From 3ec1dd13cd5b103208d70945d2ef8b5302080ad9 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Tue, 30 Sep 2025 23:18:35 +0530 Subject: [PATCH 1/4] feat: implement flexible data export utility with CSV, JSON, and XLSX support - Introduced Exporter class for handling various data formats. - Added formatters for CSV, JSON, and XLSX exports. - Created schemas for defining export fields and their transformations. - Implemented IssueExportSchema for exporting issue data with nested attributes. - Enhanced issue export task to utilize the new exporter system for better data handling. --- apps/api/plane/bgtasks/export_task.py | 389 ++--------------- apps/api/plane/utils/exporters/README.md | 413 ++++++++++++++++++ apps/api/plane/utils/exporters/__init__.py | 38 ++ apps/api/plane/utils/exporters/exporter.py | 56 +++ apps/api/plane/utils/exporters/formatters.py | 179 ++++++++ .../plane/utils/exporters/schemas/__init__.py | 30 ++ .../api/plane/utils/exporters/schemas/base.py | 187 ++++++++ .../plane/utils/exporters/schemas/issue.py | 171 ++++++++ 8 files changed, 1108 insertions(+), 355 deletions(-) create mode 100644 apps/api/plane/utils/exporters/README.md create mode 100644 apps/api/plane/utils/exporters/__init__.py create mode 100644 apps/api/plane/utils/exporters/exporter.py create mode 100644 apps/api/plane/utils/exporters/formatters.py create mode 100644 apps/api/plane/utils/exporters/schemas/__init__.py create mode 100644 apps/api/plane/utils/exporters/schemas/base.py create mode 100644 apps/api/plane/utils/exporters/schemas/issue.py diff --git a/apps/api/plane/bgtasks/export_task.py b/apps/api/plane/bgtasks/export_task.py index fc41d7bbf93..f3b5c89b118 100644 --- a/apps/api/plane/bgtasks/export_task.py +++ b/apps/api/plane/bgtasks/export_task.py @@ -1,82 +1,24 @@ # Python imports -import csv import io -import json import zipfile from typing import List +from collections import defaultdict import boto3 from botocore.client import Config from uuid import UUID -from datetime import datetime, date # Third party imports from celery import shared_task - # Django imports from django.conf import settings from django.utils import timezone -from openpyxl import Workbook -from django.db.models import F, Prefetch - -from collections import defaultdict +from django.db.models import Prefetch # Module imports -from plane.db.models import ExporterHistory, Issue, FileAsset, Label, User, IssueComment +from plane.db.models import ExporterHistory, Issue, Label, User from plane.utils.exception_logger import log_exception - - -def dateTimeConverter(time: datetime) -> str | None: - """ - Convert a datetime object to a formatted string. - """ - if time: - return time.strftime("%a, %d %b %Y %I:%M:%S %Z%z") - - -def dateConverter(time: date) -> str | None: - """ - Convert a date object to a formatted string. - """ - if time: - return time.strftime("%a, %d %b %Y") - - -def create_csv_file(data: List[List[str]]) -> str: - """ - Create a CSV file from the provided data. - """ - csv_buffer = io.StringIO() - csv_writer = csv.writer(csv_buffer, delimiter=",", quoting=csv.QUOTE_ALL) - - for row in data: - csv_writer.writerow(row) - - csv_buffer.seek(0) - return csv_buffer.getvalue() - - -def create_json_file(data: List[dict]) -> str: - """ - Create a JSON file from the provided data. - """ - return json.dumps(data) - - -def create_xlsx_file(data: List[List[str]]) -> bytes: - """ - Create an XLSX file from the provided data. - """ - workbook = Workbook() - sheet = workbook.active - - for row in data: - sheet.append(row) - - xlsx_buffer = io.BytesIO() - workbook.save(xlsx_buffer) - xlsx_buffer.seek(0) - return xlsx_buffer.getvalue() +from plane.utils.exporters import Exporter, IssueExportSchema def create_zip_file(files: List[tuple[str, str | bytes]]) -> io.BytesIO: @@ -118,7 +60,9 @@ def upload_to_s3(zip_file: io.BytesIO, workspace_id: UUID, token_id: str, slug: # Generate presigned url for the uploaded file with different base presign_s3 = boto3.client( "s3", - endpoint_url=f"{settings.AWS_S3_URL_PROTOCOL}//{str(settings.AWS_S3_CUSTOM_DOMAIN).replace('/uploads', '')}/", # noqa: E501 + endpoint_url=( + f"{settings.AWS_S3_URL_PROTOCOL}//{str(settings.AWS_S3_CUSTOM_DOMAIN).replace('/uploads', '')}/" + ), aws_access_key_id=settings.AWS_ACCESS_KEY_ID, aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, config=Config(signature_version="s3v4"), @@ -176,187 +120,6 @@ def upload_to_s3(zip_file: io.BytesIO, workspace_id: UUID, token_id: str, slug: exporter_instance.save(update_fields=["status", "url", "key"]) -def generate_table_row(issue: dict) -> List[str]: - """ - Generate a table row from an issue dictionary. - """ - return [ - f"""{issue["project_identifier"]}-{issue["sequence_id"]}""", - issue["project_name"], - issue["name"], - issue["description"], - issue["state_name"], - dateConverter(issue["start_date"]), - dateConverter(issue["target_date"]), - issue["priority"], - issue["created_by"], - ", ".join(issue["labels"]) if issue["labels"] else "", - issue["cycle_name"], - issue["cycle_start_date"], - issue["cycle_end_date"], - ", ".join(issue.get("module_name", "")) if issue.get("module_name") else "", - dateTimeConverter(issue["created_at"]), - dateTimeConverter(issue["updated_at"]), - dateTimeConverter(issue["completed_at"]), - dateTimeConverter(issue["archived_at"]), - ( - ", ".join( - [ - f"{comment['comment']} ({comment['created_at']} by {comment['created_by']})" - for comment in issue["comments"] - ] - ) - if issue["comments"] - else "" - ), - issue["estimate"] if issue["estimate"] else "", - ", ".join(issue["link"]) if issue["link"] else "", - ", ".join(issue["assignees"]) if issue["assignees"] else "", - issue["subscribers_count"] if issue["subscribers_count"] else "", - issue["attachment_count"] if issue["attachment_count"] else "", - ", ".join(issue["attachment_links"]) if issue["attachment_links"] else "", - ] - - -def generate_json_row(issue: dict) -> dict: - """ - Generate a JSON row from an issue dictionary. - """ - return { - "ID": f"""{issue["project_identifier"]}-{issue["sequence_id"]}""", - "Project": issue["project_name"], - "Name": issue["name"], - "Description": issue["description"], - "State": issue["state_name"], - "Start Date": dateConverter(issue["start_date"]), - "Target Date": dateConverter(issue["target_date"]), - "Priority": issue["priority"], - "Created By": (f"{issue['created_by']}" if issue["created_by"] else ""), - "Assignee": issue["assignees"], - "Labels": issue["labels"], - "Cycle Name": issue["cycle_name"], - "Cycle Start Date": issue["cycle_start_date"], - "Cycle End Date": issue["cycle_end_date"], - "Module Name": issue["module_name"], - "Created At": dateTimeConverter(issue["created_at"]), - "Updated At": dateTimeConverter(issue["updated_at"]), - "Completed At": dateTimeConverter(issue["completed_at"]), - "Archived At": dateTimeConverter(issue["archived_at"]), - "Comments": issue["comments"], - "Estimate": issue["estimate"], - "Link": issue["link"], - "Subscribers Count": issue["subscribers_count"], - "Attachment Count": issue["attachment_count"], - "Attachment Links": issue["attachment_links"], - } - - -def update_json_row(rows: List[dict], row: dict) -> None: - """ - Update the json row with the new assignee and label. - """ - matched_index = next( - (index for index, existing_row in enumerate(rows) if existing_row["ID"] == row["ID"]), - None, - ) - - if matched_index is not None: - existing_assignees, existing_labels = ( - rows[matched_index]["Assignee"], - rows[matched_index]["Labels"], - ) - assignee, label = row["Assignee"], row["Labels"] - - if assignee is not None and (existing_assignees is None or label not in existing_assignees): - rows[matched_index]["Assignee"] += f", {assignee}" - if label is not None and (existing_labels is None or label not in existing_labels): - rows[matched_index]["Labels"] += f", {label}" - else: - rows.append(row) - - -def update_table_row(rows: List[List[str]], row: List[str]) -> None: - """ - Update the table row with the new assignee and label. - """ - matched_index = next( - (index for index, existing_row in enumerate(rows) if existing_row[0] == row[0]), - None, - ) - - if matched_index is not None: - existing_assignees, existing_labels = rows[matched_index][7:9] - assignee, label = row[7:9] - - if assignee is not None and (existing_assignees is None or label not in existing_assignees): - rows[matched_index][8] += f", {assignee}" - if label is not None and (existing_labels is None or label not in existing_labels): - rows[matched_index][8] += f", {label}" - else: - rows.append(row) - - -def generate_csv( - header: List[str], - project_id: str, - issues: List[dict], - files: List[tuple[str, str | bytes]], -) -> None: - """ - Generate CSV export for all the passed issues. - """ - rows = [header] - for issue in issues: - row = generate_table_row(issue) - update_table_row(rows, row) - csv_file = create_csv_file(rows) - files.append((f"{project_id}.csv", csv_file)) - - -def generate_json( - header: List[str], - project_id: str, - issues: List[dict], - files: List[tuple[str, str | bytes]], -) -> None: - """ - Generate JSON export for all the passed issues. - """ - rows = [] - for issue in issues: - row = generate_json_row(issue) - update_json_row(rows, row) - json_file = create_json_file(rows) - files.append((f"{project_id}.json", json_file)) - - -def generate_xlsx( - header: List[str], - project_id: str, - issues: List[dict], - files: List[tuple[str, str | bytes]], -) -> None: - """ - Generate XLSX export for all the passed issues. - """ - rows = [header] - for issue in issues: - row = generate_table_row(issue) - - update_table_row(rows, row) - xlsx_file = create_xlsx_file(rows) - files.append((f"{project_id}.xlsx", xlsx_file)) - - -def get_created_by(obj: Issue | IssueComment) -> str: - """ - Get the created by user for the given object. - """ - if obj.created_by: - return f"{obj.created_by.first_name} {obj.created_by.last_name}" - return "" - - @shared_task def issue_export_task( provider: str, @@ -377,7 +140,7 @@ def issue_export_task( exporter_instance.status = "processing" exporter_instance.save(update_fields=["status"]) - # Base query to get the issues + # Build base queryset for issues workspace_issues = ( Issue.objects.filter( workspace__id=workspace_id, @@ -415,111 +178,24 @@ def issue_export_task( ) ) - # Get the attachments for the issues - file_assets = FileAsset.objects.filter( - issue_id__in=workspace_issues.values_list("id", flat=True), - entity_type=FileAsset.EntityTypeContext.ISSUE_ATTACHMENT, - ).annotate(work_item_id=F("issue_id"), asset_id=F("id")) - - # Create a dictionary to store the attachments for the issues - attachment_dict = defaultdict(list) - for asset in file_assets: - attachment_dict[asset.work_item_id].append(asset.asset_id) - - # Create a list to store the issues data - issues_data = [] - - # Iterate over the issues - for issue in workspace_issues: - attachments = attachment_dict.get(issue.id, []) - - issue_data = { - "id": issue.id, - "project_identifier": issue.project.identifier, - "project_name": issue.project.name, - "project_id": issue.project.id, - "sequence_id": issue.sequence_id, - "name": issue.name, - "description": issue.description_stripped, - "priority": issue.priority, - "start_date": issue.start_date, - "target_date": issue.target_date, - "state_name": issue.state.name if issue.state else None, - "created_at": issue.created_at, - "updated_at": issue.updated_at, - "completed_at": issue.completed_at, - "archived_at": issue.archived_at, - "module_name": [module.module.name for module in issue.issue_module.all()], - "created_by": get_created_by(issue), - "labels": [label.name for label in issue.label_details], - "comments": [ - { - "comment": comment.comment_stripped, - "created_at": dateConverter(comment.created_at), - "created_by": get_created_by(comment), - } - for comment in issue.issue_comments.all() - ], - "estimate": issue.estimate_point.value if issue.estimate_point and issue.estimate_point.value else "", - "link": [link.url for link in issue.issue_link.all()], - "assignees": [f"{assignee.first_name} {assignee.last_name}" for assignee in issue.assignee_details], - "subscribers_count": issue.issue_subscribers.count(), - "attachment_count": len(attachments), - "attachment_links": [ - f"/api/assets/v2/workspaces/{issue.workspace.slug}/projects/{issue.project_id}/issues/{issue.id}/attachments/{asset}/" - for asset in attachments - ], - } + # Serialize issues using the schema + # TODO: Add support for custom field selection from request/settings + issues_data = IssueExportSchema.serialize_issues(workspace_issues) - # Get Cycles data for the issue - cycle = issue.issue_cycle.last() - if cycle: - # Update cycle data - issue_data["cycle_name"] = cycle.cycle.name - issue_data["cycle_start_date"] = dateConverter(cycle.cycle.start_date) - issue_data["cycle_end_date"] = dateConverter(cycle.cycle.end_date) - else: - issue_data["cycle_name"] = "" - issue_data["cycle_start_date"] = "" - issue_data["cycle_end_date"] = "" - - issues_data.append(issue_data) - - # CSV header - header = [ - "ID", - "Project", - "Name", - "Description", - "State", - "Start Date", - "Target Date", - "Priority", - "Created By", - "Labels", - "Cycle Name", - "Cycle Start Date", - "Cycle End Date", - "Module Name", - "Created At", - "Updated At", - "Completed At", - "Archived At", - "Comments", - "Estimate", - "Link", - "Assignees", - "Subscribers Count", - "Attachment Count", - "Attachment Links", - ] - - # Map the provider to the function - EXPORTER_MAPPER = { - "csv": generate_csv, - "json": generate_json, - "xlsx": generate_xlsx, - } + # Create exporter for the specified format + try: + exporter = Exporter( + format_type=provider, + schema_class=IssueExportSchema, + options={"list_joiner": ", "}, + ) + except ValueError as e: + # Invalid format type + exporter_instance = ExporterHistory.objects.get(token=token_id) + exporter_instance.status = "failed" + exporter_instance.reason = str(e) + exporter_instance.save(update_fields=["status", "reason"]) + return files = [] if multiple: @@ -530,14 +206,17 @@ def issue_export_task( for project_id in project_ids: issues = project_dict.get(str(project_id), []) - exporter = EXPORTER_MAPPER.get(provider) - if exporter is not None: - exporter(header, project_id, issues, files) + if issues: # Only export if there are issues for this project + # Generate filename for each project export + export_filename = f"{slug}-{project_id}" + filename, content = exporter.export(export_filename, issues) + files.append((filename, content)) else: - exporter = EXPORTER_MAPPER.get(provider) - if exporter is not None: - exporter(header, workspace_id, issues_data, files) + # Generate filename for workspace export + export_filename = f"{slug}-{workspace_id}" + filename, content = exporter.export(export_filename, issues_data) + files.append((filename, content)) zip_buffer = create_zip_file(files) upload_to_s3(zip_buffer, workspace_id, token_id, slug) diff --git a/apps/api/plane/utils/exporters/README.md b/apps/api/plane/utils/exporters/README.md new file mode 100644 index 00000000000..601ae2d1243 --- /dev/null +++ b/apps/api/plane/utils/exporters/README.md @@ -0,0 +1,413 @@ +# ๐Ÿ“Š Exporters + +A flexible and extensible data export utility for exporting Django model data in multiple formats (CSV, JSON, XLSX). + +## ๐ŸŽฏ Overview + +The exporters module provides a schema-based approach to exporting data with support for: + +- **๐Ÿ“„ Multiple formats**: CSV, JSON, and XLSX (Excel) +- **๐Ÿ”’ Type-safe field definitions**: StringField, NumberField, DateField, DateTimeField, BooleanField, ListField, JSONField +- **โšก Custom transformations**: Field-level transformations and custom preparer methods +- **๐Ÿ”— Dotted path notation**: Easy access to nested attributes and related models +- **๐ŸŽจ Format-specific handling**: Automatic formatting based on export format (e.g., lists as arrays in JSON, comma-separated in CSV) + +## ๐Ÿš€ Quick Start + +### Basic Usage + +```python +from plane.utils.exporters import Exporter, ExportSchema, StringField, NumberField + +# Define a schema +class UserExportSchema(ExportSchema): + name = StringField(source="username", label="User Name") + email = StringField(source="email", label="Email Address") + posts_count = NumberField(label="Total Posts") + + def prepare_posts_count(self, obj): + return obj.posts.count() + +# Export data +users = User.objects.all() +exporter = Exporter(format_type="csv", schema_class=UserExportSchema) +filename, content = exporter.export("users_export", users) +``` + +### Exporting Issues + +```python +from plane.utils.exporters import Exporter, IssueExportSchema + +# Get issues with prefetched relations +issues = Issue.objects.filter(project_id=project_id).prefetch_related( + 'assignee_details', + 'label_details', + 'issue_module', + # ... other relations +) + +# Export as XLSX +exporter = Exporter(format_type="xlsx", schema_class=IssueExportSchema) +filename, content = exporter.export("issues", issues) + +# Export with custom fields only +issues_data = IssueExportSchema.serialize_issues( + issues, + fields=["id", "name", "state_name", "assignees"] +) +exporter = Exporter(format_type="json", schema_class=IssueExportSchema) +filename, content = exporter.export("issues_filtered", issues_data) +``` + +## ๐Ÿ“ Schema Definition + +### Field Types + +#### ๐Ÿ“ StringField + +Converts values to strings. + +```python +name = StringField(source="name", label="Name", default="N/A") +``` + +#### ๐Ÿ”ข NumberField + +Handles numeric values (int, float). + +```python +count = NumberField(source="items_count", label="Count", default=0) +``` + +#### ๐Ÿ“… DateField + +Formats date objects as `%a, %d %b %Y` (e.g., "Mon, 01 Jan 2024"). + +```python +start_date = DateField(source="start_date", label="Start Date") +``` + +#### โฐ DateTimeField + +Formats datetime objects as `%a, %d %b %Y %I:%M:%S %Z%z`. + +```python +created_at = DateTimeField(source="created_at", label="Created At") +``` + +#### โœ… BooleanField + +Converts values to boolean. + +```python +is_active = BooleanField(source="is_active", label="Active", default=False) +``` + +#### ๐Ÿ“‹ ListField + +Handles list/array values. In CSV/XLSX, lists are joined with a separator (default: `", "`). In JSON, they remain as arrays. + +```python +tags = ListField(source="tags", label="Tags") +assignees = ListField(label="Assignees") # Custom preparer can populate this +``` + +#### ๐Ÿ—‚๏ธ JSONField + +Handles complex JSON-serializable objects (dicts, lists of dicts). In CSV/XLSX, they're serialized as JSON strings. In JSON, they remain as objects. + +```python +metadata = JSONField(source="metadata", label="Metadata") +comments = JSONField(label="Comments") +``` + +### โš™๏ธ Field Parameters + +All field types support these parameters: + +- **`source`**: Dotted path to the attribute (e.g., `"project.name"`) or a callable +- **`transform`**: Custom transformation function +- **`default`**: Default value when field is None +- **`label`**: Display name in export headers + +### ๐Ÿ”— Dotted Path Notation + +Access nested attributes using dot notation: + +```python +project_name = StringField(source="project.name", label="Project") +owner_email = StringField(source="created_by.email", label="Owner Email") +``` + +### ๐ŸŽฏ Custom Preparers + +For complex logic, define `prepare_{field_name}` methods: + +```python +class MySchema(ExportSchema): + assignees = ListField(label="Assignees") + + def prepare_assignees(self, obj): + return [f"{u.first_name} {u.last_name}" for u in obj.assignee_details] +``` + +Preparers take precedence over field definitions. + +### ๐Ÿ”„ Using Callables + +Use callable sources for dynamic values: + +```python +class MySchema(ExportSchema): + status = StringField( + source=lambda obj: "Active" if obj.is_active else "Inactive", + label="Status" + ) +``` + +### โšก Transform Functions + +Apply transformations to values: + +```python +class MySchema(ExportSchema): + name = StringField( + source="name", + transform=lambda val: val.upper(), + label="Name (Uppercase)" + ) +``` + +## ๐Ÿ“ฆ Export Formats + +### ๐Ÿ“Š CSV Format + +- Fields are quoted with `QUOTE_ALL` +- Lists are joined with `", "` (customizable with `list_joiner` option) +- JSON objects are serialized as JSON strings +- File extension: `.csv` + +```python +exporter = Exporter( + format_type="csv", + schema_class=MySchema, + options={"list_joiner": "; "} # Custom separator +) +``` + +### ๐Ÿ“‹ JSON Format + +- Lists remain as arrays +- Objects remain as nested structures +- Preserves data types +- File extension: `.json` + +```python +exporter = Exporter(format_type="json", schema_class=MySchema) +filename, content = exporter.export("data", records) +# content is a JSON string: '[{"field": "value"}, ...]' +``` + +### ๐Ÿ“— XLSX Format + +- Creates Excel-compatible files using openpyxl +- Lists are joined with `", "` (customizable with `list_joiner` option) +- JSON objects are serialized as JSON strings +- File extension: `.xlsx` +- Returns binary content (bytes) + +```python +exporter = Exporter(format_type="xlsx", schema_class=MySchema) +filename, content = exporter.export("data", records) +# content is bytes +``` + +## ๐Ÿ”ง Advanced Usage + +### ๐Ÿ“ฆ Using Context + +Pass context data to schemas for additional information: + +```python +class MySchema(ExportSchema): + attachment_count = NumberField(label="Attachments") + + def prepare_attachment_count(self, obj): + attachments_dict = self.context.get("attachments_dict", {}) + return len(attachments_dict.get(obj.id, [])) + +# Create schema with context +attachments_dict = get_attachments_dict(queryset) +schema = MySchema(context={"attachments_dict": attachments_dict}) + +# Serialize +data = [schema.serialize(obj) for obj in queryset] +``` + +### ๐Ÿ”Œ Registering Custom Formatters + +Add support for new export formats: + +```python +from plane.utils.exporters import Exporter, BaseFormatter + +class XMLFormatter(BaseFormatter): + def format(self, filename, records, schema_class, options=None): + # Implementation + return (f"{filename}.xml", xml_content) + +# Register the formatter +Exporter.register_formatter("xml", XMLFormatter) + +# Use it +exporter = Exporter(format_type="xml", schema_class=MySchema) +``` + +### โœ… Checking Available Formats + +```python +formats = Exporter.get_available_formats() +# Returns: ['csv', 'json', 'xlsx'] +``` + +### ๐Ÿ” Filtering Fields + +Use the `serialize_issues` class method pattern to filter fields: + +```python +@classmethod +def serialize_records(cls, queryset, fields=None): + fields_to_extract = set(fields) if fields else set(cls._declared_fields.keys()) + + schema = cls() + records_data = [] + for record in queryset: + record_data = schema.serialize(record) + filtered_data = { + field: record_data.get(field, "") + for field in fields_to_extract + if field in record_data + } + records_data.append(filtered_data) + + return records_data +``` + +## ๐Ÿ’ก Example: IssueExportSchema + +The `IssueExportSchema` demonstrates a complete implementation: + +```python +from plane.utils.exporters import IssueExportSchema + +# Serialize issues +issues_data = IssueExportSchema.serialize_issues( + issues_queryset, + fields=["id", "name", "state_name", "assignees", "labels"] +) + +# Export as CSV +exporter = Exporter(format_type="csv", schema_class=IssueExportSchema) +filename, content = exporter.export("issues", issues_data) +``` + +Key features: + +- ๐Ÿ”— Access to related models via dotted paths +- ๐ŸŽฏ Custom preparers for complex fields +- ๐Ÿ“Ž Context-based attachment handling +- ๐Ÿ“‹ List and JSON field handling +- ๐Ÿ“… Date/datetime formatting + +## โœจ Best Practices + +1. **๐Ÿš„ Prefetch Relations**: Always prefetch related data before exporting to avoid N+1 queries: + + ```python + issues = Issue.objects.prefetch_related('assignee_details', 'label_details') + ``` + +2. **๐Ÿท๏ธ Use Labels**: Provide descriptive labels for better export headers: + + ```python + created_at = DateTimeField(source="created_at", label="Created At") + ``` + +3. **๐Ÿ›ก๏ธ Handle None Values**: Set appropriate defaults for fields that might be None: + + ```python + count = NumberField(source="count", default=0) + ``` + +4. **๐ŸŽฏ Use Preparers for Complex Logic**: Keep field definitions simple and use preparers for complex transformations: + + ```python + def prepare_assignees(self, obj): + return [f"{u.first_name} {u.last_name}" for u in obj.assignee_details] + ``` + +5. **โšก Context for Expensive Operations**: Use context to pass pre-computed data and avoid redundant queries: + ```python + schema = MySchema(context={"computed_data": precomputed_dict}) + ``` + +## ๐Ÿ“š API Reference + +### ๐Ÿ“Š Exporter + +**`__init__(format_type, schema_class, options=None)`** + +- `format_type`: Export format ('csv', 'json', 'xlsx') +- `schema_class`: Schema class defining fields +- `options`: Optional dict of format-specific options + +**`export(filename, records)`** + +- Returns: `(filename_with_extension, content)` +- `content` is str for CSV/JSON, bytes for XLSX + +**`get_available_formats()`** (class method) + +- Returns: List of available format types + +**`register_formatter(format_type, formatter_class)`** (class method) + +- Register a custom formatter + +### ๐Ÿ“ ExportSchema + +**`__init__(context=None)`** + +- `context`: Optional dict passed to field transformations and preparers + +**`serialize(obj)`** + +- Returns: Dict of serialized field values + +### ๐Ÿ”ง ExportField + +Base class for all field types. Subclass to create custom field types. + +**`get_value(obj, context)`** + +- Returns: Formatted value for the field + +**`_format_value(raw)`** + +- Override in subclasses for type-specific formatting + +## ๐Ÿงช Testing + +```python +# Test schema serialization +schema = MySchema() +result = schema.serialize(obj) +assert result["field_name"] == expected_value + +# Test export +exporter = Exporter(format_type="json", schema_class=MySchema) +filename, content = exporter.export("test", [{"data": "value"}]) +assert filename == "test.json" +assert isinstance(content, str) +``` diff --git a/apps/api/plane/utils/exporters/__init__.py b/apps/api/plane/utils/exporters/__init__.py new file mode 100644 index 00000000000..9e7b1a9d51d --- /dev/null +++ b/apps/api/plane/utils/exporters/__init__.py @@ -0,0 +1,38 @@ +"""Export utilities for various data formats.""" + +from .exporter import Exporter +from .formatters import BaseFormatter, CSVFormatter, JSONFormatter, XLSXFormatter +from .schemas import ( + BooleanField, + DateField, + DateTimeField, + ExportField, + ExportSchema, + IssueExportSchema, + JSONField, + ListField, + NumberField, + StringField, +) + +__all__ = [ + # Core Exporter + "Exporter", + # Schemas + "ExportSchema", + "ExportField", + "StringField", + "NumberField", + "DateField", + "DateTimeField", + "BooleanField", + "ListField", + "JSONField", + # Formatters + "BaseFormatter", + "CSVFormatter", + "JSONFormatter", + "XLSXFormatter", + # Issue Schema + "IssueExportSchema", +] diff --git a/apps/api/plane/utils/exporters/exporter.py b/apps/api/plane/utils/exporters/exporter.py new file mode 100644 index 00000000000..ee51c60bf2d --- /dev/null +++ b/apps/api/plane/utils/exporters/exporter.py @@ -0,0 +1,56 @@ +from typing import Any, Dict, List, Type + +from .formatters import CSVFormatter, JSONFormatter, XLSXFormatter + + +class Exporter: + """Generic exporter class that handles data exports using different formatters.""" + + # Available formatters + FORMATTERS = { + "csv": CSVFormatter, + "json": JSONFormatter, + "xlsx": XLSXFormatter, + } + + def __init__(self, format_type: str, schema_class: Type, options: Dict[str, Any] = None): + """Initialize exporter with specified format type and schema. + + Args: + format_type: The export format (csv, json, xlsx) + schema_class: The schema class to use for field definitions + options: Optional formatting options + """ + if format_type not in self.FORMATTERS: + raise ValueError(f"Unsupported format: {format_type}. Available: {list(self.FORMATTERS.keys())}") + + self.format_type = format_type + self.schema_class = schema_class + self.formatter = self.FORMATTERS[format_type]() + self.options = options or {} + + def export( + self, + filename: str, + records: List[dict], + ) -> tuple[str, str | bytes]: + """Export records using the configured formatter and return (filename, content). + + Args: + filename: The filename for the export (without extension) + records: List of data dictionaries + + Returns: + Tuple of (filename_with_extension, content) + """ + return self.formatter.format(filename, records, self.schema_class, self.options) + + @classmethod + def get_available_formats(cls) -> List[str]: + """Get list of available export formats.""" + return list(cls.FORMATTERS.keys()) + + @classmethod + def register_formatter(cls, format_type: str, formatter_class: type) -> None: + """Register a new formatter for a format type.""" + cls.FORMATTERS[format_type] = formatter_class diff --git a/apps/api/plane/utils/exporters/formatters.py b/apps/api/plane/utils/exporters/formatters.py new file mode 100644 index 00000000000..c20f535ebbc --- /dev/null +++ b/apps/api/plane/utils/exporters/formatters.py @@ -0,0 +1,179 @@ +import csv +import io +import json +from typing import Any, Dict, List, Type + +from openpyxl import Workbook + + +class BaseFormatter: + """Base class for export formatters.""" + + def format( + self, + filename: str, + records: List[dict], + schema_class: Type, + options: Dict[str, Any] | None = None, + ) -> tuple[str, str | bytes]: + """Format records for export. + + Args: + filename: The filename for the export (without extension) + records: List of records to export + schema_class: Schema class to extract field order and labels + options: Optional formatting options + + Returns: + Tuple of (filename_with_extension, content) + """ + raise NotImplementedError + + @staticmethod + def _get_field_info(schema_class: Type) -> tuple[List[str], Dict[str, str]]: + """Extract field order and labels from schema. + + Args: + schema_class: Schema class with field definitions + + Returns: + Tuple of (field_order, field_labels) + """ + if not hasattr(schema_class, "_declared_fields"): + raise ValueError(f"Schema class {schema_class.__name__} must have _declared_fields attribute") + + # Get order and labels from schema + field_order = list(schema_class._declared_fields.keys()) + field_labels = { + name: field.label if field.label else name.replace("_", " ").title() + for name, field in schema_class._declared_fields.items() + } + + return field_order, field_labels + + +class CSVFormatter(BaseFormatter): + """Formatter for CSV exports.""" + + @staticmethod + def _format_field_value(value: Any, list_joiner: str = ", ") -> str: + """Format a field value for CSV output.""" + if value is None: + return "" + if isinstance(value, list): + return list_joiner.join(str(v) for v in value) + if isinstance(value, dict): + # For complex objects, serialize as JSON + return json.dumps(value) + return str(value) + + def _generate_table_row( + self, record: dict, field_order: List[str], options: Dict[str, Any] | None = None + ) -> List[str]: + """Generate a CSV row from a record.""" + opts = options or {} + list_joiner = opts.get("list_joiner", ", ") + return [self._format_field_value(record.get(field, ""), list_joiner) for field in field_order] + + def _create_csv_file(self, data: List[List[str]]) -> str: + """Create CSV file content from row data.""" + buf = io.StringIO() + writer = csv.writer(buf, delimiter=",", quoting=csv.QUOTE_ALL) + for row in data: + writer.writerow(row) + buf.seek(0) + return buf.getvalue() + + def format(self, filename, records, schema_class, options: Dict[str, Any] | None = None) -> tuple[str, str]: + if not records: + return (f"{filename}.csv", "") + + # Get field order and labels from schema + field_order, field_labels = self._get_field_info(schema_class) + header = [field_labels[field] for field in field_order] + + rows = [header] + for record in records: + row = self._generate_table_row(record, field_order, options) + rows.append(row) + content = self._create_csv_file(rows) + return (f"{filename}.csv", content) + + +class JSONFormatter(BaseFormatter): + """Formatter for JSON exports.""" + + def _generate_json_row( + self, record: dict, field_labels: Dict[str, str], field_order: List[str], options: Dict[str, Any] | None = None + ) -> dict: + """Generate a JSON object from a record. + + Preserves data types - lists stay as arrays, dicts stay as objects. + """ + return {field_labels[field]: record.get(field) for field in field_order if field in record} + + def format(self, filename, records, schema_class, options: Dict[str, Any] | None = None) -> tuple[str, str]: + if not records: + return (f"{filename}.json", "[]") + + # Get field order and labels from schema + field_order, field_labels = self._get_field_info(schema_class) + + rows: List[dict] = [] + for record in records: + row = self._generate_json_row(record, field_labels, field_order, options) + rows.append(row) + content = json.dumps(rows) + return (f"{filename}.json", content) + + +class XLSXFormatter(BaseFormatter): + """Formatter for XLSX (Excel) exports.""" + + @staticmethod + def _format_field_value(value: Any, list_joiner: str = ", ") -> str: + """Format a field value for XLSX output.""" + if value is None: + return "" + if isinstance(value, list): + return list_joiner.join(str(v) for v in value) + if isinstance(value, dict): + # For complex objects, serialize as JSON + return json.dumps(value) + return str(value) + + def _generate_table_row( + self, record: dict, field_order: List[str], options: Dict[str, Any] | None = None + ) -> List[str]: + """Generate an XLSX row from a record.""" + opts = options or {} + list_joiner = opts.get("list_joiner", ", ") + return [self._format_field_value(record.get(field, ""), list_joiner) for field in field_order] + + def _create_xlsx_file(self, data: List[List[str]]) -> bytes: + """Create XLSX file content from row data.""" + wb = Workbook() + sh = wb.active + for row in data: + sh.append(row) + out = io.BytesIO() + wb.save(out) + out.seek(0) + return out.getvalue() + + def format(self, filename, records, schema_class, options: Dict[str, Any] | None = None) -> tuple[str, bytes]: + if not records: + # Create empty workbook + content = self._create_xlsx_file([]) + return (f"{filename}.xlsx", content) + + # Get field order and labels from schema + field_order, field_labels = self._get_field_info(schema_class) + header = [field_labels[field] for field in field_order] + + rows = [header] + for record in records: + row = self._generate_table_row(record, field_order, options) + rows.append(row) + content = self._create_xlsx_file(rows) + return (f"{filename}.xlsx", content) diff --git a/apps/api/plane/utils/exporters/schemas/__init__.py b/apps/api/plane/utils/exporters/schemas/__init__.py new file mode 100644 index 00000000000..98b2623aed2 --- /dev/null +++ b/apps/api/plane/utils/exporters/schemas/__init__.py @@ -0,0 +1,30 @@ +"""Export schemas for various data types.""" + +from .base import ( + BooleanField, + DateField, + DateTimeField, + ExportField, + ExportSchema, + JSONField, + ListField, + NumberField, + StringField, +) +from .issue import IssueExportSchema + +__all__ = [ + # Base field types + "ExportField", + "StringField", + "NumberField", + "DateField", + "DateTimeField", + "BooleanField", + "ListField", + "JSONField", + # Base schema + "ExportSchema", + # Issue schema + "IssueExportSchema", +] diff --git a/apps/api/plane/utils/exporters/schemas/base.py b/apps/api/plane/utils/exporters/schemas/base.py new file mode 100644 index 00000000000..caef29f48d0 --- /dev/null +++ b/apps/api/plane/utils/exporters/schemas/base.py @@ -0,0 +1,187 @@ +from dataclasses import dataclass, field +from typing import Any, Callable, Dict, List, Optional + + +@dataclass +class ExportField: + """Base export field class for generic fields.""" + + source: Optional[str | Callable[[Any, Dict[str, Any]], Any]] = None + transform: Optional[Callable[[Any, Dict[str, Any]], Any]] = None + default: Any = "" + label: Optional[str] = None # Display name for export headers + + def get_value(self, obj: Any, context: Dict[str, Any]) -> Any: + raw: Any + if callable(self.source): + try: + raw = self.source(obj, context) + except TypeError: + raw = self.source(obj) + elif isinstance(self.source, str) and self.source: + raw = self._resolve_dotted_path(obj, self.source) + else: + raw = obj + + if self.transform is not None: + try: + return self.transform(raw, context) + except TypeError: + return self.transform(raw) + + return self._format_value(raw) + + def _format_value(self, raw: Any) -> Any: + """Format the raw value. Override in subclasses for type-specific formatting.""" + return raw if raw is not None else self.default + + def _resolve_dotted_path(self, obj: Any, path: str) -> Any: + current = obj + for part in path.split("."): + if current is None: + return None + if hasattr(current, part): + current = getattr(current, part) + elif isinstance(current, dict): + current = current.get(part) + else: + return None + return current + + +@dataclass +class StringField(ExportField): + """Export field for string values.""" + + default: str = "" + + def _format_value(self, raw: Any) -> str: + if raw is None: + return self.default + return str(raw) + + +@dataclass +class DateField(ExportField): + """Export field for date values with automatic conversion.""" + + default: str = "" + + def _format_value(self, raw: Any) -> str: + if raw is None: + return self.default + # Convert date to formatted string + if hasattr(raw, "strftime"): + return raw.strftime("%a, %d %b %Y") + return str(raw) + + +@dataclass +class DateTimeField(ExportField): + """Export field for datetime values with automatic conversion.""" + + default: str = "" + + def _format_value(self, raw: Any) -> str: + if raw is None: + return self.default + # Convert datetime to formatted string + if hasattr(raw, "strftime"): + return raw.strftime("%a, %d %b %Y %I:%M:%S %Z%z") + return str(raw) + + +@dataclass +class NumberField(ExportField): + """Export field for numeric values.""" + + default: Any = "" + + def _format_value(self, raw: Any) -> Any: + if raw is None: + return self.default + return raw + + +@dataclass +class BooleanField(ExportField): + """Export field for boolean values.""" + + default: bool = False + + def _format_value(self, raw: Any) -> bool: + if raw is None: + return self.default + return bool(raw) + + +@dataclass +class ListField(ExportField): + """Export field for list/array values. + + Returns the list as-is by default. The formatter will handle conversion to strings + when needed (e.g., CSV/XLSX will join with separator, JSON will keep as array). + """ + + default: Optional[List] = field(default_factory=list) + + def _format_value(self, raw: Any) -> List[Any]: + if raw is None: + return self.default if self.default is not None else [] + if isinstance(raw, (list, tuple)): + return list(raw) + return [raw] # Wrap single items in a list + + +@dataclass +class JSONField(ExportField): + """Export field for complex JSON-serializable values (dicts, lists of dicts, etc). + + Preserves the structure as-is for JSON exports. For CSV/XLSX, the formatter + will handle serialization (e.g., JSON stringify). + """ + + default: Any = field(default_factory=dict) + + def _format_value(self, raw: Any) -> Any: + if raw is None: + return self.default + # Return as-is - should be JSON-serializable + return raw + + +class ExportSchemaMeta(type): + def __new__(mcls, name, bases, attrs): + declared: Dict[str, ExportField] = { + key: value for key, value in list(attrs.items()) if isinstance(value, ExportField) + } + for key in declared.keys(): + attrs.pop(key) + cls = super().__new__(mcls, name, bases, attrs) + base_fields: Dict[str, ExportField] = {} + for base in bases: + if hasattr(base, "_declared_fields"): + base_fields.update(base._declared_fields) + base_fields.update(declared) + cls._declared_fields = base_fields + return cls + + +class ExportSchema(metaclass=ExportSchemaMeta): + def __init__(self, context: Optional[Dict[str, Any]] = None) -> None: + self.context = context or {} + + def serialize(self, obj: Any) -> Dict[str, Any]: + output: Dict[str, Any] = {} + for field_name, export_field in self._declared_fields.items(): + # Prefer explicit preparer methods if present + preparer = getattr(self, f"prepare_{field_name}", None) + if callable(preparer): + try: + output[field_name] = preparer(obj) + except TypeError: + output[field_name] = preparer(obj, self.context) + continue + + output[field_name] = export_field.get_value(obj, self.context) + return output diff --git a/apps/api/plane/utils/exporters/schemas/issue.py b/apps/api/plane/utils/exporters/schemas/issue.py new file mode 100644 index 00000000000..b132c6a14d5 --- /dev/null +++ b/apps/api/plane/utils/exporters/schemas/issue.py @@ -0,0 +1,171 @@ +from collections import defaultdict +from typing import Any, Dict, List + +from django.db.models import F, QuerySet + +from plane.db.models import FileAsset + +from .base import ( + DateField, + DateTimeField, + ExportSchema, + JSONField, + ListField, + NumberField, + StringField, +) + + +def get_issue_attachments_dict(issues_queryset: QuerySet) -> Dict[str, List[str]]: + """Get attachments dictionary for the given issues queryset. + + Args: + issues_queryset: Queryset of Issue objects + + Returns: + Dictionary mapping issue IDs to lists of attachment IDs + """ + file_assets = FileAsset.objects.filter( + issue_id__in=issues_queryset.values_list("id", flat=True), + entity_type=FileAsset.EntityTypeContext.ISSUE_ATTACHMENT, + ).annotate(work_item_id=F("issue_id"), asset_id=F("id")) + + attachment_dict = defaultdict(list) + for asset in file_assets: + attachment_dict[asset.work_item_id].append(asset.asset_id) + + return attachment_dict + + +class IssueExportSchema(ExportSchema): + """Schema for exporting issue data in various formats.""" + + @staticmethod + def _get_created_by(obj) -> str: + """Get the created by user for the given object.""" + try: + if getattr(obj, "created_by", None): + return f"{obj.created_by.first_name} {obj.created_by.last_name}" + except Exception: + pass + return "" + + @staticmethod + def _format_date(date_obj) -> str: + """Format date object to string.""" + if date_obj and hasattr(date_obj, "strftime"): + return date_obj.strftime("%a, %d %b %Y") + return "" + + # Field definitions with display labels + id = StringField(label="ID") + project_identifier = StringField(source="project.identifier", label="Project Identifier") + project_name = StringField(source="project.name", label="Project") + project_id = StringField(source="project.id", label="Project ID") + sequence_id = NumberField(source="sequence_id", label="Sequence ID") + name = StringField(source="name", label="Name") + description = StringField(source="description_stripped", label="Description") + priority = StringField(source="priority", label="Priority") + start_date = DateField(source="start_date", label="Start Date") + target_date = DateField(source="target_date", label="Target Date") + state_name = StringField(label="State") + created_at = DateTimeField(source="created_at", label="Created At") + updated_at = DateTimeField(source="updated_at", label="Updated At") + completed_at = DateTimeField(source="completed_at", label="Completed At") + archived_at = DateTimeField(source="archived_at", label="Archived At") + module_name = ListField(label="Module Name") + created_by = StringField(label="Created By") + labels = ListField(label="Labels") + comments = JSONField(label="Comments") + estimate = StringField(label="Estimate") + link = ListField(label="Link") + assignees = ListField(label="Assignees") + subscribers_count = NumberField(label="Subscribers Count") + attachment_count = NumberField(label="Attachment Count") + attachment_links = ListField(label="Attachment Links") + cycle_name = StringField(label="Cycle Name") + cycle_start_date = DateField(label="Cycle Start Date") + cycle_end_date = DateField(label="Cycle End Date") + + def prepare_id(self, i): + return f"{i.project.identifier}-{i.sequence_id}" + + def prepare_state_name(self, i): + return i.state.name if i.state else None + + def prepare_module_name(self, i): + return [m.module.name for m in i.issue_module.all()] + + def prepare_created_by(self, i): + return self._get_created_by(i) + + def prepare_labels(self, i): + return [label.name for label in i.label_details] + + def prepare_comments(self, i): + return [ + { + "comment": comment.comment_stripped, + "created_at": self._format_date(comment.created_at), + "created_by": self._get_created_by(comment), + } + for comment in i.issue_comments.all() + ] + + def prepare_estimate(self, i): + return i.estimate_point.value if i.estimate_point and i.estimate_point.value else "" + + def prepare_link(self, i): + return [link.url for link in i.issue_link.all()] + + def prepare_assignees(self, i): + return [f"{u.first_name} {u.last_name}" for u in i.assignee_details] + + def prepare_subscribers_count(self, i): + return i.issue_subscribers.count() + + def prepare_attachment_count(self, i): + return len((self.context.get("attachments_dict") or {}).get(i.id, [])) + + def prepare_attachment_links(self, i): + return [ + f"/api/assets/v2/workspaces/{i.workspace.slug}/projects/{i.project_id}/issues/{i.id}/attachments/{asset}/" + for asset in (self.context.get("attachments_dict") or {}).get(i.id, []) + ] + + def prepare_cycle_name(self, i): + return i.issue_cycle.last().cycle.name if i.issue_cycle.last() else "" + + def prepare_cycle_start_date(self, i): + return i.issue_cycle.last().cycle.start_date if i.issue_cycle.last() else None + + def prepare_cycle_end_date(self, i): + return i.issue_cycle.last().cycle.end_date if i.issue_cycle.last() else None + + @classmethod + def serialize_issues(cls, issues_queryset: QuerySet, fields: List[str] = None) -> List[Dict[str, Any]]: + """Serialize a queryset of issues to export data. + + Args: + issues_queryset: Queryset of Issue objects + fields: Optional list of field names to include. Defaults to all fields. + + Returns: + List of dictionaries containing serialized issue data + """ + # Get attachments for all issues + attachments_dict = get_issue_attachments_dict(issues_queryset) + + # Determine which fields to extract + fields_to_extract = set(fields) if fields else set(cls._declared_fields.keys()) + + # Serialize each issue + schema = cls(context={"attachments_dict": attachments_dict}) + issues_data = [] + for issue in issues_queryset: + issue_data = schema.serialize(issue) + # Filter to only requested fields + filtered_data = {field: issue_data.get(field, "") for field in fields_to_extract if field in issue_data} + issues_data.append(filtered_data) + + return issues_data From 8e1e8d118aa76d45e49135efad7c9ad05cff0727 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Mon, 6 Oct 2025 18:41:58 +0530 Subject: [PATCH 2/4] feat: enhance issue export functionality with new relations and context handling - Updated issue export task to utilize new IssueRelation model for better relationship management. - Refactored Exporter class to accept QuerySets directly, improving performance and flexibility. - Enhanced IssueExportSchema to include parent issues and relations in the export. - Improved documentation for exporting multiple projects and filtering fields during export. --- apps/api/plane/bgtasks/export_task.py | 46 ++--- apps/api/plane/db/models/issue.py | 15 ++ apps/api/plane/utils/exporters/README.md | 195 +++++++++++++----- apps/api/plane/utils/exporters/exporter.py | 19 +- .../api/plane/utils/exporters/schemas/base.py | 57 +++++ .../plane/utils/exporters/schemas/issue.py | 58 +++--- 6 files changed, 278 insertions(+), 112 deletions(-) diff --git a/apps/api/plane/bgtasks/export_task.py b/apps/api/plane/bgtasks/export_task.py index f3b5c89b118..d8aad5f69c8 100644 --- a/apps/api/plane/bgtasks/export_task.py +++ b/apps/api/plane/bgtasks/export_task.py @@ -16,7 +16,7 @@ from django.db.models import Prefetch # Module imports -from plane.db.models import ExporterHistory, Issue, Label, User +from plane.db.models import ExporterHistory, Issue, IssueRelation from plane.utils.exception_logger import log_exception from plane.utils.exporters import Exporter, IssueExportSchema @@ -153,7 +153,6 @@ def issue_export_task( "project", "workspace", "state", - "parent", "created_by", "estimate_point", ) @@ -163,25 +162,23 @@ def issue_export_task( "issue_module__module", "issue_comments", "assignees", + "issue_subscribers", + "issue_link", Prefetch( - "assignees", - queryset=User.objects.only("first_name", "last_name").distinct(), - to_attr="assignee_details", + "issue_relation", + queryset=IssueRelation.objects.select_related("related_issue", "related_issue__project"), ), Prefetch( - "labels", - queryset=Label.objects.only("name").distinct(), - to_attr="label_details", + "issue_related", + queryset=IssueRelation.objects.select_related("issue", "issue__project"), + ), + Prefetch( + "parent", + queryset=Issue.objects.select_related("type", "project"), ), - "issue_subscribers", - "issue_link", ) ) - # Serialize issues using the schema - # TODO: Add support for custom field selection from request/settings - issues_data = IssueExportSchema.serialize_issues(workspace_issues) - # Create exporter for the specified format try: exporter = Exporter( @@ -199,23 +196,16 @@ def issue_export_task( files = [] if multiple: - project_dict = defaultdict(list) - for issue in issues_data: - project_dict[str(issue["project_id"])].append(issue) - + # Export each project separately with its own queryset for project_id in project_ids: - issues = project_dict.get(str(project_id), []) - - if issues: # Only export if there are issues for this project - # Generate filename for each project export - export_filename = f"{slug}-{project_id}" - filename, content = exporter.export(export_filename, issues) - files.append((filename, content)) - + project_issues = workspace_issues.filter(project_id=project_id) + export_filename = f"{slug}-{project_id}" + filename, content = exporter.export(export_filename, project_issues) + files.append((filename, content)) else: - # Generate filename for workspace export + # Export all issues in a single file export_filename = f"{slug}-{workspace_id}" - filename, content = exporter.export(export_filename, issues_data) + filename, content = exporter.export(export_filename, workspace_issues) files.append((filename, content)) zip_buffer = create_zip_file(files) diff --git a/apps/api/plane/db/models/issue.py b/apps/api/plane/db/models/issue.py index 9412a57c62e..c495fdb5741 100644 --- a/apps/api/plane/db/models/issue.py +++ b/apps/api/plane/db/models/issue.py @@ -273,6 +273,21 @@ class IssueRelationChoices(models.TextChoices): IMPLEMENTED_BY = "implemented_by", "Implemented By" +# Bidirectional relation pairs: (forward, reverse) +# Defined after class to avoid enum metaclass conflicts +IssueRelationChoices._RELATION_PAIRS = ( + ("blocked_by", "blocking"), + ("relates_to", "relates_to"), # symmetric + ("duplicate", "duplicate"), # symmetric + ("start_before", "start_after"), + ("finish_before", "finish_after"), + ("implemented_by", "implements"), +) + +# Generate reverse mapping from pairs +IssueRelationChoices._REVERSE_MAPPING = {forward: reverse for forward, reverse in IssueRelationChoices._RELATION_PAIRS} + + class IssueRelation(ProjectBaseModel): issue = models.ForeignKey(Issue, related_name="issue_relation", on_delete=models.CASCADE) related_issue = models.ForeignKey(Issue, related_name="issue_related", on_delete=models.CASCADE) diff --git a/apps/api/plane/utils/exporters/README.md b/apps/api/plane/utils/exporters/README.md index 601ae2d1243..02bcffd7810 100644 --- a/apps/api/plane/utils/exporters/README.md +++ b/apps/api/plane/utils/exporters/README.md @@ -28,7 +28,7 @@ class UserExportSchema(ExportSchema): def prepare_posts_count(self, obj): return obj.posts.count() -# Export data +# Export data - just pass the queryset! users = User.objects.all() exporter = Exporter(format_type="csv", schema_class=UserExportSchema) filename, content = exporter.export("users_export", users) @@ -47,17 +47,24 @@ issues = Issue.objects.filter(project_id=project_id).prefetch_related( # ... other relations ) -# Export as XLSX +# Export as XLSX - pass the queryset directly! exporter = Exporter(format_type="xlsx", schema_class=IssueExportSchema) filename, content = exporter.export("issues", issues) # Export with custom fields only -issues_data = IssueExportSchema.serialize_issues( - issues, - fields=["id", "name", "state_name", "assignees"] -) exporter = Exporter(format_type="json", schema_class=IssueExportSchema) -filename, content = exporter.export("issues_filtered", issues_data) +filename, content = exporter.export("issues_filtered", issues, fields=["id", "name", "state_name", "assignees"]) +``` + +### Exporting Multiple Projects Separately + +```python +# Export each project to a separate file +for project_id in project_ids: + project_issues = issues.filter(project_id=project_id) + exporter = Exporter(format_type="csv", schema_class=IssueExportSchema) + filename, content = exporter.export(f"issues-{project_id}", project_issues) + # Save or upload the file ``` ## ๐Ÿ“ Schema Definition @@ -225,9 +232,9 @@ filename, content = exporter.export("data", records) ## ๐Ÿ”ง Advanced Usage -### ๐Ÿ“ฆ Using Context +### ๐Ÿ“ฆ Using Context for Pre-fetched Data -Pass context data to schemas for additional information: +Pass context data to schemas to avoid N+1 queries. Override `get_context_data()` in your schema: ```python class MySchema(ExportSchema): @@ -237,12 +244,16 @@ class MySchema(ExportSchema): attachments_dict = self.context.get("attachments_dict", {}) return len(attachments_dict.get(obj.id, [])) -# Create schema with context -attachments_dict = get_attachments_dict(queryset) -schema = MySchema(context={"attachments_dict": attachments_dict}) + @classmethod + def get_context_data(cls, queryset): + """Pre-fetch all attachments in one query.""" + attachments_dict = get_attachments_dict(queryset) + return {"attachments_dict": attachments_dict} -# Serialize -data = [schema.serialize(obj) for obj in queryset] +# The Exporter automatically uses get_context_data() when serializing +queryset = MyModel.objects.all() +exporter = Exporter(format_type="csv", schema_class=MySchema) +filename, content = exporter.export("data", queryset) ``` ### ๐Ÿ”Œ Registering Custom Formatters @@ -273,25 +284,52 @@ formats = Exporter.get_available_formats() ### ๐Ÿ” Filtering Fields -Use the `serialize_issues` class method pattern to filter fields: +Pass a `fields` parameter to export only specific fields: + +```python +# Export only specific fields +exporter = Exporter(format_type="csv", schema_class=MySchema) +filename, content = exporter.export( + "filtered_data", + queryset, + fields=["id", "name", "email"] +) +``` + +### ๐ŸŽฏ Extending Schemas + +Create extended schemas by inheriting from existing ones and overriding `get_context_data()`: ```python -@classmethod -def serialize_records(cls, queryset, fields=None): - fields_to_extract = set(fields) if fields else set(cls._declared_fields.keys()) - - schema = cls() - records_data = [] - for record in queryset: - record_data = schema.serialize(record) - filtered_data = { - field: record_data.get(field, "") - for field in fields_to_extract - if field in record_data - } - records_data.append(filtered_data) - - return records_data +class ExtendedIssueExportSchema(IssueExportSchema): + custom_field = JSONField(label="Custom Data") + + def prepare_custom_field(self, obj): + # Use pre-fetched data from context + return self.context.get("custom_data", {}).get(obj.id, {}) + + @classmethod + def get_context_data(cls, queryset): + # Get parent context (attachments, etc.) + context = super().get_context_data(queryset) + + # Add your custom pre-fetched data + context["custom_data"] = fetch_custom_data(queryset) + + return context +``` + +### ๐Ÿ’พ Manual Serialization + +If you need to serialize data without exporting, you can use the schema directly: + +```python +# Serialize a queryset to a list of dicts +data = MySchema.serialize_queryset(queryset, fields=["id", "name"]) + +# Or serialize a single object +schema = MySchema() +obj_data = schema.serialize(obj) ``` ## ๐Ÿ’ก Example: IssueExportSchema @@ -299,33 +337,46 @@ def serialize_records(cls, queryset, fields=None): The `IssueExportSchema` demonstrates a complete implementation: ```python -from plane.utils.exporters import IssueExportSchema +from plane.utils.exporters import Exporter, IssueExportSchema + +# Simple export - just pass the queryset! +issues = Issue.objects.filter(project_id=project_id) +exporter = Exporter(format_type="csv", schema_class=IssueExportSchema) +filename, content = exporter.export("issues", issues) -# Serialize issues -issues_data = IssueExportSchema.serialize_issues( - issues_queryset, +# Export specific fields only +filename, content = exporter.export( + "issues_filtered", + issues, fields=["id", "name", "state_name", "assignees", "labels"] ) -# Export as CSV -exporter = Exporter(format_type="csv", schema_class=IssueExportSchema) -filename, content = exporter.export("issues", issues_data) +# Export multiple projects to separate files +for project_id in project_ids: + project_issues = issues.filter(project_id=project_id) + filename, content = exporter.export(f"issues-{project_id}", project_issues) + # Save or upload each file ``` Key features: - ๐Ÿ”— Access to related models via dotted paths - ๐ŸŽฏ Custom preparers for complex fields -- ๐Ÿ“Ž Context-based attachment handling +- ๐Ÿ“Ž Context-based attachment handling via `get_context_data()` - ๐Ÿ“‹ List and JSON field handling - ๐Ÿ“… Date/datetime formatting ## โœจ Best Practices -1. **๐Ÿš„ Prefetch Relations**: Always prefetch related data before exporting to avoid N+1 queries: +1. **๐Ÿš„ Avoid N+1 Queries**: Override `get_context_data()` to pre-fetch related data: ```python - issues = Issue.objects.prefetch_related('assignee_details', 'label_details') + @classmethod + def get_context_data(cls, queryset): + return { + "attachments": get_attachments_dict(queryset), + "comments": get_comments_dict(queryset), + } ``` 2. **๐Ÿท๏ธ Use Labels**: Provide descriptive labels for better export headers: @@ -347,9 +398,30 @@ Key features: return [f"{u.first_name} {u.last_name}" for u in obj.assignee_details] ``` -5. **โšก Context for Expensive Operations**: Use context to pass pre-computed data and avoid redundant queries: +5. **โšก Pass QuerySets Directly**: Let the Exporter handle serialization: + ```python - schema = MySchema(context={"computed_data": precomputed_dict}) + # Good - Exporter handles serialization + exporter.export("data", queryset) + + # Avoid - Manual serialization unless needed + data = MySchema.serialize_queryset(queryset) + exporter.export("data", data) + ``` + +6. **๐Ÿ“ฆ Filter QuerySets, Not Data**: For multiple exports, filter the queryset instead of the serialized data: + + ```python + # Good - efficient, only serializes what's needed + for project_id in project_ids: + project_issues = issues.filter(project_id=project_id) + exporter.export(f"project-{project_id}", project_issues) + + # Avoid - serializes all data upfront + all_data = MySchema.serialize_queryset(issues) + for project_id in project_ids: + project_data = [d for d in all_data if d['project_id'] == project_id] + exporter.export(f"project-{project_id}", project_data) ``` ## ๐Ÿ“š API Reference @@ -362,8 +434,11 @@ Key features: - `schema_class`: Schema class defining fields - `options`: Optional dict of format-specific options -**`export(filename, records)`** +**`export(filename, data, fields=None)`** +- `filename`: Filename without extension +- `data`: Django QuerySet or list of dicts +- `fields`: Optional list of field names to include - Returns: `(filename_with_extension, content)` - `content` is str for CSV/JSON, bytes for XLSX @@ -383,7 +458,18 @@ Key features: **`serialize(obj)`** -- Returns: Dict of serialized field values +- Returns: Dict of serialized field values for a single object + +**`serialize_queryset(queryset, fields=None)`** (class method) + +- `queryset`: QuerySet of objects to serialize +- `fields`: Optional list of field names to include +- Returns: List of dicts with serialized data + +**`get_context_data(queryset)`** (class method) + +- Override to pre-fetch related data for the queryset +- Returns: Dict of context data ### ๐Ÿ”ง ExportField @@ -400,14 +486,19 @@ Base class for all field types. Subclass to create custom field types. ## ๐Ÿงช Testing ```python -# Test schema serialization -schema = MySchema() -result = schema.serialize(obj) -assert result["field_name"] == expected_value - -# Test export +# Test exporting a queryset +queryset = MyModel.objects.all() exporter = Exporter(format_type="json", schema_class=MySchema) -filename, content = exporter.export("test", [{"data": "value"}]) +filename, content = exporter.export("test", queryset) assert filename == "test.json" assert isinstance(content, str) + +# Test with field filtering +filename, content = exporter.export("test", queryset, fields=["id", "name"]) +data = json.loads(content) +assert all(set(item.keys()) == {"id", "name"} for item in data) + +# Test manual serialization +data = MySchema.serialize_queryset(queryset) +assert len(data) == queryset.count() ``` diff --git a/apps/api/plane/utils/exporters/exporter.py b/apps/api/plane/utils/exporters/exporter.py index ee51c60bf2d..e48d2b0190a 100644 --- a/apps/api/plane/utils/exporters/exporter.py +++ b/apps/api/plane/utils/exporters/exporter.py @@ -1,4 +1,6 @@ -from typing import Any, Dict, List, Type +from typing import Any, Dict, List, Type, Union + +from django.db.models import QuerySet from .formatters import CSVFormatter, JSONFormatter, XLSXFormatter @@ -32,17 +34,26 @@ def __init__(self, format_type: str, schema_class: Type, options: Dict[str, Any] def export( self, filename: str, - records: List[dict], + data: Union[QuerySet, List[dict]], + fields: List[str] = None, ) -> tuple[str, str | bytes]: - """Export records using the configured formatter and return (filename, content). + """Export data using the configured formatter and return (filename, content). Args: filename: The filename for the export (without extension) - records: List of data dictionaries + data: Either a Django QuerySet or a list of already-serialized dicts + fields: Optional list of field names to include in export Returns: Tuple of (filename_with_extension, content) """ + # Serialize the queryset if needed + if isinstance(data, QuerySet): + records = self.schema_class.serialize_queryset(data, fields=fields) + else: + # Already serialized data + records = data + return self.formatter.format(filename, records, self.schema_class, self.options) @classmethod diff --git a/apps/api/plane/utils/exporters/schemas/base.py b/apps/api/plane/utils/exporters/schemas/base.py index caef29f48d0..bd968ca92d1 100644 --- a/apps/api/plane/utils/exporters/schemas/base.py +++ b/apps/api/plane/utils/exporters/schemas/base.py @@ -1,6 +1,8 @@ from dataclasses import dataclass, field from typing import Any, Callable, Dict, List, Optional +from django.db.models import QuerySet + @dataclass class ExportField: @@ -168,10 +170,25 @@ def __new__(mcls, name, bases, attrs): class ExportSchema(metaclass=ExportSchemaMeta): + """Base schema for exporting data in various formats. + + Subclasses should define fields as class attributes and can override: + - prepare_ methods for custom field serialization + - get_context_data() class method to pre-fetch related data for the queryset + """ + def __init__(self, context: Optional[Dict[str, Any]] = None) -> None: self.context = context or {} def serialize(self, obj: Any) -> Dict[str, Any]: + """Serialize a single object. + + Args: + obj: The object to serialize + + Returns: + Dictionary of serialized data + """ output: Dict[str, Any] = {} for field_name, export_field in self._declared_fields.items(): # Prefer explicit preparer methods if present @@ -185,3 +202,43 @@ def serialize(self, obj: Any) -> Dict[str, Any]: output[field_name] = export_field.get_value(obj, self.context) return output + + @classmethod + def get_context_data(cls, queryset: QuerySet) -> Dict[str, Any]: + """Get context data for serialization. Override in subclasses to pre-fetch related data. + + Args: + queryset: QuerySet of objects to be serialized + + Returns: + Dictionary of context data to be passed to the schema instance + """ + return {} + + @classmethod + def serialize_queryset(cls, queryset: QuerySet, fields: List[str] = None) -> List[Dict[str, Any]]: + """Serialize a queryset of objects to export data. + + Args: + queryset: QuerySet of objects to serialize + fields: Optional list of field names to include. Defaults to all fields. + + Returns: + List of dictionaries containing serialized data + """ + # Get context data (can be extended by subclasses) + context = cls.get_context_data(queryset) + + # Determine which fields to extract + fields_to_extract = set(fields) if fields else set(cls._declared_fields.keys()) + + # Serialize each object + schema = cls(context=context) + data = [] + for obj in queryset: + obj_data = schema.serialize(obj) + # Filter to only requested fields + filtered_data = {field: obj_data.get(field, "") for field in fields_to_extract if field in obj_data} + data.append(filtered_data) + + return data diff --git a/apps/api/plane/utils/exporters/schemas/issue.py b/apps/api/plane/utils/exporters/schemas/issue.py index b132c6a14d5..64f57772fcb 100644 --- a/apps/api/plane/utils/exporters/schemas/issue.py +++ b/apps/api/plane/utils/exporters/schemas/issue.py @@ -86,6 +86,8 @@ def _format_date(date_obj) -> str: cycle_name = StringField(label="Cycle Name") cycle_start_date = DateField(label="Cycle Start Date") cycle_end_date = DateField(label="Cycle End Date") + parent = StringField(label="Parent") + relations = JSONField(label="Relations") def prepare_id(self, i): return f"{i.project.identifier}-{i.sequence_id}" @@ -100,7 +102,7 @@ def prepare_created_by(self, i): return self._get_created_by(i) def prepare_labels(self, i): - return [label.name for label in i.label_details] + return [label.name for label in i.labels.all()] def prepare_comments(self, i): return [ @@ -119,7 +121,7 @@ def prepare_link(self, i): return [link.url for link in i.issue_link.all()] def prepare_assignees(self, i): - return [f"{u.first_name} {u.last_name}" for u in i.assignee_details] + return [f"{u.first_name} {u.last_name}" for u in i.assignees.all()] def prepare_subscribers_count(self, i): return i.issue_subscribers.count() @@ -142,30 +144,30 @@ def prepare_cycle_start_date(self, i): def prepare_cycle_end_date(self, i): return i.issue_cycle.last().cycle.end_date if i.issue_cycle.last() else None + def prepare_parent(self, i): + if not i.parent: + return "" + return f"{i.parent.project.identifier}-{i.parent.sequence_id}" + + def prepare_relations(self, i): + # Should show reverse relation as well + from plane.db.models.issue import IssueRelationChoices + + relations = { + r.relation_type: f"{r.related_issue.project.identifier}-{r.related_issue.sequence_id}" + for r in i.issue_relation.all() + } + reverse_relations = {} + for relation in i.issue_related.all(): + reverse_relations[IssueRelationChoices._REVERSE_MAPPING[relation.relation_type]] = ( + f"{relation.issue.project.identifier}-{relation.issue.sequence_id}" + ) + relations.update(reverse_relations) + return relations + @classmethod - def serialize_issues(cls, issues_queryset: QuerySet, fields: List[str] = None) -> List[Dict[str, Any]]: - """Serialize a queryset of issues to export data. - - Args: - issues_queryset: Queryset of Issue objects - fields: Optional list of field names to include. Defaults to all fields. - - Returns: - List of dictionaries containing serialized issue data - """ - # Get attachments for all issues - attachments_dict = get_issue_attachments_dict(issues_queryset) - - # Determine which fields to extract - fields_to_extract = set(fields) if fields else set(cls._declared_fields.keys()) - - # Serialize each issue - schema = cls(context={"attachments_dict": attachments_dict}) - issues_data = [] - for issue in issues_queryset: - issue_data = schema.serialize(issue) - # Filter to only requested fields - filtered_data = {field: issue_data.get(field, "") for field in fields_to_extract if field in issue_data} - issues_data.append(filtered_data) - - return issues_data + def get_context_data(cls, queryset: QuerySet) -> Dict[str, Any]: + """Get context data for issue serialization.""" + return { + "attachments_dict": get_issue_attachments_dict(queryset), + } From 284455f6beb9ba8aa66a08fcfc1cb38b6bbc0af4 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Tue, 7 Oct 2025 15:20:01 +0530 Subject: [PATCH 3/4] feat: enhance export functionality with field filtering and context support - Updated Exporter class to merge fields into options for formatting. - Modified formatters to filter fields based on specified options. - Enhanced ExportSchema to support optional field selection during serialization. - Improved documentation for the serialize method to clarify field filtering capabilities. --- apps/api/plane/utils/exporters/README.md | 34 +++++-------- apps/api/plane/utils/exporters/exporter.py | 7 ++- apps/api/plane/utils/exporters/formatters.py | 20 ++++++++ .../api/plane/utils/exporters/schemas/base.py | 48 ++++++++----------- .../plane/utils/exporters/schemas/issue.py | 43 +++++++++++++++-- 5 files changed, 96 insertions(+), 56 deletions(-) diff --git a/apps/api/plane/utils/exporters/README.md b/apps/api/plane/utils/exporters/README.md index 02bcffd7810..cbecaaa4b9f 100644 --- a/apps/api/plane/utils/exporters/README.md +++ b/apps/api/plane/utils/exporters/README.md @@ -133,8 +133,7 @@ comments = JSONField(label="Comments") All field types support these parameters: -- **`source`**: Dotted path to the attribute (e.g., `"project.name"`) or a callable -- **`transform`**: Custom transformation function +- **`source`**: Dotted path string to the attribute (e.g., `"project.name"`) - **`default`**: Default value when field is None - **`label`**: Display name in export headers @@ -161,29 +160,22 @@ class MySchema(ExportSchema): Preparers take precedence over field definitions. -### ๐Ÿ”„ Using Callables +### โšก Custom Transformations with Preparer Methods -Use callable sources for dynamic values: +For any custom logic or transformations, use `prepare_` methods: ```python class MySchema(ExportSchema): - status = StringField( - source=lambda obj: "Active" if obj.is_active else "Inactive", - label="Status" - ) -``` - -### โšก Transform Functions + name = StringField(source="name", label="Name (Uppercase)") + status = StringField(label="Status") -Apply transformations to values: + def prepare_name(self, obj): + """Transform the name field to uppercase.""" + return obj.name.upper() if obj.name else "" -```python -class MySchema(ExportSchema): - name = StringField( - source="name", - transform=lambda val: val.upper(), - label="Name (Uppercase)" - ) + def prepare_status(self, obj): + """Compute status based on model state.""" + return "Active" if obj.is_active else "Inactive" ``` ## ๐Ÿ“ฆ Export Formats @@ -454,9 +446,9 @@ Key features: **`__init__(context=None)`** -- `context`: Optional dict passed to field transformations and preparers +- `context`: Optional dict accessible in preparer methods via `self.context` for pre-fetched data -**`serialize(obj)`** +**`serialize(obj, fields=None)`** - Returns: Dict of serialized field values for a single object diff --git a/apps/api/plane/utils/exporters/exporter.py b/apps/api/plane/utils/exporters/exporter.py index e48d2b0190a..75b396cb4eb 100644 --- a/apps/api/plane/utils/exporters/exporter.py +++ b/apps/api/plane/utils/exporters/exporter.py @@ -54,7 +54,12 @@ def export( # Already serialized data records = data - return self.formatter.format(filename, records, self.schema_class, self.options) + # Merge fields into options for the formatter + format_options = {**self.options} + if fields: + format_options["fields"] = fields + + return self.formatter.format(filename, records, self.schema_class, format_options) @classmethod def get_available_formats(cls) -> List[str]: diff --git a/apps/api/plane/utils/exporters/formatters.py b/apps/api/plane/utils/exporters/formatters.py index c20f535ebbc..fc7c23528b0 100644 --- a/apps/api/plane/utils/exporters/formatters.py +++ b/apps/api/plane/utils/exporters/formatters.py @@ -90,6 +90,13 @@ def format(self, filename, records, schema_class, options: Dict[str, Any] | None # Get field order and labels from schema field_order, field_labels = self._get_field_info(schema_class) + + # Filter to requested fields if specified + opts = options or {} + requested_fields = opts.get("fields") + if requested_fields: + field_order = [f for f in field_order if f in requested_fields] + header = [field_labels[field] for field in field_order] rows = [header] @@ -119,6 +126,12 @@ def format(self, filename, records, schema_class, options: Dict[str, Any] | None # Get field order and labels from schema field_order, field_labels = self._get_field_info(schema_class) + # Filter to requested fields if specified + opts = options or {} + requested_fields = opts.get("fields") + if requested_fields: + field_order = [f for f in field_order if f in requested_fields] + rows: List[dict] = [] for record in records: row = self._generate_json_row(record, field_labels, field_order, options) @@ -169,6 +182,13 @@ def format(self, filename, records, schema_class, options: Dict[str, Any] | None # Get field order and labels from schema field_order, field_labels = self._get_field_info(schema_class) + + # Filter to requested fields if specified + opts = options or {} + requested_fields = opts.get("fields") + if requested_fields: + field_order = [f for f in field_order if f in requested_fields] + header = [field_labels[field] for field in field_order] rows = [header] diff --git a/apps/api/plane/utils/exporters/schemas/base.py b/apps/api/plane/utils/exporters/schemas/base.py index bd968ca92d1..4e67c6980c5 100644 --- a/apps/api/plane/utils/exporters/schemas/base.py +++ b/apps/api/plane/utils/exporters/schemas/base.py @@ -1,5 +1,5 @@ from dataclasses import dataclass, field -from typing import Any, Callable, Dict, List, Optional +from typing import Any, Dict, List, Optional from django.db.models import QuerySet @@ -8,29 +8,17 @@ class ExportField: """Base export field class for generic fields.""" - source: Optional[str | Callable[[Any, Dict[str, Any]], Any]] = None - transform: Optional[Callable[[Any, Dict[str, Any]], Any]] = None + source: Optional[str] = None default: Any = "" label: Optional[str] = None # Display name for export headers def get_value(self, obj: Any, context: Dict[str, Any]) -> Any: raw: Any - if callable(self.source): - try: - raw = self.source(obj, context) - except TypeError: - raw = self.source(obj) - elif isinstance(self.source, str) and self.source: + if self.source: raw = self._resolve_dotted_path(obj, self.source) else: raw = obj - if self.transform is not None: - try: - return self.transform(raw, context) - except TypeError: - return self.transform(raw) - return self._format_value(raw) def _format_value(self, raw: Any) -> Any: @@ -180,24 +168,31 @@ class ExportSchema(metaclass=ExportSchemaMeta): def __init__(self, context: Optional[Dict[str, Any]] = None) -> None: self.context = context or {} - def serialize(self, obj: Any) -> Dict[str, Any]: + def serialize(self, obj: Any, fields: Optional[List[str]] = None) -> Dict[str, Any]: """Serialize a single object. Args: obj: The object to serialize + fields: Optional list of field names to include. If None, all fields are serialized. Returns: Dictionary of serialized data """ output: Dict[str, Any] = {} - for field_name, export_field in self._declared_fields.items(): + # Determine which fields to process + fields_to_process = fields if fields else list(self._declared_fields.keys()) + + for field_name in fields_to_process: + # Skip if field doesn't exist in schema + if field_name not in self._declared_fields: + continue + + export_field = self._declared_fields[field_name] + # Prefer explicit preparer methods if present preparer = getattr(self, f"prepare_{field_name}", None) if callable(preparer): - try: - output[field_name] = preparer(obj) - except TypeError: - output[field_name] = preparer(obj, self.context) + output[field_name] = preparer(obj) continue output[field_name] = export_field.get_value(obj, self.context) @@ -229,16 +224,11 @@ def serialize_queryset(cls, queryset: QuerySet, fields: List[str] = None) -> Lis # Get context data (can be extended by subclasses) context = cls.get_context_data(queryset) - # Determine which fields to extract - fields_to_extract = set(fields) if fields else set(cls._declared_fields.keys()) - - # Serialize each object + # Serialize each object, passing fields to only process requested fields schema = cls(context=context) data = [] for obj in queryset: - obj_data = schema.serialize(obj) - # Filter to only requested fields - filtered_data = {field: obj_data.get(field, "") for field in fields_to_extract if field in obj_data} - data.append(filtered_data) + obj_data = schema.serialize(obj, fields=fields) + data.append(obj_data) return data diff --git a/apps/api/plane/utils/exporters/schemas/issue.py b/apps/api/plane/utils/exporters/schemas/issue.py index 64f57772fcb..98e213c0beb 100644 --- a/apps/api/plane/utils/exporters/schemas/issue.py +++ b/apps/api/plane/utils/exporters/schemas/issue.py @@ -1,9 +1,9 @@ from collections import defaultdict -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional from django.db.models import F, QuerySet -from plane.db.models import FileAsset +from plane.db.models import CycleIssue, FileAsset from .base import ( DateField, @@ -37,6 +37,32 @@ def get_issue_attachments_dict(issues_queryset: QuerySet) -> Dict[str, List[str] return attachment_dict +def get_issue_last_cycles_dict(issues_queryset: QuerySet) -> Dict[str, Optional["CycleIssue"]]: + """Get the last cycle for each issue in the given queryset. + + Args: + issues_queryset: Queryset of Issue objects + + Returns: + Dictionary mapping issue IDs to their last CycleIssue object + """ + # Fetch all cycle issues for the given issues, ordered by created_at descending + # select_related is used to fetch cycle data in the same query + cycle_issues = ( + CycleIssue.objects.filter(issue_id__in=issues_queryset.values_list("id", flat=True)) + .select_related("cycle") + .order_by("issue_id", "-created_at") + ) + + # Keep only the last (most recent) cycle for each issue + last_cycles_dict = {} + for cycle_issue in cycle_issues: + if cycle_issue.issue_id not in last_cycles_dict: + last_cycles_dict[cycle_issue.issue_id] = cycle_issue + + return last_cycles_dict + + class IssueExportSchema(ExportSchema): """Schema for exporting issue data in various formats.""" @@ -136,13 +162,19 @@ def prepare_attachment_links(self, i): ] def prepare_cycle_name(self, i): - return i.issue_cycle.last().cycle.name if i.issue_cycle.last() else "" + cycles_dict = self.context.get("cycles_dict") or {} + last_cycle = cycles_dict.get(i.id) + return last_cycle.cycle.name if last_cycle else "" def prepare_cycle_start_date(self, i): - return i.issue_cycle.last().cycle.start_date if i.issue_cycle.last() else None + cycles_dict = self.context.get("cycles_dict") or {} + last_cycle = cycles_dict.get(i.id) + return last_cycle.cycle.start_date if last_cycle else None def prepare_cycle_end_date(self, i): - return i.issue_cycle.last().cycle.end_date if i.issue_cycle.last() else None + cycles_dict = self.context.get("cycles_dict") or {} + last_cycle = cycles_dict.get(i.id) + return last_cycle.cycle.end_date if last_cycle else None def prepare_parent(self, i): if not i.parent: @@ -170,4 +202,5 @@ def get_context_data(cls, queryset: QuerySet) -> Dict[str, Any]: """Get context data for issue serialization.""" return { "attachments_dict": get_issue_attachments_dict(queryset), + "cycles_dict": get_issue_last_cycles_dict(queryset), } From 697ba8ae2cf1fe937c71e8168aa5947f64223e89 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Tue, 7 Oct 2025 17:34:03 +0530 Subject: [PATCH 4/4] fixed type --- apps/api/plane/utils/exporters/schemas/issue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/plane/utils/exporters/schemas/issue.py b/apps/api/plane/utils/exporters/schemas/issue.py index 98e213c0beb..ae7ee8535c7 100644 --- a/apps/api/plane/utils/exporters/schemas/issue.py +++ b/apps/api/plane/utils/exporters/schemas/issue.py @@ -37,7 +37,7 @@ def get_issue_attachments_dict(issues_queryset: QuerySet) -> Dict[str, List[str] return attachment_dict -def get_issue_last_cycles_dict(issues_queryset: QuerySet) -> Dict[str, Optional["CycleIssue"]]: +def get_issue_last_cycles_dict(issues_queryset: QuerySet) -> Dict[str, Optional[CycleIssue]]: """Get the last cycle for each issue in the given queryset. Args: