From 837320dce9b85dc8a2d7b0ba2420bbcc0967e88d Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Fri, 22 Nov 2024 15:50:19 +0530 Subject: [PATCH] - Change batch size to 50 for inserting issues - Fallback to server when mentions filter is used - Split load workspace into multiple transactions --- web/core/local-db/storage.sqlite.ts | 6 +++--- web/core/local-db/utils/load-issues.ts | 8 +++++--- web/core/local-db/utils/load-workspace.ts | 16 ++++++++++++++++ web/core/services/issue/issue.service.ts | 5 ++--- web/helpers/issue.helper.ts | 15 +++++++++++++++ 5 files changed, 41 insertions(+), 9 deletions(-) diff --git a/web/core/local-db/storage.sqlite.ts b/web/core/local-db/storage.sqlite.ts index a6a9889c5c3..a1170c87600 100644 --- a/web/core/local-db/storage.sqlite.ts +++ b/web/core/local-db/storage.sqlite.ts @@ -21,7 +21,7 @@ import { clearOPFS, getGroupedIssueResults, getSubGroupedIssueResults, log, logE const DB_VERSION = 1; const PAGE_SIZE = 500; -const BATCH_SIZE = 500; +const BATCH_SIZE = 50; type TProjectStatus = { issues: { status: undefined | "loading" | "ready" | "error" | "syncing"; sync: Promise | undefined }; @@ -382,10 +382,10 @@ export class Storage { getIssue = async (issueId: string) => { try { - if (!rootStore.user.localDBEnabled) return; + if (!rootStore.user.localDBEnabled || this.status !== "ready") return; const issues = await runQuery(`select * from issues where id='${issueId}'`); - if (issues.length) { + if (Array.isArray(issues) && issues.length) { return formatLocalIssue(issues[0]); } } catch (err) { diff --git a/web/core/local-db/utils/load-issues.ts b/web/core/local-db/utils/load-issues.ts index a16f6bd0ca7..444c07894cd 100644 --- a/web/core/local-db/utils/load-issues.ts +++ b/web/core/local-db/utils/load-issues.ts @@ -15,7 +15,7 @@ export const addIssue = async (issue: any) => { await persistence.db.exec("COMMIT;"); }; -export const addIssuesBulk = async (issues: any, batchSize = 100) => { +export const addIssuesBulk = async (issues: any, batchSize = 50) => { if (!rootStore.user.localDBEnabled || !persistence.db) return; if (!issues.length) return; const insertStart = performance.now(); @@ -24,18 +24,20 @@ export const addIssuesBulk = async (issues: any, batchSize = 100) => { for (let i = 0; i < issues.length; i += batchSize) { const batch = issues.slice(i, i + batchSize); + const promises = []; for (let j = 0; j < batch.length; j++) { const issue = batch[j]; if (!issue.type_id) { issue.type_id = ""; } - await stageIssueInserts(issue); + promises.push(stageIssueInserts(issue)); } + await Promise.all(promises); } await persistence.db.exec("COMMIT;"); const insertEnd = performance.now(); - log("Inserted issues in ", `${insertEnd - insertStart}ms`); + log("Inserted issues in ", `${insertEnd - insertStart}ms`, batchSize, issues.length); }; export const deleteIssueFromLocal = async (issue_id: any) => { if (!rootStore.user.localDBEnabled || !persistence.db) return; diff --git a/web/core/local-db/utils/load-workspace.ts b/web/core/local-db/utils/load-workspace.ts index b6330832689..e9198a48990 100644 --- a/web/core/local-db/utils/load-workspace.ts +++ b/web/core/local-db/utils/load-workspace.ts @@ -120,12 +120,28 @@ export const loadWorkSpaceData = async (workspaceSlug: string) => { const start = performance.now(); await persistence.db.exec("BEGIN;"); await batchInserts(labels, "labels", labelSchema); + await persistence.db.exec("COMMIT;"); + + await persistence.db.exec("BEGIN;"); await batchInserts(modules, "modules", moduleSchema); + await persistence.db.exec("COMMIT;"); + + await persistence.db.exec("BEGIN;"); await batchInserts(cycles, "cycles", cycleSchema); + await persistence.db.exec("COMMIT;"); + + await persistence.db.exec("BEGIN;"); await batchInserts(states, "states", stateSchema); + await persistence.db.exec("COMMIT;"); + + await persistence.db.exec("BEGIN;"); await batchInserts(estimates, "estimate_points", estimatePointSchema); + await persistence.db.exec("COMMIT;"); + + await persistence.db.exec("BEGIN;"); await batchInserts(members, "members", memberSchema); await persistence.db.exec("COMMIT;"); + const end = performance.now(); log("Time taken to load workspace data", end - start); }; diff --git a/web/core/services/issue/issue.service.ts b/web/core/services/issue/issue.service.ts index 2cef113d80f..d74ce0c88ff 100644 --- a/web/core/services/issue/issue.service.ts +++ b/web/core/services/issue/issue.service.ts @@ -1,5 +1,3 @@ -import { startSpan } from "@sentry/nextjs"; -import isEmpty from "lodash/isEmpty"; // types import type { IIssueDisplayProperties, @@ -12,6 +10,7 @@ import type { } from "@plane/types"; // helpers import { API_BASE_URL } from "@/helpers/common.helper"; +import { getIssuesShouldFallbackToServer } from "@/helpers/issue.helper"; import { persistence } from "@/local-db/storage.sqlite"; // services @@ -68,7 +67,7 @@ export class IssueService extends APIService { } async getIssues(workspaceSlug: string, projectId: string, queries?: any, config = {}): Promise { - if (!isEmpty(queries.expand as string) && !queries.group_by) + if (getIssuesShouldFallbackToServer(queries)) return await this.getIssuesFromServer(workspaceSlug, projectId, queries, config); const response = await persistence.getIssues(workspaceSlug, projectId, queries, config); diff --git a/web/helpers/issue.helper.ts b/web/helpers/issue.helper.ts index d1fc970185e..695d124d049 100644 --- a/web/helpers/issue.helper.ts +++ b/web/helpers/issue.helper.ts @@ -1,4 +1,5 @@ import differenceInCalendarDays from "date-fns/differenceInCalendarDays"; +import isEmpty from "lodash/isEmpty"; import set from "lodash/set"; import { v4 as uuidv4 } from "uuid"; // types @@ -308,3 +309,17 @@ export const getComputedDisplayProperties = ( cycle: displayProperties?.cycle ?? true, issue_type: displayProperties?.issue_type ?? true, }); + +/** + * This is to check if the issues list api should fall back to server or use local db + * @param queries + * @returns + */ +export const getIssuesShouldFallbackToServer = (queries: any) => { + // If there is expand query and is not grouped then fallback to server + if (!isEmpty(queries.expand as string) && !queries.group_by) return true; + // If query has mentions then fallback to server + if (!isEmpty(queries.mentions)) return true; + + return false; +};