From be8f069fbe1cedd363bdc7d587b11b099359940d Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Thu, 26 Sep 2024 18:55:49 +0530 Subject: [PATCH 1/6] - Handle single quotes in load workspace queries - Add IS null where condition in query utils --- web/core/local-db/utils/load-workspace.ts | 4 ++-- web/core/local-db/utils/query.utils.ts | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/web/core/local-db/utils/load-workspace.ts b/web/core/local-db/utils/load-workspace.ts index 36d2aabce95..87231161c6f 100644 --- a/web/core/local-db/utils/load-workspace.ts +++ b/web/core/local-db/utils/load-workspace.ts @@ -27,10 +27,10 @@ const stageInserts = (table: string, schema: Schema, data: any) => { return ""; } if (typeof value === "object") { - return `'${JSON.stringify(value)}'`; + return `'${JSON.stringify(value).replace(/'/g, "''")}'`; } if (typeof value === "string") { - return `'${value}'`; + return `'${value.replace(/'/g, "''")}'`; } return value; }) diff --git a/web/core/local-db/utils/query.utils.ts b/web/core/local-db/utils/query.utils.ts index e95174bc7fa..eaaf11a1c87 100644 --- a/web/core/local-db/utils/query.utils.ts +++ b/web/core/local-db/utils/query.utils.ts @@ -238,7 +238,10 @@ export const singleFilterConstructor = (queries: any) => { keys.forEach((key) => { const value = filters[key] ? filters[key].split(",") : ""; - if (!value) return; + if (!value) { + sql += ` AND ${key} IS NULL`; + return; + } if (!ARRAY_FIELDS.includes(key)) { sql += ` AND ${key} in ('${value.join("','")}') `; From 215889c9dfd96ced873b65f134a21ea64543ef79 Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Thu, 26 Sep 2024 19:09:43 +0530 Subject: [PATCH 2/6] Fix description_html being lost --- web/core/local-db/utils/utils.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/web/core/local-db/utils/utils.ts b/web/core/local-db/utils/utils.ts index 619ec9af8d5..f1e05274e82 100644 --- a/web/core/local-db/utils/utils.ts +++ b/web/core/local-db/utils/utils.ts @@ -47,6 +47,7 @@ export const updatePersistentLayer = async (issueIds: string | string[]) => { "label_ids", "module_ids", "type_id", + "description_html", ]); updateIssue({ ...issuePartial, is_local_update: 1 }); } From 00c48c385d8b5aca34d54776f8e9c31c70b7e40c Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Fri, 27 Sep 2024 08:16:24 +0530 Subject: [PATCH 3/6] Change secondary order to sequence_id --- web/core/local-db/utils/indexes.ts | 1 + web/core/local-db/utils/query.utils.ts | 12 ++++-------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/web/core/local-db/utils/indexes.ts b/web/core/local-db/utils/indexes.ts index aeff6992e9a..214bacb44f5 100644 --- a/web/core/local-db/utils/indexes.ts +++ b/web/core/local-db/utils/indexes.ts @@ -10,6 +10,7 @@ export const createIssueIndexes = async () => { "project_id", "created_by", "cycle_id", + "sequence_id", ]; const promises: Promise[] = []; diff --git a/web/core/local-db/utils/query.utils.ts b/web/core/local-db/utils/query.utils.ts index eaaf11a1c87..ad012a30a90 100644 --- a/web/core/local-db/utils/query.utils.ts +++ b/web/core/local-db/utils/query.utils.ts @@ -47,9 +47,9 @@ export const getOrderByFragment = (order_by: string, table = "") => { if (!order_by) return orderByString; if (order_by.startsWith("-")) { - orderByString += ` ORDER BY ${wrapDateTime(order_by.slice(1))} DESC NULLS LAST, datetime(${table}created_at) DESC`; + orderByString += ` ORDER BY ${wrapDateTime(order_by.slice(1))} DESC NULLS LAST, ${table}sequence_id DESC`; } else { - orderByString += ` ORDER BY ${wrapDateTime(order_by)} ASC NULLS LAST, datetime(${table}created_at) DESC`; + orderByString += ` ORDER BY ${wrapDateTime(order_by)} ASC NULLS LAST, ${table}sequence_id DESC`; } return orderByString; }; @@ -130,7 +130,7 @@ export const getFilteredRowsForGrouping = (projectId: string, queries: any) => { let sql = ""; if (!joinsRequired) { - sql = `WITH fi as (SELECT i.id,i.created_at ${issueTableFilterFields}`; + sql = `WITH fi as (SELECT i.id,i.created_at, i.sequence_id ${issueTableFilterFields}`; if (group_by) { if (group_by === "target_date") { sql += `, date(i.${group_by}) as group_id`; @@ -153,7 +153,7 @@ export const getFilteredRowsForGrouping = (projectId: string, queries: any) => { } sql = `WITH fi AS (`; - sql += `SELECT i.id,i.created_at ${issueTableFilterFields} `; + sql += `SELECT i.id,i.created_at,i.sequence_id ${issueTableFilterFields} `; if (group_by) { if (ARRAY_FIELDS.includes(group_by)) { sql += `, ${group_by}.value as group_id @@ -252,10 +252,6 @@ export const singleFilterConstructor = (queries: any) => { return sql; }; -// let q = '2_months;after;fromnow,1_months;after;fromnow,2024-09-01;after,2024-10-06;after,2_weeks;after;fromnow' - -// ["2_months;after;fromnow", "1_months;after;fromnow", "2024-09-01;after", "2024-10-06;before", "2_weeks;after;fromnow"]; - const createDateFilter = (key: string, q: string) => { let sql = " "; // get todays date in YYYY-MM-DD format From 5cece0f37dce74828a0990d24a970d3d1ef2a83c Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Fri, 27 Sep 2024 08:58:40 +0530 Subject: [PATCH 4/6] Fix update persistence layer --- web/core/local-db/utils/utils.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/web/core/local-db/utils/utils.ts b/web/core/local-db/utils/utils.ts index f1e05274e82..368a19e7fa2 100644 --- a/web/core/local-db/utils/utils.ts +++ b/web/core/local-db/utils/utils.ts @@ -1,6 +1,7 @@ import pick from "lodash/pick"; import { TIssue } from "@plane/types"; import { rootStore } from "@/lib/store-context"; +import { persistence } from "../storage.sqlite"; import { updateIssue } from "./load-issues"; export const log = (...args: any) => { @@ -15,11 +16,13 @@ export const updatePersistentLayer = async (issueIds: string | string[]) => { if (typeof issueIds === "string") { issueIds = [issueIds]; } - issueIds.forEach((issueId) => { + issueIds.forEach(async (issueId) => { + const dbIssue = await persistence.getIssue(issueId); const issue = rootStore.issue.issues.getIssueById(issueId); if (issue) { - const issuePartial = pick(JSON.parse(JSON.stringify(issue)), [ + // JSON.parse(JSON.stringify(issue)) is used to remove the mobx observables + const issuePartial = pick({ ...dbIssue, ...JSON.parse(JSON.stringify(issue)) }, [ "id", "name", "state_id", From 53e9d64abb85a88df9343024347bf890e33449be Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Fri, 27 Sep 2024 11:47:04 +0530 Subject: [PATCH 5/6] Add instrumentation --- web/core/local-db/storage.sqlite.ts | 32 +++++++++++++++++++++--- web/core/local-db/utils/utils.ts | 6 ++++- web/core/services/issue/issue.service.ts | 6 ++++- 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/web/core/local-db/storage.sqlite.ts b/web/core/local-db/storage.sqlite.ts index 70a96942fe4..e66011e4dda 100644 --- a/web/core/local-db/storage.sqlite.ts +++ b/web/core/local-db/storage.sqlite.ts @@ -1,3 +1,4 @@ +import * as Sentry from "@sentry/react"; import set from "lodash/set"; // plane import { EIssueGroupBYServerToProperty } from "@plane/constants"; @@ -15,7 +16,7 @@ import { loadWorkSpaceData } from "./utils/load-workspace"; import { issueFilterCountQueryConstructor, issueFilterQueryConstructor } from "./utils/query-constructor"; import { runQuery } from "./utils/query-executor"; import { createTables } from "./utils/tables"; -import { logError, getGroupedIssueResults, getSubGroupedIssueResults, logInfo, log } from "./utils/utils"; +import { getGroupedIssueResults, getSubGroupedIssueResults, log, logError, logInfo } from "./utils/utils"; declare module "@sqlite.org/sqlite-wasm" { export function sqlite3Worker1Promiser(...args: any): any; @@ -66,7 +67,7 @@ export class Storage { this.reset(); } try { - await this._initialize(workspaceSlug); + await Sentry.startSpan({ name: "INIT_DB" }, async () => await this._initialize(workspaceSlug)); return true; } catch (err) { logError(err); @@ -147,7 +148,9 @@ export class Storage { syncWorkspace = async () => { if (document.hidden || !rootStore.user.localDBEnabled) return; // return if the window gets hidden - loadWorkSpaceData(this.workspaceSlug); + await Sentry.startSpan({ name: "LOAD_WS", attributes: { slug: this.workspaceSlug } }, async () => { + await loadWorkSpaceData(this.workspaceSlug); + }); }; syncProject = async (projectId: string) => { @@ -172,7 +175,7 @@ export class Storage { if (document.hidden || !rootStore.user.localDBEnabled) return false; // return if the window gets hidden try { - const sync = this._syncIssues(projectId); + const sync = Sentry.startSpan({ name: `SYNC_ISSUES` }, () => this._syncIssues(projectId)); this.setSync(projectId, sync); await sync; } catch (e) { @@ -182,6 +185,8 @@ export class Storage { }; _syncIssues = async (projectId: string) => { + const activeSpan = Sentry.getActiveSpan(); + log("### Sync started"); let status = this.getStatus(projectId); if (status === "loading" || status === "syncing") { @@ -241,6 +246,11 @@ export class Storage { this.setOption(projectId, "ready"); this.setStatus(projectId, "ready"); this.setSync(projectId, undefined); + + activeSpan?.setAttributes({ + projectId: projectId, + count: response.total_count, + }); }; getIssueCount = async (projectId: string) => { @@ -317,6 +327,8 @@ export class Storage { issueResults = getGroupedIssueResults(issueResults); } } + const groupCount = group_by ? Object.keys(issueResults).length : undefined; + const subGroupCount = sub_group_by ? Object.keys(issueResults[Object.keys(issueResults)[0]]).length : undefined; const groupingEnd = performance.now(); const times = { @@ -340,6 +352,17 @@ export class Storage { total_pages, }; + const activeSpan = Sentry.getActiveSpan(); + activeSpan?.setAttributes({ + projectId, + count: total_count, + groupBy: group_by, + subGroupBy: sub_group_by, + queries: queries, + local: true, + groupCount, + subGroupCount, + }); return out; }; @@ -352,6 +375,7 @@ export class Storage { return formatLocalIssue(issues[0]); } } catch (err) { + logError(err); console.warn("unable to fetch issue from local db"); } diff --git a/web/core/local-db/utils/utils.ts b/web/core/local-db/utils/utils.ts index 368a19e7fa2..bc9f9a42b2a 100644 --- a/web/core/local-db/utils/utils.ts +++ b/web/core/local-db/utils/utils.ts @@ -1,3 +1,4 @@ +import * as Sentry from "@sentry/react"; import pick from "lodash/pick"; import { TIssue } from "@plane/types"; import { rootStore } from "@/lib/store-context"; @@ -9,7 +10,10 @@ export const log = (...args: any) => { console.log(...args); } }; -export const logError = console.error; +export const logError = (e: any) => { + Sentry.captureException(e); + console.error(e); +}; export const logInfo = console.info; export const updatePersistentLayer = async (issueIds: string | string[]) => { diff --git a/web/core/services/issue/issue.service.ts b/web/core/services/issue/issue.service.ts index 6b57c7bad19..10725de743f 100644 --- a/web/core/services/issue/issue.service.ts +++ b/web/core/services/issue/issue.service.ts @@ -1,3 +1,4 @@ +import * as Sentry from "@sentry/react"; // types import type { IIssueDisplayProperties, @@ -69,7 +70,10 @@ export class IssueService extends APIService { } async getIssues(workspaceSlug: string, projectId: string, queries?: any, config = {}): Promise { - const response = await persistence.getIssues(workspaceSlug, projectId, queries, config); + const response = await Sentry.startSpan({ name: "GET_ISSUES" }, async () => { + const res = await persistence.getIssues(workspaceSlug, projectId, queries, config); + return res; + }); return response as TIssuesResponse; } From 8ae412e6896b924f8a569f02f84c32beb005afd8 Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Mon, 30 Sep 2024 20:29:55 +0530 Subject: [PATCH 6/6] - Fallback to server incase of any error --- web/core/local-db/storage.sqlite.ts | 16 +++++++++++++--- web/core/local-db/utils/utils.ts | 12 ++++++++++-- web/core/services/issue/issue.service.ts | 2 +- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/web/core/local-db/storage.sqlite.ts b/web/core/local-db/storage.sqlite.ts index e66011e4dda..4daa86e16d0 100644 --- a/web/core/local-db/storage.sqlite.ts +++ b/web/core/local-db/storage.sqlite.ts @@ -1,4 +1,4 @@ -import * as Sentry from "@sentry/react"; +import * as Sentry from "@sentry/nextjs"; import set from "lodash/set"; // plane import { EIssueGroupBYServerToProperty } from "@plane/constants"; @@ -288,7 +288,9 @@ export class Storage { currentProjectStatus === "error" || !rootStore.user.localDBEnabled ) { - logInfo(`Project ${projectId} is loading, falling back to server`); + if (rootStore.user.localDBEnabled) { + logInfo(`Project ${projectId} is loading, falling back to server`); + } const issueService = new IssueService(); return await issueService.getIssuesFromServer(workspaceSlug, projectId, queries); } @@ -298,7 +300,15 @@ export class Storage { const query = issueFilterQueryConstructor(this.workspaceSlug, projectId, queries); const countQuery = issueFilterCountQueryConstructor(this.workspaceSlug, projectId, queries); const start = performance.now(); - const [issuesRaw, count] = await Promise.all([runQuery(query), runQuery(countQuery)]); + let issuesRaw: any[] = []; + let count: any[]; + try { + [issuesRaw, count] = await Promise.all([runQuery(query), runQuery(countQuery)]); + } catch (e) { + logError(e); + const issueService = new IssueService(); + return await issueService.getIssuesFromServer(workspaceSlug, projectId, queries); + } // const issuesRaw = await runQuery(query); const end = performance.now(); diff --git a/web/core/local-db/utils/utils.ts b/web/core/local-db/utils/utils.ts index bc9f9a42b2a..fca007d3451 100644 --- a/web/core/local-db/utils/utils.ts +++ b/web/core/local-db/utils/utils.ts @@ -1,4 +1,4 @@ -import * as Sentry from "@sentry/react"; +import * as Sentry from "@sentry/nextjs"; import pick from "lodash/pick"; import { TIssue } from "@plane/types"; import { rootStore } from "@/lib/store-context"; @@ -11,8 +11,11 @@ export const log = (...args: any) => { } }; export const logError = (e: any) => { + if (e?.result?.errorClass === "SQLite3Error") { + e = parseSQLite3Error(e); + } Sentry.captureException(e); - console.error(e); + console.log(e); }; export const logInfo = console.info; @@ -144,3 +147,8 @@ export const getSubGroupedIssueResults = ( }; export const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +const parseSQLite3Error = (error: any) => { + error.result = JSON.stringify(error.result); + return error; +}; diff --git a/web/core/services/issue/issue.service.ts b/web/core/services/issue/issue.service.ts index 10725de743f..107ad9923ba 100644 --- a/web/core/services/issue/issue.service.ts +++ b/web/core/services/issue/issue.service.ts @@ -1,4 +1,4 @@ -import * as Sentry from "@sentry/react"; +import * as Sentry from "@sentry/nextjs"; // types import type { IIssueDisplayProperties,