From 957e981168d2fa5c5ed17cf659da981373f33759 Mon Sep 17 00:00:00 2001 From: rahulramesha Date: Mon, 29 Jul 2024 18:41:46 +0530 Subject: [PATCH 001/111] use common getIssues from issue service instead of multiple different services for modules and cycles --- web/core/store/issue/cycle/filter.store.ts | 7 ++++++- web/core/store/issue/cycle/issue.store.ts | 4 ++-- web/core/store/issue/module/filter.store.ts | 7 ++++++- web/core/store/issue/module/issue.store.ts | 4 ++-- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/web/core/store/issue/cycle/filter.store.ts b/web/core/store/issue/cycle/filter.store.ts index 490393a43a6..2b35ee74632 100644 --- a/web/core/store/issue/cycle/filter.store.ts +++ b/web/core/store/issue/cycle/filter.store.ts @@ -119,7 +119,12 @@ export class CycleIssuesFilter extends IssueFilterHelperStore implements ICycleI groupId: string | undefined, subGroupId: string | undefined ) => { - const filterParams = this.getAppliedFilters(cycleId); + let filterParams = this.getAppliedFilters(cycleId); + + if (!filterParams) { + filterParams = {}; + } + filterParams["cycle"] = cycleId; const paginationParams = this.getPaginationParams(filterParams, options, cursor, groupId, subGroupId); return paginationParams; diff --git a/web/core/store/issue/cycle/issue.store.ts b/web/core/store/issue/cycle/issue.store.ts index 41dacd8a0f5..250e786ae45 100644 --- a/web/core/store/issue/cycle/issue.store.ts +++ b/web/core/store/issue/cycle/issue.store.ts @@ -164,7 +164,7 @@ export class CycleIssues extends BaseIssuesStore implements ICycleIssues { // get params from pagination options const params = this.issueFilterStore?.getFilterParams(options, cycleId, undefined, undefined, undefined); // call the fetch issues API with the params - const response = await this.cycleService.getCycleIssues(workspaceSlug, projectId, cycleId, params, { + const response = await this.issueService.getIssues(workspaceSlug, projectId, params, { signal: this.controller.signal, }); @@ -212,7 +212,7 @@ export class CycleIssues extends BaseIssuesStore implements ICycleIssues { subGroupId ); // call the fetch issues API with the params for next page in issues - const response = await this.cycleService.getCycleIssues(workspaceSlug, projectId, cycleId, params); + const response = await this.issueService.getIssues(workspaceSlug, projectId, cycleId, params); // after the next page of issues are fetched, call the base method to process the response this.onfetchNexIssues(response, groupId, subGroupId); diff --git a/web/core/store/issue/module/filter.store.ts b/web/core/store/issue/module/filter.store.ts index 1e02ba13adf..d64a4c0f015 100644 --- a/web/core/store/issue/module/filter.store.ts +++ b/web/core/store/issue/module/filter.store.ts @@ -119,7 +119,12 @@ export class ModuleIssuesFilter extends IssueFilterHelperStore implements IModul groupId: string | undefined, subGroupId: string | undefined ) => { - const filterParams = this.getAppliedFilters(moduleId); + let filterParams = this.getAppliedFilters(moduleId); + + if (!filterParams) { + filterParams = {}; + } + filterParams["module"] = moduleId; const paginationParams = this.getPaginationParams(filterParams, options, cursor, groupId, subGroupId); return paginationParams; diff --git a/web/core/store/issue/module/issue.store.ts b/web/core/store/issue/module/issue.store.ts index ee78b3d7c44..a69d5ee77cf 100644 --- a/web/core/store/issue/module/issue.store.ts +++ b/web/core/store/issue/module/issue.store.ts @@ -118,7 +118,7 @@ export class ModuleIssues extends BaseIssuesStore implements IModuleIssues { // get params from pagination options const params = this.issueFilterStore?.getFilterParams(options, moduleId, undefined, undefined, undefined); // call the fetch issues API with the params - const response = await this.moduleService.getModuleIssues(workspaceSlug, projectId, moduleId, params, { + const response = await this.issueService.getIssues(workspaceSlug, projectId, params, { signal: this.controller.signal, }); @@ -166,7 +166,7 @@ export class ModuleIssues extends BaseIssuesStore implements IModuleIssues { subGroupId ); // call the fetch issues API with the params for next page in issues - const response = await this.moduleService.getModuleIssues(workspaceSlug, projectId, moduleId, params); + const response = await this.issueService.getIssues(workspaceSlug, projectId, params); // after the next page of issues are fetched, call the base method to process the response this.onfetchNexIssues(response, groupId, subGroupId); From cc0d229c02230594af3cd53b0065054b0e14cf05 Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Wed, 31 Jul 2024 13:45:36 +0530 Subject: [PATCH 002/111] Use SQLite to store issues locally and load issues from it. --- .../layouts/auth-layout/project-wrapper.tsx | 10 ++- web/core/local-db/indexes.ts | 46 ++++++++++ web/core/local-db/load-issues.ts | 85 +++++++++++++++++++ web/core/local-db/queries/issues.ts | 60 +++++++++++++ web/core/local-db/query-constructor.ts | 51 +++++++++++ web/core/local-db/query-executor.ts | 15 ++++ web/core/local-db/sqlite.ts | 81 ++++++++++++++++++ web/core/local-db/tables.ts | 42 +++++++++ web/core/services/issue/issue.service.ts | 12 ++- web/next.config.js | 9 +- web/package.json | 3 +- yarn.lock | 7 +- 12 files changed, 413 insertions(+), 8 deletions(-) create mode 100644 web/core/local-db/indexes.ts create mode 100644 web/core/local-db/load-issues.ts create mode 100644 web/core/local-db/queries/issues.ts create mode 100644 web/core/local-db/query-constructor.ts create mode 100644 web/core/local-db/query-executor.ts create mode 100644 web/core/local-db/sqlite.ts create mode 100644 web/core/local-db/tables.ts diff --git a/web/core/layouts/auth-layout/project-wrapper.tsx b/web/core/layouts/auth-layout/project-wrapper.tsx index 8e3a88eb4cc..969097b1475 100644 --- a/web/core/layouts/auth-layout/project-wrapper.tsx +++ b/web/core/layouts/auth-layout/project-wrapper.tsx @@ -1,4 +1,4 @@ -import { FC, ReactNode } from "react"; +import { FC, ReactNode, useEffect } from "react"; import { observer } from "mobx-react"; import { useParams } from "next/navigation"; import useSWR from "swr"; @@ -20,6 +20,8 @@ import { useCommandPalette, } from "@/hooks/store"; // images +import { loadIssues } from "@/local-db/load-issues"; +import { initializeSQLite } from "@/local-db/sqlite"; import emptyProject from "@/public/empty-state/project.svg"; interface IProjectAuthWrapper { @@ -48,6 +50,12 @@ export const ProjectAuthWrapper: FC = observer((props) => { // router const { workspaceSlug, projectId } = useParams(); + useEffect(() => { + (async () => { + await initializeSQLite(); + await loadIssues(workspaceSlug.toString(), projectId.toString()); + })(); + }, [workspaceSlug, projectId]); // fetching project details useSWR( workspaceSlug && projectId ? `PROJECT_DETAILS_${workspaceSlug.toString()}_${projectId.toString()}` : null, diff --git a/web/core/local-db/indexes.ts b/web/core/local-db/indexes.ts new file mode 100644 index 00000000000..8ace6dd59bc --- /dev/null +++ b/web/core/local-db/indexes.ts @@ -0,0 +1,46 @@ +import { SQL } from "./sqlite"; + +export const createIssueIndexes = async () => { + const columns = [ + "state_id", + "sort_order", + // "priority", + "project_id", + "created_by", + "cycle_id", + ]; + + // Drop indexes + const dropPromises = []; + dropPromises.push(SQL.db.exec({ sql: `DROP INDEX IF EXISTS issues_issue_id_idx` })); + dropPromises.push(SQL.db.exec({ sql: `DROP INDEX IF EXISTS issues_priority_idx` })); + columns.forEach((column) => { + dropPromises.push(SQL.db.exec({ sql: `DROP INDEX IF EXISTS issues_issue_${column}_idx` })); + }); + await Promise.all(dropPromises); + + const promises = []; + + promises.push(SQL.db.exec({ sql: `CREATE UNIQUE INDEX issues_issue_id_idx ON issues (id)` })); + + columns.forEach((column) => { + promises.push(SQL.db.exec({ sql: `CREATE INDEX issues_issue_${column}_idx ON issues (project_id, ${column})` })); + }); + await Promise.all(promises); +}; + +export const createIssueMetaIndexes = async () => { + // Drop indexes + await SQL.db.exec({ sql: `DROP INDEX IF EXISTS issue_meta_all_idx` }); + + await SQL.db.exec({ sql: `CREATE INDEX issue_meta_all_idx ON issue_meta (issue_id,key,value)` }); +}; + +const createIndexes = async () => { + console.log("### Creating indexes"); + const promises = [createIssueIndexes(), createIssueMetaIndexes()]; + await Promise.all(promises); + console.log("### Indexes created"); +}; + +export default createIndexes; diff --git a/web/core/local-db/load-issues.ts b/web/core/local-db/load-issues.ts new file mode 100644 index 00000000000..4c6c636700f --- /dev/null +++ b/web/core/local-db/load-issues.ts @@ -0,0 +1,85 @@ +import { TBaseIssue } from "@plane/types"; +import { IssueService } from "@/services/issue"; +import createIndexes from "./indexes"; +import { runQuery } from "./query-executor"; +import { SQL } from "./sqlite"; + +const arrayFields = ["label_ids", "assignee_ids", "module_ids"]; + +export const addIssue = async (issue: any) => { + const issue_id = issue.id; + const { label_ids, assignee_ids, module_ids, ...otherProps } = issue; + const keys = Object.keys(issue).join(","); + const values = Object.values(issue).map((val) => { + if (val === null) { + return ""; + } + if (typeof val === "object") { + return JSON.stringify(val); + } + return val; + }); // Will fail when the values have a comma + + const promises = []; + SQL.db.exec("BEGIN TRANSACTION;"); + + promises.push( + SQL.db.exec({ + sql: `insert into issues(${keys}) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, + bind: values, + }) + ); + + arrayFields.forEach((field) => { + const values = issue[field]; + if (values) { + values.forEach((val) => { + // promises.push( + SQL.db.exec({ + sql: `insert into issue_meta(issue_id,key,value) values (?,?,?)`, + bind: [issue_id, field, val], + }); + // ); + }); + } + }); + SQL.db.exec("COMMIT;"); + + // await Promise.all(promises); + console.log("### Added issue", issue.id); +}; + +export const loadIssues = async (workspaceId: string, projectId: string) => { + // Load issues from the API + const issueService = new IssueService(); + + const PAGE_SIZE = 100; + let cursor = `${PAGE_SIZE}:0:0`; + let results; + let breakLoop = false; + let count = await runQuery(`select count(*) as count from issues where project_id='${projectId}'`); + console.log("### Count", count); + count = count[0]["count"]; + + console.log("### Count", count, typeof count); + if (!count) { + do { + const response = await issueService.getIssuesFromServer(workspaceId, projectId, { cursor }); + cursor = response.next_cursor; + results = response.results as TBaseIssue[]; + console.log("#### Loading issues", results.length); + results.map(async (issue) => { + try { + await addIssue(issue); + } catch (e) { + console.log("###Error", e, issue); + breakLoop = true; + } + }); + } while (results.length >= PAGE_SIZE && !breakLoop); + } else { + console.log("### Project already stored locally, call update issues"); + } + + createIndexes(); +}; diff --git a/web/core/local-db/queries/issues.ts b/web/core/local-db/queries/issues.ts new file mode 100644 index 00000000000..d031fa5d914 --- /dev/null +++ b/web/core/local-db/queries/issues.ts @@ -0,0 +1,60 @@ +import { TIssue } from "@plane/types"; +import { issueFilterCountQueryConstructor, issueFilterQueryConstructor } from "../query-constructor"; +import { runQuery } from "../query-executor"; + +const arrayFields = ["label_ids", "assignee_ids", "module_ids"]; + +export const getIssues = async (workspaceSlug: string, projectId: string, queries?: any, config = {}): Promise => { + // { + // "priority": "urgent,high,medium", + // "state": "5d572a70-aa2f-409c-ade0-6a49e3c052c7,f0c8572a-164a-4ae0-98b4-329e89e22d86", + // "order_by": "-created_at", + // "sub_issue": true, + // "cursor": "100:0:0", + // "per_page": "100" + // } + // https://dexie.org/docs/MultiEntry-Index + + // console.log("#### queries", queries); + const { cursor } = queries; + + const start = performance.now(); + const query = issueFilterQueryConstructor(workspaceSlug, projectId, queries); + const countQuery = issueFilterCountQueryConstructor(workspaceSlug, projectId, queries); + let issues = await runQuery(query); + const { total_count } = (await runQuery(countQuery))[0]; + const end = performance.now(); + + console.log("#### Local time", end - start); + + const [pageSize, page, offset] = cursor.split(":"); + + issues = issues.map((issue: TIssue) => { + arrayFields.forEach((field) => { + issue[field] = JSON.parse(issue[field]); + }); + + return issue; + }); + + const out = { + results: issues, + next_cursor: `${pageSize}:${page}:${Number(offset) + Number(pageSize)}`, + prev_cursor: `${pageSize}:${page}:${Number(offset) - Number(pageSize)}`, + total_results: total_count, + count: issues.length, + total_count, + total_pages: Math.ceil(total_count / Number(pageSize)), + }; + + console.log(out); + return { + results: issues, + next_cursor: `${pageSize}:${page}:${Number(offset) + Number(pageSize)}`, + prev_cursor: `${pageSize}:${page}:${Number(offset) - Number(pageSize)}`, + total_results: total_count, + count: issues.length, + total_count, + total_pages: Math.ceil(total_count / Number(pageSize)), + }; +}; diff --git a/web/core/local-db/query-constructor.ts b/web/core/local-db/query-constructor.ts new file mode 100644 index 00000000000..46e53971e1f --- /dev/null +++ b/web/core/local-db/query-constructor.ts @@ -0,0 +1,51 @@ +export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: string, queries) => { + const { order_by, cursor, per_page, labels, sub_issue, state, cycle, group_by, module, ...otherProps } = queries; + + if (state) otherProps.state_id = state; + if (cycle) otherProps.cycle_id = cycle; + if (module) otherProps.module_ids = module; + if (labels) otherProps.label_ids = labels; + + const arrayFields = ["label_ids", "assignee_ids", "module_ids"]; + let sql = `SELECT * FROM issues i LEFT JOIN issue_meta im ON i.id = im.issue_id WHERE 1=1 AND project_id='${projectId}' `; + + const keys = Object.keys(otherProps); + + keys.forEach((key) => { + const value = otherProps[key] ? otherProps[key].split(",") : ""; + if (!value) return; + if (arrayFields.includes(key)) { + sql += ` AND key='${key}' AND value IN ('${value.join("','")}')`; + } else { + sql += ` AND ${key} in ('${value.join("','")}')`; + } + }); + + sql += ` group by i.id`; + + if (order_by) { + //if order_by starts with "-" then sort in descending order + if (order_by.startsWith("-")) { + sql += ` ORDER BY ${order_by.slice(1)} DESC`; + } else { + sql += ` ORDER BY ${order_by} ASC`; + } + } + + const [pageSize, page, offset] = cursor.split(":"); + // Add offset and paging to query + sql += ` LIMIT ${pageSize} OFFSET ${offset * 1 + page * pageSize};`; + + console.log("$$$", sql); + return sql; +}; + +export const issueFilterCountQueryConstructor = (workspaceSlug: string, projectId: string, queries) => { + let sql = issueFilterQueryConstructor(workspaceSlug, projectId, queries); + + sql = sql.replace("SELECT *", "SELECT COUNT(*) as total_count"); + // Remove everything after group by i.id + sql = `${sql.split("group by i.id")[0]} group by i.id`; + + return sql; +}; diff --git a/web/core/local-db/query-executor.ts b/web/core/local-db/query-executor.ts new file mode 100644 index 00000000000..81c69df80cc --- /dev/null +++ b/web/core/local-db/query-executor.ts @@ -0,0 +1,15 @@ +import { SQL } from "./sqlite"; + +export const runQuery = async (sql) => { + const data = await SQL.db.exec({ + // sql: `SELECT name, '[' || GROUP_CONCAT('"' || value || '"') || ']' AS label_ids FROM issues LEFT JOIN issue_meta ON issues.id = issue_meta.issue_id WHERE key='label_ids' AND value IN ('2e2a79d1-241a-4708-bf95-95eefa33a501','67970b7d-0525-48fa-9f7e-10fc086f5122')`, + // sql: `SELECT * FROM issues LEFT JOIN issue_meta ON issues.id = issue_meta.issue_id WHERE key='label_ids' AND value IN ('2e2a79d1-241a-4708-bf95-95eefa33a501','67970b7d-0525-48fa-9f7e-10fc086f5122')`, + sql, + + // key=label_ids AND value in ('2e2a79d1-241a-4708-bf95-95eefa33a501','67970b7d-0525-48fa-9f7e-10fc086f5122')`, + rowMode: "object", + returnValue: "resultRows", + }); + + return data.result.resultRows; +}; diff --git a/web/core/local-db/sqlite.ts b/web/core/local-db/sqlite.ts new file mode 100644 index 00000000000..3403a066051 --- /dev/null +++ b/web/core/local-db/sqlite.ts @@ -0,0 +1,81 @@ +// import sqlite3InitModule from "@sqlite.org/sqlite-wasm"; +import { sqlite3Worker1Promiser } from "@sqlite.org/sqlite-wasm"; +import { createTables } from "./tables"; + +declare module "@sqlite.org/sqlite-wasm" { + export function sqlite3Worker1Promiser(...args: any): any; +} + +const log = console.log; +const error = console.error; + +const SQL = {}; +const start = async (sqlite3: any) => { + log("Running SQLite3 version", sqlite3.version.libVersion); + SQL.db = new sqlite3.oo1.DB("/mydb.sqlite3", "ct"); + createTables(SQL.db); +}; + +const initializeSQLiteMemory = async () => { + try { + log("Loading and initializing SQLite3 module..."); + const sqlite3 = await sqlite3InitModule({ + print: log, + printErr: error, + }); + log("Done initializing. Running demo..."); + await start(sqlite3); + } catch (err) { + error("Initialization error:", err.name, err.message); + } +}; + +const initializeSQLite = async () => { + if (SQL.db) { + console.info("Instance already initialized"); + return; + } + try { + log("Loading and initializing SQLite3 module..."); + + const promiser = await new Promise((resolve) => { + const _promiser = sqlite3Worker1Promiser({ + onready: () => resolve(_promiser), + }); + }); + + log("Done initializing. Running demo..."); + + const configResponse = await promiser("config-get", {}); + log("Running SQLite3 version", configResponse.result.version.libVersion); + + const openResponse = await promiser("open", { + filename: "file:mydb.sqlite3?vfs=opfs", + }); + const { dbId } = openResponse; + SQL.db = { + dbId, + exec: async (val) => { + if (typeof val === "string") { + val = { sql: val }; + } + return promiser("exec", { dbId, ...val }); + }, + }; + log( + "OPFS is available, created persisted database at", + openResponse.result.filename.replace(/^file:(.*?)\?vfs=opfs$/, "$1") + ); + // Your SQLite code here. + await createTables(SQL.db); + } catch (err) { + if (!(err instanceof Error)) { + err = new Error(err.result.message); + } + error(err.name, err.message); + } +}; + +// initializeSQLite(); + +export { SQL, initializeSQLite }; diff --git a/web/core/local-db/tables.ts b/web/core/local-db/tables.ts new file mode 100644 index 00000000000..b147f80abc5 --- /dev/null +++ b/web/core/local-db/tables.ts @@ -0,0 +1,42 @@ +export const createIssuesTable = (SQLITE) => { + const sqlstr = `CREATE TABLE IF NOT EXISTS issues ( + id TEXT, + name TEXT, + state_id TEXT, + sort_order REAL, + completed_at TEXT, + estimate_point REAL, + priority TEXT, + start_date TEXT, + target_date TEXT, + sequence_id INTEGER, + project_id TEXT, + parent_id TEXT, + created_at TEXT, + updated_at TEXT, + created_by TEXT, + updated_by TEXT, + is_draft INTEGER, + archived_at TEXT, + state__group TEXT, + sub_issues_count INTEGER, + cycle_id TEXT, + link_count INTEGER, + attachment_count INTEGER, + label_ids TEXT, + assignee_ids TEXT, + module_ids TEXT);`; + SQLITE.exec(sqlstr); +}; + +export const createIssueMetaTable = (SQLITE) => { + const sqlstr = `CREATE TABLE IF NOT EXISTS issue_meta ( + issue_id TEXT, + key TEXT, + value TEXT);`; + SQLITE.exec(sqlstr); +}; +export const createTables = async (SQLITE) => { + await createIssuesTable(SQLITE); + await createIssueMetaTable(SQLITE); +}; diff --git a/web/core/services/issue/issue.service.ts b/web/core/services/issue/issue.service.ts index 69ef1ed5c3a..01e1808d034 100644 --- a/web/core/services/issue/issue.service.ts +++ b/web/core/services/issue/issue.service.ts @@ -11,6 +11,7 @@ import type { // helpers import { API_BASE_URL } from "@/helpers/common.helper"; // services +import { getIssues } from "@/local-db/queries/issues"; import { APIService } from "@/services/api.service"; export class IssueService extends APIService { @@ -26,7 +27,12 @@ export class IssueService extends APIService { }); } - async getIssues(workspaceSlug: string, projectId: string, queries?: any, config = {}): Promise { + async getIssuesFromServer( + workspaceSlug: string, + projectId: string, + queries?: any, + config = {} + ): Promise { return this.get( `/api/workspaces/${workspaceSlug}/projects/${projectId}/issues/`, { @@ -40,6 +46,10 @@ export class IssueService extends APIService { }); } + async getIssues(workspaceSlug: string, projectId: string, queries?: any, config = {}): Promise { + return await getIssues(workspaceSlug, projectId, queries, config); + } + async getIssuesWithParams( workspaceSlug: string, projectId: string, diff --git a/web/next.config.js b/web/next.config.js index ad2914df0d7..25f7136e02c 100644 --- a/web/next.config.js +++ b/web/next.config.js @@ -18,6 +18,8 @@ const nextConfig = { key: "Referrer-Policy", value: "origin-when-cross-origin", }, + { key: "Cross-Origin-Opener-Policy", value: "same-origin" }, + { key: "Cross-Origin-Embedder-Policy", value: "require-corp" }, ], }, ]; @@ -56,7 +58,7 @@ const nextConfig = { ]; }, async rewrites() { - const posthogHost = process.env.NEXT_PUBLIC_POSTHOG_HOST || "https://app.posthog.com" + const posthogHost = process.env.NEXT_PUBLIC_POSTHOG_HOST || "https://app.posthog.com"; const rewrites = [ { source: "/ingest/static/:path*", @@ -74,7 +76,7 @@ const nextConfig = { rewrites.push({ source: "/god-mode", destination: `${GOD_MODE_BASE_URL}/`, - }) + }); rewrites.push({ source: "/god-mode/:path*", destination: `${GOD_MODE_BASE_URL}/:path*`, @@ -113,8 +115,7 @@ const sentryConfig = { // https://docs.sentry.io/product/crons/ // https://vercel.com/docs/cron-jobs automaticVercelMonitors: true, -} - +}; if (parseInt(process.env.SENTRY_MONITORING_ENABLED || "0", 10)) { module.exports = withSentryConfig(nextConfig, sentryConfig); diff --git a/web/package.json b/web/package.json index 2e4d74aefd9..2c5f9a06910 100644 --- a/web/package.json +++ b/web/package.json @@ -24,12 +24,13 @@ "@nivo/line": "0.80.0", "@nivo/pie": "0.80.0", "@nivo/scatterplot": "0.80.0", + "@plane/constants": "*", "@plane/editor": "*", "@plane/types": "*", "@plane/ui": "*", - "@plane/constants": "*", "@popperjs/core": "^2.11.8", "@sentry/nextjs": "^8", + "@sqlite.org/sqlite-wasm": "^3.46.0-build2", "axios": "^1.1.3", "clsx": "^2.0.0", "cmdk": "^1.0.0", diff --git a/yarn.lock b/yarn.lock index 4e18b24bfbf..c048ccf6e34 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2910,6 +2910,11 @@ resolved "https://registry.yarnpkg.com/@sindresorhus/merge-streams/-/merge-streams-2.3.0.tgz#719df7fb41766bc143369eaa0dd56d8dc87c9958" integrity sha512-LtoMMhxAlorcGhmFYI+LhPgbPZCkgP6ra1YL604EeF6U98pLlQ3iWIGMdWSC+vWmPBWBNgmDBAhnAobLROJmwg== +"@sqlite.org/sqlite-wasm@^3.46.0-build2": + version "3.46.0-build2" + resolved "https://registry.yarnpkg.com/@sqlite.org/sqlite-wasm/-/sqlite-wasm-3.46.0-build2.tgz#f84c3014f3fed6db08fc585d67e386d39e3956bf" + integrity sha512-10s/u/Main1RGO+jjzK+mgC/zh1ls1CEnq3Dujr03TwvzLg+j4FAohOmlYkQj8KQOj1vGR9cuB9F8tVBTwVGVA== + "@storybook/addon-actions@8.1.6": version "8.1.6" resolved "https://registry.yarnpkg.com/@storybook/addon-actions/-/addon-actions-8.1.6.tgz#ac05b295ee88ddba1f5a96499438d997d761392d" @@ -4446,7 +4451,7 @@ dependencies: "@types/react" "*" -"@types/react@*", "@types/react@18.2.48", "@types/react@^16.8.0 || ^17.0.0 || ^18.0.0", "@types/react@^18.2.42", "@types/react@^18.2.48": +"@types/react@*", "@types/react@^16.8.0 || ^17.0.0 || ^18.0.0", "@types/react@^18.2.42", "@types/react@^18.2.48": version "18.2.48" resolved "https://registry.yarnpkg.com/@types/react/-/react-18.2.48.tgz#11df5664642d0bd879c1f58bc1d37205b064e8f1" integrity sha512-qboRCl6Ie70DQQG9hhNREz81jqC1cs9EVNcjQ1AU+jH6NFfSAhVVbrrY/+nSF+Bsk4AOwm9Qa61InvMCyV+H3w== From 915757c17af4adb5edf86e6e622c6fb88c9d45e1 Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Wed, 31 Jul 2024 16:14:36 +0530 Subject: [PATCH 003/111] Fix incorrect total count and filtering on assignees. --- web/core/local-db/queries/issues.ts | 29 +++++++++++++------------- web/core/local-db/query-constructor.ts | 8 ++++--- web/core/local-db/sqlite.ts | 5 +++-- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/web/core/local-db/queries/issues.ts b/web/core/local-db/queries/issues.ts index d031fa5d914..9ebee9bc3bc 100644 --- a/web/core/local-db/queries/issues.ts +++ b/web/core/local-db/queries/issues.ts @@ -5,30 +5,21 @@ import { runQuery } from "../query-executor"; const arrayFields = ["label_ids", "assignee_ids", "module_ids"]; export const getIssues = async (workspaceSlug: string, projectId: string, queries?: any, config = {}): Promise => { - // { - // "priority": "urgent,high,medium", - // "state": "5d572a70-aa2f-409c-ade0-6a49e3c052c7,f0c8572a-164a-4ae0-98b4-329e89e22d86", - // "order_by": "-created_at", - // "sub_issue": true, - // "cursor": "100:0:0", - // "per_page": "100" - // } - // https://dexie.org/docs/MultiEntry-Index - - // console.log("#### queries", queries); const { cursor } = queries; - const start = performance.now(); const query = issueFilterQueryConstructor(workspaceSlug, projectId, queries); const countQuery = issueFilterCountQueryConstructor(workspaceSlug, projectId, queries); + const start = performance.now(); let issues = await runQuery(query); - const { total_count } = (await runQuery(countQuery))[0]; const end = performance.now(); - console.log("#### Local time", end - start); + const countStart = performance.now(); + const { total_count } = (await runQuery(countQuery))[0]; + const countEnd = performance.now(); const [pageSize, page, offset] = cursor.split(":"); + const parsingStart = performance.now(); issues = issues.map((issue: TIssue) => { arrayFields.forEach((field) => { issue[field] = JSON.parse(issue[field]); @@ -37,6 +28,16 @@ export const getIssues = async (workspaceSlug: string, projectId: string, querie return issue; }); + const parsingEnd = performance.now(); + + const times = { + IssueQuery: end - start, + Count: countEnd - countStart, + Parsing: parsingEnd - parsingStart, + }; + + console.table(times); + const out = { results: issues, next_cursor: `${pageSize}:${page}:${Number(offset) + Number(pageSize)}`, diff --git a/web/core/local-db/query-constructor.ts b/web/core/local-db/query-constructor.ts index 46e53971e1f..664e03600cd 100644 --- a/web/core/local-db/query-constructor.ts +++ b/web/core/local-db/query-constructor.ts @@ -1,10 +1,12 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: string, queries) => { - const { order_by, cursor, per_page, labels, sub_issue, state, cycle, group_by, module, ...otherProps } = queries; + const { order_by, cursor, per_page, labels, sub_issue, assignees, state, cycle, group_by, module, ...otherProps } = + queries; if (state) otherProps.state_id = state; if (cycle) otherProps.cycle_id = cycle; if (module) otherProps.module_ids = module; if (labels) otherProps.label_ids = labels; + if (assignees) otherProps.assignee_ids = assignees; const arrayFields = ["label_ids", "assignee_ids", "module_ids"]; let sql = `SELECT * FROM issues i LEFT JOIN issue_meta im ON i.id = im.issue_id WHERE 1=1 AND project_id='${projectId}' `; @@ -43,9 +45,9 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st export const issueFilterCountQueryConstructor = (workspaceSlug: string, projectId: string, queries) => { let sql = issueFilterQueryConstructor(workspaceSlug, projectId, queries); - sql = sql.replace("SELECT *", "SELECT COUNT(*) as total_count"); + sql = sql.replace("SELECT *", "SELECT COUNT(DISTINCT i.id) as total_count"); // Remove everything after group by i.id - sql = `${sql.split("group by i.id")[0]} group by i.id`; + sql = `${sql.split("group by i.id")[0]};`; return sql; }; diff --git a/web/core/local-db/sqlite.ts b/web/core/local-db/sqlite.ts index 3403a066051..fff2173b159 100644 --- a/web/core/local-db/sqlite.ts +++ b/web/core/local-db/sqlite.ts @@ -9,7 +9,7 @@ declare module "@sqlite.org/sqlite-wasm" { const log = console.log; const error = console.error; -const SQL = {}; +const SQL = { initialized: false }; const start = async (sqlite3: any) => { log("Running SQLite3 version", sqlite3.version.libVersion); SQL.db = new sqlite3.oo1.DB("/mydb.sqlite3", "ct"); @@ -31,7 +31,7 @@ const initializeSQLiteMemory = async () => { }; const initializeSQLite = async () => { - if (SQL.db) { + if (SQL.initialized) { console.info("Instance already initialized"); return; } @@ -66,6 +66,7 @@ const initializeSQLite = async () => { "OPFS is available, created persisted database at", openResponse.result.filename.replace(/^file:(.*?)\?vfs=opfs$/, "$1") ); + SQL.initialized = true; // Your SQLite code here. await createTables(SQL.db); } catch (err) { From 0656de2f9be77f972169a18aec54734d022a59fb Mon Sep 17 00:00:00 2001 From: rahulramesha Date: Wed, 31 Jul 2024 18:49:49 +0530 Subject: [PATCH 004/111] enable parallel API calls --- .../issue-layouts/kanban/base-kanban-root.tsx | 4 +- .../issues/issue-layouts/kanban/default.tsx | 2 +- .../kanban/headers/group-by-card.tsx | 15 +++-- .../issue-layouts/kanban/kanban-group.tsx | 6 +- .../issues/issue-layouts/kanban/swimlanes.tsx | 4 +- .../issue-layouts/list/base-list-root.tsx | 42 ++++++------ .../list/headers/group-by-card.tsx | 13 ++-- .../issues/issue-layouts/list/list-group.tsx | 5 +- .../components/issues/issue-layouts/utils.tsx | 3 +- .../layouts/auth-layout/project-wrapper.tsx | 28 +++++--- web/core/local-db/queries/issues.ts | 18 +++-- web/core/store/issue/cycle/issue.store.ts | 64 ++++++++++++++++-- .../store/issue/helpers/base-issues.store.ts | 65 ++++++++++++++++--- .../helpers/issue-filter-helper.store.ts | 2 +- web/core/store/issue/module/issue.store.ts | 64 ++++++++++++++++-- .../store/issue/project-views/issue.store.ts | 61 +++++++++++++++-- web/core/store/issue/project/issue.store.ts | 61 +++++++++++++++-- 17 files changed, 370 insertions(+), 87 deletions(-) diff --git a/web/core/components/issues/issue-layouts/kanban/base-kanban-root.tsx b/web/core/components/issues/issue-layouts/kanban/base-kanban-root.tsx index 7121169ee6c..4ba7a7a21ec 100644 --- a/web/core/components/issues/issue-layouts/kanban/base-kanban-root.tsx +++ b/web/core/components/issues/issue-layouts/kanban/base-kanban-root.tsx @@ -222,7 +222,7 @@ export const BaseKanBanRoot: React.FC = observer((props: IBas const kanbanFilters = issuesFilter?.issueFilters?.kanbanFilters || { group_by: [], sub_group_by: [] }; return ( - + <> = observer((props: IBas - + ); }); diff --git a/web/core/components/issues/issue-layouts/kanban/default.tsx b/web/core/components/issues/issue-layouts/kanban/default.tsx index f28396367cf..368d30c9c47 100644 --- a/web/core/components/issues/issue-layouts/kanban/default.tsx +++ b/web/core/components/issues/issue-layouts/kanban/default.tsx @@ -156,7 +156,7 @@ export const KanBan: React.FC = observer((props) => { column_id={subList.id} icon={subList.icon} title={subList.name} - count={getGroupIssueCount(subList.id, undefined, false) ?? 0} + count={getGroupIssueCount(subList.id, undefined, false)} issuePayload={subList.payload} disableIssueCreation={disableIssueCreation || isGroupByCreatedBy} addIssuesToView={addIssuesToView} diff --git a/web/core/components/issues/issue-layouts/kanban/headers/group-by-card.tsx b/web/core/components/issues/issue-layouts/kanban/headers/group-by-card.tsx index d7b1a04a399..e5602155ce1 100644 --- a/web/core/components/issues/issue-layouts/kanban/headers/group-by-card.tsx +++ b/web/core/components/issues/issue-layouts/kanban/headers/group-by-card.tsx @@ -15,6 +15,7 @@ import { CreateUpdateIssueModal } from "@/components/issues"; // hooks import { useEventTracker } from "@/hooks/store"; import { useIssueStoreType } from "@/hooks/use-issue-layout-store"; +import isNil from "lodash/isNil"; // types interface IHeaderGroupByCard { @@ -23,7 +24,7 @@ interface IHeaderGroupByCard { column_id: string; icon?: React.ReactNode; title: string; - count: number; + count: number | undefined; kanbanFilters: TIssueKanbanFilters; handleKanbanFilters: any; issuePayload: Partial; @@ -123,11 +124,13 @@ export const HeaderGroupByCard: FC = observer((props) => { > {title} -
- {count || 0} -
+ {!isNil(count) && ( +
+ {count || 0} +
+ )} {sub_group_by === null && ( diff --git a/web/core/components/issues/issue-layouts/kanban/kanban-group.tsx b/web/core/components/issues/issue-layouts/kanban/kanban-group.tsx index d272bb949c2..66b9ec772c0 100644 --- a/web/core/components/issues/issue-layouts/kanban/kanban-group.tsx +++ b/web/core/components/issues/issue-layouts/kanban/kanban-group.tsx @@ -28,6 +28,7 @@ import { GroupDragOverlay } from "../group-drag-overlay"; import { TRenderQuickActions } from "../list/list-view-types"; import { GroupDropLocation, getSourceFromDropPayload, getDestinationFromDropPayload, getIssueBlockId } from "../utils"; import { KanbanIssueBlocksList, KanBanQuickAddIssueForm } from "."; +import isNil from "lodash/isNil"; interface IKanbanGroup { groupId: string; @@ -218,7 +219,7 @@ export const KanbanGroup = observer((props: IKanbanGroup) => { ? (groupedIssueIds as TSubGroupedIssues)?.[groupId]?.[sub_group_id] ?? [] : (groupedIssueIds as TGroupedIssues)?.[groupId] ?? []; - const groupIssueCount = getGroupIssueCount(groupId, sub_group_id, false) ?? 0; + const groupIssueCount = getGroupIssueCount(groupId, sub_group_id, false); const nextPageResults = getPaginationData(groupId, sub_group_id)?.nextPageResults; @@ -234,7 +235,8 @@ export const KanbanGroup = observer((props: IKanbanGroup) => { ); - const shouldLoadMore = nextPageResults === undefined ? issueIds?.length < groupIssueCount : !!nextPageResults; + const shouldLoadMore = + nextPageResults === undefined ? isNil(groupIssueCount) || issueIds?.length < groupIssueCount : !!nextPageResults; const canOverlayBeVisible = orderBy !== "sort_order" || isDropDisabled; const shouldOverlayBeVisible = isDraggingOverColumn && canOverlayBeVisible; diff --git a/web/core/components/issues/issue-layouts/kanban/swimlanes.tsx b/web/core/components/issues/issue-layouts/kanban/swimlanes.tsx index 1c50dc26762..d8a232721a9 100644 --- a/web/core/components/issues/issue-layouts/kanban/swimlanes.tsx +++ b/web/core/components/issues/issue-layouts/kanban/swimlanes.tsx @@ -56,9 +56,9 @@ const SubGroupSwimlaneHeader: React.FC = observer( {list && list.length > 0 && list.map((_list: IGroupByColumn) => { - const groupCount = getGroupIssueCount(_list?.id, undefined, false) ?? 0; + const groupCount = getGroupIssueCount(_list?.id, undefined, false); - const subGroupByVisibilityToggle = visibilitySubGroupByGroupCount(groupCount, showEmptyGroup); + const subGroupByVisibilityToggle = visibilitySubGroupByGroupCount(groupCount ?? 0, showEmptyGroup); if (subGroupByVisibilityToggle === false) return <>; diff --git a/web/core/components/issues/issue-layouts/list/base-list-root.tsx b/web/core/components/issues/issue-layouts/list/base-list-root.tsx index 09ddd44d7f2..76e452a9282 100644 --- a/web/core/components/issues/issue-layouts/list/base-list-root.tsx +++ b/web/core/components/issues/issue-layouts/list/base-list-root.tsx @@ -107,27 +107,25 @@ export const BaseListRoot = observer((props: IBaseListRoot) => { ); return ( - -
- -
- +
+ +
); }); diff --git a/web/core/components/issues/issue-layouts/list/headers/group-by-card.tsx b/web/core/components/issues/issue-layouts/list/headers/group-by-card.tsx index 1a87e51b895..97720f0e664 100644 --- a/web/core/components/issues/issue-layouts/list/headers/group-by-card.tsx +++ b/web/core/components/issues/issue-layouts/list/headers/group-by-card.tsx @@ -18,12 +18,13 @@ import { cn } from "@/helpers/common.helper"; import { useEventTracker } from "@/hooks/store"; import { useIssueStoreType } from "@/hooks/use-issue-layout-store"; import { TSelectionHelper } from "@/hooks/use-multiple-select"; +import isNil from "lodash/isNil"; interface IHeaderGroupByCard { groupID: string; icon?: React.ReactNode; title: string; - count: number; + count: number | undefined; issuePayload: Partial; canEditProperties: (projectId: string | undefined) => boolean; toggleListGroup: () => void; @@ -98,7 +99,7 @@ export const HeaderGroupByCard = observer((props: IHeaderGroupByCard) => { )} groupID={groupID} selectionHelpers={selectionHelpers} - disabled={count === 0} + disabled={!count} /> )} @@ -106,10 +107,12 @@ export const HeaderGroupByCard = observer((props: IHeaderGroupByCard) => { {icon ?? } -
+
{title}
-
{count || 0}
+ {!isNil(count) &&
{count || 0}
}
{!disableIssueCreation && diff --git a/web/core/components/issues/issue-layouts/list/list-group.tsx b/web/core/components/issues/issue-layouts/list/list-group.tsx index 0f1b8069458..84be7f0d26b 100644 --- a/web/core/components/issues/issue-layouts/list/list-group.tsx +++ b/web/core/components/issues/issue-layouts/list/list-group.tsx @@ -37,6 +37,7 @@ import { IssueBlocksList } from "./blocks-list"; import { HeaderGroupByCard } from "./headers/group-by-card"; import { TRenderQuickActions } from "./list-view-types"; import { ListQuickAddIssueForm } from "./quick-add-issue-form"; +import isNil from "lodash/isNil"; interface Props { groupIssueIds: string[] | undefined; @@ -98,7 +99,7 @@ export const ListGroup = observer((props: Props) => { const [intersectionElement, setIntersectionElement] = useState(null); - const groupIssueCount = getGroupIssueCount(group.id, undefined, false) ?? 0; + const groupIssueCount = getGroupIssueCount(group.id, undefined, false); const nextPageResults = getPaginationData(group.id, undefined)?.nextPageResults; const isPaginating = !!getIssueLoader(group.id); @@ -106,7 +107,7 @@ export const ListGroup = observer((props: Props) => { const shouldLoadMore = nextPageResults === undefined && groupIssueCount !== undefined && groupIssueIds - ? groupIssueIds.length < groupIssueCount + ? isNil(groupIssueCount) || groupIssueIds.length < groupIssueCount : !!nextPageResults; const loadMore = isPaginating ? ( diff --git a/web/core/components/issues/issue-layouts/utils.tsx b/web/core/components/issues/issue-layouts/utils.tsx index 7840bb46217..1332e8cea78 100644 --- a/web/core/components/issues/issue-layouts/utils.tsx +++ b/web/core/components/issues/issue-layouts/utils.tsx @@ -39,6 +39,7 @@ import { IMemberRootStore } from "@/store/member"; import { IModuleStore } from "@/store/module.store"; import { IProjectStore } from "@/store/project/project.store"; import { IStateStore } from "@/store/state.store"; +import { ALL_ISSUES } from "@plane/constants"; export const HIGHLIGHT_CLASS = "highlight"; export const HIGHLIGHT_WITH_LINE = "highlight-with-line"; @@ -92,7 +93,7 @@ export const getGroupByColumns = ( case "created_by": return getCreatedByColumns(member) as any; default: - if (includeNone) return [{ id: `All Issues`, name: `All Issues`, payload: {}, icon: undefined }]; + if (includeNone) return [{ id: ALL_ISSUES, name: ALL_ISSUES, payload: {}, icon: undefined }]; } }; diff --git a/web/core/layouts/auth-layout/project-wrapper.tsx b/web/core/layouts/auth-layout/project-wrapper.tsx index 969097b1475..a59352154ef 100644 --- a/web/core/layouts/auth-layout/project-wrapper.tsx +++ b/web/core/layouts/auth-layout/project-wrapper.tsx @@ -1,4 +1,4 @@ -import { FC, ReactNode, useEffect } from "react"; +import { FC, ReactNode, useEffect, useState } from "react"; import { observer } from "mobx-react"; import { useParams } from "next/navigation"; import useSWR from "swr"; @@ -34,6 +34,7 @@ export const ProjectAuthWrapper: FC = observer((props) => { // const { fetchInboxes } = useInbox(); const { toggleCreateProjectModal } = useCommandPalette(); const { setTrackElement } = useEventTracker(); + const [areIssuesLoading, setIssuesLoading] = useState(false); const { membership: { fetchUserProjectInfo, projectMemberInfo, hasPermissionToProject }, } = useUser(); @@ -52,8 +53,10 @@ export const ProjectAuthWrapper: FC = observer((props) => { useEffect(() => { (async () => { + setIssuesLoading(true); await initializeSQLite(); await loadIssues(workspaceSlug.toString(), projectId.toString()); + setIssuesLoading(false); })(); }, [workspaceSlug, projectId]); // fetching project details @@ -67,37 +70,37 @@ export const ProjectAuthWrapper: FC = observer((props) => { workspaceSlug && projectId ? () => fetchUserProjectInfo(workspaceSlug.toString(), projectId.toString()) : null ); // fetching project labels - useSWR( + const { isLoading: isLabelsLoading } = useSWR( workspaceSlug && projectId ? `PROJECT_LABELS_${workspaceSlug}_${projectId}` : null, workspaceSlug && projectId ? () => fetchProjectLabels(workspaceSlug.toString(), projectId.toString()) : null, { revalidateIfStale: false, revalidateOnFocus: false } ); // fetching project members - useSWR( + const { isLoading: isMembersLoading } = useSWR( workspaceSlug && projectId ? `PROJECT_MEMBERS_${workspaceSlug}_${projectId}` : null, workspaceSlug && projectId ? () => fetchProjectMembers(workspaceSlug.toString(), projectId.toString()) : null, { revalidateIfStale: false, revalidateOnFocus: false } ); // fetching project states - useSWR( + const { isLoading: isStateLoading } = useSWR( workspaceSlug && projectId ? `PROJECT_STATES_${workspaceSlug}_${projectId}` : null, workspaceSlug && projectId ? () => fetchProjectStates(workspaceSlug.toString(), projectId.toString()) : null, { revalidateIfStale: false, revalidateOnFocus: false } ); // fetching project estimates - useSWR( + const { isLoading: isEstimatesLoading } = useSWR( workspaceSlug && projectId ? `PROJECT_ESTIMATES_${workspaceSlug}_${projectId}` : null, workspaceSlug && projectId ? () => getProjectEstimates(workspaceSlug.toString(), projectId.toString()) : null, { revalidateIfStale: false, revalidateOnFocus: false } ); // fetching project cycles - useSWR( + const { isLoading: isCyclesLoading } = useSWR( workspaceSlug && projectId ? `PROJECT_ALL_CYCLES_${workspaceSlug}_${projectId}` : null, workspaceSlug && projectId ? () => fetchAllCycles(workspaceSlug.toString(), projectId.toString()) : null, { revalidateIfStale: false, revalidateOnFocus: false } ); // fetching project modules - useSWR( + const { isLoading: isModulesLoading } = useSWR( workspaceSlug && projectId ? `PROJECT_MODULES_${workspaceSlug}_${projectId}` : null, workspaceSlug && projectId ? () => fetchModules(workspaceSlug.toString(), projectId.toString()) : null, { revalidateIfStale: false, revalidateOnFocus: false } @@ -110,8 +113,17 @@ export const ProjectAuthWrapper: FC = observer((props) => { ); const projectExists = projectId ? getProjectById(projectId.toString()) : null; + const isLoading = + isLabelsLoading || + isMembersLoading || + isStateLoading || + isEstimatesLoading || + isCyclesLoading || + isModulesLoading || + areIssuesLoading; + // check if the project member apis is loading - if (!projectMemberInfo && projectId && hasPermissionToProject[projectId.toString()] === null) + if ((!projectMemberInfo && projectId && hasPermissionToProject[projectId.toString()] === null) || isLoading) return (
diff --git a/web/core/local-db/queries/issues.ts b/web/core/local-db/queries/issues.ts index 9ebee9bc3bc..c68f503bfaf 100644 --- a/web/core/local-db/queries/issues.ts +++ b/web/core/local-db/queries/issues.ts @@ -7,7 +7,9 @@ const arrayFields = ["label_ids", "assignee_ids", "module_ids"]; export const getIssues = async (workspaceSlug: string, projectId: string, queries?: any, config = {}): Promise => { const { cursor } = queries; + console.log("## queries", queries); const query = issueFilterQueryConstructor(workspaceSlug, projectId, queries); + console.log("SQL", query); const countQuery = issueFilterCountQueryConstructor(workspaceSlug, projectId, queries); const start = performance.now(); let issues = await runQuery(query); @@ -38,6 +40,9 @@ export const getIssues = async (workspaceSlug: string, projectId: string, querie console.table(times); + const total_pages = Math.ceil(total_count / Number(pageSize)); + const next_page_results = total_pages > parseInt(page) + 1; + const out = { results: issues, next_cursor: `${pageSize}:${page}:${Number(offset) + Number(pageSize)}`, @@ -45,17 +50,10 @@ export const getIssues = async (workspaceSlug: string, projectId: string, querie total_results: total_count, count: issues.length, total_count, - total_pages: Math.ceil(total_count / Number(pageSize)), + next_page_results, + total_pages, }; console.log(out); - return { - results: issues, - next_cursor: `${pageSize}:${page}:${Number(offset) + Number(pageSize)}`, - prev_cursor: `${pageSize}:${page}:${Number(offset) - Number(pageSize)}`, - total_results: total_count, - count: issues.length, - total_count, - total_pages: Math.ceil(total_count / Number(pageSize)), - }; + return out; }; diff --git a/web/core/store/issue/cycle/issue.store.ts b/web/core/store/issue/cycle/issue.store.ts index 250e786ae45..fae8fc4b74f 100644 --- a/web/core/store/issue/cycle/issue.store.ts +++ b/web/core/store/issue/cycle/issue.store.ts @@ -153,23 +153,79 @@ export class CycleIssues extends BaseIssuesStore implements ICycleIssues { options: IssuePaginationOptions, cycleId: string, isExistingPaginationOptions: boolean = false + ) => { + try { + const groups = this.getGroups(); + const subGroups = this.getSubGroups(); + + this.clear(!isExistingPaginationOptions); + + if (!groups || (!groups && !subGroups)) + return await this.fetchParallelIssues(workspaceSlug, projectId, loadType, options, cycleId); + + const promises = []; + + for (const group of groups) { + if (!subGroups) { + promises.push(this.fetchParallelIssues(workspaceSlug, projectId, loadType, options, cycleId, group.id)); + continue; + } + + for (const subGroup of subGroups) { + promises.push( + this.fetchParallelIssues(workspaceSlug, projectId, loadType, options, cycleId, group.id, subGroup.id) + ); + } + } + + await Promise.all(promises); + + // fetch parent stats if required, to be handled in the Implemented class + this.fetchParentStats(workspaceSlug, projectId, cycleId); + } catch (error) { + // set loader to undefined if errored out + this.setLoader(undefined); + throw error; + } + }; + + /** + * This method is called to fetch the first issues of pagination + * @param workspaceSlug + * @param projectId + * @param loadType + * @param options + * @returns + */ + fetchParallelIssues = async ( + workspaceSlug: string, + projectId: string, + loadType: TLoader = "init-loader", + options: IssuePaginationOptions, + cycleId: string, + groupId?: string, + subGroupId?: string ) => { try { // set loader and clear store runInAction(() => { - this.setLoader(loadType); + // set Loader + if (groupId || subGroupId) { + this.setLoader("pagination", groupId, subGroupId); + } else { + this.setLoader(loadType); + } }); - this.clear(!isExistingPaginationOptions); // get params from pagination options - const params = this.issueFilterStore?.getFilterParams(options, cycleId, undefined, undefined, undefined); + const params = this.issueFilterStore?.getFilterParams(options, cycleId, undefined, groupId, subGroupId); // call the fetch issues API with the params const response = await this.issueService.getIssues(workspaceSlug, projectId, params, { signal: this.controller.signal, }); // after fetching issues, call the base method to process the response further - this.onfetchIssues(response, options, workspaceSlug, projectId, cycleId); + this.onfetchIssues(response, options, groupId, subGroupId); return response; } catch (error) { // set loader to undefined once errored out diff --git a/web/core/store/issue/helpers/base-issues.store.ts b/web/core/store/issue/helpers/base-issues.store.ts index a50d1585f51..3a644666278 100644 --- a/web/core/store/issue/helpers/base-issues.store.ts +++ b/web/core/store/issue/helpers/base-issues.store.ts @@ -44,6 +44,7 @@ import { getSubGroupIssueKeyActions, } from "./base-issues-utils"; import { IBaseIssueFilterStore } from "./issue-filter-helper.store"; +import { getGroupByColumns } from "@/components/issues/issue-layouts/utils"; // constants // helpers // services @@ -419,6 +420,56 @@ export abstract class BaseIssuesStore implements IBaseIssuesStore { } ); + getGroups = (isWorkspaceLevel = false) => { + if (!this.groupBy || this.groupBy === "target_date") return; + + const { + projectRoot, + cycle, + module: moduleInfo, + label, + state: projectState, + memberRoot, + } = this.rootIssueStore.rootStore; + + return getGroupByColumns( + this.groupBy, + projectRoot.project, + cycle, + moduleInfo, + label, + projectState, + memberRoot, + false, + isWorkspaceLevel + ); + }; + + getSubGroups = (isWorkspaceLevel = false) => { + if (!this.subGroupBy || this.subGroupBy === "target_date") return; + + const { + projectRoot, + cycle, + module: moduleInfo, + label, + state: projectState, + memberRoot, + } = this.rootIssueStore.rootStore; + + return getGroupByColumns( + this.subGroupBy, + projectRoot.project, + cycle, + moduleInfo, + label, + projectState, + memberRoot, + false, + isWorkspaceLevel + ); + }; + /** * Gets the next page cursor based on number of issues currently available * @param groupId groupId for the cursor @@ -451,9 +502,8 @@ export abstract class BaseIssuesStore implements IBaseIssuesStore { onfetchIssues( issuesResponse: TIssuesResponse, options: IssuePaginationOptions, - workspaceSlug: string, - projectId?: string, - id?: string + groupId?: string, + subGroupId?: string ) { // Process the Issue Response to get the following data from it const { issueList, groupedIssues, groupedIssueCount } = this.processIssueResponse(issuesResponse); @@ -463,15 +513,12 @@ export abstract class BaseIssuesStore implements IBaseIssuesStore { // Update all the GroupIds to this Store's groupedIssueIds and update Individual group issue counts runInAction(() => { - this.updateGroupedIssueIds(groupedIssues, groupedIssueCount); - this.loader[getGroupKey()] = undefined; + this.updateGroupedIssueIds(groupedIssues, groupedIssueCount, groupId, subGroupId); + this.loader[getGroupKey(groupId, subGroupId)] = undefined; }); - // fetch parent stats if required, to be handled in the Implemented class - this.fetchParentStats(workspaceSlug, projectId, id); - // store Pagination options for next subsequent calls and data like next cursor etc - this.storePreviousPaginationValues(issuesResponse, options); + this.storePreviousPaginationValues(issuesResponse, options, groupId, subGroupId); } /** diff --git a/web/core/store/issue/helpers/issue-filter-helper.store.ts b/web/core/store/issue/helpers/issue-filter-helper.store.ts index 8ba9e47ea9f..64e6eaa94f9 100644 --- a/web/core/store/issue/helpers/issue-filter-helper.store.ts +++ b/web/core/store/issue/helpers/issue-filter-helper.store.ts @@ -282,7 +282,7 @@ export class IssueFilterHelperStore implements IIssueFilterHelperStore { subGroupId?: string ) { // if cursor exists, use the cursor. If it doesn't exist construct the cursor based on per page count - const pageCursor = cursor ? cursor : groupId ? `${options.perPageCount}:1:0` : `${options.perPageCount}:0:0`; + const pageCursor = cursor ? cursor : `${options.perPageCount}:0:0`; // pagination params const paginationParams: Partial> = { diff --git a/web/core/store/issue/module/issue.store.ts b/web/core/store/issue/module/issue.store.ts index a69d5ee77cf..a44d692b1ae 100644 --- a/web/core/store/issue/module/issue.store.ts +++ b/web/core/store/issue/module/issue.store.ts @@ -107,23 +107,79 @@ export class ModuleIssues extends BaseIssuesStore implements IModuleIssues { options: IssuePaginationOptions, moduleId: string, isExistingPaginationOptions: boolean = false + ) => { + try { + const groups = this.getGroups(); + const subGroups = this.getSubGroups(); + + this.clear(!isExistingPaginationOptions); + + if (!groups || (!groups && !subGroups)) + return await this.fetchParallelIssues(workspaceSlug, projectId, loadType, options, moduleId); + + const promises = []; + + for (const group of groups) { + if (!subGroups) { + promises.push(this.fetchParallelIssues(workspaceSlug, projectId, loadType, options, moduleId, group.id)); + continue; + } + + for (const subGroup of subGroups) { + promises.push( + this.fetchParallelIssues(workspaceSlug, projectId, loadType, options, moduleId, group.id, subGroup.id) + ); + } + } + + await Promise.all(promises); + + // fetch parent stats if required, to be handled in the Implemented class + this.fetchParentStats(workspaceSlug, projectId, moduleId); + } catch (error) { + // set loader to undefined if errored out + this.setLoader(undefined); + throw error; + } + }; + + /** + * This method is called to fetch the first issues of pagination + * @param workspaceSlug + * @param projectId + * @param loadType + * @param options + * @returns + */ + fetchParallelIssues = async ( + workspaceSlug: string, + projectId: string, + loadType: TLoader = "init-loader", + options: IssuePaginationOptions, + moduleId: string, + groupId?: string, + subGroupId?: string ) => { try { // set loader and clear store runInAction(() => { - this.setLoader(loadType); + // set Loader + if (groupId || subGroupId) { + this.setLoader("pagination", groupId, subGroupId); + } else { + this.setLoader(loadType); + } }); - this.clear(!isExistingPaginationOptions); // get params from pagination options - const params = this.issueFilterStore?.getFilterParams(options, moduleId, undefined, undefined, undefined); + const params = this.issueFilterStore?.getFilterParams(options, moduleId, undefined, groupId, subGroupId); // call the fetch issues API with the params const response = await this.issueService.getIssues(workspaceSlug, projectId, params, { signal: this.controller.signal, }); // after fetching issues, call the base method to process the response further - this.onfetchIssues(response, options, workspaceSlug, projectId, moduleId); + this.onfetchIssues(response, options, groupId, subGroupId); return response; } catch (error) { // set loader to undefined once errored out diff --git a/web/core/store/issue/project-views/issue.store.ts b/web/core/store/issue/project-views/issue.store.ts index 07824dcd2ae..64757babfe8 100644 --- a/web/core/store/issue/project-views/issue.store.ts +++ b/web/core/store/issue/project-views/issue.store.ts @@ -85,23 +85,76 @@ export class ProjectViewIssues extends BaseIssuesStore implements IProjectViewIs loadType: TLoader, options: IssuePaginationOptions, isExistingPaginationOptions: boolean = false + ) => { + try { + const groups = this.getGroups(); + const subGroups = this.getSubGroups(); + + this.clear(!isExistingPaginationOptions); + + if (!groups || (!groups && !subGroups)) + return await this.fetchParallelIssues(workspaceSlug, projectId, loadType, options, viewId); + + const promises = []; + + for (const group of groups) { + if (!subGroups) { + promises.push(this.fetchParallelIssues(workspaceSlug, projectId, loadType, options, viewId, group.id)); + continue; + } + + for (const subGroup of subGroups) { + promises.push( + this.fetchParallelIssues(workspaceSlug, projectId, loadType, options, viewId, group.id, subGroup.id) + ); + } + } + + await Promise.all(promises); + } catch (error) { + // set loader to undefined if errored out + this.setLoader(undefined); + throw error; + } + }; + + /** + * This method is called to fetch the first issues of pagination + * @param workspaceSlug + * @param projectId + * @param loadType + * @param options + * @returns + */ + fetchParallelIssues = async ( + workspaceSlug: string, + projectId: string, + loadType: TLoader = "init-loader", + options: IssuePaginationOptions, + viewId: string, + groupId?: string, + subGroupId?: string ) => { try { // set loader and clear store runInAction(() => { - this.setLoader(loadType); + // set Loader + if (groupId || subGroupId) { + this.setLoader("pagination", groupId, subGroupId); + } else { + this.setLoader(loadType); + } }); - this.clear(!isExistingPaginationOptions); // get params from pagination options - const params = this.issueFilterStore?.getFilterParams(options, viewId, undefined, undefined, undefined); + const params = this.issueFilterStore?.getFilterParams(options, viewId, undefined, groupId, subGroupId); // call the fetch issues API with the params const response = await this.issueService.getIssues(workspaceSlug, projectId, params, { signal: this.controller.signal, }); // after fetching issues, call the base method to process the response further - this.onfetchIssues(response, options, workspaceSlug, projectId); + this.onfetchIssues(response, options, groupId, subGroupId); return response; } catch (error) { // set loader to undefined if errored out diff --git a/web/core/store/issue/project/issue.store.ts b/web/core/store/issue/project/issue.store.ts index e1b013e4afc..86f7cf2a26e 100644 --- a/web/core/store/issue/project/issue.store.ts +++ b/web/core/store/issue/project/issue.store.ts @@ -84,23 +84,76 @@ export class ProjectIssues extends BaseIssuesStore implements IProjectIssues { loadType: TLoader = "init-loader", options: IssuePaginationOptions, isExistingPaginationOptions: boolean = false + ) => { + try { + const groups = this.getGroups(); + const subGroups = this.getSubGroups(); + + this.clear(!isExistingPaginationOptions); + + if (!groups || (!groups && !subGroups)) + return await this.fetchParallelIssues(workspaceSlug, projectId, loadType, options); + + const promises = []; + + for (const group of groups) { + if (!subGroups) { + promises.push(this.fetchParallelIssues(workspaceSlug, projectId, loadType, options, group.id)); + continue; + } + + for (const subGroup of subGroups) { + promises.push(this.fetchParallelIssues(workspaceSlug, projectId, loadType, options, group.id, subGroup.id)); + } + } + + await Promise.all(promises); + + // fetch parent stats if required, to be handled in the Implemented class + this.fetchParentStats(workspaceSlug, projectId); + } catch (error) { + // set loader to undefined if errored out + this.setLoader(undefined); + throw error; + } + }; + + /** + * This method is called to fetch the first issues of pagination + * @param workspaceSlug + * @param projectId + * @param loadType + * @param options + * @returns + */ + fetchParallelIssues = async ( + workspaceSlug: string, + projectId: string, + loadType: TLoader = "init-loader", + options: IssuePaginationOptions, + groupId?: string, + subGroupId?: string ) => { try { // set loader and clear store runInAction(() => { - this.setLoader(loadType); + // set Loader + if (groupId || subGroupId) { + this.setLoader("pagination", groupId, subGroupId); + } else { + this.setLoader(loadType); + } }); - this.clear(!isExistingPaginationOptions); // get params from pagination options - const params = this.issueFilterStore?.getFilterParams(options, projectId, undefined, undefined, undefined); + const params = this.issueFilterStore?.getFilterParams(options, projectId, undefined, groupId, subGroupId); // call the fetch issues API with the params const response = await this.issueService.getIssues(workspaceSlug, projectId, params, { signal: this.controller.signal, }); // after fetching issues, call the base method to process the response further - this.onfetchIssues(response, options, workspaceSlug, projectId); + this.onfetchIssues(response, options, groupId, subGroupId); return response; } catch (error) { // set loader to undefined if errored out From f8d78acfeed66154723fe10e6fcf84716ce22ed9 Mon Sep 17 00:00:00 2001 From: rahulramesha Date: Mon, 29 Jul 2024 18:41:46 +0530 Subject: [PATCH 005/111] use common getIssues from issue service instead of multiple different services for modules and cycles --- web/core/store/issue/cycle/filter.store.ts | 7 ++++++- web/core/store/issue/cycle/issue.store.ts | 4 ++-- web/core/store/issue/module/filter.store.ts | 7 ++++++- web/core/store/issue/module/issue.store.ts | 4 ++-- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/web/core/store/issue/cycle/filter.store.ts b/web/core/store/issue/cycle/filter.store.ts index 490393a43a6..2b35ee74632 100644 --- a/web/core/store/issue/cycle/filter.store.ts +++ b/web/core/store/issue/cycle/filter.store.ts @@ -119,7 +119,12 @@ export class CycleIssuesFilter extends IssueFilterHelperStore implements ICycleI groupId: string | undefined, subGroupId: string | undefined ) => { - const filterParams = this.getAppliedFilters(cycleId); + let filterParams = this.getAppliedFilters(cycleId); + + if (!filterParams) { + filterParams = {}; + } + filterParams["cycle"] = cycleId; const paginationParams = this.getPaginationParams(filterParams, options, cursor, groupId, subGroupId); return paginationParams; diff --git a/web/core/store/issue/cycle/issue.store.ts b/web/core/store/issue/cycle/issue.store.ts index 7f6ac53ec21..624f14048ee 100644 --- a/web/core/store/issue/cycle/issue.store.ts +++ b/web/core/store/issue/cycle/issue.store.ts @@ -181,7 +181,7 @@ export class CycleIssues extends BaseIssuesStore implements ICycleIssues { // get params from pagination options const params = this.issueFilterStore?.getFilterParams(options, cycleId, undefined, undefined, undefined); // call the fetch issues API with the params - const response = await this.cycleService.getCycleIssues(workspaceSlug, projectId, cycleId, params, { + const response = await this.issueService.getIssues(workspaceSlug, projectId, params, { signal: this.controller.signal, }); @@ -229,7 +229,7 @@ export class CycleIssues extends BaseIssuesStore implements ICycleIssues { subGroupId ); // call the fetch issues API with the params for next page in issues - const response = await this.cycleService.getCycleIssues(workspaceSlug, projectId, cycleId, params); + const response = await this.issueService.getIssues(workspaceSlug, projectId, cycleId, params); // after the next page of issues are fetched, call the base method to process the response this.onfetchNexIssues(response, groupId, subGroupId); diff --git a/web/core/store/issue/module/filter.store.ts b/web/core/store/issue/module/filter.store.ts index 1e02ba13adf..d64a4c0f015 100644 --- a/web/core/store/issue/module/filter.store.ts +++ b/web/core/store/issue/module/filter.store.ts @@ -119,7 +119,12 @@ export class ModuleIssuesFilter extends IssueFilterHelperStore implements IModul groupId: string | undefined, subGroupId: string | undefined ) => { - const filterParams = this.getAppliedFilters(moduleId); + let filterParams = this.getAppliedFilters(moduleId); + + if (!filterParams) { + filterParams = {}; + } + filterParams["module"] = moduleId; const paginationParams = this.getPaginationParams(filterParams, options, cursor, groupId, subGroupId); return paginationParams; diff --git a/web/core/store/issue/module/issue.store.ts b/web/core/store/issue/module/issue.store.ts index 266c9d66146..76cf6981d40 100644 --- a/web/core/store/issue/module/issue.store.ts +++ b/web/core/store/issue/module/issue.store.ts @@ -138,7 +138,7 @@ export class ModuleIssues extends BaseIssuesStore implements IModuleIssues { // get params from pagination options const params = this.issueFilterStore?.getFilterParams(options, moduleId, undefined, undefined, undefined); // call the fetch issues API with the params - const response = await this.moduleService.getModuleIssues(workspaceSlug, projectId, moduleId, params, { + const response = await this.issueService.getIssues(workspaceSlug, projectId, params, { signal: this.controller.signal, }); @@ -186,7 +186,7 @@ export class ModuleIssues extends BaseIssuesStore implements IModuleIssues { subGroupId ); // call the fetch issues API with the params for next page in issues - const response = await this.moduleService.getModuleIssues(workspaceSlug, projectId, moduleId, params); + const response = await this.issueService.getIssues(workspaceSlug, projectId, params); // after the next page of issues are fetched, call the base method to process the response this.onfetchNexIssues(response, groupId, subGroupId); From 5edfb5cc866ad256db6707d57ebeb328c93e8517 Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Wed, 31 Jul 2024 13:45:36 +0530 Subject: [PATCH 006/111] Use SQLite to store issues locally and load issues from it. --- .../layouts/auth-layout/project-wrapper.tsx | 10 ++- web/core/local-db/indexes.ts | 46 ++++++++++ web/core/local-db/load-issues.ts | 85 +++++++++++++++++++ web/core/local-db/queries/issues.ts | 60 +++++++++++++ web/core/local-db/query-constructor.ts | 51 +++++++++++ web/core/local-db/query-executor.ts | 15 ++++ web/core/local-db/sqlite.ts | 81 ++++++++++++++++++ web/core/local-db/tables.ts | 42 +++++++++ web/core/services/issue/issue.service.ts | 12 ++- web/next.config.js | 2 + web/package.json | 1 + yarn.lock | 7 +- 12 files changed, 409 insertions(+), 3 deletions(-) create mode 100644 web/core/local-db/indexes.ts create mode 100644 web/core/local-db/load-issues.ts create mode 100644 web/core/local-db/queries/issues.ts create mode 100644 web/core/local-db/query-constructor.ts create mode 100644 web/core/local-db/query-executor.ts create mode 100644 web/core/local-db/sqlite.ts create mode 100644 web/core/local-db/tables.ts diff --git a/web/core/layouts/auth-layout/project-wrapper.tsx b/web/core/layouts/auth-layout/project-wrapper.tsx index 8e3a88eb4cc..969097b1475 100644 --- a/web/core/layouts/auth-layout/project-wrapper.tsx +++ b/web/core/layouts/auth-layout/project-wrapper.tsx @@ -1,4 +1,4 @@ -import { FC, ReactNode } from "react"; +import { FC, ReactNode, useEffect } from "react"; import { observer } from "mobx-react"; import { useParams } from "next/navigation"; import useSWR from "swr"; @@ -20,6 +20,8 @@ import { useCommandPalette, } from "@/hooks/store"; // images +import { loadIssues } from "@/local-db/load-issues"; +import { initializeSQLite } from "@/local-db/sqlite"; import emptyProject from "@/public/empty-state/project.svg"; interface IProjectAuthWrapper { @@ -48,6 +50,12 @@ export const ProjectAuthWrapper: FC = observer((props) => { // router const { workspaceSlug, projectId } = useParams(); + useEffect(() => { + (async () => { + await initializeSQLite(); + await loadIssues(workspaceSlug.toString(), projectId.toString()); + })(); + }, [workspaceSlug, projectId]); // fetching project details useSWR( workspaceSlug && projectId ? `PROJECT_DETAILS_${workspaceSlug.toString()}_${projectId.toString()}` : null, diff --git a/web/core/local-db/indexes.ts b/web/core/local-db/indexes.ts new file mode 100644 index 00000000000..8ace6dd59bc --- /dev/null +++ b/web/core/local-db/indexes.ts @@ -0,0 +1,46 @@ +import { SQL } from "./sqlite"; + +export const createIssueIndexes = async () => { + const columns = [ + "state_id", + "sort_order", + // "priority", + "project_id", + "created_by", + "cycle_id", + ]; + + // Drop indexes + const dropPromises = []; + dropPromises.push(SQL.db.exec({ sql: `DROP INDEX IF EXISTS issues_issue_id_idx` })); + dropPromises.push(SQL.db.exec({ sql: `DROP INDEX IF EXISTS issues_priority_idx` })); + columns.forEach((column) => { + dropPromises.push(SQL.db.exec({ sql: `DROP INDEX IF EXISTS issues_issue_${column}_idx` })); + }); + await Promise.all(dropPromises); + + const promises = []; + + promises.push(SQL.db.exec({ sql: `CREATE UNIQUE INDEX issues_issue_id_idx ON issues (id)` })); + + columns.forEach((column) => { + promises.push(SQL.db.exec({ sql: `CREATE INDEX issues_issue_${column}_idx ON issues (project_id, ${column})` })); + }); + await Promise.all(promises); +}; + +export const createIssueMetaIndexes = async () => { + // Drop indexes + await SQL.db.exec({ sql: `DROP INDEX IF EXISTS issue_meta_all_idx` }); + + await SQL.db.exec({ sql: `CREATE INDEX issue_meta_all_idx ON issue_meta (issue_id,key,value)` }); +}; + +const createIndexes = async () => { + console.log("### Creating indexes"); + const promises = [createIssueIndexes(), createIssueMetaIndexes()]; + await Promise.all(promises); + console.log("### Indexes created"); +}; + +export default createIndexes; diff --git a/web/core/local-db/load-issues.ts b/web/core/local-db/load-issues.ts new file mode 100644 index 00000000000..4c6c636700f --- /dev/null +++ b/web/core/local-db/load-issues.ts @@ -0,0 +1,85 @@ +import { TBaseIssue } from "@plane/types"; +import { IssueService } from "@/services/issue"; +import createIndexes from "./indexes"; +import { runQuery } from "./query-executor"; +import { SQL } from "./sqlite"; + +const arrayFields = ["label_ids", "assignee_ids", "module_ids"]; + +export const addIssue = async (issue: any) => { + const issue_id = issue.id; + const { label_ids, assignee_ids, module_ids, ...otherProps } = issue; + const keys = Object.keys(issue).join(","); + const values = Object.values(issue).map((val) => { + if (val === null) { + return ""; + } + if (typeof val === "object") { + return JSON.stringify(val); + } + return val; + }); // Will fail when the values have a comma + + const promises = []; + SQL.db.exec("BEGIN TRANSACTION;"); + + promises.push( + SQL.db.exec({ + sql: `insert into issues(${keys}) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, + bind: values, + }) + ); + + arrayFields.forEach((field) => { + const values = issue[field]; + if (values) { + values.forEach((val) => { + // promises.push( + SQL.db.exec({ + sql: `insert into issue_meta(issue_id,key,value) values (?,?,?)`, + bind: [issue_id, field, val], + }); + // ); + }); + } + }); + SQL.db.exec("COMMIT;"); + + // await Promise.all(promises); + console.log("### Added issue", issue.id); +}; + +export const loadIssues = async (workspaceId: string, projectId: string) => { + // Load issues from the API + const issueService = new IssueService(); + + const PAGE_SIZE = 100; + let cursor = `${PAGE_SIZE}:0:0`; + let results; + let breakLoop = false; + let count = await runQuery(`select count(*) as count from issues where project_id='${projectId}'`); + console.log("### Count", count); + count = count[0]["count"]; + + console.log("### Count", count, typeof count); + if (!count) { + do { + const response = await issueService.getIssuesFromServer(workspaceId, projectId, { cursor }); + cursor = response.next_cursor; + results = response.results as TBaseIssue[]; + console.log("#### Loading issues", results.length); + results.map(async (issue) => { + try { + await addIssue(issue); + } catch (e) { + console.log("###Error", e, issue); + breakLoop = true; + } + }); + } while (results.length >= PAGE_SIZE && !breakLoop); + } else { + console.log("### Project already stored locally, call update issues"); + } + + createIndexes(); +}; diff --git a/web/core/local-db/queries/issues.ts b/web/core/local-db/queries/issues.ts new file mode 100644 index 00000000000..d031fa5d914 --- /dev/null +++ b/web/core/local-db/queries/issues.ts @@ -0,0 +1,60 @@ +import { TIssue } from "@plane/types"; +import { issueFilterCountQueryConstructor, issueFilterQueryConstructor } from "../query-constructor"; +import { runQuery } from "../query-executor"; + +const arrayFields = ["label_ids", "assignee_ids", "module_ids"]; + +export const getIssues = async (workspaceSlug: string, projectId: string, queries?: any, config = {}): Promise => { + // { + // "priority": "urgent,high,medium", + // "state": "5d572a70-aa2f-409c-ade0-6a49e3c052c7,f0c8572a-164a-4ae0-98b4-329e89e22d86", + // "order_by": "-created_at", + // "sub_issue": true, + // "cursor": "100:0:0", + // "per_page": "100" + // } + // https://dexie.org/docs/MultiEntry-Index + + // console.log("#### queries", queries); + const { cursor } = queries; + + const start = performance.now(); + const query = issueFilterQueryConstructor(workspaceSlug, projectId, queries); + const countQuery = issueFilterCountQueryConstructor(workspaceSlug, projectId, queries); + let issues = await runQuery(query); + const { total_count } = (await runQuery(countQuery))[0]; + const end = performance.now(); + + console.log("#### Local time", end - start); + + const [pageSize, page, offset] = cursor.split(":"); + + issues = issues.map((issue: TIssue) => { + arrayFields.forEach((field) => { + issue[field] = JSON.parse(issue[field]); + }); + + return issue; + }); + + const out = { + results: issues, + next_cursor: `${pageSize}:${page}:${Number(offset) + Number(pageSize)}`, + prev_cursor: `${pageSize}:${page}:${Number(offset) - Number(pageSize)}`, + total_results: total_count, + count: issues.length, + total_count, + total_pages: Math.ceil(total_count / Number(pageSize)), + }; + + console.log(out); + return { + results: issues, + next_cursor: `${pageSize}:${page}:${Number(offset) + Number(pageSize)}`, + prev_cursor: `${pageSize}:${page}:${Number(offset) - Number(pageSize)}`, + total_results: total_count, + count: issues.length, + total_count, + total_pages: Math.ceil(total_count / Number(pageSize)), + }; +}; diff --git a/web/core/local-db/query-constructor.ts b/web/core/local-db/query-constructor.ts new file mode 100644 index 00000000000..46e53971e1f --- /dev/null +++ b/web/core/local-db/query-constructor.ts @@ -0,0 +1,51 @@ +export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: string, queries) => { + const { order_by, cursor, per_page, labels, sub_issue, state, cycle, group_by, module, ...otherProps } = queries; + + if (state) otherProps.state_id = state; + if (cycle) otherProps.cycle_id = cycle; + if (module) otherProps.module_ids = module; + if (labels) otherProps.label_ids = labels; + + const arrayFields = ["label_ids", "assignee_ids", "module_ids"]; + let sql = `SELECT * FROM issues i LEFT JOIN issue_meta im ON i.id = im.issue_id WHERE 1=1 AND project_id='${projectId}' `; + + const keys = Object.keys(otherProps); + + keys.forEach((key) => { + const value = otherProps[key] ? otherProps[key].split(",") : ""; + if (!value) return; + if (arrayFields.includes(key)) { + sql += ` AND key='${key}' AND value IN ('${value.join("','")}')`; + } else { + sql += ` AND ${key} in ('${value.join("','")}')`; + } + }); + + sql += ` group by i.id`; + + if (order_by) { + //if order_by starts with "-" then sort in descending order + if (order_by.startsWith("-")) { + sql += ` ORDER BY ${order_by.slice(1)} DESC`; + } else { + sql += ` ORDER BY ${order_by} ASC`; + } + } + + const [pageSize, page, offset] = cursor.split(":"); + // Add offset and paging to query + sql += ` LIMIT ${pageSize} OFFSET ${offset * 1 + page * pageSize};`; + + console.log("$$$", sql); + return sql; +}; + +export const issueFilterCountQueryConstructor = (workspaceSlug: string, projectId: string, queries) => { + let sql = issueFilterQueryConstructor(workspaceSlug, projectId, queries); + + sql = sql.replace("SELECT *", "SELECT COUNT(*) as total_count"); + // Remove everything after group by i.id + sql = `${sql.split("group by i.id")[0]} group by i.id`; + + return sql; +}; diff --git a/web/core/local-db/query-executor.ts b/web/core/local-db/query-executor.ts new file mode 100644 index 00000000000..81c69df80cc --- /dev/null +++ b/web/core/local-db/query-executor.ts @@ -0,0 +1,15 @@ +import { SQL } from "./sqlite"; + +export const runQuery = async (sql) => { + const data = await SQL.db.exec({ + // sql: `SELECT name, '[' || GROUP_CONCAT('"' || value || '"') || ']' AS label_ids FROM issues LEFT JOIN issue_meta ON issues.id = issue_meta.issue_id WHERE key='label_ids' AND value IN ('2e2a79d1-241a-4708-bf95-95eefa33a501','67970b7d-0525-48fa-9f7e-10fc086f5122')`, + // sql: `SELECT * FROM issues LEFT JOIN issue_meta ON issues.id = issue_meta.issue_id WHERE key='label_ids' AND value IN ('2e2a79d1-241a-4708-bf95-95eefa33a501','67970b7d-0525-48fa-9f7e-10fc086f5122')`, + sql, + + // key=label_ids AND value in ('2e2a79d1-241a-4708-bf95-95eefa33a501','67970b7d-0525-48fa-9f7e-10fc086f5122')`, + rowMode: "object", + returnValue: "resultRows", + }); + + return data.result.resultRows; +}; diff --git a/web/core/local-db/sqlite.ts b/web/core/local-db/sqlite.ts new file mode 100644 index 00000000000..3403a066051 --- /dev/null +++ b/web/core/local-db/sqlite.ts @@ -0,0 +1,81 @@ +// import sqlite3InitModule from "@sqlite.org/sqlite-wasm"; +import { sqlite3Worker1Promiser } from "@sqlite.org/sqlite-wasm"; +import { createTables } from "./tables"; + +declare module "@sqlite.org/sqlite-wasm" { + export function sqlite3Worker1Promiser(...args: any): any; +} + +const log = console.log; +const error = console.error; + +const SQL = {}; +const start = async (sqlite3: any) => { + log("Running SQLite3 version", sqlite3.version.libVersion); + SQL.db = new sqlite3.oo1.DB("/mydb.sqlite3", "ct"); + createTables(SQL.db); +}; + +const initializeSQLiteMemory = async () => { + try { + log("Loading and initializing SQLite3 module..."); + const sqlite3 = await sqlite3InitModule({ + print: log, + printErr: error, + }); + log("Done initializing. Running demo..."); + await start(sqlite3); + } catch (err) { + error("Initialization error:", err.name, err.message); + } +}; + +const initializeSQLite = async () => { + if (SQL.db) { + console.info("Instance already initialized"); + return; + } + try { + log("Loading and initializing SQLite3 module..."); + + const promiser = await new Promise((resolve) => { + const _promiser = sqlite3Worker1Promiser({ + onready: () => resolve(_promiser), + }); + }); + + log("Done initializing. Running demo..."); + + const configResponse = await promiser("config-get", {}); + log("Running SQLite3 version", configResponse.result.version.libVersion); + + const openResponse = await promiser("open", { + filename: "file:mydb.sqlite3?vfs=opfs", + }); + const { dbId } = openResponse; + SQL.db = { + dbId, + exec: async (val) => { + if (typeof val === "string") { + val = { sql: val }; + } + return promiser("exec", { dbId, ...val }); + }, + }; + log( + "OPFS is available, created persisted database at", + openResponse.result.filename.replace(/^file:(.*?)\?vfs=opfs$/, "$1") + ); + // Your SQLite code here. + await createTables(SQL.db); + } catch (err) { + if (!(err instanceof Error)) { + err = new Error(err.result.message); + } + error(err.name, err.message); + } +}; + +// initializeSQLite(); + +export { SQL, initializeSQLite }; diff --git a/web/core/local-db/tables.ts b/web/core/local-db/tables.ts new file mode 100644 index 00000000000..b147f80abc5 --- /dev/null +++ b/web/core/local-db/tables.ts @@ -0,0 +1,42 @@ +export const createIssuesTable = (SQLITE) => { + const sqlstr = `CREATE TABLE IF NOT EXISTS issues ( + id TEXT, + name TEXT, + state_id TEXT, + sort_order REAL, + completed_at TEXT, + estimate_point REAL, + priority TEXT, + start_date TEXT, + target_date TEXT, + sequence_id INTEGER, + project_id TEXT, + parent_id TEXT, + created_at TEXT, + updated_at TEXT, + created_by TEXT, + updated_by TEXT, + is_draft INTEGER, + archived_at TEXT, + state__group TEXT, + sub_issues_count INTEGER, + cycle_id TEXT, + link_count INTEGER, + attachment_count INTEGER, + label_ids TEXT, + assignee_ids TEXT, + module_ids TEXT);`; + SQLITE.exec(sqlstr); +}; + +export const createIssueMetaTable = (SQLITE) => { + const sqlstr = `CREATE TABLE IF NOT EXISTS issue_meta ( + issue_id TEXT, + key TEXT, + value TEXT);`; + SQLITE.exec(sqlstr); +}; +export const createTables = async (SQLITE) => { + await createIssuesTable(SQLITE); + await createIssueMetaTable(SQLITE); +}; diff --git a/web/core/services/issue/issue.service.ts b/web/core/services/issue/issue.service.ts index 69ef1ed5c3a..01e1808d034 100644 --- a/web/core/services/issue/issue.service.ts +++ b/web/core/services/issue/issue.service.ts @@ -11,6 +11,7 @@ import type { // helpers import { API_BASE_URL } from "@/helpers/common.helper"; // services +import { getIssues } from "@/local-db/queries/issues"; import { APIService } from "@/services/api.service"; export class IssueService extends APIService { @@ -26,7 +27,12 @@ export class IssueService extends APIService { }); } - async getIssues(workspaceSlug: string, projectId: string, queries?: any, config = {}): Promise { + async getIssuesFromServer( + workspaceSlug: string, + projectId: string, + queries?: any, + config = {} + ): Promise { return this.get( `/api/workspaces/${workspaceSlug}/projects/${projectId}/issues/`, { @@ -40,6 +46,10 @@ export class IssueService extends APIService { }); } + async getIssues(workspaceSlug: string, projectId: string, queries?: any, config = {}): Promise { + return await getIssues(workspaceSlug, projectId, queries, config); + } + async getIssuesWithParams( workspaceSlug: string, projectId: string, diff --git a/web/next.config.js b/web/next.config.js index 47b81e0870a..9c0b24792e7 100644 --- a/web/next.config.js +++ b/web/next.config.js @@ -24,6 +24,8 @@ const nextConfig = { key: "Referrer-Policy", value: "origin-when-cross-origin", }, + { key: "Cross-Origin-Opener-Policy", value: "same-origin" }, + { key: "Cross-Origin-Embedder-Policy", value: "require-corp" }, ], }, ]; diff --git a/web/package.json b/web/package.json index 5b861daf025..faae68f074a 100644 --- a/web/package.json +++ b/web/package.json @@ -31,6 +31,7 @@ "@plane/ui": "*", "@popperjs/core": "^2.11.8", "@sentry/nextjs": "^8", + "@sqlite.org/sqlite-wasm": "^3.46.0-build2", "axios": "^1.1.3", "clsx": "^2.0.0", "cmdk": "^1.0.0", diff --git a/yarn.lock b/yarn.lock index 5a026323127..6a2f1e8eeaf 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2915,6 +2915,11 @@ resolved "https://registry.yarnpkg.com/@sindresorhus/merge-streams/-/merge-streams-2.3.0.tgz#719df7fb41766bc143369eaa0dd56d8dc87c9958" integrity sha512-LtoMMhxAlorcGhmFYI+LhPgbPZCkgP6ra1YL604EeF6U98pLlQ3iWIGMdWSC+vWmPBWBNgmDBAhnAobLROJmwg== +"@sqlite.org/sqlite-wasm@^3.46.0-build2": + version "3.46.0-build2" + resolved "https://registry.yarnpkg.com/@sqlite.org/sqlite-wasm/-/sqlite-wasm-3.46.0-build2.tgz#f84c3014f3fed6db08fc585d67e386d39e3956bf" + integrity sha512-10s/u/Main1RGO+jjzK+mgC/zh1ls1CEnq3Dujr03TwvzLg+j4FAohOmlYkQj8KQOj1vGR9cuB9F8tVBTwVGVA== + "@storybook/addon-actions@8.1.6": version "8.1.6" resolved "https://registry.yarnpkg.com/@storybook/addon-actions/-/addon-actions-8.1.6.tgz#ac05b295ee88ddba1f5a96499438d997d761392d" @@ -4451,7 +4456,7 @@ dependencies: "@types/react" "*" -"@types/react@*", "@types/react@18.2.48", "@types/react@^16.8.0 || ^17.0.0 || ^18.0.0", "@types/react@^18.2.42", "@types/react@^18.2.48": +"@types/react@*", "@types/react@^16.8.0 || ^17.0.0 || ^18.0.0", "@types/react@^18.2.42", "@types/react@^18.2.48": version "18.2.48" resolved "https://registry.yarnpkg.com/@types/react/-/react-18.2.48.tgz#11df5664642d0bd879c1f58bc1d37205b064e8f1" integrity sha512-qboRCl6Ie70DQQG9hhNREz81jqC1cs9EVNcjQ1AU+jH6NFfSAhVVbrrY/+nSF+Bsk4AOwm9Qa61InvMCyV+H3w== From 52b495d2a2c3ccbfa8b9476588a3b9278ee57fab Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Wed, 31 Jul 2024 16:14:36 +0530 Subject: [PATCH 007/111] Fix incorrect total count and filtering on assignees. --- web/core/local-db/queries/issues.ts | 29 +++++++++++++------------- web/core/local-db/query-constructor.ts | 8 ++++--- web/core/local-db/sqlite.ts | 5 +++-- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/web/core/local-db/queries/issues.ts b/web/core/local-db/queries/issues.ts index d031fa5d914..9ebee9bc3bc 100644 --- a/web/core/local-db/queries/issues.ts +++ b/web/core/local-db/queries/issues.ts @@ -5,30 +5,21 @@ import { runQuery } from "../query-executor"; const arrayFields = ["label_ids", "assignee_ids", "module_ids"]; export const getIssues = async (workspaceSlug: string, projectId: string, queries?: any, config = {}): Promise => { - // { - // "priority": "urgent,high,medium", - // "state": "5d572a70-aa2f-409c-ade0-6a49e3c052c7,f0c8572a-164a-4ae0-98b4-329e89e22d86", - // "order_by": "-created_at", - // "sub_issue": true, - // "cursor": "100:0:0", - // "per_page": "100" - // } - // https://dexie.org/docs/MultiEntry-Index - - // console.log("#### queries", queries); const { cursor } = queries; - const start = performance.now(); const query = issueFilterQueryConstructor(workspaceSlug, projectId, queries); const countQuery = issueFilterCountQueryConstructor(workspaceSlug, projectId, queries); + const start = performance.now(); let issues = await runQuery(query); - const { total_count } = (await runQuery(countQuery))[0]; const end = performance.now(); - console.log("#### Local time", end - start); + const countStart = performance.now(); + const { total_count } = (await runQuery(countQuery))[0]; + const countEnd = performance.now(); const [pageSize, page, offset] = cursor.split(":"); + const parsingStart = performance.now(); issues = issues.map((issue: TIssue) => { arrayFields.forEach((field) => { issue[field] = JSON.parse(issue[field]); @@ -37,6 +28,16 @@ export const getIssues = async (workspaceSlug: string, projectId: string, querie return issue; }); + const parsingEnd = performance.now(); + + const times = { + IssueQuery: end - start, + Count: countEnd - countStart, + Parsing: parsingEnd - parsingStart, + }; + + console.table(times); + const out = { results: issues, next_cursor: `${pageSize}:${page}:${Number(offset) + Number(pageSize)}`, diff --git a/web/core/local-db/query-constructor.ts b/web/core/local-db/query-constructor.ts index 46e53971e1f..664e03600cd 100644 --- a/web/core/local-db/query-constructor.ts +++ b/web/core/local-db/query-constructor.ts @@ -1,10 +1,12 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: string, queries) => { - const { order_by, cursor, per_page, labels, sub_issue, state, cycle, group_by, module, ...otherProps } = queries; + const { order_by, cursor, per_page, labels, sub_issue, assignees, state, cycle, group_by, module, ...otherProps } = + queries; if (state) otherProps.state_id = state; if (cycle) otherProps.cycle_id = cycle; if (module) otherProps.module_ids = module; if (labels) otherProps.label_ids = labels; + if (assignees) otherProps.assignee_ids = assignees; const arrayFields = ["label_ids", "assignee_ids", "module_ids"]; let sql = `SELECT * FROM issues i LEFT JOIN issue_meta im ON i.id = im.issue_id WHERE 1=1 AND project_id='${projectId}' `; @@ -43,9 +45,9 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st export const issueFilterCountQueryConstructor = (workspaceSlug: string, projectId: string, queries) => { let sql = issueFilterQueryConstructor(workspaceSlug, projectId, queries); - sql = sql.replace("SELECT *", "SELECT COUNT(*) as total_count"); + sql = sql.replace("SELECT *", "SELECT COUNT(DISTINCT i.id) as total_count"); // Remove everything after group by i.id - sql = `${sql.split("group by i.id")[0]} group by i.id`; + sql = `${sql.split("group by i.id")[0]};`; return sql; }; diff --git a/web/core/local-db/sqlite.ts b/web/core/local-db/sqlite.ts index 3403a066051..fff2173b159 100644 --- a/web/core/local-db/sqlite.ts +++ b/web/core/local-db/sqlite.ts @@ -9,7 +9,7 @@ declare module "@sqlite.org/sqlite-wasm" { const log = console.log; const error = console.error; -const SQL = {}; +const SQL = { initialized: false }; const start = async (sqlite3: any) => { log("Running SQLite3 version", sqlite3.version.libVersion); SQL.db = new sqlite3.oo1.DB("/mydb.sqlite3", "ct"); @@ -31,7 +31,7 @@ const initializeSQLiteMemory = async () => { }; const initializeSQLite = async () => { - if (SQL.db) { + if (SQL.initialized) { console.info("Instance already initialized"); return; } @@ -66,6 +66,7 @@ const initializeSQLite = async () => { "OPFS is available, created persisted database at", openResponse.result.filename.replace(/^file:(.*?)\?vfs=opfs$/, "$1") ); + SQL.initialized = true; // Your SQLite code here. await createTables(SQL.db); } catch (err) { From 322e9a9ea6295c8315b9a415585c5158546f2d5b Mon Sep 17 00:00:00 2001 From: rahulramesha Date: Wed, 31 Jul 2024 18:49:49 +0530 Subject: [PATCH 008/111] enable parallel API calls --- .../issue-layouts/kanban/base-kanban-root.tsx | 4 +- .../issues/issue-layouts/kanban/default.tsx | 2 +- .../kanban/headers/group-by-card.tsx | 15 +++-- .../issue-layouts/kanban/kanban-group.tsx | 6 +- .../issues/issue-layouts/kanban/swimlanes.tsx | 4 +- .../issue-layouts/list/base-list-root.tsx | 42 ++++++------ .../list/headers/group-by-card.tsx | 13 ++-- .../issues/issue-layouts/list/list-group.tsx | 5 +- .../components/issues/issue-layouts/utils.tsx | 3 +- .../layouts/auth-layout/project-wrapper.tsx | 28 +++++--- web/core/local-db/queries/issues.ts | 18 +++-- web/core/store/issue/cycle/issue.store.ts | 64 ++++++++++++++++-- .../store/issue/helpers/base-issues.store.ts | 65 ++++++++++++++++--- .../helpers/issue-filter-helper.store.ts | 2 +- web/core/store/issue/module/issue.store.ts | 64 ++++++++++++++++-- .../store/issue/project-views/issue.store.ts | 61 +++++++++++++++-- web/core/store/issue/project/issue.store.ts | 61 +++++++++++++++-- 17 files changed, 370 insertions(+), 87 deletions(-) diff --git a/web/core/components/issues/issue-layouts/kanban/base-kanban-root.tsx b/web/core/components/issues/issue-layouts/kanban/base-kanban-root.tsx index 7121169ee6c..4ba7a7a21ec 100644 --- a/web/core/components/issues/issue-layouts/kanban/base-kanban-root.tsx +++ b/web/core/components/issues/issue-layouts/kanban/base-kanban-root.tsx @@ -222,7 +222,7 @@ export const BaseKanBanRoot: React.FC = observer((props: IBas const kanbanFilters = issuesFilter?.issueFilters?.kanbanFilters || { group_by: [], sub_group_by: [] }; return ( - + <> = observer((props: IBas
- + ); }); diff --git a/web/core/components/issues/issue-layouts/kanban/default.tsx b/web/core/components/issues/issue-layouts/kanban/default.tsx index f28396367cf..368d30c9c47 100644 --- a/web/core/components/issues/issue-layouts/kanban/default.tsx +++ b/web/core/components/issues/issue-layouts/kanban/default.tsx @@ -156,7 +156,7 @@ export const KanBan: React.FC = observer((props) => { column_id={subList.id} icon={subList.icon} title={subList.name} - count={getGroupIssueCount(subList.id, undefined, false) ?? 0} + count={getGroupIssueCount(subList.id, undefined, false)} issuePayload={subList.payload} disableIssueCreation={disableIssueCreation || isGroupByCreatedBy} addIssuesToView={addIssuesToView} diff --git a/web/core/components/issues/issue-layouts/kanban/headers/group-by-card.tsx b/web/core/components/issues/issue-layouts/kanban/headers/group-by-card.tsx index d7b1a04a399..e5602155ce1 100644 --- a/web/core/components/issues/issue-layouts/kanban/headers/group-by-card.tsx +++ b/web/core/components/issues/issue-layouts/kanban/headers/group-by-card.tsx @@ -15,6 +15,7 @@ import { CreateUpdateIssueModal } from "@/components/issues"; // hooks import { useEventTracker } from "@/hooks/store"; import { useIssueStoreType } from "@/hooks/use-issue-layout-store"; +import isNil from "lodash/isNil"; // types interface IHeaderGroupByCard { @@ -23,7 +24,7 @@ interface IHeaderGroupByCard { column_id: string; icon?: React.ReactNode; title: string; - count: number; + count: number | undefined; kanbanFilters: TIssueKanbanFilters; handleKanbanFilters: any; issuePayload: Partial; @@ -123,11 +124,13 @@ export const HeaderGroupByCard: FC = observer((props) => { > {title} -
- {count || 0} -
+ {!isNil(count) && ( +
+ {count || 0} +
+ )} {sub_group_by === null && ( diff --git a/web/core/components/issues/issue-layouts/kanban/kanban-group.tsx b/web/core/components/issues/issue-layouts/kanban/kanban-group.tsx index d272bb949c2..66b9ec772c0 100644 --- a/web/core/components/issues/issue-layouts/kanban/kanban-group.tsx +++ b/web/core/components/issues/issue-layouts/kanban/kanban-group.tsx @@ -28,6 +28,7 @@ import { GroupDragOverlay } from "../group-drag-overlay"; import { TRenderQuickActions } from "../list/list-view-types"; import { GroupDropLocation, getSourceFromDropPayload, getDestinationFromDropPayload, getIssueBlockId } from "../utils"; import { KanbanIssueBlocksList, KanBanQuickAddIssueForm } from "."; +import isNil from "lodash/isNil"; interface IKanbanGroup { groupId: string; @@ -218,7 +219,7 @@ export const KanbanGroup = observer((props: IKanbanGroup) => { ? (groupedIssueIds as TSubGroupedIssues)?.[groupId]?.[sub_group_id] ?? [] : (groupedIssueIds as TGroupedIssues)?.[groupId] ?? []; - const groupIssueCount = getGroupIssueCount(groupId, sub_group_id, false) ?? 0; + const groupIssueCount = getGroupIssueCount(groupId, sub_group_id, false); const nextPageResults = getPaginationData(groupId, sub_group_id)?.nextPageResults; @@ -234,7 +235,8 @@ export const KanbanGroup = observer((props: IKanbanGroup) => { ); - const shouldLoadMore = nextPageResults === undefined ? issueIds?.length < groupIssueCount : !!nextPageResults; + const shouldLoadMore = + nextPageResults === undefined ? isNil(groupIssueCount) || issueIds?.length < groupIssueCount : !!nextPageResults; const canOverlayBeVisible = orderBy !== "sort_order" || isDropDisabled; const shouldOverlayBeVisible = isDraggingOverColumn && canOverlayBeVisible; diff --git a/web/core/components/issues/issue-layouts/kanban/swimlanes.tsx b/web/core/components/issues/issue-layouts/kanban/swimlanes.tsx index 1c50dc26762..d8a232721a9 100644 --- a/web/core/components/issues/issue-layouts/kanban/swimlanes.tsx +++ b/web/core/components/issues/issue-layouts/kanban/swimlanes.tsx @@ -56,9 +56,9 @@ const SubGroupSwimlaneHeader: React.FC = observer( {list && list.length > 0 && list.map((_list: IGroupByColumn) => { - const groupCount = getGroupIssueCount(_list?.id, undefined, false) ?? 0; + const groupCount = getGroupIssueCount(_list?.id, undefined, false); - const subGroupByVisibilityToggle = visibilitySubGroupByGroupCount(groupCount, showEmptyGroup); + const subGroupByVisibilityToggle = visibilitySubGroupByGroupCount(groupCount ?? 0, showEmptyGroup); if (subGroupByVisibilityToggle === false) return <>; diff --git a/web/core/components/issues/issue-layouts/list/base-list-root.tsx b/web/core/components/issues/issue-layouts/list/base-list-root.tsx index 09ddd44d7f2..76e452a9282 100644 --- a/web/core/components/issues/issue-layouts/list/base-list-root.tsx +++ b/web/core/components/issues/issue-layouts/list/base-list-root.tsx @@ -107,27 +107,25 @@ export const BaseListRoot = observer((props: IBaseListRoot) => { ); return ( - -
- -
- +
+ +
); }); diff --git a/web/core/components/issues/issue-layouts/list/headers/group-by-card.tsx b/web/core/components/issues/issue-layouts/list/headers/group-by-card.tsx index 1a87e51b895..97720f0e664 100644 --- a/web/core/components/issues/issue-layouts/list/headers/group-by-card.tsx +++ b/web/core/components/issues/issue-layouts/list/headers/group-by-card.tsx @@ -18,12 +18,13 @@ import { cn } from "@/helpers/common.helper"; import { useEventTracker } from "@/hooks/store"; import { useIssueStoreType } from "@/hooks/use-issue-layout-store"; import { TSelectionHelper } from "@/hooks/use-multiple-select"; +import isNil from "lodash/isNil"; interface IHeaderGroupByCard { groupID: string; icon?: React.ReactNode; title: string; - count: number; + count: number | undefined; issuePayload: Partial; canEditProperties: (projectId: string | undefined) => boolean; toggleListGroup: () => void; @@ -98,7 +99,7 @@ export const HeaderGroupByCard = observer((props: IHeaderGroupByCard) => { )} groupID={groupID} selectionHelpers={selectionHelpers} - disabled={count === 0} + disabled={!count} /> )} @@ -106,10 +107,12 @@ export const HeaderGroupByCard = observer((props: IHeaderGroupByCard) => { {icon ?? } -
+
{title}
-
{count || 0}
+ {!isNil(count) &&
{count || 0}
}
{!disableIssueCreation && diff --git a/web/core/components/issues/issue-layouts/list/list-group.tsx b/web/core/components/issues/issue-layouts/list/list-group.tsx index 0f1b8069458..84be7f0d26b 100644 --- a/web/core/components/issues/issue-layouts/list/list-group.tsx +++ b/web/core/components/issues/issue-layouts/list/list-group.tsx @@ -37,6 +37,7 @@ import { IssueBlocksList } from "./blocks-list"; import { HeaderGroupByCard } from "./headers/group-by-card"; import { TRenderQuickActions } from "./list-view-types"; import { ListQuickAddIssueForm } from "./quick-add-issue-form"; +import isNil from "lodash/isNil"; interface Props { groupIssueIds: string[] | undefined; @@ -98,7 +99,7 @@ export const ListGroup = observer((props: Props) => { const [intersectionElement, setIntersectionElement] = useState(null); - const groupIssueCount = getGroupIssueCount(group.id, undefined, false) ?? 0; + const groupIssueCount = getGroupIssueCount(group.id, undefined, false); const nextPageResults = getPaginationData(group.id, undefined)?.nextPageResults; const isPaginating = !!getIssueLoader(group.id); @@ -106,7 +107,7 @@ export const ListGroup = observer((props: Props) => { const shouldLoadMore = nextPageResults === undefined && groupIssueCount !== undefined && groupIssueIds - ? groupIssueIds.length < groupIssueCount + ? isNil(groupIssueCount) || groupIssueIds.length < groupIssueCount : !!nextPageResults; const loadMore = isPaginating ? ( diff --git a/web/core/components/issues/issue-layouts/utils.tsx b/web/core/components/issues/issue-layouts/utils.tsx index 7840bb46217..1332e8cea78 100644 --- a/web/core/components/issues/issue-layouts/utils.tsx +++ b/web/core/components/issues/issue-layouts/utils.tsx @@ -39,6 +39,7 @@ import { IMemberRootStore } from "@/store/member"; import { IModuleStore } from "@/store/module.store"; import { IProjectStore } from "@/store/project/project.store"; import { IStateStore } from "@/store/state.store"; +import { ALL_ISSUES } from "@plane/constants"; export const HIGHLIGHT_CLASS = "highlight"; export const HIGHLIGHT_WITH_LINE = "highlight-with-line"; @@ -92,7 +93,7 @@ export const getGroupByColumns = ( case "created_by": return getCreatedByColumns(member) as any; default: - if (includeNone) return [{ id: `All Issues`, name: `All Issues`, payload: {}, icon: undefined }]; + if (includeNone) return [{ id: ALL_ISSUES, name: ALL_ISSUES, payload: {}, icon: undefined }]; } }; diff --git a/web/core/layouts/auth-layout/project-wrapper.tsx b/web/core/layouts/auth-layout/project-wrapper.tsx index 969097b1475..a59352154ef 100644 --- a/web/core/layouts/auth-layout/project-wrapper.tsx +++ b/web/core/layouts/auth-layout/project-wrapper.tsx @@ -1,4 +1,4 @@ -import { FC, ReactNode, useEffect } from "react"; +import { FC, ReactNode, useEffect, useState } from "react"; import { observer } from "mobx-react"; import { useParams } from "next/navigation"; import useSWR from "swr"; @@ -34,6 +34,7 @@ export const ProjectAuthWrapper: FC = observer((props) => { // const { fetchInboxes } = useInbox(); const { toggleCreateProjectModal } = useCommandPalette(); const { setTrackElement } = useEventTracker(); + const [areIssuesLoading, setIssuesLoading] = useState(false); const { membership: { fetchUserProjectInfo, projectMemberInfo, hasPermissionToProject }, } = useUser(); @@ -52,8 +53,10 @@ export const ProjectAuthWrapper: FC = observer((props) => { useEffect(() => { (async () => { + setIssuesLoading(true); await initializeSQLite(); await loadIssues(workspaceSlug.toString(), projectId.toString()); + setIssuesLoading(false); })(); }, [workspaceSlug, projectId]); // fetching project details @@ -67,37 +70,37 @@ export const ProjectAuthWrapper: FC = observer((props) => { workspaceSlug && projectId ? () => fetchUserProjectInfo(workspaceSlug.toString(), projectId.toString()) : null ); // fetching project labels - useSWR( + const { isLoading: isLabelsLoading } = useSWR( workspaceSlug && projectId ? `PROJECT_LABELS_${workspaceSlug}_${projectId}` : null, workspaceSlug && projectId ? () => fetchProjectLabels(workspaceSlug.toString(), projectId.toString()) : null, { revalidateIfStale: false, revalidateOnFocus: false } ); // fetching project members - useSWR( + const { isLoading: isMembersLoading } = useSWR( workspaceSlug && projectId ? `PROJECT_MEMBERS_${workspaceSlug}_${projectId}` : null, workspaceSlug && projectId ? () => fetchProjectMembers(workspaceSlug.toString(), projectId.toString()) : null, { revalidateIfStale: false, revalidateOnFocus: false } ); // fetching project states - useSWR( + const { isLoading: isStateLoading } = useSWR( workspaceSlug && projectId ? `PROJECT_STATES_${workspaceSlug}_${projectId}` : null, workspaceSlug && projectId ? () => fetchProjectStates(workspaceSlug.toString(), projectId.toString()) : null, { revalidateIfStale: false, revalidateOnFocus: false } ); // fetching project estimates - useSWR( + const { isLoading: isEstimatesLoading } = useSWR( workspaceSlug && projectId ? `PROJECT_ESTIMATES_${workspaceSlug}_${projectId}` : null, workspaceSlug && projectId ? () => getProjectEstimates(workspaceSlug.toString(), projectId.toString()) : null, { revalidateIfStale: false, revalidateOnFocus: false } ); // fetching project cycles - useSWR( + const { isLoading: isCyclesLoading } = useSWR( workspaceSlug && projectId ? `PROJECT_ALL_CYCLES_${workspaceSlug}_${projectId}` : null, workspaceSlug && projectId ? () => fetchAllCycles(workspaceSlug.toString(), projectId.toString()) : null, { revalidateIfStale: false, revalidateOnFocus: false } ); // fetching project modules - useSWR( + const { isLoading: isModulesLoading } = useSWR( workspaceSlug && projectId ? `PROJECT_MODULES_${workspaceSlug}_${projectId}` : null, workspaceSlug && projectId ? () => fetchModules(workspaceSlug.toString(), projectId.toString()) : null, { revalidateIfStale: false, revalidateOnFocus: false } @@ -110,8 +113,17 @@ export const ProjectAuthWrapper: FC = observer((props) => { ); const projectExists = projectId ? getProjectById(projectId.toString()) : null; + const isLoading = + isLabelsLoading || + isMembersLoading || + isStateLoading || + isEstimatesLoading || + isCyclesLoading || + isModulesLoading || + areIssuesLoading; + // check if the project member apis is loading - if (!projectMemberInfo && projectId && hasPermissionToProject[projectId.toString()] === null) + if ((!projectMemberInfo && projectId && hasPermissionToProject[projectId.toString()] === null) || isLoading) return (
diff --git a/web/core/local-db/queries/issues.ts b/web/core/local-db/queries/issues.ts index 9ebee9bc3bc..c68f503bfaf 100644 --- a/web/core/local-db/queries/issues.ts +++ b/web/core/local-db/queries/issues.ts @@ -7,7 +7,9 @@ const arrayFields = ["label_ids", "assignee_ids", "module_ids"]; export const getIssues = async (workspaceSlug: string, projectId: string, queries?: any, config = {}): Promise => { const { cursor } = queries; + console.log("## queries", queries); const query = issueFilterQueryConstructor(workspaceSlug, projectId, queries); + console.log("SQL", query); const countQuery = issueFilterCountQueryConstructor(workspaceSlug, projectId, queries); const start = performance.now(); let issues = await runQuery(query); @@ -38,6 +40,9 @@ export const getIssues = async (workspaceSlug: string, projectId: string, querie console.table(times); + const total_pages = Math.ceil(total_count / Number(pageSize)); + const next_page_results = total_pages > parseInt(page) + 1; + const out = { results: issues, next_cursor: `${pageSize}:${page}:${Number(offset) + Number(pageSize)}`, @@ -45,17 +50,10 @@ export const getIssues = async (workspaceSlug: string, projectId: string, querie total_results: total_count, count: issues.length, total_count, - total_pages: Math.ceil(total_count / Number(pageSize)), + next_page_results, + total_pages, }; console.log(out); - return { - results: issues, - next_cursor: `${pageSize}:${page}:${Number(offset) + Number(pageSize)}`, - prev_cursor: `${pageSize}:${page}:${Number(offset) - Number(pageSize)}`, - total_results: total_count, - count: issues.length, - total_count, - total_pages: Math.ceil(total_count / Number(pageSize)), - }; + return out; }; diff --git a/web/core/store/issue/cycle/issue.store.ts b/web/core/store/issue/cycle/issue.store.ts index 624f14048ee..6812afefa75 100644 --- a/web/core/store/issue/cycle/issue.store.ts +++ b/web/core/store/issue/cycle/issue.store.ts @@ -170,23 +170,79 @@ export class CycleIssues extends BaseIssuesStore implements ICycleIssues { options: IssuePaginationOptions, cycleId: string, isExistingPaginationOptions: boolean = false + ) => { + try { + const groups = this.getGroups(); + const subGroups = this.getSubGroups(); + + this.clear(!isExistingPaginationOptions); + + if (!groups || (!groups && !subGroups)) + return await this.fetchParallelIssues(workspaceSlug, projectId, loadType, options, cycleId); + + const promises = []; + + for (const group of groups) { + if (!subGroups) { + promises.push(this.fetchParallelIssues(workspaceSlug, projectId, loadType, options, cycleId, group.id)); + continue; + } + + for (const subGroup of subGroups) { + promises.push( + this.fetchParallelIssues(workspaceSlug, projectId, loadType, options, cycleId, group.id, subGroup.id) + ); + } + } + + await Promise.all(promises); + + // fetch parent stats if required, to be handled in the Implemented class + this.fetchParentStats(workspaceSlug, projectId, cycleId); + } catch (error) { + // set loader to undefined if errored out + this.setLoader(undefined); + throw error; + } + }; + + /** + * This method is called to fetch the first issues of pagination + * @param workspaceSlug + * @param projectId + * @param loadType + * @param options + * @returns + */ + fetchParallelIssues = async ( + workspaceSlug: string, + projectId: string, + loadType: TLoader = "init-loader", + options: IssuePaginationOptions, + cycleId: string, + groupId?: string, + subGroupId?: string ) => { try { // set loader and clear store runInAction(() => { - this.setLoader(loadType); + // set Loader + if (groupId || subGroupId) { + this.setLoader("pagination", groupId, subGroupId); + } else { + this.setLoader(loadType); + } }); - this.clear(!isExistingPaginationOptions); // get params from pagination options - const params = this.issueFilterStore?.getFilterParams(options, cycleId, undefined, undefined, undefined); + const params = this.issueFilterStore?.getFilterParams(options, cycleId, undefined, groupId, subGroupId); // call the fetch issues API with the params const response = await this.issueService.getIssues(workspaceSlug, projectId, params, { signal: this.controller.signal, }); // after fetching issues, call the base method to process the response further - this.onfetchIssues(response, options, workspaceSlug, projectId, cycleId); + this.onfetchIssues(response, options, groupId, subGroupId); return response; } catch (error) { // set loader to undefined once errored out diff --git a/web/core/store/issue/helpers/base-issues.store.ts b/web/core/store/issue/helpers/base-issues.store.ts index 5ba71080f4f..d1b9cdadeee 100644 --- a/web/core/store/issue/helpers/base-issues.store.ts +++ b/web/core/store/issue/helpers/base-issues.store.ts @@ -44,6 +44,7 @@ import { getSubGroupIssueKeyActions, } from "./base-issues-utils"; import { IBaseIssueFilterStore } from "./issue-filter-helper.store"; +import { getGroupByColumns } from "@/components/issues/issue-layouts/utils"; // constants // helpers // services @@ -421,6 +422,56 @@ export abstract class BaseIssuesStore implements IBaseIssuesStore { } ); + getGroups = (isWorkspaceLevel = false) => { + if (!this.groupBy || this.groupBy === "target_date") return; + + const { + projectRoot, + cycle, + module: moduleInfo, + label, + state: projectState, + memberRoot, + } = this.rootIssueStore.rootStore; + + return getGroupByColumns( + this.groupBy, + projectRoot.project, + cycle, + moduleInfo, + label, + projectState, + memberRoot, + false, + isWorkspaceLevel + ); + }; + + getSubGroups = (isWorkspaceLevel = false) => { + if (!this.subGroupBy || this.subGroupBy === "target_date") return; + + const { + projectRoot, + cycle, + module: moduleInfo, + label, + state: projectState, + memberRoot, + } = this.rootIssueStore.rootStore; + + return getGroupByColumns( + this.subGroupBy, + projectRoot.project, + cycle, + moduleInfo, + label, + projectState, + memberRoot, + false, + isWorkspaceLevel + ); + }; + /** * Gets the next page cursor based on number of issues currently available * @param groupId groupId for the cursor @@ -453,9 +504,8 @@ export abstract class BaseIssuesStore implements IBaseIssuesStore { onfetchIssues( issuesResponse: TIssuesResponse, options: IssuePaginationOptions, - workspaceSlug: string, - projectId?: string, - id?: string + groupId?: string, + subGroupId?: string ) { // Process the Issue Response to get the following data from it const { issueList, groupedIssues, groupedIssueCount } = this.processIssueResponse(issuesResponse); @@ -465,15 +515,12 @@ export abstract class BaseIssuesStore implements IBaseIssuesStore { // Update all the GroupIds to this Store's groupedIssueIds and update Individual group issue counts runInAction(() => { - this.updateGroupedIssueIds(groupedIssues, groupedIssueCount); - this.loader[getGroupKey()] = undefined; + this.updateGroupedIssueIds(groupedIssues, groupedIssueCount, groupId, subGroupId); + this.loader[getGroupKey(groupId, subGroupId)] = undefined; }); - // fetch parent stats if required, to be handled in the Implemented class - this.fetchParentStats(workspaceSlug, projectId, id); - // store Pagination options for next subsequent calls and data like next cursor etc - this.storePreviousPaginationValues(issuesResponse, options); + this.storePreviousPaginationValues(issuesResponse, options, groupId, subGroupId); } /** diff --git a/web/core/store/issue/helpers/issue-filter-helper.store.ts b/web/core/store/issue/helpers/issue-filter-helper.store.ts index 8ba9e47ea9f..64e6eaa94f9 100644 --- a/web/core/store/issue/helpers/issue-filter-helper.store.ts +++ b/web/core/store/issue/helpers/issue-filter-helper.store.ts @@ -282,7 +282,7 @@ export class IssueFilterHelperStore implements IIssueFilterHelperStore { subGroupId?: string ) { // if cursor exists, use the cursor. If it doesn't exist construct the cursor based on per page count - const pageCursor = cursor ? cursor : groupId ? `${options.perPageCount}:1:0` : `${options.perPageCount}:0:0`; + const pageCursor = cursor ? cursor : `${options.perPageCount}:0:0`; // pagination params const paginationParams: Partial> = { diff --git a/web/core/store/issue/module/issue.store.ts b/web/core/store/issue/module/issue.store.ts index 76cf6981d40..35c37703614 100644 --- a/web/core/store/issue/module/issue.store.ts +++ b/web/core/store/issue/module/issue.store.ts @@ -127,23 +127,79 @@ export class ModuleIssues extends BaseIssuesStore implements IModuleIssues { options: IssuePaginationOptions, moduleId: string, isExistingPaginationOptions: boolean = false + ) => { + try { + const groups = this.getGroups(); + const subGroups = this.getSubGroups(); + + this.clear(!isExistingPaginationOptions); + + if (!groups || (!groups && !subGroups)) + return await this.fetchParallelIssues(workspaceSlug, projectId, loadType, options, moduleId); + + const promises = []; + + for (const group of groups) { + if (!subGroups) { + promises.push(this.fetchParallelIssues(workspaceSlug, projectId, loadType, options, moduleId, group.id)); + continue; + } + + for (const subGroup of subGroups) { + promises.push( + this.fetchParallelIssues(workspaceSlug, projectId, loadType, options, moduleId, group.id, subGroup.id) + ); + } + } + + await Promise.all(promises); + + // fetch parent stats if required, to be handled in the Implemented class + this.fetchParentStats(workspaceSlug, projectId, moduleId); + } catch (error) { + // set loader to undefined if errored out + this.setLoader(undefined); + throw error; + } + }; + + /** + * This method is called to fetch the first issues of pagination + * @param workspaceSlug + * @param projectId + * @param loadType + * @param options + * @returns + */ + fetchParallelIssues = async ( + workspaceSlug: string, + projectId: string, + loadType: TLoader = "init-loader", + options: IssuePaginationOptions, + moduleId: string, + groupId?: string, + subGroupId?: string ) => { try { // set loader and clear store runInAction(() => { - this.setLoader(loadType); + // set Loader + if (groupId || subGroupId) { + this.setLoader("pagination", groupId, subGroupId); + } else { + this.setLoader(loadType); + } }); - this.clear(!isExistingPaginationOptions); // get params from pagination options - const params = this.issueFilterStore?.getFilterParams(options, moduleId, undefined, undefined, undefined); + const params = this.issueFilterStore?.getFilterParams(options, moduleId, undefined, groupId, subGroupId); // call the fetch issues API with the params const response = await this.issueService.getIssues(workspaceSlug, projectId, params, { signal: this.controller.signal, }); // after fetching issues, call the base method to process the response further - this.onfetchIssues(response, options, workspaceSlug, projectId, moduleId); + this.onfetchIssues(response, options, groupId, subGroupId); return response; } catch (error) { // set loader to undefined once errored out diff --git a/web/core/store/issue/project-views/issue.store.ts b/web/core/store/issue/project-views/issue.store.ts index f2e9ba02eb4..f7871e699d9 100644 --- a/web/core/store/issue/project-views/issue.store.ts +++ b/web/core/store/issue/project-views/issue.store.ts @@ -88,23 +88,76 @@ export class ProjectViewIssues extends BaseIssuesStore implements IProjectViewIs loadType: TLoader, options: IssuePaginationOptions, isExistingPaginationOptions: boolean = false + ) => { + try { + const groups = this.getGroups(); + const subGroups = this.getSubGroups(); + + this.clear(!isExistingPaginationOptions); + + if (!groups || (!groups && !subGroups)) + return await this.fetchParallelIssues(workspaceSlug, projectId, loadType, options, viewId); + + const promises = []; + + for (const group of groups) { + if (!subGroups) { + promises.push(this.fetchParallelIssues(workspaceSlug, projectId, loadType, options, viewId, group.id)); + continue; + } + + for (const subGroup of subGroups) { + promises.push( + this.fetchParallelIssues(workspaceSlug, projectId, loadType, options, viewId, group.id, subGroup.id) + ); + } + } + + await Promise.all(promises); + } catch (error) { + // set loader to undefined if errored out + this.setLoader(undefined); + throw error; + } + }; + + /** + * This method is called to fetch the first issues of pagination + * @param workspaceSlug + * @param projectId + * @param loadType + * @param options + * @returns + */ + fetchParallelIssues = async ( + workspaceSlug: string, + projectId: string, + loadType: TLoader = "init-loader", + options: IssuePaginationOptions, + viewId: string, + groupId?: string, + subGroupId?: string ) => { try { // set loader and clear store runInAction(() => { - this.setLoader(loadType); + // set Loader + if (groupId || subGroupId) { + this.setLoader("pagination", groupId, subGroupId); + } else { + this.setLoader(loadType); + } }); - this.clear(!isExistingPaginationOptions); // get params from pagination options - const params = this.issueFilterStore?.getFilterParams(options, viewId, undefined, undefined, undefined); + const params = this.issueFilterStore?.getFilterParams(options, viewId, undefined, groupId, subGroupId); // call the fetch issues API with the params const response = await this.issueService.getIssues(workspaceSlug, projectId, params, { signal: this.controller.signal, }); // after fetching issues, call the base method to process the response further - this.onfetchIssues(response, options, workspaceSlug, projectId); + this.onfetchIssues(response, options, groupId, subGroupId); return response; } catch (error) { // set loader to undefined if errored out diff --git a/web/core/store/issue/project/issue.store.ts b/web/core/store/issue/project/issue.store.ts index 66f152ac65d..7546a08bc7e 100644 --- a/web/core/store/issue/project/issue.store.ts +++ b/web/core/store/issue/project/issue.store.ts @@ -96,23 +96,76 @@ export class ProjectIssues extends BaseIssuesStore implements IProjectIssues { loadType: TLoader = "init-loader", options: IssuePaginationOptions, isExistingPaginationOptions: boolean = false + ) => { + try { + const groups = this.getGroups(); + const subGroups = this.getSubGroups(); + + this.clear(!isExistingPaginationOptions); + + if (!groups || (!groups && !subGroups)) + return await this.fetchParallelIssues(workspaceSlug, projectId, loadType, options); + + const promises = []; + + for (const group of groups) { + if (!subGroups) { + promises.push(this.fetchParallelIssues(workspaceSlug, projectId, loadType, options, group.id)); + continue; + } + + for (const subGroup of subGroups) { + promises.push(this.fetchParallelIssues(workspaceSlug, projectId, loadType, options, group.id, subGroup.id)); + } + } + + await Promise.all(promises); + + // fetch parent stats if required, to be handled in the Implemented class + this.fetchParentStats(workspaceSlug, projectId); + } catch (error) { + // set loader to undefined if errored out + this.setLoader(undefined); + throw error; + } + }; + + /** + * This method is called to fetch the first issues of pagination + * @param workspaceSlug + * @param projectId + * @param loadType + * @param options + * @returns + */ + fetchParallelIssues = async ( + workspaceSlug: string, + projectId: string, + loadType: TLoader = "init-loader", + options: IssuePaginationOptions, + groupId?: string, + subGroupId?: string ) => { try { // set loader and clear store runInAction(() => { - this.setLoader(loadType); + // set Loader + if (groupId || subGroupId) { + this.setLoader("pagination", groupId, subGroupId); + } else { + this.setLoader(loadType); + } }); - this.clear(!isExistingPaginationOptions); // get params from pagination options - const params = this.issueFilterStore?.getFilterParams(options, projectId, undefined, undefined, undefined); + const params = this.issueFilterStore?.getFilterParams(options, projectId, undefined, groupId, subGroupId); // call the fetch issues API with the params const response = await this.issueService.getIssues(workspaceSlug, projectId, params, { signal: this.controller.signal, }); // after fetching issues, call the base method to process the response further - this.onfetchIssues(response, options, workspaceSlug, projectId); + this.onfetchIssues(response, options, groupId, subGroupId); return response; } catch (error) { // set loader to undefined if errored out From 705f23fe2d537c046990164be1a642d59ed4b674 Mon Sep 17 00:00:00 2001 From: NarayanBavisetti Date: Tue, 6 Aug 2024 14:11:08 +0530 Subject: [PATCH 009/111] chore: deleted issue list --- apiserver/plane/app/urls/issue.py | 6 ++++++ apiserver/plane/app/views/__init__.py | 1 + apiserver/plane/app/views/issue/base.py | 22 +++++++++++++++++++++- 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/apiserver/plane/app/urls/issue.py b/apiserver/plane/app/urls/issue.py index aa6a8e2f06c..a8526191614 100644 --- a/apiserver/plane/app/urls/issue.py +++ b/apiserver/plane/app/urls/issue.py @@ -21,6 +21,7 @@ LabelViewSet, BulkIssueOperationsEndpoint, BulkArchiveIssuesEndpoint, + DeletedIssuesListViewSet, ) urlpatterns = [ @@ -310,4 +311,9 @@ BulkIssueOperationsEndpoint.as_view(), name="bulk-operations-issues", ), + path( + "workspaces//projects//deleted-issues/", + DeletedIssuesListViewSet.as_view(), + name="deleted-issues", + ), ] diff --git a/apiserver/plane/app/views/__init__.py b/apiserver/plane/app/views/__init__.py index 9d8929fda51..a537f104bfa 100644 --- a/apiserver/plane/app/views/__init__.py +++ b/apiserver/plane/app/views/__init__.py @@ -112,6 +112,7 @@ IssueViewSet, IssueUserDisplayPropertyEndpoint, BulkDeleteIssuesEndpoint, + DeletedIssuesListViewSet, ) from .issue.activity import ( diff --git a/apiserver/plane/app/views/issue/base.py b/apiserver/plane/app/views/issue/base.py index e0ec0193616..c491557e4b5 100644 --- a/apiserver/plane/app/views/issue/base.py +++ b/apiserver/plane/app/views/issue/base.py @@ -234,10 +234,15 @@ def get_queryset(self): @method_decorator(gzip_page) def list(self, request, slug, project_id): + extra_filters = {} + if request.GET.get("updated_at__gt", None) is not None: + extra_filters = { + "updated_at__gt": request.GET.get("updated_at__gt") + } filters = issue_filters(request.query_params, "GET") order_by_param = request.GET.get("order_by", "-created_at") - issue_queryset = self.get_queryset().filter(**filters) + issue_queryset = self.get_queryset().filter(**filters, **extra_filters) # Custom ordering for priority and state # Issue queryset @@ -652,3 +657,18 @@ def delete(self, request, slug, project_id): {"message": f"{total_issues} issues were deleted"}, status=status.HTTP_200_OK, ) + + +class DeletedIssuesListViewSet(BaseAPIView): + permission_classes = [ + ProjectEntityPermission, + ] + + def get(self, request, slug, project_id): + deleted_issues = Issue.all_objects.filter( + workspace__slug=slug, + project_id=project_id, + deleted_at__isnull=False, + ).values_list("id", flat=True) + + return Response(deleted_issues, status=status.HTTP_200_OK) From 8ecdd12e482e661a870dc37a0408d6dac89f540b Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Wed, 7 Aug 2024 16:52:39 +0530 Subject: [PATCH 010/111] - Handle local mutations - Implement getting the updates - Use SWR to update/sync data --- apiserver/plane/app/views/issue/base.py | 2 + apiserver/plane/utils/paginator.py | 10 +-- .../issue-layouts/kanban/base-kanban-root.tsx | 3 +- .../layouts/auth-layout/project-wrapper.tsx | 30 ++++++-- web/core/local-db/indexes.ts | 5 +- web/core/local-db/load-issues.ts | 77 ++++++++++++++++--- web/core/local-db/queries/issues.ts | 14 +--- web/core/local-db/utils.ts | 48 ++++++++++++ web/core/services/issue/issue.service.ts | 33 ++++++-- 9 files changed, 180 insertions(+), 42 deletions(-) create mode 100644 web/core/local-db/utils.ts diff --git a/apiserver/plane/app/views/issue/base.py b/apiserver/plane/app/views/issue/base.py index c491557e4b5..86896d8d2d3 100644 --- a/apiserver/plane/app/views/issue/base.py +++ b/apiserver/plane/app/views/issue/base.py @@ -239,6 +239,8 @@ def list(self, request, slug, project_id): extra_filters = { "updated_at__gt": request.GET.get("updated_at__gt") } + + print (extra_filters) filters = issue_filters(request.query_params, "GET") order_by_param = request.GET.get("order_by", "-created_at") diff --git a/apiserver/plane/utils/paginator.py b/apiserver/plane/utils/paginator.py index 3ea74bf9be4..65f0aa7f746 100644 --- a/apiserver/plane/utils/paginator.py +++ b/apiserver/plane/utils/paginator.py @@ -82,7 +82,7 @@ def __repr__(self): return f"<{type(self).__name__}: results={len(self.results)}>" -MAX_LIMIT = 100 +MAX_LIMIT = 1000 class BadPaginationError(Exception): @@ -118,7 +118,7 @@ def __init__( self.max_offset = max_offset self.on_results = on_results - def get_result(self, limit=100, cursor=None): + def get_result(self, limit=1000, cursor=None): # offset is page # # value is page limit if cursor is None: @@ -727,7 +727,7 @@ class BasePaginator: cursor_name = "cursor" # get the per page parameter from request - def get_per_page(self, request, default_per_page=100, max_per_page=100): + def get_per_page(self, request, default_per_page=1000, max_per_page=1000): try: per_page = int(request.GET.get("per_page", default_per_page)) except ValueError: @@ -747,8 +747,8 @@ def paginate( on_results=None, paginator=None, paginator_cls=OffsetPaginator, - default_per_page=100, - max_per_page=100, + default_per_page=1000, + max_per_page=1000, cursor_cls=Cursor, extra_stats=None, controller=None, diff --git a/web/core/components/issues/issue-layouts/kanban/base-kanban-root.tsx b/web/core/components/issues/issue-layouts/kanban/base-kanban-root.tsx index 4ba7a7a21ec..dfdd802f603 100644 --- a/web/core/components/issues/issue-layouts/kanban/base-kanban-root.tsx +++ b/web/core/components/issues/issue-layouts/kanban/base-kanban-root.tsx @@ -10,7 +10,7 @@ import { useParams, usePathname } from "next/navigation"; import { DeleteIssueModal } from "@/components/issues"; //constants import { ISSUE_DELETED } from "@/constants/event-tracker"; -import { EIssueFilterType, EIssueLayoutTypes, EIssuesStoreType } from "@/constants/issue"; +import { EIssueFilterType, EIssuesStoreType } from "@/constants/issue"; import { EUserProjectRoles } from "@/constants/project"; //hooks import { useEventTracker, useIssueDetail, useIssues, useKanbanView, useUser } from "@/hooks/store"; @@ -20,7 +20,6 @@ import { useIssuesActions } from "@/hooks/use-issues-actions"; // store // ui // types -import { IssueLayoutHOC } from "../issue-layout-HOC"; import { IQuickActionProps, TRenderQuickActions } from "../list/list-view-types"; //components import { getSourceFromDropPayload } from "../utils"; diff --git a/web/core/layouts/auth-layout/project-wrapper.tsx b/web/core/layouts/auth-layout/project-wrapper.tsx index a59352154ef..88846f1e9c2 100644 --- a/web/core/layouts/auth-layout/project-wrapper.tsx +++ b/web/core/layouts/auth-layout/project-wrapper.tsx @@ -34,7 +34,7 @@ export const ProjectAuthWrapper: FC = observer((props) => { // const { fetchInboxes } = useInbox(); const { toggleCreateProjectModal } = useCommandPalette(); const { setTrackElement } = useEventTracker(); - const [areIssuesLoading, setIssuesLoading] = useState(false); + // const [areIssuesLoading, setIssuesLoading] = useState(false); const { membership: { fetchUserProjectInfo, projectMemberInfo, hasPermissionToProject }, } = useUser(); @@ -51,14 +51,28 @@ export const ProjectAuthWrapper: FC = observer((props) => { // router const { workspaceSlug, projectId } = useParams(); - useEffect(() => { - (async () => { - setIssuesLoading(true); + const { isLoading: seedingInProgress, isValidating } = useSWR( + workspaceSlug && projectId ? `PROJECT_${workspaceSlug}_${projectId}` : null, + async () => { await initializeSQLite(); await loadIssues(workspaceSlug.toString(), projectId.toString()); - setIssuesLoading(false); - })(); - }, [workspaceSlug, projectId]); + }, + { + revalidateIfStale: true, + revalidateOnFocus: true, + revalidateOnReconnect: true, + refreshInterval: 5 * 60 * 1000, + } + ); + // useEffect(() => { + // (async () => { + // setIssuesLoading(true); + // await initializeSQLite(); + + // await loadIssues(workspaceSlug.toString(), projectId.toString()); + // setIssuesLoading(false); + // })(); + // }, [workspaceSlug, projectId]); // fetching project details useSWR( workspaceSlug && projectId ? `PROJECT_DETAILS_${workspaceSlug.toString()}_${projectId.toString()}` : null, @@ -120,7 +134,7 @@ export const ProjectAuthWrapper: FC = observer((props) => { isEstimatesLoading || isCyclesLoading || isModulesLoading || - areIssuesLoading; + seedingInProgress; // check if the project member apis is loading if ((!projectMemberInfo && projectId && hasPermissionToProject[projectId.toString()] === null) || isLoading) diff --git a/web/core/local-db/indexes.ts b/web/core/local-db/indexes.ts index 8ace6dd59bc..8c35edf2304 100644 --- a/web/core/local-db/indexes.ts +++ b/web/core/local-db/indexes.ts @@ -1,5 +1,6 @@ import { SQL } from "./sqlite"; +const log = console.log; export const createIssueIndexes = async () => { const columns = [ "state_id", @@ -37,10 +38,10 @@ export const createIssueMetaIndexes = async () => { }; const createIndexes = async () => { - console.log("### Creating indexes"); + log("### Creating indexes"); const promises = [createIssueIndexes(), createIssueMetaIndexes()]; await Promise.all(promises); - console.log("### Indexes created"); + log("### Indexes created"); }; export default createIndexes; diff --git a/web/core/local-db/load-issues.ts b/web/core/local-db/load-issues.ts index 4c6c636700f..0d26339e79a 100644 --- a/web/core/local-db/load-issues.ts +++ b/web/core/local-db/load-issues.ts @@ -3,8 +3,10 @@ import { IssueService } from "@/services/issue"; import createIndexes from "./indexes"; import { runQuery } from "./query-executor"; import { SQL } from "./sqlite"; +import { log } from "./utils"; const arrayFields = ["label_ids", "assignee_ids", "module_ids"]; +const PAGE_SIZE = 1000; export const addIssue = async (issue: any) => { const issue_id = issue.id; @@ -46,40 +48,95 @@ export const addIssue = async (issue: any) => { SQL.db.exec("COMMIT;"); // await Promise.all(promises); - console.log("### Added issue", issue.id); + // log("### Added issue", issue.id); }; +export const deleteIssueFromLocal = async (issue_id: any) => { + const deleteQuery = `delete from issues where id='${issue_id}'`; + const deleteMetaQuery = `delete from issue_meta where issue_id='${issue_id}'`; + + SQL.db.exec("BEGIN TRANSACTION;"); + SQL.db.exec(deleteQuery); + SQL.db.exec(deleteMetaQuery); + SQL.db.exec("COMMIT;"); +}; + +export const updateIssue = async (issue: any) => { + const issue_id = issue.id; + // delete the issue and its meta data + await deleteIssueFromLocal(issue_id); + addIssue(issue); +}; export const loadIssues = async (workspaceId: string, projectId: string) => { // Load issues from the API const issueService = new IssueService(); - const PAGE_SIZE = 100; - let cursor = `${PAGE_SIZE}:0:0`; - let results; - let breakLoop = false; let count = await runQuery(`select count(*) as count from issues where project_id='${projectId}'`); - console.log("### Count", count); + log("### Count", count); count = count[0]["count"]; - console.log("### Count", count, typeof count); + log("### Count", count, typeof count); if (!count) { + let cursor = `${PAGE_SIZE}:0:0`; + let results; + let breakLoop = false; do { const response = await issueService.getIssuesFromServer(workspaceId, projectId, { cursor }); cursor = response.next_cursor; results = response.results as TBaseIssue[]; - console.log("#### Loading issues", results.length); + log("#### Loading issues", results.length); results.map(async (issue) => { try { await addIssue(issue); } catch (e) { - console.log("###Error", e, issue); + log("###Error", e, issue); breakLoop = true; } }); } while (results.length >= PAGE_SIZE && !breakLoop); } else { - console.log("### Project already stored locally, call update issues"); + syncLocalData(workspaceId, projectId); } createIndexes(); }; + +export const syncUpdatesToLocal = async (workspaceId: string, projectId: string) => { + let cursor = `${PAGE_SIZE}:0:0`; + let results; + let breakLoop = false; + + // get the last updated issue + const lastUpdatedIssue = await runQuery( + `select id, name, updated_at , sequence_id from issues where project_id='${projectId}' order by date(updated_at) desc limit 1` + ); + + // eslint-disable-next-line @typescript-eslint/naming-convention + const updated_at__gt = lastUpdatedIssue[0]["updated_at"]; + const issueService = new IssueService(); + + do { + const response = await issueService.getIssuesFromServer(workspaceId, projectId, { cursor, updated_at__gt }); + cursor = response.next_cursor; + results = response.results as TBaseIssue[]; + log("#### Loading issues", results.length); + results.map(async (issue) => { + try { + await updateIssue(issue); + } catch (e) { + log("###Error", e, issue); + breakLoop = true; + } + }); + } while (results.length >= PAGE_SIZE && !breakLoop); +}; + +export const syncDeletesToLocal = async (workspaceId: string, projectId: string) => { + const issueService = new IssueService(); + const response = await issueService.getDeletedIssues(workspaceId, projectId); + response.map(async (issue) => deleteIssueFromLocal(issue)); +}; + +export const syncLocalData = async (workspaceId: string, projectId: string) => { + await Promise.all([syncDeletesToLocal(workspaceId, projectId), syncUpdatesToLocal(workspaceId, projectId)]); +}; diff --git a/web/core/local-db/queries/issues.ts b/web/core/local-db/queries/issues.ts index c68f503bfaf..edb1ac1b73e 100644 --- a/web/core/local-db/queries/issues.ts +++ b/web/core/local-db/queries/issues.ts @@ -7,24 +7,20 @@ const arrayFields = ["label_ids", "assignee_ids", "module_ids"]; export const getIssues = async (workspaceSlug: string, projectId: string, queries?: any, config = {}): Promise => { const { cursor } = queries; - console.log("## queries", queries); const query = issueFilterQueryConstructor(workspaceSlug, projectId, queries); - console.log("SQL", query); const countQuery = issueFilterCountQueryConstructor(workspaceSlug, projectId, queries); const start = performance.now(); - let issues = await runQuery(query); + const [issuesRaw, count] = await Promise.all([runQuery(query), runQuery(countQuery)]); const end = performance.now(); - const countStart = performance.now(); - const { total_count } = (await runQuery(countQuery))[0]; - const countEnd = performance.now(); + const { total_count } = count[0]; const [pageSize, page, offset] = cursor.split(":"); const parsingStart = performance.now(); - issues = issues.map((issue: TIssue) => { + const issues = issuesRaw.map((issue: TIssue) => { arrayFields.forEach((field) => { - issue[field] = JSON.parse(issue[field]); + issue[field] = issue[field] ? JSON.parse(issue[field]) : []; }); return issue; @@ -34,7 +30,6 @@ export const getIssues = async (workspaceSlug: string, projectId: string, querie const times = { IssueQuery: end - start, - Count: countEnd - countStart, Parsing: parsingEnd - parsingStart, }; @@ -54,6 +49,5 @@ export const getIssues = async (workspaceSlug: string, projectId: string, querie total_pages, }; - console.log(out); return out; }; diff --git a/web/core/local-db/utils.ts b/web/core/local-db/utils.ts new file mode 100644 index 00000000000..6993ee75031 --- /dev/null +++ b/web/core/local-db/utils.ts @@ -0,0 +1,48 @@ +import pick from "lodash/pick"; +import { rootStore } from "@/lib/store-context"; +import { updateIssue } from "./load-issues"; + +export const log = console.log; + +// export const log = () => {}; + +export const updatePersistentLayer = async (issueIds: string | string[]) => { + if (typeof issueIds === "string") { + issueIds = [issueIds]; + } + issueIds.forEach((issueId) => { + const issue = rootStore.issue.issues.getIssueById(issueId); + + if (issue) { + const issuePartial = pick(JSON.parse(JSON.stringify(issue)), [ + "id", + "name", + "state_id", + "sort_order", + "completed_at", + "estimate_point", + "priority", + "start_date", + "target_date", + "sequence_id", + "project_id", + "parent_id", + "created_at", + "updated_at", + "created_by", + "updated_by", + "is_draft", + "archived_at", + "state__group", + "cycle_id", + "link_count", + "attachment_count", + "sub_issues_count", + "assignee_ids", + "label_ids", + "module_ids", + ]); + updateIssue(issuePartial); + } + }); +}; diff --git a/web/core/services/issue/issue.service.ts b/web/core/services/issue/issue.service.ts index 01e1808d034..cda111b2716 100644 --- a/web/core/services/issue/issue.service.ts +++ b/web/core/services/issue/issue.service.ts @@ -1,17 +1,19 @@ // types import type { - TIssue, IIssueDisplayProperties, - TIssueLink, - TIssueSubIssues, + TBulkOperationsPayload, + TIssue, TIssueActivity, + TIssueLink, TIssuesResponse, - TBulkOperationsPayload, + TIssueSubIssues, } from "@plane/types"; // helpers import { API_BASE_URL } from "@/helpers/common.helper"; // services +import { deleteIssueFromLocal } from "@/local-db/load-issues"; import { getIssues } from "@/local-db/queries/issues"; +import { updatePersistentLayer } from "@/local-db/utils"; import { APIService } from "@/services/api.service"; export class IssueService extends APIService { @@ -21,7 +23,10 @@ export class IssueService extends APIService { async createIssue(workspaceSlug: string, projectId: string, data: Partial): Promise { return this.post(`/api/workspaces/${workspaceSlug}/projects/${projectId}/issues/`, data) - .then((response) => response?.data) + .then((response) => { + updatePersistentLayer(response?.data?.id); + return response?.data; + }) .catch((error) => { throw error?.response?.data; }); @@ -50,6 +55,16 @@ export class IssueService extends APIService { return await getIssues(workspaceSlug, projectId, queries, config); } + async getDeletedIssues(workspaceSlug: string, projectId: string, queries?: any): Promise { + return this.get(`/api/workspaces/${workspaceSlug}/projects/${projectId}/deleted-issues/`, { + params: queries, + }) + .then((response) => response?.data) + .catch((error) => { + throw error?.response?.data; + }); + } + async getIssuesWithParams( workspaceSlug: string, projectId: string, @@ -100,6 +115,7 @@ export class IssueService extends APIService { issues: string[]; } ) { + updatePersistentLayer(data.issues); return this.post(`/api/workspaces/${workspaceSlug}/projects/${projectId}/cycles/${cycleId}/cycle-issues/`, data) .then((response) => response?.data) .catch((error) => { @@ -129,6 +145,7 @@ export class IssueService extends APIService { relation?: "blocking" | null; } ) { + updatePersistentLayer(issueId); return this.post(`/api/workspaces/${workspaceSlug}/projects/${projectId}/issues/${issueId}/issue-relation/`, data) .then((response) => response?.data) .catch((error) => { @@ -169,6 +186,7 @@ export class IssueService extends APIService { } async patchIssue(workspaceSlug: string, projectId: string, issueId: string, data: Partial): Promise { + updatePersistentLayer(issueId); return this.patch(`/api/workspaces/${workspaceSlug}/projects/${projectId}/issues/${issueId}/`, data) .then((response) => response?.data) .catch((error) => { @@ -177,6 +195,7 @@ export class IssueService extends APIService { } async deleteIssue(workspaceSlug: string, projectId: string, issuesId: string): Promise { + deleteIssueFromLocal(issuesId); return this.delete(`/api/workspaces/${workspaceSlug}/projects/${projectId}/issues/${issuesId}/`) .then((response) => response?.data) .catch((error) => { @@ -198,6 +217,7 @@ export class IssueService extends APIService { issueId: string, data: { sub_issue_ids: string[] } ): Promise { + updatePersistentLayer([issueId, ...data.sub_issue_ids]); return this.post(`/api/workspaces/${workspaceSlug}/projects/${projectId}/issues/${issueId}/sub-issues/`, data) .then((response) => response?.data) .catch((error) => { @@ -219,6 +239,7 @@ export class IssueService extends APIService { issueId: string, data: Partial ): Promise { + updatePersistentLayer(issueId); return this.post(`/api/workspaces/${workspaceSlug}/projects/${projectId}/issues/${issueId}/issue-links/`, data) .then((response) => response?.data) .catch((error) => { @@ -233,6 +254,7 @@ export class IssueService extends APIService { linkId: string, data: Partial ): Promise { + updatePersistentLayer(issueId); return this.patch( `/api/workspaces/${workspaceSlug}/projects/${projectId}/issues/${issueId}/issue-links/${linkId}/`, data @@ -244,6 +266,7 @@ export class IssueService extends APIService { } async deleteIssueLink(workspaceSlug: string, projectId: string, issueId: string, linkId: string): Promise { + updatePersistentLayer(issueId); return this.delete( `/api/workspaces/${workspaceSlug}/projects/${projectId}/issues/${issueId}/issue-links/${linkId}/` ) From 5e120f7f3bf1969c50e4b6c7643e0782aa08a9cf Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Wed, 7 Aug 2024 18:13:59 +0530 Subject: [PATCH 011/111] Wait for sync to complete in get issues --- .../layouts/auth-layout/project-wrapper.tsx | 43 +++++++------------ web/core/local-db/load-issues.ts | 15 +++++-- web/core/local-db/queries/issues.ts | 2 + web/core/local-db/sqlite.ts | 22 +++------- 4 files changed, 37 insertions(+), 45 deletions(-) diff --git a/web/core/layouts/auth-layout/project-wrapper.tsx b/web/core/layouts/auth-layout/project-wrapper.tsx index 88846f1e9c2..708518c8420 100644 --- a/web/core/layouts/auth-layout/project-wrapper.tsx +++ b/web/core/layouts/auth-layout/project-wrapper.tsx @@ -1,4 +1,4 @@ -import { FC, ReactNode, useEffect, useState } from "react"; +import { FC, ReactNode } from "react"; import { observer } from "mobx-react"; import { useParams } from "next/navigation"; import useSWR from "swr"; @@ -7,17 +7,17 @@ import { JoinProject } from "@/components/auth-screens"; import { EmptyState, LogoSpinner } from "@/components/common"; // hooks import { - useEventTracker, + useCommandPalette, useCycle, - useProjectEstimates, + useEventTracker, useLabel, useMember, useModule, useProject, + useProjectEstimates, useProjectState, useProjectView, useUser, - useCommandPalette, } from "@/hooks/store"; // images import { loadIssues } from "@/local-db/load-issues"; @@ -34,7 +34,6 @@ export const ProjectAuthWrapper: FC = observer((props) => { // const { fetchInboxes } = useInbox(); const { toggleCreateProjectModal } = useCommandPalette(); const { setTrackElement } = useEventTracker(); - // const [areIssuesLoading, setIssuesLoading] = useState(false); const { membership: { fetchUserProjectInfo, projectMemberInfo, hasPermissionToProject }, } = useUser(); @@ -51,12 +50,16 @@ export const ProjectAuthWrapper: FC = observer((props) => { // router const { workspaceSlug, projectId } = useParams(); - const { isLoading: seedingInProgress, isValidating } = useSWR( - workspaceSlug && projectId ? `PROJECT_${workspaceSlug}_${projectId}` : null, - async () => { - await initializeSQLite(); - await loadIssues(workspaceSlug.toString(), projectId.toString()); - }, + useSWR( + workspaceSlug && projectId ? `PROJECT_${workspaceSlug.toString()}_${projectId.toString()}` : null, + workspaceSlug && projectId + ? () => { + (async () => { + await initializeSQLite(); + await loadIssues(workspaceSlug.toString(), projectId.toString()); + })(); + } + : null, { revalidateIfStale: true, revalidateOnFocus: true, @@ -64,20 +67,12 @@ export const ProjectAuthWrapper: FC = observer((props) => { refreshInterval: 5 * 60 * 1000, } ); - // useEffect(() => { - // (async () => { - // setIssuesLoading(true); - // await initializeSQLite(); - // await loadIssues(workspaceSlug.toString(), projectId.toString()); - // setIssuesLoading(false); - // })(); - // }, [workspaceSlug, projectId]); - // fetching project details useSWR( workspaceSlug && projectId ? `PROJECT_DETAILS_${workspaceSlug.toString()}_${projectId.toString()}` : null, workspaceSlug && projectId ? () => fetchProjectDetails(workspaceSlug.toString(), projectId.toString()) : null ); + // fetching user project member information useSWR( workspaceSlug && projectId ? `PROJECT_MEMBERS_ME_${workspaceSlug}_${projectId}` : null, @@ -128,13 +123,7 @@ export const ProjectAuthWrapper: FC = observer((props) => { const projectExists = projectId ? getProjectById(projectId.toString()) : null; const isLoading = - isLabelsLoading || - isMembersLoading || - isStateLoading || - isEstimatesLoading || - isCyclesLoading || - isModulesLoading || - seedingInProgress; + isLabelsLoading || isMembersLoading || isStateLoading || isEstimatesLoading || isCyclesLoading || isModulesLoading; // check if the project member apis is loading if ((!projectMemberInfo && projectId && hasPermissionToProject[projectId.toString()] === null) || isLoading) diff --git a/web/core/local-db/load-issues.ts b/web/core/local-db/load-issues.ts index 0d26339e79a..96eadd85913 100644 --- a/web/core/local-db/load-issues.ts +++ b/web/core/local-db/load-issues.ts @@ -67,7 +67,7 @@ export const updateIssue = async (issue: any) => { await deleteIssueFromLocal(issue_id); addIssue(issue); }; -export const loadIssues = async (workspaceId: string, projectId: string) => { +export const loadIssuesPrivate = async (workspaceId: string, projectId: string) => { // Load issues from the API const issueService = new IssueService(); @@ -94,11 +94,15 @@ export const loadIssues = async (workspaceId: string, projectId: string) => { } }); } while (results.length >= PAGE_SIZE && !breakLoop); + createIndexes(); } else { syncLocalData(workspaceId, projectId); } +}; - createIndexes(); +export const loadIssues = async (workspaceId: string, projectId: string) => { + SQL.syncInProgress = loadIssuesPrivate(workspaceId, projectId); + await SQL.syncInProgress; }; export const syncUpdatesToLocal = async (workspaceId: string, projectId: string) => { @@ -138,5 +142,10 @@ export const syncDeletesToLocal = async (workspaceId: string, projectId: string) }; export const syncLocalData = async (workspaceId: string, projectId: string) => { - await Promise.all([syncDeletesToLocal(workspaceId, projectId), syncUpdatesToLocal(workspaceId, projectId)]); + SQL.syncInProgress = Promise.all([ + syncDeletesToLocal(workspaceId, projectId), + syncUpdatesToLocal(workspaceId, projectId), + ]); + + await SQL.syncInProgress; }; diff --git a/web/core/local-db/queries/issues.ts b/web/core/local-db/queries/issues.ts index edb1ac1b73e..4e092dac745 100644 --- a/web/core/local-db/queries/issues.ts +++ b/web/core/local-db/queries/issues.ts @@ -1,12 +1,14 @@ import { TIssue } from "@plane/types"; import { issueFilterCountQueryConstructor, issueFilterQueryConstructor } from "../query-constructor"; import { runQuery } from "../query-executor"; +import { SQL } from "../sqlite"; const arrayFields = ["label_ids", "assignee_ids", "module_ids"]; export const getIssues = async (workspaceSlug: string, projectId: string, queries?: any, config = {}): Promise => { const { cursor } = queries; + await SQL.syncInProgress; const query = issueFilterQueryConstructor(workspaceSlug, projectId, queries); const countQuery = issueFilterCountQueryConstructor(workspaceSlug, projectId, queries); const start = performance.now(); diff --git a/web/core/local-db/sqlite.ts b/web/core/local-db/sqlite.ts index fff2173b159..633b74c74a5 100644 --- a/web/core/local-db/sqlite.ts +++ b/web/core/local-db/sqlite.ts @@ -6,30 +6,22 @@ declare module "@sqlite.org/sqlite-wasm" { export function sqlite3Worker1Promiser(...args: any): any; } +type TSQL = { + db?: any; + initialized: boolean; + syncInProgress: boolean | Promise; +}; + const log = console.log; const error = console.error; -const SQL = { initialized: false }; +const SQL: TSQL = { initialized: false, syncInProgress: false }; const start = async (sqlite3: any) => { log("Running SQLite3 version", sqlite3.version.libVersion); SQL.db = new sqlite3.oo1.DB("/mydb.sqlite3", "ct"); createTables(SQL.db); }; -const initializeSQLiteMemory = async () => { - try { - log("Loading and initializing SQLite3 module..."); - const sqlite3 = await sqlite3InitModule({ - print: log, - printErr: error, - }); - log("Done initializing. Running demo..."); - await start(sqlite3); - } catch (err) { - error("Initialization error:", err.name, err.message); - } -}; - const initializeSQLite = async () => { if (SQL.initialized) { console.info("Instance already initialized"); From 620bd0c300eb7a22109cd7cb93e56db2e543cf21 Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Wed, 7 Aug 2024 18:35:41 +0530 Subject: [PATCH 012/111] Fix build errors --- web/core/local-db/load-issues.ts | 6 ++++-- web/core/local-db/queries/issues.ts | 5 ++--- web/core/local-db/query-constructor.ts | 4 ++-- web/core/local-db/query-executor.ts | 2 +- web/core/local-db/sqlite.ts | 14 +++----------- web/core/local-db/tables.ts | 6 +++--- 6 files changed, 15 insertions(+), 22 deletions(-) diff --git a/web/core/local-db/load-issues.ts b/web/core/local-db/load-issues.ts index 96eadd85913..3741e6eb65f 100644 --- a/web/core/local-db/load-issues.ts +++ b/web/core/local-db/load-issues.ts @@ -35,7 +35,7 @@ export const addIssue = async (issue: any) => { arrayFields.forEach((field) => { const values = issue[field]; if (values) { - values.forEach((val) => { + values.forEach((val: any) => { // promises.push( SQL.db.exec({ sql: `insert into issue_meta(issue_id,key,value) values (?,?,?)`, @@ -138,7 +138,9 @@ export const syncUpdatesToLocal = async (workspaceId: string, projectId: string) export const syncDeletesToLocal = async (workspaceId: string, projectId: string) => { const issueService = new IssueService(); const response = await issueService.getDeletedIssues(workspaceId, projectId); - response.map(async (issue) => deleteIssueFromLocal(issue)); + if (Array.isArray(response)) { + response.map(async (issue) => deleteIssueFromLocal(issue)); + } }; export const syncLocalData = async (workspaceId: string, projectId: string) => { diff --git a/web/core/local-db/queries/issues.ts b/web/core/local-db/queries/issues.ts index 4e092dac745..cc551298be1 100644 --- a/web/core/local-db/queries/issues.ts +++ b/web/core/local-db/queries/issues.ts @@ -1,4 +1,3 @@ -import { TIssue } from "@plane/types"; import { issueFilterCountQueryConstructor, issueFilterQueryConstructor } from "../query-constructor"; import { runQuery } from "../query-executor"; import { SQL } from "../sqlite"; @@ -20,8 +19,8 @@ export const getIssues = async (workspaceSlug: string, projectId: string, querie const [pageSize, page, offset] = cursor.split(":"); const parsingStart = performance.now(); - const issues = issuesRaw.map((issue: TIssue) => { - arrayFields.forEach((field) => { + const issues = issuesRaw.map((issue: any) => { + arrayFields.forEach((field: string) => { issue[field] = issue[field] ? JSON.parse(issue[field]) : []; }); diff --git a/web/core/local-db/query-constructor.ts b/web/core/local-db/query-constructor.ts index 664e03600cd..919e63c6273 100644 --- a/web/core/local-db/query-constructor.ts +++ b/web/core/local-db/query-constructor.ts @@ -1,4 +1,4 @@ -export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: string, queries) => { +export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: string, queries: any) => { const { order_by, cursor, per_page, labels, sub_issue, assignees, state, cycle, group_by, module, ...otherProps } = queries; @@ -42,7 +42,7 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st return sql; }; -export const issueFilterCountQueryConstructor = (workspaceSlug: string, projectId: string, queries) => { +export const issueFilterCountQueryConstructor = (workspaceSlug: string, projectId: string, queries: any) => { let sql = issueFilterQueryConstructor(workspaceSlug, projectId, queries); sql = sql.replace("SELECT *", "SELECT COUNT(DISTINCT i.id) as total_count"); diff --git a/web/core/local-db/query-executor.ts b/web/core/local-db/query-executor.ts index 81c69df80cc..a86a280b621 100644 --- a/web/core/local-db/query-executor.ts +++ b/web/core/local-db/query-executor.ts @@ -1,6 +1,6 @@ import { SQL } from "./sqlite"; -export const runQuery = async (sql) => { +export const runQuery = async (sql: string) => { const data = await SQL.db.exec({ // sql: `SELECT name, '[' || GROUP_CONCAT('"' || value || '"') || ']' AS label_ids FROM issues LEFT JOIN issue_meta ON issues.id = issue_meta.issue_id WHERE key='label_ids' AND value IN ('2e2a79d1-241a-4708-bf95-95eefa33a501','67970b7d-0525-48fa-9f7e-10fc086f5122')`, // sql: `SELECT * FROM issues LEFT JOIN issue_meta ON issues.id = issue_meta.issue_id WHERE key='label_ids' AND value IN ('2e2a79d1-241a-4708-bf95-95eefa33a501','67970b7d-0525-48fa-9f7e-10fc086f5122')`, diff --git a/web/core/local-db/sqlite.ts b/web/core/local-db/sqlite.ts index 633b74c74a5..3160a8b374d 100644 --- a/web/core/local-db/sqlite.ts +++ b/web/core/local-db/sqlite.ts @@ -16,11 +16,6 @@ const log = console.log; const error = console.error; const SQL: TSQL = { initialized: false, syncInProgress: false }; -const start = async (sqlite3: any) => { - log("Running SQLite3 version", sqlite3.version.libVersion); - SQL.db = new sqlite3.oo1.DB("/mydb.sqlite3", "ct"); - createTables(SQL.db); -}; const initializeSQLite = async () => { if (SQL.initialized) { @@ -30,7 +25,7 @@ const initializeSQLite = async () => { try { log("Loading and initializing SQLite3 module..."); - const promiser = await new Promise((resolve) => { + const promiser: any = await new Promise((resolve) => { const _promiser = sqlite3Worker1Promiser({ onready: () => resolve(_promiser), }); @@ -47,7 +42,7 @@ const initializeSQLite = async () => { const { dbId } = openResponse; SQL.db = { dbId, - exec: async (val) => { + exec: async (val: any) => { if (typeof val === "string") { val = { sql: val }; } @@ -62,10 +57,7 @@ const initializeSQLite = async () => { // Your SQLite code here. await createTables(SQL.db); } catch (err) { - if (!(err instanceof Error)) { - err = new Error(err.result.message); - } - error(err.name, err.message); + error(err); } }; diff --git a/web/core/local-db/tables.ts b/web/core/local-db/tables.ts index b147f80abc5..fcb8d038bda 100644 --- a/web/core/local-db/tables.ts +++ b/web/core/local-db/tables.ts @@ -1,4 +1,4 @@ -export const createIssuesTable = (SQLITE) => { +export const createIssuesTable = (SQLITE: any) => { const sqlstr = `CREATE TABLE IF NOT EXISTS issues ( id TEXT, name TEXT, @@ -29,14 +29,14 @@ export const createIssuesTable = (SQLITE) => { SQLITE.exec(sqlstr); }; -export const createIssueMetaTable = (SQLITE) => { +export const createIssueMetaTable = (SQLITE: any) => { const sqlstr = `CREATE TABLE IF NOT EXISTS issue_meta ( issue_id TEXT, key TEXT, value TEXT);`; SQLITE.exec(sqlstr); }; -export const createTables = async (SQLITE) => { +export const createTables = async (SQLITE: any) => { await createIssuesTable(SQLITE); await createIssueMetaTable(SQLITE); }; From 4e85f006c6aef73748143e911c548d6db960d4a0 Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Wed, 7 Aug 2024 19:28:58 +0530 Subject: [PATCH 013/111] Fix build issue --- web/core/local-db/sqlite.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/web/core/local-db/sqlite.ts b/web/core/local-db/sqlite.ts index 3160a8b374d..80aefc4140c 100644 --- a/web/core/local-db/sqlite.ts +++ b/web/core/local-db/sqlite.ts @@ -1,5 +1,5 @@ // import sqlite3InitModule from "@sqlite.org/sqlite-wasm"; -import { sqlite3Worker1Promiser } from "@sqlite.org/sqlite-wasm"; +// import { sqlite3Worker1Promiser } from "@sqlite.org/sqlite-wasm"; import { createTables } from "./tables"; declare module "@sqlite.org/sqlite-wasm" { @@ -22,6 +22,8 @@ const initializeSQLite = async () => { console.info("Instance already initialized"); return; } + const { sqlite3Worker1Promiser } = await import("@sqlite.org/sqlite-wasm"); + try { log("Loading and initializing SQLite3 module..."); From 48a80434c9190faca778be55e3f98ac84769861b Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Fri, 9 Aug 2024 13:58:11 +0530 Subject: [PATCH 014/111] - Sync updates to local-db - Fallback to server when the local data is loading - Wait when the updates are being fetched --- .../layouts/auth-layout/project-wrapper.tsx | 13 ++++---- web/core/local-db/load-issues.ts | 32 +++++++++++-------- web/core/local-db/queries/issues.ts | 8 +++++ web/core/local-db/query-constructor.ts | 6 ++-- web/core/local-db/utils.ts | 9 ++++++ 5 files changed, 46 insertions(+), 22 deletions(-) diff --git a/web/core/layouts/auth-layout/project-wrapper.tsx b/web/core/layouts/auth-layout/project-wrapper.tsx index 7b19e6751b5..d4ebc68e156 100644 --- a/web/core/layouts/auth-layout/project-wrapper.tsx +++ b/web/core/layouts/auth-layout/project-wrapper.tsx @@ -1,4 +1,6 @@ -import { FC, ReactNode, useEffect, useState } from "react"; +"use client"; + +import { FC, ReactNode, useState } from "react"; import { observer } from "mobx-react"; import { useParams } from "next/navigation"; import useSWR from "swr"; @@ -34,7 +36,6 @@ export const ProjectAuthWrapper: FC = observer((props) => { // const { fetchInboxes } = useInbox(); const { toggleCreateProjectModal } = useCommandPalette(); const { setTrackElement } = useEventTracker(); - const [areIssuesLoading, setIssuesLoading] = useState(false); const { membership: { fetchUserProjectInfo, projectMemberInfo, hasPermissionToProject }, } = useUser(); @@ -54,11 +55,9 @@ export const ProjectAuthWrapper: FC = observer((props) => { useSWR( workspaceSlug && projectId ? `PROJECT_${workspaceSlug.toString()}_${projectId.toString()}` : null, workspaceSlug && projectId - ? () => { - (async () => { - await initializeSQLite(); - await loadIssues(workspaceSlug.toString(), projectId.toString()); - })(); + ? async () => { + await initializeSQLite(); + await loadIssues(workspaceSlug.toString(), projectId.toString()); } : null, { diff --git a/web/core/local-db/load-issues.ts b/web/core/local-db/load-issues.ts index 3741e6eb65f..5fd9ca31011 100644 --- a/web/core/local-db/load-issues.ts +++ b/web/core/local-db/load-issues.ts @@ -8,9 +8,9 @@ import { log } from "./utils"; const arrayFields = ["label_ids", "assignee_ids", "module_ids"]; const PAGE_SIZE = 1000; +export const PROJECT_OFFLINE_STATUS: Record = {}; export const addIssue = async (issue: any) => { const issue_id = issue.id; - const { label_ids, assignee_ids, module_ids, ...otherProps } = issue; const keys = Object.keys(issue).join(","); const values = Object.values(issue).map((val) => { if (val === null) { @@ -36,19 +36,14 @@ export const addIssue = async (issue: any) => { const values = issue[field]; if (values) { values.forEach((val: any) => { - // promises.push( SQL.db.exec({ sql: `insert into issue_meta(issue_id,key,value) values (?,?,?)`, bind: [issue_id, field, val], }); - // ); }); } }); SQL.db.exec("COMMIT;"); - - // await Promise.all(promises); - // log("### Added issue", issue.id); }; export const deleteIssueFromLocal = async (issue_id: any) => { @@ -67,16 +62,22 @@ export const updateIssue = async (issue: any) => { await deleteIssueFromLocal(issue_id); addIssue(issue); }; + export const loadIssuesPrivate = async (workspaceId: string, projectId: string) => { // Load issues from the API - const issueService = new IssueService(); + if (PROJECT_OFFLINE_STATUS[projectId] === undefined) { + PROJECT_OFFLINE_STATUS[projectId] = false; + } else { + // Load issues is already in progress + return; + } let count = await runQuery(`select count(*) as count from issues where project_id='${projectId}'`); - log("### Count", count); count = count[0]["count"]; - log("### Count", count, typeof count); if (!count) { + log("### Loading issues from the server"); + const issueService = new IssueService(); let cursor = `${PAGE_SIZE}:0:0`; let results; let breakLoop = false; @@ -84,7 +85,6 @@ export const loadIssuesPrivate = async (workspaceId: string, projectId: string) const response = await issueService.getIssuesFromServer(workspaceId, projectId, { cursor }); cursor = response.next_cursor; results = response.results as TBaseIssue[]; - log("#### Loading issues", results.length); results.map(async (issue) => { try { await addIssue(issue); @@ -94,10 +94,13 @@ export const loadIssuesPrivate = async (workspaceId: string, projectId: string) } }); } while (results.length >= PAGE_SIZE && !breakLoop); - createIndexes(); + await createIndexes(); } else { + log(`### issues already present in the db ${count}`); syncLocalData(workspaceId, projectId); } + + PROJECT_OFFLINE_STATUS[projectId] = true; }; export const loadIssues = async (workspaceId: string, projectId: string) => { @@ -112,11 +115,14 @@ export const syncUpdatesToLocal = async (workspaceId: string, projectId: string) // get the last updated issue const lastUpdatedIssue = await runQuery( - `select id, name, updated_at , sequence_id from issues where project_id='${projectId}' order by date(updated_at) desc limit 1` + `select id, name, updated_at , sequence_id from issues where project_id='${projectId}' order by datetime(updated_at) desc limit 1` ); + console.log("#### Last Updated Issue", lastUpdatedIssue); + // eslint-disable-next-line @typescript-eslint/naming-convention - const updated_at__gt = lastUpdatedIssue[0]["updated_at"]; + const { updated_at: updated_at__gt, sequence_id } = lastUpdatedIssue[0]; + log("#### Last Updated Issue", sequence_id, updated_at__gt); const issueService = new IssueService(); do { diff --git a/web/core/local-db/queries/issues.ts b/web/core/local-db/queries/issues.ts index cc551298be1..004cafa4b01 100644 --- a/web/core/local-db/queries/issues.ts +++ b/web/core/local-db/queries/issues.ts @@ -1,3 +1,5 @@ +import { IssueService } from "@/services/issue/issue.service"; +import { PROJECT_OFFLINE_STATUS } from "../load-issues"; import { issueFilterCountQueryConstructor, issueFilterQueryConstructor } from "../query-constructor"; import { runQuery } from "../query-executor"; import { SQL } from "../sqlite"; @@ -5,6 +7,12 @@ import { SQL } from "../sqlite"; const arrayFields = ["label_ids", "assignee_ids", "module_ids"]; export const getIssues = async (workspaceSlug: string, projectId: string, queries?: any, config = {}): Promise => { + if (!PROJECT_OFFLINE_STATUS[projectId]) { + console.log(`Project ${projectId} is not offline, fetching from server.`); + const issueService = new IssueService(); + return await issueService.getIssuesFromServer(workspaceSlug, projectId, queries, config); + } + const { cursor } = queries; await SQL.syncInProgress; diff --git a/web/core/local-db/query-constructor.ts b/web/core/local-db/query-constructor.ts index 919e63c6273..fb8b31d0764 100644 --- a/web/core/local-db/query-constructor.ts +++ b/web/core/local-db/query-constructor.ts @@ -1,3 +1,5 @@ +import { wrapDateTime } from "./utils"; + export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: string, queries: any) => { const { order_by, cursor, per_page, labels, sub_issue, assignees, state, cycle, group_by, module, ...otherProps } = queries; @@ -28,9 +30,9 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st if (order_by) { //if order_by starts with "-" then sort in descending order if (order_by.startsWith("-")) { - sql += ` ORDER BY ${order_by.slice(1)} DESC`; + sql += ` ORDER BY ${wrapDateTime(order_by.slice(1))} DESC`; } else { - sql += ` ORDER BY ${order_by} ASC`; + sql += ` ORDER BY ${wrapDateTime(order_by)} ASC`; } } diff --git a/web/core/local-db/utils.ts b/web/core/local-db/utils.ts index 6993ee75031..c60900863db 100644 --- a/web/core/local-db/utils.ts +++ b/web/core/local-db/utils.ts @@ -46,3 +46,12 @@ export const updatePersistentLayer = async (issueIds: string | string[]) => { } }); }; + +export const wrapDateTime = (field: string) => { + const DATE_TIME_FIELDS = ["created_at", "updated_at", "completed_at", "start_date", "target_date"]; + + if (DATE_TIME_FIELDS.includes(field)) { + return `datetime(${field})`; + } + return field; +}; From d315728c5113b97fdb93c6c16e4f41edd26980de Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Mon, 12 Aug 2024 15:22:11 +0530 Subject: [PATCH 015/111] Add issues in batches --- web/core/local-db/load-issues.ts | 61 +++++++++----------------- web/core/local-db/query-constructor.ts | 33 ++++++++++++++ 2 files changed, 53 insertions(+), 41 deletions(-) diff --git a/web/core/local-db/load-issues.ts b/web/core/local-db/load-issues.ts index 5fd9ca31011..39a037a320f 100644 --- a/web/core/local-db/load-issues.ts +++ b/web/core/local-db/load-issues.ts @@ -1,51 +1,32 @@ import { TBaseIssue } from "@plane/types"; import { IssueService } from "@/services/issue"; import createIndexes from "./indexes"; +import { stageIssueInserts } from "./query-constructor"; import { runQuery } from "./query-executor"; import { SQL } from "./sqlite"; import { log } from "./utils"; -const arrayFields = ["label_ids", "assignee_ids", "module_ids"]; const PAGE_SIZE = 1000; export const PROJECT_OFFLINE_STATUS: Record = {}; + export const addIssue = async (issue: any) => { - const issue_id = issue.id; - const keys = Object.keys(issue).join(","); - const values = Object.values(issue).map((val) => { - if (val === null) { - return ""; - } - if (typeof val === "object") { - return JSON.stringify(val); - } - return val; - }); // Will fail when the values have a comma - - const promises = []; SQL.db.exec("BEGIN TRANSACTION;"); - - promises.push( - SQL.db.exec({ - sql: `insert into issues(${keys}) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, - bind: values, - }) - ); - - arrayFields.forEach((field) => { - const values = issue[field]; - if (values) { - values.forEach((val: any) => { - SQL.db.exec({ - sql: `insert into issue_meta(issue_id,key,value) values (?,?,?)`, - bind: [issue_id, field, val], - }); - }); - } - }); + stageIssueInserts(issue); SQL.db.exec("COMMIT;"); }; +export const addIssuesBulk = async (issues: any, batchSize = 100) => { + for (let i = 0; i < issues.length; i += batchSize) { + const batch = issues.slice(i, i + batchSize); + + SQL.db.exec("BEGIN TRANSACTION;"); + batch.forEach((issue: any) => { + stageIssueInserts(issue); + }); + await SQL.db.exec("COMMIT;"); + } +}; export const deleteIssueFromLocal = async (issue_id: any) => { const deleteQuery = `delete from issues where id='${issue_id}'`; const deleteMetaQuery = `delete from issue_meta where issue_id='${issue_id}'`; @@ -85,14 +66,12 @@ export const loadIssuesPrivate = async (workspaceId: string, projectId: string) const response = await issueService.getIssuesFromServer(workspaceId, projectId, { cursor }); cursor = response.next_cursor; results = response.results as TBaseIssue[]; - results.map(async (issue) => { - try { - await addIssue(issue); - } catch (e) { - log("###Error", e, issue); - breakLoop = true; - } - }); + try { + await addIssuesBulk(results); + } catch (e) { + log("###Error", e, results); + breakLoop = true; + } } while (results.length >= PAGE_SIZE && !breakLoop); await createIndexes(); } else { diff --git a/web/core/local-db/query-constructor.ts b/web/core/local-db/query-constructor.ts index fb8b31d0764..43468ee6be7 100644 --- a/web/core/local-db/query-constructor.ts +++ b/web/core/local-db/query-constructor.ts @@ -1,3 +1,4 @@ +import { SQL } from "./sqlite"; import { wrapDateTime } from "./utils"; export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: string, queries: any) => { @@ -53,3 +54,35 @@ export const issueFilterCountQueryConstructor = (workspaceSlug: string, projectI return sql; }; + +const arrayFields = ["label_ids", "assignee_ids", "module_ids"]; + +export const stageIssueInserts = (issue: any) => { + const issue_id = issue.id; + const keys = Object.keys(issue).join(","); + const values = Object.values(issue).map((val) => { + if (val === null) { + return ""; + } + if (typeof val === "object") { + return JSON.stringify(val); + } + return val; + }); // Will fail when the values have a comma + + SQL.db.exec({ + sql: `insert into issues(${keys}) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, + bind: values, + }); + arrayFields.forEach((field) => { + const values = issue[field]; + if (values) { + values.forEach((val: any) => { + SQL.db.exec({ + sql: `insert into issue_meta(issue_id,key,value) values (?,?,?)`, + bind: [issue_id, field, val], + }); + }); + } + }); +}; From 537ed927afd5cc2a5c2e47b1f4f05121c53be37f Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Tue, 13 Aug 2024 18:00:19 +0530 Subject: [PATCH 016/111] Disable skeleton loaders for first 10 issues --- web/core/components/core/render-if-visible-HOC.tsx | 4 +++- web/core/components/issues/issue-layouts/kanban/block.tsx | 3 +++ .../components/issues/issue-layouts/kanban/blocks-list.tsx | 3 ++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/web/core/components/core/render-if-visible-HOC.tsx b/web/core/components/core/render-if-visible-HOC.tsx index a2259c6ca56..5480aeb590f 100644 --- a/web/core/components/core/render-if-visible-HOC.tsx +++ b/web/core/components/core/render-if-visible-HOC.tsx @@ -10,6 +10,7 @@ type Props = { as?: keyof JSX.IntrinsicElements; classNames?: string; placeholderChildren?: ReactNode; + shouldRenderByDefault?: boolean; }; const RenderIfVisible: React.FC = (props) => { @@ -22,8 +23,9 @@ const RenderIfVisible: React.FC = (props) => { children, classNames = "", placeholderChildren = null, //placeholder children + shouldRenderByDefault = false, } = props; - const [shouldVisible, setShouldVisible] = useState(); + const [shouldVisible, setShouldVisible] = useState(shouldRenderByDefault); const placeholderHeight = useRef(defaultHeight); const intersectionRef = useRef(null); diff --git a/web/core/components/issues/issue-layouts/kanban/block.tsx b/web/core/components/issues/issue-layouts/kanban/block.tsx index 0d9ea8a13f1..53f8e376108 100644 --- a/web/core/components/issues/issue-layouts/kanban/block.tsx +++ b/web/core/components/issues/issue-layouts/kanban/block.tsx @@ -32,6 +32,7 @@ interface IssueBlockProps { displayProperties: IIssueDisplayProperties | undefined; draggableId: string; canDropOverIssue: boolean; + shouldRenderByDefault?: boolean; updateIssue: ((projectId: string | null, issueId: string, data: Partial) => Promise) | undefined; quickActions: TRenderQuickActions; canEditProperties: (projectId: string | undefined) => boolean; @@ -115,6 +116,7 @@ export const KanbanIssueBlock: React.FC = observer((props) => { quickActions, canEditProperties, scrollableContainerRef, + shouldRenderByDefault, } = props; const cardRef = useRef(null); @@ -226,6 +228,7 @@ export const KanbanIssueBlock: React.FC = observer((props) => { defaultHeight="100px" horizontalOffset={100} verticalOffset={200} + shouldRenderByDefault={shouldRenderByDefault} > = observer((p <> {issueIds && issueIds.length > 0 ? ( <> - {issueIds.map((issueId) => { + {issueIds.map((issueId, index) => { if (!issueId) return null; let draggableId = issueId; @@ -50,6 +50,7 @@ export const KanbanIssueBlocksList: React.FC = observer((p issueId={issueId} groupId={groupId} subGroupId={sub_group_id} + shouldRenderByDefault={index <= 10} issuesMap={issuesMap} displayProperties={displayProperties} updateIssue={updateIssue} From 21b3924150a12b1d7f6dfeccddb451ad80a36f7b Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Tue, 13 Aug 2024 18:59:29 +0530 Subject: [PATCH 017/111] Load issues in bulk --- web/core/local-db/indexes.ts | 3 ++- web/core/local-db/load-issues.ts | 36 ++++++++++++++++---------------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/web/core/local-db/indexes.ts b/web/core/local-db/indexes.ts index 8c35edf2304..b5a1b29ab56 100644 --- a/web/core/local-db/indexes.ts +++ b/web/core/local-db/indexes.ts @@ -39,9 +39,10 @@ export const createIssueMetaIndexes = async () => { const createIndexes = async () => { log("### Creating indexes"); + const start = performance.now(); const promises = [createIssueIndexes(), createIssueMetaIndexes()]; await Promise.all(promises); - log("### Indexes created"); + log("### Indexes created in", `${performance.now() - start}ms`); }; export default createIndexes; diff --git a/web/core/local-db/load-issues.ts b/web/core/local-db/load-issues.ts index 39a037a320f..6fdf3ce46ef 100644 --- a/web/core/local-db/load-issues.ts +++ b/web/core/local-db/load-issues.ts @@ -57,28 +57,28 @@ export const loadIssuesPrivate = async (workspaceId: string, projectId: string) count = count[0]["count"]; if (!count) { - log("### Loading issues from the server"); + const start = performance.now(); + log("### Adding issues to local db"); const issueService = new IssueService(); - let cursor = `${PAGE_SIZE}:0:0`; - let results; - let breakLoop = false; - do { - const response = await issueService.getIssuesFromServer(workspaceId, projectId, { cursor }); - cursor = response.next_cursor; - results = response.results as TBaseIssue[]; - try { - await addIssuesBulk(results); - } catch (e) { - log("###Error", e, results); - breakLoop = true; + const cursor = `${PAGE_SIZE}:0:0`; + + const response = await issueService.getIssuesFromServer(workspaceId, projectId, { cursor }); + addIssuesBulk(response.results, 500); + + if (response.total_pages > 1) { + const promiseArray = []; + for (let i = 1; i < response.total_pages; i++) { + promiseArray.push(issueService.getIssuesFromServer(workspaceId, projectId, { cursor: `${PAGE_SIZE}:${i}:0` })); } - } while (results.length >= PAGE_SIZE && !breakLoop); + const pages = await Promise.all(promiseArray); + for (const page of pages) { + await addIssuesBulk(page.results, 500); + } + } + + console.log("### Time taken to add issues", performance.now() - start); await createIndexes(); - } else { - log(`### issues already present in the db ${count}`); - syncLocalData(workspaceId, projectId); } - PROJECT_OFFLINE_STATUS[projectId] = true; }; From d2f530f2f854f90880bafb7715443ca7d24efafd Mon Sep 17 00:00:00 2001 From: rahulramesha Date: Wed, 14 Aug 2024 19:18:48 +0530 Subject: [PATCH 018/111] working version of sql lite with grouped issues --- packages/constants/issue.ts | 13 ++++ .../issue-layouts/kanban/base-kanban-root.tsx | 7 +- .../issues/issue-layouts/kanban/default.tsx | 2 +- .../kanban/headers/group-by-card.tsx | 15 ++--- .../issue-layouts/kanban/kanban-group.tsx | 6 +- .../issues/issue-layouts/kanban/swimlanes.tsx | 4 +- .../issue-layouts/list/base-list-root.tsx | 42 ++++++------ .../list/headers/group-by-card.tsx | 13 ++-- .../issues/issue-layouts/list/list-group.tsx | 5 +- .../components/issues/issue-layouts/utils.tsx | 3 +- web/core/local-db/queries/issues.ts | 48 +++++++++++--- web/core/local-db/query-constructor.ts | 65 +++++++++++++----- web/core/store/issue/cycle/issue.store.ts | 66 ++----------------- .../store/issue/helpers/base-issues.store.ts | 65 +++--------------- web/core/store/issue/module/issue.store.ts | 64 ++---------------- .../store/issue/project-views/issue.store.ts | 61 ++--------------- web/core/store/issue/project/issue.store.ts | 61 ++--------------- 17 files changed, 171 insertions(+), 369 deletions(-) diff --git a/packages/constants/issue.ts b/packages/constants/issue.ts index 67f8af56fd6..5db398c7634 100644 --- a/packages/constants/issue.ts +++ b/packages/constants/issue.ts @@ -13,6 +13,19 @@ export enum EIssueGroupByToServerOptions { "created_by" = "created_by", } +export enum EIssueGroupBYServerToProperty { + "state_id" = "state_id", + "priority" = "priority", + "labels__id" = "label_ids", + "state__group" = "state__group", + "assignees__id" = "assignee_ids", + "cycle_id" = "cycle_id", + "issue_module__module_id" = "module_ids", + "target_date" = "target_date", + "project_id" = "project_id", + "created_by" = "created_by", +} + export enum EServerGroupByToFilterOptions { "state_id" = "state", "priority" = "priority", diff --git a/web/core/components/issues/issue-layouts/kanban/base-kanban-root.tsx b/web/core/components/issues/issue-layouts/kanban/base-kanban-root.tsx index dfdd802f603..7121169ee6c 100644 --- a/web/core/components/issues/issue-layouts/kanban/base-kanban-root.tsx +++ b/web/core/components/issues/issue-layouts/kanban/base-kanban-root.tsx @@ -10,7 +10,7 @@ import { useParams, usePathname } from "next/navigation"; import { DeleteIssueModal } from "@/components/issues"; //constants import { ISSUE_DELETED } from "@/constants/event-tracker"; -import { EIssueFilterType, EIssuesStoreType } from "@/constants/issue"; +import { EIssueFilterType, EIssueLayoutTypes, EIssuesStoreType } from "@/constants/issue"; import { EUserProjectRoles } from "@/constants/project"; //hooks import { useEventTracker, useIssueDetail, useIssues, useKanbanView, useUser } from "@/hooks/store"; @@ -20,6 +20,7 @@ import { useIssuesActions } from "@/hooks/use-issues-actions"; // store // ui // types +import { IssueLayoutHOC } from "../issue-layout-HOC"; import { IQuickActionProps, TRenderQuickActions } from "../list/list-view-types"; //components import { getSourceFromDropPayload } from "../utils"; @@ -221,7 +222,7 @@ export const BaseKanBanRoot: React.FC = observer((props: IBas const kanbanFilters = issuesFilter?.issueFilters?.kanbanFilters || { group_by: [], sub_group_by: [] }; return ( - <> + = observer((props: IBas
- + ); }); diff --git a/web/core/components/issues/issue-layouts/kanban/default.tsx b/web/core/components/issues/issue-layouts/kanban/default.tsx index 368d30c9c47..f28396367cf 100644 --- a/web/core/components/issues/issue-layouts/kanban/default.tsx +++ b/web/core/components/issues/issue-layouts/kanban/default.tsx @@ -156,7 +156,7 @@ export const KanBan: React.FC = observer((props) => { column_id={subList.id} icon={subList.icon} title={subList.name} - count={getGroupIssueCount(subList.id, undefined, false)} + count={getGroupIssueCount(subList.id, undefined, false) ?? 0} issuePayload={subList.payload} disableIssueCreation={disableIssueCreation || isGroupByCreatedBy} addIssuesToView={addIssuesToView} diff --git a/web/core/components/issues/issue-layouts/kanban/headers/group-by-card.tsx b/web/core/components/issues/issue-layouts/kanban/headers/group-by-card.tsx index e5602155ce1..d7b1a04a399 100644 --- a/web/core/components/issues/issue-layouts/kanban/headers/group-by-card.tsx +++ b/web/core/components/issues/issue-layouts/kanban/headers/group-by-card.tsx @@ -15,7 +15,6 @@ import { CreateUpdateIssueModal } from "@/components/issues"; // hooks import { useEventTracker } from "@/hooks/store"; import { useIssueStoreType } from "@/hooks/use-issue-layout-store"; -import isNil from "lodash/isNil"; // types interface IHeaderGroupByCard { @@ -24,7 +23,7 @@ interface IHeaderGroupByCard { column_id: string; icon?: React.ReactNode; title: string; - count: number | undefined; + count: number; kanbanFilters: TIssueKanbanFilters; handleKanbanFilters: any; issuePayload: Partial; @@ -124,13 +123,11 @@ export const HeaderGroupByCard: FC = observer((props) => { > {title} - {!isNil(count) && ( -
- {count || 0} -
- )} +
+ {count || 0} +
{sub_group_by === null && ( diff --git a/web/core/components/issues/issue-layouts/kanban/kanban-group.tsx b/web/core/components/issues/issue-layouts/kanban/kanban-group.tsx index 66b9ec772c0..d272bb949c2 100644 --- a/web/core/components/issues/issue-layouts/kanban/kanban-group.tsx +++ b/web/core/components/issues/issue-layouts/kanban/kanban-group.tsx @@ -28,7 +28,6 @@ import { GroupDragOverlay } from "../group-drag-overlay"; import { TRenderQuickActions } from "../list/list-view-types"; import { GroupDropLocation, getSourceFromDropPayload, getDestinationFromDropPayload, getIssueBlockId } from "../utils"; import { KanbanIssueBlocksList, KanBanQuickAddIssueForm } from "."; -import isNil from "lodash/isNil"; interface IKanbanGroup { groupId: string; @@ -219,7 +218,7 @@ export const KanbanGroup = observer((props: IKanbanGroup) => { ? (groupedIssueIds as TSubGroupedIssues)?.[groupId]?.[sub_group_id] ?? [] : (groupedIssueIds as TGroupedIssues)?.[groupId] ?? []; - const groupIssueCount = getGroupIssueCount(groupId, sub_group_id, false); + const groupIssueCount = getGroupIssueCount(groupId, sub_group_id, false) ?? 0; const nextPageResults = getPaginationData(groupId, sub_group_id)?.nextPageResults; @@ -235,8 +234,7 @@ export const KanbanGroup = observer((props: IKanbanGroup) => { ); - const shouldLoadMore = - nextPageResults === undefined ? isNil(groupIssueCount) || issueIds?.length < groupIssueCount : !!nextPageResults; + const shouldLoadMore = nextPageResults === undefined ? issueIds?.length < groupIssueCount : !!nextPageResults; const canOverlayBeVisible = orderBy !== "sort_order" || isDropDisabled; const shouldOverlayBeVisible = isDraggingOverColumn && canOverlayBeVisible; diff --git a/web/core/components/issues/issue-layouts/kanban/swimlanes.tsx b/web/core/components/issues/issue-layouts/kanban/swimlanes.tsx index d8a232721a9..1c50dc26762 100644 --- a/web/core/components/issues/issue-layouts/kanban/swimlanes.tsx +++ b/web/core/components/issues/issue-layouts/kanban/swimlanes.tsx @@ -56,9 +56,9 @@ const SubGroupSwimlaneHeader: React.FC = observer( {list && list.length > 0 && list.map((_list: IGroupByColumn) => { - const groupCount = getGroupIssueCount(_list?.id, undefined, false); + const groupCount = getGroupIssueCount(_list?.id, undefined, false) ?? 0; - const subGroupByVisibilityToggle = visibilitySubGroupByGroupCount(groupCount ?? 0, showEmptyGroup); + const subGroupByVisibilityToggle = visibilitySubGroupByGroupCount(groupCount, showEmptyGroup); if (subGroupByVisibilityToggle === false) return <>; diff --git a/web/core/components/issues/issue-layouts/list/base-list-root.tsx b/web/core/components/issues/issue-layouts/list/base-list-root.tsx index 76e452a9282..09ddd44d7f2 100644 --- a/web/core/components/issues/issue-layouts/list/base-list-root.tsx +++ b/web/core/components/issues/issue-layouts/list/base-list-root.tsx @@ -107,25 +107,27 @@ export const BaseListRoot = observer((props: IBaseListRoot) => { ); return ( -
- -
+ +
+ +
+ ); }); diff --git a/web/core/components/issues/issue-layouts/list/headers/group-by-card.tsx b/web/core/components/issues/issue-layouts/list/headers/group-by-card.tsx index 97720f0e664..1a87e51b895 100644 --- a/web/core/components/issues/issue-layouts/list/headers/group-by-card.tsx +++ b/web/core/components/issues/issue-layouts/list/headers/group-by-card.tsx @@ -18,13 +18,12 @@ import { cn } from "@/helpers/common.helper"; import { useEventTracker } from "@/hooks/store"; import { useIssueStoreType } from "@/hooks/use-issue-layout-store"; import { TSelectionHelper } from "@/hooks/use-multiple-select"; -import isNil from "lodash/isNil"; interface IHeaderGroupByCard { groupID: string; icon?: React.ReactNode; title: string; - count: number | undefined; + count: number; issuePayload: Partial; canEditProperties: (projectId: string | undefined) => boolean; toggleListGroup: () => void; @@ -99,7 +98,7 @@ export const HeaderGroupByCard = observer((props: IHeaderGroupByCard) => { )} groupID={groupID} selectionHelpers={selectionHelpers} - disabled={!count} + disabled={count === 0} /> )} @@ -107,12 +106,10 @@ export const HeaderGroupByCard = observer((props: IHeaderGroupByCard) => { {icon ?? } -
+
{title}
- {!isNil(count) &&
{count || 0}
} +
{count || 0}
{!disableIssueCreation && diff --git a/web/core/components/issues/issue-layouts/list/list-group.tsx b/web/core/components/issues/issue-layouts/list/list-group.tsx index 84be7f0d26b..0f1b8069458 100644 --- a/web/core/components/issues/issue-layouts/list/list-group.tsx +++ b/web/core/components/issues/issue-layouts/list/list-group.tsx @@ -37,7 +37,6 @@ import { IssueBlocksList } from "./blocks-list"; import { HeaderGroupByCard } from "./headers/group-by-card"; import { TRenderQuickActions } from "./list-view-types"; import { ListQuickAddIssueForm } from "./quick-add-issue-form"; -import isNil from "lodash/isNil"; interface Props { groupIssueIds: string[] | undefined; @@ -99,7 +98,7 @@ export const ListGroup = observer((props: Props) => { const [intersectionElement, setIntersectionElement] = useState(null); - const groupIssueCount = getGroupIssueCount(group.id, undefined, false); + const groupIssueCount = getGroupIssueCount(group.id, undefined, false) ?? 0; const nextPageResults = getPaginationData(group.id, undefined)?.nextPageResults; const isPaginating = !!getIssueLoader(group.id); @@ -107,7 +106,7 @@ export const ListGroup = observer((props: Props) => { const shouldLoadMore = nextPageResults === undefined && groupIssueCount !== undefined && groupIssueIds - ? isNil(groupIssueCount) || groupIssueIds.length < groupIssueCount + ? groupIssueIds.length < groupIssueCount : !!nextPageResults; const loadMore = isPaginating ? ( diff --git a/web/core/components/issues/issue-layouts/utils.tsx b/web/core/components/issues/issue-layouts/utils.tsx index 1332e8cea78..7840bb46217 100644 --- a/web/core/components/issues/issue-layouts/utils.tsx +++ b/web/core/components/issues/issue-layouts/utils.tsx @@ -39,7 +39,6 @@ import { IMemberRootStore } from "@/store/member"; import { IModuleStore } from "@/store/module.store"; import { IProjectStore } from "@/store/project/project.store"; import { IStateStore } from "@/store/state.store"; -import { ALL_ISSUES } from "@plane/constants"; export const HIGHLIGHT_CLASS = "highlight"; export const HIGHLIGHT_WITH_LINE = "highlight-with-line"; @@ -93,7 +92,7 @@ export const getGroupByColumns = ( case "created_by": return getCreatedByColumns(member) as any; default: - if (includeNone) return [{ id: ALL_ISSUES, name: ALL_ISSUES, payload: {}, icon: undefined }]; + if (includeNone) return [{ id: `All Issues`, name: `All Issues`, payload: {}, icon: undefined }]; } }; diff --git a/web/core/local-db/queries/issues.ts b/web/core/local-db/queries/issues.ts index 004cafa4b01..16894f8cee8 100644 --- a/web/core/local-db/queries/issues.ts +++ b/web/core/local-db/queries/issues.ts @@ -3,6 +3,8 @@ import { PROJECT_OFFLINE_STATUS } from "../load-issues"; import { issueFilterCountQueryConstructor, issueFilterQueryConstructor } from "../query-constructor"; import { runQuery } from "../query-executor"; import { SQL } from "../sqlite"; +import { EIssueGroupBYServerToProperty } from "@plane/constants"; +import { TIssue, TIssues } from "@plane/types"; const arrayFields = ["label_ids", "assignee_ids", "module_ids"]; @@ -13,21 +15,25 @@ export const getIssues = async (workspaceSlug: string, projectId: string, querie return await issueService.getIssuesFromServer(workspaceSlug, projectId, queries, config); } - const { cursor } = queries; + const { cursor, group_by } = queries; await SQL.syncInProgress; - const query = issueFilterQueryConstructor(workspaceSlug, projectId, queries); - const countQuery = issueFilterCountQueryConstructor(workspaceSlug, projectId, queries); + const query = issueFilterQueryConstructor(queries); + //const countQuery = issueFilterCountQueryConstructor(queries); const start = performance.now(); - const [issuesRaw, count] = await Promise.all([runQuery(query), runQuery(countQuery)]); + //const [issuesRaw, count] = await Promise.all([runQuery(query), runQuery(countQuery)]); + const issuesRaw = await runQuery(query); const end = performance.now(); - const { total_count } = count[0]; + //const { total_count } = count[0]; + const total_count = 2300; const [pageSize, page, offset] = cursor.split(":"); + const groupByProperty = EIssueGroupBYServerToProperty[group_by as typeof EIssueGroupBYServerToProperty]; + const parsingStart = performance.now(); - const issues = issuesRaw.map((issue: any) => { + let issueResults = issuesRaw.map((issue: any) => { arrayFields.forEach((field: string) => { issue[field] = issue[field] ? JSON.parse(issue[field]) : []; }); @@ -42,17 +48,21 @@ export const getIssues = async (workspaceSlug: string, projectId: string, querie Parsing: parsingEnd - parsingStart, }; + if (groupByProperty && page === "0") { + issueResults = getGroupedIssueResults(issueResults); + } + + console.log(issueResults); console.table(times); const total_pages = Math.ceil(total_count / Number(pageSize)); const next_page_results = total_pages > parseInt(page) + 1; const out = { - results: issues, + results: issueResults, next_cursor: `${pageSize}:${page}:${Number(offset) + Number(pageSize)}`, prev_cursor: `${pageSize}:${page}:${Number(offset) - Number(pageSize)}`, total_results: total_count, - count: issues.length, total_count, next_page_results, total_pages, @@ -60,3 +70,25 @@ export const getIssues = async (workspaceSlug: string, projectId: string, querie return out; }; + +function getGroupedIssueResults(issueResults: (TIssue & { group_id: string; total_issues: number })[]): any { + const groupedResults: { + [key: string]: { + results: TIssue[]; + total_results: number; + }; + } = {}; + + for (const issue of issueResults) { + const { group_id, total_issues } = issue; + const groupId = group_id ? group_id : "None"; + if (groupedResults?.[groupId] !== undefined && Array.isArray(groupedResults?.[groupId]?.results)) { + groupedResults?.[groupId]?.results.push(issue); + } else { + groupedResults[groupId] = { results: [issue], total_results: total_issues }; + } + } + + return groupedResults; +} + diff --git a/web/core/local-db/query-constructor.ts b/web/core/local-db/query-constructor.ts index fb8b31d0764..fd4f3be3e52 100644 --- a/web/core/local-db/query-constructor.ts +++ b/web/core/local-db/query-constructor.ts @@ -1,6 +1,8 @@ +import { EIssueGroupBYServerToProperty } from "@plane/constants"; import { wrapDateTime } from "./utils"; +import { TIssue } from "@plane/types"; -export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: string, queries: any) => { +export const issueFilterQueryConstructor = (queries: any) => { const { order_by, cursor, per_page, labels, sub_issue, assignees, state, cycle, group_by, module, ...otherProps } = queries; @@ -10,8 +12,36 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st if (labels) otherProps.label_ids = labels; if (assignees) otherProps.assignee_ids = assignees; + const [pageSize, page, offset] = cursor.split(":"); + + let orderBY = ""; + + if (order_by) { + //if order_by starts with "-" then sort in descending order + if (order_by.startsWith("-")) { + orderBY += ` ORDER BY ${wrapDateTime(order_by.slice(1))} DESC`; + } else { + orderBY += ` ORDER BY ${wrapDateTime(order_by)} ASC`; + } + } + + const groupByProperty = EIssueGroupBYServerToProperty[group_by as typeof EIssueGroupBYServerToProperty]; + console.log(group_by, groupByProperty, page); + const arrayFields = ["label_ids", "assignee_ids", "module_ids"]; - let sql = `SELECT * FROM issues i LEFT JOIN issue_meta im ON i.id = im.issue_id WHERE 1=1 AND project_id='${projectId}' `; + let sql = `SELECT * FROM issues i LEFT JOIN issue_meta im ON i.id = im.issue_id WHERE 1=1 `; + if (groupByProperty && page === "0") { + sql = `SELECT im.issue_id, + im.value AS group_id, + RANK() OVER ( + PARTITION BY im.value + ${orderBY} + ) AS rank, + COUNT(*) OVER (PARTITION BY im.value) AS total_issues + FROM issue_meta im + JOIN issues i ON im.issue_id = i.id + WHERE im.key = '${groupByProperty}'`; + } const keys = Object.keys(otherProps); @@ -25,31 +55,30 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st } }); - sql += ` group by i.id`; + if (groupByProperty && page === "0") + sql = `SELECT i.*, rs.group_id, rs.total_issues + FROM ( + ${sql} + ) rs + JOIN issues i ON rs.issue_id = i.id + WHERE rs.rank <= ${pageSize}`; - if (order_by) { - //if order_by starts with "-" then sort in descending order - if (order_by.startsWith("-")) { - sql += ` ORDER BY ${wrapDateTime(order_by.slice(1))} DESC`; - } else { - sql += ` ORDER BY ${wrapDateTime(order_by)} ASC`; - } - } + // sql += ` group by i.id`; - const [pageSize, page, offset] = cursor.split(":"); // Add offset and paging to query - sql += ` LIMIT ${pageSize} OFFSET ${offset * 1 + page * pageSize};`; + //sql += ` LIMIT ${pageSize} OFFSET ${offset * 1 + page * pageSize};`; console.log("$$$", sql); return sql; }; -export const issueFilterCountQueryConstructor = (workspaceSlug: string, projectId: string, queries: any) => { - let sql = issueFilterQueryConstructor(workspaceSlug, projectId, queries); +export const issueFilterCountQueryConstructor = (queries: any) => { + let sql = issueFilterQueryConstructor(queries); - sql = sql.replace("SELECT *", "SELECT COUNT(DISTINCT i.id) as total_count"); - // Remove everything after group by i.id - sql = `${sql.split("group by i.id")[0]};`; + // sql = sql.replace("SELECT *", "SELECT COUNT(DISTINCT i.id) as total_count"); + // // Remove everything after group by i.id + // sql = `${sql.split("group by i.id")[0]};`; + console.log(sql); return sql; }; diff --git a/web/core/store/issue/cycle/issue.store.ts b/web/core/store/issue/cycle/issue.store.ts index 6812afefa75..110e39ac915 100644 --- a/web/core/store/issue/cycle/issue.store.ts +++ b/web/core/store/issue/cycle/issue.store.ts @@ -170,79 +170,23 @@ export class CycleIssues extends BaseIssuesStore implements ICycleIssues { options: IssuePaginationOptions, cycleId: string, isExistingPaginationOptions: boolean = false - ) => { - try { - const groups = this.getGroups(); - const subGroups = this.getSubGroups(); - - this.clear(!isExistingPaginationOptions); - - if (!groups || (!groups && !subGroups)) - return await this.fetchParallelIssues(workspaceSlug, projectId, loadType, options, cycleId); - - const promises = []; - - for (const group of groups) { - if (!subGroups) { - promises.push(this.fetchParallelIssues(workspaceSlug, projectId, loadType, options, cycleId, group.id)); - continue; - } - - for (const subGroup of subGroups) { - promises.push( - this.fetchParallelIssues(workspaceSlug, projectId, loadType, options, cycleId, group.id, subGroup.id) - ); - } - } - - await Promise.all(promises); - - // fetch parent stats if required, to be handled in the Implemented class - this.fetchParentStats(workspaceSlug, projectId, cycleId); - } catch (error) { - // set loader to undefined if errored out - this.setLoader(undefined); - throw error; - } - }; - - /** - * This method is called to fetch the first issues of pagination - * @param workspaceSlug - * @param projectId - * @param loadType - * @param options - * @returns - */ - fetchParallelIssues = async ( - workspaceSlug: string, - projectId: string, - loadType: TLoader = "init-loader", - options: IssuePaginationOptions, - cycleId: string, - groupId?: string, - subGroupId?: string ) => { try { // set loader and clear store runInAction(() => { - // set Loader - if (groupId || subGroupId) { - this.setLoader("pagination", groupId, subGroupId); - } else { - this.setLoader(loadType); - } + this.setLoader(loadType); }); + this.clear(!isExistingPaginationOptions); // get params from pagination options - const params = this.issueFilterStore?.getFilterParams(options, cycleId, undefined, groupId, subGroupId); + const params = this.issueFilterStore?.getFilterParams(options, cycleId, undefined, undefined, undefined); // call the fetch issues API with the params const response = await this.issueService.getIssues(workspaceSlug, projectId, params, { signal: this.controller.signal, }); // after fetching issues, call the base method to process the response further - this.onfetchIssues(response, options, groupId, subGroupId); + this.onfetchIssues(response, options, workspaceSlug, projectId, cycleId); return response; } catch (error) { // set loader to undefined once errored out @@ -285,7 +229,7 @@ export class CycleIssues extends BaseIssuesStore implements ICycleIssues { subGroupId ); // call the fetch issues API with the params for next page in issues - const response = await this.issueService.getIssues(workspaceSlug, projectId, cycleId, params); + const response = await this.issueService.getIssues(workspaceSlug, projectId, params); // after the next page of issues are fetched, call the base method to process the response this.onfetchNexIssues(response, groupId, subGroupId); diff --git a/web/core/store/issue/helpers/base-issues.store.ts b/web/core/store/issue/helpers/base-issues.store.ts index d1b9cdadeee..5ba71080f4f 100644 --- a/web/core/store/issue/helpers/base-issues.store.ts +++ b/web/core/store/issue/helpers/base-issues.store.ts @@ -44,7 +44,6 @@ import { getSubGroupIssueKeyActions, } from "./base-issues-utils"; import { IBaseIssueFilterStore } from "./issue-filter-helper.store"; -import { getGroupByColumns } from "@/components/issues/issue-layouts/utils"; // constants // helpers // services @@ -422,56 +421,6 @@ export abstract class BaseIssuesStore implements IBaseIssuesStore { } ); - getGroups = (isWorkspaceLevel = false) => { - if (!this.groupBy || this.groupBy === "target_date") return; - - const { - projectRoot, - cycle, - module: moduleInfo, - label, - state: projectState, - memberRoot, - } = this.rootIssueStore.rootStore; - - return getGroupByColumns( - this.groupBy, - projectRoot.project, - cycle, - moduleInfo, - label, - projectState, - memberRoot, - false, - isWorkspaceLevel - ); - }; - - getSubGroups = (isWorkspaceLevel = false) => { - if (!this.subGroupBy || this.subGroupBy === "target_date") return; - - const { - projectRoot, - cycle, - module: moduleInfo, - label, - state: projectState, - memberRoot, - } = this.rootIssueStore.rootStore; - - return getGroupByColumns( - this.subGroupBy, - projectRoot.project, - cycle, - moduleInfo, - label, - projectState, - memberRoot, - false, - isWorkspaceLevel - ); - }; - /** * Gets the next page cursor based on number of issues currently available * @param groupId groupId for the cursor @@ -504,8 +453,9 @@ export abstract class BaseIssuesStore implements IBaseIssuesStore { onfetchIssues( issuesResponse: TIssuesResponse, options: IssuePaginationOptions, - groupId?: string, - subGroupId?: string + workspaceSlug: string, + projectId?: string, + id?: string ) { // Process the Issue Response to get the following data from it const { issueList, groupedIssues, groupedIssueCount } = this.processIssueResponse(issuesResponse); @@ -515,12 +465,15 @@ export abstract class BaseIssuesStore implements IBaseIssuesStore { // Update all the GroupIds to this Store's groupedIssueIds and update Individual group issue counts runInAction(() => { - this.updateGroupedIssueIds(groupedIssues, groupedIssueCount, groupId, subGroupId); - this.loader[getGroupKey(groupId, subGroupId)] = undefined; + this.updateGroupedIssueIds(groupedIssues, groupedIssueCount); + this.loader[getGroupKey()] = undefined; }); + // fetch parent stats if required, to be handled in the Implemented class + this.fetchParentStats(workspaceSlug, projectId, id); + // store Pagination options for next subsequent calls and data like next cursor etc - this.storePreviousPaginationValues(issuesResponse, options, groupId, subGroupId); + this.storePreviousPaginationValues(issuesResponse, options); } /** diff --git a/web/core/store/issue/module/issue.store.ts b/web/core/store/issue/module/issue.store.ts index 35c37703614..76cf6981d40 100644 --- a/web/core/store/issue/module/issue.store.ts +++ b/web/core/store/issue/module/issue.store.ts @@ -127,79 +127,23 @@ export class ModuleIssues extends BaseIssuesStore implements IModuleIssues { options: IssuePaginationOptions, moduleId: string, isExistingPaginationOptions: boolean = false - ) => { - try { - const groups = this.getGroups(); - const subGroups = this.getSubGroups(); - - this.clear(!isExistingPaginationOptions); - - if (!groups || (!groups && !subGroups)) - return await this.fetchParallelIssues(workspaceSlug, projectId, loadType, options, moduleId); - - const promises = []; - - for (const group of groups) { - if (!subGroups) { - promises.push(this.fetchParallelIssues(workspaceSlug, projectId, loadType, options, moduleId, group.id)); - continue; - } - - for (const subGroup of subGroups) { - promises.push( - this.fetchParallelIssues(workspaceSlug, projectId, loadType, options, moduleId, group.id, subGroup.id) - ); - } - } - - await Promise.all(promises); - - // fetch parent stats if required, to be handled in the Implemented class - this.fetchParentStats(workspaceSlug, projectId, moduleId); - } catch (error) { - // set loader to undefined if errored out - this.setLoader(undefined); - throw error; - } - }; - - /** - * This method is called to fetch the first issues of pagination - * @param workspaceSlug - * @param projectId - * @param loadType - * @param options - * @returns - */ - fetchParallelIssues = async ( - workspaceSlug: string, - projectId: string, - loadType: TLoader = "init-loader", - options: IssuePaginationOptions, - moduleId: string, - groupId?: string, - subGroupId?: string ) => { try { // set loader and clear store runInAction(() => { - // set Loader - if (groupId || subGroupId) { - this.setLoader("pagination", groupId, subGroupId); - } else { - this.setLoader(loadType); - } + this.setLoader(loadType); }); + this.clear(!isExistingPaginationOptions); // get params from pagination options - const params = this.issueFilterStore?.getFilterParams(options, moduleId, undefined, groupId, subGroupId); + const params = this.issueFilterStore?.getFilterParams(options, moduleId, undefined, undefined, undefined); // call the fetch issues API with the params const response = await this.issueService.getIssues(workspaceSlug, projectId, params, { signal: this.controller.signal, }); // after fetching issues, call the base method to process the response further - this.onfetchIssues(response, options, groupId, subGroupId); + this.onfetchIssues(response, options, workspaceSlug, projectId, moduleId); return response; } catch (error) { // set loader to undefined once errored out diff --git a/web/core/store/issue/project-views/issue.store.ts b/web/core/store/issue/project-views/issue.store.ts index f7871e699d9..f2e9ba02eb4 100644 --- a/web/core/store/issue/project-views/issue.store.ts +++ b/web/core/store/issue/project-views/issue.store.ts @@ -88,76 +88,23 @@ export class ProjectViewIssues extends BaseIssuesStore implements IProjectViewIs loadType: TLoader, options: IssuePaginationOptions, isExistingPaginationOptions: boolean = false - ) => { - try { - const groups = this.getGroups(); - const subGroups = this.getSubGroups(); - - this.clear(!isExistingPaginationOptions); - - if (!groups || (!groups && !subGroups)) - return await this.fetchParallelIssues(workspaceSlug, projectId, loadType, options, viewId); - - const promises = []; - - for (const group of groups) { - if (!subGroups) { - promises.push(this.fetchParallelIssues(workspaceSlug, projectId, loadType, options, viewId, group.id)); - continue; - } - - for (const subGroup of subGroups) { - promises.push( - this.fetchParallelIssues(workspaceSlug, projectId, loadType, options, viewId, group.id, subGroup.id) - ); - } - } - - await Promise.all(promises); - } catch (error) { - // set loader to undefined if errored out - this.setLoader(undefined); - throw error; - } - }; - - /** - * This method is called to fetch the first issues of pagination - * @param workspaceSlug - * @param projectId - * @param loadType - * @param options - * @returns - */ - fetchParallelIssues = async ( - workspaceSlug: string, - projectId: string, - loadType: TLoader = "init-loader", - options: IssuePaginationOptions, - viewId: string, - groupId?: string, - subGroupId?: string ) => { try { // set loader and clear store runInAction(() => { - // set Loader - if (groupId || subGroupId) { - this.setLoader("pagination", groupId, subGroupId); - } else { - this.setLoader(loadType); - } + this.setLoader(loadType); }); + this.clear(!isExistingPaginationOptions); // get params from pagination options - const params = this.issueFilterStore?.getFilterParams(options, viewId, undefined, groupId, subGroupId); + const params = this.issueFilterStore?.getFilterParams(options, viewId, undefined, undefined, undefined); // call the fetch issues API with the params const response = await this.issueService.getIssues(workspaceSlug, projectId, params, { signal: this.controller.signal, }); // after fetching issues, call the base method to process the response further - this.onfetchIssues(response, options, groupId, subGroupId); + this.onfetchIssues(response, options, workspaceSlug, projectId); return response; } catch (error) { // set loader to undefined if errored out diff --git a/web/core/store/issue/project/issue.store.ts b/web/core/store/issue/project/issue.store.ts index 7546a08bc7e..66f152ac65d 100644 --- a/web/core/store/issue/project/issue.store.ts +++ b/web/core/store/issue/project/issue.store.ts @@ -96,76 +96,23 @@ export class ProjectIssues extends BaseIssuesStore implements IProjectIssues { loadType: TLoader = "init-loader", options: IssuePaginationOptions, isExistingPaginationOptions: boolean = false - ) => { - try { - const groups = this.getGroups(); - const subGroups = this.getSubGroups(); - - this.clear(!isExistingPaginationOptions); - - if (!groups || (!groups && !subGroups)) - return await this.fetchParallelIssues(workspaceSlug, projectId, loadType, options); - - const promises = []; - - for (const group of groups) { - if (!subGroups) { - promises.push(this.fetchParallelIssues(workspaceSlug, projectId, loadType, options, group.id)); - continue; - } - - for (const subGroup of subGroups) { - promises.push(this.fetchParallelIssues(workspaceSlug, projectId, loadType, options, group.id, subGroup.id)); - } - } - - await Promise.all(promises); - - // fetch parent stats if required, to be handled in the Implemented class - this.fetchParentStats(workspaceSlug, projectId); - } catch (error) { - // set loader to undefined if errored out - this.setLoader(undefined); - throw error; - } - }; - - /** - * This method is called to fetch the first issues of pagination - * @param workspaceSlug - * @param projectId - * @param loadType - * @param options - * @returns - */ - fetchParallelIssues = async ( - workspaceSlug: string, - projectId: string, - loadType: TLoader = "init-loader", - options: IssuePaginationOptions, - groupId?: string, - subGroupId?: string ) => { try { // set loader and clear store runInAction(() => { - // set Loader - if (groupId || subGroupId) { - this.setLoader("pagination", groupId, subGroupId); - } else { - this.setLoader(loadType); - } + this.setLoader(loadType); }); + this.clear(!isExistingPaginationOptions); // get params from pagination options - const params = this.issueFilterStore?.getFilterParams(options, projectId, undefined, groupId, subGroupId); + const params = this.issueFilterStore?.getFilterParams(options, projectId, undefined, undefined, undefined); // call the fetch issues API with the params const response = await this.issueService.getIssues(workspaceSlug, projectId, params, { signal: this.controller.signal, }); // after fetching issues, call the base method to process the response further - this.onfetchIssues(response, options, groupId, subGroupId); + this.onfetchIssues(response, options, workspaceSlug, projectId); return response; } catch (error) { // set loader to undefined if errored out From 2fdf4f4c12b83d8a037367ce911e9d622cd26d5b Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Fri, 16 Aug 2024 16:11:02 +0530 Subject: [PATCH 019/111] Use window queries for group by --- web/core/local-db/constants.ts | 12 +++ web/core/local-db/queries/issues.ts | 3 +- web/core/local-db/query-constructor.ts | 109 +++++++++++++------------ web/core/local-db/utils.ts | 17 ++++ 4 files changed, 87 insertions(+), 54 deletions(-) create mode 100644 web/core/local-db/constants.ts diff --git a/web/core/local-db/constants.ts b/web/core/local-db/constants.ts new file mode 100644 index 00000000000..f52eaa0fc16 --- /dev/null +++ b/web/core/local-db/constants.ts @@ -0,0 +1,12 @@ +export const ARRAY_FIELDS = ["label_ids", "assignee_ids", "module_ids"]; + +export const GROUP_BY_MAP = { + state_id: "state_id", + priority: "priority", + cycle_id: "cycle_id", + created_by: "", + // Array Props + issue_module__module_id: "module_ids", + labels__id: "label_ids", + assignees__id: "assignee_ids", +}; diff --git a/web/core/local-db/queries/issues.ts b/web/core/local-db/queries/issues.ts index 16894f8cee8..8346a164522 100644 --- a/web/core/local-db/queries/issues.ts +++ b/web/core/local-db/queries/issues.ts @@ -18,7 +18,7 @@ export const getIssues = async (workspaceSlug: string, projectId: string, querie const { cursor, group_by } = queries; await SQL.syncInProgress; - const query = issueFilterQueryConstructor(queries); + const query = issueFilterQueryConstructor(workspaceSlug, projectId, queries); //const countQuery = issueFilterCountQueryConstructor(queries); const start = performance.now(); //const [issuesRaw, count] = await Promise.all([runQuery(query), runQuery(countQuery)]); @@ -91,4 +91,3 @@ function getGroupedIssueResults(issueResults: (TIssue & { group_id: string; tota return groupedResults; } - diff --git a/web/core/local-db/query-constructor.ts b/web/core/local-db/query-constructor.ts index 37b3f03bed5..4689827b367 100644 --- a/web/core/local-db/query-constructor.ts +++ b/web/core/local-db/query-constructor.ts @@ -1,9 +1,8 @@ -import { EIssueGroupBYServerToProperty } from "@plane/constants"; +import { ARRAY_FIELDS, GROUP_BY_MAP } from "./constants"; import { SQL } from "./sqlite"; -import { wrapDateTime } from "./utils"; -import { TIssue } from "@plane/types"; +import { filterConstructor, wrapDateTime } from "./utils"; -export const issueFilterQueryConstructor = (queries: any) => { +export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: string, queries: any) => { const { order_by, cursor, per_page, labels, sub_issue, assignees, state, cycle, group_by, module, ...otherProps } = queries; @@ -13,74 +12,80 @@ export const issueFilterQueryConstructor = (queries: any) => { if (labels) otherProps.label_ids = labels; if (assignees) otherProps.assignee_ids = assignees; - const [pageSize, page, offset] = cursor.split(":"); - - let orderBY = ""; - + let orderByString = ""; if (order_by) { //if order_by starts with "-" then sort in descending order if (order_by.startsWith("-")) { - orderBY += ` ORDER BY ${wrapDateTime(order_by.slice(1))} DESC`; + orderByString += ` ORDER BY ${wrapDateTime(order_by.slice(1))} DESC`; } else { - orderBY += ` ORDER BY ${wrapDateTime(order_by)} ASC`; + orderByString += ` ORDER BY ${wrapDateTime(order_by)} ASC`; } } + const [pageSize, page, offset] = cursor.split(":"); + const filterString = filterConstructor(otherProps); + + let sql = ""; + if (group_by) { + console.log("###", group_by); + + const translatedGroupBy = GROUP_BY_MAP[group_by]; + // Check if group by is by array field + if (ARRAY_FIELDS.includes(translatedGroupBy)) { + sql = ` + SELECT i.*, rs.group_id, rs.total_issues + FROM ( + SELECT im.issue_id, + im.value AS group_id, + RANK() OVER ( + PARTITION BY im.value + ${orderByString} + ) AS rank, + COUNT(*) OVER (PARTITION BY im.value) AS total_issues + FROM issue_meta im + JOIN issues i ON im.issue_id = i.id + WHERE im.key = '${translatedGroupBy}' -- param for group + AND i.project_id = '${projectId}' -- Project ID + ${filterString} + ) rs + JOIN issues i ON rs.issue_id = i.id + WHERE rs.rank <= ${per_page};`; + } else { + sql = ` + SELECT * + FROM ( + SELECT i.*, + i.${translatedGroupBy} AS group_id, -- param for group + COUNT(*) OVER (PARTITION BY i.${translatedGroupBy}) AS total_issues, + ROW_NUMBER() OVER (PARTITION BY i.${translatedGroupBy} ${orderByString}) AS rank FROM issues i + WHERE i.project_id = '${projectId}' ${filterString} + ) ranked_issues + WHERE rank <= 10;`; + } - const groupByProperty = EIssueGroupBYServerToProperty[group_by as typeof EIssueGroupBYServerToProperty]; - console.log(group_by, groupByProperty, page); - - const arrayFields = ["label_ids", "assignee_ids", "module_ids"]; - let sql = `SELECT * FROM issues i LEFT JOIN issue_meta im ON i.id = im.issue_id WHERE 1=1 `; - if (groupByProperty && page === "0") { - sql = `SELECT im.issue_id, - im.value AS group_id, - RANK() OVER ( - PARTITION BY im.value - ${orderBY} - ) AS rank, - COUNT(*) OVER (PARTITION BY im.value) AS total_issues - FROM issue_meta im - JOIN issues i ON im.issue_id = i.id - WHERE im.key = '${groupByProperty}'`; + console.log("####", sql); + return sql; } - const keys = Object.keys(otherProps); - - keys.forEach((key) => { - const value = otherProps[key] ? otherProps[key].split(",") : ""; - if (!value) return; - if (arrayFields.includes(key)) { - sql += ` AND key='${key}' AND value IN ('${value.join("','")}')`; - } else { - sql += ` AND ${key} in ('${value.join("','")}')`; - } - }); + sql = `SELECT * FROM issues i LEFT JOIN issue_meta im ON i.id = im.issue_id WHERE 1=1 AND project_id='${projectId}' `; - if (groupByProperty && page === "0") - sql = `SELECT i.*, rs.group_id, rs.total_issues - FROM ( - ${sql} - ) rs - JOIN issues i ON rs.issue_id = i.id - WHERE rs.rank <= ${pageSize}`; + sql += filterString; - // sql += ` group by i.id`; + sql += ` group by i.id`; // Add offset and paging to query - //sql += ` LIMIT ${pageSize} OFFSET ${offset * 1 + page * pageSize};`; + sql += ` LIMIT ${pageSize} OFFSET ${offset * 1 + page * pageSize};`; console.log("$$$", sql); return sql; }; -export const issueFilterCountQueryConstructor = (queries: any) => { - let sql = issueFilterQueryConstructor(queries); +export const issueFilterCountQueryConstructor = (workspaceSlug: string, projectId: string, queries: any) => { + let sql = issueFilterQueryConstructor(workspaceSlug, projectId, queries); - // sql = sql.replace("SELECT *", "SELECT COUNT(DISTINCT i.id) as total_count"); - // // Remove everything after group by i.id - // sql = `${sql.split("group by i.id")[0]};`; + sql = sql.replace("SELECT *", "SELECT COUNT(DISTINCT i.id) as total_count"); + // Remove everything after group by i.id + sql = `${sql.split("group by i.id")[0]};`; - console.log(sql); return sql; }; diff --git a/web/core/local-db/utils.ts b/web/core/local-db/utils.ts index c60900863db..8e8220dd092 100644 --- a/web/core/local-db/utils.ts +++ b/web/core/local-db/utils.ts @@ -1,5 +1,6 @@ import pick from "lodash/pick"; import { rootStore } from "@/lib/store-context"; +import { ARRAY_FIELDS } from "./constants"; import { updateIssue } from "./load-issues"; export const log = console.log; @@ -55,3 +56,19 @@ export const wrapDateTime = (field: string) => { } return field; }; + +export const filterConstructor = (filters: any) => { + let sql = ""; + const keys = Object.keys(filters); + + keys.forEach((key) => { + const value = filters[key] ? filters[key].split(",") : ""; + if (!value) return; + if (ARRAY_FIELDS.includes(key)) { + sql += ` AND key='${key}' AND value IN ('${value.join("','")}')`; + } else { + sql += ` AND ${key} in ('${value.join("','")}')`; + } + }); + return sql; +}; From 35d2397b615a2c0499aa8e24e49671e31ec6d6f7 Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Fri, 16 Aug 2024 17:10:34 +0530 Subject: [PATCH 020/111] - Fix sort by date fields - Fix the total count --- web/core/local-db/queries/issues.ts | 14 +++++++------- web/core/local-db/query-constructor.ts | 6 ++++-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/web/core/local-db/queries/issues.ts b/web/core/local-db/queries/issues.ts index 8346a164522..812a0168ed9 100644 --- a/web/core/local-db/queries/issues.ts +++ b/web/core/local-db/queries/issues.ts @@ -1,10 +1,10 @@ +import { EIssueGroupBYServerToProperty } from "@plane/constants"; +import { TIssue } from "@plane/types"; import { IssueService } from "@/services/issue/issue.service"; import { PROJECT_OFFLINE_STATUS } from "../load-issues"; import { issueFilterCountQueryConstructor, issueFilterQueryConstructor } from "../query-constructor"; import { runQuery } from "../query-executor"; import { SQL } from "../sqlite"; -import { EIssueGroupBYServerToProperty } from "@plane/constants"; -import { TIssue, TIssues } from "@plane/types"; const arrayFields = ["label_ids", "assignee_ids", "module_ids"]; @@ -19,14 +19,14 @@ export const getIssues = async (workspaceSlug: string, projectId: string, querie await SQL.syncInProgress; const query = issueFilterQueryConstructor(workspaceSlug, projectId, queries); - //const countQuery = issueFilterCountQueryConstructor(queries); + const countQuery = issueFilterCountQueryConstructor(workspaceSlug, projectId, queries); const start = performance.now(); - //const [issuesRaw, count] = await Promise.all([runQuery(query), runQuery(countQuery)]); - const issuesRaw = await runQuery(query); + const [issuesRaw, count] = await Promise.all([runQuery(query), runQuery(countQuery)]); + // const issuesRaw = await runQuery(query); const end = performance.now(); - //const { total_count } = count[0]; - const total_count = 2300; + const { total_count } = count[0]; + // const total_count = 2300; const [pageSize, page, offset] = cursor.split(":"); diff --git a/web/core/local-db/query-constructor.ts b/web/core/local-db/query-constructor.ts index 4689827b367..543e0dc16a5 100644 --- a/web/core/local-db/query-constructor.ts +++ b/web/core/local-db/query-constructor.ts @@ -36,7 +36,7 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st FROM ( SELECT im.issue_id, im.value AS group_id, - RANK() OVER ( + ROW_NUMBER() OVER ( PARTITION BY im.value ${orderByString} ) AS rank, @@ -80,7 +80,9 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st }; export const issueFilterCountQueryConstructor = (workspaceSlug: string, projectId: string, queries: any) => { - let sql = issueFilterQueryConstructor(workspaceSlug, projectId, queries); + // Remove group by from the query to fallback to non group query + const { group_by, ...otherProps } = queries; + let sql = issueFilterQueryConstructor(workspaceSlug, projectId, otherProps); sql = sql.replace("SELECT *", "SELECT COUNT(DISTINCT i.id) as total_count"); // Remove everything after group by i.id From 3617ae78cad6c3d7576e7c8329773e6e04878c65 Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Fri, 16 Aug 2024 18:07:52 +0530 Subject: [PATCH 021/111] - Fix grouping by created by - Fix order by and limit --- web/core/local-db/constants.ts | 2 +- web/core/local-db/query-constructor.ts | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/web/core/local-db/constants.ts b/web/core/local-db/constants.ts index f52eaa0fc16..3d61f3e3b66 100644 --- a/web/core/local-db/constants.ts +++ b/web/core/local-db/constants.ts @@ -4,7 +4,7 @@ export const GROUP_BY_MAP = { state_id: "state_id", priority: "priority", cycle_id: "cycle_id", - created_by: "", + created_by: "created_by", // Array Props issue_module__module_id: "module_ids", labels__id: "label_ids", diff --git a/web/core/local-db/query-constructor.ts b/web/core/local-db/query-constructor.ts index 543e0dc16a5..bc87a375e57 100644 --- a/web/core/local-db/query-constructor.ts +++ b/web/core/local-db/query-constructor.ts @@ -1,3 +1,4 @@ +import { sq } from "date-fns/locale"; import { ARRAY_FIELDS, GROUP_BY_MAP } from "./constants"; import { SQL } from "./sqlite"; import { filterConstructor, wrapDateTime } from "./utils"; @@ -59,7 +60,7 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st ROW_NUMBER() OVER (PARTITION BY i.${translatedGroupBy} ${orderByString}) AS rank FROM issues i WHERE i.project_id = '${projectId}' ${filterString} ) ranked_issues - WHERE rank <= 10;`; + WHERE rank <= ${per_page};`; } console.log("####", sql); @@ -71,6 +72,7 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st sql += filterString; sql += ` group by i.id`; + sql += orderByString; // Add offset and paging to query sql += ` LIMIT ${pageSize} OFFSET ${offset * 1 + page * pageSize};`; From 0cbb164f062a9decef8ef56ca901b43e7c88fcfc Mon Sep 17 00:00:00 2001 From: rahulramesha Date: Fri, 16 Aug 2024 19:08:31 +0530 Subject: [PATCH 022/111] fix pagination --- web/core/local-db/queries/issues.ts | 4 ++-- web/core/store/issue/helpers/issue-filter-helper.store.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/web/core/local-db/queries/issues.ts b/web/core/local-db/queries/issues.ts index 812a0168ed9..3db11e8587b 100644 --- a/web/core/local-db/queries/issues.ts +++ b/web/core/local-db/queries/issues.ts @@ -60,8 +60,8 @@ export const getIssues = async (workspaceSlug: string, projectId: string, querie const out = { results: issueResults, - next_cursor: `${pageSize}:${page}:${Number(offset) + Number(pageSize)}`, - prev_cursor: `${pageSize}:${page}:${Number(offset) - Number(pageSize)}`, + next_cursor: `${pageSize}:${parseInt(page) + 1}:${Number(offset) + Number(pageSize)}`, + prev_cursor: `${pageSize}:${parseInt(page) - 1}:${Number(offset) - Number(pageSize)}`, total_results: total_count, total_count, next_page_results, diff --git a/web/core/store/issue/helpers/issue-filter-helper.store.ts b/web/core/store/issue/helpers/issue-filter-helper.store.ts index 64e6eaa94f9..8ba9e47ea9f 100644 --- a/web/core/store/issue/helpers/issue-filter-helper.store.ts +++ b/web/core/store/issue/helpers/issue-filter-helper.store.ts @@ -282,7 +282,7 @@ export class IssueFilterHelperStore implements IIssueFilterHelperStore { subGroupId?: string ) { // if cursor exists, use the cursor. If it doesn't exist construct the cursor based on per page count - const pageCursor = cursor ? cursor : `${options.perPageCount}:0:0`; + const pageCursor = cursor ? cursor : groupId ? `${options.perPageCount}:1:0` : `${options.perPageCount}:0:0`; // pagination params const paginationParams: Partial> = { From d67d5c015053f3aca21944eacd37af47633e6552 Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Fri, 16 Aug 2024 19:08:13 +0530 Subject: [PATCH 023/111] Fix sorting on issue priority --- web/core/local-db/constants.ts | 10 +++++++++- web/core/local-db/indexes.ts | 1 + web/core/local-db/query-constructor.ts | 6 ++++-- web/core/local-db/tables.ts | 1 + web/core/local-db/utils.ts | 6 +++++- 5 files changed, 20 insertions(+), 4 deletions(-) diff --git a/web/core/local-db/constants.ts b/web/core/local-db/constants.ts index 3d61f3e3b66..1d9552de68a 100644 --- a/web/core/local-db/constants.ts +++ b/web/core/local-db/constants.ts @@ -2,7 +2,7 @@ export const ARRAY_FIELDS = ["label_ids", "assignee_ids", "module_ids"]; export const GROUP_BY_MAP = { state_id: "state_id", - priority: "priority", + priority: "priority_proxy", cycle_id: "cycle_id", created_by: "created_by", // Array Props @@ -10,3 +10,11 @@ export const GROUP_BY_MAP = { labels__id: "label_ids", assignees__id: "assignee_ids", }; + +export const PRIORITY_MAP = { + low: 1, + medium: 2, + high: 3, + urgent: 4, + none: 0, +}; diff --git a/web/core/local-db/indexes.ts b/web/core/local-db/indexes.ts index b5a1b29ab56..f5e360909d3 100644 --- a/web/core/local-db/indexes.ts +++ b/web/core/local-db/indexes.ts @@ -6,6 +6,7 @@ export const createIssueIndexes = async () => { "state_id", "sort_order", // "priority", + "priority_proxy", "project_id", "created_by", "cycle_id", diff --git a/web/core/local-db/query-constructor.ts b/web/core/local-db/query-constructor.ts index bc87a375e57..8c6a3ee4f09 100644 --- a/web/core/local-db/query-constructor.ts +++ b/web/core/local-db/query-constructor.ts @@ -1,5 +1,5 @@ import { sq } from "date-fns/locale"; -import { ARRAY_FIELDS, GROUP_BY_MAP } from "./constants"; +import { ARRAY_FIELDS, GROUP_BY_MAP, PRIORITY_MAP } from "./constants"; import { SQL } from "./sqlite"; import { filterConstructor, wrapDateTime } from "./utils"; @@ -97,7 +97,9 @@ const arrayFields = ["label_ids", "assignee_ids", "module_ids"]; export const stageIssueInserts = (issue: any) => { const issue_id = issue.id; + issue.priority_proxy = PRIORITY_MAP[issue.priority]; const keys = Object.keys(issue).join(","); + const values = Object.values(issue).map((val) => { if (val === null) { return ""; @@ -109,7 +111,7 @@ export const stageIssueInserts = (issue: any) => { }); // Will fail when the values have a comma SQL.db.exec({ - sql: `insert into issues(${keys}) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, + sql: `insert into issues(${keys}) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, bind: values, }); arrayFields.forEach((field) => { diff --git a/web/core/local-db/tables.ts b/web/core/local-db/tables.ts index fcb8d038bda..bf165e5dd0f 100644 --- a/web/core/local-db/tables.ts +++ b/web/core/local-db/tables.ts @@ -7,6 +7,7 @@ export const createIssuesTable = (SQLITE: any) => { completed_at TEXT, estimate_point REAL, priority TEXT, + priority_proxy INTEGER, start_date TEXT, target_date TEXT, sequence_id INTEGER, diff --git a/web/core/local-db/utils.ts b/web/core/local-db/utils.ts index 8e8220dd092..2277c61ea2e 100644 --- a/web/core/local-db/utils.ts +++ b/web/core/local-db/utils.ts @@ -1,6 +1,6 @@ import pick from "lodash/pick"; import { rootStore } from "@/lib/store-context"; -import { ARRAY_FIELDS } from "./constants"; +import { ARRAY_FIELDS, PRIORITY_MAP } from "./constants"; import { updateIssue } from "./load-issues"; export const log = console.log; @@ -59,6 +59,10 @@ export const wrapDateTime = (field: string) => { export const filterConstructor = (filters: any) => { let sql = ""; + if (filters.priority) { + filters.priority_proxy = PRIORITY_MAP[filters.priority]; + delete filters.priority; + } const keys = Object.keys(filters); keys.forEach((key) => { From 174e079a795c7821f27879026926abc33f484348 Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Fri, 16 Aug 2024 19:56:21 +0530 Subject: [PATCH 024/111] - Add secondary sort order - Fix group by priority --- web/core/local-db/constants.ts | 2 +- web/core/local-db/query-constructor.ts | 4 ++-- web/core/local-db/utils.ts | 4 ++++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/web/core/local-db/constants.ts b/web/core/local-db/constants.ts index 1d9552de68a..5e3dcfe8943 100644 --- a/web/core/local-db/constants.ts +++ b/web/core/local-db/constants.ts @@ -2,7 +2,7 @@ export const ARRAY_FIELDS = ["label_ids", "assignee_ids", "module_ids"]; export const GROUP_BY_MAP = { state_id: "state_id", - priority: "priority_proxy", + priority: "priority", cycle_id: "cycle_id", created_by: "created_by", // Array Props diff --git a/web/core/local-db/query-constructor.ts b/web/core/local-db/query-constructor.ts index 8c6a3ee4f09..9710681fd04 100644 --- a/web/core/local-db/query-constructor.ts +++ b/web/core/local-db/query-constructor.ts @@ -17,9 +17,9 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st if (order_by) { //if order_by starts with "-" then sort in descending order if (order_by.startsWith("-")) { - orderByString += ` ORDER BY ${wrapDateTime(order_by.slice(1))} DESC`; + orderByString += ` ORDER BY ${wrapDateTime(order_by.slice(1))} DESC, created_at DESC`; } else { - orderByString += ` ORDER BY ${wrapDateTime(order_by)} ASC`; + orderByString += ` ORDER BY ${wrapDateTime(order_by)} ASC, created_at DESC`; } } const [pageSize, page, offset] = cursor.split(":"); diff --git a/web/core/local-db/utils.ts b/web/core/local-db/utils.ts index 2277c61ea2e..e5cd60721f1 100644 --- a/web/core/local-db/utils.ts +++ b/web/core/local-db/utils.ts @@ -66,6 +66,10 @@ export const filterConstructor = (filters: any) => { const keys = Object.keys(filters); keys.forEach((key) => { + if (key === "priority_proxy") { + sql += ` AND priority_proxy=${filters[key]} `; + return; + } const value = filters[key] ? filters[key].split(",") : ""; if (!value) return; if (ARRAY_FIELDS.includes(key)) { From 7816667d465b71e2a3084a3cc37e211d490429d3 Mon Sep 17 00:00:00 2001 From: NarayanBavisetti Date: Tue, 20 Aug 2024 17:05:55 +0530 Subject: [PATCH 025/111] chore: added timestamp filter for deleted issues --- apiserver/plane/app/views/issue/base.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/apiserver/plane/app/views/issue/base.py b/apiserver/plane/app/views/issue/base.py index 86896d8d2d3..96238a2daf4 100644 --- a/apiserver/plane/app/views/issue/base.py +++ b/apiserver/plane/app/views/issue/base.py @@ -240,7 +240,6 @@ def list(self, request, slug, project_id): "updated_at__gt": request.GET.get("updated_at__gt") } - print (extra_filters) filters = issue_filters(request.query_params, "GET") order_by_param = request.GET.get("order_by", "-created_at") @@ -667,10 +666,17 @@ class DeletedIssuesListViewSet(BaseAPIView): ] def get(self, request, slug, project_id): - deleted_issues = Issue.all_objects.filter( - workspace__slug=slug, - project_id=project_id, - deleted_at__isnull=False, - ).values_list("id", flat=True) + filters = {} + if request.GET.get("updated_at__gt", None) is not None: + filters = {"updated_at__gt": request.GET.get("updated_at__gt")} + deleted_issues = ( + Issue.all_objects.filter( + workspace__slug=slug, + project_id=project_id, + deleted_at__isnull=False, + ) + .filter(**filters) + .values_list("id", flat=True) + ) return Response(deleted_issues, status=status.HTTP_200_OK) From 107ba8f7b2ee04206a983473a1c6e72875e56435 Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Tue, 20 Aug 2024 17:06:18 +0530 Subject: [PATCH 026/111] - Extract local DB into its own class - Implement sorting by label names --- .../layouts/auth-layout/project-wrapper.tsx | 22 +- .../layouts/auth-layout/workspace-wrapper.tsx | 28 +- web/core/local-db/indexes.ts | 49 ---- web/core/local-db/load-issues.ts | 138 --------- web/core/local-db/queries/issues.ts | 93 ------ web/core/local-db/query-executor.ts | 15 - web/core/local-db/sqlite.ts | 68 ----- web/core/local-db/storage.sqlite.ts | 264 ++++++++++++++++++ web/core/local-db/{ => utils}/constants.ts | 0 web/core/local-db/utils/indexes.ts | 52 ++++ web/core/local-db/utils/load-issues.ts | 58 ++++ web/core/local-db/utils/load-labels.ts | 22 ++ .../local-db/{ => utils}/query-constructor.ts | 65 +++-- web/core/local-db/utils/query-executor.ts | 13 + web/core/local-db/{ => utils}/tables.ts | 21 ++ web/core/local-db/{ => utils}/utils.ts | 33 ++- web/core/services/issue/issue.service.ts | 9 +- 17 files changed, 550 insertions(+), 400 deletions(-) delete mode 100644 web/core/local-db/indexes.ts delete mode 100644 web/core/local-db/load-issues.ts delete mode 100644 web/core/local-db/queries/issues.ts delete mode 100644 web/core/local-db/query-executor.ts delete mode 100644 web/core/local-db/sqlite.ts create mode 100644 web/core/local-db/storage.sqlite.ts rename web/core/local-db/{ => utils}/constants.ts (100%) create mode 100644 web/core/local-db/utils/indexes.ts create mode 100644 web/core/local-db/utils/load-issues.ts create mode 100644 web/core/local-db/utils/load-labels.ts rename web/core/local-db/{ => utils}/query-constructor.ts (66%) create mode 100644 web/core/local-db/utils/query-executor.ts rename web/core/local-db/{ => utils}/tables.ts (72%) rename web/core/local-db/{ => utils}/utils.ts (67%) diff --git a/web/core/layouts/auth-layout/project-wrapper.tsx b/web/core/layouts/auth-layout/project-wrapper.tsx index d4ebc68e156..9c6e678129d 100644 --- a/web/core/layouts/auth-layout/project-wrapper.tsx +++ b/web/core/layouts/auth-layout/project-wrapper.tsx @@ -1,9 +1,11 @@ "use client"; -import { FC, ReactNode, useState } from "react"; +import { FC, ReactNode } from "react"; import { observer } from "mobx-react"; import { useParams } from "next/navigation"; import useSWR from "swr"; +import useSWRImmutable from "swr/immutable"; + // components import { JoinProject } from "@/components/auth-screens"; import { EmptyState, LogoSpinner } from "@/components/common"; @@ -22,8 +24,7 @@ import { useUser, } from "@/hooks/store"; // images -import { loadIssues } from "@/local-db/load-issues"; -import { initializeSQLite } from "@/local-db/sqlite"; +import { persistence } from "@/local-db/storage.sqlite"; import emptyProject from "@/public/empty-state/project.svg"; interface IProjectAuthWrapper { @@ -52,12 +53,19 @@ export const ProjectAuthWrapper: FC = observer((props) => { // router const { workspaceSlug, projectId } = useParams(); - useSWR( - workspaceSlug && projectId ? `PROJECT_${workspaceSlug.toString()}_${projectId.toString()}` : null, + useSWRImmutable( + workspaceSlug && projectId ? `PROJECT_SYNC_${workspaceSlug.toString()}_${projectId.toString()}` : null, workspaceSlug && projectId ? async () => { - await initializeSQLite(); - await loadIssues(workspaceSlug.toString(), projectId.toString()); + await persistence.syncProject(projectId.toString()); + } + : null + ); + useSWR( + workspaceSlug && projectId ? `PROJECT_SYNC_ISSUES_${workspaceSlug.toString()}_${projectId.toString()}` : null, + workspaceSlug && projectId + ? () => { + persistence.syncIssues(projectId.toString()); } : null, { diff --git a/web/core/layouts/auth-layout/workspace-wrapper.tsx b/web/core/layouts/auth-layout/workspace-wrapper.tsx index 37be07fcecc..ec5584271b1 100644 --- a/web/core/layouts/auth-layout/workspace-wrapper.tsx +++ b/web/core/layouts/auth-layout/workspace-wrapper.tsx @@ -7,14 +7,17 @@ import Link from "next/link"; import { useParams } from "next/navigation"; import { useTheme } from "next-themes"; import useSWR from "swr"; +import useSWRImmutable from "swr/immutable"; + import { LogOut } from "lucide-react"; // hooks -import { Button, TOAST_TYPE, setToast, Tooltip } from "@plane/ui"; +import { Button, setToast, TOAST_TYPE, Tooltip } from "@plane/ui"; import { LogoSpinner } from "@/components/common"; import { useMember, useProject, useUser, useWorkspace } from "@/hooks/store"; import { useFavorite } from "@/hooks/store/use-favorite"; import { usePlatformOS } from "@/hooks/use-platform-os"; // images +import { persistence } from "@/local-db/storage.sqlite"; import PlaneBlackLogo from "@/public/plane-logos/black-horizontal-with-blue-logo.png"; import PlaneWhiteLogo from "@/public/plane-logos/white-horizontal-with-blue-logo.png"; import WorkSpaceNotAvailable from "@/public/workspace/workspace-not-available.png"; @@ -77,6 +80,27 @@ export const WorkspaceAuthWrapper: FC = observer((props) { revalidateIfStale: false, revalidateOnFocus: false } ); + // initialize the local database + const { isLoading: isDBInitializing } = useSWRImmutable( + workspaceSlug ? `WORKSPACE_DB_${workspaceSlug}` : null, + workspaceSlug + ? async () => { + persistence.reset(); + await persistence.initialize(workspaceSlug.toString()); + } + : null + ); + + // Load common data + useSWRImmutable( + workspaceSlug ? `WORKSPACE_SYNC_${workspaceSlug}` : null, + workspaceSlug + ? async () => { + await persistence.syncWorkspace(); + } + : null + ); + const handleSignOut = async () => { await signOut().catch(() => setToast({ @@ -88,7 +112,7 @@ export const WorkspaceAuthWrapper: FC = observer((props) }; // if list of workspaces are not there then we have to render the spinner - if (allWorkspaces === undefined) { + if (allWorkspaces === undefined || isDBInitializing) { return (
diff --git a/web/core/local-db/indexes.ts b/web/core/local-db/indexes.ts deleted file mode 100644 index f5e360909d3..00000000000 --- a/web/core/local-db/indexes.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { SQL } from "./sqlite"; - -const log = console.log; -export const createIssueIndexes = async () => { - const columns = [ - "state_id", - "sort_order", - // "priority", - "priority_proxy", - "project_id", - "created_by", - "cycle_id", - ]; - - // Drop indexes - const dropPromises = []; - dropPromises.push(SQL.db.exec({ sql: `DROP INDEX IF EXISTS issues_issue_id_idx` })); - dropPromises.push(SQL.db.exec({ sql: `DROP INDEX IF EXISTS issues_priority_idx` })); - columns.forEach((column) => { - dropPromises.push(SQL.db.exec({ sql: `DROP INDEX IF EXISTS issues_issue_${column}_idx` })); - }); - await Promise.all(dropPromises); - - const promises = []; - - promises.push(SQL.db.exec({ sql: `CREATE UNIQUE INDEX issues_issue_id_idx ON issues (id)` })); - - columns.forEach((column) => { - promises.push(SQL.db.exec({ sql: `CREATE INDEX issues_issue_${column}_idx ON issues (project_id, ${column})` })); - }); - await Promise.all(promises); -}; - -export const createIssueMetaIndexes = async () => { - // Drop indexes - await SQL.db.exec({ sql: `DROP INDEX IF EXISTS issue_meta_all_idx` }); - - await SQL.db.exec({ sql: `CREATE INDEX issue_meta_all_idx ON issue_meta (issue_id,key,value)` }); -}; - -const createIndexes = async () => { - log("### Creating indexes"); - const start = performance.now(); - const promises = [createIssueIndexes(), createIssueMetaIndexes()]; - await Promise.all(promises); - log("### Indexes created in", `${performance.now() - start}ms`); -}; - -export default createIndexes; diff --git a/web/core/local-db/load-issues.ts b/web/core/local-db/load-issues.ts deleted file mode 100644 index 6fdf3ce46ef..00000000000 --- a/web/core/local-db/load-issues.ts +++ /dev/null @@ -1,138 +0,0 @@ -import { TBaseIssue } from "@plane/types"; -import { IssueService } from "@/services/issue"; -import createIndexes from "./indexes"; -import { stageIssueInserts } from "./query-constructor"; -import { runQuery } from "./query-executor"; -import { SQL } from "./sqlite"; -import { log } from "./utils"; - -const PAGE_SIZE = 1000; - -export const PROJECT_OFFLINE_STATUS: Record = {}; - -export const addIssue = async (issue: any) => { - SQL.db.exec("BEGIN TRANSACTION;"); - stageIssueInserts(issue); - SQL.db.exec("COMMIT;"); -}; - -export const addIssuesBulk = async (issues: any, batchSize = 100) => { - for (let i = 0; i < issues.length; i += batchSize) { - const batch = issues.slice(i, i + batchSize); - - SQL.db.exec("BEGIN TRANSACTION;"); - batch.forEach((issue: any) => { - stageIssueInserts(issue); - }); - await SQL.db.exec("COMMIT;"); - } -}; -export const deleteIssueFromLocal = async (issue_id: any) => { - const deleteQuery = `delete from issues where id='${issue_id}'`; - const deleteMetaQuery = `delete from issue_meta where issue_id='${issue_id}'`; - - SQL.db.exec("BEGIN TRANSACTION;"); - SQL.db.exec(deleteQuery); - SQL.db.exec(deleteMetaQuery); - SQL.db.exec("COMMIT;"); -}; - -export const updateIssue = async (issue: any) => { - const issue_id = issue.id; - // delete the issue and its meta data - await deleteIssueFromLocal(issue_id); - addIssue(issue); -}; - -export const loadIssuesPrivate = async (workspaceId: string, projectId: string) => { - // Load issues from the API - if (PROJECT_OFFLINE_STATUS[projectId] === undefined) { - PROJECT_OFFLINE_STATUS[projectId] = false; - } else { - // Load issues is already in progress - return; - } - - let count = await runQuery(`select count(*) as count from issues where project_id='${projectId}'`); - count = count[0]["count"]; - - if (!count) { - const start = performance.now(); - log("### Adding issues to local db"); - const issueService = new IssueService(); - const cursor = `${PAGE_SIZE}:0:0`; - - const response = await issueService.getIssuesFromServer(workspaceId, projectId, { cursor }); - addIssuesBulk(response.results, 500); - - if (response.total_pages > 1) { - const promiseArray = []; - for (let i = 1; i < response.total_pages; i++) { - promiseArray.push(issueService.getIssuesFromServer(workspaceId, projectId, { cursor: `${PAGE_SIZE}:${i}:0` })); - } - const pages = await Promise.all(promiseArray); - for (const page of pages) { - await addIssuesBulk(page.results, 500); - } - } - - console.log("### Time taken to add issues", performance.now() - start); - await createIndexes(); - } - PROJECT_OFFLINE_STATUS[projectId] = true; -}; - -export const loadIssues = async (workspaceId: string, projectId: string) => { - SQL.syncInProgress = loadIssuesPrivate(workspaceId, projectId); - await SQL.syncInProgress; -}; - -export const syncUpdatesToLocal = async (workspaceId: string, projectId: string) => { - let cursor = `${PAGE_SIZE}:0:0`; - let results; - let breakLoop = false; - - // get the last updated issue - const lastUpdatedIssue = await runQuery( - `select id, name, updated_at , sequence_id from issues where project_id='${projectId}' order by datetime(updated_at) desc limit 1` - ); - - console.log("#### Last Updated Issue", lastUpdatedIssue); - - // eslint-disable-next-line @typescript-eslint/naming-convention - const { updated_at: updated_at__gt, sequence_id } = lastUpdatedIssue[0]; - log("#### Last Updated Issue", sequence_id, updated_at__gt); - const issueService = new IssueService(); - - do { - const response = await issueService.getIssuesFromServer(workspaceId, projectId, { cursor, updated_at__gt }); - cursor = response.next_cursor; - results = response.results as TBaseIssue[]; - log("#### Loading issues", results.length); - results.map(async (issue) => { - try { - await updateIssue(issue); - } catch (e) { - log("###Error", e, issue); - breakLoop = true; - } - }); - } while (results.length >= PAGE_SIZE && !breakLoop); -}; - -export const syncDeletesToLocal = async (workspaceId: string, projectId: string) => { - const issueService = new IssueService(); - const response = await issueService.getDeletedIssues(workspaceId, projectId); - if (Array.isArray(response)) { - response.map(async (issue) => deleteIssueFromLocal(issue)); - } -}; - -export const syncLocalData = async (workspaceId: string, projectId: string) => { - SQL.syncInProgress = Promise.all([ - syncDeletesToLocal(workspaceId, projectId), - syncUpdatesToLocal(workspaceId, projectId), - ]); - - await SQL.syncInProgress; -}; diff --git a/web/core/local-db/queries/issues.ts b/web/core/local-db/queries/issues.ts deleted file mode 100644 index 3db11e8587b..00000000000 --- a/web/core/local-db/queries/issues.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { EIssueGroupBYServerToProperty } from "@plane/constants"; -import { TIssue } from "@plane/types"; -import { IssueService } from "@/services/issue/issue.service"; -import { PROJECT_OFFLINE_STATUS } from "../load-issues"; -import { issueFilterCountQueryConstructor, issueFilterQueryConstructor } from "../query-constructor"; -import { runQuery } from "../query-executor"; -import { SQL } from "../sqlite"; - -const arrayFields = ["label_ids", "assignee_ids", "module_ids"]; - -export const getIssues = async (workspaceSlug: string, projectId: string, queries?: any, config = {}): Promise => { - if (!PROJECT_OFFLINE_STATUS[projectId]) { - console.log(`Project ${projectId} is not offline, fetching from server.`); - const issueService = new IssueService(); - return await issueService.getIssuesFromServer(workspaceSlug, projectId, queries, config); - } - - const { cursor, group_by } = queries; - - await SQL.syncInProgress; - const query = issueFilterQueryConstructor(workspaceSlug, projectId, queries); - const countQuery = issueFilterCountQueryConstructor(workspaceSlug, projectId, queries); - const start = performance.now(); - const [issuesRaw, count] = await Promise.all([runQuery(query), runQuery(countQuery)]); - // const issuesRaw = await runQuery(query); - const end = performance.now(); - - const { total_count } = count[0]; - // const total_count = 2300; - - const [pageSize, page, offset] = cursor.split(":"); - - const groupByProperty = EIssueGroupBYServerToProperty[group_by as typeof EIssueGroupBYServerToProperty]; - - const parsingStart = performance.now(); - let issueResults = issuesRaw.map((issue: any) => { - arrayFields.forEach((field: string) => { - issue[field] = issue[field] ? JSON.parse(issue[field]) : []; - }); - - return issue; - }); - - const parsingEnd = performance.now(); - - const times = { - IssueQuery: end - start, - Parsing: parsingEnd - parsingStart, - }; - - if (groupByProperty && page === "0") { - issueResults = getGroupedIssueResults(issueResults); - } - - console.log(issueResults); - console.table(times); - - const total_pages = Math.ceil(total_count / Number(pageSize)); - const next_page_results = total_pages > parseInt(page) + 1; - - const out = { - results: issueResults, - next_cursor: `${pageSize}:${parseInt(page) + 1}:${Number(offset) + Number(pageSize)}`, - prev_cursor: `${pageSize}:${parseInt(page) - 1}:${Number(offset) - Number(pageSize)}`, - total_results: total_count, - total_count, - next_page_results, - total_pages, - }; - - return out; -}; - -function getGroupedIssueResults(issueResults: (TIssue & { group_id: string; total_issues: number })[]): any { - const groupedResults: { - [key: string]: { - results: TIssue[]; - total_results: number; - }; - } = {}; - - for (const issue of issueResults) { - const { group_id, total_issues } = issue; - const groupId = group_id ? group_id : "None"; - if (groupedResults?.[groupId] !== undefined && Array.isArray(groupedResults?.[groupId]?.results)) { - groupedResults?.[groupId]?.results.push(issue); - } else { - groupedResults[groupId] = { results: [issue], total_results: total_issues }; - } - } - - return groupedResults; -} diff --git a/web/core/local-db/query-executor.ts b/web/core/local-db/query-executor.ts deleted file mode 100644 index a86a280b621..00000000000 --- a/web/core/local-db/query-executor.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { SQL } from "./sqlite"; - -export const runQuery = async (sql: string) => { - const data = await SQL.db.exec({ - // sql: `SELECT name, '[' || GROUP_CONCAT('"' || value || '"') || ']' AS label_ids FROM issues LEFT JOIN issue_meta ON issues.id = issue_meta.issue_id WHERE key='label_ids' AND value IN ('2e2a79d1-241a-4708-bf95-95eefa33a501','67970b7d-0525-48fa-9f7e-10fc086f5122')`, - // sql: `SELECT * FROM issues LEFT JOIN issue_meta ON issues.id = issue_meta.issue_id WHERE key='label_ids' AND value IN ('2e2a79d1-241a-4708-bf95-95eefa33a501','67970b7d-0525-48fa-9f7e-10fc086f5122')`, - sql, - - // key=label_ids AND value in ('2e2a79d1-241a-4708-bf95-95eefa33a501','67970b7d-0525-48fa-9f7e-10fc086f5122')`, - rowMode: "object", - returnValue: "resultRows", - }); - - return data.result.resultRows; -}; diff --git a/web/core/local-db/sqlite.ts b/web/core/local-db/sqlite.ts deleted file mode 100644 index 80aefc4140c..00000000000 --- a/web/core/local-db/sqlite.ts +++ /dev/null @@ -1,68 +0,0 @@ -// import sqlite3InitModule from "@sqlite.org/sqlite-wasm"; -// import { sqlite3Worker1Promiser } from "@sqlite.org/sqlite-wasm"; -import { createTables } from "./tables"; - -declare module "@sqlite.org/sqlite-wasm" { - export function sqlite3Worker1Promiser(...args: any): any; -} - -type TSQL = { - db?: any; - initialized: boolean; - syncInProgress: boolean | Promise; -}; - -const log = console.log; -const error = console.error; - -const SQL: TSQL = { initialized: false, syncInProgress: false }; - -const initializeSQLite = async () => { - if (SQL.initialized) { - console.info("Instance already initialized"); - return; - } - const { sqlite3Worker1Promiser } = await import("@sqlite.org/sqlite-wasm"); - - try { - log("Loading and initializing SQLite3 module..."); - - const promiser: any = await new Promise((resolve) => { - const _promiser = sqlite3Worker1Promiser({ - onready: () => resolve(_promiser), - }); - }); - - log("Done initializing. Running demo..."); - - const configResponse = await promiser("config-get", {}); - log("Running SQLite3 version", configResponse.result.version.libVersion); - - const openResponse = await promiser("open", { - filename: "file:mydb.sqlite3?vfs=opfs", - }); - const { dbId } = openResponse; - SQL.db = { - dbId, - exec: async (val: any) => { - if (typeof val === "string") { - val = { sql: val }; - } - return promiser("exec", { dbId, ...val }); - }, - }; - log( - "OPFS is available, created persisted database at", - openResponse.result.filename.replace(/^file:(.*?)\?vfs=opfs$/, "$1") - ); - SQL.initialized = true; - // Your SQLite code here. - await createTables(SQL.db); - } catch (err) { - error(err); - } -}; - -// initializeSQLite(); - -export { SQL, initializeSQLite }; diff --git a/web/core/local-db/storage.sqlite.ts b/web/core/local-db/storage.sqlite.ts new file mode 100644 index 00000000000..f17001f207e --- /dev/null +++ b/web/core/local-db/storage.sqlite.ts @@ -0,0 +1,264 @@ +import set from "lodash/set"; +import { EIssueGroupBYServerToProperty } from "@plane/constants"; +import { IssueService } from "@/services/issue/issue.service"; +import { ARRAY_FIELDS } from "./utils/constants"; +import { addIssuesBulk } from "./utils/load-issues"; +import { loadLabels } from "./utils/load-labels"; +import { issueFilterQueryConstructor, issueFilterCountQueryConstructor } from "./utils/query-constructor"; +import { runQuery } from "./utils/query-executor"; +import { createTables } from "./utils/tables"; +import { getGroupedIssueResults } from "./utils/utils"; + +const PAGE_SIZE = 1000; +const log = console.log; +const error = console.error; +const info = console.info; + +type TProjectStatus = { + issues: { status: undefined | "loading" | "ready" | "error" | "syncing"; sync: Promise | undefined }; +}; + +type TDBStatus = "initializing" | "ready" | "error" | undefined; +export class Storage { + db: any; + status: TDBStatus = undefined; + dbName = "plane"; + projectStatus: Record = {}; + workspaceSlug: string = ""; + workspaceInitPromise: Promise | undefined; + // issueService: any; + + constructor() { + this.db = null; + // this.issueService = new IssueService(); + } + + reset = () => { + this.db = null; + this.status = undefined; + this.projectStatus = {}; + this.workspaceSlug = ""; + }; + + initialize = async (workspaceSlug: string): Promise => { + this.workspaceInitPromise = this._initialize(workspaceSlug); + await this.workspaceInitPromise; + }; + + _initialize = async (workspaceSlug: string): Promise => { + if (this.status === "initializing") { + console.warn(`Initialization already in progress for workspace ${workspaceSlug}`); + return true; + } + if (this.status === "ready") { + console.warn(`Already initialized for workspace ${workspaceSlug}`); + return true; + } + if (this.status === "error") { + console.warn(`Initialization failed for workspace ${workspaceSlug}`); + return false; + } + + info("Loading and initializing SQLite3 module..."); + + this.workspaceSlug = workspaceSlug; + this.dbName = workspaceSlug; + const { sqlite3Worker1Promiser } = await import("@sqlite.org/sqlite-wasm"); + + try { + const promiser: any = await new Promise((resolve) => { + const _promiser = sqlite3Worker1Promiser({ + onready: () => resolve(_promiser), + }); + }); + + const configResponse = await promiser("config-get", {}); + log("Running SQLite3 version", configResponse.result.version.libVersion); + + const openResponse = await promiser("open", { + filename: `file:${this.dbName}.sqlite3?vfs=opfs`, + }); + const { dbId } = openResponse; + this.db = { + dbId, + exec: async (val: any) => { + if (typeof val === "string") { + val = { sql: val }; + } + return promiser("exec", { dbId, ...val }); + }, + }; + log( + "OPFS is available, created persisted database at", + openResponse.result.filename.replace(/^file:(.*?)\?vfs=opfs$/, "$1") + ); + this.status = "ready"; + // Your SQLite code here. + await createTables(this.db); + } catch (err) { + error(err); + } + + return true; + }; + + syncWorkspace = async () => { + await this.workspaceInitPromise; + loadLabels(this.workspaceSlug); + }; + + syncProject = async (projectId: string) => { + // Load labels, members, states, modules, cycles + + const sync = this.syncIssues(projectId); + if (this.getStatus(projectId) === "syncing") { + this.setSync(projectId, sync); + } + }; + + syncIssues = async (projectId: string) => { + console.log("### Sync started"); + if (this.getStatus(projectId) === "loading" || this.getStatus(projectId) === "syncing") { + info(`Project ${projectId} is already loading or syncing`); + return; + } + + const queryParams: { cursor: string; updated_at__gt?: string } = { + cursor: `${PAGE_SIZE}:0:0`, + }; + + const syncedAt = await this.getLastSyncTime(projectId); + + if (syncedAt) { + queryParams["updated_at__gt"] = syncedAt; + } + + this.setStatus(projectId, syncedAt ? "syncing" : "loading"); + + log(`### ${syncedAt ? "Syncing" : "Loading"} issues to local db for project ${projectId}`); + + const start = performance.now(); + const issueService = new IssueService(); + + const response = await issueService.getIssuesFromServer(this.workspaceSlug, projectId, queryParams); + addIssuesBulk(response.results, 500); + + if (response.total_pages > 1) { + const promiseArray = []; + for (let i = 1; i < response.total_pages; i++) { + promiseArray.push( + issueService.getIssuesFromServer(this.workspaceSlug, projectId, { cursor: `${PAGE_SIZE}:${i}:0` }) + ); + } + const pages = await Promise.all(promiseArray); + for (const page of pages) { + await addIssuesBulk(page.results, 500); + } + } + + console.log("### Time taken to add issues", performance.now() - start); + + this.setStatus(projectId, "ready"); + }; + + getIssueCount = async (projectId: string) => { + const count = await runQuery(`select count(*) as count from issues where project_id='${projectId}'`); + return count[0]["count"]; + }; + + getLastUpdatedIssue = async (projectId: string) => { + const lastUpdatedIssue = await runQuery( + `select id, name, updated_at , sequence_id from issues where project_id='${projectId}' order by datetime(updated_at) desc limit 1` + ); + + if (lastUpdatedIssue.length) { + return lastUpdatedIssue[0]; + } + return; + }; + + getLastSyncTime = async (projectId: string) => { + const issue = await this.getLastUpdatedIssue(projectId); + if (!issue) { + return false; + } + return issue.updated_at; + }; + + getIssues = async (projectId: string, queries: any, config: any) => { + console.log("#### Queries", queries); + if (this.getStatus(projectId) === "loading" || window.DISABLE_LOCAL) { + info(`Project ${projectId} is loading, falling back to server`); + const issueService = new IssueService(); + return await issueService.getIssuesFromServer(this.workspaceSlug, projectId, queries); + } + await this.getSync(projectId); + + const { cursor, group_by } = queries; + + const query = issueFilterQueryConstructor(this.workspaceSlug, projectId, queries); + const countQuery = issueFilterCountQueryConstructor(this.workspaceSlug, projectId, queries); + const start = performance.now(); + const [issuesRaw, count] = await Promise.all([runQuery(query), runQuery(countQuery)]); + // const issuesRaw = await runQuery(query); + const end = performance.now(); + + const { total_count } = count[0]; + // const total_count = 2300; + + const [pageSize, page, offset] = cursor.split(":"); + + const groupByProperty = EIssueGroupBYServerToProperty[group_by as typeof EIssueGroupBYServerToProperty]; + + const parsingStart = performance.now(); + let issueResults = issuesRaw.map((issue: any) => { + ARRAY_FIELDS.forEach((field: string) => { + issue[field] = issue[field] ? JSON.parse(issue[field]) : []; + }); + + return issue; + }); + + const parsingEnd = performance.now(); + + const times = { + IssueQuery: end - start, + Parsing: parsingEnd - parsingStart, + }; + + if (groupByProperty && page === "0") { + issueResults = getGroupedIssueResults(issueResults); + } + + console.log(issueResults); + console.table(times); + + const total_pages = Math.ceil(total_count / Number(pageSize)); + const next_page_results = total_pages > parseInt(page) + 1; + + const out = { + results: issueResults, + next_cursor: `${pageSize}:${parseInt(page) + 1}:${Number(offset) + Number(pageSize)}`, + prev_cursor: `${pageSize}:${parseInt(page) - 1}:${Number(offset) - Number(pageSize)}`, + total_results: total_count, + total_count, + next_page_results, + total_pages, + }; + console.log("#### OUT", out); + + return out; + }; + + getStatus = (projectId: string) => this.projectStatus[projectId]?.issues?.status || undefined; + setStatus = (projectId: string, status: "loading" | "ready" | "error" | "syncing" | undefined = undefined) => { + set(this.projectStatus, `${projectId}.issues.status`, status); + }; + + getSync = (projectId: string) => this.projectStatus[projectId]?.issues?.sync; + setSync = (projectId: string, sync: Promise) => { + this.projectStatus[projectId].issues.sync = sync; + }; +} + +export const persistence = new Storage(); diff --git a/web/core/local-db/constants.ts b/web/core/local-db/utils/constants.ts similarity index 100% rename from web/core/local-db/constants.ts rename to web/core/local-db/utils/constants.ts diff --git a/web/core/local-db/utils/indexes.ts b/web/core/local-db/utils/indexes.ts new file mode 100644 index 00000000000..b5e8cffeae2 --- /dev/null +++ b/web/core/local-db/utils/indexes.ts @@ -0,0 +1,52 @@ +import { persistence } from "../storage.sqlite"; + +const log = console.log; +export const createIssueIndexes = async () => { + const columns = [ + "state_id", + "sort_order", + // "priority", + "priority_proxy", + "project_id", + "created_by", + "cycle_id", + ]; + + const promises = []; + + promises.push(persistence.db.exec({ sql: `CREATE UNIQUE INDEX issues_issue_id_idx ON issues (id)` })); + + columns.forEach((column) => { + promises.push( + persistence.db.exec({ sql: `CREATE INDEX issues_issue_${column}_idx ON issues (project_id, ${column})` }) + ); + }); + await Promise.all(promises); +}; + +export const createIssueMetaIndexes = async () => { + // Drop indexes + await persistence.db.exec({ sql: `CREATE INDEX issue_meta_all_idx ON issue_meta (issue_id,key,value)` }); +}; + +export const createLabelIndexes = async () => { + const columns = ["name", "id", "project_id"]; + const promises = []; + columns.forEach((column) => { + promises.push(persistence.db.exec({ sql: `CREATE INDEX labels_${column}_idx ON labels (${column})` })); + }); + await Promise.all(promises); +}; +const createIndexes = async () => { + log("### Creating indexes"); + const start = performance.now(); + const promises = [createIssueIndexes(), createIssueMetaIndexes(), createLabelIndexes()]; + try { + await Promise.all(promises); + } catch (e) { + console.log(e.result.message); + } + log("### Indexes created in", `${performance.now() - start}ms`); +}; + +export default createIndexes; diff --git a/web/core/local-db/utils/load-issues.ts b/web/core/local-db/utils/load-issues.ts new file mode 100644 index 00000000000..9263878e9b4 --- /dev/null +++ b/web/core/local-db/utils/load-issues.ts @@ -0,0 +1,58 @@ +import { IssueService } from "@/services/issue"; +import { persistence } from "../storage.sqlite"; +import { stageIssueInserts } from "./query-constructor"; + +const PAGE_SIZE = 1000; + +export const PROJECT_OFFLINE_STATUS: Record = {}; + +export const addIssue = async (issue: any) => { + persistence.db.exec("BEGIN TRANSACTION;"); + stageIssueInserts(issue); + persistence.db.exec("COMMIT;"); +}; + +export const addIssuesBulk = async (issues: any, batchSize = 100) => { + for (let i = 0; i < issues.length; i += batchSize) { + const batch = issues.slice(i, i + batchSize); + + persistence.db.exec("BEGIN TRANSACTION;"); + batch.forEach((issue: any) => { + stageIssueInserts(issue); + }); + await persistence.db.exec("COMMIT;"); + } +}; +export const deleteIssueFromLocal = async (issue_id: any) => { + const deleteQuery = `delete from issues where id='${issue_id}'`; + const deleteMetaQuery = `delete from issue_meta where issue_id='${issue_id}'`; + + persistence.db.exec("BEGIN TRANSACTION;"); + persistence.db.exec(deleteQuery); + persistence.db.exec(deleteMetaQuery); + persistence.db.exec("COMMIT;"); +}; + +export const updateIssue = async (issue: any) => { + const issue_id = issue.id; + // delete the issue and its meta data + await deleteIssueFromLocal(issue_id); + addIssue(issue); +}; + +export const syncDeletesToLocal = async (workspaceId: string, projectId: string) => { + const issueService = new IssueService(); + const response = await issueService.getDeletedIssues(workspaceId, projectId); + if (Array.isArray(response)) { + response.map(async (issue) => deleteIssueFromLocal(issue)); + } +}; + +// export const syncLocalData = async (workspaceId: string, projectId: string) => { +// persistence.syncInProgress = Promise.all([ +// syncDeletesToLocal(workspaceId, projectId), +// syncUpdatesToLocal(workspaceId, projectId), +// ]); + +// await persistence.syncInProgress; +// }; diff --git a/web/core/local-db/utils/load-labels.ts b/web/core/local-db/utils/load-labels.ts new file mode 100644 index 00000000000..ac8ab21920e --- /dev/null +++ b/web/core/local-db/utils/load-labels.ts @@ -0,0 +1,22 @@ +import { IssueLabelService } from "@/services/issue/issue_label.service"; +import { persistence } from "../storage.sqlite"; + +const stageLabelInserts = (label: any) => { + const { id, name, color, parent = "", project_id, sort_order, workspace_id } = label; + + const query = `INSERT INTO labels (id, name, color, parent, project_id, sort_order, workspace_id) VALUES ('${id}', '${name}', '${color}', '${parent}', '${project_id}', '${sort_order}', '${workspace_id}');`; + persistence.db.exec(query); +}; +export const loadLabels = async (workspaceSlug: string, batchSize = 500) => { + const issueLabelService = new IssueLabelService(); + const labels = await issueLabelService.getWorkspaceIssueLabels(workspaceSlug); + for (let i = 0; i < labels.length; i += batchSize) { + const batch = labels.slice(i, i + batchSize); + + persistence.db.exec("BEGIN TRANSACTION;"); + batch.forEach((issue: any) => { + stageLabelInserts(issue); + }); + await persistence.db.exec("COMMIT;"); + } +}; diff --git a/web/core/local-db/query-constructor.ts b/web/core/local-db/utils/query-constructor.ts similarity index 66% rename from web/core/local-db/query-constructor.ts rename to web/core/local-db/utils/query-constructor.ts index 9710681fd04..6d82ab58872 100644 --- a/web/core/local-db/query-constructor.ts +++ b/web/core/local-db/utils/query-constructor.ts @@ -1,8 +1,14 @@ -import { sq } from "date-fns/locale"; -import { ARRAY_FIELDS, GROUP_BY_MAP, PRIORITY_MAP } from "./constants"; -import { SQL } from "./sqlite"; -import { filterConstructor, wrapDateTime } from "./utils"; - +import { persistence } from "../storage.sqlite"; +import { GROUP_BY_MAP, ARRAY_FIELDS, PRIORITY_MAP } from "./constants"; +import { wrapDateTime, filterConstructor } from "./utils"; +const SPECIAL_ORDER_BY = [ + "labels__name", + "-labels__name", + "assignee__name", + "-assignee__name", + "module__name", + "-module__name", +]; export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: string, queries: any) => { const { order_by, cursor, per_page, labels, sub_issue, assignees, state, cycle, group_by, module, ...otherProps } = queries; @@ -52,22 +58,45 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st WHERE rs.rank <= ${per_page};`; } else { sql = ` - SELECT * - FROM ( - SELECT i.*, - i.${translatedGroupBy} AS group_id, -- param for group - COUNT(*) OVER (PARTITION BY i.${translatedGroupBy}) AS total_issues, - ROW_NUMBER() OVER (PARTITION BY i.${translatedGroupBy} ${orderByString}) AS rank FROM issues i - WHERE i.project_id = '${projectId}' ${filterString} - ) ranked_issues - WHERE rank <= ${per_page};`; + SELECT j.* FROM ( + SELECT i.*,i.${translatedGroupBy} as group_id, + ROW_NUMBER() OVER (PARTITION BY i.${translatedGroupBy} ${orderByString} ) as rank, + COUNT(*) OVER (PARTITION by i.${translatedGroupBy}) as total_issues + + FROM issues AS i LEFT JOIN issue_meta as im ON i.id = im.issue_id + WHERE i.project_id = '${projectId}' -- Project ID + ${filterString} + GROUP BY i.id + ) AS j WHERE rank <= ${per_page}; + + `; } console.log("####", sql); return sql; } - sql = `SELECT * FROM issues i LEFT JOIN issue_meta im ON i.id = im.issue_id WHERE 1=1 AND project_id='${projectId}' `; + // sql = `SELECT * FROM issues i LEFT JOIN issue_meta im ON i.id = im.issue_id WHERE 1=1 AND project_id='${projectId}' `; + + // sql += filterString; + + // sql += ` group by i.id`; + // sql += orderByString; + + // // Add offset and paging to query + // sql += ` LIMIT ${pageSize} OFFSET ${offset * 1 + page * pageSize};`; + + // console.log("$$$", sql); + // return sql; + + if (order_by && SPECIAL_ORDER_BY.includes(order_by)) { + const name = order_by.replace("-", ""); + sql = `SELECT i.*,im.*, s.name as ${name} FROM issues i LEFT JOIN issue_meta im ON i.id = im.issue_id LEFT JOIN labels s ON s.id = im.value `; + } else { + sql = `SELECT * FROM issues i LEFT JOIN issue_meta im ON i.id = im.issue_id `; + } + + sql += ` WHERE 1=1 AND i.project_id='${projectId}' `; sql += filterString; @@ -83,7 +112,7 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st export const issueFilterCountQueryConstructor = (workspaceSlug: string, projectId: string, queries: any) => { // Remove group by from the query to fallback to non group query - const { group_by, ...otherProps } = queries; + const { group_by, order_by, ...otherProps } = queries; let sql = issueFilterQueryConstructor(workspaceSlug, projectId, otherProps); sql = sql.replace("SELECT *", "SELECT COUNT(DISTINCT i.id) as total_count"); @@ -110,7 +139,7 @@ export const stageIssueInserts = (issue: any) => { return val; }); // Will fail when the values have a comma - SQL.db.exec({ + persistence.db.exec({ sql: `insert into issues(${keys}) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, bind: values, }); @@ -118,7 +147,7 @@ export const stageIssueInserts = (issue: any) => { const values = issue[field]; if (values) { values.forEach((val: any) => { - SQL.db.exec({ + persistence.db.exec({ sql: `insert into issue_meta(issue_id,key,value) values (?,?,?)`, bind: [issue_id, field, val], }); diff --git a/web/core/local-db/utils/query-executor.ts b/web/core/local-db/utils/query-executor.ts new file mode 100644 index 00000000000..9002d5a7bfc --- /dev/null +++ b/web/core/local-db/utils/query-executor.ts @@ -0,0 +1,13 @@ +// import { SQL } from "./sqlite"; + +import { persistence } from "../storage.sqlite"; + +export const runQuery = async (sql: string) => { + const data = await persistence.db.exec({ + sql, + rowMode: "object", + returnValue: "resultRows", + }); + + return data.result.resultRows; +}; diff --git a/web/core/local-db/tables.ts b/web/core/local-db/utils/tables.ts similarity index 72% rename from web/core/local-db/tables.ts rename to web/core/local-db/utils/tables.ts index bf165e5dd0f..b5af29eda6f 100644 --- a/web/core/local-db/tables.ts +++ b/web/core/local-db/utils/tables.ts @@ -1,3 +1,5 @@ +import createIndexes from "./indexes"; + export const createIssuesTable = (SQLITE: any) => { const sqlstr = `CREATE TABLE IF NOT EXISTS issues ( id TEXT, @@ -37,7 +39,26 @@ export const createIssueMetaTable = (SQLITE: any) => { value TEXT);`; SQLITE.exec(sqlstr); }; + +export const createLabelsTable = (SQLITE: any) => { + const sqlstr = `CREATE TABLE IF NOT EXISTS labels ( + id TEXT UNIQUE, + name TEXT, + color TEXT, + parent TEXT, + project_id TEXT, + sort_order INTEGER, + workspace_id TEXT);`; + SQLITE.exec(sqlstr); +}; export const createTables = async (SQLITE: any) => { await createIssuesTable(SQLITE); await createIssueMetaTable(SQLITE); + await createLabelsTable(SQLITE); + + try { + await createIndexes(); + } catch (e) { + console.error(e); + } }; diff --git a/web/core/local-db/utils.ts b/web/core/local-db/utils/utils.ts similarity index 67% rename from web/core/local-db/utils.ts rename to web/core/local-db/utils/utils.ts index e5cd60721f1..0b6ce568bc7 100644 --- a/web/core/local-db/utils.ts +++ b/web/core/local-db/utils/utils.ts @@ -1,5 +1,6 @@ import pick from "lodash/pick"; import { rootStore } from "@/lib/store-context"; +import { TIssue } from "@plane/types"; import { ARRAY_FIELDS, PRIORITY_MAP } from "./constants"; import { updateIssue } from "./load-issues"; @@ -60,23 +61,43 @@ export const wrapDateTime = (field: string) => { export const filterConstructor = (filters: any) => { let sql = ""; if (filters.priority) { - filters.priority_proxy = PRIORITY_MAP[filters.priority]; + filters.priority_proxy = filters.priority + .split(",") + .map((priority: string) => PRIORITY_MAP[priority]) + .join(","); delete filters.priority; } const keys = Object.keys(filters); keys.forEach((key) => { - if (key === "priority_proxy") { - sql += ` AND priority_proxy=${filters[key]} `; - return; - } const value = filters[key] ? filters[key].split(",") : ""; if (!value) return; if (ARRAY_FIELDS.includes(key)) { - sql += ` AND key='${key}' AND value IN ('${value.join("','")}')`; + sql += ` AND im.key='${key}' AND value IN ('${value.join("','")}')`; } else { sql += ` AND ${key} in ('${value.join("','")}')`; } }); return sql; }; + +export const getGroupedIssueResults = (issueResults: (TIssue & { group_id: string; total_issues: number })[]): any => { + const groupedResults: { + [key: string]: { + results: TIssue[]; + total_results: number; + }; + } = {}; + + for (const issue of issueResults) { + const { group_id, total_issues } = issue; + const groupId = group_id ? group_id : "None"; + if (groupedResults?.[groupId] !== undefined && Array.isArray(groupedResults?.[groupId]?.results)) { + groupedResults?.[groupId]?.results.push(issue); + } else { + groupedResults[groupId] = { results: [issue], total_results: total_issues }; + } + } + + return groupedResults; +}; diff --git a/web/core/services/issue/issue.service.ts b/web/core/services/issue/issue.service.ts index cda111b2716..1a835b50c84 100644 --- a/web/core/services/issue/issue.service.ts +++ b/web/core/services/issue/issue.service.ts @@ -10,10 +10,11 @@ import type { } from "@plane/types"; // helpers import { API_BASE_URL } from "@/helpers/common.helper"; +import { persistence } from "@/local-db/storage.sqlite"; // services -import { deleteIssueFromLocal } from "@/local-db/load-issues"; -import { getIssues } from "@/local-db/queries/issues"; -import { updatePersistentLayer } from "@/local-db/utils"; + +import { deleteIssueFromLocal } from "@/local-db/utils/load-issues"; +import { updatePersistentLayer } from "@/local-db/utils/utils"; import { APIService } from "@/services/api.service"; export class IssueService extends APIService { @@ -52,7 +53,7 @@ export class IssueService extends APIService { } async getIssues(workspaceSlug: string, projectId: string, queries?: any, config = {}): Promise { - return await getIssues(workspaceSlug, projectId, queries, config); + return await persistence.getIssues(projectId, queries, config); } async getDeletedIssues(workspaceSlug: string, projectId: string, queries?: any): Promise { From e75f7b1cd13a956db0e8f0a122f3f0f4fba22193 Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Tue, 20 Aug 2024 19:37:58 +0530 Subject: [PATCH 027/111] Implement subgroup by --- web/core/local-db/utils/query-constructor.ts | 21 ++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/web/core/local-db/utils/query-constructor.ts b/web/core/local-db/utils/query-constructor.ts index 6d82ab58872..2551cf69b9f 100644 --- a/web/core/local-db/utils/query-constructor.ts +++ b/web/core/local-db/utils/query-constructor.ts @@ -10,8 +10,20 @@ const SPECIAL_ORDER_BY = [ "-module__name", ]; export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: string, queries: any) => { - const { order_by, cursor, per_page, labels, sub_issue, assignees, state, cycle, group_by, module, ...otherProps } = - queries; + const { + order_by, + cursor, + per_page, + labels, + sub_issue, + assignees, + state, + cycle, + group_by, + sub_group_by, + module, + ...otherProps + } = queries; if (state) otherProps.state_id = state; if (cycle) otherProps.cycle_id = cycle; @@ -36,6 +48,7 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st console.log("###", group_by); const translatedGroupBy = GROUP_BY_MAP[group_by]; + const translatedSubGroupBy = GROUP_BY_MAP[sub_group_by]; // Check if group by is by array field if (ARRAY_FIELDS.includes(translatedGroupBy)) { sql = ` @@ -59,8 +72,8 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st } else { sql = ` SELECT j.* FROM ( - SELECT i.*,i.${translatedGroupBy} as group_id, - ROW_NUMBER() OVER (PARTITION BY i.${translatedGroupBy} ${orderByString} ) as rank, + SELECT i.*,i.${translatedGroupBy} as group_id, i.${translatedSubGroupBy} as sub_group_id, + ROW_NUMBER() OVER (PARTITION BY i.${translatedGroupBy}, i.${translatedSubGroupBy} ${orderByString} ) as rank, COUNT(*) OVER (PARTITION by i.${translatedGroupBy}) as total_issues FROM issues AS i LEFT JOIN issue_meta as im ON i.id = im.issue_id From 3ac9132999ded44c06995e615bc11fc111208253 Mon Sep 17 00:00:00 2001 From: rahulramesha Date: Tue, 20 Aug 2024 20:52:46 +0530 Subject: [PATCH 028/111] sub group by changes --- web/core/local-db/storage.sqlite.ts | 11 ++++-- web/core/local-db/utils/utils.ts | 54 +++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 3 deletions(-) diff --git a/web/core/local-db/storage.sqlite.ts b/web/core/local-db/storage.sqlite.ts index f17001f207e..2917f8eff6f 100644 --- a/web/core/local-db/storage.sqlite.ts +++ b/web/core/local-db/storage.sqlite.ts @@ -7,7 +7,7 @@ import { loadLabels } from "./utils/load-labels"; import { issueFilterQueryConstructor, issueFilterCountQueryConstructor } from "./utils/query-constructor"; import { runQuery } from "./utils/query-executor"; import { createTables } from "./utils/tables"; -import { getGroupedIssueResults } from "./utils/utils"; +import { getGroupedIssueResults, getSubGroupedIssueResults } from "./utils/utils"; const PAGE_SIZE = 1000; const log = console.log; @@ -194,7 +194,7 @@ export class Storage { } await this.getSync(projectId); - const { cursor, group_by } = queries; + const { cursor, group_by, sub_group_by } = queries; const query = issueFilterQueryConstructor(this.workspaceSlug, projectId, queries); const countQuery = issueFilterCountQueryConstructor(this.workspaceSlug, projectId, queries); @@ -209,6 +209,7 @@ export class Storage { const [pageSize, page, offset] = cursor.split(":"); const groupByProperty = EIssueGroupBYServerToProperty[group_by as typeof EIssueGroupBYServerToProperty]; + const subGroupByProperty = EIssueGroupBYServerToProperty[sub_group_by as typeof EIssueGroupBYServerToProperty]; const parsingStart = performance.now(); let issueResults = issuesRaw.map((issue: any) => { @@ -227,7 +228,11 @@ export class Storage { }; if (groupByProperty && page === "0") { - issueResults = getGroupedIssueResults(issueResults); + if (subGroupByProperty) { + issueResults = getSubGroupedIssueResults(issueResults); + } else { + issueResults = getGroupedIssueResults(issueResults); + } } console.log(issueResults); diff --git a/web/core/local-db/utils/utils.ts b/web/core/local-db/utils/utils.ts index 0b6ce568bc7..b604631018b 100644 --- a/web/core/local-db/utils/utils.ts +++ b/web/core/local-db/utils/utils.ts @@ -101,3 +101,57 @@ export const getGroupedIssueResults = (issueResults: (TIssue & { group_id: strin return groupedResults; }; + + +export const getSubGroupedIssueResults = ( + issueResults: (TIssue & { group_id: string; total_issues: number; sub_group_id: string })[] +): any => { + const subGroupedResults: { + [key: string]: { + results: { + [key: string]: { + results: TIssue[]; + total_results: number; + }; + }; + total_results: number; + }; + } = {}; + + for (const issue of issueResults) { + const { group_id, total_issues, sub_group_id } = issue; + const groupId = group_id ? group_id : "None"; + const subGroupId = sub_group_id ? sub_group_id : "None"; + + if (subGroupedResults?.[groupId] === undefined) { + subGroupedResults[groupId] = { results: {}, total_results: 0 }; + } + + if ( + subGroupedResults[groupId].results[subGroupId] !== undefined && + Array.isArray(subGroupedResults[groupId].results[subGroupId]?.results) + ) { + subGroupedResults[groupId].results[subGroupId]?.results.push(issue); + } else { + subGroupedResults[groupId].results[subGroupId] = { results: [issue], total_results: total_issues }; + } + } + + const groupByKeys = Object.keys(subGroupedResults); + + for (const groupByKey of groupByKeys) { + let totalIssues = 0; + const groupedResults = subGroupedResults[groupByKey]?.results ?? {}; + const subGroupByKeys = Object.keys(groupedResults); + + for (const subGroupByKey of subGroupByKeys) { + const subGroupedResultsCount = groupedResults[subGroupByKey].total_results ?? 0; + totalIssues += subGroupedResultsCount; + } + + subGroupedResults[groupByKey].total_results = totalIssues; + } + + return subGroupedResults; +}; + From 673b2fb1b53bd36159a0e0868757b540e7009cfe Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Wed, 21 Aug 2024 18:43:43 +0530 Subject: [PATCH 029/111] Refactor query constructor --- web/core/local-db/storage.sqlite.ts | 14 ++-- web/core/local-db/utils/query-constructor.ts | 74 +++++++++++--------- web/core/local-db/utils/query.utils.ts | 25 +++++++ 3 files changed, 74 insertions(+), 39 deletions(-) create mode 100644 web/core/local-db/utils/query.utils.ts diff --git a/web/core/local-db/storage.sqlite.ts b/web/core/local-db/storage.sqlite.ts index 2917f8eff6f..f1e7e9557b2 100644 --- a/web/core/local-db/storage.sqlite.ts +++ b/web/core/local-db/storage.sqlite.ts @@ -222,11 +222,7 @@ export class Storage { const parsingEnd = performance.now(); - const times = { - IssueQuery: end - start, - Parsing: parsingEnd - parsingStart, - }; - + const grouping = performance.now(); if (groupByProperty && page === "0") { if (subGroupByProperty) { issueResults = getSubGroupedIssueResults(issueResults); @@ -234,7 +230,13 @@ export class Storage { issueResults = getGroupedIssueResults(issueResults); } } + const groupingEnd = performance.now(); + const times = { + IssueQuery: end - start, + Parsing: parsingEnd - parsingStart, + Grouping: groupingEnd - grouping, + }; console.log(issueResults); console.table(times); @@ -250,7 +252,7 @@ export class Storage { next_page_results, total_pages, }; - console.log("#### OUT", out); + // console.log("#### OUT", out); return out; }; diff --git a/web/core/local-db/utils/query-constructor.ts b/web/core/local-db/utils/query-constructor.ts index 2551cf69b9f..ba8d110bb2b 100644 --- a/web/core/local-db/utils/query-constructor.ts +++ b/web/core/local-db/utils/query-constructor.ts @@ -1,5 +1,6 @@ import { persistence } from "../storage.sqlite"; import { GROUP_BY_MAP, ARRAY_FIELDS, PRIORITY_MAP } from "./constants"; +import { getOrderByFragment, translateQueryParams } from "./query.utils"; import { wrapDateTime, filterConstructor } from "./utils"; const SPECIAL_ORDER_BY = [ "labels__name", @@ -10,38 +11,12 @@ const SPECIAL_ORDER_BY = [ "-module__name", ]; export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: string, queries: any) => { - const { - order_by, - cursor, - per_page, - labels, - sub_issue, - assignees, - state, - cycle, - group_by, - sub_group_by, - module, - ...otherProps - } = queries; - - if (state) otherProps.state_id = state; - if (cycle) otherProps.cycle_id = cycle; - if (module) otherProps.module_ids = module; - if (labels) otherProps.label_ids = labels; - if (assignees) otherProps.assignee_ids = assignees; - - let orderByString = ""; - if (order_by) { - //if order_by starts with "-" then sort in descending order - if (order_by.startsWith("-")) { - orderByString += ` ORDER BY ${wrapDateTime(order_by.slice(1))} DESC, created_at DESC`; - } else { - orderByString += ` ORDER BY ${wrapDateTime(order_by)} ASC, created_at DESC`; - } - } + const { order_by, cursor, per_page, group_by, sub_group_by, ...otherProps } = translateQueryParams(queries); + const orderByString = getOrderByFragment(order_by); const [pageSize, page, offset] = cursor.split(":"); + const filterString = filterConstructor(otherProps); + const subFilterString = subFilterConstructor(queries); let sql = ""; if (group_by) { @@ -65,15 +40,18 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st JOIN issues i ON im.issue_id = i.id WHERE im.key = '${translatedGroupBy}' -- param for group AND i.project_id = '${projectId}' -- Project ID - ${filterString} + ${filterString} ${subFilterString} ) rs JOIN issues i ON rs.issue_id = i.id WHERE rs.rank <= ${per_page};`; } else { sql = ` SELECT j.* FROM ( - SELECT i.*,i.${translatedGroupBy} as group_id, i.${translatedSubGroupBy} as sub_group_id, - ROW_NUMBER() OVER (PARTITION BY i.${translatedGroupBy}, i.${translatedSubGroupBy} ${orderByString} ) as rank, + SELECT i.*,i.${translatedGroupBy} as group_id, + -- i.${translatedSubGroupBy} as sub_group_id, + ROW_NUMBER() OVER (PARTITION BY i.${translatedGroupBy} + -- , i.${translatedSubGroupBy} + ${orderByString} ) as rank, COUNT(*) OVER (PARTITION by i.${translatedGroupBy}) as total_issues FROM issues AS i LEFT JOIN issue_meta as im ON i.id = im.issue_id @@ -168,3 +146,33 @@ export const stageIssueInserts = (issue: any) => { } }); }; + +const subFilterConstructor = (queries: any) => { + // return ""; + let { group_by, sub_group_by } = queries; + group_by = GROUP_BY_MAP[group_by]; + sub_group_by = GROUP_BY_MAP[sub_group_by]; + const fields = new Set(); + if (ARRAY_FIELDS.includes(sub_group_by)) { + fields.add(sub_group_by); + } + if (ARRAY_FIELDS.includes(group_by)) { + fields.add(group_by); + } + + ARRAY_FIELDS.forEach((field: string) => { + if (queries[field]) { + fields.add(field); + } + }); + + if (fields.size === 0) { + return ""; + } + + let sql = ""; + + sql += 'AND im.key IN ("' + Array.from(fields).join('","') + '")'; + + return sql; +}; diff --git a/web/core/local-db/utils/query.utils.ts b/web/core/local-db/utils/query.utils.ts new file mode 100644 index 00000000000..88ce8596542 --- /dev/null +++ b/web/core/local-db/utils/query.utils.ts @@ -0,0 +1,25 @@ +import { wrapDateTime } from "./utils"; + +export const translateQueryParams = (queries: any) => { + const { labels, assignees, state, cycle, module, ...otherProps } = queries; + + if (state) otherProps.state_id = state; + if (cycle) otherProps.cycle_id = cycle; + if (module) otherProps.module_ids = module; + if (labels) otherProps.label_ids = labels; + if (assignees) otherProps.assignee_ids = assignees; + + return otherProps; +}; + +export const getOrderByFragment = (order_by: string) => { + let orderByString = ""; + if (!order_by) return orderByString; + + if (order_by.startsWith("-")) { + orderByString += ` ORDER BY ${wrapDateTime(order_by.slice(1))} DESC, created_at DESC`; + } else { + orderByString += ` ORDER BY ${wrapDateTime(order_by)} ASC, created_at DESC`; + } + return orderByString; +}; From 3c8b27dbf010915c752a4aba4c25fe68c31ec7ae Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Thu, 22 Aug 2024 12:11:36 +0530 Subject: [PATCH 030/111] Insert or update issues instead of directly adding them. --- web/core/local-db/utils/load-labels.ts | 2 +- web/core/local-db/utils/query-constructor.ts | 97 ++++++++++++++++++-- web/core/local-db/utils/tables.ts | 7 +- 3 files changed, 93 insertions(+), 13 deletions(-) diff --git a/web/core/local-db/utils/load-labels.ts b/web/core/local-db/utils/load-labels.ts index ac8ab21920e..dea8520d83e 100644 --- a/web/core/local-db/utils/load-labels.ts +++ b/web/core/local-db/utils/load-labels.ts @@ -4,7 +4,7 @@ import { persistence } from "../storage.sqlite"; const stageLabelInserts = (label: any) => { const { id, name, color, parent = "", project_id, sort_order, workspace_id } = label; - const query = `INSERT INTO labels (id, name, color, parent, project_id, sort_order, workspace_id) VALUES ('${id}', '${name}', '${color}', '${parent}', '${project_id}', '${sort_order}', '${workspace_id}');`; + const query = `INSERT OR REPLACE INTO labels (id, name, color, parent, project_id, sort_order, workspace_id) VALUES ('${id}', '${name}', '${color}', '${parent}', '${project_id}', '${sort_order}', '${workspace_id}');`; persistence.db.exec(query); }; export const loadLabels = async (workspaceSlug: string, batchSize = 500) => { diff --git a/web/core/local-db/utils/query-constructor.ts b/web/core/local-db/utils/query-constructor.ts index ba8d110bb2b..6826dd4343c 100644 --- a/web/core/local-db/utils/query-constructor.ts +++ b/web/core/local-db/utils/query-constructor.ts @@ -1,7 +1,7 @@ import { persistence } from "../storage.sqlite"; -import { GROUP_BY_MAP, ARRAY_FIELDS, PRIORITY_MAP } from "./constants"; -import { getOrderByFragment, translateQueryParams } from "./query.utils"; -import { wrapDateTime, filterConstructor } from "./utils"; +import { ARRAY_FIELDS, GROUP_BY_MAP, PRIORITY_MAP } from "./constants"; +import { getOrderByFragment, isMetaJoinRequired, translateQueryParams } from "./query.utils"; +import { filterConstructor } from "./utils"; const SPECIAL_ORDER_BY = [ "labels__name", "-labels__name", @@ -11,19 +11,91 @@ const SPECIAL_ORDER_BY = [ "-module__name", ]; export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: string, queries: any) => { - const { order_by, cursor, per_page, group_by, sub_group_by, ...otherProps } = translateQueryParams(queries); + const { order_by, cursor, per_page, group_by, sub_group_by, sub_issue, ...otherProps } = + translateQueryParams(queries); const orderByString = getOrderByFragment(order_by); const [pageSize, page, offset] = cursor.split(":"); const filterString = filterConstructor(otherProps); const subFilterString = subFilterConstructor(queries); + const translatedGroupBy = GROUP_BY_MAP[group_by]; + const translatedSubGroupBy = GROUP_BY_MAP[sub_group_by]; + // If group by or sub_group_by is present, check if we need a join. + const metaJoinRequired = isMetaJoinRequired(translatedGroupBy, translatedSubGroupBy); let sql = ""; + + if (sub_group_by) { + // CASE #1 Both are multi fields + if (ARRAY_FIELDS.includes(translatedGroupBy) && ARRAY_FIELDS.includes(translatedSubGroupBy)) { + sql = ` + --- ARRAY ARRAY + SELECT DISTINCT i.*,rs.group_id, rs.sub_group_id, rs.total_issues FROM ( + SELECT m.issue_id, m.value as group_id, msg.value as sub_group_id, + RANK() OVER (PARTITION BY m.value,msg.value ${orderByString}) as rank, + COUNT(*) OVER (PARTITION BY m.value, msg.value) as total_issues + FROM issues i + LEFT JOIN issue_meta m ON i.id = m.issue_id + LEFT JOIN issue_meta msg ON i.id = msg.issue_id + + WHERE i.project_id = '${projectId}' AND m.key = '${GROUP_BY_MAP[group_by]}' AND msg.key = '${GROUP_BY_MAP[sub_group_by]}' + ${filterString} + ) rs + JOIN issues i ON i.id = rs.issue_id + JOIN issue_meta im ON i.id = im.issue_id + WHERE rs.rank <= ${per_page} + `; + console.log("###", sql); + return sql; + } + + debugger; + // CASE #2 Group by is multi field && sub group by is single field + if (ARRAY_FIELDS.includes(translatedGroupBy) && !ARRAY_FIELDS.includes(translatedSubGroupBy)) { + sql = ` + --- ARRAY SINGLE + SELECT DISTINCT i.*,rs.group_id, rs.sub_group_id, rs.total_issues FROM ( + SELECT m.issue_id, m.value as group_id, i.${translatedSubGroupBy} as sub_group_id, + RANK() OVER (PARTITION BY m.value, i.${translatedSubGroupBy} ${orderByString}) as rank, + COUNT(*) OVER (PARTITION BY m.value, i.${translatedSubGroupBy}) as total_issues + FROM issues i + LEFT JOIN issue_meta m ON i.id = m.issue_id + + WHERE i.project_id = '${projectId}' AND m.key = '${GROUP_BY_MAP[group_by]}' + ${filterString} + ) rs + RIGHT JOIN issues i ON i.id = rs.issue_id + JOIN issue_meta im ON i.id = im.issue_id + WHERE rs.rank <= ${per_page} + `; + console.log("###", sql); + return sql; + } + } + + // if (group_by) { + // if (metaJoinRequired) { + // sql = ` + // SELECT DISTINCT i.*,rs.group_id, rs.total_issues FROM ( + // SELECT m.issue_id, m.value as group_id, + // RANK() OVER (PARTITION BY m.value ${orderByString}) as rank, + // COUNT(*) OVER (PARTITION BY m.value) as total_issues + // FROM issue_meta m + // LEFT JOIN issues i ON i.id = m.issue_id + // WHERE i.project_id = '${projectId}' AND m.key = '${GROUP_BY_MAP[group_by]}' + // ${filterString} + // ) rs + // JOIN issues i ON i.id = rs.issue_id + // JOIN issue_meta im ON i.id = im.issue_id + // WHERE rs.rank <= ${per_page} + // `; + // } + // console.log("###", sql); + // return sql; + // } if (group_by) { console.log("###", group_by); - const translatedGroupBy = GROUP_BY_MAP[group_by]; - const translatedSubGroupBy = GROUP_BY_MAP[sub_group_by]; // Check if group by is by array field if (ARRAY_FIELDS.includes(translatedGroupBy)) { sql = ` @@ -109,7 +181,7 @@ export const issueFilterCountQueryConstructor = (workspaceSlug: string, projectI sql = sql.replace("SELECT *", "SELECT COUNT(DISTINCT i.id) as total_count"); // Remove everything after group by i.id sql = `${sql.split("group by i.id")[0]};`; - + console.log("### COUNT", sql); return sql; }; @@ -131,18 +203,23 @@ export const stageIssueInserts = (issue: any) => { }); // Will fail when the values have a comma persistence.db.exec({ - sql: `insert into issues(${keys}) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, + sql: `INSERT OR REPLACE into issues(${keys}) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, bind: values, }); arrayFields.forEach((field) => { const values = issue[field]; - if (values) { + if (values && values.length) { values.forEach((val: any) => { persistence.db.exec({ - sql: `insert into issue_meta(issue_id,key,value) values (?,?,?)`, + sql: `INSERT OR REPLACE into issue_meta(issue_id,key,value) values (?,?,?) `, bind: [issue_id, field, val], }); }); + } else { + persistence.db.exec({ + sql: `INSERT OR REPLACE into issue_meta(issue_id,key,value) values (?,?,?) `, + bind: [issue_id, field, ""], + }); } }); }; diff --git a/web/core/local-db/utils/tables.ts b/web/core/local-db/utils/tables.ts index b5af29eda6f..26de77042fc 100644 --- a/web/core/local-db/utils/tables.ts +++ b/web/core/local-db/utils/tables.ts @@ -2,7 +2,7 @@ import createIndexes from "./indexes"; export const createIssuesTable = (SQLITE: any) => { const sqlstr = `CREATE TABLE IF NOT EXISTS issues ( - id TEXT, + id TEXT UNIQUE, name TEXT, state_id TEXT, sort_order REAL, @@ -36,7 +36,10 @@ export const createIssueMetaTable = (SQLITE: any) => { const sqlstr = `CREATE TABLE IF NOT EXISTS issue_meta ( issue_id TEXT, key TEXT, - value TEXT);`; + value TEXT, + UNIQUE(issue_id, key,value) +) + ;`; SQLITE.exec(sqlstr); }; From 1a1eb0def9e21f112079dd5946b26d67219ef5ea Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Thu, 22 Aug 2024 19:40:51 +0530 Subject: [PATCH 031/111] Segregated queries. Not working though!! --- web/core/local-db/storage.sqlite.ts | 6 +- web/core/local-db/utils/query-constructor.ts | 169 ++++++++++--------- web/core/local-db/utils/query.utils.ts | 52 +++++- web/core/local-db/utils/utils.ts | 10 +- 4 files changed, 146 insertions(+), 91 deletions(-) diff --git a/web/core/local-db/storage.sqlite.ts b/web/core/local-db/storage.sqlite.ts index f1e7e9557b2..3a4850d583f 100644 --- a/web/core/local-db/storage.sqlite.ts +++ b/web/core/local-db/storage.sqlite.ts @@ -111,9 +111,7 @@ export class Storage { // Load labels, members, states, modules, cycles const sync = this.syncIssues(projectId); - if (this.getStatus(projectId) === "syncing") { - this.setSync(projectId, sync); - } + this.setSync(projectId, sync); }; syncIssues = async (projectId: string) => { @@ -123,6 +121,8 @@ export class Storage { return; } + await this.getSync(projectId); + const queryParams: { cursor: string; updated_at__gt?: string } = { cursor: `${PAGE_SIZE}:0:0`, }; diff --git a/web/core/local-db/utils/query-constructor.ts b/web/core/local-db/utils/query-constructor.ts index 6826dd4343c..79d243967f3 100644 --- a/web/core/local-db/utils/query-constructor.ts +++ b/web/core/local-db/utils/query-constructor.ts @@ -1,7 +1,7 @@ import { persistence } from "../storage.sqlite"; import { ARRAY_FIELDS, GROUP_BY_MAP, PRIORITY_MAP } from "./constants"; -import { getOrderByFragment, isMetaJoinRequired, translateQueryParams } from "./query.utils"; -import { filterConstructor } from "./utils"; +import { getMetaKeysFragment, getOrderByFragment, translateQueryParams } from "./query.utils"; +import { filterConstructor, isFilterJoinRequired } from "./utils"; const SPECIAL_ORDER_BY = [ "labels__name", "-labels__name", @@ -19,10 +19,11 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st const filterString = filterConstructor(otherProps); const subFilterString = subFilterConstructor(queries); + const filterJoinRequired = isFilterJoinRequired(otherProps); + + const metaKeysFragment = getMetaKeysFragment(queries); const translatedGroupBy = GROUP_BY_MAP[group_by]; const translatedSubGroupBy = GROUP_BY_MAP[sub_group_by]; - // If group by or sub_group_by is present, check if we need a join. - const metaJoinRequired = isMetaJoinRequired(translatedGroupBy, translatedSubGroupBy); let sql = ""; if (sub_group_by) { @@ -37,19 +38,17 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st FROM issues i LEFT JOIN issue_meta m ON i.id = m.issue_id LEFT JOIN issue_meta msg ON i.id = msg.issue_id - WHERE i.project_id = '${projectId}' AND m.key = '${GROUP_BY_MAP[group_by]}' AND msg.key = '${GROUP_BY_MAP[sub_group_by]}' - ${filterString} ) rs JOIN issues i ON i.id = rs.issue_id - JOIN issue_meta im ON i.id = im.issue_id - WHERE rs.rank <= ${per_page} + ${filterJoinRequired ? "JOIN issue_meta m ON i.id = m.issue_id" : ""} + WHERE rs.rank <= ${per_page} ${filterString} + `; console.log("###", sql); return sql; } - debugger; // CASE #2 Group by is multi field && sub group by is single field if (ARRAY_FIELDS.includes(translatedGroupBy) && !ARRAY_FIELDS.includes(translatedSubGroupBy)) { sql = ` @@ -60,103 +59,107 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st COUNT(*) OVER (PARTITION BY m.value, i.${translatedSubGroupBy}) as total_issues FROM issues i LEFT JOIN issue_meta m ON i.id = m.issue_id - - WHERE i.project_id = '${projectId}' AND m.key = '${GROUP_BY_MAP[group_by]}' - ${filterString} - ) rs + WHERE i.project_id = '${projectId}' AND m.key in ${metaKeysFragment} + ${filterString} + ) rs RIGHT JOIN issues i ON i.id = rs.issue_id - JOIN issue_meta im ON i.id = im.issue_id - WHERE rs.rank <= ${per_page} + -- ${filterJoinRequired ? "LEFT JOIN issue_meta m ON i.id = m.issue_id" : ""} + WHERE rs.rank <= ${per_page} + `; console.log("###", sql); return sql; } + + debugger; + // CASE #3 Group by single field && sub group by is multi field + if (!ARRAY_FIELDS.includes(translatedGroupBy) && ARRAY_FIELDS.includes(translatedSubGroupBy)) { + sql = ` + --- SINGLE ARRAY + SELECT DISTINCT i.*,rs.group_id, rs.sub_group_id, rs.total_issues FROM ( + SELECT m.issue_id, i.${translatedGroupBy} as group_id, m.value as sub_group_id, + RANK() OVER (PARTITION BY i.${translatedGroupBy}, m.value ${orderByString}) as rank, + COUNT(*) OVER (PARTITION BY i.${translatedGroupBy}, m.value) as total_issues + FROM issues i + -- LEFT JOIN issue_meta msg ON i.id = msg.issue_id + LEFT JOIN issue_meta m ON i.id = m.issue_id + WHERE i.project_id = '${projectId}' AND m.key = '${GROUP_BY_MAP[sub_group_by]}' + ) rs + RIGHT JOIN issues i ON i.id = rs.issue_id + JOIN issue_meta m ON i.id = m.issue_id + WHERE rs.rank <= ${per_page} ${filterString} + + `; + console.log("###", sql); + return sql; + } + + // CASE #4 Both are single fields + if (!ARRAY_FIELDS.includes(translatedGroupBy) && !ARRAY_FIELDS.includes(translatedSubGroupBy)) { + sql = ` + --- SINGLE SINGLE + SELECT DISTINCT i.*,rs.group_id, rs.sub_group_id, rs.total_issues FROM ( + SELECT i.id, i.${translatedGroupBy} as group_id, i.${translatedSubGroupBy} as sub_group_id, + RANK() OVER (PARTITION BY i.${translatedGroupBy}, i.${translatedSubGroupBy} ${orderByString}) as rank, + COUNT(*) OVER (PARTITION BY i.${translatedGroupBy}, i.${translatedSubGroupBy}) as total_issues + FROM issues i + WHERE i.project_id = '${projectId}' + ${filterString} + ) rs + RIGHT JOIN issues i ON i.id = rs.id + JOIN issue_meta im ON i.id = im.issue_id + WHERE rs.rank <= ${per_page} + `; + + return sql; + } } - // if (group_by) { - // if (metaJoinRequired) { - // sql = ` - // SELECT DISTINCT i.*,rs.group_id, rs.total_issues FROM ( - // SELECT m.issue_id, m.value as group_id, - // RANK() OVER (PARTITION BY m.value ${orderByString}) as rank, - // COUNT(*) OVER (PARTITION BY m.value) as total_issues - // FROM issue_meta m - // LEFT JOIN issues i ON i.id = m.issue_id - // WHERE i.project_id = '${projectId}' AND m.key = '${GROUP_BY_MAP[group_by]}' - // ${filterString} - // ) rs - // JOIN issues i ON i.id = rs.issue_id - // JOIN issue_meta im ON i.id = im.issue_id - // WHERE rs.rank <= ${per_page} - // `; - // } - // console.log("###", sql); - // return sql; - // } if (group_by) { - console.log("###", group_by); - - // Check if group by is by array field if (ARRAY_FIELDS.includes(translatedGroupBy)) { sql = ` - SELECT i.*, rs.group_id, rs.total_issues + -- ARRAY + SELECT DISTINCT i.*, rs.group_id, rs.total_issues FROM ( - SELECT im.issue_id, - im.value AS group_id, - ROW_NUMBER() OVER ( - PARTITION BY im.value - ${orderByString} - ) AS rank, - COUNT(*) OVER (PARTITION BY im.value) AS total_issues - FROM issue_meta im - JOIN issues i ON im.issue_id = i.id - WHERE im.key = '${translatedGroupBy}' -- param for group - AND i.project_id = '${projectId}' -- Project ID - ${filterString} ${subFilterString} + SELECT m.issue_id, + m.value AS group_id, + RANK() OVER (PARTITION BY m.value ${orderByString}) AS rank, + COUNT(*) OVER (PARTITION BY m.value) AS total_issues + FROM issues i + LEFT JOIN issue_meta im ON m.issue_id = i.id + WHERE m.key = '${translatedGroupBy}' -- param for group + AND i.project_id = '${projectId}' -- Project ID + ${filterString} ) rs JOIN issues i ON rs.issue_id = i.id WHERE rs.rank <= ${per_page};`; } else { sql = ` - SELECT j.* FROM ( - SELECT i.*,i.${translatedGroupBy} as group_id, - -- i.${translatedSubGroupBy} as sub_group_id, - ROW_NUMBER() OVER (PARTITION BY i.${translatedGroupBy} - -- , i.${translatedSubGroupBy} - ${orderByString} ) as rank, - COUNT(*) OVER (PARTITION by i.${translatedGroupBy}) as total_issues - - FROM issues AS i LEFT JOIN issue_meta as im ON i.id = im.issue_id - WHERE i.project_id = '${projectId}' -- Project ID - ${filterString} - GROUP BY i.id - ) AS j WHERE rank <= ${per_page}; - - `; + -- SINGLE + SELECT DISTINCT i.*, rs.group_id, rs.total_issues + FROM ( + SELECT i.id as issue_id, + i.${translatedGroupBy} AS group_id, + RANK() OVER (PARTITION BY i.${translatedGroupBy} ${orderByString}) AS rank, + COUNT(*) OVER (PARTITION BY i.${translatedGroupBy}) AS total_issues + FROM issues i + ${filterJoinRequired ? "LEFT JOIN issue_meta im ON i.id = m.issue_id" : ""} + WHERE i.project_id = '${projectId}' -- Project ID + ${filterString} + ) rs + JOIN issues i ON rs.issue_id = i.id + WHERE rs.rank <= ${per_page};`; } - console.log("####", sql); + console.log("###", sql); return sql; } - // sql = `SELECT * FROM issues i LEFT JOIN issue_meta im ON i.id = im.issue_id WHERE 1=1 AND project_id='${projectId}' `; - - // sql += filterString; - - // sql += ` group by i.id`; - // sql += orderByString; - - // // Add offset and paging to query - // sql += ` LIMIT ${pageSize} OFFSET ${offset * 1 + page * pageSize};`; - - // console.log("$$$", sql); - // return sql; - if (order_by && SPECIAL_ORDER_BY.includes(order_by)) { const name = order_by.replace("-", ""); - sql = `SELECT i.*,im.*, s.name as ${name} FROM issues i LEFT JOIN issue_meta im ON i.id = im.issue_id LEFT JOIN labels s ON s.id = im.value `; + sql = `SELECT i.*,m.*, s.name as ${name} FROM issues i LEFT JOIN issue_meta m ON i.id = m.issue_id LEFT JOIN labels s ON s.id = m.value `; } else { - sql = `SELECT * FROM issues i LEFT JOIN issue_meta im ON i.id = im.issue_id `; + sql = `SELECT * FROM issues i LEFT JOIN issue_meta m ON i.id = m.issue_id `; } sql += ` WHERE 1=1 AND i.project_id='${projectId}' `; @@ -175,7 +178,7 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st export const issueFilterCountQueryConstructor = (workspaceSlug: string, projectId: string, queries: any) => { // Remove group by from the query to fallback to non group query - const { group_by, order_by, ...otherProps } = queries; + const { group_by, sub_group_by, order_by, ...otherProps } = queries; let sql = issueFilterQueryConstructor(workspaceSlug, projectId, otherProps); sql = sql.replace("SELECT *", "SELECT COUNT(DISTINCT i.id) as total_count"); diff --git a/web/core/local-db/utils/query.utils.ts b/web/core/local-db/utils/query.utils.ts index 88ce8596542..7bdf941d9bd 100644 --- a/web/core/local-db/utils/query.utils.ts +++ b/web/core/local-db/utils/query.utils.ts @@ -1,3 +1,4 @@ +import { ARRAY_FIELDS, GROUP_BY_MAP } from "./constants"; import { wrapDateTime } from "./utils"; export const translateQueryParams = (queries: any) => { @@ -16,10 +17,57 @@ export const getOrderByFragment = (order_by: string) => { let orderByString = ""; if (!order_by) return orderByString; + if (order_by.includes("priority")) { + order_by = order_by.replace("priority", "priority_proxy"); + } if (order_by.startsWith("-")) { - orderByString += ` ORDER BY ${wrapDateTime(order_by.slice(1))} DESC, created_at DESC`; + orderByString += ` ORDER BY ${wrapDateTime(order_by.slice(1))} DESC NULLS LAST, created_at DESC`; } else { - orderByString += ` ORDER BY ${wrapDateTime(order_by)} ASC, created_at DESC`; + orderByString += ` ORDER BY ${wrapDateTime(order_by)} ASC NULLS LAST, created_at DESC`; } return orderByString; }; + +export const getSubGroupByFragment = (sub_group_by: string) => { + let subGroupByString = ""; + if (!sub_group_by) return subGroupByString; + + if (ARRAY_FIELDS.includes(sub_group_by)) { + subGroupByString = `,m.value ${sub_group_by} `; + } else { + subGroupByString = `, im.value as sub_group_id`; + } +}; + +export const isMetaJoinRequired = (groupBy: string, subGroupBy: string) => + ARRAY_FIELDS.includes(groupBy) || ARRAY_FIELDS.includes(subGroupBy); + +export const getMetaKeysFragment = (queries: any) => { + const { group_by, sub_group_by, ...otherProps } = translateQueryParams(queries); + + const translatedGroupBy = GROUP_BY_MAP[group_by]; + const translatedSubGroupBy = GROUP_BY_MAP[sub_group_by]; + + const fields = new Set(); + if (ARRAY_FIELDS.includes(translatedGroupBy)) { + fields.add(translatedGroupBy); + } + + if (ARRAY_FIELDS.includes(translatedSubGroupBy)) { + fields.add(translatedSubGroupBy); + } + + const keys = Object.keys(otherProps); + + keys.forEach((field: string) => { + if (ARRAY_FIELDS.includes(field)) { + fields.add(field); + } + }); + + let sql; + + sql = ` ('${Array.from(fields).join("','")}')`; + + return sql; +}; diff --git a/web/core/local-db/utils/utils.ts b/web/core/local-db/utils/utils.ts index b604631018b..5c6e6518d9f 100644 --- a/web/core/local-db/utils/utils.ts +++ b/web/core/local-db/utils/utils.ts @@ -73,14 +73,20 @@ export const filterConstructor = (filters: any) => { const value = filters[key] ? filters[key].split(",") : ""; if (!value) return; if (ARRAY_FIELDS.includes(key)) { - sql += ` AND im.key='${key}' AND value IN ('${value.join("','")}')`; + sql += ` AND m.key='${key}' AND value IN ('${value.join("','")}')`; } else { sql += ` AND ${key} in ('${value.join("','")}')`; } }); + debugger; return sql; }; +export const isFilterJoinRequired = (filters: any) => { + const keys = Object.keys(filters); + return keys.some((key) => ARRAY_FIELDS.includes(key)); +}; + export const getGroupedIssueResults = (issueResults: (TIssue & { group_id: string; total_issues: number })[]): any => { const groupedResults: { [key: string]: { @@ -102,7 +108,6 @@ export const getGroupedIssueResults = (issueResults: (TIssue & { group_id: strin return groupedResults; }; - export const getSubGroupedIssueResults = ( issueResults: (TIssue & { group_id: string; total_issues: number; sub_group_id: string })[] ): any => { @@ -154,4 +159,3 @@ export const getSubGroupedIssueResults = ( return subGroupedResults; }; - From 59b19fa4547e287a94718d2d7df7594e909ca654 Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Sun, 25 Aug 2024 15:15:01 +0530 Subject: [PATCH 032/111] - Get filtered issues and then group them. - Cleanup code. - Implement order by labels. --- web/core/local-db/storage.sqlite.ts | 3 +- web/core/local-db/utils/query-constructor.ts | 171 ++++------------- web/core/local-db/utils/query.utils.ts | 186 ++++++++++++++++--- web/core/local-db/utils/utils.ts | 32 +--- 4 files changed, 202 insertions(+), 190 deletions(-) diff --git a/web/core/local-db/storage.sqlite.ts b/web/core/local-db/storage.sqlite.ts index 3a4850d583f..c539c672d8d 100644 --- a/web/core/local-db/storage.sqlite.ts +++ b/web/core/local-db/storage.sqlite.ts @@ -220,6 +220,8 @@ export class Storage { return issue; }); + console.log("#### Issue Results", issueResults.length); + const parsingEnd = performance.now(); const grouping = performance.now(); @@ -252,7 +254,6 @@ export class Storage { next_page_results, total_pages, }; - // console.log("#### OUT", out); return out; }; diff --git a/web/core/local-db/utils/query-constructor.ts b/web/core/local-db/utils/query-constructor.ts index 79d243967f3..40a51835971 100644 --- a/web/core/local-db/utils/query-constructor.ts +++ b/web/core/local-db/utils/query-constructor.ts @@ -1,8 +1,13 @@ import { persistence } from "../storage.sqlite"; import { ARRAY_FIELDS, GROUP_BY_MAP, PRIORITY_MAP } from "./constants"; -import { getMetaKeysFragment, getOrderByFragment, translateQueryParams } from "./query.utils"; -import { filterConstructor, isFilterJoinRequired } from "./utils"; -const SPECIAL_ORDER_BY = [ +import { + getFilteredRowsForGrouping, + getMetaKeys, + getOrderByFragment, + singleFilterConstructor, + translateQueryParams, +} from "./query.utils"; +export const SPECIAL_ORDER_BY = [ "labels__name", "-labels__name", "assignee__name", @@ -16,157 +21,53 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st const orderByString = getOrderByFragment(order_by); const [pageSize, page, offset] = cursor.split(":"); - const filterString = filterConstructor(otherProps); - const subFilterString = subFilterConstructor(queries); - - const filterJoinRequired = isFilterJoinRequired(otherProps); - - const metaKeysFragment = getMetaKeysFragment(queries); - const translatedGroupBy = GROUP_BY_MAP[group_by]; - const translatedSubGroupBy = GROUP_BY_MAP[sub_group_by]; let sql = ""; if (sub_group_by) { - // CASE #1 Both are multi fields - if (ARRAY_FIELDS.includes(translatedGroupBy) && ARRAY_FIELDS.includes(translatedSubGroupBy)) { - sql = ` - --- ARRAY ARRAY - SELECT DISTINCT i.*,rs.group_id, rs.sub_group_id, rs.total_issues FROM ( - SELECT m.issue_id, m.value as group_id, msg.value as sub_group_id, - RANK() OVER (PARTITION BY m.value,msg.value ${orderByString}) as rank, - COUNT(*) OVER (PARTITION BY m.value, msg.value) as total_issues - FROM issues i - LEFT JOIN issue_meta m ON i.id = m.issue_id - LEFT JOIN issue_meta msg ON i.id = msg.issue_id - WHERE i.project_id = '${projectId}' AND m.key = '${GROUP_BY_MAP[group_by]}' AND msg.key = '${GROUP_BY_MAP[sub_group_by]}' - ) rs - JOIN issues i ON i.id = rs.issue_id - ${filterJoinRequired ? "JOIN issue_meta m ON i.id = m.issue_id" : ""} - WHERE rs.rank <= ${per_page} ${filterString} - - `; - console.log("###", sql); - return sql; - } - - // CASE #2 Group by is multi field && sub group by is single field - if (ARRAY_FIELDS.includes(translatedGroupBy) && !ARRAY_FIELDS.includes(translatedSubGroupBy)) { - sql = ` - --- ARRAY SINGLE - SELECT DISTINCT i.*,rs.group_id, rs.sub_group_id, rs.total_issues FROM ( - SELECT m.issue_id, m.value as group_id, i.${translatedSubGroupBy} as sub_group_id, - RANK() OVER (PARTITION BY m.value, i.${translatedSubGroupBy} ${orderByString}) as rank, - COUNT(*) OVER (PARTITION BY m.value, i.${translatedSubGroupBy}) as total_issues - FROM issues i - LEFT JOIN issue_meta m ON i.id = m.issue_id - WHERE i.project_id = '${projectId}' AND m.key in ${metaKeysFragment} - ${filterString} - ) rs - RIGHT JOIN issues i ON i.id = rs.issue_id - -- ${filterJoinRequired ? "LEFT JOIN issue_meta m ON i.id = m.issue_id" : ""} - WHERE rs.rank <= ${per_page} - - `; - console.log("###", sql); - return sql; - } - - debugger; - // CASE #3 Group by single field && sub group by is multi field - if (!ARRAY_FIELDS.includes(translatedGroupBy) && ARRAY_FIELDS.includes(translatedSubGroupBy)) { - sql = ` - --- SINGLE ARRAY - SELECT DISTINCT i.*,rs.group_id, rs.sub_group_id, rs.total_issues FROM ( - SELECT m.issue_id, i.${translatedGroupBy} as group_id, m.value as sub_group_id, - RANK() OVER (PARTITION BY i.${translatedGroupBy}, m.value ${orderByString}) as rank, - COUNT(*) OVER (PARTITION BY i.${translatedGroupBy}, m.value) as total_issues - FROM issues i - -- LEFT JOIN issue_meta msg ON i.id = msg.issue_id - LEFT JOIN issue_meta m ON i.id = m.issue_id - WHERE i.project_id = '${projectId}' AND m.key = '${GROUP_BY_MAP[sub_group_by]}' - ) rs - RIGHT JOIN issues i ON i.id = rs.issue_id - JOIN issue_meta m ON i.id = m.issue_id - WHERE rs.rank <= ${per_page} ${filterString} + sql = getFilteredRowsForGrouping(projectId, queries); + sql += `, rn AS ( SELECT fi.*, + RANK() OVER (PARTITION BY group_id, sub_group_id ${orderByString}) as rank, + COUNT(*) OVER (PARTITION by group_id, sub_group_id) as total_issues from fi) SELECT * FROM rn + WHERE rank <= ${per_page} `; - console.log("###", sql); - return sql; - } - // CASE #4 Both are single fields - if (!ARRAY_FIELDS.includes(translatedGroupBy) && !ARRAY_FIELDS.includes(translatedSubGroupBy)) { - sql = ` - --- SINGLE SINGLE - SELECT DISTINCT i.*,rs.group_id, rs.sub_group_id, rs.total_issues FROM ( - SELECT i.id, i.${translatedGroupBy} as group_id, i.${translatedSubGroupBy} as sub_group_id, - RANK() OVER (PARTITION BY i.${translatedGroupBy}, i.${translatedSubGroupBy} ${orderByString}) as rank, - COUNT(*) OVER (PARTITION BY i.${translatedGroupBy}, i.${translatedSubGroupBy}) as total_issues - FROM issues i - WHERE i.project_id = '${projectId}' - ${filterString} - ) rs - RIGHT JOIN issues i ON i.id = rs.id - JOIN issue_meta im ON i.id = im.issue_id - WHERE rs.rank <= ${per_page} - `; + console.log("###", sql); - return sql; - } + return sql; } - if (group_by) { - if (ARRAY_FIELDS.includes(translatedGroupBy)) { - sql = ` - -- ARRAY - SELECT DISTINCT i.*, rs.group_id, rs.total_issues - FROM ( - SELECT m.issue_id, - m.value AS group_id, - RANK() OVER (PARTITION BY m.value ${orderByString}) AS rank, - COUNT(*) OVER (PARTITION BY m.value) AS total_issues - FROM issues i - LEFT JOIN issue_meta im ON m.issue_id = i.id - WHERE m.key = '${translatedGroupBy}' -- param for group - AND i.project_id = '${projectId}' -- Project ID - ${filterString} - ) rs - JOIN issues i ON rs.issue_id = i.id - WHERE rs.rank <= ${per_page};`; - } else { - sql = ` - -- SINGLE - SELECT DISTINCT i.*, rs.group_id, rs.total_issues - FROM ( - SELECT i.id as issue_id, - i.${translatedGroupBy} AS group_id, - RANK() OVER (PARTITION BY i.${translatedGroupBy} ${orderByString}) AS rank, - COUNT(*) OVER (PARTITION BY i.${translatedGroupBy}) AS total_issues - FROM issues i - ${filterJoinRequired ? "LEFT JOIN issue_meta im ON i.id = m.issue_id" : ""} - WHERE i.project_id = '${projectId}' -- Project ID - ${filterString} - ) rs - JOIN issues i ON rs.issue_id = i.id - WHERE rs.rank <= ${per_page};`; - } + sql = getFilteredRowsForGrouping(projectId, queries); + sql += `, rn AS ( SELECT fi.*, + RANK() OVER (PARTITION BY group_id ${orderByString}) as rank, + COUNT(*) OVER (PARTITION by group_id) as total_issues from fi) SELECT * FROM rn + WHERE rank <= ${per_page} + `; console.log("###", sql); + return sql; } + const filterJoinFields = getMetaKeys(queries); if (order_by && SPECIAL_ORDER_BY.includes(order_by)) { const name = order_by.replace("-", ""); - sql = `SELECT i.*,m.*, s.name as ${name} FROM issues i LEFT JOIN issue_meta m ON i.id = m.issue_id LEFT JOIN labels s ON s.id = m.value `; + sql = `SELECT i.*, s.name as ${name} from issues i`; } else { - sql = `SELECT * FROM issues i LEFT JOIN issue_meta m ON i.id = m.issue_id `; + sql = `SELECT * from issues i`; } + filterJoinFields.forEach((field: string) => { + const value = otherProps[field] || ""; + sql += ` INNER JOIN issue_meta ${field} ON i.id = ${field}.issue_id AND ${field}.key = '${field}' AND ${field}.value IN ('${value.split(",").join("','")}') + `; + }); - sql += ` WHERE 1=1 AND i.project_id='${projectId}' `; - - sql += filterString; - - sql += ` group by i.id`; + if (order_by && SPECIAL_ORDER_BY.includes(order_by)) { + sql += ` + LEFT JOIN issue_meta label_ids ON i.id = label_ids.issue_id AND label_ids.key = 'label_ids' + INNER JOIN labels s ON s.id = label_ids.value`; + } + sql += ` WHERE i.project_id = '${projectId}' ${singleFilterConstructor(otherProps)} group by i.id`; sql += orderByString; // Add offset and paging to query diff --git a/web/core/local-db/utils/query.utils.ts b/web/core/local-db/utils/query.utils.ts index 7bdf941d9bd..53cbe71c726 100644 --- a/web/core/local-db/utils/query.utils.ts +++ b/web/core/local-db/utils/query.utils.ts @@ -1,25 +1,44 @@ -import { ARRAY_FIELDS, GROUP_BY_MAP } from "./constants"; +import { ARRAY_FIELDS, GROUP_BY_MAP, PRIORITY_MAP } from "./constants"; import { wrapDateTime } from "./utils"; export const translateQueryParams = (queries: any) => { - const { labels, assignees, state, cycle, module, ...otherProps } = queries; + const { group_by, sub_group_by, labels, assignees, state, cycle, module, priority, ...otherProps } = queries; + const order_by = queries.order_by; if (state) otherProps.state_id = state; if (cycle) otherProps.cycle_id = cycle; if (module) otherProps.module_ids = module; if (labels) otherProps.label_ids = labels; if (assignees) otherProps.assignee_ids = assignees; + if (group_by) otherProps.group_by = GROUP_BY_MAP[group_by]; + if (sub_group_by) otherProps.sub_group_by = GROUP_BY_MAP[sub_group_by]; + if (priority) { + otherProps.priority_proxy = priority + .split(",") + .map((priority: string) => PRIORITY_MAP[priority]) + .join(","); + } + + if (order_by?.includes("priority")) { + otherProps.order_by = order_by.replace("priority", "priority_proxy"); + } + + // For each property value, replace None with empty string + Object.keys(otherProps).forEach((key) => { + if (otherProps[key] === "None") { + otherProps[key] = ""; + } + }); return otherProps; }; export const getOrderByFragment = (order_by: string) => { + debugger; + let orderByString = ""; if (!order_by) return orderByString; - if (order_by.includes("priority")) { - order_by = order_by.replace("priority", "priority_proxy"); - } if (order_by.startsWith("-")) { orderByString += ` ORDER BY ${wrapDateTime(order_by.slice(1))} DESC NULLS LAST, created_at DESC`; } else { @@ -28,33 +47,19 @@ export const getOrderByFragment = (order_by: string) => { return orderByString; }; -export const getSubGroupByFragment = (sub_group_by: string) => { - let subGroupByString = ""; - if (!sub_group_by) return subGroupByString; - - if (ARRAY_FIELDS.includes(sub_group_by)) { - subGroupByString = `,m.value ${sub_group_by} `; - } else { - subGroupByString = `, im.value as sub_group_id`; - } -}; - export const isMetaJoinRequired = (groupBy: string, subGroupBy: string) => ARRAY_FIELDS.includes(groupBy) || ARRAY_FIELDS.includes(subGroupBy); export const getMetaKeysFragment = (queries: any) => { const { group_by, sub_group_by, ...otherProps } = translateQueryParams(queries); - const translatedGroupBy = GROUP_BY_MAP[group_by]; - const translatedSubGroupBy = GROUP_BY_MAP[sub_group_by]; - const fields = new Set(); - if (ARRAY_FIELDS.includes(translatedGroupBy)) { - fields.add(translatedGroupBy); + if (ARRAY_FIELDS.includes(group_by)) { + fields.add(group_by); } - if (ARRAY_FIELDS.includes(translatedSubGroupBy)) { - fields.add(translatedSubGroupBy); + if (ARRAY_FIELDS.includes(sub_group_by)) { + fields.add(sub_group_by); } const keys = Object.keys(otherProps); @@ -71,3 +76,138 @@ export const getMetaKeysFragment = (queries: any) => { return sql; }; + +export const getMetaKeys = (queries: any) => { + const { group_by, sub_group_by, ...otherProps } = translateQueryParams(queries); + + const fields = new Set(); + if (ARRAY_FIELDS.includes(group_by)) { + fields.add(group_by); + } + + if (ARRAY_FIELDS.includes(sub_group_by)) { + fields.add(sub_group_by); + } + + const keys = Object.keys(otherProps); + + keys.forEach((field: string) => { + if (ARRAY_FIELDS.includes(field)) { + fields.add(field); + } + }); + + return Array.from(fields); +}; + +const areJoinsRequired = (queries: any) => { + const { group_by, sub_group_by, ...otherProps } = translateQueryParams(queries); + + if (ARRAY_FIELDS.includes(group_by) || ARRAY_FIELDS.includes(sub_group_by)) { + return true; + } + if (Object.keys(otherProps).some((field) => ARRAY_FIELDS.includes(field))) { + return true; + } + return false; +}; + +// Apply filters to the query +export const getFilteredRowsForGrouping = (projectId: string, queries: any) => { + const { group_by, sub_group_by, ...otherProps } = translateQueryParams(queries); + + const filterJoinFields = getMetaKeys(otherProps); + + let issueTableFilterFields = getSingleFilterFields(queries); + issueTableFilterFields = issueTableFilterFields.length ? "," + issueTableFilterFields.join(",") : ""; + + const joinsRequired = areJoinsRequired(queries); + + let sql = ""; + if (!joinsRequired) { + sql = `WITH fi as (SELECT *`; + if (group_by) { + sql += `, i.${group_by} as group_id`; + } + if (sub_group_by) { + sql += `, i.${sub_group_by} as sub_group_id`; + } + sql += ` FROM issues i WHERE project_id = '${projectId}' ${singleFilterConstructor(otherProps)}) `; + return sql; + } + + sql = `WITH fi AS (SELECT * FROM (`; + sql += `SELECT i.id,i.created_at ${issueTableFilterFields} `; + if (group_by) { + if (ARRAY_FIELDS.includes(group_by)) { + sql += `, ${group_by}.value as group_id`; + } else { + sql += `, i.${group_by} as group_id`; + } + } + if (sub_group_by) { + if (ARRAY_FIELDS.includes(sub_group_by)) { + sql += `, ${sub_group_by}.value as sub_group_id`; + } else { + sql += `, i.${sub_group_by} as sub_group_id`; + } + } + + sql += ` from issues i + `; + filterJoinFields.forEach((field: string) => { + sql += ` INNER JOIN issue_meta ${field} ON i.id = ${field}.issue_id AND ${field}.key = '${field}' AND ${field}.value IN ('${otherProps[field].split(",").join("','")}') + `; + }); + + // If group by field is not already joined, join it + if (ARRAY_FIELDS.includes(group_by) && !filterJoinFields.includes(group_by)) { + sql += ` LEFT JOIN issue_meta ${group_by} ON i.id = ${group_by}.issue_id AND ${group_by}.key = '${group_by}' + `; + } + if (ARRAY_FIELDS.includes(sub_group_by) && !filterJoinFields.includes(sub_group_by)) { + sql += ` LEFT JOIN issue_meta ${sub_group_by} ON i.id = ${sub_group_by}.issue_id AND ${sub_group_by}.key = '${sub_group_by}' + `; + } + + sql += ` WHERE i.project_id = '${projectId}' ${singleFilterConstructor(otherProps)} group by i.id`; + + sql += `) AS slim LEFT JOIN issues ON slim.id = issues.id) + `; + return sql; +}; + +export const singleFilterConstructor = (queries: any) => { + const { order_by, cursor, per_page, group_by, sub_group_by, sub_issue, ...filters } = translateQueryParams(queries); + + let sql = ""; + const keys = Object.keys(filters); + + keys.forEach((key) => { + const value = filters[key] ? filters[key].split(",") : ""; + if (!value) return; + if (!ARRAY_FIELDS.includes(key)) { + sql += ` AND ${key} in ('${value.join("','")}')`; + } + }); + return sql; +}; + +const getSingleFilterFields = (queries) => { + debugger; + const { order_by, cursor, per_page, group_by, sub_group_by, sub_issue, ...otherProps } = + translateQueryParams(queries); + + const fields = new Set(); + if (order_by) fields.add(order_by); + + const keys = Object.keys(otherProps); + + keys.forEach((field: string) => { + if (!ARRAY_FIELDS.includes(field)) { + fields.add(field); + } + }); + + return Array.from(fields); +}; diff --git a/web/core/local-db/utils/utils.ts b/web/core/local-db/utils/utils.ts index 5c6e6518d9f..e4724ab78e8 100644 --- a/web/core/local-db/utils/utils.ts +++ b/web/core/local-db/utils/utils.ts @@ -1,7 +1,6 @@ import pick from "lodash/pick"; -import { rootStore } from "@/lib/store-context"; import { TIssue } from "@plane/types"; -import { ARRAY_FIELDS, PRIORITY_MAP } from "./constants"; +import { rootStore } from "@/lib/store-context"; import { updateIssue } from "./load-issues"; export const log = console.log; @@ -58,35 +57,6 @@ export const wrapDateTime = (field: string) => { return field; }; -export const filterConstructor = (filters: any) => { - let sql = ""; - if (filters.priority) { - filters.priority_proxy = filters.priority - .split(",") - .map((priority: string) => PRIORITY_MAP[priority]) - .join(","); - delete filters.priority; - } - const keys = Object.keys(filters); - - keys.forEach((key) => { - const value = filters[key] ? filters[key].split(",") : ""; - if (!value) return; - if (ARRAY_FIELDS.includes(key)) { - sql += ` AND m.key='${key}' AND value IN ('${value.join("','")}')`; - } else { - sql += ` AND ${key} in ('${value.join("','")}')`; - } - }); - debugger; - return sql; -}; - -export const isFilterJoinRequired = (filters: any) => { - const keys = Object.keys(filters); - return keys.some((key) => ARRAY_FIELDS.includes(key)); -}; - export const getGroupedIssueResults = (issueResults: (TIssue & { group_id: string; total_issues: number })[]): any => { const groupedResults: { [key: string]: { From ac40489eea7b72e7332a61cf51f081e248e392b0 Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Sun, 25 Aug 2024 15:55:05 +0530 Subject: [PATCH 033/111] Fix build issues --- web/core/local-db/storage.sqlite.ts | 21 +++++++++--- web/core/local-db/utils/indexes.ts | 6 ++-- web/core/local-db/utils/query-constructor.ts | 34 ++------------------ web/core/local-db/utils/query.utils.ts | 18 +++++------ web/core/services/issue/issue.service.ts | 3 +- 5 files changed, 32 insertions(+), 50 deletions(-) diff --git a/web/core/local-db/storage.sqlite.ts b/web/core/local-db/storage.sqlite.ts index c539c672d8d..71494ded617 100644 --- a/web/core/local-db/storage.sqlite.ts +++ b/web/core/local-db/storage.sqlite.ts @@ -8,7 +8,9 @@ import { issueFilterQueryConstructor, issueFilterCountQueryConstructor } from ". import { runQuery } from "./utils/query-executor"; import { createTables } from "./utils/tables"; import { getGroupedIssueResults, getSubGroupedIssueResults } from "./utils/utils"; - +declare module "@sqlite.org/sqlite-wasm" { + export function sqlite3Worker1Promiser(...args: any): any; +} const PAGE_SIZE = 1000; const log = console.log; const error = console.error; @@ -42,7 +44,14 @@ export class Storage { initialize = async (workspaceSlug: string): Promise => { this.workspaceInitPromise = this._initialize(workspaceSlug); - await this.workspaceInitPromise; + try { + await this.workspaceInitPromise; + return true; + } catch (err) { + error(err); + this.status = "error"; + return false; + } }; _initialize = async (workspaceSlug: string): Promise => { @@ -187,7 +196,7 @@ export class Storage { getIssues = async (projectId: string, queries: any, config: any) => { console.log("#### Queries", queries); - if (this.getStatus(projectId) === "loading" || window.DISABLE_LOCAL) { + if (this.getStatus(projectId) === "loading" || (window as any).DISABLE_LOCAL) { info(`Project ${projectId} is loading, falling back to server`); const issueService = new IssueService(); return await issueService.getIssuesFromServer(this.workspaceSlug, projectId, queries); @@ -208,8 +217,10 @@ export class Storage { const [pageSize, page, offset] = cursor.split(":"); - const groupByProperty = EIssueGroupBYServerToProperty[group_by as typeof EIssueGroupBYServerToProperty]; - const subGroupByProperty = EIssueGroupBYServerToProperty[sub_group_by as typeof EIssueGroupBYServerToProperty]; + const groupByProperty: string = + EIssueGroupBYServerToProperty[group_by as keyof typeof EIssueGroupBYServerToProperty]; + const subGroupByProperty = + EIssueGroupBYServerToProperty[sub_group_by as keyof typeof EIssueGroupBYServerToProperty]; const parsingStart = performance.now(); let issueResults = issuesRaw.map((issue: any) => { diff --git a/web/core/local-db/utils/indexes.ts b/web/core/local-db/utils/indexes.ts index b5e8cffeae2..379c24e103f 100644 --- a/web/core/local-db/utils/indexes.ts +++ b/web/core/local-db/utils/indexes.ts @@ -12,7 +12,7 @@ export const createIssueIndexes = async () => { "cycle_id", ]; - const promises = []; + const promises: Promise[] = []; promises.push(persistence.db.exec({ sql: `CREATE UNIQUE INDEX issues_issue_id_idx ON issues (id)` })); @@ -31,7 +31,7 @@ export const createIssueMetaIndexes = async () => { export const createLabelIndexes = async () => { const columns = ["name", "id", "project_id"]; - const promises = []; + const promises: Promise[] = []; columns.forEach((column) => { promises.push(persistence.db.exec({ sql: `CREATE INDEX labels_${column}_idx ON labels (${column})` })); }); @@ -44,7 +44,7 @@ const createIndexes = async () => { try { await Promise.all(promises); } catch (e) { - console.log(e.result.message); + console.log((e as Error).message); } log("### Indexes created in", `${performance.now() - start}ms`); }; diff --git a/web/core/local-db/utils/query-constructor.ts b/web/core/local-db/utils/query-constructor.ts index 40a51835971..b15f0519e03 100644 --- a/web/core/local-db/utils/query-constructor.ts +++ b/web/core/local-db/utils/query-constructor.ts @@ -1,5 +1,5 @@ import { persistence } from "../storage.sqlite"; -import { ARRAY_FIELDS, GROUP_BY_MAP, PRIORITY_MAP } from "./constants"; +import { PRIORITY_MAP } from "./constants"; import { getFilteredRowsForGrouping, getMetaKeys, @@ -93,7 +93,7 @@ const arrayFields = ["label_ids", "assignee_ids", "module_ids"]; export const stageIssueInserts = (issue: any) => { const issue_id = issue.id; - issue.priority_proxy = PRIORITY_MAP[issue.priority]; + issue.priority_proxy = PRIORITY_MAP[issue.priority as keyof typeof PRIORITY_MAP]; const keys = Object.keys(issue).join(","); const values = Object.values(issue).map((val) => { @@ -127,33 +127,3 @@ export const stageIssueInserts = (issue: any) => { } }); }; - -const subFilterConstructor = (queries: any) => { - // return ""; - let { group_by, sub_group_by } = queries; - group_by = GROUP_BY_MAP[group_by]; - sub_group_by = GROUP_BY_MAP[sub_group_by]; - const fields = new Set(); - if (ARRAY_FIELDS.includes(sub_group_by)) { - fields.add(sub_group_by); - } - if (ARRAY_FIELDS.includes(group_by)) { - fields.add(group_by); - } - - ARRAY_FIELDS.forEach((field: string) => { - if (queries[field]) { - fields.add(field); - } - }); - - if (fields.size === 0) { - return ""; - } - - let sql = ""; - - sql += 'AND im.key IN ("' + Array.from(fields).join('","') + '")'; - - return sql; -}; diff --git a/web/core/local-db/utils/query.utils.ts b/web/core/local-db/utils/query.utils.ts index 53cbe71c726..ab9c9f7141a 100644 --- a/web/core/local-db/utils/query.utils.ts +++ b/web/core/local-db/utils/query.utils.ts @@ -10,12 +10,12 @@ export const translateQueryParams = (queries: any) => { if (module) otherProps.module_ids = module; if (labels) otherProps.label_ids = labels; if (assignees) otherProps.assignee_ids = assignees; - if (group_by) otherProps.group_by = GROUP_BY_MAP[group_by]; - if (sub_group_by) otherProps.sub_group_by = GROUP_BY_MAP[sub_group_by]; + if (group_by) otherProps.group_by = GROUP_BY_MAP[group_by as keyof typeof GROUP_BY_MAP]; + if (sub_group_by) otherProps.sub_group_by = GROUP_BY_MAP[sub_group_by as keyof typeof GROUP_BY_MAP]; if (priority) { otherProps.priority_proxy = priority .split(",") - .map((priority: string) => PRIORITY_MAP[priority]) + .map((priority: string) => PRIORITY_MAP[priority as keyof typeof PRIORITY_MAP]) .join(","); } @@ -53,7 +53,7 @@ export const isMetaJoinRequired = (groupBy: string, subGroupBy: string) => export const getMetaKeysFragment = (queries: any) => { const { group_by, sub_group_by, ...otherProps } = translateQueryParams(queries); - const fields = new Set(); + const fields: Set = new Set(); if (ARRAY_FIELDS.includes(group_by)) { fields.add(group_by); } @@ -77,10 +77,10 @@ export const getMetaKeysFragment = (queries: any) => { return sql; }; -export const getMetaKeys = (queries: any) => { +export const getMetaKeys = (queries: any): string[] => { const { group_by, sub_group_by, ...otherProps } = translateQueryParams(queries); - const fields = new Set(); + const fields: Set = new Set(); if (ARRAY_FIELDS.includes(group_by)) { fields.add(group_by); } @@ -118,8 +118,8 @@ export const getFilteredRowsForGrouping = (projectId: string, queries: any) => { const filterJoinFields = getMetaKeys(otherProps); - let issueTableFilterFields = getSingleFilterFields(queries); - issueTableFilterFields = issueTableFilterFields.length ? "," + issueTableFilterFields.join(",") : ""; + const temp = getSingleFilterFields(queries); + const issueTableFilterFields = temp.length ? "," + temp.join(",") : ""; const joinsRequired = areJoinsRequired(queries); @@ -193,7 +193,7 @@ export const singleFilterConstructor = (queries: any) => { return sql; }; -const getSingleFilterFields = (queries) => { +const getSingleFilterFields = (queries: any) => { debugger; const { order_by, cursor, per_page, group_by, sub_group_by, sub_issue, ...otherProps } = translateQueryParams(queries); diff --git a/web/core/services/issue/issue.service.ts b/web/core/services/issue/issue.service.ts index 1a835b50c84..d5c11c5ad3b 100644 --- a/web/core/services/issue/issue.service.ts +++ b/web/core/services/issue/issue.service.ts @@ -53,7 +53,8 @@ export class IssueService extends APIService { } async getIssues(workspaceSlug: string, projectId: string, queries?: any, config = {}): Promise { - return await persistence.getIssues(projectId, queries, config); + const response = await persistence.getIssues(projectId, queries, config); + return response as TIssuesResponse; } async getDeletedIssues(workspaceSlug: string, projectId: string, queries?: any): Promise { From 4215094f61c19fe95e26e2b0d291f87f0f550b65 Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Sun, 25 Aug 2024 16:01:34 +0530 Subject: [PATCH 034/111] Remove debuggers --- web/core/local-db/utils/query.utils.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/web/core/local-db/utils/query.utils.ts b/web/core/local-db/utils/query.utils.ts index ab9c9f7141a..63749cdc737 100644 --- a/web/core/local-db/utils/query.utils.ts +++ b/web/core/local-db/utils/query.utils.ts @@ -34,8 +34,6 @@ export const translateQueryParams = (queries: any) => { }; export const getOrderByFragment = (order_by: string) => { - debugger; - let orderByString = ""; if (!order_by) return orderByString; @@ -194,7 +192,6 @@ export const singleFilterConstructor = (queries: any) => { }; const getSingleFilterFields = (queries: any) => { - debugger; const { order_by, cursor, per_page, group_by, sub_group_by, sub_issue, ...otherProps } = translateQueryParams(queries); From 5dc8a82fe8d4284fac33ebb0f8267e33ca20a992 Mon Sep 17 00:00:00 2001 From: rahulramesha Date: Mon, 26 Aug 2024 13:30:07 +0530 Subject: [PATCH 035/111] remove loaders while changing sorting or applying filters --- web/core/store/issue/archived/issue.store.ts | 2 +- web/core/store/issue/cycle/issue.store.ts | 3 +-- web/core/store/issue/draft/issue.store.ts | 2 +- web/core/store/issue/helpers/base-issues.store.ts | 4 +++- web/core/store/issue/module/issue.store.ts | 3 +-- web/core/store/issue/profile/issue.store.ts | 2 +- web/core/store/issue/project-views/issue.store.ts | 3 +-- web/core/store/issue/project/issue.store.ts | 3 +-- web/core/store/issue/workspace/issue.store.ts | 2 +- 9 files changed, 11 insertions(+), 13 deletions(-) diff --git a/web/core/store/issue/archived/issue.store.ts b/web/core/store/issue/archived/issue.store.ts index c3091d5c51c..267c23cc7e4 100644 --- a/web/core/store/issue/archived/issue.store.ts +++ b/web/core/store/issue/archived/issue.store.ts @@ -106,7 +106,7 @@ export class ArchivedIssues extends BaseIssuesStore implements IArchivedIssues { }); // after fetching issues, call the base method to process the response further - this.onfetchIssues(response, options, workspaceSlug, projectId); + this.onfetchIssues(response, options, workspaceSlug, projectId, undefined, !isExistingPaginationOptions); return response; } catch (error) { // set loader to undefined if errored out diff --git a/web/core/store/issue/cycle/issue.store.ts b/web/core/store/issue/cycle/issue.store.ts index 110e39ac915..511eb470847 100644 --- a/web/core/store/issue/cycle/issue.store.ts +++ b/web/core/store/issue/cycle/issue.store.ts @@ -176,7 +176,6 @@ export class CycleIssues extends BaseIssuesStore implements ICycleIssues { runInAction(() => { this.setLoader(loadType); }); - this.clear(!isExistingPaginationOptions); // get params from pagination options const params = this.issueFilterStore?.getFilterParams(options, cycleId, undefined, undefined, undefined); @@ -186,7 +185,7 @@ export class CycleIssues extends BaseIssuesStore implements ICycleIssues { }); // after fetching issues, call the base method to process the response further - this.onfetchIssues(response, options, workspaceSlug, projectId, cycleId); + this.onfetchIssues(response, options, workspaceSlug, projectId, cycleId, !isExistingPaginationOptions); return response; } catch (error) { // set loader to undefined once errored out diff --git a/web/core/store/issue/draft/issue.store.ts b/web/core/store/issue/draft/issue.store.ts index ac1a424d7d6..6dfbeac80a1 100644 --- a/web/core/store/issue/draft/issue.store.ts +++ b/web/core/store/issue/draft/issue.store.ts @@ -103,7 +103,7 @@ export class DraftIssues extends BaseIssuesStore implements IDraftIssues { }); // after fetching issues, call the base method to process the response further - this.onfetchIssues(response, options, workspaceSlug, projectId); + this.onfetchIssues(response, options, workspaceSlug, projectId, undefined, !isExistingPaginationOptions); return response; } catch (error) { // set loader to undefined if errored out diff --git a/web/core/store/issue/helpers/base-issues.store.ts b/web/core/store/issue/helpers/base-issues.store.ts index 5ba71080f4f..26828217f2c 100644 --- a/web/core/store/issue/helpers/base-issues.store.ts +++ b/web/core/store/issue/helpers/base-issues.store.ts @@ -455,7 +455,8 @@ export abstract class BaseIssuesStore implements IBaseIssuesStore { options: IssuePaginationOptions, workspaceSlug: string, projectId?: string, - id?: string + id?: string, + shouldClearPaginationOptions = true ) { // Process the Issue Response to get the following data from it const { issueList, groupedIssues, groupedIssueCount } = this.processIssueResponse(issuesResponse); @@ -465,6 +466,7 @@ export abstract class BaseIssuesStore implements IBaseIssuesStore { // Update all the GroupIds to this Store's groupedIssueIds and update Individual group issue counts runInAction(() => { + this.clear(shouldClearPaginationOptions); this.updateGroupedIssueIds(groupedIssues, groupedIssueCount); this.loader[getGroupKey()] = undefined; }); diff --git a/web/core/store/issue/module/issue.store.ts b/web/core/store/issue/module/issue.store.ts index 76cf6981d40..0c0a191921a 100644 --- a/web/core/store/issue/module/issue.store.ts +++ b/web/core/store/issue/module/issue.store.ts @@ -133,7 +133,6 @@ export class ModuleIssues extends BaseIssuesStore implements IModuleIssues { runInAction(() => { this.setLoader(loadType); }); - this.clear(!isExistingPaginationOptions); // get params from pagination options const params = this.issueFilterStore?.getFilterParams(options, moduleId, undefined, undefined, undefined); @@ -143,7 +142,7 @@ export class ModuleIssues extends BaseIssuesStore implements IModuleIssues { }); // after fetching issues, call the base method to process the response further - this.onfetchIssues(response, options, workspaceSlug, projectId, moduleId); + this.onfetchIssues(response, options, workspaceSlug, projectId, moduleId, !isExistingPaginationOptions); return response; } catch (error) { // set loader to undefined once errored out diff --git a/web/core/store/issue/profile/issue.store.ts b/web/core/store/issue/profile/issue.store.ts index 855dbcd86b3..d8728a93585 100644 --- a/web/core/store/issue/profile/issue.store.ts +++ b/web/core/store/issue/profile/issue.store.ts @@ -140,7 +140,7 @@ export class ProfileIssues extends BaseIssuesStore implements IProfileIssues { }); // after fetching issues, call the base method to process the response further - this.onfetchIssues(response, options, workspaceSlug); + this.onfetchIssues(response, options, workspaceSlug, undefined, undefined, !isExistingPaginationOptions); return response; } catch (error) { // set loader to undefined if errored out diff --git a/web/core/store/issue/project-views/issue.store.ts b/web/core/store/issue/project-views/issue.store.ts index f2e9ba02eb4..5dd69d763ac 100644 --- a/web/core/store/issue/project-views/issue.store.ts +++ b/web/core/store/issue/project-views/issue.store.ts @@ -94,7 +94,6 @@ export class ProjectViewIssues extends BaseIssuesStore implements IProjectViewIs runInAction(() => { this.setLoader(loadType); }); - this.clear(!isExistingPaginationOptions); // get params from pagination options const params = this.issueFilterStore?.getFilterParams(options, viewId, undefined, undefined, undefined); @@ -104,7 +103,7 @@ export class ProjectViewIssues extends BaseIssuesStore implements IProjectViewIs }); // after fetching issues, call the base method to process the response further - this.onfetchIssues(response, options, workspaceSlug, projectId); + this.onfetchIssues(response, options, workspaceSlug, projectId, viewId, !isExistingPaginationOptions); return response; } catch (error) { // set loader to undefined if errored out diff --git a/web/core/store/issue/project/issue.store.ts b/web/core/store/issue/project/issue.store.ts index 66f152ac65d..9d88d0f7094 100644 --- a/web/core/store/issue/project/issue.store.ts +++ b/web/core/store/issue/project/issue.store.ts @@ -102,7 +102,6 @@ export class ProjectIssues extends BaseIssuesStore implements IProjectIssues { runInAction(() => { this.setLoader(loadType); }); - this.clear(!isExistingPaginationOptions); // get params from pagination options const params = this.issueFilterStore?.getFilterParams(options, projectId, undefined, undefined, undefined); @@ -112,7 +111,7 @@ export class ProjectIssues extends BaseIssuesStore implements IProjectIssues { }); // after fetching issues, call the base method to process the response further - this.onfetchIssues(response, options, workspaceSlug, projectId); + this.onfetchIssues(response, options, workspaceSlug, projectId, undefined, !isExistingPaginationOptions); return response; } catch (error) { // set loader to undefined if errored out diff --git a/web/core/store/issue/workspace/issue.store.ts b/web/core/store/issue/workspace/issue.store.ts index 3ac1fac1df5..2d31a94acbe 100644 --- a/web/core/store/issue/workspace/issue.store.ts +++ b/web/core/store/issue/workspace/issue.store.ts @@ -109,7 +109,7 @@ export class WorkspaceIssues extends BaseIssuesStore implements IWorkspaceIssues }); // after fetching issues, call the base method to process the response further - this.onfetchIssues(response, options, workspaceSlug); + this.onfetchIssues(response, options, workspaceSlug, undefined, undefined, !isExistingPaginationOptions); return response; } catch (error) { // set loader to undefined if errored out From 83e7b4f1fc77df06967fc58042acbc547cb56fc1 Mon Sep 17 00:00:00 2001 From: rahulramesha Date: Mon, 26 Aug 2024 16:00:07 +0530 Subject: [PATCH 036/111] fix loader while clearing all filters --- web/core/store/issue/cycle/filter.store.ts | 14 ++++++-------- web/core/store/issue/module/filter.store.ts | 4 +--- web/core/store/issue/profile/filter.store.ts | 8 +------- web/core/store/issue/project-views/filter.store.ts | 4 +--- web/core/store/issue/project/filter.store.ts | 8 +------- 5 files changed, 10 insertions(+), 28 deletions(-) diff --git a/web/core/store/issue/cycle/filter.store.ts b/web/core/store/issue/cycle/filter.store.ts index 2b35ee74632..4f3506ff6f7 100644 --- a/web/core/store/issue/cycle/filter.store.ts +++ b/web/core/store/issue/cycle/filter.store.ts @@ -195,14 +195,12 @@ export class CycleIssuesFilter extends IssueFilterHelperStore implements ICycleI }); }); - const appliedFilters = _filters.filters || {}; - const filteredFilters = pickBy(appliedFilters, (value) => value && isArray(value) && value.length > 0); - this.rootIssueStore.cycleIssues.fetchIssuesWithExistingPagination( - workspaceSlug, - projectId, - isEmpty(filteredFilters) ? "init-loader" : "mutation", - cycleId - ); + this.rootIssueStore.cycleIssues.fetchIssuesWithExistingPagination( + workspaceSlug, + projectId, + "mutation", + cycleId + ); await this.issueFilterService.patchCycleIssueFilters(workspaceSlug, projectId, cycleId, { filters: _filters.filters, }); diff --git a/web/core/store/issue/module/filter.store.ts b/web/core/store/issue/module/filter.store.ts index d64a4c0f015..4cbecb6977e 100644 --- a/web/core/store/issue/module/filter.store.ts +++ b/web/core/store/issue/module/filter.store.ts @@ -194,12 +194,10 @@ export class ModuleIssuesFilter extends IssueFilterHelperStore implements IModul set(this.filters, [moduleId, "filters", _key], updatedFilters[_key as keyof IIssueFilterOptions]); }); }); - const appliedFilters = _filters.filters || {}; - const filteredFilters = pickBy(appliedFilters, (value) => value && isArray(value) && value.length > 0); this.rootIssueStore.moduleIssues.fetchIssuesWithExistingPagination( workspaceSlug, projectId, - isEmpty(filteredFilters) ? "init-loader" : "mutation", + "mutation", moduleId ); await this.issueFilterService.patchModuleIssueFilters(workspaceSlug, projectId, moduleId, { diff --git a/web/core/store/issue/profile/filter.store.ts b/web/core/store/issue/profile/filter.store.ts index 8e27a108167..99cd3544d23 100644 --- a/web/core/store/issue/profile/filter.store.ts +++ b/web/core/store/issue/profile/filter.store.ts @@ -180,13 +180,7 @@ export class ProfileIssuesFilter extends IssueFilterHelperStore implements IProf }); }); - const appliedFilters = _filters.filters || {}; - const filteredFilters = pickBy(appliedFilters, (value) => value && isArray(value) && value.length > 0); - this.rootIssueStore.profileIssues.fetchIssuesWithExistingPagination( - workspaceSlug, - userId, - isEmpty(filteredFilters) ? "init-loader" : "mutation" - ); + this.rootIssueStore.profileIssues.fetchIssuesWithExistingPagination(workspaceSlug, userId, "mutation"); this.handleIssuesLocalFilters.set(EIssuesStoreType.PROFILE, type, workspaceSlug, userId, undefined, { filters: _filters.filters, diff --git a/web/core/store/issue/project-views/filter.store.ts b/web/core/store/issue/project-views/filter.store.ts index 511ce850824..6b50a90bb7a 100644 --- a/web/core/store/issue/project-views/filter.store.ts +++ b/web/core/store/issue/project-views/filter.store.ts @@ -188,13 +188,11 @@ export class ProjectViewIssuesFilter extends IssueFilterHelperStore implements I }); }); - const appliedFilters = _filters.filters || {}; - const filteredFilters = pickBy(appliedFilters, (value) => value && isArray(value) && value.length > 0); this.rootIssueStore.projectViewIssues.fetchIssuesWithExistingPagination( workspaceSlug, projectId, viewId, - isEmpty(filteredFilters) ? "init-loader" : "mutation" + "mutation" ); break; } diff --git a/web/core/store/issue/project/filter.store.ts b/web/core/store/issue/project/filter.store.ts index a8e6110164b..20041d1a9d8 100644 --- a/web/core/store/issue/project/filter.store.ts +++ b/web/core/store/issue/project/filter.store.ts @@ -183,13 +183,7 @@ export class ProjectIssuesFilter extends IssueFilterHelperStore implements IProj }); }); - const appliedFilters = _filters.filters || {}; - const filteredFilters = pickBy(appliedFilters, (value) => value && isArray(value) && value.length > 0); - this.rootIssueStore.projectIssues.fetchIssuesWithExistingPagination( - workspaceSlug, - projectId, - isEmpty(filteredFilters) ? "init-loader" : "mutation" - ); + this.rootIssueStore.projectIssues.fetchIssuesWithExistingPagination(workspaceSlug, projectId, "mutation"); await this.issueFilterService.patchProjectIssueFilters(workspaceSlug, projectId, { filters: _filters.filters, }); From d3b9e101dfdf503dde2cb3ae27dad040c55e105b Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Mon, 26 Aug 2024 16:39:02 +0530 Subject: [PATCH 037/111] Fix issue with project being synced twice --- web/core/local-db/storage.sqlite.ts | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/web/core/local-db/storage.sqlite.ts b/web/core/local-db/storage.sqlite.ts index 71494ded617..5dc69ea9894 100644 --- a/web/core/local-db/storage.sqlite.ts +++ b/web/core/local-db/storage.sqlite.ts @@ -57,7 +57,7 @@ export class Storage { _initialize = async (workspaceSlug: string): Promise => { if (this.status === "initializing") { console.warn(`Initialization already in progress for workspace ${workspaceSlug}`); - return true; + return false; } if (this.status === "ready") { console.warn(`Already initialized for workspace ${workspaceSlug}`); @@ -116,7 +116,7 @@ export class Storage { loadLabels(this.workspaceSlug); }; - syncProject = async (projectId: string) => { + syncProject = (projectId: string) => { // Load labels, members, states, modules, cycles const sync = this.syncIssues(projectId); @@ -129,8 +129,12 @@ export class Storage { info(`Project ${projectId} is already loading or syncing`); return; } + const syncPromise = this.getSync(projectId); - await this.getSync(projectId); + if (syncPromise) { + // Redundant check? + return; + } const queryParams: { cursor: string; updated_at__gt?: string } = { cursor: `${PAGE_SIZE}:0:0`, @@ -168,6 +172,7 @@ export class Storage { console.log("### Time taken to add issues", performance.now() - start); this.setStatus(projectId, "ready"); + this.setSync(projectId, undefined); }; getIssueCount = async (projectId: string) => { @@ -275,8 +280,8 @@ export class Storage { }; getSync = (projectId: string) => this.projectStatus[projectId]?.issues?.sync; - setSync = (projectId: string, sync: Promise) => { - this.projectStatus[projectId].issues.sync = sync; + setSync = (projectId: string, sync: Promise | undefined) => { + set(this.projectStatus, `${projectId}.issues.sync`, sync); }; } From 40704c1052b51fae1c955e1f19a0df98e387e802 Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Mon, 26 Aug 2024 18:44:05 +0530 Subject: [PATCH 038/111] Improve project sync --- .../layouts/auth-layout/project-wrapper.tsx | 8 ----- web/core/local-db/storage.sqlite.ts | 30 +++++++++++++++---- web/core/local-db/utils/query-constructor.ts | 6 ++++ web/core/local-db/utils/tables.ts | 10 +++---- 4 files changed, 35 insertions(+), 19 deletions(-) diff --git a/web/core/layouts/auth-layout/project-wrapper.tsx b/web/core/layouts/auth-layout/project-wrapper.tsx index 9c6e678129d..4ae79a9af7f 100644 --- a/web/core/layouts/auth-layout/project-wrapper.tsx +++ b/web/core/layouts/auth-layout/project-wrapper.tsx @@ -53,14 +53,6 @@ export const ProjectAuthWrapper: FC = observer((props) => { // router const { workspaceSlug, projectId } = useParams(); - useSWRImmutable( - workspaceSlug && projectId ? `PROJECT_SYNC_${workspaceSlug.toString()}_${projectId.toString()}` : null, - workspaceSlug && projectId - ? async () => { - await persistence.syncProject(projectId.toString()); - } - : null - ); useSWR( workspaceSlug && projectId ? `PROJECT_SYNC_ISSUES_${workspaceSlug.toString()}_${projectId.toString()}` : null, workspaceSlug && projectId diff --git a/web/core/local-db/storage.sqlite.ts b/web/core/local-db/storage.sqlite.ts index 5dc69ea9894..40a603ab721 100644 --- a/web/core/local-db/storage.sqlite.ts +++ b/web/core/local-db/storage.sqlite.ts @@ -1,10 +1,12 @@ -import set from "lodash/set"; -import { EIssueGroupBYServerToProperty } from "@plane/constants"; import { IssueService } from "@/services/issue/issue.service"; +import { EIssueGroupBYServerToProperty } from "@plane/constants"; +import { setToast, TOAST_TYPE } from "@plane/ui"; +import set from "lodash/set"; import { ARRAY_FIELDS } from "./utils/constants"; +import createIndexes from "./utils/indexes"; import { addIssuesBulk } from "./utils/load-issues"; import { loadLabels } from "./utils/load-labels"; -import { issueFilterQueryConstructor, issueFilterCountQueryConstructor } from "./utils/query-constructor"; +import { issueFilterCountQueryConstructor, issueFilterQueryConstructor } from "./utils/query-constructor"; import { runQuery } from "./utils/query-executor"; import { createTables } from "./utils/tables"; import { getGroupedIssueResults, getSubGroupedIssueResults } from "./utils/utils"; @@ -32,6 +34,7 @@ export class Storage { constructor() { this.db = null; + // this.issueService = new IssueService(); } @@ -119,13 +122,18 @@ export class Storage { syncProject = (projectId: string) => { // Load labels, members, states, modules, cycles - const sync = this.syncIssues(projectId); + this.syncIssues(projectId); + }; + + syncIssues = (projectId: string) => { + const sync = this._syncIssues(projectId); this.setSync(projectId, sync); }; - syncIssues = async (projectId: string) => { + _syncIssues = async (projectId: string) => { console.log("### Sync started"); - if (this.getStatus(projectId) === "loading" || this.getStatus(projectId) === "syncing") { + let status = this.getStatus(projectId); + if (status === "loading" || status === "syncing") { info(`Project ${projectId} is already loading or syncing`); return; } @@ -147,6 +155,7 @@ export class Storage { } this.setStatus(projectId, syncedAt ? "syncing" : "loading"); + status = this.getStatus(projectId); log(`### ${syncedAt ? "Syncing" : "Loading"} issues to local db for project ${projectId}`); @@ -171,6 +180,14 @@ export class Storage { console.log("### Time taken to add issues", performance.now() - start); + if (status === "loading") { + await createIndexes(); + setToast({ + title: "Project synced", + message: `in ${Math.round((performance.now() - start) / 1000)}s`, + type: TOAST_TYPE.SUCCESS, + }); + } this.setStatus(projectId, "ready"); this.setSync(projectId, undefined); }; @@ -201,6 +218,7 @@ export class Storage { getIssues = async (projectId: string, queries: any, config: any) => { console.log("#### Queries", queries); + if (this.getStatus(projectId) === "loading" || (window as any).DISABLE_LOCAL) { info(`Project ${projectId} is loading, falling back to server`); const issueService = new IssueService(); diff --git a/web/core/local-db/utils/query-constructor.ts b/web/core/local-db/utils/query-constructor.ts index b15f0519e03..76d907c7b19 100644 --- a/web/core/local-db/utils/query-constructor.ts +++ b/web/core/local-db/utils/query-constructor.ts @@ -110,6 +110,11 @@ export const stageIssueInserts = (issue: any) => { sql: `INSERT OR REPLACE into issues(${keys}) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, bind: values, }); + + persistence.db.exec({ + sql: `DELETE from issue_meta where issue_id='${issue_id}'`, + }); + arrayFields.forEach((field) => { const values = issue[field]; if (values && values.length) { @@ -120,6 +125,7 @@ export const stageIssueInserts = (issue: any) => { }); }); } else { + // Added for empty fields? persistence.db.exec({ sql: `INSERT OR REPLACE into issue_meta(issue_id,key,value) values (?,?,?) `, bind: [issue_id, field, ""], diff --git a/web/core/local-db/utils/tables.ts b/web/core/local-db/utils/tables.ts index 26de77042fc..a1d4fa14110 100644 --- a/web/core/local-db/utils/tables.ts +++ b/web/core/local-db/utils/tables.ts @@ -59,9 +59,9 @@ export const createTables = async (SQLITE: any) => { await createIssueMetaTable(SQLITE); await createLabelsTable(SQLITE); - try { - await createIndexes(); - } catch (e) { - console.error(e); - } + // try { + // await createIndexes(); + // } catch (e) { + // console.error(e); + // } }; From 7f3ec53bbd13fcfecfdff9520c3aa49ef95c3845 Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Tue, 27 Aug 2024 15:13:59 +0530 Subject: [PATCH 039/111] Optimize the queries --- web/core/local-db/utils/query-constructor.ts | 28 ++++++++++++-------- web/core/local-db/utils/query.utils.ts | 10 ++++--- web/core/local-db/utils/tables.ts | 1 + 3 files changed, 24 insertions(+), 15 deletions(-) diff --git a/web/core/local-db/utils/query-constructor.ts b/web/core/local-db/utils/query-constructor.ts index 76d907c7b19..735a73a887c 100644 --- a/web/core/local-db/utils/query-constructor.ts +++ b/web/core/local-db/utils/query-constructor.ts @@ -25,9 +25,12 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st if (sub_group_by) { sql = getFilteredRowsForGrouping(projectId, queries); - sql += `, rn AS ( SELECT fi.*, - RANK() OVER (PARTITION BY group_id, sub_group_id ${orderByString}) as rank, - COUNT(*) OVER (PARTITION by group_id, sub_group_id) as total_issues from fi) SELECT * FROM rn + sql += `, ranked_issues AS ( SELECT fi.*, + ROW_NUMBER() OVER (PARTITION BY group_id, sub_group_id ${orderByString}) as rank, + COUNT(*) OVER (PARTITION by group_id, sub_group_id) as total_issues from fi) + SELECT ri.*, i.* + FROM ranked_issues ri + JOIN issues i ON ri.id = i.id WHERE rank <= ${per_page} `; @@ -38,10 +41,13 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st } if (group_by) { sql = getFilteredRowsForGrouping(projectId, queries); - sql += `, rn AS ( SELECT fi.*, - RANK() OVER (PARTITION BY group_id ${orderByString}) as rank, - COUNT(*) OVER (PARTITION by group_id) as total_issues from fi) SELECT * FROM rn - WHERE rank <= ${per_page} + sql += `, ranked_issues AS ( SELECT fi.*, + ROW_NUMBER() OVER (PARTITION BY group_id ${orderByString}) as rank, + COUNT(*) OVER (PARTITION by group_id) as total_issues FROM fi) + SELECT ri.*, i.* + FROM ranked_issues ri + JOIN issues i ON ri.id = i.id + WHERE rank <= ${per_page} `; console.log("###", sql); @@ -54,7 +60,7 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st const name = order_by.replace("-", ""); sql = `SELECT i.*, s.name as ${name} from issues i`; } else { - sql = `SELECT * from issues i`; + sql = `SELECT i.* from issues i`; } filterJoinFields.forEach((field: string) => { const value = otherProps[field] || ""; @@ -67,7 +73,7 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st LEFT JOIN issue_meta label_ids ON i.id = label_ids.issue_id AND label_ids.key = 'label_ids' INNER JOIN labels s ON s.id = label_ids.value`; } - sql += ` WHERE i.project_id = '${projectId}' ${singleFilterConstructor(otherProps)} group by i.id`; + sql += ` WHERE i.project_id = '${projectId}' ${singleFilterConstructor(otherProps)} group by i.id `; sql += orderByString; // Add offset and paging to query @@ -82,7 +88,7 @@ export const issueFilterCountQueryConstructor = (workspaceSlug: string, projectI const { group_by, sub_group_by, order_by, ...otherProps } = queries; let sql = issueFilterQueryConstructor(workspaceSlug, projectId, otherProps); - sql = sql.replace("SELECT *", "SELECT COUNT(DISTINCT i.id) as total_count"); + sql = sql.replace("SELECT i.*", "SELECT COUNT(DISTINCT i.id) as total_count"); // Remove everything after group by i.id sql = `${sql.split("group by i.id")[0]};`; console.log("### COUNT", sql); @@ -107,7 +113,7 @@ export const stageIssueInserts = (issue: any) => { }); // Will fail when the values have a comma persistence.db.exec({ - sql: `INSERT OR REPLACE into issues(${keys}) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, + sql: `INSERT OR REPLACE into issues(${keys}) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, bind: values, }); diff --git a/web/core/local-db/utils/query.utils.ts b/web/core/local-db/utils/query.utils.ts index 63749cdc737..28d02c92b73 100644 --- a/web/core/local-db/utils/query.utils.ts +++ b/web/core/local-db/utils/query.utils.ts @@ -121,6 +121,7 @@ export const getFilteredRowsForGrouping = (projectId: string, queries: any) => { const joinsRequired = areJoinsRequired(queries); + const orderByFragment = getOrderByFragment(queries.order_by); let sql = ""; if (!joinsRequired) { sql = `WITH fi as (SELECT *`; @@ -134,7 +135,7 @@ export const getFilteredRowsForGrouping = (projectId: string, queries: any) => { return sql; } - sql = `WITH fi AS (SELECT * FROM (`; + sql = `WITH fi AS (`; sql += `SELECT i.id,i.created_at ${issueTableFilterFields} `; if (group_by) { if (ARRAY_FIELDS.includes(group_by)) { @@ -168,9 +169,10 @@ export const getFilteredRowsForGrouping = (projectId: string, queries: any) => { `; } - sql += ` WHERE i.project_id = '${projectId}' ${singleFilterConstructor(otherProps)} group by i.id`; + sql += ` WHERE i.project_id = '${projectId}' ${singleFilterConstructor(otherProps)} + `; - sql += `) AS slim LEFT JOIN issues ON slim.id = issues.id) + sql += `) `; return sql; }; @@ -196,7 +198,7 @@ const getSingleFilterFields = (queries: any) => { translateQueryParams(queries); const fields = new Set(); - if (order_by) fields.add(order_by); + if (order_by && !order_by.includes("created_at")) fields.add(order_by.replace("-", "")); const keys = Object.keys(otherProps); diff --git a/web/core/local-db/utils/tables.ts b/web/core/local-db/utils/tables.ts index a1d4fa14110..a3d8b1c1dc2 100644 --- a/web/core/local-db/utils/tables.ts +++ b/web/core/local-db/utils/tables.ts @@ -26,6 +26,7 @@ export const createIssuesTable = (SQLITE: any) => { cycle_id TEXT, link_count INTEGER, attachment_count INTEGER, + type_id TEXT, label_ids TEXT, assignee_ids TEXT, module_ids TEXT);`; From 317ae96d561b532d4136bc53d16683775ea750e7 Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Tue, 27 Aug 2024 19:16:11 +0530 Subject: [PATCH 040/111] Make create dummy data more realistic --- apiserver/plane/bgtasks/dummy_data_task.py | 27 ++++++++++++------- .../management/commands/create_dummy_data.py | 2 +- web/core/local-db/utils/load-issues.ts | 3 +++ 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/apiserver/plane/bgtasks/dummy_data_task.py b/apiserver/plane/bgtasks/dummy_data_task.py index 74e210de64a..0d973b6dc8a 100644 --- a/apiserver/plane/bgtasks/dummy_data_task.py +++ b/apiserver/plane/bgtasks/dummy_data_task.py @@ -347,7 +347,7 @@ def create_issues(workspace, project, user_id, issue_count): ) ) - text = fake.text(max_nb_chars=60000) + text = fake.text(max_nb_chars=3000) issues.append( Issue( state_id=states[random.randint(0, len(states) - 1)], @@ -496,12 +496,14 @@ def create_issue_labels(workspace, project, user_id, issue_count): ), int(issue_count / 2), ) + shuffled_labels = list(labels) # Bulk issue bulk_issue_labels = [] for issue in issues: + random.shuffle(shuffled_labels) for label in random.sample( - list(labels), random.randint(0, len(labels) - 1) + shuffled_labels, random.randint(0, 3) ): bulk_issue_labels.append( IssueLabel( @@ -559,18 +561,23 @@ def create_module_issues(workspace, project, user_id, issue_count): int(issue_count / 2), ) + shuffled_modules = list(modules) + # Bulk issue bulk_module_issues = [] for issue in issues: - module = modules[random.randint(0, len(modules) - 1)] - bulk_module_issues.append( - ModuleIssue( - module_id=module, - issue_id=issue, - project=project, - workspace=workspace, + random.shuffle(shuffled_modules) + for module in random.sample( + shuffled_modules, random.randint(0, 3) + ): + bulk_module_issues.append( + ModuleIssue( + module_id=module, + issue_id=issue, + project=project, + workspace=workspace, + ) ) - ) # Issue assignees ModuleIssue.objects.bulk_create( bulk_module_issues, batch_size=1000, ignore_conflicts=True diff --git a/apiserver/plane/db/management/commands/create_dummy_data.py b/apiserver/plane/db/management/commands/create_dummy_data.py index dde1411fe99..f71d90f0eee 100644 --- a/apiserver/plane/db/management/commands/create_dummy_data.py +++ b/apiserver/plane/db/management/commands/create_dummy_data.py @@ -73,7 +73,7 @@ def handle(self, *args: Any, **options: Any) -> str | None: from plane.bgtasks.dummy_data_task import create_dummy_data - create_dummy_data.delay( + create_dummy_data( slug=workspace_slug, email=creator, members=members, diff --git a/web/core/local-db/utils/load-issues.ts b/web/core/local-db/utils/load-issues.ts index 9263878e9b4..409092a73b5 100644 --- a/web/core/local-db/utils/load-issues.ts +++ b/web/core/local-db/utils/load-issues.ts @@ -18,6 +18,9 @@ export const addIssuesBulk = async (issues: any, batchSize = 100) => { persistence.db.exec("BEGIN TRANSACTION;"); batch.forEach((issue: any) => { + if (!issue.type_id) { + issue.type_id = ""; + } stageIssueInserts(issue); }); await persistence.db.exec("COMMIT;"); From fae431419008390bf5f885d03132e74b6115c7f8 Mon Sep 17 00:00:00 2001 From: gurusainath Date: Tue, 27 Aug 2024 20:36:13 +0530 Subject: [PATCH 041/111] dev: added total pages in the global paginator --- apiserver/plane/app/views/issue/base.py | 3 ++- apiserver/plane/utils/global_paginator.py | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/apiserver/plane/app/views/issue/base.py b/apiserver/plane/app/views/issue/base.py index 4f2d1d0855e..68435e6bbdc 100644 --- a/apiserver/plane/app/views/issue/base.py +++ b/apiserver/plane/app/views/issue/base.py @@ -660,7 +660,6 @@ def delete(self, request, slug, project_id): class DeletedIssuesListViewSet(BaseAPIView): - @allow_permission([ROLE.ADMIN, ROLE.MEMBER, ROLE.GUEST, ROLE.VIEWER]) def get(self, request, slug, project_id): filters = {} @@ -677,6 +676,8 @@ def get(self, request, slug, project_id): ) return Response(deleted_issues, status=status.HTTP_200_OK) + + class IssuePaginatedViewSet(BaseViewSet): def get_queryset(self): workspace_slug = self.kwargs.get("slug") diff --git a/apiserver/plane/utils/global_paginator.py b/apiserver/plane/utils/global_paginator.py index db918e08552..66912ede7db 100644 --- a/apiserver/plane/utils/global_paginator.py +++ b/apiserver/plane/utils/global_paginator.py @@ -1,3 +1,6 @@ +# python imports +from math import ceil + # constants PAGINATOR_MAX_LIMIT = 1000 @@ -36,6 +39,9 @@ def paginate(base_queryset, queryset, cursor, on_result): total_results = base_queryset.count() page_size = min(cursor_object.current_page_size, PAGINATOR_MAX_LIMIT) + # getting the total pages available based on the page size + total_pages = total_results / page_size + # Calculate the start and end index for the paginated data start_index = 0 if cursor_object.current_page > 0: @@ -72,6 +78,7 @@ def paginate(base_queryset, queryset, cursor, on_result): "next_page_results": next_page_results, "page_count": len(paginated_data), "total_results": total_results, + "total_pages": ceil(total_pages), "results": paginated_data, } From 921c142af9f51f434a8f4e6fa8542980b7e47d5e Mon Sep 17 00:00:00 2001 From: gurusainath Date: Wed, 28 Aug 2024 02:06:06 +0530 Subject: [PATCH 042/111] chore: updated total_paged count --- apiserver/plane/utils/global_paginator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apiserver/plane/utils/global_paginator.py b/apiserver/plane/utils/global_paginator.py index 66912ede7db..e9ed735d533 100644 --- a/apiserver/plane/utils/global_paginator.py +++ b/apiserver/plane/utils/global_paginator.py @@ -40,7 +40,7 @@ def paginate(base_queryset, queryset, cursor, on_result): page_size = min(cursor_object.current_page_size, PAGINATOR_MAX_LIMIT) # getting the total pages available based on the page size - total_pages = total_results / page_size + total_pages = ceil(total_results / page_size) # Calculate the start and end index for the paginated data start_index = 0 @@ -78,7 +78,7 @@ def paginate(base_queryset, queryset, cursor, on_result): "next_page_results": next_page_results, "page_count": len(paginated_data), "total_results": total_results, - "total_pages": ceil(total_pages), + "total_pages": total_pages, "results": paginated_data, } From 06b1ecfe37866b0c220b5fa9c3e66cd49226e352 Mon Sep 17 00:00:00 2001 From: gurusainath Date: Wed, 28 Aug 2024 02:17:34 +0530 Subject: [PATCH 043/111] chore: added state_group in the issues pagination --- apiserver/plane/app/views/issue/base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apiserver/plane/app/views/issue/base.py b/apiserver/plane/app/views/issue/base.py index 68435e6bbdc..5b39b5edd87 100644 --- a/apiserver/plane/app/views/issue/base.py +++ b/apiserver/plane/app/views/issue/base.py @@ -759,6 +759,7 @@ def list(self, request, slug, project_id): "link_count", "attachment_count", "sub_issues_count", + "state_group", ] if is_description_required: @@ -802,6 +803,8 @@ def list(self, request, slug, project_id): ), Value([], output_field=ArrayField(UUIDField())), ), + ).annotate( + state_group=F("state__group"), ) paginated_data = paginate( From 8bc1924f37cf9779a3a62247ff3c2971288ba0cb Mon Sep 17 00:00:00 2001 From: gurusainath Date: Wed, 28 Aug 2024 02:19:10 +0530 Subject: [PATCH 044/111] chore: removed deleted_at from the issue pagination payload --- apiserver/plane/app/views/issue/base.py | 1 - yarn.lock | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/apiserver/plane/app/views/issue/base.py b/apiserver/plane/app/views/issue/base.py index 5b39b5edd87..41dca8a3245 100644 --- a/apiserver/plane/app/views/issue/base.py +++ b/apiserver/plane/app/views/issue/base.py @@ -752,7 +752,6 @@ def list(self, request, slug, project_id): "updated_by", "is_draft", "archived_at", - "deleted_at", "module_ids", "label_ids", "assignee_ids", diff --git a/yarn.lock b/yarn.lock index 8bef87f01bd..add5512f4dc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4382,7 +4382,7 @@ dependencies: "@types/react" "*" -"@types/react@*", "@types/react@^16.8.0 || ^17.0.0 || ^18.0.0", "@types/react@^18.2.42", "@types/react@^18.2.48": +"@types/react@*", "@types/react@18.2.48", "@types/react@^16.8.0 || ^17.0.0 || ^18.0.0", "@types/react@^18.2.42", "@types/react@^18.2.48": version "18.2.48" resolved "https://registry.yarnpkg.com/@types/react/-/react-18.2.48.tgz#11df5664642d0bd879c1f58bc1d37205b064e8f1" integrity sha512-qboRCl6Ie70DQQG9hhNREz81jqC1cs9EVNcjQ1AU+jH6NFfSAhVVbrrY/+nSF+Bsk4AOwm9Qa61InvMCyV+H3w== From 5600257fe42c5f13cd6234fa650dd236b79ff8d7 Mon Sep 17 00:00:00 2001 From: gurusainath Date: Wed, 28 Aug 2024 12:12:30 +0530 Subject: [PATCH 045/111] chore: replaced state_group with state__group --- apiserver/plane/app/views/issue/base.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/apiserver/plane/app/views/issue/base.py b/apiserver/plane/app/views/issue/base.py index 41dca8a3245..474e99bdab4 100644 --- a/apiserver/plane/app/views/issue/base.py +++ b/apiserver/plane/app/views/issue/base.py @@ -736,6 +736,7 @@ def list(self, request, slug, project_id): "id", "name", "state_id", + "state__group", "sort_order", "completed_at", "estimate_point", @@ -758,7 +759,6 @@ def list(self, request, slug, project_id): "link_count", "attachment_count", "sub_issues_count", - "state_group", ] if is_description_required: @@ -802,8 +802,6 @@ def list(self, request, slug, project_id): ), Value([], output_field=ArrayField(UUIDField())), ), - ).annotate( - state_group=F("state__group"), ) paginated_data = paginate( From 83d2384e9b4beadea389591bdc62bff0619c0e0c Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Wed, 28 Aug 2024 16:00:30 +0530 Subject: [PATCH 046/111] Integrate new getIssues API, and fix sync issues bug. --- web/core/local-db/storage.sqlite.ts | 11 +++++------ web/core/local-db/utils/load-issues.ts | 11 ----------- web/core/services/issue/issue.service.ts | 19 +++++++++++++++++++ 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/web/core/local-db/storage.sqlite.ts b/web/core/local-db/storage.sqlite.ts index 40a603ab721..750e2a209b8 100644 --- a/web/core/local-db/storage.sqlite.ts +++ b/web/core/local-db/storage.sqlite.ts @@ -144,14 +144,14 @@ export class Storage { return; } - const queryParams: { cursor: string; updated_at__gt?: string } = { + const queryParams: { cursor: string; updated_at__gte?: string } = { cursor: `${PAGE_SIZE}:0:0`, }; const syncedAt = await this.getLastSyncTime(projectId); if (syncedAt) { - queryParams["updated_at__gt"] = syncedAt; + queryParams["updated_at__gte"] = syncedAt; } this.setStatus(projectId, syncedAt ? "syncing" : "loading"); @@ -162,15 +162,14 @@ export class Storage { const start = performance.now(); const issueService = new IssueService(); - const response = await issueService.getIssuesFromServer(this.workspaceSlug, projectId, queryParams); + const response = await issueService.getIssuesForSync(this.workspaceSlug, projectId, queryParams); addIssuesBulk(response.results, 500); if (response.total_pages > 1) { const promiseArray = []; for (let i = 1; i < response.total_pages; i++) { - promiseArray.push( - issueService.getIssuesFromServer(this.workspaceSlug, projectId, { cursor: `${PAGE_SIZE}:${i}:0` }) - ); + queryParams.cursor = `${PAGE_SIZE}:${i}:0`; + promiseArray.push(issueService.getIssuesForSync(this.workspaceSlug, projectId, queryParams)); } const pages = await Promise.all(promiseArray); for (const page of pages) { diff --git a/web/core/local-db/utils/load-issues.ts b/web/core/local-db/utils/load-issues.ts index 409092a73b5..93bb5f79aa2 100644 --- a/web/core/local-db/utils/load-issues.ts +++ b/web/core/local-db/utils/load-issues.ts @@ -2,8 +2,6 @@ import { IssueService } from "@/services/issue"; import { persistence } from "../storage.sqlite"; import { stageIssueInserts } from "./query-constructor"; -const PAGE_SIZE = 1000; - export const PROJECT_OFFLINE_STATUS: Record = {}; export const addIssue = async (issue: any) => { @@ -50,12 +48,3 @@ export const syncDeletesToLocal = async (workspaceId: string, projectId: string) response.map(async (issue) => deleteIssueFromLocal(issue)); } }; - -// export const syncLocalData = async (workspaceId: string, projectId: string) => { -// persistence.syncInProgress = Promise.all([ -// syncDeletesToLocal(workspaceId, projectId), -// syncUpdatesToLocal(workspaceId, projectId), -// ]); - -// await persistence.syncInProgress; -// }; diff --git a/web/core/services/issue/issue.service.ts b/web/core/services/issue/issue.service.ts index d5c11c5ad3b..151e6e20814 100644 --- a/web/core/services/issue/issue.service.ts +++ b/web/core/services/issue/issue.service.ts @@ -52,6 +52,25 @@ export class IssueService extends APIService { }); } + async getIssuesForSync( + workspaceSlug: string, + projectId: string, + queries?: any, + config = {} + ): Promise { + return this.get( + `/api/workspaces/${workspaceSlug}/projects/${projectId}/v2/issues/`, + { + params: queries, + }, + config + ) + .then((response) => response?.data) + .catch((error) => { + throw error?.response?.data; + }); + } + async getIssues(workspaceSlug: string, projectId: string, queries?: any, config = {}): Promise { const response = await persistence.getIssues(projectId, queries, config); return response as TIssuesResponse; From 9dc26b636879b9d4567ac1a205dc365bb572f055 Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Wed, 28 Aug 2024 16:19:01 +0530 Subject: [PATCH 047/111] Fix issue with SWR running twice in workspace wrapper --- web/core/layouts/auth-layout/workspace-wrapper.tsx | 3 ++- web/core/local-db/storage.sqlite.ts | 11 +++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/web/core/layouts/auth-layout/workspace-wrapper.tsx b/web/core/layouts/auth-layout/workspace-wrapper.tsx index 9f1d0a68582..d4b66b334ff 100644 --- a/web/core/layouts/auth-layout/workspace-wrapper.tsx +++ b/web/core/layouts/auth-layout/workspace-wrapper.tsx @@ -89,12 +89,13 @@ export const WorkspaceAuthWrapper: FC = observer((props) workspaceSlug ? `WORKSPACE_DB_${workspaceSlug}` : null, workspaceSlug ? async () => { - persistence.reset(); + // persistence.reset(); await persistence.initialize(workspaceSlug.toString()); } : null ); + console.log("#### Workspace Wrapper isLoading", isDBInitializing); // Load common data useSWRImmutable( workspaceSlug ? `WORKSPACE_SYNC_${workspaceSlug}` : null, diff --git a/web/core/local-db/storage.sqlite.ts b/web/core/local-db/storage.sqlite.ts index 750e2a209b8..64cf4eeed25 100644 --- a/web/core/local-db/storage.sqlite.ts +++ b/web/core/local-db/storage.sqlite.ts @@ -1,7 +1,7 @@ -import { IssueService } from "@/services/issue/issue.service"; +import set from "lodash/set"; import { EIssueGroupBYServerToProperty } from "@plane/constants"; import { setToast, TOAST_TYPE } from "@plane/ui"; -import set from "lodash/set"; +import { IssueService } from "@/services/issue/issue.service"; import { ARRAY_FIELDS } from "./utils/constants"; import createIndexes from "./utils/indexes"; import { addIssuesBulk } from "./utils/load-issues"; @@ -43,9 +43,16 @@ export class Storage { this.status = undefined; this.projectStatus = {}; this.workspaceSlug = ""; + this.workspaceInitPromise = undefined; }; initialize = async (workspaceSlug: string): Promise => { + if (workspaceSlug !== this.workspaceSlug) { + this.reset(); + } + if (this.workspaceInitPromise) { + return this.workspaceInitPromise; + } this.workspaceInitPromise = this._initialize(workspaceSlug); try { await this.workspaceInitPromise; From 0305e7dd254bb488a02c2db0ae97cad3ce293d0a Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Wed, 28 Aug 2024 18:21:00 +0530 Subject: [PATCH 048/111] Fix DB initialization called when opening project for the first time. --- web/core/layouts/auth-layout/workspace-wrapper.tsx | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/web/core/layouts/auth-layout/workspace-wrapper.tsx b/web/core/layouts/auth-layout/workspace-wrapper.tsx index d4b66b334ff..bd33759245d 100644 --- a/web/core/layouts/auth-layout/workspace-wrapper.tsx +++ b/web/core/layouts/auth-layout/workspace-wrapper.tsx @@ -91,17 +91,9 @@ export const WorkspaceAuthWrapper: FC = observer((props) ? async () => { // persistence.reset(); await persistence.initialize(workspaceSlug.toString()); - } - : null - ); - - console.log("#### Workspace Wrapper isLoading", isDBInitializing); - // Load common data - useSWRImmutable( - workspaceSlug ? `WORKSPACE_SYNC_${workspaceSlug}` : null, - workspaceSlug - ? async () => { - await persistence.syncWorkspace(); + // Load common data + persistence.syncWorkspace(); + return true; } : null ); From ef7ca4e4449057045771c2ad1e9e22e85522ccfd Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Thu, 29 Aug 2024 15:26:35 +0530 Subject: [PATCH 049/111] Add all the tables required for sorting --- web/core/local-db/storage.sqlite.ts | 7 +- web/core/local-db/utils/load-labels.ts | 22 --- web/core/local-db/utils/load-workspace.ts | 142 +++++++++++++++++++ web/core/local-db/utils/query-constructor.ts | 106 ++++++-------- web/core/local-db/utils/query.utils.ts | 11 +- web/core/local-db/utils/schemas.ts | 130 +++++++++++++++++ web/core/local-db/utils/tables.ts | 93 ++++-------- 7 files changed, 360 insertions(+), 151 deletions(-) delete mode 100644 web/core/local-db/utils/load-labels.ts create mode 100644 web/core/local-db/utils/load-workspace.ts create mode 100644 web/core/local-db/utils/schemas.ts diff --git a/web/core/local-db/storage.sqlite.ts b/web/core/local-db/storage.sqlite.ts index 64cf4eeed25..a4b6c6e8b58 100644 --- a/web/core/local-db/storage.sqlite.ts +++ b/web/core/local-db/storage.sqlite.ts @@ -5,7 +5,7 @@ import { IssueService } from "@/services/issue/issue.service"; import { ARRAY_FIELDS } from "./utils/constants"; import createIndexes from "./utils/indexes"; import { addIssuesBulk } from "./utils/load-issues"; -import { loadLabels } from "./utils/load-labels"; +import { loadWorkSpaceData } from "./utils/load-workspace"; import { issueFilterCountQueryConstructor, issueFilterQueryConstructor } from "./utils/query-constructor"; import { runQuery } from "./utils/query-executor"; import { createTables } from "./utils/tables"; @@ -123,7 +123,7 @@ export class Storage { syncWorkspace = async () => { await this.workspaceInitPromise; - loadLabels(this.workspaceSlug); + loadWorkSpaceData(this.workspaceSlug); }; syncProject = (projectId: string) => { @@ -151,8 +151,9 @@ export class Storage { return; } - const queryParams: { cursor: string; updated_at__gte?: string } = { + const queryParams: { cursor: string; updated_at__gte?: string; description: boolean } = { cursor: `${PAGE_SIZE}:0:0`, + description: true, }; const syncedAt = await this.getLastSyncTime(projectId); diff --git a/web/core/local-db/utils/load-labels.ts b/web/core/local-db/utils/load-labels.ts deleted file mode 100644 index dea8520d83e..00000000000 --- a/web/core/local-db/utils/load-labels.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { IssueLabelService } from "@/services/issue/issue_label.service"; -import { persistence } from "../storage.sqlite"; - -const stageLabelInserts = (label: any) => { - const { id, name, color, parent = "", project_id, sort_order, workspace_id } = label; - - const query = `INSERT OR REPLACE INTO labels (id, name, color, parent, project_id, sort_order, workspace_id) VALUES ('${id}', '${name}', '${color}', '${parent}', '${project_id}', '${sort_order}', '${workspace_id}');`; - persistence.db.exec(query); -}; -export const loadLabels = async (workspaceSlug: string, batchSize = 500) => { - const issueLabelService = new IssueLabelService(); - const labels = await issueLabelService.getWorkspaceIssueLabels(workspaceSlug); - for (let i = 0; i < labels.length; i += batchSize) { - const batch = labels.slice(i, i + batchSize); - - persistence.db.exec("BEGIN TRANSACTION;"); - batch.forEach((issue: any) => { - stageLabelInserts(issue); - }); - await persistence.db.exec("COMMIT;"); - } -}; diff --git a/web/core/local-db/utils/load-workspace.ts b/web/core/local-db/utils/load-workspace.ts new file mode 100644 index 00000000000..ac72fa927e9 --- /dev/null +++ b/web/core/local-db/utils/load-workspace.ts @@ -0,0 +1,142 @@ +import { IEstimate, IEstimatePoint, IWorkspaceMember } from "@plane/types"; +import { API_BASE_URL } from "@/helpers/common.helper"; +import { EstimateService } from "@/plane-web/services/project/estimate.service"; +import { CycleService } from "@/services/cycle.service"; +import { IssueLabelService } from "@/services/issue/issue_label.service"; +import { ModuleService } from "@/services/module.service"; +import { ProjectStateService } from "@/services/project"; +import { WorkspaceService } from "@/services/workspace.service"; +import { persistence } from "../storage.sqlite"; +import { cycleSchema, estimatePointSchema, labelSchema, memberSchema, Schema, stateSchema } from "./schemas"; + +const stageInserts = (table: string, schema: Schema, data: object) => { + const keys = Object.keys(schema); + // Pick only the keys that are in the schema + const filteredData = keys.reduce((acc: any, key) => { + if (data[key]) { + acc[key] = data[key]; + } + return acc; + }, {}); + const columns = "'" + Object.keys(filteredData).join("','") + "'"; + // Add quotes to column names + + const values = Object.values(filteredData) + .map((value) => { + if (value === null) { + return ""; + } + if (typeof value === "object") { + return `'${JSON.stringify(value)}'`; + } + if (typeof value === "string") { + return `'${value}'`; + } + return value; + }) + .join(", "); + const query = `INSERT OR REPLACE INTO ${table} (${columns}) VALUES (${values});`; + persistence.db.exec(query); +}; + +export const loadLabels = async (workspaceSlug: string, batchSize = 500) => { + const issueLabelService = new IssueLabelService(); + const objects = await issueLabelService.getWorkspaceIssueLabels(workspaceSlug); + for (let i = 0; i < objects.length; i += batchSize) { + const batch = objects.slice(i, i + batchSize); + + persistence.db.exec("BEGIN TRANSACTION;"); + batch.forEach((label: any) => { + stageInserts("labels", labelSchema, label); + }); + await persistence.db.exec("COMMIT;"); + } +}; + +export const loadModules = async (workspaceSlug: string, batchSize = 500) => { + const moduleService = new ModuleService(); + const objects = await moduleService.getWorkspaceModules(workspaceSlug); + for (let i = 0; i < objects.length; i += batchSize) { + const batch = objects.slice(i, i + batchSize); + + persistence.db.exec("BEGIN TRANSACTION;"); + batch.forEach((label: any) => { + stageInserts("modules", labelSchema, label); + }); + await persistence.db.exec("COMMIT;"); + } +}; + +export const loadCycles = async (workspaceSlug: string, batchSize = 500) => { + const cycleService = new CycleService(); + + const objects = await cycleService.getWorkspaceCycles(workspaceSlug); + for (let i = 0; i < objects.length; i += batchSize) { + const batch = objects.slice(i, i + batchSize); + + persistence.db.exec("BEGIN TRANSACTION;"); + batch.forEach((cycle: any) => { + stageInserts("cycles", cycleSchema, cycle); + }); + await persistence.db.exec("COMMIT;"); + } +}; + +export const loadStates = async (workspaceSlug: string, batchSize = 500) => { + const stateService = new ProjectStateService(); + const objects = await stateService.getWorkspaceStates(workspaceSlug); + for (let i = 0; i < objects.length; i += batchSize) { + const batch = objects.slice(i, i + batchSize); + + persistence.db.exec("BEGIN TRANSACTION;"); + batch.forEach((state: any) => { + stageInserts("states", stateSchema, state); + }); + await persistence.db.exec("COMMIT;"); + } +}; + +export const loadEstimatePoints = async (workspaceSlug: string, batchSize = 500) => { + const estimateService = new EstimateService(); + const estimates = await estimateService.fetchWorkspaceEstimates(workspaceSlug); + const objects: IEstimatePoint[] = []; + (estimates || []).forEach((estimate: IEstimate) => { + if (estimate?.points) { + objects.concat(estimate.points); + } + }); + for (let i = 0; i < objects.length; i += batchSize) { + const batch = objects.slice(i, i + batchSize); + + persistence.db.exec("BEGIN TRANSACTION;"); + batch.forEach((point: any) => { + stageInserts("estimate_points", estimatePointSchema, point); + }); + await persistence.db.exec("COMMIT;"); + } +}; + +export const loadMembers = async (workspaceSlug: string, batchSize = 500) => { + const workspaceService = new WorkspaceService(API_BASE_URL); + const members = await workspaceService.fetchWorkspaceMembers(workspaceSlug); + const objects = members.map((member: IWorkspaceMember) => member.member); + for (let i = 0; i < objects.length; i += batchSize) { + const batch = objects.slice(i, i + batchSize); + persistence.db.exec("BEGIN TRANSACTION;"); + batch.forEach((member: any) => { + stageInserts("members", memberSchema, member); + }); + await persistence.db.exec("COMMIT;"); + } +}; + +export const loadWorkSpaceData = async (workspaceSlug: string) => { + const promises = []; + promises.push(loadLabels(workspaceSlug)); + promises.push(loadModules(workspaceSlug)); + promises.push(loadCycles(workspaceSlug)); + promises.push(loadStates(workspaceSlug)); + promises.push(loadEstimatePoints(workspaceSlug)); + promises.push(loadMembers(workspaceSlug)); + await Promise.all(promises); +}; diff --git a/web/core/local-db/utils/query-constructor.ts b/web/core/local-db/utils/query-constructor.ts index 735a73a887c..551c311131e 100644 --- a/web/core/local-db/utils/query-constructor.ts +++ b/web/core/local-db/utils/query-constructor.ts @@ -1,5 +1,3 @@ -import { persistence } from "../storage.sqlite"; -import { PRIORITY_MAP } from "./constants"; import { getFilteredRowsForGrouping, getMetaKeys, @@ -7,17 +5,20 @@ import { singleFilterConstructor, translateQueryParams, } from "./query.utils"; -export const SPECIAL_ORDER_BY = [ - "labels__name", - "-labels__name", - "assignee__name", - "-assignee__name", - "module__name", - "-module__name", -]; +export const SPECIAL_ORDER_BY = { + labels__name: "labels", + "-labels__name": "labels", + assignees__first_name: "members", + "-assignees__first_name": "members", + issue_module__module__name: "modules", + "-issue_module__module__name": "modules", + issue_cycle__cycle__name: "cycles", + "-issue_cycle__cycle__name": "cycles", + state__name: "states", + "-state__name": "states", +}; export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: string, queries: any) => { - const { order_by, cursor, per_page, group_by, sub_group_by, sub_issue, ...otherProps } = - translateQueryParams(queries); + const { order_by, cursor, per_page, group_by, sub_group_by, ...otherProps } = translateQueryParams(queries); const orderByString = getOrderByFragment(order_by); const [pageSize, page, offset] = cursor.split(":"); @@ -56,7 +57,7 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st } const filterJoinFields = getMetaKeys(queries); - if (order_by && SPECIAL_ORDER_BY.includes(order_by)) { + if (order_by && Object.keys(SPECIAL_ORDER_BY).includes(order_by)) { const name = order_by.replace("-", ""); sql = `SELECT i.*, s.name as ${name} from issues i`; } else { @@ -68,10 +69,35 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st `; }); - if (order_by && SPECIAL_ORDER_BY.includes(order_by)) { - sql += ` - LEFT JOIN issue_meta label_ids ON i.id = label_ids.issue_id AND label_ids.key = 'label_ids' - INNER JOIN labels s ON s.id = label_ids.value`; + if (order_by && Object.keys(SPECIAL_ORDER_BY).includes(order_by)) { + if (order_by.includes("cycle")) { + sql += ` + LEFT JOIN cycles s on i.cycle_id = s.id`; + } + if (order_by.includes("estimate_point")) { + sql += ` + LEFT JOIN estimate_points s on i.estimate_point = s.id`; + } + if (order_by.includes("state")) { + sql += ` + LEFT JOIN states s on i.state_id = s.id`; + } + if (order_by.includes("label")) { + sql += ` + LEFT JOIN issue_meta sm ON i.id = sm.issue_id AND sm.key = 'label_ids' + INNER JOIN labels s ON s.id = sm.value`; + } + if (order_by.includes("module")) { + sql += ` + LEFT JOIN issue_meta sm ON i.id = sm.issue_id AND sm.key = 'module_ids' + INNER JOIN modules s ON s.id = sm.value`; + } + + if (order_by.includes("assignee")) { + sql += ` + LEFT JOIN issue_meta sm ON i.id = sm.issue_id AND sm.key = 'assignee_ids' + INNER JOIN members s ON s.id = sm.value`; + } } sql += ` WHERE i.project_id = '${projectId}' ${singleFilterConstructor(otherProps)} group by i.id `; sql += orderByString; @@ -91,51 +117,5 @@ export const issueFilterCountQueryConstructor = (workspaceSlug: string, projectI sql = sql.replace("SELECT i.*", "SELECT COUNT(DISTINCT i.id) as total_count"); // Remove everything after group by i.id sql = `${sql.split("group by i.id")[0]};`; - console.log("### COUNT", sql); return sql; }; - -const arrayFields = ["label_ids", "assignee_ids", "module_ids"]; - -export const stageIssueInserts = (issue: any) => { - const issue_id = issue.id; - issue.priority_proxy = PRIORITY_MAP[issue.priority as keyof typeof PRIORITY_MAP]; - const keys = Object.keys(issue).join(","); - - const values = Object.values(issue).map((val) => { - if (val === null) { - return ""; - } - if (typeof val === "object") { - return JSON.stringify(val); - } - return val; - }); // Will fail when the values have a comma - - persistence.db.exec({ - sql: `INSERT OR REPLACE into issues(${keys}) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, - bind: values, - }); - - persistence.db.exec({ - sql: `DELETE from issue_meta where issue_id='${issue_id}'`, - }); - - arrayFields.forEach((field) => { - const values = issue[field]; - if (values && values.length) { - values.forEach((val: any) => { - persistence.db.exec({ - sql: `INSERT OR REPLACE into issue_meta(issue_id,key,value) values (?,?,?) `, - bind: [issue_id, field, val], - }); - }); - } else { - // Added for empty fields? - persistence.db.exec({ - sql: `INSERT OR REPLACE into issue_meta(issue_id,key,value) values (?,?,?) `, - bind: [issue_id, field, ""], - }); - } - }); -}; diff --git a/web/core/local-db/utils/query.utils.ts b/web/core/local-db/utils/query.utils.ts index 28d02c92b73..0d552731220 100644 --- a/web/core/local-db/utils/query.utils.ts +++ b/web/core/local-db/utils/query.utils.ts @@ -1,4 +1,5 @@ import { ARRAY_FIELDS, GROUP_BY_MAP, PRIORITY_MAP } from "./constants"; +import { issueSchema } from "./schemas"; import { wrapDateTime } from "./utils"; export const translateQueryParams = (queries: any) => { @@ -121,7 +122,6 @@ export const getFilteredRowsForGrouping = (projectId: string, queries: any) => { const joinsRequired = areJoinsRequired(queries); - const orderByFragment = getOrderByFragment(queries.order_by); let sql = ""; if (!joinsRequired) { sql = `WITH fi as (SELECT *`; @@ -181,6 +181,9 @@ export const singleFilterConstructor = (queries: any) => { const { order_by, cursor, per_page, group_by, sub_group_by, sub_issue, ...filters } = translateQueryParams(queries); let sql = ""; + if (!sub_issue) { + sql += ` AND parent_id IS NOT NULL `; + } const keys = Object.keys(filters); keys.forEach((key) => { @@ -210,3 +213,9 @@ const getSingleFilterFields = (queries: any) => { return Array.from(fields); }; + +export const getIssueFieldsFragment = () => { + const [description_html, ...keys] = Object.keys(issueSchema); + const sql = ` ('${keys.join("','")}')`; + return sql; +}; diff --git a/web/core/local-db/utils/schemas.ts b/web/core/local-db/utils/schemas.ts new file mode 100644 index 00000000000..841034ee182 --- /dev/null +++ b/web/core/local-db/utils/schemas.ts @@ -0,0 +1,130 @@ +export type Schema = { + [key: string]: string; +}; + +export const issueSchema: Schema = { + id: "TEXT UNIQUE", + name: "TEXT", + state_id: "TEXT", + sort_order: "REAL", + completed_at: "TEXT", + estimate_point: "REAL", + priority: "TEXT", + priority_proxy: "INTEGER", + start_date: "TEXT", + target_date: "TEXT", + sequence_id: "INTEGER", + project_id: "TEXT", + parent_id: "TEXT", + created_at: "TEXT", + updated_at: "TEXT", + created_by: "TEXT", + updated_by: "TEXT", + is_draft: "INTEGER", + archived_at: "TEXT", + state__group: "TEXT", + sub_issues_count: "INTEGER", + cycle_id: "TEXT", + link_count: "INTEGER", + attachment_count: "INTEGER", + type_id: "TEXT", + label_ids: "TEXT", + assignee_ids: "TEXT", + module_ids: "TEXT", + description_html: "TEXT", +}; + +export const issueMetaSchema: Schema = { + issue_id: "TEXT", + key: "TEXT", + value: "TEXT", +}; +export const moduleSchema: Schema = { + id: "TEXT UNIQUE", + workspace_id: "TEXT", + project_id: "TEXT", + name: "TEXT", + description: "TEXT", + description_text: "TEXT", + description_html: "TEXT", + start_date: "TEXT", + target_date: "TEXT", + status: "TEXT", + lead_id: "TEXT", + member_ids: "TEXT", + view_props: "TEXT", + sort_order: "INTEGER", + external_source: "TEXT", + external_id: "TEXT", + logo_props: "TEXT", + total_issues: "INTEGER", + cancelled_issues: "INTEGER", + completed_issues: "INTEGER", + started_issues: "INTEGER", + unstarted_issues: "INTEGER", + backlog_issues: "INTEGER", + created_at: "TEXT", + updated_at: "TEXT", + archived_at: "TEXT", +}; + +export const labelSchema: Schema = { + id: "TEXT UNIQUE", + name: "TEXT", + color: "TEXT", + parent: "TEXT", + project_id: "TEXT", + workspace_id: "TEXT", + sort_order: "INTEGER", +}; + +export const cycleSchema: Schema = { + id: "TEXT UNIQUE", + workspace_id: "TEXT", + project_id: "TEXT", + name: "TEXT", + description: "TEXT", + start_date: "TEXT", + end_date: "TEXT", + owned_by_id: "TEXT", + view_props: "TEXT", + sort_order: "INTEGER", + external_source: "TEXT", + external_id: "TEXT", + progress_snapshot: "TEXT", + logo_props: "TEXT", + total_issues: "INTEGER", + cancelled_issues: "INTEGER", + completed_issues: "INTEGER", + started_issues: "INTEGER", + unstarted_issues: "INTEGER", + backlog_issues: "INTEGER", +}; + +export const stateSchema: Schema = { + id: "TEXT UNIQUE", + project_id: "TEXT", + workspace_id: "TEXT", + name: "TEXT", + color: "TEXT", + group: "TEXT", + default: "BOOLEAN", + description: "TEXT", + sequence: "INTEGER", +}; + +export const estimatePointSchema: Schema = { + id: "TEXT UNIQUE", + key: "TEXT", + value: "REAL", +}; + +export const memberSchema: Schema = { + id: "TEXT UNIQUE", + first_name: "TEXT", + last_name: "TEXT", + avatar: "TEXT", + is_bot: "BOOLEAN", + display_name: "TEXT", + email: "TEXT", +}; diff --git a/web/core/local-db/utils/tables.ts b/web/core/local-db/utils/tables.ts index a3d8b1c1dc2..050f976be80 100644 --- a/web/core/local-db/utils/tables.ts +++ b/web/core/local-db/utils/tables.ts @@ -1,68 +1,37 @@ -import createIndexes from "./indexes"; +import { persistence } from "../storage.sqlite"; +import { + labelSchema, + moduleSchema, + Schema, + issueMetaSchema, + issueSchema, + stateSchema, + cycleSchema, + estimatePointSchema, + memberSchema, +} from "./schemas"; -export const createIssuesTable = (SQLITE: any) => { - const sqlstr = `CREATE TABLE IF NOT EXISTS issues ( - id TEXT UNIQUE, - name TEXT, - state_id TEXT, - sort_order REAL, - completed_at TEXT, - estimate_point REAL, - priority TEXT, - priority_proxy INTEGER, - start_date TEXT, - target_date TEXT, - sequence_id INTEGER, - project_id TEXT, - parent_id TEXT, - created_at TEXT, - updated_at TEXT, - created_by TEXT, - updated_by TEXT, - is_draft INTEGER, - archived_at TEXT, - state__group TEXT, - sub_issues_count INTEGER, - cycle_id TEXT, - link_count INTEGER, - attachment_count INTEGER, - type_id TEXT, - label_ids TEXT, - assignee_ids TEXT, - module_ids TEXT);`; - SQLITE.exec(sqlstr); +const createTableSQLfromSchema = (tableName: string, schema: Schema) => { + let sql = `CREATE TABLE IF NOT EXISTS ${tableName} (`; + sql += Object.keys(schema) + .map((key) => `'${key}' ${schema[key]}`) + .join(", "); + sql += `);`; + console.log("#####", sql); + return sql; }; -export const createIssueMetaTable = (SQLITE: any) => { - const sqlstr = `CREATE TABLE IF NOT EXISTS issue_meta ( - issue_id TEXT, - key TEXT, - value TEXT, - UNIQUE(issue_id, key,value) -) - ;`; - SQLITE.exec(sqlstr); -}; +export const createTables = async () => { + persistence.db.exec("BEGIN TRANSACTION;"); -export const createLabelsTable = (SQLITE: any) => { - const sqlstr = `CREATE TABLE IF NOT EXISTS labels ( - id TEXT UNIQUE, - name TEXT, - color TEXT, - parent TEXT, - project_id TEXT, - sort_order INTEGER, - workspace_id TEXT);`; - SQLITE.exec(sqlstr); -}; -export const createTables = async (SQLITE: any) => { - await createIssuesTable(SQLITE); - await createIssueMetaTable(SQLITE); - await createLabelsTable(SQLITE); + persistence.db.exec(createTableSQLfromSchema("modules", moduleSchema)); + persistence.db.exec(createTableSQLfromSchema("labels", labelSchema)); + persistence.db.exec(createTableSQLfromSchema("issue_meta", issueMetaSchema)); + persistence.db.exec(createTableSQLfromSchema("issues", issueSchema)); + persistence.db.exec(createTableSQLfromSchema("states", stateSchema)); + persistence.db.exec(createTableSQLfromSchema("cycles", cycleSchema)); + persistence.db.exec(createTableSQLfromSchema("estimate_points", estimatePointSchema)); + persistence.db.exec(createTableSQLfromSchema("members", memberSchema)); - // try { - // await createIndexes(); - // } catch (e) { - // console.error(e); - // } + persistence.db.exec("COMMIT;"); }; From 79a2b94b5be362a8ece9d04e402e8aa86e41d5b5 Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Thu, 29 Aug 2024 15:51:00 +0530 Subject: [PATCH 050/111] Exclude description from getIssues --- web/core/local-db/utils/load-issues.ts | 58 +++++++++++++++++++- web/core/local-db/utils/query-constructor.ts | 7 ++- web/core/local-db/utils/query.utils.ts | 5 +- web/core/local-db/utils/tables.ts | 4 +- 4 files changed, 67 insertions(+), 7 deletions(-) diff --git a/web/core/local-db/utils/load-issues.ts b/web/core/local-db/utils/load-issues.ts index 93bb5f79aa2..b9eae508ca9 100644 --- a/web/core/local-db/utils/load-issues.ts +++ b/web/core/local-db/utils/load-issues.ts @@ -1,6 +1,7 @@ import { IssueService } from "@/services/issue"; import { persistence } from "../storage.sqlite"; -import { stageIssueInserts } from "./query-constructor"; +import { ARRAY_FIELDS, PRIORITY_MAP } from "./constants"; +import { issueSchema } from "./schemas"; export const PROJECT_OFFLINE_STATUS: Record = {}; @@ -48,3 +49,58 @@ export const syncDeletesToLocal = async (workspaceId: string, projectId: string) response.map(async (issue) => deleteIssueFromLocal(issue)); } }; + +export const stageIssueInserts = (issue: any) => { + const issue_id = issue.id; + issue.priority_proxy = PRIORITY_MAP[issue.priority as keyof typeof PRIORITY_MAP]; + + const keys = Object.keys(issueSchema); + const sanitizedIssue = keys.reduce((acc: any, key) => { + if (issue[key]) { + acc[key] = issue[key]; + } + return acc; + }, {}); + + const columns = "'" + Object.keys(sanitizedIssue).join("','") + "'"; + + const values = Object.values(sanitizedIssue) + .map((value) => { + if (value === null) { + return ""; + } + if (typeof value === "object") { + return `'${JSON.stringify(value)}'`; + } + if (typeof value === "string") { + return `'${value}'`; + } + return value; + }) + .join(", "); + + const query = `INSERT OR REPLACE INTO issues (${columns}) VALUES (${values});`; + persistence.db.exec(query); + + persistence.db.exec({ + sql: `DELETE from issue_meta where issue_id='${issue_id}'`, + }); + + ARRAY_FIELDS.forEach((field) => { + const values = issue[field]; + if (values && values.length) { + values.forEach((val: any) => { + persistence.db.exec({ + sql: `INSERT OR REPLACE into issue_meta(issue_id,key,value) values (?,?,?) `, + bind: [issue_id, field, val], + }); + }); + } else { + // Added for empty fields? + persistence.db.exec({ + sql: `INSERT OR REPLACE into issue_meta(issue_id,key,value) values (?,?,?) `, + bind: [issue_id, field, ""], + }); + } + }); +}; diff --git a/web/core/local-db/utils/query-constructor.ts b/web/core/local-db/utils/query-constructor.ts index 551c311131e..3618101bcab 100644 --- a/web/core/local-db/utils/query-constructor.ts +++ b/web/core/local-db/utils/query-constructor.ts @@ -1,5 +1,6 @@ import { getFilteredRowsForGrouping, + getIssueFieldsFragment, getMetaKeys, getOrderByFragment, singleFilterConstructor, @@ -24,12 +25,14 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st let sql = ""; + const fieldsFragment = getIssueFieldsFragment(); + if (sub_group_by) { sql = getFilteredRowsForGrouping(projectId, queries); sql += `, ranked_issues AS ( SELECT fi.*, ROW_NUMBER() OVER (PARTITION BY group_id, sub_group_id ${orderByString}) as rank, COUNT(*) OVER (PARTITION by group_id, sub_group_id) as total_issues from fi) - SELECT ri.*, i.* + SELECT ri.*, ${fieldsFragment} FROM ranked_issues ri JOIN issues i ON ri.id = i.id WHERE rank <= ${per_page} @@ -45,7 +48,7 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st sql += `, ranked_issues AS ( SELECT fi.*, ROW_NUMBER() OVER (PARTITION BY group_id ${orderByString}) as rank, COUNT(*) OVER (PARTITION by group_id) as total_issues FROM fi) - SELECT ri.*, i.* + SELECT ri.*, ${fieldsFragment} FROM ranked_issues ri JOIN issues i ON ri.id = i.id WHERE rank <= ${per_page} diff --git a/web/core/local-db/utils/query.utils.ts b/web/core/local-db/utils/query.utils.ts index 0d552731220..2d5592caa2c 100644 --- a/web/core/local-db/utils/query.utils.ts +++ b/web/core/local-db/utils/query.utils.ts @@ -215,7 +215,8 @@ const getSingleFilterFields = (queries: any) => { }; export const getIssueFieldsFragment = () => { - const [description_html, ...keys] = Object.keys(issueSchema); - const sql = ` ('${keys.join("','")}')`; + const { description_html, ...filtered } = issueSchema; + const keys = Object.keys(filtered); + const sql = ` ${keys.map((key) => `i.${key}`).join(",")}`; return sql; }; diff --git a/web/core/local-db/utils/tables.ts b/web/core/local-db/utils/tables.ts index 050f976be80..f8097d74699 100644 --- a/web/core/local-db/utils/tables.ts +++ b/web/core/local-db/utils/tables.ts @@ -24,10 +24,10 @@ const createTableSQLfromSchema = (tableName: string, schema: Schema) => { export const createTables = async () => { persistence.db.exec("BEGIN TRANSACTION;"); + persistence.db.exec(createTableSQLfromSchema("issues", issueSchema)); + persistence.db.exec(createTableSQLfromSchema("issue_meta", issueMetaSchema)); persistence.db.exec(createTableSQLfromSchema("modules", moduleSchema)); persistence.db.exec(createTableSQLfromSchema("labels", labelSchema)); - persistence.db.exec(createTableSQLfromSchema("issue_meta", issueMetaSchema)); - persistence.db.exec(createTableSQLfromSchema("issues", issueSchema)); persistence.db.exec(createTableSQLfromSchema("states", stateSchema)); persistence.db.exec(createTableSQLfromSchema("cycles", cycleSchema)); persistence.db.exec(createTableSQLfromSchema("estimate_points", estimatePointSchema)); From ec0a2a5c50ea610680a308004595f0a409b3cb4d Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Thu, 29 Aug 2024 15:52:55 +0530 Subject: [PATCH 051/111] Add getIssue function. --- web/core/local-db/storage.sqlite.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/web/core/local-db/storage.sqlite.ts b/web/core/local-db/storage.sqlite.ts index a4b6c6e8b58..f6893b98f73 100644 --- a/web/core/local-db/storage.sqlite.ts +++ b/web/core/local-db/storage.sqlite.ts @@ -299,6 +299,13 @@ export class Storage { return out; }; + getIssue = async (issueId: string) => { + const issue = await runQuery(`select * from issues where id='${issueId}'`); + if (issue.length) { + return issue[0]; + } + return; + }; getStatus = (projectId: string) => this.projectStatus[projectId]?.issues?.status || undefined; setStatus = (projectId: string, status: "loading" | "ready" | "error" | "syncing" | undefined = undefined) => { set(this.projectStatus, `${projectId}.issues.status`, status); From 87a9c382a2b2b426b2e77d9a1f977c9a8c739fe8 Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Thu, 29 Aug 2024 17:28:51 +0530 Subject: [PATCH 052/111] Add only selected fields to get query. --- web/core/local-db/utils/query-constructor.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/core/local-db/utils/query-constructor.ts b/web/core/local-db/utils/query-constructor.ts index 3618101bcab..d98637979fd 100644 --- a/web/core/local-db/utils/query-constructor.ts +++ b/web/core/local-db/utils/query-constructor.ts @@ -62,9 +62,9 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st const filterJoinFields = getMetaKeys(queries); if (order_by && Object.keys(SPECIAL_ORDER_BY).includes(order_by)) { const name = order_by.replace("-", ""); - sql = `SELECT i.*, s.name as ${name} from issues i`; + sql = `SELECT ${fieldsFragment} , s.name as ${name} from issues i`; } else { - sql = `SELECT i.* from issues i`; + sql = `SELECT ${fieldsFragment} from issues i`; } filterJoinFields.forEach((field: string) => { const value = otherProps[field] || ""; From 43fe5af9e2590bdd33bc0e2a515a4781a946b8b3 Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Thu, 29 Aug 2024 19:45:04 +0530 Subject: [PATCH 053/111] Fix the count query --- web/core/local-db/utils/query-constructor.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/web/core/local-db/utils/query-constructor.ts b/web/core/local-db/utils/query-constructor.ts index d98637979fd..18df6cb08b6 100644 --- a/web/core/local-db/utils/query-constructor.ts +++ b/web/core/local-db/utils/query-constructor.ts @@ -113,11 +113,13 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st }; export const issueFilterCountQueryConstructor = (workspaceSlug: string, projectId: string, queries: any) => { + //@todo Very crude way to extract count from the actual query. Needs to be refactored // Remove group by from the query to fallback to non group query const { group_by, sub_group_by, order_by, ...otherProps } = queries; let sql = issueFilterQueryConstructor(workspaceSlug, projectId, otherProps); + const fieldsFragment = getIssueFieldsFragment(); - sql = sql.replace("SELECT i.*", "SELECT COUNT(DISTINCT i.id) as total_count"); + sql = sql.replace(`SELECT ${fieldsFragment}`, "SELECT COUNT(DISTINCT i.id) as total_count"); // Remove everything after group by i.id sql = `${sql.split("group by i.id")[0]};`; return sql; From 24e6b3c39d1fedccd95c0f1d59a743f9cf77b267 Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Fri, 30 Aug 2024 11:43:42 +0530 Subject: [PATCH 054/111] Minor query optimization when no joins are required. --- web/core/local-db/utils/query.utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/core/local-db/utils/query.utils.ts b/web/core/local-db/utils/query.utils.ts index 2d5592caa2c..dfae1084fd1 100644 --- a/web/core/local-db/utils/query.utils.ts +++ b/web/core/local-db/utils/query.utils.ts @@ -124,7 +124,7 @@ export const getFilteredRowsForGrouping = (projectId: string, queries: any) => { let sql = ""; if (!joinsRequired) { - sql = `WITH fi as (SELECT *`; + sql = `WITH fi as (SELECT i.id,i.created_at ${issueTableFilterFields}`; if (group_by) { sql += `, i.${group_by} as group_id`; } From cb682ef7b9635f3b4b16fa6b03d52f0b375114f8 Mon Sep 17 00:00:00 2001 From: rahulramesha Date: Fri, 30 Aug 2024 14:46:03 +0530 Subject: [PATCH 055/111] fetch issue description from local db --- .../(detail)/[archivedIssueId]/page.tsx | 6 +- .../issues/(detail)/[issueId]/page.tsx | 10 +- .../components/issues/peek-overview/root.tsx | 8 +- web/core/local-db/storage.sqlite.ts | 37 +++++--- web/core/local-db/utils/load-issues.ts | 3 +- .../store/issue/issue-details/issue.store.ts | 93 ++++++++++++------- 6 files changed, 93 insertions(+), 64 deletions(-) diff --git a/web/app/[workspaceSlug]/(projects)/projects/(detail)/[projectId]/archives/issues/(detail)/[archivedIssueId]/page.tsx b/web/app/[workspaceSlug]/(projects)/projects/(detail)/[projectId]/archives/issues/(detail)/[archivedIssueId]/page.tsx index 74dbb76491a..35896a3870e 100644 --- a/web/app/[workspaceSlug]/(projects)/projects/(detail)/[projectId]/archives/issues/(detail)/[archivedIssueId]/page.tsx +++ b/web/app/[workspaceSlug]/(projects)/projects/(detail)/[projectId]/archives/issues/(detail)/[archivedIssueId]/page.tsx @@ -19,12 +19,12 @@ const ArchivedIssueDetailsPage = observer(() => { // hooks const { fetchIssue, - issue: { getIssueById }, + issue: { getIssueById, isFetchingIssueDetails }, } = useIssueDetail(); const { getProjectById } = useProject(); - const { isLoading, data: swrArchivedIssueDetails } = useSWR( + const { data: swrArchivedIssueDetails } = useSWR( workspaceSlug && projectId && archivedIssueId ? `ARCHIVED_ISSUE_DETAIL_${workspaceSlug}_${projectId}_${archivedIssueId}` : null, @@ -40,7 +40,7 @@ const ArchivedIssueDetailsPage = observer(() => { if (!issue) return <>; - const issueLoader = !issue || isLoading ? true : false; + const issueLoader = !issue || isFetchingIssueDetails ? true : false; return ( <> diff --git a/web/app/[workspaceSlug]/(projects)/projects/(detail)/[projectId]/issues/(detail)/[issueId]/page.tsx b/web/app/[workspaceSlug]/(projects)/projects/(detail)/[projectId]/issues/(detail)/[issueId]/page.tsx index 27b87509051..0814e38061a 100644 --- a/web/app/[workspaceSlug]/(projects)/projects/(detail)/[projectId]/issues/(detail)/[issueId]/page.tsx +++ b/web/app/[workspaceSlug]/(projects)/projects/(detail)/[projectId]/issues/(detail)/[issueId]/page.tsx @@ -27,16 +27,12 @@ const IssueDetailsPage = observer(() => { // store hooks const { fetchIssue, - issue: { getIssueById }, + issue: { getIssueById, isFetchingIssueDetails }, } = useIssueDetail(); const { getProjectById } = useProject(); const { toggleIssueDetailSidebar, issueDetailSidebarCollapsed } = useAppTheme(); // fetching issue details - const { - isLoading, - data: swrIssueDetails, - error, - } = useSWR( + const { data: swrIssueDetails, error } = useSWR( workspaceSlug && projectId && issueId ? `ISSUE_DETAIL_${workspaceSlug}_${projectId}_${issueId}` : null, workspaceSlug && projectId && issueId ? () => fetchIssue(workspaceSlug.toString(), projectId.toString(), issueId.toString()) @@ -45,7 +41,7 @@ const IssueDetailsPage = observer(() => { // derived values const issue = getIssueById(issueId?.toString() || "") || undefined; const project = (issue?.project_id && getProjectById(issue?.project_id)) || undefined; - const issueLoader = !issue || isLoading ? true : false; + const issueLoader = !issue || isFetchingIssueDetails ? true : false; const pageTitle = project && issue ? `${project?.identifier}-${issue?.sequence_id} ${issue?.name}` : undefined; useEffect(() => { diff --git a/web/core/components/issues/peek-overview/root.tsx b/web/core/components/issues/peek-overview/root.tsx index a55b0d5464f..864a10eae5f 100644 --- a/web/core/components/issues/peek-overview/root.tsx +++ b/web/core/components/issues/peek-overview/root.tsx @@ -35,14 +35,13 @@ export const IssuePeekOverview: FC = observer((props) => { const { peekIssue, setPeekIssue, - issue: { fetchIssue }, + issue: { fetchIssue, isFetchingIssueDetails }, fetchActivities, } = useIssueDetail(); const { issues } = useIssuesStore(); const { captureIssueEvent } = useEventTracker(); // state - const [loader, setLoader] = useState(true); const [error, setError] = useState(false); const removeRoutePeekId = () => { @@ -54,7 +53,6 @@ export const IssuePeekOverview: FC = observer((props) => { () => ({ fetch: async (workspaceSlug: string, projectId: string, issueId: string, loader = true) => { try { - setLoader(loader); setError(false); await fetchIssue( workspaceSlug, @@ -62,10 +60,8 @@ export const IssuePeekOverview: FC = observer((props) => { issueId, is_archived ? "ARCHIVED" : is_draft ? "DRAFT" : "DEFAULT" ); - setLoader(false); setError(false); } catch (error) { - setLoader(false); setError(true); console.error("Error fetching the parent issue"); } @@ -344,7 +340,7 @@ export const IssuePeekOverview: FC = observer((props) => { workspaceSlug={peekIssue.workspaceSlug} projectId={peekIssue.projectId} issueId={peekIssue.issueId} - isLoading={loader} + isLoading={isFetchingIssueDetails} isError={error} is_archived={is_archived} disabled={!isEditable} diff --git a/web/core/local-db/storage.sqlite.ts b/web/core/local-db/storage.sqlite.ts index f6893b98f73..f82ed2f823d 100644 --- a/web/core/local-db/storage.sqlite.ts +++ b/web/core/local-db/storage.sqlite.ts @@ -1,7 +1,11 @@ import set from "lodash/set"; +// plane import { EIssueGroupBYServerToProperty } from "@plane/constants"; +import { TIssue } from "@plane/types"; import { setToast, TOAST_TYPE } from "@plane/ui"; +// services import { IssueService } from "@/services/issue/issue.service"; +// import { ARRAY_FIELDS } from "./utils/constants"; import createIndexes from "./utils/indexes"; import { addIssuesBulk } from "./utils/load-issues"; @@ -10,9 +14,12 @@ import { issueFilterCountQueryConstructor, issueFilterQueryConstructor } from ". import { runQuery } from "./utils/query-executor"; import { createTables } from "./utils/tables"; import { getGroupedIssueResults, getSubGroupedIssueResults } from "./utils/utils"; + + declare module "@sqlite.org/sqlite-wasm" { export function sqlite3Worker1Promiser(...args: any): any; } + const PAGE_SIZE = 1000; const log = console.log; const error = console.error; @@ -30,12 +37,9 @@ export class Storage { projectStatus: Record = {}; workspaceSlug: string = ""; workspaceInitPromise: Promise | undefined; - // issueService: any; constructor() { this.db = null; - - // this.issueService = new IssueService(); } reset = () => { @@ -113,7 +117,7 @@ export class Storage { ); this.status = "ready"; // Your SQLite code here. - await createTables(this.db); + await createTables(); } catch (err) { error(err); } @@ -254,11 +258,7 @@ export class Storage { const parsingStart = performance.now(); let issueResults = issuesRaw.map((issue: any) => { - ARRAY_FIELDS.forEach((field: string) => { - issue[field] = issue[field] ? JSON.parse(issue[field]) : []; - }); - - return issue; + return formatLocalIssue(issue); }); console.log("#### Issue Results", issueResults.length); @@ -300,9 +300,9 @@ export class Storage { }; getIssue = async (issueId: string) => { - const issue = await runQuery(`select * from issues where id='${issueId}'`); - if (issue.length) { - return issue[0]; + const issues = await runQuery(`select * from issues where id='${issueId}'`); + if (issues.length) { + return formatLocalIssue(issues[0]); } return; }; @@ -318,3 +318,16 @@ export class Storage { } export const persistence = new Storage(); + +/** + * format the issue fetched from local db into an issue + * @param issue + * @returns + */ +export const formatLocalIssue = (issue: any) => { + const currIssue = issue; + ARRAY_FIELDS.forEach((field: string) => { + currIssue[field] = currIssue[field] ? JSON.parse(currIssue[field]) : []; + }); + return currIssue as TIssue; +} \ No newline at end of file diff --git a/web/core/local-db/utils/load-issues.ts b/web/core/local-db/utils/load-issues.ts index b9eae508ca9..765cf54beda 100644 --- a/web/core/local-db/utils/load-issues.ts +++ b/web/core/local-db/utils/load-issues.ts @@ -1,3 +1,4 @@ +import { TIssue } from "@plane/types"; import { IssueService } from "@/services/issue"; import { persistence } from "../storage.sqlite"; import { ARRAY_FIELDS, PRIORITY_MAP } from "./constants"; @@ -35,7 +36,7 @@ export const deleteIssueFromLocal = async (issue_id: any) => { persistence.db.exec("COMMIT;"); }; -export const updateIssue = async (issue: any) => { +export const updateIssue = async (issue: TIssue) => { const issue_id = issue.id; // delete the issue and its meta data await deleteIssueFromLocal(issue_id); diff --git a/web/core/store/issue/issue-details/issue.store.ts b/web/core/store/issue/issue-details/issue.store.ts index 500d8e034b8..a5a88cd22e8 100644 --- a/web/core/store/issue/issue-details/issue.store.ts +++ b/web/core/store/issue/issue-details/issue.store.ts @@ -1,7 +1,10 @@ -import { makeObservable } from "mobx"; +import { makeObservable, observable } from "mobx"; import { computedFn } from "mobx-utils"; // types import { TIssue } from "@plane/types"; +// local +import { persistence } from "@/local-db/storage.sqlite"; +import { updateIssue } from "@/local-db/utils/load-issues"; // services import { IssueArchiveService, IssueDraftService, IssueService } from "@/services/issue"; // types @@ -32,11 +35,13 @@ export interface IIssueStoreActions { } export interface IIssueStore extends IIssueStoreActions { + isFetchingIssueDetails: boolean; // helper methods getIssueById: (issueId: string) => TIssue | undefined; } export class IssueStore implements IIssueStore { + isFetchingIssueDetails: boolean = false; // root store rootIssueDetailStore: IIssueDetail; // services @@ -45,7 +50,9 @@ export class IssueStore implements IIssueStore { issueDraftService; constructor(rootStore: IIssueDetail) { - makeObservable(this, {}); + makeObservable(this, { + isFetchingIssueDetails: observable.ref, + }); // root store this.rootIssueDetailStore = rootStore; // services @@ -66,7 +73,16 @@ export class IssueStore implements IIssueStore { expand: "issue_reactions,issue_attachment,issue_link,parent", }; - let issue: TIssue; + let issue: TIssue | undefined; + + // fetch issue from local db + issue = await persistence.getIssue(issueId); + + this.isFetchingIssueDetails = true; + + if (issue) { + this.addIssueToStore(issue); + } if (issueType === "ARCHIVED") issue = await this.issueArchiveService.retrieveArchivedIssue(workspaceSlug, projectId, issueId, query); @@ -76,38 +92,9 @@ export class IssueStore implements IIssueStore { if (!issue) throw new Error("Issue not found"); - const issuePayload: TIssue = { - id: issue?.id, - sequence_id: issue?.sequence_id, - name: issue?.name, - description_html: issue?.description_html, - sort_order: issue?.sort_order, - state_id: issue?.state_id, - priority: issue?.priority, - label_ids: issue?.label_ids, - assignee_ids: issue?.assignee_ids, - estimate_point: issue?.estimate_point, - sub_issues_count: issue?.sub_issues_count, - attachment_count: issue?.attachment_count, - link_count: issue?.link_count, - project_id: issue?.project_id, - parent_id: issue?.parent_id, - cycle_id: issue?.cycle_id, - module_ids: issue?.module_ids, - type_id: issue?.type_id, - created_at: issue?.created_at, - updated_at: issue?.updated_at, - start_date: issue?.start_date, - target_date: issue?.target_date, - completed_at: issue?.completed_at, - archived_at: issue?.archived_at, - created_by: issue?.created_by, - updated_by: issue?.updated_by, - is_draft: issue?.is_draft, - is_subscribed: issue?.is_subscribed, - }; - - this.rootIssueDetailStore.rootIssueStore.issues.addIssue([issuePayload], true); + this.addIssueToStore(issue); + // adds issue to local store + updateIssue(issue); // store handlers from issue detail // parent @@ -150,6 +137,42 @@ export class IssueStore implements IIssueStore { return issue; }; + addIssueToStore = (issue: TIssue) => { + const issuePayload: TIssue = { + id: issue?.id, + sequence_id: issue?.sequence_id, + name: issue?.name, + description_html: issue?.description_html, + sort_order: issue?.sort_order, + state_id: issue?.state_id, + priority: issue?.priority, + label_ids: issue?.label_ids, + assignee_ids: issue?.assignee_ids, + estimate_point: issue?.estimate_point, + sub_issues_count: issue?.sub_issues_count, + attachment_count: issue?.attachment_count, + link_count: issue?.link_count, + project_id: issue?.project_id, + parent_id: issue?.parent_id, + cycle_id: issue?.cycle_id, + module_ids: issue?.module_ids, + type_id: issue?.type_id, + created_at: issue?.created_at, + updated_at: issue?.updated_at, + start_date: issue?.start_date, + target_date: issue?.target_date, + completed_at: issue?.completed_at, + archived_at: issue?.archived_at, + created_by: issue?.created_by, + updated_by: issue?.updated_by, + is_draft: issue?.is_draft, + is_subscribed: issue?.is_subscribed, + }; + + this.rootIssueDetailStore.rootIssueStore.issues.addIssue([issuePayload], true); + this.isFetchingIssueDetails = false; + }; + updateIssue = async (workspaceSlug: string, projectId: string, issueId: string, data: Partial) => { await this.rootIssueDetailStore.rootIssueStore.projectIssues.updateIssue(workspaceSlug, projectId, issueId, data); await this.rootIssueDetailStore.activity.fetchActivities(workspaceSlug, projectId, issueId); From 6bf6a069067ccc20d60f33be47e199a04ac5bc30 Mon Sep 17 00:00:00 2001 From: rahulramesha Date: Fri, 30 Aug 2024 14:54:50 +0530 Subject: [PATCH 056/111] clear local db on signout --- web/core/local-db/storage.sqlite.ts | 11 +++++++++++ web/core/store/user/index.ts | 3 +++ 2 files changed, 14 insertions(+) diff --git a/web/core/local-db/storage.sqlite.ts b/web/core/local-db/storage.sqlite.ts index f82ed2f823d..9c0aedf7f66 100644 --- a/web/core/local-db/storage.sqlite.ts +++ b/web/core/local-db/storage.sqlite.ts @@ -50,6 +50,17 @@ export class Storage { this.workspaceInitPromise = undefined; }; + clearStorage = async () => { + try { + const storageManager = window.navigator.storage; + const fileSystemDirectoryHandle = await storageManager.getDirectory(); + //@ts-ignore + await fileSystemDirectoryHandle.remove({ recursive: true }); + } catch (e) { + console.error("Error clearing sqlite sync storage", e); + } + }; + initialize = async (workspaceSlug: string): Promise => { if (workspaceSlug !== this.workspaceSlug) { this.reset(); diff --git a/web/core/store/user/index.ts b/web/core/store/user/index.ts index 56eee659aa4..dbe2d891352 100644 --- a/web/core/store/user/index.ts +++ b/web/core/store/user/index.ts @@ -8,6 +8,8 @@ import { EUserProjectRoles } from "@/constants/project"; import { EUserWorkspaceRoles } from "@/constants/workspace"; // helpers import { API_BASE_URL } from "@/helpers/common.helper"; +// local +import { persistence } from "@/local-db/storage.sqlite"; // services import { AuthService } from "@/services/auth.service"; import { UserService } from "@/services/user.service"; @@ -268,6 +270,7 @@ export class UserStore implements IUserStore { */ signOut = async (): Promise => { await this.authService.signOut(API_BASE_URL); + await persistence.clearStorage(); this.store.resetOnSignOut(); }; From 14101e89d3f21b867b1e7b8b2eb8c917ff5e7cfa Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Fri, 30 Aug 2024 15:21:40 +0530 Subject: [PATCH 057/111] Correct dummy data creation --- apiserver/plane/bgtasks/dummy_data_task.py | 30 +++++++++++++--------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/apiserver/plane/bgtasks/dummy_data_task.py b/apiserver/plane/bgtasks/dummy_data_task.py index 0d973b6dc8a..21ca32afb37 100644 --- a/apiserver/plane/bgtasks/dummy_data_task.py +++ b/apiserver/plane/bgtasks/dummy_data_task.py @@ -490,12 +490,15 @@ def create_issue_assignees(workspace, project, user_id, issue_count): def create_issue_labels(workspace, project, user_id, issue_count): # labels labels = Label.objects.filter(project=project).values_list("id", flat=True) - issues = random.sample( - list( + # issues = random.sample( + # list( + # Issue.objects.filter(project=project).values_list("id", flat=True) + # ), + # int(issue_count / 2), + # ) + issues = list( Issue.objects.filter(project=project).values_list("id", flat=True) - ), - int(issue_count / 2), - ) + ) shuffled_labels = list(labels) # Bulk issue @@ -503,7 +506,7 @@ def create_issue_labels(workspace, project, user_id, issue_count): for issue in issues: random.shuffle(shuffled_labels) for label in random.sample( - shuffled_labels, random.randint(0, 3) + shuffled_labels, random.randint(0, 5) ): bulk_issue_labels.append( IssueLabel( @@ -554,12 +557,15 @@ def create_module_issues(workspace, project, user_id, issue_count): modules = Module.objects.filter(project=project).values_list( "id", flat=True ) - issues = random.sample( - list( + # issues = random.sample( + # list( + # Issue.objects.filter(project=project).values_list("id", flat=True) + # ), + # int(issue_count / 2), + # ) + issues = list( Issue.objects.filter(project=project).values_list("id", flat=True) - ), - int(issue_count / 2), - ) + ) shuffled_modules = list(modules) @@ -568,7 +574,7 @@ def create_module_issues(workspace, project, user_id, issue_count): for issue in issues: random.shuffle(shuffled_modules) for module in random.sample( - shuffled_modules, random.randint(0, 3) + shuffled_modules, random.randint(0, 5) ): bulk_module_issues.append( ModuleIssue( From 02700ba3eef69c4ec501c2e6e7676d03a8167bcf Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Fri, 30 Aug 2024 16:13:41 +0530 Subject: [PATCH 058/111] Fix sort by assignee --- web/core/local-db/utils/query-constructor.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/web/core/local-db/utils/query-constructor.ts b/web/core/local-db/utils/query-constructor.ts index 18df6cb08b6..430309f6292 100644 --- a/web/core/local-db/utils/query-constructor.ts +++ b/web/core/local-db/utils/query-constructor.ts @@ -62,7 +62,11 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st const filterJoinFields = getMetaKeys(queries); if (order_by && Object.keys(SPECIAL_ORDER_BY).includes(order_by)) { const name = order_by.replace("-", ""); - sql = `SELECT ${fieldsFragment} , s.name as ${name} from issues i`; + if (order_by.includes("assignee")) { + sql = `SELECT ${fieldsFragment} , s.first_name as ${name} from issues i`; + } else { + sql = `SELECT ${fieldsFragment} , s.name as ${name} from issues i`; + } } else { sql = `SELECT ${fieldsFragment} from issues i`; } From 7d5ab1b3ce3f19eab02ae2ab6b0a2eaa55182a16 Mon Sep 17 00:00:00 2001 From: rahulramesha Date: Fri, 30 Aug 2024 17:04:09 +0530 Subject: [PATCH 059/111] sync to local changes --- web/core/services/issue/issue.service.ts | 16 +++++++++++++--- .../store/issue/issue-details/issue.store.ts | 2 -- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/web/core/services/issue/issue.service.ts b/web/core/services/issue/issue.service.ts index 151e6e20814..7e3978c4fce 100644 --- a/web/core/services/issue/issue.service.ts +++ b/web/core/services/issue/issue.service.ts @@ -13,7 +13,7 @@ import { API_BASE_URL } from "@/helpers/common.helper"; import { persistence } from "@/local-db/storage.sqlite"; // services -import { deleteIssueFromLocal } from "@/local-db/utils/load-issues"; +import { addIssue, addIssuesBulk, deleteIssueFromLocal, updateIssue } from "@/local-db/utils/load-issues"; import { updatePersistentLayer } from "@/local-db/utils/utils"; import { APIService } from "@/services/api.service"; @@ -104,7 +104,12 @@ export class IssueService extends APIService { return this.get(`/api/workspaces/${workspaceSlug}/projects/${projectId}/issues/${issueId}/`, { params: queries, }) - .then((response) => response?.data) + .then((response) => { + if (response.data) { + addIssue(response?.data); + } + return response?.data; + }) .catch((error) => { throw error?.response?.data; }); @@ -114,7 +119,12 @@ export class IssueService extends APIService { return this.get(`/api/workspaces/${workspaceSlug}/projects/${projectId}/issues/list/`, { params: { issues: issueIds.join(",") }, }) - .then((response) => response?.data) + .then((response) => { + if (response?.data && Array.isArray(response?.data)) { + addIssuesBulk(response.data); + } + return response?.data; + }) .catch((error) => { throw error?.response?.data; }); diff --git a/web/core/store/issue/issue-details/issue.store.ts b/web/core/store/issue/issue-details/issue.store.ts index a5a88cd22e8..1b676b8d378 100644 --- a/web/core/store/issue/issue-details/issue.store.ts +++ b/web/core/store/issue/issue-details/issue.store.ts @@ -93,8 +93,6 @@ export class IssueStore implements IIssueStore { if (!issue) throw new Error("Issue not found"); this.addIssueToStore(issue); - // adds issue to local store - updateIssue(issue); // store handlers from issue detail // parent From d430b6834a1c987800bebbaa517df5db808b382d Mon Sep 17 00:00:00 2001 From: NarayanBavisetti Date: Fri, 30 Aug 2024 17:19:06 +0530 Subject: [PATCH 060/111] chore: added archived issues in the deleted endpoint --- apiserver/plane/app/views/issue/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apiserver/plane/app/views/issue/base.py b/apiserver/plane/app/views/issue/base.py index 474e99bdab4..b70d4fdf81d 100644 --- a/apiserver/plane/app/views/issue/base.py +++ b/apiserver/plane/app/views/issue/base.py @@ -669,8 +669,8 @@ def get(self, request, slug, project_id): Issue.all_objects.filter( workspace__slug=slug, project_id=project_id, - deleted_at__isnull=False, ) + .filter(Q(archived_at__isnull=False) | Q(deleted_at__isnull=False)) .filter(**filters) .values_list("id", flat=True) ) From ae5dfe090e2b3064e272eccdd0952634b8e79ec0 Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Mon, 2 Sep 2024 10:47:08 +0530 Subject: [PATCH 061/111] Sync deletes to local db. --- web/core/local-db/storage.sqlite.ts | 13 +++++++------ web/core/local-db/utils/load-issues.ts | 4 ++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/web/core/local-db/storage.sqlite.ts b/web/core/local-db/storage.sqlite.ts index 9c0aedf7f66..b8bfed5baaa 100644 --- a/web/core/local-db/storage.sqlite.ts +++ b/web/core/local-db/storage.sqlite.ts @@ -8,14 +8,13 @@ import { IssueService } from "@/services/issue/issue.service"; // import { ARRAY_FIELDS } from "./utils/constants"; import createIndexes from "./utils/indexes"; -import { addIssuesBulk } from "./utils/load-issues"; +import { addIssuesBulk, syncDeletesToLocal } from "./utils/load-issues"; import { loadWorkSpaceData } from "./utils/load-workspace"; import { issueFilterCountQueryConstructor, issueFilterQueryConstructor } from "./utils/query-constructor"; import { runQuery } from "./utils/query-executor"; import { createTables } from "./utils/tables"; import { getGroupedIssueResults, getSubGroupedIssueResults } from "./utils/utils"; - declare module "@sqlite.org/sqlite-wasm" { export function sqlite3Worker1Promiser(...args: any): any; } @@ -200,6 +199,8 @@ export class Storage { } } + await syncDeletesToLocal(this.workspaceSlug, projectId, { updated_at__gt: syncedAt }); + console.log("### Time taken to add issues", performance.now() - start); if (status === "loading") { @@ -332,13 +333,13 @@ export const persistence = new Storage(); /** * format the issue fetched from local db into an issue - * @param issue - * @returns + * @param issue + * @returns */ -export const formatLocalIssue = (issue: any) => { +export const formatLocalIssue = (issue: any) => { const currIssue = issue; ARRAY_FIELDS.forEach((field: string) => { currIssue[field] = currIssue[field] ? JSON.parse(currIssue[field]) : []; }); return currIssue as TIssue; -} \ No newline at end of file +}; diff --git a/web/core/local-db/utils/load-issues.ts b/web/core/local-db/utils/load-issues.ts index 765cf54beda..3013efab81c 100644 --- a/web/core/local-db/utils/load-issues.ts +++ b/web/core/local-db/utils/load-issues.ts @@ -43,9 +43,9 @@ export const updateIssue = async (issue: TIssue) => { addIssue(issue); }; -export const syncDeletesToLocal = async (workspaceId: string, projectId: string) => { +export const syncDeletesToLocal = async (workspaceId: string, projectId: string, queries: any) => { const issueService = new IssueService(); - const response = await issueService.getDeletedIssues(workspaceId, projectId); + const response = await issueService.getDeletedIssues(workspaceId, projectId, queries); if (Array.isArray(response)) { response.map(async (issue) => deleteIssueFromLocal(issue)); } From d9a0ae20a516fb121c1c1c28be7a89a878264014 Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Mon, 2 Sep 2024 15:55:07 +0530 Subject: [PATCH 062/111] - Add missing indexes for tables used in sorting in spreadsheet layout. - Add options table --- web/core/local-db/storage.sqlite.ts | 24 ++++++++++++++++++++++-- web/core/local-db/utils/indexes.ts | 28 ++++++++++++++++++++++------ web/core/local-db/utils/schemas.ts | 5 +++++ web/core/local-db/utils/tables.ts | 2 ++ 4 files changed, 51 insertions(+), 8 deletions(-) diff --git a/web/core/local-db/storage.sqlite.ts b/web/core/local-db/storage.sqlite.ts index b8bfed5baaa..16cc0a5437e 100644 --- a/web/core/local-db/storage.sqlite.ts +++ b/web/core/local-db/storage.sqlite.ts @@ -199,8 +199,9 @@ export class Storage { } } - await syncDeletesToLocal(this.workspaceSlug, projectId, { updated_at__gt: syncedAt }); - + if (syncedAt) { + await syncDeletesToLocal(this.workspaceSlug, projectId, { updated_at__gt: syncedAt }); + } console.log("### Time taken to add issues", performance.now() - start); if (status === "loading") { @@ -327,6 +328,25 @@ export class Storage { setSync = (projectId: string, sync: Promise | undefined) => { set(this.projectStatus, `${projectId}.issues.sync`, sync); }; + + getOption = async (name: string, fallback: any) => { + const options = await runQuery(`select * from options where name='${name}'`); + if (options.length) { + return options[0].value; + } + return fallback; + }; + setOption = async (name: string, value: string) => { + await runQuery(`insert or replace into options (name, value) values ('${name}', '${value}')`); + }; + + getOptions = async (names: string[]) => { + const options = await runQuery(`select * from options where name in ('${names.join("','")}')`); + return options.reduce((acc: any, option: any) => { + acc[option.name] = option.value; + return acc; + }, {}); + }; } export const persistence = new Storage(); diff --git a/web/core/local-db/utils/indexes.ts b/web/core/local-db/utils/indexes.ts index 379c24e103f..aeff6992e9a 100644 --- a/web/core/local-db/utils/indexes.ts +++ b/web/core/local-db/utils/indexes.ts @@ -29,18 +29,34 @@ export const createIssueMetaIndexes = async () => { await persistence.db.exec({ sql: `CREATE INDEX issue_meta_all_idx ON issue_meta (issue_id,key,value)` }); }; -export const createLabelIndexes = async () => { - const columns = ["name", "id", "project_id"]; +export const createWorkSpaceIndexes = async () => { const promises: Promise[] = []; - columns.forEach((column) => { - promises.push(persistence.db.exec({ sql: `CREATE INDEX labels_${column}_idx ON labels (${column})` })); - }); + // Labels + promises.push(persistence.db.exec({ sql: `CREATE INDEX labels_name_idx ON labels (id,name,project_id)` })); + + // Modules + promises.push(persistence.db.exec({ sql: `CREATE INDEX modules_name_idx ON modules (id,name,project_id)` })); + + // States + promises.push(persistence.db.exec({ sql: `CREATE INDEX states_name_idx ON states (id,name,project_id)` })); + // Cycles + promises.push(persistence.db.exec({ sql: `CREATE INDEX cycles_name_idx ON cycles (id,name,project_id)` })); + + // Members + promises.push(persistence.db.exec({ sql: `CREATE INDEX members_name_idx ON members (id,first_name)` })); + + // Estimate Points @todo + promises.push(persistence.db.exec({ sql: `CREATE INDEX estimate_points_name_idx ON estimate_points (id,value)` })); + // Options + promises.push(persistence.db.exec({ sql: `CREATE INDEX options_name_idx ON options (name)` })); + await Promise.all(promises); }; + const createIndexes = async () => { log("### Creating indexes"); const start = performance.now(); - const promises = [createIssueIndexes(), createIssueMetaIndexes(), createLabelIndexes()]; + const promises = [createIssueIndexes(), createIssueMetaIndexes(), createWorkSpaceIndexes()]; try { await Promise.all(promises); } catch (e) { diff --git a/web/core/local-db/utils/schemas.ts b/web/core/local-db/utils/schemas.ts index 841034ee182..20501fbaf0d 100644 --- a/web/core/local-db/utils/schemas.ts +++ b/web/core/local-db/utils/schemas.ts @@ -128,3 +128,8 @@ export const memberSchema: Schema = { display_name: "TEXT", email: "TEXT", }; + +export const optionsSchema: Schema = { + key: "TEXT UNIQUE", + value: "TEXT", +}; diff --git a/web/core/local-db/utils/tables.ts b/web/core/local-db/utils/tables.ts index f8097d74699..d4ef132f4e1 100644 --- a/web/core/local-db/utils/tables.ts +++ b/web/core/local-db/utils/tables.ts @@ -9,6 +9,7 @@ import { cycleSchema, estimatePointSchema, memberSchema, + optionsSchema, } from "./schemas"; const createTableSQLfromSchema = (tableName: string, schema: Schema) => { @@ -32,6 +33,7 @@ export const createTables = async () => { persistence.db.exec(createTableSQLfromSchema("cycles", cycleSchema)); persistence.db.exec(createTableSQLfromSchema("estimate_points", estimatePointSchema)); persistence.db.exec(createTableSQLfromSchema("members", memberSchema)); + persistence.db.exec(createTableSQLfromSchema("options", optionsSchema)); persistence.db.exec("COMMIT;"); }; From d43c34c4d7d91a1a80799428a23fc0cb7f8bf84b Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Mon, 2 Sep 2024 15:57:04 +0530 Subject: [PATCH 063/111] Make fallback optional in getOption --- web/core/local-db/storage.sqlite.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/core/local-db/storage.sqlite.ts b/web/core/local-db/storage.sqlite.ts index 16cc0a5437e..c6314793552 100644 --- a/web/core/local-db/storage.sqlite.ts +++ b/web/core/local-db/storage.sqlite.ts @@ -329,7 +329,7 @@ export class Storage { set(this.projectStatus, `${projectId}.issues.sync`, sync); }; - getOption = async (name: string, fallback: any) => { + getOption = async (name: string, fallback = "") => { const options = await runQuery(`select * from options where name='${name}'`); if (options.length) { return options[0].value; From 45d8538dc20511d66e1656d4899059a42936012f Mon Sep 17 00:00:00 2001 From: rahulramesha Date: Mon, 2 Sep 2024 16:03:39 +0530 Subject: [PATCH 064/111] Kanban column virtualization --- .../components/core/render-if-visible-HOC.tsx | 29 ++++---- .../issues/issue-layouts/kanban/block.tsx | 2 +- .../issues/issue-layouts/kanban/default.tsx | 74 +++++++++++++------ .../components/issues/issue-layouts/utils.tsx | 46 ++++++++++++ .../loader/layouts/kanban-layout-loader.tsx | 48 ++++++++---- 5 files changed, 147 insertions(+), 52 deletions(-) diff --git a/web/core/components/core/render-if-visible-HOC.tsx b/web/core/components/core/render-if-visible-HOC.tsx index 529bbbbea59..226e6c235d5 100644 --- a/web/core/components/core/render-if-visible-HOC.tsx +++ b/web/core/components/core/render-if-visible-HOC.tsx @@ -10,8 +10,8 @@ type Props = { as?: keyof JSX.IntrinsicElements; classNames?: string; placeholderChildren?: ReactNode; - shouldRenderByDefault?: boolean; defaultValue?: boolean; + useIdletime?: boolean; }; const RenderIfVisible: React.FC = (props) => { @@ -22,12 +22,12 @@ const RenderIfVisible: React.FC = (props) => { horizontalOffset = 0, as = "div", children, - defaultValue = false, classNames = "", placeholderChildren = null, //placeholder children - shouldRenderByDefault = false, + defaultValue = false, + useIdletime = false, } = props; - const [shouldVisible, setShouldVisible] = useState(shouldRenderByDefault); + const [shouldVisible, setShouldVisible] = useState(defaultValue); const placeholderHeight = useRef(defaultHeight); const intersectionRef = useRef(null); @@ -39,14 +39,13 @@ const RenderIfVisible: React.FC = (props) => { const observer = new IntersectionObserver( (entries) => { //DO no remove comments for future - // if (typeof window !== undefined && window.requestIdleCallback) { - // window.requestIdleCallback(() => setShouldVisible(entries[0].isIntersecting), { - // timeout: 300, - // }); - // } else { - // setShouldVisible(entries[0].isIntersecting); - // } - setShouldVisible(entries[entries.length - 1].isIntersecting); + if (typeof window !== undefined && window.requestIdleCallback && useIdletime) { + window.requestIdleCallback(() => setShouldVisible(entries[entries.length - 1].isIntersecting), { + timeout: 300, + }); + } else { + setShouldVisible(entries[entries.length - 1].isIntersecting); + } }, { root: root?.current, @@ -71,8 +70,10 @@ const RenderIfVisible: React.FC = (props) => { }, [isVisible, intersectionRef]); const child = isVisible ? <>{children} : placeholderChildren; - const style = isVisible ? {} : { height: placeholderHeight.current, width: "100%" }; - const className = isVisible ? classNames : cn(classNames, "bg-custom-background-80"); + const style: { width?: string; height?: string } = isVisible + ? {} + : { height: placeholderHeight.current, width: "100%" }; + const className = isVisible || placeholderChildren ? classNames : cn(classNames, "bg-custom-background-80"); return React.createElement(as, { ref: intersectionRef, style, className }, child); }; diff --git a/web/core/components/issues/issue-layouts/kanban/block.tsx b/web/core/components/issues/issue-layouts/kanban/block.tsx index 5b146aa8cce..da050491285 100644 --- a/web/core/components/issues/issue-layouts/kanban/block.tsx +++ b/web/core/components/issues/issue-layouts/kanban/block.tsx @@ -226,7 +226,7 @@ export const KanbanIssueBlock: React.FC = observer((props) => { defaultHeight="100px" horizontalOffset={100} verticalOffset={200} - shouldRenderByDefault={shouldRenderByDefault} + defaultValue={shouldRenderByDefault} > = observer((props) => { }; const isGroupByCreatedBy = group_by === "created_by"; + const appxCardHeight = getApproximateKanbanCardHeight(displayProperties); + const isSubGroup = !!sub_group_id && sub_group_id !== "null"; return (
@@ -139,6 +145,11 @@ export const KanBan: React.FC = observer((props) => { list.length > 0 && list.map((subList: IGroupByColumn) => { const groupByVisibilityToggle = visibilityGroupBy(subList); + const issueIds = isSubGroup + ? (groupedIssueIds as TSubGroupedIssues)?.[subList.id]?.[sub_group_id] ?? [] + : (groupedIssueIds as TGroupedIssues)?.[subList.id] ?? []; + const issueLength = issueIds?.length as number; + const groupHeight = issueLength * appxCardHeight; if (groupByVisibilityToggle.showGroup === false) return <>; return ( @@ -167,28 +178,44 @@ export const KanBan: React.FC = observer((props) => { )} {groupByVisibilityToggle.showIssues && ( - + + } + useIdletime + > + + )}
); @@ -196,3 +223,4 @@ export const KanBan: React.FC = observer((props) => {
); }); + diff --git a/web/core/components/issues/issue-layouts/utils.tsx b/web/core/components/issues/issue-layouts/utils.tsx index ec333ef9230..b36f2a776d9 100644 --- a/web/core/components/issues/issue-layouts/utils.tsx +++ b/web/core/components/issues/issue-layouts/utils.tsx @@ -619,3 +619,49 @@ export const isIssueNew = (issue: TIssue) => { const diff = currentDate.getTime() - createdDate.getTime(); return diff < 30000; }; + +/** + * Get approximate height of card based on display Properties + * @param displayProperties + * @returns + */ +export function getApproximateKanbanCardHeight(displayProperties: IIssueDisplayProperties | undefined) { + if (!displayProperties) return 100; + + let defaultCardHeight = 46; + + const clonedProperties = clone(displayProperties); + + if (clonedProperties.key) { + defaultCardHeight += 24; + } + + const ignoredProperties: (keyof IIssueDisplayProperties)[] = [ + "key", + "sub_issue_count", + "link", + "attachment_count", + "created_on", + "updated_on", + ]; + + ignoredProperties.forEach((key: keyof IIssueDisplayProperties) => { + delete clonedProperties[key]; + }); + + let propertyCount = 0; + + (Object.keys(clonedProperties) as (keyof IIssueDisplayProperties)[]).forEach((key: keyof IIssueDisplayProperties) => { + if (clonedProperties[key]) { + propertyCount++; + } + }); + + if (propertyCount > 3) { + defaultCardHeight += 60; + } else if (propertyCount > 0) { + defaultCardHeight += 32; + } + + return defaultCardHeight; +} diff --git a/web/core/components/ui/loader/layouts/kanban-layout-loader.tsx b/web/core/components/ui/loader/layouts/kanban-layout-loader.tsx index 817a708768b..0bb1f96eb49 100644 --- a/web/core/components/ui/loader/layouts/kanban-layout-loader.tsx +++ b/web/core/components/ui/loader/layouts/kanban-layout-loader.tsx @@ -1,25 +1,45 @@ import { forwardRef } from "react"; -export const KanbanIssueBlockLoader = forwardRef((props, ref) => ( - -)); +export const KanbanIssueBlockLoader = forwardRef( + ({ cardHeight = 100 }, ref) => ( + + ) +); + +export const KanbanColumnLoader = ({ + cardsInColumn = 3, + ignoreHeader = false, + cardHeight = 100, +}: { + cardsInColumn?: number; + ignoreHeader?: boolean; + cardHeight?: number; +}) => ( +
+ {!ignoreHeader && ( +
+
+ + +
+
+ )} + {Array.from({ length: cardsInColumn }, (_, cardIndex) => ( + + ))} +
+); KanbanIssueBlockLoader.displayName = "KanbanIssueBlockLoader"; export const KanbanLayoutLoader = ({ cardsInEachColumn = [2, 3, 2, 4, 3] }: { cardsInEachColumn?: number[] }) => (
{cardsInEachColumn.map((cardsInColumn, columnIndex) => ( -
-
-
- - -
-
- {Array.from({ length: cardsInColumn }, (_, cardIndex) => ( - - ))} -
+ ))}
); From c63ea986fa142d8edecd5c4832bfc3ea01da902b Mon Sep 17 00:00:00 2001 From: rahulramesha Date: Mon, 2 Sep 2024 18:07:00 +0530 Subject: [PATCH 065/111] persist project sync readiness to sqlite and use that as the source of truth for the project issues to be ready --- web/core/local-db/storage.sqlite.ts | 51 +++++++++++++++++++---------- 1 file changed, 34 insertions(+), 17 deletions(-) diff --git a/web/core/local-db/storage.sqlite.ts b/web/core/local-db/storage.sqlite.ts index c6314793552..fbfe43d775f 100644 --- a/web/core/local-db/storage.sqlite.ts +++ b/web/core/local-db/storage.sqlite.ts @@ -146,9 +146,14 @@ export class Storage { this.syncIssues(projectId); }; - syncIssues = (projectId: string) => { - const sync = this._syncIssues(projectId); - this.setSync(projectId, sync); + syncIssues = async (projectId: string) => { + try { + const sync = this._syncIssues(projectId); + this.setSync(projectId, sync); + await sync; + } catch (e) { + this.setStatus(projectId, "error"); + } }; _syncIssues = async (projectId: string) => { @@ -171,15 +176,16 @@ export class Storage { }; const syncedAt = await this.getLastSyncTime(projectId); + const projectSync = await this.getOption(projectId); if (syncedAt) { queryParams["updated_at__gte"] = syncedAt; } - this.setStatus(projectId, syncedAt ? "syncing" : "loading"); + this.setStatus(projectId, projectSync === "ready" ? "syncing" : "loading"); status = this.getStatus(projectId); - log(`### ${syncedAt ? "Syncing" : "Loading"} issues to local db for project ${projectId}`); + log(`### ${projectSync === "ready" ? "Syncing" : "Loading"} issues to local db for project ${projectId}`); const start = performance.now(); const issueService = new IssueService(); @@ -212,6 +218,7 @@ export class Storage { type: TOAST_TYPE.SUCCESS, }); } + this.setOption(projectId, "ready"); this.setStatus(projectId, "ready"); this.setSync(projectId, undefined); }; @@ -243,12 +250,17 @@ export class Storage { getIssues = async (projectId: string, queries: any, config: any) => { console.log("#### Queries", queries); - if (this.getStatus(projectId) === "loading" || (window as any).DISABLE_LOCAL) { + const currentProjectStatus = this.getStatus(projectId); + if ( + !currentProjectStatus || + currentProjectStatus === "loading" || + currentProjectStatus === "error" || + (window as any).DISABLE_LOCAL + ) { info(`Project ${projectId} is loading, falling back to server`); const issueService = new IssueService(); return await issueService.getIssuesFromServer(this.workspaceSlug, projectId, queries); } - await this.getSync(projectId); const { cursor, group_by, sub_group_by } = queries; @@ -329,21 +341,26 @@ export class Storage { set(this.projectStatus, `${projectId}.issues.sync`, sync); }; - getOption = async (name: string, fallback = "") => { - const options = await runQuery(`select * from options where name='${name}'`); - if (options.length) { - return options[0].value; + getOption = async (key: string, fallback = "") => { + try { + const options = await runQuery(`select * from options where key='${key}'`); + if (options.length) { + return options[0].value; + } + + return fallback; + } catch (e) { + return fallback; } - return fallback; }; - setOption = async (name: string, value: string) => { - await runQuery(`insert or replace into options (name, value) values ('${name}', '${value}')`); + setOption = async (key: string, value: string) => { + await runQuery(`insert or replace into options (key, value) values ('${key}', '${value}')`); }; - getOptions = async (names: string[]) => { - const options = await runQuery(`select * from options where name in ('${names.join("','")}')`); + getOptions = async (keys: string[]) => { + const options = await runQuery(`select * from options where key in ('${keys.join("','")}')`); return options.reduce((acc: any, option: any) => { - acc[option.name] = option.value; + acc[option.key] = option.value; return acc; }, {}); }; From d93a1d412dd54362df52b8a8f4a72394487767a7 Mon Sep 17 00:00:00 2001 From: rahulramesha Date: Mon, 2 Sep 2024 18:23:35 +0530 Subject: [PATCH 066/111] fix build errors --- web/core/local-db/utils/load-workspace.ts | 2 +- web/core/local-db/utils/utils.ts | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/web/core/local-db/utils/load-workspace.ts b/web/core/local-db/utils/load-workspace.ts index ac72fa927e9..8840ef83c9f 100644 --- a/web/core/local-db/utils/load-workspace.ts +++ b/web/core/local-db/utils/load-workspace.ts @@ -9,7 +9,7 @@ import { WorkspaceService } from "@/services/workspace.service"; import { persistence } from "../storage.sqlite"; import { cycleSchema, estimatePointSchema, labelSchema, memberSchema, Schema, stateSchema } from "./schemas"; -const stageInserts = (table: string, schema: Schema, data: object) => { +const stageInserts = (table: string, schema: Schema, data: any) => { const keys = Object.keys(schema); // Pick only the keys that are in the schema const filteredData = keys.reduce((acc: any, key) => { diff --git a/web/core/local-db/utils/utils.ts b/web/core/local-db/utils/utils.ts index e4724ab78e8..0fb20f2999d 100644 --- a/web/core/local-db/utils/utils.ts +++ b/web/core/local-db/utils/utils.ts @@ -42,6 +42,7 @@ export const updatePersistentLayer = async (issueIds: string | string[]) => { "assignee_ids", "label_ids", "module_ids", + "type_id", ]); updateIssue(issuePartial); } From 1263650eab2ba6f6e3472f895421b84cae51087c Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Tue, 3 Sep 2024 13:21:06 +0530 Subject: [PATCH 067/111] Fix calendar view --- web/core/local-db/utils/constants.ts | 1 + web/core/local-db/utils/query.utils.ts | 22 +++++++++++++++++-- .../store/issue/issue-details/issue.store.ts | 2 +- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/web/core/local-db/utils/constants.ts b/web/core/local-db/utils/constants.ts index 5e3dcfe8943..cb61fbdf44d 100644 --- a/web/core/local-db/utils/constants.ts +++ b/web/core/local-db/utils/constants.ts @@ -9,6 +9,7 @@ export const GROUP_BY_MAP = { issue_module__module_id: "module_ids", labels__id: "label_ids", assignees__id: "assignee_ids", + target_date: "target_date", }; export const PRIORITY_MAP = { diff --git a/web/core/local-db/utils/query.utils.ts b/web/core/local-db/utils/query.utils.ts index dfae1084fd1..f7c48d98458 100644 --- a/web/core/local-db/utils/query.utils.ts +++ b/web/core/local-db/utils/query.utils.ts @@ -1,3 +1,4 @@ +import { start } from "nprogress"; import { ARRAY_FIELDS, GROUP_BY_MAP, PRIORITY_MAP } from "./constants"; import { issueSchema } from "./schemas"; import { wrapDateTime } from "./utils"; @@ -126,7 +127,11 @@ export const getFilteredRowsForGrouping = (projectId: string, queries: any) => { if (!joinsRequired) { sql = `WITH fi as (SELECT i.id,i.created_at ${issueTableFilterFields}`; if (group_by) { - sql += `, i.${group_by} as group_id`; + if (group_by === "target_date") { + sql += `, date(i.${group_by}) as group_id`; + } else { + sql += `, i.${group_by} as group_id`; + } } if (sub_group_by) { sql += `, i.${sub_group_by} as sub_group_id`; @@ -140,6 +145,8 @@ export const getFilteredRowsForGrouping = (projectId: string, queries: any) => { if (group_by) { if (ARRAY_FIELDS.includes(group_by)) { sql += `, ${group_by}.value as group_id`; + } else if (group_by === "target_date") { + sql += `, date(i.${group_by}) as group_id`; } else { sql += `, i.${group_by} as group_id`; } @@ -178,12 +185,21 @@ export const getFilteredRowsForGrouping = (projectId: string, queries: any) => { }; export const singleFilterConstructor = (queries: any) => { - const { order_by, cursor, per_page, group_by, sub_group_by, sub_issue, ...filters } = translateQueryParams(queries); + const { order_by, cursor, per_page, group_by, sub_group_by, sub_issue, target_date, ...filters } = + translateQueryParams(queries); let sql = ""; if (!sub_issue) { sql += ` AND parent_id IS NOT NULL `; } + if (target_date) { + // "2024-09-29;after,2024-11-02;before" + const dates = target_date.split(","); + const start = dates[0].split(";")[0]; + const end = dates[1].split(";")[0]; + + sql += ` AND target_date >= date('${start}') AND target_date <= date('${end}') `; + } const keys = Object.keys(filters); keys.forEach((key) => { @@ -193,6 +209,8 @@ export const singleFilterConstructor = (queries: any) => { sql += ` AND ${key} in ('${value.join("','")}')`; } }); + // + return sql; }; diff --git a/web/core/store/issue/issue-details/issue.store.ts b/web/core/store/issue/issue-details/issue.store.ts index 87d7a88d24b..3c4b4794f81 100644 --- a/web/core/store/issue/issue-details/issue.store.ts +++ b/web/core/store/issue/issue-details/issue.store.ts @@ -97,7 +97,7 @@ export class IssueStore implements IIssueStore { else issue = await this.issueService.retrieve(workspaceSlug, projectId, issueId, query); if (!issue) throw new Error("Issue not found"); - + this.addIssueToStore(issue); const issuePayload: TIssue = { id: issue?.id, sequence_id: issue?.sequence_id, From 7ee6fea4216bd061ba9fc34022a17c2b55892c08 Mon Sep 17 00:00:00 2001 From: rahulramesha Date: Tue, 3 Sep 2024 16:27:20 +0530 Subject: [PATCH 068/111] fetch slimed down version of modules in project wrapper --- .../layouts/auth-layout/project-wrapper.tsx | 21 +++++++-------- web/core/store/module.store.ts | 27 +++++++++++++++++++ 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/web/core/layouts/auth-layout/project-wrapper.tsx b/web/core/layouts/auth-layout/project-wrapper.tsx index b8c1488dc2c..662a2f12ccd 100644 --- a/web/core/layouts/auth-layout/project-wrapper.tsx +++ b/web/core/layouts/auth-layout/project-wrapper.tsx @@ -42,7 +42,7 @@ export const ProjectAuthWrapper: FC = observer((props) => { } = useUser(); const { loader, getProjectById, fetchProjectDetails } = useProject(); const { fetchAllCycles } = useCycle(); - const { fetchModules } = useModule(); + const { fetchModulesSlim } = useModule(); const { fetchViews } = useProjectView(); const { project: { fetchProjectMembers }, @@ -79,39 +79,39 @@ export const ProjectAuthWrapper: FC = observer((props) => { workspaceSlug && projectId ? () => fetchUserProjectInfo(workspaceSlug.toString(), projectId.toString()) : null ); // fetching project labels - const { isLoading: isLabelsLoading } = useSWR( + useSWR( workspaceSlug && projectId ? `PROJECT_LABELS_${workspaceSlug}_${projectId}` : null, workspaceSlug && projectId ? () => fetchProjectLabels(workspaceSlug.toString(), projectId.toString()) : null, { revalidateIfStale: false, revalidateOnFocus: false } ); // fetching project members - const { isLoading: isMembersLoading } = useSWR( + useSWR( workspaceSlug && projectId ? `PROJECT_MEMBERS_${workspaceSlug}_${projectId}` : null, workspaceSlug && projectId ? () => fetchProjectMembers(workspaceSlug.toString(), projectId.toString()) : null, { revalidateIfStale: false, revalidateOnFocus: false } ); // fetching project states - const { isLoading: isStateLoading } = useSWR( + useSWR( workspaceSlug && projectId ? `PROJECT_STATES_${workspaceSlug}_${projectId}` : null, workspaceSlug && projectId ? () => fetchProjectStates(workspaceSlug.toString(), projectId.toString()) : null, { revalidateIfStale: false, revalidateOnFocus: false } ); // fetching project estimates - const { isLoading: isEstimatesLoading } = useSWR( + useSWR( workspaceSlug && projectId ? `PROJECT_ESTIMATES_${workspaceSlug}_${projectId}` : null, workspaceSlug && projectId ? () => getProjectEstimates(workspaceSlug.toString(), projectId.toString()) : null, { revalidateIfStale: false, revalidateOnFocus: false } ); // fetching project cycles - const { isLoading: isCyclesLoading } = useSWR( + useSWR( workspaceSlug && projectId ? `PROJECT_ALL_CYCLES_${workspaceSlug}_${projectId}` : null, workspaceSlug && projectId ? () => fetchAllCycles(workspaceSlug.toString(), projectId.toString()) : null, { revalidateIfStale: false, revalidateOnFocus: false } ); // fetching project modules - const { isLoading: isModulesLoading } = useSWR( + useSWR( workspaceSlug && projectId ? `PROJECT_MODULES_${workspaceSlug}_${projectId}` : null, - workspaceSlug && projectId ? () => fetchModules(workspaceSlug.toString(), projectId.toString()) : null, + workspaceSlug && projectId ? () => fetchModulesSlim(workspaceSlug.toString(), projectId.toString()) : null, { revalidateIfStale: false, revalidateOnFocus: false } ); // fetching project views @@ -122,11 +122,8 @@ export const ProjectAuthWrapper: FC = observer((props) => { ); const projectExists = projectId ? getProjectById(projectId.toString()) : null; - const isLoading = - isLabelsLoading || isMembersLoading || isStateLoading || isEstimatesLoading || isCyclesLoading || isModulesLoading; - // check if the project member apis is loading - if ((!projectMemberInfo && projectId && hasPermissionToProject[projectId.toString()] === null) || isLoading) + if (!projectMemberInfo && projectId && hasPermissionToProject[projectId.toString()] === null) return (
diff --git a/web/core/store/module.store.ts b/web/core/store/module.store.ts index c3401dfbe92..f7a99a5c431 100644 --- a/web/core/store/module.store.ts +++ b/web/core/store/module.store.ts @@ -39,6 +39,7 @@ export interface IModuleStore { updateModuleDistribution: (distributionUpdates: DistributionUpdates, moduleId: string) => void; fetchWorkspaceModules: (workspaceSlug: string) => Promise; fetchModules: (workspaceSlug: string, projectId: string) => Promise; + fetchModulesSlim: (workspaceSlug: string, projectId: string) => Promise fetchArchivedModules: (workspaceSlug: string, projectId: string) => Promise; fetchArchivedModuleDetails: (workspaceSlug: string, projectId: string, moduleId: string) => Promise; fetchModuleDetails: (workspaceSlug: string, projectId: string, moduleId: string) => Promise; @@ -281,6 +282,32 @@ export class ModulesStore implements IModuleStore { } }; + /** + * @description fetch all modules + * @param workspaceSlug + * @param projectId + * @returns IModule[] + */ + fetchModulesSlim = async (workspaceSlug: string, projectId: string) => { + try { + this.loader = true; + await this.moduleService.getWorkspaceModules(workspaceSlug).then((response) => { + const projectModules = response.filter((module) => module.project_id === projectId); + runInAction(() => { + projectModules.forEach((module) => { + set(this.moduleMap, [module.id], { ...this.moduleMap[module.id], ...module }); + }); + set(this.fetchedMap, projectId, true); + this.loader = false; + }); + return projectModules; + }); + } catch (error) { + this.loader = false; + return undefined; + } + }; + /** * @description fetch all archived modules * @param workspaceSlug From 42fb1d76d1b41b58b26a65c61dec4554d5f82f4b Mon Sep 17 00:00:00 2001 From: rahulramesha Date: Tue, 3 Sep 2024 16:41:22 +0530 Subject: [PATCH 069/111] fetch toned down modules and then fetch complete modules --- web/core/layouts/auth-layout/project-wrapper.tsx | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/web/core/layouts/auth-layout/project-wrapper.tsx b/web/core/layouts/auth-layout/project-wrapper.tsx index 662a2f12ccd..6174fec630e 100644 --- a/web/core/layouts/auth-layout/project-wrapper.tsx +++ b/web/core/layouts/auth-layout/project-wrapper.tsx @@ -42,7 +42,7 @@ export const ProjectAuthWrapper: FC = observer((props) => { } = useUser(); const { loader, getProjectById, fetchProjectDetails } = useProject(); const { fetchAllCycles } = useCycle(); - const { fetchModulesSlim } = useModule(); + const { fetchModulesSlim, fetchModules } = useModule(); const { fetchViews } = useProjectView(); const { project: { fetchProjectMembers }, @@ -111,7 +111,12 @@ export const ProjectAuthWrapper: FC = observer((props) => { // fetching project modules useSWR( workspaceSlug && projectId ? `PROJECT_MODULES_${workspaceSlug}_${projectId}` : null, - workspaceSlug && projectId ? () => fetchModulesSlim(workspaceSlug.toString(), projectId.toString()) : null, + workspaceSlug && projectId + ? async () => { + await fetchModulesSlim(workspaceSlug.toString(), projectId.toString()); + await fetchModules(workspaceSlug.toString(), projectId.toString()); + } + : null, { revalidateIfStale: false, revalidateOnFocus: false } ); // fetching project views From 89ae8465086cda04d0ba91c6c1a2db7fe022d313 Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Tue, 3 Sep 2024 23:12:21 +0530 Subject: [PATCH 070/111] Fix multi value order by in spread sheet layout --- web/core/local-db/utils/query-constructor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/core/local-db/utils/query-constructor.ts b/web/core/local-db/utils/query-constructor.ts index 430309f6292..871248c9f3c 100644 --- a/web/core/local-db/utils/query-constructor.ts +++ b/web/core/local-db/utils/query-constructor.ts @@ -65,7 +65,7 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st if (order_by.includes("assignee")) { sql = `SELECT ${fieldsFragment} , s.first_name as ${name} from issues i`; } else { - sql = `SELECT ${fieldsFragment} , s.name as ${name} from issues i`; + sql = `SELECT ${fieldsFragment} , group_concat(s.name) as ${name} from issues i`; } } else { sql = `SELECT ${fieldsFragment} from issues i`; From 5bc359d19bc8b9a7464c65317e43a5cc2634b9a8 Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Wed, 4 Sep 2024 00:07:03 +0530 Subject: [PATCH 071/111] Fix sort by --- web/core/local-db/utils/query.utils.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/core/local-db/utils/query.utils.ts b/web/core/local-db/utils/query.utils.ts index f7c48d98458..453aba464d6 100644 --- a/web/core/local-db/utils/query.utils.ts +++ b/web/core/local-db/utils/query.utils.ts @@ -40,9 +40,9 @@ export const getOrderByFragment = (order_by: string) => { if (!order_by) return orderByString; if (order_by.startsWith("-")) { - orderByString += ` ORDER BY ${wrapDateTime(order_by.slice(1))} DESC NULLS LAST, created_at DESC`; + orderByString += ` ORDER BY ${wrapDateTime(order_by.slice(1))} DESC NULLS LAST, i.created_at DESC`; } else { - orderByString += ` ORDER BY ${wrapDateTime(order_by)} ASC NULLS LAST, created_at DESC`; + orderByString += ` ORDER BY ${wrapDateTime(order_by)} ASC NULLS LAST, i.created_at DESC`; } return orderByString; }; From 89f43ad5678d8e166cf2a8ec2e0fde7f61ea03fb Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Wed, 4 Sep 2024 11:56:07 +0530 Subject: [PATCH 072/111] Fix the query when ordering by multi field names --- web/core/local-db/utils/query-constructor.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/web/core/local-db/utils/query-constructor.ts b/web/core/local-db/utils/query-constructor.ts index 871248c9f3c..1d21f67cbcc 100644 --- a/web/core/local-db/utils/query-constructor.ts +++ b/web/core/local-db/utils/query-constructor.ts @@ -92,18 +92,18 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st if (order_by.includes("label")) { sql += ` LEFT JOIN issue_meta sm ON i.id = sm.issue_id AND sm.key = 'label_ids' - INNER JOIN labels s ON s.id = sm.value`; + LEFT JOIN labels s ON s.id = sm.value`; } if (order_by.includes("module")) { sql += ` LEFT JOIN issue_meta sm ON i.id = sm.issue_id AND sm.key = 'module_ids' - INNER JOIN modules s ON s.id = sm.value`; + LEFT JOIN modules s ON s.id = sm.value`; } if (order_by.includes("assignee")) { sql += ` LEFT JOIN issue_meta sm ON i.id = sm.issue_id AND sm.key = 'assignee_ids' - INNER JOIN members s ON s.id = sm.value`; + LEFT JOIN members s ON s.id = sm.value`; } } sql += ` WHERE i.project_id = '${projectId}' ${singleFilterConstructor(otherProps)} group by i.id `; From 2c4c2411ee797283de073901fc69a3e716ffa1e6 Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Wed, 4 Sep 2024 14:12:27 +0530 Subject: [PATCH 073/111] Remove unused import --- web/core/local-db/utils/query.utils.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/web/core/local-db/utils/query.utils.ts b/web/core/local-db/utils/query.utils.ts index 453aba464d6..057c92cbcfc 100644 --- a/web/core/local-db/utils/query.utils.ts +++ b/web/core/local-db/utils/query.utils.ts @@ -1,4 +1,3 @@ -import { start } from "nprogress"; import { ARRAY_FIELDS, GROUP_BY_MAP, PRIORITY_MAP } from "./constants"; import { issueSchema } from "./schemas"; import { wrapDateTime } from "./utils"; From 7e66115e85c7ae21addbb9b451a90c5538ef962b Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Wed, 4 Sep 2024 15:56:07 +0530 Subject: [PATCH 074/111] Fix sort by multi value fields --- web/core/local-db/utils/query-constructor.ts | 84 ++++++++++++-------- web/core/local-db/utils/query.utils.ts | 22 ++++- 2 files changed, 69 insertions(+), 37 deletions(-) diff --git a/web/core/local-db/utils/query-constructor.ts b/web/core/local-db/utils/query-constructor.ts index 1d21f67cbcc..629f0c51172 100644 --- a/web/core/local-db/utils/query-constructor.ts +++ b/web/core/local-db/utils/query-constructor.ts @@ -59,53 +59,67 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st return sql; } - const filterJoinFields = getMetaKeys(queries); if (order_by && Object.keys(SPECIAL_ORDER_BY).includes(order_by)) { const name = order_by.replace("-", ""); + + sql = `WITH sorted_issues AS (`; + sql += getFilteredRowsForGrouping(projectId, queries); + sql += `SELECT fi.* , `; if (order_by.includes("assignee")) { - sql = `SELECT ${fieldsFragment} , s.first_name as ${name} from issues i`; + sql += ` s.first_name as ${name} `; } else { - sql = `SELECT ${fieldsFragment} , group_concat(s.name) as ${name} from issues i`; + sql += ` s.name as ${name} `; + } + sql += `FROM fi `; + if (order_by && Object.keys(SPECIAL_ORDER_BY).includes(order_by)) { + if (order_by.includes("cycle")) { + sql += ` + LEFT JOIN cycles s on fi.cycle_id = s.id`; + } + if (order_by.includes("estimate_point")) { + sql += ` + LEFT JOIN estimate_points s on fi.estimate_point = s.id`; + } + if (order_by.includes("state")) { + sql += ` + LEFT JOIN states s on fi.state_id = s.id`; + } + if (order_by.includes("label")) { + sql += ` + LEFT JOIN issue_meta sm ON fi.id = sm.issue_id AND sm.key = 'label_ids' + LEFT JOIN labels s ON s.id = sm.value`; + } + if (order_by.includes("module")) { + sql += ` + LEFT JOIN issue_meta sm ON fi.id = sm.issue_id AND sm.key = 'module_ids' + LEFT JOIN modules s ON s.id = sm.value`; + } + + if (order_by.includes("assignee")) { + sql += ` + LEFT JOIN issue_meta sm ON fi.id = sm.issue_id AND sm.key = 'assignee_ids' + LEFT JOIN members s ON s.id = sm.value`; + } + + sql += ` ORDER BY ${name} ASC NULLS LAST`; } - } else { - sql = `SELECT ${fieldsFragment} from issues i`; + sql += `)`; + + sql += `SELECT ${fieldsFragment}, group_concat(si.${name}) as ${name} from sorted_issues si JOIN issues i ON si.id = i.id group by i.id ${orderByString} LIMIT ${pageSize} OFFSET ${offset * 1 + page * pageSize};`; + + console.log("######$$$", sql); + return sql; } + const filterJoinFields = getMetaKeys(queries); + + sql = `SELECT ${fieldsFragment} from issues i`; + filterJoinFields.forEach((field: string) => { const value = otherProps[field] || ""; sql += ` INNER JOIN issue_meta ${field} ON i.id = ${field}.issue_id AND ${field}.key = '${field}' AND ${field}.value IN ('${value.split(",").join("','")}') `; }); - if (order_by && Object.keys(SPECIAL_ORDER_BY).includes(order_by)) { - if (order_by.includes("cycle")) { - sql += ` - LEFT JOIN cycles s on i.cycle_id = s.id`; - } - if (order_by.includes("estimate_point")) { - sql += ` - LEFT JOIN estimate_points s on i.estimate_point = s.id`; - } - if (order_by.includes("state")) { - sql += ` - LEFT JOIN states s on i.state_id = s.id`; - } - if (order_by.includes("label")) { - sql += ` - LEFT JOIN issue_meta sm ON i.id = sm.issue_id AND sm.key = 'label_ids' - LEFT JOIN labels s ON s.id = sm.value`; - } - if (order_by.includes("module")) { - sql += ` - LEFT JOIN issue_meta sm ON i.id = sm.issue_id AND sm.key = 'module_ids' - LEFT JOIN modules s ON s.id = sm.value`; - } - - if (order_by.includes("assignee")) { - sql += ` - LEFT JOIN issue_meta sm ON i.id = sm.issue_id AND sm.key = 'assignee_ids' - LEFT JOIN members s ON s.id = sm.value`; - } - } sql += ` WHERE i.project_id = '${projectId}' ${singleFilterConstructor(otherProps)} group by i.id `; sql += orderByString; diff --git a/web/core/local-db/utils/query.utils.ts b/web/core/local-db/utils/query.utils.ts index 057c92cbcfc..18ee2de930f 100644 --- a/web/core/local-db/utils/query.utils.ts +++ b/web/core/local-db/utils/query.utils.ts @@ -1,4 +1,5 @@ import { ARRAY_FIELDS, GROUP_BY_MAP, PRIORITY_MAP } from "./constants"; +import { SPECIAL_ORDER_BY } from "./query-constructor"; import { issueSchema } from "./schemas"; import { wrapDateTime } from "./utils"; @@ -218,7 +219,9 @@ const getSingleFilterFields = (queries: any) => { translateQueryParams(queries); const fields = new Set(); - if (order_by && !order_by.includes("created_at")) fields.add(order_by.replace("-", "")); + + if (order_by && !order_by.includes("created_at") && !Object.keys(SPECIAL_ORDER_BY).includes(order_by)) + fields.add(order_by.replace("-", "")); const keys = Object.keys(otherProps); @@ -228,12 +231,27 @@ const getSingleFilterFields = (queries: any) => { } }); + if (order_by.includes("state__name")) { + fields.add("state_id"); + } + if (order_by.includes("cycle__name")) { + fields.add("cycle_id"); + } + return Array.from(fields); }; export const getIssueFieldsFragment = () => { const { description_html, ...filtered } = issueSchema; const keys = Object.keys(filtered); - const sql = ` ${keys.map((key) => `i.${key}`).join(",")}`; + const sql = ` ${keys + .map((key, index) => { + if (index % 5 === 0) { + return `i.${key} + `; + } + return `i.${key}`; + }) + .join(",")}`; return sql; }; From e63bcc4a9ad3e7722fafc5aaef643c0eec981d4b Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Wed, 4 Sep 2024 17:39:11 +0530 Subject: [PATCH 075/111] Format queries and fix order by --- web/core/local-db/utils/query-constructor.ts | 10 +++- web/core/local-db/utils/query.utils.ts | 49 +++++++++++--------- 2 files changed, 35 insertions(+), 24 deletions(-) diff --git a/web/core/local-db/utils/query-constructor.ts b/web/core/local-db/utils/query-constructor.ts index 629f0c51172..aa782e9f28b 100644 --- a/web/core/local-db/utils/query-constructor.ts +++ b/web/core/local-db/utils/query-constructor.ts @@ -20,7 +20,6 @@ export const SPECIAL_ORDER_BY = { }; export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: string, queries: any) => { const { order_by, cursor, per_page, group_by, sub_group_by, ...otherProps } = translateQueryParams(queries); - const orderByString = getOrderByFragment(order_by); const [pageSize, page, offset] = cursor.split(":"); let sql = ""; @@ -28,6 +27,7 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st const fieldsFragment = getIssueFieldsFragment(); if (sub_group_by) { + const orderByString = getOrderByFragment(order_by); sql = getFilteredRowsForGrouping(projectId, queries); sql += `, ranked_issues AS ( SELECT fi.*, ROW_NUMBER() OVER (PARTITION BY group_id, sub_group_id ${orderByString}) as rank, @@ -44,6 +44,7 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st return sql; } if (group_by) { + const orderByString = getOrderByFragment(order_by); sql = getFilteredRowsForGrouping(projectId, queries); sql += `, ranked_issues AS ( SELECT fi.*, ROW_NUMBER() OVER (PARTITION BY group_id ${orderByString}) as rank, @@ -61,6 +62,7 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st if (order_by && Object.keys(SPECIAL_ORDER_BY).includes(order_by)) { const name = order_by.replace("-", ""); + const orderByString = getOrderByFragment(order_by, "i."); sql = `WITH sorted_issues AS (`; sql += getFilteredRowsForGrouping(projectId, queries); @@ -105,12 +107,16 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st } sql += `)`; - sql += `SELECT ${fieldsFragment}, group_concat(si.${name}) as ${name} from sorted_issues si JOIN issues i ON si.id = i.id group by i.id ${orderByString} LIMIT ${pageSize} OFFSET ${offset * 1 + page * pageSize};`; + sql += `SELECT ${fieldsFragment}, group_concat(si.${name}) as ${name} from sorted_issues si JOIN issues i ON si.id = i.id + `; + sql += ` group by i.id ${orderByString} LIMIT ${pageSize} OFFSET ${offset * 1 + page * pageSize};`; console.log("######$$$", sql); return sql; } + const filterJoinFields = getMetaKeys(queries); + const orderByString = getOrderByFragment(order_by); sql = `SELECT ${fieldsFragment} from issues i`; diff --git a/web/core/local-db/utils/query.utils.ts b/web/core/local-db/utils/query.utils.ts index 18ee2de930f..b0dd3c63839 100644 --- a/web/core/local-db/utils/query.utils.ts +++ b/web/core/local-db/utils/query.utils.ts @@ -35,14 +35,14 @@ export const translateQueryParams = (queries: any) => { return otherProps; }; -export const getOrderByFragment = (order_by: string) => { +export const getOrderByFragment = (order_by: string, table = "") => { let orderByString = ""; if (!order_by) return orderByString; if (order_by.startsWith("-")) { - orderByString += ` ORDER BY ${wrapDateTime(order_by.slice(1))} DESC NULLS LAST, i.created_at DESC`; + orderByString += ` ORDER BY ${wrapDateTime(order_by.slice(1))} DESC NULLS LAST, ${table}created_at DESC`; } else { - orderByString += ` ORDER BY ${wrapDateTime(order_by)} ASC NULLS LAST, i.created_at DESC`; + orderByString += ` ORDER BY ${wrapDateTime(order_by)} ASC NULLS LAST, ${table}created_at DESC`; } return orderByString; }; @@ -136,7 +136,10 @@ export const getFilteredRowsForGrouping = (projectId: string, queries: any) => { if (sub_group_by) { sql += `, i.${sub_group_by} as sub_group_id`; } - sql += ` FROM issues i WHERE project_id = '${projectId}' ${singleFilterConstructor(otherProps)}) `; + sql += ` FROM issues i WHERE project_id = '${projectId}' + `; + sql += `${singleFilterConstructor(otherProps)}) + `; return sql; } @@ -144,18 +147,23 @@ export const getFilteredRowsForGrouping = (projectId: string, queries: any) => { sql += `SELECT i.id,i.created_at ${issueTableFilterFields} `; if (group_by) { if (ARRAY_FIELDS.includes(group_by)) { - sql += `, ${group_by}.value as group_id`; + sql += `, ${group_by}.value as group_id + `; } else if (group_by === "target_date") { - sql += `, date(i.${group_by}) as group_id`; + sql += `, date(i.${group_by}) as group_id + `; } else { - sql += `, i.${group_by} as group_id`; + sql += `, i.${group_by} as group_id + `; } } if (sub_group_by) { if (ARRAY_FIELDS.includes(sub_group_by)) { - sql += `, ${sub_group_by}.value as sub_group_id`; + sql += `, ${sub_group_by}.value as sub_group_id + `; } else { - sql += `, i.${sub_group_by} as sub_group_id`; + sql += `, i.${sub_group_by} as sub_group_id + `; } } @@ -176,8 +184,9 @@ export const getFilteredRowsForGrouping = (projectId: string, queries: any) => { `; } - sql += ` WHERE i.project_id = '${projectId}' ${singleFilterConstructor(otherProps)} + sql += ` WHERE i.project_id = '${projectId}' `; + sql += singleFilterConstructor(otherProps); sql += `) `; @@ -190,7 +199,8 @@ export const singleFilterConstructor = (queries: any) => { let sql = ""; if (!sub_issue) { - sql += ` AND parent_id IS NOT NULL `; + sql += ` AND parent_id IS NOT NULL + `; } if (target_date) { // "2024-09-29;after,2024-11-02;before" @@ -198,7 +208,8 @@ export const singleFilterConstructor = (queries: any) => { const start = dates[0].split(";")[0]; const end = dates[1].split(";")[0]; - sql += ` AND target_date >= date('${start}') AND target_date <= date('${end}') `; + sql += ` AND target_date >= date('${start}') AND target_date <= date('${end}') + `; } const keys = Object.keys(filters); @@ -206,7 +217,8 @@ export const singleFilterConstructor = (queries: any) => { const value = filters[key] ? filters[key].split(",") : ""; if (!value) return; if (!ARRAY_FIELDS.includes(key)) { - sql += ` AND ${key} in ('${value.join("','")}')`; + sql += ` AND ${key} in ('${value.join("','")}') + `; } }); // @@ -244,14 +256,7 @@ const getSingleFilterFields = (queries: any) => { export const getIssueFieldsFragment = () => { const { description_html, ...filtered } = issueSchema; const keys = Object.keys(filtered); - const sql = ` ${keys - .map((key, index) => { - if (index % 5 === 0) { - return `i.${key} - `; - } - return `i.${key}`; - }) - .join(",")}`; + const sql = ` ${keys.map((key, index) => `i.${key}`).join(`, + `)}`; return sql; }; From 7238b187c25b032d695e81d3f05be58fa71d0735 Mon Sep 17 00:00:00 2001 From: rahulramesha Date: Wed, 4 Sep 2024 17:58:37 +0530 Subject: [PATCH 076/111] fix order by for multi issue --- web/core/store/issue/helpers/base-issues.store.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/core/store/issue/helpers/base-issues.store.ts b/web/core/store/issue/helpers/base-issues.store.ts index 26828217f2c..b397238ce27 100644 --- a/web/core/store/issue/helpers/base-issues.store.ts +++ b/web/core/store/issue/helpers/base-issues.store.ts @@ -1773,7 +1773,7 @@ export abstract class BaseIssuesStore implements IBaseIssuesStore { } } - return isDataIdsArray ? (order ? orderBy(dataValues, undefined, [order])[0] : dataValues) : dataValues[0]; + return isDataIdsArray ? (order ? orderBy(dataValues, undefined, [order]) : dataValues) : dataValues; } issuesSortWithOrderBy = (issueIds: string[], key: TIssueOrderByOptions | undefined): string[] => { From a1522b1173b366c8cb89323ab4ff35543ac818c6 Mon Sep 17 00:00:00 2001 From: rahulramesha Date: Wed, 4 Sep 2024 18:41:54 +0530 Subject: [PATCH 077/111] fix loaders for spreadsheet --- web/core/store/issue/cycle/issue.store.ts | 1 + web/core/store/issue/module/issue.store.ts | 1 + web/core/store/issue/project-views/issue.store.ts | 1 + web/core/store/issue/project/issue.store.ts | 1 + 4 files changed, 4 insertions(+) diff --git a/web/core/store/issue/cycle/issue.store.ts b/web/core/store/issue/cycle/issue.store.ts index 974ded65d36..6a0beb2f91f 100644 --- a/web/core/store/issue/cycle/issue.store.ts +++ b/web/core/store/issue/cycle/issue.store.ts @@ -179,6 +179,7 @@ export class CycleIssues extends BaseIssuesStore implements ICycleIssues { // set loader and clear store runInAction(() => { this.setLoader(loadType); + if(!options.canGroup) this.clear(!isExistingPaginationOptions); }); // get params from pagination options diff --git a/web/core/store/issue/module/issue.store.ts b/web/core/store/issue/module/issue.store.ts index c8583e9e0a5..583237894da 100644 --- a/web/core/store/issue/module/issue.store.ts +++ b/web/core/store/issue/module/issue.store.ts @@ -136,6 +136,7 @@ export class ModuleIssues extends BaseIssuesStore implements IModuleIssues { // set loader and clear store runInAction(() => { this.setLoader(loadType); + if(!options.canGroup) this.clear(!isExistingPaginationOptions); }); // get params from pagination options diff --git a/web/core/store/issue/project-views/issue.store.ts b/web/core/store/issue/project-views/issue.store.ts index 5dd69d763ac..7f5bb8b24c5 100644 --- a/web/core/store/issue/project-views/issue.store.ts +++ b/web/core/store/issue/project-views/issue.store.ts @@ -93,6 +93,7 @@ export class ProjectViewIssues extends BaseIssuesStore implements IProjectViewIs // set loader and clear store runInAction(() => { this.setLoader(loadType); + if(!options.canGroup) this.clear(!isExistingPaginationOptions); }); // get params from pagination options diff --git a/web/core/store/issue/project/issue.store.ts b/web/core/store/issue/project/issue.store.ts index 9d88d0f7094..f013ed38101 100644 --- a/web/core/store/issue/project/issue.store.ts +++ b/web/core/store/issue/project/issue.store.ts @@ -101,6 +101,7 @@ export class ProjectIssues extends BaseIssuesStore implements IProjectIssues { // set loader and clear store runInAction(() => { this.setLoader(loadType); + if(!options.canGroup) this.clear(!isExistingPaginationOptions); }); // get params from pagination options From 0f45b56c6f69ad11eeabdfcb950185d7bc15472a Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Wed, 4 Sep 2024 18:58:45 +0530 Subject: [PATCH 078/111] Fallback to manual order whn moving away from spreadsheet layout --- web/core/local-db/utils/query-constructor.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/web/core/local-db/utils/query-constructor.ts b/web/core/local-db/utils/query-constructor.ts index aa782e9f28b..a07d9e9f855 100644 --- a/web/core/local-db/utils/query-constructor.ts +++ b/web/core/local-db/utils/query-constructor.ts @@ -19,13 +19,18 @@ export const SPECIAL_ORDER_BY = { "-state__name": "states", }; export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: string, queries: any) => { - const { order_by, cursor, per_page, group_by, sub_group_by, ...otherProps } = translateQueryParams(queries); + const { cursor, per_page, group_by, sub_group_by, ...rest } = translateQueryParams(queries); + // eslint-disable-next-line prefer-const + let { order_by, ...otherProps } = rest; const [pageSize, page, offset] = cursor.split(":"); let sql = ""; const fieldsFragment = getIssueFieldsFragment(); + if ((group_by || sub_group_by) && Object.keys(SPECIAL_ORDER_BY).includes(order_by)) { + order_by = "sort_order"; + } if (sub_group_by) { const orderByString = getOrderByFragment(order_by); sql = getFilteredRowsForGrouping(projectId, queries); From f6e58cee6ca67387219decb8baff0a75120f5e09 Mon Sep 17 00:00:00 2001 From: rahulramesha Date: Wed, 4 Sep 2024 21:05:08 +0530 Subject: [PATCH 079/111] fix minor bug --- web/core/local-db/utils/query.utils.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/core/local-db/utils/query.utils.ts b/web/core/local-db/utils/query.utils.ts index b0dd3c63839..db8b50d0cb4 100644 --- a/web/core/local-db/utils/query.utils.ts +++ b/web/core/local-db/utils/query.utils.ts @@ -243,10 +243,10 @@ const getSingleFilterFields = (queries: any) => { } }); - if (order_by.includes("state__name")) { + if (order_by?.includes("state__name")) { fields.add("state_id"); } - if (order_by.includes("cycle__name")) { + if (order_by?.includes("cycle__name")) { fields.add("cycle_id"); } From a1da35f0ce954df744a6a5dc1d0654295c72183f Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Thu, 5 Sep 2024 13:00:03 +0530 Subject: [PATCH 080/111] Move fix for order_by when switching from spreadsheet layout to translateQueryParams --- web/core/local-db/utils/query-constructor.ts | 8 ++------ web/core/local-db/utils/query.utils.ts | 4 ++++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/web/core/local-db/utils/query-constructor.ts b/web/core/local-db/utils/query-constructor.ts index a07d9e9f855..0893fb5ce08 100644 --- a/web/core/local-db/utils/query-constructor.ts +++ b/web/core/local-db/utils/query-constructor.ts @@ -19,18 +19,14 @@ export const SPECIAL_ORDER_BY = { "-state__name": "states", }; export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: string, queries: any) => { - const { cursor, per_page, group_by, sub_group_by, ...rest } = translateQueryParams(queries); - // eslint-disable-next-line prefer-const - let { order_by, ...otherProps } = rest; + const { cursor, per_page, group_by, sub_group_by, order_by, ...otherProps } = translateQueryParams(queries); + const [pageSize, page, offset] = cursor.split(":"); let sql = ""; const fieldsFragment = getIssueFieldsFragment(); - if ((group_by || sub_group_by) && Object.keys(SPECIAL_ORDER_BY).includes(order_by)) { - order_by = "sort_order"; - } if (sub_group_by) { const orderByString = getOrderByFragment(order_by); sql = getFilteredRowsForGrouping(projectId, queries); diff --git a/web/core/local-db/utils/query.utils.ts b/web/core/local-db/utils/query.utils.ts index b0dd3c63839..ef5e67d7b18 100644 --- a/web/core/local-db/utils/query.utils.ts +++ b/web/core/local-db/utils/query.utils.ts @@ -25,6 +25,10 @@ export const translateQueryParams = (queries: any) => { otherProps.order_by = order_by.replace("priority", "priority_proxy"); } + // Fix invalid orderby when switching from spreadsheet layout + if ((group_by || sub_group_by) && Object.keys(SPECIAL_ORDER_BY).includes(order_by)) { + otherProps.order_by = "sort_order"; + } // For each property value, replace None with empty string Object.keys(otherProps).forEach((key) => { if (otherProps[key] === "None") { From 3527a7b87ea105f5edbc7e1ed1fcb2c688f0e93f Mon Sep 17 00:00:00 2001 From: rahulramesha Date: Thu, 5 Sep 2024 13:20:25 +0530 Subject: [PATCH 081/111] fix default rendering of kanban groups --- web/core/components/issues/issue-layouts/kanban/default.tsx | 5 ++++- .../components/issues/issue-layouts/kanban/swimlanes.tsx | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/web/core/components/issues/issue-layouts/kanban/default.tsx b/web/core/components/issues/issue-layouts/kanban/default.tsx index d0eb5c42c5f..de6f02bcc22 100644 --- a/web/core/components/issues/issue-layouts/kanban/default.tsx +++ b/web/core/components/issues/issue-layouts/kanban/default.tsx @@ -43,6 +43,7 @@ export interface IKanBan { isDropDisabled?: boolean; dropErrorMessage?: string | undefined; sub_group_id?: string; + sub_group_index?: number updateIssue: ((projectId: string | null, issueId: string, data: Partial) => Promise) | undefined; quickActions: TRenderQuickActions; kanbanFilters: TIssueKanbanFilters; @@ -83,6 +84,7 @@ export const KanBan: React.FC = observer((props) => { orderBy, isDropDisabled, dropErrorMessage, + sub_group_index = 0, } = props; const storeType = useIssueStoreType(); @@ -143,7 +145,7 @@ export const KanBan: React.FC = observer((props) => {
{list && list.length > 0 && - list.map((subList: IGroupByColumn) => { + list.map((subList: IGroupByColumn, index) => { const groupByVisibilityToggle = visibilityGroupBy(subList); const issueIds = isSubGroup ? (groupedIssueIds as TSubGroupedIssues)?.[subList.id]?.[sub_group_id] ?? [] @@ -191,6 +193,7 @@ export const KanBan: React.FC = observer((props) => { cardsInColumn={issueLength !== undefined && issueLength < 3 ? issueLength : 3} /> } + defaultValue={index < 5 && sub_group_index < 2} useIdletime > = observer((props) => {
{list && list.length > 0 && - list.map((_list: IGroupByColumn) => { + list.map((_list: IGroupByColumn, index) => { const issueCount = getGroupIssueCount(undefined, _list.id, true) ?? 0; const subGroupByVisibilityToggle = visibilitySubGroupBy(_list, issueCount); if (subGroupByVisibilityToggle.showGroup === false) return <>; @@ -182,6 +182,7 @@ const SubGroupSwimlane: React.FC = observer((props) => { sub_group_by={sub_group_by} group_by={group_by} sub_group_id={_list.id} + sub_group_index={index} updateIssue={updateIssue} quickActions={quickActions} kanbanFilters={kanbanFilters} From ece0b2145614be1e4ff7cd10df0194cc029adbb1 Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Thu, 5 Sep 2024 15:31:02 +0530 Subject: [PATCH 082/111] Fix none priority being saved as null --- web/core/local-db/utils/load-issues.ts | 3 ++- web/core/local-db/utils/load-workspace.ts | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/web/core/local-db/utils/load-issues.ts b/web/core/local-db/utils/load-issues.ts index 3013efab81c..9ee4e3ab16c 100644 --- a/web/core/local-db/utils/load-issues.ts +++ b/web/core/local-db/utils/load-issues.ts @@ -57,7 +57,7 @@ export const stageIssueInserts = (issue: any) => { const keys = Object.keys(issueSchema); const sanitizedIssue = keys.reduce((acc: any, key) => { - if (issue[key]) { + if (typeof issue[key] !== "undefined") { acc[key] = issue[key]; } return acc; @@ -81,6 +81,7 @@ export const stageIssueInserts = (issue: any) => { .join(", "); const query = `INSERT OR REPLACE INTO issues (${columns}) VALUES (${values});`; + debugger; persistence.db.exec(query); persistence.db.exec({ diff --git a/web/core/local-db/utils/load-workspace.ts b/web/core/local-db/utils/load-workspace.ts index 8840ef83c9f..1241b1cd8b1 100644 --- a/web/core/local-db/utils/load-workspace.ts +++ b/web/core/local-db/utils/load-workspace.ts @@ -13,7 +13,7 @@ const stageInserts = (table: string, schema: Schema, data: any) => { const keys = Object.keys(schema); // Pick only the keys that are in the schema const filteredData = keys.reduce((acc: any, key) => { - if (data[key]) { + if (typeof data[key] !== "undefined") { acc[key] = data[key]; } return acc; From 08f57a8428cea1fd4b3772589eb58e8dd26e3ff7 Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Thu, 5 Sep 2024 15:49:47 +0530 Subject: [PATCH 083/111] Remove debugger statement --- web/core/local-db/utils/load-issues.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/web/core/local-db/utils/load-issues.ts b/web/core/local-db/utils/load-issues.ts index 9ee4e3ab16c..a13beb7df58 100644 --- a/web/core/local-db/utils/load-issues.ts +++ b/web/core/local-db/utils/load-issues.ts @@ -81,7 +81,6 @@ export const stageIssueInserts = (issue: any) => { .join(", "); const query = `INSERT OR REPLACE INTO issues (${columns}) VALUES (${values});`; - debugger; persistence.db.exec(query); persistence.db.exec({ From 2fc8ab56e8fd239c3a8745398db25ce4b48917fe Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Thu, 5 Sep 2024 16:13:08 +0530 Subject: [PATCH 084/111] Fix issue load --- web/core/local-db/utils/load-issues.ts | 2 +- web/core/local-db/utils/load-workspace.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/web/core/local-db/utils/load-issues.ts b/web/core/local-db/utils/load-issues.ts index a13beb7df58..f1d8212c6d7 100644 --- a/web/core/local-db/utils/load-issues.ts +++ b/web/core/local-db/utils/load-issues.ts @@ -57,7 +57,7 @@ export const stageIssueInserts = (issue: any) => { const keys = Object.keys(issueSchema); const sanitizedIssue = keys.reduce((acc: any, key) => { - if (typeof issue[key] !== "undefined") { + if (issue[key] || issue[key] === 0) { acc[key] = issue[key]; } return acc; diff --git a/web/core/local-db/utils/load-workspace.ts b/web/core/local-db/utils/load-workspace.ts index 1241b1cd8b1..36d2aabce95 100644 --- a/web/core/local-db/utils/load-workspace.ts +++ b/web/core/local-db/utils/load-workspace.ts @@ -13,7 +13,7 @@ const stageInserts = (table: string, schema: Schema, data: any) => { const keys = Object.keys(schema); // Pick only the keys that are in the schema const filteredData = keys.reduce((acc: any, key) => { - if (typeof data[key] !== "undefined") { + if (data[key] || data[key] === 0) { acc[key] = data[key]; } return acc; From b791b31b54038aeaf24b020f7b62c652a9bcaea5 Mon Sep 17 00:00:00 2001 From: gurusainath Date: Mon, 9 Sep 2024 12:46:24 +0530 Subject: [PATCH 085/111] chore: updated isue paginated query from to --- apiserver/plane/app/views/issue/base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apiserver/plane/app/views/issue/base.py b/apiserver/plane/app/views/issue/base.py index c4dbf4ff6a2..fcd4615df1e 100644 --- a/apiserver/plane/app/views/issue/base.py +++ b/apiserver/plane/app/views/issue/base.py @@ -748,7 +748,7 @@ def process_paginated_result(self, fields, results, timezone): def list(self, request, slug, project_id): cursor = request.GET.get("cursor", None) is_description_required = request.GET.get("description", False) - updated_at = request.GET.get("updated_at__gte", None) + updated_at = request.GET.get("updated_at__gt", None) # required fields required_fields = [ @@ -791,8 +791,8 @@ def list(self, request, slug, project_id): # filtering issues by greater then updated_at given by the user if updated_at: - base_queryset = base_queryset.filter(updated_at__gte=updated_at) - queryset = queryset.filter(updated_at__gte=updated_at) + base_queryset = base_queryset.filter(updated_at__gt=updated_at) + queryset = queryset.filter(updated_at__gt=updated_at) queryset = queryset.annotate( label_ids=Coalesce( From 6b2d5ad034354b9fd0c5c18bb265014f42c3e63b Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Fri, 6 Sep 2024 16:55:05 +0530 Subject: [PATCH 086/111] Fix sub issues and start and target date filters --- web/core/local-db/utils/query.utils.ts | 56 +++++++++++++++++++++----- 1 file changed, 47 insertions(+), 9 deletions(-) diff --git a/web/core/local-db/utils/query.utils.ts b/web/core/local-db/utils/query.utils.ts index c34320019e6..a14962d2af5 100644 --- a/web/core/local-db/utils/query.utils.ts +++ b/web/core/local-db/utils/query.utils.ts @@ -198,22 +198,19 @@ export const getFilteredRowsForGrouping = (projectId: string, queries: any) => { }; export const singleFilterConstructor = (queries: any) => { - const { order_by, cursor, per_page, group_by, sub_group_by, sub_issue, target_date, ...filters } = + const { order_by, cursor, per_page, group_by, sub_group_by, sub_issue, target_date, start_date, ...filters } = translateQueryParams(queries); let sql = ""; if (!sub_issue) { - sql += ` AND parent_id IS NOT NULL + sql += ` AND parent_id IS NULL `; } if (target_date) { - // "2024-09-29;after,2024-11-02;before" - const dates = target_date.split(","); - const start = dates[0].split(";")[0]; - const end = dates[1].split(";")[0]; - - sql += ` AND target_date >= date('${start}') AND target_date <= date('${end}') - `; + sql += createDateFilter("target_date", target_date); + } + if (start_date) { + sql += createDateFilter("start_date", start_date); } const keys = Object.keys(filters); @@ -230,6 +227,47 @@ export const singleFilterConstructor = (queries: any) => { return sql; }; +// let q = '2_months;after;fromnow,1_months;after;fromnow,2024-09-01;after,2024-10-06;after,2_weeks;after;fromnow' + +// ["2_months;after;fromnow", "1_months;after;fromnow", "2024-09-01;after", "2024-10-06;before", "2_weeks;after;fromnow"]; + +const createDateFilter = (key: string, q: string) => { + let sql = " "; + // get todays date in YYYY-MM-DD format + const queries = q.split(","); + const customRange: string[] = []; + let isAnd = true; + queries.forEach((query: string) => { + const [date, type, from] = query.split(";"); + if (from) { + // Assuming type is always after + let after = ""; + const [length, unit] = date.split("_"); + if (unit === "weeks") { + // get date in yyyy-mm-dd format one week from now + after = new Date(new Date().setDate(new Date().getDate() + length * 7)).toISOString().split("T")[0]; + } + if (unit === "months") { + after = new Date(new Date().setDate(new Date().getDate() + length * 30)).toISOString().split("T")[0]; + } + sql += ` ${isAnd ? "AND" : "OR"} ${key} >= date('${after}')`; + isAnd = false; + // sql += ` AND ${key} ${type === "after" ? ">=" : "<="} date('${date}', '${today}')`; + } else { + customRange.push(query); + } + }); + + if (customRange.length === 2) { + const end = customRange.find((date) => date.includes("before"))?.split(";")[0]; + const start = customRange.find((date) => date.includes("after"))?.split(";")[0]; + if (end && start) { + sql += ` ${isAnd ? "AND" : "OR"} ${key} BETWEEN date('${start}') AND date('${end}')`; + } + } + + return sql; +}; const getSingleFilterFields = (queries: any) => { const { order_by, cursor, per_page, group_by, sub_group_by, sub_issue, ...otherProps } = translateQueryParams(queries); From ffd18cdd75434a1a26699822550ce8d7a4156fe8 Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Fri, 6 Sep 2024 18:05:55 +0530 Subject: [PATCH 087/111] Fix active and backlog filter --- web/core/local-db/utils/query-constructor.ts | 10 +++++++++- web/core/local-db/utils/query.utils.ts | 4 +++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/web/core/local-db/utils/query-constructor.ts b/web/core/local-db/utils/query-constructor.ts index 0893fb5ce08..ed065ebd95c 100644 --- a/web/core/local-db/utils/query-constructor.ts +++ b/web/core/local-db/utils/query-constructor.ts @@ -119,8 +119,16 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st const filterJoinFields = getMetaKeys(queries); const orderByString = getOrderByFragment(order_by); - sql = `SELECT ${fieldsFragment} from issues i`; + sql = `SELECT ${fieldsFragment}`; + if (otherProps.state_group) { + sql += `, states.'group' as state_group`; + } + sql += ` from issues i + `; + if (otherProps.state_group) { + sql += `LEFT JOIN states ON i.state_id = states.id `; + } filterJoinFields.forEach((field: string) => { const value = otherProps[field] || ""; sql += ` INNER JOIN issue_meta ${field} ON i.id = ${field}.issue_id AND ${field}.key = '${field}' AND ${field}.value IN ('${value.split(",").join("','")}') diff --git a/web/core/local-db/utils/query.utils.ts b/web/core/local-db/utils/query.utils.ts index a14962d2af5..d9f6d0dbc97 100644 --- a/web/core/local-db/utils/query.utils.ts +++ b/web/core/local-db/utils/query.utils.ts @@ -242,7 +242,9 @@ const createDateFilter = (key: string, q: string) => { if (from) { // Assuming type is always after let after = ""; - const [length, unit] = date.split("_"); + const [_length, unit] = date.split("_"); + const length = parseInt(_length); + if (unit === "weeks") { // get date in yyyy-mm-dd format one week from now after = new Date(new Date().setDate(new Date().getDate() + length * 7)).toISOString().split("T")[0]; From e39d17c6bc9f852abf12a850594e5839d41197e7 Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Fri, 6 Sep 2024 18:30:23 +0530 Subject: [PATCH 088/111] Add default order by --- web/core/local-db/utils/query-constructor.ts | 9 ++++- web/core/local-db/utils/query.utils.ts | 41 +++++++++++++++++--- 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/web/core/local-db/utils/query-constructor.ts b/web/core/local-db/utils/query-constructor.ts index ed065ebd95c..556bbe521bb 100644 --- a/web/core/local-db/utils/query-constructor.ts +++ b/web/core/local-db/utils/query-constructor.ts @@ -19,7 +19,14 @@ export const SPECIAL_ORDER_BY = { "-state__name": "states", }; export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: string, queries: any) => { - const { cursor, per_page, group_by, sub_group_by, order_by, ...otherProps } = translateQueryParams(queries); + const { + cursor, + per_page, + group_by, + sub_group_by, + order_by = "created_at", + ...otherProps + } = translateQueryParams(queries); const [pageSize, page, offset] = cursor.split(":"); diff --git a/web/core/local-db/utils/query.utils.ts b/web/core/local-db/utils/query.utils.ts index d9f6d0dbc97..ec834f7d3fd 100644 --- a/web/core/local-db/utils/query.utils.ts +++ b/web/core/local-db/utils/query.utils.ts @@ -4,7 +4,7 @@ import { issueSchema } from "./schemas"; import { wrapDateTime } from "./utils"; export const translateQueryParams = (queries: any) => { - const { group_by, sub_group_by, labels, assignees, state, cycle, module, priority, ...otherProps } = queries; + const { group_by, sub_group_by, labels, assignees, state, cycle, module, priority, type, ...otherProps } = queries; const order_by = queries.order_by; if (state) otherProps.state_id = state; @@ -20,6 +20,9 @@ export const translateQueryParams = (queries: any) => { .map((priority: string) => PRIORITY_MAP[priority as keyof typeof PRIORITY_MAP]) .join(","); } + if (type) { + otherProps.state_group = type === "backlog" ? "backlog" : "unstarted,started"; + } if (order_by?.includes("priority")) { otherProps.order_by = order_by.replace("priority", "priority_proxy"); @@ -140,7 +143,11 @@ export const getFilteredRowsForGrouping = (projectId: string, queries: any) => { if (sub_group_by) { sql += `, i.${sub_group_by} as sub_group_id`; } - sql += ` FROM issues i WHERE project_id = '${projectId}' + sql += ` FROM issues i `; + if (otherProps.state_group) { + sql += `LEFT JOIN states ON i.state_id = states.id `; + } + sql += `WHERE i.project_id = '${projectId}' `; sql += `${singleFilterConstructor(otherProps)}) `; @@ -173,6 +180,9 @@ export const getFilteredRowsForGrouping = (projectId: string, queries: any) => { sql += ` from issues i `; + if (otherProps.state_group) { + sql += `LEFT JOIN states ON i.state_id = states.id `; + } filterJoinFields.forEach((field: string) => { sql += ` INNER JOIN issue_meta ${field} ON i.id = ${field}.issue_id AND ${field}.key = '${field}' AND ${field}.value IN ('${otherProps[field].split(",").join("','")}') `; @@ -198,8 +208,18 @@ export const getFilteredRowsForGrouping = (projectId: string, queries: any) => { }; export const singleFilterConstructor = (queries: any) => { - const { order_by, cursor, per_page, group_by, sub_group_by, sub_issue, target_date, start_date, ...filters } = - translateQueryParams(queries); + const { + order_by, + cursor, + per_page, + group_by, + sub_group_by, + state_group, + sub_issue, + target_date, + start_date, + ...filters + } = translateQueryParams(queries); let sql = ""; if (!sub_issue) { @@ -212,6 +232,10 @@ export const singleFilterConstructor = (queries: any) => { if (start_date) { sql += createDateFilter("start_date", start_date); } + if (state_group) { + sql += ` AND state_group in ('${state_group.split(",").join("','")}') + `; + } const keys = Object.keys(filters); keys.forEach((key) => { @@ -267,11 +291,14 @@ const createDateFilter = (key: string, q: string) => { sql += ` ${isAnd ? "AND" : "OR"} ${key} BETWEEN date('${start}') AND date('${end}')`; } } + if (customRange.length === 1) { + sql += ` AND ${key}=date('${customRange[0].split(";")[0]}')`; + } return sql; }; const getSingleFilterFields = (queries: any) => { - const { order_by, cursor, per_page, group_by, sub_group_by, sub_issue, ...otherProps } = + const { order_by, cursor, per_page, group_by, sub_group_by, sub_issue, state_group, ...otherProps } = translateQueryParams(queries); const fields = new Set(); @@ -293,7 +320,9 @@ const getSingleFilterFields = (queries: any) => { if (order_by?.includes("cycle__name")) { fields.add("cycle_id"); } - + if (state_group) { + fields.add("states.'group' as state_group"); + } return Array.from(fields); }; From bce8f2daeda83ffaea782dfac3b4edcbbdd78e2b Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Mon, 9 Sep 2024 13:32:15 +0530 Subject: [PATCH 089/111] Update the Query param to match with backend. --- web/core/local-db/storage.sqlite.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/core/local-db/storage.sqlite.ts b/web/core/local-db/storage.sqlite.ts index fbfe43d775f..7edda3923aa 100644 --- a/web/core/local-db/storage.sqlite.ts +++ b/web/core/local-db/storage.sqlite.ts @@ -170,7 +170,7 @@ export class Storage { return; } - const queryParams: { cursor: string; updated_at__gte?: string; description: boolean } = { + const queryParams: { cursor: string; updated_at__gt?: string; description: boolean } = { cursor: `${PAGE_SIZE}:0:0`, description: true, }; @@ -179,7 +179,7 @@ export class Storage { const projectSync = await this.getOption(projectId); if (syncedAt) { - queryParams["updated_at__gte"] = syncedAt; + queryParams["updated_at__gt"] = syncedAt; } this.setStatus(projectId, projectSync === "ready" ? "syncing" : "loading"); From 799aeab1b4d86b29eb6d17c1dff176c0a4e9a9c6 Mon Sep 17 00:00:00 2001 From: rahulramesha Date: Tue, 10 Sep 2024 20:27:03 +0530 Subject: [PATCH 090/111] local sqlite db versioning --- web/core/local-db/storage.sqlite.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/web/core/local-db/storage.sqlite.ts b/web/core/local-db/storage.sqlite.ts index 7edda3923aa..c38eb89aa39 100644 --- a/web/core/local-db/storage.sqlite.ts +++ b/web/core/local-db/storage.sqlite.ts @@ -19,6 +19,7 @@ declare module "@sqlite.org/sqlite-wasm" { export function sqlite3Worker1Promiser(...args: any): any; } +const DB_VERSION = "1"; const PAGE_SIZE = 1000; const log = console.log; const error = console.error; @@ -121,6 +122,16 @@ export class Storage { return promiser("exec", { dbId, ...val }); }, }; + + // dump DB of db version is matching + const dbVersion = await this.getOption("DB_VERSION"); + if (dbVersion !== "" && dbVersion !== DB_VERSION) { + await this.clearStorage(); + this.reset(); + this.workspaceInitPromise = this._initialize(workspaceSlug); + return false; + } + log( "OPFS is available, created persisted database at", openResponse.result.filename.replace(/^file:(.*?)\?vfs=opfs$/, "$1") @@ -128,6 +139,8 @@ export class Storage { this.status = "ready"; // Your SQLite code here. await createTables(); + + await this.setOption("DB_VERSION", DB_VERSION); } catch (err) { error(err); } @@ -148,6 +161,7 @@ export class Storage { syncIssues = async (projectId: string) => { try { + await this.workspaceInitPromise; const sync = this._syncIssues(projectId); this.setSync(projectId, sync); await sync; @@ -251,10 +265,12 @@ export class Storage { console.log("#### Queries", queries); const currentProjectStatus = this.getStatus(projectId); + const dbVersion = await this.getOption("DB_VERSION"); if ( !currentProjectStatus || currentProjectStatus === "loading" || currentProjectStatus === "error" || + dbVersion !== DB_VERSION || (window as any).DISABLE_LOCAL ) { info(`Project ${projectId} is loading, falling back to server`); From 13a7a5f1debda36a6b2df3b32fbee967261d688b Mon Sep 17 00:00:00 2001 From: rahulramesha Date: Tue, 10 Sep 2024 20:29:51 +0530 Subject: [PATCH 091/111] When window is hidden, do not perform any db versioning --- web/core/local-db/storage.sqlite.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/web/core/local-db/storage.sqlite.ts b/web/core/local-db/storage.sqlite.ts index c38eb89aa39..e4dfaa45cf3 100644 --- a/web/core/local-db/storage.sqlite.ts +++ b/web/core/local-db/storage.sqlite.ts @@ -62,6 +62,8 @@ export class Storage { }; initialize = async (workspaceSlug: string): Promise => { + if (document.hidden) return false; // return if the window gets hidden + if (workspaceSlug !== this.workspaceSlug) { this.reset(); } @@ -143,23 +145,29 @@ export class Storage { await this.setOption("DB_VERSION", DB_VERSION); } catch (err) { error(err); + throw err; } return true; }; syncWorkspace = async () => { + if (document.hidden) return; // return if the window gets hidden + await this.workspaceInitPromise; loadWorkSpaceData(this.workspaceSlug); }; syncProject = (projectId: string) => { - // Load labels, members, states, modules, cycles + if (document.hidden) return false; // return if the window gets hidden + // Load labels, members, states, modules, cycles this.syncIssues(projectId); }; syncIssues = async (projectId: string) => { + if (document.hidden) return false; // return if the window gets hidden + try { await this.workspaceInitPromise; const sync = this._syncIssues(projectId); From f9833370290d8b8b364309194539270fc1220782 Mon Sep 17 00:00:00 2001 From: rahulramesha Date: Tue, 10 Sep 2024 20:44:14 +0530 Subject: [PATCH 092/111] fix error handling and fall back to server when database errors out --- web/core/local-db/storage.sqlite.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/web/core/local-db/storage.sqlite.ts b/web/core/local-db/storage.sqlite.ts index e4dfaa45cf3..bf8f5052861 100644 --- a/web/core/local-db/storage.sqlite.ts +++ b/web/core/local-db/storage.sqlite.ts @@ -276,6 +276,7 @@ export class Storage { const dbVersion = await this.getOption("DB_VERSION"); if ( !currentProjectStatus || + this.status !== "ready" || currentProjectStatus === "loading" || currentProjectStatus === "error" || dbVersion !== DB_VERSION || @@ -349,10 +350,15 @@ export class Storage { }; getIssue = async (issueId: string) => { - const issues = await runQuery(`select * from issues where id='${issueId}'`); - if (issues.length) { - return formatLocalIssue(issues[0]); + try { + const issues = await runQuery(`select * from issues where id='${issueId}'`); + if (issues.length) { + return formatLocalIssue(issues[0]); + } + } catch (err) { + console.warn("unable to fetch issue from local db"); } + return; }; getStatus = (projectId: string) => this.projectStatus[projectId]?.issues?.status || undefined; From ea83e3c16e428b7abe08457e1962888668c75a14 Mon Sep 17 00:00:00 2001 From: rahulramesha Date: Wed, 11 Sep 2024 15:59:31 +0530 Subject: [PATCH 093/111] Add ability to disable local db cache --- web/app/profile/page.tsx | 44 +++++++++++++++++-- web/core/hooks/use-local-storage.tsx | 12 ++++- web/core/local-db/storage.sqlite.ts | 18 +++++--- web/core/local-db/utils/load-issues.ts | 13 +++++- web/core/services/issue/issue.service.ts | 2 +- web/core/store/issue/cycle/filter.store.ts | 18 ++++---- web/core/store/issue/cycle/issue.store.ts | 3 +- .../store/issue/helpers/base-issues.store.ts | 30 ++++++++----- .../helpers/issue-filter-helper.store.ts | 14 ++++++ web/core/store/issue/module/filter.store.ts | 6 ++- web/core/store/issue/module/issue.store.ts | 3 +- web/core/store/issue/profile/filter.store.ts | 2 - .../store/issue/project-views/filter.store.ts | 6 ++- .../store/issue/project-views/issue.store.ts | 3 +- web/core/store/issue/project/filter.store.ts | 6 ++- web/core/store/issue/project/issue.store.ts | 3 +- web/core/store/user/index.ts | 8 +++- web/core/store/user/settings.store.ts | 33 ++++++++++++++ 18 files changed, 178 insertions(+), 46 deletions(-) diff --git a/web/app/profile/page.tsx b/web/app/profile/page.tsx index 89a67ff7a67..b9795d06a40 100644 --- a/web/app/profile/page.tsx +++ b/web/app/profile/page.tsx @@ -10,7 +10,16 @@ import { Disclosure, Transition } from "@headlessui/react"; // layouts // components import type { IUser } from "@plane/types"; -import { Button, CustomSelect, CustomSearchSelect, Input, TOAST_TYPE, setPromiseToast, setToast } from "@plane/ui"; +import { + Button, + CustomSelect, + CustomSearchSelect, + Input, + TOAST_TYPE, + setPromiseToast, + setToast, + ToggleSwitch, +} from "@plane/ui"; import { DeactivateAccountModal } from "@/components/account"; import { LogoSpinner } from "@/components/common"; import { ImagePickerPopover, UserImageUploadModal, PageHead } from "@/components/core"; @@ -22,7 +31,7 @@ import { ProfileSettingContentWrapper } from "@/components/profile"; import { TIME_ZONES } from "@/constants/timezones"; import { USER_ROLES } from "@/constants/workspace"; // hooks -import { useUser } from "@/hooks/store"; +import { useUser, useUserSettings } from "@/hooks/store"; // import { ProfileSettingsLayout } from "@/layouts/settings-layout"; // layouts import { FileService } from "@/services/file.service"; @@ -59,6 +68,7 @@ const ProfileSettingsPage = observer(() => { } = useForm({ defaultValues }); // store hooks const { data: currentUser, updateCurrentUser } = useUser(); + const { canUseLocalDB, toggleLocalDB } = useUserSettings(); useEffect(() => { reset({ ...defaultValues, ...currentUser }); @@ -387,7 +397,7 @@ const ProfileSettingsPage = observer(() => { render={({ field: { value, onChange } }) => ( t.value === value)?.label ?? value : "Select a timezone"} + label={value ? (TIME_ZONES.find((t) => t.value === value)?.label ?? value) : "Select a timezone"} options={timeZoneOptions} onChange={onChange} buttonClassName={errors.user_timezone ? "border-red-500" : "border-none"} @@ -407,6 +417,34 @@ const ProfileSettingsPage = observer(() => {
+ + {({ open }) => ( + <> + + Local Cache + + + + +
+ + Enabling this will let the application cache data on system for faster loading experience + + toggleLocalDB()} /> +
+ + + + )} + {({ open }) => ( <> diff --git a/web/core/hooks/use-local-storage.tsx b/web/core/hooks/use-local-storage.tsx index e13165bf80e..538b8a93b77 100644 --- a/web/core/hooks/use-local-storage.tsx +++ b/web/core/hooks/use-local-storage.tsx @@ -1,6 +1,6 @@ import { useState, useEffect, useCallback } from "react"; -const getValueFromLocalStorage = (key: string, defaultValue: any) => { +export const getValueFromLocalStorage = (key: string, defaultValue: any) => { if (typeof window === undefined || typeof window === "undefined") return defaultValue; try { const item = window.localStorage.getItem(key); @@ -11,6 +11,16 @@ const getValueFromLocalStorage = (key: string, defaultValue: any) => { } }; +export const setValueIntoLocalStorage = (key: string, value: any) => { + if (typeof window === undefined || typeof window === "undefined") return false; + try { + window.localStorage.setItem(key, JSON.stringify(value)); + return true; + } catch (error) { + return false; + } +}; + const useLocalStorage = (key: string, initialValue: T) => { const [storedValue, setStoredValue] = useState(() => getValueFromLocalStorage(key, initialValue)); diff --git a/web/core/local-db/storage.sqlite.ts b/web/core/local-db/storage.sqlite.ts index bf8f5052861..f82d3f0a955 100644 --- a/web/core/local-db/storage.sqlite.ts +++ b/web/core/local-db/storage.sqlite.ts @@ -3,6 +3,8 @@ import set from "lodash/set"; import { EIssueGroupBYServerToProperty } from "@plane/constants"; import { TIssue } from "@plane/types"; import { setToast, TOAST_TYPE } from "@plane/ui"; +// lib +import { rootStore } from "@/lib/store-context"; // services import { IssueService } from "@/services/issue/issue.service"; // @@ -62,7 +64,7 @@ export class Storage { }; initialize = async (workspaceSlug: string): Promise => { - if (document.hidden) return false; // return if the window gets hidden + if (document.hidden || !rootStore.user.localDBEnabled) return false; // return if the window gets hidden if (workspaceSlug !== this.workspaceSlug) { this.reset(); @@ -152,21 +154,21 @@ export class Storage { }; syncWorkspace = async () => { - if (document.hidden) return; // return if the window gets hidden + if (document.hidden || !rootStore.user.localDBEnabled) return; // return if the window gets hidden await this.workspaceInitPromise; loadWorkSpaceData(this.workspaceSlug); }; syncProject = (projectId: string) => { - if (document.hidden) return false; // return if the window gets hidden + if (document.hidden || !rootStore.user.localDBEnabled) return false; // return if the window gets hidden // Load labels, members, states, modules, cycles this.syncIssues(projectId); }; syncIssues = async (projectId: string) => { - if (document.hidden) return false; // return if the window gets hidden + if (document.hidden || !rootStore.user.localDBEnabled) return false; // return if the window gets hidden try { await this.workspaceInitPromise; @@ -269,7 +271,7 @@ export class Storage { return issue.updated_at; }; - getIssues = async (projectId: string, queries: any, config: any) => { + getIssues = async (workspaceSlug: string, projectId: string, queries: any, config: any) => { console.log("#### Queries", queries); const currentProjectStatus = this.getStatus(projectId); @@ -280,11 +282,11 @@ export class Storage { currentProjectStatus === "loading" || currentProjectStatus === "error" || dbVersion !== DB_VERSION || - (window as any).DISABLE_LOCAL + !rootStore.user.localDBEnabled ) { info(`Project ${projectId} is loading, falling back to server`); const issueService = new IssueService(); - return await issueService.getIssuesFromServer(this.workspaceSlug, projectId, queries); + return await issueService.getIssuesFromServer(workspaceSlug, projectId, queries); } const { cursor, group_by, sub_group_by } = queries; @@ -351,6 +353,8 @@ export class Storage { getIssue = async (issueId: string) => { try { + if (!rootStore.user.localDBEnabled) return; + const issues = await runQuery(`select * from issues where id='${issueId}'`); if (issues.length) { return formatLocalIssue(issues[0]); diff --git a/web/core/local-db/utils/load-issues.ts b/web/core/local-db/utils/load-issues.ts index f1d8212c6d7..3f33a75bf7f 100644 --- a/web/core/local-db/utils/load-issues.ts +++ b/web/core/local-db/utils/load-issues.ts @@ -1,4 +1,5 @@ import { TIssue } from "@plane/types"; +import { rootStore } from "@/lib/store-context"; import { IssueService } from "@/services/issue"; import { persistence } from "../storage.sqlite"; import { ARRAY_FIELDS, PRIORITY_MAP } from "./constants"; @@ -7,12 +8,16 @@ import { issueSchema } from "./schemas"; export const PROJECT_OFFLINE_STATUS: Record = {}; export const addIssue = async (issue: any) => { + if (document.hidden || !rootStore.user.localDBEnabled) return; + persistence.db.exec("BEGIN TRANSACTION;"); stageIssueInserts(issue); persistence.db.exec("COMMIT;"); }; export const addIssuesBulk = async (issues: any, batchSize = 100) => { + if (document.hidden || !rootStore.user.localDBEnabled) return; + for (let i = 0; i < issues.length; i += batchSize) { const batch = issues.slice(i, i + batchSize); @@ -27,6 +32,8 @@ export const addIssuesBulk = async (issues: any, batchSize = 100) => { } }; export const deleteIssueFromLocal = async (issue_id: any) => { + if (document.hidden || !rootStore.user.localDBEnabled) return; + const deleteQuery = `delete from issues where id='${issue_id}'`; const deleteMetaQuery = `delete from issue_meta where issue_id='${issue_id}'`; @@ -37,6 +44,8 @@ export const deleteIssueFromLocal = async (issue_id: any) => { }; export const updateIssue = async (issue: TIssue) => { + if (document.hidden || !rootStore.user.localDBEnabled) return; + const issue_id = issue.id; // delete the issue and its meta data await deleteIssueFromLocal(issue_id); @@ -44,6 +53,8 @@ export const updateIssue = async (issue: TIssue) => { }; export const syncDeletesToLocal = async (workspaceId: string, projectId: string, queries: any) => { + if (document.hidden || !rootStore.user.localDBEnabled) return; + const issueService = new IssueService(); const response = await issueService.getDeletedIssues(workspaceId, projectId, queries); if (Array.isArray(response)) { @@ -51,7 +62,7 @@ export const syncDeletesToLocal = async (workspaceId: string, projectId: string, } }; -export const stageIssueInserts = (issue: any) => { +const stageIssueInserts = (issue: any) => { const issue_id = issue.id; issue.priority_proxy = PRIORITY_MAP[issue.priority as keyof typeof PRIORITY_MAP]; diff --git a/web/core/services/issue/issue.service.ts b/web/core/services/issue/issue.service.ts index 7e3978c4fce..b0a0687dc84 100644 --- a/web/core/services/issue/issue.service.ts +++ b/web/core/services/issue/issue.service.ts @@ -72,7 +72,7 @@ export class IssueService extends APIService { } async getIssues(workspaceSlug: string, projectId: string, queries?: any, config = {}): Promise { - const response = await persistence.getIssues(projectId, queries, config); + const response = await persistence.getIssues(workspaceSlug, projectId, queries, config); return response as TIssuesResponse; } diff --git a/web/core/store/issue/cycle/filter.store.ts b/web/core/store/issue/cycle/filter.store.ts index 4f3506ff6f7..cca800c9fac 100644 --- a/web/core/store/issue/cycle/filter.store.ts +++ b/web/core/store/issue/cycle/filter.store.ts @@ -1,6 +1,4 @@ -import isArray from "lodash/isArray"; import isEmpty from "lodash/isEmpty"; -import pickBy from "lodash/pickBy"; import set from "lodash/set"; import { action, computed, makeObservable, observable, runInAction } from "mobx"; // base class @@ -195,12 +193,12 @@ export class CycleIssuesFilter extends IssueFilterHelperStore implements ICycleI }); }); - this.rootIssueStore.cycleIssues.fetchIssuesWithExistingPagination( - workspaceSlug, - projectId, - "mutation", - cycleId - ); + this.rootIssueStore.cycleIssues.fetchIssuesWithExistingPagination( + workspaceSlug, + projectId, + "mutation", + cycleId + ); await this.issueFilterService.patchCycleIssueFilters(workspaceSlug, projectId, cycleId, { filters: _filters.filters, }); @@ -239,6 +237,10 @@ export class CycleIssuesFilter extends IssueFilterHelperStore implements ICycleI }); }); + if (this.getShouldClearIssues(updatedDisplayFilters)) { + this.rootIssueStore.cycleIssues.clear(true, true); // clear issues for local db when some filters like layout changes + } + if (this.getShouldReFetchIssues(updatedDisplayFilters)) { this.rootIssueStore.cycleIssues.fetchIssuesWithExistingPagination( workspaceSlug, diff --git a/web/core/store/issue/cycle/issue.store.ts b/web/core/store/issue/cycle/issue.store.ts index 6a0beb2f91f..ac1ab48c7b4 100644 --- a/web/core/store/issue/cycle/issue.store.ts +++ b/web/core/store/issue/cycle/issue.store.ts @@ -179,7 +179,8 @@ export class CycleIssues extends BaseIssuesStore implements ICycleIssues { // set loader and clear store runInAction(() => { this.setLoader(loadType); - if(!options.canGroup) this.clear(!isExistingPaginationOptions); + this.clear(!isExistingPaginationOptions, false); // clear while fetching from server. + if (!this.groupBy) this.clear(!isExistingPaginationOptions, true); // clear while using local to have the no load effect. }); // get params from pagination options diff --git a/web/core/store/issue/helpers/base-issues.store.ts b/web/core/store/issue/helpers/base-issues.store.ts index b397238ce27..422c5a82158 100644 --- a/web/core/store/issue/helpers/base-issues.store.ts +++ b/web/core/store/issue/helpers/base-issues.store.ts @@ -65,6 +65,7 @@ export interface IBaseIssuesStore { //actions removeIssue: (workspaceSlug: string, projectId: string, issueId: string) => Promise; + clear(shouldClearPaginationOptions?: boolean, clearForLocal?: boolean): void; // helper methods getIssueIds: (groupId?: string, subGroupId?: string) => string[] | undefined; issuesSortWithOrderBy(issueIds: string[], key: Partial): string[]; @@ -466,7 +467,7 @@ export abstract class BaseIssuesStore implements IBaseIssuesStore { // Update all the GroupIds to this Store's groupedIssueIds and update Individual group issue counts runInAction(() => { - this.clear(shouldClearPaginationOptions); + this.clear(shouldClearPaginationOptions, true); this.updateGroupedIssueIds(groupedIssues, groupedIssueCount); this.loader[getGroupKey()] = undefined; }); @@ -1218,17 +1219,22 @@ export abstract class BaseIssuesStore implements IBaseIssuesStore { /** * Method called to clear out the current store */ - clear(shouldClearPaginationOptions = true) { - runInAction(() => { - this.groupedIssueIds = undefined; - this.issuePaginationData = {}; - this.groupedIssueCount = {}; - if (shouldClearPaginationOptions) { - this.paginationOptions = undefined; - } - }); - this.controller.abort(); - this.controller = new AbortController(); + clear(shouldClearPaginationOptions = true, clearForLocal = false) { + if ( + (this.rootIssueStore.rootStore.user?.localDBEnabled && clearForLocal) || + (!this.rootIssueStore.rootStore.user?.localDBEnabled && !clearForLocal) + ) { + runInAction(() => { + this.groupedIssueIds = undefined; + this.issuePaginationData = {}; + this.groupedIssueCount = {}; + if (shouldClearPaginationOptions) { + this.paginationOptions = undefined; + } + }); + this.controller.abort(); + this.controller = new AbortController(); + } } /** diff --git a/web/core/store/issue/helpers/issue-filter-helper.store.ts b/web/core/store/issue/helpers/issue-filter-helper.store.ts index b3b066ad2d9..1f77863b6a5 100644 --- a/web/core/store/issue/helpers/issue-filter-helper.store.ts +++ b/web/core/store/issue/helpers/issue-filter-helper.store.ts @@ -267,6 +267,20 @@ export class IssueFilterHelperStore implements IIssueFilterHelperStore { ); }; + /** + * This Method returns true if the display properties changed requires a server side update + * @param displayFilters + * @returns + */ + getShouldClearIssues = (displayFilters: IIssueDisplayFilterOptions) => { + const NON_SERVER_DISPLAY_FILTERS = ["layout"]; + const displayFilterKeys = Object.keys(displayFilters); + + return NON_SERVER_DISPLAY_FILTERS.some((serverDisplayfilter: string) => + displayFilterKeys.includes(serverDisplayfilter) + ); + }; + /** * This Method is used to construct the url params along with paginated values * @param filterParams params generated from filters diff --git a/web/core/store/issue/module/filter.store.ts b/web/core/store/issue/module/filter.store.ts index 4cbecb6977e..d04e714b9f6 100644 --- a/web/core/store/issue/module/filter.store.ts +++ b/web/core/store/issue/module/filter.store.ts @@ -1,6 +1,4 @@ -import isArray from "lodash/isArray"; import isEmpty from "lodash/isEmpty"; -import pickBy from "lodash/pickBy"; import set from "lodash/set"; import { action, computed, makeObservable, observable, runInAction } from "mobx"; // base class @@ -238,6 +236,10 @@ export class ModuleIssuesFilter extends IssueFilterHelperStore implements IModul }); }); + if (this.getShouldClearIssues(updatedDisplayFilters)) { + this.rootIssueStore.moduleIssues.clear(true, true); // clear issues for local db when some filters like layout changes + } + if (this.getShouldReFetchIssues(updatedDisplayFilters)) { this.rootIssueStore.moduleIssues.fetchIssuesWithExistingPagination( workspaceSlug, diff --git a/web/core/store/issue/module/issue.store.ts b/web/core/store/issue/module/issue.store.ts index 583237894da..8d9dc6be902 100644 --- a/web/core/store/issue/module/issue.store.ts +++ b/web/core/store/issue/module/issue.store.ts @@ -136,7 +136,8 @@ export class ModuleIssues extends BaseIssuesStore implements IModuleIssues { // set loader and clear store runInAction(() => { this.setLoader(loadType); - if(!options.canGroup) this.clear(!isExistingPaginationOptions); + this.clear(!isExistingPaginationOptions, false); // clear while fetching from server. + if (!this.groupBy) this.clear(!isExistingPaginationOptions, true); // clear while using local to have the no load effect. }); // get params from pagination options diff --git a/web/core/store/issue/profile/filter.store.ts b/web/core/store/issue/profile/filter.store.ts index 99cd3544d23..5bbbb88f69c 100644 --- a/web/core/store/issue/profile/filter.store.ts +++ b/web/core/store/issue/profile/filter.store.ts @@ -1,6 +1,4 @@ -import isArray from "lodash/isArray"; import isEmpty from "lodash/isEmpty"; -import pickBy from "lodash/pickBy"; import set from "lodash/set"; import { action, computed, makeObservable, observable, runInAction } from "mobx"; // base class diff --git a/web/core/store/issue/project-views/filter.store.ts b/web/core/store/issue/project-views/filter.store.ts index 6b50a90bb7a..a6d3c47317a 100644 --- a/web/core/store/issue/project-views/filter.store.ts +++ b/web/core/store/issue/project-views/filter.store.ts @@ -1,6 +1,4 @@ -import isArray from "lodash/isArray"; import isEmpty from "lodash/isEmpty"; -import pickBy from "lodash/pickBy"; import set from "lodash/set"; import { action, computed, makeObservable, observable, runInAction } from "mobx"; // base class @@ -229,6 +227,10 @@ export class ProjectViewIssuesFilter extends IssueFilterHelperStore implements I }); }); + if (this.getShouldClearIssues(updatedDisplayFilters)) { + this.rootIssueStore.projectIssues.clear(true, true); // clear issues for local db when some filters like layout changes + } + if (this.getShouldReFetchIssues(updatedDisplayFilters)) { this.rootIssueStore.projectViewIssues.fetchIssuesWithExistingPagination( workspaceSlug, diff --git a/web/core/store/issue/project-views/issue.store.ts b/web/core/store/issue/project-views/issue.store.ts index 7f5bb8b24c5..8c50519ab3c 100644 --- a/web/core/store/issue/project-views/issue.store.ts +++ b/web/core/store/issue/project-views/issue.store.ts @@ -93,7 +93,8 @@ export class ProjectViewIssues extends BaseIssuesStore implements IProjectViewIs // set loader and clear store runInAction(() => { this.setLoader(loadType); - if(!options.canGroup) this.clear(!isExistingPaginationOptions); + this.clear(!isExistingPaginationOptions, false); // clear while fetching from server. + if (!this.groupBy) this.clear(!isExistingPaginationOptions, true); // clear while using local to have the no load effect. }); // get params from pagination options diff --git a/web/core/store/issue/project/filter.store.ts b/web/core/store/issue/project/filter.store.ts index 20041d1a9d8..7973e439ab2 100644 --- a/web/core/store/issue/project/filter.store.ts +++ b/web/core/store/issue/project/filter.store.ts @@ -1,6 +1,4 @@ -import isArray from "lodash/isArray"; import isEmpty from "lodash/isEmpty"; -import pickBy from "lodash/pickBy"; import set from "lodash/set"; import { action, computed, makeObservable, observable, runInAction } from "mobx"; // base class @@ -222,6 +220,10 @@ export class ProjectIssuesFilter extends IssueFilterHelperStore implements IProj }); }); + if (this.getShouldClearIssues(updatedDisplayFilters)) { + this.rootIssueStore.projectIssues.clear(true, true); // clear issues for local db when some filters like layout changes + } + if (this.getShouldReFetchIssues(updatedDisplayFilters)) { this.rootIssueStore.projectIssues.fetchIssuesWithExistingPagination(workspaceSlug, projectId, "mutation"); } diff --git a/web/core/store/issue/project/issue.store.ts b/web/core/store/issue/project/issue.store.ts index f013ed38101..052ab7c9bc9 100644 --- a/web/core/store/issue/project/issue.store.ts +++ b/web/core/store/issue/project/issue.store.ts @@ -101,7 +101,8 @@ export class ProjectIssues extends BaseIssuesStore implements IProjectIssues { // set loader and clear store runInAction(() => { this.setLoader(loadType); - if(!options.canGroup) this.clear(!isExistingPaginationOptions); + this.clear(!isExistingPaginationOptions, false); // clear while fetching from server. + if (!this.groupBy) this.clear(!isExistingPaginationOptions, true); // clear while using local to have the no load effect. }); // get params from pagination options diff --git a/web/core/store/user/index.ts b/web/core/store/user/index.ts index dbe2d891352..488aa78bba6 100644 --- a/web/core/store/user/index.ts +++ b/web/core/store/user/index.ts @@ -44,7 +44,7 @@ export interface IUserStore { reset: () => void; signOut: () => Promise; // computed - + localDBEnabled: boolean; // workspace level canPerformWorkspaceAdminActions: boolean; canPerformWorkspaceMemberActions: boolean; @@ -115,6 +115,8 @@ export class UserStore implements IUserStore { canPerformAnyCreateAction: computed, projectsWithCreatePermissions: computed, + + localDBEnabled: computed, }); } @@ -356,4 +358,8 @@ export class UserStore implements IUserStore { get canPerformProjectGuestActions() { return !!this.membership.currentProjectRole && this.membership.currentProjectRole >= EUserProjectRoles.GUEST; } + + get localDBEnabled() { + return this.userSettings.canUseLocalDB; + } } diff --git a/web/core/store/user/settings.store.ts b/web/core/store/user/settings.store.ts index 970c2faf89f..95e8e3942af 100644 --- a/web/core/store/user/settings.store.ts +++ b/web/core/store/user/settings.store.ts @@ -1,5 +1,9 @@ import { action, makeObservable, observable, runInAction } from "mobx"; import { IUserSettings } from "@plane/types"; +// hooks +import { getValueFromLocalStorage, setValueIntoLocalStorage } from "@/hooks/use-local-storage"; +// local +import { persistence } from "@/local-db/storage.sqlite"; // services import { UserService } from "@/services/user.service"; @@ -8,13 +12,17 @@ type TError = { message: string; }; +const LOCAL_DB_ENABLED = "LOCAL_DB_ENABLED"; + export interface IUserSettingsStore { // observables isLoading: boolean; error: TError | undefined; data: IUserSettings; + canUseLocalDB: boolean; // actions fetchCurrentUserSettings: () => Promise; + toggleLocalDB: () => Promise; } export class UserSettingsStore implements IUserSettingsStore { @@ -32,6 +40,7 @@ export class UserSettingsStore implements IUserSettingsStore { invites: undefined, }, }; + canUseLocalDB: boolean = getValueFromLocalStorage(LOCAL_DB_ENABLED, true); // services userService: UserService; @@ -41,13 +50,37 @@ export class UserSettingsStore implements IUserSettingsStore { isLoading: observable.ref, error: observable, data: observable, + canUseLocalDB: observable.ref, // actions fetchCurrentUserSettings: action, + toggleLocalDB: action, }); // services this.userService = new UserService(); } + toggleLocalDB = async () => { + const currentLocalDBValue = this.canUseLocalDB; + try { + runInAction(() => { + this.canUseLocalDB = !currentLocalDBValue; + }); + + const transactionResult = setValueIntoLocalStorage(LOCAL_DB_ENABLED, !currentLocalDBValue); + + if (!transactionResult) { + throw new Error("error while toggling local DB"); + } else if (currentLocalDBValue) { + await persistence.clearStorage(); + } + } catch (e) { + console.warn("error while toggling local DB"); + runInAction(() => { + this.canUseLocalDB = currentLocalDBValue; + }); + } + }; + // actions /** * @description fetches user profile information From a903aec711f21a990580614232c75a15a1e89ff5 Mon Sep 17 00:00:00 2001 From: rahulramesha Date: Wed, 11 Sep 2024 16:03:41 +0530 Subject: [PATCH 094/111] remove db version check from getIssues function --- web/core/local-db/storage.sqlite.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/web/core/local-db/storage.sqlite.ts b/web/core/local-db/storage.sqlite.ts index f82d3f0a955..8e395b18882 100644 --- a/web/core/local-db/storage.sqlite.ts +++ b/web/core/local-db/storage.sqlite.ts @@ -275,13 +275,11 @@ export class Storage { console.log("#### Queries", queries); const currentProjectStatus = this.getStatus(projectId); - const dbVersion = await this.getOption("DB_VERSION"); if ( !currentProjectStatus || this.status !== "ready" || currentProjectStatus === "loading" || currentProjectStatus === "error" || - dbVersion !== DB_VERSION || !rootStore.user.localDBEnabled ) { info(`Project ${projectId} is loading, falling back to server`); From 052d5ec749e512eb7295374394e8c221da2478bd Mon Sep 17 00:00:00 2001 From: rahulramesha Date: Wed, 11 Sep 2024 18:14:21 +0530 Subject: [PATCH 095/111] change db version to number and remove workspaceInitPromise in storage.sqlite --- web/core/local-db/storage.sqlite.ts | 19 +++++-------------- web/core/local-db/utils/load-issues.ts | 6 +++--- 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/web/core/local-db/storage.sqlite.ts b/web/core/local-db/storage.sqlite.ts index 8e395b18882..0cfcadfd28a 100644 --- a/web/core/local-db/storage.sqlite.ts +++ b/web/core/local-db/storage.sqlite.ts @@ -21,7 +21,7 @@ declare module "@sqlite.org/sqlite-wasm" { export function sqlite3Worker1Promiser(...args: any): any; } -const DB_VERSION = "1"; +const DB_VERSION = 1; const PAGE_SIZE = 1000; const log = console.log; const error = console.error; @@ -38,7 +38,6 @@ export class Storage { dbName = "plane"; projectStatus: Record = {}; workspaceSlug: string = ""; - workspaceInitPromise: Promise | undefined; constructor() { this.db = null; @@ -49,7 +48,6 @@ export class Storage { this.status = undefined; this.projectStatus = {}; this.workspaceSlug = ""; - this.workspaceInitPromise = undefined; }; clearStorage = async () => { @@ -69,12 +67,8 @@ export class Storage { if (workspaceSlug !== this.workspaceSlug) { this.reset(); } - if (this.workspaceInitPromise) { - return this.workspaceInitPromise; - } - this.workspaceInitPromise = this._initialize(workspaceSlug); try { - await this.workspaceInitPromise; + await this._initialize(workspaceSlug); return true; } catch (err) { error(err); @@ -129,10 +123,10 @@ export class Storage { // dump DB of db version is matching const dbVersion = await this.getOption("DB_VERSION"); - if (dbVersion !== "" && dbVersion !== DB_VERSION) { + if (dbVersion !== "" && parseInt(dbVersion) !== DB_VERSION) { await this.clearStorage(); this.reset(); - this.workspaceInitPromise = this._initialize(workspaceSlug); + await this._initialize(workspaceSlug); return false; } @@ -144,7 +138,7 @@ export class Storage { // Your SQLite code here. await createTables(); - await this.setOption("DB_VERSION", DB_VERSION); + await this.setOption("DB_VERSION", DB_VERSION.toString()); } catch (err) { error(err); throw err; @@ -155,8 +149,6 @@ export class Storage { syncWorkspace = async () => { if (document.hidden || !rootStore.user.localDBEnabled) return; // return if the window gets hidden - - await this.workspaceInitPromise; loadWorkSpaceData(this.workspaceSlug); }; @@ -171,7 +163,6 @@ export class Storage { if (document.hidden || !rootStore.user.localDBEnabled) return false; // return if the window gets hidden try { - await this.workspaceInitPromise; const sync = this._syncIssues(projectId); this.setSync(projectId, sync); await sync; diff --git a/web/core/local-db/utils/load-issues.ts b/web/core/local-db/utils/load-issues.ts index 3f33a75bf7f..057e82747b8 100644 --- a/web/core/local-db/utils/load-issues.ts +++ b/web/core/local-db/utils/load-issues.ts @@ -16,7 +16,7 @@ export const addIssue = async (issue: any) => { }; export const addIssuesBulk = async (issues: any, batchSize = 100) => { - if (document.hidden || !rootStore.user.localDBEnabled) return; + if (!rootStore.user.localDBEnabled) return; for (let i = 0; i < issues.length; i += batchSize) { const batch = issues.slice(i, i + batchSize); @@ -32,7 +32,7 @@ export const addIssuesBulk = async (issues: any, batchSize = 100) => { } }; export const deleteIssueFromLocal = async (issue_id: any) => { - if (document.hidden || !rootStore.user.localDBEnabled) return; + if (!rootStore.user.localDBEnabled) return; const deleteQuery = `delete from issues where id='${issue_id}'`; const deleteMetaQuery = `delete from issue_meta where issue_id='${issue_id}'`; @@ -53,7 +53,7 @@ export const updateIssue = async (issue: TIssue) => { }; export const syncDeletesToLocal = async (workspaceId: string, projectId: string, queries: any) => { - if (document.hidden || !rootStore.user.localDBEnabled) return; + if (!rootStore.user.localDBEnabled) return; const issueService = new IssueService(); const response = await issueService.getDeletedIssues(workspaceId, projectId, queries); From 1e74d88bf4b0ecaca2e00094a41b9019810b8b2a Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Fri, 13 Sep 2024 13:09:17 +0530 Subject: [PATCH 096/111] - Sync the entire workspace in the background - Add get sub issue method with distribution --- web/core/local-db/storage.sqlite.ts | 39 ++++++++++++++++++++------- web/core/local-db/utils/data.utils.ts | 30 +++++++++++++++++++++ web/core/local-db/utils/utils.ts | 2 ++ 3 files changed, 62 insertions(+), 9 deletions(-) create mode 100644 web/core/local-db/utils/data.utils.ts diff --git a/web/core/local-db/storage.sqlite.ts b/web/core/local-db/storage.sqlite.ts index 0cfcadfd28a..74ad0b455be 100644 --- a/web/core/local-db/storage.sqlite.ts +++ b/web/core/local-db/storage.sqlite.ts @@ -9,13 +9,14 @@ import { rootStore } from "@/lib/store-context"; import { IssueService } from "@/services/issue/issue.service"; // import { ARRAY_FIELDS } from "./utils/constants"; +import { getProjectIds, getSubIssuesWithDistribution } from "./utils/data.utils"; import createIndexes from "./utils/indexes"; import { addIssuesBulk, syncDeletesToLocal } from "./utils/load-issues"; import { loadWorkSpaceData } from "./utils/load-workspace"; import { issueFilterCountQueryConstructor, issueFilterQueryConstructor } from "./utils/query-constructor"; import { runQuery } from "./utils/query-executor"; import { createTables } from "./utils/tables"; -import { getGroupedIssueResults, getSubGroupedIssueResults } from "./utils/utils"; +import { delay, getGroupedIssueResults, getSubGroupedIssueResults } from "./utils/utils"; declare module "@sqlite.org/sqlite-wasm" { export function sqlite3Worker1Promiser(...args: any): any; @@ -23,6 +24,7 @@ declare module "@sqlite.org/sqlite-wasm" { const DB_VERSION = 1; const PAGE_SIZE = 1000; +const BATCH_SIZE = 200; const log = console.log; const error = console.error; const info = console.info; @@ -54,7 +56,7 @@ export class Storage { try { const storageManager = window.navigator.storage; const fileSystemDirectoryHandle = await storageManager.getDirectory(); - //@ts-ignore + //@ts-expect-error await fileSystemDirectoryHandle.remove({ recursive: true }); } catch (e) { console.error("Error clearing sqlite sync storage", e); @@ -152,11 +154,22 @@ export class Storage { loadWorkSpaceData(this.workspaceSlug); }; - syncProject = (projectId: string) => { + syncProject = async (projectId: string) => { if (document.hidden || !rootStore.user.localDBEnabled) return false; // return if the window gets hidden // Load labels, members, states, modules, cycles - this.syncIssues(projectId); + await this.syncIssues(projectId); + + // Sync rest of the projects + const projects = await getProjectIds(); + + // Exclude the one we just synced + const projectsToSync = projects.filter((p) => p !== projectId); + for (const project of projectsToSync) { + await delay(8000); + await this.syncIssues(project); + } + this.setOption("workspace_synced_at", new Date().toISOString()); }; syncIssues = async (projectId: string) => { @@ -206,7 +219,7 @@ export class Storage { const issueService = new IssueService(); const response = await issueService.getIssuesForSync(this.workspaceSlug, projectId, queryParams); - addIssuesBulk(response.results, 500); + addIssuesBulk(response.results, BATCH_SIZE); if (response.total_pages > 1) { const promiseArray = []; @@ -216,7 +229,7 @@ export class Storage { } const pages = await Promise.all(promiseArray); for (const page of pages) { - await addIssuesBulk(page.results, 500); + await addIssuesBulk(page.results, BATCH_SIZE); } } @@ -298,9 +311,7 @@ export class Storage { EIssueGroupBYServerToProperty[sub_group_by as keyof typeof EIssueGroupBYServerToProperty]; const parsingStart = performance.now(); - let issueResults = issuesRaw.map((issue: any) => { - return formatLocalIssue(issue); - }); + let issueResults = issuesRaw.map((issue: any) => formatLocalIssue(issue)); console.log("#### Issue Results", issueResults.length); @@ -354,6 +365,16 @@ export class Storage { return; }; + + getSubIssues = async (workspaceSlug: string, projectId: string, issueId: string) => { + const workspace_synced_at = await this.getOption("workspace_synced_at"); + if (!workspace_synced_at) { + const issueService = new IssueService(); + return await issueService.subIssues(workspaceSlug, projectId, issueId); + } + return await getSubIssuesWithDistribution(issueId); + }; + getStatus = (projectId: string) => this.projectStatus[projectId]?.issues?.status || undefined; setStatus = (projectId: string, status: "loading" | "ready" | "error" | "syncing" | undefined = undefined) => { set(this.projectStatus, `${projectId}.issues.status`, status); diff --git a/web/core/local-db/utils/data.utils.ts b/web/core/local-db/utils/data.utils.ts new file mode 100644 index 00000000000..532bb630c2d --- /dev/null +++ b/web/core/local-db/utils/data.utils.ts @@ -0,0 +1,30 @@ +import { runQuery } from "./query-executor"; + +export const getProjectIds = async () => { + const q = `select project_id from states where project_id is not null group by project_id`; + return await runQuery(q); +}; + +export const getSubIssues = async (issueId: string) => { + const q = `select * from issues where parent_id = '${issueId}'`; + return await runQuery(q); +}; + +export const getSubIssueDistribution = async (issueId: string) => { + const q = `select s.'group', group_concat(i.id) as issues from issues i left join states s on s.id = i.state_id where i.parent_id = '${issueId}' group by s.'group'`; + + const result = await runQuery(q); + if (!result.length) { + return {}; + } + return result.reduce((acc: Record, item: { group: string; issues: string }) => { + acc[item.group] = item.issues.split(","); + return acc; + }, {}); +}; + +export const getSubIssuesWithDistribution = async (issueId: string) => { + const promises = [getSubIssues(issueId), getSubIssueDistribution(issueId)]; + const [sub_issues, state_distribution] = await Promise.all(promises); + return { sub_issues, state_distribution }; +}; diff --git a/web/core/local-db/utils/utils.ts b/web/core/local-db/utils/utils.ts index 0fb20f2999d..4c355ac95c9 100644 --- a/web/core/local-db/utils/utils.ts +++ b/web/core/local-db/utils/utils.ts @@ -130,3 +130,5 @@ export const getSubGroupedIssueResults = ( return subGroupedResults; }; + +export const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); From 8f0dee96cd1cb67726a3e8aa962acf5ca4370a23 Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Fri, 13 Sep 2024 13:18:50 +0530 Subject: [PATCH 097/111] Make changes to get issues for sync to match backend. --- web/core/services/issue/issue.service.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/web/core/services/issue/issue.service.ts b/web/core/services/issue/issue.service.ts index b0a0687dc84..551ab48edd8 100644 --- a/web/core/services/issue/issue.service.ts +++ b/web/core/services/issue/issue.service.ts @@ -58,8 +58,9 @@ export class IssueService extends APIService { queries?: any, config = {} ): Promise { + queries.project_id = projectId; return this.get( - `/api/workspaces/${workspaceSlug}/projects/${projectId}/v2/issues/`, + `/api/workspaces/${workspaceSlug}/v2/issues/`, { params: queries, }, From e53bc007b7c0ba8e2ef1188ae2de926323eafcfd Mon Sep 17 00:00:00 2001 From: gurusainath Date: Fri, 13 Sep 2024 13:30:30 +0530 Subject: [PATCH 098/111] chore: handled workspace and project in v2 paginted issues --- apiserver/plane/app/urls/issue.py | 4 ++-- apiserver/plane/app/views/issue/base.py | 29 +++++++++++++++++-------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/apiserver/plane/app/urls/issue.py b/apiserver/plane/app/urls/issue.py index edc7c409dd0..910eb91078c 100644 --- a/apiserver/plane/app/urls/issue.py +++ b/apiserver/plane/app/urls/issue.py @@ -40,9 +40,9 @@ ), name="project-issue", ), - # updated v1 paginated issues + # updated v2 paginated issues path( - "workspaces//projects//v2/issues/", + "workspaces//v2/issues/", IssuePaginatedViewSet.as_view({"get": "list"}), name="project-issues-paginated", ), diff --git a/apiserver/plane/app/views/issue/base.py b/apiserver/plane/app/views/issue/base.py index fcd4615df1e..b7a8ef55b79 100644 --- a/apiserver/plane/app/views/issue/base.py +++ b/apiserver/plane/app/views/issue/base.py @@ -700,13 +700,21 @@ def get(self, request, slug, project_id): class IssuePaginatedViewSet(BaseViewSet): def get_queryset(self): workspace_slug = self.kwargs.get("slug") - project_id = self.kwargs.get("project_id") + + # getting the project_id from the request params + project_id = self.request.GET.get("project_id", None) + + issue_queryset = Issue.issue_objects.filter( + workspace__slug=workspace_slug + ) + + if project_id: + issue_queryset = issue_queryset.filter(project_id=project_id) return ( - Issue.issue_objects.filter( - workspace__slug=workspace_slug, project_id=project_id + issue_queryset.select_related( + "workspace", "project", "state", "parent" ) - .select_related("workspace", "project", "state", "parent") .prefetch_related("assignees", "labels", "issue_module__module") .annotate(cycle_id=F("issue_cycle__cycle_id")) .annotate( @@ -744,8 +752,8 @@ def process_paginated_result(self, fields, results, timezone): return paginated_data - @allow_permission([ROLE.ADMIN, ROLE.MEMBER, ROLE.VIEWER, ROLE.GUEST]) - def list(self, request, slug, project_id): + def list(self, request, slug): + project_id = self.request.GET.get("project_id", None) cursor = request.GET.get("cursor", None) is_description_required = request.GET.get("description", False) updated_at = request.GET.get("updated_at__gt", None) @@ -784,9 +792,12 @@ def list(self, request, slug, project_id): required_fields.append("description_html") # querying issues - base_queryset = Issue.issue_objects.filter( - workspace__slug=slug, project_id=project_id - ).order_by("updated_at") + base_queryset = Issue.issue_objects.filter(workspace__slug=slug) + + if project_id: + base_queryset = base_queryset.filter(project_id=project_id) + + base_queryset = base_queryset.order_by("updated_at") queryset = self.get_queryset().order_by("updated_at") # filtering issues by greater then updated_at given by the user From cb3e8be4827720f98cd7d8eb93a7491621770655 Mon Sep 17 00:00:00 2001 From: rahulramesha Date: Fri, 13 Sep 2024 19:27:59 +0530 Subject: [PATCH 099/111] disable issue description and title until fetched from server --- .../components/issues/peek-overview/view.tsx | 6 +-- .../store/issue/issue-details/issue.store.ts | 39 ++++--------------- 2 files changed, 11 insertions(+), 34 deletions(-) diff --git a/web/core/components/issues/peek-overview/view.tsx b/web/core/components/issues/peek-overview/view.tsx index c03582979c1..2c64ecaaf4a 100644 --- a/web/core/components/issues/peek-overview/view.tsx +++ b/web/core/components/issues/peek-overview/view.tsx @@ -60,7 +60,7 @@ export const IssueView: FC = observer((props) => { isArchiveIssueModalOpen, toggleDeleteIssueModal, toggleArchiveIssueModal, - issue: { getIssueById }, + issue: { getIssueById, isLocalDBIssueDescription }, } = useIssueDetail(); const issue = getIssueById(issueId); // remove peek id @@ -178,7 +178,7 @@ export const IssueView: FC = observer((props) => { projectId={projectId} issueId={issueId} issueOperations={issueOperations} - disabled={disabled || is_archived} + disabled={disabled || is_archived || isLocalDBIssueDescription} isArchived={is_archived} isSubmitting={isSubmitting} setIsSubmitting={(value) => setIsSubmitting(value)} @@ -217,7 +217,7 @@ export const IssueView: FC = observer((props) => { projectId={projectId} issueId={issueId} issueOperations={issueOperations} - disabled={disabled || is_archived} + disabled={disabled || is_archived || isLocalDBIssueDescription} isArchived={is_archived} isSubmitting={isSubmitting} setIsSubmitting={(value) => setIsSubmitting(value)} diff --git a/web/core/store/issue/issue-details/issue.store.ts b/web/core/store/issue/issue-details/issue.store.ts index 3c4b4794f81..f19a823506e 100644 --- a/web/core/store/issue/issue-details/issue.store.ts +++ b/web/core/store/issue/issue-details/issue.store.ts @@ -36,12 +36,14 @@ export interface IIssueStoreActions { export interface IIssueStore extends IIssueStoreActions { isFetchingIssueDetails: boolean; + isLocalDBIssueDescription: boolean; // helper methods getIssueById: (issueId: string) => TIssue | undefined; } export class IssueStore implements IIssueStore { isFetchingIssueDetails: boolean = false; + isLocalDBIssueDescription: boolean = false; // root store rootIssueDetailStore: IIssueDetail; // services @@ -88,6 +90,7 @@ export class IssueStore implements IIssueStore { if (issue) { this.addIssueToStore(issue); + this.isLocalDBIssueDescription = true; } if (issueType === "ARCHIVED") @@ -97,37 +100,9 @@ export class IssueStore implements IIssueStore { else issue = await this.issueService.retrieve(workspaceSlug, projectId, issueId, query); if (!issue) throw new Error("Issue not found"); - this.addIssueToStore(issue); - const issuePayload: TIssue = { - id: issue?.id, - sequence_id: issue?.sequence_id, - name: issue?.name, - description_html: issue?.description_html, - sort_order: issue?.sort_order, - state_id: issue?.state_id, - priority: issue?.priority, - label_ids: issue?.label_ids, - assignee_ids: issue?.assignee_ids, - estimate_point: issue?.estimate_point, - sub_issues_count: issue?.sub_issues_count, - attachment_count: issue?.attachment_count, - link_count: issue?.link_count, - project_id: issue?.project_id, - parent_id: issue?.parent_id, - cycle_id: issue?.cycle_id, - module_ids: issue?.module_ids, - type_id: issue?.type_id, - created_at: issue?.created_at, - updated_at: issue?.updated_at, - start_date: issue?.start_date, - target_date: issue?.target_date, - completed_at: issue?.completed_at, - archived_at: issue?.archived_at, - created_by: issue?.created_by, - updated_by: issue?.updated_by, - is_draft: issue?.is_draft, - is_subscribed: issue?.is_subscribed, - }; + + const issuePayload = this.addIssueToStore(issue); + this.isLocalDBIssueDescription = false; this.rootIssueDetailStore.rootIssueStore.issues.addIssue([issuePayload], shouldReplace); @@ -206,6 +181,8 @@ export class IssueStore implements IIssueStore { this.rootIssueDetailStore.rootIssueStore.issues.addIssue([issuePayload], true); this.isFetchingIssueDetails = false; + + return issuePayload; }; updateIssue = async (workspaceSlug: string, projectId: string, issueId: string, data: Partial) => { From 6ccdaa45a635d244c73b8eebd4c9fa87876a5bd8 Mon Sep 17 00:00:00 2001 From: rahulramesha Date: Fri, 13 Sep 2024 20:09:20 +0530 Subject: [PATCH 100/111] sync issues post bulk operations --- web/core/services/issue/issue.service.ts | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/web/core/services/issue/issue.service.ts b/web/core/services/issue/issue.service.ts index 551ab48edd8..0d925f6b97b 100644 --- a/web/core/services/issue/issue.service.ts +++ b/web/core/services/issue/issue.service.ts @@ -13,7 +13,7 @@ import { API_BASE_URL } from "@/helpers/common.helper"; import { persistence } from "@/local-db/storage.sqlite"; // services -import { addIssue, addIssuesBulk, deleteIssueFromLocal, updateIssue } from "@/local-db/utils/load-issues"; +import { addIssue, addIssuesBulk, deleteIssueFromLocal } from "@/local-db/utils/load-issues"; import { updatePersistentLayer } from "@/local-db/utils/utils"; import { APIService } from "@/services/api.service"; @@ -310,7 +310,10 @@ export class IssueService extends APIService { async bulkOperations(workspaceSlug: string, projectId: string, data: TBulkOperationsPayload): Promise { return this.post(`/api/workspaces/${workspaceSlug}/projects/${projectId}/bulk-operation-issues/`, data) - .then((response) => response?.data) + .then((response) => { + persistence.syncIssues(projectId); + return response?.data; + }) .catch((error) => { throw error?.response?.data; }); @@ -324,7 +327,10 @@ export class IssueService extends APIService { } ): Promise { return this.delete(`/api/workspaces/${workspaceSlug}/projects/${projectId}/bulk-delete-issues/`, data) - .then((response) => response?.data) + .then((response) => { + persistence.syncIssues(projectId); + return response?.data; + }) .catch((error) => { throw error?.response?.data; }); @@ -340,7 +346,10 @@ export class IssueService extends APIService { archived_at: string; }> { return this.post(`/api/workspaces/${workspaceSlug}/projects/${projectId}/bulk-archive-issues/`, data) - .then((response) => response?.data) + .then((response) => { + persistence.syncIssues(projectId); + return response?.data; + }) .catch((error) => { throw error?.response?.data; }); From 5b957f4e0d146f441b61c00e3f6e9c688b86013e Mon Sep 17 00:00:00 2001 From: rahulramesha Date: Fri, 13 Sep 2024 20:53:09 +0530 Subject: [PATCH 101/111] fix server error --- apiserver/plane/app/views/issue/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apiserver/plane/app/views/issue/base.py b/apiserver/plane/app/views/issue/base.py index 7911783bf94..8adc5a84200 100644 --- a/apiserver/plane/app/views/issue/base.py +++ b/apiserver/plane/app/views/issue/base.py @@ -720,7 +720,7 @@ def delete(self, request, slug, project_id): class DeletedIssuesListViewSet(BaseAPIView): - @allow_permission([ROLE.ADMIN, ROLE.MEMBER, ROLE.GUEST, ROLE.VIEWER]) + @allow_permission([ROLE.ADMIN, ROLE.MEMBER, ROLE.GUEST]) def get(self, request, slug, project_id): filters = {} if request.GET.get("updated_at__gt", None) is not None: From c2fd7814ddadf834be01d33cba3c15aa5d8f2525 Mon Sep 17 00:00:00 2001 From: rahulramesha Date: Fri, 13 Sep 2024 20:53:21 +0530 Subject: [PATCH 102/111] fix front end build --- web/core/local-db/storage.sqlite.ts | 2 +- web/core/store/issue/issue-details/issue.store.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/web/core/local-db/storage.sqlite.ts b/web/core/local-db/storage.sqlite.ts index 74ad0b455be..1d976e88010 100644 --- a/web/core/local-db/storage.sqlite.ts +++ b/web/core/local-db/storage.sqlite.ts @@ -164,7 +164,7 @@ export class Storage { const projects = await getProjectIds(); // Exclude the one we just synced - const projectsToSync = projects.filter((p) => p !== projectId); + const projectsToSync = projects.filter((p: string) => p !== projectId); for (const project of projectsToSync) { await delay(8000); await this.syncIssues(project); diff --git a/web/core/store/issue/issue-details/issue.store.ts b/web/core/store/issue/issue-details/issue.store.ts index 4cc944e3244..8833b6ea114 100644 --- a/web/core/store/issue/issue-details/issue.store.ts +++ b/web/core/store/issue/issue-details/issue.store.ts @@ -172,7 +172,7 @@ export class IssueStore implements IIssueStore { is_subscribed: issue?.is_subscribed, }; - this.rootIssueDetailStore.rootIssueStore.issues.addIssue([issuePayload], true); + this.rootIssueDetailStore.rootIssueStore.issues.addIssue([issuePayload]); this.isFetchingIssueDetails = false; return issuePayload; From 8390f6dde9aec251dedce89701e9192de9a7fa7f Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Wed, 18 Sep 2024 17:38:04 +0530 Subject: [PATCH 103/111] Remove full workspace sync --- web/core/local-db/storage.sqlite.ts | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/web/core/local-db/storage.sqlite.ts b/web/core/local-db/storage.sqlite.ts index 1d976e88010..1555020a538 100644 --- a/web/core/local-db/storage.sqlite.ts +++ b/web/core/local-db/storage.sqlite.ts @@ -9,14 +9,14 @@ import { rootStore } from "@/lib/store-context"; import { IssueService } from "@/services/issue/issue.service"; // import { ARRAY_FIELDS } from "./utils/constants"; -import { getProjectIds, getSubIssuesWithDistribution } from "./utils/data.utils"; +import { getSubIssuesWithDistribution } from "./utils/data.utils"; import createIndexes from "./utils/indexes"; import { addIssuesBulk, syncDeletesToLocal } from "./utils/load-issues"; import { loadWorkSpaceData } from "./utils/load-workspace"; import { issueFilterCountQueryConstructor, issueFilterQueryConstructor } from "./utils/query-constructor"; import { runQuery } from "./utils/query-executor"; import { createTables } from "./utils/tables"; -import { delay, getGroupedIssueResults, getSubGroupedIssueResults } from "./utils/utils"; +import { getGroupedIssueResults, getSubGroupedIssueResults } from "./utils/utils"; declare module "@sqlite.org/sqlite-wasm" { export function sqlite3Worker1Promiser(...args: any): any; @@ -160,16 +160,16 @@ export class Storage { // Load labels, members, states, modules, cycles await this.syncIssues(projectId); - // Sync rest of the projects - const projects = await getProjectIds(); + // // Sync rest of the projects + // const projects = await getProjectIds(); - // Exclude the one we just synced - const projectsToSync = projects.filter((p: string) => p !== projectId); - for (const project of projectsToSync) { - await delay(8000); - await this.syncIssues(project); - } - this.setOption("workspace_synced_at", new Date().toISOString()); + // // Exclude the one we just synced + // const projectsToSync = projects.filter((p: string) => p !== projectId); + // for (const project of projectsToSync) { + // await delay(8000); + // await this.syncIssues(project); + // } + // this.setOption("workspace_synced_at", new Date().toISOString()); }; syncIssues = async (projectId: string) => { From d4e71f507f91a8360587c9b641e50ee68c0e7b2f Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Mon, 23 Sep 2024 18:33:50 +0530 Subject: [PATCH 104/111] - Remove the toast message on sync. - Update the disable local message. --- web/app/profile/page.tsx | 4 +++- web/core/local-db/storage.sqlite.ts | 6 ------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/web/app/profile/page.tsx b/web/app/profile/page.tsx index b9795d06a40..b675b188740 100644 --- a/web/app/profile/page.tsx +++ b/web/app/profile/page.tsx @@ -417,6 +417,7 @@ const ProfileSettingsPage = observer(() => {
+ {({ open }) => ( <> @@ -436,7 +437,8 @@ const ProfileSettingsPage = observer(() => {
- Enabling this will let the application cache data on system for faster loading experience + Toggled on by default to keep Plane performant. Disable this if you are facing any issues with + Plane. Applicable only to this device. toggleLocalDB()} />
diff --git a/web/core/local-db/storage.sqlite.ts b/web/core/local-db/storage.sqlite.ts index 1555020a538..1a2bf9219ba 100644 --- a/web/core/local-db/storage.sqlite.ts +++ b/web/core/local-db/storage.sqlite.ts @@ -2,7 +2,6 @@ import set from "lodash/set"; // plane import { EIssueGroupBYServerToProperty } from "@plane/constants"; import { TIssue } from "@plane/types"; -import { setToast, TOAST_TYPE } from "@plane/ui"; // lib import { rootStore } from "@/lib/store-context"; // services @@ -240,11 +239,6 @@ export class Storage { if (status === "loading") { await createIndexes(); - setToast({ - title: "Project synced", - message: `in ${Math.round((performance.now() - start) / 1000)}s`, - type: TOAST_TYPE.SUCCESS, - }); } this.setOption(projectId, "ready"); this.setStatus(projectId, "ready"); From 9aa8a7cfd4215c9cc8c680f2e01184e4b0e07b35 Mon Sep 17 00:00:00 2001 From: rahulramesha Date: Mon, 23 Sep 2024 19:47:00 +0530 Subject: [PATCH 105/111] Add Hardcoded constant to disable the local db caching --- web/app/profile/page.tsx | 62 +++++++++++++++++++----------------- web/ce/constants/issues.ts | 3 ++ web/core/store/user/index.ts | 3 +- 3 files changed, 37 insertions(+), 31 deletions(-) diff --git a/web/app/profile/page.tsx b/web/app/profile/page.tsx index b675b188740..a516c4f9bbc 100644 --- a/web/app/profile/page.tsx +++ b/web/app/profile/page.tsx @@ -35,6 +35,7 @@ import { useUser, useUserSettings } from "@/hooks/store"; // import { ProfileSettingsLayout } from "@/layouts/settings-layout"; // layouts import { FileService } from "@/services/file.service"; +import { ENABLE_LOCAL_DB_CACHE } from "ee/constants/issues"; // services // types @@ -417,36 +418,37 @@ const ProfileSettingsPage = observer(() => {
- - - {({ open }) => ( - <> - - Local Cache - - - - -
- - Toggled on by default to keep Plane performant. Disable this if you are facing any issues with - Plane. Applicable only to this device. - - toggleLocalDB()} /> -
- - - - )} - + {ENABLE_LOCAL_DB_CACHE && ( + + {({ open }) => ( + <> + + Local Cache + + + + +
+ + Toggled on by default to keep Plane performant. Disable this if you are facing any issues with + Plane. Applicable only to this device. + + toggleLocalDB()} /> +
+ + + + )} + + )} {({ open }) => ( <> diff --git a/web/ce/constants/issues.ts b/web/ce/constants/issues.ts index ac31c1ec9e6..a139dc86a14 100644 --- a/web/ce/constants/issues.ts +++ b/web/ce/constants/issues.ts @@ -30,3 +30,6 @@ export const filterActivityOnSelectedFilters = ( filter: TActivityFilters[] ): TIssueActivityComment[] => activity.filter((activity) => filter.includes(activity.activity_type as TActivityFilters)); + +// boolean to decide if the local db cache is enabled +export const ENABLE_LOCAL_DB_CACHE = false; diff --git a/web/core/store/user/index.ts b/web/core/store/user/index.ts index eac764a3db3..4bc58f5d68a 100644 --- a/web/core/store/user/index.ts +++ b/web/core/store/user/index.ts @@ -19,6 +19,7 @@ import { IAccountStore } from "@/store/user/account.store"; import { ProfileStore, IUserProfileStore } from "@/store/user/profile.store"; import { IUserPermissionStore, UserPermissionStore } from "./permissions.store"; import { IUserSettingsStore, UserSettingsStore } from "./settings.store"; +import { ENABLE_LOCAL_DB_CACHE } from "@/plane-web/constants/issues"; type TUserErrorStatus = { status: string; @@ -277,6 +278,6 @@ export class UserStore implements IUserStore { } get localDBEnabled() { - return this.userSettings.canUseLocalDB; + return ENABLE_LOCAL_DB_CACHE && this.userSettings.canUseLocalDB; } } From f2f2fca14e731ab2e6d6ef5b81adc00c098cb339 Mon Sep 17 00:00:00 2001 From: rahulramesha Date: Mon, 23 Sep 2024 19:47:13 +0530 Subject: [PATCH 106/111] fix lint errors --- .../archives/issues/(detail)/[archivedIssueId]/page.tsx | 2 +- .../(detail)/[projectId]/issues/(detail)/[issueId]/page.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/web/app/[workspaceSlug]/(projects)/projects/(detail)/[projectId]/archives/issues/(detail)/[archivedIssueId]/page.tsx b/web/app/[workspaceSlug]/(projects)/projects/(detail)/[projectId]/archives/issues/(detail)/[archivedIssueId]/page.tsx index cff8b0f8e94..69d24b167b2 100644 --- a/web/app/[workspaceSlug]/(projects)/projects/(detail)/[projectId]/archives/issues/(detail)/[archivedIssueId]/page.tsx +++ b/web/app/[workspaceSlug]/(projects)/projects/(detail)/[projectId]/archives/issues/(detail)/[archivedIssueId]/page.tsx @@ -24,7 +24,7 @@ const ArchivedIssueDetailsPage = observer(() => { const { getProjectById } = useProject(); - const { isLoading } = useSWR( + useSWR( workspaceSlug && projectId && archivedIssueId ? `ARCHIVED_ISSUE_DETAIL_${workspaceSlug}_${projectId}_${archivedIssueId}` : null, diff --git a/web/app/[workspaceSlug]/(projects)/projects/(detail)/[projectId]/issues/(detail)/[issueId]/page.tsx b/web/app/[workspaceSlug]/(projects)/projects/(detail)/[projectId]/issues/(detail)/[issueId]/page.tsx index 84d9981cd32..7956c96fb86 100644 --- a/web/app/[workspaceSlug]/(projects)/projects/(detail)/[projectId]/issues/(detail)/[issueId]/page.tsx +++ b/web/app/[workspaceSlug]/(projects)/projects/(detail)/[projectId]/issues/(detail)/[issueId]/page.tsx @@ -32,7 +32,7 @@ const IssueDetailsPage = observer(() => { const { getProjectById } = useProject(); const { toggleIssueDetailSidebar, issueDetailSidebarCollapsed } = useAppTheme(); // fetching issue details - const { isLoading, error } = useSWR( + const { error } = useSWR( workspaceSlug && projectId && issueId ? `ISSUE_DETAIL_${workspaceSlug}_${projectId}_${issueId}` : null, workspaceSlug && projectId && issueId ? () => fetchIssue(workspaceSlug.toString(), projectId.toString(), issueId.toString()) From 00685a8b7ac5ab984d651ca1beeb5c20137a5785 Mon Sep 17 00:00:00 2001 From: Satish Gandham Date: Tue, 24 Sep 2024 13:57:20 +0530 Subject: [PATCH 107/111] Fix order by in grouping --- web/core/local-db/utils/query.utils.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/core/local-db/utils/query.utils.ts b/web/core/local-db/utils/query.utils.ts index ec834f7d3fd..eea6fc16ce9 100644 --- a/web/core/local-db/utils/query.utils.ts +++ b/web/core/local-db/utils/query.utils.ts @@ -47,9 +47,9 @@ export const getOrderByFragment = (order_by: string, table = "") => { if (!order_by) return orderByString; if (order_by.startsWith("-")) { - orderByString += ` ORDER BY ${wrapDateTime(order_by.slice(1))} DESC NULLS LAST, ${table}created_at DESC`; + orderByString += ` ORDER BY ${wrapDateTime(order_by.slice(1))} DESC NULLS LAST, datetime(${table}created_at) DESC`; } else { - orderByString += ` ORDER BY ${wrapDateTime(order_by)} ASC NULLS LAST, ${table}created_at DESC`; + orderByString += ` ORDER BY ${wrapDateTime(order_by)} ASC NULLS LAST, datetime(${table}created_at) DESC`; } return orderByString; }; From b1d8edbc976212c59ff3d4b3c2efb4b401810cd3 Mon Sep 17 00:00:00 2001 From: rahulramesha Date: Tue, 24 Sep 2024 15:13:19 +0530 Subject: [PATCH 108/111] update yarn lock --- yarn.lock | 39 +++++++-------------------------------- 1 file changed, 7 insertions(+), 32 deletions(-) diff --git a/yarn.lock b/yarn.lock index 103617e01af..0c4a00c79ac 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2924,10 +2924,10 @@ resolved "https://registry.yarnpkg.com/@sqlite.org/sqlite-wasm/-/sqlite-wasm-3.46.0-build2.tgz#f84c3014f3fed6db08fc585d67e386d39e3956bf" integrity sha512-10s/u/Main1RGO+jjzK+mgC/zh1ls1CEnq3Dujr03TwvzLg+j4FAohOmlYkQj8KQOj1vGR9cuB9F8tVBTwVGVA== -"@storybook/addon-actions@8.2.9": - version "8.2.9" - resolved "https://registry.yarnpkg.com/@storybook/addon-actions/-/addon-actions-8.2.9.tgz#5a27f07f276ec776fb768f5da9bfe2c43fe3e851" - integrity sha512-eh2teOqjga7aoClDVV+/b1gHJqsPwjiU1t+Hg/l4i2CkaBUNdYMEL90nR6fgReOdvvL5YhcPwJ8w38f9TrQcoQ== +"@storybook/addon-actions@8.3.2": + version "8.3.2" + resolved "https://registry.yarnpkg.com/@storybook/addon-actions/-/addon-actions-8.3.2.tgz#bded2d778f3c9309334d8e378a55723d25907f50" + integrity sha512-Ds2lNyEpeVO0TexoXEHpE3kRcA7rJm5X5nWz4PdvF7kiC1aX5ZMy2qEPZOH6Jvalysm+PChw4Ib+lCaoIFGOJg== dependencies: "@storybook/global" "^5.0.0" "@types/uuid" "^9.0.1" @@ -10868,16 +10868,7 @@ streamx@^2.15.0, streamx@^2.20.0: optionalDependencies: bare-events "^2.2.0" -"string-width-cjs@npm:string-width@^4.2.0": - version "4.2.3" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" - -string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -10964,14 +10955,7 @@ string_decoder@^1.1.1, string_decoder@^1.3.0: dependencies: safe-buffer "~5.2.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1": - version "6.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - -strip-ansi@^6.0.0, strip-ansi@^6.0.1: +"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -12155,16 +12139,7 @@ word-wrap@^1.2.5: resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.5.tgz#d2c45c6dd4fbce621a66f136cbe328afd0410b34" integrity sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA== -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": - version "7.0.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" - integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - -wrap-ansi@^7.0.0: +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== From 7fd7e678727b38b50f68e106aec41e108b825178 Mon Sep 17 00:00:00 2001 From: rahulramesha Date: Tue, 24 Sep 2024 15:13:27 +0530 Subject: [PATCH 109/111] fix build --- web/core/components/issues/issue-layouts/kanban/block.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/web/core/components/issues/issue-layouts/kanban/block.tsx b/web/core/components/issues/issue-layouts/kanban/block.tsx index e676d58c6cf..29696b3d4f7 100644 --- a/web/core/components/issues/issue-layouts/kanban/block.tsx +++ b/web/core/components/issues/issue-layouts/kanban/block.tsx @@ -35,7 +35,6 @@ interface IssueBlockProps { displayProperties: IIssueDisplayProperties | undefined; draggableId: string; canDropOverIssue: boolean; - shouldRenderByDefault?: boolean; updateIssue: ((projectId: string | null, issueId: string, data: Partial) => Promise) | undefined; quickActions: TRenderQuickActions; canEditProperties: (projectId: string | undefined) => boolean; From 5a16b2e0c31531358b203eb5f0f7c49202e5172d Mon Sep 17 00:00:00 2001 From: rahulramesha Date: Tue, 24 Sep 2024 15:35:08 +0530 Subject: [PATCH 110/111] fix plane-web imports --- web/app/profile/page.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/app/profile/page.tsx b/web/app/profile/page.tsx index a516c4f9bbc..ccbd62446db 100644 --- a/web/app/profile/page.tsx +++ b/web/app/profile/page.tsx @@ -35,7 +35,7 @@ import { useUser, useUserSettings } from "@/hooks/store"; // import { ProfileSettingsLayout } from "@/layouts/settings-layout"; // layouts import { FileService } from "@/services/file.service"; -import { ENABLE_LOCAL_DB_CACHE } from "ee/constants/issues"; +import { ENABLE_LOCAL_DB_CACHE } from "@/plane-web/constants/issues"; // services // types From f220592883470ef0e9a22581b39579a7c343379c Mon Sep 17 00:00:00 2001 From: rahulramesha Date: Tue, 24 Sep 2024 15:49:55 +0530 Subject: [PATCH 111/111] address review comments --- web/core/components/issues/issue-layouts/kanban/default.tsx | 1 - web/core/store/issue/cycle/filter.store.ts | 2 +- web/core/store/issue/module/filter.store.ts | 2 +- web/core/store/issue/project-views/filter.store.ts | 2 +- web/core/store/issue/project/filter.store.ts | 2 +- 5 files changed, 4 insertions(+), 5 deletions(-) diff --git a/web/core/components/issues/issue-layouts/kanban/default.tsx b/web/core/components/issues/issue-layouts/kanban/default.tsx index cf43c37b64a..37897d602cb 100644 --- a/web/core/components/issues/issue-layouts/kanban/default.tsx +++ b/web/core/components/issues/issue-layouts/kanban/default.tsx @@ -1,5 +1,4 @@ import { MutableRefObject } from "react"; -import clone from "lodash/clone"; import { observer } from "mobx-react"; import { GroupByColumnTypes, diff --git a/web/core/store/issue/cycle/filter.store.ts b/web/core/store/issue/cycle/filter.store.ts index 8284a510627..d8073bd52cd 100644 --- a/web/core/store/issue/cycle/filter.store.ts +++ b/web/core/store/issue/cycle/filter.store.ts @@ -234,7 +234,7 @@ export class CycleIssuesFilter extends IssueFilterHelperStore implements ICycleI }); if (this.getShouldClearIssues(updatedDisplayFilters)) { - this.rootIssueStore.cycleIssues.clear(true, true); // clear issues for local db when some filters like layout changes + this.rootIssueStore.cycleIssues.clear(true, true); // clear issues for local store when some filters like layout changes } if (this.getShouldReFetchIssues(updatedDisplayFilters)) { diff --git a/web/core/store/issue/module/filter.store.ts b/web/core/store/issue/module/filter.store.ts index d04e714b9f6..3b9ca78c47f 100644 --- a/web/core/store/issue/module/filter.store.ts +++ b/web/core/store/issue/module/filter.store.ts @@ -237,7 +237,7 @@ export class ModuleIssuesFilter extends IssueFilterHelperStore implements IModul }); if (this.getShouldClearIssues(updatedDisplayFilters)) { - this.rootIssueStore.moduleIssues.clear(true, true); // clear issues for local db when some filters like layout changes + this.rootIssueStore.moduleIssues.clear(true, true); // clear issues for local store when some filters like layout changes } if (this.getShouldReFetchIssues(updatedDisplayFilters)) { diff --git a/web/core/store/issue/project-views/filter.store.ts b/web/core/store/issue/project-views/filter.store.ts index a6d3c47317a..18f19fb2f28 100644 --- a/web/core/store/issue/project-views/filter.store.ts +++ b/web/core/store/issue/project-views/filter.store.ts @@ -228,7 +228,7 @@ export class ProjectViewIssuesFilter extends IssueFilterHelperStore implements I }); if (this.getShouldClearIssues(updatedDisplayFilters)) { - this.rootIssueStore.projectIssues.clear(true, true); // clear issues for local db when some filters like layout changes + this.rootIssueStore.projectIssues.clear(true, true); // clear issues for local store when some filters like layout changes } if (this.getShouldReFetchIssues(updatedDisplayFilters)) { diff --git a/web/core/store/issue/project/filter.store.ts b/web/core/store/issue/project/filter.store.ts index 7973e439ab2..54b109d12af 100644 --- a/web/core/store/issue/project/filter.store.ts +++ b/web/core/store/issue/project/filter.store.ts @@ -221,7 +221,7 @@ export class ProjectIssuesFilter extends IssueFilterHelperStore implements IProj }); if (this.getShouldClearIssues(updatedDisplayFilters)) { - this.rootIssueStore.projectIssues.clear(true, true); // clear issues for local db when some filters like layout changes + this.rootIssueStore.projectIssues.clear(true, true); // clear issues for local store when some filters like layout changes } if (this.getShouldReFetchIssues(updatedDisplayFilters)) {