From ade2812e0257f9e58039c33b7ad73f464b15b590 Mon Sep 17 00:00:00 2001 From: Aaryan Khandelwal Date: Fri, 3 Jan 2025 18:50:57 +0530 Subject: [PATCH 1/2] chore: new convert doucment endpoint created --- live/src/core/helpers/convert.ts | 43 ++++++ live/src/core/types/common.d.ts | 5 + live/src/server.ts | 44 ++++-- packages/editor/src/core/helpers/yjs-utils.ts | 136 ++++++++++++++++++ packages/editor/src/core/helpers/yjs.ts | 16 --- packages/editor/src/index.ts | 2 +- packages/editor/src/lib.ts | 1 + 7 files changed, 216 insertions(+), 31 deletions(-) create mode 100644 live/src/core/helpers/convert.ts create mode 100644 packages/editor/src/core/helpers/yjs-utils.ts delete mode 100644 packages/editor/src/core/helpers/yjs.ts diff --git a/live/src/core/helpers/convert.ts b/live/src/core/helpers/convert.ts new file mode 100644 index 00000000000..b0a7cf18f3d --- /dev/null +++ b/live/src/core/helpers/convert.ts @@ -0,0 +1,43 @@ +// plane types +import { + getAllDocumentFormatsFromDocumentEditorBinaryData, + getAllDocumentFormatsFromRichTextEditorBinaryData, + getBinaryDataFromDocumentEditorHTMLString, + getBinaryDataFromRichTextEditorHTMLString, +} from "@plane/editor"; +import { TDocumentPayload } from "@plane/types"; + +type TArgs = { + document_html: string; + variant: "rich" | "document"; +}; + +export const convertHTMLDocumentToAllFormats = (args: TArgs): TDocumentPayload => { + const { document_html, variant } = args; + + let allFormats: TDocumentPayload; + + if (variant === "rich") { + const contentBinary = getBinaryDataFromRichTextEditorHTMLString(document_html); + const { contentBinaryEncoded, contentHTML, contentJSON } = + getAllDocumentFormatsFromRichTextEditorBinaryData(contentBinary); + allFormats = { + description: contentJSON, + description_html: contentHTML, + description_binary: contentBinaryEncoded, + }; + } else if (variant === "document") { + const contentBinary = getBinaryDataFromDocumentEditorHTMLString(document_html); + const { contentBinaryEncoded, contentHTML, contentJSON } = + getAllDocumentFormatsFromDocumentEditorBinaryData(contentBinary); + allFormats = { + description: contentJSON, + description_html: contentHTML, + description_binary: contentBinaryEncoded, + }; + } else { + throw new Error(`Invalid variant provided: ${variant}`); + } + + return allFormats; +}; diff --git a/live/src/core/types/common.d.ts b/live/src/core/types/common.d.ts index 3156060efb3..7c241605934 100644 --- a/live/src/core/types/common.d.ts +++ b/live/src/core/types/common.d.ts @@ -6,3 +6,8 @@ export type TDocumentTypes = "project_page" | TAdditionalDocumentTypes; export type HocusPocusServerContext = { cookie: string; }; + +export type TConvertDocumentRequestBody = { + document_html: string; + variant: "rich" | "document"; +}; diff --git a/live/src/server.ts b/live/src/server.ts index 1868b86c198..010b03fafa7 100644 --- a/live/src/server.ts +++ b/live/src/server.ts @@ -1,20 +1,19 @@ -import "@/core/config/sentry-config.js"; - -import express from "express"; -import expressWs from "express-ws"; import * as Sentry from "@sentry/node"; import compression from "compression"; -import helmet from "helmet"; - -// cors import cors from "cors"; - -// core hocuspocus server +import expressWs from "express-ws"; +import express from "express"; +import helmet from "helmet"; +// config +import "@/core/config/sentry-config.js"; +// hocuspocus server import { getHocusPocusServer } from "@/core/hocuspocus-server.js"; - // helpers +import { convertHTMLDocumentToAllFormats } from "@/core/helpers/convert.js"; import { logger, manualLogger } from "@/core/helpers/logger.js"; import { errorHandler } from "@/core/helpers/error-handler.js"; +// types +import { TConvertDocumentRequestBody } from "@/core/types/common.js"; const app = express(); expressWs(app); @@ -29,7 +28,7 @@ app.use( compression({ level: 6, threshold: 5 * 1000, - }), + }) ); // Logging middleware @@ -62,6 +61,25 @@ router.ws("/collaboration", (ws, req) => { } }); +router.post("/convert-document", (req, res) => { + const { document_html, variant } = req.body as TConvertDocumentRequestBody; + try { + if (document_html === undefined || variant === undefined) { + res.status(400).send({ + message: "Missing required fields", + }); + return; + } + const convertedDocument = convertHTMLDocumentToAllFormats(req.body); + res.status(200).json(convertedDocument); + } catch (error) { + manualLogger.error("Error in /resolve-document-conflicts endpoint:", error); + res.status(500).send({ + message: `Internal server error. ${error}`, + }); + } +}); + app.use(process.env.LIVE_BASE_PATH || "/live", router); app.use((_req, res) => { @@ -82,9 +100,7 @@ const gracefulShutdown = async () => { try { // Close the HocusPocus server WebSocket connections await HocusPocusServer.destroy(); - manualLogger.info( - "HocusPocus server WebSocket connections closed gracefully.", - ); + manualLogger.info("HocusPocus server WebSocket connections closed gracefully."); // Close the Express server liveServer.close(() => { diff --git a/packages/editor/src/core/helpers/yjs-utils.ts b/packages/editor/src/core/helpers/yjs-utils.ts new file mode 100644 index 00000000000..d06636fc342 --- /dev/null +++ b/packages/editor/src/core/helpers/yjs-utils.ts @@ -0,0 +1,136 @@ +import { getSchema } from "@tiptap/core"; +import { generateHTML, generateJSON } from "@tiptap/html"; +import { prosemirrorJSONToYDoc, yXmlFragmentToProseMirrorRootNode } from "y-prosemirror"; +import * as Y from "yjs"; +// extensions +import { + CoreEditorExtensionsWithoutProps, + DocumentEditorExtensionsWithoutProps, +} from "@/extensions/core-without-props"; + +// editor extension configs +const RICH_TEXT_EDITOR_EXTENSIONS = CoreEditorExtensionsWithoutProps; +const DOCUMENT_EDITOR_EXTENSIONS = [...CoreEditorExtensionsWithoutProps, ...DocumentEditorExtensionsWithoutProps]; +// editor schemas +const richTextEditorSchema = getSchema(RICH_TEXT_EDITOR_EXTENSIONS); +const documentEditorSchema = getSchema(DOCUMENT_EDITOR_EXTENSIONS); + +/** + * @description apply updates to a doc and return the updated doc in binary format + * @param {Uint8Array} document + * @param {Uint8Array} updates + * @returns {Uint8Array} + */ +export const applyUpdates = (document: Uint8Array, updates?: Uint8Array): Uint8Array => { + const yDoc = new Y.Doc(); + Y.applyUpdate(yDoc, document); + if (updates) { + Y.applyUpdate(yDoc, updates); + } + + const encodedDoc = Y.encodeStateAsUpdate(yDoc); + return encodedDoc; +}; + +/** + * @description this function encodes binary data to base64 string + * @param {Uint8Array} document + * @returns {string} + */ +export const convertBinaryDataToBase64String = (document: Uint8Array): string => + Buffer.from(document).toString("base64"); + +/** + * @description this function decodes base64 string to binary data + * @param {string} document + * @returns {ArrayBuffer} + */ +export const convertBase64StringToBinaryData = (document: string): ArrayBuffer => Buffer.from(document, "base64"); + +/** + * @description this function generates the binary equivalent of html content for the rich text editor + * @param {string} descriptionHTML + * @returns {Uint8Array} + */ +export const getBinaryDataFromRichTextEditorHTMLString = (descriptionHTML: string): Uint8Array => { + // convert HTML to JSON + const contentJSON = generateJSON(descriptionHTML ?? "

", RICH_TEXT_EDITOR_EXTENSIONS); + // convert JSON to Y.Doc format + const transformedData = prosemirrorJSONToYDoc(richTextEditorSchema, contentJSON, "default"); + // convert Y.Doc to Uint8Array format + const encodedData = Y.encodeStateAsUpdate(transformedData); + return encodedData; +}; + +/** + * @description this function generates the binary equivalent of html content for the document editor + * @param {string} descriptionHTML + * @returns {Uint8Array} + */ +export const getBinaryDataFromDocumentEditorHTMLString = (descriptionHTML: string): Uint8Array => { + // convert HTML to JSON + const contentJSON = generateJSON(descriptionHTML ?? "

", DOCUMENT_EDITOR_EXTENSIONS); + // convert JSON to Y.Doc format + const transformedData = prosemirrorJSONToYDoc(documentEditorSchema, contentJSON, "default"); + // convert Y.Doc to Uint8Array format + const encodedData = Y.encodeStateAsUpdate(transformedData); + return encodedData; +}; + +/** + * @description this function generates all document formats for the provided binary data for the rich text editor + * @param {Uint8Array} description + * @returns + */ +export const getAllDocumentFormatsFromRichTextEditorBinaryData = ( + description: Uint8Array +): { + contentBinaryEncoded: string; + contentJSON: object; + contentHTML: string; +} => { + // encode binary description data + const base64Data = convertBinaryDataToBase64String(description); + const yDoc = new Y.Doc(); + Y.applyUpdate(yDoc, description); + // convert to JSON + const type = yDoc.getXmlFragment("default"); + const contentJSON = yXmlFragmentToProseMirrorRootNode(type, richTextEditorSchema).toJSON(); + // convert to HTML + const contentHTML = generateHTML(contentJSON, RICH_TEXT_EDITOR_EXTENSIONS); + + return { + contentBinaryEncoded: base64Data, + contentJSON, + contentHTML, + }; +}; + +/** + * @description this function generates all document formats for the provided binary data for the document editor + * @param {Uint8Array} description + * @returns + */ +export const getAllDocumentFormatsFromDocumentEditorBinaryData = ( + description: Uint8Array +): { + contentBinaryEncoded: string; + contentJSON: object; + contentHTML: string; +} => { + // encode binary description data + const base64Data = convertBinaryDataToBase64String(description); + const yDoc = new Y.Doc(); + Y.applyUpdate(yDoc, description); + // convert to JSON + const type = yDoc.getXmlFragment("default"); + const contentJSON = yXmlFragmentToProseMirrorRootNode(type, documentEditorSchema).toJSON(); + // convert to HTML + const contentHTML = generateHTML(contentJSON, DOCUMENT_EDITOR_EXTENSIONS); + + return { + contentBinaryEncoded: base64Data, + contentJSON, + contentHTML, + }; +}; diff --git a/packages/editor/src/core/helpers/yjs.ts b/packages/editor/src/core/helpers/yjs.ts deleted file mode 100644 index ffd9367107d..00000000000 --- a/packages/editor/src/core/helpers/yjs.ts +++ /dev/null @@ -1,16 +0,0 @@ -import * as Y from "yjs"; - -/** - * @description apply updates to a doc and return the updated doc in base64(binary) format - * @param {Uint8Array} document - * @param {Uint8Array} updates - * @returns {string} base64(binary) form of the updated doc - */ -export const applyUpdates = (document: Uint8Array, updates: Uint8Array): Uint8Array => { - const yDoc = new Y.Doc(); - Y.applyUpdate(yDoc, document); - Y.applyUpdate(yDoc, updates); - - const encodedDoc = Y.encodeStateAsUpdate(yDoc); - return encodedDoc; -}; diff --git a/packages/editor/src/index.ts b/packages/editor/src/index.ts index 9dd0db267f2..a2a9afaf92a 100644 --- a/packages/editor/src/index.ts +++ b/packages/editor/src/index.ts @@ -24,7 +24,7 @@ export * from "@/constants/common"; // helpers export * from "@/helpers/common"; export * from "@/helpers/editor-commands"; -export * from "@/helpers/yjs"; +export * from "@/helpers/yjs-utils"; export * from "@/extensions/table/table"; // components diff --git a/packages/editor/src/lib.ts b/packages/editor/src/lib.ts index e32fa078508..44388a00eae 100644 --- a/packages/editor/src/lib.ts +++ b/packages/editor/src/lib.ts @@ -1,4 +1,5 @@ export * from "@/extensions/core-without-props"; export * from "@/constants/document-collaborative-events"; export * from "@/helpers/get-document-server-event"; +export * from "@/helpers/yjs-utils"; export * from "@/types/document-collaborative-events"; From 3c61b9734d17d4bf5294747b956b34898854de3f Mon Sep 17 00:00:00 2001 From: Aaryan Khandelwal Date: Fri, 3 Jan 2025 19:26:39 +0530 Subject: [PATCH 2/2] chore: update types --- .../{convert.ts => convert-document.ts} | 3 ++- live/src/core/types/common.d.ts | 2 +- live/src/server.ts | 18 ++++++++++++------ 3 files changed, 15 insertions(+), 8 deletions(-) rename live/src/core/helpers/{convert.ts => convert-document.ts} (98%) diff --git a/live/src/core/helpers/convert.ts b/live/src/core/helpers/convert-document.ts similarity index 98% rename from live/src/core/helpers/convert.ts rename to live/src/core/helpers/convert-document.ts index b0a7cf18f3d..12398919014 100644 --- a/live/src/core/helpers/convert.ts +++ b/live/src/core/helpers/convert-document.ts @@ -1,10 +1,11 @@ -// plane types +// plane editor import { getAllDocumentFormatsFromDocumentEditorBinaryData, getAllDocumentFormatsFromRichTextEditorBinaryData, getBinaryDataFromDocumentEditorHTMLString, getBinaryDataFromRichTextEditorHTMLString, } from "@plane/editor"; +// plane types import { TDocumentPayload } from "@plane/types"; type TArgs = { diff --git a/live/src/core/types/common.d.ts b/live/src/core/types/common.d.ts index 7c241605934..90fd335ae45 100644 --- a/live/src/core/types/common.d.ts +++ b/live/src/core/types/common.d.ts @@ -8,6 +8,6 @@ export type HocusPocusServerContext = { }; export type TConvertDocumentRequestBody = { - document_html: string; + description_html: string; variant: "rich" | "document"; }; diff --git a/live/src/server.ts b/live/src/server.ts index 010b03fafa7..93f56bdb572 100644 --- a/live/src/server.ts +++ b/live/src/server.ts @@ -9,7 +9,7 @@ import "@/core/config/sentry-config.js"; // hocuspocus server import { getHocusPocusServer } from "@/core/hocuspocus-server.js"; // helpers -import { convertHTMLDocumentToAllFormats } from "@/core/helpers/convert.js"; +import { convertHTMLDocumentToAllFormats } from "@/core/helpers/convert-document.js"; import { logger, manualLogger } from "@/core/helpers/logger.js"; import { errorHandler } from "@/core/helpers/error-handler.js"; // types @@ -62,18 +62,24 @@ router.ws("/collaboration", (ws, req) => { }); router.post("/convert-document", (req, res) => { - const { document_html, variant } = req.body as TConvertDocumentRequestBody; + const { description_html, variant } = req.body as TConvertDocumentRequestBody; try { - if (document_html === undefined || variant === undefined) { + if (description_html === undefined || variant === undefined) { res.status(400).send({ message: "Missing required fields", }); return; } - const convertedDocument = convertHTMLDocumentToAllFormats(req.body); - res.status(200).json(convertedDocument); + const { description, description_binary } = convertHTMLDocumentToAllFormats({ + document_html: description_html, + variant, + }); + res.status(200).json({ + description, + description_binary, + }); } catch (error) { - manualLogger.error("Error in /resolve-document-conflicts endpoint:", error); + manualLogger.error("Error in /convert-document endpoint:", error); res.status(500).send({ message: `Internal server error. ${error}`, });