From 3b91be8bae55a26d859aac448d998347c1f9639f Mon Sep 17 00:00:00 2001 From: Weihan Li Date: Sat, 26 Jul 2025 10:31:50 +0800 Subject: [PATCH 01/13] feat: json-records content type support --- src/model/events/body-formatting.ts | 41 ++++++++++++++++++++++++++-- src/model/events/content-types.ts | 19 ++++++++++++- src/model/events/stream-message.ts | 8 +++++- src/services/ui-worker-formatters.ts | 21 +++++++++++++- src/util/buffer.ts | 29 ++++++++++++++++++++ 5 files changed, 113 insertions(+), 5 deletions(-) diff --git a/src/model/events/body-formatting.ts b/src/model/events/body-formatting.ts index 82c557d1..b75af2ce 100644 --- a/src/model/events/body-formatting.ts +++ b/src/model/events/body-formatting.ts @@ -1,9 +1,9 @@ import { Headers } from '../../types'; import { styled } from '../../styles'; -import { ViewableContentType } from '../events/content-types'; +import { jsonRecordsSeparators, ViewableContentType } from '../events/content-types'; import { ObservablePromise, observablePromise } from '../../util/observable'; -import { bufferToString, bufferToHex } from '../../util/buffer'; +import { bufferToString, bufferToHex, splitBuffer } from '../../util/buffer'; import type { WorkerFormatterKey } from '../../services/ui-worker-formatters'; import { formatBufferAsync } from '../../services/ui-worker-api'; @@ -127,6 +127,43 @@ export const Formatters: { [key in ViewableContentType]: Formatter } = { } } }, + 'json-records': { + language: 'json', + cacheKey: Symbol('json-records'), + isEditApplicable: false, + render: (input: Buffer, headers?: Headers) => { + if (input.byteLength < 10_000) { + const inputAsString = bufferToString(input); + + try { + let records = new Array(); + jsonRecordsSeparators.forEach((separator) => { + splitBuffer(input, separator).forEach((recordBuffer: Buffer) => { + if (recordBuffer.length > 0) { + const record = recordBuffer.toString('utf-8'); + records.push(JSON.parse(record.trim())); + } + }); + }); + // For short-ish inputs, we return synchronously - conveniently this avoids + // showing the loading spinner that churns the layout in short content cases. + return JSON.stringify( + records, + null, + 2 + ); + // ^ Same logic as in UI-worker-formatter + } catch (e) { + // Fallback to showing the raw un-formatted: + return inputAsString; + } + } else { + return observablePromise( + formatBufferAsync(input, 'json', headers) + ); + } + } + }, javascript: { language: 'javascript', cacheKey: Symbol('javascript'), diff --git a/src/model/events/content-types.ts b/src/model/events/content-types.ts index 09520bc3..cd353294 100644 --- a/src/model/events/content-types.ts +++ b/src/model/events/content-types.ts @@ -51,7 +51,9 @@ export type ViewableContentType = | 'yaml' | 'image' | 'protobuf' - | 'grpc-proto'; + | 'grpc-proto' + | 'json-records' + ; export const EditableContentTypes = [ 'text', @@ -122,6 +124,10 @@ const mimeTypeToContentTypeMap: { [mimeType: string]: ViewableContentType } = { 'application/octet-stream': 'raw' } as const; +export const jsonRecordsSeparators = [ + 0x1E +]; + export function getContentType(mimeType: string | undefined): ViewableContentType | undefined { const baseContentType = getBaseContentType(mimeType); return mimeTypeToContentTypeMap[baseContentType!]; @@ -141,6 +147,7 @@ export function getEditableContentType(mimeType: string | undefined): EditableCo export function getContentEditorName(contentType: ViewableContentType): string { return contentType === 'raw' ? 'Hex' + : contentType === 'json-records' ? 'JSON Records' : contentType === 'json' ? 'JSON' : contentType === 'css' ? 'CSS' : contentType === 'url-encoded' ? 'URL-Encoded' @@ -189,6 +196,16 @@ export function getCompatibleTypes( // Examine the first char of the body, assuming it's ascii const firstChar = body && body.subarray(0, 1).toString('ascii'); + // Allow optionally formatting non-JSON-records as JSON-records, if it looks like it might be + if (body && body.length > 2 && firstChar === '{' + && jsonRecordsSeparators.indexOf(body[body.length - 1]) > -1 + ) { + const secondToLastChar = body.subarray(body.length - 2, body.length - 1).toString('ascii'); + if (secondToLastChar === '}') { + types.add('json-records'); + } + } + // Allow optionally formatting non-JSON as JSON, if it looks like it might be if (firstChar === '{' || firstChar === '[') { types.add('json'); diff --git a/src/model/events/stream-message.ts b/src/model/events/stream-message.ts index 3b1a79ab..0be8df9f 100644 --- a/src/model/events/stream-message.ts +++ b/src/model/events/stream-message.ts @@ -3,6 +3,7 @@ import { computed, observable } from 'mobx'; import { InputStreamMessage } from "../../types"; import { asBuffer } from '../../util/buffer'; import { ObservableCache } from '../observable-cache'; +import { jsonRecordsSeparators } from './content-types'; export class StreamMessage { @@ -57,7 +58,12 @@ export class StreamMessage { startOfMessage.includes('{') || startOfMessage.includes('[') || this.subprotocol?.includes('json') - ) return 'json'; + ) { + if (jsonRecordsSeparators.indexOf(this.content[this.content.length - 1]) > -1) + return 'json-records'; + else + return 'json'; + } else return 'text'; } diff --git a/src/services/ui-worker-formatters.ts b/src/services/ui-worker-formatters.ts index 96a2e632..303548fd 100644 --- a/src/services/ui-worker-formatters.ts +++ b/src/services/ui-worker-formatters.ts @@ -6,8 +6,9 @@ import { import * as beautifyXml from 'xml-beautifier'; import { Headers } from '../types'; -import { bufferToHex, bufferToString, getReadableSize } from '../util/buffer'; +import { bufferToHex, bufferToString, getReadableSize, splitBuffer } from '../util/buffer'; import { parseRawProtobuf, extractProtobufFromGrpc } from '../util/protobuf'; +import { jsonRecordsSeparators } from '../model/events/content-types'; const truncationMarker = (size: string) => `\n[-- Truncated to ${size} --]`; const FIVE_MB = 1024 * 1024 * 5; @@ -80,6 +81,24 @@ const WorkerFormatters = { return asString; } }, + 'json-records': (content: Buffer) => { + const asString = content.toString('utf8'); + + try { + let records = new Array(); + jsonRecordsSeparators.forEach((separator) => { + splitBuffer(content, separator).forEach((recordBuffer: Buffer) => { + if (recordBuffer.length > 0) { + const record = recordBuffer.toString('utf-8'); + records.push(JSON.parse(record.trim())); + } + }); + }); + return JSON.stringify(records, null, 2); + } catch (e) { + return asString; + } + }, javascript: (content: Buffer) => { return beautifyJs(content.toString('utf8'), { indent_size: 2 diff --git a/src/util/buffer.ts b/src/util/buffer.ts index 8769f2ac..f9ad085c 100644 --- a/src/util/buffer.ts +++ b/src/util/buffer.ts @@ -149,4 +149,33 @@ export function getReadableSize(input: number | Buffer | string, siUnits = true) let unitName = bytes === 1 ? 'byte' : units[unitIndex]; return (bytes / Math.pow(thresh, unitIndex)).toFixed(1).replace(/\.0$/, '') + ' ' + unitName; +} + +/** + * Splits a Buffer into an array of Buffers using a specified separator. + * @param buffer The Buffer to split. + * @param separator The byte or Buffer sequence to split by. + * @returns An array of Buffers. + */ +export function splitBuffer(buffer: Buffer, separator: number | Buffer): Buffer[] { + const result: Buffer[] = []; + let currentOffset = 0; + let separatorIndex: number; + + // Handle single byte separator vs. multi-byte separator + const separatorLength = typeof separator === 'number' ? 1 : separator.length; + + while ((separatorIndex = buffer.indexOf(separator, currentOffset)) !== -1) { + // Add the chunk before the separator + result.push(buffer.slice(currentOffset, separatorIndex)); + // Move the offset past the separator + currentOffset = separatorIndex + separatorLength; + } + + // Add the last chunk (or the whole buffer if no separator was found) + if (currentOffset <= buffer.length) { + result.push(buffer.slice(currentOffset)); + } + + return result; } \ No newline at end of file From cce16cd0c2cdf4f2642eb18d8ab853eb66f600ec Mon Sep 17 00:00:00 2001 From: Weihan Li Date: Sat, 26 Jul 2025 10:44:48 +0800 Subject: [PATCH 02/13] refactor: remove unnecessary toString convert --- src/model/events/body-formatting.ts | 17 +++++++---------- src/services/ui-worker-formatters.ts | 17 +++++++---------- 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/src/model/events/body-formatting.ts b/src/model/events/body-formatting.ts index b75af2ce..47fd793e 100644 --- a/src/model/events/body-formatting.ts +++ b/src/model/events/body-formatting.ts @@ -133,17 +133,14 @@ export const Formatters: { [key in ViewableContentType]: Formatter } = { isEditApplicable: false, render: (input: Buffer, headers?: Headers) => { if (input.byteLength < 10_000) { - const inputAsString = bufferToString(input); - try { let records = new Array(); - jsonRecordsSeparators.forEach((separator) => { - splitBuffer(input, separator).forEach((recordBuffer: Buffer) => { - if (recordBuffer.length > 0) { - const record = recordBuffer.toString('utf-8'); - records.push(JSON.parse(record.trim())); - } - }); + const separator = input[input.length - 1]; + splitBuffer(input, separator).forEach((recordBuffer: Buffer) => { + if (recordBuffer.length > 0) { + const record = recordBuffer.toString('utf-8'); + records.push(JSON.parse(record.trim())); + } }); // For short-ish inputs, we return synchronously - conveniently this avoids // showing the loading spinner that churns the layout in short content cases. @@ -155,7 +152,7 @@ export const Formatters: { [key in ViewableContentType]: Formatter } = { // ^ Same logic as in UI-worker-formatter } catch (e) { // Fallback to showing the raw un-formatted: - return inputAsString; + return bufferToString(input); } } else { return observablePromise( diff --git a/src/services/ui-worker-formatters.ts b/src/services/ui-worker-formatters.ts index 303548fd..35a6fa4e 100644 --- a/src/services/ui-worker-formatters.ts +++ b/src/services/ui-worker-formatters.ts @@ -82,21 +82,18 @@ const WorkerFormatters = { } }, 'json-records': (content: Buffer) => { - const asString = content.toString('utf8'); - try { let records = new Array(); - jsonRecordsSeparators.forEach((separator) => { - splitBuffer(content, separator).forEach((recordBuffer: Buffer) => { - if (recordBuffer.length > 0) { - const record = recordBuffer.toString('utf-8'); - records.push(JSON.parse(record.trim())); - } - }); + const separator = content[content.length - 1]; + splitBuffer(content, separator).forEach((recordBuffer: Buffer) => { + if (recordBuffer.length > 0) { + const record = recordBuffer.toString('utf-8'); + records.push(JSON.parse(record.trim())); + } }); return JSON.stringify(records, null, 2); } catch (e) { - return asString; + return content.toString('utf8'); } }, javascript: (content: Buffer) => { From 62c666b9fccddc4c77d60c2ad8fc3d791ab6ed0c Mon Sep 17 00:00:00 2001 From: Weihan Li Date: Sat, 26 Jul 2025 10:50:03 +0800 Subject: [PATCH 03/13] refactor: remove unnecessary import --- src/model/events/body-formatting.ts | 2 +- src/services/ui-worker-formatters.ts | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/model/events/body-formatting.ts b/src/model/events/body-formatting.ts index 47fd793e..56bd96c9 100644 --- a/src/model/events/body-formatting.ts +++ b/src/model/events/body-formatting.ts @@ -1,7 +1,7 @@ import { Headers } from '../../types'; import { styled } from '../../styles'; -import { jsonRecordsSeparators, ViewableContentType } from '../events/content-types'; +import { ViewableContentType } from '../events/content-types'; import { ObservablePromise, observablePromise } from '../../util/observable'; import { bufferToString, bufferToHex, splitBuffer } from '../../util/buffer'; diff --git a/src/services/ui-worker-formatters.ts b/src/services/ui-worker-formatters.ts index 35a6fa4e..a945a8c3 100644 --- a/src/services/ui-worker-formatters.ts +++ b/src/services/ui-worker-formatters.ts @@ -8,7 +8,6 @@ import * as beautifyXml from 'xml-beautifier'; import { Headers } from '../types'; import { bufferToHex, bufferToString, getReadableSize, splitBuffer } from '../util/buffer'; import { parseRawProtobuf, extractProtobufFromGrpc } from '../util/protobuf'; -import { jsonRecordsSeparators } from '../model/events/content-types'; const truncationMarker = (size: string) => `\n[-- Truncated to ${size} --]`; const FIVE_MB = 1024 * 1024 * 5; From c0744756f394fd0b75d11f6b33d9e8fd66620afd Mon Sep 17 00:00:00 2001 From: Weihan Li Date: Sat, 26 Jul 2025 10:59:40 +0800 Subject: [PATCH 04/13] docs: add a comment for the SignalR record separator --- src/model/events/content-types.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/model/events/content-types.ts b/src/model/events/content-types.ts index cd353294..31346ed7 100644 --- a/src/model/events/content-types.ts +++ b/src/model/events/content-types.ts @@ -125,7 +125,7 @@ const mimeTypeToContentTypeMap: { [mimeType: string]: ViewableContentType } = { } as const; export const jsonRecordsSeparators = [ - 0x1E + 0x1E, // SignalR record separator https://github.com/dotnet/aspnetcore/blob/v8.0.0/src/SignalR/docs/specs/HubProtocol.md#json-encoding ]; export function getContentType(mimeType: string | undefined): ViewableContentType | undefined { From f2dafbfc7fd328ae7297317206cdd74223ce25ee Mon Sep 17 00:00:00 2001 From: Weihan Li Date: Tue, 29 Jul 2025 07:40:45 +0800 Subject: [PATCH 05/13] feat: akip add json when already has json-records --- src/model/events/content-types.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/model/events/content-types.ts b/src/model/events/content-types.ts index 31346ed7..969d19c2 100644 --- a/src/model/events/content-types.ts +++ b/src/model/events/content-types.ts @@ -207,7 +207,7 @@ export function getCompatibleTypes( } // Allow optionally formatting non-JSON as JSON, if it looks like it might be - if (firstChar === '{' || firstChar === '[') { + if ((firstChar === '{' && !types.has('json-records')) || firstChar === '[') { types.add('json'); } From 549367b630b800c13fbcc3fee85029135e580ab1 Mon Sep 17 00:00:00 2001 From: Weihan Li Date: Sat, 2 Aug 2025 23:27:25 +0800 Subject: [PATCH 06/13] feat: add json-records language --- .../editor/json-records-validation.ts | 69 +++++++++++++++++++ src/components/editor/monaco.ts | 6 ++ src/model/events/body-formatting.ts | 20 +++--- src/services/ui-worker-formatters.ts | 7 +- 4 files changed, 89 insertions(+), 13 deletions(-) create mode 100644 src/components/editor/json-records-validation.ts diff --git a/src/components/editor/json-records-validation.ts b/src/components/editor/json-records-validation.ts new file mode 100644 index 00000000..35faca34 --- /dev/null +++ b/src/components/editor/json-records-validation.ts @@ -0,0 +1,69 @@ +import type * as MonacoTypes from 'monaco-editor' +import * as json_parser from 'jsonc-parser' + +export function setupJsonRecordsValidation(monaco: typeof MonacoTypes) { + const markerId = 'json-records-validation' + + function validate(model: MonacoTypes.editor.ITextModel) { + const markers: MonacoTypes.editor.IMarkerData[] = [] + const text = model.getValue(); + + if (text) { + const separator = text[text.length - 1]; + const splits = text.split(separator); + let offset = 0; + for (let i = 0; i < splits.length; i++) { + const part = splits[i]; + if (part) { + let errors: json_parser.ParseError[] = [] + json_parser.parse(part, errors, { allowTrailingComma: false, disallowComments: true }); + console.log(`Validating JSON record part [${i}] `, part, `error ${JSON.stringify(errors)}`); + if (errors) { + errors.forEach((err) => { + markers.push({ + severity: monaco.MarkerSeverity.Error, + startLineNumber: i, + startColumn: offset, // +1 for the separator + endLineNumber: i, + endColumn: offset + err.length, + message: err.error.toString(), + }) + }); + } + } + + offset += part.length + 1; // +1 for the separator + } + } + + monaco.editor.setModelMarkers(model, markerId, markers) + } + + const contentChangeListeners = new Map() + function manageContentChangeListener(model: MonacoTypes.editor.ITextModel) { + const isJsonRecords = model.getModeId() === 'json-records' + const listener = contentChangeListeners.get(model) + + if (isJsonRecords && !listener) { + contentChangeListeners.set( + model, + model.onDidChangeContent(() => validate(model)) + ) + validate(model) + } else if (!isJsonRecords && listener) { + listener.dispose() + contentChangeListeners.delete(model) + monaco.editor.setModelMarkers(model, markerId, []) + } + } + + monaco.editor.onWillDisposeModel(model => { + contentChangeListeners.delete(model) + }) + monaco.editor.onDidChangeModelLanguage(({ model }) => { + manageContentChangeListener(model) + }) + monaco.editor.onDidCreateModel(model => { + manageContentChangeListener(model) + }) +} diff --git a/src/components/editor/monaco.ts b/src/components/editor/monaco.ts index 245345dd..98a8fad2 100644 --- a/src/components/editor/monaco.ts +++ b/src/components/editor/monaco.ts @@ -7,6 +7,7 @@ import { delay } from '../../util/promise'; import { asError } from '../../util/error'; import { observable, runInAction } from 'mobx'; import { setupXMLValidation } from './xml-validation'; +import { setupJsonRecordsValidation } from './json-records-validation'; export type { MonacoTypes, @@ -75,7 +76,12 @@ async function loadMonacoEditor(retries = 5): Promise { }, }); + monaco.languages.register({ + id: 'json-records' + }); + setupXMLValidation(monaco); + setupJsonRecordsValidation(monaco); MonacoEditor = rmeModule.default; } catch (err) { diff --git a/src/model/events/body-formatting.ts b/src/model/events/body-formatting.ts index 56bd96c9..b95a728e 100644 --- a/src/model/events/body-formatting.ts +++ b/src/model/events/body-formatting.ts @@ -9,6 +9,7 @@ import type { WorkerFormatterKey } from '../../services/ui-worker-formatters'; import { formatBufferAsync } from '../../services/ui-worker-api'; import { ReadOnlyParams } from '../../components/common/editable-params'; import { ImageViewer } from '../../components/editor/image-viewer'; +import { Buffer } from 'buffer'; export interface EditorFormatter { language: string; @@ -128,27 +129,24 @@ export const Formatters: { [key in ViewableContentType]: Formatter } = { } }, 'json-records': { - language: 'json', + language: 'json-records', cacheKey: Symbol('json-records'), isEditApplicable: false, render: (input: Buffer, headers?: Headers) => { if (input.byteLength < 10_000) { try { - let records = new Array(); + let records = new Array(); const separator = input[input.length - 1]; + const separatorString = Buffer.of(separator).toString('utf8'); + splitBuffer(input, separator).forEach((recordBuffer: Buffer) => { if (recordBuffer.length > 0) { const record = recordBuffer.toString('utf-8'); - records.push(JSON.parse(record.trim())); + records.push(record + separatorString); } }); - // For short-ish inputs, we return synchronously - conveniently this avoids - // showing the loading spinner that churns the layout in short content cases. - return JSON.stringify( - records, - null, - 2 - ); + + return records.join(''); // ^ Same logic as in UI-worker-formatter } catch (e) { // Fallback to showing the raw un-formatted: @@ -156,7 +154,7 @@ export const Formatters: { [key in ViewableContentType]: Formatter } = { } } else { return observablePromise( - formatBufferAsync(input, 'json', headers) + formatBufferAsync(input, 'json-records', headers) ); } } diff --git a/src/services/ui-worker-formatters.ts b/src/services/ui-worker-formatters.ts index a945a8c3..a426f138 100644 --- a/src/services/ui-worker-formatters.ts +++ b/src/services/ui-worker-formatters.ts @@ -84,13 +84,16 @@ const WorkerFormatters = { try { let records = new Array(); const separator = content[content.length - 1]; + const separatorString = Buffer.of(separator).toString('utf8'); + splitBuffer(content, separator).forEach((recordBuffer: Buffer) => { if (recordBuffer.length > 0) { const record = recordBuffer.toString('utf-8'); - records.push(JSON.parse(record.trim())); + records.push(record + separatorString); } }); - return JSON.stringify(records, null, 2); + + return records.join(''); } catch (e) { return content.toString('utf8'); } From 6b97ffea7e9f6439eccba6b5ef5f434b1c9144b7 Mon Sep 17 00:00:00 2001 From: Weihan Li Date: Sat, 2 Aug 2025 23:32:16 +0800 Subject: [PATCH 07/13] update line number --- src/components/editor/json-records-validation.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/components/editor/json-records-validation.ts b/src/components/editor/json-records-validation.ts index 35faca34..d812b409 100644 --- a/src/components/editor/json-records-validation.ts +++ b/src/components/editor/json-records-validation.ts @@ -22,9 +22,9 @@ export function setupJsonRecordsValidation(monaco: typeof MonacoTypes) { errors.forEach((err) => { markers.push({ severity: monaco.MarkerSeverity.Error, - startLineNumber: i, + startLineNumber: 0, startColumn: offset, // +1 for the separator - endLineNumber: i, + endLineNumber: 0, endColumn: offset + err.length, message: err.error.toString(), }) From 02ccb3becd3a62f567dbb37406356627aada4865 Mon Sep 17 00:00:00 2001 From: Weihan Li Date: Sun, 3 Aug 2025 00:11:14 +0800 Subject: [PATCH 08/13] update line number as splits index --- src/components/editor/json-records-validation.ts | 6 +++--- src/model/events/body-formatting.ts | 2 +- src/services/ui-worker-formatters.ts | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/components/editor/json-records-validation.ts b/src/components/editor/json-records-validation.ts index d812b409..e50f370b 100644 --- a/src/components/editor/json-records-validation.ts +++ b/src/components/editor/json-records-validation.ts @@ -22,9 +22,9 @@ export function setupJsonRecordsValidation(monaco: typeof MonacoTypes) { errors.forEach((err) => { markers.push({ severity: monaco.MarkerSeverity.Error, - startLineNumber: 0, - startColumn: offset, // +1 for the separator - endLineNumber: 0, + startLineNumber: i, + startColumn: offset, + endLineNumber: i, endColumn: offset + err.length, message: err.error.toString(), }) diff --git a/src/model/events/body-formatting.ts b/src/model/events/body-formatting.ts index b95a728e..33d2d310 100644 --- a/src/model/events/body-formatting.ts +++ b/src/model/events/body-formatting.ts @@ -146,7 +146,7 @@ export const Formatters: { [key in ViewableContentType]: Formatter } = { } }); - return records.join(''); + return records.join('\n'); // ^ Same logic as in UI-worker-formatter } catch (e) { // Fallback to showing the raw un-formatted: diff --git a/src/services/ui-worker-formatters.ts b/src/services/ui-worker-formatters.ts index a426f138..f19eee62 100644 --- a/src/services/ui-worker-formatters.ts +++ b/src/services/ui-worker-formatters.ts @@ -93,7 +93,7 @@ const WorkerFormatters = { } }); - return records.join(''); + return records.join('\n'); } catch (e) { return content.toString('utf8'); } From f982a195ecc18c82647ef07d91919ab00b4433f3 Mon Sep 17 00:00:00 2001 From: Tim Perry Date: Tue, 5 Aug 2025 16:35:05 +0200 Subject: [PATCH 09/13] Use JSON record line & character details in validation --- package-lock.json | 31 +++++-- package.json | 1 + .../editor/json-records-validation.ts | 80 +++++++++++++++---- 3 files changed, 88 insertions(+), 24 deletions(-) diff --git a/package-lock.json b/package-lock.json index dd78a9b4..f86f71f7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -73,6 +73,7 @@ "har-validator": "^5.1.3", "http-encoding": "^2.0.1", "js-beautify": "^1.8.8", + "jsonc-parser": "^3.3.1", "jsonwebtoken": "^8.4.0", "localforage": "^1.7.3", "lodash": "^4.17.21", @@ -12394,10 +12395,10 @@ } }, "node_modules/jsonc-parser": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-1.0.3.tgz", - "integrity": "sha512-hk/69oAeaIzchq/v3lS50PXuzn5O2ynldopMC+SWBql7J2WtdptfB9dy8Y7+Og5rPkTCpn83zTiO8FMcqlXJ/g==", - "dev": true + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.3.1.tgz", + "integrity": "sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ==", + "license": "MIT" }, "node_modules/jsonp": { "version": "0.2.1", @@ -20854,6 +20855,13 @@ "vscode-languageserver-types": "^3.6.0-next.1" } }, + "node_modules/vscode-emmet-helper/node_modules/jsonc-parser": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-1.0.3.tgz", + "integrity": "sha512-hk/69oAeaIzchq/v3lS50PXuzn5O2ynldopMC+SWBql7J2WtdptfB9dy8Y7+Og5rPkTCpn83zTiO8FMcqlXJ/g==", + "dev": true, + "license": "MIT" + }, "node_modules/vscode-languageserver-types": { "version": "3.15.1", "resolved": "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.15.1.tgz", @@ -31539,10 +31547,9 @@ "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==" }, "jsonc-parser": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-1.0.3.tgz", - "integrity": "sha512-hk/69oAeaIzchq/v3lS50PXuzn5O2ynldopMC+SWBql7J2WtdptfB9dy8Y7+Og5rPkTCpn83zTiO8FMcqlXJ/g==", - "dev": true + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.3.1.tgz", + "integrity": "sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ==" }, "jsonp": { "version": "0.2.1", @@ -37912,6 +37919,14 @@ "@emmetio/extract-abbreviation": "0.1.6", "jsonc-parser": "^1.0.0", "vscode-languageserver-types": "^3.6.0-next.1" + }, + "dependencies": { + "jsonc-parser": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-1.0.3.tgz", + "integrity": "sha512-hk/69oAeaIzchq/v3lS50PXuzn5O2ynldopMC+SWBql7J2WtdptfB9dy8Y7+Og5rPkTCpn83zTiO8FMcqlXJ/g==", + "dev": true + } } }, "vscode-languageserver-types": { diff --git a/package.json b/package.json index ec0f18aa..409c791e 100644 --- a/package.json +++ b/package.json @@ -98,6 +98,7 @@ "har-validator": "^5.1.3", "http-encoding": "^2.0.1", "js-beautify": "^1.8.8", + "jsonc-parser": "^3.3.1", "jsonwebtoken": "^8.4.0", "localforage": "^1.7.3", "lodash": "^4.17.21", diff --git a/src/components/editor/json-records-validation.ts b/src/components/editor/json-records-validation.ts index e50f370b..5ad23c07 100644 --- a/src/components/editor/json-records-validation.ts +++ b/src/components/editor/json-records-validation.ts @@ -11,28 +11,24 @@ export function setupJsonRecordsValidation(monaco: typeof MonacoTypes) { if (text) { const separator = text[text.length - 1]; const splits = text.split(separator); - let offset = 0; + for (let i = 0; i < splits.length; i++) { const part = splits[i]; if (part) { - let errors: json_parser.ParseError[] = [] - json_parser.parse(part, errors, { allowTrailingComma: false, disallowComments: true }); - console.log(`Validating JSON record part [${i}] `, part, `error ${JSON.stringify(errors)}`); - if (errors) { - errors.forEach((err) => { - markers.push({ - severity: monaco.MarkerSeverity.Error, - startLineNumber: i, - startColumn: offset, - endLineNumber: i, - endColumn: offset + err.length, - message: err.error.toString(), - }) + let errors: ParserError[] = []; + parseJson(part, errors, { allowTrailingComma: false, disallowComments: true }); + if (errors.length) { + const firstError = errors[0]; + markers.push({ + severity: monaco.MarkerSeverity.Error, + startLineNumber: i + 1, + startColumn: firstError.startCharacter + 1, + endLineNumber: i + 1, + endColumn: firstError.startCharacter + firstError.length + 1, + message: json_parser.printParseErrorCode(firstError.error) }); } } - - offset += part.length + 1; // +1 for the separator } } @@ -67,3 +63,55 @@ export function setupJsonRecordsValidation(monaco: typeof MonacoTypes) { manageContentChangeListener(model) }) } + + +interface ParserError extends json_parser.ParseError { + startLine: number; + startCharacter: number; +} + +function parseJson(text: string, errors: ParserError[] = [], options: json_parser.ParseOptions = {}): any { + let currentProperty: string | null = null; + let currentParent: any = []; + const previousParents: any[] = []; + + function onValue(value: any) { + if (Array.isArray(currentParent)) { + (currentParent).push(value); + } else if (currentProperty !== null) { + currentParent[currentProperty] = value; + } + } + + const visitor: json_parser.JSONVisitor = { + onObjectBegin: () => { + const object = {}; + onValue(object); + previousParents.push(currentParent); + currentParent = object; + currentProperty = null; + }, + onObjectProperty: (name: string) => { + currentProperty = name; + }, + onObjectEnd: () => { + currentParent = previousParents.pop(); + }, + onArrayBegin: () => { + const array: any[] = []; + onValue(array); + previousParents.push(currentParent); + currentParent = array; + currentProperty = null; + }, + onArrayEnd: () => { + currentParent = previousParents.pop(); + }, + onLiteralValue: onValue, + onError: (error: json_parser.ParseErrorCode, offset: number, length: number, startLine: number, startCharacter: number) => { + errors.push({ error, offset, length, startLine, startCharacter }); + } + }; + json_parser.visit(text, visitor, options); + return currentParent[0]; +} From 1f844726f6d77e498c3d2a11a5851bfc1db1d8d3 Mon Sep 17 00:00:00 2001 From: Tim Perry Date: Wed, 6 Aug 2025 20:45:18 +0200 Subject: [PATCH 10/13] Extract content validation + extend JSON record validation & detection This now uses parseTree with model.getPosition to get the exact correct line & column for errors, and to handle multiple errors in the same line or elsewhere, and accepts JSON lines & NDJSON in addition to 0x1E separated records like SignalR. Detection is extended to match, and uses some slightly broader heuristics to catch a few more cases. --- .../editor/json-records-validation.ts | 117 ----------------- src/components/editor/monaco.ts | 47 ++++++- src/components/editor/xml-validation.ts | 58 --------- src/model/events/content-types.ts | 31 ++--- src/model/events/content-validation.ts | 107 +++++++++++++++ src/model/events/stream-message.ts | 16 +-- src/util/json.ts | 58 +++++++++ src/util/text.ts | 7 + .../model/events/content-validation.spec.tsx | 123 ++++++++++++++++++ 9 files changed, 356 insertions(+), 208 deletions(-) delete mode 100644 src/components/editor/json-records-validation.ts delete mode 100644 src/components/editor/xml-validation.ts create mode 100644 src/model/events/content-validation.ts create mode 100644 src/util/json.ts create mode 100644 test/unit/model/events/content-validation.spec.tsx diff --git a/src/components/editor/json-records-validation.ts b/src/components/editor/json-records-validation.ts deleted file mode 100644 index 5ad23c07..00000000 --- a/src/components/editor/json-records-validation.ts +++ /dev/null @@ -1,117 +0,0 @@ -import type * as MonacoTypes from 'monaco-editor' -import * as json_parser from 'jsonc-parser' - -export function setupJsonRecordsValidation(monaco: typeof MonacoTypes) { - const markerId = 'json-records-validation' - - function validate(model: MonacoTypes.editor.ITextModel) { - const markers: MonacoTypes.editor.IMarkerData[] = [] - const text = model.getValue(); - - if (text) { - const separator = text[text.length - 1]; - const splits = text.split(separator); - - for (let i = 0; i < splits.length; i++) { - const part = splits[i]; - if (part) { - let errors: ParserError[] = []; - parseJson(part, errors, { allowTrailingComma: false, disallowComments: true }); - if (errors.length) { - const firstError = errors[0]; - markers.push({ - severity: monaco.MarkerSeverity.Error, - startLineNumber: i + 1, - startColumn: firstError.startCharacter + 1, - endLineNumber: i + 1, - endColumn: firstError.startCharacter + firstError.length + 1, - message: json_parser.printParseErrorCode(firstError.error) - }); - } - } - } - } - - monaco.editor.setModelMarkers(model, markerId, markers) - } - - const contentChangeListeners = new Map() - function manageContentChangeListener(model: MonacoTypes.editor.ITextModel) { - const isJsonRecords = model.getModeId() === 'json-records' - const listener = contentChangeListeners.get(model) - - if (isJsonRecords && !listener) { - contentChangeListeners.set( - model, - model.onDidChangeContent(() => validate(model)) - ) - validate(model) - } else if (!isJsonRecords && listener) { - listener.dispose() - contentChangeListeners.delete(model) - monaco.editor.setModelMarkers(model, markerId, []) - } - } - - monaco.editor.onWillDisposeModel(model => { - contentChangeListeners.delete(model) - }) - monaco.editor.onDidChangeModelLanguage(({ model }) => { - manageContentChangeListener(model) - }) - monaco.editor.onDidCreateModel(model => { - manageContentChangeListener(model) - }) -} - - -interface ParserError extends json_parser.ParseError { - startLine: number; - startCharacter: number; -} - -function parseJson(text: string, errors: ParserError[] = [], options: json_parser.ParseOptions = {}): any { - let currentProperty: string | null = null; - let currentParent: any = []; - const previousParents: any[] = []; - - function onValue(value: any) { - if (Array.isArray(currentParent)) { - (currentParent).push(value); - } else if (currentProperty !== null) { - currentParent[currentProperty] = value; - } - } - - const visitor: json_parser.JSONVisitor = { - onObjectBegin: () => { - const object = {}; - onValue(object); - previousParents.push(currentParent); - currentParent = object; - currentProperty = null; - }, - onObjectProperty: (name: string) => { - currentProperty = name; - }, - onObjectEnd: () => { - currentParent = previousParents.pop(); - }, - onArrayBegin: () => { - const array: any[] = []; - onValue(array); - previousParents.push(currentParent); - currentParent = array; - currentProperty = null; - }, - onArrayEnd: () => { - currentParent = previousParents.pop(); - }, - onLiteralValue: onValue, - onError: (error: json_parser.ParseErrorCode, offset: number, length: number, startLine: number, startCharacter: number) => { - errors.push({ error, offset, length, startLine, startCharacter }); - } - }; - json_parser.visit(text, visitor, options); - return currentParent[0]; -} diff --git a/src/components/editor/monaco.ts b/src/components/editor/monaco.ts index 98a8fad2..8ef7914d 100644 --- a/src/components/editor/monaco.ts +++ b/src/components/editor/monaco.ts @@ -1,13 +1,13 @@ import type * as MonacoTypes from 'monaco-editor'; import type { default as _MonacoEditor, MonacoEditorProps } from 'react-monaco-editor'; +import { observable, runInAction } from 'mobx'; import { defineMonacoThemes } from '../../styles'; import { delay } from '../../util/promise'; import { asError } from '../../util/error'; -import { observable, runInAction } from 'mobx'; -import { setupXMLValidation } from './xml-validation'; -import { setupJsonRecordsValidation } from './json-records-validation'; + +import { ContentValidator, validateJsonRecords, validateXml } from '../../model/events/content-validation'; export type { MonacoTypes, @@ -80,8 +80,8 @@ async function loadMonacoEditor(retries = 5): Promise { id: 'json-records' }); - setupXMLValidation(monaco); - setupJsonRecordsValidation(monaco); + addValidator(monaco, 'xml', validateXml); + addValidator(monaco, 'json-records', validateJsonRecords); MonacoEditor = rmeModule.default; } catch (err) { @@ -95,6 +95,43 @@ async function loadMonacoEditor(retries = 5): Promise { } } +function addValidator(monaco: typeof MonacoTypes, modeId: string, validator: ContentValidator) { + function validate(model: MonacoTypes.editor.ITextModel) { + const text = model.getValue(); + const markers = validator(text, model); + monaco.editor.setModelMarkers(model, modeId, markers); + } + + const contentChangeListeners = new Map(); + + function manageContentChangeListener(model: MonacoTypes.editor.ITextModel) { + const isActiveMode = model.getModeId() === modeId; + const listener = contentChangeListeners.get(model); + + if (isActiveMode && !listener) { + contentChangeListeners.set(model, model.onDidChangeContent(() => + validate(model) + )); + validate(model); + } else if (!isActiveMode && listener) { + listener.dispose(); + contentChangeListeners.delete(model); + monaco.editor.setModelMarkers(model, modeId, []); + } + } + + monaco.editor.onWillDisposeModel(model => { + contentChangeListeners.delete(model); + }); + monaco.editor.onDidChangeModelLanguage(({ model }) => { + manageContentChangeListener(model); + }); + monaco.editor.onDidCreateModel(model => { + manageContentChangeListener(model); + }); + +} + export function reloadMonacoEditor() { return monacoLoadingPromise = loadMonacoEditor(0); } diff --git a/src/components/editor/xml-validation.ts b/src/components/editor/xml-validation.ts deleted file mode 100644 index d71f632d..00000000 --- a/src/components/editor/xml-validation.ts +++ /dev/null @@ -1,58 +0,0 @@ -import type * as MonacoTypes from 'monaco-editor' -import { XMLValidator } from 'fast-xml-parser' - -export function setupXMLValidation(monaco: typeof MonacoTypes) { - const markerId = 'xml-validation' - - function validate(model: MonacoTypes.editor.ITextModel) { - const markers: MonacoTypes.editor.IMarkerData[] = [] - const text = model.getValue() - - if (text.trim()) { - const validationResult = XMLValidator.validate(text, { - allowBooleanAttributes: true, - }) - - if (validationResult !== true) { - markers.push({ - severity: monaco.MarkerSeverity.Error, - startLineNumber: validationResult.err.line, - startColumn: validationResult.err.col, - endLineNumber: validationResult.err.line, - endColumn: model.getLineContent(validationResult.err.line).length + 1, - message: validationResult.err.msg, - }) - } - } - - monaco.editor.setModelMarkers(model, markerId, markers) - } - - const contentChangeListeners = new Map() - function manageContentChangeListener(model: MonacoTypes.editor.ITextModel) { - const isXml = model.getModeId() === 'xml' - const listener = contentChangeListeners.get(model) - - if (isXml && !listener) { - contentChangeListeners.set( - model, - model.onDidChangeContent(() => validate(model)) - ) - validate(model) - } else if (!isXml && listener) { - listener.dispose() - contentChangeListeners.delete(model) - monaco.editor.setModelMarkers(model, markerId, []) - } - } - - monaco.editor.onWillDisposeModel(model => { - contentChangeListeners.delete(model) - }) - monaco.editor.onDidChangeModelLanguage(({ model }) => { - manageContentChangeListener(model) - }) - monaco.editor.onDidCreateModel(model => { - manageContentChangeListener(model) - }) -} diff --git a/src/model/events/content-types.ts b/src/model/events/content-types.ts index 969d19c2..376001b6 100644 --- a/src/model/events/content-types.ts +++ b/src/model/events/content-types.ts @@ -7,6 +7,7 @@ import { isProbablyGrpcProto, isValidGrpcProto, } from '../../util/protobuf'; +import { isProbablyJson, isProbablyJsonRecords } from '../../util/json'; // Simplify a mime type as much as we can, without throwing any errors export const getBaseContentType = (mimeType: string | undefined) => { @@ -121,13 +122,17 @@ const mimeTypeToContentTypeMap: { [mimeType: string]: ViewableContentType } = { 'application/grpc-proto': 'grpc-proto', 'application/grpc-protobuf': 'grpc-proto', + // Nobody can quite agree on the names for the various sequence-of-JSON formats: + 'application/jsonlines': 'json-records', + 'application/json-lines': 'json-records', + 'application/x-jsonlines': 'json-records', + 'application/jsonl': 'json-records', + 'application/x-ndjson': 'json-records', + 'application/json-seq': 'json-records', + 'application/octet-stream': 'raw' } as const; -export const jsonRecordsSeparators = [ - 0x1E, // SignalR record separator https://github.com/dotnet/aspnetcore/blob/v8.0.0/src/SignalR/docs/specs/HubProtocol.md#json-encoding -]; - export function getContentType(mimeType: string | undefined): ViewableContentType | undefined { const baseContentType = getBaseContentType(mimeType); return mimeTypeToContentTypeMap[baseContentType!]; @@ -193,26 +198,18 @@ export function getCompatibleTypes( body = body.decodedData; } - // Examine the first char of the body, assuming it's ascii - const firstChar = body && body.subarray(0, 1).toString('ascii'); - // Allow optionally formatting non-JSON-records as JSON-records, if it looks like it might be - if (body && body.length > 2 && firstChar === '{' - && jsonRecordsSeparators.indexOf(body[body.length - 1]) > -1 - ) { - const secondToLastChar = body.subarray(body.length - 2, body.length - 1).toString('ascii'); - if (secondToLastChar === '}') { - types.add('json-records'); - } + if (!types.has('json-records') && isProbablyJsonRecords(body)) { + types.add('json-records'); } - // Allow optionally formatting non-JSON as JSON, if it looks like it might be - if ((firstChar === '{' && !types.has('json-records')) || firstChar === '[') { + if (!types.has('json-records') && isProbablyJson(body)) { + // Allow optionally formatting non-JSON as JSON, if it's anything remotely close types.add('json'); } // Allow optionally formatting non-XML as XML, if it looks like it might be - if (firstChar === '<') { + if (body?.subarray(0, 1).toString('ascii') === '<') { types.add('xml'); } diff --git a/src/model/events/content-validation.ts b/src/model/events/content-validation.ts new file mode 100644 index 00000000..9bf99ea3 --- /dev/null +++ b/src/model/events/content-validation.ts @@ -0,0 +1,107 @@ +import type * as MonacoTypes from 'monaco-editor'; + +import { XMLValidator } from 'fast-xml-parser'; +import * as jsonCParser from 'jsonc-parser'; + +import { camelToSentenceCase } from '../../util/text'; +import { RECORD_SEPARATOR_CHARS } from '../../util/json'; + +export interface ContentValidator { + (text: string, model: MonacoTypes.editor.ITextModel): ValidationMarker[]; +} + +// A minimal more generic version of Monaco's MonacoTypes.editor.IMarkerData type: +export interface ValidationMarker { + severity: MarkerSeverity; + message: string; + startLineNumber: number; + startColumn: number; + endLineNumber: number; + endColumn: number; +} + +enum MarkerSeverity { + Hint = 1, + Info = 2, + Warning = 4, + Error = 8 +} + +export function validateXml(text: string): ValidationMarker[] { + const markers: ValidationMarker[] = []; + + if (!text.trim()) return markers; + + const validationResult = XMLValidator.validate(text, { + allowBooleanAttributes: true, + }) + + if (validationResult !== true) { + markers.push({ + severity: MarkerSeverity.Error, + startLineNumber: validationResult.err.line, + startColumn: validationResult.err.col, + endLineNumber: validationResult.err.line, + endColumn: Infinity, + message: validationResult.err.msg, + }) + } + + return markers; +} + +export function validateJsonRecords(text: string, model: MonacoTypes.editor.ITextModel): ValidationMarker[] { + const markers: ValidationMarker[] = []; + if (!text.trim()) return markers; + + let offset = 0; + let remainingText = text.trimEnd(); + while (remainingText) { + if (RECORD_SEPARATOR_CHARS.includes(remainingText[0])) { + remainingText = remainingText.slice(1); + offset++; + continue; + } + + const errors: jsonCParser.ParseError[] = []; + const result = jsonCParser.parseTree(remainingText, errors, { + allowTrailingComma: false, + disallowComments: true + }); + + const parsedContentLength = result + ? result.offset + result.length + : Math.max(...errors.map((err) => err.offset + err.length)); + + // We show the first error for each record, except any errors after the end of + // a completely parsed value (i.e. due to hitting the subsequent record instead + // of an expected EOF). We'll handle that in the next iteration one way or another. + if (errors.length) { + const firstError = errors[0]; + if (firstError && firstError.offset < parsedContentLength) { + const position = model.getPositionAt(offset + firstError.offset); + markers.push({ + severity: MarkerSeverity.Error, + startLineNumber: position.lineNumber, + endLineNumber: position.lineNumber, + startColumn: position.column, + endColumn: position.column + firstError.length, + message: camelToSentenceCase(jsonCParser.printParseErrorCode(firstError.error)) + }); + } + } + + if (parsedContentLength === 0) { + // Should never happen, but this should at least give us some debug info if it ever does: + console.log(`Parsing ${remainingText} (${remainingText.length}) parsedContentLength was 0 with ${ + result + } result and ${JSON.stringify(errors)} errors`); + throw new Error(`JSON record parsed content length was 0`); + } + + remainingText = remainingText.slice(parsedContentLength); + offset += parsedContentLength; + } + + return markers; +} diff --git a/src/model/events/stream-message.ts b/src/model/events/stream-message.ts index 0be8df9f..0f01b569 100644 --- a/src/model/events/stream-message.ts +++ b/src/model/events/stream-message.ts @@ -3,7 +3,7 @@ import { computed, observable } from 'mobx'; import { InputStreamMessage } from "../../types"; import { asBuffer } from '../../util/buffer'; import { ObservableCache } from '../observable-cache'; -import { jsonRecordsSeparators } from './content-types'; +import { isProbablyJson, isProbablyJsonRecords } from '../../util/json'; export class StreamMessage { @@ -53,16 +53,10 @@ export class StreamMessage { } // prefix+JSON is very common, so we try to parse anything JSON-ish optimistically: - const startOfMessage = this.content.subarray(0, 10).toString('utf-8').trim(); - if ( - startOfMessage.includes('{') || - startOfMessage.includes('[') || - this.subprotocol?.includes('json') - ) { - if (jsonRecordsSeparators.indexOf(this.content[this.content.length - 1]) > -1) - return 'json-records'; - else - return 'json'; + if (isProbablyJsonRecords(this.content)) { + return 'json-records'; + } else if (isProbablyJson(this.content) || this.subprotocol?.includes('json')) { + return 'json'; } else return 'text'; diff --git a/src/util/json.ts b/src/util/json.ts new file mode 100644 index 00000000..832126c7 --- /dev/null +++ b/src/util/json.ts @@ -0,0 +1,58 @@ +const JSON_START_REGEX = /^\s*[\[\{tfn"\d-]/; // Optional whitespace, then start array/object/true/false/null/string/number + +const JSON_TEXT_SEQ_START_REGEX = /^\u001E\s*[\[\{]/; // Record separate, optional whitespace, then array/object + +const SIGNALR_HUB_START_REGEX = /^\s*\{/; // Optional whitespace, then start object +const SIGNALR_HUB_END_REGEX = /\}\s*\u001E$/; // End object, optional whitespace, then record separator + +const JSON_LINES_END_REGEX = /[\]\}](\r?\n)+$/; // Array/object end, then optional newline(s) +const JSON_LINES_INNER_REGEX = /[\]\}](\r?\n)+[\{\[]/; // Object/array end, then newline(s), then start object/array +const JSON_LINES_SCAN_LIMIT = 16 * 1024; + +export const isProbablyJson = (text: Buffer | undefined) => { + if (!text || text.length < 2) return false; + + const startChunk = text.subarray(0, 6).toString('utf8'); + return JSON_START_REGEX.test(startChunk); +} + +export const isProbablyJsonRecords = (text: Buffer | undefined) => { + if (!text || text.length < 3) return false; + + // This has some false negatives: e.g. standalone JSON primitive values, or unusual + // extra whitespace in certain places, but I think it should provide pretty good coverage + // the rest of the time. + + const startChunk = text.subarray(0, 6).toString('utf8'); + const endChunk = text.subarray(-6).toString('utf8'); + + if (JSON_TEXT_SEQ_START_REGEX.test(startChunk)) { + // JSON text sequence: https://www.rfc-editor.org/rfc/rfc7464.html + return true; + } + + if (SIGNALR_HUB_START_REGEX.test(startChunk) && SIGNALR_HUB_END_REGEX.test(endChunk)) { + // SignalR hub protocol: + // https://github.com/dotnet/aspnetcore/blob/main/src/SignalR/docs/specs/HubProtocol.md#json-encoding + return true; + } + + if ( + JSON_START_REGEX.test(startChunk) && + // Technically for JSON Lines the end newline is optional, but strongly recommended & common AFAICT + JSON_LINES_END_REGEX.test(endChunk) && + // If the end looks like JSON lines/NDJSON, so scan a bigger chunk to check: + JSON_LINES_INNER_REGEX.test(text.subarray(0, JSON_LINES_SCAN_LIMIT).toString('utf8')) + ) { + // JSON Lines or NDJSON: https://jsonlines.org/ + return true; + } + + return false; +} + +export const RECORD_SEPARATOR_CHARS = [ + '\u001E', // ASCII Record Separator (used by SignalR and others) + '\n', + '\r' +]; diff --git a/src/util/text.ts b/src/util/text.ts index f59f7e98..e319b603 100644 --- a/src/util/text.ts +++ b/src/util/text.ts @@ -20,4 +20,11 @@ export function aOrAn(value: string) { export function uppercaseFirst(value: string) { return value[0].toUpperCase() + value.slice(1); +} + +export function camelToSentenceCase(value: string) { + return uppercaseFirst( + value.replace(/([a-z])([A-Z])/g, '$1 $2') + .toLowerCase() + ); } \ No newline at end of file diff --git a/test/unit/model/events/content-validation.spec.tsx b/test/unit/model/events/content-validation.spec.tsx new file mode 100644 index 00000000..32b47532 --- /dev/null +++ b/test/unit/model/events/content-validation.spec.tsx @@ -0,0 +1,123 @@ +import { expect } from "chai"; +import * as monaco from 'monaco-editor'; + +import { + validateJsonRecords, + validateXml +} from "../../../../src/model/events/content-validation"; + +const contentModel = (content: string) => monaco.editor.createModel(content, "json-records"); + +describe("XML validation", () => { + it("should validate correct XML", () => { + const text = `Content`; + const markers = validateXml(text); + expect(markers).to.deep.equal([]); + }); + + it("should reject incorrect XML", () => { + const text = ` + Content + `; + const markers = validateXml(text); + expect(markers).to.deep.equal([ + { + severity: 8, + message: "Expected closing tag 'root' (opened in line 1, col 1) instead of closing tag 'wrong'.", + startLineNumber: 3, + endLineNumber: 3, + startColumn: 9, + endColumn: Infinity + } + ]); + }); +}) + +describe("JSON Records Validation", () => { + it("should validate correct normal JSON", () => { + const text = '{"name":"John"}'; + const markers = validateJsonRecords(text, contentModel(text)); + expect(markers).to.deep.equal([]); + }); + + it("should validate correct JSON records", () => { + const text = '{"name":"John"}\n\u001E\n{"name":"Jane"}'; + const markers = validateJsonRecords(text, contentModel(text)); + expect(markers).to.deep.equal([]); + }); + + it("should reject incorrect JSON", () => { + const text = '{"name":}'; + const markers = validateJsonRecords(text, contentModel(text)); + expect(markers).to.deep.equal([ + { + severity: 8, + message: "Value expected", + startLineNumber: 1, + startColumn: 9, + endLineNumber: 1, + endColumn: 10 + } + ]); + }); + + it("should reject incorrect record-separator JSON records", () => { + const text = '\u001E{"name":"John"}\u001E{"name":}'; + const markers = validateJsonRecords(text, contentModel(text)); + expect(markers).to.deep.equal([ + { + severity: 8, + message: "Value expected", + startLineNumber: 1, + startColumn: 26, + endLineNumber: 1, + endColumn: 27 + } + ]); + }); + + it("should reject incorrect newline-separator JSON records", () => { + const text = '{"name":"John"}\n{"name":}'; + const markers = validateJsonRecords(text, contentModel(text)); + expect(markers).to.deep.equal([ + { + severity: 8, + message: "Value expected", + startLineNumber: 2, + startColumn: 9, + endLineNumber: 2, + endColumn: 10 + } + ]); + }); + + it("should reject incorrect record-separator JSON records with newlines separators too", () => { + const text = '\n{"name":"John"}\n\u001E\n{"name":}\n'; + const markers = validateJsonRecords(text, contentModel(text)); + expect(markers).to.deep.equal([ + { + severity: 8, + message: "Value expected", + startLineNumber: 4, + startColumn: 9, + endLineNumber: 4, + endColumn: 10 + } + ]); + }); + + it("should reject incorrect JSON records with good line numbers despite spurious newlines", () => { + const text = '\n{"name":\n\n"John"\n}\n\u001E\n\u001E\n{"name":}'; + const markers = validateJsonRecords(text, contentModel(text)); + expect(markers).to.deep.equal([ + { + severity: 8, + message: "Value expected", + startLineNumber: 8, + startColumn: 9, + endLineNumber: 8, + endColumn: 10 + } + ]); + }); +}); \ No newline at end of file From 0ae61fe0f725a220ee177fc8c3583f741a25a7d0 Mon Sep 17 00:00:00 2001 From: Tim Perry Date: Wed, 6 Aug 2025 21:06:17 +0200 Subject: [PATCH 11/13] Add magic forgiving JSON formatting for normal & record JSON --- src/model/events/body-formatting.ts | 37 +-- src/services/ui-worker-formatters.ts | 27 +-- src/util/json.ts | 213 +++++++++++++++++ test/unit/util/json.spec.ts | 339 +++++++++++++++++++++++++++ 4 files changed, 561 insertions(+), 55 deletions(-) create mode 100644 test/unit/util/json.spec.ts diff --git a/src/model/events/body-formatting.ts b/src/model/events/body-formatting.ts index 33d2d310..c56991f5 100644 --- a/src/model/events/body-formatting.ts +++ b/src/model/events/body-formatting.ts @@ -9,7 +9,7 @@ import type { WorkerFormatterKey } from '../../services/ui-worker-formatters'; import { formatBufferAsync } from '../../services/ui-worker-api'; import { ReadOnlyParams } from '../../components/common/editable-params'; import { ImageViewer } from '../../components/editor/image-viewer'; -import { Buffer } from 'buffer'; +import { formatJson } from '../../util/json'; export interface EditorFormatter { language: string; @@ -107,20 +107,7 @@ export const Formatters: { [key in ViewableContentType]: Formatter } = { render: (input: Buffer, headers?: Headers) => { if (input.byteLength < 10_000) { const inputAsString = bufferToString(input); - - try { - // For short-ish inputs, we return synchronously - conveniently this avoids - // showing the loading spinner that churns the layout in short content cases. - return JSON.stringify( - JSON.parse(inputAsString), - null, - 2 - ); - // ^ Same logic as in UI-worker-formatter - } catch (e) { - // Fallback to showing the raw un-formatted JSON: - return inputAsString; - } + return formatJson(inputAsString, { formatRecords: false }); } else { return observablePromise( formatBufferAsync(input, 'json', headers) @@ -134,24 +121,8 @@ export const Formatters: { [key in ViewableContentType]: Formatter } = { isEditApplicable: false, render: (input: Buffer, headers?: Headers) => { if (input.byteLength < 10_000) { - try { - let records = new Array(); - const separator = input[input.length - 1]; - const separatorString = Buffer.of(separator).toString('utf8'); - - splitBuffer(input, separator).forEach((recordBuffer: Buffer) => { - if (recordBuffer.length > 0) { - const record = recordBuffer.toString('utf-8'); - records.push(record + separatorString); - } - }); - - return records.join('\n'); - // ^ Same logic as in UI-worker-formatter - } catch (e) { - // Fallback to showing the raw un-formatted: - return bufferToString(input); - } + const inputAsString = bufferToString(input); + return formatJson(inputAsString, { formatRecords: true }); } else { return observablePromise( formatBufferAsync(input, 'json-records', headers) diff --git a/src/services/ui-worker-formatters.ts b/src/services/ui-worker-formatters.ts index f19eee62..53110a61 100644 --- a/src/services/ui-worker-formatters.ts +++ b/src/services/ui-worker-formatters.ts @@ -6,8 +6,9 @@ import { import * as beautifyXml from 'xml-beautifier'; import { Headers } from '../types'; -import { bufferToHex, bufferToString, getReadableSize, splitBuffer } from '../util/buffer'; +import { bufferToHex, bufferToString, getReadableSize } from '../util/buffer'; import { parseRawProtobuf, extractProtobufFromGrpc } from '../util/protobuf'; +import { formatJson } from '../util/json'; const truncationMarker = (size: string) => `\n[-- Truncated to ${size} --]`; const FIVE_MB = 1024 * 1024 * 5; @@ -74,29 +75,11 @@ const WorkerFormatters = { }, json: (content: Buffer) => { const asString = content.toString('utf8'); - try { - return JSON.stringify(JSON.parse(asString), null, 2); - } catch (e) { - return asString; - } + return formatJson(asString, { formatRecords: false }); }, 'json-records': (content: Buffer) => { - try { - let records = new Array(); - const separator = content[content.length - 1]; - const separatorString = Buffer.of(separator).toString('utf8'); - - splitBuffer(content, separator).forEach((recordBuffer: Buffer) => { - if (recordBuffer.length > 0) { - const record = recordBuffer.toString('utf-8'); - records.push(record + separatorString); - } - }); - - return records.join('\n'); - } catch (e) { - return content.toString('utf8'); - } + const asString = content.toString('utf8'); + return formatJson(asString, { formatRecords: true }); }, javascript: (content: Buffer) => { return beautifyJs(content.toString('utf8'), { diff --git a/src/util/json.ts b/src/util/json.ts index 832126c7..29333e8f 100644 --- a/src/util/json.ts +++ b/src/util/json.ts @@ -1,3 +1,8 @@ +import { + createScanner as createJsonScanner, + SyntaxKind as JsonSyntaxKind +} from 'jsonc-parser'; + const JSON_START_REGEX = /^\s*[\[\{tfn"\d-]/; // Optional whitespace, then start array/object/true/false/null/string/number const JSON_TEXT_SEQ_START_REGEX = /^\u001E\s*[\[\{]/; // Record separate, optional whitespace, then array/object @@ -56,3 +61,211 @@ export const RECORD_SEPARATOR_CHARS = [ '\n', '\r' ]; + +// A *very* forgiving & flexible JSON formatter. This will correctly format even things that +// will fail validation later, such as trailing commas, unquoted keys, comments, etc. +export function formatJson(text: string, options: { formatRecords: boolean } = { formatRecords: false }): string { + const scanner = createJsonScanner(text); + + let result = ""; + let indent = 0; + let token: JsonSyntaxKind; + + const indentString = ' '; + let needsIndent = false; + let previousToken: JsonSyntaxKind | null = null; + + let betweenRecords = false; + + while ((token = scanner.scan()) !== JsonSyntaxKind.EOF) { + const tokenOffset = scanner.getTokenOffset(); + const tokenLength = scanner.getTokenLength(); + const tokenText = text.slice(tokenOffset, tokenOffset + tokenLength); + + if (options.formatRecords && indent === 0) { + betweenRecords = true; + } + + // Skip over explicit 'record separator' characters, which can cause parsing problems + // when parsing JSON records: + if (betweenRecords && tokenText[0] === '\u001E') { + scanner.setPosition(tokenOffset + 1); + continue; + } + + // Ignore irrelevant whitespace (internally or between records) - we'll handle that ourselves + if (token === JsonSyntaxKind.Trivia || token === JsonSyntaxKind.LineBreakTrivia) { + continue; + } + + if (betweenRecords) { + // We've finished one record and we have another coming that won't get a newline + // automatically: add an extra newline to properly separate records if required. + betweenRecords = false; + if (result && result[result.length - 1] !== '\n' && !isValueToken(token)) { + result += '\n'; + } + } + + if (needsIndent) { + result += indentString.repeat(indent); + needsIndent = false; + } + + switch (token) { + case JsonSyntaxKind.OpenBraceToken: + case JsonSyntaxKind.OpenBracketToken: + result += tokenText; + indent++; + + const afterOpener = scanAhead(scanner); + const isClosing = afterOpener === JsonSyntaxKind.CloseBraceToken || + afterOpener === JsonSyntaxKind.CloseBracketToken; + if ( + !isClosing && + afterOpener !== JsonSyntaxKind.EOF && + afterOpener !== JsonSyntaxKind.LineCommentTrivia + ) { + result += '\n'; + needsIndent = true; + } + break; + + case JsonSyntaxKind.CloseBraceToken: + case JsonSyntaxKind.CloseBracketToken: + const wasEmpty = previousToken === JsonSyntaxKind.OpenBraceToken || + previousToken === JsonSyntaxKind.OpenBracketToken; + + let indentUnderflow = indent === 0; + indent = Math.max(0, indent - 1); + + if (!wasEmpty) { + if (!result.endsWith('\n')) { + result += '\n'; + } + result += indentString.repeat(indent); + } + + result += tokenText; + if (indentUnderflow) result += '\n'; + + break; + + case JsonSyntaxKind.CommaToken: + result += tokenText; + + const afterComma = scanAhead(scanner); + if ( + afterComma !== JsonSyntaxKind.LineCommentTrivia && + afterComma !== JsonSyntaxKind.BlockCommentTrivia && + afterComma !== JsonSyntaxKind.CloseBraceToken && + afterComma !== JsonSyntaxKind.CloseBracketToken && + afterComma !== JsonSyntaxKind.EOF && + afterComma !== JsonSyntaxKind.CommaToken + ) { + result += '\n'; + needsIndent = true; + } + break; + + case JsonSyntaxKind.ColonToken: + result += tokenText; + result += ' '; + break; + + case JsonSyntaxKind.LineCommentTrivia: + const needsNewlineBefore = ( + previousToken === JsonSyntaxKind.OpenBraceToken || + previousToken === JsonSyntaxKind.OpenBracketToken + ) && !result.endsWith('\n'); + + if (needsNewlineBefore) { + result += '\n'; + needsIndent = true; + } + + if (needsIndent) { + result += indentString.repeat(indent); + needsIndent = false; + result += tokenText; + } else { + const trimmedResult = result.trimEnd(); + if (result.length > trimmedResult.length) { + result = trimmedResult + tokenText; + } else { + result += ' ' + tokenText; + } + } + + const afterComment = scanAhead(scanner); + if ( + afterComment !== JsonSyntaxKind.CloseBraceToken && + afterComment !== JsonSyntaxKind.CloseBracketToken && + afterComment !== JsonSyntaxKind.EOF + ) { + result += '\n'; + needsIndent = true; + } + break; + + case JsonSyntaxKind.BlockCommentTrivia: + const prevChar = result[result.length - 1]; + if (prevChar === '\n' || (prevChar === ' ' && result[result.length - 2] === '\n')) { + result += tokenText; + } else { + result += ' ' + tokenText; + } + + const afterBlock = scanAhead(scanner); + if ( + afterBlock !== JsonSyntaxKind.CommaToken && + afterBlock !== JsonSyntaxKind.CloseBraceToken && + afterBlock !== JsonSyntaxKind.CloseBracketToken && + afterBlock !== JsonSyntaxKind.EOF + ) { + result += '\n'; + needsIndent = true; + } + break; + + default: + const followsValue = isValueToken(previousToken); + if (followsValue && isValueToken(token) && !result.endsWith('\n')) { + // Missing comma detected between sequential values, + // so add a newline for readability + result += '\n'; + result += indentString.repeat(indent); + } + + result += tokenText; + break; + } + + previousToken = token; + } + + return result; +} + +function isValueToken(token: JsonSyntaxKind | null): boolean { + return token === JsonSyntaxKind.StringLiteral || + token === JsonSyntaxKind.NumericLiteral || + token === JsonSyntaxKind.TrueKeyword || + token === JsonSyntaxKind.FalseKeyword || + token === JsonSyntaxKind.NullKeyword || + token === JsonSyntaxKind.CloseBraceToken || + token === JsonSyntaxKind.CloseBracketToken; +} + +function scanAhead(scanner: any): JsonSyntaxKind { + const savedPosition = scanner.getPosition(); + + let nextToken = scanner.scan(); + while (nextToken === JsonSyntaxKind.Trivia || + nextToken === JsonSyntaxKind.LineBreakTrivia) { + nextToken = scanner.scan(); + } + + scanner.setPosition(savedPosition); + return nextToken; +} diff --git a/test/unit/util/json.spec.ts b/test/unit/util/json.spec.ts new file mode 100644 index 00000000..45bb3604 --- /dev/null +++ b/test/unit/util/json.spec.ts @@ -0,0 +1,339 @@ +import { expect } from "chai"; + +import { formatJson } from "../../../src/util/json"; + +describe("JSON formatting", () => { + + describe("given valid JSON", () => { + it("should format a simple object", () => { + const input = '{"b":2,"a":1}'; + const expected = `{ + "b": 2, + "a": 1 +}`; + expect(formatJson(input)).to.equal(expected); + }); + + it("should correctly format an object with various data types", () => { + const input = '{"str":"hello world","num":-123.45,"bool":false,"n":null,"emp_str":""}'; + const expected = `{ + "str": "hello world", + "num": -123.45, + "bool": false, + "n": null, + "emp_str": "" +}`; + expect(formatJson(input)).to.equal(expected); + }); + + it("should correctly format an array with mixed data types", () => { + const input = '["a string",-123.45,false,null,{"key":"value"},["nested"]]'; + const expected = `[ + "a string", + -123.45, + false, + null, + { + "key": "value" + }, + [ + "nested" + ] +]`; + expect(formatJson(input)).to.equal(expected); + }); + + it("should correctly format strings with escaped characters", () => { + const input = '{"path":"C:\\\\Users\\\\Test","quote":"\\"hello\\""}'; + const expected = `{ + "path": "C:\\\\Users\\\\Test", + "quote": "\\"hello\\"" +}`; + expect(formatJson(input)).to.equal(expected); + }); + + it("should handle deeply nested structures correctly", () => { + const input = '{"a":{"b":{"c":{"d":[1,{"e":2},3]}}}}'; + const expected = `{ + "a": { + "b": { + "c": { + "d": [ + 1, + { + "e": 2 + }, + 3 + ] + } + } + } +}`; + expect(formatJson(input)).to.equal(expected); + }); + + it("should normalize inconsistent whitespace and newlines", () => { + const input = ` + { + "a" : 1, + "b" : [ 2, + 3 ] + } + `; + const expected = `{ + "a": 1, + "b": [ + 2, + 3 + ] +}`; + expect(formatJson(input)).to.equal(expected); + }); + }); + + describe("given invalid JSON", () => { + + it("should preserve missing quotes", () => { + const invalidInput = '{a: 1}'; + const expected = `{ + a: 1 +}`; + expect(() => formatJson(invalidInput)).to.not.throw(); + expect(formatJson(invalidInput)).to.equal(expected); + }); + + it("should preserve single quotes", () => { + const invalidInput = '{\'a\': 1}'; + const expected = `{ + 'a': 1 +}`; + expect(() => formatJson(invalidInput)).to.not.throw(); + expect(formatJson(invalidInput)).to.equal(expected); + }); + + it("should preserve a trailing comma", () => { + const input = '{"a":1, "b":2,}'; + const expected = `{ + "a": 1, + "b": 2, +}`; + expect(formatJson(input)).to.equal(expected); + }); + + it("should preserve a single-line comment at the end of a line", () => { + const input = '{"a": 1, // My line comment\n"b": 2}'; + const expected = `{ + "a": 1, // My line comment + "b": 2 +}`; + expect(formatJson(input)).to.equal(expected); + }); + + it("should preserve a block comment at the end of a line", () => { + const input = '{"a": 1 /* comment */, "b": 2}'; + const expected = `{ + "a": 1 /* comment */, + "b": 2 +}`; + expect(formatJson(input)).to.equal(expected); + }); + + it("should preserve a block comment before a closing brace", () => { + const input = '{ "a": 1 /* final comment */ }'; + const expected = `{ + "a": 1 /* final comment */ +}`; + expect(formatJson(input)).to.equal(expected); + }); + + it("should preserve both single-line and block comments", () => { + const input = `{ + // This is a comment to be kept + "a": 1, /* This should be kept */ + "b": 2 // This should also be kept + }`; + const expected = `{ + // This is a comment to be kept + "a": 1, /* This should be kept */ + "b": 2 // This should also be kept +}`; + expect(formatJson(input)).to.equal(expected); + }); + + it("should preserve a trailing comma when followed by comments", () => { + const input = '{"a":1, "b":2, // trailing comma\n}'; + const expected = `{ + "a": 1, + "b": 2, // trailing comma +}`; + expect(formatJson(input)).to.equal(expected); + }); + + it("should format lines correctly despite a missing comma between object properties", () => { + const input = '{"a": 1 "b": 2}'; + const expected = `{ + "a": 1 + "b": 2 +}`; + expect(formatJson(input)).to.equal(expected); + }); + + it("should format lines correctly despite a missing comma between array elements", () => { + const input = '[1 "a" false]'; + const expected = `[ + 1 + "a" + false +]`; + expect(formatJson(input)).to.equal(expected); + }); + + it("should preserve multiple consecutive or trailing commas", () => { + const input = '{"a":1,,"b":2,,}'; + const expected = `{ + "a": 1,, + "b": 2,, +}`; + expect(formatJson(input)).to.equal(expected); + }); + + it("should format an unclosed object without adding the closing brace", () => { + const input = '{"a": 1, "b": 2'; + const expected = `{ + "a": 1, + "b": 2`; + expect(formatJson(input)).to.equal(expected); + }); + + it("should format an unclosed array without adding the closing bracket", () => { + const input = '[1, {"a": 2'; + const expected = `[ + 1, + { + "a": 2`; + expect(formatJson(input)).to.equal(expected); + }); + + it("should handle overly closed JSON (negative indents)", () => { + const input = '{}}} []]]'; + const expected = `{} +} +} +[] +] +] +`; + expect(formatJson(input)).to.equal(expected); + }); + + it("should not format complicated JSON records at all if formatRecords is false", () => { + const input = '\u001E{"a":1}\n\u001Enull\n\u001Etrue\n\u001E"hi"\n\u001E[1,2,3]\n\u001E{"b":2}\n'; + const expected = `\u001E{ + "a": 1 +}\u001Enull\u001Etrue\u001E"hi"\u001E[ + 1, + 2, + 3 +]\u001E{ + "b": 2 +}`; + expect(formatJson(input, { formatRecords: false })).to.equal(expected); + }); + }); + + describe("with record formatting enabled", () => { + + it("should correctly format newline-separated JSON records", () => { + const input = '{"a":1}\n{"b":2}'; + const expected = `{ + "a": 1 +} +{ + "b": 2 +}`; + expect(formatJson(input, { formatRecords: true })).to.equal(expected); + }); + + it("should correctly format newline-separated JSON records with weird newlines", () => { + const input = '\n{\n"a"\n:1}\r\n\n\n{\n"b"\n:\n2\r}\n\r\n'; + const expected = `{ + "a": 1 +} +{ + "b": 2 +}`; + expect(formatJson(input, { formatRecords: true })).to.equal(expected); + }); + + it("should correctly format record-separator-separated JSON records (SignalR style)", () => { + const input = '{"a":1}\u001E{"b":2}'; + const expected = `{ + "a": 1 +} +{ + "b": 2 +}`; + expect(formatJson(input, { formatRecords: true })).to.equal(expected); + }); + + it("should correctly format heavily record-separator-separated JSON records", () => { + const input = '\u001E\u001E{"a":1}\u001E\u001E{"b":2}\u001E'; + const expected = `{ + "a": 1 +} +{ + "b": 2 +}`; + expect(formatJson(input, { formatRecords: true })).to.equal(expected); + }); + + it("should correctly format heavily record-separator-separated JSON records with funky newlines", () => { + const input = '\r\n\u001E\n{"a":1}\n\u001E{"b":2}\n\u001E'; + const expected = `{ + "a": 1 +} +{ + "b": 2 +}`; + expect(formatJson(input, { formatRecords: true })).to.equal(expected); + }); + + it("should correctly format record-separator-separated JSON records with literal values (json-seq style)", () => { + const input = '\u001E{"a":1}\n\u001Enull\n\u001Etrue\n\u001E"hi"\n\u001E[1,2,3]\n\u001E{"b":2}\n'; + const expected = `{ + "a": 1 +} +null +true +"hi" +[ + 1, + 2, + 3 +] +{ + "b": 2 +}`; + expect(formatJson(input, { formatRecords: true })).to.equal(expected); + }); + + it("should still correctly format an array with mixed data types", () => { + const input = '["a string",-123.45,false,null,{"key":"value"},["nested"]]'; + const expected = `[ + "a string", + -123.45, + false, + null, + { + "key": "value" + }, + [ + "nested" + ] +]`; + expect(formatJson(input, { formatRecords: true })).to.equal(expected); + }); + + }); + +}); \ No newline at end of file From 5e317c7a841f8dba5e4a4e183d68a4697cfb12dc Mon Sep 17 00:00:00 2001 From: Weihan Li Date: Thu, 7 Aug 2025 08:08:44 +0800 Subject: [PATCH 12/13] cleanup unused splitBuffer method --- src/model/events/body-formatting.ts | 2 +- src/util/buffer.ts | 29 ----------------------------- 2 files changed, 1 insertion(+), 30 deletions(-) diff --git a/src/model/events/body-formatting.ts b/src/model/events/body-formatting.ts index c56991f5..cff17b45 100644 --- a/src/model/events/body-formatting.ts +++ b/src/model/events/body-formatting.ts @@ -3,7 +3,7 @@ import { styled } from '../../styles'; import { ViewableContentType } from '../events/content-types'; import { ObservablePromise, observablePromise } from '../../util/observable'; -import { bufferToString, bufferToHex, splitBuffer } from '../../util/buffer'; +import { bufferToString, bufferToHex } from '../../util/buffer'; import type { WorkerFormatterKey } from '../../services/ui-worker-formatters'; import { formatBufferAsync } from '../../services/ui-worker-api'; diff --git a/src/util/buffer.ts b/src/util/buffer.ts index f9ad085c..7bc70901 100644 --- a/src/util/buffer.ts +++ b/src/util/buffer.ts @@ -150,32 +150,3 @@ export function getReadableSize(input: number | Buffer | string, siUnits = true) return (bytes / Math.pow(thresh, unitIndex)).toFixed(1).replace(/\.0$/, '') + ' ' + unitName; } - -/** - * Splits a Buffer into an array of Buffers using a specified separator. - * @param buffer The Buffer to split. - * @param separator The byte or Buffer sequence to split by. - * @returns An array of Buffers. - */ -export function splitBuffer(buffer: Buffer, separator: number | Buffer): Buffer[] { - const result: Buffer[] = []; - let currentOffset = 0; - let separatorIndex: number; - - // Handle single byte separator vs. multi-byte separator - const separatorLength = typeof separator === 'number' ? 1 : separator.length; - - while ((separatorIndex = buffer.indexOf(separator, currentOffset)) !== -1) { - // Add the chunk before the separator - result.push(buffer.slice(currentOffset, separatorIndex)); - // Move the offset past the separator - currentOffset = separatorIndex + separatorLength; - } - - // Add the last chunk (or the whole buffer if no separator was found) - if (currentOffset <= buffer.length) { - result.push(buffer.slice(currentOffset)); - } - - return result; -} \ No newline at end of file From 1adda66156af572a8584c910d73650272260e60c Mon Sep 17 00:00:00 2001 From: Weihan Li Date: Thu, 7 Aug 2025 08:11:10 +0800 Subject: [PATCH 13/13] revert buffer.ts --- src/util/buffer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/buffer.ts b/src/util/buffer.ts index 7bc70901..8769f2ac 100644 --- a/src/util/buffer.ts +++ b/src/util/buffer.ts @@ -149,4 +149,4 @@ export function getReadableSize(input: number | Buffer | string, siUnits = true) let unitName = bytes === 1 ? 'byte' : units[unitIndex]; return (bytes / Math.pow(thresh, unitIndex)).toFixed(1).replace(/\.0$/, '') + ' ' + unitName; -} +} \ No newline at end of file