From 65b2b2bc0f913ddfba9dbb7a8bdde996381f9435 Mon Sep 17 00:00:00 2001 From: Sylvain Wallez Date: Fri, 27 Sep 2024 20:26:17 +0200 Subject: [PATCH 1/3] Use a singleton row proxy handler to reduce memory usage --- js/src/row/struct.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/js/src/row/struct.ts b/js/src/row/struct.ts index bc3869cb8d0..074ec91fd64 100644 --- a/js/src/row/struct.ts +++ b/js/src/row/struct.ts @@ -39,7 +39,7 @@ export class StructRow { constructor(parent: Data>, rowIndex: number) { this[kParent] = parent; this[kRowIndex] = rowIndex; - return new Proxy(this, new StructRowProxyHandler()); + return new Proxy(this, structRowProxyHandler); } public toArray() { return Object.values(this.toJSON()); } @@ -157,3 +157,5 @@ class StructRowProxyHandler implements ProxyHandler Date: Fri, 27 Sep 2024 20:27:28 +0200 Subject: [PATCH 2/3] Add a lightweight table array view that only allocates a proxy object --- js/perf/index.ts | 4 ++ js/src/table.ts | 91 +++++++++++++++++++++++++++++++- js/test/unit/table/table-test.ts | 58 ++++++++++++++++++++ 3 files changed, 151 insertions(+), 2 deletions(-) diff --git a/js/perf/index.ts b/js/perf/index.ts index 2869470b469..d96d73429be 100755 --- a/js/perf/index.ts +++ b/js/perf/index.ts @@ -205,6 +205,10 @@ for (const { name, table, counts } of config) { table.toArray(); }), + b.add(`toArrayView, dataset: ${name}, numRows: ${formatNumber(table.numRows)}`, () => { + table.toArrayView(); + }), + b.add(`get, dataset: ${name}, numRows: ${formatNumber(table.numRows)}`, () => { for (let i = -1, n = table.numRows; ++i < n;) { table.get(i); diff --git a/js/src/table.ts b/js/src/table.ts index 2aab2b7ec2e..ea2c3350026 100644 --- a/js/src/table.ts +++ b/js/src/table.ts @@ -238,17 +238,30 @@ export class Table { * * @returns An Array of Table rows. */ - public toArray() { + public toArray(): Array['TValue']> { return [...this]; } + /** + * Return a JavaScript Array view of the Table rows. + * + * It is a lightweight read-only proxy that delegates to the table. Accessing elements has some + * overhead compared to the regular array returned by `toArray()` because of this indirection, + * but it avoids potentially large memory allocation. + * + * @returns An Array proxy to the Table rows. + */ + public toArrayView(): Array['TValue']> { + return new Proxy([] as Array['TValue']>, new TableArrayProxyHandler(this)); + } + /** * Returns a string representation of the Table rows. * * @returns A string representation of the Table rows. */ public toString() { - return `[\n ${this.toArray().join(',\n ')}\n]`; + return `[\n ${this.toArrayView().join(',\n ')}\n]`; } /** @@ -444,3 +457,77 @@ export function tableFromArrays(vecs); } + +class TableArrayProxyHandler implements ProxyHandler['TValue']>> { + table: Table; + + constructor(table: Table) { + this.table = table; + } + + // Traps that aren't implemented: + // - apply + // - construct + // - defineProperty + // - deleteProperty + // - getPrototypeOf + // - isExtensible + // - preventExtensions + // - set + // - setPrototypeOf + + get(target: Array['TValue']>, p: string | symbol, receiver: any): any { + if (typeof p === 'string') { + const i = Number(p); + if (Number.isInteger(i)) { + return this.table.get(i); + } + if (p === 'at') { + return (i: number): Struct['TValue'] | undefined => { + return this.table.at(i) ?? undefined; + }; + } + if (p === 'length') { + return this.table.numRows; + } + } else if (p === Symbol('keys')) { + const end = this.table.numRows; + return function * () { + let i = 0; + while(i < end) { + yield i++; + } + return; + }; + } + return Reflect.get(target, p, receiver); + } + + getOwnPropertyDescriptor(target: Array['TValue']>, p: string | symbol): PropertyDescriptor | undefined { + if (typeof p === 'string') { + const i = Number(p); + if (Number.isInteger(i) && i >= 0 && i < this.table.numRows) { + return { enumerable: true, configurable: true }; + } + } + return Reflect.getOwnPropertyDescriptor(target, p); + } + + has(target: Array['TValue']>, p: string | symbol): boolean { + if (typeof p === 'string') { + const i = Number(p); + if (Number.isInteger(i)) { + return i >= 0 && i < this.table.numRows; + } + } + return Reflect.has(target, p); + } + + ownKeys(_target: Array['TValue']>): ArrayLike { + // Can be expensive as we allocate an array with all index numbers as strings + const keys = Array.from({length: this.table.numRows}, (_, i) => String(i)); + keys.push('length'); + return keys; + } +} + diff --git a/js/test/unit/table/table-test.ts b/js/test/unit/table/table-test.ts index 01af4090987..908526892a4 100644 --- a/js/test/unit/table/table-test.ts +++ b/js/test/unit/table/table-test.ts @@ -79,3 +79,61 @@ describe('tableFromJSON()', () => { expect(table.getChild('c')!.type).toBeInstanceOf(Dictionary); }); }); + +describe('table views', () => { + const table = tableFromJSON([{ + a: 42, + b: true, + c: 'foo', + }, { + a: 12, + b: false, + c: 'bar', + }]); + + function checkArrayValues(arr: Array) { + expect(arr).toHaveLength(2); + expect(arr[0].a).toBe(42); + expect(arr[0].b).toBe(true); + expect(arr[0].c).toBe('foo'); + expect(arr[1].a).toBe(12); + expect(arr[1].b).toBe(false); + expect(arr[1].c).toBe('bar'); + } + + function checkArray(arr: Array) { + test('Wrapper', () => checkArrayValues(arr)); + + test('Iterator', () => { + const arr2 = []; + for (let item of arr) { + arr2.push(item); + } + checkArrayValues(arr); + }); + + test('Array index', () => { + const arr2 = new Array(arr.length); + for (let i = 0; i < arr2.length; i++) { + arr2[i] = arr[i]; + } + checkArrayValues(arr2); + }); + + test('Keys', () => { + const arr2: any[] = new Array(arr.length); + let keys = Object.keys(arr); + for (let k in keys) { + arr2[k] = arr[k]; + } + checkArrayValues(arr2); + }); + } + + describe('table.toArray()', () => { + checkArray(table.toArray()); + }); + describe('table.toArrayView()', () => { + checkArray(table.toArrayView()); + }); +}); From 4b706a904000bd97a9b270c8609cf6c62565e436 Mon Sep 17 00:00:00 2001 From: Sylvain Wallez Date: Tue, 1 Oct 2024 17:55:25 +0200 Subject: [PATCH 3/3] Remove type annotation --- js/src/table.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/js/src/table.ts b/js/src/table.ts index ea2c3350026..d9ee825f99b 100644 --- a/js/src/table.ts +++ b/js/src/table.ts @@ -238,7 +238,7 @@ export class Table { * * @returns An Array of Table rows. */ - public toArray(): Array['TValue']> { + public toArray() { return [...this]; } @@ -251,7 +251,7 @@ export class Table { * * @returns An Array proxy to the Table rows. */ - public toArrayView(): Array['TValue']> { + public toArrayView() { return new Proxy([] as Array['TValue']>, new TableArrayProxyHandler(this)); }