From 6621252e70ebf44ed0ffcb6043a12db856db289f Mon Sep 17 00:00:00 2001 From: Erik Marks Date: Fri, 17 Jan 2025 21:39:34 -0800 Subject: [PATCH 01/27] refactor: Move sqlite-wasm code to extension --- packages/extension/package.json | 1 + packages/extension/src/iframe.ts | 3 +++ .../src/kernel-integration/kernel-worker.ts | 3 ++- .../src/kernel-integration}/sqlite-kv-store.ts | 11 +---------- packages/kernel/package.json | 1 - packages/kernel/src/Kernel.test.ts | 2 +- packages/kernel/src/Kernel.ts | 3 +-- packages/kernel/src/VatSupervisor.test.ts | 2 ++ packages/kernel/src/VatSupervisor.ts | 10 +++++++--- packages/kernel/src/index.test.ts | 1 - packages/kernel/src/index.ts | 3 +-- packages/kernel/src/store/kernel-store.test.ts | 2 +- packages/kernel/src/store/kernel-store.ts | 16 +++++++++++++++- packages/kernel/src/syscall.ts | 2 +- packages/kernel/test/storage.ts | 2 +- packages/nodejs/src/vat/vat-worker.ts | 3 +++ vitest.config.ts | 16 ++++++++-------- yarn.lock | 10 +++++----- 18 files changed, 53 insertions(+), 38 deletions(-) rename packages/{kernel/src/store => extension/src/kernel-integration}/sqlite-kv-store.ts (94%) diff --git a/packages/extension/package.json b/packages/extension/package.json index f3879c653..bf5c6a8df 100644 --- a/packages/extension/package.json +++ b/packages/extension/package.json @@ -53,6 +53,7 @@ "@ocap/shims": "workspace:^", "@ocap/streams": "workspace:^", "@ocap/utils": "workspace:^", + "@sqlite.org/sqlite-wasm": "^3.48.0-build1", "react": "^18.3.1", "react-dom": "^18.3.1", "ses": "^1.9.0" diff --git a/packages/extension/src/iframe.ts b/packages/extension/src/iframe.ts index fd8a563cd..a8b2f04b3 100644 --- a/packages/extension/src/iframe.ts +++ b/packages/extension/src/iframe.ts @@ -2,6 +2,8 @@ import { isVatCommand, VatSupervisor } from '@ocap/kernel'; import type { VatCommand, VatCommandReply } from '@ocap/kernel'; import { MessagePortMultiplexer, receiveMessagePort } from '@ocap/streams'; +import { makeSQLKVStore } from './kernel-integration/sqlite-kv-store.js'; + main().catch(console.error); /** @@ -21,6 +23,7 @@ async function main(): Promise { new VatSupervisor({ id: 'iframe', commandStream, + makeKVStore: makeSQLKVStore, }); await multiplexer.start(); diff --git a/packages/extension/src/kernel-integration/kernel-worker.ts b/packages/extension/src/kernel-integration/kernel-worker.ts index 784734a94..28545d7ec 100644 --- a/packages/extension/src/kernel-integration/kernel-worker.ts +++ b/packages/extension/src/kernel-integration/kernel-worker.ts @@ -3,7 +3,7 @@ import type { KernelCommandReply, ClusterConfig, } from '@ocap/kernel'; -import { isKernelCommand, Kernel, makeSQLKVStore } from '@ocap/kernel'; +import { isKernelCommand, Kernel } from '@ocap/kernel'; import { MessagePortDuplexStream, receiveMessagePort, @@ -15,6 +15,7 @@ import { makeLogger } from '@ocap/utils'; import { handlePanelMessage } from './handle-panel-message.js'; import { isKernelControlCommand } from './messages.js'; import type { KernelControlCommand, KernelControlReply } from './messages.js'; +import { makeSQLKVStore } from './sqlite-kv-store.js'; import { ExtensionVatWorkerClient } from './VatWorkerClient.js'; const bundleHost = 'http://localhost:3000'; // XXX placeholder diff --git a/packages/kernel/src/store/sqlite-kv-store.ts b/packages/extension/src/kernel-integration/sqlite-kv-store.ts similarity index 94% rename from packages/kernel/src/store/sqlite-kv-store.ts rename to packages/extension/src/kernel-integration/sqlite-kv-store.ts index a055c24a7..176108685 100644 --- a/packages/kernel/src/store/sqlite-kv-store.ts +++ b/packages/extension/src/kernel-integration/sqlite-kv-store.ts @@ -1,17 +1,8 @@ +import type { KVStore } from '@ocap/kernel'; import { makeLogger } from '@ocap/utils'; import type { Database } from '@sqlite.org/sqlite-wasm'; import sqlite3InitModule from '@sqlite.org/sqlite-wasm'; -export type KVStore = { - get(key: string): string | undefined; - getRequired(key: string): string; - getNextKey(previousKey: string): string | undefined; - set(key: string, value: string): void; - delete(key: string): void; - clear(): void; - executeQuery(sql: string): Record[]; -}; - /** * Ensure that SQLite is initialized. * diff --git a/packages/kernel/package.json b/packages/kernel/package.json index 4fa0de43a..d9eed3839 100644 --- a/packages/kernel/package.json +++ b/packages/kernel/package.json @@ -59,7 +59,6 @@ "@ocap/shims": "workspace:^", "@ocap/streams": "workspace:^", "@ocap/utils": "workspace:^", - "@sqlite.org/sqlite-wasm": "3.46.1-build3", "ses": "^1.9.0", "setimmediate": "^1.0.5" }, diff --git a/packages/kernel/src/Kernel.test.ts b/packages/kernel/src/Kernel.test.ts index cee9b10ad..4f093de63 100644 --- a/packages/kernel/src/Kernel.test.ts +++ b/packages/kernel/src/Kernel.test.ts @@ -13,7 +13,7 @@ import type { KernelCommandReply, VatCommand, } from './messages/index.js'; -import type { KVStore } from './store/sqlite-kv-store.js'; +import type { KVStore } from './store/kernel-store.js'; import type { VatId, VatConfig, VatWorkerService } from './types.js'; import { VatHandle } from './VatHandle.js'; import { makeMapKVStore } from '../test/storage.js'; diff --git a/packages/kernel/src/Kernel.ts b/packages/kernel/src/Kernel.ts index 2d257d52f..52539e8db 100644 --- a/packages/kernel/src/Kernel.ts +++ b/packages/kernel/src/Kernel.ts @@ -29,13 +29,12 @@ import type { VatCommandReply, VatCommandReturnType, } from './messages/index.js'; -import type { KernelStore } from './store/kernel-store.js'; import { parseRef, isPromiseRef, makeKernelStore, } from './store/kernel-store.js'; -import type { KVStore } from './store/sqlite-kv-store.js'; +import type { KernelStore, KVStore } from './store/kernel-store.js'; import type { VatId, VRef, diff --git a/packages/kernel/src/VatSupervisor.test.ts b/packages/kernel/src/VatSupervisor.test.ts index f4c8740bd..72d233c14 100644 --- a/packages/kernel/src/VatSupervisor.test.ts +++ b/packages/kernel/src/VatSupervisor.test.ts @@ -31,6 +31,8 @@ const makeVatSupervisor = async ( supervisor: new VatSupervisor({ id: 'test-id', commandStream, + // @ts-expect-error Mock + makeKVStore: async () => ({}), }), stream, }; diff --git a/packages/kernel/src/VatSupervisor.ts b/packages/kernel/src/VatSupervisor.ts index b190edde2..38b9edf03 100644 --- a/packages/kernel/src/VatSupervisor.ts +++ b/packages/kernel/src/VatSupervisor.ts @@ -17,7 +17,7 @@ import type { import { makeDummyMeterControl } from './dummyMeterControl.js'; import type { VatCommand, VatCommandReply } from './messages/index.js'; import { VatCommandMethod } from './messages/index.js'; -import { makeSQLKVStore } from './store/sqlite-kv-store.js'; +import type { MakeKVStore } from './store/kernel-store.js'; import { makeSupervisorSyscall } from './syscall.js'; import type { VatConfig } from './types.js'; import { ROOT_OBJECT_VREF, isVatConfig } from './types.js'; @@ -32,6 +32,7 @@ const makeLiveSlots: (...args: unknown[]) => LiveSlots = localMakeLiveSlots; // type SupervisorConstructorProps = { id: string; commandStream: DuplexStream; + makeKVStore: MakeKVStore; }; const marshal = makeMarshal(undefined, undefined, { @@ -52,11 +53,14 @@ export class VatSupervisor { #dispatch: DispatchFn | null; + readonly #makeKVStore: MakeKVStore; + readonly #syscallsInFlight: Promise[] = []; - constructor({ id, commandStream }: SupervisorConstructorProps) { + constructor({ id, commandStream, makeKVStore }: SupervisorConstructorProps) { this.id = id; this.#commandStream = commandStream; + this.#makeKVStore = makeKVStore; this.#dispatch = null; Promise.all([ @@ -189,7 +193,7 @@ export class VatSupervisor { } this.#loaded = true; - const kvStore = await makeSQLKVStore(`[vat-${this.id}]`, true); + const kvStore = await this.#makeKVStore(`[vat-${this.id}]`, true); const syscall = makeSupervisorSyscall(this, kvStore); const vatPowers = {}; // XXX should be something more real const liveSlotsOptions = {}; // XXX should be something more real diff --git a/packages/kernel/src/index.test.ts b/packages/kernel/src/index.test.ts index d238a2a74..8311437c7 100644 --- a/packages/kernel/src/index.test.ts +++ b/packages/kernel/src/index.test.ts @@ -24,7 +24,6 @@ describe('index', () => { 'isVatId', 'isVatWorkerServiceCommand', 'isVatWorkerServiceReply', - 'makeSQLKVStore', ]); }); }); diff --git a/packages/kernel/src/index.ts b/packages/kernel/src/index.ts index ba8aacd6a..1ba543bb2 100644 --- a/packages/kernel/src/index.ts +++ b/packages/kernel/src/index.ts @@ -1,7 +1,6 @@ export * from './messages/index.js'; export { Kernel } from './Kernel.js'; -export type { KVStore } from './store/sqlite-kv-store.js'; -export { makeSQLKVStore } from './store/sqlite-kv-store.js'; +export type { KVStore, MakeKVStore } from './store/kernel-store.js'; export { VatHandle } from './VatHandle.js'; export { VatSupervisor } from './VatSupervisor.js'; // XXX Once the packaging of liveslots is fixed, this should be imported from there diff --git a/packages/kernel/src/store/kernel-store.test.ts b/packages/kernel/src/store/kernel-store.test.ts index c16196db2..b4c3441f9 100644 --- a/packages/kernel/src/store/kernel-store.test.ts +++ b/packages/kernel/src/store/kernel-store.test.ts @@ -1,7 +1,7 @@ import { describe, it, expect, beforeEach } from 'vitest'; import { makeKernelStore } from './kernel-store.js'; -import type { KVStore } from './sqlite-kv-store.js'; +import type { KVStore } from './kernel-store.js'; import { makeMapKVStore } from '../../test/storage.js'; // XXX Once the packaging of liveslots is fixed this should be imported from there import type { Message } from '../ag-types.js'; diff --git a/packages/kernel/src/store/kernel-store.ts b/packages/kernel/src/store/kernel-store.ts index aee9cd0ed..a92b42efc 100644 --- a/packages/kernel/src/store/kernel-store.ts +++ b/packages/kernel/src/store/kernel-store.ts @@ -55,7 +55,6 @@ import type { CapData } from '@endo/marshal'; -import type { KVStore } from './sqlite-kv-store.js'; // XXX Once the packaging of liveslots is fixed this should be imported from there import type { Message } from '../ag-types.js'; import type { @@ -69,6 +68,21 @@ import type { KernelPromise, } from '../types.js'; +export type KVStore = { + get(key: string): string | undefined; + getRequired(key: string): string; + getNextKey(previousKey: string): string | undefined; + set(key: string, value: string): void; + delete(key: string): void; + clear(): void; + executeQuery(sql: string): Record[]; +}; + +export type MakeKVStore = ( + label: string, + beEphemeral: boolean, +) => Promise; + type StoredValue = { get(): string | undefined; set(newValue: string): void; diff --git a/packages/kernel/src/syscall.ts b/packages/kernel/src/syscall.ts index 427d0fe31..913072eef 100644 --- a/packages/kernel/src/syscall.ts +++ b/packages/kernel/src/syscall.ts @@ -12,7 +12,7 @@ import type { VatOneResolution, SwingSetCapData, } from './ag-types-index.js'; -import type { KVStore } from './store/sqlite-kv-store.js'; +import type { KVStore } from './store/kernel-store.js'; import type { VatSupervisor } from './VatSupervisor.ts'; export type SyscallResult = SwingSetCapData | string | string[] | null; diff --git a/packages/kernel/test/storage.ts b/packages/kernel/test/storage.ts index 246f56138..15cd00e9e 100644 --- a/packages/kernel/test/storage.ts +++ b/packages/kernel/test/storage.ts @@ -1,4 +1,4 @@ -import type { KVStore } from '../src/store/sqlite-kv-store.js'; +import type { KVStore } from '../src/store/kernel-store.js'; /** * A mock key/value store realized as a Map. diff --git a/packages/nodejs/src/vat/vat-worker.ts b/packages/nodejs/src/vat/vat-worker.ts index efde1bad6..313090c40 100644 --- a/packages/nodejs/src/vat/vat-worker.ts +++ b/packages/nodejs/src/vat/vat-worker.ts @@ -6,6 +6,8 @@ import { NodeWorkerMultiplexer } from '@ocap/streams'; import { makeLogger } from '@ocap/utils'; import { parentPort } from 'node:worker_threads'; +import { makeSQLKVStore } from '../kernel/sqlite-kv-store.js'; + // eslint-disable-next-line n/no-process-env const logger = makeLogger(`[vat-worker (${process.env.NODE_VAT_ID})]`); @@ -29,5 +31,6 @@ async function main(): Promise { void new VatSupervisor({ id: 'iframe', commandStream, + makeKVStore: makeSQLKVStore, }); } diff --git a/vitest.config.ts b/vitest.config.ts index 97dabb886..2dfb2f545 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -61,16 +61,16 @@ export default defineConfig({ lines: 100, }, 'packages/extension/**': { - statements: 74.89, - functions: 74.7, - branches: 78.62, - lines: 74.79, + statements: 66.48, + functions: 70.55, + branches: 69.12, + lines: 66.48, }, 'packages/kernel/**': { - statements: 39.05, - functions: 50.31, - branches: 28.17, - lines: 39.33, + statements: 42.2, + functions: 53.69, + branches: 30.03, + lines: 42.47, }, 'packages/nodejs/**': { statements: 4.12, diff --git a/yarn.lock b/yarn.lock index 5173b3e73..d29819940 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2108,6 +2108,7 @@ __metadata: "@ocap/test-utils": "workspace:^" "@ocap/utils": "workspace:^" "@playwright/test": "npm:^1.49.1" + "@sqlite.org/sqlite-wasm": "npm:^3.48.0-build1" "@testing-library/jest-dom": "npm:^6.6.3" "@testing-library/react": "npm:^16.1.0" "@testing-library/user-event": "npm:^14.5.2" @@ -2173,7 +2174,6 @@ __metadata: "@ocap/streams": "workspace:^" "@ocap/test-utils": "workspace:^" "@ocap/utils": "workspace:^" - "@sqlite.org/sqlite-wasm": "npm:3.46.1-build3" "@ts-bridge/cli": "npm:^0.6.2" "@ts-bridge/shims": "npm:^0.1.1" "@types/setimmediate": "npm:^1.0.4" @@ -3013,12 +3013,12 @@ __metadata: languageName: node linkType: hard -"@sqlite.org/sqlite-wasm@npm:3.46.1-build3": - version: 3.46.1-build3 - resolution: "@sqlite.org/sqlite-wasm@npm:3.46.1-build3" +"@sqlite.org/sqlite-wasm@npm:^3.48.0-build1": + version: 3.48.0-build1 + resolution: "@sqlite.org/sqlite-wasm@npm:3.48.0-build1" bin: sqlite-wasm: bin/index.js - checksum: 10/a64225fd784ed2ee8c8bf82f042d05b567b4707fba22a9508fbe3ac42cf1cf7e722d2ede0e1d91556bfec66b140aeb3cf3967546249693f39cc81475d7be90ff + checksum: 10/e163ac23f79176ca777db285dd19384b52feac64725bcfef24ec925a0e6f1f0b80acfbcc238656cd695b096e95cf63f38d4f0c05178367cfa5c6a75bd9aeb35c languageName: node linkType: hard From 01026803df7a2819c919bcfd18126758db1c60a3 Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Thu, 9 Jan 2025 14:55:13 -0600 Subject: [PATCH 02/27] draft --- packages/nodejs/src/vat/inside.test.ts | 66 ++++++++++++++++++++++++++ packages/nodejs/src/vat/vat-worker.ts | 2 + 2 files changed, 68 insertions(+) create mode 100644 packages/nodejs/src/vat/inside.test.ts diff --git a/packages/nodejs/src/vat/inside.test.ts b/packages/nodejs/src/vat/inside.test.ts new file mode 100644 index 000000000..cd4f60cbe --- /dev/null +++ b/packages/nodejs/src/vat/inside.test.ts @@ -0,0 +1,66 @@ +import '@ocap/shims/endoify'; + +import { VatSupervisor } from '@ocap/kernel'; +import { NodeWorkerMultiplexer } from '@ocap/streams'; +import { makeLogger } from '@ocap/utils'; +import { describe, expect, it, vi } from 'vitest'; + +import { main } from './inside.js'; + +vi.mock('node:worker_threads', () => ({ + parentPort: {}, +})); + +vi.mock('@ocap/kernel', () => { + const MockVatSupervisor = vi.fn(); + vi.spyOn(MockVatSupervisor.prototype, 'evaluate').mockImplementation(); + return { + VatSupervisor: MockVatSupervisor, + }; +}); + +vi.mock('@ocap/streams', () => { + const MockNodeWorkerMultiplexer = vi.fn(); + vi.spyOn(MockNodeWorkerMultiplexer.prototype, 'start') + .mockImplementation() + .mockResolvedValue(undefined); + vi.spyOn( + MockNodeWorkerMultiplexer.prototype, + 'createChannel', + ).mockImplementation(); + + return { + NodeWorkerMultiplexer: MockNodeWorkerMultiplexer, + }; +}); + +describe('inside', () => { + it('reads vat id from NODE_VAT_ID', async () => { + const vatId = 'v20'; + vi.stubEnv('NODE_VAT_ID', vatId); + await main(); + expect(makeLogger).toHaveBeenCalledWith(`[${vatId} (inside)]`); + }); + + it('creates a VatSupervisor and call its evaluate method', async () => { + const MockNodeWorkerMultiplexer = vi.mocked(NodeWorkerMultiplexer); + const MockVatSupervisor = vi.mocked(VatSupervisor); + + await main(); + + expect(MockNodeWorkerMultiplexer).toHaveBeenCalledOnce(); + expect(MockNodeWorkerMultiplexer.mock.instances.at(0)).toBeDefined(); + expect( + MockNodeWorkerMultiplexer.mock.instances.at(0)?.start, + ).toHaveBeenCalledOnce(); + expect( + MockNodeWorkerMultiplexer.mock.instances.at(0)?.createChannel, + ).toHaveBeenCalledTimes(2); + + expect(MockVatSupervisor).toHaveBeenCalledOnce(); + expect(MockVatSupervisor.mock.instances.at(0)).toBeDefined(); + expect( + MockVatSupervisor.mock.instances.at(0)?.evaluate, + ).toHaveBeenCalledWith('["Hello", "world!"].join(" ");'); + }); +}); diff --git a/packages/nodejs/src/vat/vat-worker.ts b/packages/nodejs/src/vat/vat-worker.ts index 313090c40..f3ef3e919 100644 --- a/packages/nodejs/src/vat/vat-worker.ts +++ b/packages/nodejs/src/vat/vat-worker.ts @@ -23,7 +23,9 @@ async function main(): Promise { throw new Error(errMsg); } const multiplexer = new NodeWorkerMultiplexer(parentPort, 'vat'); + console.debug('premulti'); multiplexer.start().catch(logger.error); + console.debug('postmulti'); const commandStream = multiplexer.createChannel( 'command', ); From f55c9ce51f972c7138fb0c44c2d922de72cec7e6 Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Fri, 10 Jan 2025 13:49:41 -0600 Subject: [PATCH 03/27] fix vitest endoify resolution --- packages/nodejs/vitest.config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/nodejs/vitest.config.ts b/packages/nodejs/vitest.config.ts index 7e149b0e5..e460f88d4 100644 --- a/packages/nodejs/vitest.config.ts +++ b/packages/nodejs/vitest.config.ts @@ -13,7 +13,7 @@ const config = mergeConfig( alias: [ { find: '@ocap/shims/endoify', - replacement: path.resolve('../shims/src/endoify.js'), + replacement: path.resolve(__dirname, '../shims/src/endoify.js'), customResolver: (id) => ({ external: true, id }), }, ], From 6df657b376b16b324c305669baafa3b09bf2f842 Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Fri, 10 Jan 2025 18:24:54 -0600 Subject: [PATCH 04/27] align nodejs vat-worker with extension vat iframe --- packages/nodejs/src/vat/inside.test.ts | 66 ---------------------- packages/nodejs/src/vat/vat-worker.test.ts | 59 +++++++++++++++++++ packages/nodejs/src/vat/vat-worker.ts | 16 +++--- 3 files changed, 68 insertions(+), 73 deletions(-) delete mode 100644 packages/nodejs/src/vat/inside.test.ts create mode 100644 packages/nodejs/src/vat/vat-worker.test.ts diff --git a/packages/nodejs/src/vat/inside.test.ts b/packages/nodejs/src/vat/inside.test.ts deleted file mode 100644 index cd4f60cbe..000000000 --- a/packages/nodejs/src/vat/inside.test.ts +++ /dev/null @@ -1,66 +0,0 @@ -import '@ocap/shims/endoify'; - -import { VatSupervisor } from '@ocap/kernel'; -import { NodeWorkerMultiplexer } from '@ocap/streams'; -import { makeLogger } from '@ocap/utils'; -import { describe, expect, it, vi } from 'vitest'; - -import { main } from './inside.js'; - -vi.mock('node:worker_threads', () => ({ - parentPort: {}, -})); - -vi.mock('@ocap/kernel', () => { - const MockVatSupervisor = vi.fn(); - vi.spyOn(MockVatSupervisor.prototype, 'evaluate').mockImplementation(); - return { - VatSupervisor: MockVatSupervisor, - }; -}); - -vi.mock('@ocap/streams', () => { - const MockNodeWorkerMultiplexer = vi.fn(); - vi.spyOn(MockNodeWorkerMultiplexer.prototype, 'start') - .mockImplementation() - .mockResolvedValue(undefined); - vi.spyOn( - MockNodeWorkerMultiplexer.prototype, - 'createChannel', - ).mockImplementation(); - - return { - NodeWorkerMultiplexer: MockNodeWorkerMultiplexer, - }; -}); - -describe('inside', () => { - it('reads vat id from NODE_VAT_ID', async () => { - const vatId = 'v20'; - vi.stubEnv('NODE_VAT_ID', vatId); - await main(); - expect(makeLogger).toHaveBeenCalledWith(`[${vatId} (inside)]`); - }); - - it('creates a VatSupervisor and call its evaluate method', async () => { - const MockNodeWorkerMultiplexer = vi.mocked(NodeWorkerMultiplexer); - const MockVatSupervisor = vi.mocked(VatSupervisor); - - await main(); - - expect(MockNodeWorkerMultiplexer).toHaveBeenCalledOnce(); - expect(MockNodeWorkerMultiplexer.mock.instances.at(0)).toBeDefined(); - expect( - MockNodeWorkerMultiplexer.mock.instances.at(0)?.start, - ).toHaveBeenCalledOnce(); - expect( - MockNodeWorkerMultiplexer.mock.instances.at(0)?.createChannel, - ).toHaveBeenCalledTimes(2); - - expect(MockVatSupervisor).toHaveBeenCalledOnce(); - expect(MockVatSupervisor.mock.instances.at(0)).toBeDefined(); - expect( - MockVatSupervisor.mock.instances.at(0)?.evaluate, - ).toHaveBeenCalledWith('["Hello", "world!"].join(" ");'); - }); -}); diff --git a/packages/nodejs/src/vat/vat-worker.test.ts b/packages/nodejs/src/vat/vat-worker.test.ts new file mode 100644 index 000000000..f96d15fa5 --- /dev/null +++ b/packages/nodejs/src/vat/vat-worker.test.ts @@ -0,0 +1,59 @@ +import '@ocap/shims/endoify'; + +import * as ocapKernel from '@ocap/kernel'; +import * as ocapStreams from '@ocap/streams'; +import * as ocapUtils from '@ocap/utils'; +import { describe, expect, it, vi } from 'vitest'; + +import { main } from './vat-worker.js'; + +vi.mock('node:worker_threads', () => ({ + parentPort: '{- parentPort -}', +})); + +vi.mock('@ocap/kernel', async (importOriginal: () => Promise) => { + const actual = await importOriginal(); + return { + ...actual, + VatSupervisor: vi.fn(() => ({ + evaluate: vi.fn(), + })), + } +}); + +vi.mock('@ocap/streams', () => ({ + NodeWorkerMultiplexer: vi.fn(() => ({ + start: vi.fn().mockRejectedValue(undefined), + createChannel: vi.fn(), + })), +})); + +describe('inside', () => { + it('creates a VatSupervisor and call its evaluate method', async () => { + const spyNodeWorkerMultiplexer = vi.spyOn(ocapStreams, 'NodeWorkerMultiplexer'); + const spyVatSupervisor = vi.spyOn(ocapKernel, 'VatSupervisor'); + + const vatId = 'v20'; + vi.stubEnv('NODE_VAT_ID', vatId); + + await main(); + + expect(spyNodeWorkerMultiplexer).toHaveBeenCalledOnce(); + expect(spyNodeWorkerMultiplexer.mock.instances.at(0)).toBeDefined(); + expect( + spyNodeWorkerMultiplexer.mock.instances.at(0)?.start, + ).toHaveBeenCalledOnce(); + expect( + spyNodeWorkerMultiplexer.mock.instances.at(0)?.createChannel, + ).toHaveBeenCalledTimes(2); + + expect(spyVatSupervisor).toHaveBeenCalledOnce(); + expect(spyVatSupervisor).toHaveBeenCalledWith({ + id: vatId, + }); + expect(spyVatSupervisor.mock.instances.at(0)).toBeDefined(); + expect( + spyVatSupervisor.mock.instances.at(0)?.evaluate, + ).toHaveBeenCalledWith('["Hello", "world!"].join(" ");'); + }); +}); diff --git a/packages/nodejs/src/vat/vat-worker.ts b/packages/nodejs/src/vat/vat-worker.ts index f3ef3e919..b43bc42bb 100644 --- a/packages/nodejs/src/vat/vat-worker.ts +++ b/packages/nodejs/src/vat/vat-worker.ts @@ -1,36 +1,38 @@ import '@ocap/shims/endoify'; -import { VatSupervisor } from '@ocap/kernel'; +import { isVatCommand, VatSupervisor } from '@ocap/kernel'; import type { VatCommand, VatCommandReply } from '@ocap/kernel'; import { NodeWorkerMultiplexer } from '@ocap/streams'; import { makeLogger } from '@ocap/utils'; import { parentPort } from 'node:worker_threads'; import { makeSQLKVStore } from '../kernel/sqlite-kv-store.js'; +const vatId = process.env.NODE_VAT_ID // eslint-disable-next-line n/no-process-env -const logger = makeLogger(`[vat-worker (${process.env.NODE_VAT_ID})]`); +const logger = makeLogger(`[vat-worker ${vatId}]`); main().catch(logger.error); /** * The main function for the iframe. */ -async function main(): Promise { +export async function main(): Promise { + logger.debug('started main'); + if (!parentPort) { const errMsg = 'Expected to run in Node Worker with parentPort.'; logger.error(errMsg); throw new Error(errMsg); } const multiplexer = new NodeWorkerMultiplexer(parentPort, 'vat'); - console.debug('premulti'); multiplexer.start().catch(logger.error); - console.debug('postmulti'); const commandStream = multiplexer.createChannel( 'command', + isVatCommand, ); - // eslint-disable-next-line no-void - void new VatSupervisor({ + + const supervisor = new VatSupervisor({ id: 'iframe', commandStream, makeKVStore: makeSQLKVStore, From 9f034bbc8d6d00cc802adeb0501c8bd506a54122 Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Tue, 14 Jan 2025 12:04:03 -0600 Subject: [PATCH 05/27] draft --- eslint.config.mjs | 8 +++ packages/nodejs/package.json | 2 +- packages/nodejs/src/vat-worker.ts | 25 +++++++ .../nodejs/src/vat/make-multiplexer.test.ts | 43 ++++++++++++ packages/nodejs/src/vat/make-multiplexer.ts | 28 ++++++++ .../nodejs/src/vat/make-vat-worker.test.ts | 67 +++++++++++++++++++ packages/nodejs/src/vat/make-vat-worker.ts | 47 +++++++++++++ packages/nodejs/src/vat/vat-worker.test.ts | 59 ---------------- packages/nodejs/src/vat/vat-worker.ts | 40 ----------- 9 files changed, 219 insertions(+), 100 deletions(-) create mode 100644 packages/nodejs/src/vat-worker.ts create mode 100644 packages/nodejs/src/vat/make-multiplexer.test.ts create mode 100644 packages/nodejs/src/vat/make-multiplexer.ts create mode 100644 packages/nodejs/src/vat/make-vat-worker.test.ts create mode 100644 packages/nodejs/src/vat/make-vat-worker.ts delete mode 100644 packages/nodejs/src/vat/vat-worker.test.ts delete mode 100644 packages/nodejs/src/vat/vat-worker.ts diff --git a/eslint.config.mjs b/eslint.config.mjs index 21a89f057..4e7bbc754 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -179,6 +179,14 @@ const config = createConfig([ globals: { lockdown: 'readonly' }, }, }, + + { + files: ['packages/nodejs/**/*-worker.ts'], + rules: { + // Node workers have reasonable cause to read from process.env + 'n/no-process-env': 'off', + }, + }, ]); export default config; diff --git a/packages/nodejs/package.json b/packages/nodejs/package.json index 443ee59aa..e72432af5 100644 --- a/packages/nodejs/package.json +++ b/packages/nodejs/package.json @@ -34,7 +34,7 @@ "publish:preview": "yarn npm publish --tag preview", "test": "vitest run --config vitest.config.ts", "test:e2e": "vitest run --config vitest.config.e2e.ts", - "test:e2e:ci": "echo 'skipped tests' || ./scripts/test-e2e-ci.sh", + "test:e2e:ci": "./scripts/test-e2e-ci.sh", "test:clean": "yarn test --no-cache --coverage.clean", "test:dev": "yarn test --coverage false", "test:verbose": "yarn test --reporter verbose", diff --git a/packages/nodejs/src/vat-worker.ts b/packages/nodejs/src/vat-worker.ts new file mode 100644 index 000000000..b38aaac6f --- /dev/null +++ b/packages/nodejs/src/vat-worker.ts @@ -0,0 +1,25 @@ +import '@ocap/shims/endoify'; + +import type { VatId } from '@ocap/kernel'; +import { makeLogger } from '@ocap/utils'; + +import { makeSQLKVStore } from './kernel/sqlite-kv-store.js'; +import { makeMultiplexer } from './vat/make-multiplexer.js'; +import { makeVatWorker } from './vat/make-vat-worker.js'; + +const vatId = process.env.NODE_VAT_ID as VatId; + +if (vatId) { + const logger = makeLogger(`[vat-worker (${vatId})]`); + logger.debug('starting worker...'); + const { start, stop } = makeVatWorker(vatId, makeMultiplexer, makeSQLKVStore); + try { + await start(); + } catch (problem) { + logger.error(problem); + } finally { + await stop(); + } +} else { + console.log('no vatId set for env variable NODE_VAT_ID'); +} diff --git a/packages/nodejs/src/vat/make-multiplexer.test.ts b/packages/nodejs/src/vat/make-multiplexer.test.ts new file mode 100644 index 000000000..640c030b7 --- /dev/null +++ b/packages/nodejs/src/vat/make-multiplexer.test.ts @@ -0,0 +1,43 @@ +import '@ocap/shims/endoify'; + +import { describe, expect, it, vi } from 'vitest'; + +describe('getPort', () => { + it('returns a port', async () => { + const mockParentPort = {}; + vi.doMock('node:worker_threads', () => ({ + parentPort: mockParentPort, + })); + vi.resetModules(); + + const { getPort } = await import('./make-multiplexer.js'); + + const port = getPort(); + + expect(port).toStrictEqual(mockParentPort); + }); + + it('throws if parentPort is not defined', async () => { + vi.doMock('node:worker_threads', () => ({ + parentPort: undefined, + })); + vi.resetModules(); + + const { getPort } = await import('./make-multiplexer.js'); + + expect(getPort).toThrow(/parentPort/u); + }); +}); + +describe('makeMultiplexer', () => { + it('returns a NodeWorkerMultiplexer', async () => { + vi.doMock('node:worker_threads', () => ({ + parentPort: new MessageChannel().port1, + })); + vi.resetModules(); + const { NodeWorkerMultiplexer } = await import('@ocap/streams'); + const { makeMultiplexer } = await import('./make-multiplexer.js'); + const multiplexer = makeMultiplexer('v0'); + expect(multiplexer).toBeInstanceOf(NodeWorkerMultiplexer); + }); +}); diff --git a/packages/nodejs/src/vat/make-multiplexer.ts b/packages/nodejs/src/vat/make-multiplexer.ts new file mode 100644 index 000000000..c75d3f007 --- /dev/null +++ b/packages/nodejs/src/vat/make-multiplexer.ts @@ -0,0 +1,28 @@ +import { NodeWorkerMultiplexer } from '@ocap/streams'; +import { parentPort } from 'node:worker_threads'; + +/** + * Return the parent port of the Node.js worker if it exists; otherwise throw. + * + * @returns The parent port. + * @throws If not called from within a Node.js worker. + */ +// eslint-disable-next-line @typescript-eslint/explicit-function-return-type +export function getPort() { + if (!parentPort) { + const errMsg = 'Expected to run in Node Worker with parentPort.'; + throw new Error(errMsg); + } + return parentPort; +} + +/** + * When called from within Node.js worker, returns a Multiplexer which + * communicates over the parentPort. + * + * @param name - The name to give this multiplexer (for traffic logging). + * @returns A NodeWorkerMultiplexer + */ +export function makeMultiplexer(name?: string): NodeWorkerMultiplexer { + return new NodeWorkerMultiplexer(getPort(), name); +} diff --git a/packages/nodejs/src/vat/make-vat-worker.test.ts b/packages/nodejs/src/vat/make-vat-worker.test.ts new file mode 100644 index 000000000..ad6063728 --- /dev/null +++ b/packages/nodejs/src/vat/make-vat-worker.test.ts @@ -0,0 +1,67 @@ +import '@ocap/shims/endoify'; + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { makeMultiplexer } from './make-multiplexer.js'; +import { makeVatWorker as makeVatWorkerDecl } from './make-vat-worker.js'; + +type MakeVatWorker = typeof makeVatWorkerDecl; + +describe('makeVatWorker', () => { + const testVatId = 'v0'; + let makeVatWorker: MakeVatWorker; + let mockMakeMultiplexer: typeof makeMultiplexer; + + beforeEach(async () => { + mockMakeMultiplexer = vi.fn().mockImplementation(() => ({ + start: vi.fn().mockResolvedValue(undefined), + return: vi.fn().mockResolvedValue(undefined), + createChannel: vi.fn(), + })); + vi.doMock('@ocap/streams', () => ({ + NodeWorkerMultiplexer: vi.fn(), + })); + vi.doMock('@ocap/kernel', () => ({ + VatSupervisor: vi.fn().mockImplementation(() => ({ + terminate: vi.fn().mockResolvedValue(undefined), + })), + isVatCommand: vi.fn(), + })); + vi.resetModules(); + makeVatWorker = (await import('./make-vat-worker.js')).makeVatWorker; + }); + + it('returns an object with start and stop methods', async () => { + const vatWorker = makeVatWorker(testVatId, mockMakeMultiplexer); + + expect(vatWorker).toHaveProperty('start'); + expect(vatWorker).toHaveProperty('stop'); + }); + + describe('start', () => { + it('starts the multiplexer', async () => { + const vatWorker = makeVatWorker(testVatId, mockMakeMultiplexer); + + await vatWorker.start(); + + expect(mockMakeMultiplexer.mock.results.at(0)).toBeDefined(); + expect( + mockMakeMultiplexer.mock.results.at(0).value.start, + ).toHaveBeenCalledOnce(); + }); + }); + + describe('stop', () => { + it('calls supervisor.terminate and multiplexer.return', async () => { + const vatWorker = makeVatWorker(testVatId, mockMakeMultiplexer); + + await vatWorker.start(); + await vatWorker.stop(); + + expect(mockMakeMultiplexer.mock.results.at(0)).toBeDefined(); + expect( + mockMakeMultiplexer.mock.results.at(0).value.return, + ).toHaveBeenCalledOnce(); + }); + }); +}); diff --git a/packages/nodejs/src/vat/make-vat-worker.ts b/packages/nodejs/src/vat/make-vat-worker.ts new file mode 100644 index 000000000..7260ceddc --- /dev/null +++ b/packages/nodejs/src/vat/make-vat-worker.ts @@ -0,0 +1,47 @@ +import { isVatCommand, VatSupervisor } from '@ocap/kernel'; +import type { + MakeKVStore, + VatCommand, + VatCommandReply, + VatId, +} from '@ocap/kernel'; +import type { StreamMultiplexer } from '@ocap/streams'; + +/** + * Assemble a vat worker for the target environment. + * + * @param vatId - The id of the vat inside the worker. + * @param makeMultiplexer - A routine to make a Multiplexer for the VatSupervisor. + * @param makeKVStore - A routine to make a KVStore for the VatSupervisor. + * @returns A vat worker object with awaitable start and stop methods. + */ +export function makeVatWorker( + vatId: VatId, + makeMultiplexer: (name?: string) => StreamMultiplexer, + makeKVStore: MakeKVStore, +): { + start: () => Promise; + stop: () => Promise; +} { + const multiplexer = makeMultiplexer(vatId); + const commandStream = multiplexer.createChannel( + 'command', + isVatCommand, + ); + + const supervisor = new VatSupervisor({ + id: `S${vatId}`, + commandStream, + makeKVStore, + }); + + return { + start: async () => { + await multiplexer.start(); + }, + stop: async () => { + await supervisor.terminate(); + await multiplexer.return(); + }, + }; +} diff --git a/packages/nodejs/src/vat/vat-worker.test.ts b/packages/nodejs/src/vat/vat-worker.test.ts deleted file mode 100644 index f96d15fa5..000000000 --- a/packages/nodejs/src/vat/vat-worker.test.ts +++ /dev/null @@ -1,59 +0,0 @@ -import '@ocap/shims/endoify'; - -import * as ocapKernel from '@ocap/kernel'; -import * as ocapStreams from '@ocap/streams'; -import * as ocapUtils from '@ocap/utils'; -import { describe, expect, it, vi } from 'vitest'; - -import { main } from './vat-worker.js'; - -vi.mock('node:worker_threads', () => ({ - parentPort: '{- parentPort -}', -})); - -vi.mock('@ocap/kernel', async (importOriginal: () => Promise) => { - const actual = await importOriginal(); - return { - ...actual, - VatSupervisor: vi.fn(() => ({ - evaluate: vi.fn(), - })), - } -}); - -vi.mock('@ocap/streams', () => ({ - NodeWorkerMultiplexer: vi.fn(() => ({ - start: vi.fn().mockRejectedValue(undefined), - createChannel: vi.fn(), - })), -})); - -describe('inside', () => { - it('creates a VatSupervisor and call its evaluate method', async () => { - const spyNodeWorkerMultiplexer = vi.spyOn(ocapStreams, 'NodeWorkerMultiplexer'); - const spyVatSupervisor = vi.spyOn(ocapKernel, 'VatSupervisor'); - - const vatId = 'v20'; - vi.stubEnv('NODE_VAT_ID', vatId); - - await main(); - - expect(spyNodeWorkerMultiplexer).toHaveBeenCalledOnce(); - expect(spyNodeWorkerMultiplexer.mock.instances.at(0)).toBeDefined(); - expect( - spyNodeWorkerMultiplexer.mock.instances.at(0)?.start, - ).toHaveBeenCalledOnce(); - expect( - spyNodeWorkerMultiplexer.mock.instances.at(0)?.createChannel, - ).toHaveBeenCalledTimes(2); - - expect(spyVatSupervisor).toHaveBeenCalledOnce(); - expect(spyVatSupervisor).toHaveBeenCalledWith({ - id: vatId, - }); - expect(spyVatSupervisor.mock.instances.at(0)).toBeDefined(); - expect( - spyVatSupervisor.mock.instances.at(0)?.evaluate, - ).toHaveBeenCalledWith('["Hello", "world!"].join(" ");'); - }); -}); diff --git a/packages/nodejs/src/vat/vat-worker.ts b/packages/nodejs/src/vat/vat-worker.ts deleted file mode 100644 index b43bc42bb..000000000 --- a/packages/nodejs/src/vat/vat-worker.ts +++ /dev/null @@ -1,40 +0,0 @@ -import '@ocap/shims/endoify'; - -import { isVatCommand, VatSupervisor } from '@ocap/kernel'; -import type { VatCommand, VatCommandReply } from '@ocap/kernel'; -import { NodeWorkerMultiplexer } from '@ocap/streams'; -import { makeLogger } from '@ocap/utils'; -import { parentPort } from 'node:worker_threads'; - -import { makeSQLKVStore } from '../kernel/sqlite-kv-store.js'; -const vatId = process.env.NODE_VAT_ID - -// eslint-disable-next-line n/no-process-env -const logger = makeLogger(`[vat-worker ${vatId}]`); - -main().catch(logger.error); - -/** - * The main function for the iframe. - */ -export async function main(): Promise { - logger.debug('started main'); - - if (!parentPort) { - const errMsg = 'Expected to run in Node Worker with parentPort.'; - logger.error(errMsg); - throw new Error(errMsg); - } - const multiplexer = new NodeWorkerMultiplexer(parentPort, 'vat'); - multiplexer.start().catch(logger.error); - const commandStream = multiplexer.createChannel( - 'command', - isVatCommand, - ); - - const supervisor = new VatSupervisor({ - id: 'iframe', - commandStream, - makeKVStore: makeSQLKVStore, - }); -} From 13e860ce53467df8603fedb98ba72d074bcb4730 Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Wed, 22 Jan 2025 02:50:54 -0600 Subject: [PATCH 06/27] XtrEme uNiT tESTing --- eslint.config.mjs | 5 +- package.json | 1 + packages/nodejs/package.json | 2 +- .../src/kernel/VatWorkerService.test.ts | 64 ++++++++++ .../nodejs/src/kernel/VatWorkerService.ts | 14 ++- packages/nodejs/src/kernel/kernel-worker.ts | 87 ------------- .../nodejs/src/kernel/make-kernel.test.ts | 24 ++++ packages/nodejs/src/kernel/make-kernel.ts | 34 +++++ packages/nodejs/src/vat-worker.ts | 25 ---- .../nodejs/src/vat/make-multiplexer.test.ts | 63 ++++++---- packages/nodejs/src/vat/make-multiplexer.ts | 6 +- .../nodejs/src/vat/make-vat-worker.test.ts | 67 +++++----- packages/nodejs/src/vat/make-vat-worker.ts | 22 +--- packages/nodejs/src/vat/vat-worker.ts | 19 +++ .../nodejs/test/e2e/kernel-worker.test.ts | 118 +++++++++++++++++- packages/nodejs/test/workers/hello-world.mjs | 3 + packages/nodejs/test/workers/index.ts | 2 + packages/nodejs/test/workers/ping-pong.mjs | 13 ++ vitest.config.ts | 10 +- 19 files changed, 368 insertions(+), 211 deletions(-) delete mode 100644 packages/nodejs/src/kernel/kernel-worker.ts create mode 100644 packages/nodejs/src/kernel/make-kernel.test.ts create mode 100644 packages/nodejs/src/kernel/make-kernel.ts delete mode 100644 packages/nodejs/src/vat-worker.ts create mode 100644 packages/nodejs/src/vat/vat-worker.ts create mode 100644 packages/nodejs/test/workers/hello-world.mjs create mode 100644 packages/nodejs/test/workers/index.ts create mode 100644 packages/nodejs/test/workers/ping-pong.mjs diff --git a/eslint.config.mjs b/eslint.config.mjs index 4e7bbc754..ebddd69cf 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -181,7 +181,10 @@ const config = createConfig([ }, { - files: ['packages/nodejs/**/*-worker.ts'], + files: [ + 'packages/nodejs/**/*-worker.ts', + 'packages/nodejs/test/workers/**/*.mjs', + ], rules: { // Node workers have reasonable cause to read from process.env 'n/no-process-env': 'off', diff --git a/package.json b/package.json index 0850a26b5..559e8baf3 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "test": "vitest run", "test:clean": "yarn test --no-cache --coverage.clean", "test:dev": "yarn test --coverage false", + "test:e2e": "yarn workspaces foreach --all run test:e2e", "test:e2e:ci": "yarn workspaces foreach --all run test:e2e:ci", "test:verbose": "yarn test --reporter verbose", "test:watch": "vitest", diff --git a/packages/nodejs/package.json b/packages/nodejs/package.json index e72432af5..443ee59aa 100644 --- a/packages/nodejs/package.json +++ b/packages/nodejs/package.json @@ -34,7 +34,7 @@ "publish:preview": "yarn npm publish --tag preview", "test": "vitest run --config vitest.config.ts", "test:e2e": "vitest run --config vitest.config.e2e.ts", - "test:e2e:ci": "./scripts/test-e2e-ci.sh", + "test:e2e:ci": "echo 'skipped tests' || ./scripts/test-e2e-ci.sh", "test:clean": "yarn test --no-cache --coverage.clean", "test:dev": "yarn test --coverage false", "test:verbose": "yarn test --reporter verbose", diff --git a/packages/nodejs/src/kernel/VatWorkerService.test.ts b/packages/nodejs/src/kernel/VatWorkerService.test.ts index d665cecfb..e26bf3c08 100644 --- a/packages/nodejs/src/kernel/VatWorkerService.test.ts +++ b/packages/nodejs/src/kernel/VatWorkerService.test.ts @@ -1,12 +1,76 @@ import '@ocap/shims/endoify'; +import type { VatId } from '@ocap/kernel'; +import { NodeWorkerMultiplexer } from '@ocap/streams'; +import { makeCounter } from '@ocap/utils'; import { describe, expect, it } from 'vitest'; import { NodejsVatWorkerService } from './VatWorkerService.js'; +import { getTestWorkerFile } from '../../test/workers'; describe('NodejsVatWorkerService', () => { it('constructs an instance without any arguments', () => { const instance = new NodejsVatWorkerService(); expect(instance).toBeInstanceOf(NodejsVatWorkerService); }); + + const vatIdCounter = makeCounter(); + const getTestVatId = (): VatId => `v${vatIdCounter()}`; + + describe('launch', () => { + it('creates a NodeWorker and returns a NodeWorkerMultiplexer', async () => { + const service = new NodejsVatWorkerService( + getTestWorkerFile('hello-world'), + ); + const testVatId: VatId = getTestVatId(); + const multiplexer = await service.launch(testVatId); + + expect(multiplexer).toBeInstanceOf(NodeWorkerMultiplexer); + }); + }); + + describe('terminate', () => { + it('terminates the target vat', async () => { + const service = new NodejsVatWorkerService( + getTestWorkerFile('hello-world'), + ); + const testVatId: VatId = getTestVatId(); + + await service.launch(testVatId); + expect(service.workers.has(testVatId)).toBe(true); + + await service.terminate(testVatId); + expect(service.workers.has(testVatId)).toBe(false); + }); + + it('throws when terminating an unknown vat', async () => { + const service = new NodejsVatWorkerService( + getTestWorkerFile('hello-world'), + ); + const testVatId: VatId = getTestVatId(); + + await expect( + async () => await service.terminate(testVatId), + ).rejects.toThrow(/No worker found/u); + }); + }); + + describe('terminateAll', () => { + it('terminates all vats', async () => { + const service = new NodejsVatWorkerService( + getTestWorkerFile('hello-world'), + ); + const vatIds: VatId[] = [getTestVatId(), getTestVatId(), getTestVatId()]; + + await Promise.all( + vatIds.map(async (vatId) => await service.launch(vatId)), + ); + + expect(Array.from(service.workers.values())).toHaveLength(vatIds.length); + + await service.terminateAll(); + + expect(Array.from(service.workers.values())).toHaveLength(0); + }); + }); }); diff --git a/packages/nodejs/src/kernel/VatWorkerService.ts b/packages/nodejs/src/kernel/VatWorkerService.ts index 4692ebb9b..42c8d860e 100644 --- a/packages/nodejs/src/kernel/VatWorkerService.ts +++ b/packages/nodejs/src/kernel/VatWorkerService.ts @@ -7,12 +7,16 @@ import { Worker as NodeWorker } from 'node:worker_threads'; // Worker file loads from the built dist directory, requires rebuild after change // Note: Worker runs in same process and may be subject to spectre-style attacks -const workerFileURL = new URL('../../dist/vat/vat-worker.mjs', import.meta.url) - .pathname; +const DEFAULT_WORKER_FILE = new URL( + '../../dist/vat/vat-worker.mjs', + import.meta.url, +).pathname; export class NodejsVatWorkerService implements VatWorkerService { readonly #logger: Logger; + readonly #workerFilePath: string; + workers = new Map< VatId, { worker: NodeWorker; multiplexer: StreamMultiplexer } @@ -22,16 +26,18 @@ export class NodejsVatWorkerService implements VatWorkerService { * The vat worker service, intended to be constructed in * the kernel worker. * + * @param workerFilePath - The path to a file defining the worker's routine. * @param logger - An optional {@link Logger}. Defaults to a new logger labeled '[vat worker client]'. */ - constructor(logger?: Logger) { + constructor(workerFilePath: string = DEFAULT_WORKER_FILE, logger?: Logger) { + this.#workerFilePath = workerFilePath; this.#logger = logger ?? makeLogger('[vat worker service]'); } async launch(vatId: VatId): Promise { const { promise, resolve } = makePromiseKit(); this.#logger.debug('launching', vatId); - const worker = new NodeWorker(workerFileURL, { + const worker = new NodeWorker(this.#workerFilePath, { env: { NODE_VAT_ID: vatId, }, diff --git a/packages/nodejs/src/kernel/kernel-worker.ts b/packages/nodejs/src/kernel/kernel-worker.ts deleted file mode 100644 index 302bcb80f..000000000 --- a/packages/nodejs/src/kernel/kernel-worker.ts +++ /dev/null @@ -1,87 +0,0 @@ -import '@ocap/shims/endoify'; -import type { NonEmptyArray } from '@metamask/utils'; -import type { KernelCommand, KernelCommandReply, VatId } from '@ocap/kernel'; -import { Kernel, VatCommandMethod } from '@ocap/kernel'; -import { NodeWorkerDuplexStream } from '@ocap/streams'; -import { MessagePort as NodeMessagePort } from 'worker_threads'; - -import { makeSQLKVStore } from './sqlite-kv-store.js'; -import { NodejsVatWorkerService } from './VatWorkerService.js'; - -/** - * The main function for the kernel worker. - * - * @param port - The kernel's end of a node:worker_threads MessageChannel - * @returns The kernel, initialized. - */ -export async function makeKernel(port: NodeMessagePort): Promise { - const nodeStream = new NodeWorkerDuplexStream< - KernelCommand, - KernelCommandReply - >(port); - const vatWorkerClient = new NodejsVatWorkerService(); - - // Initialize kernel store. - const kvStore = await makeSQLKVStore(); - - // Create and start kernel. - const kernel = new Kernel(nodeStream, vatWorkerClient, kvStore); - await kernel.init(); - - return kernel; -} - -/** - * Runs the full lifecycle of an array of vats, including their creation, - * restart, message passing, and termination. - * - * @param kernel The kernel instance. - * @param vats An array of VatIds to be managed. - */ -export async function runVatLifecycle( - kernel: Kernel, - vats: NonEmptyArray, -): Promise { - console.log('runVatLifecycle Start...'); - const vatLabel = vats.join(', '); - console.time(`Created vats: ${vatLabel}`); - await Promise.all( - vats.map( - async () => - await kernel.launchVat({ - bundleSpec: 'http://localhost:3000/sample-vat.bundle', - parameters: { name: 'Nodeen' }, - }), - ), - ); - console.timeEnd(`Created vats: ${vatLabel}`); - const knownVats = kernel.getVatIds() as NonEmptyArray; - const knownVatsLabel = knownVats.join(', '); - console.log('Kernel vats:', knownVatsLabel); - - // Restart a randomly selected vat from the array. - console.time(`Restart vats: ${knownVatsLabel}`); - await Promise.all( - knownVats.map(async (vatId: VatId) => await kernel.restartVat(vatId)), - ); - console.timeEnd(`Restart vats: ${knownVatsLabel}`); - - // Send a "Ping" message to a randomly selected vat. - console.time(`Ping vats: ${knownVatsLabel}`); - await Promise.all( - knownVats.map( - async (vatId: VatId) => - await kernel.sendMessage(vatId, { - method: VatCommandMethod.ping, - params: null, - }), - ), - ); - console.timeEnd(`Ping vats "${knownVatsLabel}"`); - - console.time(`Terminated vats: ${knownVatsLabel}`); - await kernel.terminateAllVats(); - console.timeEnd(`Terminated vats: ${knownVatsLabel}`); - - console.log(`Kernel has ${kernel.getVatIds().length} vats`); -} diff --git a/packages/nodejs/src/kernel/make-kernel.test.ts b/packages/nodejs/src/kernel/make-kernel.test.ts new file mode 100644 index 000000000..b76718de9 --- /dev/null +++ b/packages/nodejs/src/kernel/make-kernel.test.ts @@ -0,0 +1,24 @@ +import '@ocap/shims/endoify'; + +import { Kernel } from '@ocap/kernel'; +import { + MessagePort as NodeMessagePort, + MessageChannel as NodeMessageChannel, +} from 'node:worker_threads'; +import { beforeEach, describe, expect, it } from 'vitest'; + +import { makeKernel } from './make-kernel.js'; + +describe('makeKernel', () => { + let kernelPort: NodeMessagePort; + + beforeEach(() => { + kernelPort = new NodeMessageChannel().port1; + }); + + it('should return a Kernel', async () => { + const kernel = await makeKernel(kernelPort); + + expect(kernel).toBeInstanceOf(Kernel); + }); +}); diff --git a/packages/nodejs/src/kernel/make-kernel.ts b/packages/nodejs/src/kernel/make-kernel.ts new file mode 100644 index 000000000..710ddd607 --- /dev/null +++ b/packages/nodejs/src/kernel/make-kernel.ts @@ -0,0 +1,34 @@ +import type { KernelCommand, KernelCommandReply } from '@ocap/kernel'; +import { Kernel } from '@ocap/kernel'; +import { NodeWorkerDuplexStream } from '@ocap/streams'; +import { MessagePort as NodeMessagePort } from 'node:worker_threads'; + +import { makeSQLKVStore } from './sqlite-kv-store.js'; +import { NodejsVatWorkerService } from './VatWorkerService.js'; + +/** + * The main function for the kernel worker. + * + * @param port - The kernel's end of a node:worker_threads MessageChannel + * @param workerFilePath - The path to a file defining each vat worker's routine. + * @returns The kernel, initialized. + */ +export async function makeKernel( + port: NodeMessagePort, + workerFilePath?: string, +): Promise { + const nodeStream = new NodeWorkerDuplexStream< + KernelCommand, + KernelCommandReply + >(port); + const vatWorkerClient = new NodejsVatWorkerService(workerFilePath); + + // Initialize kernel store. + const kvStore = await makeSQLKVStore(); + + // Create and start kernel. + const kernel = new Kernel(nodeStream, vatWorkerClient, kvStore); + await kernel.init(); + + return kernel; +} diff --git a/packages/nodejs/src/vat-worker.ts b/packages/nodejs/src/vat-worker.ts deleted file mode 100644 index b38aaac6f..000000000 --- a/packages/nodejs/src/vat-worker.ts +++ /dev/null @@ -1,25 +0,0 @@ -import '@ocap/shims/endoify'; - -import type { VatId } from '@ocap/kernel'; -import { makeLogger } from '@ocap/utils'; - -import { makeSQLKVStore } from './kernel/sqlite-kv-store.js'; -import { makeMultiplexer } from './vat/make-multiplexer.js'; -import { makeVatWorker } from './vat/make-vat-worker.js'; - -const vatId = process.env.NODE_VAT_ID as VatId; - -if (vatId) { - const logger = makeLogger(`[vat-worker (${vatId})]`); - logger.debug('starting worker...'); - const { start, stop } = makeVatWorker(vatId, makeMultiplexer, makeSQLKVStore); - try { - await start(); - } catch (problem) { - logger.error(problem); - } finally { - await stop(); - } -} else { - console.log('no vatId set for env variable NODE_VAT_ID'); -} diff --git a/packages/nodejs/src/vat/make-multiplexer.test.ts b/packages/nodejs/src/vat/make-multiplexer.test.ts index 640c030b7..6b17f8c38 100644 --- a/packages/nodejs/src/vat/make-multiplexer.test.ts +++ b/packages/nodejs/src/vat/make-multiplexer.test.ts @@ -2,28 +2,44 @@ import '@ocap/shims/endoify'; import { describe, expect, it, vi } from 'vitest'; -describe('getPort', () => { - it('returns a port', async () => { - const mockParentPort = {}; - vi.doMock('node:worker_threads', () => ({ - parentPort: mockParentPort, - })); - vi.resetModules(); - - const { getPort } = await import('./make-multiplexer.js'); - - const port = getPort(); +import type { + getPort as getPortImpl, + makeMultiplexer as makeMultiplexerImpl, +} from './make-multiplexer.js'; + +type GetPort = typeof getPortImpl; +type MakeMultiplexer = typeof makeMultiplexerImpl; + +// eslint-disable-next-line @typescript-eslint/explicit-function-return-type +const doMockParentPort = (value: unknown) => { + vi.doMock('node:worker_threads', () => ({ + parentPort: value, + })); + vi.resetModules(); +}; - expect(port).toStrictEqual(mockParentPort); - }); +describe('getPort', () => { + it( + 'returns a port', + async () => { + const mockParentPort = {}; + doMockParentPort(mockParentPort); + + const { getPort } = await vi.importActual('./make-multiplexer.js'); + const port = (getPort as GetPort)(); + + expect(port).toStrictEqual(mockParentPort); + }, + { + // Extra time is needed when running yarn test from monorepo root. + timeout: 5000, + }, + ); it('throws if parentPort is not defined', async () => { - vi.doMock('node:worker_threads', () => ({ - parentPort: undefined, - })); - vi.resetModules(); + doMockParentPort(undefined); - const { getPort } = await import('./make-multiplexer.js'); + const { getPort } = await vi.importActual('./make-multiplexer.js'); expect(getPort).toThrow(/parentPort/u); }); @@ -31,13 +47,10 @@ describe('getPort', () => { describe('makeMultiplexer', () => { it('returns a NodeWorkerMultiplexer', async () => { - vi.doMock('node:worker_threads', () => ({ - parentPort: new MessageChannel().port1, - })); - vi.resetModules(); - const { NodeWorkerMultiplexer } = await import('@ocap/streams'); - const { makeMultiplexer } = await import('./make-multiplexer.js'); - const multiplexer = makeMultiplexer('v0'); + doMockParentPort(new MessageChannel().port1); + const { NodeWorkerMultiplexer } = await vi.importActual('@ocap/streams'); + const { makeMultiplexer } = await vi.importActual('./make-multiplexer.js'); + const multiplexer = (makeMultiplexer as MakeMultiplexer)(); expect(multiplexer).toBeInstanceOf(NodeWorkerMultiplexer); }); }); diff --git a/packages/nodejs/src/vat/make-multiplexer.ts b/packages/nodejs/src/vat/make-multiplexer.ts index c75d3f007..f4153316f 100644 --- a/packages/nodejs/src/vat/make-multiplexer.ts +++ b/packages/nodejs/src/vat/make-multiplexer.ts @@ -11,6 +11,7 @@ import { parentPort } from 'node:worker_threads'; export function getPort() { if (!parentPort) { const errMsg = 'Expected to run in Node Worker with parentPort.'; + console.error(errMsg); throw new Error(errMsg); } return parentPort; @@ -20,9 +21,8 @@ export function getPort() { * When called from within Node.js worker, returns a Multiplexer which * communicates over the parentPort. * - * @param name - The name to give this multiplexer (for traffic logging). * @returns A NodeWorkerMultiplexer */ -export function makeMultiplexer(name?: string): NodeWorkerMultiplexer { - return new NodeWorkerMultiplexer(getPort(), name); +export function makeMultiplexer(): NodeWorkerMultiplexer { + return new NodeWorkerMultiplexer(getPort(), 'vat'); } diff --git a/packages/nodejs/src/vat/make-vat-worker.test.ts b/packages/nodejs/src/vat/make-vat-worker.test.ts index ad6063728..a9b2d8009 100644 --- a/packages/nodejs/src/vat/make-vat-worker.test.ts +++ b/packages/nodejs/src/vat/make-vat-worker.test.ts @@ -1,16 +1,19 @@ import '@ocap/shims/endoify'; +import type { VatSupervisor } from '@ocap/kernel'; import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { Mock } from 'vitest'; import { makeMultiplexer } from './make-multiplexer.js'; -import { makeVatWorker as makeVatWorkerDecl } from './make-vat-worker.js'; +import { startVatWorker as startVatWorkerDecl } from './make-vat-worker.js'; -type MakeVatWorker = typeof makeVatWorkerDecl; +type MakeVatWorker = typeof startVatWorkerDecl; -describe('makeVatWorker', () => { +describe('startVatWorker', () => { const testVatId = 'v0'; - let makeVatWorker: MakeVatWorker; - let mockMakeMultiplexer: typeof makeMultiplexer; + let startVatWorker: MakeVatWorker; + let mockMakeMultiplexer: Mock; + let MockVatSupervisor: Mock<() => VatSupervisor>; beforeEach(async () => { mockMakeMultiplexer = vi.fn().mockImplementation(() => ({ @@ -18,50 +21,38 @@ describe('makeVatWorker', () => { return: vi.fn().mockResolvedValue(undefined), createChannel: vi.fn(), })); + MockVatSupervisor = vi.fn().mockImplementation(() => ({ + terminate: vi.fn().mockResolvedValue(undefined), + })); vi.doMock('@ocap/streams', () => ({ NodeWorkerMultiplexer: vi.fn(), })); vi.doMock('@ocap/kernel', () => ({ - VatSupervisor: vi.fn().mockImplementation(() => ({ - terminate: vi.fn().mockResolvedValue(undefined), - })), + VatSupervisor: MockVatSupervisor, isVatCommand: vi.fn(), })); vi.resetModules(); - makeVatWorker = (await import('./make-vat-worker.js')).makeVatWorker; + startVatWorker = (await import('./make-vat-worker.js')).startVatWorker; }); - it('returns an object with start and stop methods', async () => { - const vatWorker = makeVatWorker(testVatId, mockMakeMultiplexer); - - expect(vatWorker).toHaveProperty('start'); - expect(vatWorker).toHaveProperty('stop'); + it('creates a multiplexer and channel and calls start', async () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + await startVatWorker(testVatId, mockMakeMultiplexer, {} as any); + + expect(mockMakeMultiplexer).toHaveBeenCalledOnce(); + expect(mockMakeMultiplexer.mock.results.at(0)).toBeDefined(); + expect( + mockMakeMultiplexer.mock.results.at(0)?.value.createChannel, + ).toHaveBeenCalledOnce(); + expect( + mockMakeMultiplexer.mock.results.at(0)?.value.start, + ).toHaveBeenCalledOnce(); }); - describe('start', () => { - it('starts the multiplexer', async () => { - const vatWorker = makeVatWorker(testVatId, mockMakeMultiplexer); - - await vatWorker.start(); - - expect(mockMakeMultiplexer.mock.results.at(0)).toBeDefined(); - expect( - mockMakeMultiplexer.mock.results.at(0).value.start, - ).toHaveBeenCalledOnce(); - }); - }); - - describe('stop', () => { - it('calls supervisor.terminate and multiplexer.return', async () => { - const vatWorker = makeVatWorker(testVatId, mockMakeMultiplexer); - - await vatWorker.start(); - await vatWorker.stop(); + it('creates a VatSupervisor', async () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + await startVatWorker(testVatId, mockMakeMultiplexer, {} as any); - expect(mockMakeMultiplexer.mock.results.at(0)).toBeDefined(); - expect( - mockMakeMultiplexer.mock.results.at(0).value.return, - ).toHaveBeenCalledOnce(); - }); + expect(MockVatSupervisor.mock.instances).toHaveLength(1); }); }); diff --git a/packages/nodejs/src/vat/make-vat-worker.ts b/packages/nodejs/src/vat/make-vat-worker.ts index 7260ceddc..f7f500f91 100644 --- a/packages/nodejs/src/vat/make-vat-worker.ts +++ b/packages/nodejs/src/vat/make-vat-worker.ts @@ -15,33 +15,23 @@ import type { StreamMultiplexer } from '@ocap/streams'; * @param makeKVStore - A routine to make a KVStore for the VatSupervisor. * @returns A vat worker object with awaitable start and stop methods. */ -export function makeVatWorker( +export async function startVatWorker( vatId: VatId, makeMultiplexer: (name?: string) => StreamMultiplexer, makeKVStore: MakeKVStore, -): { - start: () => Promise; - stop: () => Promise; -} { +): Promise { const multiplexer = makeMultiplexer(vatId); + // We must start the multiplexer here, not later. + multiplexer.start().catch(console.error); const commandStream = multiplexer.createChannel( 'command', isVatCommand, ); - const supervisor = new VatSupervisor({ + // eslint-disable-next-line no-new + new VatSupervisor({ id: `S${vatId}`, commandStream, makeKVStore, }); - - return { - start: async () => { - await multiplexer.start(); - }, - stop: async () => { - await supervisor.terminate(); - await multiplexer.return(); - }, - }; } diff --git a/packages/nodejs/src/vat/vat-worker.ts b/packages/nodejs/src/vat/vat-worker.ts new file mode 100644 index 000000000..2767c3cce --- /dev/null +++ b/packages/nodejs/src/vat/vat-worker.ts @@ -0,0 +1,19 @@ +import '@ocap/shims/endoify'; + +import type { VatId } from '@ocap/kernel'; +import { makeLogger } from '@ocap/utils'; + +import { makeMultiplexer } from './make-multiplexer.js'; +import { startVatWorker } from './make-vat-worker.js'; +import { makeSQLKVStore } from '../kernel/sqlite-kv-store.js'; + +const vatId = process.env.NODE_VAT_ID as VatId; + +if (vatId) { + console.log('vatId', vatId); + const logger = makeLogger(`[vat-worker (${vatId})]`); + logger.debug('starting worker...'); + startVatWorker(vatId, makeMultiplexer, makeSQLKVStore).catch(logger.error); +} else { + console.log('no vatId set for env variable NODE_VAT_ID'); +} diff --git a/packages/nodejs/test/e2e/kernel-worker.test.ts b/packages/nodejs/test/e2e/kernel-worker.test.ts index 25dc13f4c..65a3f4971 100644 --- a/packages/nodejs/test/e2e/kernel-worker.test.ts +++ b/packages/nodejs/test/e2e/kernel-worker.test.ts @@ -1,9 +1,19 @@ import '@ocap/shims/endoify'; -import { MessageChannel as NodeMessageChannel } from 'node:worker_threads'; -import { describe, it, expect, vi } from 'vitest'; +import type { NonEmptyArray } from '@metamask/utils'; +import { Kernel, VatCommandMethod } from '@ocap/kernel'; +import type { VatConfig, VatId } from '@ocap/kernel'; +import { + MessageChannel as NodeMessageChannel, + MessagePort as NodePort, + Worker as NodeWorker, +} from 'node:worker_threads'; +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; -import { makeKernel, runVatLifecycle } from '../../src/kernel/kernel-worker.js'; +import { makeKernel } from '../../src/kernel/make-kernel.js'; + +const workerFileURL = new URL('../../dist/vat-worker.mjs', import.meta.url) + .pathname; vi.mock('node:process', () => ({ exit: vi.fn((reason) => { @@ -12,12 +22,53 @@ vi.mock('node:process', () => ({ })); describe('Kernel Worker', () => { + let kernelPort: NodePort; + let kernel: Kernel; + + const testVatConfig: VatConfig = { + bundleSpec: 'http://localhost:3000/sample-vat.bundle', + parameters: { name: 'Nodeen' }, + }; + + beforeEach(() => { + if (kernelPort) { + kernelPort.close(); + } + kernelPort = new NodeMessageChannel().port1; + }); + + afterEach(async () => { + if (kernel) { + await kernel.terminateAllVats(); + await kernel.clearStorage(); + } + }); + + it('starts a NodeWorker', async () => { + const worker = new NodeWorker(workerFileURL); + expect(worker).toBeInstanceOf(NodeWorker); + }); + + it('makes a Kernel', async () => { + kernel = await makeKernel(kernelPort); + expect(kernel).toBeInstanceOf(Kernel); + }); + + it('creates a vat', async () => { + kernel = await makeKernel(kernelPort); + let vatIds: VatId[] = kernel.getVatIds(); + expect(vatIds).toHaveLength(0); + + const kRef = await kernel.launchVat(testVatConfig); + expect(kRef).toBeInstanceOf(String); + vatIds = kernel.getVatIds(); + expect(vatIds).toHaveLength(1); + }); + it('should handle the lifecycle of multiple vats', async () => { console.log('Started test.'); - const kernelChannel = new NodeMessageChannel(); - const { port1: kernelPort } = kernelChannel; console.log('Creating kernel...'); - const kernel = await makeKernel(kernelPort); + kernel = await makeKernel(kernelPort); console.log('Kernel created.'); console.log('Handling the lifecycle of multiple vats...'); @@ -28,3 +79,58 @@ describe('Kernel Worker', () => { expect(true).toBe(true); }); }); + +/** + * Runs the full lifecycle of an array of vats, including their creation, + * restart, message passing, and termination. + * + * @param kernel The kernel instance. + * @param vats An array of VatIds to be managed. + * @param vatConfig The config to pass for vat initialization. + */ +export async function runVatLifecycle( + kernel: Kernel, + vats: NonEmptyArray, + vatConfig: VatConfig = { + bundleSpec: 'http://localhost:3000/sample-vat.bundle', + parameters: { name: 'Nodeen' }, + }, +): Promise { + console.log('runVatLifecycle Start...'); + const vatLabel = vats.join(', '); + console.time(`Created vats: ${vatLabel}`); + const kRef = await kernel.launchVat(vatConfig); + console.debug('kref', kRef); + + await Promise.all(vats.map(async () => await kernel.launchVat(vatConfig))); + console.timeEnd(`Created vats: ${vatLabel}`); + const knownVats = kernel.getVatIds() as NonEmptyArray; + const knownVatsLabel = knownVats.join(', '); + console.log('Kernel vats:', knownVatsLabel); + + // Restart a randomly selected vat from the array. + console.time(`Restart vats: ${knownVatsLabel}`); + await Promise.all( + knownVats.map(async (vatId: VatId) => await kernel.restartVat(vatId)), + ); + console.timeEnd(`Restart vats: ${knownVatsLabel}`); + + // Send a "Ping" message to a randomly selected vat. + console.time(`Ping vats: ${knownVatsLabel}`); + await Promise.all( + knownVats.map( + async (vatId: VatId) => + await kernel.sendMessage(vatId, { + method: VatCommandMethod.ping, + params: null, + }), + ), + ); + console.timeEnd(`Ping vats "${knownVatsLabel}"`); + + console.time(`Terminated vats: ${knownVatsLabel}`); + await kernel.terminateAllVats(); + console.timeEnd(`Terminated vats: ${knownVatsLabel}`); + + console.log(`Kernel has ${kernel.getVatIds().length} vats`); +} diff --git a/packages/nodejs/test/workers/hello-world.mjs b/packages/nodejs/test/workers/hello-world.mjs new file mode 100644 index 000000000..fa438c34a --- /dev/null +++ b/packages/nodejs/test/workers/hello-world.mjs @@ -0,0 +1,3 @@ +import '@ocap/shims/endoify'; + +console.debug('hello, world computer'); diff --git a/packages/nodejs/test/workers/index.ts b/packages/nodejs/test/workers/index.ts new file mode 100644 index 000000000..60c82437b --- /dev/null +++ b/packages/nodejs/test/workers/index.ts @@ -0,0 +1,2 @@ +export const getTestWorkerFile = (name: string): string => + new URL(`./${name}.mjs`, import.meta.url).pathname; diff --git a/packages/nodejs/test/workers/ping-pong.mjs b/packages/nodejs/test/workers/ping-pong.mjs new file mode 100644 index 000000000..9533bf1b2 --- /dev/null +++ b/packages/nodejs/test/workers/ping-pong.mjs @@ -0,0 +1,13 @@ +import '@ocap/shims/endoify'; + +import { makeMultiplexer } from '../../src/vat/make-multiplexer.mjs'; + +main().catch(console.error); + +/** + * The main function for the worker. TODO: support pinging and ponging. + */ +async function main() { + const multiplexer = makeMultiplexer('v0'); + await multiplexer.start(); +} diff --git a/vitest.config.ts b/vitest.config.ts index 2dfb2f545..ce4945637 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -73,10 +73,10 @@ export default defineConfig({ lines: 42.47, }, 'packages/nodejs/**': { - statements: 4.12, - functions: 4.76, - branches: 13.33, - lines: 4.12, + statements: 14.67, + functions: 24, + branches: 23.52, + lines: 14.67, }, 'packages/shims/**': { statements: 0, @@ -105,4 +105,4 @@ export default defineConfig({ }, }, }, -}); +}); \ No newline at end of file From 2ecbebd4628ed82260a7645b881480e30918400c Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Wed, 22 Jan 2025 03:37:03 -0600 Subject: [PATCH 07/27] mock make-kernel.test kvStore --- packages/nodejs/src/kernel/make-kernel.test.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/nodejs/src/kernel/make-kernel.test.ts b/packages/nodejs/src/kernel/make-kernel.test.ts index b76718de9..0a304077a 100644 --- a/packages/nodejs/src/kernel/make-kernel.test.ts +++ b/packages/nodejs/src/kernel/make-kernel.test.ts @@ -5,10 +5,17 @@ import { MessagePort as NodeMessagePort, MessageChannel as NodeMessageChannel, } from 'node:worker_threads'; -import { beforeEach, describe, expect, it } from 'vitest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; import { makeKernel } from './make-kernel.js'; +vi.mock('./sqlite-kv-store.js', async () => { + const { makeMapKVStore } = await import('../../../kernel/test/storage.js'); + return { + makeSQLKVStore: makeMapKVStore, + }; +}); + describe('makeKernel', () => { let kernelPort: NodeMessagePort; From ca850777cdf08f9df5918bcf4fe50ecd15da79a6 Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Wed, 22 Jan 2025 11:55:43 -0600 Subject: [PATCH 08/27] thresholds --- vitest.config.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/vitest.config.ts b/vitest.config.ts index d2d4737a9..a7863db62 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -74,16 +74,16 @@ export default defineConfig({ lines: 65.92, }, 'packages/kernel/**': { - statements: 42.66, - functions: 54.48, - branches: 29.72, - lines: 42.95, + statements: 44.12, + functions: 55.17, + branches: 30.5, + lines: 44.41, }, 'packages/nodejs/**': { - statements: 14.67, - functions: 24, - branches: 23.52, - lines: 14.67, + statements: 47.61, + functions: 47.36, + branches: 27.77, + lines: 47.61, }, 'packages/shims/**': { statements: 0, @@ -112,4 +112,4 @@ export default defineConfig({ }, }, }, -}); \ No newline at end of file +}); From eea968480192dc31ab7b27002f65cf98288674ff Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Wed, 22 Jan 2025 11:56:01 -0600 Subject: [PATCH 09/27] give getPort test more time --- packages/nodejs/src/vat/make-multiplexer.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/nodejs/src/vat/make-multiplexer.test.ts b/packages/nodejs/src/vat/make-multiplexer.test.ts index 6b17f8c38..13f794e06 100644 --- a/packages/nodejs/src/vat/make-multiplexer.test.ts +++ b/packages/nodejs/src/vat/make-multiplexer.test.ts @@ -32,7 +32,7 @@ describe('getPort', () => { }, { // Extra time is needed when running yarn test from monorepo root. - timeout: 5000, + timeout: 12000, }, ); From df25f8ddfb156345381905364a48cb67ab3b0a22 Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Wed, 22 Jan 2025 16:53:54 -0600 Subject: [PATCH 10/27] more refined --- packages/nodejs/package.json | 2 +- .../src/kernel/VatWorkerService.test.ts | 23 +++---- .../nodejs/src/kernel/VatWorkerService.ts | 1 + packages/nodejs/src/kernel/map-kv-store.ts | 41 ++++++++++++ .../nodejs/src/vat/make-multiplexer.test.ts | 56 ---------------- packages/nodejs/src/vat/make-multiplexer.ts | 28 -------- .../nodejs/src/vat/make-vat-worker.test.ts | 58 ---------------- packages/nodejs/src/vat/make-vat-worker.ts | 37 ----------- packages/nodejs/src/vat/streams.test.ts | 42 ++++++++++++ packages/nodejs/src/vat/streams.ts | 36 ++++++++++ packages/nodejs/src/vat/vat-worker.ts | 66 +++++++++---------- .../nodejs/test/e2e/kernel-worker.test.ts | 13 +++- packages/nodejs/test/workers/hello-world.mjs | 2 +- packages/nodejs/test/workers/ping-pong.mjs | 13 +++- packages/nodejs/test/workers/stream-sync.mjs | 15 +++++ 15 files changed, 197 insertions(+), 236 deletions(-) create mode 100644 packages/nodejs/src/kernel/map-kv-store.ts delete mode 100644 packages/nodejs/src/vat/make-multiplexer.test.ts delete mode 100644 packages/nodejs/src/vat/make-multiplexer.ts delete mode 100644 packages/nodejs/src/vat/make-vat-worker.test.ts delete mode 100644 packages/nodejs/src/vat/make-vat-worker.ts create mode 100644 packages/nodejs/src/vat/streams.test.ts create mode 100644 packages/nodejs/src/vat/streams.ts create mode 100644 packages/nodejs/test/workers/stream-sync.mjs diff --git a/packages/nodejs/package.json b/packages/nodejs/package.json index 98e18f26d..253b377e2 100644 --- a/packages/nodejs/package.json +++ b/packages/nodejs/package.json @@ -34,7 +34,7 @@ "publish:preview": "yarn npm publish --tag preview", "test": "vitest run --config vitest.config.ts", "test:e2e": "vitest run --config vitest.config.e2e.ts", - "test:e2e:ci": "echo 'skipped tests' || ./scripts/test-e2e-ci.sh", + "test:e2e:ci": "./scripts/test-e2e-ci.sh", "test:clean": "yarn test --no-cache --coverage.clean", "test:dev": "yarn test --coverage false", "test:verbose": "yarn test --reporter verbose", diff --git a/packages/nodejs/src/kernel/VatWorkerService.test.ts b/packages/nodejs/src/kernel/VatWorkerService.test.ts index e26bf3c08..d48f53d28 100644 --- a/packages/nodejs/src/kernel/VatWorkerService.test.ts +++ b/packages/nodejs/src/kernel/VatWorkerService.test.ts @@ -1,7 +1,7 @@ import '@ocap/shims/endoify'; import type { VatId } from '@ocap/kernel'; -import { NodeWorkerMultiplexer } from '@ocap/streams'; +import { NodeWorkerDuplexStream } from '@ocap/streams'; import { makeCounter } from '@ocap/utils'; import { describe, expect, it } from 'vitest'; @@ -14,26 +14,23 @@ describe('NodejsVatWorkerService', () => { expect(instance).toBeInstanceOf(NodejsVatWorkerService); }); + const helloWorld = getTestWorkerFile('hello-world'); const vatIdCounter = makeCounter(); const getTestVatId = (): VatId => `v${vatIdCounter()}`; describe('launch', () => { - it('creates a NodeWorker and returns a NodeWorkerMultiplexer', async () => { - const service = new NodejsVatWorkerService( - getTestWorkerFile('hello-world'), - ); + it('creates a NodeWorker and returns a NodeWorkerDuplexStream', async () => { + const service = new NodejsVatWorkerService(helloWorld); const testVatId: VatId = getTestVatId(); const multiplexer = await service.launch(testVatId); - expect(multiplexer).toBeInstanceOf(NodeWorkerMultiplexer); + expect(multiplexer).toBeInstanceOf(NodeWorkerDuplexStream); }); }); describe('terminate', () => { it('terminates the target vat', async () => { - const service = new NodejsVatWorkerService( - getTestWorkerFile('hello-world'), - ); + const service = new NodejsVatWorkerService(helloWorld); const testVatId: VatId = getTestVatId(); await service.launch(testVatId); @@ -44,9 +41,7 @@ describe('NodejsVatWorkerService', () => { }); it('throws when terminating an unknown vat', async () => { - const service = new NodejsVatWorkerService( - getTestWorkerFile('hello-world'), - ); + const service = new NodejsVatWorkerService(helloWorld); const testVatId: VatId = getTestVatId(); await expect( @@ -57,9 +52,7 @@ describe('NodejsVatWorkerService', () => { describe('terminateAll', () => { it('terminates all vats', async () => { - const service = new NodejsVatWorkerService( - getTestWorkerFile('hello-world'), - ); + const service = new NodejsVatWorkerService(helloWorld); const vatIds: VatId[] = [getTestVatId(), getTestVatId(), getTestVatId()]; await Promise.all( diff --git a/packages/nodejs/src/kernel/VatWorkerService.ts b/packages/nodejs/src/kernel/VatWorkerService.ts index 80face1f8..a20a14f2b 100644 --- a/packages/nodejs/src/kernel/VatWorkerService.ts +++ b/packages/nodejs/src/kernel/VatWorkerService.ts @@ -51,6 +51,7 @@ export class NodejsVatWorkerService implements VatWorkerService { env: { NODE_VAT_ID: vatId, }, + // execArgv: ["--require", "ts-node/register"], }); this.#logger.debug('launched', vatId); worker.once('online', () => { diff --git a/packages/nodejs/src/kernel/map-kv-store.ts b/packages/nodejs/src/kernel/map-kv-store.ts new file mode 100644 index 000000000..042c5fee8 --- /dev/null +++ b/packages/nodejs/src/kernel/map-kv-store.ts @@ -0,0 +1,41 @@ +/** + * This file is a copy of `@ocap/kernel/test/storage.ts` + * Because the test dir isn't shipped, it is easier just to copy than to + * do battle with the existing build configuration. + */ +import type { KVStore } from '@ocap/kernel'; + +/** + * A mock key/value store realized as a Map. + * + * @returns The mock {@link KVStore}. + */ +export function makeMapKVStore(): KVStore { + const map = new Map(); + + /** + * Like `get`, but fail if the key isn't there. + * + * @param key - The key to fetch. + * @returns The value at `key`. + */ + function getRequired(key: string): string { + const result = map.get(key); + if (result === undefined) { + throw Error(`No value found for key ${key}.`); + } + return result; + } + + return { + get: map.get.bind(map), + getNextKey: (_key: string): string | undefined => { + throw Error(`mock store does not (yet) support getNextKey`); + }, + getRequired, + set: map.set.bind(map), + delete: map.delete.bind(map), + clear: map.clear.bind(map), + executeQuery: () => [], + }; +} diff --git a/packages/nodejs/src/vat/make-multiplexer.test.ts b/packages/nodejs/src/vat/make-multiplexer.test.ts deleted file mode 100644 index 13f794e06..000000000 --- a/packages/nodejs/src/vat/make-multiplexer.test.ts +++ /dev/null @@ -1,56 +0,0 @@ -import '@ocap/shims/endoify'; - -import { describe, expect, it, vi } from 'vitest'; - -import type { - getPort as getPortImpl, - makeMultiplexer as makeMultiplexerImpl, -} from './make-multiplexer.js'; - -type GetPort = typeof getPortImpl; -type MakeMultiplexer = typeof makeMultiplexerImpl; - -// eslint-disable-next-line @typescript-eslint/explicit-function-return-type -const doMockParentPort = (value: unknown) => { - vi.doMock('node:worker_threads', () => ({ - parentPort: value, - })); - vi.resetModules(); -}; - -describe('getPort', () => { - it( - 'returns a port', - async () => { - const mockParentPort = {}; - doMockParentPort(mockParentPort); - - const { getPort } = await vi.importActual('./make-multiplexer.js'); - const port = (getPort as GetPort)(); - - expect(port).toStrictEqual(mockParentPort); - }, - { - // Extra time is needed when running yarn test from monorepo root. - timeout: 12000, - }, - ); - - it('throws if parentPort is not defined', async () => { - doMockParentPort(undefined); - - const { getPort } = await vi.importActual('./make-multiplexer.js'); - - expect(getPort).toThrow(/parentPort/u); - }); -}); - -describe('makeMultiplexer', () => { - it('returns a NodeWorkerMultiplexer', async () => { - doMockParentPort(new MessageChannel().port1); - const { NodeWorkerMultiplexer } = await vi.importActual('@ocap/streams'); - const { makeMultiplexer } = await vi.importActual('./make-multiplexer.js'); - const multiplexer = (makeMultiplexer as MakeMultiplexer)(); - expect(multiplexer).toBeInstanceOf(NodeWorkerMultiplexer); - }); -}); diff --git a/packages/nodejs/src/vat/make-multiplexer.ts b/packages/nodejs/src/vat/make-multiplexer.ts deleted file mode 100644 index f4153316f..000000000 --- a/packages/nodejs/src/vat/make-multiplexer.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { NodeWorkerMultiplexer } from '@ocap/streams'; -import { parentPort } from 'node:worker_threads'; - -/** - * Return the parent port of the Node.js worker if it exists; otherwise throw. - * - * @returns The parent port. - * @throws If not called from within a Node.js worker. - */ -// eslint-disable-next-line @typescript-eslint/explicit-function-return-type -export function getPort() { - if (!parentPort) { - const errMsg = 'Expected to run in Node Worker with parentPort.'; - console.error(errMsg); - throw new Error(errMsg); - } - return parentPort; -} - -/** - * When called from within Node.js worker, returns a Multiplexer which - * communicates over the parentPort. - * - * @returns A NodeWorkerMultiplexer - */ -export function makeMultiplexer(): NodeWorkerMultiplexer { - return new NodeWorkerMultiplexer(getPort(), 'vat'); -} diff --git a/packages/nodejs/src/vat/make-vat-worker.test.ts b/packages/nodejs/src/vat/make-vat-worker.test.ts deleted file mode 100644 index a9b2d8009..000000000 --- a/packages/nodejs/src/vat/make-vat-worker.test.ts +++ /dev/null @@ -1,58 +0,0 @@ -import '@ocap/shims/endoify'; - -import type { VatSupervisor } from '@ocap/kernel'; -import { beforeEach, describe, expect, it, vi } from 'vitest'; -import type { Mock } from 'vitest'; - -import { makeMultiplexer } from './make-multiplexer.js'; -import { startVatWorker as startVatWorkerDecl } from './make-vat-worker.js'; - -type MakeVatWorker = typeof startVatWorkerDecl; - -describe('startVatWorker', () => { - const testVatId = 'v0'; - let startVatWorker: MakeVatWorker; - let mockMakeMultiplexer: Mock; - let MockVatSupervisor: Mock<() => VatSupervisor>; - - beforeEach(async () => { - mockMakeMultiplexer = vi.fn().mockImplementation(() => ({ - start: vi.fn().mockResolvedValue(undefined), - return: vi.fn().mockResolvedValue(undefined), - createChannel: vi.fn(), - })); - MockVatSupervisor = vi.fn().mockImplementation(() => ({ - terminate: vi.fn().mockResolvedValue(undefined), - })); - vi.doMock('@ocap/streams', () => ({ - NodeWorkerMultiplexer: vi.fn(), - })); - vi.doMock('@ocap/kernel', () => ({ - VatSupervisor: MockVatSupervisor, - isVatCommand: vi.fn(), - })); - vi.resetModules(); - startVatWorker = (await import('./make-vat-worker.js')).startVatWorker; - }); - - it('creates a multiplexer and channel and calls start', async () => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - await startVatWorker(testVatId, mockMakeMultiplexer, {} as any); - - expect(mockMakeMultiplexer).toHaveBeenCalledOnce(); - expect(mockMakeMultiplexer.mock.results.at(0)).toBeDefined(); - expect( - mockMakeMultiplexer.mock.results.at(0)?.value.createChannel, - ).toHaveBeenCalledOnce(); - expect( - mockMakeMultiplexer.mock.results.at(0)?.value.start, - ).toHaveBeenCalledOnce(); - }); - - it('creates a VatSupervisor', async () => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - await startVatWorker(testVatId, mockMakeMultiplexer, {} as any); - - expect(MockVatSupervisor.mock.instances).toHaveLength(1); - }); -}); diff --git a/packages/nodejs/src/vat/make-vat-worker.ts b/packages/nodejs/src/vat/make-vat-worker.ts deleted file mode 100644 index f7f500f91..000000000 --- a/packages/nodejs/src/vat/make-vat-worker.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { isVatCommand, VatSupervisor } from '@ocap/kernel'; -import type { - MakeKVStore, - VatCommand, - VatCommandReply, - VatId, -} from '@ocap/kernel'; -import type { StreamMultiplexer } from '@ocap/streams'; - -/** - * Assemble a vat worker for the target environment. - * - * @param vatId - The id of the vat inside the worker. - * @param makeMultiplexer - A routine to make a Multiplexer for the VatSupervisor. - * @param makeKVStore - A routine to make a KVStore for the VatSupervisor. - * @returns A vat worker object with awaitable start and stop methods. - */ -export async function startVatWorker( - vatId: VatId, - makeMultiplexer: (name?: string) => StreamMultiplexer, - makeKVStore: MakeKVStore, -): Promise { - const multiplexer = makeMultiplexer(vatId); - // We must start the multiplexer here, not later. - multiplexer.start().catch(console.error); - const commandStream = multiplexer.createChannel( - 'command', - isVatCommand, - ); - - // eslint-disable-next-line no-new - new VatSupervisor({ - id: `S${vatId}`, - commandStream, - makeKVStore, - }); -} diff --git a/packages/nodejs/src/vat/streams.test.ts b/packages/nodejs/src/vat/streams.test.ts new file mode 100644 index 000000000..26fc7f8f2 --- /dev/null +++ b/packages/nodejs/src/vat/streams.test.ts @@ -0,0 +1,42 @@ +import '../../../test-utils/src/env/mock-endoify.js'; + +import { describe, expect, it, vi } from 'vitest'; + +// eslint-disable-next-line @typescript-eslint/explicit-function-return-type +const doMockParentPort = (value: unknown) => { + vi.doMock('node:worker_threads', () => ({ + parentPort: value, + })); + vi.resetModules(); +}; + +describe('getPort', () => { + it('returns a port', async () => { + const mockParentPort = {}; + doMockParentPort(mockParentPort); + + const { getPort } = await import('./streams.js'); + const port = getPort(); + + expect(port).toStrictEqual(mockParentPort); + }, 4000); // Extra time is needed when running yarn test from monorepo root. + + it('throws if parentPort is not defined', async () => { + doMockParentPort(undefined); + + const { getPort } = await import('./streams.js'); + + expect(getPort).toThrow(/parentPort/u); + }); +}); + +describe('makeCommandStream', () => { + it('returns a NodeWorkerDuplexStream', async () => { + doMockParentPort(new MessageChannel().port1); + + const { NodeWorkerDuplexStream } = await import('@ocap/streams'); + const { makeCommandStream } = await import('./streams.js'); + const commandStream = makeCommandStream(); + expect(commandStream).toBeInstanceOf(NodeWorkerDuplexStream); + }); +}); diff --git a/packages/nodejs/src/vat/streams.ts b/packages/nodejs/src/vat/streams.ts new file mode 100644 index 000000000..4c19e7eb6 --- /dev/null +++ b/packages/nodejs/src/vat/streams.ts @@ -0,0 +1,36 @@ +import '@ocap/shims/endoify'; + +import { isVatCommand } from '@ocap/kernel'; +import type { VatCommand, VatCommandReply } from '@ocap/kernel'; +import { NodeWorkerDuplexStream } from '@ocap/streams'; +import { parentPort } from 'node:worker_threads'; + +/** + * Return the parent port of the Node.js worker if it exists; otherwise throw. + * + * @returns The parent port. + * @throws If not called from within a Node.js worker. + */ +// eslint-disable-next-line @typescript-eslint/explicit-function-return-type +export function getPort() { + if (!parentPort) { + throw new Error('Expected to run in a Node.js worker with parentPort.'); + } + return parentPort; +} + +/** + * When called from within Node.js worker, returns a DuplexStream which + * communicates over the parentPort. + * + * @returns A NodeWorkerDuplexStream + */ +export function makeCommandStream(): NodeWorkerDuplexStream< + VatCommand, + VatCommandReply +> { + return new NodeWorkerDuplexStream( + getPort(), + isVatCommand, + ); +} diff --git a/packages/nodejs/src/vat/vat-worker.ts b/packages/nodejs/src/vat/vat-worker.ts index 2ac11492f..142a9f60f 100644 --- a/packages/nodejs/src/vat/vat-worker.ts +++ b/packages/nodejs/src/vat/vat-worker.ts @@ -1,44 +1,42 @@ import '@ocap/shims/endoify'; import type { VatId } from '@ocap/kernel'; -import { isVatCommand, VatSupervisor } from '@ocap/kernel'; -import type { VatCommand, VatCommandReply } from '@ocap/kernel'; -import { NodeWorkerDuplexStream } from '@ocap/streams'; -import { makeLogger } from '@ocap/utils'; +const streams = '../../dist/vat/streams.mjs'; +const store = '../../dist/kernel/sqlite-kv-store.mjs'; -import { makeMultiplexer } from './make-multiplexer.js'; -import { startVatWorker } from './make-vat-worker.js'; -import { makeSQLKVStore } from '../kernel/sqlite-kv-store.js'; +try { + console.debug(`[vat-worker (${process.env.NODE_VAT_ID})] begin imports`); -const vatId = process.env.NODE_VAT_ID as VatId; + const { VatSupervisor } = await import('@ocap/kernel'); + const { makeLogger } = await import('@ocap/utils'); + const { makeCommandStream } = await import(streams); + const { makeSQLKVStore } = await import(store); -if (vatId) { - console.log('vatId', vatId); - const logger = makeLogger(`[vat-worker (${vatId})]`); - logger.debug('starting worker...'); - main().catch(logger.error) -} else { - console.log('no vatId set for env variable NODE_VAT_ID'); -} + console.debug(`[vat-worker (${process.env.NODE_VAT_ID})] imports complete`); + + const vatId = process.env.NODE_VAT_ID as VatId; + + if (vatId) { + console.log('vatId', vatId); + const logger = makeLogger(`[vat-worker (${vatId})]`); + logger.debug('starting worker...'); + main().catch(logger.error); + } else { + console.log('no vatId set for env variable NODE_VAT_ID'); + } -/** - * The main function for the iframe. - */ -async function main(): Promise { - if (!parentPort) { - const errMsg = 'Expected to run in Node Worker with parentPort.'; - logger.error(errMsg); - throw new Error(errMsg); + /** + * The main function for the iframe. + */ + async function main(): Promise { + // eslint-disable-next-line no-void + void new VatSupervisor({ + id: 'iframe', + commandStream: makeCommandStream(), + makeKVStore: makeSQLKVStore, + }); } - const commandStream = new NodeWorkerDuplexStream( - parentPort, - isVatCommand, - ); - // eslint-disable-next-line no-void - void new VatSupervisor({ - id: 'iframe', - commandStream, - makeKVStore: makeSQLKVStore, - }); +} catch (problem) { + console.error(problem); } diff --git a/packages/nodejs/test/e2e/kernel-worker.test.ts b/packages/nodejs/test/e2e/kernel-worker.test.ts index 65a3f4971..7b15bcd80 100644 --- a/packages/nodejs/test/e2e/kernel-worker.test.ts +++ b/packages/nodejs/test/e2e/kernel-worker.test.ts @@ -1,4 +1,5 @@ import '@ocap/shims/endoify'; +// import '../../../test-utils/src/env/mock-endoify.js'; import type { NonEmptyArray } from '@metamask/utils'; import { Kernel, VatCommandMethod } from '@ocap/kernel'; @@ -11,6 +12,7 @@ import { import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import { makeKernel } from '../../src/kernel/make-kernel.js'; +import { getTestWorkerFile } from 'test/workers/index.js'; const workerFileURL = new URL('../../dist/vat-worker.mjs', import.meta.url) .pathname; @@ -44,8 +46,13 @@ describe('Kernel Worker', () => { } }); + it('trivial', () => { + console.debug('TRIVIAL'); + expect(true).toBe(true); + }); + it('starts a NodeWorker', async () => { - const worker = new NodeWorker(workerFileURL); + const worker = new NodeWorker(getTestWorkerFile('hello-world')); expect(worker).toBeInstanceOf(NodeWorker); }); @@ -63,9 +70,9 @@ describe('Kernel Worker', () => { expect(kRef).toBeInstanceOf(String); vatIds = kernel.getVatIds(); expect(vatIds).toHaveLength(1); - }); + }, 10000); - it('should handle the lifecycle of multiple vats', async () => { + it.skip('should handle the lifecycle of multiple vats', async () => { console.log('Started test.'); console.log('Creating kernel...'); kernel = await makeKernel(kernelPort); diff --git a/packages/nodejs/test/workers/hello-world.mjs b/packages/nodejs/test/workers/hello-world.mjs index fa438c34a..93c4f5091 100644 --- a/packages/nodejs/test/workers/hello-world.mjs +++ b/packages/nodejs/test/workers/hello-world.mjs @@ -1,3 +1,3 @@ import '@ocap/shims/endoify'; -console.debug('hello, world computer'); +console.debug('hello, world computer', process.env.NODE_VAT_ID); diff --git a/packages/nodejs/test/workers/ping-pong.mjs b/packages/nodejs/test/workers/ping-pong.mjs index 9533bf1b2..f5182149a 100644 --- a/packages/nodejs/test/workers/ping-pong.mjs +++ b/packages/nodejs/test/workers/ping-pong.mjs @@ -1,6 +1,10 @@ import '@ocap/shims/endoify'; -import { makeMultiplexer } from '../../src/vat/make-multiplexer.mjs'; +import { VatSupervisor } from '../../../kernel/dist/VatSupervisor.mjs'; +import { makeMapKVStore } from '../../dist/kernel/map-kv-store.mjs'; +import { makeCommandStream } from '../../dist/vat/streams.mjs'; + +console.debug('ping pong'); main().catch(console.error); @@ -8,6 +12,9 @@ main().catch(console.error); * The main function for the worker. TODO: support pinging and ponging. */ async function main() { - const multiplexer = makeMultiplexer('v0'); - await multiplexer.start(); + void new VatSupervisor({ + id: 'iframe', + commandStream: makeCommandStream(), + makeKVStore: makeMapKVStore, + }); } diff --git a/packages/nodejs/test/workers/stream-sync.mjs b/packages/nodejs/test/workers/stream-sync.mjs new file mode 100644 index 000000000..51c635697 --- /dev/null +++ b/packages/nodejs/test/workers/stream-sync.mjs @@ -0,0 +1,15 @@ +import '@ocap/shims/endoify'; + +import { makeCommandStream } from '../../dist/vat/streams.mjs'; + +main().catch(console.error); + +/** + * The main function for the worker. + */ +async function main() { + console.debug('top', process.env.NODE_VAT_ID); + const stream = makeCommandStream(); + await stream.synchronize(); + console.debug('bot', process.env.NODE_VAT_ID); +} From a18237c804cda8cfb0f22dedab435c4c43ad7661 Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Thu, 23 Jan 2025 12:24:21 -0600 Subject: [PATCH 11/27] fix endo mocking --- packages/nodejs/src/vat/streams.test.ts | 10 +++++++++- packages/nodejs/src/vat/streams.ts | 10 ++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/packages/nodejs/src/vat/streams.test.ts b/packages/nodejs/src/vat/streams.test.ts index 26fc7f8f2..729f606b1 100644 --- a/packages/nodejs/src/vat/streams.test.ts +++ b/packages/nodejs/src/vat/streams.test.ts @@ -1,4 +1,4 @@ -import '../../../test-utils/src/env/mock-endoify.js'; +import '@ocap/test-utils/mock-endoify'; import { describe, expect, it, vi } from 'vitest'; @@ -10,6 +10,14 @@ const doMockParentPort = (value: unknown) => { vi.resetModules(); }; +vi.mock('@ocap/kernel', async () => { + return { isVatCommand: vi.fn(() => true) }; +}); + +vi.mock('@ocap/streams', () => ({ + NodeWorkerDuplexStream: vi.fn(), +})) + describe('getPort', () => { it('returns a port', async () => { const mockParentPort = {}; diff --git a/packages/nodejs/src/vat/streams.ts b/packages/nodejs/src/vat/streams.ts index 4c19e7eb6..5f453fbde 100644 --- a/packages/nodejs/src/vat/streams.ts +++ b/packages/nodejs/src/vat/streams.ts @@ -1,9 +1,9 @@ -import '@ocap/shims/endoify'; +import '@ocap/test-utils/mock-endoify'; import { isVatCommand } from '@ocap/kernel'; import type { VatCommand, VatCommandReply } from '@ocap/kernel'; import { NodeWorkerDuplexStream } from '@ocap/streams'; -import { parentPort } from 'node:worker_threads'; +import { type MessagePort as NodePort, parentPort } from 'node:worker_threads'; /** * Return the parent port of the Node.js worker if it exists; otherwise throw. @@ -12,7 +12,7 @@ import { parentPort } from 'node:worker_threads'; * @throws If not called from within a Node.js worker. */ // eslint-disable-next-line @typescript-eslint/explicit-function-return-type -export function getPort() { +export function getPort(): NodePort { if (!parentPort) { throw new Error('Expected to run in a Node.js worker with parentPort.'); } @@ -25,7 +25,9 @@ export function getPort() { * * @returns A NodeWorkerDuplexStream */ -export function makeCommandStream(): NodeWorkerDuplexStream< +export function makeCommandStream( + +): NodeWorkerDuplexStream< VatCommand, VatCommandReply > { From 5ce6c792d376d521ce6a4166892eee99f430e3a3 Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Thu, 23 Jan 2025 12:38:17 -0600 Subject: [PATCH 12/27] remove endoify from Kernel.ts --- packages/kernel/src/Kernel.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/kernel/src/Kernel.ts b/packages/kernel/src/Kernel.ts index 1d902e0ac..0ae10ad24 100644 --- a/packages/kernel/src/Kernel.ts +++ b/packages/kernel/src/Kernel.ts @@ -1,4 +1,3 @@ -import '@ocap/shims/endoify'; import { passStyleOf } from '@endo/far'; import type { CapData } from '@endo/marshal'; import { makePromiseKit } from '@endo/promise-kit'; From 4f7fdcef2afab3028f78df5dfe119d88529e6ae4 Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Fri, 24 Jan 2025 12:27:43 -0600 Subject: [PATCH 13/27] further e2e testing --- .../src/kernel/VatWorkerService.test.ts | 2 +- .../nodejs/src/kernel/VatWorkerService.ts | 1 + packages/nodejs/src/kernel/sqlite-kv-store.ts | 20 ++++-- packages/nodejs/src/vat/streams.ts | 2 - packages/nodejs/src/vat/vat-worker.ts | 66 +++++++++---------- .../nodejs/test/e2e/kernel-worker.test.ts | 9 ++- vitest.config.ts | 10 +-- 7 files changed, 57 insertions(+), 53 deletions(-) diff --git a/packages/nodejs/src/kernel/VatWorkerService.test.ts b/packages/nodejs/src/kernel/VatWorkerService.test.ts index d48f53d28..a665deda4 100644 --- a/packages/nodejs/src/kernel/VatWorkerService.test.ts +++ b/packages/nodejs/src/kernel/VatWorkerService.test.ts @@ -14,7 +14,7 @@ describe('NodejsVatWorkerService', () => { expect(instance).toBeInstanceOf(NodejsVatWorkerService); }); - const helloWorld = getTestWorkerFile('hello-world'); + const helloWorld = getTestWorkerFile('stream-sync'); const vatIdCounter = makeCounter(); const getTestVatId = (): VatId => `v${vatIdCounter()}`; diff --git a/packages/nodejs/src/kernel/VatWorkerService.ts b/packages/nodejs/src/kernel/VatWorkerService.ts index a20a14f2b..d9d761e24 100644 --- a/packages/nodejs/src/kernel/VatWorkerService.ts +++ b/packages/nodejs/src/kernel/VatWorkerService.ts @@ -60,6 +60,7 @@ export class NodejsVatWorkerService implements VatWorkerService { isVatCommandReply, ); this.workers.set(vatId, { worker, stream }); + console.debug('synchronizing...'); stream .synchronize() .then(() => { diff --git a/packages/nodejs/src/kernel/sqlite-kv-store.ts b/packages/nodejs/src/kernel/sqlite-kv-store.ts index d8f2db7b0..66e86720e 100644 --- a/packages/nodejs/src/kernel/sqlite-kv-store.ts +++ b/packages/nodejs/src/kernel/sqlite-kv-store.ts @@ -2,11 +2,14 @@ import { hasProperty, isObject } from '@metamask/utils'; import type { KVStore } from '@ocap/kernel'; import { makeLogger } from '@ocap/utils'; // eslint-disable-next-line @typescript-eslint/naming-convention -import Sqlite from 'better-sqlite3'; +// import Sqlite from 'better-sqlite3'; import { mkdir } from 'fs/promises'; import { tmpdir } from 'os'; import { join } from 'path'; +import type { Database } from 'better-sqlite3'; +const Sqlite = require('better-sqlite3'); + const dbRoot = join(tmpdir(), './db'); /** @@ -17,7 +20,7 @@ const dbRoot = join(tmpdir(), './db'); */ async function initDB( logger?: ReturnType, -): Promise { +): Promise { const dbPath = join(dbRoot, 'store.db'); console.log('dbPath:', dbPath); await mkdir(dbRoot, { recursive: true }); @@ -91,11 +94,16 @@ export async function makeSQLKVStore( * last key in the store. */ function kvGetNextKey(previousKey: string): string | undefined { - if (typeof previousKey !== 'string') { - // eslint-disable-next-line @typescript-eslint/restrict-template-expressions - throw new Error(`previousKey ${previousKey} must be a string`); + try { + if (typeof previousKey !== 'string') { + // eslint-disable-next-line @typescript-eslint/restrict-template-expressions + throw new Error(`previousKey ${previousKey} must be a string`); + } + return sqlKVGetNextKey.get(previousKey) as string | undefined; + } catch (problem) { + console.error('BIG PROBLEM', problem); + throw problem; } - return sqlKVGetNextKey.get(previousKey) as string | undefined; } const sqlKVSet = db.prepare(` diff --git a/packages/nodejs/src/vat/streams.ts b/packages/nodejs/src/vat/streams.ts index 5f453fbde..5df3dd526 100644 --- a/packages/nodejs/src/vat/streams.ts +++ b/packages/nodejs/src/vat/streams.ts @@ -1,5 +1,3 @@ -import '@ocap/test-utils/mock-endoify'; - import { isVatCommand } from '@ocap/kernel'; import type { VatCommand, VatCommandReply } from '@ocap/kernel'; import { NodeWorkerDuplexStream } from '@ocap/streams'; diff --git a/packages/nodejs/src/vat/vat-worker.ts b/packages/nodejs/src/vat/vat-worker.ts index 142a9f60f..705695022 100644 --- a/packages/nodejs/src/vat/vat-worker.ts +++ b/packages/nodejs/src/vat/vat-worker.ts @@ -5,38 +5,36 @@ import type { VatId } from '@ocap/kernel'; const streams = '../../dist/vat/streams.mjs'; const store = '../../dist/kernel/sqlite-kv-store.mjs'; -try { - console.debug(`[vat-worker (${process.env.NODE_VAT_ID})] begin imports`); - - const { VatSupervisor } = await import('@ocap/kernel'); - const { makeLogger } = await import('@ocap/utils'); - const { makeCommandStream } = await import(streams); - const { makeSQLKVStore } = await import(store); - - console.debug(`[vat-worker (${process.env.NODE_VAT_ID})] imports complete`); - - const vatId = process.env.NODE_VAT_ID as VatId; - - if (vatId) { - console.log('vatId', vatId); - const logger = makeLogger(`[vat-worker (${vatId})]`); - logger.debug('starting worker...'); - main().catch(logger.error); - } else { - console.log('no vatId set for env variable NODE_VAT_ID'); - } - - /** - * The main function for the iframe. - */ - async function main(): Promise { - // eslint-disable-next-line no-void - void new VatSupervisor({ - id: 'iframe', - commandStream: makeCommandStream(), - makeKVStore: makeSQLKVStore, - }); - } -} catch (problem) { - console.error(problem); +const { VatSupervisor } = await import('@ocap/kernel'); +const { makeLogger } = await import('@ocap/utils'); +const { makeCommandStream } = await import(streams); +const { makeSQLKVStore } = await import(store); + +console.debug(`[vat-worker (${process.env.NODE_VAT_ID})] imports complete`); + +const vatId = process.env.NODE_VAT_ID as VatId; + +if (vatId) { + console.log('vatId', vatId); + const logger = makeLogger(`[vat-worker (${vatId})]`); + logger.debug('starting worker...'); + main().catch(logger.error); +} else { + console.log('no vatId set for env variable NODE_VAT_ID'); } + +/** + * The main function for the iframe. + */ +async function main(): Promise { + // eslint-disable-next-line no-void + console.debug('entered main'); + const commandStream = makeCommandStream(); + await commandStream.synchronize(); + void new VatSupervisor({ + id: vatId, + commandStream, + makeKVStore: makeSQLKVStore, + }); + console.debug('created supervisor'); +} \ No newline at end of file diff --git a/packages/nodejs/test/e2e/kernel-worker.test.ts b/packages/nodejs/test/e2e/kernel-worker.test.ts index 7b15bcd80..d33f5e18f 100644 --- a/packages/nodejs/test/e2e/kernel-worker.test.ts +++ b/packages/nodejs/test/e2e/kernel-worker.test.ts @@ -63,13 +63,12 @@ describe('Kernel Worker', () => { it('creates a vat', async () => { kernel = await makeKernel(kernelPort); - let vatIds: VatId[] = kernel.getVatIds(); - expect(vatIds).toHaveLength(0); - + expect(kernel.getVatIds()).toHaveLength(0); + console.debug('preparing to launch vat') const kRef = await kernel.launchVat(testVatConfig); + console.debug('succsefully to launch vat') expect(kRef).toBeInstanceOf(String); - vatIds = kernel.getVatIds(); - expect(vatIds).toHaveLength(1); + expect(kernel.getVatIds()).toHaveLength(1); }, 10000); it.skip('should handle the lifecycle of multiple vats', async () => { diff --git a/vitest.config.ts b/vitest.config.ts index 8e81f19dd..7dadad60e 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -74,10 +74,10 @@ export default defineConfig({ lines: 68.73, }, 'packages/kernel/**': { - statements: 42.51, - functions: 54.86, - branches: 29.27, - lines: 42.74, + statements: 44.12, + functions: 55.55, + branches: 30.03, + lines: 44.35, }, 'packages/nodejs/**': { statements: 47.61, @@ -112,4 +112,4 @@ export default defineConfig({ }, }, }, -}); +}); \ No newline at end of file From 5ff625f9075228779b430bcbe944bd15dd801d82 Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Fri, 24 Jan 2025 12:27:58 -0600 Subject: [PATCH 14/27] trace dispatch error --- packages/kernel/src/Kernel.ts | 2 ++ packages/kernel/src/VatHandle.ts | 2 ++ packages/kernel/src/VatSupervisor.ts | 7 +++++++ 3 files changed, 11 insertions(+) diff --git a/packages/kernel/src/Kernel.ts b/packages/kernel/src/Kernel.ts index 0ae10ad24..93d4d3a00 100644 --- a/packages/kernel/src/Kernel.ts +++ b/packages/kernel/src/Kernel.ts @@ -278,7 +278,9 @@ export class Kernel { }); this.#vats.set(vatId, vat); this.#storage.initEndpoint(vatId); + console.debug('starting vat init'); await vat.init(); + console.debug('init complete'); const rootRef = this.exportFromVat(vatId, ROOT_OBJECT_VREF); return rootRef; } diff --git a/packages/kernel/src/VatHandle.ts b/packages/kernel/src/VatHandle.ts index 268aaf3f8..93cf40858 100644 --- a/packages/kernel/src/VatHandle.ts +++ b/packages/kernel/src/VatHandle.ts @@ -365,7 +365,9 @@ export class VatHandle { }, ); + this.#logger.debug('pinging...'); await this.sendMessage({ method: VatCommandMethod.ping, params: null }); + this.#logger.debug('pinged!'); await this.sendMessage({ method: VatCommandMethod.initVat, params: this.config, diff --git a/packages/kernel/src/VatSupervisor.ts b/packages/kernel/src/VatSupervisor.ts index 712c57823..2ec81bd41 100644 --- a/packages/kernel/src/VatSupervisor.ts +++ b/packages/kernel/src/VatSupervisor.ts @@ -98,6 +98,7 @@ export class VatSupervisor { * @param message.payload - The payload to handle. */ async handleMessage({ id, payload }: VatCommand): Promise { + console.debug('HANDLEMESSAGE', { id, payload }); switch (payload.method) { case VatCommandMethod.deliver: { if (!this.#dispatch) { @@ -115,7 +116,9 @@ export class VatSupervisor { } case VatCommandMethod.initVat: { + console.debug('CALLING INIT VAT'); const rootObjectVref = await this.#initVat(payload.params); + console.debug('CALLED INIT VAT'); await this.replyToMessage(id, { method: VatCommandMethod.initVat, params: rootObjectVref, @@ -225,7 +228,9 @@ export class VatSupervisor { if (!fetched.ok) { throw Error(`fetch of user code ${bundleSpec} failed: ${fetched.status}`); } + console.debug('fetched bundle'); const bundle = await fetched.json(); + console.debug('and made it json'); const buildVatNamespace = async ( lsEndowments: object, @@ -251,7 +256,9 @@ export class VatSupervisor { this.#dispatch = liveslots.dispatch; const serParam = marshal.toCapData(harden(parameters)) as CapData; + console.debug('CALLING DISPATCH'); await this.#dispatch(harden(['startVat', serParam])); + console.debug('AWAITED DISPATCH'); return ROOT_OBJECT_VREF; } From 2bd4b2c240db274f5fd5816d32003992add82a20 Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Fri, 24 Jan 2025 13:37:14 -0600 Subject: [PATCH 15/27] get plucky --- packages/nodejs/src/kernel/sqlite-kv-store.ts | 12 ++++-------- packages/nodejs/test/e2e/kernel-worker.test.ts | 6 +++--- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/packages/nodejs/src/kernel/sqlite-kv-store.ts b/packages/nodejs/src/kernel/sqlite-kv-store.ts index 66e86720e..c5088a0e9 100644 --- a/packages/nodejs/src/kernel/sqlite-kv-store.ts +++ b/packages/nodejs/src/kernel/sqlite-kv-store.ts @@ -56,6 +56,7 @@ export async function makeSQLKVStore( FROM kv WHERE key = ? `); + sqlKVGet.pluck(true); /** * Read a key's value from the database. @@ -66,16 +67,10 @@ export async function makeSQLKVStore( */ function kvGet(key: string, required: boolean): string { const result = sqlKVGet.get(key); - if (isObject(result) && hasProperty(result, 'value')) { - const value = result.value as string; - logger.debug(`kernel get '${key}' as '${value}'`); - return value; - } - if (required) { + if (required && !result) { throw Error(`no record matching key '${key}'`); } - // Sometimes, we really lean on TypeScript's unsoundness - return undefined as unknown as string; + return result as string; } const sqlKVGetNextKey = db.prepare(` @@ -84,6 +79,7 @@ export async function makeSQLKVStore( WHERE key > ? LIMIT 1 `); + sqlKVGetNextKey.pluck(true); /** * Get the lexicographically next key in the KV store after a given key. diff --git a/packages/nodejs/test/e2e/kernel-worker.test.ts b/packages/nodejs/test/e2e/kernel-worker.test.ts index d33f5e18f..2144d3406 100644 --- a/packages/nodejs/test/e2e/kernel-worker.test.ts +++ b/packages/nodejs/test/e2e/kernel-worker.test.ts @@ -67,11 +67,11 @@ describe('Kernel Worker', () => { console.debug('preparing to launch vat') const kRef = await kernel.launchVat(testVatConfig); console.debug('succsefully to launch vat') - expect(kRef).toBeInstanceOf(String); + expect(typeof kRef).toStrictEqual('string'); expect(kernel.getVatIds()).toHaveLength(1); - }, 10000); + }); - it.skip('should handle the lifecycle of multiple vats', async () => { + it('should handle the lifecycle of multiple vats', async () => { console.log('Started test.'); console.log('Creating kernel...'); kernel = await makeKernel(kernelPort); From 196f216ddc506119f9e4f62015491b71419f6ac7 Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Fri, 24 Jan 2025 14:36:14 -0600 Subject: [PATCH 16/27] cleanup --- packages/nodejs/package.json | 2 +- .../src/kernel/VatWorkerService.test.ts | 35 +++-- .../nodejs/src/kernel/VatWorkerService.ts | 7 +- packages/nodejs/src/kernel/sqlite-kv-store.ts | 23 +--- packages/nodejs/src/vat/streams.test.ts | 2 +- packages/nodejs/src/vat/streams.ts | 8 +- packages/nodejs/src/vat/vat-worker.ts | 21 +-- .../nodejs/test/e2e/kernel-worker.test.ts | 130 ++++++------------ packages/nodejs/test/workers/hello-world.mjs | 3 - packages/nodejs/test/workers/ping-pong.mjs | 20 --- packages/nodejs/test/workers/stream-sync.mjs | 8 +- yarn.lock | 2 +- 12 files changed, 90 insertions(+), 171 deletions(-) delete mode 100644 packages/nodejs/test/workers/hello-world.mjs delete mode 100644 packages/nodejs/test/workers/ping-pong.mjs diff --git a/packages/nodejs/package.json b/packages/nodejs/package.json index 253b377e2..8854840c1 100644 --- a/packages/nodejs/package.json +++ b/packages/nodejs/package.json @@ -47,6 +47,7 @@ "@metamask/eslint-config-nodejs": "^14.0.0", "@metamask/eslint-config-typescript": "^14.0.0", "@ocap/cli": "workspace:^", + "@ocap/test-utils": "workspace:^", "@ts-bridge/cli": "^0.6.2", "@ts-bridge/shims": "^0.1.1", "@types/better-sqlite3": "^7.6.12", @@ -77,7 +78,6 @@ }, "dependencies": { "@endo/promise-kit": "^1.1.6", - "@metamask/utils": "^11.0.1", "@ocap/kernel": "workspace:^", "@ocap/shims": "workspace:^", "@ocap/streams": "workspace:^", diff --git a/packages/nodejs/src/kernel/VatWorkerService.test.ts b/packages/nodejs/src/kernel/VatWorkerService.test.ts index a665deda4..5002a5df3 100644 --- a/packages/nodejs/src/kernel/VatWorkerService.test.ts +++ b/packages/nodejs/src/kernel/VatWorkerService.test.ts @@ -3,7 +3,7 @@ import '@ocap/shims/endoify'; import type { VatId } from '@ocap/kernel'; import { NodeWorkerDuplexStream } from '@ocap/streams'; import { makeCounter } from '@ocap/utils'; -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; import { NodejsVatWorkerService } from './VatWorkerService.js'; import { getTestWorkerFile } from '../../test/workers'; @@ -14,23 +14,42 @@ describe('NodejsVatWorkerService', () => { expect(instance).toBeInstanceOf(NodejsVatWorkerService); }); - const helloWorld = getTestWorkerFile('stream-sync'); + const testWorkerFile = getTestWorkerFile('stream-sync'); const vatIdCounter = makeCounter(); const getTestVatId = (): VatId => `v${vatIdCounter()}`; describe('launch', () => { it('creates a NodeWorker and returns a NodeWorkerDuplexStream', async () => { - const service = new NodejsVatWorkerService(helloWorld); + const service = new NodejsVatWorkerService(testWorkerFile); const testVatId: VatId = getTestVatId(); - const multiplexer = await service.launch(testVatId); + const stream = await service.launch(testVatId); - expect(multiplexer).toBeInstanceOf(NodeWorkerDuplexStream); + expect(stream).toBeInstanceOf(NodeWorkerDuplexStream); + }); + + it('rejects if synchronize fails', async () => { + const rejected = 'test-reject-value'; + + vi.doMock('@ocap/streams', () => ({ + NodeWorkerDuplexStream: vi.fn().mockImplementation(() => ({ + synchronize: vi.fn().mockRejectedValue(rejected), + })), + })); + vi.resetModules(); + const NVWS = (await import('./VatWorkerService.js')) + .NodejsVatWorkerService; + + const service = new NVWS(testWorkerFile); + const testVatId: VatId = getTestVatId(); + await expect(async () => await service.launch(testVatId)).rejects.toThrow( + rejected, + ); }); }); describe('terminate', () => { it('terminates the target vat', async () => { - const service = new NodejsVatWorkerService(helloWorld); + const service = new NodejsVatWorkerService(testWorkerFile); const testVatId: VatId = getTestVatId(); await service.launch(testVatId); @@ -41,7 +60,7 @@ describe('NodejsVatWorkerService', () => { }); it('throws when terminating an unknown vat', async () => { - const service = new NodejsVatWorkerService(helloWorld); + const service = new NodejsVatWorkerService(testWorkerFile); const testVatId: VatId = getTestVatId(); await expect( @@ -52,7 +71,7 @@ describe('NodejsVatWorkerService', () => { describe('terminateAll', () => { it('terminates all vats', async () => { - const service = new NodejsVatWorkerService(helloWorld); + const service = new NodejsVatWorkerService(testWorkerFile); const vatIds: VatId[] = [getTestVatId(), getTestVatId(), getTestVatId()]; await Promise.all( diff --git a/packages/nodejs/src/kernel/VatWorkerService.ts b/packages/nodejs/src/kernel/VatWorkerService.ts index d9d761e24..15bf0363c 100644 --- a/packages/nodejs/src/kernel/VatWorkerService.ts +++ b/packages/nodejs/src/kernel/VatWorkerService.ts @@ -44,28 +44,25 @@ export class NodejsVatWorkerService implements VatWorkerService { async launch( vatId: VatId, ): Promise> { + this.#logger.debug('launching vat', vatId); const { promise, resolve, reject } = makePromiseKit>(); - this.#logger.debug('launching', vatId); const worker = new NodeWorker(this.#workerFilePath, { env: { NODE_VAT_ID: vatId, }, - // execArgv: ["--require", "ts-node/register"], }); - this.#logger.debug('launched', vatId); worker.once('online', () => { const stream = new NodeWorkerDuplexStream( worker, isVatCommandReply, ); this.workers.set(vatId, { worker, stream }); - console.debug('synchronizing...'); stream .synchronize() .then(() => { resolve(stream); - this.#logger.debug('connected', vatId); + this.#logger.debug('connected to kernel'); return undefined; }) .catch((error) => { diff --git a/packages/nodejs/src/kernel/sqlite-kv-store.ts b/packages/nodejs/src/kernel/sqlite-kv-store.ts index c5088a0e9..0cf698bb0 100644 --- a/packages/nodejs/src/kernel/sqlite-kv-store.ts +++ b/packages/nodejs/src/kernel/sqlite-kv-store.ts @@ -1,13 +1,12 @@ -import { hasProperty, isObject } from '@metamask/utils'; import type { KVStore } from '@ocap/kernel'; import { makeLogger } from '@ocap/utils'; -// eslint-disable-next-line @typescript-eslint/naming-convention -// import Sqlite from 'better-sqlite3'; +import type { Database } from 'better-sqlite3'; import { mkdir } from 'fs/promises'; import { tmpdir } from 'os'; import { join } from 'path'; -import type { Database } from 'better-sqlite3'; +// We require require because the ESM import does not work properly. +// eslint-disable-next-line @typescript-eslint/no-require-imports const Sqlite = require('better-sqlite3'); const dbRoot = join(tmpdir(), './db'); @@ -90,16 +89,11 @@ export async function makeSQLKVStore( * last key in the store. */ function kvGetNextKey(previousKey: string): string | undefined { - try { - if (typeof previousKey !== 'string') { - // eslint-disable-next-line @typescript-eslint/restrict-template-expressions - throw new Error(`previousKey ${previousKey} must be a string`); - } - return sqlKVGetNextKey.get(previousKey) as string | undefined; - } catch (problem) { - console.error('BIG PROBLEM', problem); - throw problem; + if (typeof previousKey !== 'string') { + // eslint-disable-next-line @typescript-eslint/restrict-template-expressions + throw new Error(`previousKey ${previousKey} must be a string`); } + return sqlKVGetNextKey.get(previousKey) as string | undefined; } const sqlKVSet = db.prepare(` @@ -115,7 +109,6 @@ export async function makeSQLKVStore( * @param value - The value to assign to it. */ function kvSet(key: string, value: string): void { - logger.debug(`kernel set '${key}' to '${value}'`); sqlKVSet.run(key, value); } @@ -130,7 +123,6 @@ export async function makeSQLKVStore( * @param key - The key to remove. */ function kvDelete(key: string): void { - logger.debug(`kernel delete '${key}'`); sqlKVDelete.run(key); } @@ -142,7 +134,6 @@ export async function makeSQLKVStore( * Delete all keys and values from the database. */ function kvClear(): void { - logger.debug(`kernel clear`); sqlKVDrop.run(); sqlKVInit.run(); } diff --git a/packages/nodejs/src/vat/streams.test.ts b/packages/nodejs/src/vat/streams.test.ts index 729f606b1..bf9668b15 100644 --- a/packages/nodejs/src/vat/streams.test.ts +++ b/packages/nodejs/src/vat/streams.test.ts @@ -16,7 +16,7 @@ vi.mock('@ocap/kernel', async () => { vi.mock('@ocap/streams', () => ({ NodeWorkerDuplexStream: vi.fn(), -})) +})); describe('getPort', () => { it('returns a port', async () => { diff --git a/packages/nodejs/src/vat/streams.ts b/packages/nodejs/src/vat/streams.ts index 5df3dd526..b3475c356 100644 --- a/packages/nodejs/src/vat/streams.ts +++ b/packages/nodejs/src/vat/streams.ts @@ -1,7 +1,8 @@ import { isVatCommand } from '@ocap/kernel'; import type { VatCommand, VatCommandReply } from '@ocap/kernel'; import { NodeWorkerDuplexStream } from '@ocap/streams'; -import { type MessagePort as NodePort, parentPort } from 'node:worker_threads'; +import { parentPort } from 'node:worker_threads'; +import type { MessagePort as NodePort } from 'node:worker_threads'; /** * Return the parent port of the Node.js worker if it exists; otherwise throw. @@ -9,7 +10,6 @@ import { type MessagePort as NodePort, parentPort } from 'node:worker_threads'; * @returns The parent port. * @throws If not called from within a Node.js worker. */ -// eslint-disable-next-line @typescript-eslint/explicit-function-return-type export function getPort(): NodePort { if (!parentPort) { throw new Error('Expected to run in a Node.js worker with parentPort.'); @@ -23,9 +23,7 @@ export function getPort(): NodePort { * * @returns A NodeWorkerDuplexStream */ -export function makeCommandStream( - -): NodeWorkerDuplexStream< +export function makeCommandStream(): NodeWorkerDuplexStream< VatCommand, VatCommandReply > { diff --git a/packages/nodejs/src/vat/vat-worker.ts b/packages/nodejs/src/vat/vat-worker.ts index 705695022..944a59722 100644 --- a/packages/nodejs/src/vat/vat-worker.ts +++ b/packages/nodejs/src/vat/vat-worker.ts @@ -1,23 +1,16 @@ import '@ocap/shims/endoify'; import type { VatId } from '@ocap/kernel'; +import { VatSupervisor } from '@ocap/kernel'; +import { makeLogger } from '@ocap/utils'; -const streams = '../../dist/vat/streams.mjs'; -const store = '../../dist/kernel/sqlite-kv-store.mjs'; - -const { VatSupervisor } = await import('@ocap/kernel'); -const { makeLogger } = await import('@ocap/utils'); -const { makeCommandStream } = await import(streams); -const { makeSQLKVStore } = await import(store); - -console.debug(`[vat-worker (${process.env.NODE_VAT_ID})] imports complete`); +import { makeCommandStream } from './streams'; +import { makeSQLKVStore } from '../kernel/sqlite-kv-store'; const vatId = process.env.NODE_VAT_ID as VatId; if (vatId) { - console.log('vatId', vatId); const logger = makeLogger(`[vat-worker (${vatId})]`); - logger.debug('starting worker...'); main().catch(logger.error); } else { console.log('no vatId set for env variable NODE_VAT_ID'); @@ -27,14 +20,12 @@ if (vatId) { * The main function for the iframe. */ async function main(): Promise { - // eslint-disable-next-line no-void - console.debug('entered main'); const commandStream = makeCommandStream(); await commandStream.synchronize(); + // eslint-disable-next-line no-void void new VatSupervisor({ id: vatId, commandStream, makeKVStore: makeSQLKVStore, }); - console.debug('created supervisor'); -} \ No newline at end of file +} diff --git a/packages/nodejs/test/e2e/kernel-worker.test.ts b/packages/nodejs/test/e2e/kernel-worker.test.ts index 2144d3406..7fc07dc1e 100644 --- a/packages/nodejs/test/e2e/kernel-worker.test.ts +++ b/packages/nodejs/test/e2e/kernel-worker.test.ts @@ -1,21 +1,14 @@ import '@ocap/shims/endoify'; -// import '../../../test-utils/src/env/mock-endoify.js'; -import type { NonEmptyArray } from '@metamask/utils'; import { Kernel, VatCommandMethod } from '@ocap/kernel'; import type { VatConfig, VatId } from '@ocap/kernel'; import { MessageChannel as NodeMessageChannel, MessagePort as NodePort, - Worker as NodeWorker, } from 'node:worker_threads'; import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import { makeKernel } from '../../src/kernel/make-kernel.js'; -import { getTestWorkerFile } from 'test/workers/index.js'; - -const workerFileURL = new URL('../../dist/vat-worker.mjs', import.meta.url) - .pathname; vi.mock('node:process', () => ({ exit: vi.fn((reason) => { @@ -27,16 +20,20 @@ describe('Kernel Worker', () => { let kernelPort: NodePort; let kernel: Kernel; + // Tests below assume these are sorted for convenience. + const testVatIds = ['v1', 'v2', 'v3'].sort(); + const testVatConfig: VatConfig = { bundleSpec: 'http://localhost:3000/sample-vat.bundle', parameters: { name: 'Nodeen' }, }; - beforeEach(() => { + beforeEach(async () => { if (kernelPort) { kernelPort.close(); } kernelPort = new NodeMessageChannel().port1; + kernel = await makeKernel(kernelPort); }); afterEach(async () => { @@ -46,97 +43,48 @@ describe('Kernel Worker', () => { } }); - it('trivial', () => { - console.debug('TRIVIAL'); - expect(true).toBe(true); + it('launches a vat', async () => { + expect(kernel.getVatIds()).toHaveLength(0); + const kRef = await kernel.launchVat(testVatConfig); + expect(typeof kRef).toBe('string'); + expect(kernel.getVatIds()).toHaveLength(1); }); - it('starts a NodeWorker', async () => { - const worker = new NodeWorker(getTestWorkerFile('hello-world')); - expect(worker).toBeInstanceOf(NodeWorker); - }); + const launchTestVats = async (): Promise => { + await Promise.all( + testVatIds.map(async () => await kernel.launchVat(testVatConfig)), + ); + }; - it('makes a Kernel', async () => { - kernel = await makeKernel(kernelPort); - expect(kernel).toBeInstanceOf(Kernel); - }); + it('restarts vats', async () => { + await launchTestVats(); + expect(kernel.getVatIds().sort()).toStrictEqual(testVatIds); - it('creates a vat', async () => { - kernel = await makeKernel(kernelPort); - expect(kernel.getVatIds()).toHaveLength(0); - console.debug('preparing to launch vat') - const kRef = await kernel.launchVat(testVatConfig); - console.debug('succsefully to launch vat') - expect(typeof kRef).toStrictEqual('string'); - expect(kernel.getVatIds()).toHaveLength(1); + await Promise.all(testVatIds.map(kernel.restartVat.bind(kernel))); + expect(kernel.getVatIds().sort()).toStrictEqual(testVatIds); }); - it('should handle the lifecycle of multiple vats', async () => { - console.log('Started test.'); - console.log('Creating kernel...'); - kernel = await makeKernel(kernelPort); - console.log('Kernel created.'); + it('terminates all vats', async () => { + await launchTestVats(); + expect(kernel.getVatIds().sort()).toStrictEqual(testVatIds); - console.log('Handling the lifecycle of multiple vats...'); - await runVatLifecycle(kernel, ['v1', 'v2', 'v3']); - console.log('Lifecycle of multiple vats handled.'); + await kernel.terminateAllVats(); + expect(kernel.getVatIds()).toHaveLength(0); + }); - console.log('Test passed.'); + it('pings vats', async () => { + await launchTestVats(); + expect(kernel.getVatIds().sort()).toStrictEqual(testVatIds); + + await Promise.all( + testVatIds.map( + async (vatId: VatId) => + await kernel.sendMessage(vatId, { + method: VatCommandMethod.ping, + params: null, + }), + ), + ); expect(true).toBe(true); }); }); - -/** - * Runs the full lifecycle of an array of vats, including their creation, - * restart, message passing, and termination. - * - * @param kernel The kernel instance. - * @param vats An array of VatIds to be managed. - * @param vatConfig The config to pass for vat initialization. - */ -export async function runVatLifecycle( - kernel: Kernel, - vats: NonEmptyArray, - vatConfig: VatConfig = { - bundleSpec: 'http://localhost:3000/sample-vat.bundle', - parameters: { name: 'Nodeen' }, - }, -): Promise { - console.log('runVatLifecycle Start...'); - const vatLabel = vats.join(', '); - console.time(`Created vats: ${vatLabel}`); - const kRef = await kernel.launchVat(vatConfig); - console.debug('kref', kRef); - - await Promise.all(vats.map(async () => await kernel.launchVat(vatConfig))); - console.timeEnd(`Created vats: ${vatLabel}`); - const knownVats = kernel.getVatIds() as NonEmptyArray; - const knownVatsLabel = knownVats.join(', '); - console.log('Kernel vats:', knownVatsLabel); - - // Restart a randomly selected vat from the array. - console.time(`Restart vats: ${knownVatsLabel}`); - await Promise.all( - knownVats.map(async (vatId: VatId) => await kernel.restartVat(vatId)), - ); - console.timeEnd(`Restart vats: ${knownVatsLabel}`); - - // Send a "Ping" message to a randomly selected vat. - console.time(`Ping vats: ${knownVatsLabel}`); - await Promise.all( - knownVats.map( - async (vatId: VatId) => - await kernel.sendMessage(vatId, { - method: VatCommandMethod.ping, - params: null, - }), - ), - ); - console.timeEnd(`Ping vats "${knownVatsLabel}"`); - - console.time(`Terminated vats: ${knownVatsLabel}`); - await kernel.terminateAllVats(); - console.timeEnd(`Terminated vats: ${knownVatsLabel}`); - - console.log(`Kernel has ${kernel.getVatIds().length} vats`); -} diff --git a/packages/nodejs/test/workers/hello-world.mjs b/packages/nodejs/test/workers/hello-world.mjs deleted file mode 100644 index 93c4f5091..000000000 --- a/packages/nodejs/test/workers/hello-world.mjs +++ /dev/null @@ -1,3 +0,0 @@ -import '@ocap/shims/endoify'; - -console.debug('hello, world computer', process.env.NODE_VAT_ID); diff --git a/packages/nodejs/test/workers/ping-pong.mjs b/packages/nodejs/test/workers/ping-pong.mjs deleted file mode 100644 index f5182149a..000000000 --- a/packages/nodejs/test/workers/ping-pong.mjs +++ /dev/null @@ -1,20 +0,0 @@ -import '@ocap/shims/endoify'; - -import { VatSupervisor } from '../../../kernel/dist/VatSupervisor.mjs'; -import { makeMapKVStore } from '../../dist/kernel/map-kv-store.mjs'; -import { makeCommandStream } from '../../dist/vat/streams.mjs'; - -console.debug('ping pong'); - -main().catch(console.error); - -/** - * The main function for the worker. TODO: support pinging and ponging. - */ -async function main() { - void new VatSupervisor({ - id: 'iframe', - commandStream: makeCommandStream(), - makeKVStore: makeMapKVStore, - }); -} diff --git a/packages/nodejs/test/workers/stream-sync.mjs b/packages/nodejs/test/workers/stream-sync.mjs index 51c635697..8cb0d4d01 100644 --- a/packages/nodejs/test/workers/stream-sync.mjs +++ b/packages/nodejs/test/workers/stream-sync.mjs @@ -1,15 +1,13 @@ -import '@ocap/shims/endoify'; - +import '../../dist/env/endoify.mjs'; import { makeCommandStream } from '../../dist/vat/streams.mjs'; main().catch(console.error); /** - * The main function for the worker. + * The main function for the test worker. + * No supervisor is created, but the stream is synchronized for comms testing. */ async function main() { - console.debug('top', process.env.NODE_VAT_ID); const stream = makeCommandStream(); await stream.synchronize(); - console.debug('bot', process.env.NODE_VAT_ID); } diff --git a/yarn.lock b/yarn.lock index 0a83cbd05..920fbfc77 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2286,11 +2286,11 @@ __metadata: "@metamask/eslint-config": "npm:^14.0.0" "@metamask/eslint-config-nodejs": "npm:^14.0.0" "@metamask/eslint-config-typescript": "npm:^14.0.0" - "@metamask/utils": "npm:^11.0.1" "@ocap/cli": "workspace:^" "@ocap/kernel": "workspace:^" "@ocap/shims": "workspace:^" "@ocap/streams": "workspace:^" + "@ocap/test-utils": "workspace:^" "@ocap/utils": "workspace:^" "@ts-bridge/cli": "npm:^0.6.2" "@ts-bridge/shims": "npm:^0.1.1" From 9c257dbd4383332c3f97f16d214d0e740236143d Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Fri, 24 Jan 2025 14:46:19 -0600 Subject: [PATCH 17/27] remove debug logs --- packages/kernel/src/Kernel.ts | 2 -- packages/kernel/src/VatHandle.ts | 2 -- packages/kernel/src/VatSupervisor.ts | 7 ------- 3 files changed, 11 deletions(-) diff --git a/packages/kernel/src/Kernel.ts b/packages/kernel/src/Kernel.ts index 93d4d3a00..0ae10ad24 100644 --- a/packages/kernel/src/Kernel.ts +++ b/packages/kernel/src/Kernel.ts @@ -278,9 +278,7 @@ export class Kernel { }); this.#vats.set(vatId, vat); this.#storage.initEndpoint(vatId); - console.debug('starting vat init'); await vat.init(); - console.debug('init complete'); const rootRef = this.exportFromVat(vatId, ROOT_OBJECT_VREF); return rootRef; } diff --git a/packages/kernel/src/VatHandle.ts b/packages/kernel/src/VatHandle.ts index 93cf40858..268aaf3f8 100644 --- a/packages/kernel/src/VatHandle.ts +++ b/packages/kernel/src/VatHandle.ts @@ -365,9 +365,7 @@ export class VatHandle { }, ); - this.#logger.debug('pinging...'); await this.sendMessage({ method: VatCommandMethod.ping, params: null }); - this.#logger.debug('pinged!'); await this.sendMessage({ method: VatCommandMethod.initVat, params: this.config, diff --git a/packages/kernel/src/VatSupervisor.ts b/packages/kernel/src/VatSupervisor.ts index 2ec81bd41..712c57823 100644 --- a/packages/kernel/src/VatSupervisor.ts +++ b/packages/kernel/src/VatSupervisor.ts @@ -98,7 +98,6 @@ export class VatSupervisor { * @param message.payload - The payload to handle. */ async handleMessage({ id, payload }: VatCommand): Promise { - console.debug('HANDLEMESSAGE', { id, payload }); switch (payload.method) { case VatCommandMethod.deliver: { if (!this.#dispatch) { @@ -116,9 +115,7 @@ export class VatSupervisor { } case VatCommandMethod.initVat: { - console.debug('CALLING INIT VAT'); const rootObjectVref = await this.#initVat(payload.params); - console.debug('CALLED INIT VAT'); await this.replyToMessage(id, { method: VatCommandMethod.initVat, params: rootObjectVref, @@ -228,9 +225,7 @@ export class VatSupervisor { if (!fetched.ok) { throw Error(`fetch of user code ${bundleSpec} failed: ${fetched.status}`); } - console.debug('fetched bundle'); const bundle = await fetched.json(); - console.debug('and made it json'); const buildVatNamespace = async ( lsEndowments: object, @@ -256,9 +251,7 @@ export class VatSupervisor { this.#dispatch = liveslots.dispatch; const serParam = marshal.toCapData(harden(parameters)) as CapData; - console.debug('CALLING DISPATCH'); await this.#dispatch(harden(['startVat', serParam])); - console.debug('AWAITED DISPATCH'); return ROOT_OBJECT_VREF; } From e1fc71d1ec0989bc1e44ba42c1e09719044057a7 Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Fri, 24 Jan 2025 15:08:10 -0600 Subject: [PATCH 18/27] more cleanup --- .../src/kernel/VatWorkerService.test.ts | 20 ++++++--- .../nodejs/src/kernel/VatWorkerService.ts | 14 ++++--- packages/nodejs/src/kernel/make-kernel.ts | 2 +- packages/nodejs/src/kernel/map-kv-store.ts | 41 ------------------- 4 files changed, 24 insertions(+), 53 deletions(-) delete mode 100644 packages/nodejs/src/kernel/map-kv-store.ts diff --git a/packages/nodejs/src/kernel/VatWorkerService.test.ts b/packages/nodejs/src/kernel/VatWorkerService.test.ts index 5002a5df3..b53b77f04 100644 --- a/packages/nodejs/src/kernel/VatWorkerService.test.ts +++ b/packages/nodejs/src/kernel/VatWorkerService.test.ts @@ -10,7 +10,7 @@ import { getTestWorkerFile } from '../../test/workers'; describe('NodejsVatWorkerService', () => { it('constructs an instance without any arguments', () => { - const instance = new NodejsVatWorkerService(); + const instance = new NodejsVatWorkerService({}); expect(instance).toBeInstanceOf(NodejsVatWorkerService); }); @@ -20,7 +20,9 @@ describe('NodejsVatWorkerService', () => { describe('launch', () => { it('creates a NodeWorker and returns a NodeWorkerDuplexStream', async () => { - const service = new NodejsVatWorkerService(testWorkerFile); + const service = new NodejsVatWorkerService({ + workerFilePath: testWorkerFile, + }); const testVatId: VatId = getTestVatId(); const stream = await service.launch(testVatId); @@ -39,7 +41,7 @@ describe('NodejsVatWorkerService', () => { const NVWS = (await import('./VatWorkerService.js')) .NodejsVatWorkerService; - const service = new NVWS(testWorkerFile); + const service = new NVWS({ workerFilePath: testWorkerFile }); const testVatId: VatId = getTestVatId(); await expect(async () => await service.launch(testVatId)).rejects.toThrow( rejected, @@ -49,7 +51,9 @@ describe('NodejsVatWorkerService', () => { describe('terminate', () => { it('terminates the target vat', async () => { - const service = new NodejsVatWorkerService(testWorkerFile); + const service = new NodejsVatWorkerService({ + workerFilePath: testWorkerFile, + }); const testVatId: VatId = getTestVatId(); await service.launch(testVatId); @@ -60,7 +64,9 @@ describe('NodejsVatWorkerService', () => { }); it('throws when terminating an unknown vat', async () => { - const service = new NodejsVatWorkerService(testWorkerFile); + const service = new NodejsVatWorkerService({ + workerFilePath: testWorkerFile, + }); const testVatId: VatId = getTestVatId(); await expect( @@ -71,7 +77,9 @@ describe('NodejsVatWorkerService', () => { describe('terminateAll', () => { it('terminates all vats', async () => { - const service = new NodejsVatWorkerService(testWorkerFile); + const service = new NodejsVatWorkerService({ + workerFilePath: testWorkerFile, + }); const vatIds: VatId[] = [getTestVatId(), getTestVatId(), getTestVatId()]; await Promise.all( diff --git a/packages/nodejs/src/kernel/VatWorkerService.ts b/packages/nodejs/src/kernel/VatWorkerService.ts index 15bf0363c..6ddcecc7c 100644 --- a/packages/nodejs/src/kernel/VatWorkerService.ts +++ b/packages/nodejs/src/kernel/VatWorkerService.ts @@ -33,12 +33,16 @@ export class NodejsVatWorkerService implements VatWorkerService { * The vat worker service, intended to be constructed in * the kernel worker. * - * @param workerFilePath - The path to a file defining the worker's routine. - * @param logger - An optional {@link Logger}. Defaults to a new logger labeled '[vat worker client]'. + * @param args - A bag of optional arguments. + * @param args.workerFilePath - An optional path to a file defining the worker's routine. Defaults to 'vat-worker.mjs'. + * @param args.logger - An optional {@link Logger}. Defaults to a new logger labeled '[vat worker client]'. */ - constructor(workerFilePath: string = DEFAULT_WORKER_FILE, logger?: Logger) { - this.#workerFilePath = workerFilePath; - this.#logger = logger ?? makeLogger('[vat worker service]'); + constructor(args: { + workerFilePath?: string | undefined; + logger?: Logger | undefined; + }) { + this.#workerFilePath = args.workerFilePath ?? DEFAULT_WORKER_FILE; + this.#logger = args.logger ?? makeLogger('[vat worker service]'); } async launch( diff --git a/packages/nodejs/src/kernel/make-kernel.ts b/packages/nodejs/src/kernel/make-kernel.ts index 710ddd607..aff23f67a 100644 --- a/packages/nodejs/src/kernel/make-kernel.ts +++ b/packages/nodejs/src/kernel/make-kernel.ts @@ -21,7 +21,7 @@ export async function makeKernel( KernelCommand, KernelCommandReply >(port); - const vatWorkerClient = new NodejsVatWorkerService(workerFilePath); + const vatWorkerClient = new NodejsVatWorkerService({ workerFilePath }); // Initialize kernel store. const kvStore = await makeSQLKVStore(); diff --git a/packages/nodejs/src/kernel/map-kv-store.ts b/packages/nodejs/src/kernel/map-kv-store.ts deleted file mode 100644 index 042c5fee8..000000000 --- a/packages/nodejs/src/kernel/map-kv-store.ts +++ /dev/null @@ -1,41 +0,0 @@ -/** - * This file is a copy of `@ocap/kernel/test/storage.ts` - * Because the test dir isn't shipped, it is easier just to copy than to - * do battle with the existing build configuration. - */ -import type { KVStore } from '@ocap/kernel'; - -/** - * A mock key/value store realized as a Map. - * - * @returns The mock {@link KVStore}. - */ -export function makeMapKVStore(): KVStore { - const map = new Map(); - - /** - * Like `get`, but fail if the key isn't there. - * - * @param key - The key to fetch. - * @returns The value at `key`. - */ - function getRequired(key: string): string { - const result = map.get(key); - if (result === undefined) { - throw Error(`No value found for key ${key}.`); - } - return result; - } - - return { - get: map.get.bind(map), - getNextKey: (_key: string): string | undefined => { - throw Error(`mock store does not (yet) support getNextKey`); - }, - getRequired, - set: map.set.bind(map), - delete: map.delete.bind(map), - clear: map.clear.bind(map), - executeQuery: () => [], - }; -} From 6cab7576eebbbebb78dfe8d089e8387fd775d14c Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Fri, 24 Jan 2025 15:11:30 -0600 Subject: [PATCH 19/27] return type test func --- packages/nodejs/src/vat/streams.test.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/nodejs/src/vat/streams.test.ts b/packages/nodejs/src/vat/streams.test.ts index bf9668b15..00a263387 100644 --- a/packages/nodejs/src/vat/streams.test.ts +++ b/packages/nodejs/src/vat/streams.test.ts @@ -2,8 +2,7 @@ import '@ocap/test-utils/mock-endoify'; import { describe, expect, it, vi } from 'vitest'; -// eslint-disable-next-line @typescript-eslint/explicit-function-return-type -const doMockParentPort = (value: unknown) => { +const doMockParentPort = (value: unknown): void => { vi.doMock('node:worker_threads', () => ({ parentPort: value, })); From 73dc899b2f6ee0e4075b5800ea49f3e52ee7ef66 Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Fri, 24 Jan 2025 15:15:59 -0600 Subject: [PATCH 20/27] appease linter --- eslint.config.mjs | 9 ++++++++- .../test/workers/{stream-sync.mjs => stream-sync.js} | 0 vitest.config.ts | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) rename packages/nodejs/test/workers/{stream-sync.mjs => stream-sync.js} (100%) diff --git a/eslint.config.mjs b/eslint.config.mjs index ebddd69cf..bb98851c3 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -183,13 +183,20 @@ const config = createConfig([ { files: [ 'packages/nodejs/**/*-worker.ts', - 'packages/nodejs/test/workers/**/*.mjs', + 'packages/nodejs/test/workers/**/*.js', ], rules: { // Node workers have reasonable cause to read from process.env 'n/no-process-env': 'off', }, }, + { + files: ['packages/nodejs/test/workers/**/*.js'], + rules: { + // Test node worker files can resolve these imports, even if eslint cannot. + 'import-x/no-unresolved': 'off', + }, + }, ]); export default config; diff --git a/packages/nodejs/test/workers/stream-sync.mjs b/packages/nodejs/test/workers/stream-sync.js similarity index 100% rename from packages/nodejs/test/workers/stream-sync.mjs rename to packages/nodejs/test/workers/stream-sync.js diff --git a/vitest.config.ts b/vitest.config.ts index 7dadad60e..7f900b917 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -112,4 +112,4 @@ export default defineConfig({ }, }, }, -}); \ No newline at end of file +}); From 2b33bafe794542a1a5fd04ac9ea4a7aea3e75962 Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Fri, 24 Jan 2025 15:29:05 -0600 Subject: [PATCH 21/27] update extensions --- eslint.config.mjs | 4 ++-- packages/nodejs/test/workers/index.ts | 2 +- vitest.config.ts | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/eslint.config.mjs b/eslint.config.mjs index bb98851c3..99fc4818c 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -183,7 +183,7 @@ const config = createConfig([ { files: [ 'packages/nodejs/**/*-worker.ts', - 'packages/nodejs/test/workers/**/*.js', + 'packages/nodejs/test/workers/**/*', ], rules: { // Node workers have reasonable cause to read from process.env @@ -191,7 +191,7 @@ const config = createConfig([ }, }, { - files: ['packages/nodejs/test/workers/**/*.js'], + files: ['packages/nodejs/test/workers/**/*'], rules: { // Test node worker files can resolve these imports, even if eslint cannot. 'import-x/no-unresolved': 'off', diff --git a/packages/nodejs/test/workers/index.ts b/packages/nodejs/test/workers/index.ts index 60c82437b..0fd6a4a00 100644 --- a/packages/nodejs/test/workers/index.ts +++ b/packages/nodejs/test/workers/index.ts @@ -1,2 +1,2 @@ export const getTestWorkerFile = (name: string): string => - new URL(`./${name}.mjs`, import.meta.url).pathname; + new URL(`./${name}.js`, import.meta.url).pathname; diff --git a/vitest.config.ts b/vitest.config.ts index 7f900b917..167566484 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -81,8 +81,8 @@ export default defineConfig({ }, 'packages/nodejs/**': { statements: 47.61, - functions: 47.36, - branches: 27.77, + functions: 47.61, + branches: 35.29, lines: 47.61, }, 'packages/shims/**': { From 90c0d2a5089aeef780f346d8a64d717f2ee6d590 Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Fri, 24 Jan 2025 15:32:10 -0600 Subject: [PATCH 22/27] thresholds --- vitest.config.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vitest.config.ts b/vitest.config.ts index 167566484..bbc9c69ad 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -74,16 +74,16 @@ export default defineConfig({ lines: 68.73, }, 'packages/kernel/**': { - statements: 44.12, + statements: 43.97, functions: 55.55, branches: 30.03, - lines: 44.35, + lines: 44.2, }, 'packages/nodejs/**': { - statements: 47.61, + statements: 46.75, functions: 47.61, branches: 35.29, - lines: 47.61, + lines: 46.75, }, 'packages/shims/**': { statements: 0, From 82bb2bd94c4ada14ed9e8ca73c7ea11b1550b77f Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Fri, 24 Jan 2025 15:48:48 -0600 Subject: [PATCH 23/27] simplify test a bit --- packages/nodejs/test/e2e/kernel-worker.test.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/packages/nodejs/test/e2e/kernel-worker.test.ts b/packages/nodejs/test/e2e/kernel-worker.test.ts index 7fc07dc1e..9b3ce8226 100644 --- a/packages/nodejs/test/e2e/kernel-worker.test.ts +++ b/packages/nodejs/test/e2e/kernel-worker.test.ts @@ -54,28 +54,23 @@ describe('Kernel Worker', () => { await Promise.all( testVatIds.map(async () => await kernel.launchVat(testVatConfig)), ); + expect(kernel.getVatIds().sort()).toStrictEqual(testVatIds); }; it('restarts vats', async () => { await launchTestVats(); - expect(kernel.getVatIds().sort()).toStrictEqual(testVatIds); - await Promise.all(testVatIds.map(kernel.restartVat.bind(kernel))); expect(kernel.getVatIds().sort()).toStrictEqual(testVatIds); }); it('terminates all vats', async () => { await launchTestVats(); - expect(kernel.getVatIds().sort()).toStrictEqual(testVatIds); - await kernel.terminateAllVats(); expect(kernel.getVatIds()).toHaveLength(0); }); it('pings vats', async () => { await launchTestVats(); - expect(kernel.getVatIds().sort()).toStrictEqual(testVatIds); - await Promise.all( testVatIds.map( async (vatId: VatId) => From d760a69df397ae6b9439e6264c26025aadc33a96 Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Fri, 24 Jan 2025 16:04:29 -0600 Subject: [PATCH 24/27] isolate CI failure --- .../src/kernel/VatWorkerService.test.ts | 29 ++++++++------- .../nodejs/src/kernel/VatWorkerService.ts | 1 + packages/nodejs/src/vat/streams.test.ts | 6 +-- .../nodejs/test/e2e/kernel-worker.test.ts | 2 +- packages/nodejs/test/e2e/vat-worker.test.ts | 37 +++++++++++++++++++ packages/nodejs/test/workers/hello-world.js | 2 + packages/nodejs/tsconfig.json | 3 +- vitest.config.ts | 6 +-- 8 files changed, 65 insertions(+), 21 deletions(-) create mode 100644 packages/nodejs/test/e2e/vat-worker.test.ts create mode 100644 packages/nodejs/test/workers/hello-world.js diff --git a/packages/nodejs/src/kernel/VatWorkerService.test.ts b/packages/nodejs/src/kernel/VatWorkerService.test.ts index b53b77f04..58e35ec97 100644 --- a/packages/nodejs/src/kernel/VatWorkerService.test.ts +++ b/packages/nodejs/src/kernel/VatWorkerService.test.ts @@ -19,17 +19,20 @@ describe('NodejsVatWorkerService', () => { const getTestVatId = (): VatId => `v${vatIdCounter()}`; describe('launch', () => { - it('creates a NodeWorker and returns a NodeWorkerDuplexStream', async () => { - const service = new NodejsVatWorkerService({ - workerFilePath: testWorkerFile, - }); - const testVatId: VatId = getTestVatId(); - const stream = await service.launch(testVatId); - - expect(stream).toBeInstanceOf(NodeWorkerDuplexStream); - }); - - it('rejects if synchronize fails', async () => { + it.todo( + 'creates a NodeWorker and returns a NodeWorkerDuplexStream', + async () => { + const service = new NodejsVatWorkerService({ + workerFilePath: testWorkerFile, + }); + const testVatId: VatId = getTestVatId(); + const stream = await service.launch(testVatId); + + expect(stream).toBeInstanceOf(NodeWorkerDuplexStream); + }, + ); + + it.todo('rejects if synchronize fails', async () => { const rejected = 'test-reject-value'; vi.doMock('@ocap/streams', () => ({ @@ -49,7 +52,7 @@ describe('NodejsVatWorkerService', () => { }); }); - describe('terminate', () => { + describe.todo('terminate', () => { it('terminates the target vat', async () => { const service = new NodejsVatWorkerService({ workerFilePath: testWorkerFile, @@ -75,7 +78,7 @@ describe('NodejsVatWorkerService', () => { }); }); - describe('terminateAll', () => { + describe.todo('terminateAll', () => { it('terminates all vats', async () => { const service = new NodejsVatWorkerService({ workerFilePath: testWorkerFile, diff --git a/packages/nodejs/src/kernel/VatWorkerService.ts b/packages/nodejs/src/kernel/VatWorkerService.ts index 6ddcecc7c..99d147d33 100644 --- a/packages/nodejs/src/kernel/VatWorkerService.ts +++ b/packages/nodejs/src/kernel/VatWorkerService.ts @@ -56,6 +56,7 @@ export class NodejsVatWorkerService implements VatWorkerService { NODE_VAT_ID: vatId, }, }); + worker.stdout.pipe(process.stdout); worker.once('online', () => { const stream = new NodeWorkerDuplexStream( worker, diff --git a/packages/nodejs/src/vat/streams.test.ts b/packages/nodejs/src/vat/streams.test.ts index 00a263387..cb1c642de 100644 --- a/packages/nodejs/src/vat/streams.test.ts +++ b/packages/nodejs/src/vat/streams.test.ts @@ -9,9 +9,9 @@ const doMockParentPort = (value: unknown): void => { vi.resetModules(); }; -vi.mock('@ocap/kernel', async () => { - return { isVatCommand: vi.fn(() => true) }; -}); +vi.mock('@ocap/kernel', async () => ({ + isVatCommand: vi.fn(() => true), +})); vi.mock('@ocap/streams', () => ({ NodeWorkerDuplexStream: vi.fn(), diff --git a/packages/nodejs/test/e2e/kernel-worker.test.ts b/packages/nodejs/test/e2e/kernel-worker.test.ts index 9b3ce8226..2febd10d1 100644 --- a/packages/nodejs/test/e2e/kernel-worker.test.ts +++ b/packages/nodejs/test/e2e/kernel-worker.test.ts @@ -61,7 +61,7 @@ describe('Kernel Worker', () => { await launchTestVats(); await Promise.all(testVatIds.map(kernel.restartVat.bind(kernel))); expect(kernel.getVatIds().sort()).toStrictEqual(testVatIds); - }); + }, 5000); it('terminates all vats', async () => { await launchTestVats(); diff --git a/packages/nodejs/test/e2e/vat-worker.test.ts b/packages/nodejs/test/e2e/vat-worker.test.ts new file mode 100644 index 000000000..40f50162e --- /dev/null +++ b/packages/nodejs/test/e2e/vat-worker.test.ts @@ -0,0 +1,37 @@ +import '@ocap/shims/endoify'; + +import type { VatId } from '@ocap/kernel'; +import { makePromiseKitMock } from '@ocap/test-utils'; +import { makeCounter } from '@ocap/utils'; +import { describe, expect, it } from 'vitest'; +import { Worker as NodeWorker } from 'worker_threads'; + +import { getTestWorkerFile } from '../workers'; + +const { makePromiseKit } = makePromiseKitMock(); + +describe('NodejsVatWorkerService', () => { + let testWorkerFile: string; + const vatIdCounter = makeCounter(); + const getTestVatId = (): VatId => `v${vatIdCounter()}`; + + describe('hello-world', () => { + testWorkerFile = getTestWorkerFile('hello-world'); + it('can start in a Node.js worker', async () => { + const vatId = getTestVatId(); + const worker = new NodeWorker(testWorkerFile, { + env: { + NODE_VAT_ID: vatId, + }, + }); + const { resolve, reject, promise } = makePromiseKit(); + worker.once('online', (error: Error) => { + if (error) { + reject(error); + } + resolve(vatId); + }); + expect(await promise).toStrictEqual(vatId); + }); + }); +}); diff --git a/packages/nodejs/test/workers/hello-world.js b/packages/nodejs/test/workers/hello-world.js new file mode 100644 index 000000000..216a0d654 --- /dev/null +++ b/packages/nodejs/test/workers/hello-world.js @@ -0,0 +1,2 @@ +console.debug('hello world computer', process.env.NODE_VAT_ID); +export {}; diff --git a/packages/nodejs/tsconfig.json b/packages/nodejs/tsconfig.json index 0490ddc26..74ba30066 100644 --- a/packages/nodejs/tsconfig.json +++ b/packages/nodejs/tsconfig.json @@ -23,6 +23,7 @@ "./test/**/*.ts", "./vitest.config.ts", "./vitest.config.e2e.ts", - "../../types" + "../../types", + "./test/workers/*.js" ] } diff --git a/vitest.config.ts b/vitest.config.ts index bbc9c69ad..6ac1bf107 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -80,10 +80,10 @@ export default defineConfig({ lines: 44.2, }, 'packages/nodejs/**': { - statements: 46.75, - functions: 47.61, + statements: 19.23, + functions: 19.04, branches: 35.29, - lines: 46.75, + lines: 19.23, }, 'packages/shims/**': { statements: 0, From a5b9154434cdcd0363ba272054ace60f11f32c68 Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Mon, 27 Jan 2025 13:56:41 -0600 Subject: [PATCH 25/27] test(nodejs): reallocate coverage between unit and e2e phases --- .../src/kernel/VatWorkerService.test.ts | 71 ++++++++------- .../nodejs/src/kernel/VatWorkerService.ts | 1 - .../nodejs/test/e2e/VatWorkerService.test.ts | 91 +++++++++++++++++++ 3 files changed, 130 insertions(+), 33 deletions(-) create mode 100644 packages/nodejs/test/e2e/VatWorkerService.test.ts diff --git a/packages/nodejs/src/kernel/VatWorkerService.test.ts b/packages/nodejs/src/kernel/VatWorkerService.test.ts index 58e35ec97..e786b5855 100644 --- a/packages/nodejs/src/kernel/VatWorkerService.test.ts +++ b/packages/nodejs/src/kernel/VatWorkerService.test.ts @@ -1,12 +1,31 @@ import '@ocap/shims/endoify'; import type { VatId } from '@ocap/kernel'; -import { NodeWorkerDuplexStream } from '@ocap/streams'; import { makeCounter } from '@ocap/utils'; import { describe, expect, it, vi } from 'vitest'; import { NodejsVatWorkerService } from './VatWorkerService.js'; -import { getTestWorkerFile } from '../../test/workers'; + +const mocks = vi.hoisted(() => ({ + worker: { + once: (_: string, callback: () => unknown) => { + callback(); + }, + terminate: vi.fn(async () => undefined), + }, + stream: { + synchronize: vi.fn(async () => undefined).mockResolvedValue(undefined), + return: vi.fn(async () => ({})), + }, +})); + +vi.mock('@ocap/streams', () => ({ + NodeWorkerDuplexStream: vi.fn(() => mocks.stream), +})); + +vi.mock('node:worker_threads', () => ({ + Worker: vi.fn(() => mocks.worker), +})); describe('NodejsVatWorkerService', () => { it('constructs an instance without any arguments', () => { @@ -14,37 +33,25 @@ describe('NodejsVatWorkerService', () => { expect(instance).toBeInstanceOf(NodejsVatWorkerService); }); - const testWorkerFile = getTestWorkerFile('stream-sync'); + const workerFilePath = 'unused'; const vatIdCounter = makeCounter(); const getTestVatId = (): VatId => `v${vatIdCounter()}`; describe('launch', () => { - it.todo( - 'creates a NodeWorker and returns a NodeWorkerDuplexStream', - async () => { - const service = new NodejsVatWorkerService({ - workerFilePath: testWorkerFile, - }); - const testVatId: VatId = getTestVatId(); - const stream = await service.launch(testVatId); - - expect(stream).toBeInstanceOf(NodeWorkerDuplexStream); - }, - ); - - it.todo('rejects if synchronize fails', async () => { - const rejected = 'test-reject-value'; + it('creates a NodeWorker and returns a NodeWorkerDuplexStream', async () => { + const service = new NodejsVatWorkerService({ + workerFilePath, + }); + const testVatId: VatId = getTestVatId(); + const stream = await service.launch(testVatId); - vi.doMock('@ocap/streams', () => ({ - NodeWorkerDuplexStream: vi.fn().mockImplementation(() => ({ - synchronize: vi.fn().mockRejectedValue(rejected), - })), - })); - vi.resetModules(); - const NVWS = (await import('./VatWorkerService.js')) - .NodejsVatWorkerService; + expect(stream).toStrictEqual(mocks.stream); + }); - const service = new NVWS({ workerFilePath: testWorkerFile }); + it('rejects if synchronize fails', async () => { + const rejected = 'test-reject-value'; + mocks.stream.synchronize.mockRejectedValue(rejected); + const service = new NodejsVatWorkerService({ workerFilePath }); const testVatId: VatId = getTestVatId(); await expect(async () => await service.launch(testVatId)).rejects.toThrow( rejected, @@ -52,10 +59,10 @@ describe('NodejsVatWorkerService', () => { }); }); - describe.todo('terminate', () => { + describe('terminate', () => { it('terminates the target vat', async () => { const service = new NodejsVatWorkerService({ - workerFilePath: testWorkerFile, + workerFilePath, }); const testVatId: VatId = getTestVatId(); @@ -68,7 +75,7 @@ describe('NodejsVatWorkerService', () => { it('throws when terminating an unknown vat', async () => { const service = new NodejsVatWorkerService({ - workerFilePath: testWorkerFile, + workerFilePath, }); const testVatId: VatId = getTestVatId(); @@ -78,10 +85,10 @@ describe('NodejsVatWorkerService', () => { }); }); - describe.todo('terminateAll', () => { + describe('terminateAll', () => { it('terminates all vats', async () => { const service = new NodejsVatWorkerService({ - workerFilePath: testWorkerFile, + workerFilePath, }); const vatIds: VatId[] = [getTestVatId(), getTestVatId(), getTestVatId()]; diff --git a/packages/nodejs/src/kernel/VatWorkerService.ts b/packages/nodejs/src/kernel/VatWorkerService.ts index 99d147d33..6ddcecc7c 100644 --- a/packages/nodejs/src/kernel/VatWorkerService.ts +++ b/packages/nodejs/src/kernel/VatWorkerService.ts @@ -56,7 +56,6 @@ export class NodejsVatWorkerService implements VatWorkerService { NODE_VAT_ID: vatId, }, }); - worker.stdout.pipe(process.stdout); worker.once('online', () => { const stream = new NodeWorkerDuplexStream( worker, diff --git a/packages/nodejs/test/e2e/VatWorkerService.test.ts b/packages/nodejs/test/e2e/VatWorkerService.test.ts new file mode 100644 index 000000000..e776c8f77 --- /dev/null +++ b/packages/nodejs/test/e2e/VatWorkerService.test.ts @@ -0,0 +1,91 @@ +import '@ocap/shims/endoify'; + +import type { VatId } from '@ocap/kernel'; +import { NodeWorkerDuplexStream } from '@ocap/streams'; +import { makeCounter } from '@ocap/utils'; +import { describe, expect, it, vi } from 'vitest'; + +import { NodejsVatWorkerService } from '../../src/kernel/VatWorkerService.js'; +import { getTestWorkerFile } from '../workers'; + +describe('NodejsVatWorkerService', () => { + const testWorkerFile = getTestWorkerFile('stream-sync'); + const vatIdCounter = makeCounter(); + const getTestVatId = (): VatId => `v${vatIdCounter()}`; + + describe('launch', () => { + it('creates a NodeWorker and returns a NodeWorkerDuplexStream', async () => { + const service = new NodejsVatWorkerService({ + workerFilePath: testWorkerFile, + }); + const testVatId: VatId = getTestVatId(); + const stream = await service.launch(testVatId); + + expect(stream).toBeInstanceOf(NodeWorkerDuplexStream); + }); + + it('rejects if synchronize fails', async () => { + const rejected = 'test-reject-value'; + + vi.doMock('@ocap/streams', () => ({ + NodeWorkerDuplexStream: vi.fn().mockImplementation(() => ({ + synchronize: vi.fn(() => 'no').mockRejectedValue(rejected), + })), + })); + vi.resetModules(); + const NVWS = (await import('../../src/kernel/VatWorkerService.js')) + .NodejsVatWorkerService; + + const service = new NVWS({ workerFilePath: testWorkerFile }); + const testVatId: VatId = getTestVatId(); + await expect(async () => await service.launch(testVatId)).rejects.toThrow( + rejected, + ); + }); + }); + + describe('terminate', () => { + it('terminates the target vat', async () => { + const service = new NodejsVatWorkerService({ + workerFilePath: testWorkerFile, + }); + const testVatId: VatId = getTestVatId(); + + await service.launch(testVatId); + expect(service.workers.has(testVatId)).toBe(true); + + await service.terminate(testVatId); + expect(service.workers.has(testVatId)).toBe(false); + }); + + it('throws when terminating an unknown vat', async () => { + const service = new NodejsVatWorkerService({ + workerFilePath: testWorkerFile, + }); + const testVatId: VatId = getTestVatId(); + + await expect( + async () => await service.terminate(testVatId), + ).rejects.toThrow(/No worker found/u); + }); + }); + + describe('terminateAll', () => { + it('terminates all vats', async () => { + const service = new NodejsVatWorkerService({ + workerFilePath: testWorkerFile, + }); + const vatIds: VatId[] = [getTestVatId(), getTestVatId(), getTestVatId()]; + + await Promise.all( + vatIds.map(async (vatId) => await service.launch(vatId)), + ); + + expect(Array.from(service.workers.values())).toHaveLength(vatIds.length); + + await service.terminateAll(); + + expect(Array.from(service.workers.values())).toHaveLength(0); + }); + }); +}); From 7fc4b8ee28a24812aba943aa5cb17103df98fa4c Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Mon, 27 Jan 2025 14:05:27 -0600 Subject: [PATCH 26/27] thresholds --- vitest.config.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vitest.config.ts b/vitest.config.ts index 6ac1bf107..bbc9c69ad 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -80,10 +80,10 @@ export default defineConfig({ lines: 44.2, }, 'packages/nodejs/**': { - statements: 19.23, - functions: 19.04, + statements: 46.75, + functions: 47.61, branches: 35.29, - lines: 19.23, + lines: 46.75, }, 'packages/shims/**': { statements: 0, From d95a4ff99f47b76faa9a68f3eb3c3988d27d01aa Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Tue, 28 Jan 2025 10:12:24 -0600 Subject: [PATCH 27/27] tidy files --- packages/nodejs/src/env/kernel-worker-trusted-prelude.js | 2 -- packages/nodejs/test/e2e/VatWorkerService.test.ts | 2 +- packages/nodejs/test/e2e/vat-worker.test.ts | 2 +- packages/nodejs/test/get-test-worker.ts | 8 ++++++++ packages/nodejs/test/workers/index.ts | 2 -- 5 files changed, 10 insertions(+), 6 deletions(-) delete mode 100644 packages/nodejs/src/env/kernel-worker-trusted-prelude.js create mode 100644 packages/nodejs/test/get-test-worker.ts delete mode 100644 packages/nodejs/test/workers/index.ts diff --git a/packages/nodejs/src/env/kernel-worker-trusted-prelude.js b/packages/nodejs/src/env/kernel-worker-trusted-prelude.js deleted file mode 100644 index dad2cb379..000000000 --- a/packages/nodejs/src/env/kernel-worker-trusted-prelude.js +++ /dev/null @@ -1,2 +0,0 @@ -// eslint-disable-next-line import-x/no-unresolved -import './endoify.js'; diff --git a/packages/nodejs/test/e2e/VatWorkerService.test.ts b/packages/nodejs/test/e2e/VatWorkerService.test.ts index e776c8f77..c948713d6 100644 --- a/packages/nodejs/test/e2e/VatWorkerService.test.ts +++ b/packages/nodejs/test/e2e/VatWorkerService.test.ts @@ -6,7 +6,7 @@ import { makeCounter } from '@ocap/utils'; import { describe, expect, it, vi } from 'vitest'; import { NodejsVatWorkerService } from '../../src/kernel/VatWorkerService.js'; -import { getTestWorkerFile } from '../workers'; +import { getTestWorkerFile } from '../get-test-worker.js'; describe('NodejsVatWorkerService', () => { const testWorkerFile = getTestWorkerFile('stream-sync'); diff --git a/packages/nodejs/test/e2e/vat-worker.test.ts b/packages/nodejs/test/e2e/vat-worker.test.ts index 40f50162e..8e1f7c2e6 100644 --- a/packages/nodejs/test/e2e/vat-worker.test.ts +++ b/packages/nodejs/test/e2e/vat-worker.test.ts @@ -6,7 +6,7 @@ import { makeCounter } from '@ocap/utils'; import { describe, expect, it } from 'vitest'; import { Worker as NodeWorker } from 'worker_threads'; -import { getTestWorkerFile } from '../workers'; +import { getTestWorkerFile } from '../get-test-worker.js'; const { makePromiseKit } = makePromiseKitMock(); diff --git a/packages/nodejs/test/get-test-worker.ts b/packages/nodejs/test/get-test-worker.ts new file mode 100644 index 000000000..2ad3e594c --- /dev/null +++ b/packages/nodejs/test/get-test-worker.ts @@ -0,0 +1,8 @@ +/** + * Get a path for a node worker file from its name. + * + * @param name - The name of the test worker file to retrieve. + * @returns The path for a test worker file. + */ +export const getTestWorkerFile = (name: string): string => + new URL(`./workers/${name}.js`, import.meta.url).pathname; diff --git a/packages/nodejs/test/workers/index.ts b/packages/nodejs/test/workers/index.ts deleted file mode 100644 index 0fd6a4a00..000000000 --- a/packages/nodejs/test/workers/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export const getTestWorkerFile = (name: string): string => - new URL(`./${name}.js`, import.meta.url).pathname;