From 2b4ae6fc241943bf8a4cff04a019b387a346d517 Mon Sep 17 00:00:00 2001 From: Chip Morningstar Date: Thu, 6 Mar 2025 15:37:24 -0800 Subject: [PATCH] feat: Implement persistent storage for vats. Closes #431. --- packages/extension/src/iframe.ts | 2 - .../kernel-integration/command-registry.ts | 28 ++- .../handle-panel-message.test.ts | 49 ++-- .../handle-panel-message.ts | 13 +- .../handlers/clear-state.test.ts | 8 +- .../handlers/execute-db-query.test.ts | 18 +- .../handlers/execute-db-query.ts | 6 +- .../handlers/get-status.test.ts | 6 +- .../handlers/launch-vat.test.ts | 8 +- .../kernel-integration/handlers/launch-vat.ts | 4 +- .../handlers/reload-config.test.ts | 8 +- .../handlers/restart-vat.test.ts | 8 +- .../handlers/restart-vat.ts | 4 +- .../handlers/send-vat-command.test.ts | 12 +- .../handlers/send-vat-command.ts | 4 +- .../handlers/terminate-all-vats.test.ts | 12 +- .../handlers/terminate-vat.test.ts | 12 +- .../handlers/terminate-vat.ts | 4 +- .../handlers/update-cluster-config.test.ts | 6 +- .../handlers/update-cluster-config.ts | 4 +- .../src/kernel-integration/kernel-worker.ts | 25 +- .../middlewares/logging.test.ts | 18 +- packages/kernel-test/package.json | 2 + packages/kernel-test/src/vatstore-vat.js | 44 ++++ packages/kernel-test/src/vatstore.test.ts | 218 ++++++++++++++++++ packages/kernel/src/Kernel.test.ts | 50 ++-- packages/kernel/src/Kernel.ts | 71 +++--- packages/kernel/src/VatHandle.test.ts | 16 +- packages/kernel/src/VatHandle.ts | 42 ++-- packages/kernel/src/VatKVStore.test.ts | 46 ++++ packages/kernel/src/VatKVStore.ts | 60 +++++ packages/kernel/src/VatSupervisor.ts | 38 ++- packages/kernel/src/index.ts | 1 + packages/kernel/src/messages/vat.ts | 11 +- .../kernel/src/store/kernel-store.test.ts | 26 ++- packages/kernel/src/store/kernel-store.ts | 47 +++- packages/kernel/src/types.ts | 10 + packages/kernel/test/storage.ts | 66 +++++- .../nodejs/src/kernel/make-kernel.test.ts | 6 +- packages/nodejs/src/kernel/make-kernel.ts | 15 +- packages/nodejs/src/vat/vat-worker.ts | 2 - packages/store/src/index.ts | 2 +- packages/store/src/sqlite/common.test.ts | 10 + packages/store/src/sqlite/common.ts | 37 +++ packages/store/src/sqlite/nodejs.test.ts | 69 ++++-- packages/store/src/sqlite/nodejs.ts | 122 ++++++++-- packages/store/src/sqlite/wasm.test.ts | 106 +++++++-- packages/store/src/sqlite/wasm.ts | 158 +++++++++++-- packages/store/src/types.ts | 18 +- vitest.config.ts | 16 +- yarn.lock | 2 + 51 files changed, 1226 insertions(+), 344 deletions(-) create mode 100644 packages/kernel-test/src/vatstore-vat.js create mode 100644 packages/kernel-test/src/vatstore.test.ts create mode 100644 packages/kernel/src/VatKVStore.test.ts create mode 100644 packages/kernel/src/VatKVStore.ts diff --git a/packages/extension/src/iframe.ts b/packages/extension/src/iframe.ts index 4b9af57ac..61d3d3eac 100644 --- a/packages/extension/src/iframe.ts +++ b/packages/extension/src/iframe.ts @@ -1,6 +1,5 @@ import { isVatCommand, VatSupervisor } from '@ocap/kernel'; import type { VatCommand, VatCommandReply } from '@ocap/kernel'; -import { makeSQLKVStore } from '@ocap/store/sqlite/wasm'; import { MessagePortDuplexStream, receiveMessagePort, @@ -29,6 +28,5 @@ async function main(): Promise { new VatSupervisor({ id: vatId, commandStream, - makeKVStore: makeSQLKVStore, }); } diff --git a/packages/extension/src/kernel-integration/command-registry.ts b/packages/extension/src/kernel-integration/command-registry.ts index 1b16b4e37..d89e00414 100644 --- a/packages/extension/src/kernel-integration/command-registry.ts +++ b/packages/extension/src/kernel-integration/command-registry.ts @@ -2,7 +2,7 @@ import { assert } from '@metamask/superstruct'; import type { Infer, Struct } from '@metamask/superstruct'; import type { Json } from '@metamask/utils'; import type { Kernel } from '@ocap/kernel'; -import type { KVStore } from '@ocap/store'; +import type { KernelDatabase } from '@ocap/store'; import type { KernelControlMethod } from './handlers/index.ts'; import type { KernelCommandPayloadStructs } from './messages.ts'; @@ -25,20 +25,28 @@ export type CommandHandler = { * Implementation of the command. * * @param kernel - The kernel instance. - * @param kvStore - The KV store instance. + * @param kernelDatabase - The kernel database instance. * @param params - The parameters. * @returns The result of the command. */ implementation: ( kernel: Kernel, - kvStore: KVStore, + kernelDatabase: KernelDatabase, params: CommandParams[Method], ) => Promise; }; export type Middleware = ( - next: (kernel: Kernel, kvStore: KVStore, params: unknown) => Promise, -) => (kernel: Kernel, kvStore: KVStore, params: unknown) => Promise; + next: ( + kernel: Kernel, + kernelDatabase: KernelDatabase, + params: unknown, + ) => Promise, +) => ( + kernel: Kernel, + kernelDatabase: KernelDatabase, + params: unknown, +) => Promise; /** * A registry for kernel commands. @@ -77,14 +85,14 @@ export class KernelCommandRegistry { * Execute a command. * * @param kernel - The kernel. - * @param kvStore - The KV store. + * @param kernelDatabase - The kernel database. * @param method - The method name. * @param params - The parameters. * @returns The result. */ async execute( kernel: Kernel, - kvStore: KVStore, + kernelDatabase: KernelDatabase, method: Method, params: CommandParams[Method], ): Promise { @@ -95,11 +103,11 @@ export class KernelCommandRegistry { let chain = async ( k: Kernel, - kv: KVStore, + kdb: KernelDatabase, param: unknown, ): Promise => { assert(param, handler.schema); - return handler.implementation(k, kv, param); + return handler.implementation(k, kdb, param); }; // Apply middlewares in reverse order @@ -107,6 +115,6 @@ export class KernelCommandRegistry { chain = middleware(chain); } - return chain(kernel, kvStore, params); + return chain(kernel, kernelDatabase, params); } } diff --git a/packages/extension/src/kernel-integration/handle-panel-message.test.ts b/packages/extension/src/kernel-integration/handle-panel-message.test.ts index 307986ed2..c0de6d4ac 100644 --- a/packages/extension/src/kernel-integration/handle-panel-message.test.ts +++ b/packages/extension/src/kernel-integration/handle-panel-message.test.ts @@ -1,5 +1,5 @@ import type { Kernel, KernelCommand, VatId, VatConfig } from '@ocap/kernel'; -import type { KVStore } from '@ocap/store'; +import type { KernelDatabase } from '@ocap/store'; import { setupOcapKernelMock } from '@ocap/test-utils'; import { describe, it, expect, vi, beforeEach } from 'vitest'; @@ -17,20 +17,23 @@ const { setMockBehavior, resetMocks } = setupOcapKernelMock(); describe('handlePanelMessage', () => { let mockKernel: Kernel; - let mockKVStore: KVStore; + let mockKernelDatabase: KernelDatabase; beforeEach(() => { vi.resetModules(); resetMocks(); - mockKVStore = { - get: vi.fn(), - getRequired: vi.fn(), - getNextKey: vi.fn(), - set: vi.fn(), - delete: vi.fn(), + mockKernelDatabase = { + kernelKVStore: { + get: vi.fn(), + getRequired: vi.fn(), + getNextKey: vi.fn(), + set: vi.fn(), + delete: vi.fn(), + }, clear: vi.fn(), executeQuery: vi.fn(), + makeVatStore: vi.fn(), }; // Create mock kernel @@ -74,7 +77,7 @@ describe('handlePanelMessage', () => { const response = await handlePanelMessage( mockKernel, - mockKVStore, + mockKernelDatabase, message, ); @@ -104,7 +107,7 @@ describe('handlePanelMessage', () => { const response = await handlePanelMessage( mockKernel, - mockKVStore, + mockKernelDatabase, message, ); @@ -132,7 +135,7 @@ describe('handlePanelMessage', () => { const response = await handlePanelMessage( mockKernel, - mockKVStore, + mockKernelDatabase, message, ); @@ -160,7 +163,7 @@ describe('handlePanelMessage', () => { const response = await handlePanelMessage( mockKernel, - mockKVStore, + mockKernelDatabase, message, ); @@ -188,7 +191,7 @@ describe('handlePanelMessage', () => { const response = await handlePanelMessage( mockKernel, - mockKVStore, + mockKernelDatabase, message, ); @@ -214,7 +217,7 @@ describe('handlePanelMessage', () => { const response = await handlePanelMessage( mockKernel, - mockKVStore, + mockKernelDatabase, message, ); @@ -242,7 +245,7 @@ describe('handlePanelMessage', () => { const response = await handlePanelMessage( mockKernel, - mockKVStore, + mockKernelDatabase, message, ); @@ -289,7 +292,7 @@ describe('handlePanelMessage', () => { const response = await handlePanelMessage( mockKernel, - mockKVStore, + mockKernelDatabase, message, ); @@ -324,7 +327,7 @@ describe('handlePanelMessage', () => { const response = await handlePanelMessage( mockKernel, - mockKVStore, + mockKernelDatabase, message, ); @@ -355,7 +358,7 @@ describe('handlePanelMessage', () => { const response = await handlePanelMessage( mockKernel, - mockKVStore, + mockKernelDatabase, message, ); @@ -382,7 +385,7 @@ describe('handlePanelMessage', () => { const response = await handlePanelMessage( mockKernel, - mockKVStore, + mockKernelDatabase, message, ); @@ -410,7 +413,7 @@ describe('handlePanelMessage', () => { const response = await handlePanelMessage( mockKernel, - mockKVStore, + mockKernelDatabase, message, ); @@ -426,7 +429,7 @@ describe('handlePanelMessage', () => { const response2 = await handlePanelMessage( mockKernel, - mockKVStore, + mockKernelDatabase, message, ); @@ -453,7 +456,7 @@ describe('handlePanelMessage', () => { const response = await handlePanelMessage( mockKernel, - mockKVStore, + mockKernelDatabase, message, ); @@ -481,7 +484,7 @@ describe('handlePanelMessage', () => { const response = await handlePanelMessage( mockKernel, - mockKVStore, + mockKernelDatabase, message, ); diff --git a/packages/extension/src/kernel-integration/handle-panel-message.ts b/packages/extension/src/kernel-integration/handle-panel-message.ts index 9e519ca14..e7a74f033 100644 --- a/packages/extension/src/kernel-integration/handle-panel-message.ts +++ b/packages/extension/src/kernel-integration/handle-panel-message.ts @@ -1,5 +1,5 @@ import type { Kernel } from '@ocap/kernel'; -import type { KVStore } from '@ocap/store'; +import type { KernelDatabase } from '@ocap/store'; import { makeLogger } from '@ocap/utils'; import { KernelCommandRegistry } from './command-registry.ts'; @@ -23,19 +23,24 @@ handlers.forEach((handler) => * Handles a message from the panel. * * @param kernel - The kernel instance. - * @param kvStore - The KV store instance. + * @param kernelDatabase - The kernel database instance. * @param message - The message to handle. * @returns The reply to the message. */ export async function handlePanelMessage( kernel: Kernel, - kvStore: KVStore, + kernelDatabase: KernelDatabase, message: KernelControlCommand, ): Promise { const { method, params } = message.payload; try { - const result = await registry.execute(kernel, kvStore, method, params); + const result = await registry.execute( + kernel, + kernelDatabase, + method, + params, + ); return { id: message.id, diff --git a/packages/extension/src/kernel-integration/handlers/clear-state.test.ts b/packages/extension/src/kernel-integration/handlers/clear-state.test.ts index 76ecd6cdd..13d683b3c 100644 --- a/packages/extension/src/kernel-integration/handlers/clear-state.test.ts +++ b/packages/extension/src/kernel-integration/handlers/clear-state.test.ts @@ -1,5 +1,5 @@ import type { Kernel } from '@ocap/kernel'; -import type { KVStore } from '@ocap/store'; +import type { KernelDatabase } from '@ocap/store'; import { describe, it, expect, vi } from 'vitest'; import { clearStateHandler } from './clear-state.ts'; @@ -9,7 +9,7 @@ describe('clearStateHandler', () => { reset: vi.fn().mockResolvedValue(undefined), } as unknown as Kernel; - const mockKVStore = {} as unknown as KVStore; + const mockKernelDatabase = {} as unknown as KernelDatabase; it('should have the correct method', () => { expect(clearStateHandler.method).toBe('clearState'); @@ -22,7 +22,7 @@ describe('clearStateHandler', () => { it('should call kernel.reset() and return null', async () => { const result = await clearStateHandler.implementation( mockKernel, - mockKVStore, + mockKernelDatabase, null, ); expect(mockKernel.reset).toHaveBeenCalledOnce(); @@ -33,7 +33,7 @@ describe('clearStateHandler', () => { const error = new Error('Reset failed'); vi.mocked(mockKernel.reset).mockRejectedValueOnce(error); await expect( - clearStateHandler.implementation(mockKernel, mockKVStore, null), + clearStateHandler.implementation(mockKernel, mockKernelDatabase, null), ).rejects.toThrow(error); }); }); diff --git a/packages/extension/src/kernel-integration/handlers/execute-db-query.test.ts b/packages/extension/src/kernel-integration/handlers/execute-db-query.test.ts index 4c424e59b..590e9690a 100644 --- a/packages/extension/src/kernel-integration/handlers/execute-db-query.test.ts +++ b/packages/extension/src/kernel-integration/handlers/execute-db-query.test.ts @@ -1,13 +1,13 @@ import type { Kernel } from '@ocap/kernel'; -import type { KVStore } from '@ocap/store'; +import type { KernelDatabase } from '@ocap/store'; import { describe, it, expect, vi } from 'vitest'; import { executeDBQueryHandler } from './execute-db-query.ts'; describe('executeDBQueryHandler', () => { - const mockKVStore = { + const mockKernelDatabase = { executeQuery: vi.fn(() => 'test'), - } as unknown as KVStore; + } as unknown as KernelDatabase; const mockKernel = {} as unknown as Kernel; @@ -19,19 +19,23 @@ describe('executeDBQueryHandler', () => { const params = { sql: 'SELECT * FROM test' }; const result = await executeDBQueryHandler.implementation( mockKernel, - mockKVStore, + mockKernelDatabase, params, ); - expect(mockKVStore.executeQuery).toHaveBeenCalledWith(params.sql); + expect(mockKernelDatabase.executeQuery).toHaveBeenCalledWith(params.sql); expect(result).toBe('test'); }); it('should propagate errors from executeQuery', async () => { const error = new Error('Query failed'); - vi.mocked(mockKVStore.executeQuery).mockRejectedValueOnce(error); + vi.mocked(mockKernelDatabase.executeQuery).mockRejectedValueOnce(error); const params = { sql: 'SELECT * FROM test' }; await expect( - executeDBQueryHandler.implementation(mockKernel, mockKVStore, params), + executeDBQueryHandler.implementation( + mockKernel, + mockKernelDatabase, + params, + ), ).rejects.toThrow(error); }); }); diff --git a/packages/extension/src/kernel-integration/handlers/execute-db-query.ts b/packages/extension/src/kernel-integration/handlers/execute-db-query.ts index 582b70075..619741871 100644 --- a/packages/extension/src/kernel-integration/handlers/execute-db-query.ts +++ b/packages/extension/src/kernel-integration/handlers/execute-db-query.ts @@ -1,6 +1,6 @@ import type { Json } from '@metamask/utils'; import type { Kernel } from '@ocap/kernel'; -import type { KVStore } from '@ocap/store'; +import type { KernelDatabase } from '@ocap/store'; import type { CommandHandler, CommandParams } from '../command-registry.ts'; import { KernelCommandPayloadStructs } from '../messages.ts'; @@ -10,9 +10,9 @@ export const executeDBQueryHandler: CommandHandler<'executeDBQuery'> = { schema: KernelCommandPayloadStructs.executeDBQuery.schema.params, implementation: async ( _kernel: Kernel, - kvStore: KVStore, + kdb: KernelDatabase, params: CommandParams['executeDBQuery'], ): Promise => { - return kvStore.executeQuery(params.sql); + return kdb.executeQuery(params.sql); }, }; diff --git a/packages/extension/src/kernel-integration/handlers/get-status.test.ts b/packages/extension/src/kernel-integration/handlers/get-status.test.ts index 656e14e05..5f0a5b2ad 100644 --- a/packages/extension/src/kernel-integration/handlers/get-status.test.ts +++ b/packages/extension/src/kernel-integration/handlers/get-status.test.ts @@ -1,5 +1,5 @@ import type { Kernel } from '@ocap/kernel'; -import type { KVStore } from '@ocap/store'; +import type { KernelDatabase } from '@ocap/store'; import { describe, it, expect, vi } from 'vitest'; import { getStatusHandler } from './get-status.ts'; @@ -16,7 +16,7 @@ describe('getStatusHandler', () => { getVats: vi.fn(() => mockVats), } as unknown as Kernel; - const mockKVStore = {} as unknown as KVStore; + const mockKernelDatabase = {} as unknown as KernelDatabase; it('should have the correct method', () => { expect(getStatusHandler.method).toBe('getStatus'); @@ -29,7 +29,7 @@ describe('getStatusHandler', () => { it('should return vats status and cluster config', async () => { const result = await getStatusHandler.implementation( mockKernel, - mockKVStore, + mockKernelDatabase, null, ); expect(mockKernel.getVats).toHaveBeenCalledOnce(); diff --git a/packages/extension/src/kernel-integration/handlers/launch-vat.test.ts b/packages/extension/src/kernel-integration/handlers/launch-vat.test.ts index e5da3bb10..433a049a9 100644 --- a/packages/extension/src/kernel-integration/handlers/launch-vat.test.ts +++ b/packages/extension/src/kernel-integration/handlers/launch-vat.test.ts @@ -1,5 +1,5 @@ import type { Kernel } from '@ocap/kernel'; -import type { KVStore } from '@ocap/store'; +import type { KernelDatabase } from '@ocap/store'; import { describe, it, expect, vi } from 'vitest'; import { launchVatHandler } from './launch-vat.ts'; @@ -9,7 +9,7 @@ describe('launchVatHandler', () => { launchVat: vi.fn().mockResolvedValue(undefined), } as unknown as Kernel; - const mockKVStore = {} as unknown as KVStore; + const mockKernelDatabase = {} as unknown as KernelDatabase; it('should have the correct method', () => { expect(launchVatHandler.method).toBe('launchVat'); @@ -23,7 +23,7 @@ describe('launchVatHandler', () => { const params = { sourceSpec: 'test.js' }; const result = await launchVatHandler.implementation( mockKernel, - mockKVStore, + mockKernelDatabase, params, ); expect(mockKernel.launchVat).toHaveBeenCalledWith(params); @@ -35,7 +35,7 @@ describe('launchVatHandler', () => { vi.mocked(mockKernel.launchVat).mockRejectedValueOnce(error); const params = { sourceSpec: 'test.js' }; await expect( - launchVatHandler.implementation(mockKernel, mockKVStore, params), + launchVatHandler.implementation(mockKernel, mockKernelDatabase, params), ).rejects.toThrow(error); }); }); diff --git a/packages/extension/src/kernel-integration/handlers/launch-vat.ts b/packages/extension/src/kernel-integration/handlers/launch-vat.ts index c030cedf0..cff4fde8c 100644 --- a/packages/extension/src/kernel-integration/handlers/launch-vat.ts +++ b/packages/extension/src/kernel-integration/handlers/launch-vat.ts @@ -1,6 +1,6 @@ import type { Json } from '@metamask/utils'; import type { Kernel } from '@ocap/kernel'; -import type { KVStore } from '@ocap/store'; +import type { KernelDatabase } from '@ocap/store'; import type { CommandHandler, CommandParams } from '../command-registry.ts'; import { KernelCommandPayloadStructs } from '../messages.ts'; @@ -10,7 +10,7 @@ export const launchVatHandler: CommandHandler<'launchVat'> = { schema: KernelCommandPayloadStructs.launchVat.schema.params, implementation: async ( kernel: Kernel, - _kvStore: KVStore, + _kdb: KernelDatabase, params: CommandParams['launchVat'], ): Promise => { await kernel.launchVat(params); diff --git a/packages/extension/src/kernel-integration/handlers/reload-config.test.ts b/packages/extension/src/kernel-integration/handlers/reload-config.test.ts index ba923de1c..622668dde 100644 --- a/packages/extension/src/kernel-integration/handlers/reload-config.test.ts +++ b/packages/extension/src/kernel-integration/handlers/reload-config.test.ts @@ -1,5 +1,5 @@ import type { Kernel } from '@ocap/kernel'; -import type { KVStore } from '@ocap/store'; +import type { KernelDatabase } from '@ocap/store'; import { describe, it, expect, vi, beforeEach } from 'vitest'; import { reloadConfigHandler } from './reload-config.ts'; @@ -9,7 +9,7 @@ describe('reloadConfigHandler', () => { reload: vi.fn().mockResolvedValue(undefined), } as Partial; - const mockKVStore = {} as KVStore; + const mockKernelDatabase = {} as KernelDatabase; beforeEach(() => { vi.clearAllMocks(); @@ -18,7 +18,7 @@ describe('reloadConfigHandler', () => { it('should call kernel.reload() and return null', async () => { const result = await reloadConfigHandler.implementation( mockKernel as Kernel, - mockKVStore, + mockKernelDatabase, null, ); @@ -41,7 +41,7 @@ describe('reloadConfigHandler', () => { await expect( reloadConfigHandler.implementation( mockKernel as Kernel, - mockKVStore, + mockKernelDatabase, null, ), ).rejects.toThrow(error); diff --git a/packages/extension/src/kernel-integration/handlers/restart-vat.test.ts b/packages/extension/src/kernel-integration/handlers/restart-vat.test.ts index bc185cdc2..c5eb0f137 100644 --- a/packages/extension/src/kernel-integration/handlers/restart-vat.test.ts +++ b/packages/extension/src/kernel-integration/handlers/restart-vat.test.ts @@ -1,5 +1,5 @@ import type { Kernel } from '@ocap/kernel'; -import type { KVStore } from '@ocap/store'; +import type { KernelDatabase } from '@ocap/store'; import { describe, it, expect, vi } from 'vitest'; import { restartVatHandler } from './restart-vat.ts'; @@ -9,7 +9,7 @@ describe('restartVatHandler', () => { restartVat: vi.fn().mockResolvedValue(undefined), } as unknown as Kernel; - const mockKVStore = {} as unknown as KVStore; + const mockKernelDatabase = {} as unknown as KernelDatabase; it('should have the correct method', () => { expect(restartVatHandler.method).toBe('restartVat'); @@ -23,7 +23,7 @@ describe('restartVatHandler', () => { const params = { id: 'v0' } as const; const result = await restartVatHandler.implementation( mockKernel, - mockKVStore, + mockKernelDatabase, params, ); expect(mockKernel.restartVat).toHaveBeenCalledWith(params.id); @@ -35,7 +35,7 @@ describe('restartVatHandler', () => { vi.mocked(mockKernel.restartVat).mockRejectedValueOnce(error); const params = { id: 'v0' } as const; await expect( - restartVatHandler.implementation(mockKernel, mockKVStore, params), + restartVatHandler.implementation(mockKernel, mockKernelDatabase, params), ).rejects.toThrow(error); }); }); diff --git a/packages/extension/src/kernel-integration/handlers/restart-vat.ts b/packages/extension/src/kernel-integration/handlers/restart-vat.ts index 091b708ef..53a3a0162 100644 --- a/packages/extension/src/kernel-integration/handlers/restart-vat.ts +++ b/packages/extension/src/kernel-integration/handlers/restart-vat.ts @@ -1,6 +1,6 @@ import type { Json } from '@metamask/utils'; import type { Kernel } from '@ocap/kernel'; -import type { KVStore } from '@ocap/store'; +import type { KernelDatabase } from '@ocap/store'; import type { CommandHandler, CommandParams } from '../command-registry.ts'; import { KernelCommandPayloadStructs } from '../messages.ts'; @@ -10,7 +10,7 @@ export const restartVatHandler: CommandHandler<'restartVat'> = { schema: KernelCommandPayloadStructs.restartVat.schema.params, implementation: async ( kernel: Kernel, - _kvStore: KVStore, + _kdb: KernelDatabase, params: CommandParams['restartVat'], ): Promise => { await kernel.restartVat(params.id); diff --git a/packages/extension/src/kernel-integration/handlers/send-vat-command.test.ts b/packages/extension/src/kernel-integration/handlers/send-vat-command.test.ts index 51f6369df..2df5049ab 100644 --- a/packages/extension/src/kernel-integration/handlers/send-vat-command.test.ts +++ b/packages/extension/src/kernel-integration/handlers/send-vat-command.test.ts @@ -1,5 +1,5 @@ import type { Kernel } from '@ocap/kernel'; -import type { KVStore } from '@ocap/store'; +import type { KernelDatabase } from '@ocap/store'; import { describe, it, expect, vi } from 'vitest'; import { sendVatCommandHandler } from './send-vat-command.ts'; @@ -9,7 +9,7 @@ describe('sendVatCommandHandler', () => { sendVatCommand: vi.fn(() => 'success'), } as unknown as Kernel; - const mockKVStore = {} as unknown as KVStore; + const mockKernelDatabase = {} as unknown as KernelDatabase; it('should have the correct method', () => { expect(sendVatCommandHandler.method).toBe('sendVatCommand'); @@ -22,7 +22,7 @@ describe('sendVatCommandHandler', () => { } as const; const result = await sendVatCommandHandler.implementation( mockKernel, - mockKVStore, + mockKernelDatabase, params, ); expect(mockKernel.sendVatCommand).toHaveBeenCalledWith('v0', { @@ -37,7 +37,11 @@ describe('sendVatCommandHandler', () => { payload: { method: 'ping', params: null }, }; await expect( - sendVatCommandHandler.implementation(mockKernel, mockKVStore, params), + sendVatCommandHandler.implementation( + mockKernel, + mockKernelDatabase, + params, + ), ).rejects.toThrow('Vat ID required for this command'); }); }); diff --git a/packages/extension/src/kernel-integration/handlers/send-vat-command.ts b/packages/extension/src/kernel-integration/handlers/send-vat-command.ts index b4e73e4c5..ee6b787fe 100644 --- a/packages/extension/src/kernel-integration/handlers/send-vat-command.ts +++ b/packages/extension/src/kernel-integration/handlers/send-vat-command.ts @@ -1,7 +1,7 @@ import type { Json } from '@metamask/utils'; import { isKernelCommand } from '@ocap/kernel'; import type { Kernel } from '@ocap/kernel'; -import type { KVStore } from '@ocap/store'; +import type { KernelDatabase } from '@ocap/store'; import type { CommandHandler, CommandParams } from '../command-registry.ts'; import { KernelCommandPayloadStructs } from '../messages.ts'; @@ -11,7 +11,7 @@ export const sendVatCommandHandler: CommandHandler<'sendVatCommand'> = { schema: KernelCommandPayloadStructs.sendVatCommand.schema.params, implementation: async ( kernel: Kernel, - _kvStore: KVStore, + _kdb: KernelDatabase, params: CommandParams['sendVatCommand'], ): Promise => { if (!isKernelCommand(params.payload)) { diff --git a/packages/extension/src/kernel-integration/handlers/terminate-all-vats.test.ts b/packages/extension/src/kernel-integration/handlers/terminate-all-vats.test.ts index b25ed17eb..8c54b1b0a 100644 --- a/packages/extension/src/kernel-integration/handlers/terminate-all-vats.test.ts +++ b/packages/extension/src/kernel-integration/handlers/terminate-all-vats.test.ts @@ -1,5 +1,5 @@ import type { Kernel } from '@ocap/kernel'; -import type { KVStore } from '@ocap/store'; +import type { KernelDatabase } from '@ocap/store'; import { describe, it, expect, vi } from 'vitest'; import { terminateAllVatsHandler } from './terminate-all-vats.ts'; @@ -9,7 +9,7 @@ describe('terminateAllVatsHandler', () => { terminateAllVats: vi.fn().mockResolvedValue(undefined), } as unknown as Kernel; - const mockKVStore = {} as unknown as KVStore; + const mockKernelDatabase = {} as unknown as KernelDatabase; it('should have the correct method', () => { expect(terminateAllVatsHandler.method).toBe('terminateAllVats'); @@ -18,7 +18,7 @@ describe('terminateAllVatsHandler', () => { it('should terminate all vats and return null', async () => { const result = await terminateAllVatsHandler.implementation( mockKernel, - mockKVStore, + mockKernelDatabase, null, ); expect(mockKernel.terminateAllVats).toHaveBeenCalledOnce(); @@ -29,7 +29,11 @@ describe('terminateAllVatsHandler', () => { const error = new Error('Termination failed'); vi.mocked(mockKernel.terminateAllVats).mockRejectedValueOnce(error); await expect( - terminateAllVatsHandler.implementation(mockKernel, mockKVStore, null), + terminateAllVatsHandler.implementation( + mockKernel, + mockKernelDatabase, + null, + ), ).rejects.toThrow(error); }); }); diff --git a/packages/extension/src/kernel-integration/handlers/terminate-vat.test.ts b/packages/extension/src/kernel-integration/handlers/terminate-vat.test.ts index 2a15621c7..5f6b98701 100644 --- a/packages/extension/src/kernel-integration/handlers/terminate-vat.test.ts +++ b/packages/extension/src/kernel-integration/handlers/terminate-vat.test.ts @@ -1,5 +1,5 @@ import type { Kernel } from '@ocap/kernel'; -import type { KVStore } from '@ocap/store'; +import type { KernelDatabase } from '@ocap/store'; import { describe, it, expect, vi } from 'vitest'; import { terminateVatHandler } from './terminate-vat.ts'; @@ -9,7 +9,7 @@ describe('terminateVatHandler', () => { terminateVat: vi.fn().mockResolvedValue(undefined), } as unknown as Kernel; - const mockKVStore = {} as unknown as KVStore; + const mockKernelDatabase = {} as unknown as KernelDatabase; it('should have the correct method', () => { expect(terminateVatHandler.method).toBe('terminateVat'); @@ -19,7 +19,7 @@ describe('terminateVatHandler', () => { const params = { id: 'v0' } as const; const result = await terminateVatHandler.implementation( mockKernel, - mockKVStore, + mockKernelDatabase, params, ); expect(mockKernel.terminateVat).toHaveBeenCalledWith(params.id); @@ -31,7 +31,11 @@ describe('terminateVatHandler', () => { vi.mocked(mockKernel.terminateVat).mockRejectedValueOnce(error); const params = { id: 'v0' } as const; await expect( - terminateVatHandler.implementation(mockKernel, mockKVStore, params), + terminateVatHandler.implementation( + mockKernel, + mockKernelDatabase, + params, + ), ).rejects.toThrow(error); }); }); diff --git a/packages/extension/src/kernel-integration/handlers/terminate-vat.ts b/packages/extension/src/kernel-integration/handlers/terminate-vat.ts index 3372c1f7c..dacf11b9a 100644 --- a/packages/extension/src/kernel-integration/handlers/terminate-vat.ts +++ b/packages/extension/src/kernel-integration/handlers/terminate-vat.ts @@ -1,6 +1,6 @@ import type { Json } from '@metamask/utils'; import type { Kernel } from '@ocap/kernel'; -import type { KVStore } from '@ocap/store'; +import type { KernelDatabase } from '@ocap/store'; import type { CommandHandler, CommandParams } from '../command-registry.ts'; import { KernelCommandPayloadStructs } from '../messages.ts'; @@ -10,7 +10,7 @@ export const terminateVatHandler: CommandHandler<'terminateVat'> = { schema: KernelCommandPayloadStructs.terminateVat.schema.params, implementation: async ( kernel: Kernel, - _kvStore: KVStore, + _kdb: KernelDatabase, params: CommandParams['terminateVat'], ): Promise => { await kernel.terminateVat(params.id); diff --git a/packages/extension/src/kernel-integration/handlers/update-cluster-config.test.ts b/packages/extension/src/kernel-integration/handlers/update-cluster-config.test.ts index 808d09090..bc0916c08 100644 --- a/packages/extension/src/kernel-integration/handlers/update-cluster-config.test.ts +++ b/packages/extension/src/kernel-integration/handlers/update-cluster-config.test.ts @@ -1,5 +1,5 @@ import type { Kernel } from '@ocap/kernel'; -import type { KVStore } from '@ocap/store'; +import type { KernelDatabase } from '@ocap/store'; import { describe, it, expect } from 'vitest'; import { updateClusterConfigHandler } from './update-cluster-config.ts'; @@ -9,7 +9,7 @@ describe('updateClusterConfigHandler', () => { clusterConfig: null, } as Partial; - const mockKvStore = {} as KVStore; + const mockKernelDatabase = {} as KernelDatabase; const testConfig = { bootstrap: 'testVat', @@ -23,7 +23,7 @@ describe('updateClusterConfigHandler', () => { it('should update kernel cluster config', async () => { const result = await updateClusterConfigHandler.implementation( mockKernel as Kernel, - mockKvStore, + mockKernelDatabase, { config: testConfig }, ); diff --git a/packages/extension/src/kernel-integration/handlers/update-cluster-config.ts b/packages/extension/src/kernel-integration/handlers/update-cluster-config.ts index 37a3b6c3b..f87190e65 100644 --- a/packages/extension/src/kernel-integration/handlers/update-cluster-config.ts +++ b/packages/extension/src/kernel-integration/handlers/update-cluster-config.ts @@ -1,6 +1,6 @@ import type { Json } from '@metamask/utils'; import type { ClusterConfig, Kernel } from '@ocap/kernel'; -import type { KVStore } from '@ocap/store'; +import type { KernelDatabase } from '@ocap/store'; import type { CommandHandler } from '../command-registry.ts'; import { KernelCommandPayloadStructs } from '../messages.ts'; @@ -11,7 +11,7 @@ export const updateClusterConfigHandler: CommandHandler<'updateClusterConfig'> = schema: KernelCommandPayloadStructs.updateClusterConfig.schema.params, implementation: async ( kernel: Kernel, - _kvStore: KVStore, + _kdb: KernelDatabase, params: { config: ClusterConfig }, ): Promise => { kernel.clusterConfig = params.config; diff --git a/packages/extension/src/kernel-integration/kernel-worker.ts b/packages/extension/src/kernel-integration/kernel-worker.ts index 72c5ad54e..120d7d545 100644 --- a/packages/extension/src/kernel-integration/kernel-worker.ts +++ b/packages/extension/src/kernel-integration/kernel-worker.ts @@ -4,7 +4,7 @@ import type { KernelCommandReply, } from '@ocap/kernel'; import { ClusterConfigStruct, isKernelCommand, Kernel } from '@ocap/kernel'; -import { makeSQLKVStore } from '@ocap/store/sqlite/wasm'; +import { makeSQLKernelDatabase } from '@ocap/store/sqlite/wasm'; import type { PostMessageTarget } from '@ocap/streams/browser'; import { MessagePortDuplexStream, @@ -38,17 +38,22 @@ async function main(): Promise { const vatWorkerClient = ExtensionVatWorkerClient.make( globalThis as PostMessageTarget, ); - const kvStore = await makeSQLKVStore(); + const kernelDatabase = await makeSQLKernelDatabase(); - const kernel = await Kernel.make(kernelStream, vatWorkerClient, kvStore, { - // XXX Warning: Clearing storage here is a hack to aid development - // debugging, wherein extension reloads are almost exclusively used for - // retrying after tweaking some fix. The following line will prevent - // the accumulation of long term kernel state. - resetStorage: true, - }); + const kernel = await Kernel.make( + kernelStream, + vatWorkerClient, + kernelDatabase, + { + // XXX Warning: Clearing storage here is a hack to aid development + // debugging, wherein extension reloads are almost exclusively used for + // retrying after tweaking some fix. The following line will prevent + // the accumulation of long term kernel state. + resetStorage: true, + }, + ); receiveUiConnections( - async (message) => handlePanelMessage(kernel, kvStore, message), + async (message) => handlePanelMessage(kernel, kernelDatabase, message), logger, ); diff --git a/packages/extension/src/kernel-integration/middlewares/logging.test.ts b/packages/extension/src/kernel-integration/middlewares/logging.test.ts index 35b8dbc14..8ef65ade0 100644 --- a/packages/extension/src/kernel-integration/middlewares/logging.test.ts +++ b/packages/extension/src/kernel-integration/middlewares/logging.test.ts @@ -1,26 +1,26 @@ import type { Kernel } from '@ocap/kernel'; -import type { KVStore } from '@ocap/store'; +import type { KernelDatabase } from '@ocap/store'; import { describe, it, expect, vi } from 'vitest'; import { loggingMiddleware, logger } from './logging.ts'; describe('loggingMiddleware', () => { - const mockKVStore = {} as unknown as KVStore; + const mockKernelDatabase = {} as unknown as KernelDatabase; const mockKernel = {} as unknown as Kernel; it('should call the next function with the provided arguments', async () => { const next = vi.fn(); const middleware = loggingMiddleware(next); const params = { arg1: 'arg1', arg2: 'arg2' }; - await middleware(mockKernel, mockKVStore, params); - expect(next).toHaveBeenCalledWith(mockKernel, mockKVStore, params); + await middleware(mockKernel, mockKernelDatabase, params); + expect(next).toHaveBeenCalledWith(mockKernel, mockKernelDatabase, params); }); it('should return the result from the next function', async () => { const expectedResult = 'test result'; const next = vi.fn().mockResolvedValue(expectedResult); const middleware = loggingMiddleware(next); - const result = await middleware(mockKernel, mockKVStore, {}); + const result = await middleware(mockKernel, mockKernelDatabase, {}); expect(result).toBe(expectedResult); }); @@ -33,7 +33,7 @@ describe('loggingMiddleware', () => { }), ); const middleware = loggingMiddleware(next); - await middleware(mockKernel, mockKVStore, {}); + await middleware(mockKernel, mockKernelDatabase, {}); expect(debugSpy).toHaveBeenCalledWith( expect.stringMatching(/Command executed in \d*\.?\d+ms/u), ); @@ -44,9 +44,9 @@ describe('loggingMiddleware', () => { const error = new Error('Test error'); const next = vi.fn().mockRejectedValue(error); const middleware = loggingMiddleware(next); - await expect(middleware(mockKernel, mockKVStore, {})).rejects.toThrow( - error, - ); + await expect( + middleware(mockKernel, mockKernelDatabase, {}), + ).rejects.toThrow(error); expect(debugSpy).toHaveBeenCalledWith( expect.stringMatching(/Command executed in \d*\.?\d+ms/u), ); diff --git a/packages/kernel-test/package.json b/packages/kernel-test/package.json index 90337a9a6..9a2eca517 100644 --- a/packages/kernel-test/package.json +++ b/packages/kernel-test/package.json @@ -53,6 +53,8 @@ "@metamask/eslint-config-nodejs": "^14.0.0", "@metamask/eslint-config-typescript": "^14.0.0", "@ocap/cli": "workspace:^", + "@ocap/store": "workspace:^", + "@ocap/streams": "workspace:^", "@ts-bridge/cli": "^0.6.2", "@ts-bridge/shims": "^0.1.1", "@typescript-eslint/eslint-plugin": "^8.26.1", diff --git a/packages/kernel-test/src/vatstore-vat.js b/packages/kernel-test/src/vatstore-vat.js new file mode 100644 index 000000000..3ddd83d36 --- /dev/null +++ b/packages/kernel-test/src/vatstore-vat.js @@ -0,0 +1,44 @@ +import { E } from '@endo/eventual-send'; +import { Far } from '@endo/marshal'; + +/** + * Build function for running a test of the vatstore. + * + * @param {unknown} _vatPowers - Special powers granted to this vat (not used here). + * @param {unknown} parameters - Initialization parameters from the vat's config object. + * @param {unknown} baggage - Root of vat's persistent state. + * @returns {unknown} The root object for the new vat. + */ +export function buildRootObject(_vatPowers, parameters, baggage) { + const name = parameters?.name ?? 'anonymous'; + console.log(`buildRootObject "${name}"`); + + const testKey1 = 'thing'; + const testKey2 = 'goAway'; + + return Far('root', { + async bootstrap(vats) { + console.log(`vat ${name} is bootstrap`); + if (!baggage.has(testKey1)) { + baggage.init(testKey1, 1); + } + baggage.init(testKey2, 'now you see me'); + const pb = E(vats.bob).go(name, vats.alice); + const pc = E(vats.carol).go(name, vats.alice); + console.log(`vat ${name} got "go" answer from Bob: '${await pb}'`); + console.log(`vat ${name} got "go" answer from Carol: '${await pc}'`); + baggage.delete(testKey2); + }, + bump(bumper) { + const value = baggage.get(testKey1); + baggage.set(testKey1, value + 1); + console.log(`${bumper} bumps ${testKey1} from ${value} to ${value + 1}`); + }, + go(from, bumpee) { + const message = `vat ${name} got "go" from ${from}`; + console.log(message); + E(bumpee).bump(name); + return message; + }, + }); +} diff --git a/packages/kernel-test/src/vatstore.test.ts b/packages/kernel-test/src/vatstore.test.ts new file mode 100644 index 000000000..52a5f9f34 --- /dev/null +++ b/packages/kernel-test/src/vatstore.test.ts @@ -0,0 +1,218 @@ +import '@ocap/shims/endoify'; +import { makePromiseKit } from '@endo/promise-kit'; +import type { + KernelCommand, + KernelCommandReply, + ClusterConfig, + VatCheckpoint, +} from '@ocap/kernel'; +import { Kernel } from '@ocap/kernel'; +import type { KernelDatabase, VatStore } from '@ocap/store'; +import { makeSQLKernelDatabase } from '@ocap/store/sqlite/nodejs'; +import { NodeWorkerDuplexStream } from '@ocap/streams'; +import { + MessagePort as NodeMessagePort, + MessageChannel as NodeMessageChannel, +} from 'node:worker_threads'; +import { describe, vi, expect, it } from 'vitest'; + +import { kunser } from '../../kernel/src/kernel-marshal.ts'; +import { NodejsVatWorkerService } from '../../nodejs/src/kernel/VatWorkerService.ts'; + +/** + * Construct a bundle path URL from a bundle name. + * + * @param bundleName - The name of the bundle. + * + * @returns a path string for the named bundle. + */ +function bundleSpec(bundleName: string): string { + return new URL(`${bundleName}.bundle`, import.meta.url).toString(); +} + +const testSubcluster = { + bootstrap: 'alice', + forceReset: true, + vats: { + alice: { + bundleSpec: bundleSpec('vatstore-vat'), + parameters: { + name: 'Alice', + }, + }, + bob: { + bundleSpec: bundleSpec('vatstore-vat'), + parameters: { + name: 'Bob', + }, + }, + carol: { + bundleSpec: bundleSpec('vatstore-vat'), + parameters: { + name: 'Carol', + }, + }, + }, +}; + +/** + * Handle all the boilerplate to set up a kernel instance. + * + * @param kernelDatabase - The database that will hold the persistent state. + * @param resetStorage - If true, reset the database as part of setting up. + * + * @returns the new kernel instance. + */ +async function makeKernel( + kernelDatabase: KernelDatabase, + resetStorage: boolean, +): Promise { + const kernelPort: NodeMessagePort = new NodeMessageChannel().port1; + const nodeStream = new NodeWorkerDuplexStream< + KernelCommand, + KernelCommandReply + >(kernelPort); + const vatWorkerClient = new NodejsVatWorkerService({}); + const kernel = await Kernel.make( + nodeStream, + vatWorkerClient, + kernelDatabase, + { + resetStorage, + }, + ); + return kernel; +} + +/** + * Run the set of test vats. + * + * @param kernel - The kernel to run in. + * @param config - Subcluster configuration telling what vats to run. + * + * @returns the bootstrap result. + */ +async function runTestVats( + kernel: Kernel, + config: ClusterConfig, +): Promise { + const bootstrapResultRaw = await kernel.launchSubcluster(config); + + const { promise, resolve } = makePromiseKit(); + setTimeout(() => resolve(null), 0); + await promise; + if (bootstrapResultRaw === undefined) { + throw Error(`this can't happen but eslint is stupid`); + } + return kunser(bootstrapResultRaw); +} + +const emptyMap = new Map(); +const emptySet = new Set(); + +// prettier-ignore +const referenceKVUpdates = [ + [ + // initVat initializes built-in tables and empty baggage + new Map([ + ['baggageID', 'o+d6/1'], + ['idCounters', '{"exportID":10,"collectionID":5,"promiseID":5}'], + ['kindIDID', '1'], + ['storeKindIDTable', '{"scalarMapStore":2,"scalarWeakMapStore":3,"scalarSetStore":4,"scalarWeakSetStore":5,"scalarDurableMapStore":6,"scalarDurableWeakMapStore":7,"scalarDurableSetStore":8,"scalarDurableWeakSetStore":9}'], + ['vc.1.|entryCount', '0'], + ['vc.1.|nextOrdinal', '1'], + ['vc.1.|schemata', '{"body":"#{\\"keyShape\\":{\\"#tag\\":\\"match:string\\",\\"payload\\":[]},\\"label\\":\\"baggage\\"}","slots":[]}'], + ['vc.2.|entryCount', '0'], + ['vc.2.|nextOrdinal', '1'], + ['vc.2.|schemata', '{"body":"#{\\"keyShape\\":{\\"#tag\\":\\"match:scalar\\",\\"payload\\":\\"#undefined\\"},\\"label\\":\\"promiseRegistrations\\"}","slots":[]}'], + ['vc.3.|entryCount', '0'], + ['vc.3.|nextOrdinal', '1'], + ['vc.3.|schemata', '{"body":"#{\\"keyShape\\":{\\"#tag\\":\\"match:scalar\\",\\"payload\\":\\"#undefined\\"},\\"label\\":\\"promiseWatcherByKind\\"}","slots":[]}'], + ['vc.4.|entryCount', '0'], + ['vc.4.|nextOrdinal', '1'], + ['vc.4.|schemata', '{"body":"#{\\"keyShape\\":{\\"#tag\\":\\"match:and\\",\\"payload\\":[{\\"#tag\\":\\"match:scalar\\",\\"payload\\":\\"#undefined\\"},{\\"#tag\\":\\"match:string\\",\\"payload\\":[]}]},\\"label\\":\\"watchedPromises\\"}","slots":[]}'], + ['vom.rc.o+d6/1', '1'], + ['vom.rc.o+d6/3', '1'], + ['vom.rc.o+d6/4', '1'], + ['watchedPromiseTableID', 'o+d6/4'], + ['watcherTableID', 'o+d6/3'], + ]), + emptySet, + ], + // execution of 'bootstrap' initializes baggage, setting "thing" to 1 and + // "goAway" to the string "now you see me", (and thus the baggage entry count + // to 2). + [ + new Map([ + ['idCounters', '{"exportID":10,"collectionID":5,"promiseID":7}'], + ['vc.1.sgoAway', '{"body":"#\\"now you see me\\"","slots":[]}'], + ['vc.1.sthing', '{"body":"#1","slots":[]}'], + ['vc.1.|entryCount', '2'], + ]), + emptySet, + ], + // first 'bump' (from Bob) increments "thing" to 2 + [ + new Map([ + ['vc.1.sthing', '{"body":"#2","slots":[]}'], + ]), + emptySet, + ], + // notification of 'go' result from Bob changes nothing + [emptyMap, emptySet], + // second 'bump' (from Carol) increments "thing" to 3 + [ + new Map([ + ['vc.1.sthing', '{"body":"#3","slots":[]}'], + ]), + emptySet, + ], + // notification of 'go' result from Carol allows 'bootstrap' method to + // complete, deleting "goAway" from baggage and dropping the baggage entry + // count to 1. + [ + new Map([['vc.1.|entryCount', '1']]), + new Set(['vc.1.sgoAway']), + ] +] + +describe('exercise vatstore', async () => { + it('exercise vatstore', async () => { + const kernelDatabase = await makeSQLKernelDatabase(); + const origMakeVatStore = kernelDatabase.makeVatStore; + const kvUpdates: VatCheckpoint[] = []; + vi.spyOn(kernelDatabase, 'makeVatStore').mockImplementation( + (vatID: string): VatStore => { + const result = origMakeVatStore(vatID); + if (vatID === 'v1') { + const origUpdateKVData = result.updateKVData; + vi.spyOn(result, 'updateKVData').mockImplementation( + (sets: Map, deletes: Set): void => { + kvUpdates.push([sets, deletes]); + origUpdateKVData(sets, deletes); + }, + ); + } + return result; + }, + ); + const kernel = await makeKernel(kernelDatabase, true); + await runTestVats(kernel, testSubcluster); + + type VSRecord = { key: string; value: string }; + const vsContents = kernelDatabase.executeQuery( + `SELECT key, value from kv_vatStore where vatID = 'v1'`, + ) as VSRecord[]; + const vsKv = new Map(); + for (const entry of vsContents) { + vsKv.set(entry.key, entry.value); + } + expect(vsKv.get('idCounters')).toBe( + '{"exportID":10,"collectionID":5,"promiseID":7}', + ); + expect(vsKv.get('vc.1.sthing')).toBe('{"body":"#3","slots":[]}'); + expect(vsKv.get('vc.1.|entryCount')).toBe('1'); + + expect(kvUpdates).toStrictEqual(referenceKVUpdates); + }, 30000); +}); diff --git a/packages/kernel/src/Kernel.test.ts b/packages/kernel/src/Kernel.test.ts index 5e35f9338..647715033 100644 --- a/packages/kernel/src/Kernel.test.ts +++ b/packages/kernel/src/Kernel.test.ts @@ -1,5 +1,5 @@ import { VatNotFoundError } from '@ocap/errors'; -import type { KVStore } from '@ocap/store'; +import type { KernelDatabase } from '@ocap/store'; import type { DuplexStream } from '@ocap/streams'; import type { Mocked, MockInstance } from 'vitest'; import { describe, it, expect, vi, beforeEach } from 'vitest'; @@ -18,7 +18,7 @@ import type { ClusterConfig, } from './types.ts'; import { VatHandle } from './VatHandle.ts'; -import { makeMapKVStore } from '../test/storage.ts'; +import { makeMapKernelDatabase } from '../test/storage.ts'; describe('Kernel', () => { let mockStream: DuplexStream; @@ -27,7 +27,7 @@ describe('Kernel', () => { let terminateWorkerMock: MockInstance; let makeVatHandleMock: MockInstance; let vatHandles: Mocked[]; - let mockKVStore: KVStore; + let mockKernelDatabase: KernelDatabase; const makeMockVatConfig = (): VatConfig => ({ sourceSpec: 'not-really-there.js', @@ -86,7 +86,7 @@ describe('Kernel', () => { return vatHandle; }); - mockKVStore = makeMapKVStore(); + mockKernelDatabase = makeMapKernelDatabase(); }); describe('getVatIds()', () => { @@ -94,7 +94,7 @@ describe('Kernel', () => { const kernel = await Kernel.make( mockStream, mockWorkerService, - mockKVStore, + mockKernelDatabase, ); expect(kernel.getVatIds()).toStrictEqual([]); }); @@ -103,7 +103,7 @@ describe('Kernel', () => { const kernel = await Kernel.make( mockStream, mockWorkerService, - mockKVStore, + mockKernelDatabase, ); await kernel.launchVat(makeMockVatConfig()); expect(kernel.getVatIds()).toStrictEqual(['v1']); @@ -113,7 +113,7 @@ describe('Kernel', () => { const kernel = await Kernel.make( mockStream, mockWorkerService, - mockKVStore, + mockKernelDatabase, ); await kernel.launchVat(makeMockVatConfig()); await kernel.launchVat(makeMockVatConfig()); @@ -126,7 +126,7 @@ describe('Kernel', () => { const kernel = await Kernel.make( mockStream, mockWorkerService, - mockKVStore, + mockKernelDatabase, ); await kernel.launchVat(makeMockVatConfig()); expect(makeVatHandleMock).toHaveBeenCalledOnce(); @@ -138,7 +138,7 @@ describe('Kernel', () => { const kernel = await Kernel.make( mockStream, mockWorkerService, - mockKVStore, + mockKernelDatabase, ); await kernel.launchVat(makeMockVatConfig()); await kernel.launchVat(makeMockVatConfig()); @@ -153,7 +153,7 @@ describe('Kernel', () => { const kernel = await Kernel.make( mockStream, mockWorkerService, - mockKVStore, + mockKernelDatabase, ); await kernel.launchVat(makeMockVatConfig()); expect(kernel.getVatIds()).toStrictEqual(['v1']); @@ -167,7 +167,7 @@ describe('Kernel', () => { const kernel = await Kernel.make( mockStream, mockWorkerService, - mockKVStore, + mockKernelDatabase, ); const nonExistentVatId: VatId = 'v9'; await expect(async () => @@ -180,7 +180,7 @@ describe('Kernel', () => { const kernel = await Kernel.make( mockStream, mockWorkerService, - mockKVStore, + mockKernelDatabase, ); await kernel.launchVat(makeMockVatConfig()); vatHandles[0]?.terminate.mockRejectedValueOnce('Test error'); @@ -198,7 +198,7 @@ describe('Kernel', () => { const kernel = await Kernel.make( mockStream, mockWorkerService, - mockKVStore, + mockKernelDatabase, ); await kernel.launchVat(makeMockVatConfig()); await kernel.launchVat(makeMockVatConfig()); @@ -217,7 +217,7 @@ describe('Kernel', () => { const kernel = await Kernel.make( mockStream, mockWorkerService, - mockKVStore, + mockKernelDatabase, ); await kernel.launchVat(makeMockVatConfig()); await kernel.restartVat('v1'); @@ -239,7 +239,7 @@ describe('Kernel', () => { const kernel = await Kernel.make( mockStream, mockWorkerService, - mockKVStore, + mockKernelDatabase, ); await kernel.launchVat(makeMockVatConfig()); expect(kernel.getVatIds()).toStrictEqual(['v1']); @@ -259,7 +259,7 @@ describe('Kernel', () => { const kernel = await Kernel.make( mockStream, mockWorkerService, - mockKVStore, + mockKernelDatabase, ); await expect(kernel.restartVat('v999')).rejects.toThrow(VatNotFoundError); expect(vatHandles).toHaveLength(0); @@ -270,7 +270,7 @@ describe('Kernel', () => { const kernel = await Kernel.make( mockStream, mockWorkerService, - mockKVStore, + mockKernelDatabase, ); await kernel.launchVat(makeMockVatConfig()); vatHandles[0]?.terminate.mockRejectedValueOnce( @@ -286,7 +286,7 @@ describe('Kernel', () => { const kernel = await Kernel.make( mockStream, mockWorkerService, - mockKVStore, + mockKernelDatabase, ); await kernel.launchVat(makeMockVatConfig()); launchWorkerMock.mockRejectedValueOnce(new Error('Launch failed')); @@ -301,7 +301,7 @@ describe('Kernel', () => { const kernel = await Kernel.make( mockStream, mockWorkerService, - mockKVStore, + mockKernelDatabase, ); await kernel.launchVat(makeMockVatConfig()); vatHandles[0]?.sendVatCommand.mockResolvedValueOnce('test'); @@ -317,7 +317,7 @@ describe('Kernel', () => { const kernel = await Kernel.make( mockStream, mockWorkerService, - mockKVStore, + mockKernelDatabase, ); const nonExistentVatId: VatId = 'v9'; await expect(async () => @@ -329,7 +329,7 @@ describe('Kernel', () => { const kernel = await Kernel.make( mockStream, mockWorkerService, - mockKVStore, + mockKernelDatabase, ); await kernel.launchVat(makeMockVatConfig()); vatHandles[0]?.sendVatCommand.mockRejectedValueOnce('error'); @@ -343,7 +343,7 @@ describe('Kernel', () => { it('initializes the kernel without errors', () => { expect( async () => - await Kernel.make(mockStream, mockWorkerService, mockKVStore), + await Kernel.make(mockStream, mockWorkerService, mockKernelDatabase), ).not.toThrow(); }); }); @@ -361,7 +361,7 @@ describe('Kernel', () => { const kernel = await Kernel.make( mockStream, mockWorkerService, - mockKVStore, + mockKernelDatabase, ); kernel.clusterConfig = makeMockClusterConfig(); await kernel.launchVat(makeMockVatConfig()); @@ -381,7 +381,7 @@ describe('Kernel', () => { const kernel = await Kernel.make( mockStream, mockWorkerService, - mockKVStore, + mockKernelDatabase, ); await expect(kernel.reload()).rejects.toThrow('no subcluster to reload'); }); @@ -390,7 +390,7 @@ describe('Kernel', () => { const kernel = await Kernel.make( mockStream, mockWorkerService, - mockKVStore, + mockKernelDatabase, ); kernel.clusterConfig = makeMockClusterConfig(); const error = new Error('Termination failed'); diff --git a/packages/kernel/src/Kernel.ts b/packages/kernel/src/Kernel.ts index 852044dbd..570483142 100644 --- a/packages/kernel/src/Kernel.ts +++ b/packages/kernel/src/Kernel.ts @@ -7,7 +7,7 @@ import { VatAlreadyExistsError, VatNotFoundError, } from '@ocap/errors'; -import type { KVStore } from '@ocap/store'; +import type { KernelDatabase } from '@ocap/store'; import type { DuplexStream } from '@ocap/streams'; import type { Logger } from '@ocap/utils'; import { makeLogger } from '@ocap/utils'; @@ -77,8 +77,8 @@ export class Kernel { /** Service to spawn workers (in iframes) for vats to run in */ readonly #vatWorkerService: VatWorkerService; - /** Storage holding the kernel's persistent state */ - readonly #storage: KernelStore; + /** Storage holding the kernel's own persistent state */ + readonly #kernelStore: KernelStore; /** Logger for outputting messages (such as errors) to the console */ readonly #logger: Logger; @@ -101,7 +101,7 @@ export class Kernel { * * @param commandStream - Command channel from whatever external software is driving the kernel. * @param vatWorkerService - Service to create a worker in which a new vat can run. - * @param rawStorage - A KV store for holding the kernel's persistent state. + * @param kernelDatabase - Database holding the kernel's persistent state. * @param options - Options for the kernel constructor. * @param options.resetStorage - If true, the storage will be cleared. * @param options.logger - Optional logger for error and diagnostic output. @@ -110,7 +110,7 @@ export class Kernel { private constructor( commandStream: DuplexStream, vatWorkerService: VatWorkerService, - rawStorage: KVStore, + kernelDatabase: KernelDatabase, options: { resetStorage?: boolean; logger?: Logger; @@ -122,12 +122,12 @@ export class Kernel { this.#vatWorkerService = vatWorkerService; if (options.resetStorage) { - rawStorage.clear(); + kernelDatabase.clear(); } - this.#storage = makeKernelStore(rawStorage); + this.#kernelStore = makeKernelStore(kernelDatabase); this.#logger = options.logger ?? makeLogger('[ocap kernel]'); - this.#runQueueLength = this.#storage.runQueueLength(); + this.#runQueueLength = this.#kernelStore.runQueueLength(); this.#wakeUpTheRunQueue = null; } @@ -136,7 +136,7 @@ export class Kernel { * * @param commandStream - Command channel from whatever external software is driving the kernel. * @param vatWorkerService - Service to create a worker in which a new vat can run. - * @param rawStorage - A KV store for holding the kernel's persistent state. + * @param kernelDatabase - Database holding the kernel's persistent state. * @param options - Options for the kernel constructor. * @param options.resetStorage - If true, the storage will be cleared. * @param options.logger - Optional logger for error and diagnostic output. @@ -145,7 +145,7 @@ export class Kernel { static async make( commandStream: DuplexStream, vatWorkerService: VatWorkerService, - rawStorage: KVStore, + kernelDatabase: KernelDatabase, options: { resetStorage?: boolean; logger?: Logger; @@ -154,7 +154,7 @@ export class Kernel { const kernel = new Kernel( commandStream, vatWorkerService, - rawStorage, + kernelDatabase, options, ); await kernel.#init(); @@ -268,12 +268,12 @@ export class Kernel { assert(direction === 'export', `${vref} is not an export reference`); let kref; if (isPromise) { - kref = this.#storage.initKernelPromise()[0]; - this.#storage.setPromiseDecider(kref, vatId); + kref = this.#kernelStore.initKernelPromise()[0]; + this.#kernelStore.setPromiseDecider(kref, vatId); } else { - kref = this.#storage.initKernelObject(vatId); + kref = this.#kernelStore.initKernelObject(vatId); } - this.#storage.addClistEntry(vatId, kref, vref); + this.#kernelStore.addClistEntry(vatId, kref, vref); return kref; } @@ -321,10 +321,10 @@ export class Kernel { vatId, vatConfig, vatStream: commandStream, - storage: this.#storage, + kernelStore: this.#kernelStore, }); this.#vats.set(vatId, vat); - this.#storage.initEndpoint(vatId); + this.#kernelStore.initEndpoint(vatId); const rootRef = this.exportFromVat(vatId, ROOT_OBJECT_VREF); return rootRef; } @@ -337,7 +337,7 @@ export class Kernel { * @returns a promise for the KRef of the new vat's root object. */ async launchVat(vatConfig: VatConfig): Promise { - return this.#startVat(this.#storage.getNextVatId(), vatConfig); + return this.#startVat(this.#kernelStore.getNextVatId(), vatConfig); } /** @@ -351,10 +351,10 @@ export class Kernel { * @returns the VRef corresponding to `kref` in `vatId`. */ #translateRefKtoV(vatId: VatId, kref: KRef, importIfNeeded: boolean): VRef { - let eref = this.#storage.krefToEref(vatId, kref); + let eref = this.#kernelStore.krefToEref(vatId, kref); if (!eref) { if (importIfNeeded) { - eref = this.#storage.allocateErefForKref(vatId, kref); + eref = this.#kernelStore.allocateErefForKref(vatId, kref); } else { throw Fail`unmapped kref ${kref} vat=${vatId}`; } @@ -404,7 +404,7 @@ export class Kernel { * @param item - The item to add. */ enqueueRun(item: RunQueueItem): void { - this.#storage.enqueueRun(item); + this.#kernelStore.enqueueRun(item); this.#runQueueLength += 1; if (this.#runQueueLength === 1 && this.#wakeUpTheRunQueue) { const wakeUpTheRunQueue = this.#wakeUpTheRunQueue; @@ -420,7 +420,7 @@ export class Kernel { */ #dequeueRun(): RunQueueItem | undefined { this.#runQueueLength -= 1; - const result = this.#storage.dequeueRun(); + const result = this.#kernelStore.dequeueRun(); return result; } @@ -451,7 +451,7 @@ export class Kernel { return null; }; const routeAsSend = (targetObject: KRef): MessageRoute => { - const vatId = this.#storage.getOwner(targetObject); + const vatId = this.#kernelStore.getOwner(targetObject); if (!vatId) { return routeAsSplat(kser('no vat')); } @@ -462,7 +462,7 @@ export class Kernel { }; if (isPromiseRef(target)) { - const promise = this.#storage.getKernelPromise(target); + const promise = this.#kernelStore.getKernelPromise(target); switch (promise.state) { case 'fulfilled': { if (promise.value) { @@ -523,7 +523,7 @@ export class Kernel { if (typeof message.result !== 'string') { throw TypeError('message result must be a string'); } - this.#storage.setPromiseDecider(message.result, vatId); + this.#kernelStore.setPromiseDecider(message.result, vatId); } const vatTarget = this.#translateRefKtoV(vatId, target, false); const vatMessage = this.#translateMessageKtoV(vatId, message); @@ -532,7 +532,7 @@ export class Kernel { Fail`no owner for kernel object ${target}`; } } else { - this.#storage.enqueuePromiseMessage(target, message); + this.#kernelStore.enqueuePromiseMessage(target, message); } log(`@@@@ done ${vatId} send ${target}<-${JSON.stringify(message)}`); } @@ -547,13 +547,13 @@ export class Kernel { `${kpid} is not a kernel promise`, ); log(`@@@@ deliver ${vatId} notify ${kpid}`); - const promise = this.#storage.getKernelPromise(kpid); + const promise = this.#kernelStore.getKernelPromise(kpid); const { state, value } = promise; assert(value, `no value for promise ${kpid}`); if (state === 'unresolved') { Fail`notification on unresolved promise ${kpid}`; } - if (!this.#storage.krefToEref(vatId, kpid)) { + if (!this.#kernelStore.krefToEref(vatId, kpid)) { // no c-list entry, already done return; } @@ -564,7 +564,7 @@ export class Kernel { } const resolutions: VatOneResolution[] = []; for (const toResolve of targets) { - const tPromise = this.#storage.getKernelPromise(toResolve); + const tPromise = this.#kernelStore.getKernelPromise(toResolve); if (tPromise.state === 'unresolved') { Fail`target promise ${toResolve} is unresolved`; } @@ -613,7 +613,7 @@ export class Kernel { for (const slot of value.slots) { if (isPromiseRef(slot)) { if (!seen.has(slot)) { - const promise = this.#storage.getKernelPromise(slot); + const promise = this.#kernelStore.getKernelPromise(slot); if (promise.state !== 'unresolved') { if (promise.value) { scanPromise(slot, promise.value); @@ -653,7 +653,7 @@ export class Kernel { for (const resolution of resolutions) { const [kpid, rejected, dataRaw] = resolution; const data = dataRaw as CapData; - const promise = this.#storage.getKernelPromise(kpid); + const promise = this.#kernelStore.getKernelPromise(kpid); const { state, decider, subscribers } = promise; if (state !== 'unresolved') { Fail`${kpid} was already resolved`; @@ -668,7 +668,7 @@ export class Kernel { for (const subscriber of subscribers) { this.notify(subscriber, kpid); } - this.#storage.resolveKernelPromise(kpid, rejected, data); + this.#kernelStore.resolveKernelPromise(kpid, rejected, data); const kernelResolve = this.#kernelSubscriptions.get(kpid); if (kernelResolve) { this.#kernelSubscriptions.delete(kpid); @@ -691,7 +691,7 @@ export class Kernel { method: string, args: unknown[], ): Promise> { - const result = this.#storage.initKernelPromise()[0]; + const result = this.#kernelStore.initKernelPromise()[0]; const message: Message = { methargs: kser([method, args]), result, @@ -804,7 +804,7 @@ export class Kernel { * Clear the database. */ async clearStorage(): Promise { - this.#storage.reset(); + this.#kernelStore.clear(); } /** @@ -827,7 +827,8 @@ export class Kernel { */ async reset(): Promise { await this.terminateAllVats(); - this.#storage.reset(); + this.#kernelStore.clear(); + this.#kernelStore.reset(); } /** diff --git a/packages/kernel/src/VatHandle.test.ts b/packages/kernel/src/VatHandle.test.ts index e8a98f188..9e71fa392 100644 --- a/packages/kernel/src/VatHandle.test.ts +++ b/packages/kernel/src/VatHandle.test.ts @@ -9,7 +9,9 @@ import { Kernel } from './Kernel.ts'; import { isVatCommandReply, VatCommandMethod } from './messages/index.ts'; import type { VatCommand, VatCommandReply } from './messages/index.ts'; import type { KernelStore } from './store/kernel-store.ts'; +import { makeKernelStore } from './store/kernel-store.ts'; import { VatHandle } from './VatHandle.ts'; +import { makeMapKernelDatabase } from '../test/storage.ts'; vi.mock('@endo/eventual-send', () => ({ E: () => ({ @@ -19,6 +21,8 @@ vi.mock('@endo/eventual-send', () => ({ }), })); +let mockKernelStore: KernelStore; + const makeVat = async ( logger?: Logger, ): Promise<{ @@ -34,7 +38,7 @@ const makeVat = async ( return { vat: await VatHandle.make({ kernel: null as unknown as Kernel, - storage: null as unknown as KernelStore, + kernelStore: mockKernelStore, vatId: 'v0', vatConfig: { sourceSpec: 'not-really-there.js' }, vatStream: commandStream, @@ -48,10 +52,11 @@ describe('VatHandle', () => { let sendVatCommandMock: MockInstance; beforeEach(() => { + mockKernelStore = makeKernelStore(makeMapKernelDatabase()); sendVatCommandMock = vi .spyOn(VatHandle.prototype, 'sendVatCommand') - .mockResolvedValueOnce(null) - .mockResolvedValueOnce(null); + .mockResolvedValueOnce('fake') + .mockResolvedValueOnce('fake'); }); describe('init', () => { @@ -65,7 +70,10 @@ describe('VatHandle', () => { expect(sendVatCommandMock).toHaveBeenCalledWith({ method: VatCommandMethod.initVat, params: { - sourceSpec: 'not-really-there.js', + state: new Map(), + vatConfig: { + sourceSpec: 'not-really-there.js', + }, }, }); }); diff --git a/packages/kernel/src/VatHandle.ts b/packages/kernel/src/VatHandle.ts index 1af3a3a8d..d8b269c8b 100644 --- a/packages/kernel/src/VatHandle.ts +++ b/packages/kernel/src/VatHandle.ts @@ -6,6 +6,7 @@ import type { import type { CapData } from '@endo/marshal'; import { makePromiseKit } from '@endo/promise-kit'; import { VatDeletedError, StreamReadError } from '@ocap/errors'; +import type { VatStore } from '@ocap/store'; import type { DuplexStream } from '@ocap/streams'; import type { Logger } from '@ocap/utils'; import { makeLogger, makeCounter } from '@ocap/utils'; @@ -32,7 +33,7 @@ type VatConstructorProps = { vatId: VatId; vatConfig: VatConfig; vatStream: DuplexStream; - storage: KernelStore; + kernelStore: KernelStore; logger?: Logger | undefined; }; @@ -53,7 +54,10 @@ export class VatHandle { readonly #messageCounter: () => number; /** Storage holding the kernel's persistent state */ - readonly #storage: KernelStore; + readonly #kernelStore: KernelStore; + + /** Storage holding this vat's persistent state */ + readonly #vatStore: VatStore; /** The kernel we are working for. */ readonly #kernel: Kernel; @@ -70,7 +74,7 @@ export class VatHandle { * @param params.vatId - Our vat ID. * @param params.vatConfig - The configuration for this vat. * @param params.vatStream - Communications channel connected to the vat worker. - * @param params.storage - The kernel's persistent state store. + * @param params.kernelStore - The kernel's persistent state store. * @param params.logger - Optional logger for error and diagnostic output. */ // eslint-disable-next-line no-restricted-syntax @@ -79,7 +83,7 @@ export class VatHandle { vatId, vatConfig, vatStream, - storage, + kernelStore, logger, }: VatConstructorProps) { this.#kernel = kernel; @@ -88,7 +92,8 @@ export class VatHandle { this.#logger = logger ?? makeLogger(`[vat ${vatId}]`); this.#messageCounter = makeCounter(); this.#vatStream = vatStream; - this.#storage = storage; + this.#kernelStore = kernelStore; + this.#vatStore = kernelStore.makeVatStore(vatId); } /** @@ -99,7 +104,7 @@ export class VatHandle { * @param params.vatId - Our vat ID. * @param params.vatConfig - The configuration for this vat. * @param params.vatStream - Communications channel connected to the vat worker. - * @param params.storage - The kernel's persistent state store. + * @param params.kernelStore - The kernel's persistent state store. * @param params.logger - Optional logger for error and diagnostic output. * @returns A promise for the new VatHandle instance. */ @@ -108,7 +113,7 @@ export class VatHandle { vatId, vatConfig, vatStream, - storage, + kernelStore, logger, }: VatConstructorProps): Promise { const vat = new VatHandle({ @@ -116,7 +121,7 @@ export class VatHandle { vatId, vatConfig, vatStream, - storage, + kernelStore, logger, }); await vat.#init(); @@ -145,7 +150,7 @@ export class VatHandle { await this.sendVatCommand({ method: VatCommandMethod.ping, params: null }); await this.sendVatCommand({ method: VatCommandMethod.initVat, - params: this.config, + params: { vatConfig: this.config, state: this.#vatStore.getKVData() }, }); } @@ -157,7 +162,7 @@ export class VatHandle { * @returns the KRef corresponding to `vref` in this vat. */ #translateRefVtoK(vref: VRef): KRef { - let kref = this.#storage.erefToKref(this.vatId, vref); + let kref = this.#kernelStore.erefToKref(this.vatId, vref); if (!kref) { kref = this.#kernel.exportFromVat(this.vatId, vref); } @@ -306,9 +311,9 @@ export class VatHandle { * @param kpid - The KRef of the promise being subscribed to. */ #handleSyscallSubscribe(kpid: KRef): void { - const kp = this.#storage.getKernelPromise(kpid); + const kp = this.#kernelStore.getKernelPromise(kpid); if (kp.state === 'unresolved') { - this.#storage.addPromiseSubscriber(this.vatId, kpid); + this.#kernelStore.addPromiseSubscriber(this.vatId, kpid); } else { this.#kernel.notify(this.vatId, kpid); } @@ -406,12 +411,23 @@ export class VatHandle { if (payload.method === VatCommandMethod.syscall) { await this.#handleSyscall(payload.params as VatSyscallObject); } else { + let result; + if ( + payload.method === VatCommandMethod.deliver || + payload.method === VatCommandMethod.initVat + ) { + result = null; + const [sets, deletes] = payload.params; + this.#vatStore.updateKVData(sets, deletes); + } else { + result = payload.params; + } const promiseCallbacks = this.#unresolvedMessages.get(id); if (promiseCallbacks === undefined) { this.#logger.error(`No unresolved message with id "${id}".`); } else { this.#unresolvedMessages.delete(id); - promiseCallbacks.resolve(payload.params); + promiseCallbacks.resolve(result); } } } diff --git a/packages/kernel/src/VatKVStore.test.ts b/packages/kernel/src/VatKVStore.test.ts new file mode 100644 index 000000000..748e6da52 --- /dev/null +++ b/packages/kernel/src/VatKVStore.test.ts @@ -0,0 +1,46 @@ +import { describe, it, expect } from 'vitest'; + +import { makeVatKVStore } from './VatKVStore.ts'; + +describe('VatKVStore', () => { + it('working VatKVStore', () => { + const backingStore = new Map([ + ['key1', 'value1'], + ['key2', 'value2'], + ['key3', 'value3'], + ]); + const vatstore = makeVatKVStore(backingStore); + + expect(vatstore.get('key1')).toBe('value1'); + expect(vatstore.get('key4')).toBeUndefined(); + + vatstore.set('key2', 'revisedValue2'); + expect(vatstore.get('key2')).toBe('revisedValue2'); + + vatstore.set('key4', 'value4'); + expect(vatstore.get('key4')).toBe('value4'); + + vatstore.delete('key1'); + expect(vatstore.get('key1')).toBeUndefined(); + + const checkpoint = vatstore.checkpoint(); + expect(checkpoint).toStrictEqual([ + new Map([ + ['key2', 'revisedValue2'], + ['key4', 'value4'], + ]), + new Set(['key1']), + ]); + + const checkpoint2 = vatstore.checkpoint(); + expect(checkpoint2).toStrictEqual([new Map(), new Set()]); + + expect(backingStore).toStrictEqual( + new Map([ + ['key2', 'revisedValue2'], + ['key3', 'value3'], + ['key4', 'value4'], + ]), + ); + }); +}); diff --git a/packages/kernel/src/VatKVStore.ts b/packages/kernel/src/VatKVStore.ts new file mode 100644 index 000000000..ace35601c --- /dev/null +++ b/packages/kernel/src/VatKVStore.ts @@ -0,0 +1,60 @@ +import type { KVStore } from '@ocap/store'; + +import type { VatCheckpoint } from './types.ts'; + +export type VatKVStore = KVStore & { + checkpoint(): VatCheckpoint; +}; + +/** + * Create an in-memory VatKVStore for a vat, backed by a Map and tracking + * changes so that they can be reported at the end of a crank. + * + * @param state - The state to begin with. + * + * @returns a VatKVStore wrapped around `state`. + */ +export function makeVatKVStore(state: Map): VatKVStore { + let sets: Map = new Map(); + let deletes: Set = new Set(); + + return { + get(key: string): string | undefined { + return state.get(key); + }, + getRequired(key: string): string { + const result = state.get(key); + if (result) { + return result; + } + throw Error(`no record matching key '${key}'`); + }, + getNextKey(_previousKey: string): string | undefined { + // WARNING: this is a VERY expensive and complicated operation to + // implement if the backing store is an ordinary Map object fronted by the + // sets & deletes tables as we are doing here. However, the only customer + // of this KVStore is Liveslots, which does not use this operation, so it + // is not actually required. This "implementation" simply returns + // undefined, solely in interest of making the compiler happy -- it does + // not actually work! If you try to use it expecting something useful, it + // will go badly for you. + return undefined; + }, + set(key: string, value: string): void { + state.set(key, value); + sets.set(key, value); + deletes.delete(key); + }, + delete(key: string): void { + state.delete(key); + sets.delete(key); + deletes.add(key); + }, + checkpoint(): VatCheckpoint { + const result: VatCheckpoint = [sets, deletes]; + sets = new Map(); + deletes = new Set(); + return result; + }, + }; +} diff --git a/packages/kernel/src/VatSupervisor.ts b/packages/kernel/src/VatSupervisor.ts index 4c5925338..d09a0c438 100644 --- a/packages/kernel/src/VatSupervisor.ts +++ b/packages/kernel/src/VatSupervisor.ts @@ -8,7 +8,6 @@ import { importBundle } from '@endo/import-bundle'; import { makeMarshal } from '@endo/marshal'; import type { CapData } from '@endo/marshal'; import { StreamReadError } from '@ocap/errors'; -import type { MakeKVStore } from '@ocap/store'; import type { DuplexStream } from '@ocap/streams'; import type { @@ -22,6 +21,8 @@ import { VatCommandMethod } from './messages/index.ts'; import { makeSupervisorSyscall } from './syscall.ts'; import type { VatConfig, VatId, VRef } from './types.ts'; import { ROOT_OBJECT_VREF, isVatConfig } from './types.ts'; +import type { VatKVStore } from './VatKVStore.ts'; +import { makeVatKVStore } from './VatKVStore.ts'; import { waitUntilQuiescent } from './waitUntilQuiescent.ts'; const makeLiveSlots: MakeLiveSlotsFn = localMakeLiveSlots; @@ -32,7 +33,6 @@ type FetchBlob = (bundleURL: string) => Promise; type SupervisorConstructorProps = { id: VatId; commandStream: DuplexStream; - makeKVStore: MakeKVStore; fetchBlob?: FetchBlob; }; @@ -53,8 +53,8 @@ export class VatSupervisor { /** Function to dispatch deliveries into liveslots */ #dispatch: DispatchFn | null; - /** Capability to create the store for this vat. */ - readonly #makeKVStore: MakeKVStore; + /** In-memory KVStore cache for this vat. */ + #vatKVStore: VatKVStore | undefined; /** Capability to fetch the bundle of code to run in this vat. */ readonly #fetchBlob: FetchBlob; @@ -68,18 +68,11 @@ export class VatSupervisor { * @param params - Named constructor parameters. * @param params.id - The id of the vat being supervised. * @param params.commandStream - Communications channel connected to the kernel. - * @param params.makeKVStore - Capability to create the store for this vat. * @param params.fetchBlob - Function to fetch the user code bundle for this vat. */ - constructor({ - id, - commandStream, - makeKVStore, - fetchBlob, - }: SupervisorConstructorProps) { + constructor({ id, commandStream, fetchBlob }: SupervisorConstructorProps) { this.id = id; this.#commandStream = commandStream; - this.#makeKVStore = makeKVStore; this.#dispatch = null; const defaultFetchBlob: FetchBlob = async (bundleURL: string) => fetch(bundleURL); @@ -124,16 +117,18 @@ export class VatSupervisor { this.#syscallsInFlight.length = 0; await this.replyToMessage(id, { method: VatCommandMethod.deliver, - params: null, // XXX eventually this should be the actual result? + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + params: this.#vatKVStore!.checkpoint(), }); break; } case VatCommandMethod.initVat: { - const rootObjectVref = await this.#initVat(payload.params); + await this.#initVat(payload.params.vatConfig, payload.params.state); await this.replyToMessage(id, { method: VatCommandMethod.initVat, - params: rootObjectVref, + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + params: this.#vatKVStore!.checkpoint(), }); break; } @@ -194,10 +189,14 @@ export class VatSupervisor { * instance to manage it. * * @param vatConfig - Configuration object describing the vat to be intialized. + * @param state - A Map representing the current persistent state of the vat. * * @returns a promise for the VRef of the new vat's root object. */ - async #initVat(vatConfig: VatConfig): Promise { + async #initVat( + vatConfig: VatConfig, + state: Map, + ): Promise { if (this.#loaded) { throw Error( 'VatSupervisor received initVat after user code already loaded', @@ -214,11 +213,8 @@ export class VatSupervisor { } this.#loaded = true; - const kvStore = await this.#makeKVStore( - `vat-${this.id}.db`, - `[vat-${this.id}]`, - ); - const syscall = makeSupervisorSyscall(this, kvStore); + this.#vatKVStore = makeVatKVStore(state); + const syscall = makeSupervisorSyscall(this, this.#vatKVStore); const vatPowers = {}; // XXX should be something more real const liveSlotsOptions = {}; // XXX should be something more real diff --git a/packages/kernel/src/index.ts b/packages/kernel/src/index.ts index 05a6db667..af0af1bfe 100644 --- a/packages/kernel/src/index.ts +++ b/packages/kernel/src/index.ts @@ -8,6 +8,7 @@ export type { VatWorkerService, ClusterConfig, VatConfig, + VatCheckpoint, KRef, } from './types.ts'; export { diff --git a/packages/kernel/src/messages/vat.ts b/packages/kernel/src/messages/vat.ts index ec01c263c..4a3820649 100644 --- a/packages/kernel/src/messages/vat.ts +++ b/packages/kernel/src/messages/vat.ts @@ -5,6 +5,7 @@ import { unknown, tuple, union, + map, literal, refine, string, @@ -17,6 +18,7 @@ import { isVatId, MessageStruct, VatConfigStruct, + VatCheckpointStruct, CapDataStruct, } from '../types.ts'; import type { VatId } from '../types.ts'; @@ -198,7 +200,10 @@ export const VatMethodStructs = { ...VatTestMethodStructs, [VatCommandMethod.initVat]: object({ method: literal(VatCommandMethod.initVat), - params: VatConfigStruct, + params: object({ + vatConfig: VatConfigStruct, + state: map(string(), string()), + }), }), [VatCommandMethod.deliver]: object({ method: literal(VatCommandMethod.deliver), @@ -223,11 +228,11 @@ const VatReplyStructs = { ...VatTestReplyStructs, [VatCommandMethod.initVat]: object({ method: literal(VatCommandMethod.initVat), - params: string(), + params: VatCheckpointStruct, }), [VatCommandMethod.deliver]: object({ method: literal(VatCommandMethod.deliver), - params: literal(null), + params: VatCheckpointStruct, }), [VatCommandMethod.syscall]: object({ method: literal(VatCommandMethod.syscall), diff --git a/packages/kernel/src/store/kernel-store.test.ts b/packages/kernel/src/store/kernel-store.test.ts index 1560a4978..4b407faf7 100644 --- a/packages/kernel/src/store/kernel-store.test.ts +++ b/packages/kernel/src/store/kernel-store.test.ts @@ -1,9 +1,9 @@ import type { Message } from '@agoric/swingset-liveslots'; -import type { KVStore } from '@ocap/store'; +import type { KernelDatabase } from '@ocap/store'; import { describe, it, expect, beforeEach } from 'vitest'; import { makeKernelStore } from './kernel-store.ts'; -import { makeMapKVStore } from '../../test/storage.ts'; +import { makeMapKernelDatabase } from '../../test/storage.ts'; import type { RunQueueItem } from '../types.ts'; /** @@ -31,15 +31,15 @@ function tm(str: string): RunQueueItem { } describe('kernel store', () => { - let mockKVStore: KVStore; + let mockKernelDatabase: KernelDatabase; beforeEach(() => { - mockKVStore = makeMapKVStore(); + mockKernelDatabase = makeMapKernelDatabase(); }); describe('initialization', () => { it('has a working KV store', () => { - const ks = makeKernelStore(mockKVStore); + const ks = makeKernelStore(mockKernelDatabase); const { kv } = ks; expect(kv.get('foo')).toBeUndefined(); kv.set('foo', 'some value'); @@ -51,11 +51,12 @@ describe('kernel store', () => { ); }); it('has all the expected parts', () => { - const ks = makeKernelStore(mockKVStore); + const ks = makeKernelStore(mockKernelDatabase); expect(Object.keys(ks).sort()).toStrictEqual([ 'addClistEntry', 'addPromiseSubscriber', 'allocateErefForKref', + 'clear', 'decRefCount', 'deleteKernelObject', 'deleteKernelPromise', @@ -77,6 +78,7 @@ describe('kernel store', () => { 'initKernelPromise', 'krefToEref', 'kv', + 'makeVatStore', 'reset', 'resolveKernelPromise', 'runQueueLength', @@ -87,7 +89,7 @@ describe('kernel store', () => { describe('kernel entity management', () => { it('generates IDs', () => { - const ks = makeKernelStore(mockKVStore); + const ks = makeKernelStore(mockKernelDatabase); expect(ks.getNextVatId()).toBe('v1'); expect(ks.getNextVatId()).toBe('v2'); expect(ks.getNextVatId()).toBe('v3'); @@ -96,7 +98,7 @@ describe('kernel store', () => { expect(ks.getNextRemoteId()).toBe('r3'); }); it('manages kernel objects', () => { - const ks = makeKernelStore(mockKVStore); + const ks = makeKernelStore(mockKernelDatabase); const ko1Owner = 'v47'; const ko2Owner = 'r23'; expect(ks.initKernelObject(ko1Owner)).toBe('ko1'); @@ -116,7 +118,7 @@ describe('kernel store', () => { expect(() => ks.getOwner('ko99')).toThrow('unknown kernel object ko99'); }); it('manages kernel promises', () => { - const ks = makeKernelStore(mockKVStore); + const ks = makeKernelStore(mockKernelDatabase); const kp1 = { state: 'unresolved', subscribers: [], @@ -160,7 +162,7 @@ describe('kernel store', () => { ); }); it('manages the run queue', () => { - const ks = makeKernelStore(mockKVStore); + const ks = makeKernelStore(mockKernelDatabase); ks.enqueueRun(tm('first message')); ks.enqueueRun(tm('second message')); expect(ks.dequeueRun()).toBe('first message'); @@ -173,7 +175,7 @@ describe('kernel store', () => { expect(ks.dequeueRun()).toBeUndefined(); }); it('manages clists', () => { - const ks = makeKernelStore(mockKVStore); + const ks = makeKernelStore(mockKernelDatabase); ks.addClistEntry('v2', 'ko42', 'o-63'); ks.addClistEntry('v2', 'ko51', 'o-74'); ks.addClistEntry('v2', 'kp60', 'p+85'); @@ -200,7 +202,7 @@ describe('kernel store', () => { describe('reset', () => { it('clears store and resets counters', () => { - const ks = makeKernelStore(mockKVStore); + const ks = makeKernelStore(mockKernelDatabase); ks.getNextVatId(); ks.getNextVatId(); ks.getNextRemoteId(); diff --git a/packages/kernel/src/store/kernel-store.ts b/packages/kernel/src/store/kernel-store.ts index e64b31862..106ecf7e6 100644 --- a/packages/kernel/src/store/kernel-store.ts +++ b/packages/kernel/src/store/kernel-store.ts @@ -56,7 +56,7 @@ import type { Message } from '@agoric/swingset-liveslots'; import { Fail } from '@endo/errors'; import type { CapData } from '@endo/marshal'; -import type { KVStore } from '@ocap/store'; +import type { KVStore, VatStore, KernelDatabase } from '@ocap/store'; import type { VatId, @@ -152,23 +152,26 @@ export function parseRef(ref: string): RefParts { } /** - * Create a new KernelStore object wrapped around a simple string-to-string - * key/value store. The resulting object provides a variety of operations for - * accessing various kernel-relevent persistent data structure abstractions on - * their own terms, without burdening the kernel with the particular details of - * how they are stored. It is our hope that these operations may be later - * reimplemented on top of a more sophisticated storage layer that can realize + * Create a new KernelStore object wrapped around a raw kernel database. The + * resulting object provides a variety of operations for accessing various + * kernel-relevent persistent data structure abstractions on their own terms, + * without burdening the kernel with the particular details of how they are + * represented in storage. It is our hope that these operations may be later + * reimplemented on top of a more sophisticated database layer that can realize * them more directly (and thus, one hopes, more efficiently) without requiring * the kernel itself to be any the wiser. * - * @param kv - A key/value store to provide the underlying persistence mechanism. + * @param kdb - The kernel database this store is based on. * @returns A KernelStore object that maps various persistent kernel data - * structures onto `kv`. + * structures onto `kdb`. */ // eslint-disable-next-line @typescript-eslint/explicit-function-return-type -export function makeKernelStore(kv: KVStore) { +export function makeKernelStore(kdb: KernelDatabase) { // Initialize core state + /** KV store in which all the kernel's own state is kept. */ + const kv: KVStore = kdb.kernelKVStore; + /** The kernel's run queue. */ let runQueue = createStoredQueue('run', true); /** Counter for allocating VatIDs */ @@ -764,10 +767,28 @@ export function makeKernelStore(kv: KVStore) { } /** - * Clear the kernel's persistent state and reset all counters. + * Delete everything from the database. + */ + function clear(): void { + kdb.clear(); + } + + /** + * Create a new VatStore for a vat. + * + * @param vatID - The vat for which this is being done. + * + * @returns a a VatStore object for the given vat. + */ + function makeVatStore(vatID: string): VatStore { + return kdb.makeVatStore(vatID); + } + + /** + * Reset the kernel's persistent queues and counters. */ function reset(): void { - kv.clear(); + kdb.clear(); runQueue = createStoredQueue('run', true); nextVatId = provideCachedStoredValue('nextVatId', '1'); nextRemoteId = provideCachedStoredValue('nextRemoteId', '1'); @@ -802,6 +823,8 @@ export function makeKernelStore(kv: KVStore) { addClistEntry, forgetEref, forgetKref, + clear, + makeVatStore, reset, kv, }); diff --git a/packages/kernel/src/types.ts b/packages/kernel/src/types.ts index 7dde076bc..5d99ed44e 100644 --- a/packages/kernel/src/types.ts +++ b/packages/kernel/src/types.ts @@ -12,6 +12,9 @@ import { array, record, union, + tuple, + map, + set, literal, boolean, } from '@metamask/superstruct'; @@ -279,3 +282,10 @@ export const isClusterConfig = (value: unknown): value is ClusterConfig => is(value, ClusterConfigStruct); export type UserCodeStartFn = (parameters?: Record) => object; + +export type VatCheckpoint = [Map, Set]; + +export const VatCheckpointStruct = tuple([ + map(string(), string()), + set(string()), +]); diff --git a/packages/kernel/test/storage.ts b/packages/kernel/test/storage.ts index 22b82d693..e507905f1 100644 --- a/packages/kernel/test/storage.ts +++ b/packages/kernel/test/storage.ts @@ -1,4 +1,4 @@ -import type { KVStore } from '@ocap/store'; +import type { KVStore, KernelDatabase, VatStore } from '@ocap/store'; /** * A mock key/value store realized as a Map. @@ -6,8 +6,18 @@ import type { KVStore } from '@ocap/store'; * @returns The mock {@link KVStore}. */ export function makeMapKVStore(): KVStore { - const map = new Map(); + return makeMapKVStoreInternal(new Map()); +} +/** + * Internal helper function to build mock key/value stores, where the backing + * map is injected so it can be manipulated externally. + * + * @param map - The Map that will hold the mock store's state. + * + * @returns The mock {@link KVStore}. + */ +function makeMapKVStoreInternal(map: Map): KVStore { /** * Like `get`, but fail if the key isn't there. * @@ -30,7 +40,57 @@ export function makeMapKVStore(): KVStore { getRequired, set: map.set.bind(map), delete: map.delete.bind(map), - clear: map.clear.bind(map), + }; +} + +type ClearableVatStore = VatStore & { + clear: () => void; +}; + +/** + * Make a mock VatStore backed by a Map. + * + * @param _vatID - The vat ID of the vat whose store this will be (not used here). + * + * @returns the mock {@link VatStore}. + */ +function makeMapVatStore(_vatID: string): ClearableVatStore { + const map = new Map(); + return { + getKVData: () => map, + updateKVData: (sets: Map, deletes: Set) => { + for (const [key, value] of sets.entries()) { + map.set(key, value); + } + for (const key of deletes.values()) { + map.delete(key); + } + }, + clear: () => map.clear(), + }; +} + +/** + * Make a mock Kernel database using Maps. + * + * @returns the mock {@link KernelDatabase}. + */ +export function makeMapKernelDatabase(): KernelDatabase { + const map = new Map(); + const vatStores = new Set(); + return { + kernelKVStore: makeMapKVStoreInternal(map), + clear: () => { + map.clear(); + for (const vs of vatStores) { + vs.clear(); + } + }, executeQuery: () => [], + makeVatStore: (vatID: string) => { + const store = makeMapVatStore(vatID); + vatStores.add(store); + return store; + }, }; } diff --git a/packages/nodejs/src/kernel/make-kernel.test.ts b/packages/nodejs/src/kernel/make-kernel.test.ts index e45982464..1a9cd09cf 100644 --- a/packages/nodejs/src/kernel/make-kernel.test.ts +++ b/packages/nodejs/src/kernel/make-kernel.test.ts @@ -10,9 +10,11 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import { makeKernel } from './make-kernel.ts'; vi.mock('@ocap/store/sqlite/nodejs', async () => { - const { makeMapKVStore } = await import('../../../kernel/test/storage.ts'); + const { makeMapKernelDatabase } = await import( + '../../../kernel/test/storage.ts' + ); return { - makeSQLKVStore: makeMapKVStore, + makeSQLKernelDatabase: makeMapKernelDatabase, }; }); diff --git a/packages/nodejs/src/kernel/make-kernel.ts b/packages/nodejs/src/kernel/make-kernel.ts index 2e6a9ad44..aabcf3868 100644 --- a/packages/nodejs/src/kernel/make-kernel.ts +++ b/packages/nodejs/src/kernel/make-kernel.ts @@ -1,6 +1,6 @@ import type { KernelCommand, KernelCommandReply } from '@ocap/kernel'; import { Kernel } from '@ocap/kernel'; -import { makeSQLKVStore } from '@ocap/store/sqlite/nodejs'; +import { makeSQLKernelDatabase } from '@ocap/store/sqlite/nodejs'; import { NodeWorkerDuplexStream } from '@ocap/streams'; import { MessagePort as NodeMessagePort } from 'node:worker_threads'; @@ -26,12 +26,17 @@ export async function makeKernel( const vatWorkerClient = new NodejsVatWorkerService({ workerFilePath }); // Initialize kernel store. - const kvStore = await makeSQLKVStore(); + const kernelDatabase = await makeSQLKernelDatabase(); // Create and start kernel. - const kernel = await Kernel.make(nodeStream, vatWorkerClient, kvStore, { - resetStorage, - }); + const kernel = await Kernel.make( + nodeStream, + vatWorkerClient, + kernelDatabase, + { + resetStorage, + }, + ); return kernel; } diff --git a/packages/nodejs/src/vat/vat-worker.ts b/packages/nodejs/src/vat/vat-worker.ts index 96cecba92..9fa821ef0 100644 --- a/packages/nodejs/src/vat/vat-worker.ts +++ b/packages/nodejs/src/vat/vat-worker.ts @@ -2,7 +2,6 @@ import '@ocap/shims/endoify'; import type { VatId } from '@ocap/kernel'; import { VatSupervisor } from '@ocap/kernel'; -import { makeSQLKVStore } from '@ocap/store/sqlite/nodejs'; import { makeLogger } from '@ocap/utils'; import fs from 'node:fs/promises'; import url from 'node:url'; @@ -48,7 +47,6 @@ async function main(): Promise { void new VatSupervisor({ id: vatId, commandStream, - makeKVStore: makeSQLKVStore, fetchBlob, }); } diff --git a/packages/store/src/index.ts b/packages/store/src/index.ts index fff1bc8d4..b2bbc9c8c 100644 --- a/packages/store/src/index.ts +++ b/packages/store/src/index.ts @@ -1 +1 @@ -export type { KVStore, MakeKVStore } from './types.ts'; +export type { KVStore, VatStore, KernelDatabase } from './types.ts'; diff --git a/packages/store/src/sqlite/common.test.ts b/packages/store/src/sqlite/common.test.ts index 6eaf695ab..fd27e0133 100644 --- a/packages/store/src/sqlite/common.test.ts +++ b/packages/store/src/sqlite/common.test.ts @@ -3,6 +3,7 @@ import { describe, it, expect } from 'vitest'; import { SQL_QUERIES } from './common.ts'; describe('SQL_QUERIES', () => { + // XXX Is this test actually useful? It's basically testing that the source code matches itself. it.each([ [ 'CREATE_TABLE', @@ -38,13 +39,22 @@ describe('SQL_QUERIES', () => { it('has all expected query properties', () => { expect(Object.keys(SQL_QUERIES).sort()).toStrictEqual([ + 'ABORT_TRANSACTION', + 'BEGIN_TRANSACTION', 'CLEAR', + 'CLEAR_VS', + 'COMMIT_TRANSACTION', 'CREATE_TABLE', + 'CREATE_TABLE_VS', 'DELETE', + 'DELETE_VS', 'DROP', + 'DROP_VS', 'GET', + 'GET_ALL_VS', 'GET_NEXT', 'SET', + 'SET_VS', ]); }); }); diff --git a/packages/store/src/sqlite/common.ts b/packages/store/src/sqlite/common.ts index 50573728b..a65706120 100644 --- a/packages/store/src/sqlite/common.ts +++ b/packages/store/src/sqlite/common.ts @@ -6,6 +6,14 @@ export const SQL_QUERIES = { PRIMARY KEY(key) ) `, + CREATE_TABLE_VS: ` + CREATE TABLE IF NOT EXISTS kv_vatstore ( + vatID TEXT, + key TEXT, + value TEXT, + PRIMARY KEY(vatID, key) + ) + `, GET: ` SELECT value FROM kv @@ -17,19 +25,48 @@ export const SQL_QUERIES = { WHERE key > ? LIMIT 1 `, + GET_ALL_VS: ` + SELECT key, value + FROM kv_vatstore + WHERE vatID = ? + `, SET: ` INSERT INTO kv (key, value) VALUES (?, ?) ON CONFLICT DO UPDATE SET value = excluded.value `, + SET_VS: ` + INSERT INTO kv_vatstore (vatID, key, value) + VALUES (?, ?, ?) + ON CONFLICT DO UPDATE SET value = excluded.value + `, DELETE: ` DELETE FROM kv WHERE key = ? `, + DELETE_VS: ` + DELETE FROM kv_vatstore + WHERE vatID = ? AND key = ? + `, CLEAR: ` DELETE FROM kv `, + CLEAR_VS: ` + DELETE FROM kv_vatstore + `, DROP: ` DROP TABLE kv `, + DROP_VS: ` + DROP TABLE kv_vatstore + `, + BEGIN_TRANSACTION: ` + BEGIN TRANSACTION + `, + COMMIT_TRANSACTION: ` + COMMIT TRANSACTION + `, + ABORT_TRANSACTION: ` + ROLLBACK TRANSACTION + `, } as const; diff --git a/packages/store/src/sqlite/nodejs.test.ts b/packages/store/src/sqlite/nodejs.test.ts index aa9346229..e22628010 100644 --- a/packages/store/src/sqlite/nodejs.test.ts +++ b/packages/store/src/sqlite/nodejs.test.ts @@ -2,13 +2,24 @@ import { mkdir } from 'fs/promises'; import { describe, it, expect, vi, beforeEach } from 'vitest'; import { SQL_QUERIES } from './common.ts'; -import { makeSQLKVStore, getDBFilename } from './nodejs.ts'; +import { makeSQLKernelDatabase, getDBFilename } from './nodejs.ts'; + +const mockKVData = [ + { key: 'key1', value: 'value1' }, + { key: 'key2', value: 'value2' }, +]; + +const mockKVDataForMap: [string, string][] = [ + ['key1', 'value1'], + ['key2', 'value2'], +]; const mockStatement = { run: vi.fn(), get: vi.fn(), all: vi.fn(), pluck: vi.fn(), + iterate: vi.fn(() => mockKVData), }; const mockDb = { @@ -28,7 +39,7 @@ vi.mock('os', () => ({ tmpdir: vi.fn(() => '/mock-tmpdir'), })); -describe('makeSQLKVStore', () => { +describe('makeSQLKernelDatabase', () => { const mockMkdir = vi.mocked(mkdir).mockResolvedValue(''); beforeEach(() => { @@ -36,14 +47,16 @@ describe('makeSQLKVStore', () => { }); it('creates kv table', async () => { - await makeSQLKVStore(); + await makeSQLKernelDatabase(); expect(mockDb.prepare).toHaveBeenCalledWith(SQL_QUERIES.CREATE_TABLE); + expect(mockDb.prepare).toHaveBeenCalledWith(SQL_QUERIES.CREATE_TABLE_VS); }); it('get retrieves a value by key', async () => { const mockValue = 'test-value'; mockStatement.get.mockReturnValue(mockValue); - const store = await makeSQLKVStore(); + const db = await makeSQLKernelDatabase(); + const store = db.kernelKVStore; const result = store.get('test-key'); expect(result).toBe(mockValue); expect(mockStatement.get).toHaveBeenCalledWith('test-key'); @@ -51,34 +64,37 @@ describe('makeSQLKVStore', () => { it('getRequired throws when key not found', async () => { mockStatement.get.mockReturnValue(undefined); - const store = await makeSQLKVStore(); + const db = await makeSQLKernelDatabase(); + const store = db.kernelKVStore; expect(() => store.getRequired('missing-key')).toThrow( "no record matching key 'missing-key'", ); }); it('set inserts or updates a value', async () => { - const store = await makeSQLKVStore(); + const db = await makeSQLKernelDatabase(); + const store = db.kernelKVStore; store.set('test-key', 'test-value'); expect(mockStatement.run).toHaveBeenCalledWith('test-key', 'test-value'); }); it('delete removes a key-value pair', async () => { - const store = await makeSQLKVStore(); + const db = await makeSQLKernelDatabase(); + const store = db.kernelKVStore; store.delete('test-key'); expect(mockStatement.run).toHaveBeenCalledWith('test-key'); }); it('clear drops and recreates the table', async () => { - const store = await makeSQLKVStore(); + const store = await makeSQLKernelDatabase(); store.clear(); - expect(mockStatement.run).toHaveBeenCalledTimes(3); + expect(mockStatement.run).toHaveBeenCalledTimes(4); }); it('executeQuery runs arbitrary SQL queries', async () => { const mockResults = [{ key: 'value' }]; mockStatement.all.mockReturnValue(mockResults); - const store = await makeSQLKVStore(); + const store = await makeSQLKernelDatabase(); const result = store.executeQuery('SELECT * FROM kv'); expect(result).toStrictEqual(mockResults); }); @@ -86,16 +102,39 @@ describe('makeSQLKVStore', () => { it('getNextKey returns the next key in sequence', async () => { const mockNextKey = 'next-key'; mockStatement.get.mockReturnValue(mockNextKey); - const store = await makeSQLKVStore(); + const db = await makeSQLKernelDatabase(); + const store = db.kernelKVStore; const result = store.getNextKey('current-key'); expect(result).toBe(mockNextKey); expect(mockStatement.get).toHaveBeenCalledWith('current-key'); }); - it('getNextKey throws if previousKey is not a string', async () => { - const store = await makeSQLKVStore(); - // @ts-expect-error Testing invalid input - expect(() => store.getNextKey(123)).toThrow('must be a string'); + it('makeVatStore returns a VatStore', async () => { + const db = await makeSQLKernelDatabase(); + const vatStore = db.makeVatStore('vvat'); + expect(Object.keys(vatStore).sort()).toStrictEqual([ + 'getKVData', + 'updateKVData', + ]); + }); + + it('vatStore.getKVData returns a map of the data', async () => { + const db = await makeSQLKernelDatabase(); + const vatStore = db.makeVatStore('vvat'); + const data = vatStore.getKVData(); + expect(data).toStrictEqual(new Map(mockKVDataForMap)); + }); + + it('vatStore.updateKVData updates the database', async () => { + const db = await makeSQLKernelDatabase(); + const vatStore = db.makeVatStore('vvat'); + vatStore.updateKVData(new Map(mockKVDataForMap), new Set(['del1', 'del2'])); + expect(mockStatement.run).toHaveBeenCalled(); // begin transaction + expect(mockStatement.run).toHaveBeenCalledWith('vvat', 'key1', 'value1'); // set + expect(mockStatement.run).toHaveBeenCalledWith('vvat', 'key2', 'value2'); // set + expect(mockStatement.run).toHaveBeenCalledWith('vvat', 'del1'); // delete + expect(mockStatement.run).toHaveBeenCalledWith('vvat', 'del2'); // delete + expect(mockStatement.run).toHaveBeenCalled(); // commit transaction }); describe('getDBFilename', () => { diff --git a/packages/store/src/sqlite/nodejs.ts b/packages/store/src/sqlite/nodejs.ts index 5275db987..7235b14bd 100644 --- a/packages/store/src/sqlite/nodejs.ts +++ b/packages/store/src/sqlite/nodejs.ts @@ -7,7 +7,7 @@ import { tmpdir } from 'os'; import { join } from 'path'; import { SQL_QUERIES } from './common.ts'; -import type { KVStore } from '../types.ts'; +import type { KVStore, VatStore, KernelDatabase } from '../types.ts'; /** * Ensure that SQLite is initialized. @@ -30,23 +30,13 @@ async function initDB( } /** - * Makes a {@link KVStore} for low-level persistent storage. + * Makes a persistent {@link KVStore} on top of a SQLite database. * - * @param dbFilename - The filename of the database to use. Defaults to 'store.db'. - * @param label - A logger prefix label. Defaults to '[sqlite]'. - * @param verbose - If true, generate logger output; if false, be quiet. - * @returns The key/value store to base the kernel store on. + * @param db - The (open) database to use. + * @returns A key/value store using the given database. */ -export async function makeSQLKVStore( - dbFilename: string = 'store.db', - label: string = '[sqlite]', - verbose: boolean = false, -): Promise { - const logger = makeLogger(label); - const db = await initDB(dbFilename, logger, verbose); - +function makeKVStore(db: Database): KVStore { const sqlKVInit = db.prepare(SQL_QUERIES.CREATE_TABLE); - sqlKVInit.run(); const sqlKVGet = db.prepare<[string], string>(SQL_QUERIES.GET); @@ -109,14 +99,45 @@ export async function makeSQLKVStore( sqlKVDelete.run(key); } - const sqlKVDrop = db.prepare(SQL_QUERIES.DROP); + return { + get: (key) => kvGet(key, false), + getNextKey: kvGetNextKey, + getRequired: (key) => kvGet(key, true) as string, + set: kvSet, + delete: kvDelete, + }; +} + +/** + * Makes a {@link KernelDatabase} for low-level persistent storage. + * + * @param dbFilename - The filename of the database to use. Defaults to 'store.db'. + * @param label - A logger prefix label. Defaults to '[sqlite]'. + * @param verbose - If true, generate logger output; if false, be quiet. + * @returns The key/value store to base the kernel store on. + */ +export async function makeSQLKernelDatabase( + dbFilename: string = 'store.db', + label: string = '[sqlite]', + verbose: boolean = false, +): Promise { + const logger = makeLogger(label); + const db = await initDB(dbFilename, logger, verbose); + + const kvStore = makeKVStore(db); + + const sqlKVInitVS = db.prepare(SQL_QUERIES.CREATE_TABLE_VS); + sqlKVInitVS.run(); + + const sqlKVClear = db.prepare(SQL_QUERIES.CLEAR); + const sqlKVClearVS = db.prepare(SQL_QUERIES.CLEAR_VS); /** - * Delete all keys and values from the database. + * Delete everything from the database. */ function kvClear(): void { - sqlKVDrop.run(); - sqlKVInit.run(); + sqlKVClear.run(); + sqlKVClearVS.run(); } /** @@ -130,14 +151,67 @@ export async function makeSQLKVStore( return query.all() as Record[]; } + const sqlVatstoreGetAll = db.prepare(SQL_QUERIES.GET_ALL_VS); + const sqlVatstoreSet = db.prepare(SQL_QUERIES.SET_VS); + const sqlVatstoreDelete = db.prepare(SQL_QUERIES.DELETE_VS); + + /** + * Create a new VatStore for a vat. + * + * @param vatID - The vat for which this is being done. + * + * @returns a a VatStore object for the given vat. + */ + function makeVatStore(vatID: string): VatStore { + /** + * Fetch all the data in the vatstore. + * + * @returns the vatstore contents as a key-value Map. + */ + function getKVData(): Map { + const result = new Map(); + type KVPair = { + key: string; + value: string; + }; + for (const kvPair of sqlVatstoreGetAll.iterate(vatID)) { + const { key, value } = kvPair as KVPair; + result.set(key, value); + } + return result; + } + + /** + * Update the state of the vatstore + * + * @param sets - A map of key values that have been changed. + * @param deletes - A set of keys that have been deleted. + */ + function updateKVData( + sets: Map, + deletes: Set, + ): void { + db.transaction(() => { + for (const [key, value] of sets.entries()) { + sqlVatstoreSet.run(vatID, key, value); + } + for (const value of deletes.values()) { + sqlVatstoreDelete.run(vatID, value); + } + })(); + } + + return { + getKVData, + updateKVData, + }; + } + return { - get: (key) => kvGet(key, false), - getNextKey: kvGetNextKey, - getRequired: (key) => kvGet(key, true) as string, - set: kvSet, - delete: kvDelete, + kernelKVStore: kvStore, executeQuery: kvExecuteQuery, clear: db.transaction(kvClear), + makeVatStore, }; } diff --git a/packages/store/src/sqlite/wasm.test.ts b/packages/store/src/sqlite/wasm.test.ts index d4198ef7c..0ec746809 100644 --- a/packages/store/src/sqlite/wasm.test.ts +++ b/packages/store/src/sqlite/wasm.test.ts @@ -2,7 +2,17 @@ import type { Sqlite3Static } from '@sqlite.org/sqlite-wasm'; import { describe, it, expect, vi, beforeEach } from 'vitest'; import { SQL_QUERIES } from './common.ts'; -import { makeSQLKVStore } from './wasm.ts'; +import { makeSQLKernelDatabase } from './wasm.ts'; + +const mockKVData = [ + { key: 'key1', value: 'value1' }, + { key: 'key2', value: 'value2' }, +] as const; + +const mockKVDataForMap: [string, string][] = [ + ['key1', 'value1'], + ['key2', 'value2'], +]; const mockStatement = { bind: vi.fn(), @@ -28,7 +38,7 @@ vi.mock('@sqlite.org/sqlite-wasm', () => ({ })), })); -describe('makeSQLKVStore', () => { +describe('makeSQLKernelDatabase', () => { beforeEach(() => { Object.values(mockStatement) .filter( @@ -39,7 +49,7 @@ describe('makeSQLKVStore', () => { }); it('initializes with OPFS when available', async () => { - await makeSQLKVStore(); + await makeSQLKernelDatabase(); expect(mockDb.exec).toHaveBeenCalledWith(SQL_QUERIES.CREATE_TABLE); }); @@ -56,7 +66,7 @@ describe('makeSQLKVStore', () => { }) as unknown as Sqlite3Static, ); const consoleSpy = vi.spyOn(console, 'warn'); - await makeSQLKVStore(); + await makeSQLKernelDatabase(); expect(consoleSpy).toHaveBeenCalledWith( 'OPFS not enabled, database will be ephemeral', ); @@ -67,7 +77,8 @@ describe('makeSQLKVStore', () => { const mockValue = 'test-value'; mockStatement.step.mockReturnValueOnce(true); mockStatement.getString.mockReturnValueOnce(mockValue); - const store = await makeSQLKVStore(); + const db = await makeSQLKernelDatabase(); + const store = db.kernelKVStore; const result = store.get('test-key'); expect(result).toBe(mockValue); expect(mockStatement.bind).toHaveBeenCalledWith(['test-key']); @@ -75,14 +86,16 @@ describe('makeSQLKVStore', () => { it('getRequired throws when key not found', async () => { mockStatement.step.mockReturnValueOnce(false); - const store = await makeSQLKVStore(); + const db = await makeSQLKernelDatabase(); + const store = db.kernelKVStore; expect(() => store.getRequired('missing-key')).toThrow( "no record matching key 'missing-key'", ); }); it('set inserts or updates a value', async () => { - const store = await makeSQLKVStore(); + const db = await makeSQLKernelDatabase(); + const store = db.kernelKVStore; store.set('test-key', 'test-value'); expect(mockStatement.bind).toHaveBeenCalledWith(['test-key', 'test-value']); expect(mockStatement.step).toHaveBeenCalled(); @@ -90,7 +103,8 @@ describe('makeSQLKVStore', () => { }); it('delete removes a key-value pair', async () => { - const store = await makeSQLKVStore(); + const db = await makeSQLKernelDatabase(); + const store = db.kernelKVStore; store.delete('test-key'); expect(mockStatement.bind).toHaveBeenCalledWith(['test-key']); expect(mockStatement.step).toHaveBeenCalled(); @@ -98,7 +112,7 @@ describe('makeSQLKVStore', () => { }); it('clear removes all entries', async () => { - const store = await makeSQLKVStore(); + const store = await makeSQLKernelDatabase(); store.clear(); expect(mockStatement.step).toHaveBeenCalled(); expect(mockStatement.reset).toHaveBeenCalled(); @@ -108,12 +122,66 @@ describe('makeSQLKVStore', () => { const mockNextKey = 'next-key'; mockStatement.step.mockReturnValueOnce(true); mockStatement.getString.mockReturnValueOnce(mockNextKey); - const store = await makeSQLKVStore(); + const db = await makeSQLKernelDatabase(); + const store = db.kernelKVStore; const result = store.getNextKey('current-key'); expect(result).toBe(mockNextKey); expect(mockStatement.bind).toHaveBeenCalledWith(['current-key']); }); + it('makeVatStore returns a VatStore', async () => { + const db = await makeSQLKernelDatabase(); + const vatStore = db.makeVatStore('vvat'); + expect(Object.keys(vatStore).sort()).toStrictEqual([ + 'getKVData', + 'updateKVData', + ]); + }); + + it('vatStore.getKVData returns a map of the data', async () => { + const db = await makeSQLKernelDatabase(); + const vatStore = db.makeVatStore('vvat'); + mockStatement.step + .mockReturnValueOnce(true) + .mockReturnValueOnce(true) + .mockReturnValueOnce(false); + mockStatement.getString + .mockReturnValueOnce(mockKVData[0].key) + .mockReturnValueOnce(mockKVData[0].value) + .mockReturnValueOnce(mockKVData[1].key) + .mockReturnValueOnce(mockKVData[1].value); + const data = vatStore.getKVData(); + expect(data).toStrictEqual(new Map(mockKVDataForMap)); + }); + + it('vatStore.updateKVData updates the database', async () => { + const db = await makeSQLKernelDatabase(); + const vatStore = db.makeVatStore('vvat'); + vatStore.updateKVData(new Map(mockKVDataForMap), new Set(['del1', 'del2'])); + // begin transaction + expect(mockStatement.step).toHaveBeenCalled(); + expect(mockStatement.reset).toHaveBeenCalled(); + // set + expect(mockStatement.bind).toHaveBeenCalledWith(['vvat', 'key1', 'value1']); + expect(mockStatement.step).toHaveBeenCalled(); + expect(mockStatement.reset).toHaveBeenCalled(); + // set + expect(mockStatement.bind).toHaveBeenCalledWith(['vvat', 'key2', 'value2']); + expect(mockStatement.step).toHaveBeenCalled(); + expect(mockStatement.reset).toHaveBeenCalled(); + // delete + expect(mockStatement.bind).toHaveBeenCalledWith(['vvat', 'del1']); + expect(mockStatement.step).toHaveBeenCalled(); + expect(mockStatement.reset).toHaveBeenCalled(); + // delete + expect(mockStatement.bind).toHaveBeenCalledWith(['vvat', 'del2']); + expect(mockStatement.step).toHaveBeenCalled(); + expect(mockStatement.reset).toHaveBeenCalled(); + // commit transaction + expect(mockStatement.step).toHaveBeenCalled(); + expect(mockStatement.reset).toHaveBeenCalled(); + }); + it('executeQuery executes arbitrary SQL queries', async () => { mockStatement.step .mockReturnValueOnce(true) @@ -129,7 +197,7 @@ describe('makeSQLKVStore', () => { .mockReturnValueOnce('first') .mockReturnValueOnce('2') .mockReturnValueOnce('second'); - const store = await makeSQLKVStore(); + const store = await makeSQLKernelDatabase(); const results = store.executeQuery('SELECT * FROM kv'); expect(results).toStrictEqual([ { id: '1', value: 'first' }, @@ -140,7 +208,8 @@ describe('makeSQLKVStore', () => { it('get returns undefined when step() returns false', async () => { mockStatement.step.mockReturnValueOnce(false); - const store = await makeSQLKVStore(); + const db = await makeSQLKernelDatabase(); + const store = db.kernelKVStore; const result = store.get('test-key'); expect(result).toBeUndefined(); expect(mockStatement.bind).toHaveBeenCalledWith(['test-key']); @@ -150,7 +219,8 @@ describe('makeSQLKVStore', () => { it('get returns undefined when getString() returns falsy value', async () => { mockStatement.step.mockReturnValueOnce(true); mockStatement.getString.mockReturnValueOnce(''); - const store = await makeSQLKVStore(); + const db = await makeSQLKernelDatabase(); + const store = db.kernelKVStore; const result = store.get('test-key'); expect(result).toBeUndefined(); expect(mockStatement.bind).toHaveBeenCalledWith(['test-key']); @@ -167,7 +237,7 @@ describe('makeSQLKVStore', () => { .mockReturnValueOnce('1') .mockReturnValueOnce('ignored') .mockReturnValueOnce('also-ignored'); - const store = await makeSQLKVStore(); + const store = await makeSQLKernelDatabase(); const results = store.executeQuery('SELECT * FROM kv'); expect(results).toStrictEqual([{ id: '1' }]); expect(mockStatement.reset).toHaveBeenCalled(); @@ -179,7 +249,7 @@ describe('makeSQLKVStore', () => { .mockReturnValueOnce('id') .mockReturnValueOnce('number'); mockStatement.get.mockReturnValueOnce('1').mockReturnValueOnce(42); - const store = await makeSQLKVStore(); + const store = await makeSQLKernelDatabase(); const results = store.executeQuery('SELECT * FROM kv'); expect(results).toStrictEqual([ { @@ -193,7 +263,8 @@ describe('makeSQLKVStore', () => { describe('KVStore operations', () => { it('getNextKey returns undefined when no next key exists', async () => { mockStatement.step.mockReturnValueOnce(false); - const store = await makeSQLKVStore(); + const db = await makeSQLKernelDatabase(); + const store = db.kernelKVStore; const result = store.getNextKey('last-key'); expect(result).toBeUndefined(); expect(mockStatement.bind).toHaveBeenCalledWith(['last-key']); @@ -203,7 +274,8 @@ describe('makeSQLKVStore', () => { it('getNextKey returns undefined when getString returns falsy', async () => { mockStatement.step.mockReturnValueOnce(true); mockStatement.getString.mockReturnValueOnce(''); - const store = await makeSQLKVStore(); + const db = await makeSQLKernelDatabase(); + const store = db.kernelKVStore; const result = store.getNextKey('current-key'); expect(result).toBeUndefined(); expect(mockStatement.bind).toHaveBeenCalledWith(['current-key']); diff --git a/packages/store/src/sqlite/wasm.ts b/packages/store/src/sqlite/wasm.ts index 76068dfac..7515a6f87 100644 --- a/packages/store/src/sqlite/wasm.ts +++ b/packages/store/src/sqlite/wasm.ts @@ -1,9 +1,10 @@ +import type { Logger } from '@ocap/utils'; import { makeLogger } from '@ocap/utils'; -import type { Database } from '@sqlite.org/sqlite-wasm'; +import type { Database, PreparedStatement } from '@sqlite.org/sqlite-wasm'; import sqlite3InitModule from '@sqlite.org/sqlite-wasm'; import { SQL_QUERIES } from './common.ts'; -import type { KVStore } from '../types.ts'; +import type { KVStore, VatStore, KernelDatabase } from '../types.ts'; /** * Ensure that SQLite is initialized. @@ -22,25 +23,30 @@ async function initDB(dbFilename: string): Promise { } /** - * Makes a {@link KVStore} for low-level persistent storage. + * Helper function to paper over SQLite-wasm awfulness. Runs a prepared + * statement as it would be run in a more sensible API. * - * @param dbFilename - The filename of the database to use. Defaults to 'store.db'. - * @param label - A logger prefix label. Defaults to '[sqlite]'. - * @param verbose - If true, generate logger output; if false, be quiet. - * @returns A key/value store to base higher level stores on. + * @param stmt - A prepared statement to run. + * @param bindings - Optional parameters to bind for execution. */ -export async function makeSQLKVStore( - dbFilename: string = 'store.db', - label: string = '[sqlite]', - verbose: boolean = false, -): Promise { - const logger = makeLogger(label); - const db = await initDB(dbFilename); - - if (verbose) { - logger.log('Initializing kv store'); +// eslint-disable-next-line @typescript-eslint/no-unused-vars +function run(stmt: PreparedStatement, ...bindings: string[]): void { + if (bindings && bindings.length > 0) { + stmt.bind(bindings); } + stmt.step(); + stmt.reset(); +} +/** + * Makes a {@link KVStore} on top of a SQLite database + * + * @param db - The (open) database to use. + * @param logger - A logger object for recording activity. + * @param label - Label string for this store, for use in log messages. + * @returns A key/value store using the given database. + */ +function makeKVStore(db: Database, logger: Logger, label: string): KVStore { db.exec(SQL_QUERIES.CREATE_TABLE); const sqlKVGet = db.prepare(SQL_QUERIES.GET); @@ -122,10 +128,44 @@ export async function makeSQLKVStore( sqlKVDelete.reset(); } + return { + get: (key) => kvGet(key, false), + getNextKey: kvGetNextKey, + getRequired: (key) => kvGet(key, true) as string, + set: kvSet, + delete: kvDelete, + }; +} + +/** + * Makes a {@link KernelDatabase} for low-level persistent storage. + * + * @param dbFilename - The filename of the database to use. Defaults to 'store.db'. + * @param label - A logger prefix label. Defaults to '[sqlite]'. + * @param verbose - If true, generate logger output; if false, be quiet. + * @returns A key/value store to base higher level stores on. + */ +export async function makeSQLKernelDatabase( + dbFilename: string = 'store.db', + label: string = '[sqlite]', + verbose: boolean = false, +): Promise { + const logger = makeLogger(label); + const db = await initDB(dbFilename); + + if (verbose) { + logger.log('Initializing kernel store'); + } + + const kvStore = makeKVStore(db, logger, label); + + db.exec(SQL_QUERIES.CREATE_TABLE_VS); + const sqlKVClear = db.prepare(SQL_QUERIES.CLEAR); + const sqlKVClearVS = db.prepare(SQL_QUERIES.CLEAR_VS); /** - * Delete all entries from the database. + * Delete everything from the database. */ function kvClear(): void { if (verbose) { @@ -133,6 +173,8 @@ export async function makeSQLKVStore( } sqlKVClear.step(); sqlKVClear.reset(); + sqlKVClearVS.step(); + sqlKVClearVS.reset(); } /** @@ -162,13 +204,83 @@ export async function makeSQLKVStore( return results; } + const sqlVatstoreGetAll = db.prepare(SQL_QUERIES.GET_ALL_VS); + const sqlVatstoreSet = db.prepare(SQL_QUERIES.SET_VS); + const sqlVatstoreDelete = db.prepare(SQL_QUERIES.DELETE_VS); + const sqlBeginTransaction = db.prepare(SQL_QUERIES.BEGIN_TRANSACTION); + const sqlCommitTransaction = db.prepare(SQL_QUERIES.COMMIT_TRANSACTION); + const sqlAbortTransaction = db.prepare(SQL_QUERIES.ABORT_TRANSACTION); + + /** + * Create a new VatStore for a vat. + * + * @param vatID - The vat for which this is being done. + * + * @returns a a VatStore object for the given vat. + */ + function makeVatStore(vatID: string): VatStore { + /** + * Fetch all the data in the vatstore. + * + * @returns the vatstore contents as a key-value Map. + */ + function getKVData(): Map { + const result = new Map(); + sqlVatstoreGetAll.bind([vatID]); + try { + while (sqlVatstoreGetAll.step()) { + const key = sqlVatstoreGetAll.getString(0) as string; + const value = sqlVatstoreGetAll.getString(1) as string; + result.set(key, value); + } + } finally { + sqlVatstoreGetAll.reset(); + } + return result; + } + + /** + * Update the state of the vatstore + * + * @param sets - A map of key values that have been changed. + * @param deletes - A set of keys that have been deleted. + */ + function updateKVData( + sets: Map, + deletes: Set, + ): void { + try { + sqlBeginTransaction.step(); + sqlBeginTransaction.reset(); + for (const [key, value] of sets.entries()) { + sqlVatstoreSet.bind([vatID, key, value]); + sqlVatstoreSet.step(); + sqlVatstoreSet.reset(); + } + for (const value of deletes.values()) { + sqlVatstoreDelete.bind([vatID, value]); + sqlVatstoreDelete.step(); + sqlVatstoreDelete.reset(); + } + sqlCommitTransaction.step(); + sqlCommitTransaction.reset(); + } catch (problem) { + sqlAbortTransaction.step(); + sqlAbortTransaction.reset(); + throw problem; + } + } + + return { + getKVData, + updateKVData, + }; + } + return { - get: (key) => kvGet(key, false), - getNextKey: kvGetNextKey, - getRequired: (key) => kvGet(key, true) as string, - set: kvSet, - delete: kvDelete, + kernelKVStore: kvStore, clear: kvClear, executeQuery, + makeVatStore, }; } diff --git a/packages/store/src/types.ts b/packages/store/src/types.ts index d0a4c444c..330e91f0a 100644 --- a/packages/store/src/types.ts +++ b/packages/store/src/types.ts @@ -4,12 +4,16 @@ export type KVStore = { getNextKey(previousKey: string): string | undefined; set(key: string, value: string): void; delete(key: string): void; - clear(): void; - executeQuery(sql: string): Record[]; }; -export type MakeKVStore = ( - dbFilename?: string, - label?: string, - verbose?: boolean, -) => Promise; +export type VatStore = { + getKVData(): Map; + updateKVData(sets: Map, deletes: Set): void; +}; + +export type KernelDatabase = { + kernelKVStore: KVStore; + executeQuery(sql: string): Record[]; + clear(): void; + makeVatStore(vatID: string): VatStore; +}; diff --git a/vitest.config.ts b/vitest.config.ts index 16c957eee..696bb894a 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -87,10 +87,10 @@ export default defineConfig({ lines: 82.92, }, 'packages/kernel/**': { - statements: 74.69, - functions: 69.48, - branches: 59.72, - lines: 75.06, + statements: 74.73, + functions: 69.32, + branches: 59.86, + lines: 75.08, }, 'packages/nodejs/**': { statements: 72.91, @@ -105,10 +105,10 @@ export default defineConfig({ lines: 0, }, 'packages/store/**': { - statements: 97.02, - functions: 95.45, - branches: 91.17, - lines: 97, + statements: 93.2, + functions: 93.75, + branches: 78.94, + lines: 93.16, }, 'packages/streams/**': { statements: 100, diff --git a/yarn.lock b/yarn.lock index 32138a846..ea462a70e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2062,6 +2062,8 @@ __metadata: "@ocap/cli": "workspace:^" "@ocap/kernel": "workspace:^" "@ocap/shims": "workspace:^" + "@ocap/store": "workspace:^" + "@ocap/streams": "workspace:^" "@ts-bridge/cli": "npm:^0.6.2" "@ts-bridge/shims": "npm:^0.1.1" "@typescript-eslint/eslint-plugin": "npm:^8.26.1"