From 60880fed5c202825b9e074c5389519157387e551 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Wed, 5 Feb 2025 21:08:05 +0100 Subject: [PATCH 01/13] chore: Abstract KVStore to a new store package --- packages/extension/package.json | 2 +- packages/extension/src/iframe.ts | 3 +- .../kernel-integration/command-registry.ts | 3 +- .../handle-panel-message.test.ts | 9 +- .../handle-panel-message.ts | 3 +- .../handlers/clear-state.test.ts | 3 +- .../handlers/execute-db-query.test.ts | 3 +- .../handlers/execute-db-query.ts | 3 +- .../handlers/get-status.test.ts | 3 +- .../handlers/launch-vat.test.ts | 3 +- .../kernel-integration/handlers/launch-vat.ts | 3 +- .../handlers/reload-config.test.ts | 3 +- .../handlers/restart-vat.test.ts | 3 +- .../handlers/restart-vat.ts | 3 +- .../handlers/send-vat-command.test.ts | 3 +- .../handlers/send-vat-command.ts | 3 +- .../handlers/terminate-all-vats.test.ts | 3 +- .../handlers/terminate-vat.test.ts | 3 +- .../handlers/terminate-vat.ts | 3 +- .../handlers/update-cluster-config.test.ts | 3 +- .../handlers/update-cluster-config.ts | 3 +- .../src/kernel-integration/kernel-worker.ts | 2 +- .../middlewares/logging.test.ts | 3 +- packages/extension/tsconfig.build.json | 3 +- packages/extension/tsconfig.json | 6 +- packages/kernel/package.json | 1 + packages/kernel/src/Kernel.test.ts | 2 +- packages/kernel/src/Kernel.ts | 3 +- packages/kernel/src/VatSupervisor.ts | 4 +- packages/kernel/src/index.ts | 1 - .../kernel/src/store/kernel-store.test.ts | 2 +- packages/kernel/src/store/kernel-store.ts | 16 +- packages/kernel/src/syscall.ts | 2 +- packages/kernel/test/storage.ts | 2 +- packages/kernel/tsconfig.build.json | 5 +- packages/kernel/tsconfig.json | 3 +- packages/nodejs/package.json | 19 +- .../nodejs/src/kernel/make-kernel.test.ts | 2 +- packages/nodejs/src/kernel/make-kernel.ts | 2 +- packages/nodejs/src/vat/vat-worker.ts | 2 +- packages/nodejs/tsconfig.build.json | 17 +- packages/nodejs/tsconfig.json | 7 +- packages/nodejs/vitest.config.e2e.ts | 40 +--- packages/nodejs/vitest.config.ts | 37 +-- packages/store/CHANGELOG.md | 10 + packages/store/README.md | 15 ++ packages/store/package.json | 108 +++++++++ packages/store/src/index.ts | 1 + packages/store/src/sqlite/common.test.ts | 50 ++++ packages/store/src/sqlite/common.ts | 35 +++ packages/store/src/sqlite/native.test.ts | 93 ++++++++ .../src/sqlite/native.ts} | 67 ++---- packages/store/src/sqlite/wasm.test.ts | 213 ++++++++++++++++++ .../src/sqlite/wasm.ts} | 69 ++---- packages/store/src/types.ts | 14 ++ packages/store/tsconfig.build.json | 12 + packages/store/tsconfig.json | 15 ++ packages/store/tsconfig.lint.json | 9 + packages/store/typedoc.json | 8 + packages/store/vitest.config.ts | 18 ++ .../package-template/package.json | 14 +- tsconfig.build.json | 1 + tsconfig.json | 1 + vitest.config.ts | 28 ++- yarn.lock | 45 +++- 65 files changed, 827 insertions(+), 248 deletions(-) create mode 100644 packages/store/CHANGELOG.md create mode 100644 packages/store/README.md create mode 100644 packages/store/package.json create mode 100644 packages/store/src/index.ts create mode 100644 packages/store/src/sqlite/common.test.ts create mode 100644 packages/store/src/sqlite/common.ts create mode 100644 packages/store/src/sqlite/native.test.ts rename packages/{nodejs/src/kernel/sqlite-kv-store.ts => store/src/sqlite/native.ts} (72%) create mode 100644 packages/store/src/sqlite/wasm.test.ts rename packages/{extension/src/kernel-integration/sqlite-kv-store.ts => store/src/sqlite/wasm.ts} (72%) create mode 100644 packages/store/src/types.ts create mode 100644 packages/store/tsconfig.build.json create mode 100644 packages/store/tsconfig.json create mode 100644 packages/store/tsconfig.lint.json create mode 100644 packages/store/typedoc.json create mode 100644 packages/store/vitest.config.ts diff --git a/packages/extension/package.json b/packages/extension/package.json index 320f6bd1b..1178093c3 100644 --- a/packages/extension/package.json +++ b/packages/extension/package.json @@ -51,9 +51,9 @@ "@ocap/errors": "workspace:^", "@ocap/kernel": "workspace:^", "@ocap/shims": "workspace:^", + "@ocap/store": "workspace:^", "@ocap/streams": "workspace:^", "@ocap/utils": "workspace:^", - "@sqlite.org/sqlite-wasm": "^3.48.0-build1", "nanoid": "^3.3.8", "react": "^18.3.1", "react-dom": "^18.3.1", diff --git a/packages/extension/src/iframe.ts b/packages/extension/src/iframe.ts index 125b31689..8d71efccb 100644 --- a/packages/extension/src/iframe.ts +++ b/packages/extension/src/iframe.ts @@ -1,9 +1,8 @@ import { isVatCommand, VatSupervisor } from '@ocap/kernel'; import type { VatCommand, VatCommandReply } from '@ocap/kernel'; +import { makeSQLKVStore } from '@ocap/store/sqlite/wasm'; import { MessagePortDuplexStream, receiveMessagePort } from '@ocap/streams'; -import { makeSQLKVStore } from './kernel-integration/sqlite-kv-store.js'; - main().catch(console.error); /** diff --git a/packages/extension/src/kernel-integration/command-registry.ts b/packages/extension/src/kernel-integration/command-registry.ts index 54c09b346..e60e8d5f9 100644 --- a/packages/extension/src/kernel-integration/command-registry.ts +++ b/packages/extension/src/kernel-integration/command-registry.ts @@ -1,7 +1,8 @@ import { assert } from '@metamask/superstruct'; import type { Infer, Struct } from '@metamask/superstruct'; import type { Json } from '@metamask/utils'; -import type { Kernel, KVStore } from '@ocap/kernel'; +import type { Kernel } from '@ocap/kernel'; +import type { KVStore } from '@ocap/store'; import type { KernelCommandPayloadStructs, KernelMethods } from './messages.js'; diff --git a/packages/extension/src/kernel-integration/handle-panel-message.test.ts b/packages/extension/src/kernel-integration/handle-panel-message.test.ts index 5d3d0ad03..567c9234c 100644 --- a/packages/extension/src/kernel-integration/handle-panel-message.test.ts +++ b/packages/extension/src/kernel-integration/handle-panel-message.test.ts @@ -1,10 +1,5 @@ -import type { - Kernel, - KernelCommand, - VatId, - VatConfig, - KVStore, -} from '@ocap/kernel'; +import type { Kernel, KernelCommand, VatId, VatConfig } from '@ocap/kernel'; +import type { KVStore } from '@ocap/store'; import { setupOcapKernelMock } from '@ocap/test-utils'; import { describe, it, expect, vi, beforeEach } from 'vitest'; diff --git a/packages/extension/src/kernel-integration/handle-panel-message.ts b/packages/extension/src/kernel-integration/handle-panel-message.ts index 6fb23a364..265b51149 100644 --- a/packages/extension/src/kernel-integration/handle-panel-message.ts +++ b/packages/extension/src/kernel-integration/handle-panel-message.ts @@ -1,4 +1,5 @@ -import type { Kernel, KVStore } from '@ocap/kernel'; +import type { Kernel } from '@ocap/kernel'; +import type { KVStore } from '@ocap/store'; import { makeLogger } from '@ocap/utils'; import { KernelCommandRegistry } from './command-registry.js'; diff --git a/packages/extension/src/kernel-integration/handlers/clear-state.test.ts b/packages/extension/src/kernel-integration/handlers/clear-state.test.ts index 1c5916adf..7a7a05260 100644 --- a/packages/extension/src/kernel-integration/handlers/clear-state.test.ts +++ b/packages/extension/src/kernel-integration/handlers/clear-state.test.ts @@ -1,4 +1,5 @@ -import type { Kernel, KVStore } from '@ocap/kernel'; +import type { Kernel } from '@ocap/kernel'; +import type { KVStore } from '@ocap/store'; import { describe, it, expect, vi } from 'vitest'; import { clearStateHandler } from './clear-state.js'; diff --git a/packages/extension/src/kernel-integration/handlers/execute-db-query.test.ts b/packages/extension/src/kernel-integration/handlers/execute-db-query.test.ts index be0476995..74bc6c7dc 100644 --- a/packages/extension/src/kernel-integration/handlers/execute-db-query.test.ts +++ b/packages/extension/src/kernel-integration/handlers/execute-db-query.test.ts @@ -1,4 +1,5 @@ -import type { Kernel, KVStore } from '@ocap/kernel'; +import type { Kernel } from '@ocap/kernel'; +import type { KVStore } from '@ocap/store'; import { describe, it, expect, vi } from 'vitest'; import { executeDBQueryHandler } from './execute-db-query.js'; diff --git a/packages/extension/src/kernel-integration/handlers/execute-db-query.ts b/packages/extension/src/kernel-integration/handlers/execute-db-query.ts index c2d61189a..4b550c4a7 100644 --- a/packages/extension/src/kernel-integration/handlers/execute-db-query.ts +++ b/packages/extension/src/kernel-integration/handlers/execute-db-query.ts @@ -1,5 +1,6 @@ import type { Json } from '@metamask/utils'; -import type { Kernel, KVStore } from '@ocap/kernel'; +import type { Kernel } from '@ocap/kernel'; +import type { KVStore } from '@ocap/store'; import type { CommandHandler, CommandParams } from '../command-registry.js'; import { diff --git a/packages/extension/src/kernel-integration/handlers/get-status.test.ts b/packages/extension/src/kernel-integration/handlers/get-status.test.ts index 4a58d8984..694613786 100644 --- a/packages/extension/src/kernel-integration/handlers/get-status.test.ts +++ b/packages/extension/src/kernel-integration/handlers/get-status.test.ts @@ -1,4 +1,5 @@ -import type { Kernel, KVStore } from '@ocap/kernel'; +import type { Kernel } from '@ocap/kernel'; +import type { KVStore } from '@ocap/store'; import { describe, it, expect, vi } from 'vitest'; import { getStatusHandler } from './get-status.js'; diff --git a/packages/extension/src/kernel-integration/handlers/launch-vat.test.ts b/packages/extension/src/kernel-integration/handlers/launch-vat.test.ts index fe0c6f9f2..d999fef93 100644 --- a/packages/extension/src/kernel-integration/handlers/launch-vat.test.ts +++ b/packages/extension/src/kernel-integration/handlers/launch-vat.test.ts @@ -1,4 +1,5 @@ -import type { Kernel, KVStore } from '@ocap/kernel'; +import type { Kernel } from '@ocap/kernel'; +import type { KVStore } from '@ocap/store'; import { describe, it, expect, vi } from 'vitest'; import { launchVatHandler } from './launch-vat.js'; diff --git a/packages/extension/src/kernel-integration/handlers/launch-vat.ts b/packages/extension/src/kernel-integration/handlers/launch-vat.ts index 56ba68000..c45184142 100644 --- a/packages/extension/src/kernel-integration/handlers/launch-vat.ts +++ b/packages/extension/src/kernel-integration/handlers/launch-vat.ts @@ -1,5 +1,6 @@ import type { Json } from '@metamask/utils'; -import type { Kernel, KVStore } from '@ocap/kernel'; +import type { Kernel } from '@ocap/kernel'; +import type { KVStore } from '@ocap/store'; import type { CommandHandler, CommandParams } from '../command-registry.js'; import { diff --git a/packages/extension/src/kernel-integration/handlers/reload-config.test.ts b/packages/extension/src/kernel-integration/handlers/reload-config.test.ts index 2e75d3918..cb232de08 100644 --- a/packages/extension/src/kernel-integration/handlers/reload-config.test.ts +++ b/packages/extension/src/kernel-integration/handlers/reload-config.test.ts @@ -1,4 +1,5 @@ -import type { Kernel, KVStore } from '@ocap/kernel'; +import type { Kernel } from '@ocap/kernel'; +import type { KVStore } from '@ocap/store'; import { describe, it, expect, vi, beforeEach } from 'vitest'; import { reloadConfigHandler } from './reload-config.js'; diff --git a/packages/extension/src/kernel-integration/handlers/restart-vat.test.ts b/packages/extension/src/kernel-integration/handlers/restart-vat.test.ts index 130835424..7e1c76c4f 100644 --- a/packages/extension/src/kernel-integration/handlers/restart-vat.test.ts +++ b/packages/extension/src/kernel-integration/handlers/restart-vat.test.ts @@ -1,4 +1,5 @@ -import type { Kernel, KVStore } from '@ocap/kernel'; +import type { Kernel } from '@ocap/kernel'; +import type { KVStore } from '@ocap/store'; import { describe, it, expect, vi } from 'vitest'; import { restartVatHandler } from './restart-vat.js'; diff --git a/packages/extension/src/kernel-integration/handlers/restart-vat.ts b/packages/extension/src/kernel-integration/handlers/restart-vat.ts index 4f26b53eb..ba455d0f3 100644 --- a/packages/extension/src/kernel-integration/handlers/restart-vat.ts +++ b/packages/extension/src/kernel-integration/handlers/restart-vat.ts @@ -1,5 +1,6 @@ import type { Json } from '@metamask/utils'; -import type { Kernel, KVStore } from '@ocap/kernel'; +import type { Kernel } from '@ocap/kernel'; +import type { KVStore } from '@ocap/store'; import type { CommandHandler, CommandParams } from '../command-registry.js'; import { diff --git a/packages/extension/src/kernel-integration/handlers/send-vat-command.test.ts b/packages/extension/src/kernel-integration/handlers/send-vat-command.test.ts index 2bd5c21d5..b82a47e2c 100644 --- a/packages/extension/src/kernel-integration/handlers/send-vat-command.test.ts +++ b/packages/extension/src/kernel-integration/handlers/send-vat-command.test.ts @@ -1,4 +1,5 @@ -import type { Kernel, KVStore } from '@ocap/kernel'; +import type { Kernel } from '@ocap/kernel'; +import type { KVStore } from '@ocap/store'; import { describe, it, expect, vi } from 'vitest'; import { sendVatCommandHandler } from './send-vat-command.js'; diff --git a/packages/extension/src/kernel-integration/handlers/send-vat-command.ts b/packages/extension/src/kernel-integration/handlers/send-vat-command.ts index 223aa43a0..a4d09c1ae 100644 --- a/packages/extension/src/kernel-integration/handlers/send-vat-command.ts +++ b/packages/extension/src/kernel-integration/handlers/send-vat-command.ts @@ -1,6 +1,7 @@ import type { Json } from '@metamask/utils'; import { isKernelCommand } from '@ocap/kernel'; -import type { Kernel, KVStore } from '@ocap/kernel'; +import type { Kernel } from '@ocap/kernel'; +import type { KVStore } from '@ocap/store'; import type { CommandHandler, CommandParams } from '../command-registry.js'; import { diff --git a/packages/extension/src/kernel-integration/handlers/terminate-all-vats.test.ts b/packages/extension/src/kernel-integration/handlers/terminate-all-vats.test.ts index e00440a14..e9f8bc15d 100644 --- a/packages/extension/src/kernel-integration/handlers/terminate-all-vats.test.ts +++ b/packages/extension/src/kernel-integration/handlers/terminate-all-vats.test.ts @@ -1,4 +1,5 @@ -import type { Kernel, KVStore } from '@ocap/kernel'; +import type { Kernel } from '@ocap/kernel'; +import type { KVStore } from '@ocap/store'; import { describe, it, expect, vi } from 'vitest'; import { terminateAllVatsHandler } from './terminate-all-vats.js'; diff --git a/packages/extension/src/kernel-integration/handlers/terminate-vat.test.ts b/packages/extension/src/kernel-integration/handlers/terminate-vat.test.ts index 44d50a4f1..6b18d67b4 100644 --- a/packages/extension/src/kernel-integration/handlers/terminate-vat.test.ts +++ b/packages/extension/src/kernel-integration/handlers/terminate-vat.test.ts @@ -1,4 +1,5 @@ -import type { Kernel, KVStore } from '@ocap/kernel'; +import type { Kernel } from '@ocap/kernel'; +import type { KVStore } from '@ocap/store'; import { describe, it, expect, vi } from 'vitest'; import { terminateVatHandler } from './terminate-vat.js'; diff --git a/packages/extension/src/kernel-integration/handlers/terminate-vat.ts b/packages/extension/src/kernel-integration/handlers/terminate-vat.ts index f2edb1759..b834071db 100644 --- a/packages/extension/src/kernel-integration/handlers/terminate-vat.ts +++ b/packages/extension/src/kernel-integration/handlers/terminate-vat.ts @@ -1,5 +1,6 @@ import type { Json } from '@metamask/utils'; -import type { Kernel, KVStore } from '@ocap/kernel'; +import type { Kernel } from '@ocap/kernel'; +import type { KVStore } from '@ocap/store'; import type { CommandHandler, CommandParams } from '../command-registry.js'; import { diff --git a/packages/extension/src/kernel-integration/handlers/update-cluster-config.test.ts b/packages/extension/src/kernel-integration/handlers/update-cluster-config.test.ts index c11fc9090..5c58b034b 100644 --- a/packages/extension/src/kernel-integration/handlers/update-cluster-config.test.ts +++ b/packages/extension/src/kernel-integration/handlers/update-cluster-config.test.ts @@ -1,4 +1,5 @@ -import type { Kernel, KVStore } from '@ocap/kernel'; +import type { Kernel } from '@ocap/kernel'; +import type { KVStore } from '@ocap/store'; import { describe, it, expect } from 'vitest'; import { updateClusterConfigHandler } from './update-cluster-config.js'; diff --git a/packages/extension/src/kernel-integration/handlers/update-cluster-config.ts b/packages/extension/src/kernel-integration/handlers/update-cluster-config.ts index 0ec016cb8..cec8d73ea 100644 --- a/packages/extension/src/kernel-integration/handlers/update-cluster-config.ts +++ b/packages/extension/src/kernel-integration/handlers/update-cluster-config.ts @@ -1,5 +1,6 @@ import type { Json } from '@metamask/utils'; -import type { ClusterConfig, Kernel, KVStore } from '@ocap/kernel'; +import type { ClusterConfig, Kernel } from '@ocap/kernel'; +import type { KVStore } from '@ocap/store'; import type { CommandHandler } from '../command-registry.js'; import { diff --git a/packages/extension/src/kernel-integration/kernel-worker.ts b/packages/extension/src/kernel-integration/kernel-worker.ts index 30eb9c8f6..88faa7ad7 100644 --- a/packages/extension/src/kernel-integration/kernel-worker.ts +++ b/packages/extension/src/kernel-integration/kernel-worker.ts @@ -4,12 +4,12 @@ import type { KernelCommandReply, } from '@ocap/kernel'; import { ClusterConfigStruct, isKernelCommand, Kernel } from '@ocap/kernel'; +import { makeSQLKVStore } from '@ocap/store/sqlite/wasm'; import type { PostMessageTarget } from '@ocap/streams'; import { MessagePortDuplexStream, receiveMessagePort } from '@ocap/streams'; import { fetchValidatedJson, makeLogger } from '@ocap/utils'; import { handlePanelMessage } from './handle-panel-message.js'; -import { makeSQLKVStore } from './sqlite-kv-store.js'; import { receiveUiConnections } from './ui-connections.js'; import { ExtensionVatWorkerClient } from './VatWorkerClient.js'; diff --git a/packages/extension/src/kernel-integration/middlewares/logging.test.ts b/packages/extension/src/kernel-integration/middlewares/logging.test.ts index fed99b215..d249985a3 100644 --- a/packages/extension/src/kernel-integration/middlewares/logging.test.ts +++ b/packages/extension/src/kernel-integration/middlewares/logging.test.ts @@ -1,4 +1,5 @@ -import type { KVStore, Kernel } from '@ocap/kernel'; +import type { Kernel } from '@ocap/kernel'; +import type { KVStore } from '@ocap/store'; import { describe, it, expect, vi } from 'vitest'; import { loggingMiddleware, logger } from './logging.js'; diff --git a/packages/extension/tsconfig.build.json b/packages/extension/tsconfig.build.json index d8a815537..606e20287 100644 --- a/packages/extension/tsconfig.build.json +++ b/packages/extension/tsconfig.build.json @@ -16,7 +16,8 @@ { "path": "../shims/tsconfig.build.json" }, { "path": "../streams/tsconfig.build.json" }, { "path": "../kernel/tsconfig.build.json" }, - { "path": "../utils/tsconfig.build.json" } + { "path": "../utils/tsconfig.build.json" }, + { "path": "../store/tsconfig.build.json" } ], "include": [ "./src/**/*.ts", diff --git a/packages/extension/tsconfig.json b/packages/extension/tsconfig.json index 758d498f1..aec5d70c8 100644 --- a/packages/extension/tsconfig.json +++ b/packages/extension/tsconfig.json @@ -25,7 +25,8 @@ { "path": "../streams" }, { "path": "../test-utils" }, { "path": "../utils" }, - { "path": "../kernel" } + { "path": "../kernel" }, + { "path": "../store" } ], "include": [ "../../vitest.config.ts", @@ -41,6 +42,7 @@ "./vite-plugins/*.ts", "./vite.config.ts", "./vitest.config.ts", - "../../types" + "../../types", + "../store/src/sqlite-kv-store.ts" ] } diff --git a/packages/kernel/package.json b/packages/kernel/package.json index ffd6927cf..57f9d355b 100644 --- a/packages/kernel/package.json +++ b/packages/kernel/package.json @@ -53,6 +53,7 @@ "@metamask/superstruct": "^3.1.0", "@metamask/utils": "^11.0.1", "@ocap/errors": "workspace:^", + "@ocap/store": "workspace:^", "@ocap/streams": "workspace:^", "@ocap/utils": "workspace:^", "ses": "^1.9.0", diff --git a/packages/kernel/src/Kernel.test.ts b/packages/kernel/src/Kernel.test.ts index f2c5909eb..0bddbe012 100644 --- a/packages/kernel/src/Kernel.test.ts +++ b/packages/kernel/src/Kernel.test.ts @@ -1,4 +1,5 @@ import { VatNotFoundError } from '@ocap/errors'; +import type { KVStore } from '@ocap/store'; import type { MessagePortDuplexStream, DuplexStream } from '@ocap/streams'; import type { MockInstance } from 'vitest'; import { describe, it, expect, vi, beforeEach } from 'vitest'; @@ -10,7 +11,6 @@ import type { VatCommand, VatCommandReply, } from './messages/index.js'; -import type { KVStore } from './store/kernel-store.js'; import type { VatId, VatConfig, VatWorkerService } from './types.js'; import { VatHandle } from './VatHandle.js'; import { makeMapKVStore } from '../test/storage.js'; diff --git a/packages/kernel/src/Kernel.ts b/packages/kernel/src/Kernel.ts index b26765be7..ea65378e4 100644 --- a/packages/kernel/src/Kernel.ts +++ b/packages/kernel/src/Kernel.ts @@ -7,6 +7,7 @@ import { VatAlreadyExistsError, VatNotFoundError, } from '@ocap/errors'; +import type { KVStore } from '@ocap/store'; import type { DuplexStream } from '@ocap/streams'; import type { Logger } from '@ocap/utils'; import { makeLogger } from '@ocap/utils'; @@ -26,7 +27,7 @@ import { isPromiseRef, makeKernelStore, } from './store/kernel-store.js'; -import type { KernelStore, KVStore } from './store/kernel-store.js'; +import type { KernelStore } from './store/kernel-store.js'; import type { VatId, VRef, diff --git a/packages/kernel/src/VatSupervisor.ts b/packages/kernel/src/VatSupervisor.ts index 3e4cec47b..dfcd9b725 100644 --- a/packages/kernel/src/VatSupervisor.ts +++ b/packages/kernel/src/VatSupervisor.ts @@ -8,6 +8,7 @@ import { importBundle } from '@endo/import-bundle'; import { makeMarshal } from '@endo/marshal'; import type { CapData } from '@endo/marshal'; import { StreamReadError } from '@ocap/errors'; +import type { MakeKVStore } from '@ocap/store'; import type { DuplexStream } from '@ocap/streams'; import type { @@ -18,7 +19,6 @@ import type { import { makeDummyMeterControl } from './dummyMeterControl.js'; import type { VatCommand, VatCommandReply } from './messages/index.js'; import { VatCommandMethod } from './messages/index.js'; -import type { MakeKVStore } from './store/kernel-store.js'; import { makeSupervisorSyscall } from './syscall.js'; import type { VatConfig, VatId, VRef } from './types.js'; import { ROOT_OBJECT_VREF, isVatConfig } from './types.js'; @@ -198,7 +198,7 @@ export class VatSupervisor { } this.#loaded = true; - const kvStore = await this.#makeKVStore(`[vat-${this.id}]`, true); + const kvStore = await this.#makeKVStore(`[vat-${this.id}]`, this.id); const syscall = makeSupervisorSyscall(this, kvStore); const vatPowers = {}; // XXX should be something more real const liveSlotsOptions = {}; // XXX should be something more real diff --git a/packages/kernel/src/index.ts b/packages/kernel/src/index.ts index 1f1986e38..12d38287d 100644 --- a/packages/kernel/src/index.ts +++ b/packages/kernel/src/index.ts @@ -1,6 +1,5 @@ export * from './messages/index.js'; export { Kernel } from './Kernel.js'; -export type { KVStore, MakeKVStore } from './store/kernel-store.js'; export { VatHandle } from './VatHandle.js'; export { VatSupervisor } from './VatSupervisor.js'; export type { Message } from '@agoric/swingset-liveslots'; diff --git a/packages/kernel/src/store/kernel-store.test.ts b/packages/kernel/src/store/kernel-store.test.ts index 14dc5da0a..3fcc6f726 100644 --- a/packages/kernel/src/store/kernel-store.test.ts +++ b/packages/kernel/src/store/kernel-store.test.ts @@ -1,8 +1,8 @@ import type { Message } from '@agoric/swingset-liveslots'; +import type { KVStore } from '@ocap/store'; import { describe, it, expect, beforeEach } from 'vitest'; import { makeKernelStore } from './kernel-store.js'; -import type { KVStore } from './kernel-store.js'; import { makeMapKVStore } from '../../test/storage.js'; import type { RunQueueItem } from '../types.js'; diff --git a/packages/kernel/src/store/kernel-store.ts b/packages/kernel/src/store/kernel-store.ts index ababb9110..e4d1569b5 100644 --- a/packages/kernel/src/store/kernel-store.ts +++ b/packages/kernel/src/store/kernel-store.ts @@ -56,6 +56,7 @@ import type { Message } from '@agoric/swingset-liveslots'; import { Fail } from '@endo/errors'; import type { CapData } from '@endo/marshal'; +import type { KVStore } from '@ocap/store'; import type { VatId, @@ -69,21 +70,6 @@ import type { } from '../types.js'; import { insistVatId } from '../types.js'; -export type KVStore = { - get(key: string): string | undefined; - getRequired(key: string): string; - getNextKey(previousKey: string): string | undefined; - set(key: string, value: string): void; - delete(key: string): void; - clear(): void; - executeQuery(sql: string): Record[]; -}; - -export type MakeKVStore = ( - label: string, - beEphemeral: boolean, -) => Promise; - type StoredValue = { get(): string | undefined; set(newValue: string): void; diff --git a/packages/kernel/src/syscall.ts b/packages/kernel/src/syscall.ts index 9153d0838..f379823be 100644 --- a/packages/kernel/src/syscall.ts +++ b/packages/kernel/src/syscall.ts @@ -7,9 +7,9 @@ import type { VatOneResolution, } from '@agoric/swingset-liveslots'; import type { CapData } from '@endo/marshal'; +import type { KVStore } from '@ocap/store'; import type { Syscall, SyscallResult } from './ag-liveslots-types.js'; -import type { KVStore } from './store/kernel-store.js'; import type { VatSupervisor } from './VatSupervisor.ts'; /** diff --git a/packages/kernel/test/storage.ts b/packages/kernel/test/storage.ts index 15cd00e9e..22b82d693 100644 --- a/packages/kernel/test/storage.ts +++ b/packages/kernel/test/storage.ts @@ -1,4 +1,4 @@ -import type { KVStore } from '../src/store/kernel-store.js'; +import type { KVStore } from '@ocap/store'; /** * A mock key/value store realized as a Map. diff --git a/packages/kernel/tsconfig.build.json b/packages/kernel/tsconfig.build.json index 7ee75495a..94f13fc76 100644 --- a/packages/kernel/tsconfig.build.json +++ b/packages/kernel/tsconfig.build.json @@ -10,7 +10,8 @@ "references": [ { "path": "../streams/tsconfig.build.json" }, { "path": "../utils/tsconfig.build.json" }, - { "path": "../errors/tsconfig.build.json" } + { "path": "../errors/tsconfig.build.json" }, + { "path": "../store/tsconfig.build.json" } ], - "include": ["./src", "../../types"] + "include": ["./src"] } diff --git a/packages/kernel/tsconfig.json b/packages/kernel/tsconfig.json index b429699bf..5f8cdd350 100644 --- a/packages/kernel/tsconfig.json +++ b/packages/kernel/tsconfig.json @@ -7,7 +7,8 @@ { "path": "../errors" }, { "path": "../streams" }, { "path": "../utils" }, - { "path": "../test-utils" } + { "path": "../test-utils" }, + { "path": "../store" } ], "include": [ "./src", diff --git a/packages/nodejs/package.json b/packages/nodejs/package.json index e178ff38d..a5842f91a 100644 --- a/packages/nodejs/package.json +++ b/packages/nodejs/package.json @@ -40,6 +40,15 @@ "test:verbose": "yarn test --reporter verbose", "test:watch": "vitest --config vitest.config.ts" }, + "dependencies": { + "@endo/promise-kit": "^1.1.6", + "@ocap/kernel": "workspace:^", + "@ocap/shims": "workspace:^", + "@ocap/store": "workspace:^", + "@ocap/streams": "workspace:^", + "@ocap/utils": "workspace:^", + "ses": "^1.9.0" + }, "devDependencies": { "@arethetypeswrong/cli": "^0.17.3", "@metamask/auto-changelog": "^4.0.0", @@ -50,7 +59,6 @@ "@ocap/test-utils": "workspace:^", "@ts-bridge/cli": "^0.6.2", "@ts-bridge/shims": "^0.1.1", - "@types/better-sqlite3": "^7.6.12", "@typescript-eslint/eslint-plugin": "^8.8.1", "@typescript-eslint/parser": "^8.8.1", "@typescript-eslint/utils": "^8.8.1", @@ -75,14 +83,5 @@ }, "engines": { "node": "^20 || >=22" - }, - "dependencies": { - "@endo/promise-kit": "^1.1.6", - "@ocap/kernel": "workspace:^", - "@ocap/shims": "workspace:^", - "@ocap/streams": "workspace:^", - "@ocap/utils": "workspace:^", - "better-sqlite3": "^11.7.2", - "ses": "^1.9.0" } } diff --git a/packages/nodejs/src/kernel/make-kernel.test.ts b/packages/nodejs/src/kernel/make-kernel.test.ts index 0a304077a..f79f7870c 100644 --- a/packages/nodejs/src/kernel/make-kernel.test.ts +++ b/packages/nodejs/src/kernel/make-kernel.test.ts @@ -9,7 +9,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import { makeKernel } from './make-kernel.js'; -vi.mock('./sqlite-kv-store.js', async () => { +vi.mock('@ocap/store/sqlite/native', async () => { const { makeMapKVStore } = await import('../../../kernel/test/storage.js'); return { makeSQLKVStore: makeMapKVStore, diff --git a/packages/nodejs/src/kernel/make-kernel.ts b/packages/nodejs/src/kernel/make-kernel.ts index aff23f67a..3591aaf6c 100644 --- a/packages/nodejs/src/kernel/make-kernel.ts +++ b/packages/nodejs/src/kernel/make-kernel.ts @@ -1,9 +1,9 @@ import type { KernelCommand, KernelCommandReply } from '@ocap/kernel'; import { Kernel } from '@ocap/kernel'; +import { makeSQLKVStore } from '@ocap/store/sqlite/native'; import { NodeWorkerDuplexStream } from '@ocap/streams'; import { MessagePort as NodeMessagePort } from 'node:worker_threads'; -import { makeSQLKVStore } from './sqlite-kv-store.js'; import { NodejsVatWorkerService } from './VatWorkerService.js'; /** diff --git a/packages/nodejs/src/vat/vat-worker.ts b/packages/nodejs/src/vat/vat-worker.ts index 944a59722..6980e27e7 100644 --- a/packages/nodejs/src/vat/vat-worker.ts +++ b/packages/nodejs/src/vat/vat-worker.ts @@ -2,10 +2,10 @@ import '@ocap/shims/endoify'; import type { VatId } from '@ocap/kernel'; import { VatSupervisor } from '@ocap/kernel'; +import { makeSQLKVStore } from '@ocap/store/sqlite/native'; import { makeLogger } from '@ocap/utils'; import { makeCommandStream } from './streams'; -import { makeSQLKVStore } from '../kernel/sqlite-kv-store'; const vatId = process.env.NODE_VAT_ID as VatId; diff --git a/packages/nodejs/tsconfig.build.json b/packages/nodejs/tsconfig.build.json index 7799c75f6..aed373d2f 100644 --- a/packages/nodejs/tsconfig.build.json +++ b/packages/nodejs/tsconfig.build.json @@ -1,22 +1,17 @@ { "extends": "../../tsconfig.packages.build.json", "compilerOptions": { - "allowJs": true, "baseUrl": "./", - "emitDeclarationOnly": false, - "lib": ["DOM", "ES2022"], - "noEmit": true, + "outDir": "./dist", "rootDir": "./src", + "lib": ["DOM", "ES2022"], "types": ["chrome", "ses"] }, "references": [ + { "path": "../utils/tsconfig.build.json" }, { "path": "../streams/tsconfig.build.json" }, - { "path": "../kernel/tsconfig.build.json" }, - { "path": "../utils/tsconfig.build.json" } + { "path": "../store/tsconfig.build.json" }, + { "path": "../kernel/tsconfig.build.json" } ], - "include": [ - "./src/**/*.ts", - "./src/**/*-trusted-prelude.js", - "./src/env/dev-console.js" - ] + "include": ["./src"] } diff --git a/packages/nodejs/tsconfig.json b/packages/nodejs/tsconfig.json index 1288b3e03..257e52f13 100644 --- a/packages/nodejs/tsconfig.json +++ b/packages/nodejs/tsconfig.json @@ -13,7 +13,9 @@ { "path": "../kernel" }, { "path": "../streams" }, { "path": "../utils" }, - { "path": "../test-utils" } + { "path": "../test-utils" }, + { "path": "../kernel" }, + { "path": "../store" } ], "include": [ "../../vitest.config.packages.ts", @@ -24,6 +26,7 @@ "./vitest.config.ts", "./vitest.config.e2e.ts", "../../types", - "./test/workers/*.js" + "./test/workers/*.js", + "../store/src/native.ts" ] } diff --git a/packages/nodejs/vitest.config.e2e.ts b/packages/nodejs/vitest.config.e2e.ts index b8ed52fb6..2a084d69a 100644 --- a/packages/nodejs/vitest.config.e2e.ts +++ b/packages/nodejs/vitest.config.e2e.ts @@ -1,31 +1,15 @@ -// eslint-disable-next-line spaced-comment -/// - -import { mergeConfig } from '@ocap/test-utils/vitest-config'; -import path from 'path'; -import { defineConfig, defineProject } from 'vitest/config'; +import { defineConfig, mergeConfig } from 'vite'; import defaultConfig from '../../vitest.config.js'; -export default defineConfig((args) => { - return mergeConfig( - args, - defaultConfig, - defineProject({ - optimizeDeps: { include: ['better-sqlite3'] }, - test: { - name: 'nodejs:e2e', - pool: 'forks', - alias: [ - { - find: '@ocap/shims/endoify', - replacement: path.resolve('../shims/src/endoify.js'), - customResolver: (id) => ({ external: true, id }), - }, - ], - include: ['./test/e2e/**/*.test.ts'], - exclude: ['./src/**/*'], - }, - }), - ); -}); +export default mergeConfig( + defaultConfig, + defineConfig({ + test: { + name: 'nodejs:e2e', + pool: 'forks', + include: ['./test/e2e/**/*.test.ts'], + exclude: ['./src/**/*'], + }, + }), +); diff --git a/packages/nodejs/vitest.config.ts b/packages/nodejs/vitest.config.ts index 5ac197921..30a618f2e 100644 --- a/packages/nodejs/vitest.config.ts +++ b/packages/nodejs/vitest.config.ts @@ -1,28 +1,15 @@ -import { mergeConfig } from '@ocap/test-utils/vitest-config'; -import path from 'node:path'; -import { defineConfig, defineProject } from 'vitest/config'; +import { defineProject, mergeConfig } from 'vitest/config'; import defaultConfig from '../../vitest.config.js'; -export default defineConfig((args) => { - return mergeConfig( - args, - defaultConfig, - defineProject({ - optimizeDeps: { include: ['better-sqlite3'] }, - test: { - name: 'nodejs', - pool: 'forks', - alias: [ - { - find: '@ocap/shims/endoify', - replacement: path.resolve(__dirname, '../shims/src/endoify.js'), - customResolver: (id) => ({ external: true, id }), - }, - ], - include: ['./src/**/*.test.ts'], - exclude: ['./test/e2e/'], - }, - }), - ); -}); +export default mergeConfig( + defaultConfig, + defineProject({ + test: { + name: 'nodejs', + pool: 'forks', + include: ['./src/**/*.test.ts'], + exclude: ['./test/e2e/'], + }, + }), +); diff --git a/packages/store/CHANGELOG.md b/packages/store/CHANGELOG.md new file mode 100644 index 000000000..0c82cb1ed --- /dev/null +++ b/packages/store/CHANGELOG.md @@ -0,0 +1,10 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +[Unreleased]: https://github.com/MetaMask/ocap-kernel/ diff --git a/packages/store/README.md b/packages/store/README.md new file mode 100644 index 000000000..9ef74d49f --- /dev/null +++ b/packages/store/README.md @@ -0,0 +1,15 @@ +# `@ocap/store` + +Ocap Kernel storage abstractions and implementations. + +## Installation + +`yarn add @ocap/store` + +or + +`npm install @ocap/store` + +## Contributing + +This package is part of a monorepo. Instructions for contributing can be found in the [monorepo README](https://github.com/MetaMask/ocap-kernel#readme). diff --git a/packages/store/package.json b/packages/store/package.json new file mode 100644 index 000000000..b67f25d78 --- /dev/null +++ b/packages/store/package.json @@ -0,0 +1,108 @@ +{ + "name": "@ocap/store", + "version": "0.0.0", + "private": true, + "description": "Ocap Kernel storage abstractions and implementations", + "homepage": "https://github.com/MetaMask/ocap-kernel/tree/main/packages/store#readme", + "bugs": { + "url": "https://github.com/MetaMask/ocap-kernel/issues" + }, + "repository": { + "type": "git", + "url": "https://github.com/MetaMask/ocap-kernel.git" + }, + "type": "module", + "exports": { + ".": { + "import": { + "types": "./dist/index.d.mts", + "default": "./dist/index.mjs" + }, + "require": { + "types": "./dist/index.d.cts", + "default": "./dist/index.cjs" + } + }, + "./sqlite/native": { + "import": { + "types": "./dist/sqlite/native.d.mts", + "default": "./dist/sqlite/native.mjs" + }, + "require": { + "types": "./dist/sqlite/native.d.cts", + "default": "./dist/sqlite/native.cjs" + } + }, + "./sqlite/wasm": { + "import": { + "types": "./dist/sqlite/wasm.d.mts", + "default": "./dist/sqlite/wasm.mjs" + }, + "require": { + "types": "./dist/sqlite/wasm.d.cts", + "default": "./dist/sqlite/wasm.cjs" + } + }, + "./package.json": "./package.json" + }, + "files": [ + "dist/" + ], + "scripts": { + "build": "ts-bridge --project tsconfig.build.json --clean", + "build:docs": "typedoc", + "changelog:validate": "../../scripts/validate-changelog.sh @ocap/store", + "clean": "rimraf --glob './*.tsbuildinfo' ./.eslintcache ./coverage ./dist", + "lint": "yarn lint:ts && yarn lint:eslint && yarn lint:misc --check && yarn constraints && yarn lint:dependencies", + "lint:dependencies": "depcheck", + "lint:eslint": "eslint . --cache", + "lint:fix": "yarn lint:ts && yarn lint:eslint --fix && yarn lint:misc --write && yarn constraints --fix && yarn lint:dependencies", + "lint:misc": "prettier --no-error-on-unmatched-pattern '**/*.json' '**/*.md' '**/*.html' '!**/CHANGELOG.old.md' '**/*.yml' '!.yarnrc.yml' '!merged-packages/**' --ignore-path ../../.gitignore", + "lint:ts": "tsc --project tsconfig.lint.json", + "publish:preview": "yarn npm publish --tag preview", + "test": "vitest run --config vitest.config.ts", + "test:clean": "yarn test --no-cache --coverage.clean", + "test:dev": "yarn test --coverage false", + "test:verbose": "yarn test --reporter verbose", + "test:watch": "vitest --config vitest.config.ts" + }, + "dependencies": { + "@ocap/utils": "workspace:^", + "@sqlite.org/sqlite-wasm": "^3.48.0-build1", + "better-sqlite3": "^11.7.2", + "ses": "^1.9.0" + }, + "devDependencies": { + "@arethetypeswrong/cli": "^0.17.3", + "@metamask/auto-changelog": "^4.0.0", + "@metamask/eslint-config": "^14.0.0", + "@metamask/eslint-config-nodejs": "^14.0.0", + "@metamask/eslint-config-typescript": "^14.0.0", + "@ts-bridge/cli": "^0.6.2", + "@ts-bridge/shims": "^0.1.1", + "@types/better-sqlite3": "^7.6.12", + "@typescript-eslint/eslint-plugin": "^8.8.1", + "@typescript-eslint/parser": "^8.8.1", + "@typescript-eslint/utils": "^8.8.1", + "@vitest/eslint-plugin": "^1.1.25", + "depcheck": "^1.4.7", + "eslint": "^9.12.0", + "eslint-config-prettier": "^9.1.0", + "eslint-import-resolver-typescript": "^3.6.3", + "eslint-plugin-import-x": "^4.3.1", + "eslint-plugin-jsdoc": "^50.3.1", + "eslint-plugin-n": "^17.11.1", + "eslint-plugin-prettier": "^5.2.1", + "eslint-plugin-promise": "^7.1.0", + "prettier": "^3.3.3", + "rimraf": "^6.0.1", + "typedoc": "^0.27.6", + "typescript": "~5.5.4", + "typescript-eslint": "^8.8.1", + "vite": "^6.0.11", + "vitest": "3.0.4" + }, + "engines": { + "node": "^20 || >=22" + } +} diff --git a/packages/store/src/index.ts b/packages/store/src/index.ts new file mode 100644 index 000000000..623f278ae --- /dev/null +++ b/packages/store/src/index.ts @@ -0,0 +1 @@ +export type { KVStore, MakeKVStore } from './types.js'; diff --git a/packages/store/src/sqlite/common.test.ts b/packages/store/src/sqlite/common.test.ts new file mode 100644 index 000000000..332e43d04 --- /dev/null +++ b/packages/store/src/sqlite/common.test.ts @@ -0,0 +1,50 @@ +import { describe, it, expect } from 'vitest'; + +import { SQL_QUERIES } from './common'; + +describe('SQL_QUERIES', () => { + it.each([ + [ + 'CREATE_TABLE', + 'CREATE TABLE IF NOT EXISTS kv ( key TEXT, value TEXT, PRIMARY KEY(key) )', + 'creates a key-value table with proper schema', + ], + ['GET', 'SELECT value FROM kv WHERE key = ?', 'retrieves a value by key'], + [ + 'GET_NEXT', + 'SELECT key FROM kv WHERE key > ? LIMIT 1', + 'gets the next key in sequence', + ], + [ + 'SET', + 'INSERT INTO kv (key, value) VALUES (?, ?) ON CONFLICT DO UPDATE SET value = excluded.value', + 'inserts or updates a key-value pair', + ], + [ + 'DELETE', + 'DELETE FROM kv WHERE key = ?', + 'deletes a specific key-value pair', + ], + ['CLEAR', 'DELETE FROM kv', 'deletes all key-value pairs'], + ['DROP', 'DROP TABLE kv', 'drops the entire table'], + ] as const)( + 'has the expected %s query (%s)', + (queryName, expectedSql, _description) => { + expect(SQL_QUERIES[queryName].trim().replace(/\s+/gu, ' ')).toBe( + expectedSql, + ); + }, + ); + + it('has all expected query properties', () => { + expect(Object.keys(SQL_QUERIES).sort()).toStrictEqual([ + 'CLEAR', + 'CREATE_TABLE', + 'DELETE', + 'DROP', + 'GET', + 'GET_NEXT', + 'SET', + ]); + }); +}); diff --git a/packages/store/src/sqlite/common.ts b/packages/store/src/sqlite/common.ts new file mode 100644 index 000000000..50573728b --- /dev/null +++ b/packages/store/src/sqlite/common.ts @@ -0,0 +1,35 @@ +export const SQL_QUERIES = { + CREATE_TABLE: ` + CREATE TABLE IF NOT EXISTS kv ( + key TEXT, + value TEXT, + PRIMARY KEY(key) + ) + `, + GET: ` + SELECT value + FROM kv + WHERE key = ? + `, + GET_NEXT: ` + SELECT key + FROM kv + WHERE key > ? + LIMIT 1 + `, + SET: ` + INSERT INTO kv (key, value) + VALUES (?, ?) + ON CONFLICT DO UPDATE SET value = excluded.value + `, + DELETE: ` + DELETE FROM kv + WHERE key = ? + `, + CLEAR: ` + DELETE FROM kv + `, + DROP: ` + DROP TABLE kv + `, +} as const; diff --git a/packages/store/src/sqlite/native.test.ts b/packages/store/src/sqlite/native.test.ts new file mode 100644 index 000000000..d00ad4c6e --- /dev/null +++ b/packages/store/src/sqlite/native.test.ts @@ -0,0 +1,93 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +import { SQL_QUERIES } from './common.js'; +import { makeSQLKVStore } from './native.js'; + +const mockStatement = { + run: vi.fn(), + get: vi.fn(), + all: vi.fn(), + pluck: vi.fn(), +}; + +const mockDb = { + prepare: vi.fn(() => mockStatement), + transaction: vi.fn((fn) => fn), +}; + +vi.mock('better-sqlite3', () => ({ + default: vi.fn(() => mockDb), +})); + +vi.mock('fs/promises', () => ({ + mkdir: vi.fn(), +})); + +describe('makeSQLKVStore', () => { + beforeEach(() => { + Object.values(mockStatement).forEach((mock) => mock.mockReset()); + }); + + it('creates kv table', async () => { + await makeSQLKVStore(); + expect(mockDb.prepare).toHaveBeenCalledWith(SQL_QUERIES.CREATE_TABLE); + }); + + it('get retrieves a value by key', async () => { + const mockValue = 'test-value'; + mockStatement.get.mockReturnValue(mockValue); + const store = await makeSQLKVStore(); + const result = store.get('test-key'); + expect(result).toBe(mockValue); + expect(mockStatement.get).toHaveBeenCalledWith('test-key'); + }); + + it('getRequired throws when key not found', async () => { + mockStatement.get.mockReturnValue(undefined); + const store = await makeSQLKVStore(); + expect(() => store.getRequired('missing-key')).toThrow( + "no record matching key 'missing-key'", + ); + }); + + it('set inserts or updates a value', async () => { + const store = await makeSQLKVStore(); + store.set('test-key', 'test-value'); + expect(mockStatement.run).toHaveBeenCalledWith('test-key', 'test-value'); + }); + + it('delete removes a key-value pair', async () => { + const store = await makeSQLKVStore(); + store.delete('test-key'); + expect(mockStatement.run).toHaveBeenCalledWith('test-key'); + }); + + it('clear drops and recreates the table', async () => { + const store = await makeSQLKVStore(); + store.clear(); + expect(mockStatement.run).toHaveBeenCalledTimes(3); + }); + + it('executeQuery runs arbitrary SQL queries', async () => { + const mockResults = [{ key: 'value' }]; + mockStatement.all.mockReturnValue(mockResults); + const store = await makeSQLKVStore(); + const result = store.executeQuery('SELECT * FROM kv'); + expect(result).toStrictEqual(mockResults); + }); + + it('getNextKey returns the next key in sequence', async () => { + const mockNextKey = 'next-key'; + mockStatement.get.mockReturnValue(mockNextKey); + const store = await makeSQLKVStore(); + const result = store.getNextKey('current-key'); + expect(result).toBe(mockNextKey); + expect(mockStatement.get).toHaveBeenCalledWith('current-key'); + }); + + it('getNextKey throws if previousKey is not a string', async () => { + const store = await makeSQLKVStore(); + // @ts-expect-error Testing invalid input + expect(() => store.getNextKey(123)).toThrow('must be a string'); + }); +}); diff --git a/packages/nodejs/src/kernel/sqlite-kv-store.ts b/packages/store/src/sqlite/native.ts similarity index 72% rename from packages/nodejs/src/kernel/sqlite-kv-store.ts rename to packages/store/src/sqlite/native.ts index 0cf698bb0..d4effae3f 100644 --- a/packages/nodejs/src/kernel/sqlite-kv-store.ts +++ b/packages/store/src/sqlite/native.ts @@ -1,30 +1,35 @@ -import type { KVStore } from '@ocap/kernel'; import { makeLogger } from '@ocap/utils'; import type { Database } from 'better-sqlite3'; +// eslint-disable-next-line @typescript-eslint/naming-convention +import Sqlite from 'better-sqlite3'; import { mkdir } from 'fs/promises'; import { tmpdir } from 'os'; import { join } from 'path'; +import { SQL_QUERIES } from './common.js'; +import type { KVStore } from '../types.js'; // We require require because the ESM import does not work properly. -// eslint-disable-next-line @typescript-eslint/no-require-imports -const Sqlite = require('better-sqlite3'); -const dbRoot = join(tmpdir(), './db'); +// const Sqlite = require('better-sqlite3'); + +const dbRoot = join(tmpdir(), './ocap-sqlite'); /** * Ensure that SQLite is initialized. * + * @param dbFilename - The filename of the database to use. * @param logger - An optional logger to pass to the Sqlite constructor. * @returns The SQLite database object. */ async function initDB( - logger?: ReturnType, + dbFilename: string, + logger: ReturnType, ): Promise { - const dbPath = join(dbRoot, 'store.db'); - console.log('dbPath:', dbPath); + const dbPath = join(dbRoot, `${dbFilename}.db`); + logger.debug('dbPath:', dbPath); await mkdir(dbRoot, { recursive: true }); return new Sqlite(dbPath, { - verbose: (logger ?? console).info, + verbose: logger.info, }); } @@ -32,29 +37,21 @@ async function initDB( * Makes a {@link KVStore} for low-level persistent storage. * * @param label - A logger prefix label. Defaults to '[sqlite]'. + * @param dbFilename - The filename of the database to use. Defaults to 'store.db'. * @returns The key/value store to base the kernel store on. */ export async function makeSQLKVStore( label: string = '[sqlite]', + dbFilename: string = 'store', ): Promise { const logger = makeLogger(label); - const db = await initDB(logger); + const db = await initDB(dbFilename, logger); - const sqlKVInit = db.prepare(` - CREATE TABLE IF NOT EXISTS kv ( - key TEXT, - value TEXT, - PRIMARY KEY(key) - ) - `); + const sqlKVInit = db.prepare(SQL_QUERIES.CREATE_TABLE); sqlKVInit.run(); - const sqlKVGet = db.prepare(` - SELECT value - FROM kv - WHERE key = ? - `); + const sqlKVGet = db.prepare<[string], string>(SQL_QUERIES.GET); sqlKVGet.pluck(true); /** @@ -64,20 +61,15 @@ export async function makeSQLKVStore( * @param required - True if it is an error for the entry not to be there. * @returns The value at that key. */ - function kvGet(key: string, required: boolean): string { + function kvGet(key: string, required: boolean): string | undefined { const result = sqlKVGet.get(key); if (required && !result) { throw Error(`no record matching key '${key}'`); } - return result as string; + return result; } - const sqlKVGetNextKey = db.prepare(` - SELECT key - FROM kv - WHERE key > ? - LIMIT 1 - `); + const sqlKVGetNextKey = db.prepare(SQL_QUERIES.GET_NEXT); sqlKVGetNextKey.pluck(true); /** @@ -96,11 +88,7 @@ export async function makeSQLKVStore( return sqlKVGetNextKey.get(previousKey) as string | undefined; } - const sqlKVSet = db.prepare(` - INSERT INTO kv (key, value) - VALUES (?, ?) - ON CONFLICT DO UPDATE SET value = excluded.value - `); + const sqlKVSet = db.prepare(SQL_QUERIES.SET); /** * Set the value associated with a key in the database. @@ -112,10 +100,7 @@ export async function makeSQLKVStore( sqlKVSet.run(key, value); } - const sqlKVDelete = db.prepare(` - DELETE FROM kv - WHERE key = ? - `); + const sqlKVDelete = db.prepare(SQL_QUERIES.DELETE); /** * Delete a key from the database. @@ -126,9 +111,7 @@ export async function makeSQLKVStore( sqlKVDelete.run(key); } - const sqlKVDrop = db.prepare(` - DROP TABLE kv - `); + const sqlKVDrop = db.prepare(SQL_QUERIES.DROP); /** * Delete all keys and values from the database. @@ -152,7 +135,7 @@ export async function makeSQLKVStore( return { get: (key) => kvGet(key, false), getNextKey: kvGetNextKey, - getRequired: (key) => kvGet(key, true), + getRequired: (key) => kvGet(key, true) as string, set: kvSet, delete: kvDelete, executeQuery: kvExecuteQuery, diff --git a/packages/store/src/sqlite/wasm.test.ts b/packages/store/src/sqlite/wasm.test.ts new file mode 100644 index 000000000..57f2894dd --- /dev/null +++ b/packages/store/src/sqlite/wasm.test.ts @@ -0,0 +1,213 @@ +import type { Sqlite3Static } from '@sqlite.org/sqlite-wasm'; +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +import { SQL_QUERIES } from './common.js'; +import { makeSQLKVStore } from './wasm.js'; + +const mockStatement = { + bind: vi.fn(), + step: vi.fn(), + getString: vi.fn(), + reset: vi.fn(), + get: vi.fn(), + getColumnName: vi.fn(), + columnCount: 2, +}; + +const mockDb = { + exec: vi.fn(), + prepare: vi.fn(() => mockStatement), +}; + +vi.mock('@sqlite.org/sqlite-wasm', () => ({ + default: vi.fn(async () => ({ + oo1: { + OpfsDb: vi.fn(() => mockDb), + DB: vi.fn(() => mockDb), + }, + })), +})); + +describe('makeSQLKVStore', () => { + beforeEach(() => { + Object.values(mockStatement) + .filter( + (value): value is ReturnType => + typeof value === 'function', + ) + .forEach((mockFn) => mockFn.mockReset()); + }); + + it('initializes with OPFS when available', async () => { + await makeSQLKVStore(); + expect(mockDb.exec).toHaveBeenCalledWith(SQL_QUERIES.CREATE_TABLE); + }); + + it('falls back to in-memory when OPFS is not available', async () => { + vi.mocked( + await import('@sqlite.org/sqlite-wasm'), + ).default.mockImplementationOnce( + async () => + ({ + oo1: { + OpfsDb: undefined, + DB: vi.fn(() => mockDb), + }, + }) as unknown as Sqlite3Static, + ); + const consoleSpy = vi.spyOn(console, 'warn'); + await makeSQLKVStore(); + expect(consoleSpy).toHaveBeenCalledWith( + 'OPFS not enabled, database will be ephemeral', + ); + expect(mockDb.exec).toHaveBeenCalledWith(SQL_QUERIES.CREATE_TABLE); + }); + + it('get retrieves a value by key', async () => { + const mockValue = 'test-value'; + mockStatement.step.mockReturnValueOnce(true); + mockStatement.getString.mockReturnValueOnce(mockValue); + const store = await makeSQLKVStore(); + const result = store.get('test-key'); + expect(result).toBe(mockValue); + expect(mockStatement.bind).toHaveBeenCalledWith(['test-key']); + }); + + it('getRequired throws when key not found', async () => { + mockStatement.step.mockReturnValueOnce(false); + const store = await makeSQLKVStore(); + expect(() => store.getRequired('missing-key')).toThrow( + "no record matching key 'missing-key'", + ); + }); + + it('set inserts or updates a value', async () => { + const store = await makeSQLKVStore(); + store.set('test-key', 'test-value'); + expect(mockStatement.bind).toHaveBeenCalledWith(['test-key', 'test-value']); + expect(mockStatement.step).toHaveBeenCalled(); + expect(mockStatement.reset).toHaveBeenCalled(); + }); + + it('delete removes a key-value pair', async () => { + const store = await makeSQLKVStore(); + store.delete('test-key'); + expect(mockStatement.bind).toHaveBeenCalledWith(['test-key']); + expect(mockStatement.step).toHaveBeenCalled(); + expect(mockStatement.reset).toHaveBeenCalled(); + }); + + it('clear removes all entries', async () => { + const store = await makeSQLKVStore(); + store.clear(); + expect(mockStatement.step).toHaveBeenCalled(); + expect(mockStatement.reset).toHaveBeenCalled(); + }); + + it('getNextKey returns the next key in sequence', async () => { + const mockNextKey = 'next-key'; + mockStatement.step.mockReturnValueOnce(true); + mockStatement.getString.mockReturnValueOnce(mockNextKey); + const store = await makeSQLKVStore(); + const result = store.getNextKey('current-key'); + expect(result).toBe(mockNextKey); + expect(mockStatement.bind).toHaveBeenCalledWith(['current-key']); + }); + + it('executeQuery executes arbitrary SQL queries', async () => { + mockStatement.step + .mockReturnValueOnce(true) + .mockReturnValueOnce(true) + .mockReturnValueOnce(false); + mockStatement.getColumnName + .mockReturnValueOnce('id') + .mockReturnValueOnce('value') + .mockReturnValueOnce('id') + .mockReturnValueOnce('value'); + mockStatement.get + .mockReturnValueOnce('1') + .mockReturnValueOnce('first') + .mockReturnValueOnce('2') + .mockReturnValueOnce('second'); + const store = await makeSQLKVStore(); + const results = store.executeQuery('SELECT * FROM kv'); + expect(results).toStrictEqual([ + { id: '1', value: 'first' }, + { id: '2', value: 'second' }, + ]); + expect(mockStatement.reset).toHaveBeenCalled(); + }); + + it('get returns undefined when step() returns false', async () => { + mockStatement.step.mockReturnValueOnce(false); + const store = await makeSQLKVStore(); + const result = store.get('test-key'); + expect(result).toBeUndefined(); + expect(mockStatement.bind).toHaveBeenCalledWith(['test-key']); + expect(mockStatement.reset).toHaveBeenCalled(); + }); + + it('get returns undefined when getString() returns falsy value', async () => { + mockStatement.step.mockReturnValueOnce(true); + mockStatement.getString.mockReturnValueOnce(''); + const store = await makeSQLKVStore(); + const result = store.get('test-key'); + expect(result).toBeUndefined(); + expect(mockStatement.bind).toHaveBeenCalledWith(['test-key']); + expect(mockStatement.reset).toHaveBeenCalled(); + }); + + it('executeQuery skips columns with null/undefined names', async () => { + mockStatement.step.mockReturnValueOnce(true).mockReturnValueOnce(false); + mockStatement.getColumnName + .mockReturnValueOnce('id') + .mockReturnValueOnce(null) + .mockReturnValueOnce(undefined); + mockStatement.get + .mockReturnValueOnce('1') + .mockReturnValueOnce('ignored') + .mockReturnValueOnce('also-ignored'); + const store = await makeSQLKVStore(); + const results = store.executeQuery('SELECT * FROM kv'); + expect(results).toStrictEqual([{ id: '1' }]); + expect(mockStatement.reset).toHaveBeenCalled(); + }); + + it('executeQuery handles non-string values by converting them to strings', async () => { + mockStatement.step.mockReturnValueOnce(true).mockReturnValueOnce(false); + mockStatement.getColumnName + .mockReturnValueOnce('id') + .mockReturnValueOnce('number'); + mockStatement.get.mockReturnValueOnce('1').mockReturnValueOnce(42); + const store = await makeSQLKVStore(); + const results = store.executeQuery('SELECT * FROM kv'); + expect(results).toStrictEqual([ + { + id: '1', + number: '42', + }, + ]); + expect(mockStatement.reset).toHaveBeenCalled(); + }); + + describe('KVStore operations', () => { + it('getNextKey returns undefined when no next key exists', async () => { + mockStatement.step.mockReturnValueOnce(false); + const store = await makeSQLKVStore(); + const result = store.getNextKey('last-key'); + expect(result).toBeUndefined(); + expect(mockStatement.bind).toHaveBeenCalledWith(['last-key']); + expect(mockStatement.reset).toHaveBeenCalled(); + }); + + it('getNextKey returns undefined when getString returns falsy', async () => { + mockStatement.step.mockReturnValueOnce(true); + mockStatement.getString.mockReturnValueOnce(''); + const store = await makeSQLKVStore(); + const result = store.getNextKey('current-key'); + expect(result).toBeUndefined(); + expect(mockStatement.bind).toHaveBeenCalledWith(['current-key']); + expect(mockStatement.reset).toHaveBeenCalled(); + }); + }); +}); diff --git a/packages/extension/src/kernel-integration/sqlite-kv-store.ts b/packages/store/src/sqlite/wasm.ts similarity index 72% rename from packages/extension/src/kernel-integration/sqlite-kv-store.ts rename to packages/store/src/sqlite/wasm.ts index afca7b623..a73bd50ca 100644 --- a/packages/extension/src/kernel-integration/sqlite-kv-store.ts +++ b/packages/store/src/sqlite/wasm.ts @@ -1,54 +1,45 @@ -import type { KVStore } from '@ocap/kernel'; import { makeLogger } from '@ocap/utils'; import type { Database } from '@sqlite.org/sqlite-wasm'; import sqlite3InitModule from '@sqlite.org/sqlite-wasm'; +import { SQL_QUERIES } from './common.js'; +import type { KVStore } from '../types.js'; + /** * Ensure that SQLite is initialized. * - * @param beEphemeral - If true, create an ephemeral (in memory) database. + * @param dbFilename - The filename of the database to use. * @returns The SQLite database object. */ -async function initDB(beEphemeral: boolean): Promise { +async function initDB(dbFilename: string): Promise { const sqlite3 = await sqlite3InitModule(); - if (!beEphemeral) { - if (sqlite3.oo1.OpfsDb) { - return new sqlite3.oo1.OpfsDb('/testdb.sqlite', 'cw'); - } - console.warn(`OPFS not enabled, database will be ephemeral`); + if (sqlite3.oo1.OpfsDb) { + return new sqlite3.oo1.OpfsDb(`/${dbFilename}.db`, 'cw'); } - return new sqlite3.oo1.DB(':memory:', 'cw'); + console.warn(`OPFS not enabled, database will be ephemeral`); + + return new sqlite3.oo1.DB(`:memory:`, 'cw'); } /** * Makes a {@link KVStore} for low-level persistent storage. * * @param label - A logger prefix label. Defaults to '[sqlite]'. - * @param beEphemeral - If true, create an ephemeral (in memory) database. + * @param dbFilename - The filename of the database to use. Defaults to 'store.sqlite'. * @returns A key/value store to base higher level stores on. */ export async function makeSQLKVStore( label: string = '[sqlite]', - beEphemeral: boolean = false, + dbFilename: string = 'store', ): Promise { const logger = makeLogger(label); - const db = await initDB(beEphemeral); + const db = await initDB(dbFilename); logger.log('Initializing kv store'); - db.exec(` - CREATE TABLE IF NOT EXISTS kv ( - key TEXT, - value TEXT, - PRIMARY KEY(key) - ) - `); + db.exec(SQL_QUERIES.CREATE_TABLE); - const sqlKVGet = db.prepare(` - SELECT value - FROM kv - WHERE key = ? - `); + const sqlKVGet = db.prepare(SQL_QUERIES.GET); /** * Read a key's value from the database. @@ -57,7 +48,7 @@ export async function makeSQLKVStore( * @param required - True if it is an error for the entry not to be there. * @returns The value at that key. */ - function kvGet(key: string, required: boolean): string { + function kvGet(key: string, required: boolean): string | undefined { sqlKVGet.bind([key]); if (sqlKVGet.step()) { const result = sqlKVGet.getString(0); @@ -70,18 +61,11 @@ export async function makeSQLKVStore( sqlKVGet.reset(); if (required) { throw Error(`[${label}] no record matching key '${key}'`); - } else { - // Sometimes, we really lean on TypeScript's unsoundness - return undefined as unknown as string; } + return undefined; } - const sqlKVGetNextKey = db.prepare(` - SELECT key - FROM kv - WHERE key > ? - LIMIT 1 - `); + const sqlKVGetNextKey = db.prepare(SQL_QUERIES.GET_NEXT); /** * Get the lexicographically next key in the KV store after a given key. @@ -105,11 +89,7 @@ export async function makeSQLKVStore( return undefined; } - const sqlKVSet = db.prepare(` - INSERT INTO kv (key, value) - VALUES (?, ?) - ON CONFLICT DO UPDATE SET value = excluded.value - `); + const sqlKVSet = db.prepare(SQL_QUERIES.SET); /** * Set the value associated with a key in the database. @@ -124,10 +104,7 @@ export async function makeSQLKVStore( sqlKVSet.reset(); } - const sqlKVDelete = db.prepare(` - DELETE FROM kv - WHERE key = ? - `); + const sqlKVDelete = db.prepare(SQL_QUERIES.DELETE); /** * Delete a key from the database. @@ -141,9 +118,7 @@ export async function makeSQLKVStore( sqlKVDelete.reset(); } - const sqlKVClear = db.prepare(` - DELETE FROM kv - `); + const sqlKVClear = db.prepare(SQL_QUERIES.CLEAR); /** * Delete all entries from the database. @@ -184,7 +159,7 @@ export async function makeSQLKVStore( return { get: (key) => kvGet(key, false), getNextKey: kvGetNextKey, - getRequired: (key) => kvGet(key, true), + getRequired: (key) => kvGet(key, true) as string, set: kvSet, delete: kvDelete, clear: kvClear, diff --git a/packages/store/src/types.ts b/packages/store/src/types.ts new file mode 100644 index 000000000..ded9a0fcc --- /dev/null +++ b/packages/store/src/types.ts @@ -0,0 +1,14 @@ +export type KVStore = { + get(key: string): string | undefined; + getRequired(key: string): string; + getNextKey(previousKey: string): string | undefined; + set(key: string, value: string): void; + delete(key: string): void; + clear(): void; + executeQuery(sql: string): Record[]; +}; + +export type MakeKVStore = ( + label?: string, + dbFilename?: string, +) => Promise; diff --git a/packages/store/tsconfig.build.json b/packages/store/tsconfig.build.json new file mode 100644 index 000000000..0f6059b8b --- /dev/null +++ b/packages/store/tsconfig.build.json @@ -0,0 +1,12 @@ +{ + "extends": "../../tsconfig.packages.build.json", + "compilerOptions": { + "baseUrl": "./", + "lib": ["DOM", "ES2022"], + "outDir": "./dist", + "rootDir": "./src", + "types": ["ses"] + }, + "references": [{ "path": "../utils/tsconfig.build.json" }], + "include": ["./src"] +} diff --git a/packages/store/tsconfig.json b/packages/store/tsconfig.json new file mode 100644 index 000000000..dd17bde8b --- /dev/null +++ b/packages/store/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "../../tsconfig.packages.json", + "compilerOptions": { + "baseUrl": "./", + "lib": ["DOM", "ES2022"], + "types": ["vitest", "ses"] + }, + "references": [{ "path": "../utils" }], + "include": [ + "../../vitest.config.ts", + "./src", + "./vite.config.ts", + "./vitest.config.ts" + ] +} diff --git a/packages/store/tsconfig.lint.json b/packages/store/tsconfig.lint.json new file mode 100644 index 000000000..76734f289 --- /dev/null +++ b/packages/store/tsconfig.lint.json @@ -0,0 +1,9 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "composite": false, + "noEmit": true, + "skipLibCheck": true + }, + "exclude": [] +} diff --git a/packages/store/typedoc.json b/packages/store/typedoc.json new file mode 100644 index 000000000..f8eb78ae1 --- /dev/null +++ b/packages/store/typedoc.json @@ -0,0 +1,8 @@ +{ + "entryPoints": [], + "excludePrivate": true, + "hideGenerator": true, + "out": "docs", + "tsconfig": "./tsconfig.build.json", + "projectDocuments": ["documents/*.md"] +} diff --git a/packages/store/vitest.config.ts b/packages/store/vitest.config.ts new file mode 100644 index 000000000..f58e064b1 --- /dev/null +++ b/packages/store/vitest.config.ts @@ -0,0 +1,18 @@ +import path from 'node:path'; +import { defineProject, mergeConfig } from 'vitest/config'; + +import defaultConfig from '../../vitest.config.js'; + +const config = mergeConfig( + defaultConfig, + defineProject({ + test: { + name: 'store', + setupFiles: path.resolve(__dirname, '../shims/src/endoify.js'), + }, + }), +); + +config.test.coverage.thresholds = true; + +export default config; diff --git a/scripts/create-package/package-template/package.json b/scripts/create-package/package-template/package.json index 7da949f4e..e0e885df2 100644 --- a/scripts/create-package/package-template/package.json +++ b/scripts/create-package/package-template/package.json @@ -11,7 +11,6 @@ "type": "git", "url": "https://github.com/MetaMask/ocap-kernel.git" }, - "sideEffects": false, "type": "module", "exports": { ".": { @@ -33,7 +32,7 @@ "build": "ts-bridge --project tsconfig.build.json --clean", "build:docs": "typedoc", "changelog:validate": "../../scripts/validate-changelog.sh PACKAGE_NAME", - "clean": "rimraf --glob ./dist './*.tsbuildinfo'", + "clean": "rimraf --glob './*.tsbuildinfo' ./.eslintcache ./coverage ./dist", "lint": "yarn lint:ts && yarn lint:eslint && yarn lint:misc --check && yarn constraints && yarn lint:dependencies", "lint:dependencies": "depcheck", "lint:eslint": "eslint . --cache", @@ -48,14 +47,12 @@ "test:watch": "vitest --config vitest.config.ts" }, "devDependencies": { - "@arethetypeswrong/cli": "^0.16.4", - "@metamask/auto-changelog": "^3.4.4", + "@arethetypeswrong/cli": "^0.17.3", + "@metamask/auto-changelog": "^4.0.0", "@metamask/eslint-config": "^14.0.0", "@metamask/eslint-config-nodejs": "^14.0.0", "@metamask/eslint-config-typescript": "^14.0.0", - "@ocap/cli": "workspace:^", - "@ocap/test-utils": "workspace:^", - "@ts-bridge/cli": "^0.5.1", + "@ts-bridge/cli": "^0.6.2", "@ts-bridge/shims": "^0.1.1", "@typescript-eslint/eslint-plugin": "^8.8.1", "@typescript-eslint/parser": "^8.8.1", @@ -70,10 +67,9 @@ "eslint-plugin-n": "^17.11.1", "eslint-plugin-prettier": "^5.2.1", "eslint-plugin-promise": "^7.1.0", - "jsdom": "^24.1.1", "prettier": "^3.3.3", "rimraf": "^6.0.1", - "typedoc": "^0.26.8", + "typedoc": "^0.27.6", "typescript": "~5.5.4", "typescript-eslint": "^8.8.1", "vite": "^6.0.11", diff --git a/tsconfig.build.json b/tsconfig.build.json index 7a0cd69a4..ce5d59ff4 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -5,6 +5,7 @@ { "path": "./packages/errors/tsconfig.build.json" }, { "path": "./packages/kernel/tsconfig.build.json" }, { "path": "./packages/nodejs/tsconfig.build.json" }, + { "path": "./packages/store/tsconfig.build.json" }, { "path": "./packages/streams/tsconfig.build.json" }, { "path": "./packages/utils/tsconfig.build.json" } ] diff --git a/tsconfig.json b/tsconfig.json index 078ae1f90..0ccb8e7b4 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -16,6 +16,7 @@ { "path": "./packages/kernel" }, { "path": "./packages/nodejs" }, { "path": "./packages/shims" }, + { "path": "./packages/store" }, { "path": "./packages/streams" }, { "path": "./packages/test-utils" }, { "path": "./packages/utils" }, diff --git a/vitest.config.ts b/vitest.config.ts index d5cb8dc62..dfff80079 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -4,7 +4,11 @@ import { defineConfig } from 'vitest/config'; export default defineConfig({ optimizeDeps: { - include: ['@vitest/coverage-istanbul', 'vitest-fetch-mock'], + include: [ + '@vitest/coverage-istanbul', + 'vitest-fetch-mock', + 'better-sqlite3', + ], }, plugins: [ @@ -68,10 +72,10 @@ export default defineConfig({ lines: 100, }, 'packages/extension/**': { - statements: 74.07, - functions: 79.39, - branches: 70.86, - lines: 74.11, + statements: 82.86, + functions: 83.59, + branches: 80.45, + lines: 82.79, }, 'packages/kernel/**': { statements: 48.54, @@ -80,10 +84,10 @@ export default defineConfig({ lines: 48.79, }, 'packages/nodejs/**': { - statements: 46.75, - functions: 47.61, - branches: 35.29, - lines: 46.75, + statements: 81.81, + functions: 90.9, + branches: 75, + lines: 81.81, }, 'packages/shims/**': { statements: 0, @@ -91,6 +95,12 @@ export default defineConfig({ branches: 0, lines: 0, }, + 'packages/store/**': { + statements: 100, + functions: 100, + branches: 100, + lines: 100, + }, 'packages/streams/**': { statements: 100, functions: 100, diff --git a/yarn.lock b/yarn.lock index 094c25824..cfaee1777 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2118,11 +2118,11 @@ __metadata: "@ocap/errors": "workspace:^" "@ocap/kernel": "workspace:^" "@ocap/shims": "workspace:^" + "@ocap/store": "workspace:^" "@ocap/streams": "workspace:^" "@ocap/test-utils": "workspace:^" "@ocap/utils": "workspace:^" "@playwright/test": "npm:^1.50.0" - "@sqlite.org/sqlite-wasm": "npm:^3.48.0-build1" "@testing-library/jest-dom": "npm:^6.6.3" "@testing-library/react": "npm:^16.2.0" "@testing-library/user-event": "npm:^14.6.1" @@ -2183,6 +2183,7 @@ __metadata: "@metamask/utils": "npm:^11.0.1" "@ocap/cli": "workspace:^" "@ocap/errors": "workspace:^" + "@ocap/store": "workspace:^" "@ocap/streams": "workspace:^" "@ocap/test-utils": "workspace:^" "@ocap/utils": "workspace:^" @@ -2287,17 +2288,16 @@ __metadata: "@ocap/cli": "workspace:^" "@ocap/kernel": "workspace:^" "@ocap/shims": "workspace:^" + "@ocap/store": "workspace:^" "@ocap/streams": "workspace:^" "@ocap/test-utils": "workspace:^" "@ocap/utils": "workspace:^" "@ts-bridge/cli": "npm:^0.6.2" "@ts-bridge/shims": "npm:^0.1.1" - "@types/better-sqlite3": "npm:^7.6.12" "@typescript-eslint/eslint-plugin": "npm:^8.8.1" "@typescript-eslint/parser": "npm:^8.8.1" "@typescript-eslint/utils": "npm:^8.8.1" "@vitest/eslint-plugin": "npm:^1.1.25" - better-sqlite3: "npm:^11.7.2" depcheck: "npm:^1.4.7" eslint: "npm:^9.12.0" eslint-config-prettier: "npm:^9.1.0" @@ -2355,6 +2355,45 @@ __metadata: languageName: unknown linkType: soft +"@ocap/store@workspace:^, @ocap/store@workspace:packages/store": + version: 0.0.0-use.local + resolution: "@ocap/store@workspace:packages/store" + dependencies: + "@arethetypeswrong/cli": "npm:^0.17.3" + "@metamask/auto-changelog": "npm:^4.0.0" + "@metamask/eslint-config": "npm:^14.0.0" + "@metamask/eslint-config-nodejs": "npm:^14.0.0" + "@metamask/eslint-config-typescript": "npm:^14.0.0" + "@ocap/utils": "workspace:^" + "@sqlite.org/sqlite-wasm": "npm:^3.48.0-build1" + "@ts-bridge/cli": "npm:^0.6.2" + "@ts-bridge/shims": "npm:^0.1.1" + "@types/better-sqlite3": "npm:^7.6.12" + "@typescript-eslint/eslint-plugin": "npm:^8.8.1" + "@typescript-eslint/parser": "npm:^8.8.1" + "@typescript-eslint/utils": "npm:^8.8.1" + "@vitest/eslint-plugin": "npm:^1.1.25" + better-sqlite3: "npm:^11.7.2" + depcheck: "npm:^1.4.7" + eslint: "npm:^9.12.0" + eslint-config-prettier: "npm:^9.1.0" + eslint-import-resolver-typescript: "npm:^3.6.3" + eslint-plugin-import-x: "npm:^4.3.1" + eslint-plugin-jsdoc: "npm:^50.3.1" + eslint-plugin-n: "npm:^17.11.1" + eslint-plugin-prettier: "npm:^5.2.1" + eslint-plugin-promise: "npm:^7.1.0" + prettier: "npm:^3.3.3" + rimraf: "npm:^6.0.1" + ses: "npm:^1.9.0" + typedoc: "npm:^0.27.6" + typescript: "npm:~5.5.4" + typescript-eslint: "npm:^8.8.1" + vite: "npm:^6.0.11" + vitest: "npm:3.0.4" + languageName: unknown + linkType: soft + "@ocap/streams@workspace:^, @ocap/streams@workspace:packages/streams": version: 0.0.0-use.local resolution: "@ocap/streams@workspace:packages/streams" From b4d06014c6482dd45cd538b9806d4be7a4e4a12d Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Wed, 5 Feb 2025 21:18:14 +0100 Subject: [PATCH 02/13] fix ts config imports --- packages/extension/tsconfig.build.json | 3 +-- packages/extension/tsconfig.json | 4 +--- packages/kernel/tsconfig.json | 8 +------- packages/nodejs/tsconfig.json | 4 +--- 4 files changed, 4 insertions(+), 15 deletions(-) diff --git a/packages/extension/tsconfig.build.json b/packages/extension/tsconfig.build.json index 606e20287..24d929890 100644 --- a/packages/extension/tsconfig.build.json +++ b/packages/extension/tsconfig.build.json @@ -24,7 +24,6 @@ "./src/**/*.tsx", "./src/**/*.module.css", "./src/**/*-trusted-prelude.js", - "./src/env/dev-console.js", - "../../types" + "./src/env/dev-console.js" ] } diff --git a/packages/extension/tsconfig.json b/packages/extension/tsconfig.json index aec5d70c8..d3a9b3382 100644 --- a/packages/extension/tsconfig.json +++ b/packages/extension/tsconfig.json @@ -41,8 +41,6 @@ "./test/**/*.ts", "./vite-plugins/*.ts", "./vite.config.ts", - "./vitest.config.ts", - "../../types", - "../store/src/sqlite-kv-store.ts" + "./vitest.config.ts" ] } diff --git a/packages/kernel/tsconfig.json b/packages/kernel/tsconfig.json index 5f8cdd350..dce592358 100644 --- a/packages/kernel/tsconfig.json +++ b/packages/kernel/tsconfig.json @@ -10,11 +10,5 @@ { "path": "../test-utils" }, { "path": "../store" } ], - "include": [ - "./src", - "./test", - "./vite.config.ts", - "./vitest.config.ts", - "../../types" - ] + "include": ["./src", "./test", "./vite.config.ts", "./vitest.config.ts"] } diff --git a/packages/nodejs/tsconfig.json b/packages/nodejs/tsconfig.json index 257e52f13..45ecd95e1 100644 --- a/packages/nodejs/tsconfig.json +++ b/packages/nodejs/tsconfig.json @@ -25,8 +25,6 @@ "./test/**/*.ts", "./vitest.config.ts", "./vitest.config.e2e.ts", - "../../types", - "./test/workers/*.js", - "../store/src/native.ts" + "./test/workers/*.js" ] } From aa19ddad0e025a6f802a2507ca64a46e41667cf0 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Wed, 5 Feb 2025 22:00:10 +0100 Subject: [PATCH 03/13] fix lint issues --- packages/store/tsconfig.build.json | 5 +++-- tsconfig.packages.build.json | 5 ++++- tsconfig.packages.json | 2 ++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/store/tsconfig.build.json b/packages/store/tsconfig.build.json index 0f6059b8b..2b7ce0224 100644 --- a/packages/store/tsconfig.build.json +++ b/packages/store/tsconfig.build.json @@ -5,8 +5,9 @@ "lib": ["DOM", "ES2022"], "outDir": "./dist", "rootDir": "./src", - "types": ["ses"] + "types": ["ses"], + "composite": true }, "references": [{ "path": "../utils/tsconfig.build.json" }], - "include": ["./src"] + "include": ["./src/**/*.ts", "./src/sqlite/*.ts"] } diff --git a/tsconfig.packages.build.json b/tsconfig.packages.build.json index 148a0b480..512750954 100644 --- a/tsconfig.packages.build.json +++ b/tsconfig.packages.build.json @@ -8,7 +8,10 @@ "emitDeclarationOnly": true, "inlineSources": true, "skipLibCheck": true, - "sourceMap": true + "sourceMap": true, + "paths": { + "@ocap/*": ["../*/src"] + } }, "exclude": [ "**/vite.config.ts", diff --git a/tsconfig.packages.json b/tsconfig.packages.json index aa680afb2..4f2caa175 100644 --- a/tsconfig.packages.json +++ b/tsconfig.packages.json @@ -6,6 +6,8 @@ * uncompiled source code for packages that live in this repo. */ "paths": { + "@ocap/store/sqlite/native": ["../store/src/sqlite/native.ts"], + "@ocap/store/sqlite/wasm": ["../store/src/sqlite/wasm.ts"], "@ocap/*": ["../*/src"] } } From 4f494eb94222c096cfe8c96051265d466a9cc351 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 6 Feb 2025 00:03:37 +0100 Subject: [PATCH 04/13] cleanup --- packages/store/src/sqlite/native.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/store/src/sqlite/native.ts b/packages/store/src/sqlite/native.ts index d4effae3f..223752f53 100644 --- a/packages/store/src/sqlite/native.ts +++ b/packages/store/src/sqlite/native.ts @@ -8,9 +8,6 @@ import { join } from 'path'; import { SQL_QUERIES } from './common.js'; import type { KVStore } from '../types.js'; -// We require require because the ESM import does not work properly. - -// const Sqlite = require('better-sqlite3'); const dbRoot = join(tmpdir(), './ocap-sqlite'); From 290c2f2b7ea5b80331a2fc79a8543cac89b2df9d Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 6 Feb 2025 00:38:02 +0100 Subject: [PATCH 05/13] fix native label for memory --- packages/kernel/src/VatSupervisor.ts | 5 ++++- packages/store/src/sqlite/native.test.ts | 24 +++++++++++++++++++++++- packages/store/src/sqlite/native.ts | 23 ++++++++++++++++++----- packages/store/src/sqlite/wasm.ts | 6 +++--- 4 files changed, 48 insertions(+), 10 deletions(-) diff --git a/packages/kernel/src/VatSupervisor.ts b/packages/kernel/src/VatSupervisor.ts index dfcd9b725..71f0a7fed 100644 --- a/packages/kernel/src/VatSupervisor.ts +++ b/packages/kernel/src/VatSupervisor.ts @@ -198,7 +198,10 @@ export class VatSupervisor { } this.#loaded = true; - const kvStore = await this.#makeKVStore(`[vat-${this.id}]`, this.id); + const kvStore = await this.#makeKVStore( + `[vat-${this.id}]`, + `vat-${this.id}.db`, + ); const syscall = makeSupervisorSyscall(this, kvStore); const vatPowers = {}; // XXX should be something more real const liveSlotsOptions = {}; // XXX should be something more real diff --git a/packages/store/src/sqlite/native.test.ts b/packages/store/src/sqlite/native.test.ts index d00ad4c6e..0a5d9e3eb 100644 --- a/packages/store/src/sqlite/native.test.ts +++ b/packages/store/src/sqlite/native.test.ts @@ -1,7 +1,8 @@ +import { mkdir } from 'fs/promises'; import { describe, it, expect, vi, beforeEach } from 'vitest'; import { SQL_QUERIES } from './common.js'; -import { makeSQLKVStore } from './native.js'; +import { makeSQLKVStore, getDBFilename } from './native.js'; const mockStatement = { run: vi.fn(), @@ -23,7 +24,13 @@ vi.mock('fs/promises', () => ({ mkdir: vi.fn(), })); +vi.mock('os', () => ({ + tmpdir: vi.fn(() => '/mock-tmpdir'), +})); + describe('makeSQLKVStore', () => { + const mockMkdir = vi.mocked(mkdir).mockResolvedValue(''); + beforeEach(() => { Object.values(mockStatement).forEach((mock) => mock.mockReset()); }); @@ -90,4 +97,19 @@ describe('makeSQLKVStore', () => { // @ts-expect-error Testing invalid input expect(() => store.getNextKey(123)).toThrow('must be a string'); }); + + describe('getDBFilename', () => { + it('returns in-memory database path when label starts with ":"', async () => { + const result = await getDBFilename(':memory:'); + expect(result).toBe(':memory:'); + }); + + it('creates file-based database path for normal labels with .db suffix', async () => { + const result = await getDBFilename('test.db'); + expect(result).toBe('/mock-tmpdir/ocap-sqlite/test.db'); + expect(mockMkdir).toHaveBeenCalledWith('/mock-tmpdir/ocap-sqlite', { + recursive: true, + }); + }); + }); }); diff --git a/packages/store/src/sqlite/native.ts b/packages/store/src/sqlite/native.ts index 223752f53..69d31e19f 100644 --- a/packages/store/src/sqlite/native.ts +++ b/packages/store/src/sqlite/native.ts @@ -9,8 +9,6 @@ import { join } from 'path'; import { SQL_QUERIES } from './common.js'; import type { KVStore } from '../types.js'; -const dbRoot = join(tmpdir(), './ocap-sqlite'); - /** * Ensure that SQLite is initialized. * @@ -22,9 +20,8 @@ async function initDB( dbFilename: string, logger: ReturnType, ): Promise { - const dbPath = join(dbRoot, `${dbFilename}.db`); + const dbPath = await getDBFilename(dbFilename); logger.debug('dbPath:', dbPath); - await mkdir(dbRoot, { recursive: true }); return new Sqlite(dbPath, { verbose: logger.info, }); @@ -39,7 +36,7 @@ async function initDB( */ export async function makeSQLKVStore( label: string = '[sqlite]', - dbFilename: string = 'store', + dbFilename: string = 'store.db', ): Promise { const logger = makeLogger(label); const db = await initDB(dbFilename, logger); @@ -139,3 +136,19 @@ export async function makeSQLKVStore( clear: db.transaction(kvClear), }; } + +/** + * Get the filename for a database. + * + * @param label - A label for the database. + * @returns The filename for the database. + */ +export async function getDBFilename(label: string): Promise { + if (label.startsWith(':')) { + return label; + } + const dbRoot = join(tmpdir(), './ocap-sqlite'); + await mkdir(dbRoot, { recursive: true }); + console.log('dbRoot:', dbRoot); + return join(dbRoot, label); +} diff --git a/packages/store/src/sqlite/wasm.ts b/packages/store/src/sqlite/wasm.ts index a73bd50ca..6803d1cf0 100644 --- a/packages/store/src/sqlite/wasm.ts +++ b/packages/store/src/sqlite/wasm.ts @@ -14,7 +14,7 @@ import type { KVStore } from '../types.js'; async function initDB(dbFilename: string): Promise { const sqlite3 = await sqlite3InitModule(); if (sqlite3.oo1.OpfsDb) { - return new sqlite3.oo1.OpfsDb(`/${dbFilename}.db`, 'cw'); + return new sqlite3.oo1.OpfsDb(dbFilename, 'cw'); } console.warn(`OPFS not enabled, database will be ephemeral`); @@ -25,12 +25,12 @@ async function initDB(dbFilename: string): Promise { * Makes a {@link KVStore} for low-level persistent storage. * * @param label - A logger prefix label. Defaults to '[sqlite]'. - * @param dbFilename - The filename of the database to use. Defaults to 'store.sqlite'. + * @param dbFilename - The filename of the database to use. Defaults to 'store.db'. * @returns A key/value store to base higher level stores on. */ export async function makeSQLKVStore( label: string = '[sqlite]', - dbFilename: string = 'store', + dbFilename: string = 'store.db', ): Promise { const logger = makeLogger(label); const db = await initDB(dbFilename); From ce4926352b1ecb3e4a329278cc703d2f47974fcf Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 6 Feb 2025 15:55:53 +0100 Subject: [PATCH 06/13] fix test command --- packages/store/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/store/package.json b/packages/store/package.json index b67f25d78..9a99907b3 100644 --- a/packages/store/package.json +++ b/packages/store/package.json @@ -62,7 +62,7 @@ "publish:preview": "yarn npm publish --tag preview", "test": "vitest run --config vitest.config.ts", "test:clean": "yarn test --no-cache --coverage.clean", - "test:dev": "yarn test --coverage false", + "test:dev": "yarn test --mode development", "test:verbose": "yarn test --reporter verbose", "test:watch": "vitest --config vitest.config.ts" }, From a7b0263b5471e042589af870e53cb662f6d35f11 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 6 Feb 2025 15:58:46 +0100 Subject: [PATCH 07/13] remove log --- packages/store/src/sqlite/native.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/store/src/sqlite/native.ts b/packages/store/src/sqlite/native.ts index 69d31e19f..ad24a6647 100644 --- a/packages/store/src/sqlite/native.ts +++ b/packages/store/src/sqlite/native.ts @@ -149,6 +149,5 @@ export async function getDBFilename(label: string): Promise { } const dbRoot = join(tmpdir(), './ocap-sqlite'); await mkdir(dbRoot, { recursive: true }); - console.log('dbRoot:', dbRoot); return join(dbRoot, label); } From 28c46b7c7b4f3d59e6424a09df58fe612513904a Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 6 Feb 2025 16:34:49 +0100 Subject: [PATCH 08/13] fix test --- packages/nodejs/vitest.config.e2e.ts | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/packages/nodejs/vitest.config.e2e.ts b/packages/nodejs/vitest.config.e2e.ts index 2a084d69a..26d87762c 100644 --- a/packages/nodejs/vitest.config.e2e.ts +++ b/packages/nodejs/vitest.config.e2e.ts @@ -1,15 +1,19 @@ -import { defineConfig, mergeConfig } from 'vite'; +import { mergeConfig } from '@ocap/test-utils/vitest-config'; +import { defineConfig, defineProject } from 'vitest/config'; import defaultConfig from '../../vitest.config.js'; -export default mergeConfig( - defaultConfig, - defineConfig({ - test: { - name: 'nodejs:e2e', - pool: 'forks', - include: ['./test/e2e/**/*.test.ts'], - exclude: ['./src/**/*'], - }, - }), -); +export default defineConfig((args) => { + return mergeConfig( + args, + defaultConfig, + defineProject({ + test: { + name: 'nodejs:e2e', + pool: 'forks', + include: ['./test/e2e/**/*.test.ts'], + exclude: ['./src/**/*'], + }, + }), + ); +}); From 7664c6683cb9343e7d93775a21b3fd35cec455a1 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 6 Feb 2025 16:37:19 +0100 Subject: [PATCH 09/13] fix deps --- package.json | 2 +- packages/store/package.json | 2 +- packages/streams/package.json | 2 +- yarn.lock | 80 +++++++---------------------------- 4 files changed, 19 insertions(+), 67 deletions(-) diff --git a/package.json b/package.json index bfa2b4d02..6b0f53c75 100644 --- a/package.json +++ b/package.json @@ -70,7 +70,7 @@ "@typescript-eslint/eslint-plugin": "^8.8.1", "@typescript-eslint/parser": "^8.8.1", "@typescript-eslint/utils": "^8.8.1", - "@vitest/coverage-istanbul": "3.0.4", + "@vitest/coverage-istanbul": "^3.0.5", "@vitest/eslint-plugin": "^1.1.25", "@yarnpkg/types": "^4.0.0", "depcheck": "^1.4.7", diff --git a/packages/store/package.json b/packages/store/package.json index 9a99907b3..e136a93f1 100644 --- a/packages/store/package.json +++ b/packages/store/package.json @@ -100,7 +100,7 @@ "typescript": "~5.5.4", "typescript-eslint": "^8.8.1", "vite": "^6.0.11", - "vitest": "3.0.4" + "vitest": "^3.0.5" }, "engines": { "node": "^20 || >=22" diff --git a/packages/streams/package.json b/packages/streams/package.json index 21a7db314..c4a18ee28 100644 --- a/packages/streams/package.json +++ b/packages/streams/package.json @@ -69,7 +69,7 @@ "@typescript-eslint/eslint-plugin": "^8.8.1", "@typescript-eslint/parser": "^8.8.1", "@typescript-eslint/utils": "^8.8.1", - "@vitest/browser": "3.0.4", + "@vitest/browser": "^3.0.5", "@vitest/eslint-plugin": "^1.1.25", "cookie": "^1.0.2", "depcheck": "^1.4.7", diff --git a/yarn.lock b/yarn.lock index cfaee1777..cbff3e5d8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2237,7 +2237,7 @@ __metadata: "@typescript-eslint/eslint-plugin": "npm:^8.8.1" "@typescript-eslint/parser": "npm:^8.8.1" "@typescript-eslint/utils": "npm:^8.8.1" - "@vitest/coverage-istanbul": "npm:3.0.4" + "@vitest/coverage-istanbul": "npm:^3.0.5" "@vitest/eslint-plugin": "npm:^1.1.25" "@yarnpkg/types": "npm:^4.0.0" ava: "npm:^6.2.0" @@ -2390,7 +2390,7 @@ __metadata: typescript: "npm:~5.5.4" typescript-eslint: "npm:^8.8.1" vite: "npm:^6.0.11" - vitest: "npm:3.0.4" + vitest: "npm:^3.0.5" languageName: unknown linkType: soft @@ -2417,7 +2417,7 @@ __metadata: "@typescript-eslint/eslint-plugin": "npm:^8.8.1" "@typescript-eslint/parser": "npm:^8.8.1" "@typescript-eslint/utils": "npm:^8.8.1" - "@vitest/browser": "npm:3.0.4" + "@vitest/browser": "npm:^3.0.5" "@vitest/eslint-plugin": "npm:^1.1.25" cookie: "npm:^1.0.2" depcheck: "npm:^1.4.7" @@ -3599,14 +3599,14 @@ __metadata: languageName: node linkType: hard -"@vitest/browser@npm:3.0.4": - version: 3.0.4 - resolution: "@vitest/browser@npm:3.0.4" +"@vitest/browser@npm:^3.0.5": + version: 3.0.5 + resolution: "@vitest/browser@npm:3.0.5" dependencies: "@testing-library/dom": "npm:^10.4.0" "@testing-library/user-event": "npm:^14.6.1" - "@vitest/mocker": "npm:3.0.4" - "@vitest/utils": "npm:3.0.4" + "@vitest/mocker": "npm:3.0.5" + "@vitest/utils": "npm:3.0.5" magic-string: "npm:^0.30.17" msw: "npm:^2.7.0" sirv: "npm:^3.0.0" @@ -3614,7 +3614,7 @@ __metadata: ws: "npm:^8.18.0" peerDependencies: playwright: "*" - vitest: 3.0.4 + vitest: 3.0.5 webdriverio: "*" peerDependenciesMeta: playwright: @@ -3623,13 +3623,13 @@ __metadata: optional: true webdriverio: optional: true - checksum: 10/23f7a60b7ea073ad06cf3145a3416e1dd53489f26db2a497ea55d2313943797e99af807c4c077b54baa670d4c87cf028daa334af78d4298d8da9f087505e9138 + checksum: 10/8464f5ac7162e2905711fd2677ce57bab87ded5de7624cb742b2c495874c3d04ed549408dd6e088c831da37d03376dc3837998b07f7d5eea9ec85e3d0c948c6c languageName: node linkType: hard -"@vitest/coverage-istanbul@npm:3.0.4": - version: 3.0.4 - resolution: "@vitest/coverage-istanbul@npm:3.0.4" +"@vitest/coverage-istanbul@npm:^3.0.5": + version: 3.0.5 + resolution: "@vitest/coverage-istanbul@npm:3.0.5" dependencies: "@istanbuljs/schema": "npm:^0.1.3" debug: "npm:^4.4.0" @@ -3642,8 +3642,8 @@ __metadata: test-exclude: "npm:^7.0.1" tinyrainbow: "npm:^2.0.0" peerDependencies: - vitest: 3.0.4 - checksum: 10/2da1733a2ccb4eb4817ab83856304a6ae8e082ec68d314f7c89780bf5bffdb7e4edd5531b0770839591ee0bbbc2ef6a26f2d29fb6db6ce70c1f4e0bdf5cf1b11 + vitest: 3.0.5 + checksum: 10/2f69c745644f392e544f4bed1b4af7c4fc9452d8142ab903469e274a0cb247b3091bd8bcf1e2cec4c8e398bf326140bd72a2031a811e9974c59eacde291359a8 languageName: node linkType: hard @@ -3676,25 +3676,6 @@ __metadata: languageName: node linkType: hard -"@vitest/mocker@npm:3.0.4": - version: 3.0.4 - resolution: "@vitest/mocker@npm:3.0.4" - dependencies: - "@vitest/spy": "npm:3.0.4" - estree-walker: "npm:^3.0.3" - magic-string: "npm:^0.30.17" - peerDependencies: - msw: ^2.4.9 - vite: ^5.0.0 || ^6.0.0 - peerDependenciesMeta: - msw: - optional: true - vite: - optional: true - checksum: 10/f6e7a57575271b1f9f4fd8671e0760a035c31620086b694f303815aba353864b2eb3c51f5c4506e5f618ab7584b9260035e0183a4f8d7a9947a30dc7ef91c5b6 - languageName: node - linkType: hard - "@vitest/mocker@npm:3.0.5": version: 3.0.5 resolution: "@vitest/mocker@npm:3.0.5" @@ -3714,15 +3695,6 @@ __metadata: languageName: node linkType: hard -"@vitest/pretty-format@npm:3.0.4": - version: 3.0.4 - resolution: "@vitest/pretty-format@npm:3.0.4" - dependencies: - tinyrainbow: "npm:^2.0.0" - checksum: 10/8c54fc5df1e73339b5b81ad66d779c98af750a4f1609f47aecabc9af2e11620775d521ab183e9db8acf2cd018d7aa29d5fd9737bf2935369dd6f1306a6487b9f - languageName: node - linkType: hard - "@vitest/pretty-format@npm:3.0.5, @vitest/pretty-format@npm:^3.0.5": version: 3.0.5 resolution: "@vitest/pretty-format@npm:3.0.5" @@ -3753,15 +3725,6 @@ __metadata: languageName: node linkType: hard -"@vitest/spy@npm:3.0.4": - version: 3.0.4 - resolution: "@vitest/spy@npm:3.0.4" - dependencies: - tinyspy: "npm:^3.0.2" - checksum: 10/a2e03516e7f678120b03b1f1e95b587781e6c6c78781a2b37bd5b7706fb57a99f127d46d337db14477673aa811027730fe5fb5af68f03fde7e65050293810e67 - languageName: node - linkType: hard - "@vitest/spy@npm:3.0.5": version: 3.0.5 resolution: "@vitest/spy@npm:3.0.5" @@ -3771,17 +3734,6 @@ __metadata: languageName: node linkType: hard -"@vitest/utils@npm:3.0.4": - version: 3.0.4 - resolution: "@vitest/utils@npm:3.0.4" - dependencies: - "@vitest/pretty-format": "npm:3.0.4" - loupe: "npm:^3.1.2" - tinyrainbow: "npm:^2.0.0" - checksum: 10/68132cc059ac0db29e325b3e8a1ac6e0a99ea8a2d6d214bb4dc6399c3de0ffe78c42b13c733cc775a78d7ee1e7e3dcd67f75b7c35e5c28e3825cabf4ec7c50dc - languageName: node - linkType: hard - "@vitest/utils@npm:3.0.5": version: 3.0.5 resolution: "@vitest/utils@npm:3.0.5" @@ -11959,7 +11911,7 @@ __metadata: languageName: node linkType: hard -"vitest@npm:3.0.5": +"vitest@npm:3.0.5, vitest@npm:^3.0.5": version: 3.0.5 resolution: "vitest@npm:3.0.5" dependencies: From 3d882ab439a9e970dc36ced6bc9d3a02c8a52d2e Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 6 Feb 2025 17:05:30 +0100 Subject: [PATCH 10/13] fix lint --- package.json | 2 +- packages/cli/package.json | 2 +- packages/errors/package.json | 2 +- packages/extension/package.json | 2 +- packages/kernel/package.json | 2 +- packages/nodejs/package.json | 2 +- packages/shims/package.json | 2 +- packages/streams/package.json | 2 +- packages/test-utils/package.json | 2 +- packages/utils/package.json | 2 +- .../package-template/package.json | 2 +- yarn.lock | 22 +++++++++---------- 12 files changed, 22 insertions(+), 22 deletions(-) diff --git a/package.json b/package.json index 6b0f53c75..c1086ed0e 100644 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "typescript-eslint": "^8.8.1", "vite": "^6.0.11", "vite-tsconfig-paths": "^5.1.4", - "vitest": "3.0.5", + "vitest": "^3.0.5", "vitest-fetch-mock": "^0.4.3", "yargs": "^17.7.2" }, diff --git a/packages/cli/package.json b/packages/cli/package.json index e6d2ca728..08473b75f 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -79,7 +79,7 @@ "typescript": "~5.5.4", "typescript-eslint": "^8.8.1", "vite": "^6.0.11", - "vitest": "3.0.5" + "vitest": "^3.0.5" }, "engines": { "node": "^20 || >=22" diff --git a/packages/errors/package.json b/packages/errors/package.json index f6116965b..cb0a15c91 100644 --- a/packages/errors/package.json +++ b/packages/errors/package.json @@ -77,7 +77,7 @@ "typescript": "~5.5.4", "typescript-eslint": "^8.8.1", "vite": "^6.0.11", - "vitest": "3.0.5" + "vitest": "^3.0.5" }, "engines": { "node": "^20 || >=22" diff --git a/packages/extension/package.json b/packages/extension/package.json index 1178093c3..af67ce3ac 100644 --- a/packages/extension/package.json +++ b/packages/extension/package.json @@ -101,7 +101,7 @@ "vite": "^6.0.11", "vite-plugin-checker": "^0.8.0", "vite-plugin-static-copy": "^2.2.0", - "vitest": "3.0.5" + "vitest": "^3.0.5" }, "engines": { "node": "^20 || >=22" diff --git a/packages/kernel/package.json b/packages/kernel/package.json index 57f9d355b..776c9d2ef 100644 --- a/packages/kernel/package.json +++ b/packages/kernel/package.json @@ -85,7 +85,7 @@ "typescript": "~5.5.4", "typescript-eslint": "^8.8.1", "vite": "^6.0.11", - "vitest": "3.0.5" + "vitest": "^3.0.5" }, "engines": { "node": "^20 || >=22" diff --git a/packages/nodejs/package.json b/packages/nodejs/package.json index a5842f91a..ddf6bf94b 100644 --- a/packages/nodejs/package.json +++ b/packages/nodejs/package.json @@ -79,7 +79,7 @@ "typescript": "~5.5.4", "typescript-eslint": "^8.8.1", "vite": "^6.0.11", - "vitest": "3.0.5" + "vitest": "^3.0.5" }, "engines": { "node": "^20 || >=22" diff --git a/packages/shims/package.json b/packages/shims/package.json index 8c1a4e4b2..1b3b7702c 100644 --- a/packages/shims/package.json +++ b/packages/shims/package.json @@ -67,7 +67,7 @@ "typescript": "~5.5.4", "typescript-eslint": "^8.8.1", "vite": "^6.0.11", - "vitest": "3.0.5" + "vitest": "^3.0.5" }, "engines": { "node": "^20 || >=22" diff --git a/packages/streams/package.json b/packages/streams/package.json index c4a18ee28..cb7be5112 100644 --- a/packages/streams/package.json +++ b/packages/streams/package.json @@ -89,7 +89,7 @@ "typescript": "~5.5.4", "typescript-eslint": "^8.8.1", "vite": "^6.0.11", - "vitest": "3.0.5" + "vitest": "^3.0.5" }, "engines": { "node": "^20 || >=22" diff --git a/packages/test-utils/package.json b/packages/test-utils/package.json index 4a99b5a1e..5d9365731 100644 --- a/packages/test-utils/package.json +++ b/packages/test-utils/package.json @@ -49,7 +49,7 @@ "rimraf": "^6.0.1", "typescript": "~5.5.4", "typescript-eslint": "^8.8.1", - "vitest": "3.0.5", + "vitest": "^3.0.5", "vitest-fetch-mock": "^0.4.3" }, "engines": { diff --git a/packages/utils/package.json b/packages/utils/package.json index a0b8a677d..764174034 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -79,7 +79,7 @@ "typescript": "~5.5.4", "typescript-eslint": "^8.8.1", "vite": "^6.0.11", - "vitest": "3.0.5" + "vitest": "^3.0.5" }, "engines": { "node": "^20 || >=22" diff --git a/scripts/create-package/package-template/package.json b/scripts/create-package/package-template/package.json index e0e885df2..7fa106419 100644 --- a/scripts/create-package/package-template/package.json +++ b/scripts/create-package/package-template/package.json @@ -73,7 +73,7 @@ "typescript": "~5.5.4", "typescript-eslint": "^8.8.1", "vite": "^6.0.11", - "vitest": "3.0.5" + "vitest": "^3.0.5" }, "engines": { "node": "NODE_VERSIONS" diff --git a/yarn.lock b/yarn.lock index cbff3e5d8..86ae7ca13 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2052,7 +2052,7 @@ __metadata: typescript: "npm:~5.5.4" typescript-eslint: "npm:^8.8.1" vite: "npm:^6.0.11" - vitest: "npm:3.0.5" + vitest: "npm:^3.0.5" yargs: "npm:^17.7.2" bin: ocap: ./dist/app.mjs @@ -2095,7 +2095,7 @@ __metadata: typescript: "npm:~5.5.4" typescript-eslint: "npm:^8.8.1" vite: "npm:^6.0.11" - vitest: "npm:3.0.5" + vitest: "npm:^3.0.5" languageName: unknown linkType: soft @@ -2160,7 +2160,7 @@ __metadata: vite: "npm:^6.0.11" vite-plugin-checker: "npm:^0.8.0" vite-plugin-static-copy: "npm:^2.2.0" - vitest: "npm:3.0.5" + vitest: "npm:^3.0.5" languageName: unknown linkType: soft @@ -2208,7 +2208,7 @@ __metadata: typescript: "npm:~5.5.4" typescript-eslint: "npm:^8.8.1" vite: "npm:^6.0.11" - vitest: "npm:3.0.5" + vitest: "npm:^3.0.5" languageName: unknown linkType: soft @@ -2268,7 +2268,7 @@ __metadata: typescript-eslint: "npm:^8.8.1" vite: "npm:^6.0.11" vite-tsconfig-paths: "npm:^5.1.4" - vitest: "npm:3.0.5" + vitest: "npm:^3.0.5" vitest-fetch-mock: "npm:^0.4.3" webextension-polyfill: "npm:^0.12.0" yargs: "npm:^17.7.2" @@ -2315,7 +2315,7 @@ __metadata: typescript: "npm:~5.5.4" typescript-eslint: "npm:^8.8.1" vite: "npm:^6.0.11" - vitest: "npm:3.0.5" + vitest: "npm:^3.0.5" languageName: unknown linkType: soft @@ -2351,7 +2351,7 @@ __metadata: typescript: "npm:~5.5.4" typescript-eslint: "npm:^8.8.1" vite: "npm:^6.0.11" - vitest: "npm:3.0.5" + vitest: "npm:^3.0.5" languageName: unknown linkType: soft @@ -2437,7 +2437,7 @@ __metadata: typescript: "npm:~5.5.4" typescript-eslint: "npm:^8.8.1" vite: "npm:^6.0.11" - vitest: "npm:3.0.5" + vitest: "npm:^3.0.5" languageName: unknown linkType: soft @@ -2467,7 +2467,7 @@ __metadata: rimraf: "npm:^6.0.1" typescript: "npm:~5.5.4" typescript-eslint: "npm:^8.8.1" - vitest: "npm:3.0.5" + vitest: "npm:^3.0.5" vitest-fetch-mock: "npm:^0.4.3" languageName: unknown linkType: soft @@ -2510,7 +2510,7 @@ __metadata: typescript: "npm:~5.5.4" typescript-eslint: "npm:^8.8.1" vite: "npm:^6.0.11" - vitest: "npm:3.0.5" + vitest: "npm:^3.0.5" languageName: unknown linkType: soft @@ -11911,7 +11911,7 @@ __metadata: languageName: node linkType: hard -"vitest@npm:3.0.5, vitest@npm:^3.0.5": +"vitest@npm:^3.0.5": version: 3.0.5 resolution: "vitest@npm:3.0.5" dependencies: From ad889da8e0e3f6355611594fe2917769c6c14218 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 6 Feb 2025 20:06:32 +0100 Subject: [PATCH 11/13] Rename native to nodejs --- packages/nodejs/src/kernel/make-kernel.test.ts | 2 +- packages/nodejs/src/kernel/make-kernel.ts | 2 +- packages/nodejs/src/vat/vat-worker.ts | 2 +- packages/store/package.json | 10 +++++----- .../src/sqlite/{native.test.ts => nodejs.test.ts} | 2 +- packages/store/src/sqlite/{native.ts => nodejs.ts} | 0 packages/store/vitest.config.ts | 6 +----- tsconfig.packages.json | 2 +- 8 files changed, 11 insertions(+), 15 deletions(-) rename packages/store/src/sqlite/{native.test.ts => nodejs.test.ts} (98%) rename packages/store/src/sqlite/{native.ts => nodejs.ts} (100%) diff --git a/packages/nodejs/src/kernel/make-kernel.test.ts b/packages/nodejs/src/kernel/make-kernel.test.ts index f79f7870c..9c3e8fc68 100644 --- a/packages/nodejs/src/kernel/make-kernel.test.ts +++ b/packages/nodejs/src/kernel/make-kernel.test.ts @@ -9,7 +9,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import { makeKernel } from './make-kernel.js'; -vi.mock('@ocap/store/sqlite/native', async () => { +vi.mock('@ocap/store/sqlite/nodejs', async () => { const { makeMapKVStore } = await import('../../../kernel/test/storage.js'); return { makeSQLKVStore: makeMapKVStore, diff --git a/packages/nodejs/src/kernel/make-kernel.ts b/packages/nodejs/src/kernel/make-kernel.ts index 3591aaf6c..7e9b884da 100644 --- a/packages/nodejs/src/kernel/make-kernel.ts +++ b/packages/nodejs/src/kernel/make-kernel.ts @@ -1,6 +1,6 @@ import type { KernelCommand, KernelCommandReply } from '@ocap/kernel'; import { Kernel } from '@ocap/kernel'; -import { makeSQLKVStore } from '@ocap/store/sqlite/native'; +import { makeSQLKVStore } from '@ocap/store/sqlite/nodejs'; import { NodeWorkerDuplexStream } from '@ocap/streams'; import { MessagePort as NodeMessagePort } from 'node:worker_threads'; diff --git a/packages/nodejs/src/vat/vat-worker.ts b/packages/nodejs/src/vat/vat-worker.ts index 6980e27e7..155abce31 100644 --- a/packages/nodejs/src/vat/vat-worker.ts +++ b/packages/nodejs/src/vat/vat-worker.ts @@ -2,7 +2,7 @@ import '@ocap/shims/endoify'; import type { VatId } from '@ocap/kernel'; import { VatSupervisor } from '@ocap/kernel'; -import { makeSQLKVStore } from '@ocap/store/sqlite/native'; +import { makeSQLKVStore } from '@ocap/store/sqlite/nodejs'; import { makeLogger } from '@ocap/utils'; import { makeCommandStream } from './streams'; diff --git a/packages/store/package.json b/packages/store/package.json index e136a93f1..25d0b5ee8 100644 --- a/packages/store/package.json +++ b/packages/store/package.json @@ -23,14 +23,14 @@ "default": "./dist/index.cjs" } }, - "./sqlite/native": { + "./sqlite/nodejs": { "import": { - "types": "./dist/sqlite/native.d.mts", - "default": "./dist/sqlite/native.mjs" + "types": "./dist/sqlite/nodejs.d.mts", + "default": "./dist/sqlite/nodejs.mjs" }, "require": { - "types": "./dist/sqlite/native.d.cts", - "default": "./dist/sqlite/native.cjs" + "types": "./dist/sqlite/nodejs.d.cts", + "default": "./dist/sqlite/nodejs.cjs" } }, "./sqlite/wasm": { diff --git a/packages/store/src/sqlite/native.test.ts b/packages/store/src/sqlite/nodejs.test.ts similarity index 98% rename from packages/store/src/sqlite/native.test.ts rename to packages/store/src/sqlite/nodejs.test.ts index 0a5d9e3eb..4dc4a0036 100644 --- a/packages/store/src/sqlite/native.test.ts +++ b/packages/store/src/sqlite/nodejs.test.ts @@ -2,7 +2,7 @@ import { mkdir } from 'fs/promises'; import { describe, it, expect, vi, beforeEach } from 'vitest'; import { SQL_QUERIES } from './common.js'; -import { makeSQLKVStore, getDBFilename } from './native.js'; +import { makeSQLKVStore, getDBFilename } from './nodejs.js'; const mockStatement = { run: vi.fn(), diff --git a/packages/store/src/sqlite/native.ts b/packages/store/src/sqlite/nodejs.ts similarity index 100% rename from packages/store/src/sqlite/native.ts rename to packages/store/src/sqlite/nodejs.ts diff --git a/packages/store/vitest.config.ts b/packages/store/vitest.config.ts index f58e064b1..340240f0c 100644 --- a/packages/store/vitest.config.ts +++ b/packages/store/vitest.config.ts @@ -3,7 +3,7 @@ import { defineProject, mergeConfig } from 'vitest/config'; import defaultConfig from '../../vitest.config.js'; -const config = mergeConfig( +export default mergeConfig( defaultConfig, defineProject({ test: { @@ -12,7 +12,3 @@ const config = mergeConfig( }, }), ); - -config.test.coverage.thresholds = true; - -export default config; diff --git a/tsconfig.packages.json b/tsconfig.packages.json index 4f2caa175..73fa8bfe4 100644 --- a/tsconfig.packages.json +++ b/tsconfig.packages.json @@ -6,7 +6,7 @@ * uncompiled source code for packages that live in this repo. */ "paths": { - "@ocap/store/sqlite/native": ["../store/src/sqlite/native.ts"], + "@ocap/store/sqlite/nodejs": ["../store/src/sqlite/nodejs.ts"], "@ocap/store/sqlite/wasm": ["../store/src/sqlite/wasm.ts"], "@ocap/*": ["../*/src"] } From ba6940c44aeb50416d4d3ac2e5f5f7d87b8bff6a Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 6 Feb 2025 20:30:37 +0100 Subject: [PATCH 12/13] Fix store vitest config --- packages/store/package.json | 1 + packages/store/vitest.config.ts | 26 +++++++++++-------- .../package-template/package.json | 1 + 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/packages/store/package.json b/packages/store/package.json index 25d0b5ee8..b229e5332 100644 --- a/packages/store/package.json +++ b/packages/store/package.json @@ -78,6 +78,7 @@ "@metamask/eslint-config": "^14.0.0", "@metamask/eslint-config-nodejs": "^14.0.0", "@metamask/eslint-config-typescript": "^14.0.0", + "@ocap/test-utils": "workspace:^", "@ts-bridge/cli": "^0.6.2", "@ts-bridge/shims": "^0.1.1", "@types/better-sqlite3": "^7.6.12", diff --git a/packages/store/vitest.config.ts b/packages/store/vitest.config.ts index 340240f0c..045a31518 100644 --- a/packages/store/vitest.config.ts +++ b/packages/store/vitest.config.ts @@ -1,14 +1,18 @@ -import path from 'node:path'; -import { defineProject, mergeConfig } from 'vitest/config'; +import { mergeConfig } from '@ocap/test-utils/vitest-config'; +import path from 'path'; +import { defineConfig, defineProject } from 'vitest/config'; import defaultConfig from '../../vitest.config.js'; -export default mergeConfig( - defaultConfig, - defineProject({ - test: { - name: 'store', - setupFiles: path.resolve(__dirname, '../shims/src/endoify.js'), - }, - }), -); +export default defineConfig((args) => { + return mergeConfig( + args, + defaultConfig, + defineProject({ + test: { + name: 'store', + setupFiles: path.resolve(__dirname, '../shims/src/endoify.js'), + }, + }), + ); +}); diff --git a/scripts/create-package/package-template/package.json b/scripts/create-package/package-template/package.json index 7fa106419..3443e8c96 100644 --- a/scripts/create-package/package-template/package.json +++ b/scripts/create-package/package-template/package.json @@ -52,6 +52,7 @@ "@metamask/eslint-config": "^14.0.0", "@metamask/eslint-config-nodejs": "^14.0.0", "@metamask/eslint-config-typescript": "^14.0.0", + "@ocap/test-utils": "workspace:^", "@ts-bridge/cli": "^0.6.2", "@ts-bridge/shims": "^0.1.1", "@typescript-eslint/eslint-plugin": "^8.8.1", From 1bb5c67c1e44c343e49ed996a4342debfbf4f400 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 6 Feb 2025 20:33:05 +0100 Subject: [PATCH 13/13] locak --- yarn.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/yarn.lock b/yarn.lock index 86ae7ca13..21fd9451d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2364,6 +2364,7 @@ __metadata: "@metamask/eslint-config": "npm:^14.0.0" "@metamask/eslint-config-nodejs": "npm:^14.0.0" "@metamask/eslint-config-typescript": "npm:^14.0.0" + "@ocap/test-utils": "workspace:^" "@ocap/utils": "workspace:^" "@sqlite.org/sqlite-wasm": "npm:^3.48.0-build1" "@ts-bridge/cli": "npm:^0.6.2"