chore(nodejs): Use better sqlite

.depcheckrc.yml

@@ -38,6 +38,9 @@ ignores:
   - 'typedoc'
   - 'typescript'
 
+  # Used by @ocap/nodejs to build the sqlite3 bindings
+  - 'node-gyp'
+
   # These are peer dependencies of various modules we actually do
   # depend on, which have been elevated to full dependencies (even
   # though we don't actually depend on them) in order to work around a

.github/workflows/lint-build-test.yml

@@ -173,7 +173,7 @@ jobs:
           node-version: ${{ matrix.node-version }}
           cache: yarn
       - run: yarn --immutable
-      - run: yarn build
+      - run: yarn build -f
       - run: yarn test:e2e:ci
       - name: Require clean working directory
         shell: bash

package.json

@@ -16,7 +16,7 @@
     "build:clean": "yarn clean && yarn build",
     "build:docs": "yarn workspaces foreach --all --exclude @ocap/monorepo --exclude @ocap/extension --parallel --interlaced --verbose run build:docs",
     "build:source": "ts-bridge --project tsconfig.build.json --verbose && yarn build:special",
-    "build:special": "yarn workspace @ocap/shims run build && yarn workspace @ocap/extension run build",
+    "build:special": "yarn workspace @ocap/shims run build && yarn workspace @ocap/extension run build && yarn workspace @ocap/nodejs run build:sqlite3",
     "bundle": "node ./scripts/bundle-vat.js",
     "changelog:update": "yarn workspaces foreach --all --no-private --parallel --interlaced --verbose run changelog:update",
     "changelog:validate": "yarn workspaces foreach --all --no-private --parallel --interlaced --verbose run changelog:validate",
@@ -33,6 +33,7 @@
     "test": "vitest run",
     "test:clean": "yarn test --no-cache --coverage.clean",
     "test:dev": "yarn test --coverage false",
+    "test:e2e:ci": "yarn workspaces foreach -A run test:e2e:ci --no-silent",
     "test:verbose": "yarn test --reporter verbose",
     "test:watch": "vitest",
     "why:batch": "./scripts/why-batch.sh"
@@ -66,7 +67,8 @@
     "@typescript-eslint/eslint-plugin": "^8.8.1",
     "@typescript-eslint/parser": "^8.8.1",
     "@typescript-eslint/utils": "^8.8.1",
-    "@vitest/coverage-istanbul": "2.1.3",
+    "@vitest/browser": "2.1.4",
+    "@vitest/coverage-istanbul": "2.1.4",
     "@vitest/eslint-plugin": "^1.1.7",
     "@yarnpkg/types": "^4.0.0",
     "depcheck": "^1.4.7",
@@ -91,7 +93,7 @@
     "typescript-eslint": "^8.8.1",
     "vite": "^5.3.5",
     "vite-tsconfig-paths": "^4.3.2",
-    "vitest": "^2.1.2",
+    "vitest": "2.1.4",
     "vitest-fetch-mock": "^0.4.2"
   },
   "packageManager": "yarn@4.2.2",
@@ -106,7 +108,8 @@
       "simple-git-hooks": false,
       "vitest>@vitest/browser>webdriverio>@wdio/utils>edgedriver": false,
       "vitest>@vitest/browser>webdriverio>@wdio/utils>geckodriver": false,
-      "vitest>@vitest/mocker>msw": false
+      "vitest>@vitest/mocker>msw": false,
+      "@vitest/browser>msw": false
     }
   },
   "resolutions": {

packages/cli/package.json

@@ -78,7 +78,7 @@
     "typescript": "~5.5.4",
     "typescript-eslint": "^8.8.1",
     "vite": "^5.3.5",
-    "vitest": "^2.1.2"
+    "vitest": "2.1.4"
   },
   "engines": {
     "node": "^20 || >=22"

packages/errors/package.json

@@ -77,7 +77,7 @@
     "typescript": "~5.5.4",
     "typescript-eslint": "^8.8.1",
     "vite": "^5.3.5",
-    "vitest": "^2.1.2"
+    "vitest": "2.1.4"
   },
   "engines": {
     "node": "^20 || >=22"

packages/extension/package.json

@@ -91,7 +91,7 @@
     "vite": "^5.3.5",
     "vite-plugin-checker": "^0.8.0",
     "vite-plugin-static-copy": "^1.0.6",
-    "vitest": "^2.1.2"
+    "vitest": "2.1.4"
   },
   "engines": {
     "node": "^20 || >=22"

packages/kernel/package.json

@@ -80,7 +80,7 @@
     "typescript": "~5.5.4",
     "typescript-eslint": "^8.8.1",
     "vite": "^5.3.5",
-    "vitest": "^2.1.2"
+    "vitest": "2.1.4"
   },
   "engines": {
     "node": "^20 || >=22"

packages/nodejs/package.json

@@ -12,7 +12,9 @@
     "dist/"
   ],
   "scripts": {
-    "build": "ts-bridge --project tsconfig.build.json --clean",
+    "build": "ts-bridge --project tsconfig.build.json --clean && yarn build:sqlite3",
+    "build:e2e": "ts-bridge --project tsconfig.build.json --clean && yarn build:sqlite3 -f",
+    "build:sqlite3": "scripts/build-sqlite3.sh",
     "changelog:validate": "../../scripts/validate-changelog.sh @ocap/nodejs",
     "clean": "rimraf --glob ./dist './*.tsbuildinfo'",
     "lint": "yarn lint:ts && yarn lint:eslint && yarn lint:misc --check && yarn constraints && yarn lint:dependencies",
@@ -23,24 +25,25 @@
     "lint:ts": "tsc --project tsconfig.lint.json",
     "publish:preview": "yarn npm publish --tag preview",
     "start": "yarn build:vite:dev --watch",
-    "test": "yarn build && vitest",
+    "test": "vitest run --config vitest.config.ts",
+    "test:e2e": "vitest run --config vitest.config.e2e.ts --no-silent",
+    "test:e2e:ci": "./scripts/test-e2e-ci.sh",
     "test:clean": "yarn test --no-cache --coverage.clean",
     "test:dev": "yarn test --coverage false",
-    "test:verbose": "yarn test --reporter verbose"
+    "test:verbose": "yarn test --reporter verbose",
+    "build:docs": "typedoc",
+    "test:watch": "vitest --config vitest.config.ts"
   },
   "dependencies": {
-    "@endo/eventual-send": "^1.2.6",
     "@endo/exo": "^1.5.4",
     "@endo/patterns": "^1.4.4",
     "@endo/promise-kit": "^1.1.6",
-    "@metamask/snaps-utils": "^8.3.0",
     "@metamask/utils": "^9.3.0",
-    "@ocap/errors": "workspace:^",
     "@ocap/kernel": "workspace:^",
     "@ocap/shims": "workspace:^",
     "@ocap/streams": "workspace:^",
     "@ocap/utils": "workspace:^",
-    "@sqlite.org/sqlite-wasm": "3.47.0-build1",
+    "better-sqlite3": "^11.7.0",
     "ses": "^1.9.0"
   },
   "devDependencies": {
@@ -50,12 +53,14 @@
     "@metamask/eslint-config-nodejs": "^14.0.0",
     "@metamask/eslint-config-typescript": "^14.0.0",
     "@ocap/cli": "workspace:^",
-    "@ocap/test-utils": "workspace:^",
     "@ts-bridge/cli": "^0.5.1",
+    "@types/better-sqlite3": "^7.6.12",
     "@types/chrome": "^0.0.268",
     "@typescript-eslint/eslint-plugin": "^8.8.1",
     "@typescript-eslint/parser": "^8.8.1",
     "@typescript-eslint/utils": "^8.8.1",
+    "@vitest/browser": "2.1.4",
+    "@vitest/coverage-istanbul": "2.1.4",
     "depcheck": "^1.4.7",
     "eslint": "^9.12.0",
     "eslint-config-prettier": "^9.1.0",
@@ -65,15 +70,21 @@
     "eslint-plugin-n": "^17.11.1",
     "eslint-plugin-prettier": "^5.2.1",
     "eslint-plugin-promise": "^7.1.0",
+    "node-gyp": "^11.0.0",
     "prettier": "^3.3.3",
     "rimraf": "^6.0.1",
     "typedoc": "^0.26.8",
     "typescript": "~5.5.4",
     "typescript-eslint": "^8.8.1",
     "vite": "^5.3.5",
-    "vitest": "^2.1.2"
+    "vite-plugin-checker": "^0.8.0",
+    "vite-plugin-static-copy": "^1.0.6",
+    "vitest": "2.1.4"
   },
   "engines": {
-    "node": "^18.18 || >=20"
+    "node": "^20 || >=22"
+  },
+  "exports": {
+    "./package.json": "./package.json"
   }
 }

packages/nodejs/scripts/build-constants.mjs

@@ -0,0 +1,11 @@
+import path from 'path';
+
+export const sourceDir = './src';
+export const buildDir = path.resolve(sourceDir, '../dist');
+
+export const trustedPreludes = {
+  'kernel-worker': path.resolve(
+    sourceDir,
+    'env/kernel-worker-trusted-prelude.js',
+  ),
+};

packages/nodejs/scripts/build-sqlite3.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+# set -x
+set -e
+set -o pipefail
+
+dir=$(pwd)
+
+package_root="$(dirname "$0")/.."
+cd $package_root
+package_root=$(pwd)
+monorepo_root="$package_root/../.."
+
+while getopts ":ab:" opt; do
+  case $OPTARG in
+    f) force=1;;
+    \?) echo "Invalid option: -$OPTARG"; exit 1;;
+  esac
+done
+
+if ! [ "$force" = "1" ] && [ -f "node_modules/better-sqlite3/build/better_sqlite3.node" ]; then
+  echo "Found better-sqlite3 bindings."
+  exit 0
+fi
+
+echo "Building better-sqlite3 bindings."
+
+# build better-sqlite at the monorepo root
+cd $monorepo_root
+cd node_modules/better-sqlite3
+yarn build-release
+
+# move back to the source folder
+cd $package_root
+
+# copy the build to this package
+mkdir -p node_modules/better-sqlite3/build/
+cp -r ../../node_modules/better-sqlite3/build/Release/ node_modules/better-sqlite3/build/
+
+cd $dir

packages/nodejs/scripts/test-e2e-ci.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+set -x
+set -e
+set -o pipefail
+
+# force build sqlite3 so it aligns with node version
+yarn build:e2e
+
+# We borrow the vat definition from extension for now
+yarn ocap bundle "../extension/src/vats"
+
+# Start the server in background and capture its PID
+yarn ocap serve "../extension/src/vats" & 
+SERVER_PID=$!
+
+function cleanup() {
+  # Kill the server if it's still running
+  if kill -0 $SERVER_PID 2>/dev/null; then
+    kill $SERVER_PID
+  fi
+}
+# Ensure we always close the server
+trap cleanup EXIT
+
+yarn test:e2e

packages/nodejs/src/env/endoify.ts

@@ -0,0 +1 @@
+import '@ocap/shims/endoify';

packages/nodejs/src/env/kernel-worker-trusted-prelude.js

@@ -0,0 +1,2 @@
+// eslint-disable-next-line import-x/no-unresolved
+import './endoify.js';

packages/nodejs/src/kernel/VatWorkerService.ts

@@ -1,8 +1,9 @@
+import { makePromiseKit } from '@endo/promise-kit';
 import type { VatWorkerService, VatId } from '@ocap/kernel';
-import { NodeWorkerMultiplexer } from '@ocap/streams';
-import type { StreamMultiplexer } from '@ocap/streams';
+import { NodeWorkerMultiplexer, StreamMultiplexer } from '@ocap/streams';
 import { makeLogger } from '@ocap/utils';
 import type { Logger } from '@ocap/utils';
+import { readFile } from 'node:fs/promises';
 import { Worker as NodeWorker } from 'node:worker_threads';
 
 // Worker file loads from the built dist directory, requires rebuild after change
@@ -11,7 +12,6 @@ const workerFileURL = new URL('../../dist/vat/inside.mjs', import.meta.url)
   .pathname;
 
 export class NodejsVatWorkerService implements VatWorkerService {
-  // eslint-disable-next-line no-unused-private-class-members
   readonly #logger: Logger;
 
   workers = new Map<
@@ -30,10 +30,25 @@ export class NodejsVatWorkerService implements VatWorkerService {
   }
 
   async launch(vatId: VatId): Promise<StreamMultiplexer> {
-    const worker = new NodeWorker(workerFileURL);
-    const multiplexer = new NodeWorkerMultiplexer(worker);
-    this.workers.set(vatId, { worker, multiplexer });
-    return multiplexer;
+    const { promise, resolve } = makePromiseKit<StreamMultiplexer>();
+    this.#logger.debug('launching', vatId);
+    this.#logger.debug('fileUrl', workerFileURL);
+    this.#logger.debug(
+      `fileContent\n===\n${(await readFile(workerFileURL)).toString()}\n===\n`,
+    );
+    const worker = new NodeWorker(workerFileURL, {
+      env: {
+        NODE_VAT_ID: vatId,
+      },
+    });
+    this.#logger.debug('launched', vatId);
+    worker.once('online', () => {
+      const multiplexer = new NodeWorkerMultiplexer(worker, 'vat');
+      this.workers.set(vatId, { worker, multiplexer });
+      resolve(multiplexer);
+      this.#logger.debug('launched', vatId);
+    });
+    return promise;
   }
 
   async terminate(vatId: VatId): Promise<undefined> {

packages/nodejs/src/kernel/kernel-worker.ts

@@ -1,4 +1,4 @@
-import './env.js';
+import '@ocap/shims/endoify';
 import type { NonEmptyArray } from '@metamask/utils';
 import type { KernelCommand, KernelCommandReply, VatId } from '@ocap/kernel';
 import { Kernel, VatCommandMethod } from '@ocap/kernel';
@@ -31,6 +31,9 @@ export async function makeKernel(port: NodeMessagePort): Promise<Kernel> {
   return kernel;
 }
 
+const sampleOne = <Ele>(from: NonEmptyArray<Ele>): Ele =>
+  from[Math.floor(Math.random() * from.length)] as Ele;
+
 /**
  * Runs the full lifecycle of an array of vats, including their creation,
  * restart, message passing, and termination.
@@ -42,27 +45,31 @@ export async function runVatLifecycle(
   kernel: Kernel,
   vats: NonEmptyArray<VatId>,
 ): Promise<void> {
-  console.time(`Created vats: ${vats.join(', ')}`);
+  console.log('runVatLifecycle Start...');
+  const vatLabel = vats.join(', ');
+  console.time(`Created vats: ${vatLabel}`);
   await Promise.all(
-    vats.map(async () =>
-      kernel.launchVat({
-        bundleName: 'sample-vat',
-        parameters: { name: 'Nodeen' },
-      }),
+    vats.map(
+      async () =>
+        await kernel.launchVat({
+          bundleSpec: 'http://localhost:3000/sample-vat.bundle',
+          parameters: { name: 'Nodeen' },
+        }),
     ),
   );
-  console.timeEnd(`Created vats: ${vats.join(', ')}`);
-
+  console.timeEnd(`Created vats: ${vatLabel}`);
   console.log('Kernel vats:', kernel.getVatIds().join(', '));
 
+  const knownVats = kernel.getVatIds() as NonEmptyArray<VatId>;
+
   // Restart a randomly selected vat from the array.
-  const vatToRestart = vats[Math.floor(Math.random() * vats.length)] as VatId;
+  const vatToRestart = sampleOne(knownVats);
   console.time(`Vat "${vatToRestart}" restart`);
   await kernel.restartVat(vatToRestart);
   console.timeEnd(`Vat "${vatToRestart}" restart`);
 
   // Send a "Ping" message to a randomly selected vat.
-  const vatToPing = vats[Math.floor(Math.random() * vats.length)] as VatId;
+  const vatToPing = sampleOne(knownVats);
   console.time(`Ping Vat "${vatToPing}"`);
   await kernel.sendMessage(vatToPing, {
     method: VatCommandMethod.ping,