Add nodejs package

.depcheckrc.yml

@@ -39,6 +39,9 @@ ignores:
   - 'typescript'
   - 'typescript-plugin-css-modules'
 
+  # Used by @ocap/nodejs to build the sqlite3 bindings
+  - 'node-gyp'
+
   # These are peer dependencies of various modules we actually do
   # depend on, which have been elevated to full dependencies (even
   # though we don't actually depend on them) in order to work around a

package.json

@@ -34,6 +34,7 @@
     "test": "vitest run",
     "test:clean": "yarn test --no-cache --coverage.clean",
     "test:dev": "yarn test --coverage false",
+    "test:e2e:ci": "yarn workspaces foreach --all run test:e2e:ci",
     "test:verbose": "yarn test --reporter verbose",
     "test:watch": "vitest",
     "why:batch": "./scripts/why-batch.sh"

packages/nodejs/CHANGELOG.md

@@ -0,0 +1,10 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+[Unreleased]: https://github.com/MetaMask/ocap-kernel/

packages/nodejs/README.md

@@ -0,0 +1,35 @@
+# `@ocap/nodejs`
+
+For running Ocap Kernel experiments in a Node.js environment
+
+## Installation
+
+`yarn add @ocap/nodejs`
+
+or
+
+`npm install @ocap/nodejs`
+
+## Contributing
+
+This package is part of a monorepo. Instructions for contributing can be found in the [monorepo README](https://github.com/MetaMask/ocap-kernel#readme).
+
+## End-to-End Tests
+
+Navigate to package root.
+
+```sh
+cd ~/path/to/ocap-kernel/packages/nodejs
+```
+
+If it's not already running, start the `@ocap/cli` in `extension/src/vats/`.
+
+```sh
+yarn ocap start ../extension/src/vats
+```
+
+Then, run the end to end tests.
+
+```sh
+yarn test:e2e
+```

packages/nodejs/package.json

@@ -0,0 +1,90 @@
+{
+  "name": "@ocap/nodejs",
+  "version": "0.0.0",
+  "private": true,
+  "description": "For running Ocap Kernel experiments in a Node.js environment",
+  "homepage": "https://github.com/MetaMask/ocap-kernel/tree/main/packages/nodejs#readme",
+  "bugs": {
+    "url": "https://github.com/MetaMask/ocap-kernel/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/MetaMask/ocap-kernel.git"
+  },
+  "type": "module",
+  "exports": {
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "dist/"
+  ],
+  "scripts": {
+    "build": "ts-bridge --project tsconfig.build.json --clean && yarn build:sqlite3",
+    "build:e2e": "ts-bridge --project tsconfig.build.json --clean && yarn build:sqlite3 -f",
+    "build:sqlite3": "scripts/build-sqlite3.sh",
+    "build:docs": "typedoc",
+    "changelog:validate": "../../scripts/validate-changelog.sh @ocap/nodejs",
+    "clean": "rimraf --glob './*.tsbuildinfo' ./dist ./.eslintcache",
+    "lint": "yarn lint:ts && yarn lint:eslint && yarn lint:misc --check && yarn constraints && yarn lint:dependencies",
+    "lint:dependencies": "depcheck",
+    "lint:eslint": "eslint . --cache",
+    "lint:fix": "yarn lint:ts && yarn lint:eslint --fix && yarn lint:misc --write && yarn constraints --fix && yarn lint:dependencies",
+    "lint:misc": "prettier --no-error-on-unmatched-pattern '**/*.json' '**/*.md' '**/*.html' '!**/CHANGELOG.old.md' '**/*.yml' '!.yarnrc.yml' '!merged-packages/**' --ignore-path ../../.gitignore",
+    "lint:ts": "tsc --project tsconfig.lint.json",
+    "publish:preview": "yarn npm publish --tag preview",
+    "test": "vitest run --config vitest.config.ts",
+    "test:e2e": "vitest run --config vitest.config.e2e.ts",
+    "test:e2e:ci": "./scripts/test-e2e-ci.sh",
+    "test:clean": "yarn test --no-cache --coverage.clean",
+    "test:dev": "yarn test --coverage false",
+    "test:verbose": "yarn test --reporter verbose",
+    "test:watch": "vitest --config vitest.config.ts"
+  },
+  "devDependencies": {
+    "@arethetypeswrong/cli": "^0.16.4",
+    "@metamask/auto-changelog": "^4.0.0",
+    "@metamask/eslint-config": "^14.0.0",
+    "@metamask/eslint-config-nodejs": "^14.0.0",
+    "@metamask/eslint-config-typescript": "^14.0.0",
+    "@ocap/cli": "workspace:^",
+    "@ts-bridge/cli": "^0.6.2",
+    "@ts-bridge/shims": "^0.1.1",
+    "@types/better-sqlite3": "^7.6.12",
+    "@typescript-eslint/eslint-plugin": "^8.8.1",
+    "@typescript-eslint/parser": "^8.8.1",
+    "@typescript-eslint/utils": "^8.8.1",
+    "@vitest/eslint-plugin": "^1.1.24",
+    "depcheck": "^1.4.7",
+    "eslint": "^9.12.0",
+    "eslint-config-prettier": "^9.1.0",
+    "eslint-import-resolver-typescript": "^3.6.3",
+    "eslint-plugin-import-x": "^4.3.1",
+    "eslint-plugin-jsdoc": "^50.3.1",
+    "eslint-plugin-n": "^17.11.1",
+    "eslint-plugin-prettier": "^5.2.1",
+    "eslint-plugin-promise": "^7.1.0",
+    "node-gyp": "^11.0.0",
+    "prettier": "^3.3.3",
+    "rimraf": "^6.0.1",
+    "typedoc": "^0.26.8",
+    "typescript": "~5.5.4",
+    "typescript-eslint": "^8.8.1",
+    "vite": "^5.3.5",
+    "vitest": "2.1.8"
+  },
+  "engines": {
+    "node": "^20 || >=22"
+  },
+  "dependencies": {
+    "@endo/exo": "^1.5.4",
+    "@endo/patterns": "^1.4.4",
+    "@endo/promise-kit": "^1.1.6",
+    "@metamask/utils": "^11.0.1",
+    "@ocap/kernel": "workspace:^",
+    "@ocap/shims": "workspace:^",
+    "@ocap/streams": "workspace:^",
+    "@ocap/utils": "workspace:^",
+    "better-sqlite3": "^11.7.2",
+    "ses": "^1.9.0"
+  }
+}

packages/nodejs/scripts/build-sqlite3.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+# set -x
+set -e
+set -o pipefail
+
+dir=$(pwd)
+
+package_root="$(dirname "$0")/.."
+cd "$package_root"
+package_root=$(pwd)
+monorepo_root="$package_root/../.."
+
+while getopts ":ab:" _; do
+  case $OPTARG in
+    f) force=1 ;;
+    \?) echo "Invalid option: -$OPTARG"; exit 1 ;;
+  esac
+done
+
+if ! [ "$force" = "1" ] && [ -f "node_modules/better-sqlite3/build/better_sqlite3.node" ]; then
+  echo "Found better-sqlite3 bindings."
+  exit 0
+fi
+
+echo "Building better-sqlite3 bindings."
+
+# build better-sqlite at the monorepo root
+cd "$monorepo_root"
+cd node_modules/better-sqlite3
+yarn build-release
+
+# move back to the source folder
+cd "$package_root"
+
+# copy the build to this package
+mkdir -p node_modules/better-sqlite3/build/
+cp -r ../../node_modules/better-sqlite3/build/Release/ node_modules/better-sqlite3/build/
+
+cd "$dir"

packages/nodejs/scripts/test-e2e-ci.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+set -x
+set -e
+set -o pipefail
+
+# force build sqlite3 so it aligns with node version
+yarn build:e2e
+
+# We borrow the vat definition from extension for now
+yarn ocap bundle "../extension/src/vats"
+
+# Start the server in background and capture its PID
+yarn ocap serve "../extension/src/vats" & 
+SERVER_PID=$!
+
+function cleanup() {
+  # Kill the server if it's still running
+  if kill -0 $SERVER_PID 2>/dev/null; then
+    kill $SERVER_PID
+  fi
+}
+# Ensure we always close the server
+trap cleanup EXIT
+
+yarn test:e2e

packages/nodejs/src/env/endoify.ts

@@ -0,0 +1 @@
+import '@ocap/shims/endoify';

packages/nodejs/src/env/kernel-worker-trusted-prelude.js

@@ -0,0 +1,2 @@
+// eslint-disable-next-line import-x/no-unresolved
+import './endoify.js';

packages/nodejs/src/kernel/VatWorkerService.test.ts

@@ -0,0 +1,12 @@
+import '@ocap/shims/endoify';
+
+import { describe, expect, it } from 'vitest';
+
+import { NodejsVatWorkerService } from './VatWorkerService.js';
+
+describe('NodejsVatWorkerService', () => {
+  it('constructs an instance without any arguments', () => {
+    const instance = new NodejsVatWorkerService();
+    expect(instance).toBeInstanceOf(NodejsVatWorkerService);
+  });
+});

packages/nodejs/src/kernel/VatWorkerService.ts

@@ -0,0 +1,65 @@
+import { makePromiseKit } from '@endo/promise-kit';
+import type { VatWorkerService, VatId } from '@ocap/kernel';
+import { NodeWorkerMultiplexer, StreamMultiplexer } from '@ocap/streams';
+import { makeLogger } from '@ocap/utils';
+import type { Logger } from '@ocap/utils';
+import { Worker as NodeWorker } from 'node:worker_threads';
+
+// Worker file loads from the built dist directory, requires rebuild after change
+// Note: Worker runs in same process and may be subject to spectre-style attacks
+const workerFileURL = new URL('../../dist/vat/vat-worker.mjs', import.meta.url)
+  .pathname;
+
+export class NodejsVatWorkerService implements VatWorkerService {
+  readonly #logger: Logger;
+
+  workers = new Map<
+    VatId,
+    { worker: NodeWorker; multiplexer: StreamMultiplexer }
+  >();
+
+  /**
+   * The vat worker service, intended to be constructed in
+   * the kernel worker.
+   *
+   * @param logger - An optional {@link Logger}. Defaults to a new logger labeled '[vat worker client]'.
+   */
+  constructor(logger?: Logger) {
+    this.#logger = logger ?? makeLogger('[vat worker service]');
+  }
+
+  async launch(vatId: VatId): Promise<StreamMultiplexer> {
+    const { promise, resolve } = makePromiseKit<StreamMultiplexer>();
+    this.#logger.debug('launching', vatId);
+    const worker = new NodeWorker(workerFileURL, {
+      env: {
+        NODE_VAT_ID: vatId,
+      },
+    });
+    this.#logger.debug('launched', vatId);
+    worker.once('online', () => {
+      const multiplexer = new NodeWorkerMultiplexer(worker, 'vat');
+      this.workers.set(vatId, { worker, multiplexer });
+      resolve(multiplexer);
+      this.#logger.debug('connected', vatId);
+    });
+    return promise;
+  }
+
+  async terminate(vatId: VatId): Promise<undefined> {
+    const workerEntry = this.workers.get(vatId);
+    assert(workerEntry, `No worker found for vatId ${vatId}`);
+    const { worker, multiplexer } = workerEntry;
+    await multiplexer.return();
+    await worker.terminate();
+    this.workers.delete(vatId);
+    return undefined;
+  }
+
+  async terminateAll(): Promise<void> {
+    for (const vatId of this.workers.keys()) {
+      await this.terminate(vatId);
+    }
+  }
+}
+harden(NodejsVatWorkerService);

packages/nodejs/src/kernel/kernel-worker.ts

@@ -0,0 +1,87 @@
+import '@ocap/shims/endoify';
+import type { NonEmptyArray } from '@metamask/utils';
+import type { KernelCommand, KernelCommandReply, VatId } from '@ocap/kernel';
+import { Kernel, VatCommandMethod } from '@ocap/kernel';
+import { NodeWorkerDuplexStream } from '@ocap/streams';
+import { MessagePort as NodeMessagePort } from 'worker_threads';
+
+import { makeSQLKVStore } from './sqlite-kv-store.js';
+import { NodejsVatWorkerService } from './VatWorkerService.js';
+
+/**
+ * The main function for the kernel worker.
+ *
+ * @param port - The kernel's end of a node:worker_threads MessageChannel
+ * @returns The kernel, initialized.
+ */
+export async function makeKernel(port: NodeMessagePort): Promise<Kernel> {
+  const nodeStream = new NodeWorkerDuplexStream<
+    KernelCommand,
+    KernelCommandReply
+  >(port);
+  const vatWorkerClient = new NodejsVatWorkerService();
+
+  // Initialize kernel store.
+  const kvStore = await makeSQLKVStore();
+
+  // Create and start kernel.
+  const kernel = new Kernel(nodeStream, vatWorkerClient, kvStore);
+  await kernel.init();
+
+  return kernel;
+}
+
+/**
+ * Runs the full lifecycle of an array of vats, including their creation,
+ * restart, message passing, and termination.
+ *
+ * @param kernel The kernel instance.
+ * @param vats An array of VatIds to be managed.
+ */
+export async function runVatLifecycle(
+  kernel: Kernel,
+  vats: NonEmptyArray<VatId>,
+): Promise<void> {
+  console.log('runVatLifecycle Start...');
+  const vatLabel = vats.join(', ');
+  console.time(`Created vats: ${vatLabel}`);
+  await Promise.all(
+    vats.map(
+      async () =>
+        await kernel.launchVat({
+          bundleSpec: 'http://localhost:3000/sample-vat.bundle',
+          parameters: { name: 'Nodeen' },
+        }),
+    ),
+  );
+  console.timeEnd(`Created vats: ${vatLabel}`);
+  const knownVats = kernel.getVatIds() as NonEmptyArray<VatId>;
+  const knownVatsLabel = knownVats.join(', ');
+  console.log('Kernel vats:', knownVatsLabel);
+
+  // Restart a randomly selected vat from the array.
+  console.time(`Restart vats: ${knownVatsLabel}`);
+  await Promise.all(
+    knownVats.map(async (vatId: VatId) => await kernel.restartVat(vatId)),
+  );
+  console.timeEnd(`Restart vats: ${knownVatsLabel}`);
+
+  // Send a "Ping" message to a randomly selected vat.
+  console.time(`Ping vats: ${knownVatsLabel}`);
+  await Promise.all(
+    knownVats.map(
+      async (vatId: VatId) =>
+        await kernel.sendMessage(vatId, {
+          method: VatCommandMethod.ping,
+          params: null,
+        }),
+    ),
+  );
+  console.timeEnd(`Ping vats "${knownVatsLabel}"`);
+
+  console.time(`Terminated vats: ${knownVatsLabel}`);
+  await kernel.terminateAllVats();
+  console.timeEnd(`Terminated vats: ${knownVatsLabel}`);
+
+  console.log(`Kernel has ${kernel.getVatIds().length} vats`);
+}