feat(kernel): Support liveslots distributed garbage collection

.github/workflows/lint-build-test.yml

@@ -149,7 +149,6 @@ jobs:
           yarn build-release
           cd -
       - run: yarn build
-      - run: yarn build:vats
       - run: yarn run test
       - name: Require clean working directory
         shell: bash

package.json

@@ -16,7 +16,7 @@
     "build:clean": "yarn clean && yarn build",
     "build:docs": "yarn workspaces foreach --all --exclude @ocap/monorepo --exclude @ocap/extension --parallel --interlaced --verbose run build:docs",
     "build:source": "ts-bridge --project tsconfig.build.json --verbose && yarn build:special",
-    "build:special": "yarn workspace @ocap/shims run build && yarn workspace @ocap/extension run build",
+    "build:special": "yarn workspace @ocap/shims run build && yarn workspace @ocap/extension run build && yarn workspace @ocap/kernel-test run build:vats",
     "bundle": "node ./scripts/bundle-vat.js",
     "changelog:update": "yarn workspaces foreach --all --no-private --parallel --interlaced --verbose run changelog:update",
     "changelog:validate": "yarn workspaces foreach --all --no-private --parallel --interlaced --verbose run changelog:validate",

packages/kernel-test/src/liveslots.test.ts

@@ -8,7 +8,7 @@ import {
 } from 'node:worker_threads';
 import { beforeEach, describe, expect, it } from 'vitest';
 
-import { kunser } from '../../kernel/src/kernel-marshal.ts';
+import { kunser } from '../../kernel/src/services/kernel-marshal.ts';
 import { makeKernel } from '../../nodejs/src/kernel/make-kernel.ts';
 
 const origStdoutWrite = process.stdout.write.bind(process.stdout);

packages/kernel-test/src/supervisor.test.ts

@@ -5,7 +5,7 @@ import { readFile } from 'fs/promises';
 import { join } from 'path';
 import { describe, it, expect } from 'vitest';
 
-import { kser } from '../../kernel/src/kernel-marshal.ts';
+import { kser } from '../../kernel/src/services/kernel-marshal.ts';
 import { TestDuplexStream } from '../../streams/test/stream-mocks.ts';
 
 const makeVatSupervisor = async ({

packages/kernel-test/src/vatstore.test.ts

@@ -17,7 +17,7 @@ import {
 } from 'node:worker_threads';
 import { describe, vi, expect, it } from 'vitest';
 
-import { kunser } from '../../kernel/src/kernel-marshal.ts';
+import { kunser } from '../../kernel/src/services/kernel-marshal.ts';
 import { NodejsVatWorkerService } from '../../nodejs/src/kernel/VatWorkerService.ts';
 
 /**

packages/kernel/src/Kernel.ts

@@ -12,22 +12,20 @@ import type { DuplexStream } from '@ocap/streams';
 import type { Logger } from '@ocap/utils';
 import { makeLogger } from '@ocap/utils';
 
-import { assert, Fail } from './assert.ts';
-import { kser, kunser, krefOf, kslot } from './kernel-marshal.ts';
-import type { SlotValue } from './kernel-marshal.ts';
 import { isKernelCommand, KernelCommandMethod } from './messages/index.ts';
 import type {
   KernelCommand,
   KernelCommandReply,
   VatCommand,
   VatCommandReturnType,
 } from './messages/index.ts';
-import {
-  parseRef,
-  isPromiseRef,
-  makeKernelStore,
-} from './store/kernel-store.ts';
+import { processGCActionSet } from './services/garbage-collection.ts';
+import type { SlotValue } from './services/kernel-marshal.ts';
+import { kser, kunser, krefOf, kslot } from './services/kernel-marshal.ts';
+import { makeKernelStore } from './store/kernel-store.ts';
 import type { KernelStore } from './store/kernel-store.ts';
+import { parseRef } from './store/utils/parse-ref.ts';
+import { isPromiseRef } from './store/utils/promise-ref.ts';
 import type {
   VatId,
   VRef,
@@ -45,6 +43,7 @@ import {
   insistMessage,
   isClusterConfig,
 } from './types.ts';
+import { assert, Fail } from './utils/assert.ts';
 import { VatHandle } from './VatHandle.ts';
 
 /**
@@ -193,6 +192,18 @@ export class Kernel {
    */
   async *#runQueueItems(): AsyncGenerator<RunQueueItem> {
     for (;;) {
+      const gcAction = processGCActionSet(this.#kernelStore);
+      if (gcAction) {
+        yield gcAction;
+        continue;
+      }
+
+      const reapAction = this.#kernelStore.nextReapAction();
+      if (reapAction) {
+        yield reapAction;
+        continue;
+      }
+
       while (this.#runQueueLength > 0) {
         const item = this.#dequeueRun();
         if (item) {
@@ -201,6 +212,7 @@ export class Kernel {
           break;
         }
       }
+
       if (this.#runQueueLength === 0) {
         const { promise, resolve } = makePromiseKit<void>();
         if (this.#wakeUpTheRunQueue !== null) {
@@ -582,6 +594,38 @@ export class Kernel {
         log(`@@@@ done ${vatId} notify ${kpid}`);
         break;
       }
+      case 'dropExports': {
+        const { vatId, krefs } = item;
+        log(`@@@@ deliver ${vatId} dropExports`, krefs);
+        const vat = this.#getVat(vatId);
+        await vat.deliverDropExports(krefs);
+        log(`@@@@ done ${vatId} dropExports`, krefs);
+        break;
+      }
+      case 'retireExports': {
+        const { vatId, krefs } = item;
+        log(`@@@@ deliver ${vatId} retireExports`, krefs);
+        const vat = this.#getVat(vatId);
+        await vat.deliverRetireExports(krefs);
+        log(`@@@@ done ${vatId} retireExports`, krefs);
+        break;
+      }
+      case 'retireImports': {
+        const { vatId, krefs } = item;
+        log(`@@@@ deliver ${vatId} retireImports`, krefs);
+        const vat = this.#getVat(vatId);
+        await vat.deliverRetireImports(krefs);
+        log(`@@@@ done ${vatId} retireImports`, krefs);
+        break;
+      }
+      case 'bringOutYourDead': {
+        const { vatId } = item;
+        log(`@@@@ deliver ${vatId} bringOutYourDead`);
+        const vat = this.#getVat(vatId);
+        await vat.deliverBringOutYourDead();
+        log(`@@@@ done ${vatId} bringOutYourDead`);
+        break;
+      }
       default:
         // @ts-expect-error Runtime does not respect "never".
         Fail`unsupported or unknown run queue item type ${item.type}`;

packages/kernel/src/VatHandle.test.ts

@@ -8,8 +8,8 @@ import type { MockInstance } from 'vitest';
 import { Kernel } from './Kernel.ts';
 import { isVatCommandReply, VatCommandMethod } from './messages/index.ts';
 import type { VatCommand, VatCommandReply } from './messages/index.ts';
-import type { KernelStore } from './store/kernel-store.ts';
 import { makeKernelStore } from './store/kernel-store.ts';
+import type { KernelStore } from './store/kernel-store.ts';
 import { VatHandle } from './VatHandle.ts';
 import { makeMapKernelDatabase } from '../test/storage.ts';
 

packages/kernel/src/VatHandle.ts

@@ -19,6 +19,7 @@ import type {
   VatCommandReturnType,
 } from './messages/index.ts';
 import type { KernelStore } from './store/kernel-store.ts';
+import { parseRef } from './store/utils/parse-ref.ts';
 import type {
   PromiseCallbacks,
   VatId,
@@ -312,6 +313,76 @@ export class VatHandle {
     }
   }
 
+  /**
+   * Handle a 'dropImports' syscall from the vat.
+   *
+   * @param krefs - The KRefs of the imports to be dropped.
+   */
+  #handleSyscallDropImports(krefs: KRef[]): void {
+    for (const kref of krefs) {
+      const { direction, isPromise } = parseRef(kref);
+      // We validate it's an import - meaning this vat received this object from somewhere else
+      if (direction === 'export' || isPromise) {
+        throw Error(
+          `vat ${this.vatId} issued invalid syscall dropImports for ${kref}`,
+        );
+      }
+      this.#kernelStore.clearReachableFlag(this.vatId, kref);
+    }
+  }
+
+  /**
+   * Handle a 'retireImports' syscall from the vat.
+   *
+   * @param krefs - The KRefs of the imports to be retired.
+   */
+  #handleSyscallRetireImports(krefs: KRef[]): void {
+    for (const kref of krefs) {
+      const { direction, isPromise } = parseRef(kref);
+      // We validate it's an import - meaning this vat received this object from somewhere else
+      if (direction === 'export' || isPromise) {
+        throw Error(
+          `vat ${this.vatId} issued invalid syscall retireImports for ${kref}`,
+        );
+      }
+      if (this.#kernelStore.getReachableFlag(this.vatId, kref)) {
+        throw Error(`syscall.retireImports but ${kref} is still reachable`);
+      }
+      // deleting the clist entry will decrement the recognizable count, but
+      // not the reachable count (because it was unreachable, as we asserted)
+      this.#kernelStore.forgetKref(this.vatId, kref);
+    }
+  }
+
+  /**
+   * Handle retiring or abandoning exports syscall from the vat.
+   *
+   * @param krefs - The KRefs of the exports to be retired/abandoned.
+   * @param checkReachable - If true, verify the object is not reachable (retire). If false, ignore reachability (abandon).
+   */
+  #handleSyscallExportCleanup(krefs: KRef[], checkReachable: boolean): void {
+    const action = checkReachable ? 'retire' : 'abandon';
+
+    for (const kref of krefs) {
+      const { direction, isPromise } = parseRef(kref);
+      // We validate it's an export - meaning this vat created/owns this object
+      if (direction === 'import' || isPromise) {
+        throw Error(
+          `vat ${this.vatId} issued invalid syscall ${action}Exports for ${kref}`,
+        );
+      }
+      if (checkReachable) {
+        if (this.#kernelStore.getReachableFlag(this.vatId, kref)) {
+          throw Error(
+            `syscall.${action}Exports but ${kref} is still reachable`,
+          );
+        }
+      }
+      this.#kernelStore.forgetKref(this.vatId, kref);
+      this.#logger.debug(`${action}Exports: deleted object ${kref}`);
+    }
+  }
+
   /**
    * Handle a syscall from the vat.
    *
@@ -354,24 +425,28 @@ export class VatHandle {
         // [KRef[]];
         const [, refs] = kso;
         log(`@@@@ ${vatId} syscall dropImports ${JSON.stringify(refs)}`);
+        this.#handleSyscallDropImports(refs);
         break;
       }
       case 'retireImports': {
         // [KRef[]];
         const [, refs] = kso;
         log(`@@@@ ${vatId} syscall retireImports ${JSON.stringify(refs)}`);
+        this.#handleSyscallRetireImports(refs);
         break;
       }
       case 'retireExports': {
         // [KRef[]];
         const [, refs] = kso;
         log(`@@@@ ${vatId} syscall retireExports ${JSON.stringify(refs)}`);
+        this.#handleSyscallExportCleanup(refs, true);
         break;
       }
       case 'abandonExports': {
         // [KRef[]];
         const [, refs] = kso;
         log(`@@@@ ${vatId} syscall abandonExports ${JSON.stringify(refs)}`);
+        this.#handleSyscallExportCleanup(refs, false);
         break;
       }
       case 'callNow':
@@ -450,6 +525,52 @@ export class VatHandle {
     });
   }
 
+  /**
+   * Make a 'dropExports' delivery to the vat.
+   *
+   * @param krefs - The KRefs of the exports to be dropped.
+   */
+  async deliverDropExports(krefs: KRef[]): Promise<void> {
+    await this.sendVatCommand({
+      method: VatCommandMethod.deliver,
+      params: ['dropExports', krefs],
+    });
+  }
+
+  /**
+   * Make a 'retireExports' delivery to the vat.
+   *
+   * @param krefs - The KRefs of the exports to be retired.
+   */
+  async deliverRetireExports(krefs: KRef[]): Promise<void> {
+    await this.sendVatCommand({
+      method: VatCommandMethod.deliver,
+      params: ['retireExports', krefs],
+    });
+  }
+
+  /**
+   * Make a 'retireImports' delivery to the vat.
+   *
+   * @param krefs - The KRefs of the imports to be retired.
+   */
+  async deliverRetireImports(krefs: KRef[]): Promise<void> {
+    await this.sendVatCommand({
+      method: VatCommandMethod.deliver,
+      params: ['retireImports', krefs],
+    });
+  }
+
+  /**
+   * Make a 'bringOutYourDead' delivery to the vat.
+   */
+  async deliverBringOutYourDead(): Promise<void> {
+    await this.sendVatCommand({
+      method: VatCommandMethod.deliver,
+      params: ['bringOutYourDead'],
+    });
+  }
+
   /**
    * Terminates the vat.
    *

packages/kernel/src/VatSupervisor.ts

@@ -10,20 +10,16 @@ import type { CapData } from '@endo/marshal';
 import { StreamReadError } from '@ocap/errors';
 import type { DuplexStream } from '@ocap/streams';
 
-import type {
-  DispatchFn,
-  MakeLiveSlotsFn,
-  GCTools,
-} from './ag-liveslots-types.ts';
-import { makeDummyMeterControl } from './dummyMeterControl.ts';
 import type { VatCommand, VatCommandReply } from './messages/index.ts';
 import { VatCommandMethod } from './messages/index.ts';
-import { makeSupervisorSyscall } from './syscall.ts';
+import { makeDummyMeterControl } from './services/meter-control.ts';
+import { makeSupervisorSyscall } from './services/syscall.ts';
+import type { DispatchFn, MakeLiveSlotsFn, GCTools } from './services/types.ts';
 import type { VatConfig, VatId, VRef } from './types.ts';
 import { ROOT_OBJECT_VREF, isVatConfig } from './types.ts';
+import { waitUntilQuiescent } from './utils/wait-quiescent.ts';
 import type { VatKVStore } from './VatKVStore.ts';
 import { makeVatKVStore } from './VatKVStore.ts';
-import { waitUntilQuiescent } from './waitUntilQuiescent.ts';
 
 const makeLiveSlots: MakeLiveSlotsFn = localMakeLiveSlots;