From 53ce0f686a28b53b1b2824f54d4c358c10cba1fa Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Wed, 12 Feb 2025 15:00:37 +0100 Subject: [PATCH 01/21] feat(kernel): add retire/abandon syscall handlers --- packages/kernel/src/VatHandle.ts | 85 ++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/packages/kernel/src/VatHandle.ts b/packages/kernel/src/VatHandle.ts index 32e111414..4cb6a8fb7 100644 --- a/packages/kernel/src/VatHandle.ts +++ b/packages/kernel/src/VatHandle.ts @@ -18,6 +18,7 @@ import type { VatCommand, VatCommandReturnType, } from './messages/index.ts'; +import { parseRef } from './store/kernel-store.ts'; import type { KernelStore } from './store/kernel-store.ts'; import type { PromiseCallbacks, @@ -312,6 +313,86 @@ export class VatHandle { } } + /** + * Handle a 'dropImports' syscall from the vat. + * + * @param krefs - The KRefs of the imports to be dropped. + */ + #handleSyscallDropImports(krefs: KRef[]): void { + for (const kref of krefs) { + const { direction, isPromise } = parseRef(kref); + // We validate it's an import - meaning this vat received this object from somewhere else + if (direction === 'export' || isPromise) { + throw Error( + `vat ${this.vatId} issued invalid syscall dropImports for ${kref}`, + ); + } + // Decrement the refCount and delete the object if it's now zero + const newCount = this.#storage.decRefCount(kref); + if (newCount === 0) { + this.#storage.deleteKernelObject(kref); + } + } + } + + /** + * Handle a 'retireImports' syscall from the vat. + * + * @param krefs - The KRefs of the imports to be retired. + */ + #handleSyscallRetireImports(krefs: KRef[]): void { + for (const kref of krefs) { + const { direction, isPromise } = parseRef(kref); + // We validate it's an import - meaning this vat received this object from somewhere else + if (direction === 'export' || isPromise) { + throw Error( + `vat ${this.vatId} issued invalid syscall retireImports for ${kref}`, + ); + } + // Check that the refCount is 0 before retiring + const refCount = this.#storage.getRefCount(kref); + if (refCount > 0) { + throw Error( + `syscall.retireImports but ${kref} still has references (count: ${refCount})`, + ); + } + // Delete the object from storage + this.#storage.deleteKernelObject(kref); + } + } + + /** + * Handle retiring or abandoning exports syscall from the vat. + * + * @param krefs - The KRefs of the exports to be retired/abandoned. + * @param checkRefCount - If true, verify refCount is 0 (retire). If false, ignore refCount (abandon). + */ + #handleSyscallExportCleanup(krefs: KRef[], checkRefCount: boolean): void { + const action = checkRefCount ? 'retire' : 'abandon'; + + for (const kref of krefs) { + const { direction, isPromise } = parseRef(kref); + // We validate it's an export - meaning this vat created/owns this object + if (direction === 'import' || isPromise) { + throw Error( + `vat ${this.vatId} issued invalid syscall ${action}Exports for ${kref}`, + ); + } + if (checkRefCount) { + // Check that the refCount is 0 before retiring + const refCount = this.#storage.getRefCount(kref); + if (refCount > 0) { + throw Error( + `syscall.${action}Exports but ${kref} still has references (count: ${refCount})`, + ); + } + } + // Delete the object from storage + this.#storage.deleteKernelObject(kref); + this.#logger.debug(`${action}Exports: deleted object ${kref}`); + } + } + /** * Handle a syscall from the vat. * @@ -354,24 +435,28 @@ export class VatHandle { // [KRef[]]; const [, refs] = kso; log(`@@@@ ${vatId} syscall dropImports ${JSON.stringify(refs)}`); + this.#handleSyscallDropImports(refs); break; } case 'retireImports': { // [KRef[]]; const [, refs] = kso; log(`@@@@ ${vatId} syscall retireImports ${JSON.stringify(refs)}`); + this.#handleSyscallRetireImports(refs); break; } case 'retireExports': { // [KRef[]]; const [, refs] = kso; log(`@@@@ ${vatId} syscall retireExports ${JSON.stringify(refs)}`); + this.#handleSyscallExportCleanup(refs, true); break; } case 'abandonExports': { // [KRef[]]; const [, refs] = kso; log(`@@@@ ${vatId} syscall abandonExports ${JSON.stringify(refs)}`); + this.#handleSyscallExportCleanup(refs, false); break; } case 'callNow': From aa7e994901b50ba42bdfc337a31d512668db8dce Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 13 Feb 2025 21:42:45 +0100 Subject: [PATCH 02/21] Add reachability tracking --- packages/kernel/src/Kernel.ts | 7 +- packages/kernel/src/VatHandle.ts | 33 ++-- .../kernel/src/store/kernel-store.test.ts | 2 + packages/kernel/src/store/kernel-store.ts | 179 ++++++++++++------ packages/kernel/src/store/parse-ref.ts | 59 ++++++ packages/kernel/src/store/reachable.ts | 42 ++++ 6 files changed, 236 insertions(+), 86 deletions(-) create mode 100644 packages/kernel/src/store/parse-ref.ts create mode 100644 packages/kernel/src/store/reachable.ts diff --git a/packages/kernel/src/Kernel.ts b/packages/kernel/src/Kernel.ts index 570483142..a6fcaf159 100644 --- a/packages/kernel/src/Kernel.ts +++ b/packages/kernel/src/Kernel.ts @@ -22,12 +22,9 @@ import type { VatCommand, VatCommandReturnType, } from './messages/index.ts'; -import { - parseRef, - isPromiseRef, - makeKernelStore, -} from './store/kernel-store.ts'; +import { isPromiseRef, makeKernelStore } from './store/kernel-store.ts'; import type { KernelStore } from './store/kernel-store.ts'; +import { parseRef } from './store/parse-ref.ts'; import type { VatId, VRef, diff --git a/packages/kernel/src/VatHandle.ts b/packages/kernel/src/VatHandle.ts index 4cb6a8fb7..80769c8fc 100644 --- a/packages/kernel/src/VatHandle.ts +++ b/packages/kernel/src/VatHandle.ts @@ -18,8 +18,8 @@ import type { VatCommand, VatCommandReturnType, } from './messages/index.ts'; -import { parseRef } from './store/kernel-store.ts'; import type { KernelStore } from './store/kernel-store.ts'; +import { parseRef } from './store/parse-ref.ts'; import type { PromiseCallbacks, VatId, @@ -327,11 +327,7 @@ export class VatHandle { `vat ${this.vatId} issued invalid syscall dropImports for ${kref}`, ); } - // Decrement the refCount and delete the object if it's now zero - const newCount = this.#storage.decRefCount(kref); - if (newCount === 0) { - this.#storage.deleteKernelObject(kref); - } + this.#storage.clearReachableFlag(this.vatId, kref); } } @@ -349,14 +345,10 @@ export class VatHandle { `vat ${this.vatId} issued invalid syscall retireImports for ${kref}`, ); } - // Check that the refCount is 0 before retiring - const refCount = this.#storage.getRefCount(kref); - if (refCount > 0) { - throw Error( - `syscall.retireImports but ${kref} still has references (count: ${refCount})`, - ); + if (this.#storage.getReachableFlag(this.vatId, kref)) { + throw Error(`syscall.retireImports but ${kref} is still reachable`); } - // Delete the object from storage + // TODO: instead of deleting the object from storage, we should deleteCListEntry this.#storage.deleteKernelObject(kref); } } @@ -365,10 +357,10 @@ export class VatHandle { * Handle retiring or abandoning exports syscall from the vat. * * @param krefs - The KRefs of the exports to be retired/abandoned. - * @param checkRefCount - If true, verify refCount is 0 (retire). If false, ignore refCount (abandon). + * @param checkReachable - If true, verify the object is not reachable (retire). If false, ignore reachability (abandon). */ - #handleSyscallExportCleanup(krefs: KRef[], checkRefCount: boolean): void { - const action = checkRefCount ? 'retire' : 'abandon'; + #handleSyscallExportCleanup(krefs: KRef[], checkReachable: boolean): void { + const action = checkReachable ? 'retire' : 'abandon'; for (const kref of krefs) { const { direction, isPromise } = parseRef(kref); @@ -378,16 +370,15 @@ export class VatHandle { `vat ${this.vatId} issued invalid syscall ${action}Exports for ${kref}`, ); } - if (checkRefCount) { + if (checkReachable) { // Check that the refCount is 0 before retiring - const refCount = this.#storage.getRefCount(kref); - if (refCount > 0) { + if (this.#storage.getReachableFlag(this.vatId, kref)) { throw Error( - `syscall.${action}Exports but ${kref} still has references (count: ${refCount})`, + `syscall.${action}Exports but ${kref} is still reachable`, ); } } - // Delete the object from storage + // TODO: instead of deleting the object from storage, we should deleteCListEntry this.#storage.deleteKernelObject(kref); this.#logger.debug(`${action}Exports: deleted object ${kref}`); } diff --git a/packages/kernel/src/store/kernel-store.test.ts b/packages/kernel/src/store/kernel-store.test.ts index 4b407faf7..d01cd900a 100644 --- a/packages/kernel/src/store/kernel-store.test.ts +++ b/packages/kernel/src/store/kernel-store.test.ts @@ -72,7 +72,9 @@ describe('kernel store', () => { 'getNextVatId', 'getOwner', 'getRefCount', + 'getObjectRefCount', 'incRefCount', + 'setObjectRefCount', 'initEndpoint', 'initKernelObject', 'initKernelPromise', diff --git a/packages/kernel/src/store/kernel-store.ts b/packages/kernel/src/store/kernel-store.ts index 106ecf7e6..48ef39fb3 100644 --- a/packages/kernel/src/store/kernel-store.ts +++ b/packages/kernel/src/store/kernel-store.ts @@ -58,6 +58,11 @@ import { Fail } from '@endo/errors'; import type { CapData } from '@endo/marshal'; import type { KVStore, VatStore, KernelDatabase } from '@ocap/store'; +import { parseRef } from './parse-ref.ts'; +import { + buildReachableAndVatSlot, + parseReachableAndVatSlot, +} from './reachable.ts'; import type { VatId, RemoteId, @@ -82,13 +87,6 @@ type StoredQueue = { delete(): void; }; -type RefParts = { - context: 'kernel' | 'vat' | 'remote'; - direction?: 'export' | 'import'; - isPromise: boolean; - index: string; -}; - /** * Test if a KRef designates a promise. * @@ -100,57 +98,6 @@ export function isPromiseRef(kref: KRef): boolean { return kref[1] === 'p'; } -/** - * Parse an alleged ref string into its components. - * - * @param ref - The string to be parsed. - * - * @returns an object with all of the ref string components as individual properties. - */ -export function parseRef(ref: string): RefParts { - let context; - let typeIdx = 1; - - switch (ref[0]) { - case 'k': - context = 'kernel'; - break; - case 'o': - case 'p': - typeIdx = 0; - context = 'vat'; - break; - case 'r': - context = 'remote'; - break; - case undefined: - default: - Fail`invalid reference context ${ref[0]}`; - } - if (ref[typeIdx] !== 'p' && ref[typeIdx] !== 'o') { - Fail`invalid reference type ${ref[typeIdx]}`; - } - const isPromise = ref[typeIdx] === 'p'; - let direction; - let index; - if (context === 'kernel') { - index = ref.slice(2); - } else { - const dirIdx = typeIdx + 1; - if (ref[dirIdx] !== '+' && ref[dirIdx] !== '-') { - Fail`invalid reference direction ${ref[dirIdx]}`; - } - direction = ref[dirIdx] === '+' ? 'export' : 'import'; - index = ref.slice(dirIdx + 1); - } - return { - context, - direction, - isPromise, - index, - } as RefParts; -} - /** * Create a new KernelStore object wrapped around a raw kernel database. The * resulting object provides a variety of operations for accessing various @@ -486,6 +433,48 @@ export function makeKernelStore(kdb: KernelDatabase) { return newCount; } + /** + * Get the reference counts for a kernel object + * + * @param kref - The KRef of the object of interest. + * @returns The reference counts for the object. + */ + function getObjectRefCount(kref: KRef): { + reachable: number; + recognizable: number; + } { + const data = kv.get(`${kref}.refCount`); + if (!data) { + return { reachable: 0, recognizable: 0 }; + } + const [reachable = 0, recognizable = 0] = data.split(',').map(Number); + reachable <= recognizable || + Fail`refMismatch(get) ${kref} ${reachable},${recognizable}`; + return { reachable, recognizable }; + } + + /** + * Set the reference counts for a kernel object + * + * @param kref - The KRef of the object of interest. + * @param counts - The reference counts to set. + * @param counts.reachable - The reachable reference count. + * @param counts.recognizable - The recognizable reference count. + */ + function setObjectRefCount( + kref: KRef, + counts: { reachable: number; recognizable: number }, + ): void { + const { reachable, recognizable } = counts; + assert.typeof(reachable, 'number'); + assert.typeof(recognizable, 'number'); + (reachable >= 0 && recognizable >= 0) || + Fail`${kref} underflow ${reachable},${recognizable}`; + reachable <= recognizable || + Fail`refMismatch(set) ${kref} ${reachable},${recognizable}`; + kv.set(`${kref}.refCount`, `${reachable},${recognizable}`); + } + /** * Create a new kernel object. The new object will be born with reference and * recognizability counts of 1, on the assumption that the new object @@ -497,7 +486,7 @@ export function makeKernelStore(kdb: KernelDatabase) { function initKernelObject(owner: EndpointId): KRef { const koId = getNextObjectId(); kv.set(`${koId}.owner`, owner); - kv.set(refCountKey(koId), '1'); + setObjectRefCount(koId, { reachable: 1, recognizable: 1 }); return koId; } @@ -796,6 +785,71 @@ export function makeKernelStore(kdb: KernelDatabase) { nextPromiseId = provideCachedStoredValue('nextPromiseId', '1'); } + /** + * Check if a kernel object exists in the kernel's persistent state. + * + * @param kref - The KRef of the kernel object in question. + * @returns True if the kernel object exists, false otherwise. + */ + function kernelObjectExists(kref: KRef): boolean { + return Boolean(kv.get(`${kref}.refCount`)); + } + + /** + * Get the key for the reachable flag and vatSlot for a given endpoint and kref. + * + * @param endpointId - The endpoint for which the reachable flag is being set. + * @param kref - The kref. + * @returns The key for the reachable flag and vatSlot. + */ + function getReachableVatSlotKey(endpointId: EndpointId, kref: KRef): string { + return `${endpointId}.c.${kref}`; + } + + /** + * Check if a kernel object is reachable. + * + * @param endpointId - The endpoint for which the reachable flag is being checked. + * @param kref - The kref. + * @returns True if the kernel object is reachable, false otherwise. + */ + function getReachableFlag(endpointId: EndpointId, kref: KRef): boolean { + const key = getReachableVatSlotKey(endpointId, kref); + const data = kv.getRequired(key); + const { isReachable } = parseReachableAndVatSlot(data); + return isReachable; + } + + /** + * Clear the reachable flag for a given endpoint and kref. + * + * @param endpointId - The endpoint for which the reachable flag is being cleared. + * @param kref - The kref. + */ + function clearReachableFlag(endpointId: EndpointId, kref: KRef): void { + const key = getReachableVatSlotKey(endpointId, kref); + const { isReachable, vatSlot } = parseReachableAndVatSlot( + kv.getRequired(key), + ); + kv.set(key, buildReachableAndVatSlot(false, vatSlot)); + const { direction, isPromise } = parseRef(vatSlot); + // decrement 'reachable' part of refcount, but only for object imports + if ( + isReachable && + !isPromise && + direction === 'export' && + kernelObjectExists(kref) + ) { + const counts = getObjectRefCount(kref); + counts.reachable -= 1; + setObjectRefCount(kref, counts); + if (counts.reachable === 0) { + deleteKernelObject(kref); + // TODO: implement addMaybeFreeKref(kref); + } + } + } + return harden({ enqueueRun, dequeueRun, @@ -803,10 +857,15 @@ export function makeKernelStore(kdb: KernelDatabase) { getNextVatId, getNextRemoteId, initEndpoint, - getRefCount, + getRefCount, // For promises incRefCount, decRefCount, + getObjectRefCount, // For objects + setObjectRefCount, // For objects initKernelObject, + kernelObjectExists, + getReachableFlag, + clearReachableFlag, getOwner, deleteKernelObject, initKernelPromise, diff --git a/packages/kernel/src/store/parse-ref.ts b/packages/kernel/src/store/parse-ref.ts new file mode 100644 index 000000000..691fb96fc --- /dev/null +++ b/packages/kernel/src/store/parse-ref.ts @@ -0,0 +1,59 @@ +import { Fail } from '@endo/errors'; + +type RefParts = { + context: 'kernel' | 'vat' | 'remote'; + direction?: 'export' | 'import'; + isPromise: boolean; + index: string; +}; + +/** + * Parse an alleged ref string into its components. + * + * @param ref - The string to be parsed. + * + * @returns an object with all of the ref string components as individual properties. + */ +export function parseRef(ref: string): RefParts { + let context; + let typeIdx = 1; + + switch (ref[0]) { + case 'k': + context = 'kernel'; + break; + case 'o': + case 'p': + typeIdx = 0; + context = 'vat'; + break; + case 'r': + context = 'remote'; + break; + case undefined: + default: + Fail`invalid reference context ${ref[0]}`; + } + if (ref[typeIdx] !== 'p' && ref[typeIdx] !== 'o') { + Fail`invalid reference type ${ref[typeIdx]}`; + } + const isPromise = ref[typeIdx] === 'p'; + let direction; + let index; + if (context === 'kernel') { + index = ref.slice(2); + } else { + const dirIdx = typeIdx + 1; + if (ref[dirIdx] !== '+' && ref[dirIdx] !== '-') { + Fail`invalid reference direction ${ref[dirIdx]}`; + } + direction = ref[dirIdx] === '+' ? 'export' : 'import'; + index = ref.slice(dirIdx + 1); + } + return { + context, + direction, + isPromise, + index, + } as RefParts; +} diff --git a/packages/kernel/src/store/reachable.ts b/packages/kernel/src/store/reachable.ts new file mode 100644 index 000000000..64b0c7907 --- /dev/null +++ b/packages/kernel/src/store/reachable.ts @@ -0,0 +1,42 @@ +import { assert, Fail } from '../assert.ts'; + +/** + * Parse a string into an object with `isReachable` and `vatSlot` properties. + * + * @param value - The string to parse. + * @returns An object with `isReachable` and `vatSlot` properties. + */ +export function parseReachableAndVatSlot(value: string): { + isReachable: boolean; + vatSlot: string; +} { + typeof value === 'string' || Fail`non-string value: ${value}`; + const flag = value.slice(0, 1); + assert.equal(value.slice(1, 2), ' '); + const vatSlot = value.slice(2); + let isReachable; + if (flag === 'R') { + isReachable = true; + } else if (flag === '_') { + isReachable = false; + } else { + throw Fail`flag (${flag}) must be 'R' or '_'`; + } + return { isReachable, vatSlot }; +} +harden(parseReachableAndVatSlot); + +/** + * Build a string from an object with `isReachable` and `vatSlot` properties. + * + * @param isReachable - The `isReachable` property of the object. + * @param vatSlot - The `vatSlot` property of the object. + * @returns A string with the `isReachable` and `vatSlot` properties. + */ +export function buildReachableAndVatSlot( + isReachable: boolean, + vatSlot: string, +): string { + return `${isReachable ? 'R' : '_'} ${vatSlot}`; +} +harden(buildReachableAndVatSlot); From f6169179603e77bc182a032c3b72265135b0711a Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Fri, 14 Feb 2025 20:42:06 +0100 Subject: [PATCH 03/21] feat(kernel): implement reference counting and enhance c-list cleanup --- packages/kernel/src/VatHandle.ts | 9 +- packages/kernel/src/store/kernel-store.ts | 227 +++++++++++++++++----- 2 files changed, 177 insertions(+), 59 deletions(-) diff --git a/packages/kernel/src/VatHandle.ts b/packages/kernel/src/VatHandle.ts index 80769c8fc..3bf5e3590 100644 --- a/packages/kernel/src/VatHandle.ts +++ b/packages/kernel/src/VatHandle.ts @@ -348,8 +348,9 @@ export class VatHandle { if (this.#storage.getReachableFlag(this.vatId, kref)) { throw Error(`syscall.retireImports but ${kref} is still reachable`); } - // TODO: instead of deleting the object from storage, we should deleteCListEntry - this.#storage.deleteKernelObject(kref); + // deleting the clist entry will decrement the recognizable count, but + // not the reachable count (because it was unreachable, as we asserted) + this.#storage.forgetKref(this.vatId, kref); } } @@ -371,15 +372,13 @@ export class VatHandle { ); } if (checkReachable) { - // Check that the refCount is 0 before retiring if (this.#storage.getReachableFlag(this.vatId, kref)) { throw Error( `syscall.${action}Exports but ${kref} is still reachable`, ); } } - // TODO: instead of deleting the object from storage, we should deleteCListEntry - this.#storage.deleteKernelObject(kref); + this.#storage.forgetKref(this.vatId, kref); this.#logger.debug(`${action}Exports: deleted object ${kref}`); } } diff --git a/packages/kernel/src/store/kernel-store.ts b/packages/kernel/src/store/kernel-store.ts index 48ef39fb3..a1d88ece1 100644 --- a/packages/kernel/src/store/kernel-store.ts +++ b/packages/kernel/src/store/kernel-store.ts @@ -130,6 +130,18 @@ export function makeKernelStore(kdb: KernelDatabase) { /** Counter for allocating kernel promise IDs */ let nextPromiseId = provideCachedStoredValue('nextPromiseId', '1'); + // As refcounts are decremented, we accumulate a set of krefs for which + // action might need to be taken: + // * promises which are now resolved and unreferenced can be deleted + // * objects which are no longer reachable: export can be dropped + // * objects which are no longer recognizable: export can be retired + // This set is ephemeral: it lives in RAM, grows as deliveries and syscalls + // cause decrefs, and will be harvested by processRefcounts(). This needs to be + // called in the same transaction window as the syscalls/etc which prompted + // the change, else removals might be lost (not performed during the next + // replay). + const maybeFreeKrefs = new Set(); + /** * Provide a stored value object for which we keep an in-memory cache. We only * touch persistent storage if the value hasn't ever been read of if it is @@ -443,7 +455,7 @@ export function makeKernelStore(kdb: KernelDatabase) { reachable: number; recognizable: number; } { - const data = kv.get(`${kref}.refCount`); + const data = kv.get(refCountKey(kref)); if (!data) { return { reachable: 0, recognizable: 0 }; } @@ -472,7 +484,7 @@ export function makeKernelStore(kdb: KernelDatabase) { Fail`${kref} underflow ${reachable},${recognizable}`; reachable <= recognizable || Fail`refMismatch(set) ${kref} ${reachable},${recognizable}`; - kv.set(`${kref}.refCount`, `${reachable},${recognizable}`); + kv.set(refCountKey(kref), `${reachable},${recognizable}`); } /** @@ -675,6 +687,17 @@ export function makeKernelStore(kdb: KernelDatabase) { provideStoredQueue(kpid).delete(); } + /** + * Get the key for the reachable flag and vatSlot for a given endpoint and kref. + * + * @param endpointId - The endpoint for which the reachable flag is being set. + * @param kref - The kref. + * @returns The key for the reachable flag and vatSlot. + */ + function getSlotKey(endpointId: EndpointId, kref: KRef): string { + return `${endpointId}.c.${kref}`; + } + /** * Look up the ERef that and endpoint's c-list maps a KRef to. * @@ -684,7 +707,7 @@ export function makeKernelStore(kdb: KernelDatabase) { * if there is no such mapping. */ function erefToKref(endpointId: EndpointId, eref: ERef): KRef | undefined { - return kv.get(`cle.${endpointId}.${eref}`); + return kv.get(getSlotKey(endpointId, eref)); } /** @@ -696,37 +719,13 @@ export function makeKernelStore(kdb: KernelDatabase) { * there is no such mapping. */ function krefToEref(endpointId: EndpointId, kref: KRef): ERef | undefined { - return kv.get(`clk.${endpointId}.${kref}`); - } - - /** - * Add an entry to an endpoint's c-list, creating a new bidirectional mapping - * between an ERef belonging to the endpoint and a KRef belonging to the - * kernel. - * - * @param endpointId - The endpoint whose c-list is to be added to. - * @param kref - The KRef. - * @param eref - The ERef. - */ - function addClistEntry(endpointId: EndpointId, kref: KRef, eref: ERef): void { - kv.set(`clk.${endpointId}.${kref}`, eref); - kv.set(`cle.${endpointId}.${eref}`, kref); - } - - /** - * Remove an entry from an endpoint's c-list. - * - * @param endpointId - The endpoint whose c-list entry is to be removed. - * @param kref - The KRef. - * @param eref - The ERef. - */ - function deleteClistEntry( - endpointId: EndpointId, - kref: KRef, - eref: ERef, - ): void { - kv.delete(`clk.${endpointId}.${kref}`); - kv.delete(`cle.${endpointId}.${eref}`); + const key = getSlotKey(endpointId, kref); + const data = kv.get(key); + if (!data) { + return undefined; + } + const { vatSlot } = parseReachableAndVatSlot(data); + return vatSlot; } /** @@ -792,18 +791,7 @@ export function makeKernelStore(kdb: KernelDatabase) { * @returns True if the kernel object exists, false otherwise. */ function kernelObjectExists(kref: KRef): boolean { - return Boolean(kv.get(`${kref}.refCount`)); - } - - /** - * Get the key for the reachable flag and vatSlot for a given endpoint and kref. - * - * @param endpointId - The endpoint for which the reachable flag is being set. - * @param kref - The kref. - * @returns The key for the reachable flag and vatSlot. - */ - function getReachableVatSlotKey(endpointId: EndpointId, kref: KRef): string { - return `${endpointId}.c.${kref}`; + return Boolean(kv.get(refCountKey(kref))); } /** @@ -814,7 +802,7 @@ export function makeKernelStore(kdb: KernelDatabase) { * @returns True if the kernel object is reachable, false otherwise. */ function getReachableFlag(endpointId: EndpointId, kref: KRef): boolean { - const key = getReachableVatSlotKey(endpointId, kref); + const key = getSlotKey(endpointId, kref); const data = kv.getRequired(key); const { isReachable } = parseReachableAndVatSlot(data); return isReachable; @@ -827,7 +815,7 @@ export function makeKernelStore(kdb: KernelDatabase) { * @param kref - The kref. */ function clearReachableFlag(endpointId: EndpointId, kref: KRef): void { - const key = getReachableVatSlotKey(endpointId, kref); + const key = getSlotKey(endpointId, kref); const { isReachable, vatSlot } = parseReachableAndVatSlot( kv.getRequired(key), ); @@ -837,19 +825,147 @@ export function makeKernelStore(kdb: KernelDatabase) { if ( isReachable && !isPromise && - direction === 'export' && + direction === 'import' && kernelObjectExists(kref) ) { const counts = getObjectRefCount(kref); counts.reachable -= 1; setObjectRefCount(kref, counts); if (counts.reachable === 0) { - deleteKernelObject(kref); - // TODO: implement addMaybeFreeKref(kref); + maybeFreeKrefs.add(kref); } } } + /** + * Increment the reference count associated with some kernel object. + * + * We track references to promises and objects, but not devices. Promises + * have only a "reachable" count, whereas objects track both "reachable" + * and "recognizable" counts. + * + * @param kref - The kernel slot whose refcount is to be incremented. + * @param options - Options for the increment. + * @param options.isExport - True if the reference comes from a clist export, which counts for promises but not objects. + * @param options.onlyRecognizable - True if the reference provides only recognition, not reachability. + */ + function incrementRefCount( + kref: KRef, + { + isExport = false, + onlyRecognizable = false, + }: { isExport?: boolean; onlyRecognizable?: boolean }, + ): void { + kref || Fail`incrementRefCount called with empty kref`; + + const { isPromise } = parseRef(kref); + if (isPromise) { + const refCount = Number(kv.get(refCountKey(kref))) + 1; + kv.set(refCountKey(kref), `${refCount}`); + return; + } + + // If `isExport` the reference comes from a clist export, which counts for promises but not objects + if (isExport) { + return; + } + + const counts = getObjectRefCount(kref); + if (!onlyRecognizable) { + counts.reachable += 1; + } + counts.recognizable += 1; + setObjectRefCount(kref, counts); + } + + /** + * Decrement the reference count associated with some kernel object. + * + * @param kref - The kernel slot whose refcount is to be decremented. + * @param options - Options for the decrement. + * @param options.isExport - True if the reference comes from a clist export, which counts for promises but not objects. + * @param options.onlyRecognizable - True if the reference provides only recognition, not reachability. + * @returns True if the reference count has been decremented to zero, false if it is still non-zero. + * @throws if this tries to decrement the reference count below zero. + */ + function decrementRefCount( + kref: KRef, + { + isExport = false, + onlyRecognizable = false, + }: { isExport?: boolean; onlyRecognizable?: boolean }, + ): boolean { + kref || Fail`decrementRefCount called with empty kref`; + + const { isPromise } = parseRef(kref); + if (isPromise) { + let refCount = Number(kv.get(refCountKey(kref))); + refCount > 0 || Fail`refCount underflow ${kref}`; + refCount -= 1; + kv.set(refCountKey(kref), `${refCount}`); + if (refCount === 0) { + maybeFreeKrefs.add(kref); + return true; + } + return false; + } + + if (isExport || !kernelObjectExists(kref)) { + return false; + } + + const counts = getObjectRefCount(kref); + if (!onlyRecognizable) { + counts.reachable -= 1; + } + counts.recognizable -= 1; + if (!counts.reachable || !counts.recognizable) { + maybeFreeKrefs.add(kref); + } + setObjectRefCount(kref, counts); + + return false; + } + + /** + * Add an entry to an endpoint's c-list, creating a new bidirectional mapping + * between an ERef belonging to the endpoint and a KRef belonging to the + * kernel. + * + * @param endpointId - The endpoint whose c-list is to be added to. + * @param kref - The KRef. + * @param eref - The ERef. + */ + function addClistEntry(endpointId: EndpointId, kref: KRef, eref: ERef): void { + kv.set(getSlotKey(endpointId, kref), buildReachableAndVatSlot(true, eref)); + kv.set(getSlotKey(endpointId, eref), kref); + } + + /** + * Remove an entry from an endpoint's c-list. + * + * @param endpointId - The endpoint whose c-list entry is to be removed. + * @param kref - The KRef. + * @param eref - The ERef. + */ + function deleteClistEntry( + endpointId: EndpointId, + kref: KRef, + eref: ERef, + ): void { + const kernelKey = getSlotKey(endpointId, kref); + const vatKey = getSlotKey(endpointId, eref); + assert(kv.get(kernelKey)); + clearReachableFlag(endpointId, kref); + const { direction } = parseRef(eref); + decrementRefCount(kref, { + isExport: direction === 'export', + onlyRecognizable: true, + }); + kv.delete(kernelKey); + kv.delete(vatKey); + } + return harden({ enqueueRun, dequeueRun, @@ -857,17 +973,20 @@ export function makeKernelStore(kdb: KernelDatabase) { getNextVatId, getNextRemoteId, initEndpoint, - getRefCount, // For promises + getRefCount, incRefCount, decRefCount, - getObjectRefCount, // For objects - setObjectRefCount, // For objects + getObjectRefCount, + setObjectRefCount, + incrementRefCount, + decrementRefCount, initKernelObject, kernelObjectExists, getReachableFlag, clearReachableFlag, getOwner, deleteKernelObject, + deleteClistEntry, initKernelPromise, getKernelPromise, enqueuePromiseMessage, From 1fe69f46d18f29556b4c062237c78de80849ed33 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Mon, 24 Feb 2025 14:03:36 -0300 Subject: [PATCH 04/21] Implement GC delivery --- packages/kernel/src/Kernel.ts | 41 +++++ packages/kernel/src/VatHandle.ts | 98 +++++++++++ packages/kernel/src/processGCActionSet.ts | 203 ++++++++++++++++++++++ packages/kernel/src/store/kernel-slots.ts | 76 ++++++++ packages/kernel/src/store/kernel-store.ts | 102 ++++++++++- packages/kernel/src/types.ts | 94 +++++++++- 6 files changed, 604 insertions(+), 10 deletions(-) create mode 100644 packages/kernel/src/processGCActionSet.ts create mode 100644 packages/kernel/src/store/kernel-slots.ts diff --git a/packages/kernel/src/Kernel.ts b/packages/kernel/src/Kernel.ts index a6fcaf159..d5f5e70f1 100644 --- a/packages/kernel/src/Kernel.ts +++ b/packages/kernel/src/Kernel.ts @@ -22,6 +22,7 @@ import type { VatCommand, VatCommandReturnType, } from './messages/index.ts'; +import { processGCActionSet } from './processGCActionSet.ts'; import { isPromiseRef, makeKernelStore } from './store/kernel-store.ts'; import type { KernelStore } from './store/kernel-store.ts'; import { parseRef } from './store/parse-ref.ts'; @@ -179,6 +180,14 @@ export class Kernel { */ async #run(): Promise { for await (const item of this.#runQueueItems()) { + // Process GC actions + const message = + processGCActionSet(this.#storage) ?? this.#storage.nextReapAction(); + if (message) { + await this.#deliver(message); + } + + // Deliver message await this.#deliver(item); } } @@ -579,6 +588,38 @@ export class Kernel { log(`@@@@ done ${vatId} notify ${kpid}`); break; } + case 'dropExports': { + const { vatId, krefs } = item; + log(`@@@@ deliver ${vatId} dropExports`, krefs); + const vat = this.#getVat(vatId); + await vat.deliverDropExports(krefs); + log(`@@@@ done ${vatId} dropExports`, krefs); + break; + } + case 'retireExports': { + const { vatId, krefs } = item; + log(`@@@@ deliver ${vatId} retireExports`, krefs); + const vat = this.#getVat(vatId); + await vat.deliverRetireExports(krefs); + log(`@@@@ done ${vatId} retireExports`, krefs); + break; + } + case 'retireImports': { + const { vatId, krefs } = item; + log(`@@@@ deliver ${vatId} retireImports`, krefs); + const vat = this.#getVat(vatId); + await vat.deliverRetireImports(krefs); + log(`@@@@ done ${vatId} retireImports`, krefs); + break; + } + case 'bringOutYourDead': { + const { vatId } = item; + log(`@@@@ deliver ${vatId} bringOutYourDead`); + const vat = this.#getVat(vatId); + await vat.deliverBringOutYourDead(); + log(`@@@@ done ${vatId} bringOutYourDead`); + break; + } default: // @ts-expect-error Runtime does not respect "never". Fail`unsupported or unknown run queue item type ${item.type}`; diff --git a/packages/kernel/src/VatHandle.ts b/packages/kernel/src/VatHandle.ts index 3bf5e3590..707116ced 100644 --- a/packages/kernel/src/VatHandle.ts +++ b/packages/kernel/src/VatHandle.ts @@ -501,6 +501,7 @@ export class VatHandle { } /** +<<<<<<< HEAD * Make a 'message' delivery to the vat. * * @param target - The VRef of the object to which the message is addressed. @@ -523,6 +524,32 @@ export class VatHandle { method: VatCommandMethod.deliver, params: ['notify', resolutions], }); +======= + * Initializes the vat. + * + * @returns A promise that resolves when the vat is initialized. + */ + async init(): Promise { + Promise.all([this.#vatStream.drain(this.handleMessage.bind(this))]).catch( + async (error) => { + this.#logger.error(`Unexpected read error`, error); + await this.terminate(new StreamReadError({ vatId: this.vatId }, error)); + }, + ); + + // XXX This initial `ping` was originally put here as a sanity check to make + // sure that the vat was actually running and able to exchange message + // traffic with the kernel, but the addition of the `initVat` message to the + // startup flow has, I'm fairly sure, obviated the need for that as it + // effectively performs the same function. Probably this ping should be + // removed. + await this.sendVatCommand({ method: VatCommandMethod.ping, params: null }); + await this.sendVatCommand({ + method: VatCommandMethod.initVat, + params: this.config, + }); + this.#logger.debug('Created'); +>>>>>>> 59065de (Implement GC delivery) } /** @@ -564,4 +591,75 @@ export class VatHandle { readonly #nextMessageId = (): VatCommand['id'] => { return `${this.vatId}:${this.#messageCounter()}`; }; + + /** + * Make a 'message' delivery to the vat. + * + * @param target - The VRef of the object to which the message is addressed. + * @param message - The message to deliver. + */ + async deliverMessage(target: VRef, message: Message): Promise { + await this.sendVatCommand({ + method: VatCommandMethod.deliver, + params: ['message', target, message], + }); + } + + /** + * Make a 'notify' delivery to the vat. + * + * @param resolutions - One or more promise resolutions to deliver. + */ + async deliverNotify(resolutions: VatOneResolution[]): Promise { + await this.sendVatCommand({ + method: VatCommandMethod.deliver, + params: ['notify', resolutions], + }); + } + + /** + * Make a 'dropExports' delivery to the vat. + * + * @param krefs - The KRefs of the exports to be dropped. + */ + async deliverDropExports(krefs: KRef[]): Promise { + await this.sendVatCommand({ + method: VatCommandMethod.deliver, + params: ['dropExports', krefs], + }); + } + + /** + * Make a 'retireExports' delivery to the vat. + * + * @param krefs - The KRefs of the exports to be retired. + */ + async deliverRetireExports(krefs: KRef[]): Promise { + await this.sendVatCommand({ + method: VatCommandMethod.deliver, + params: ['retireExports', krefs], + }); + } + + /** + * Make a 'retireImports' delivery to the vat. + * + * @param krefs - The KRefs of the imports to be retired. + */ + async deliverRetireImports(krefs: KRef[]): Promise { + await this.sendVatCommand({ + method: VatCommandMethod.deliver, + params: ['retireImports', krefs], + }); + } + + /** + * Make a 'bringOutYourDead' delivery to the vat. + */ + async deliverBringOutYourDead(): Promise { + await this.sendVatCommand({ + method: VatCommandMethod.deliver, + params: ['bringOutYourDead'], + }); + } } diff --git a/packages/kernel/src/processGCActionSet.ts b/packages/kernel/src/processGCActionSet.ts new file mode 100644 index 000000000..efbcf0e1c --- /dev/null +++ b/packages/kernel/src/processGCActionSet.ts @@ -0,0 +1,203 @@ +import { insistKernelType } from './store/kernel-slots.ts'; +import type { KernelStore } from './store/kernel-store.ts'; +import type { + GCAction, + GCActionType, + KRef, + RunQueueItem, + VatId, +} from './types.ts'; +import { + actionTypePriorities, + insistGCActionType, + insistVatId, + queueTypeFromActionType, +} from './types.ts'; + +/** + * Parse a GC action string into a vat id, type, and kref. + * + * @param action - The GC action string to parse. + * @returns The vat id, type, and kref. + */ +function parseAction(action: GCAction): { + vatId: VatId; + type: GCActionType; + kref: KRef; +} { + const [vatId, type, kref] = action.split(' '); + insistVatId(vatId); + insistGCActionType(type); + insistKernelType('object', kref); + return { vatId, type, kref }; +} + +/** + * Process the set of GC actions. + * + * @param storage - The kernel storage. + * @returns The next action to process, or undefined if there are no actions to process. + */ +export function processGCActionSet( + storage: KernelStore, +): RunQueueItem | undefined { + const allActionsSet = storage.getGCActions(); + let actionSetUpdated = false; + + // GC actions are each one of 'dropExport', 'retireExport', or + // 'retireImport', aimed at a specific vat and affecting a specific kref. + // They are added to the durable "GC Actions" set (stored in kernelDB) when + // `processRefcounts` notices a refcount sitting at zero, which means some + // vat needs to be told that an object can be freed. Before each crank, the + // kernel calls processGCActionSet to see if there are any GC actions that + // should be taken. All such GC actions are executed before any regular vat + // delivery gets to run. + + // However, things might have changed between the time the action was + // pushed into the durable set and the time the kernel is ready to execute + // it. For example, the kref might have been re-exported: we were all set + // to tell the exporting vat that their object isn't recognizable any more + // (with a `dispatch.retireExport`), but then they sent a brand new copy to + // some importer. We must negate the `retireExport` action, because it's no + // longer the right thing to do. Alternatively, the exporting vat might + // have deleted the object itself (`syscall.retireExport`) before the + // kernel got a chance to deliver the `dispatch.retireExport`, which means + // we must bypass the action as redundant (since it's an error to delete + // the same c-list entry twice). + + /** + * Inspect a queued GC action and decide whether the current state of c-lists + * and reference counts warrants processing it, or if it should instead be + * negated/bypassed. + * + * @param vatId - The vat id of the vat that owns the kref. + * @param type - The type of GC action. + * @param kref - The kref of the object in question. + * @returns True if the action should be processed, false otherwise. + */ + function shouldProcessAction( + vatId: VatId, + type: GCActionType, + kref: KRef, + ): boolean { + const hasCList = storage.hasCListEntry(vatId, kref); + const isReachable = hasCList + ? storage.getReachableFlag(vatId, kref) + : undefined; + const exists = storage.kernelObjectExists(kref); + const { reachable, recognizable } = exists + ? storage.getObjectRefCount(kref) + : { reachable: 0, recognizable: 0 }; + + if (type === 'dropExport') { + if (!exists) { + return false; + } // already, shouldn't happen + if (reachable) { + return false; + } // negated + if (!hasCList) { + return false; + } // already, shouldn't happen + if (!isReachable) { + return false; + } // already, shouldn't happen + } + if (type === 'retireExport') { + if (!exists) { + return false; + } // already + if (reachable || recognizable) { + return false; + } // negated + if (!hasCList) { + return false; + } // already + } + if (type === 'retireImport') { + if (!hasCList) { + return false; + } // already + } + return true; + } + + // We process actions in groups (sorted first by vat, then by type), to + // make it deterministic, and to ensure that `dropExport` happens before + // `retireExport`. This examines one group at a time, filtering everything + // in that group, and returning the survivors of the first group that + // wasn't filtered out entirely. Our available dispatch functions take + // multiple krefs (`dispatch.dropExports`, rather than + // `dispatch.dropExport`), so the set of surviving krefs can all be + // delivered to a vat in a single crank. + + // Some day we may consolidate the three GC delivery methods into a single + // one, in which case we'll batch together an entire vat's worth of + // actions, instead of the narrower (vat+type) group. The filtering rules + // may need to change to support that, to ensure that `dropExport` and + // `retireExport` can both be delivered. + + /** + * Process the set of GC actions for a given vat. + * + * @param vatId - The vat id of the vat that owns the krefs. + * @param groupedActions - The set of GC actions to process. + * @returns The krefs to process. + */ + function krefsToProcess(vatId: VatId, groupedActions: Set): KRef[] { + const krefs: KRef[] = []; + for (const action of groupedActions) { + const { type, kref } = parseAction(action); + if (shouldProcessAction(vatId, type, kref)) { + krefs.push(kref); + } + allActionsSet.delete(action); + actionSetUpdated = true; + } + return krefs; + } + + const actionsByVat = new Map(); + for (const action of allActionsSet) { + const { vatId, type } = parseAction(action); + if (!actionsByVat.has(vatId)) { + actionsByVat.set(vatId, new Map()); + } + const actionsForVatByType = actionsByVat.get(vatId); + if (!actionsForVatByType.has(type)) { + actionsForVatByType.set(type, []); + } + actionsForVatByType.get(type).push(action); + } + + const vatIds = Array.from(actionsByVat.keys()); + vatIds.sort(); + for (const vatId of vatIds) { + const actionsForVatByType = actionsByVat.get(vatId); + // find the highest-priority type of work to do within this vat + for (const type of actionTypePriorities) { + if (actionsForVatByType.has(type)) { + const actions = actionsForVatByType.get(type); + const krefs = krefsToProcess(vatId, actions); + if (krefs.length) { + // at last, we act + krefs.sort(); + // remove the work we're about to do from the durable set + storage.setGCActions(allActionsSet); + const queueType = queueTypeFromActionType.get(type); + assert(queueType, `Unknown action type: ${type}`); + return harden({ type: queueType, vatId, krefs }); + } + } + } + } + + if (actionSetUpdated) { + // remove negated items from the durable set + storage.setGCActions(allActionsSet); + } + + // no GC work to do and no DB changes + return undefined; +} +harden(processGCActionSet); diff --git a/packages/kernel/src/store/kernel-slots.ts b/packages/kernel/src/store/kernel-slots.ts new file mode 100644 index 000000000..35072e8cd --- /dev/null +++ b/packages/kernel/src/store/kernel-slots.ts @@ -0,0 +1,76 @@ +import { assert, Fail } from '../assert.ts'; + +// Object/promise references (in the kernel) contain a two-tuple of (type, +// index). All object references point to entries in the kernel Object +// Table, which records the vat that owns the actual object. In that vat, +// the object reference will be expressed as 'o+NN', and the NN was +// allocated by that vat when they first exported the reference into the +// kernel. In all other vats, if/when they are given a reference to this +// object, they will receive 'o-NN', with the NN allocated by the kernel +// clist for the recipient vat. + +type KernelSlotType = 'object' | 'promise'; + +/** + * Parse a kernel slot reference string into a kernel slot object. + * + * @param slot The string to be parsed, as described above. + * @returns kernel slot object corresponding to the parameter. + * @throws if the given string is syntactically incorrect. + */ +export function parseKernelSlot(slot: string): { + type: KernelSlotType; + id: string; +} { + assert.typeof(slot, 'string'); + let type: KernelSlotType; + let idSuffix: string; + if (slot.startsWith('ko')) { + type = 'object'; + idSuffix = slot.slice(2); + } else if (slot.startsWith('kp')) { + type = 'promise'; + idSuffix = slot.slice(2); + } else { + throw Fail`invalid kernelSlot ${slot}`; + } + const id = idSuffix; + return { type, id }; +} + +/** + * Generate a kernel slot reference string given a type and id. + * + * @param type - The kernel slot type desired, a string. + * @param id - The id, a number. + * @returns the corresponding kernel slot reference string. + * @throws if type is not one of the above known types. + */ +export function makeKernelSlot(type: KernelSlotType, id: string): string { + if (type === 'object') { + return `ko${id}`; + } + if (type === 'promise') { + return `kp${id}`; + } + throw Fail`unknown type ${type}`; +} + +/** + * Assert function to ensure that a kernel slot reference string refers to a + * slot of a given type. + * + * @param type - The kernel slot type desired, a string. + * @param kernelSlot - The kernel slot reference string being tested + * @throws if kernelSlot is not of the given type or is malformed. + */ +export function insistKernelType( + type: KernelSlotType, + kernelSlot: string | undefined, +): asserts kernelSlot is KernelSlotType { + if (kernelSlot === undefined) { + throw Fail`kernelSlot is undefined`; + } + type === parseKernelSlot(kernelSlot).type || + Fail`kernelSlot ${kernelSlot} is not of type ${type}`; +} diff --git a/packages/kernel/src/store/kernel-store.ts b/packages/kernel/src/store/kernel-store.ts index a1d88ece1..3f218e692 100644 --- a/packages/kernel/src/store/kernel-store.ts +++ b/packages/kernel/src/store/kernel-store.ts @@ -58,6 +58,7 @@ import { Fail } from '@endo/errors'; import type { CapData } from '@endo/marshal'; import type { KVStore, VatStore, KernelDatabase } from '@ocap/store'; +import { insistKernelType, makeKernelSlot } from './kernel-slots.ts'; import { parseRef } from './parse-ref.ts'; import { buildReachableAndVatSlot, @@ -72,8 +73,10 @@ import type { RunQueueItem, PromiseState, KernelPromise, + GCAction, + RunQueueItemBringOutYourDead, } from '../types.ts'; -import { insistVatId } from '../types.ts'; +import { insistGCActionType, insistVatId, RunQueueItemType } from '../types.ts'; type StoredValue = { get(): string | undefined; @@ -129,6 +132,10 @@ export function makeKernelStore(kdb: KernelDatabase) { let nextObjectId = provideCachedStoredValue('nextObjectId', '1'); /** Counter for allocating kernel promise IDs */ let nextPromiseId = provideCachedStoredValue('nextPromiseId', '1'); + /** Actions to perform during garbage collection */ + let gcActions = provideCachedStoredValue('gcActions', '[]'); + /** Objects to reap */ + let reapQueue = provideCachedStoredValue('reapQueue', '[]'); // As refcounts are decremented, we accumulate a set of krefs for which // action might need to be taken: @@ -396,7 +403,7 @@ export function makeKernelStore(kdb: KernelDatabase) { * @returns The next koId use. */ function getNextObjectId(): KRef { - return `ko${incCounter(nextObjectId)}`; + return makeKernelSlot('object', incCounter(nextObjectId)); } /** @@ -532,7 +539,7 @@ export function makeKernelStore(kdb: KernelDatabase) { * @returns The next kpid use. */ function getNextPromiseId(): KRef { - return `kp${incCounter(nextPromiseId)}`; + return makeKernelSlot('promise', incCounter(nextPromiseId)); } /** @@ -782,6 +789,8 @@ export function makeKernelStore(kdb: KernelDatabase) { nextRemoteId = provideCachedStoredValue('nextRemoteId', '1'); nextObjectId = provideCachedStoredValue('nextObjectId', '1'); nextPromiseId = provideCachedStoredValue('nextPromiseId', '1'); + gcActions = provideCachedStoredValue('gcActions', '[]'); + reapQueue = provideCachedStoredValue('reapQueue', '[]'); } /** @@ -941,6 +950,17 @@ export function makeKernelStore(kdb: KernelDatabase) { kv.set(getSlotKey(endpointId, eref), kref); } + /** + * Test if there's a c-list entry for some slot. + * + * @param endpointId - The endpoint of interest + * @param slot - The slot of interest + * @returns true iff this vat has a c-list entry mapping for `slot`. + */ + function hasCListEntry(endpointId: EndpointId, slot: string): boolean { + return kv.get(getSlotKey(endpointId, slot)) !== undefined; + } + /** * Remove an entry from an endpoint's c-list. * @@ -966,6 +986,72 @@ export function makeKernelStore(kdb: KernelDatabase) { kv.delete(vatKey); } + /** + * Get the set of GC actions to perform. + * + * @returns The set of GC actions to perform. + */ + function getGCActions(): Set { + return new Set(JSON.parse(gcActions.get() ?? '[]')); + } + + /** + * Set the set of GC actions to perform. + * + * @param actions - The set of GC actions to perform. + */ + function setGCActions(actions: Set): void { + const a = Array.from(actions); + a.sort(); + gcActions.set(JSON.stringify(a)); + } + + /** + * Add a new GC action to the set of GC actions to perform. + * + * @param newActions - The new GC action to add. + */ + function addGCActions(newActions: GCAction[]): void { + const actions = getGCActions(); + for (const action of newActions) { + assert.typeof(action, 'string', 'addGCActions given bad action'); + const [vatId, type, kref] = action.split(' '); + insistVatId(vatId); + insistGCActionType(type); + insistKernelType('object', kref); + actions.add(action); + } + setGCActions(actions); + } + + /** + * Schedule a vat for reaping. + * + * @param vatId - The vat to schedule for reaping. + */ + function scheduleReap(vatId: VatId): void { + const queue = JSON.parse(reapQueue.get() ?? '[]'); + if (!queue.includes(vatId)) { + queue.push(vatId); + reapQueue.set(JSON.stringify(queue)); + } + } + + /** + * Get the next reap action. + * + * @returns The next reap action, or undefined if the queue is empty. + */ + function nextReapAction(): RunQueueItemBringOutYourDead | undefined { + const queue = JSON.parse(reapQueue.get() ?? '[]'); + if (queue.length > 0) { + const vatId = queue.shift(); + reapQueue.set(JSON.stringify(queue)); + return harden({ type: RunQueueItemType.bringOutYourDead, vatId }); + } + return undefined; + } + return harden({ enqueueRun, dequeueRun, @@ -986,7 +1072,6 @@ export function makeKernelStore(kdb: KernelDatabase) { clearReachableFlag, getOwner, deleteKernelObject, - deleteClistEntry, initKernelPromise, getKernelPromise, enqueuePromiseMessage, @@ -995,14 +1080,21 @@ export function makeKernelStore(kdb: KernelDatabase) { resolveKernelPromise, deleteKernelPromise, addPromiseSubscriber, + addClistEntry, + hasCListEntry, + deleteClistEntry, erefToKref, allocateErefForKref, krefToEref, - addClistEntry, forgetEref, forgetKref, clear, makeVatStore, + getGCActions, + setGCActions, + addGCActions, + scheduleReap, + nextReapAction, reset, kv, }); diff --git a/packages/kernel/src/types.ts b/packages/kernel/src/types.ts index 019876812..771af0a5d 100644 --- a/packages/kernel/src/types.ts +++ b/packages/kernel/src/types.ts @@ -1,5 +1,4 @@ import type { Message } from '@agoric/swingset-liveslots'; -import { Fail } from '@endo/errors'; import type { CapData } from '@endo/marshal'; import type { PromiseKit } from '@endo/promise-kit'; import { @@ -24,6 +23,7 @@ import type { Json } from '@metamask/utils'; import { UnsafeJsonStruct } from '@metamask/utils'; import type { DuplexStream } from '@ocap/streams'; +import { Fail } from './assert.ts'; import type { VatCommandReply, VatCommand } from './messages/vat.ts'; export type VatId = string; @@ -55,17 +55,39 @@ export type RunQueueItemNotify = { kpid: KRef; }; -export type RunQueueItem = RunQueueItemSend | RunQueueItemNotify; +export type RunQueueItemGCAction = { + type: GCRunQueueType; + vatId: VatId; + krefs: KRef[]; +}; + +export type RunQueueItemBringOutYourDead = { + type: 'bringOutYourDead'; + vatId: VatId; +}; + +export type RunQueueItem = + | RunQueueItemSend + | RunQueueItemNotify + | RunQueueItemGCAction + | RunQueueItemBringOutYourDead; export const MessageStruct = object({ methargs: CapDataStruct, result: union([string(), literal(undefined), literal(null)]), }); -export const insistMessage = (value: unknown): boolean => +/** + * Assert that a value is a valid message. + * + * @param value - The value to check. + * @throws if the value is not a valid message. + */ +export function insistMessage(value: unknown): asserts value is Message { is(value, MessageStruct) || Fail`not a valid message`; +} -const RunQueueItemType = { +export const RunQueueItemType = { send: 'send', notify: 'notify', dropExports: 'dropExports', @@ -96,6 +118,7 @@ const RunQueueItemStructs = { }), [RunQueueItemType.bringOutYourDead]: object({ type: literal(RunQueueItemType.bringOutYourDead), + vatId: string(), }), }; @@ -153,8 +176,15 @@ export const isVatId = (value: unknown): value is VatId => value.at(0) === 'v' && value.slice(1) === String(Number(value.slice(1))); -export const insistVatId = (value: unknown): boolean => +/** + * Assert that a value is a valid vat id. + * + * @param value - The value to check. + * @throws if the value is not a valid vat id. + */ +export function insistVatId(value: unknown): asserts value is VatId { isVatId(value) || Fail`not a valid VatId`; +} export const VatIdStruct = define('VatId', isVatId); @@ -290,3 +320,57 @@ export const VatCheckpointStruct = tuple([ map(string(), string()), set(string()), ]); + +export type GCRunQueueType = 'dropExports' | 'retireExports' | 'retireImports'; +export type GCActionType = 'dropExport' | 'retireExport' | 'retireImport'; +export const actionTypePriorities: GCActionType[] = [ + 'dropExport', + 'retireExport', + 'retireImport', +]; + +/** + * A mapping of GC action type to queue event type. + */ +export const queueTypeFromActionType = new Map([ + ['dropExport', RunQueueItemType.dropExports], + ['retireExport', RunQueueItemType.retireExports], + ['retireImport', RunQueueItemType.retireImports], +]); + +export const isGCActionType = (value: unknown): value is GCActionType => + actionTypePriorities.includes(value as GCActionType); + +/** + * Assert that a value is a valid GC action type. + * + * @param value - The value to check. + * @throws if the value is not a valid GC action type. + */ +export function insistGCActionType( + value: unknown, +): asserts value is GCActionType { + isGCActionType(value) || Fail`not a valid GCActionType ${value}`; +} + +export type GCAction = `${VatId} ${GCActionType} ${KRef}`; + +export const GCActionStruct = define('GCAction', (value: unknown) => { + if (typeof value !== 'string') { + return false; + } + const [vatId, actionType, kref] = value.split(' '); + if (!isVatId(vatId)) { + return false; + } + if (!isGCActionType(actionType)) { + return false; + } + if (typeof kref !== 'string' || !kref.startsWith('ko')) { + return false; + } + return true; +}); + +export const isGCAction = (value: unknown): value is GCAction => + is(value, GCActionStruct); From 2aa6ca8d5f29d306c21c3d3c2d51a420acd40897 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Mon, 24 Feb 2025 14:45:30 -0300 Subject: [PATCH 05/21] process GC actions in the loop --- packages/kernel/src/Kernel.ts | 24 ++++++++++++++++-------- packages/kernel/src/syscall.ts | 2 +- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/packages/kernel/src/Kernel.ts b/packages/kernel/src/Kernel.ts index d5f5e70f1..64e34f633 100644 --- a/packages/kernel/src/Kernel.ts +++ b/packages/kernel/src/Kernel.ts @@ -180,14 +180,6 @@ export class Kernel { */ async #run(): Promise { for await (const item of this.#runQueueItems()) { - // Process GC actions - const message = - processGCActionSet(this.#storage) ?? this.#storage.nextReapAction(); - if (message) { - await this.#deliver(message); - } - - // Deliver message await this.#deliver(item); } } @@ -199,6 +191,21 @@ export class Kernel { */ async *#runQueueItems(): AsyncGenerator { for (;;) { + // Check for GC actions regardless of queue state + const gcAction = processGCActionSet(this.#storage); + if (gcAction) { + yield gcAction; + continue; + } + + // Check for reap actions regardless of queue state + const reapAction = this.#storage.nextReapAction(); + if (reapAction) { + yield reapAction; + continue; + } + + // Process regular queue items if any exist while (this.#runQueueLength > 0) { const item = this.#dequeueRun(); if (item) { @@ -207,6 +214,7 @@ export class Kernel { break; } } + if (this.#runQueueLength === 0) { const { promise, resolve } = makePromiseKit(); if (this.#wakeUpTheRunQueue !== null) { diff --git a/packages/kernel/src/syscall.ts b/packages/kernel/src/syscall.ts index f36e177f6..bed8d4e37 100644 --- a/packages/kernel/src/syscall.ts +++ b/packages/kernel/src/syscall.ts @@ -40,6 +40,7 @@ function makeSupervisorSyscall( insistVatSyscallObject(vso); let syscallResult; try { + console.log('doSyscall', vso); syscallResult = supervisor.executeSyscall(vso); } catch (problem) { console.warn(`supervisor got error during syscall:`, problem); @@ -76,7 +77,6 @@ function makeSupervisorSyscall( retireImports: (vrefs: string[]) => doSyscall(['retireImports', vrefs]), retireExports: (vrefs: string[]) => doSyscall(['retireExports', vrefs]), abandonExports: (vrefs: string[]) => doSyscall(['abandonExports', vrefs]), - callNow: (_target: string, _method: string, _args: unknown[]) => { throw Error(`callNow not supported (we have no devices)`); }, From 949a41ac438e8a0324dbaab3c64e82b8e3b16354 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Tue, 25 Feb 2025 19:40:50 -0300 Subject: [PATCH 06/21] chore: restructure kernel store and services --- packages/kernel/src/Kernel.ts | 13 +- packages/kernel/src/VatHandle.ts | 2 +- packages/kernel/src/VatSupervisor.ts | 12 +- .../garbage-collection.ts} | 10 +- .../src/{ => services}/kernel-marshal.ts | 2 +- .../meter-control.ts} | 0 packages/kernel/src/{ => services}/syscall.ts | 4 +- .../types.ts} | 3 + packages/kernel/src/store/base-store.ts | 121 ++ packages/kernel/src/store/clist-store.ts | 275 +++++ packages/kernel/src/store/gc-store.ts | 172 +++ packages/kernel/src/store/id-store.ts | 65 ++ packages/kernel/src/store/kernel-store.ts | 1032 +---------------- packages/kernel/src/store/object-store.ts | 133 +++ packages/kernel/src/store/promise-store.ts | 221 ++++ packages/kernel/src/store/queue-store.ts | 171 +++ packages/kernel/src/store/refcount-store.ts | 77 ++ .../src/store/{ => utils}/kernel-slots.ts | 2 +- .../store/{ => utils}/kernel-store.test.ts | 6 +- .../kernel/src/store/{ => utils}/parse-ref.ts | 0 .../kernel/src/store/utils/promise-ref.ts | 12 + .../kernel/src/store/{ => utils}/reachable.ts | 2 +- packages/kernel/src/types.ts | 2 +- packages/kernel/src/{ => utils}/assert.ts | 0 .../wait-quiescent.ts} | 0 25 files changed, 1317 insertions(+), 1020 deletions(-) rename packages/kernel/src/{processGCActionSet.ts => services/garbage-collection.ts} (96%) rename packages/kernel/src/{ => services}/kernel-marshal.ts (99%) rename packages/kernel/src/{dummyMeterControl.ts => services/meter-control.ts} (100%) rename packages/kernel/src/{ => services}/syscall.ts (96%) rename packages/kernel/src/{ag-liveslots-types.ts => services/types.ts} (99%) create mode 100644 packages/kernel/src/store/base-store.ts create mode 100644 packages/kernel/src/store/clist-store.ts create mode 100644 packages/kernel/src/store/gc-store.ts create mode 100644 packages/kernel/src/store/id-store.ts create mode 100644 packages/kernel/src/store/object-store.ts create mode 100644 packages/kernel/src/store/promise-store.ts create mode 100644 packages/kernel/src/store/queue-store.ts create mode 100644 packages/kernel/src/store/refcount-store.ts rename packages/kernel/src/store/{ => utils}/kernel-slots.ts (97%) rename packages/kernel/src/store/{ => utils}/kernel-store.test.ts (97%) rename packages/kernel/src/store/{ => utils}/parse-ref.ts (100%) create mode 100644 packages/kernel/src/store/utils/promise-ref.ts rename packages/kernel/src/store/{ => utils}/reachable.ts (95%) rename packages/kernel/src/{ => utils}/assert.ts (100%) rename packages/kernel/src/{waitUntilQuiescent.ts => utils/wait-quiescent.ts} (100%) diff --git a/packages/kernel/src/Kernel.ts b/packages/kernel/src/Kernel.ts index 64e34f633..8785b5bba 100644 --- a/packages/kernel/src/Kernel.ts +++ b/packages/kernel/src/Kernel.ts @@ -12,9 +12,6 @@ import type { DuplexStream } from '@ocap/streams'; import type { Logger } from '@ocap/utils'; import { makeLogger } from '@ocap/utils'; -import { assert, Fail } from './assert.ts'; -import { kser, kunser, krefOf, kslot } from './kernel-marshal.ts'; -import type { SlotValue } from './kernel-marshal.ts'; import { isKernelCommand, KernelCommandMethod } from './messages/index.ts'; import type { KernelCommand, @@ -22,10 +19,13 @@ import type { VatCommand, VatCommandReturnType, } from './messages/index.ts'; -import { processGCActionSet } from './processGCActionSet.ts'; -import { isPromiseRef, makeKernelStore } from './store/kernel-store.ts'; +import { processGCActionSet } from './services/garbage-collection.ts'; +import type { SlotValue } from './services/kernel-marshal.ts'; +import { kser, kunser, krefOf, kslot } from './services/kernel-marshal.ts'; +import { makeKernelStore } from './store/kernel-store.ts'; import type { KernelStore } from './store/kernel-store.ts'; -import { parseRef } from './store/parse-ref.ts'; +import { parseRef } from './store/utils/parse-ref.ts'; +import { isPromiseRef } from './store/utils/promise-ref.ts'; import type { VatId, VRef, @@ -43,6 +43,7 @@ import { insistMessage, isClusterConfig, } from './types.ts'; +import { assert, Fail } from './utils/assert.ts'; import { VatHandle } from './VatHandle.ts'; /** diff --git a/packages/kernel/src/VatHandle.ts b/packages/kernel/src/VatHandle.ts index 707116ced..ff069f54d 100644 --- a/packages/kernel/src/VatHandle.ts +++ b/packages/kernel/src/VatHandle.ts @@ -19,7 +19,7 @@ import type { VatCommandReturnType, } from './messages/index.ts'; import type { KernelStore } from './store/kernel-store.ts'; -import { parseRef } from './store/parse-ref.ts'; +import { parseRef } from './store/utils/parse-ref.ts'; import type { PromiseCallbacks, VatId, diff --git a/packages/kernel/src/VatSupervisor.ts b/packages/kernel/src/VatSupervisor.ts index db42507f4..e226b0e39 100644 --- a/packages/kernel/src/VatSupervisor.ts +++ b/packages/kernel/src/VatSupervisor.ts @@ -10,20 +10,16 @@ import type { CapData } from '@endo/marshal'; import { StreamReadError } from '@ocap/errors'; import type { DuplexStream } from '@ocap/streams'; -import type { - DispatchFn, - MakeLiveSlotsFn, - GCTools, -} from './ag-liveslots-types.ts'; -import { makeDummyMeterControl } from './dummyMeterControl.ts'; import type { VatCommand, VatCommandReply } from './messages/index.ts'; import { VatCommandMethod } from './messages/index.ts'; -import { makeSupervisorSyscall } from './syscall.ts'; +import { makeDummyMeterControl } from './services/meter-control.ts'; +import { makeSupervisorSyscall } from './services/syscall.ts'; +import type { DispatchFn, MakeLiveSlotsFn, GCTools } from './services/types.ts'; import type { VatConfig, VatId, VRef } from './types.ts'; import { ROOT_OBJECT_VREF, isVatConfig } from './types.ts'; +import { waitUntilQuiescent } from './utils/wait-quiescent.ts'; import type { VatKVStore } from './VatKVStore.ts'; import { makeVatKVStore } from './VatKVStore.ts'; -import { waitUntilQuiescent } from './waitUntilQuiescent.ts'; const makeLiveSlots: MakeLiveSlotsFn = localMakeLiveSlots; diff --git a/packages/kernel/src/processGCActionSet.ts b/packages/kernel/src/services/garbage-collection.ts similarity index 96% rename from packages/kernel/src/processGCActionSet.ts rename to packages/kernel/src/services/garbage-collection.ts index efbcf0e1c..cd1de5785 100644 --- a/packages/kernel/src/processGCActionSet.ts +++ b/packages/kernel/src/services/garbage-collection.ts @@ -1,18 +1,18 @@ -import { insistKernelType } from './store/kernel-slots.ts'; -import type { KernelStore } from './store/kernel-store.ts'; +import type { KernelStore } from '../store/kernel-store.ts'; +import { insistKernelType } from '../store/utils/kernel-slots.ts'; import type { GCAction, GCActionType, KRef, RunQueueItem, VatId, -} from './types.ts'; +} from '../types.ts'; import { actionTypePriorities, insistGCActionType, insistVatId, queueTypeFromActionType, -} from './types.ts'; +} from '../types.ts'; /** * Parse a GC action string into a vat id, type, and kref. @@ -84,7 +84,7 @@ export function processGCActionSet( const isReachable = hasCList ? storage.getReachableFlag(vatId, kref) : undefined; - const exists = storage.kernelObjectExists(kref); + const exists = storage.kernelRefExists(kref); const { reachable, recognizable } = exists ? storage.getObjectRefCount(kref) : { reachable: 0, recognizable: 0 }; diff --git a/packages/kernel/src/kernel-marshal.ts b/packages/kernel/src/services/kernel-marshal.ts similarity index 99% rename from packages/kernel/src/kernel-marshal.ts rename to packages/kernel/src/services/kernel-marshal.ts index cfe779eab..146a0d72c 100644 --- a/packages/kernel/src/kernel-marshal.ts +++ b/packages/kernel/src/services/kernel-marshal.ts @@ -4,7 +4,7 @@ import { makeMarshal } from '@endo/marshal'; import type { CapData } from '@endo/marshal'; import type { Passable } from '@endo/pass-style'; -import type { KRef } from './types.ts'; +import type { KRef } from '../types.ts'; // Simple wrapper for serializing and unserializing marshalled values inside the // kernel, where we don't actually want to use clists nor actually allocate real diff --git a/packages/kernel/src/dummyMeterControl.ts b/packages/kernel/src/services/meter-control.ts similarity index 100% rename from packages/kernel/src/dummyMeterControl.ts rename to packages/kernel/src/services/meter-control.ts diff --git a/packages/kernel/src/syscall.ts b/packages/kernel/src/services/syscall.ts similarity index 96% rename from packages/kernel/src/syscall.ts rename to packages/kernel/src/services/syscall.ts index bed8d4e37..9132d1579 100644 --- a/packages/kernel/src/syscall.ts +++ b/packages/kernel/src/services/syscall.ts @@ -9,8 +9,8 @@ import type { import type { CapData } from '@endo/marshal'; import type { KVStore } from '@ocap/store'; -import type { Syscall, SyscallResult } from './ag-liveslots-types.ts'; -import type { VatSupervisor } from './VatSupervisor.ts'; +import type { Syscall, SyscallResult } from './types.ts'; +import type { VatSupervisor } from '../VatSupervisor.ts'; /** * This returns a function that is provided to liveslots as the 'syscall' diff --git a/packages/kernel/src/ag-liveslots-types.ts b/packages/kernel/src/services/types.ts similarity index 99% rename from packages/kernel/src/ag-liveslots-types.ts rename to packages/kernel/src/services/types.ts index 1200cdbfb..cf5c89ded 100644 --- a/packages/kernel/src/ag-liveslots-types.ts +++ b/packages/kernel/src/services/types.ts @@ -12,6 +12,7 @@ export type DispatchFn = (vdo: VatDeliveryObject) => Promise; export type LiveSlots = { dispatch: DispatchFn; }; + export type Syscall = { send: ( target: string, @@ -31,6 +32,7 @@ export type Syscall = { vatstoreSet: (key: string, value: string) => void; vatstoreDelete: (key: string) => void; }; + export type GCTools = { // eslint-disable-next-line @typescript-eslint/naming-convention WeakRef: WeakRefConstructor; @@ -40,6 +42,7 @@ export type GCTools = { gcAndFinalize: () => Promise; meterControl: MeterControl; }; + export type MakeLiveSlotsFn = ( syscall: Syscall, forVatId: string, diff --git a/packages/kernel/src/store/base-store.ts b/packages/kernel/src/store/base-store.ts new file mode 100644 index 000000000..71ea02442 --- /dev/null +++ b/packages/kernel/src/store/base-store.ts @@ -0,0 +1,121 @@ +import type { KVStore } from '@ocap/store'; + +import type { EndpointId, KRef } from '../types.ts'; + +export type StoredValue = { + get(): string | undefined; + set(newValue: string): void; + delete(): void; +}; + +/** + * Create a base store object that provides functionality for managing stored values and queues. + * + * @param kv - The key-value store to use for persistent storage. + * @returns An object with methods for managing stored values and queues. + */ +// eslint-disable-next-line @typescript-eslint/explicit-function-return-type +export function makeBaseStore(kv: KVStore) { + // As refcounts are decremented, we accumulate a set of krefs for which + // action might need to be taken: + // * promises which are now resolved and unreferenced can be deleted + // * objects which are no longer reachable: export can be dropped + // * objects which are no longer recognizable: export can be retired + // This set is ephemeral: it lives in RAM, grows as deliveries and syscalls + // cause decrefs, and will be harvested by processRefcounts(). This needs to be + // called in the same transaction window as the syscalls/etc which prompted + // the change, else removals might be lost (not performed during the next + // replay). + const maybeFreeKrefs = new Set(); + + /** + * Get the key for the reachable flag and vatSlot for a given endpoint and kref. + * + * @param endpointId - The endpoint for which the reachable flag is being set. + * @param kref - The kref. + * @returns The key for the reachable flag and vatSlot. + */ + function getSlotKey(endpointId: EndpointId, kref: KRef): string { + return `${endpointId}.c.${kref}`; + } + + /** + * Increment the value of a persistently stored counter. + * + * Note that the while the value is interpreted as an integer (in order to + * enable it to be incremented), it is stored and returned in the form of a + * string. This is because (a) our persistent storage only stores strings, and + * (b) the sole purpose of one of these counters is simply to provide an + * unending sequence of unique values; we don't actually use them as numbers + * or, indeed, even care at all if this sequence is produced using numbers. + * + * @param value - Reference to the stored value to be incremented. + * @returns The value as it was prior to being incremented. + */ + function incCounter(value: StoredValue): string { + const current = value.get(); + const next = `${Number(current) + 1}`; + value.set(next); + return current as string; + } + + /** + * Provide a stored value object for which we keep an in-memory cache. We only + * touch persistent storage if the value hasn't ever been read of if it is + * modified; otherwise we can service read requests from memory. + * + * IMPORTANT NOTE: in order for the cache to work, all subsequent accesses to + * the value MUST be made through a common stored value object. + * + * @param key - A key string that identifies the value. + * @param init - If provided, an initial setting if the stored entity does not exist. + * @returns An object for interacting with the value. + */ + function provideCachedStoredValue(key: string, init?: string): StoredValue { + let value: string | undefined = kv.get(key); + if (value === undefined && init !== undefined) { + kv.set(key, init); + value = init; + } + return harden({ + get(): string | undefined { + return value; + }, + set(newValue: string): void { + value = newValue; + kv.set(key, value); + }, + delete(): void { + value = undefined; + kv.delete(key); + }, + }); + } + + /** + * Provide a stored value object that is kept in persistent storage without caching. + * + * @param key - A key string that identifies the value. + * @param init - If provided, an initial setting if the stored entity does not exist. + * @returns An object for interacting with the value. + */ + function provideRawStoredValue(key: string, init?: string): StoredValue { + if (kv.get(key) === undefined && init !== undefined) { + kv.set(key, init); + } + return harden({ + get: () => kv.get(key), + set: (newValue: string) => kv.set(key, newValue), + delete: () => kv.delete(key), + }); + } + + return { + maybeFreeKrefs, + getSlotKey, + incCounter, + // Stored value + provideCachedStoredValue, + provideRawStoredValue, + }; +} diff --git a/packages/kernel/src/store/clist-store.ts b/packages/kernel/src/store/clist-store.ts new file mode 100644 index 000000000..6ba2b571f --- /dev/null +++ b/packages/kernel/src/store/clist-store.ts @@ -0,0 +1,275 @@ +import { Fail } from '@endo/errors'; +import type { KVStore } from '@ocap/store'; + +import type { makeBaseStore } from './base-store.ts'; +import type { makeGCStore } from './gc-store.ts'; +import type { makeObjectStore } from './object-store.ts'; +import type { makeRefCountStore } from './refcount-store.ts'; +import { parseRef } from './utils/parse-ref.ts'; +import { isPromiseRef } from './utils/promise-ref.ts'; +import { + buildReachableAndVatSlot, + parseReachableAndVatSlot, +} from './utils/reachable.ts'; +import type { EndpointId, KRef, ERef } from '../types.ts'; + +/** + * Create a store for the c-list. + * + * @param kv - The key-value store to use for persistent storage. + * @param baseStore - The base store to use for the c-list store. + * @param gcStore - The GC store to use for the c-list store. + * @param objectStore - The object store to use for the c-list store. + * @param refCountStore - The refcount store to use for the c-list store. + * @returns The c-list store. + */ +// eslint-disable-next-line @typescript-eslint/explicit-function-return-type +export function makeCListStore( + kv: KVStore, + baseStore: ReturnType, + gcStore: ReturnType, + objectStore: ReturnType, + refCountStore: ReturnType, +) { + /** + * Add an entry to an endpoint's c-list, creating a new bidirectional mapping + * between an ERef belonging to the endpoint and a KRef belonging to the + * kernel. + * + * @param endpointId - The endpoint whose c-list is to be added to. + * @param kref - The KRef. + * @param eref - The ERef. + */ + function addClistEntry(endpointId: EndpointId, kref: KRef, eref: ERef): void { + kv.set( + baseStore.getSlotKey(endpointId, kref), + buildReachableAndVatSlot(true, eref), + ); + kv.set(baseStore.getSlotKey(endpointId, eref), kref); + } + + /** + * Test if there's a c-list entry for some slot. + * + * @param endpointId - The endpoint of interest + * @param slot - The slot of interest + * @returns true iff this vat has a c-list entry mapping for `slot`. + */ + function hasCListEntry(endpointId: EndpointId, slot: string): boolean { + return kv.get(baseStore.getSlotKey(endpointId, slot)) !== undefined; + } + + /** + * Remove an entry from an endpoint's c-list. + * + * @param endpointId - The endpoint whose c-list entry is to be removed. + * @param kref - The KRef. + * @param eref - The ERef. + */ + function deleteClistEntry( + endpointId: EndpointId, + kref: KRef, + eref: ERef, + ): void { + const kernelKey = baseStore.getSlotKey(endpointId, kref); + const vatKey = baseStore.getSlotKey(endpointId, eref); + assert(kv.get(kernelKey)); + gcStore.clearReachableFlag(endpointId, kref); + const { direction } = parseRef(eref); + decrementRefCount(kref, { + isExport: direction === 'export', + onlyRecognizable: true, + }); + kv.delete(kernelKey); + kv.delete(vatKey); + } + + /** + * Generate a new eref for a kernel object or promise being imported into an + * endpoint. + * + * @param endpointId - The endpoint the kref is being imported into. + * @param kref - The kref for the kernel object or promise in question. + * + * @returns A new eref in the scope of the given endpoint for the given kernel entity. + */ + function allocateErefForKref(endpointId: EndpointId, kref: KRef): ERef { + let id; + const refTag = endpointId.startsWith('v') ? '' : endpointId[0]; + let refType; + if (isPromiseRef(kref)) { + id = kv.get(`e.nextPromiseId.${endpointId}`); + kv.set(`e.nextPromiseId.${endpointId}`, `${Number(id) + 1}`); + refType = 'p'; + } else { + id = kv.get(`e.nextObjectId.${endpointId}`); + kv.set(`e.nextObjectId.${endpointId}`, `${Number(id) + 1}`); + refType = 'o'; + } + const eref = `${refTag}${refType}-${id}`; + addClistEntry(endpointId, kref, eref); + return eref; + } + + /** + * Look up the ERef that and endpoint's c-list maps a KRef to. + * + * @param endpointId - The endpoint in question. + * @param eref - The ERef to look up. + * @returns The KRef corresponding to `eref` in the given endpoints c-list, or undefined + * if there is no such mapping. + */ + function erefToKref(endpointId: EndpointId, eref: ERef): KRef | undefined { + return kv.get(baseStore.getSlotKey(endpointId, eref)); + } + + /** + * Look up the KRef that and endpoint's c-list maps an ERef to. + * + * @param endpointId - The endpoint in question. + * @param kref - The KRef to look up. + * @returns The given endpoint's ERef corresponding to `kref`, or undefined if + * there is no such mapping. + */ + function krefToEref(endpointId: EndpointId, kref: KRef): ERef | undefined { + const key = baseStore.getSlotKey(endpointId, kref); + const data = kv.get(key); + if (!data) { + return undefined; + } + const { vatSlot } = parseReachableAndVatSlot(data); + return vatSlot; + } + + /** + * Remove an entry from an endpoint's c-list given an eref. + * + * @param endpointId - The endpoint whose c-list entry is to be removed. + * @param eref - The ERef. + */ + function forgetEref(endpointId: EndpointId, eref: ERef): void { + const kref = erefToKref(endpointId, eref); + if (kref) { + deleteClistEntry(endpointId, kref, eref); + } + } + + /** + * Remove an entry from an endpoint's c-list given a kref. + * + * @param endpointId - The endpoint whose c-list entry is to be removed. + * @param kref - The Kref. + */ + function forgetKref(endpointId: EndpointId, kref: KRef): void { + const eref = krefToEref(endpointId, kref); + if (eref) { + deleteClistEntry(endpointId, kref, eref); + } + } + + /** + * Increment the reference count associated with some kernel object. + * + * We track references to promises and objects, but not devices. Promises + * have only a "reachable" count, whereas objects track both "reachable" + * and "recognizable" counts. + * + * @param kref - The kernel slot whose refcount is to be incremented. + * @param options - Options for the increment. + * @param options.isExport - True if the reference comes from a clist export, which counts for promises but not objects. + * @param options.onlyRecognizable - True if the reference provides only recognition, not reachability. + */ + function incrementRefCount( + kref: KRef, + { + isExport = false, + onlyRecognizable = false, + }: { isExport?: boolean; onlyRecognizable?: boolean }, + ): void { + kref || Fail`incrementRefCount called with empty kref`; + + const { isPromise } = parseRef(kref); + if (isPromise) { + const refCount = Number(kv.get(refCountStore.refCountKey(kref))) + 1; + kv.set(refCountStore.refCountKey(kref), `${refCount}`); + return; + } + + // If `isExport` the reference comes from a clist export, which counts for promises but not objects + if (isExport) { + return; + } + + const counts = objectStore.getObjectRefCount(kref); + if (!onlyRecognizable) { + counts.reachable += 1; + } + counts.recognizable += 1; + objectStore.setObjectRefCount(kref, counts); + } + + /** + * Decrement the reference count associated with some kernel object. + * + * @param kref - The kernel slot whose refcount is to be decremented. + * @param options - Options for the decrement. + * @param options.isExport - True if the reference comes from a clist export, which counts for promises but not objects. + * @param options.onlyRecognizable - True if the reference provides only recognition, not reachability. + * @returns True if the reference count has been decremented to zero, false if it is still non-zero. + * @throws if this tries to decrement the reference count below zero. + */ + function decrementRefCount( + kref: KRef, + { + isExport = false, + onlyRecognizable = false, + }: { isExport?: boolean; onlyRecognizable?: boolean }, + ): boolean { + kref || Fail`decrementRefCount called with empty kref`; + + const { isPromise } = parseRef(kref); + if (isPromise) { + let refCount = Number(kv.get(refCountStore.refCountKey(kref))); + refCount > 0 || Fail`refCount underflow ${kref}`; + refCount -= 1; + kv.set(refCountStore.refCountKey(kref), `${refCount}`); + if (refCount === 0) { + baseStore.maybeFreeKrefs.add(kref); + return true; + } + return false; + } + + if (isExport || !refCountStore.kernelRefExists(kref)) { + return false; + } + + const counts = objectStore.getObjectRefCount(kref); + if (!onlyRecognizable) { + counts.reachable -= 1; + } + counts.recognizable -= 1; + if (!counts.reachable || !counts.recognizable) { + baseStore.maybeFreeKrefs.add(kref); + } + objectStore.setObjectRefCount(kref, counts); + + return false; + } + + return { + // C-List entries + addClistEntry, + hasCListEntry, + deleteClistEntry, + // Eref allocation + allocateErefForKref, + erefToKref, + krefToEref, + forgetEref, + forgetKref, + // Refcount management + incrementRefCount, + decrementRefCount, + }; +} diff --git a/packages/kernel/src/store/gc-store.ts b/packages/kernel/src/store/gc-store.ts new file mode 100644 index 000000000..e5e1e410d --- /dev/null +++ b/packages/kernel/src/store/gc-store.ts @@ -0,0 +1,172 @@ +import type { KVStore } from '@ocap/store'; + +import type { makeBaseStore } from './base-store.ts'; +import type { makeObjectStore } from './object-store.ts'; +import type { makeRefCountStore } from './refcount-store.ts'; +import { insistKernelType } from './utils/kernel-slots.ts'; +import { parseRef } from './utils/parse-ref.ts'; +import { + buildReachableAndVatSlot, + parseReachableAndVatSlot, +} from './utils/reachable.ts'; +import type { + VatId, + EndpointId, + KRef, + GCAction, + RunQueueItemBringOutYourDead, +} from '../types.ts'; +import { insistGCActionType, insistVatId, RunQueueItemType } from '../types.ts'; + +/** + * Create a store for garbage collection. + * + * @param kv - The key-value store to use for persistent storage. + * @param baseStore - The base store to use for the GC store. + * @param refCountStore - The refcount store to use for the GC store. + * @param objectStore - The object store to use for the GC store. + * @returns The GC store. + */ +// eslint-disable-next-line @typescript-eslint/explicit-function-return-type +export function makeGCStore( + kv: KVStore, + baseStore: ReturnType, + refCountStore: ReturnType, + objectStore: ReturnType, +) { + let gcActions = baseStore.provideCachedStoredValue('gcActions', '[]'); + let reapQueue = baseStore.provideCachedStoredValue('reapQueue', '[]'); + + /** + * Get the set of GC actions to perform. + * + * @returns The set of GC actions to perform. + */ + function getGCActions(): Set { + return new Set(JSON.parse(gcActions.get() ?? '[]')); + } + + /** + * Set the set of GC actions to perform. + * + * @param actions - The set of GC actions to perform. + */ + function setGCActions(actions: Set): void { + const a = Array.from(actions); + a.sort(); + gcActions.set(JSON.stringify(a)); + } + + /** + * Add a new GC action to the set of GC actions to perform. + * + * @param newActions - The new GC action to add. + */ + function addGCActions(newActions: GCAction[]): void { + const actions = getGCActions(); + for (const action of newActions) { + assert.typeof(action, 'string', 'addGCActions given bad action'); + const [vatId, type, kref] = action.split(' '); + insistVatId(vatId); + insistGCActionType(type); + insistKernelType('object', kref); + actions.add(action); + } + setGCActions(actions); + } + + /** + * Check if a kernel object is reachable. + * + * @param endpointId - The endpoint for which the reachable flag is being checked. + * @param kref - The kref. + * @returns True if the kernel object is reachable, false otherwise. + */ + function getReachableFlag(endpointId: EndpointId, kref: KRef): boolean { + const key = baseStore.getSlotKey(endpointId, kref); + const data = kv.getRequired(key); + const { isReachable } = parseReachableAndVatSlot(data); + return isReachable; + } + + /** + * Clear the reachable flag for a given endpoint and kref. + * + * @param endpointId - The endpoint for which the reachable flag is being cleared. + * @param kref - The kref. + */ + function clearReachableFlag(endpointId: EndpointId, kref: KRef): void { + const key = baseStore.getSlotKey(endpointId, kref); + const { isReachable, vatSlot } = parseReachableAndVatSlot( + kv.getRequired(key), + ); + kv.set(key, buildReachableAndVatSlot(false, vatSlot)); + const { direction, isPromise } = parseRef(vatSlot); + // decrement 'reachable' part of refcount, but only for object imports + if ( + isReachable && + !isPromise && + direction === 'import' && + refCountStore.kernelRefExists(kref) + ) { + const counts = objectStore.getObjectRefCount(kref); + counts.reachable -= 1; + objectStore.setObjectRefCount(kref, counts); + if (counts.reachable === 0) { + baseStore.maybeFreeKrefs.add(kref); + } + } + } + + /** + * Schedule a vat for reaping. + * + * @param vatId - The vat to schedule for reaping. + */ + function scheduleReap(vatId: VatId): void { + const queue = JSON.parse(reapQueue.get() ?? '[]'); + if (!queue.includes(vatId)) { + queue.push(vatId); + reapQueue.set(JSON.stringify(queue)); + } + } + + /** + * Get the next reap action. + * + * @returns The next reap action, or undefined if the queue is empty. + */ + function nextReapAction(): RunQueueItemBringOutYourDead | undefined { + const queue = JSON.parse(reapQueue.get() ?? '[]'); + if (queue.length > 0) { + const vatId = queue.shift(); + reapQueue.set(JSON.stringify(queue)); + return harden({ type: RunQueueItemType.bringOutYourDead, vatId }); + } + return undefined; + } + + /** + * Reset the GC store. + */ + function reset(): void { + baseStore.maybeFreeKrefs.clear(); + gcActions = baseStore.provideCachedStoredValue('gcActions', '[]'); + reapQueue = baseStore.provideCachedStoredValue('reapQueue', '[]'); + } + + return { + // GC actions + getGCActions, + setGCActions, + addGCActions, + // Reachability tracking + getReachableFlag, + clearReachableFlag, + // Reaping + scheduleReap, + nextReapAction, + // Reset + reset, + }; +} diff --git a/packages/kernel/src/store/id-store.ts b/packages/kernel/src/store/id-store.ts new file mode 100644 index 000000000..439343605 --- /dev/null +++ b/packages/kernel/src/store/id-store.ts @@ -0,0 +1,65 @@ +import type { KVStore } from '@ocap/store'; + +import type { makeBaseStore } from './base-store.ts'; +import type { VatId, RemoteId, EndpointId } from '../types.ts'; + +/** + * Create a store for allocating IDs. + * + * @param kv - The key-value store to use for persistent storage. + * @param baseStore - The base store to use for the ID store. + * @returns The ID store. + */ +// eslint-disable-next-line @typescript-eslint/explicit-function-return-type +export function makeIdStore( + kv: KVStore, + baseStore: ReturnType, +) { + /** Counter for allocating VatIDs */ + let nextVatId = baseStore.provideCachedStoredValue('nextVatId', '1'); + /** Counter for allocating RemoteIDs */ + let nextRemoteId = baseStore.provideCachedStoredValue('nextRemoteId', '1'); + + /** + * Obtain an ID for a new vat. + * + * @returns The next VatID use. + */ + function getNextVatId(): VatId { + return `v${baseStore.incCounter(nextVatId)}`; + } + + /** + * Obtain an ID for a new remote connection. + * + * @returns The next remote ID use. + */ + function getNextRemoteId(): RemoteId { + return `r${baseStore.incCounter(nextRemoteId)}`; + } + + /** + * Initialize persistent state for a new endpoint. + * + * @param endpointId - The ID of the endpoint being added. + */ + function initEndpoint(endpointId: EndpointId): void { + kv.set(`e.nextPromiseId.${endpointId}`, '1'); + kv.set(`e.nextObjectId.${endpointId}`, '1'); + } + + /** + * Clear the kernel's persistent state and reset all counters. + */ + function reset(): void { + nextVatId = baseStore.provideCachedStoredValue('nextVatId', '1'); + nextRemoteId = baseStore.provideCachedStoredValue('nextRemoteId', '1'); + } + + return { + getNextVatId, + getNextRemoteId, + initEndpoint, + reset, + }; +} diff --git a/packages/kernel/src/store/kernel-store.ts b/packages/kernel/src/store/kernel-store.ts index 3f218e692..1afc0c0b9 100644 --- a/packages/kernel/src/store/kernel-store.ts +++ b/packages/kernel/src/store/kernel-store.ts @@ -53,53 +53,16 @@ * k.nextPromiseId = NN // allocation counter for promise KRefs */ -import type { Message } from '@agoric/swingset-liveslots'; -import { Fail } from '@endo/errors'; -import type { CapData } from '@endo/marshal'; -import type { KVStore, VatStore, KernelDatabase } from '@ocap/store'; +import type { KernelDatabase, KVStore, VatStore } from '@ocap/store'; -import { insistKernelType, makeKernelSlot } from './kernel-slots.ts'; -import { parseRef } from './parse-ref.ts'; -import { - buildReachableAndVatSlot, - parseReachableAndVatSlot, -} from './reachable.ts'; -import type { - VatId, - RemoteId, - EndpointId, - KRef, - ERef, - RunQueueItem, - PromiseState, - KernelPromise, - GCAction, - RunQueueItemBringOutYourDead, -} from '../types.ts'; -import { insistGCActionType, insistVatId, RunQueueItemType } from '../types.ts'; - -type StoredValue = { - get(): string | undefined; - set(newValue: string): void; - delete(): void; -}; - -type StoredQueue = { - enqueue(item: object): void; - dequeue(): object | undefined; - delete(): void; -}; - -/** - * Test if a KRef designates a promise. - * - * @param kref - The KRef to test. - * - * @returns true iff the given KRef references a promise. - */ -export function isPromiseRef(kref: KRef): boolean { - return kref[1] === 'p'; -} +import { makeBaseStore } from './base-store.ts'; +import { makeCListStore } from './clist-store.ts'; +import { makeGCStore } from './gc-store.ts'; +import { makeIdStore } from './id-store.ts'; +import { makeObjectStore } from './object-store.ts'; +import { makePromiseStore } from './promise-store.ts'; +import { makeQueueStore } from './queue-store.ts'; +import { makeRefCountStore } from './refcount-store.ts'; /** * Create a new KernelStore object wrapped around a raw kernel database. The @@ -122,644 +85,25 @@ export function makeKernelStore(kdb: KernelDatabase) { /** KV store in which all the kernel's own state is kept. */ const kv: KVStore = kdb.kernelKVStore; - /** The kernel's run queue. */ - let runQueue = createStoredQueue('run', true); - /** Counter for allocating VatIDs */ - let nextVatId = provideCachedStoredValue('nextVatId', '1'); - /** Counter for allocating RemoteIDs */ - let nextRemoteId = provideCachedStoredValue('nextRemoteId', '1'); - /** Counter for allocating kernel object IDs */ - let nextObjectId = provideCachedStoredValue('nextObjectId', '1'); - /** Counter for allocating kernel promise IDs */ - let nextPromiseId = provideCachedStoredValue('nextPromiseId', '1'); - /** Actions to perform during garbage collection */ - let gcActions = provideCachedStoredValue('gcActions', '[]'); - /** Objects to reap */ - let reapQueue = provideCachedStoredValue('reapQueue', '[]'); - - // As refcounts are decremented, we accumulate a set of krefs for which - // action might need to be taken: - // * promises which are now resolved and unreferenced can be deleted - // * objects which are no longer reachable: export can be dropped - // * objects which are no longer recognizable: export can be retired - // This set is ephemeral: it lives in RAM, grows as deliveries and syscalls - // cause decrefs, and will be harvested by processRefcounts(). This needs to be - // called in the same transaction window as the syscalls/etc which prompted - // the change, else removals might be lost (not performed during the next - // replay). - const maybeFreeKrefs = new Set(); - - /** - * Provide a stored value object for which we keep an in-memory cache. We only - * touch persistent storage if the value hasn't ever been read of if it is - * modified; otherwise we can service read requests from memory. - * - * IMPORTANT NOTE: in order for the cache to work, all subsequent accesses to - * the value MUST be made through a common stored value object. - * - * @param key - A key string that identifies the value. - * @param init - If provided, an initial setting if the stored entity does not exist. - * @returns An object for interacting with the value. - */ - function provideCachedStoredValue(key: string, init?: string): StoredValue { - let value: string | undefined = kv.get(key); - if (value === undefined && init !== undefined) { - kv.set(key, init); - value = init; - } - return harden({ - get(): string | undefined { - return value; - }, - set(newValue: string): void { - value = newValue; - kv.set(key, value); - }, - delete(): void { - value = undefined; - kv.delete(key); - }, - }); - } - - /** - * Provide a stored value object that is kept in persistent storage without caching. - * - * @param key - A key string that identifies the value. - * @param init - If provided, an initial setting if the stored entity does not exist. - * @returns An object for interacting with the value. - */ - function provideRawStoredValue(key: string, init?: string): StoredValue { - if (kv.get(key) === undefined && init !== undefined) { - kv.set(key, init); - } - return harden({ - get: () => kv.get(key), - set: (newValue: string) => kv.set(key, newValue), - delete: () => kv.delete(key), - }); - } - - /** - * Increment the value of a persistently stored counter. - * - * Note that the while the value is interpreted as an integer (in order to - * enable it to be incremented), it is stored and returned in the form of a - * string. This is because (a) our persistent storage only stores strings, and - * (b) the sole purpose of one of these counters is simply to provide an - * unending sequence of unique values; we don't actually use them as numbers - * or, indeed, even care at all if this sequence is produced using numbers. - * - * @param value - Reference to the stored value to be incremented. - * @returns The value as it was prior to being incremented. - */ - function incCounter(value: StoredValue): string { - const current = value.get(); - const next = `${Number(current) + 1}`; - value.set(next); - return current as string; - } - - /** - * Create a new (empty) persistently stored queue. - * - * @param queueName - The name for the queue (must be unique among queues). - * @param cached - Optional flag: set to true if the queue should cache its - * @returns An object for interacting with the new queue. - */ - function createStoredQueue( - queueName: string, - cached: boolean = false, - ): StoredQueue { - const qk = `queue.${queueName}`; - kv.set(`${qk}.head`, '1'); - kv.set(`${qk}.tail`, '1'); - return provideStoredQueue(queueName, cached); - } - - /** - * Find out how long some queue is. - * - * @param queueName - The name of the queue of interest. - * - * @returns the number of items in the given queue. - */ - function getQueueLength(queueName: string): number { - const qk = `queue.${queueName}`; - const head = kv.get(`${qk}.head`); - const tail = kv.get(`${qk}.tail`); - if (head === undefined || tail === undefined) { - throw Error(`unknown queue ${queueName}`); - } - return Number(head) - Number(tail); - } - - /** - * Produce an object to access a persistently stored queue. - * - * @param queueName - The name for the queue (must be unique among queues). - * @param cached - Optional flag: set to true if the queue should cache its - * limit indices in memory (only do this if the queue is going to be accessed or - * checked frequently). - * @returns An object for interacting with the queue. - */ - function provideStoredQueue( - queueName: string, - cached: boolean = false, - ): StoredQueue { - const qk = `queue.${queueName}`; - // Note: cached=true ==> caches only the head & tail indices, NOT the queue entries themselves - const provideValue = cached - ? provideCachedStoredValue - : provideRawStoredValue; - const head = provideValue(`${qk}.head`); - const tail = provideValue(`${qk}.tail`); - if (head.get() === undefined || tail.get() === undefined) { - throw Error(`queue ${queueName} not initialized`); - } - return { - enqueue(item: object): void { - if (head.get() === undefined) { - throw Error(`enqueue into deleted queue ${queueName}`); - } - const entryPos = incCounter(head); - kv.set(`${qk}.${entryPos}`, JSON.stringify(item)); - }, - dequeue(): object | undefined { - const headPos = head.get(); - if (headPos === undefined) { - return undefined; - } - const tailPos = tail.get(); - if (tailPos !== headPos) { - const entry = kv.getRequired(`${qk}.${tailPos}`); - kv.delete(`${qk}.${tailPos}`); - incCounter(tail); - return JSON.parse(entry) as object; - } - return undefined; - }, - delete(): void { - const headPos = head.get(); - if (headPos !== undefined) { - let tailPos = tail.get(); - while (tailPos !== headPos) { - kv.delete(`${qk}.${tailPos}`); - tailPos = `${Number(tailPos) + 1}`; - } - head.delete(); - tail.delete(); - } - }, - }; - } - - /** - * Append a message to the kernel's run queue. - * - * @param message - The message to enqueue. - */ - function enqueueRun(message: RunQueueItem): void { - runQueue.enqueue(message); - } - - /** - * Fetch the next message on the kernel's run queue. - * - * @returns The next message on the run queue, or undefined if the queue is - * empty. - */ - function dequeueRun(): RunQueueItem | undefined { - return runQueue.dequeue() as RunQueueItem | undefined; - } - - /** - * Obtain the number of entries in the run queue. - * - * @returns the number of items in the run queue. - */ - function runQueueLength(): number { - return getQueueLength('run'); - } - - /** - * Obtain an ID for a new vat. - * - * @returns The next VatID use. - */ - function getNextVatId(): VatId { - return `v${incCounter(nextVatId)}`; - } - - /** - * Obtain an ID for a new remote connection. - * - * @returns The next remote ID use. - */ - function getNextRemoteId(): RemoteId { - return `r${incCounter(nextRemoteId)}`; - } - - /** - * Initialize persistent state for a new endpoint. - * - * @param endpointId - The ID of the endpoint being added. - */ - function initEndpoint(endpointId: EndpointId): void { - kv.set(`e.nextPromiseId.${endpointId}`, '1'); - kv.set(`e.nextObjectId.${endpointId}`, '1'); - } - - /** - * Generate a new eref for a kernel object or promise being imported into an - * endpoint. - * - * @param endpointId - The endpoint the kref is being imported into. - * @param kref - The kref for the kernel object or promise in question. - * - * @returns A new eref in the scope of the given endpoint for the given kernel entity. - */ - function allocateErefForKref(endpointId: EndpointId, kref: KRef): ERef { - let id; - const refTag = endpointId.startsWith('v') ? '' : endpointId[0]; - let refType; - if (isPromiseRef(kref)) { - id = kv.get(`e.nextPromiseId.${endpointId}`); - kv.set(`e.nextPromiseId.${endpointId}`, `${Number(id) + 1}`); - refType = 'p'; - } else { - id = kv.get(`e.nextObjectId.${endpointId}`); - kv.set(`e.nextObjectId.${endpointId}`, `${Number(id) + 1}`); - refType = 'o'; - } - const eref = `${refTag}${refType}-${id}`; - addClistEntry(endpointId, kref, eref); - return eref; - } - - /** - * Obtain a KRef for the next unallocated kernel object. - * - * @returns The next koId use. - */ - function getNextObjectId(): KRef { - return makeKernelSlot('object', incCounter(nextObjectId)); - } - - /** - * Generate the storage key for a kernel entity's reference count. - * - * @param kref - The KRef of interest. - * @returns the key to store the indicated reference count at. - */ - function refCountKey(kref: KRef): string { - return `${kref}.refCount`; - } - - /** - * Get a kernel entity's reference count. - * - * @param kref - The KRef of interest. - * @returns the reference count of the indicated kernel entity. - */ - function getRefCount(kref: KRef): number { - return Number(kv.get(refCountKey(kref))); - } - - /** - * Increment a kernel entity's reference count. - * - * @param kref - The KRef of the entity to increment the ref count of. - * @returns the new reference count after incrementing. - */ - function incRefCount(kref: KRef): number { - const key = refCountKey(kref); - const newCount = Number(kv.get(key)) + 1; - kv.set(key, `${newCount}`); - return newCount; - } - - /** - * Decrement a kernel entity's reference count. - * - * @param kref - The KRef of the entity to decrement the ref count of. - * @returns the new reference count after decrementing. - */ - function decRefCount(kref: KRef): number { - const key = refCountKey(kref); - const newCount = Number(kv.get(key)) - 1; - kv.set(key, `${newCount}`); - return newCount; - } - - /** - * Get the reference counts for a kernel object - * - * @param kref - The KRef of the object of interest. - * @returns The reference counts for the object. - */ - function getObjectRefCount(kref: KRef): { - reachable: number; - recognizable: number; - } { - const data = kv.get(refCountKey(kref)); - if (!data) { - return { reachable: 0, recognizable: 0 }; - } - const [reachable = 0, recognizable = 0] = data.split(',').map(Number); - reachable <= recognizable || - Fail`refMismatch(get) ${kref} ${reachable},${recognizable}`; - return { reachable, recognizable }; - } - - /** - * Set the reference counts for a kernel object - * - * @param kref - The KRef of the object of interest. - * @param counts - The reference counts to set. - * @param counts.reachable - The reachable reference count. - * @param counts.recognizable - The recognizable reference count. - */ - function setObjectRefCount( - kref: KRef, - counts: { reachable: number; recognizable: number }, - ): void { - const { reachable, recognizable } = counts; - assert.typeof(reachable, 'number'); - assert.typeof(recognizable, 'number'); - (reachable >= 0 && recognizable >= 0) || - Fail`${kref} underflow ${reachable},${recognizable}`; - reachable <= recognizable || - Fail`refMismatch(set) ${kref} ${reachable},${recognizable}`; - kv.set(refCountKey(kref), `${reachable},${recognizable}`); - } - - /** - * Create a new kernel object. The new object will be born with reference and - * recognizability counts of 1, on the assumption that the new object - * corresponds to an object that has just been imported from somewhere. - * - * @param owner - The endpoint that is the owner of the new object. - * @returns The new object's KRef. - */ - function initKernelObject(owner: EndpointId): KRef { - const koId = getNextObjectId(); - kv.set(`${koId}.owner`, owner); - setObjectRefCount(koId, { reachable: 1, recognizable: 1 }); - return koId; - } - - /** - * Get a kernel object's owner. - * - * @param koId - The KRef of the kernel object of interest. - * @returns The identity of the vat or remote that owns the object. - */ - function getOwner(koId: KRef): EndpointId { - const owner = kv.get(`${koId}.owner`); - if (owner === undefined) { - throw Error(`unknown kernel object ${koId}`); - } - return owner; - } - - /** - * Expunge a kernel object from the kernel's persistent state. - * - * @param koId - The KRef of the kernel object to delete. - */ - function deleteKernelObject(koId: KRef): void { - kv.delete(`${koId}.owner`); - kv.delete(refCountKey(koId)); - } - - /** - * Obtain a KRef for the next unallocated kernel promise. - * - * @returns The next kpid use. - */ - function getNextPromiseId(): KRef { - return makeKernelSlot('promise', incCounter(nextPromiseId)); - } - - /** - * Create a new, unresolved kernel promise. The new promise will be born with - * a reference count of 1 on the assumption that the promise has just been - * imported from somewhere. - * - * @returns A tuple of the new promise's KRef and an object describing the - * new promise itself. - */ - function initKernelPromise(): [KRef, KernelPromise] { - const kpr: KernelPromise = { - state: 'unresolved', - subscribers: [], - }; - const kpid = getNextPromiseId(); - createStoredQueue(kpid, false); - kv.set(`${kpid}.state`, 'unresolved'); - kv.set(`${kpid}.subscribers`, '[]'); - kv.set(refCountKey(kpid), '1'); - return [kpid, kpr]; - } - - /** - * Append a message to a promise's message queue. - * - * @param kpid - The KRef of the promise to enqueue on. - * @param message - The message to enqueue. - */ - function enqueuePromiseMessage(kpid: KRef, message: Message): void { - provideStoredQueue(kpid, false).enqueue(message); - } - - /** - * Add a new subscriber to a kernel promise's collection of subscribers. - * - * @param vatId - The vat that is subscribing. - * @param kpid - The KRef of the promise being subscribed to. - */ - function addPromiseSubscriber(vatId: VatId, kpid: KRef): void { - insistVatId(vatId); - const kp = getKernelPromise(kpid); - kp.state === 'unresolved' || - Fail`attempt to add subscriber to resolved promise ${kpid}`; - const tempSet = new Set(kp.subscribers); - tempSet.add(vatId); - const newSubscribers = Array.from(tempSet).sort(); - const key = `${kpid}.subscribers`; - kv.set(key, JSON.stringify(newSubscribers)); - } - - /** - * Assign a kernel promise's decider. - * - * @param kpid - The KRef of promise whose decider is being set. - * @param vatId - The vat which will become the decider. - */ - function setPromiseDecider(kpid: KRef, vatId: VatId): void { - insistVatId(vatId); - if (kpid) { - kv.set(`${kpid}.decider`, vatId); - } - } - - /** - * Fetch the descriptive record for a kernel promise. - * - * @param kpid - The KRef of the kernel promise of interest. - * @returns An object describing the requested kernel promise. - */ - function getKernelPromise(kpid: KRef): KernelPromise { - const { context, isPromise } = parseRef(kpid); - assert(context === 'kernel' && isPromise); - const state = kv.get(`${kpid}.state`) as PromiseState; - if (state === undefined) { - throw Error(`unknown kernel promise ${kpid}`); - } - const result: KernelPromise = { state }; - switch (state as string) { - case 'unresolved': { - const decider = kv.get(`${kpid}.decider`); - if (decider !== '' && decider !== undefined) { - result.decider = decider; - } - const subscribers = kv.getRequired(`${kpid}.subscribers`); - result.subscribers = JSON.parse(subscribers); - break; - } - case 'fulfilled': - case 'rejected': { - result.value = JSON.parse(kv.getRequired(`${kpid}.value`)); - break; - } - default: - throw Error(`unknown state for ${kpid}: ${state}`); - } - return result; - } - - /** - * Fetch the messages in a kernel promise's message queue. - * - * @param kpid - The KRef of the kernel promise of interest. - * @returns An array of all the messages in the given promise's message queue. - */ - function getKernelPromiseMessageQueue(kpid: KRef): Message[] { - const result: Message[] = []; - const queue = provideStoredQueue(kpid, false); - for (;;) { - const message = queue.dequeue() as Message; - if (message) { - result.push(message); - } else { - return result; - } - } - } - - /** - * Record the resolution of a kernel promise. - * - * @param kpid - The ref of the promise being resolved. - * @param rejected - True if the promise is being rejected, false if fulfilled. - * @param value - The value the promise is being fulfilled to or rejected with. - */ - function resolveKernelPromise( - kpid: KRef, - rejected: boolean, - value: CapData, - ): void { - const queue = provideStoredQueue(kpid, false); - for (const message of getKernelPromiseMessageQueue(kpid)) { - queue.enqueue(message); - } - kv.set(`${kpid}.state`, rejected ? 'rejected' : 'fulfilled'); - kv.set(`${kpid}.value`, JSON.stringify(value)); - kv.delete(`${kpid}.decider`); - kv.delete(`${kpid}.subscribers`); - } - - /** - * Expunge a kernel promise from the kernel's persistent state. - * - * @param kpid - The KRef of the kernel promise to delete. - */ - function deleteKernelPromise(kpid: KRef): void { - kv.delete(`${kpid}.state`); - kv.delete(`${kpid}.decider`); - kv.delete(`${kpid}.subscribers`); - kv.delete(`${kpid}.value`); - kv.delete(refCountKey(kpid)); - provideStoredQueue(kpid).delete(); - } - - /** - * Get the key for the reachable flag and vatSlot for a given endpoint and kref. - * - * @param endpointId - The endpoint for which the reachable flag is being set. - * @param kref - The kref. - * @returns The key for the reachable flag and vatSlot. - */ - function getSlotKey(endpointId: EndpointId, kref: KRef): string { - return `${endpointId}.c.${kref}`; - } - - /** - * Look up the ERef that and endpoint's c-list maps a KRef to. - * - * @param endpointId - The endpoint in question. - * @param eref - The ERef to look up. - * @returns The KRef corresponding to `eref` in the given endpoints c-list, or undefined - * if there is no such mapping. - */ - function erefToKref(endpointId: EndpointId, eref: ERef): KRef | undefined { - return kv.get(getSlotKey(endpointId, eref)); - } - - /** - * Look up the KRef that and endpoint's c-list maps an ERef to. - * - * @param endpointId - The endpoint in question. - * @param kref - The KRef to look up. - * @returns The given endpoint's ERef corresponding to `kref`, or undefined if - * there is no such mapping. - */ - function krefToEref(endpointId: EndpointId, kref: KRef): ERef | undefined { - const key = getSlotKey(endpointId, kref); - const data = kv.get(key); - if (!data) { - return undefined; - } - const { vatSlot } = parseReachableAndVatSlot(data); - return vatSlot; - } - - /** - * Remove an entry from an endpoint's c-list given an eref. - * - * @param endpointId - The endpoint whose c-list entry is to be removed. - * @param eref - The ERef. - */ - function forgetEref(endpointId: EndpointId, eref: ERef): void { - const kref = erefToKref(endpointId, eref); - if (kref) { - deleteClistEntry(endpointId, kref, eref); - } - } - - /** - * Remove an entry from an endpoint's c-list given a kref. - * - * @param endpointId - The endpoint whose c-list entry is to be removed. - * @param kref - The Kref. - */ - function forgetKref(endpointId: EndpointId, kref: KRef): void { - const eref = krefToEref(endpointId, kref); - if (eref) { - deleteClistEntry(endpointId, kref, eref); - } - } + const baseStore = makeBaseStore(kv); + const idStore = makeIdStore(kv, baseStore); + const queueStore = makeQueueStore(kv, baseStore); + const refCountStore = makeRefCountStore(kv); + const objectStore = makeObjectStore(kv, baseStore, refCountStore); + const promiseStore = makePromiseStore( + kv, + baseStore, + refCountStore, + queueStore, + ); + const gcStore = makeGCStore(kv, baseStore, refCountStore, objectStore); + const cListStore = makeCListStore( + kv, + baseStore, + gcStore, + objectStore, + refCountStore, + ); /** * Delete everything from the database. @@ -784,317 +128,23 @@ export function makeKernelStore(kdb: KernelDatabase) { */ function reset(): void { kdb.clear(); - runQueue = createStoredQueue('run', true); - nextVatId = provideCachedStoredValue('nextVatId', '1'); - nextRemoteId = provideCachedStoredValue('nextRemoteId', '1'); - nextObjectId = provideCachedStoredValue('nextObjectId', '1'); - nextPromiseId = provideCachedStoredValue('nextPromiseId', '1'); - gcActions = provideCachedStoredValue('gcActions', '[]'); - reapQueue = provideCachedStoredValue('reapQueue', '[]'); - } - - /** - * Check if a kernel object exists in the kernel's persistent state. - * - * @param kref - The KRef of the kernel object in question. - * @returns True if the kernel object exists, false otherwise. - */ - function kernelObjectExists(kref: KRef): boolean { - return Boolean(kv.get(refCountKey(kref))); - } - - /** - * Check if a kernel object is reachable. - * - * @param endpointId - The endpoint for which the reachable flag is being checked. - * @param kref - The kref. - * @returns True if the kernel object is reachable, false otherwise. - */ - function getReachableFlag(endpointId: EndpointId, kref: KRef): boolean { - const key = getSlotKey(endpointId, kref); - const data = kv.getRequired(key); - const { isReachable } = parseReachableAndVatSlot(data); - return isReachable; - } - - /** - * Clear the reachable flag for a given endpoint and kref. - * - * @param endpointId - The endpoint for which the reachable flag is being cleared. - * @param kref - The kref. - */ - function clearReachableFlag(endpointId: EndpointId, kref: KRef): void { - const key = getSlotKey(endpointId, kref); - const { isReachable, vatSlot } = parseReachableAndVatSlot( - kv.getRequired(key), - ); - kv.set(key, buildReachableAndVatSlot(false, vatSlot)); - const { direction, isPromise } = parseRef(vatSlot); - // decrement 'reachable' part of refcount, but only for object imports - if ( - isReachable && - !isPromise && - direction === 'import' && - kernelObjectExists(kref) - ) { - const counts = getObjectRefCount(kref); - counts.reachable -= 1; - setObjectRefCount(kref, counts); - if (counts.reachable === 0) { - maybeFreeKrefs.add(kref); - } - } - } - - /** - * Increment the reference count associated with some kernel object. - * - * We track references to promises and objects, but not devices. Promises - * have only a "reachable" count, whereas objects track both "reachable" - * and "recognizable" counts. - * - * @param kref - The kernel slot whose refcount is to be incremented. - * @param options - Options for the increment. - * @param options.isExport - True if the reference comes from a clist export, which counts for promises but not objects. - * @param options.onlyRecognizable - True if the reference provides only recognition, not reachability. - */ - function incrementRefCount( - kref: KRef, - { - isExport = false, - onlyRecognizable = false, - }: { isExport?: boolean; onlyRecognizable?: boolean }, - ): void { - kref || Fail`incrementRefCount called with empty kref`; - - const { isPromise } = parseRef(kref); - if (isPromise) { - const refCount = Number(kv.get(refCountKey(kref))) + 1; - kv.set(refCountKey(kref), `${refCount}`); - return; - } - - // If `isExport` the reference comes from a clist export, which counts for promises but not objects - if (isExport) { - return; - } - - const counts = getObjectRefCount(kref); - if (!onlyRecognizable) { - counts.reachable += 1; - } - counts.recognizable += 1; - setObjectRefCount(kref, counts); - } - - /** - * Decrement the reference count associated with some kernel object. - * - * @param kref - The kernel slot whose refcount is to be decremented. - * @param options - Options for the decrement. - * @param options.isExport - True if the reference comes from a clist export, which counts for promises but not objects. - * @param options.onlyRecognizable - True if the reference provides only recognition, not reachability. - * @returns True if the reference count has been decremented to zero, false if it is still non-zero. - * @throws if this tries to decrement the reference count below zero. - */ - function decrementRefCount( - kref: KRef, - { - isExport = false, - onlyRecognizable = false, - }: { isExport?: boolean; onlyRecognizable?: boolean }, - ): boolean { - kref || Fail`decrementRefCount called with empty kref`; - - const { isPromise } = parseRef(kref); - if (isPromise) { - let refCount = Number(kv.get(refCountKey(kref))); - refCount > 0 || Fail`refCount underflow ${kref}`; - refCount -= 1; - kv.set(refCountKey(kref), `${refCount}`); - if (refCount === 0) { - maybeFreeKrefs.add(kref); - return true; - } - return false; - } - - if (isExport || !kernelObjectExists(kref)) { - return false; - } - - const counts = getObjectRefCount(kref); - if (!onlyRecognizable) { - counts.reachable -= 1; - } - counts.recognizable -= 1; - if (!counts.reachable || !counts.recognizable) { - maybeFreeKrefs.add(kref); - } - setObjectRefCount(kref, counts); - - return false; - } - - /** - * Add an entry to an endpoint's c-list, creating a new bidirectional mapping - * between an ERef belonging to the endpoint and a KRef belonging to the - * kernel. - * - * @param endpointId - The endpoint whose c-list is to be added to. - * @param kref - The KRef. - * @param eref - The ERef. - */ - function addClistEntry(endpointId: EndpointId, kref: KRef, eref: ERef): void { - kv.set(getSlotKey(endpointId, kref), buildReachableAndVatSlot(true, eref)); - kv.set(getSlotKey(endpointId, eref), kref); - } - - /** - * Test if there's a c-list entry for some slot. - * - * @param endpointId - The endpoint of interest - * @param slot - The slot of interest - * @returns true iff this vat has a c-list entry mapping for `slot`. - */ - function hasCListEntry(endpointId: EndpointId, slot: string): boolean { - return kv.get(getSlotKey(endpointId, slot)) !== undefined; - } - - /** - * Remove an entry from an endpoint's c-list. - * - * @param endpointId - The endpoint whose c-list entry is to be removed. - * @param kref - The KRef. - * @param eref - The ERef. - */ - function deleteClistEntry( - endpointId: EndpointId, - kref: KRef, - eref: ERef, - ): void { - const kernelKey = getSlotKey(endpointId, kref); - const vatKey = getSlotKey(endpointId, eref); - assert(kv.get(kernelKey)); - clearReachableFlag(endpointId, kref); - const { direction } = parseRef(eref); - decrementRefCount(kref, { - isExport: direction === 'export', - onlyRecognizable: true, - }); - kv.delete(kernelKey); - kv.delete(vatKey); - } - - /** - * Get the set of GC actions to perform. - * - * @returns The set of GC actions to perform. - */ - function getGCActions(): Set { - return new Set(JSON.parse(gcActions.get() ?? '[]')); - } - - /** - * Set the set of GC actions to perform. - * - * @param actions - The set of GC actions to perform. - */ - function setGCActions(actions: Set): void { - const a = Array.from(actions); - a.sort(); - gcActions.set(JSON.stringify(a)); - } - - /** - * Add a new GC action to the set of GC actions to perform. - * - * @param newActions - The new GC action to add. - */ - function addGCActions(newActions: GCAction[]): void { - const actions = getGCActions(); - for (const action of newActions) { - assert.typeof(action, 'string', 'addGCActions given bad action'); - const [vatId, type, kref] = action.split(' '); - insistVatId(vatId); - insistGCActionType(type); - insistKernelType('object', kref); - actions.add(action); - } - setGCActions(actions); - } - - /** - * Schedule a vat for reaping. - * - * @param vatId - The vat to schedule for reaping. - */ - function scheduleReap(vatId: VatId): void { - const queue = JSON.parse(reapQueue.get() ?? '[]'); - if (!queue.includes(vatId)) { - queue.push(vatId); - reapQueue.set(JSON.stringify(queue)); - } - } - - /** - * Get the next reap action. - * - * @returns The next reap action, or undefined if the queue is empty. - */ - function nextReapAction(): RunQueueItemBringOutYourDead | undefined { - const queue = JSON.parse(reapQueue.get() ?? '[]'); - if (queue.length > 0) { - const vatId = queue.shift(); - reapQueue.set(JSON.stringify(queue)); - return harden({ type: RunQueueItemType.bringOutYourDead, vatId }); - } - return undefined; + queueStore.reset(); + objectStore.reset(); + promiseStore.reset(); + gcStore.reset(); + idStore.reset(); } return harden({ - enqueueRun, - dequeueRun, - runQueueLength, - getNextVatId, - getNextRemoteId, - initEndpoint, - getRefCount, - incRefCount, - decRefCount, - getObjectRefCount, - setObjectRefCount, - incrementRefCount, - decrementRefCount, - initKernelObject, - kernelObjectExists, - getReachableFlag, - clearReachableFlag, - getOwner, - deleteKernelObject, - initKernelPromise, - getKernelPromise, - enqueuePromiseMessage, - setPromiseDecider, - getKernelPromiseMessageQueue, - resolveKernelPromise, - deleteKernelPromise, - addPromiseSubscriber, - addClistEntry, - hasCListEntry, - deleteClistEntry, - erefToKref, - allocateErefForKref, - krefToEref, - forgetEref, - forgetKref, - clear, + ...idStore, + ...queueStore, + ...refCountStore, + ...objectStore, + ...promiseStore, + ...gcStore, + ...cListStore, makeVatStore, - getGCActions, - setGCActions, - addGCActions, - scheduleReap, - nextReapAction, + clear, reset, kv, }); diff --git a/packages/kernel/src/store/object-store.ts b/packages/kernel/src/store/object-store.ts new file mode 100644 index 000000000..f1fe2989f --- /dev/null +++ b/packages/kernel/src/store/object-store.ts @@ -0,0 +1,133 @@ +import { Fail } from '@endo/errors'; +import type { KVStore } from '@ocap/store'; + +import type { makeBaseStore } from './base-store.ts'; +import type { makeRefCountStore } from './refcount-store.ts'; +import { makeKernelSlot } from './utils/kernel-slots.ts'; +import type { EndpointId, KRef } from '../types.ts'; + +/** + * Create an object store object that provides functionality for managing kernel objects. + * + * @param kv - The key-value store to use for persistent storage. + * @param baseStore - The base store to use for the object store. + * @param refCountStore - The refcount store to use for the object store. + * @returns An object store object that maps various persistent kernel data + * structures onto `kv`. + */ +// eslint-disable-next-line @typescript-eslint/explicit-function-return-type +export function makeObjectStore( + kv: KVStore, + baseStore: ReturnType, + refCountStore: ReturnType, +) { + /** Counter for allocating kernel object IDs */ + let nextObjectId = baseStore.provideCachedStoredValue('nextObjectId', '1'); + + /** + * Create a new kernel object. The new object will be born with reference and + * recognizability counts of 1, on the assumption that the new object + * corresponds to an object that has just been imported from somewhere. + * + * @param owner - The endpoint that is the owner of the new object. + * @returns The new object's KRef. + */ + function initKernelObject(owner: EndpointId): KRef { + const koId = getNextObjectId(); + kv.set(`${koId}.owner`, owner); + setObjectRefCount(koId, { reachable: 1, recognizable: 1 }); + return koId; + } + + /** + * Get a kernel object's owner. + * + * @param koId - The KRef of the kernel object of interest. + * @returns The identity of the vat or remote that owns the object. + */ + function getOwner(koId: KRef): EndpointId { + const owner = kv.get(`${koId}.owner`); + if (owner === undefined) { + throw Error(`unknown kernel object ${koId}`); + } + return owner; + } + + /** + * Expunge a kernel object from the kernel's persistent state. + * + * @param koId - The KRef of the kernel object to delete. + */ + function deleteKernelObject(koId: KRef): void { + kv.delete(`${koId}.owner`); + kv.delete(refCountStore.refCountKey(koId)); + } + + /** + * Obtain a KRef for the next unallocated kernel object. + * + * @returns The next koId use. + */ + function getNextObjectId(): KRef { + return makeKernelSlot('object', baseStore.incCounter(nextObjectId)); + } + + /** + * Get the reference counts for a kernel object + * + * @param kref - The KRef of the object of interest. + * @returns The reference counts for the object. + */ + function getObjectRefCount(kref: KRef): { + reachable: number; + recognizable: number; + } { + const data = kv.get(refCountStore.refCountKey(kref)); + if (!data) { + return { reachable: 0, recognizable: 0 }; + } + const [reachable = 0, recognizable = 0] = data.split(',').map(Number); + reachable <= recognizable || + Fail`refMismatch(get) ${kref} ${reachable},${recognizable}`; + return { reachable, recognizable }; + } + + /** + * Set the reference counts for a kernel object + * + * @param kref - The KRef of the object of interest. + * @param counts - The reference counts to set. + * @param counts.reachable - The reachable reference count. + * @param counts.recognizable - The recognizable reference count. + */ + function setObjectRefCount( + kref: KRef, + counts: { reachable: number; recognizable: number }, + ): void { + const { reachable, recognizable } = counts; + assert.typeof(reachable, 'number'); + assert.typeof(recognizable, 'number'); + (reachable >= 0 && recognizable >= 0) || + Fail`${kref} underflow ${reachable},${recognizable}`; + reachable <= recognizable || + Fail`refMismatch(set) ${kref} ${reachable},${recognizable}`; + kv.set(refCountStore.refCountKey(kref), `${reachable},${recognizable}`); + } + + /** + * Reset the object store. + */ + function reset(): void { + nextObjectId = baseStore.provideCachedStoredValue('nextObjectId', '1'); + } + + return { + initKernelObject, + getOwner, + deleteKernelObject, + getNextObjectId, + getObjectRefCount, + setObjectRefCount, + reset, + }; +} diff --git a/packages/kernel/src/store/promise-store.ts b/packages/kernel/src/store/promise-store.ts new file mode 100644 index 000000000..9ea0eb032 --- /dev/null +++ b/packages/kernel/src/store/promise-store.ts @@ -0,0 +1,221 @@ +import type { Message } from '@agoric/swingset-liveslots'; +import { Fail } from '@endo/errors'; +import type { CapData } from '@endo/marshal'; +import type { KVStore } from '@ocap/store'; + +import type { makeBaseStore } from './base-store'; +import type { makeQueueStore } from './queue-store.ts'; +import type { makeRefCountStore } from './refcount-store'; +import { makeKernelSlot } from './utils/kernel-slots'; +import { parseRef } from './utils/parse-ref'; +import type { KRef, KernelPromise, PromiseState, VatId } from '../types.ts'; +import { insistVatId } from '../types.ts'; + +/** + * Create a promise store object that provides functionality for managing kernel promises. + * + * @param kv - The key-value store to use for persistent storage. + * @param baseStore - The base store to use for the promise store. + * @param refCountStore - The refcount store to use for the promise store. + * @param queueStore - The queue store to use for the promise store. + * @returns A promise store object that maps various persistent kernel data + * structures onto `kv`. + */ +// eslint-disable-next-line @typescript-eslint/explicit-function-return-type +export function makePromiseStore( + kv: KVStore, + baseStore: ReturnType, + refCountStore: ReturnType, + queueStore: ReturnType, +) { + /** Counter for allocating kernel promise IDs */ + let nextPromiseId = baseStore.provideCachedStoredValue('nextPromiseId', '1'); + + /** + * Create a new, unresolved kernel promise. The new promise will be born with + * a reference count of 1 on the assumption that the promise has just been + * imported from somewhere. + * + * @returns A tuple of the new promise's KRef and an object describing the + * new promise itself. + */ + function initKernelPromise(): [KRef, KernelPromise] { + const kpr: KernelPromise = { + state: 'unresolved', + subscribers: [], + }; + const kpid = getNextPromiseId(); + queueStore.createStoredQueue(kpid, false); + kv.set(`${kpid}.state`, 'unresolved'); + kv.set(`${kpid}.subscribers`, '[]'); + kv.set(refCountStore.refCountKey(kpid), '1'); + return [kpid, kpr]; + } + + /** + * Fetch the descriptive record for a kernel promise. + * + * @param kpid - The KRef of the kernel promise of interest. + * @returns An object describing the requested kernel promise. + */ + function getKernelPromise(kpid: KRef): KernelPromise { + const { context, isPromise } = parseRef(kpid); + assert(context === 'kernel' && isPromise); + const state = kv.get(`${kpid}.state`) as PromiseState; + if (state === undefined) { + throw Error(`unknown kernel promise ${kpid}`); + } + const result: KernelPromise = { state }; + switch (state as string) { + case 'unresolved': { + const decider = kv.get(`${kpid}.decider`); + if (decider !== '' && decider !== undefined) { + result.decider = decider; + } + const subscribers = kv.getRequired(`${kpid}.subscribers`); + result.subscribers = JSON.parse(subscribers); + break; + } + case 'fulfilled': + case 'rejected': { + result.value = JSON.parse(kv.getRequired(`${kpid}.value`)); + break; + } + default: + throw Error(`unknown state for ${kpid}: ${state}`); + } + return result; + } + + /** + * Expunge a kernel promise from the kernel's persistent state. + * + * @param kpid - The KRef of the kernel promise to delete. + */ + function deleteKernelPromise(kpid: KRef): void { + kv.delete(`${kpid}.state`); + kv.delete(`${kpid}.decider`); + kv.delete(`${kpid}.subscribers`); + kv.delete(`${kpid}.value`); + kv.delete(refCountStore.refCountKey(kpid)); + queueStore.provideStoredQueue(kpid).delete(); + } + + /** + * Obtain a KRef for the next unallocated kernel promise. + * + * @returns The next kpid use. + */ + function getNextPromiseId(): KRef { + return makeKernelSlot('promise', baseStore.incCounter(nextPromiseId)); + } + + /** + * Add a new subscriber to a kernel promise's collection of subscribers. + * + * @param vatId - The vat that is subscribing. + * @param kpid - The KRef of the promise being subscribed to. + */ + function addPromiseSubscriber(vatId: VatId, kpid: KRef): void { + insistVatId(vatId); + const kp = getKernelPromise(kpid); + kp.state === 'unresolved' || + Fail`attempt to add subscriber to resolved promise ${kpid}`; + const tempSet = new Set(kp.subscribers); + tempSet.add(vatId); + const newSubscribers = Array.from(tempSet).sort(); + const key = `${kpid}.subscribers`; + kv.set(key, JSON.stringify(newSubscribers)); + } + + /** + * Assign a kernel promise's decider. + * + * @param kpid - The KRef of promise whose decider is being set. + * @param vatId - The vat which will become the decider. + */ + function setPromiseDecider(kpid: KRef, vatId: VatId): void { + insistVatId(vatId); + if (kpid) { + kv.set(`${kpid}.decider`, vatId); + } + } + + /** + * Record the resolution of a kernel promise. + * + * @param kpid - The ref of the promise being resolved. + * @param rejected - True if the promise is being rejected, false if fulfilled. + * @param value - The value the promise is being fulfilled to or rejected with. + */ + function resolveKernelPromise( + kpid: KRef, + rejected: boolean, + value: CapData, + ): void { + const queue = queueStore.provideStoredQueue(kpid, false); + for (const message of getKernelPromiseMessageQueue(kpid)) { + queue.enqueue(message); + } + kv.set(`${kpid}.state`, rejected ? 'rejected' : 'fulfilled'); + kv.set(`${kpid}.value`, JSON.stringify(value)); + kv.delete(`${kpid}.decider`); + kv.delete(`${kpid}.subscribers`); + } + + /** + * Append a message to a promise's message queue. + * + * @param kpid - The KRef of the promise to enqueue on. + * @param message - The message to enqueue. + */ + function enqueuePromiseMessage(kpid: KRef, message: Message): void { + queueStore.provideStoredQueue(kpid, false).enqueue(message); + } + + /** + * Fetch the messages in a kernel promise's message queue. + * + * @param kpid - The KRef of the kernel promise of interest. + * @returns An array of all the messages in the given promise's message queue. + */ + function getKernelPromiseMessageQueue(kpid: KRef): Message[] { + const result: Message[] = []; + const queue = queueStore.provideStoredQueue(kpid, false); + for (;;) { + const message = queue.dequeue() as Message; + if (message) { + result.push(message); + } else { + return result; + } + } + } + + /** + * + */ + function reset(): void { + nextPromiseId = baseStore.provideCachedStoredValue('nextPromiseId', '1'); + } + + return { + // Promise lifecycle + initKernelPromise, + getKernelPromise, + deleteKernelPromise, + getNextPromiseId, + + // Promise state management + addPromiseSubscriber, + setPromiseDecider, + resolveKernelPromise, + + // Promise messaging + enqueuePromiseMessage, + getKernelPromiseMessageQueue, + + // Reset + reset, + }; +} diff --git a/packages/kernel/src/store/queue-store.ts b/packages/kernel/src/store/queue-store.ts new file mode 100644 index 000000000..13e2153e9 --- /dev/null +++ b/packages/kernel/src/store/queue-store.ts @@ -0,0 +1,171 @@ +import type { KVStore } from '@ocap/store'; + +import type { makeBaseStore } from './base-store.ts'; +import type { RunQueueItem } from '../types.ts'; + +export type StoredQueue = { + enqueue(item: object): void; + dequeue(): object | undefined; + delete(): void; +}; + +/** + * Create a queue store object that provides functionality for managing queues. + * + * @param kv - The key-value store to use for persistent storage. + * @param baseStore - The base store to use for the queue store. + * @returns A queue store object that maps various persistent kernel data + * structures onto `kv`. + */ +// eslint-disable-next-line @typescript-eslint/explicit-function-return-type +export function makeQueueStore( + kv: KVStore, + baseStore: ReturnType, +) { + /** The kernel's run queue. */ + let runQueue = createStoredQueue('run', true); + + /** + * Create a new (empty) persistently stored queue. + * + * @param queueName - The name for the queue (must be unique among queues). + * @param cached - Optional flag: set to true if the queue should cache its + * @returns An object for interacting with the new queue. + */ + function createStoredQueue( + queueName: string, + cached: boolean = false, + ): StoredQueue { + const qk = `queue.${queueName}`; + kv.set(`${qk}.head`, '1'); + kv.set(`${qk}.tail`, '1'); + return provideStoredQueue(queueName, cached); + } + + /** + * Produce an object to access a persistently stored queue. + * + * @param queueName - The name for the queue (must be unique among queues). + * @param cached - Optional flag: set to true if the queue should cache its + * limit indices in memory (only do this if the queue is going to be accessed or + * checked frequently). + * @returns An object for interacting with the queue. + */ + function provideStoredQueue( + queueName: string, + cached: boolean = false, + ): StoredQueue { + const qk = `queue.${queueName}`; + // Note: cached=true ==> caches only the head & tail indices, NOT the queue entries themselves + const provideValue = cached + ? baseStore.provideCachedStoredValue + : baseStore.provideRawStoredValue; + const head = provideValue(`${qk}.head`); + const tail = provideValue(`${qk}.tail`); + if (head.get() === undefined || tail.get() === undefined) { + throw Error(`queue ${queueName} not initialized`); + } + return { + enqueue(item: object): void { + if (head.get() === undefined) { + throw Error(`enqueue into deleted queue ${queueName}`); + } + const entryPos = baseStore.incCounter(head); + kv.set(`${qk}.${entryPos}`, JSON.stringify(item)); + }, + dequeue(): object | undefined { + const headPos = head.get(); + if (headPos === undefined) { + return undefined; + } + const tailPos = tail.get(); + if (tailPos !== headPos) { + const entry = kv.getRequired(`${qk}.${tailPos}`); + kv.delete(`${qk}.${tailPos}`); + baseStore.incCounter(tail); + return JSON.parse(entry) as object; + } + return undefined; + }, + delete(): void { + const headPos = head.get(); + if (headPos !== undefined) { + let tailPos = tail.get(); + while (tailPos !== headPos) { + kv.delete(`${qk}.${tailPos}`); + tailPos = `${Number(tailPos) + 1}`; + } + head.delete(); + tail.delete(); + } + }, + }; + } + + /** + * Append a message to the kernel's run queue. + * + * @param message - The message to enqueue. + */ + function enqueueRun(message: RunQueueItem): void { + runQueue.enqueue(message); + } + + /** + * Find out how long some queue is. + * + * @param queueName - The name of the queue of interest. + * + * @returns the number of items in the given queue. + */ + function getQueueLength(queueName: string): number { + const qk = `queue.${queueName}`; + const head = kv.get(`${qk}.head`); + const tail = kv.get(`${qk}.tail`); + if (head === undefined || tail === undefined) { + throw Error(`unknown queue ${queueName}`); + } + return Number(head) - Number(tail); + } + + /** + * Fetch the next message on the kernel's run queue. + * + * @returns The next message on the run queue, or undefined if the queue is + * empty. + */ + function dequeueRun(): RunQueueItem | undefined { + return runQueue.dequeue() as RunQueueItem | undefined; + } + + /** + * Obtain the number of entries in the run queue. + * + * @returns the number of items in the run queue. + */ + function runQueueLength(): number { + return getQueueLength('run'); + } + + /** + * + */ + function reset(): void { + runQueue = createStoredQueue('run', true); + } + + return { + // Queue + createStoredQueue, + provideStoredQueue, + getQueueLength, + + // Run queue operations + enqueueRun, + dequeueRun, + runQueueLength, + + // Reset + reset, + }; +} diff --git a/packages/kernel/src/store/refcount-store.ts b/packages/kernel/src/store/refcount-store.ts new file mode 100644 index 000000000..e62327740 --- /dev/null +++ b/packages/kernel/src/store/refcount-store.ts @@ -0,0 +1,77 @@ +import type { KVStore } from '@ocap/store'; + +import type { KRef } from '../types.ts'; + +/** + * Create a refcount store object that provides functionality for managing reference counts. + * + * @param kv - The key-value store to use for persistent storage. + * @returns A refcount store object that maps various persistent kernel data + * structures onto `kv`. + */ +// eslint-disable-next-line @typescript-eslint/explicit-function-return-type +export function makeRefCountStore(kv: KVStore) { + /** + * Generate the storage key for a kernel entity's reference count. + * + * @param kref - The KRef of interest. + * @returns the key to store the indicated reference count at. + */ + function refCountKey(kref: KRef): string { + return `${kref}.refCount`; + } + + /** + * Get a kernel entity's reference count. + * + * @param kref - The KRef of interest. + * @returns the reference count of the indicated kernel entity. + */ + function getRefCount(kref: KRef): number { + return Number(kv.get(refCountKey(kref))); + } + + /** + * Increment a kernel entity's reference count. + * + * @param kref - The KRef of the entity to increment the ref count of. + * @returns the new reference count after incrementing. + */ + function incRefCount(kref: KRef): number { + const key = refCountKey(kref); + const newCount = Number(kv.get(key)) + 1; + kv.set(key, `${newCount}`); + return newCount; + } + + /** + * Decrement a kernel entity's reference count. + * + * @param kref - The KRef of the entity to decrement the ref count of. + * @returns the new reference count after decrementing. + */ + function decRefCount(kref: KRef): number { + const key = refCountKey(kref); + const newCount = Number(kv.get(key)) - 1; + kv.set(key, `${newCount}`); + return newCount; + } + + /** + * Check if a kernel object exists in the kernel's persistent state. + * + * @param kref - The KRef of the kernel object in question. + * @returns True if the kernel object exists, false otherwise. + */ + function kernelRefExists(kref: KRef): boolean { + return Boolean(kv.get(refCountKey(kref))); + } + + return { + refCountKey, + getRefCount, + incRefCount, + decRefCount, + kernelRefExists, + }; +} diff --git a/packages/kernel/src/store/kernel-slots.ts b/packages/kernel/src/store/utils/kernel-slots.ts similarity index 97% rename from packages/kernel/src/store/kernel-slots.ts rename to packages/kernel/src/store/utils/kernel-slots.ts index 35072e8cd..341288b88 100644 --- a/packages/kernel/src/store/kernel-slots.ts +++ b/packages/kernel/src/store/utils/kernel-slots.ts @@ -1,4 +1,4 @@ -import { assert, Fail } from '../assert.ts'; +import { assert, Fail } from '../../utils/assert.ts'; // Object/promise references (in the kernel) contain a two-tuple of (type, // index). All object references point to entries in the kernel Object diff --git a/packages/kernel/src/store/kernel-store.test.ts b/packages/kernel/src/store/utils/kernel-store.test.ts similarity index 97% rename from packages/kernel/src/store/kernel-store.test.ts rename to packages/kernel/src/store/utils/kernel-store.test.ts index d01cd900a..3b9f61db5 100644 --- a/packages/kernel/src/store/kernel-store.test.ts +++ b/packages/kernel/src/store/utils/kernel-store.test.ts @@ -2,9 +2,9 @@ import type { Message } from '@agoric/swingset-liveslots'; import type { KernelDatabase } from '@ocap/store'; import { describe, it, expect, beforeEach } from 'vitest'; -import { makeKernelStore } from './kernel-store.ts'; -import { makeMapKernelDatabase } from '../../test/storage.ts'; -import type { RunQueueItem } from '../types.ts'; +import { makeMapKernelDatabase } from '../../../test/storage.ts'; +import type { RunQueueItem } from '../../types.ts'; +import { makeKernelStore } from '../kernel-store.ts'; /** * Mock Message: A stupid TS hack to allow trivial use of plain strings as if they diff --git a/packages/kernel/src/store/parse-ref.ts b/packages/kernel/src/store/utils/parse-ref.ts similarity index 100% rename from packages/kernel/src/store/parse-ref.ts rename to packages/kernel/src/store/utils/parse-ref.ts diff --git a/packages/kernel/src/store/utils/promise-ref.ts b/packages/kernel/src/store/utils/promise-ref.ts new file mode 100644 index 000000000..cae418daa --- /dev/null +++ b/packages/kernel/src/store/utils/promise-ref.ts @@ -0,0 +1,12 @@ +import type { KRef } from '../../types.ts'; + +/** + * Test if a KRef designates a promise. + * + * @param kref - The KRef to test. + * + * @returns true iff the given KRef references a promise. + */ +export function isPromiseRef(kref: KRef): boolean { + return kref[1] === 'p'; +} diff --git a/packages/kernel/src/store/reachable.ts b/packages/kernel/src/store/utils/reachable.ts similarity index 95% rename from packages/kernel/src/store/reachable.ts rename to packages/kernel/src/store/utils/reachable.ts index 64b0c7907..9de02dee3 100644 --- a/packages/kernel/src/store/reachable.ts +++ b/packages/kernel/src/store/utils/reachable.ts @@ -1,4 +1,4 @@ -import { assert, Fail } from '../assert.ts'; +import { assert, Fail } from '../../utils/assert.ts'; /** * Parse a string into an object with `isReachable` and `vatSlot` properties. diff --git a/packages/kernel/src/types.ts b/packages/kernel/src/types.ts index 771af0a5d..ae7a10df5 100644 --- a/packages/kernel/src/types.ts +++ b/packages/kernel/src/types.ts @@ -23,8 +23,8 @@ import type { Json } from '@metamask/utils'; import { UnsafeJsonStruct } from '@metamask/utils'; import type { DuplexStream } from '@ocap/streams'; -import { Fail } from './assert.ts'; import type { VatCommandReply, VatCommand } from './messages/vat.ts'; +import { Fail } from './utils/assert.ts'; export type VatId = string; export type RemoteId = string; diff --git a/packages/kernel/src/assert.ts b/packages/kernel/src/utils/assert.ts similarity index 100% rename from packages/kernel/src/assert.ts rename to packages/kernel/src/utils/assert.ts diff --git a/packages/kernel/src/waitUntilQuiescent.ts b/packages/kernel/src/utils/wait-quiescent.ts similarity index 100% rename from packages/kernel/src/waitUntilQuiescent.ts rename to packages/kernel/src/utils/wait-quiescent.ts From 780ef929440d04ef4d8bea74b25ef5cab600408d Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Tue, 11 Mar 2025 16:25:37 -0300 Subject: [PATCH 07/21] fix merge conflict --- packages/kernel/src/VatHandle.ts | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/packages/kernel/src/VatHandle.ts b/packages/kernel/src/VatHandle.ts index ff069f54d..1a1652ea4 100644 --- a/packages/kernel/src/VatHandle.ts +++ b/packages/kernel/src/VatHandle.ts @@ -501,30 +501,6 @@ export class VatHandle { } /** -<<<<<<< HEAD - * Make a 'message' delivery to the vat. - * - * @param target - The VRef of the object to which the message is addressed. - * @param message - The message to deliver. - */ - async deliverMessage(target: VRef, message: Message): Promise { - await this.sendVatCommand({ - method: VatCommandMethod.deliver, - params: ['message', target, message], - }); - } - - /** - * Make a 'notify' delivery to the vat. - * - * @param resolutions - One or more promise resolutions to deliver. - */ - async deliverNotify(resolutions: VatOneResolution[]): Promise { - await this.sendVatCommand({ - method: VatCommandMethod.deliver, - params: ['notify', resolutions], - }); -======= * Initializes the vat. * * @returns A promise that resolves when the vat is initialized. @@ -549,7 +525,6 @@ export class VatHandle { params: this.config, }); this.#logger.debug('Created'); ->>>>>>> 59065de (Implement GC delivery) } /** From 533e69a93eba0f7652911f8998fbf6e32f4afab7 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Tue, 11 Mar 2025 18:31:30 -0300 Subject: [PATCH 08/21] fix them tests --- packages/kernel-test/src/liveslots.test.ts | 2 +- packages/kernel/src/Kernel.ts | 3 - packages/kernel/src/services/syscall.ts | 1 - packages/kernel/src/store/base-store.test.ts | 181 ++++++++++++++++ .../store/{utils => }/kernel-store.test.ts | 121 ++++++++--- packages/kernel/src/store/promise-store.ts | 8 +- .../src/store/utils/kernel-slots.test.ts | 135 ++++++++++++ .../kernel/src/store/utils/parse-ref.test.ts | 205 ++++++++++++++++++ .../src/store/utils/promise-ref.test.ts | 23 ++ .../kernel/src/store/utils/reachable.test.ts | 109 ++++++++++ packages/nodejs/src/index.ts | 2 + vitest.config.ts | 8 +- 12 files changed, 754 insertions(+), 44 deletions(-) create mode 100644 packages/kernel/src/store/base-store.test.ts rename packages/kernel/src/store/{utils => }/kernel-store.test.ts (68%) create mode 100644 packages/kernel/src/store/utils/kernel-slots.test.ts create mode 100644 packages/kernel/src/store/utils/parse-ref.test.ts create mode 100644 packages/kernel/src/store/utils/promise-ref.test.ts create mode 100644 packages/kernel/src/store/utils/reachable.test.ts create mode 100644 packages/nodejs/src/index.ts diff --git a/packages/kernel-test/src/liveslots.test.ts b/packages/kernel-test/src/liveslots.test.ts index 0b0803386..8af65311e 100644 --- a/packages/kernel-test/src/liveslots.test.ts +++ b/packages/kernel-test/src/liveslots.test.ts @@ -8,7 +8,7 @@ import { } from 'node:worker_threads'; import { beforeEach, describe, expect, it } from 'vitest'; -import { kunser } from '../../kernel/src/kernel-marshal.ts'; +import { kunser } from '../../kernel/src/services/kernel-marshal.ts'; import { makeKernel } from '../../nodejs/src/kernel/make-kernel.ts'; const origStdoutWrite = process.stdout.write.bind(process.stdout); diff --git a/packages/kernel/src/Kernel.ts b/packages/kernel/src/Kernel.ts index 8785b5bba..fb7fd2258 100644 --- a/packages/kernel/src/Kernel.ts +++ b/packages/kernel/src/Kernel.ts @@ -192,21 +192,18 @@ export class Kernel { */ async *#runQueueItems(): AsyncGenerator { for (;;) { - // Check for GC actions regardless of queue state const gcAction = processGCActionSet(this.#storage); if (gcAction) { yield gcAction; continue; } - // Check for reap actions regardless of queue state const reapAction = this.#storage.nextReapAction(); if (reapAction) { yield reapAction; continue; } - // Process regular queue items if any exist while (this.#runQueueLength > 0) { const item = this.#dequeueRun(); if (item) { diff --git a/packages/kernel/src/services/syscall.ts b/packages/kernel/src/services/syscall.ts index 9132d1579..ede62bcf8 100644 --- a/packages/kernel/src/services/syscall.ts +++ b/packages/kernel/src/services/syscall.ts @@ -40,7 +40,6 @@ function makeSupervisorSyscall( insistVatSyscallObject(vso); let syscallResult; try { - console.log('doSyscall', vso); syscallResult = supervisor.executeSyscall(vso); } catch (problem) { console.warn(`supervisor got error during syscall:`, problem); diff --git a/packages/kernel/src/store/base-store.test.ts b/packages/kernel/src/store/base-store.test.ts new file mode 100644 index 000000000..bd86397a4 --- /dev/null +++ b/packages/kernel/src/store/base-store.test.ts @@ -0,0 +1,181 @@ +import type { KVStore } from '@ocap/store'; +import { describe, it, expect, beforeEach } from 'vitest'; + +import { makeBaseStore } from './base-store.ts'; +import { makeMapKVStore } from '../../test/storage.ts'; + +describe('base-store', () => { + let mockKVStore: KVStore; + let baseStore: ReturnType; + + beforeEach(() => { + mockKVStore = makeMapKVStore(); + baseStore = makeBaseStore(mockKVStore); + }); + + describe('getSlotKey', () => { + it('generates correct slot keys', () => { + expect(baseStore.getSlotKey('v1', 'ko123')).toBe('v1.c.ko123'); + expect(baseStore.getSlotKey('r2', 'kp456')).toBe('r2.c.kp456'); + }); + }); + + describe('incCounter', () => { + it('increments a stored counter value', () => { + // Create a stored value to increment + const storedValue = baseStore.provideCachedStoredValue( + 'test-counter', + '5', + ); + + // Increment and check return value + expect(baseStore.incCounter(storedValue)).toBe('5'); + expect(storedValue.get()).toBe('6'); + + // Increment again + expect(baseStore.incCounter(storedValue)).toBe('6'); + expect(storedValue.get()).toBe('7'); + }); + }); + + describe('provideCachedStoredValue', () => { + it('creates a new value if it does not exist', () => { + const value = baseStore.provideCachedStoredValue('new-key', 'initial'); + expect(value.get()).toBe('initial'); + expect(mockKVStore.get('new-key')).toBe('initial'); + }); + + it('retrieves an existing value', () => { + mockKVStore.set('existing-key', 'existing-value'); + const value = baseStore.provideCachedStoredValue('existing-key'); + expect(value.get()).toBe('existing-value'); + }); + + it('caches values in memory', () => { + const value = baseStore.provideCachedStoredValue('cached-key', 'initial'); + + // Change the value through the stored value object + value.set('updated'); + expect(value.get()).toBe('updated'); + expect(mockKVStore.get('cached-key')).toBe('updated'); + + // Change the value directly in the KV store + mockKVStore.set('cached-key', 'changed-externally'); + + // The cached value should still return the cached value, not the updated KV store value + // This is because the value is cached in memory + expect(value.get()).toBe('updated'); + + // But a new stored value object should see the updated KV store value + const newValue = baseStore.provideCachedStoredValue('cached-key'); + expect(newValue.get()).toBe('changed-externally'); + }); + + it('deletes values correctly', () => { + const value = baseStore.provideCachedStoredValue( + 'delete-key', + 'to-delete', + ); + expect(value.get()).toBe('to-delete'); + + value.delete(); + expect(value.get()).toBeUndefined(); + expect(mockKVStore.get('delete-key')).toBeUndefined(); + }); + }); + + describe('provideRawStoredValue', () => { + it('creates a new value if it does not exist', () => { + const value = baseStore.provideRawStoredValue('new-raw-key', 'initial'); + expect(value.get()).toBe('initial'); + expect(mockKVStore.get('new-raw-key')).toBe('initial'); + }); + + it('retrieves an existing value', () => { + mockKVStore.set('existing-raw-key', 'existing-value'); + const value = baseStore.provideRawStoredValue('existing-raw-key'); + expect(value.get()).toBe('existing-value'); + }); + + it('does not cache values in memory', () => { + const value = baseStore.provideRawStoredValue('raw-key', 'initial'); + + // Change the value through the stored value object + value.set('updated'); + expect(value.get()).toBe('updated'); + expect(mockKVStore.get('raw-key')).toBe('updated'); + + // Change the value directly in the KV store + mockKVStore.set('raw-key', 'changed-externally'); + + // The raw value should always read from the KV store + expect(value.get()).toBe('changed-externally'); + }); + + it('deletes values correctly', () => { + const value = baseStore.provideRawStoredValue( + 'delete-raw-key', + 'to-delete', + ); + expect(value.get()).toBe('to-delete'); + + value.delete(); + expect(value.get()).toBeUndefined(); + expect(mockKVStore.get('delete-raw-key')).toBeUndefined(); + }); + }); + + describe('maybeFreeKrefs', () => { + it('maintains a set of krefs that might need to be freed', () => { + expect(baseStore.maybeFreeKrefs.size).toBe(0); + + baseStore.maybeFreeKrefs.add('ko1'); + baseStore.maybeFreeKrefs.add('kp2'); + + expect(baseStore.maybeFreeKrefs.size).toBe(2); + expect(baseStore.maybeFreeKrefs.has('ko1')).toBe(true); + expect(baseStore.maybeFreeKrefs.has('kp2')).toBe(true); + + baseStore.maybeFreeKrefs.delete('ko1'); + expect(baseStore.maybeFreeKrefs.size).toBe(1); + expect(baseStore.maybeFreeKrefs.has('ko1')).toBe(false); + expect(baseStore.maybeFreeKrefs.has('kp2')).toBe(true); + + baseStore.maybeFreeKrefs.clear(); + expect(baseStore.maybeFreeKrefs.size).toBe(0); + }); + }); + + describe('integration', () => { + it('works with multiple stored values', () => { + const counter1 = baseStore.provideCachedStoredValue('counter1', '1'); + const counter2 = baseStore.provideCachedStoredValue('counter2', '10'); + + expect(baseStore.incCounter(counter1)).toBe('1'); + expect(baseStore.incCounter(counter2)).toBe('10'); + + expect(counter1.get()).toBe('2'); + expect(counter2.get()).toBe('11'); + }); + + it('supports both cached and raw stored values', () => { + const cachedValue = baseStore.provideCachedStoredValue( + 'cached', + 'cached-value', + ); + const rawValue = baseStore.provideRawStoredValue('raw', 'raw-value'); + + expect(cachedValue.get()).toBe('cached-value'); + expect(rawValue.get()).toBe('raw-value'); + + // Modify directly in KV store + mockKVStore.set('cached', 'modified-cached'); + mockKVStore.set('raw', 'modified-raw'); + + // Cached value should still return the cached value + expect(cachedValue.get()).toBe('cached-value'); + // Raw value should return the updated value + expect(rawValue.get()).toBe('modified-raw'); + }); + }); +}); diff --git a/packages/kernel/src/store/utils/kernel-store.test.ts b/packages/kernel/src/store/kernel-store.test.ts similarity index 68% rename from packages/kernel/src/store/utils/kernel-store.test.ts rename to packages/kernel/src/store/kernel-store.test.ts index 3b9f61db5..7b66579a0 100644 --- a/packages/kernel/src/store/utils/kernel-store.test.ts +++ b/packages/kernel/src/store/kernel-store.test.ts @@ -2,9 +2,9 @@ import type { Message } from '@agoric/swingset-liveslots'; import type { KernelDatabase } from '@ocap/store'; import { describe, it, expect, beforeEach } from 'vitest'; -import { makeMapKernelDatabase } from '../../../test/storage.ts'; -import type { RunQueueItem } from '../../types.ts'; -import { makeKernelStore } from '../kernel-store.ts'; +import { makeMapKernelDatabase } from '../../test/storage.ts'; +import type { RunQueueItem } from '../types.ts'; +import { makeKernelStore } from './kernel-store.ts'; /** * Mock Message: A stupid TS hack to allow trivial use of plain strings as if they @@ -54,10 +54,15 @@ describe('kernel store', () => { const ks = makeKernelStore(mockKernelDatabase); expect(Object.keys(ks).sort()).toStrictEqual([ 'addClistEntry', + 'addGCActions', 'addPromiseSubscriber', 'allocateErefForKref', 'clear', + 'clearReachableFlag', + 'createStoredQueue', 'decRefCount', + 'decrementRefCount', + 'deleteClistEntry', 'deleteKernelObject', 'deleteKernelPromise', 'dequeueRun', @@ -66,24 +71,37 @@ describe('kernel store', () => { 'erefToKref', 'forgetEref', 'forgetKref', + 'getGCActions', 'getKernelPromise', 'getKernelPromiseMessageQueue', + 'getNextObjectId', + 'getNextPromiseId', 'getNextRemoteId', 'getNextVatId', + 'getObjectRefCount', 'getOwner', + 'getQueueLength', + 'getReachableFlag', 'getRefCount', - 'getObjectRefCount', + 'hasCListEntry', 'incRefCount', - 'setObjectRefCount', + 'incrementRefCount', 'initEndpoint', 'initKernelObject', 'initKernelPromise', + 'kernelRefExists', 'krefToEref', 'kv', 'makeVatStore', + 'nextReapAction', + 'provideStoredQueue', + 'refCountKey', 'reset', 'resolveKernelPromise', 'runQueueLength', + 'scheduleReap', + 'setGCActions', + 'setObjectRefCount', 'setPromiseDecider', ]); }); @@ -104,17 +122,41 @@ describe('kernel store', () => { const ko1Owner = 'v47'; const ko2Owner = 'r23'; expect(ks.initKernelObject(ko1Owner)).toBe('ko1'); - expect(ks.getRefCount('ko1')).toBe(1); - expect(ks.incRefCount('ko1')).toBe(2); - ks.incRefCount('ko1'); - expect(ks.getRefCount('ko1')).toBe(3); - expect(ks.decRefCount('ko1')).toBe(2); - ks.decRefCount('ko1'); - ks.decRefCount('ko1'); - expect(ks.getRefCount('ko1')).toBe(0); + + // Check that the object is initialized with reachable=1, recognizable=1 + const refCounts = ks.getObjectRefCount('ko1'); + expect(refCounts.reachable).toBe(1); + expect(refCounts.recognizable).toBe(1); + + // Increment the reference count + ks.incrementRefCount('ko1', {}); + expect(ks.getObjectRefCount('ko1').reachable).toBe(2); + expect(ks.getObjectRefCount('ko1').recognizable).toBe(2); + + // Increment again + ks.incrementRefCount('ko1', {}); + expect(ks.getObjectRefCount('ko1').reachable).toBe(3); + expect(ks.getObjectRefCount('ko1').recognizable).toBe(3); + + // Decrement + ks.decrementRefCount('ko1', {}); + expect(ks.getObjectRefCount('ko1').reachable).toBe(2); + expect(ks.getObjectRefCount('ko1').recognizable).toBe(2); + + // Decrement twice more to reach 0 + ks.decrementRefCount('ko1', {}); + ks.decrementRefCount('ko1', {}); + expect(ks.getObjectRefCount('ko1').reachable).toBe(0); + expect(ks.getObjectRefCount('ko1').recognizable).toBe(0); + + // Create another object expect(ks.initKernelObject(ko2Owner)).toBe('ko2'); + + // Check owners expect(ks.getOwner('ko1')).toBe(ko1Owner); expect(ks.getOwner('ko2')).toBe(ko2Owner); + + // Delete an object ks.deleteKernelObject('ko1'); expect(() => ks.getOwner('ko1')).toThrow('unknown kernel object ko1'); expect(() => ks.getOwner('ko99')).toThrow('unknown kernel object ko99'); @@ -178,27 +220,44 @@ describe('kernel store', () => { }); it('manages clists', () => { const ks = makeKernelStore(mockKernelDatabase); - ks.addClistEntry('v2', 'ko42', 'o-63'); - ks.addClistEntry('v2', 'ko51', 'o-74'); - ks.addClistEntry('v2', 'kp60', 'p+85'); - ks.addClistEntry('r7', 'ko42', 'ro+11'); - ks.addClistEntry('r7', 'kp61', 'rp-99'); - expect(ks.krefToEref('v2', 'ko42')).toBe('o-63'); - expect(ks.erefToKref('v2', 'o-63')).toBe('ko42'); - expect(ks.krefToEref('v2', 'ko51')).toBe('o-74'); - expect(ks.erefToKref('v2', 'o-74')).toBe('ko51'); - expect(ks.krefToEref('v2', 'kp60')).toBe('p+85'); - expect(ks.erefToKref('v2', 'p+85')).toBe('kp60'); - expect(ks.krefToEref('r7', 'ko42')).toBe('ro+11'); - expect(ks.erefToKref('r7', 'ro+11')).toBe('ko42'); - expect(ks.krefToEref('r7', 'kp61')).toBe('rp-99'); - expect(ks.erefToKref('r7', 'rp-99')).toBe('kp61'); - ks.forgetKref('v2', 'ko42'); - expect(ks.krefToEref('v2', 'ko42')).toBeUndefined(); + + // Create objects first to ensure they exist in the kernel + const ko42 = ks.initKernelObject('v2'); + const ko51 = ks.initKernelObject('v2'); + const [kp60] = ks.initKernelPromise(); + const [kp61] = ks.initKernelPromise(); + + // Add C-list entries + ks.addClistEntry('v2', ko42, 'o-63'); + ks.addClistEntry('v2', ko51, 'o-74'); + ks.addClistEntry('v2', kp60, 'p+85'); + ks.addClistEntry('r7', ko42, 'ro+11'); + ks.addClistEntry('r7', kp61, 'rp-99'); + + // Verify mappings + expect(ks.krefToEref('v2', ko42)).toBe('o-63'); + expect(ks.erefToKref('v2', 'o-63')).toBe(ko42); + expect(ks.krefToEref('v2', ko51)).toBe('o-74'); + expect(ks.erefToKref('v2', 'o-74')).toBe(ko51); + expect(ks.krefToEref('v2', kp60)).toBe('p+85'); + expect(ks.erefToKref('v2', 'p+85')).toBe(kp60); + expect(ks.krefToEref('r7', ko42)).toBe('ro+11'); + expect(ks.erefToKref('r7', 'ro+11')).toBe(ko42); + expect(ks.krefToEref('r7', kp61)).toBe('rp-99'); + expect(ks.erefToKref('r7', 'rp-99')).toBe(kp61); + + // Test forgetting entries + ks.forgetKref('v2', ko42); + expect(ks.krefToEref('v2', ko42)).toBeUndefined(); expect(ks.erefToKref('v2', 'o-63')).toBeUndefined(); + ks.forgetEref('r7', 'rp-99'); - expect(ks.krefToEref('r7', 'kp61')).toBeUndefined(); + expect(ks.krefToEref('r7', kp61)).toBeUndefined(); expect(ks.erefToKref('r7', 'rp-99')).toBeUndefined(); + + // Verify C-list entry existence + expect(ks.hasCListEntry('r7', ko42)).toBe(true); + expect(ks.hasCListEntry('v2', ko42)).toBe(false); // We forgot this one }); }); diff --git a/packages/kernel/src/store/promise-store.ts b/packages/kernel/src/store/promise-store.ts index 9ea0eb032..f1566e691 100644 --- a/packages/kernel/src/store/promise-store.ts +++ b/packages/kernel/src/store/promise-store.ts @@ -3,11 +3,11 @@ import { Fail } from '@endo/errors'; import type { CapData } from '@endo/marshal'; import type { KVStore } from '@ocap/store'; -import type { makeBaseStore } from './base-store'; +import type { makeBaseStore } from './base-store.ts'; import type { makeQueueStore } from './queue-store.ts'; -import type { makeRefCountStore } from './refcount-store'; -import { makeKernelSlot } from './utils/kernel-slots'; -import { parseRef } from './utils/parse-ref'; +import type { makeRefCountStore } from './refcount-store.ts'; +import { makeKernelSlot } from './utils/kernel-slots.ts'; +import { parseRef } from './utils/parse-ref.ts'; import type { KRef, KernelPromise, PromiseState, VatId } from '../types.ts'; import { insistVatId } from '../types.ts'; diff --git a/packages/kernel/src/store/utils/kernel-slots.test.ts b/packages/kernel/src/store/utils/kernel-slots.test.ts new file mode 100644 index 000000000..1d37aab67 --- /dev/null +++ b/packages/kernel/src/store/utils/kernel-slots.test.ts @@ -0,0 +1,135 @@ +import { describe, it, expect } from 'vitest'; + +import { + parseKernelSlot, + makeKernelSlot, + insistKernelType, +} from './kernel-slots.ts'; + +describe('kernel-slots', () => { + describe('parseKernelSlot', () => { + it('parses object slots correctly', () => { + const result = parseKernelSlot('ko123'); + expect(result).toStrictEqual({ + type: 'object', + id: '123', + }); + }); + + it('parses promise slots correctly', () => { + const result = parseKernelSlot('kp456'); + expect(result).toStrictEqual({ + type: 'promise', + id: '456', + }); + }); + + it('throws for invalid slot format', () => { + expect(() => parseKernelSlot('invalid')).toThrow( + 'invalid kernelSlot "invalid"', + ); + expect(() => parseKernelSlot('k123')).toThrow( + 'invalid kernelSlot "k123"', + ); + expect(() => parseKernelSlot('kx123')).toThrow( + 'invalid kernelSlot "kx123"', + ); + }); + + it('throws for non-string input', () => { + // @ts-expect-error Testing invalid input + expect(() => parseKernelSlot(123)).toThrow('123 must be a string'); + // @ts-expect-error Testing invalid input + expect(() => parseKernelSlot(null)).toThrow('null must be a string'); + // @ts-expect-error Testing invalid input + expect(() => parseKernelSlot(undefined)).toThrow( + '"[undefined]" must be a string', + ); + // @ts-expect-error Testing invalid input + expect(() => parseKernelSlot({})).toThrow('{} must be a string'); + }); + }); + + describe('makeKernelSlot', () => { + it('creates object slots correctly', () => { + expect(makeKernelSlot('object', '123')).toBe('ko123'); + }); + + it('creates promise slots correctly', () => { + expect(makeKernelSlot('promise', '456')).toBe('kp456'); + }); + + it('throws for invalid type', () => { + // @ts-expect-error Testing invalid input + expect(() => makeKernelSlot('invalid', '123')).toThrow( + 'unknown type "invalid"', + ); + }); + }); + + describe('insistKernelType', () => { + it('passes for correct object type', () => { + expect(() => insistKernelType('object', 'ko123')).not.toThrow(); + }); + + it('passes for correct promise type', () => { + expect(() => insistKernelType('promise', 'kp456')).not.toThrow(); + }); + + it('throws for mismatched type', () => { + expect(() => insistKernelType('object', 'kp123')).toThrow( + 'kernelSlot "kp123" is not of type "object"', + ); + expect(() => insistKernelType('promise', 'ko456')).toThrow( + 'kernelSlot "ko456" is not of type "promise"', + ); + }); + + it('throws for invalid slot format', () => { + expect(() => insistKernelType('object', 'invalid')).toThrow( + 'invalid kernelSlot "invalid"', + ); + expect(() => insistKernelType('promise', 'k123')).toThrow( + 'invalid kernelSlot "k123"', + ); + }); + + it('throws for undefined input', () => { + expect(() => insistKernelType('object', undefined)).toThrow( + 'kernelSlot is undefined', + ); + }); + }); + + describe('integration', () => { + it('can round-trip object slots', () => { + const slot = makeKernelSlot('object', '123'); + const parsed = parseKernelSlot(slot); + expect(parsed.type).toBe('object'); + expect(parsed.id).toBe('123'); + expect(makeKernelSlot(parsed.type, parsed.id)).toBe(slot); + }); + + it('can round-trip promise slots', () => { + const slot = makeKernelSlot('promise', '456'); + const parsed = parseKernelSlot(slot); + expect(parsed.type).toBe('promise'); + expect(parsed.id).toBe('456'); + expect(makeKernelSlot(parsed.type, parsed.id)).toBe(slot); + }); + + it('insistKernelType works with makeKernelSlot', () => { + const objectSlot = makeKernelSlot('object', '123'); + const promiseSlot = makeKernelSlot('promise', '456'); + + expect(() => insistKernelType('object', objectSlot)).not.toThrow(); + expect(() => insistKernelType('promise', promiseSlot)).not.toThrow(); + expect(() => insistKernelType('object', promiseSlot)).toThrow( + 'kernelSlot "kp456" is not of type "object"', + ); + expect(() => insistKernelType('promise', objectSlot)).toThrow( + 'kernelSlot "ko123" is not of type "promise"', + ); + }); + }); +}); diff --git a/packages/kernel/src/store/utils/parse-ref.test.ts b/packages/kernel/src/store/utils/parse-ref.test.ts new file mode 100644 index 000000000..7e4745154 --- /dev/null +++ b/packages/kernel/src/store/utils/parse-ref.test.ts @@ -0,0 +1,205 @@ +import { describe, it, expect } from 'vitest'; + +import { parseRef } from './parse-ref.ts'; + +describe('parse-ref', () => { + describe('parseRef', () => { + describe('valid references', () => { + it.each([ + // Kernel references + [ + 'ko123', + { + context: 'kernel', + isPromise: false, + index: '123', + direction: undefined, + }, + ], + [ + 'kp456', + { + context: 'kernel', + isPromise: true, + index: '456', + direction: undefined, + }, + ], + + // Vat references + [ + 'o+789', + { + context: 'vat', + direction: 'export', + isPromise: false, + index: '789', + }, + ], + [ + 'o-321', + { + context: 'vat', + direction: 'import', + isPromise: false, + index: '321', + }, + ], + [ + 'p+654', + { + context: 'vat', + direction: 'export', + isPromise: true, + index: '654', + }, + ], + [ + 'p-987', + { + context: 'vat', + direction: 'import', + isPromise: true, + index: '987', + }, + ], + + // Remote references + [ + 'ro+111', + { + context: 'remote', + direction: 'export', + isPromise: false, + index: '111', + }, + ], + [ + 'ro-222', + { + context: 'remote', + direction: 'import', + isPromise: false, + index: '222', + }, + ], + [ + 'rp+333', + { + context: 'remote', + direction: 'export', + isPromise: true, + index: '333', + }, + ], + [ + 'rp-444', + { + context: 'remote', + direction: 'import', + isPromise: true, + index: '444', + }, + ], + + // Edge cases + [ + 'o+', + { context: 'vat', direction: 'export', isPromise: false, index: '' }, + ], + [ + 'kpabc', + { + context: 'kernel', + isPromise: true, + index: 'abc', + direction: undefined, + }, + ], + [ + 'o+', + { context: 'vat', direction: 'export', isPromise: false, index: '' }, + ], + ])('parses %s correctly', (ref, expected) => { + expect(parseRef(ref)).toStrictEqual(expected); + }); + }); + + describe('error cases', () => { + it.each([ + ['xo+123', 'invalid reference context "x"'], + ['zo-456', 'invalid reference context "z"'], + ['kx123', 'invalid reference type "x"'], + ['rx+123', 'invalid reference type "x"'], + ['o*123', 'invalid reference direction "*"'], + ['p=456', 'invalid reference direction "="'], + ['ro?789', 'invalid reference direction "?"'], + ['rp!321', 'invalid reference direction "!"'], + ['', 'invalid reference context "[undefined]"'], + ])('throws for invalid reference %s', (ref, errorMessage) => { + expect(() => parseRef(ref)).toThrow(errorMessage); + }); + }); + }); + + describe('reference patterns', () => { + describe('context identification', () => { + it.each([ + ['ko1', 'kernel'], + ['kp2', 'kernel'], + ['o+3', 'vat'], + ['o-4', 'vat'], + ['p+5', 'vat'], + ['p-6', 'vat'], + ['ro+7', 'remote'], + ['ro-8', 'remote'], + ['rp+9', 'remote'], + ['rp-10', 'remote'], + ])('identifies %s as %s context', (ref, expectedContext) => { + expect(parseRef(ref).context).toBe(expectedContext); + }); + }); + + describe('promise vs object identification', () => { + it.each([ + // Object references + ['ko1', false], + ['o+3', false], + ['o-4', false], + ['ro+7', false], + ['ro-8', false], + + // Promise references + ['kp2', true], + ['p+5', true], + ['p-6', true], + ['rp+9', true], + ['rp-10', true], + ])('identifies %s as %s for isPromise', (ref, isPromise) => { + expect(parseRef(ref).isPromise).toBe(isPromise); + }); + }); + + describe('direction identification', () => { + it.each([ + // Export references + ['o+3', 'export'], + ['p+5', 'export'], + ['ro+7', 'export'], + ['rp+9', 'export'], + + // Import references + ['o-4', 'import'], + ['p-6', 'import'], + ['ro-8', 'import'], + ['rp-10', 'import'], + ])('identifies %s as %s direction', (ref, expectedDirection) => { + expect(parseRef(ref).direction).toBe(expectedDirection); + }); + + it.each(['ko1', 'kp2'])('kernel reference %s has no direction', (ref) => { + expect(parseRef(ref).direction).toBeUndefined(); + }); + }); + }); +}); diff --git a/packages/kernel/src/store/utils/promise-ref.test.ts b/packages/kernel/src/store/utils/promise-ref.test.ts new file mode 100644 index 000000000..5dda9e82b --- /dev/null +++ b/packages/kernel/src/store/utils/promise-ref.test.ts @@ -0,0 +1,23 @@ +import { describe, it, expect } from 'vitest'; + +import { isPromiseRef } from './promise-ref.ts'; + +describe('promise-ref', () => { + describe('isPromiseRef', () => { + it.each([ + // References with 'p' as the second character (should return true) + ['kp1', true, 'kernel promise'], + ['rp+7', true, 'remote promise'], + ['xpyz', true, 'any string with p as second character'], + // References without 'p' as the second character (should return false) + ['ko2', false, 'kernel object'], + ['p+42', false, 'vat promise (p in first position)'], + ['o+3', false, 'vat object'], + ['ro+5', false, 'remote object'], + ['abc', false, 'string without p as second character'], + ['a', false, 'single character string'], + ])('returns %s for %s', (ref, expected) => { + expect(isPromiseRef(ref)).toBe(expected); + }); + }); +}); diff --git a/packages/kernel/src/store/utils/reachable.test.ts b/packages/kernel/src/store/utils/reachable.test.ts new file mode 100644 index 000000000..15fb91426 --- /dev/null +++ b/packages/kernel/src/store/utils/reachable.test.ts @@ -0,0 +1,109 @@ +import { describe, it, expect } from 'vitest'; + +import { + parseReachableAndVatSlot, + buildReachableAndVatSlot, +} from './reachable.ts'; + +describe('reachable', () => { + describe('parseReachableAndVatSlot', () => { + it('parses reachable flag correctly', () => { + expect(parseReachableAndVatSlot('R o+123')).toStrictEqual({ + isReachable: true, + vatSlot: 'o+123', + }); + + expect(parseReachableAndVatSlot('_ o+123')).toStrictEqual({ + isReachable: false, + vatSlot: 'o+123', + }); + }); + + it('works with different vat slot types', () => { + // Object slots + expect(parseReachableAndVatSlot('R o+123')).toStrictEqual({ + isReachable: true, + vatSlot: 'o+123', + }); + + expect(parseReachableAndVatSlot('_ o-456')).toStrictEqual({ + isReachable: false, + vatSlot: 'o-456', + }); + + // Promise slots + expect(parseReachableAndVatSlot('R p+789')).toStrictEqual({ + isReachable: true, + vatSlot: 'p+789', + }); + + expect(parseReachableAndVatSlot('_ p-012')).toStrictEqual({ + isReachable: false, + vatSlot: 'p-012', + }); + }); + + it('throws for invalid flag format', () => { + expect(() => parseReachableAndVatSlot('X o+123')).toThrow( + `flag ("X") must be 'R' or '_'`, + ); + expect(() => parseReachableAndVatSlot('Ro+123')).toThrow( + 'Expected "o" is same as " "', + ); + expect(() => parseReachableAndVatSlot('R')).toThrow( + 'Expected "" is same as " "', + ); + }); + + it('throws for non-string input', () => { + // @ts-expect-error Testing invalid input + expect(() => parseReachableAndVatSlot(123)).toThrow( + 'non-string value: 123', + ); + // @ts-expect-error Testing invalid input + expect(() => parseReachableAndVatSlot(null)).toThrow( + 'non-string value: null', + ); + // @ts-expect-error Testing invalid input + expect(() => parseReachableAndVatSlot(undefined)).toThrow( + 'non-string value: "[undefined]"', + ); + }); + }); + + describe('buildReachableAndVatSlot', () => { + it('builds string with reachable flag correctly', () => { + expect(buildReachableAndVatSlot(true, 'o+123')).toBe('R o+123'); + expect(buildReachableAndVatSlot(false, 'o+123')).toBe('_ o+123'); + }); + + it('works with different vat slot types', () => { + // Object slots + expect(buildReachableAndVatSlot(true, 'o+123')).toBe('R o+123'); + expect(buildReachableAndVatSlot(false, 'o-456')).toBe('_ o-456'); + + // Promise slots + expect(buildReachableAndVatSlot(true, 'p+789')).toBe('R p+789'); + expect(buildReachableAndVatSlot(false, 'p-012')).toBe('_ p-012'); + }); + }); + + describe('round-trip conversion', () => { + it('can round-trip reachable values', () => { + const testCases = [ + { isReachable: true, vatSlot: 'o+123' }, + { isReachable: false, vatSlot: 'o-456' }, + { isReachable: true, vatSlot: 'p+789' }, + { isReachable: false, vatSlot: 'p-012' }, + ]; + + for (const testCase of testCases) { + const { isReachable, vatSlot } = testCase; + const built = buildReachableAndVatSlot(isReachable, vatSlot); + const parsed = parseReachableAndVatSlot(built); + + expect(parsed).toStrictEqual(testCase); + } + }); + }); +}); diff --git a/packages/nodejs/src/index.ts b/packages/nodejs/src/index.ts new file mode 100644 index 000000000..f20a4d8bc --- /dev/null +++ b/packages/nodejs/src/index.ts @@ -0,0 +1,2 @@ +export { NodejsVatWorkerService } from './kernel/VatWorkerService.ts'; +export { makeKernel } from './kernel/make-kernel.ts'; diff --git a/vitest.config.ts b/vitest.config.ts index 244c19f1d..42f3f247b 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -88,10 +88,10 @@ export default defineConfig({ lines: 81.78, }, 'packages/kernel/**': { - statements: 80.9, - functions: 77.91, - branches: 62.12, - lines: 81.05, + statements: 0, + functions: 0, + branches: 0, + lines: 0, }, 'packages/nodejs/**': { statements: 72.91, From 28bb341dd92d2ba9aa5b49810c83cd67eec09215 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Wed, 12 Mar 2025 14:10:50 -0300 Subject: [PATCH 09/21] refactor(kernel): reorganize kernel store into method groups --- packages/kernel/src/Kernel.ts | 4 +- packages/kernel/src/VatHandle.test.ts | 4 +- packages/kernel/src/VatHandle.ts | 2 +- .../kernel/src/services/garbage-collection.ts | 2 +- .../{kernel-store.test.ts => index.test.ts} | 2 +- .../src/store/{kernel-store.ts => index.ts} | 133 +++++++++++++----- .../base.test.ts} | 59 +++----- .../store/{base-store.ts => methods/base.ts} | 28 +--- .../{clist-store.ts => methods/clist.ts} | 81 +++++------ .../src/store/{gc-store.ts => methods/gc.ts} | 70 ++++----- .../src/store/{id-store.ts => methods/id.ts} | 30 ++-- .../{object-store.ts => methods/object.ts} | 41 ++---- .../{promise-store.ts => methods/promise.ts} | 60 +++----- .../{queue-store.ts => methods/queue.ts} | 76 ++-------- .../refcount.ts} | 11 +- packages/kernel/src/store/types.ts | 25 ++++ 16 files changed, 283 insertions(+), 345 deletions(-) rename packages/kernel/src/store/{kernel-store.test.ts => index.test.ts} (99%) rename packages/kernel/src/store/{kernel-store.ts => index.ts} (56%) rename packages/kernel/src/store/{base-store.test.ts => methods/base.test.ts} (71%) rename packages/kernel/src/store/{base-store.ts => methods/base.ts} (75%) rename packages/kernel/src/store/{clist-store.ts => methods/clist.ts} (75%) rename packages/kernel/src/store/{gc-store.ts => methods/gc.ts} (68%) rename packages/kernel/src/store/{id-store.ts => methods/id.ts} (55%) rename packages/kernel/src/store/{object-store.ts => methods/object.ts} (72%) rename packages/kernel/src/store/{promise-store.ts => methods/promise.ts} (77%) rename packages/kernel/src/store/{queue-store.ts => methods/queue.ts} (65%) rename packages/kernel/src/store/{refcount-store.ts => methods/refcount.ts} (90%) create mode 100644 packages/kernel/src/store/types.ts diff --git a/packages/kernel/src/Kernel.ts b/packages/kernel/src/Kernel.ts index fb7fd2258..c177005d5 100644 --- a/packages/kernel/src/Kernel.ts +++ b/packages/kernel/src/Kernel.ts @@ -22,8 +22,8 @@ import type { import { processGCActionSet } from './services/garbage-collection.ts'; import type { SlotValue } from './services/kernel-marshal.ts'; import { kser, kunser, krefOf, kslot } from './services/kernel-marshal.ts'; -import { makeKernelStore } from './store/kernel-store.ts'; -import type { KernelStore } from './store/kernel-store.ts'; +import { makeKernelStore } from './store/index.ts'; +import type { KernelStore } from './store/index.ts'; import { parseRef } from './store/utils/parse-ref.ts'; import { isPromiseRef } from './store/utils/promise-ref.ts'; import type { diff --git a/packages/kernel/src/VatHandle.test.ts b/packages/kernel/src/VatHandle.test.ts index 92eef54fb..53671e7ad 100644 --- a/packages/kernel/src/VatHandle.test.ts +++ b/packages/kernel/src/VatHandle.test.ts @@ -8,8 +8,8 @@ import type { MockInstance } from 'vitest'; import { Kernel } from './Kernel.ts'; import { isVatCommandReply, VatCommandMethod } from './messages/index.ts'; import type { VatCommand, VatCommandReply } from './messages/index.ts'; -import type { KernelStore } from './store/kernel-store.ts'; -import { makeKernelStore } from './store/kernel-store.ts'; +import { makeKernelStore } from './store/index.ts'; +import type { KernelStore } from './store/index.ts'; import { VatHandle } from './VatHandle.ts'; import { makeMapKernelDatabase } from '../test/storage.ts'; diff --git a/packages/kernel/src/VatHandle.ts b/packages/kernel/src/VatHandle.ts index 1a1652ea4..1570f65ab 100644 --- a/packages/kernel/src/VatHandle.ts +++ b/packages/kernel/src/VatHandle.ts @@ -18,7 +18,7 @@ import type { VatCommand, VatCommandReturnType, } from './messages/index.ts'; -import type { KernelStore } from './store/kernel-store.ts'; +import type { KernelStore } from './store'; import { parseRef } from './store/utils/parse-ref.ts'; import type { PromiseCallbacks, diff --git a/packages/kernel/src/services/garbage-collection.ts b/packages/kernel/src/services/garbage-collection.ts index cd1de5785..cef4b7669 100644 --- a/packages/kernel/src/services/garbage-collection.ts +++ b/packages/kernel/src/services/garbage-collection.ts @@ -1,4 +1,4 @@ -import type { KernelStore } from '../store/kernel-store.ts'; +import type { KernelStore } from '../store'; import { insistKernelType } from '../store/utils/kernel-slots.ts'; import type { GCAction, diff --git a/packages/kernel/src/store/kernel-store.test.ts b/packages/kernel/src/store/index.test.ts similarity index 99% rename from packages/kernel/src/store/kernel-store.test.ts rename to packages/kernel/src/store/index.test.ts index 7b66579a0..c4d04a629 100644 --- a/packages/kernel/src/store/kernel-store.test.ts +++ b/packages/kernel/src/store/index.test.ts @@ -2,9 +2,9 @@ import type { Message } from '@agoric/swingset-liveslots'; import type { KernelDatabase } from '@ocap/store'; import { describe, it, expect, beforeEach } from 'vitest'; +import { makeKernelStore } from './index.ts'; import { makeMapKernelDatabase } from '../../test/storage.ts'; import type { RunQueueItem } from '../types.ts'; -import { makeKernelStore } from './kernel-store.ts'; /** * Mock Message: A stupid TS hack to allow trivial use of plain strings as if they diff --git a/packages/kernel/src/store/kernel-store.ts b/packages/kernel/src/store/index.ts similarity index 56% rename from packages/kernel/src/store/kernel-store.ts rename to packages/kernel/src/store/index.ts index 1afc0c0b9..9f6b57d4a 100644 --- a/packages/kernel/src/store/kernel-store.ts +++ b/packages/kernel/src/store/index.ts @@ -55,14 +55,16 @@ import type { KernelDatabase, KVStore, VatStore } from '@ocap/store'; -import { makeBaseStore } from './base-store.ts'; -import { makeCListStore } from './clist-store.ts'; -import { makeGCStore } from './gc-store.ts'; -import { makeIdStore } from './id-store.ts'; -import { makeObjectStore } from './object-store.ts'; -import { makePromiseStore } from './promise-store.ts'; -import { makeQueueStore } from './queue-store.ts'; -import { makeRefCountStore } from './refcount-store.ts'; +import { getBaseMethods } from './methods/base.ts'; +import { getCListMethods } from './methods/clist.ts'; +import { getGCMethods } from './methods/gc.ts'; +import { getIdMethods } from './methods/id.ts'; +import { getObjectMethods } from './methods/object.ts'; +import { getPromiseMethods } from './methods/promise.ts'; +import { getQueueMethods } from './methods/queue.ts'; +import { getRefCountMethods } from './methods/refcount.ts'; +import type { StoreContext } from './types.ts'; +import type { KRef, RunQueueItem } from '../types.ts'; /** * Create a new KernelStore object wrapped around a raw kernel database. The @@ -85,25 +87,76 @@ export function makeKernelStore(kdb: KernelDatabase) { /** KV store in which all the kernel's own state is kept. */ const kv: KVStore = kdb.kernelKVStore; - const baseStore = makeBaseStore(kv); - const idStore = makeIdStore(kv, baseStore); - const queueStore = makeQueueStore(kv, baseStore); - const refCountStore = makeRefCountStore(kv); - const objectStore = makeObjectStore(kv, baseStore, refCountStore); - const promiseStore = makePromiseStore( - kv, - baseStore, - refCountStore, - queueStore, - ); - const gcStore = makeGCStore(kv, baseStore, refCountStore, objectStore); - const cListStore = makeCListStore( + const { provideCachedStoredValue } = getBaseMethods(kv); + const queue = getQueueMethods(kv); + + /** The kernel's run queue. */ + const runQueue = queue.createStoredQueue('run', true); + /** Counter for allocating kernel object IDs */ + const nextObjectId = provideCachedStoredValue('nextObjectId', '1'); + /** Counter for allocating kernel promise IDs */ + const nextPromiseId = provideCachedStoredValue('nextPromiseId', '1'); + + // As refcounts are decremented, we accumulate a set of krefs for which + // action might need to be taken: + // * promises which are now resolved and unreferenced can be deleted + // * objects which are no longer reachable: export can be dropped + // * objects which are no longer recognizable: export can be retired + // This set is ephemeral: it lives in RAM, grows as deliveries and syscalls + // cause decrefs, and will be harvested by processRefcounts(). This needs to be + // called in the same transaction window as the syscalls/etc which prompted + // the change, else removals might be lost (not performed during the next + // replay). + const maybeFreeKrefs = new Set(); + + // Garbage collection + const gcActions = provideCachedStoredValue('gcActions', '[]'); + const reapQueue = provideCachedStoredValue('reapQueue', '[]'); + + const context: StoreContext = { kv, - baseStore, - gcStore, - objectStore, - refCountStore, - ); + runQueue, + nextObjectId, + nextPromiseId, + maybeFreeKrefs, + gcActions, + reapQueue, + }; + + const id = getIdMethods(context); + const refCount = getRefCountMethods(context); + const object = getObjectMethods(context); + const promise = getPromiseMethods(context); + const gc = getGCMethods(context); + const cList = getCListMethods(context); + + /** + * Append a message to the kernel's run queue. + * + * @param message - The message to enqueue. + */ + function enqueueRun(message: RunQueueItem): void { + context.runQueue.enqueue(message); + } + + /** + * Fetch the next message on the kernel's run queue. + * + * @returns The next message on the run queue, or undefined if the queue is + * empty. + */ + function dequeueRun(): RunQueueItem | undefined { + return context.runQueue.dequeue() as RunQueueItem | undefined; + } + + /** + * Obtain the number of entries in the run queue. + * + * @returns the number of items in the run queue. + */ + function runQueueLength(): number { + return queue.getQueueLength('run'); + } /** * Delete everything from the database. @@ -128,21 +181,25 @@ export function makeKernelStore(kdb: KernelDatabase) { */ function reset(): void { kdb.clear(); - queueStore.reset(); - objectStore.reset(); - promiseStore.reset(); - gcStore.reset(); - idStore.reset(); + context.maybeFreeKrefs.clear(); + context.runQueue = queue.createStoredQueue('run', true); + context.gcActions = provideCachedStoredValue('gcActions', '[]'); + context.reapQueue = provideCachedStoredValue('reapQueue', '[]'); + context.nextObjectId = provideCachedStoredValue('nextObjectId', '1'); + context.nextPromiseId = provideCachedStoredValue('nextPromiseId', '1'); } return harden({ - ...idStore, - ...queueStore, - ...refCountStore, - ...objectStore, - ...promiseStore, - ...gcStore, - ...cListStore, + ...id, + ...queue, + ...refCount, + ...object, + ...promise, + ...gc, + ...cList, + enqueueRun, + dequeueRun, + runQueueLength, makeVatStore, clear, reset, diff --git a/packages/kernel/src/store/base-store.test.ts b/packages/kernel/src/store/methods/base.test.ts similarity index 71% rename from packages/kernel/src/store/base-store.test.ts rename to packages/kernel/src/store/methods/base.test.ts index bd86397a4..5b09faf27 100644 --- a/packages/kernel/src/store/base-store.test.ts +++ b/packages/kernel/src/store/methods/base.test.ts @@ -1,16 +1,16 @@ import type { KVStore } from '@ocap/store'; import { describe, it, expect, beforeEach } from 'vitest'; -import { makeBaseStore } from './base-store.ts'; -import { makeMapKVStore } from '../../test/storage.ts'; +import { getBaseMethods } from './base.ts'; +import { makeMapKVStore } from '../../../test/storage.ts'; -describe('base-store', () => { - let mockKVStore: KVStore; - let baseStore: ReturnType; +describe('base-methods', () => { + let kv: KVStore; + let baseStore: ReturnType; beforeEach(() => { - mockKVStore = makeMapKVStore(); - baseStore = makeBaseStore(mockKVStore); + kv = makeMapKVStore(); + baseStore = getBaseMethods(kv); }); describe('getSlotKey', () => { @@ -42,11 +42,11 @@ describe('base-store', () => { it('creates a new value if it does not exist', () => { const value = baseStore.provideCachedStoredValue('new-key', 'initial'); expect(value.get()).toBe('initial'); - expect(mockKVStore.get('new-key')).toBe('initial'); + expect(kv.get('new-key')).toBe('initial'); }); it('retrieves an existing value', () => { - mockKVStore.set('existing-key', 'existing-value'); + kv.set('existing-key', 'existing-value'); const value = baseStore.provideCachedStoredValue('existing-key'); expect(value.get()).toBe('existing-value'); }); @@ -57,10 +57,10 @@ describe('base-store', () => { // Change the value through the stored value object value.set('updated'); expect(value.get()).toBe('updated'); - expect(mockKVStore.get('cached-key')).toBe('updated'); + expect(kv.get('cached-key')).toBe('updated'); // Change the value directly in the KV store - mockKVStore.set('cached-key', 'changed-externally'); + kv.set('cached-key', 'changed-externally'); // The cached value should still return the cached value, not the updated KV store value // This is because the value is cached in memory @@ -80,7 +80,7 @@ describe('base-store', () => { value.delete(); expect(value.get()).toBeUndefined(); - expect(mockKVStore.get('delete-key')).toBeUndefined(); + expect(kv.get('delete-key')).toBeUndefined(); }); }); @@ -88,11 +88,11 @@ describe('base-store', () => { it('creates a new value if it does not exist', () => { const value = baseStore.provideRawStoredValue('new-raw-key', 'initial'); expect(value.get()).toBe('initial'); - expect(mockKVStore.get('new-raw-key')).toBe('initial'); + expect(kv.get('new-raw-key')).toBe('initial'); }); it('retrieves an existing value', () => { - mockKVStore.set('existing-raw-key', 'existing-value'); + kv.set('existing-raw-key', 'existing-value'); const value = baseStore.provideRawStoredValue('existing-raw-key'); expect(value.get()).toBe('existing-value'); }); @@ -103,10 +103,10 @@ describe('base-store', () => { // Change the value through the stored value object value.set('updated'); expect(value.get()).toBe('updated'); - expect(mockKVStore.get('raw-key')).toBe('updated'); + expect(kv.get('raw-key')).toBe('updated'); // Change the value directly in the KV store - mockKVStore.set('raw-key', 'changed-externally'); + kv.set('raw-key', 'changed-externally'); // The raw value should always read from the KV store expect(value.get()).toBe('changed-externally'); @@ -121,28 +121,7 @@ describe('base-store', () => { value.delete(); expect(value.get()).toBeUndefined(); - expect(mockKVStore.get('delete-raw-key')).toBeUndefined(); - }); - }); - - describe('maybeFreeKrefs', () => { - it('maintains a set of krefs that might need to be freed', () => { - expect(baseStore.maybeFreeKrefs.size).toBe(0); - - baseStore.maybeFreeKrefs.add('ko1'); - baseStore.maybeFreeKrefs.add('kp2'); - - expect(baseStore.maybeFreeKrefs.size).toBe(2); - expect(baseStore.maybeFreeKrefs.has('ko1')).toBe(true); - expect(baseStore.maybeFreeKrefs.has('kp2')).toBe(true); - - baseStore.maybeFreeKrefs.delete('ko1'); - expect(baseStore.maybeFreeKrefs.size).toBe(1); - expect(baseStore.maybeFreeKrefs.has('ko1')).toBe(false); - expect(baseStore.maybeFreeKrefs.has('kp2')).toBe(true); - - baseStore.maybeFreeKrefs.clear(); - expect(baseStore.maybeFreeKrefs.size).toBe(0); + expect(kv.get('delete-raw-key')).toBeUndefined(); }); }); @@ -169,8 +148,8 @@ describe('base-store', () => { expect(rawValue.get()).toBe('raw-value'); // Modify directly in KV store - mockKVStore.set('cached', 'modified-cached'); - mockKVStore.set('raw', 'modified-raw'); + kv.set('cached', 'modified-cached'); + kv.set('raw', 'modified-raw'); // Cached value should still return the cached value expect(cachedValue.get()).toBe('cached-value'); diff --git a/packages/kernel/src/store/base-store.ts b/packages/kernel/src/store/methods/base.ts similarity index 75% rename from packages/kernel/src/store/base-store.ts rename to packages/kernel/src/store/methods/base.ts index 71ea02442..4117bda48 100644 --- a/packages/kernel/src/store/base-store.ts +++ b/packages/kernel/src/store/methods/base.ts @@ -1,33 +1,16 @@ import type { KVStore } from '@ocap/store'; -import type { EndpointId, KRef } from '../types.ts'; - -export type StoredValue = { - get(): string | undefined; - set(newValue: string): void; - delete(): void; -}; +import type { EndpointId, KRef } from '../../types.ts'; +import type { StoredValue } from '../types.ts'; /** - * Create a base store object that provides functionality for managing stored values and queues. + * Get the base store methods for managing stored values and queues. * - * @param kv - The key-value store to use for persistent storage. + * @param kv - The key/value store to provide the underlying persistence mechanism. * @returns An object with methods for managing stored values and queues. */ // eslint-disable-next-line @typescript-eslint/explicit-function-return-type -export function makeBaseStore(kv: KVStore) { - // As refcounts are decremented, we accumulate a set of krefs for which - // action might need to be taken: - // * promises which are now resolved and unreferenced can be deleted - // * objects which are no longer reachable: export can be dropped - // * objects which are no longer recognizable: export can be retired - // This set is ephemeral: it lives in RAM, grows as deliveries and syscalls - // cause decrefs, and will be harvested by processRefcounts(). This needs to be - // called in the same transaction window as the syscalls/etc which prompted - // the change, else removals might be lost (not performed during the next - // replay). - const maybeFreeKrefs = new Set(); - +export function getBaseMethods(kv: KVStore) { /** * Get the key for the reachable flag and vatSlot for a given endpoint and kref. * @@ -111,7 +94,6 @@ export function makeBaseStore(kv: KVStore) { } return { - maybeFreeKrefs, getSlotKey, incCounter, // Stored value diff --git a/packages/kernel/src/store/clist-store.ts b/packages/kernel/src/store/methods/clist.ts similarity index 75% rename from packages/kernel/src/store/clist-store.ts rename to packages/kernel/src/store/methods/clist.ts index 6ba2b571f..603031dfe 100644 --- a/packages/kernel/src/store/clist-store.ts +++ b/packages/kernel/src/store/methods/clist.ts @@ -1,36 +1,32 @@ import { Fail } from '@endo/errors'; -import type { KVStore } from '@ocap/store'; -import type { makeBaseStore } from './base-store.ts'; -import type { makeGCStore } from './gc-store.ts'; -import type { makeObjectStore } from './object-store.ts'; -import type { makeRefCountStore } from './refcount-store.ts'; -import { parseRef } from './utils/parse-ref.ts'; -import { isPromiseRef } from './utils/promise-ref.ts'; +import { getBaseMethods } from './base.ts'; +import { getGCMethods } from './gc.ts'; +import { getObjectMethods } from './object.ts'; +import { getRefCountMethods } from './refcount.ts'; +import type { EndpointId, KRef, ERef } from '../../types.ts'; +import type { StoreContext } from '../types.ts'; +import { parseRef } from '../utils/parse-ref.ts'; +import { isPromiseRef } from '../utils/promise-ref.ts'; import { buildReachableAndVatSlot, parseReachableAndVatSlot, -} from './utils/reachable.ts'; -import type { EndpointId, KRef, ERef } from '../types.ts'; +} from '../utils/reachable.ts'; /** - * Create a store for the c-list. + * Get the c-list methods that provide functionality for managing c-lists. * - * @param kv - The key-value store to use for persistent storage. - * @param baseStore - The base store to use for the c-list store. - * @param gcStore - The GC store to use for the c-list store. - * @param objectStore - The object store to use for the c-list store. - * @param refCountStore - The refcount store to use for the c-list store. + * @param ctx - The store context. * @returns The c-list store. */ // eslint-disable-next-line @typescript-eslint/explicit-function-return-type -export function makeCListStore( - kv: KVStore, - baseStore: ReturnType, - gcStore: ReturnType, - objectStore: ReturnType, - refCountStore: ReturnType, -) { +export function getCListMethods(ctx: StoreContext) { + const { kv, maybeFreeKrefs } = ctx; + const { getSlotKey } = getBaseMethods(kv); + const { clearReachableFlag } = getGCMethods(ctx); + const { getObjectRefCount, setObjectRefCount } = getObjectMethods(ctx); + const { kernelRefExists, refCountKey } = getRefCountMethods(ctx); + /** * Add an entry to an endpoint's c-list, creating a new bidirectional mapping * between an ERef belonging to the endpoint and a KRef belonging to the @@ -41,11 +37,8 @@ export function makeCListStore( * @param eref - The ERef. */ function addClistEntry(endpointId: EndpointId, kref: KRef, eref: ERef): void { - kv.set( - baseStore.getSlotKey(endpointId, kref), - buildReachableAndVatSlot(true, eref), - ); - kv.set(baseStore.getSlotKey(endpointId, eref), kref); + kv.set(getSlotKey(endpointId, kref), buildReachableAndVatSlot(true, eref)); + kv.set(getSlotKey(endpointId, eref), kref); } /** @@ -56,7 +49,7 @@ export function makeCListStore( * @returns true iff this vat has a c-list entry mapping for `slot`. */ function hasCListEntry(endpointId: EndpointId, slot: string): boolean { - return kv.get(baseStore.getSlotKey(endpointId, slot)) !== undefined; + return kv.get(getSlotKey(endpointId, slot)) !== undefined; } /** @@ -71,10 +64,10 @@ export function makeCListStore( kref: KRef, eref: ERef, ): void { - const kernelKey = baseStore.getSlotKey(endpointId, kref); - const vatKey = baseStore.getSlotKey(endpointId, eref); + const kernelKey = getSlotKey(endpointId, kref); + const vatKey = getSlotKey(endpointId, eref); assert(kv.get(kernelKey)); - gcStore.clearReachableFlag(endpointId, kref); + clearReachableFlag(endpointId, kref); const { direction } = parseRef(eref); decrementRefCount(kref, { isExport: direction === 'export', @@ -120,7 +113,7 @@ export function makeCListStore( * if there is no such mapping. */ function erefToKref(endpointId: EndpointId, eref: ERef): KRef | undefined { - return kv.get(baseStore.getSlotKey(endpointId, eref)); + return kv.get(getSlotKey(endpointId, eref)); } /** @@ -132,7 +125,7 @@ export function makeCListStore( * there is no such mapping. */ function krefToEref(endpointId: EndpointId, kref: KRef): ERef | undefined { - const key = baseStore.getSlotKey(endpointId, kref); + const key = getSlotKey(endpointId, kref); const data = kv.get(key); if (!data) { return undefined; @@ -190,8 +183,8 @@ export function makeCListStore( const { isPromise } = parseRef(kref); if (isPromise) { - const refCount = Number(kv.get(refCountStore.refCountKey(kref))) + 1; - kv.set(refCountStore.refCountKey(kref), `${refCount}`); + const refCount = Number(kv.get(refCountKey(kref))) + 1; + kv.set(refCountKey(kref), `${refCount}`); return; } @@ -200,12 +193,12 @@ export function makeCListStore( return; } - const counts = objectStore.getObjectRefCount(kref); + const counts = getObjectRefCount(kref); if (!onlyRecognizable) { counts.reachable += 1; } counts.recognizable += 1; - objectStore.setObjectRefCount(kref, counts); + setObjectRefCount(kref, counts); } /** @@ -229,30 +222,30 @@ export function makeCListStore( const { isPromise } = parseRef(kref); if (isPromise) { - let refCount = Number(kv.get(refCountStore.refCountKey(kref))); + let refCount = Number(kv.get(refCountKey(kref))); refCount > 0 || Fail`refCount underflow ${kref}`; refCount -= 1; - kv.set(refCountStore.refCountKey(kref), `${refCount}`); + kv.set(refCountKey(kref), `${refCount}`); if (refCount === 0) { - baseStore.maybeFreeKrefs.add(kref); + maybeFreeKrefs.add(kref); return true; } return false; } - if (isExport || !refCountStore.kernelRefExists(kref)) { + if (isExport || !kernelRefExists(kref)) { return false; } - const counts = objectStore.getObjectRefCount(kref); + const counts = getObjectRefCount(kref); if (!onlyRecognizable) { counts.reachable -= 1; } counts.recognizable -= 1; if (!counts.reachable || !counts.recognizable) { - baseStore.maybeFreeKrefs.add(kref); + maybeFreeKrefs.add(kref); } - objectStore.setObjectRefCount(kref, counts); + setObjectRefCount(kref, counts); return false; } diff --git a/packages/kernel/src/store/gc-store.ts b/packages/kernel/src/store/methods/gc.ts similarity index 68% rename from packages/kernel/src/store/gc-store.ts rename to packages/kernel/src/store/methods/gc.ts index e5e1e410d..0f24aa0d8 100644 --- a/packages/kernel/src/store/gc-store.ts +++ b/packages/kernel/src/store/methods/gc.ts @@ -1,41 +1,38 @@ -import type { KVStore } from '@ocap/store'; - -import type { makeBaseStore } from './base-store.ts'; -import type { makeObjectStore } from './object-store.ts'; -import type { makeRefCountStore } from './refcount-store.ts'; -import { insistKernelType } from './utils/kernel-slots.ts'; -import { parseRef } from './utils/parse-ref.ts'; -import { - buildReachableAndVatSlot, - parseReachableAndVatSlot, -} from './utils/reachable.ts'; +import { getBaseMethods } from './base.ts'; +import { getObjectMethods } from './object.ts'; +import { getRefCountMethods } from './refcount.ts'; import type { VatId, EndpointId, KRef, GCAction, RunQueueItemBringOutYourDead, -} from '../types.ts'; -import { insistGCActionType, insistVatId, RunQueueItemType } from '../types.ts'; +} from '../../types.ts'; +import { + insistGCActionType, + insistVatId, + RunQueueItemType, +} from '../../types.ts'; +import type { StoreContext } from '../types.ts'; +import { insistKernelType } from '../utils/kernel-slots.ts'; +import { parseRef } from '../utils/parse-ref.ts'; +import { + buildReachableAndVatSlot, + parseReachableAndVatSlot, +} from '../utils/reachable.ts'; /** * Create a store for garbage collection. * - * @param kv - The key-value store to use for persistent storage. - * @param baseStore - The base store to use for the GC store. - * @param refCountStore - The refcount store to use for the GC store. - * @param objectStore - The object store to use for the GC store. + * @param ctx - The store context. * @returns The GC store. */ // eslint-disable-next-line @typescript-eslint/explicit-function-return-type -export function makeGCStore( - kv: KVStore, - baseStore: ReturnType, - refCountStore: ReturnType, - objectStore: ReturnType, -) { - let gcActions = baseStore.provideCachedStoredValue('gcActions', '[]'); - let reapQueue = baseStore.provideCachedStoredValue('reapQueue', '[]'); +export function getGCMethods(ctx: StoreContext) { + const { kv, maybeFreeKrefs, gcActions, reapQueue } = ctx; + const { getSlotKey } = getBaseMethods(kv); + const { getObjectRefCount, setObjectRefCount } = getObjectMethods(ctx); + const { kernelRefExists } = getRefCountMethods(ctx); /** * Get the set of GC actions to perform. @@ -83,7 +80,7 @@ export function makeGCStore( * @returns True if the kernel object is reachable, false otherwise. */ function getReachableFlag(endpointId: EndpointId, kref: KRef): boolean { - const key = baseStore.getSlotKey(endpointId, kref); + const key = getSlotKey(endpointId, kref); const data = kv.getRequired(key); const { isReachable } = parseReachableAndVatSlot(data); return isReachable; @@ -96,7 +93,7 @@ export function makeGCStore( * @param kref - The kref. */ function clearReachableFlag(endpointId: EndpointId, kref: KRef): void { - const key = baseStore.getSlotKey(endpointId, kref); + const key = getSlotKey(endpointId, kref); const { isReachable, vatSlot } = parseReachableAndVatSlot( kv.getRequired(key), ); @@ -107,13 +104,13 @@ export function makeGCStore( isReachable && !isPromise && direction === 'import' && - refCountStore.kernelRefExists(kref) + kernelRefExists(kref) ) { - const counts = objectStore.getObjectRefCount(kref); + const counts = getObjectRefCount(kref); counts.reachable -= 1; - objectStore.setObjectRefCount(kref, counts); + setObjectRefCount(kref, counts); if (counts.reachable === 0) { - baseStore.maybeFreeKrefs.add(kref); + maybeFreeKrefs.add(kref); } } } @@ -146,15 +143,6 @@ export function makeGCStore( return undefined; } - /** - * Reset the GC store. - */ - function reset(): void { - baseStore.maybeFreeKrefs.clear(); - gcActions = baseStore.provideCachedStoredValue('gcActions', '[]'); - reapQueue = baseStore.provideCachedStoredValue('reapQueue', '[]'); - } - return { // GC actions getGCActions, @@ -166,7 +154,5 @@ export function makeGCStore( // Reaping scheduleReap, nextReapAction, - // Reset - reset, }; } diff --git a/packages/kernel/src/store/id-store.ts b/packages/kernel/src/store/methods/id.ts similarity index 55% rename from packages/kernel/src/store/id-store.ts rename to packages/kernel/src/store/methods/id.ts index 439343605..fa7171815 100644 --- a/packages/kernel/src/store/id-store.ts +++ b/packages/kernel/src/store/methods/id.ts @@ -1,24 +1,22 @@ -import type { KVStore } from '@ocap/store'; - -import type { makeBaseStore } from './base-store.ts'; -import type { VatId, RemoteId, EndpointId } from '../types.ts'; +import { getBaseMethods } from './base.ts'; +import type { VatId, RemoteId, EndpointId } from '../../types.ts'; +import type { StoreContext } from '../types.ts'; /** * Create a store for allocating IDs. * - * @param kv - The key-value store to use for persistent storage. - * @param baseStore - The base store to use for the ID store. + * @param ctx - The store context. * @returns The ID store. */ // eslint-disable-next-line @typescript-eslint/explicit-function-return-type -export function makeIdStore( - kv: KVStore, - baseStore: ReturnType, -) { +export function getIdMethods(ctx: StoreContext) { + const { kv } = ctx; + const { provideCachedStoredValue, incCounter } = getBaseMethods(kv); + /** Counter for allocating VatIDs */ - let nextVatId = baseStore.provideCachedStoredValue('nextVatId', '1'); + let nextVatId = provideCachedStoredValue('nextVatId', '1'); /** Counter for allocating RemoteIDs */ - let nextRemoteId = baseStore.provideCachedStoredValue('nextRemoteId', '1'); + let nextRemoteId = provideCachedStoredValue('nextRemoteId', '1'); /** * Obtain an ID for a new vat. @@ -26,7 +24,7 @@ export function makeIdStore( * @returns The next VatID use. */ function getNextVatId(): VatId { - return `v${baseStore.incCounter(nextVatId)}`; + return `v${incCounter(nextVatId)}`; } /** @@ -35,7 +33,7 @@ export function makeIdStore( * @returns The next remote ID use. */ function getNextRemoteId(): RemoteId { - return `r${baseStore.incCounter(nextRemoteId)}`; + return `r${incCounter(nextRemoteId)}`; } /** @@ -52,8 +50,8 @@ export function makeIdStore( * Clear the kernel's persistent state and reset all counters. */ function reset(): void { - nextVatId = baseStore.provideCachedStoredValue('nextVatId', '1'); - nextRemoteId = baseStore.provideCachedStoredValue('nextRemoteId', '1'); + nextVatId = provideCachedStoredValue('nextVatId', '1'); + nextRemoteId = provideCachedStoredValue('nextRemoteId', '1'); } return { diff --git a/packages/kernel/src/store/object-store.ts b/packages/kernel/src/store/methods/object.ts similarity index 72% rename from packages/kernel/src/store/object-store.ts rename to packages/kernel/src/store/methods/object.ts index f1fe2989f..15da1af7f 100644 --- a/packages/kernel/src/store/object-store.ts +++ b/packages/kernel/src/store/methods/object.ts @@ -1,28 +1,23 @@ import { Fail } from '@endo/errors'; -import type { KVStore } from '@ocap/store'; -import type { makeBaseStore } from './base-store.ts'; -import type { makeRefCountStore } from './refcount-store.ts'; -import { makeKernelSlot } from './utils/kernel-slots.ts'; -import type { EndpointId, KRef } from '../types.ts'; +import { getBaseMethods } from './base.ts'; +import { getRefCountMethods } from './refcount.ts'; +import type { EndpointId, KRef } from '../../types.ts'; +import type { StoreContext } from '../types.ts'; +import { makeKernelSlot } from '../utils/kernel-slots.ts'; /** * Create an object store object that provides functionality for managing kernel objects. * - * @param kv - The key-value store to use for persistent storage. - * @param baseStore - The base store to use for the object store. - * @param refCountStore - The refcount store to use for the object store. + * @param ctx - The store context. * @returns An object store object that maps various persistent kernel data * structures onto `kv`. */ // eslint-disable-next-line @typescript-eslint/explicit-function-return-type -export function makeObjectStore( - kv: KVStore, - baseStore: ReturnType, - refCountStore: ReturnType, -) { - /** Counter for allocating kernel object IDs */ - let nextObjectId = baseStore.provideCachedStoredValue('nextObjectId', '1'); +export function getObjectMethods(ctx: StoreContext) { + const { kv, nextObjectId } = ctx; + const { incCounter } = getBaseMethods(kv); + const { refCountKey } = getRefCountMethods(ctx); /** * Create a new kernel object. The new object will be born with reference and @@ -60,7 +55,7 @@ export function makeObjectStore( */ function deleteKernelObject(koId: KRef): void { kv.delete(`${koId}.owner`); - kv.delete(refCountStore.refCountKey(koId)); + kv.delete(refCountKey(koId)); } /** @@ -69,7 +64,7 @@ export function makeObjectStore( * @returns The next koId use. */ function getNextObjectId(): KRef { - return makeKernelSlot('object', baseStore.incCounter(nextObjectId)); + return makeKernelSlot('object', incCounter(nextObjectId)); } /** @@ -82,7 +77,7 @@ export function makeObjectStore( reachable: number; recognizable: number; } { - const data = kv.get(refCountStore.refCountKey(kref)); + const data = kv.get(refCountKey(kref)); if (!data) { return { reachable: 0, recognizable: 0 }; } @@ -111,14 +106,7 @@ export function makeObjectStore( Fail`${kref} underflow ${reachable},${recognizable}`; reachable <= recognizable || Fail`refMismatch(set) ${kref} ${reachable},${recognizable}`; - kv.set(refCountStore.refCountKey(kref), `${reachable},${recognizable}`); - } - - /** - * Reset the object store. - */ - function reset(): void { - nextObjectId = baseStore.provideCachedStoredValue('nextObjectId', '1'); + kv.set(refCountKey(kref), `${reachable},${recognizable}`); } return { @@ -128,6 +116,5 @@ export function makeObjectStore( getNextObjectId, getObjectRefCount, setObjectRefCount, - reset, }; } diff --git a/packages/kernel/src/store/promise-store.ts b/packages/kernel/src/store/methods/promise.ts similarity index 77% rename from packages/kernel/src/store/promise-store.ts rename to packages/kernel/src/store/methods/promise.ts index f1566e691..b3e4b5dc4 100644 --- a/packages/kernel/src/store/promise-store.ts +++ b/packages/kernel/src/store/methods/promise.ts @@ -1,35 +1,29 @@ import type { Message } from '@agoric/swingset-liveslots'; import { Fail } from '@endo/errors'; import type { CapData } from '@endo/marshal'; -import type { KVStore } from '@ocap/store'; -import type { makeBaseStore } from './base-store.ts'; -import type { makeQueueStore } from './queue-store.ts'; -import type { makeRefCountStore } from './refcount-store.ts'; -import { makeKernelSlot } from './utils/kernel-slots.ts'; -import { parseRef } from './utils/parse-ref.ts'; -import type { KRef, KernelPromise, PromiseState, VatId } from '../types.ts'; -import { insistVatId } from '../types.ts'; +import { getBaseMethods } from './base.ts'; +import { getQueueMethods } from './queue.ts'; +import { getRefCountMethods } from './refcount.ts'; +import type { KRef, KernelPromise, PromiseState, VatId } from '../../types.ts'; +import { insistVatId } from '../../types.ts'; +import type { StoreContext } from '../types.ts'; +import { makeKernelSlot } from '../utils/kernel-slots.ts'; +import { parseRef } from '../utils/parse-ref.ts'; /** * Create a promise store object that provides functionality for managing kernel promises. * - * @param kv - The key-value store to use for persistent storage. - * @param baseStore - The base store to use for the promise store. - * @param refCountStore - The refcount store to use for the promise store. - * @param queueStore - The queue store to use for the promise store. + * @param ctx - The store context. * @returns A promise store object that maps various persistent kernel data * structures onto `kv`. */ // eslint-disable-next-line @typescript-eslint/explicit-function-return-type -export function makePromiseStore( - kv: KVStore, - baseStore: ReturnType, - refCountStore: ReturnType, - queueStore: ReturnType, -) { - /** Counter for allocating kernel promise IDs */ - let nextPromiseId = baseStore.provideCachedStoredValue('nextPromiseId', '1'); +export function getPromiseMethods(ctx: StoreContext) { + const { kv, nextPromiseId } = ctx; + const { incCounter } = getBaseMethods(kv); + const { createStoredQueue, provideStoredQueue } = getQueueMethods(ctx.kv); + const { refCountKey } = getRefCountMethods(ctx); /** * Create a new, unresolved kernel promise. The new promise will be born with @@ -45,10 +39,10 @@ export function makePromiseStore( subscribers: [], }; const kpid = getNextPromiseId(); - queueStore.createStoredQueue(kpid, false); + createStoredQueue(kpid, false); kv.set(`${kpid}.state`, 'unresolved'); kv.set(`${kpid}.subscribers`, '[]'); - kv.set(refCountStore.refCountKey(kpid), '1'); + kv.set(refCountKey(kpid), '1'); return [kpid, kpr]; } @@ -97,8 +91,8 @@ export function makePromiseStore( kv.delete(`${kpid}.decider`); kv.delete(`${kpid}.subscribers`); kv.delete(`${kpid}.value`); - kv.delete(refCountStore.refCountKey(kpid)); - queueStore.provideStoredQueue(kpid).delete(); + kv.delete(refCountKey(kpid)); + provideStoredQueue(kpid).delete(); } /** @@ -107,7 +101,7 @@ export function makePromiseStore( * @returns The next kpid use. */ function getNextPromiseId(): KRef { - return makeKernelSlot('promise', baseStore.incCounter(nextPromiseId)); + return makeKernelSlot('promise', incCounter(nextPromiseId)); } /** @@ -153,7 +147,7 @@ export function makePromiseStore( rejected: boolean, value: CapData, ): void { - const queue = queueStore.provideStoredQueue(kpid, false); + const queue = provideStoredQueue(kpid, false); for (const message of getKernelPromiseMessageQueue(kpid)) { queue.enqueue(message); } @@ -170,7 +164,7 @@ export function makePromiseStore( * @param message - The message to enqueue. */ function enqueuePromiseMessage(kpid: KRef, message: Message): void { - queueStore.provideStoredQueue(kpid, false).enqueue(message); + provideStoredQueue(kpid, false).enqueue(message); } /** @@ -181,7 +175,7 @@ export function makePromiseStore( */ function getKernelPromiseMessageQueue(kpid: KRef): Message[] { const result: Message[] = []; - const queue = queueStore.provideStoredQueue(kpid, false); + const queue = provideStoredQueue(kpid, false); for (;;) { const message = queue.dequeue() as Message; if (message) { @@ -192,13 +186,6 @@ export function makePromiseStore( } } - /** - * - */ - function reset(): void { - nextPromiseId = baseStore.provideCachedStoredValue('nextPromiseId', '1'); - } - return { // Promise lifecycle initKernelPromise, @@ -214,8 +201,5 @@ export function makePromiseStore( // Promise messaging enqueuePromiseMessage, getKernelPromiseMessageQueue, - - // Reset - reset, }; } diff --git a/packages/kernel/src/store/queue-store.ts b/packages/kernel/src/store/methods/queue.ts similarity index 65% rename from packages/kernel/src/store/queue-store.ts rename to packages/kernel/src/store/methods/queue.ts index 13e2153e9..68a2ff5d6 100644 --- a/packages/kernel/src/store/queue-store.ts +++ b/packages/kernel/src/store/methods/queue.ts @@ -1,29 +1,19 @@ import type { KVStore } from '@ocap/store'; -import type { makeBaseStore } from './base-store.ts'; -import type { RunQueueItem } from '../types.ts'; - -export type StoredQueue = { - enqueue(item: object): void; - dequeue(): object | undefined; - delete(): void; -}; +import { getBaseMethods } from './base.ts'; +import type { StoredQueue } from '../types.ts'; /** - * Create a queue store object that provides functionality for managing queues. + * Get a queue store object that provides functionality for managing queues. * - * @param kv - The key-value store to use for persistent storage. - * @param baseStore - The base store to use for the queue store. + * @param kv - A key/value store to provide the underlying persistence mechanism. * @returns A queue store object that maps various persistent kernel data * structures onto `kv`. */ // eslint-disable-next-line @typescript-eslint/explicit-function-return-type -export function makeQueueStore( - kv: KVStore, - baseStore: ReturnType, -) { - /** The kernel's run queue. */ - let runQueue = createStoredQueue('run', true); +export function getQueueMethods(kv: KVStore) { + const { provideCachedStoredValue, provideRawStoredValue, incCounter } = + getBaseMethods(kv); /** * Create a new (empty) persistently stored queue. @@ -58,8 +48,8 @@ export function makeQueueStore( const qk = `queue.${queueName}`; // Note: cached=true ==> caches only the head & tail indices, NOT the queue entries themselves const provideValue = cached - ? baseStore.provideCachedStoredValue - : baseStore.provideRawStoredValue; + ? provideCachedStoredValue + : provideRawStoredValue; const head = provideValue(`${qk}.head`); const tail = provideValue(`${qk}.tail`); if (head.get() === undefined || tail.get() === undefined) { @@ -70,7 +60,7 @@ export function makeQueueStore( if (head.get() === undefined) { throw Error(`enqueue into deleted queue ${queueName}`); } - const entryPos = baseStore.incCounter(head); + const entryPos = incCounter(head); kv.set(`${qk}.${entryPos}`, JSON.stringify(item)); }, dequeue(): object | undefined { @@ -82,7 +72,7 @@ export function makeQueueStore( if (tailPos !== headPos) { const entry = kv.getRequired(`${qk}.${tailPos}`); kv.delete(`${qk}.${tailPos}`); - baseStore.incCounter(tail); + incCounter(tail); return JSON.parse(entry) as object; } return undefined; @@ -102,15 +92,6 @@ export function makeQueueStore( }; } - /** - * Append a message to the kernel's run queue. - * - * @param message - The message to enqueue. - */ - function enqueueRun(message: RunQueueItem): void { - runQueue.enqueue(message); - } - /** * Find out how long some queue is. * @@ -128,44 +109,9 @@ export function makeQueueStore( return Number(head) - Number(tail); } - /** - * Fetch the next message on the kernel's run queue. - * - * @returns The next message on the run queue, or undefined if the queue is - * empty. - */ - function dequeueRun(): RunQueueItem | undefined { - return runQueue.dequeue() as RunQueueItem | undefined; - } - - /** - * Obtain the number of entries in the run queue. - * - * @returns the number of items in the run queue. - */ - function runQueueLength(): number { - return getQueueLength('run'); - } - - /** - * - */ - function reset(): void { - runQueue = createStoredQueue('run', true); - } - return { - // Queue createStoredQueue, provideStoredQueue, getQueueLength, - - // Run queue operations - enqueueRun, - dequeueRun, - runQueueLength, - - // Reset - reset, }; } diff --git a/packages/kernel/src/store/refcount-store.ts b/packages/kernel/src/store/methods/refcount.ts similarity index 90% rename from packages/kernel/src/store/refcount-store.ts rename to packages/kernel/src/store/methods/refcount.ts index e62327740..a0a264573 100644 --- a/packages/kernel/src/store/refcount-store.ts +++ b/packages/kernel/src/store/methods/refcount.ts @@ -1,16 +1,17 @@ -import type { KVStore } from '@ocap/store'; - -import type { KRef } from '../types.ts'; +import type { KRef } from '../../types.ts'; +import type { StoreContext } from '../types.ts'; /** * Create a refcount store object that provides functionality for managing reference counts. * - * @param kv - The key-value store to use for persistent storage. + * @param context - The store context. * @returns A refcount store object that maps various persistent kernel data * structures onto `kv`. */ // eslint-disable-next-line @typescript-eslint/explicit-function-return-type -export function makeRefCountStore(kv: KVStore) { +export function getRefCountMethods(context: StoreContext) { + const { kv } = context; + /** * Generate the storage key for a kernel entity's reference count. * diff --git a/packages/kernel/src/store/types.ts b/packages/kernel/src/store/types.ts new file mode 100644 index 000000000..3b3ed2d67 --- /dev/null +++ b/packages/kernel/src/store/types.ts @@ -0,0 +1,25 @@ +import type { KVStore } from '@ocap/store'; + +import type { KRef } from '../types.ts'; + +export type StoreContext = { + kv: KVStore; + runQueue: StoredQueue; + nextObjectId: StoredValue; + nextPromiseId: StoredValue; + maybeFreeKrefs: Set; + gcActions: StoredValue; + reapQueue: StoredValue; +}; + +export type StoredValue = { + get(): string | undefined; + set(newValue: string): void; + delete(): void; +}; + +export type StoredQueue = { + enqueue(item: object): void; + dequeue(): object | undefined; + delete(): void; +}; From 823effb5b5cf0708a71cbdd499c1b96468f78f8e Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Wed, 12 Mar 2025 14:25:15 -0300 Subject: [PATCH 10/21] Fix context --- packages/kernel/src/store/index.ts | 71 +++++++++---------- packages/kernel/src/store/methods/clist.ts | 42 +++++------ packages/kernel/src/store/methods/gc.ts | 23 +++--- packages/kernel/src/store/methods/id.ts | 20 +----- packages/kernel/src/store/methods/object.ts | 17 +++-- packages/kernel/src/store/methods/promise.ts | 41 ++++++----- packages/kernel/src/store/methods/refcount.ts | 18 +++-- packages/kernel/src/store/types.ts | 2 + 8 files changed, 108 insertions(+), 126 deletions(-) diff --git a/packages/kernel/src/store/index.ts b/packages/kernel/src/store/index.ts index 9f6b57d4a..f8dddfde0 100644 --- a/packages/kernel/src/store/index.ts +++ b/packages/kernel/src/store/index.ts @@ -90,37 +90,32 @@ export function makeKernelStore(kdb: KernelDatabase) { const { provideCachedStoredValue } = getBaseMethods(kv); const queue = getQueueMethods(kv); - /** The kernel's run queue. */ - const runQueue = queue.createStoredQueue('run', true); - /** Counter for allocating kernel object IDs */ - const nextObjectId = provideCachedStoredValue('nextObjectId', '1'); - /** Counter for allocating kernel promise IDs */ - const nextPromiseId = provideCachedStoredValue('nextPromiseId', '1'); - - // As refcounts are decremented, we accumulate a set of krefs for which - // action might need to be taken: - // * promises which are now resolved and unreferenced can be deleted - // * objects which are no longer reachable: export can be dropped - // * objects which are no longer recognizable: export can be retired - // This set is ephemeral: it lives in RAM, grows as deliveries and syscalls - // cause decrefs, and will be harvested by processRefcounts(). This needs to be - // called in the same transaction window as the syscalls/etc which prompted - // the change, else removals might be lost (not performed during the next - // replay). - const maybeFreeKrefs = new Set(); - - // Garbage collection - const gcActions = provideCachedStoredValue('gcActions', '[]'); - const reapQueue = provideCachedStoredValue('reapQueue', '[]'); - const context: StoreContext = { kv, - runQueue, - nextObjectId, - nextPromiseId, - maybeFreeKrefs, - gcActions, - reapQueue, + /** The kernel's run queue. */ + runQueue: queue.createStoredQueue('run', true), + /** Counter for allocating kernel object IDs */ + nextObjectId: provideCachedStoredValue('nextObjectId', '1'), + /** Counter for allocating kernel promise IDs */ + nextPromiseId: provideCachedStoredValue('nextPromiseId', '1'), + /** Counter for allocating VatIDs */ + nextVatId: provideCachedStoredValue('nextVatId', '1'), + /** Counter for allocating RemoteIDs */ + nextRemoteId: provideCachedStoredValue('nextRemoteId', '1'), + // As refcounts are decremented, we accumulate a set of krefs for which + // action might need to be taken: + // * promises which are now resolved and unreferenced can be deleted + // * objects which are no longer reachable: export can be dropped + // * objects which are no longer recognizable: export can be retired + // This set is ephemeral: it lives in RAM, grows as deliveries and syscalls + // cause decrefs, and will be harvested by processRefcounts(). This needs to be + // called in the same transaction window as the syscalls/etc which prompted + // the change, else removals might be lost (not performed during the next + // replay). + maybeFreeKrefs: new Set(), + // Garbage collection + gcActions: provideCachedStoredValue('gcActions', '[]'), + reapQueue: provideCachedStoredValue('reapQueue', '[]'), }; const id = getIdMethods(context); @@ -158,13 +153,6 @@ export function makeKernelStore(kdb: KernelDatabase) { return queue.getQueueLength('run'); } - /** - * Delete everything from the database. - */ - function clear(): void { - kdb.clear(); - } - /** * Create a new VatStore for a vat. * @@ -177,7 +165,7 @@ export function makeKernelStore(kdb: KernelDatabase) { } /** - * Reset the kernel's persistent queues and counters. + * Reset the kernel's persistent state and reset all counters. */ function reset(): void { kdb.clear(); @@ -187,6 +175,15 @@ export function makeKernelStore(kdb: KernelDatabase) { context.reapQueue = provideCachedStoredValue('reapQueue', '[]'); context.nextObjectId = provideCachedStoredValue('nextObjectId', '1'); context.nextPromiseId = provideCachedStoredValue('nextPromiseId', '1'); + context.nextVatId = provideCachedStoredValue('nextVatId', '1'); + context.nextRemoteId = provideCachedStoredValue('nextRemoteId', '1'); + } + + /** + * Delete everything from the database. + */ + function clear(): void { + kdb.clear(); } return harden({ diff --git a/packages/kernel/src/store/methods/clist.ts b/packages/kernel/src/store/methods/clist.ts index 603031dfe..ef5076e4e 100644 --- a/packages/kernel/src/store/methods/clist.ts +++ b/packages/kernel/src/store/methods/clist.ts @@ -21,8 +21,7 @@ import { */ // eslint-disable-next-line @typescript-eslint/explicit-function-return-type export function getCListMethods(ctx: StoreContext) { - const { kv, maybeFreeKrefs } = ctx; - const { getSlotKey } = getBaseMethods(kv); + const { getSlotKey } = getBaseMethods(ctx.kv); const { clearReachableFlag } = getGCMethods(ctx); const { getObjectRefCount, setObjectRefCount } = getObjectMethods(ctx); const { kernelRefExists, refCountKey } = getRefCountMethods(ctx); @@ -37,8 +36,11 @@ export function getCListMethods(ctx: StoreContext) { * @param eref - The ERef. */ function addClistEntry(endpointId: EndpointId, kref: KRef, eref: ERef): void { - kv.set(getSlotKey(endpointId, kref), buildReachableAndVatSlot(true, eref)); - kv.set(getSlotKey(endpointId, eref), kref); + ctx.kv.set( + getSlotKey(endpointId, kref), + buildReachableAndVatSlot(true, eref), + ); + ctx.kv.set(getSlotKey(endpointId, eref), kref); } /** @@ -49,7 +51,7 @@ export function getCListMethods(ctx: StoreContext) { * @returns true iff this vat has a c-list entry mapping for `slot`. */ function hasCListEntry(endpointId: EndpointId, slot: string): boolean { - return kv.get(getSlotKey(endpointId, slot)) !== undefined; + return ctx.kv.get(getSlotKey(endpointId, slot)) !== undefined; } /** @@ -66,15 +68,15 @@ export function getCListMethods(ctx: StoreContext) { ): void { const kernelKey = getSlotKey(endpointId, kref); const vatKey = getSlotKey(endpointId, eref); - assert(kv.get(kernelKey)); + assert(ctx.kv.get(kernelKey)); clearReachableFlag(endpointId, kref); const { direction } = parseRef(eref); decrementRefCount(kref, { isExport: direction === 'export', onlyRecognizable: true, }); - kv.delete(kernelKey); - kv.delete(vatKey); + ctx.kv.delete(kernelKey); + ctx.kv.delete(vatKey); } /** @@ -91,12 +93,12 @@ export function getCListMethods(ctx: StoreContext) { const refTag = endpointId.startsWith('v') ? '' : endpointId[0]; let refType; if (isPromiseRef(kref)) { - id = kv.get(`e.nextPromiseId.${endpointId}`); - kv.set(`e.nextPromiseId.${endpointId}`, `${Number(id) + 1}`); + id = ctx.kv.get(`e.nextPromiseId.${endpointId}`); + ctx.kv.set(`e.nextPromiseId.${endpointId}`, `${Number(id) + 1}`); refType = 'p'; } else { - id = kv.get(`e.nextObjectId.${endpointId}`); - kv.set(`e.nextObjectId.${endpointId}`, `${Number(id) + 1}`); + id = ctx.kv.get(`e.nextObjectId.${endpointId}`); + ctx.kv.set(`e.nextObjectId.${endpointId}`, `${Number(id) + 1}`); refType = 'o'; } const eref = `${refTag}${refType}-${id}`; @@ -113,7 +115,7 @@ export function getCListMethods(ctx: StoreContext) { * if there is no such mapping. */ function erefToKref(endpointId: EndpointId, eref: ERef): KRef | undefined { - return kv.get(getSlotKey(endpointId, eref)); + return ctx.kv.get(getSlotKey(endpointId, eref)); } /** @@ -126,7 +128,7 @@ export function getCListMethods(ctx: StoreContext) { */ function krefToEref(endpointId: EndpointId, kref: KRef): ERef | undefined { const key = getSlotKey(endpointId, kref); - const data = kv.get(key); + const data = ctx.kv.get(key); if (!data) { return undefined; } @@ -183,8 +185,8 @@ export function getCListMethods(ctx: StoreContext) { const { isPromise } = parseRef(kref); if (isPromise) { - const refCount = Number(kv.get(refCountKey(kref))) + 1; - kv.set(refCountKey(kref), `${refCount}`); + const refCount = Number(ctx.kv.get(refCountKey(kref))) + 1; + ctx.kv.set(refCountKey(kref), `${refCount}`); return; } @@ -222,12 +224,12 @@ export function getCListMethods(ctx: StoreContext) { const { isPromise } = parseRef(kref); if (isPromise) { - let refCount = Number(kv.get(refCountKey(kref))); + let refCount = Number(ctx.kv.get(refCountKey(kref))); refCount > 0 || Fail`refCount underflow ${kref}`; refCount -= 1; - kv.set(refCountKey(kref), `${refCount}`); + ctx.kv.set(refCountKey(kref), `${refCount}`); if (refCount === 0) { - maybeFreeKrefs.add(kref); + ctx.maybeFreeKrefs.add(kref); return true; } return false; @@ -243,7 +245,7 @@ export function getCListMethods(ctx: StoreContext) { } counts.recognizable -= 1; if (!counts.reachable || !counts.recognizable) { - maybeFreeKrefs.add(kref); + ctx.maybeFreeKrefs.add(kref); } setObjectRefCount(kref, counts); diff --git a/packages/kernel/src/store/methods/gc.ts b/packages/kernel/src/store/methods/gc.ts index 0f24aa0d8..3f2c6753f 100644 --- a/packages/kernel/src/store/methods/gc.ts +++ b/packages/kernel/src/store/methods/gc.ts @@ -29,8 +29,7 @@ import { */ // eslint-disable-next-line @typescript-eslint/explicit-function-return-type export function getGCMethods(ctx: StoreContext) { - const { kv, maybeFreeKrefs, gcActions, reapQueue } = ctx; - const { getSlotKey } = getBaseMethods(kv); + const { getSlotKey } = getBaseMethods(ctx.kv); const { getObjectRefCount, setObjectRefCount } = getObjectMethods(ctx); const { kernelRefExists } = getRefCountMethods(ctx); @@ -40,7 +39,7 @@ export function getGCMethods(ctx: StoreContext) { * @returns The set of GC actions to perform. */ function getGCActions(): Set { - return new Set(JSON.parse(gcActions.get() ?? '[]')); + return new Set(JSON.parse(ctx.gcActions.get() ?? '[]')); } /** @@ -51,7 +50,7 @@ export function getGCMethods(ctx: StoreContext) { function setGCActions(actions: Set): void { const a = Array.from(actions); a.sort(); - gcActions.set(JSON.stringify(a)); + ctx.gcActions.set(JSON.stringify(a)); } /** @@ -81,7 +80,7 @@ export function getGCMethods(ctx: StoreContext) { */ function getReachableFlag(endpointId: EndpointId, kref: KRef): boolean { const key = getSlotKey(endpointId, kref); - const data = kv.getRequired(key); + const data = ctx.kv.getRequired(key); const { isReachable } = parseReachableAndVatSlot(data); return isReachable; } @@ -95,9 +94,9 @@ export function getGCMethods(ctx: StoreContext) { function clearReachableFlag(endpointId: EndpointId, kref: KRef): void { const key = getSlotKey(endpointId, kref); const { isReachable, vatSlot } = parseReachableAndVatSlot( - kv.getRequired(key), + ctx.kv.getRequired(key), ); - kv.set(key, buildReachableAndVatSlot(false, vatSlot)); + ctx.kv.set(key, buildReachableAndVatSlot(false, vatSlot)); const { direction, isPromise } = parseRef(vatSlot); // decrement 'reachable' part of refcount, but only for object imports if ( @@ -110,7 +109,7 @@ export function getGCMethods(ctx: StoreContext) { counts.reachable -= 1; setObjectRefCount(kref, counts); if (counts.reachable === 0) { - maybeFreeKrefs.add(kref); + ctx.maybeFreeKrefs.add(kref); } } } @@ -121,10 +120,10 @@ export function getGCMethods(ctx: StoreContext) { * @param vatId - The vat to schedule for reaping. */ function scheduleReap(vatId: VatId): void { - const queue = JSON.parse(reapQueue.get() ?? '[]'); + const queue = JSON.parse(ctx.reapQueue.get() ?? '[]'); if (!queue.includes(vatId)) { queue.push(vatId); - reapQueue.set(JSON.stringify(queue)); + ctx.reapQueue.set(JSON.stringify(queue)); } } @@ -134,10 +133,10 @@ export function getGCMethods(ctx: StoreContext) { * @returns The next reap action, or undefined if the queue is empty. */ function nextReapAction(): RunQueueItemBringOutYourDead | undefined { - const queue = JSON.parse(reapQueue.get() ?? '[]'); + const queue = JSON.parse(ctx.reapQueue.get() ?? '[]'); if (queue.length > 0) { const vatId = queue.shift(); - reapQueue.set(JSON.stringify(queue)); + ctx.reapQueue.set(JSON.stringify(queue)); return harden({ type: RunQueueItemType.bringOutYourDead, vatId }); } return undefined; diff --git a/packages/kernel/src/store/methods/id.ts b/packages/kernel/src/store/methods/id.ts index fa7171815..964839825 100644 --- a/packages/kernel/src/store/methods/id.ts +++ b/packages/kernel/src/store/methods/id.ts @@ -11,12 +11,7 @@ import type { StoreContext } from '../types.ts'; // eslint-disable-next-line @typescript-eslint/explicit-function-return-type export function getIdMethods(ctx: StoreContext) { const { kv } = ctx; - const { provideCachedStoredValue, incCounter } = getBaseMethods(kv); - - /** Counter for allocating VatIDs */ - let nextVatId = provideCachedStoredValue('nextVatId', '1'); - /** Counter for allocating RemoteIDs */ - let nextRemoteId = provideCachedStoredValue('nextRemoteId', '1'); + const { incCounter } = getBaseMethods(kv); /** * Obtain an ID for a new vat. @@ -24,7 +19,7 @@ export function getIdMethods(ctx: StoreContext) { * @returns The next VatID use. */ function getNextVatId(): VatId { - return `v${incCounter(nextVatId)}`; + return `v${incCounter(ctx.nextVatId)}`; } /** @@ -33,7 +28,7 @@ export function getIdMethods(ctx: StoreContext) { * @returns The next remote ID use. */ function getNextRemoteId(): RemoteId { - return `r${incCounter(nextRemoteId)}`; + return `r${incCounter(ctx.nextRemoteId)}`; } /** @@ -46,18 +41,9 @@ export function getIdMethods(ctx: StoreContext) { kv.set(`e.nextObjectId.${endpointId}`, '1'); } - /** - * Clear the kernel's persistent state and reset all counters. - */ - function reset(): void { - nextVatId = provideCachedStoredValue('nextVatId', '1'); - nextRemoteId = provideCachedStoredValue('nextRemoteId', '1'); - } - return { getNextVatId, getNextRemoteId, initEndpoint, - reset, }; } diff --git a/packages/kernel/src/store/methods/object.ts b/packages/kernel/src/store/methods/object.ts index 15da1af7f..e69a1e107 100644 --- a/packages/kernel/src/store/methods/object.ts +++ b/packages/kernel/src/store/methods/object.ts @@ -15,8 +15,7 @@ import { makeKernelSlot } from '../utils/kernel-slots.ts'; */ // eslint-disable-next-line @typescript-eslint/explicit-function-return-type export function getObjectMethods(ctx: StoreContext) { - const { kv, nextObjectId } = ctx; - const { incCounter } = getBaseMethods(kv); + const { incCounter } = getBaseMethods(ctx.kv); const { refCountKey } = getRefCountMethods(ctx); /** @@ -29,7 +28,7 @@ export function getObjectMethods(ctx: StoreContext) { */ function initKernelObject(owner: EndpointId): KRef { const koId = getNextObjectId(); - kv.set(`${koId}.owner`, owner); + ctx.kv.set(`${koId}.owner`, owner); setObjectRefCount(koId, { reachable: 1, recognizable: 1 }); return koId; } @@ -41,7 +40,7 @@ export function getObjectMethods(ctx: StoreContext) { * @returns The identity of the vat or remote that owns the object. */ function getOwner(koId: KRef): EndpointId { - const owner = kv.get(`${koId}.owner`); + const owner = ctx.kv.get(`${koId}.owner`); if (owner === undefined) { throw Error(`unknown kernel object ${koId}`); } @@ -54,8 +53,8 @@ export function getObjectMethods(ctx: StoreContext) { * @param koId - The KRef of the kernel object to delete. */ function deleteKernelObject(koId: KRef): void { - kv.delete(`${koId}.owner`); - kv.delete(refCountKey(koId)); + ctx.kv.delete(`${koId}.owner`); + ctx.kv.delete(refCountKey(koId)); } /** @@ -64,7 +63,7 @@ export function getObjectMethods(ctx: StoreContext) { * @returns The next koId use. */ function getNextObjectId(): KRef { - return makeKernelSlot('object', incCounter(nextObjectId)); + return makeKernelSlot('object', incCounter(ctx.nextObjectId)); } /** @@ -77,7 +76,7 @@ export function getObjectMethods(ctx: StoreContext) { reachable: number; recognizable: number; } { - const data = kv.get(refCountKey(kref)); + const data = ctx.kv.get(refCountKey(kref)); if (!data) { return { reachable: 0, recognizable: 0 }; } @@ -106,7 +105,7 @@ export function getObjectMethods(ctx: StoreContext) { Fail`${kref} underflow ${reachable},${recognizable}`; reachable <= recognizable || Fail`refMismatch(set) ${kref} ${reachable},${recognizable}`; - kv.set(refCountKey(kref), `${reachable},${recognizable}`); + ctx.kv.set(refCountKey(kref), `${reachable},${recognizable}`); } return { diff --git a/packages/kernel/src/store/methods/promise.ts b/packages/kernel/src/store/methods/promise.ts index b3e4b5dc4..6cbd1c044 100644 --- a/packages/kernel/src/store/methods/promise.ts +++ b/packages/kernel/src/store/methods/promise.ts @@ -20,8 +20,7 @@ import { parseRef } from '../utils/parse-ref.ts'; */ // eslint-disable-next-line @typescript-eslint/explicit-function-return-type export function getPromiseMethods(ctx: StoreContext) { - const { kv, nextPromiseId } = ctx; - const { incCounter } = getBaseMethods(kv); + const { incCounter } = getBaseMethods(ctx.kv); const { createStoredQueue, provideStoredQueue } = getQueueMethods(ctx.kv); const { refCountKey } = getRefCountMethods(ctx); @@ -40,9 +39,9 @@ export function getPromiseMethods(ctx: StoreContext) { }; const kpid = getNextPromiseId(); createStoredQueue(kpid, false); - kv.set(`${kpid}.state`, 'unresolved'); - kv.set(`${kpid}.subscribers`, '[]'); - kv.set(refCountKey(kpid), '1'); + ctx.kv.set(`${kpid}.state`, 'unresolved'); + ctx.kv.set(`${kpid}.subscribers`, '[]'); + ctx.kv.set(refCountKey(kpid), '1'); return [kpid, kpr]; } @@ -55,24 +54,24 @@ export function getPromiseMethods(ctx: StoreContext) { function getKernelPromise(kpid: KRef): KernelPromise { const { context, isPromise } = parseRef(kpid); assert(context === 'kernel' && isPromise); - const state = kv.get(`${kpid}.state`) as PromiseState; + const state = ctx.kv.get(`${kpid}.state`) as PromiseState; if (state === undefined) { throw Error(`unknown kernel promise ${kpid}`); } const result: KernelPromise = { state }; switch (state as string) { case 'unresolved': { - const decider = kv.get(`${kpid}.decider`); + const decider = ctx.kv.get(`${kpid}.decider`); if (decider !== '' && decider !== undefined) { result.decider = decider; } - const subscribers = kv.getRequired(`${kpid}.subscribers`); + const subscribers = ctx.kv.getRequired(`${kpid}.subscribers`); result.subscribers = JSON.parse(subscribers); break; } case 'fulfilled': case 'rejected': { - result.value = JSON.parse(kv.getRequired(`${kpid}.value`)); + result.value = JSON.parse(ctx.kv.getRequired(`${kpid}.value`)); break; } default: @@ -87,11 +86,11 @@ export function getPromiseMethods(ctx: StoreContext) { * @param kpid - The KRef of the kernel promise to delete. */ function deleteKernelPromise(kpid: KRef): void { - kv.delete(`${kpid}.state`); - kv.delete(`${kpid}.decider`); - kv.delete(`${kpid}.subscribers`); - kv.delete(`${kpid}.value`); - kv.delete(refCountKey(kpid)); + ctx.kv.delete(`${kpid}.state`); + ctx.kv.delete(`${kpid}.decider`); + ctx.kv.delete(`${kpid}.subscribers`); + ctx.kv.delete(`${kpid}.value`); + ctx.kv.delete(refCountKey(kpid)); provideStoredQueue(kpid).delete(); } @@ -101,7 +100,7 @@ export function getPromiseMethods(ctx: StoreContext) { * @returns The next kpid use. */ function getNextPromiseId(): KRef { - return makeKernelSlot('promise', incCounter(nextPromiseId)); + return makeKernelSlot('promise', incCounter(ctx.nextPromiseId)); } /** @@ -119,7 +118,7 @@ export function getPromiseMethods(ctx: StoreContext) { tempSet.add(vatId); const newSubscribers = Array.from(tempSet).sort(); const key = `${kpid}.subscribers`; - kv.set(key, JSON.stringify(newSubscribers)); + ctx.kv.set(key, JSON.stringify(newSubscribers)); } /** @@ -131,7 +130,7 @@ export function getPromiseMethods(ctx: StoreContext) { function setPromiseDecider(kpid: KRef, vatId: VatId): void { insistVatId(vatId); if (kpid) { - kv.set(`${kpid}.decider`, vatId); + ctx.kv.set(`${kpid}.decider`, vatId); } } @@ -151,10 +150,10 @@ export function getPromiseMethods(ctx: StoreContext) { for (const message of getKernelPromiseMessageQueue(kpid)) { queue.enqueue(message); } - kv.set(`${kpid}.state`, rejected ? 'rejected' : 'fulfilled'); - kv.set(`${kpid}.value`, JSON.stringify(value)); - kv.delete(`${kpid}.decider`); - kv.delete(`${kpid}.subscribers`); + ctx.kv.set(`${kpid}.state`, rejected ? 'rejected' : 'fulfilled'); + ctx.kv.set(`${kpid}.value`, JSON.stringify(value)); + ctx.kv.delete(`${kpid}.decider`); + ctx.kv.delete(`${kpid}.subscribers`); } /** diff --git a/packages/kernel/src/store/methods/refcount.ts b/packages/kernel/src/store/methods/refcount.ts index a0a264573..5a8c040d8 100644 --- a/packages/kernel/src/store/methods/refcount.ts +++ b/packages/kernel/src/store/methods/refcount.ts @@ -4,14 +4,12 @@ import type { StoreContext } from '../types.ts'; /** * Create a refcount store object that provides functionality for managing reference counts. * - * @param context - The store context. + * @param ctx - The store context. * @returns A refcount store object that maps various persistent kernel data * structures onto `kv`. */ // eslint-disable-next-line @typescript-eslint/explicit-function-return-type -export function getRefCountMethods(context: StoreContext) { - const { kv } = context; - +export function getRefCountMethods(ctx: StoreContext) { /** * Generate the storage key for a kernel entity's reference count. * @@ -29,7 +27,7 @@ export function getRefCountMethods(context: StoreContext) { * @returns the reference count of the indicated kernel entity. */ function getRefCount(kref: KRef): number { - return Number(kv.get(refCountKey(kref))); + return Number(ctx.kv.get(refCountKey(kref))); } /** @@ -40,8 +38,8 @@ export function getRefCountMethods(context: StoreContext) { */ function incRefCount(kref: KRef): number { const key = refCountKey(kref); - const newCount = Number(kv.get(key)) + 1; - kv.set(key, `${newCount}`); + const newCount = Number(ctx.kv.get(key)) + 1; + ctx.kv.set(key, `${newCount}`); return newCount; } @@ -53,8 +51,8 @@ export function getRefCountMethods(context: StoreContext) { */ function decRefCount(kref: KRef): number { const key = refCountKey(kref); - const newCount = Number(kv.get(key)) - 1; - kv.set(key, `${newCount}`); + const newCount = Number(ctx.kv.get(key)) - 1; + ctx.kv.set(key, `${newCount}`); return newCount; } @@ -65,7 +63,7 @@ export function getRefCountMethods(context: StoreContext) { * @returns True if the kernel object exists, false otherwise. */ function kernelRefExists(kref: KRef): boolean { - return Boolean(kv.get(refCountKey(kref))); + return Boolean(ctx.kv.get(refCountKey(kref))); } return { diff --git a/packages/kernel/src/store/types.ts b/packages/kernel/src/store/types.ts index 3b3ed2d67..75aac5bc9 100644 --- a/packages/kernel/src/store/types.ts +++ b/packages/kernel/src/store/types.ts @@ -7,6 +7,8 @@ export type StoreContext = { runQueue: StoredQueue; nextObjectId: StoredValue; nextPromiseId: StoredValue; + nextVatId: StoredValue; + nextRemoteId: StoredValue; maybeFreeKrefs: Set; gcActions: StoredValue; reapQueue: StoredValue; From a3e55189cc2c915f97ee3f31cdfd43bec9ffad7f Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Wed, 12 Mar 2025 14:59:47 -0300 Subject: [PATCH 11/21] add more tests --- .../kernel/src/store/methods/clist.test.ts | 258 +++++++++++ packages/kernel/src/store/methods/gc.test.ts | 135 ++++++ packages/kernel/src/store/methods/id.test.ts | 172 ++++++++ .../kernel/src/store/methods/object.test.ts | 278 ++++++++++++ .../kernel/src/store/methods/promise.test.ts | 400 ++++++++++++++++++ .../kernel/src/store/methods/queue.test.ts | 276 ++++++++++++ .../kernel/src/store/methods/refcount.test.ts | 164 +++++++ 7 files changed, 1683 insertions(+) create mode 100644 packages/kernel/src/store/methods/clist.test.ts create mode 100644 packages/kernel/src/store/methods/gc.test.ts create mode 100644 packages/kernel/src/store/methods/id.test.ts create mode 100644 packages/kernel/src/store/methods/object.test.ts create mode 100644 packages/kernel/src/store/methods/promise.test.ts create mode 100644 packages/kernel/src/store/methods/queue.test.ts create mode 100644 packages/kernel/src/store/methods/refcount.test.ts diff --git a/packages/kernel/src/store/methods/clist.test.ts b/packages/kernel/src/store/methods/clist.test.ts new file mode 100644 index 000000000..bcb71ce2a --- /dev/null +++ b/packages/kernel/src/store/methods/clist.test.ts @@ -0,0 +1,258 @@ +import type { KVStore } from '@ocap/store'; +import { describe, it, expect, beforeEach, vi } from 'vitest'; + +import { getCListMethods } from './clist.ts'; +import { makeMapKVStore } from '../../../test/storage.ts'; +import type { EndpointId, KRef, ERef } from '../../types.ts'; +import type { StoreContext } from '../types.ts'; + +describe('clist-methods', () => { + let kv: KVStore; + let clistMethods: ReturnType; + let maybeFreeKrefs: Set; + + // Mock dependencies + const mockGetObjectRefCount = vi.fn(); + const mockSetObjectRefCount = vi.fn(); + const mockClearReachableFlag = vi.fn(); + + beforeEach(() => { + kv = makeMapKVStore(); + maybeFreeKrefs = new Set(); + + // Initialize endpoint counters + kv.set('e.nextPromiseId.v1', '1'); + kv.set('e.nextObjectId.v1', '1'); + kv.set('e.nextPromiseId.r1', '1'); + kv.set('e.nextObjectId.r1', '1'); + + // Reset mocks + mockGetObjectRefCount.mockReset(); + mockSetObjectRefCount.mockReset(); + mockClearReachableFlag.mockReset(); + + // Default mock implementations + mockGetObjectRefCount.mockImplementation(() => ({ + reachable: 1, + recognizable: 1, + })); + + // Create the store with mocked dependencies + clistMethods = getCListMethods({ + kv, + maybeFreeKrefs, + } as StoreContext); + }); + + describe('addClistEntry', () => { + it('adds a bidirectional mapping between KRef and ERef', () => { + const endpointId: EndpointId = 'v1'; + const kref: KRef = 'ko1'; + const eref: ERef = 'o-1'; + + clistMethods.addClistEntry(endpointId, kref, eref); + + // Check that both mappings are stored + expect(kv.get(`${endpointId}.c.${kref}`)).toBe(`R ${eref}`); + expect(kv.get(`${endpointId}.c.${eref}`)).toBe(kref); + }); + + it('works with promise refs', () => { + const endpointId: EndpointId = 'v1'; + const kref: KRef = 'kp1'; + const eref: ERef = 'p+2'; + + clistMethods.addClistEntry(endpointId, kref, eref); + + expect(kv.get(`${endpointId}.c.${kref}`)).toBe(`R ${eref}`); + expect(kv.get(`${endpointId}.c.${eref}`)).toBe(kref); + }); + + it('works with remote endpoints', () => { + const endpointId: EndpointId = 'r1'; + const kref: KRef = 'ko2'; + const eref: ERef = 'ro+3'; + + clistMethods.addClistEntry(endpointId, kref, eref); + + expect(kv.get(`${endpointId}.c.${kref}`)).toBe(`R ${eref}`); + expect(kv.get(`${endpointId}.c.${eref}`)).toBe(kref); + }); + }); + + describe('hasCListEntry', () => { + it('returns true for existing entries', () => { + const endpointId: EndpointId = 'v1'; + const kref: KRef = 'ko1'; + const eref: ERef = 'o-1'; + + clistMethods.addClistEntry(endpointId, kref, eref); + + expect(clistMethods.hasCListEntry(endpointId, kref)).toBe(true); + expect(clistMethods.hasCListEntry(endpointId, eref)).toBe(true); + }); + + it('returns false for non-existent entries', () => { + const endpointId: EndpointId = 'v1'; + + expect(clistMethods.hasCListEntry(endpointId, 'ko99')).toBe(false); + expect(clistMethods.hasCListEntry(endpointId, 'o-99')).toBe(false); + }); + }); + + describe('allocateErefForKref', () => { + it('allocates a new object ERef for a KRef', () => { + const endpointId: EndpointId = 'v1'; + const kref: KRef = 'ko1'; + + const eref = clistMethods.allocateErefForKref(endpointId, kref); + + // Check the allocated ERef format + expect(eref).toBe('o-1'); + + // Check that the counter was incremented + expect(kv.get(`e.nextObjectId.${endpointId}`)).toBe('2'); + + // Check that the mapping was added + expect(kv.get(`${endpointId}.c.${kref}`)).toBe(`R ${eref}`); + expect(kv.get(`${endpointId}.c.${eref}`)).toBe(kref); + }); + + it('allocates a new promise ERef for a KRef', () => { + const endpointId: EndpointId = 'v1'; + const kref: KRef = 'kp1'; + + const eref = clistMethods.allocateErefForKref(endpointId, kref); + + // Check the allocated ERef format + expect(eref).toBe('p-1'); + + // Check that the counter was incremented + expect(kv.get(`e.nextPromiseId.${endpointId}`)).toBe('2'); + + // Check that the mapping was added + expect(kv.get(`${endpointId}.c.${kref}`)).toBe(`R ${eref}`); + expect(kv.get(`${endpointId}.c.${eref}`)).toBe(kref); + }); + + it('allocates ERefs with remote prefix for remote endpoints', () => { + const endpointId: EndpointId = 'r1'; + const kref: KRef = 'ko1'; + + const eref = clistMethods.allocateErefForKref(endpointId, kref); + + // Check the allocated ERef format (should have 'r' prefix) + expect(eref).toBe('ro-1'); + + // Check that the counter was incremented + expect(kv.get(`e.nextObjectId.${endpointId}`)).toBe('2'); + }); + }); + + describe('erefToKref and krefToEref', () => { + it('converts between ERef and KRef', () => { + const endpointId: EndpointId = 'v1'; + const kref: KRef = 'ko1'; + const eref: ERef = 'o-1'; + + clistMethods.addClistEntry(endpointId, kref, eref); + + expect(clistMethods.erefToKref(endpointId, eref)).toBe(kref); + expect(clistMethods.krefToEref(endpointId, kref)).toBe(eref); + }); + + it('returns undefined for non-existent mappings', () => { + const endpointId: EndpointId = 'v1'; + + expect(clistMethods.erefToKref(endpointId, 'o-99')).toBeUndefined(); + expect(clistMethods.krefToEref(endpointId, 'ko99')).toBeUndefined(); + }); + }); + + describe('incrementRefCount', () => { + it('increments promise reference counts', () => { + const kref: KRef = 'kp1'; + + // Set up initial refCount + kv.set(`${kref}.refCount`, '1'); + + clistMethods.incrementRefCount(kref, {}); + + // Check that the refCount was incremented + expect(kv.get(`${kref}.refCount`)).toBe('2'); + }); + + it('does not increment object counts for exports', () => { + const kref: KRef = 'ko1'; + + clistMethods.incrementRefCount(kref, { isExport: true }); + + // Should not call getObjectRefCount or setObjectRefCount + expect(mockGetObjectRefCount).not.toHaveBeenCalled(); + expect(mockSetObjectRefCount).not.toHaveBeenCalled(); + }); + + it('throws for empty kref', () => { + expect(() => clistMethods.incrementRefCount('' as KRef, {})).toThrow( + 'incrementRefCount called with empty kref', + ); + }); + }); + + describe('decrementRefCount', () => { + it('decrements promise reference counts', () => { + const kref: KRef = 'kp1'; + + // Set up initial refCount + kv.set(`${kref}.refCount`, '2'); + + const result = clistMethods.decrementRefCount(kref, {}); + + // Check that the refCount was decremented + expect(kv.get(`${kref}.refCount`)).toBe('1'); + expect(result).toBe(false); // Not zero yet + }); + + it('adds promise to maybeFreeKrefs when count reaches zero', () => { + const kref: KRef = 'kp1'; + + // Set up initial refCount + kv.set(`${kref}.refCount`, '1'); + + const result = clistMethods.decrementRefCount(kref, {}); + + // Check that the refCount was decremented to zero + expect(kv.get(`${kref}.refCount`)).toBe('0'); + expect(result).toBe(true); // Now zero + expect(maybeFreeKrefs.has(kref)).toBe(true); + }); + + it('does not decrement object counts for exports', () => { + const kref: KRef = 'ko1'; + + const result = clistMethods.decrementRefCount(kref, { isExport: true }); + + // Should not call getObjectRefCount or setObjectRefCount + expect(mockGetObjectRefCount).not.toHaveBeenCalled(); + expect(mockSetObjectRefCount).not.toHaveBeenCalled(); + expect(result).toBe(false); + }); + + it('throws for empty kref', () => { + expect(() => clistMethods.decrementRefCount('' as KRef, {})).toThrow( + 'decrementRefCount called with empty kref', + ); + }); + + it('throws for underflow on promise refCount', () => { + const kref: KRef = 'kp1'; + + // Set up initial refCount at 0 + kv.set(`${kref}.refCount`, '0'); + + expect(() => clistMethods.decrementRefCount(kref, {})).toThrow( + /refCount underflow/u, + ); + }); + }); +}); diff --git a/packages/kernel/src/store/methods/gc.test.ts b/packages/kernel/src/store/methods/gc.test.ts new file mode 100644 index 000000000..9289172aa --- /dev/null +++ b/packages/kernel/src/store/methods/gc.test.ts @@ -0,0 +1,135 @@ +import { describe, it, expect, beforeEach } from 'vitest'; + +import { makeMapKVStore } from '../../../test/storage.ts'; +import { RunQueueItemType } from '../../types.ts'; +import type { GCAction } from '../../types.ts'; +import { makeKernelStore } from '../index.ts'; + +describe('GC methods', () => { + let kernelStore: ReturnType; + + beforeEach(() => { + const mockKVStore = makeMapKVStore(); + kernelStore = makeKernelStore(mockKVStore); + }); + + describe('GC actions', () => { + it('manages all valid GC action types', () => { + const ko1 = kernelStore.initKernelObject('v1'); + const ko2 = kernelStore.initKernelObject('v1'); + const ko3 = kernelStore.initKernelObject('v2'); + + const validActions: GCAction[] = [ + `v1 dropExport ${ko1}`, + `v1 retireExport ${ko2}`, + `v2 retireImport ${ko3}`, + ]; + + kernelStore.addGCActions(validActions); + + const actions = kernelStore.getGCActions(); + expect(actions.size).toBe(3); + expect(actions).toStrictEqual(new Set(validActions)); + }); + + it('rejects invalid GC actions', () => { + const ko1 = kernelStore.initKernelObject('v1'); + + // Invalid vat ID + expect(() => { + kernelStore.addGCActions(['x1 dropExport ko1']); + }).toThrow('not a valid VatId'); + + // Invalid action type + expect(() => { + kernelStore.addGCActions([`v1 invalidAction ${ko1}`] as GCAction[]); + }).toThrow('not a valid GCActionType "invalidAction"'); + + // Invalid kref (must be kernel object, not promise) + expect(() => { + kernelStore.addGCActions(['v1 dropExport kp1']); + }).toThrow('kernelSlot "kp1" is not of type "object"'); + + // Malformed action string + expect(() => { + kernelStore.addGCActions(['v1 dropExport'] as unknown as GCAction[]); + }).toThrow('kernelSlot is undefined'); + }); + + it('maintains action order when storing', () => { + const ko1 = kernelStore.initKernelObject('v1'); + const ko2 = kernelStore.initKernelObject('v2'); + const ko3 = kernelStore.initKernelObject('v3'); + + const actions = [ + `v3 retireImport ${ko3}`, + `v1 dropExport ${ko1}`, + `v2 retireExport ${ko2}`, + ]; + + kernelStore.setGCActions(new Set(actions) as Set); + + // Actions should be sorted when retrieved + const sortedActions = Array.from(kernelStore.getGCActions()); + expect(sortedActions).toStrictEqual([ + `v1 dropExport ${ko1}`, + `v2 retireExport ${ko2}`, + `v3 retireImport ${ko3}`, + ]); + }); + }); + + describe('reachability tracking', () => { + it('manages reachable flags', () => { + const ko1 = kernelStore.initKernelObject('v1'); + kernelStore.addClistEntry('v1', ko1, 'o-1'); + + expect(kernelStore.getReachableFlag('v1', ko1)).toBe(true); + + kernelStore.clearReachableFlag('v1', ko1); + expect(kernelStore.getReachableFlag('v1', ko1)).toBe(false); + + const refCounts = kernelStore.getObjectRefCount(ko1); + expect(refCounts.reachable).toBe(0); + }); + }); + + describe('reaping', () => { + it('processes reap queue in order', () => { + const vatIds = ['v1', 'v2', 'v3']; + + // Schedule multiple vats for reaping + vatIds.forEach((vatId) => kernelStore.scheduleReap(vatId)); + + // Verify they are processed in order + vatIds.forEach((vatId) => { + expect(kernelStore.nextReapAction()).toStrictEqual({ + type: RunQueueItemType.bringOutYourDead, + vatId, + }); + }); + + // Queue should be empty after processing all items + expect(kernelStore.nextReapAction()).toBeUndefined(); + }); + + it('handles duplicate reap scheduling', () => { + kernelStore.scheduleReap('v1'); + kernelStore.scheduleReap('v1'); // Duplicate scheduling + kernelStore.scheduleReap('v2'); + + // Should only process v1 once + expect(kernelStore.nextReapAction()).toStrictEqual({ + type: RunQueueItemType.bringOutYourDead, + vatId: 'v1', + }); + + expect(kernelStore.nextReapAction()).toStrictEqual({ + type: RunQueueItemType.bringOutYourDead, + vatId: 'v2', + }); + + expect(kernelStore.nextReapAction()).toBeUndefined(); + }); + }); +}); diff --git a/packages/kernel/src/store/methods/id.test.ts b/packages/kernel/src/store/methods/id.test.ts new file mode 100644 index 000000000..2c507db25 --- /dev/null +++ b/packages/kernel/src/store/methods/id.test.ts @@ -0,0 +1,172 @@ +import type { KVStore } from '@ocap/store'; +import { describe, it, expect, beforeEach } from 'vitest'; + +import { getIdMethods } from './id.ts'; +import { makeMapKVStore } from '../../../test/storage.ts'; +import type { EndpointId } from '../../types.ts'; +import type { StoreContext } from '../types.ts'; + +describe('id-methods', () => { + let kv: KVStore; + let idStore: ReturnType; + let nextVatId: { get: () => string; set: (value: string) => void }; + let nextRemoteId: { get: () => string; set: (value: string) => void }; + + beforeEach(() => { + kv = makeMapKVStore(); + + // Initialize ID counters + kv.set('nextVatId', '0'); + kv.set('nextRemoteId', '0'); + + nextVatId = { + get: () => kv.get('nextVatId') ?? '0', + set: (value: string) => kv.set('nextVatId', value), + }; + + nextRemoteId = { + get: () => kv.get('nextRemoteId') ?? '0', + set: (value: string) => kv.set('nextRemoteId', value), + }; + + idStore = getIdMethods({ + kv, + nextVatId, + nextRemoteId, + } as StoreContext); + }); + + describe('getNextVatId', () => { + it('returns sequential vat IDs', () => { + expect(idStore.getNextVatId()).toBe('v0'); + expect(idStore.getNextVatId()).toBe('v1'); + expect(idStore.getNextVatId()).toBe('v2'); + }); + + it('increments the vat ID counter', () => { + idStore.getNextVatId(); + expect(kv.get('nextVatId')).toBe('1'); + + idStore.getNextVatId(); + expect(kv.get('nextVatId')).toBe('2'); + }); + + it('continues from existing counter value', () => { + // Set an existing counter value + kv.set('nextVatId', '42'); + + expect(idStore.getNextVatId()).toBe('v42'); + expect(idStore.getNextVatId()).toBe('v43'); + }); + }); + + describe('getNextRemoteId', () => { + it('returns sequential remote IDs', () => { + expect(idStore.getNextRemoteId()).toBe('r0'); + expect(idStore.getNextRemoteId()).toBe('r1'); + expect(idStore.getNextRemoteId()).toBe('r2'); + }); + + it('increments the remote ID counter', () => { + idStore.getNextRemoteId(); + expect(kv.get('nextRemoteId')).toBe('1'); + + idStore.getNextRemoteId(); + expect(kv.get('nextRemoteId')).toBe('2'); + }); + + it('continues from existing counter value', () => { + // Set an existing counter value + kv.set('nextRemoteId', '99'); + + expect(idStore.getNextRemoteId()).toBe('r99'); + expect(idStore.getNextRemoteId()).toBe('r100'); + }); + }); + + describe('initEndpoint', () => { + it('initializes a vat endpoint', () => { + const vatId: EndpointId = 'v1'; + + idStore.initEndpoint(vatId); + + expect(kv.get(`e.nextPromiseId.${vatId}`)).toBe('1'); + expect(kv.get(`e.nextObjectId.${vatId}`)).toBe('1'); + }); + + it('initializes a remote endpoint', () => { + const remoteId: EndpointId = 'r2'; + + idStore.initEndpoint(remoteId); + + expect(kv.get(`e.nextPromiseId.${remoteId}`)).toBe('1'); + expect(kv.get(`e.nextObjectId.${remoteId}`)).toBe('1'); + }); + + it('can initialize multiple endpoints', () => { + idStore.initEndpoint('v1'); + idStore.initEndpoint('v2'); + idStore.initEndpoint('r1'); + + expect(kv.get('e.nextPromiseId.v1')).toBe('1'); + expect(kv.get('e.nextObjectId.v1')).toBe('1'); + + expect(kv.get('e.nextPromiseId.v2')).toBe('1'); + expect(kv.get('e.nextObjectId.v2')).toBe('1'); + + expect(kv.get('e.nextPromiseId.r1')).toBe('1'); + expect(kv.get('e.nextObjectId.r1')).toBe('1'); + }); + + it('can reinitialize an existing endpoint', () => { + const vatId: EndpointId = 'v3'; + + // Initialize with default values + idStore.initEndpoint(vatId); + + // Modify the values + kv.set(`e.nextPromiseId.${vatId}`, '10'); + kv.set(`e.nextObjectId.${vatId}`, '20'); + + // Reinitialize + idStore.initEndpoint(vatId); + + // Values should be reset to 1 + expect(kv.get(`e.nextPromiseId.${vatId}`)).toBe('1'); + expect(kv.get(`e.nextObjectId.${vatId}`)).toBe('1'); + }); + }); + + describe('integration', () => { + it('supports creating and initializing endpoints', () => { + // Get new vat and remote IDs + const vatId = idStore.getNextVatId(); + const remoteId = idStore.getNextRemoteId(); + + // Initialize them as endpoints + idStore.initEndpoint(vatId); + idStore.initEndpoint(remoteId); + + // Check that they're properly initialized + expect(kv.get(`e.nextPromiseId.${vatId}`)).toBe('1'); + expect(kv.get(`e.nextObjectId.${vatId}`)).toBe('1'); + + expect(kv.get(`e.nextPromiseId.${remoteId}`)).toBe('1'); + expect(kv.get(`e.nextObjectId.${remoteId}`)).toBe('1'); + }); + + it('maintains separate counters for vats and remotes', () => { + // Generate multiple IDs of each type + const vatId1 = idStore.getNextVatId(); + const vatId2 = idStore.getNextVatId(); + const remoteId1 = idStore.getNextRemoteId(); + const remoteId2 = idStore.getNextRemoteId(); + + // Check that they follow the expected sequence + expect(vatId1).toBe('v0'); + expect(vatId2).toBe('v1'); + expect(remoteId1).toBe('r0'); + expect(remoteId2).toBe('r1'); + }); + }); +}); diff --git a/packages/kernel/src/store/methods/object.test.ts b/packages/kernel/src/store/methods/object.test.ts new file mode 100644 index 000000000..6d81a4145 --- /dev/null +++ b/packages/kernel/src/store/methods/object.test.ts @@ -0,0 +1,278 @@ +import type { KVStore } from '@ocap/store'; +import { describe, it, expect, beforeEach } from 'vitest'; + +import { getObjectMethods } from './object.ts'; +import { makeMapKVStore } from '../../../test/storage.ts'; +import type { EndpointId } from '../../types.ts'; +import type { StoreContext } from '../types.ts'; + +describe('object-methods', () => { + let kv: KVStore; + let objectStore: ReturnType; + let nextObjectId: { get: () => string; set: (value: string) => void }; + + beforeEach(() => { + kv = makeMapKVStore(); + // Initialize nextObjectId counter + kv.set('nextObjectId', '0'); + nextObjectId = { + get: () => kv.get('nextObjectId') ?? '0', + set: (value: string) => kv.set('nextObjectId', value), + }; + + objectStore = getObjectMethods({ + kv, + nextObjectId, + } as StoreContext); + }); + + describe('initKernelObject', () => { + it('creates a new kernel object with initial reference counts', () => { + const owner: EndpointId = 'v1'; + const koId = objectStore.initKernelObject(owner); + + // Check the object ID format + expect(koId).toBe('ko0'); + + // Check the owner is set correctly + expect(kv.get(`${koId}.owner`)).toBe(owner); + + // Check reference counts are initialized to 1,1 + expect(kv.get(`${koId}.refCount`)).toBe('1,1'); + + // Check via the API + const refCounts = objectStore.getObjectRefCount(koId); + expect(refCounts.reachable).toBe(1); + expect(refCounts.recognizable).toBe(1); + }); + + it('increments the object ID counter', () => { + const koId1 = objectStore.initKernelObject('v1'); + const koId2 = objectStore.initKernelObject('v2'); + const koId3 = objectStore.initKernelObject('r1'); + + expect(koId1).toBe('ko0'); + expect(koId2).toBe('ko1'); + expect(koId3).toBe('ko2'); + }); + }); + + describe('getOwner', () => { + it('returns the owner of a kernel object', () => { + const owner1: EndpointId = 'v1'; + const owner2: EndpointId = 'r2'; + + const koId1 = objectStore.initKernelObject(owner1); + const koId2 = objectStore.initKernelObject(owner2); + + expect(objectStore.getOwner(koId1)).toBe(owner1); + expect(objectStore.getOwner(koId2)).toBe(owner2); + }); + + it('throws for unknown kernel objects', () => { + expect(() => objectStore.getOwner('ko99')).toThrow( + 'unknown kernel object ko99', + ); + }); + }); + + describe('deleteKernelObject', () => { + it('removes a kernel object from storage', () => { + const koId = objectStore.initKernelObject('v1'); + + // Object exists before deletion + expect(kv.get(`${koId}.owner`)).toBeDefined(); + expect(kv.get(`${koId}.refCount`)).toBeDefined(); + + // Delete the object + objectStore.deleteKernelObject(koId); + + // Object should be gone + expect(kv.get(`${koId}.owner`)).toBeUndefined(); + expect(kv.get(`${koId}.refCount`)).toBeUndefined(); + + // getOwner should throw + expect(() => objectStore.getOwner(koId)).toThrow( + `unknown kernel object ${koId}`, + ); + }); + }); + + describe('getNextObjectId', () => { + it('returns sequential object IDs', () => { + expect(objectStore.getNextObjectId()).toBe('ko0'); + expect(objectStore.getNextObjectId()).toBe('ko1'); + expect(objectStore.getNextObjectId()).toBe('ko2'); + }); + }); + + describe('getObjectRefCount', () => { + it('returns reference counts for existing objects', () => { + const koId = objectStore.initKernelObject('v1'); + + const refCounts = objectStore.getObjectRefCount(koId); + expect(refCounts).toStrictEqual({ reachable: 1, recognizable: 1 }); + }); + + it('returns zero counts for non-existent objects', () => { + const refCounts = objectStore.getObjectRefCount('ko99'); + expect(refCounts).toStrictEqual({ reachable: 0, recognizable: 0 }); + }); + + it('parses reference counts correctly', () => { + const koId = objectStore.initKernelObject('v1'); + + // Manually set reference counts + kv.set(`${koId}.refCount`, '5,10'); + + const refCounts = objectStore.getObjectRefCount(koId); + expect(refCounts).toStrictEqual({ reachable: 5, recognizable: 10 }); + }); + }); + + describe('setObjectRefCount', () => { + it('sets reference counts for an object', () => { + const koId = objectStore.initKernelObject('v1'); + + // Set new reference counts + objectStore.setObjectRefCount(koId, { reachable: 3, recognizable: 5 }); + + // Check via direct KV access + expect(kv.get(`${koId}.refCount`)).toBe('3,5'); + + // Check via API + const refCounts = objectStore.getObjectRefCount(koId); + expect(refCounts).toStrictEqual({ reachable: 3, recognizable: 5 }); + }); + + it('allows zero reference counts', () => { + const koId = objectStore.initKernelObject('v1'); + + objectStore.setObjectRefCount(koId, { reachable: 0, recognizable: 0 }); + + const refCounts = objectStore.getObjectRefCount(koId); + expect(refCounts).toStrictEqual({ reachable: 0, recognizable: 0 }); + }); + + it('throws when reachable count exceeds recognizable count', () => { + const koId = objectStore.initKernelObject('v1'); + + expect(() => + objectStore.setObjectRefCount(koId, { reachable: 5, recognizable: 3 }), + ).toThrow(/refMismatch/u); + }); + + it('throws when counts are negative', () => { + const koId = objectStore.initKernelObject('v1'); + + expect(() => + objectStore.setObjectRefCount(koId, { reachable: -1, recognizable: 1 }), + ).toThrow(/underflow/u); + + expect(() => + objectStore.setObjectRefCount(koId, { reachable: 0, recognizable: -1 }), + ).toThrow(/underflow/u); + }); + }); + + describe('integration', () => { + it('supports the full kernel object lifecycle', () => { + // Create an object + const koId = objectStore.initKernelObject('v1'); + + // Check initial state + expect(objectStore.getOwner(koId)).toBe('v1'); + expect(objectStore.getObjectRefCount(koId)).toStrictEqual({ + reachable: 1, + recognizable: 1, + }); + + // Update reference counts + objectStore.setObjectRefCount(koId, { reachable: 2, recognizable: 3 }); + expect(objectStore.getObjectRefCount(koId)).toStrictEqual({ + reachable: 2, + recognizable: 3, + }); + + // Reduce to zero + objectStore.setObjectRefCount(koId, { reachable: 0, recognizable: 0 }); + expect(objectStore.getObjectRefCount(koId)).toStrictEqual({ + reachable: 0, + recognizable: 0, + }); + + // Object still exists with zero counts + expect(objectStore.getOwner(koId)).toBe('v1'); + + // Delete the object + objectStore.deleteKernelObject(koId); + + // Object should be gone + expect(() => objectStore.getOwner(koId)).toThrow( + `unknown kernel object ${koId}`, + ); + expect(objectStore.getObjectRefCount(koId)).toStrictEqual({ + reachable: 0, + recognizable: 0, + }); + }); + + it('handles multiple objects simultaneously', () => { + // Create multiple objects with different owners + const koId1 = objectStore.initKernelObject('v1'); + const koId2 = objectStore.initKernelObject('r2'); + + // Check owners + expect(objectStore.getOwner(koId1)).toBe('v1'); + expect(objectStore.getOwner(koId2)).toBe('r2'); + + // Set different reference counts + objectStore.setObjectRefCount(koId1, { reachable: 2, recognizable: 2 }); + objectStore.setObjectRefCount(koId2, { reachable: 3, recognizable: 5 }); + + // Check counts + expect(objectStore.getObjectRefCount(koId1)).toStrictEqual({ + reachable: 2, + recognizable: 2, + }); + expect(objectStore.getObjectRefCount(koId2)).toStrictEqual({ + reachable: 3, + recognizable: 5, + }); + + // Delete one object + objectStore.deleteKernelObject(koId1); + + // First object should be gone, second still exists + expect(() => objectStore.getOwner(koId1)).toThrow( + `unknown kernel object ${koId1}`, + ); + expect(objectStore.getOwner(koId2)).toBe('r2'); + }); + + it('handles reference count patterns correctly', () => { + const koId = objectStore.initKernelObject('v1'); + + // Equal reachable and recognizable + objectStore.setObjectRefCount(koId, { reachable: 5, recognizable: 5 }); + expect(objectStore.getObjectRefCount(koId)).toStrictEqual({ + reachable: 5, + recognizable: 5, + }); + + // Reachable less than recognizable + objectStore.setObjectRefCount(koId, { reachable: 3, recognizable: 5 }); + expect(objectStore.getObjectRefCount(koId)).toStrictEqual({ + reachable: 3, + recognizable: 5, + }); + + // Both zero + objectStore.setObjectRefCount(koId, { reachable: 0, recognizable: 0 }); + expect(objectStore.getObjectRefCount(koId)).toStrictEqual({ + reachable: 0, + recognizable: 0, + }); + }); + }); +}); diff --git a/packages/kernel/src/store/methods/promise.test.ts b/packages/kernel/src/store/methods/promise.test.ts new file mode 100644 index 000000000..96b34d0af --- /dev/null +++ b/packages/kernel/src/store/methods/promise.test.ts @@ -0,0 +1,400 @@ +import type { Message } from '@agoric/swingset-liveslots'; +import type { CapData } from '@endo/marshal'; +import type { KVStore } from '@ocap/store'; +import { describe, it, expect, beforeEach } from 'vitest'; + +import { getPromiseMethods } from './promise.ts'; +import { makeMapKVStore } from '../../../test/storage.ts'; +import type { KRef } from '../../types.ts'; +import type { StoreContext } from '../types.ts'; + +/** + * Mock Message: A helper to allow simple objects to be used as Messages for testing + * + * @param obj - An object to use as a message. + * @returns The same object coerced to type Message. + */ +function mockMessage(obj: object): Message { + return obj as unknown as Message; +} + +describe('promise-methods', () => { + let kv: KVStore; + let promiseStore: ReturnType; + let nextPromiseId: { get: () => string; set: (value: string) => void }; + + beforeEach(() => { + kv = makeMapKVStore(); + // Initialize nextPromiseId counter + kv.set('nextPromiseId', '0'); + nextPromiseId = { + get: () => kv.get('nextPromiseId') ?? '0', + set: (value: string) => kv.set('nextPromiseId', value), + }; + + promiseStore = getPromiseMethods({ + kv, + nextPromiseId, + } as StoreContext); + }); + + describe('initKernelPromise', () => { + it('creates a new unresolved kernel promise', () => { + const [kpid, kp] = promiseStore.initKernelPromise(); + + // Check the returned promise + expect(kpid).toBe('kp0'); + expect(kp).toStrictEqual({ + state: 'unresolved', + subscribers: [], + }); + + // Check the stored promise + expect(kv.get(`${kpid}.state`)).toBe('unresolved'); + expect(kv.get(`${kpid}.subscribers`)).toBe('[]'); + expect(kv.get(`${kpid}.refCount`)).toBe('1'); + }); + + it('increments the promise ID counter', () => { + const [kpid1] = promiseStore.initKernelPromise(); + const [kpid2] = promiseStore.initKernelPromise(); + const [kpid3] = promiseStore.initKernelPromise(); + + expect(kpid1).toBe('kp0'); + expect(kpid2).toBe('kp1'); + expect(kpid3).toBe('kp2'); + }); + }); + + describe('getKernelPromise', () => { + it('retrieves an unresolved promise', () => { + const [kpid] = promiseStore.initKernelPromise(); + const kp = promiseStore.getKernelPromise(kpid); + + expect(kp).toStrictEqual({ + state: 'unresolved', + subscribers: [], + }); + }); + + it('retrieves an unresolved promise with decider', () => { + const [kpid] = promiseStore.initKernelPromise(); + promiseStore.setPromiseDecider(kpid, 'v1'); + + const kp = promiseStore.getKernelPromise(kpid); + expect(kp).toStrictEqual({ + state: 'unresolved', + decider: 'v1', + subscribers: [], + }); + }); + + it('retrieves a fulfilled promise', () => { + const [kpid] = promiseStore.initKernelPromise(); + const value: CapData = { body: 'fulfilled-value', slots: [] }; + + promiseStore.resolveKernelPromise(kpid, false, value); + + const kp = promiseStore.getKernelPromise(kpid); + expect(kp).toStrictEqual({ + state: 'fulfilled', + value, + }); + }); + + it('retrieves a rejected promise', () => { + const [kpid] = promiseStore.initKernelPromise(); + const value: CapData = { body: 'error-message', slots: [] }; + + promiseStore.resolveKernelPromise(kpid, true, value); + + const kp = promiseStore.getKernelPromise(kpid); + expect(kp).toStrictEqual({ + state: 'rejected', + value, + }); + }); + + it('throws for unknown promises', () => { + expect(() => promiseStore.getKernelPromise('kp99')).toThrow( + 'unknown kernel promise kp99', + ); + }); + }); + + describe('deleteKernelPromise', () => { + it('removes a promise from storage', () => { + const [kpid] = promiseStore.initKernelPromise(); + + // Add a message to the promise queue + promiseStore.enqueuePromiseMessage( + kpid, + mockMessage({ test: 'message' }), + ); + + // Delete the promise + promiseStore.deleteKernelPromise(kpid); + + // Check that all promise data is gone + expect(kv.get(`${kpid}.state`)).toBeUndefined(); + expect(kv.get(`${kpid}.subscribers`)).toBeUndefined(); + expect(kv.get(`${kpid}.refCount`)).toBeUndefined(); + + // Check that the promise queue is gone + expect(() => + promiseStore.enqueuePromiseMessage(kpid, mockMessage({})), + ).toThrow(`queue ${kpid} not initialized`); + }); + }); + + describe('getNextPromiseId', () => { + it('returns sequential promise IDs', () => { + expect(promiseStore.getNextPromiseId()).toBe('kp0'); + expect(promiseStore.getNextPromiseId()).toBe('kp1'); + expect(promiseStore.getNextPromiseId()).toBe('kp2'); + }); + }); + + describe('addPromiseSubscriber', () => { + it('adds a subscriber to an unresolved promise', () => { + const [kpid] = promiseStore.initKernelPromise(); + + promiseStore.addPromiseSubscriber('v1', kpid); + + const kp = promiseStore.getKernelPromise(kpid); + expect(kp.subscribers).toStrictEqual(['v1']); + + // Add another subscriber + promiseStore.addPromiseSubscriber('v2', kpid); + + const kpUpdated = promiseStore.getKernelPromise(kpid); + expect(kpUpdated.subscribers).toStrictEqual(['v1', 'v2']); + }); + + it('does not add duplicate subscribers', () => { + const [kpid] = promiseStore.initKernelPromise(); + + promiseStore.addPromiseSubscriber('v1', kpid); + promiseStore.addPromiseSubscriber('v1', kpid); + + const kp = promiseStore.getKernelPromise(kpid); + expect(kp.subscribers).toStrictEqual(['v1']); + }); + + it('throws when adding a subscriber to a resolved promise', () => { + const [kpid] = promiseStore.initKernelPromise(); + + // Resolve the promise + promiseStore.resolveKernelPromise(kpid, false, { + body: 'value', + slots: [], + }); + + // Try to add a subscriber + expect(() => promiseStore.addPromiseSubscriber('v1', kpid)).toThrow( + /attempt to add subscriber to resolved promise/u, + ); + }); + }); + + describe('setPromiseDecider', () => { + it('sets the decider for a promise', () => { + const [kpid] = promiseStore.initKernelPromise(); + + promiseStore.setPromiseDecider(kpid, 'v3'); + + const kp = promiseStore.getKernelPromise(kpid); + expect(kp.decider).toBe('v3'); + }); + + it('updates the decider for a promise', () => { + const [kpid] = promiseStore.initKernelPromise(); + + promiseStore.setPromiseDecider(kpid, 'v1'); + promiseStore.setPromiseDecider(kpid, 'v2'); + + const kp = promiseStore.getKernelPromise(kpid); + expect(kp.decider).toBe('v2'); + }); + }); + + describe('resolveKernelPromise', () => { + it('fulfills a promise', () => { + const [kpid] = promiseStore.initKernelPromise(); + const value: CapData = { body: 'fulfilled-value', slots: ['ko1'] }; + + promiseStore.resolveKernelPromise(kpid, false, value); + + const kp = promiseStore.getKernelPromise(kpid); + expect(kp.state).toBe('fulfilled'); + expect(kp.value).toStrictEqual(value); + }); + + it('rejects a promise', () => { + const [kpid] = promiseStore.initKernelPromise(); + const value: CapData = { body: 'error-message', slots: [] }; + + promiseStore.resolveKernelPromise(kpid, true, value); + + const kp = promiseStore.getKernelPromise(kpid); + expect(kp.state).toBe('rejected'); + expect(kp.value).toStrictEqual(value); + }); + + it('clears decider and subscribers when resolving', () => { + const [kpid] = promiseStore.initKernelPromise(); + + // Add decider and subscribers + promiseStore.setPromiseDecider(kpid, 'v1'); + promiseStore.addPromiseSubscriber('v2', kpid); + + // Resolve the promise + promiseStore.resolveKernelPromise(kpid, false, { + body: 'value', + slots: [], + }); + + // Check that decider and subscribers are gone + expect(kv.get(`${kpid}.decider`)).toBeUndefined(); + expect(kv.get(`${kpid}.subscribers`)).toBeUndefined(); + }); + + it('preserves queued messages when resolving', () => { + const [kpid] = promiseStore.initKernelPromise(); + + // Add messages to the queue + const message1 = mockMessage({ id: 1 }); + const message2 = mockMessage({ id: 2 }); + + promiseStore.enqueuePromiseMessage(kpid, message1); + promiseStore.enqueuePromiseMessage(kpid, message2); + + // Resolve the promise + promiseStore.resolveKernelPromise(kpid, false, { + body: 'value', + slots: [], + }); + + // Check that messages are still in the queue + const messages = promiseStore.getKernelPromiseMessageQueue(kpid); + expect(messages).toHaveLength(2); + expect(messages[0]).toStrictEqual(message1); + expect(messages[1]).toStrictEqual(message2); + }); + }); + + describe('enqueuePromiseMessage and getKernelPromiseMessageQueue', () => { + it('enqueues and retrieves messages', () => { + const [kpid] = promiseStore.initKernelPromise(); + + const message1 = mockMessage({ id: 1, data: 'first' }); + const message2 = mockMessage({ id: 2, data: 'second' }); + + promiseStore.enqueuePromiseMessage(kpid, message1); + promiseStore.enqueuePromiseMessage(kpid, message2); + + const messages = promiseStore.getKernelPromiseMessageQueue(kpid); + + expect(messages).toHaveLength(2); + expect(messages[0]).toStrictEqual(message1); + expect(messages[1]).toStrictEqual(message2); + }); + + it('empties the queue when retrieving messages', () => { + const [kpid] = promiseStore.initKernelPromise(); + + promiseStore.enqueuePromiseMessage(kpid, mockMessage({ id: 1 })); + promiseStore.enqueuePromiseMessage(kpid, mockMessage({ id: 2 })); + + // First call gets all messages + const messages1 = promiseStore.getKernelPromiseMessageQueue(kpid); + expect(messages1).toHaveLength(2); + + // Second call gets empty array + const messages2 = promiseStore.getKernelPromiseMessageQueue(kpid); + expect(messages2).toHaveLength(0); + }); + + it('throws when enqueueing to a non-existent promise', () => { + expect(() => + promiseStore.enqueuePromiseMessage('kp99', mockMessage({})), + ).toThrow('queue kp99 not initialized'); + }); + }); + + describe('integration', () => { + it('supports the full promise lifecycle', () => { + // Create a promise + const [kpid] = promiseStore.initKernelPromise(); + + // Add subscribers and decider + promiseStore.addPromiseSubscriber('v1', kpid); + promiseStore.addPromiseSubscriber('v2', kpid); + promiseStore.setPromiseDecider(kpid, 'v3'); + + // Add messages + promiseStore.enqueuePromiseMessage(kpid, mockMessage({ id: 1 })); + + // Check the promise state + let kp = promiseStore.getKernelPromise(kpid); + expect(kp.state).toBe('unresolved'); + expect(kp.subscribers).toStrictEqual(['v1', 'v2']); + expect(kp.decider).toBe('v3'); + + // Resolve the promise + const value: CapData = { body: 'final-value', slots: ['ko5'] }; + promiseStore.resolveKernelPromise(kpid, false, value); + + // Check the resolved state + kp = promiseStore.getKernelPromise(kpid); + expect(kp.state).toBe('fulfilled'); + expect(kp.value).toStrictEqual(value); + + // Messages should still be available + const messages = promiseStore.getKernelPromiseMessageQueue(kpid); + expect(messages).toHaveLength(1); + + // Delete the promise + promiseStore.deleteKernelPromise(kpid); + + // Promise should be gone + expect(() => promiseStore.getKernelPromise(kpid)).toThrow( + `unknown kernel promise ${kpid}`, + ); + }); + + it('handles multiple promises simultaneously', () => { + // Create two promises + const [kpid1] = promiseStore.initKernelPromise(); + const [kpid2] = promiseStore.initKernelPromise(); + + // Set up different states + promiseStore.addPromiseSubscriber('v1', kpid1); + promiseStore.setPromiseDecider(kpid2, 'v2'); + + // Add messages to both + promiseStore.enqueuePromiseMessage(kpid1, mockMessage({ for: 'kp1' })); + promiseStore.enqueuePromiseMessage(kpid2, mockMessage({ for: 'kp2' })); + + // Resolve one promise + promiseStore.resolveKernelPromise(kpid1, false, { + body: 'resolved', + slots: [], + }); + + // Check states + const kp1 = promiseStore.getKernelPromise(kpid1); + const kp2 = promiseStore.getKernelPromise(kpid2); + + expect(kp1.state).toBe('fulfilled'); + expect(kp2.state).toBe('unresolved'); + + // Messages should be preserved + const messages1 = promiseStore.getKernelPromiseMessageQueue(kpid1); + const messages2 = promiseStore.getKernelPromiseMessageQueue(kpid2); + + expect(messages1).toHaveLength(1); + expect(messages2).toHaveLength(1); + }); + }); +}); diff --git a/packages/kernel/src/store/methods/queue.test.ts b/packages/kernel/src/store/methods/queue.test.ts new file mode 100644 index 000000000..aa7a51096 --- /dev/null +++ b/packages/kernel/src/store/methods/queue.test.ts @@ -0,0 +1,276 @@ +import type { KVStore } from '@ocap/store'; +import { describe, it, expect, beforeEach } from 'vitest'; + +import { getQueueMethods } from './queue.ts'; +import { makeMapKVStore } from '../../../test/storage.ts'; + +describe('queue-methods', () => { + let kv: KVStore; + let queueStore: ReturnType; + + beforeEach(() => { + kv = makeMapKVStore(); + queueStore = getQueueMethods(kv); + }); + + describe('createStoredQueue', () => { + it('creates a new empty queue', () => { + queueStore.createStoredQueue('test-queue'); + expect(kv.get('queue.test-queue.head')).toBe('1'); + expect(kv.get('queue.test-queue.tail')).toBe('1'); + }); + + it('creates a cached queue when specified', () => { + queueStore.createStoredQueue('cached-queue', true); + expect(kv.get('queue.cached-queue.head')).toBe('1'); + expect(kv.get('queue.cached-queue.tail')).toBe('1'); + }); + }); + + describe('provideStoredQueue', () => { + it('throws an error for uninitialized queues', () => { + expect(() => queueStore.provideStoredQueue('nonexistent')).toThrow( + 'queue nonexistent not initialized', + ); + }); + + it('provides access to an existing queue', () => { + // Create a queue first + queueStore.createStoredQueue('existing-queue'); + + // Then access it + const queue = queueStore.provideStoredQueue('existing-queue'); + expect(queue).toBeDefined(); + }); + }); + + describe('queue operations', () => { + it('enqueues and dequeues items correctly', () => { + const queue = queueStore.createStoredQueue('ops-queue'); + + // Enqueue items + queue.enqueue({ id: 1, value: 'first' }); + queue.enqueue({ id: 2, value: 'second' }); + + // Check queue length + expect(queueStore.getQueueLength('ops-queue')).toBe(2); + + // Dequeue items in FIFO order + const item1 = queue.dequeue(); + expect(item1).toStrictEqual({ id: 1, value: 'first' }); + + const item2 = queue.dequeue(); + expect(item2).toStrictEqual({ id: 2, value: 'second' }); + + // Queue should be empty now + expect(queueStore.getQueueLength('ops-queue')).toBe(0); + + // Dequeue from empty queue returns undefined + const emptyResult = queue.dequeue(); + expect(emptyResult).toBeUndefined(); + }); + + it('handles complex objects in the queue', () => { + const queue = queueStore.createStoredQueue('complex-queue'); + + const complexObject = { + id: 123, + nested: { + array: [1, 2, 3], + map: { a: 1, b: 2 }, + }, + date: new Date().toISOString(), + }; + + queue.enqueue(complexObject); + const result = queue.dequeue(); + + expect(result).toStrictEqual(complexObject); + }); + + it('deletes queues correctly', () => { + const queue = queueStore.createStoredQueue('delete-queue'); + + // Add some items + queue.enqueue({ id: 1 }); + queue.enqueue({ id: 2 }); + queue.enqueue({ id: 3 }); + + // Delete the queue + queue.delete(); + + // Queue metadata should be gone + expect(kv.get('queue.delete-queue.head')).toBeUndefined(); + expect(kv.get('queue.delete-queue.tail')).toBeUndefined(); + + // Queue entries should be gone + expect(kv.get('queue.delete-queue.1')).toBeUndefined(); + expect(kv.get('queue.delete-queue.2')).toBeUndefined(); + expect(kv.get('queue.delete-queue.3')).toBeUndefined(); + + // Operations on deleted queue should behave appropriately + expect(queue.dequeue()).toBeUndefined(); + expect(() => queue.enqueue({ id: 4 })).toThrow( + 'enqueue into deleted queue delete-queue', + ); + }); + }); + + describe('getQueueLength', () => { + it('returns the correct queue length', () => { + const queue = queueStore.createStoredQueue('length-queue'); + + expect(queueStore.getQueueLength('length-queue')).toBe(0); + + queue.enqueue({ id: 1 }); + expect(queueStore.getQueueLength('length-queue')).toBe(1); + + queue.enqueue({ id: 2 }); + queue.enqueue({ id: 3 }); + expect(queueStore.getQueueLength('length-queue')).toBe(3); + + queue.dequeue(); + expect(queueStore.getQueueLength('length-queue')).toBe(2); + + queue.dequeue(); + queue.dequeue(); + expect(queueStore.getQueueLength('length-queue')).toBe(0); + }); + + it('throws an error for unknown queues', () => { + expect(() => queueStore.getQueueLength('unknown-queue')).toThrow( + 'unknown queue unknown-queue', + ); + }); + }); + + describe('cached vs uncached queues', () => { + it('both cached and uncached queues work the same way', () => { + const cachedQueue = queueStore.createStoredQueue('cached-queue', true); + const uncachedQueue = queueStore.createStoredQueue( + 'uncached-queue', + false, + ); + + // Add same items to both queues + cachedQueue.enqueue({ id: 1 }); + uncachedQueue.enqueue({ id: 1 }); + + cachedQueue.enqueue({ id: 2 }); + uncachedQueue.enqueue({ id: 2 }); + + // Both should have same length + expect(queueStore.getQueueLength('cached-queue')).toBe(2); + expect(queueStore.getQueueLength('uncached-queue')).toBe(2); + + // Both should dequeue the same items + expect(cachedQueue.dequeue()).toStrictEqual({ id: 1 }); + expect(uncachedQueue.dequeue()).toStrictEqual({ id: 1 }); + + expect(cachedQueue.dequeue()).toStrictEqual({ id: 2 }); + expect(uncachedQueue.dequeue()).toStrictEqual({ id: 2 }); + + // Both should be empty + expect(queueStore.getQueueLength('cached-queue')).toBe(0); + expect(queueStore.getQueueLength('uncached-queue')).toBe(0); + }); + + it('cached vs uncached queues handle external changes differently', () => { + // First, let's create two separate queues + const cachedQueue = queueStore.createStoredQueue('cached-test', true); + const uncachedQueue = queueStore.createStoredQueue( + 'uncached-test', + false, + ); + + // Add an item to each queue to advance the head counter + cachedQueue.enqueue({ test: 'cached' }); + uncachedQueue.enqueue({ test: 'uncached' }); + + // Both heads should now be at 2 + expect(kv.get('queue.cached-test.head')).toBe('2'); + expect(kv.get('queue.uncached-test.head')).toBe('2'); + + // Now let's modify the KV store directly for both queues + kv.set('queue.cached-test.head', '10'); + kv.set('queue.uncached-test.head', '10'); + + // Enqueue new items + cachedQueue.enqueue({ test: 'cached-after-change' }); + uncachedQueue.enqueue({ test: 'uncached-after-change' }); + + // For the cached queue, the cached head value (2) should have been used, + // so the item should be at position 2 + expect(kv.get('queue.cached-test.2')).toBeDefined(); + + // For the uncached queue, the modified head value (10) should have been used, + // so the item should be at position 10 + expect(kv.get('queue.uncached-test.10')).toBeDefined(); + + // The heads should now be at 3 and 11 respectively + expect(kv.get('queue.cached-test.head')).toBe('3'); + expect(kv.get('queue.uncached-test.head')).toBe('11'); + }); + }); + + describe('integration', () => { + it('supports multiple queues simultaneously', () => { + const queue1 = queueStore.createStoredQueue('queue1'); + const queue2 = queueStore.createStoredQueue('queue2'); + + queue1.enqueue({ id: 'q1-1' }); + queue2.enqueue({ id: 'q2-1' }); + queue1.enqueue({ id: 'q1-2' }); + queue2.enqueue({ id: 'q2-2' }); + + expect(queueStore.getQueueLength('queue1')).toBe(2); + expect(queueStore.getQueueLength('queue2')).toBe(2); + + expect(queue1.dequeue()).toStrictEqual({ id: 'q1-1' }); + expect(queue2.dequeue()).toStrictEqual({ id: 'q2-1' }); + + expect(queueStore.getQueueLength('queue1')).toBe(1); + expect(queueStore.getQueueLength('queue2')).toBe(1); + + queue1.delete(); + expect(() => queueStore.getQueueLength('queue1')).toThrow( + 'unknown queue queue1', + ); + expect(queueStore.getQueueLength('queue2')).toBe(1); + }); + + it('handles a large number of queue operations', () => { + const queue = queueStore.createStoredQueue('large-queue'); + + // Enqueue 100 items + for (let i = 0; i < 100; i++) { + queue.enqueue({ index: i }); + } + + expect(queueStore.getQueueLength('large-queue')).toBe(100); + + // Dequeue 50 items + for (let i = 0; i < 50; i++) { + const item = queue.dequeue(); + expect(item).toStrictEqual({ index: i }); + } + + expect(queueStore.getQueueLength('large-queue')).toBe(50); + + // Enqueue 50 more + for (let i = 100; i < 150; i++) { + queue.enqueue({ index: i }); + } + + expect(queueStore.getQueueLength('large-queue')).toBe(100); + + // Dequeue all remaining + for (let i = 50; i < 150; i++) { + const item = queue.dequeue(); + expect(item).toStrictEqual({ index: i }); + } + + expect(queueStore.getQueueLength('large-queue')).toBe(0); + }); + }); +}); diff --git a/packages/kernel/src/store/methods/refcount.test.ts b/packages/kernel/src/store/methods/refcount.test.ts new file mode 100644 index 000000000..aa6e5c852 --- /dev/null +++ b/packages/kernel/src/store/methods/refcount.test.ts @@ -0,0 +1,164 @@ +import type { KVStore } from '@ocap/store'; +import { describe, it, expect, beforeEach } from 'vitest'; + +import { getRefCountMethods } from './refcount.ts'; +import { makeMapKVStore } from '../../../test/storage.ts'; +import type { KRef } from '../../types.ts'; +import type { StoreContext } from '../types.ts'; + +describe('refcount-methods', () => { + let kv: KVStore; + let refCountStore: ReturnType; + + beforeEach(() => { + kv = makeMapKVStore(); + refCountStore = getRefCountMethods({ kv } as StoreContext); + }); + + describe('refCountKey', () => { + it('generates correct reference count keys', () => { + expect(refCountStore.refCountKey('ko1')).toBe('ko1.refCount'); + expect(refCountStore.refCountKey('kp42')).toBe('kp42.refCount'); + expect(refCountStore.refCountKey('v7')).toBe('v7.refCount'); + }); + }); + + describe('getRefCount', () => { + it('returns 0 for non-existent references', () => { + kv.set(refCountStore.refCountKey('ko99'), '0'); + expect(refCountStore.getRefCount('ko99')).toBe(0); + }); + + it('handles undefined values by returning NaN', () => { + expect(Number.isNaN(refCountStore.getRefCount('nonexistent'))).toBe(true); + }); + + it('returns the correct reference count for existing references', () => { + const kref: KRef = 'ko1'; + kv.set(`${kref}.refCount`, '5'); + expect(refCountStore.getRefCount(kref)).toBe(5); + }); + }); + + describe('incRefCount', () => { + it('initializes and increments reference count for new references', () => { + const kref: KRef = 'ko1'; + + kv.set(refCountStore.refCountKey(kref), '0'); + + expect(refCountStore.incRefCount(kref)).toBe(1); + expect(kv.get(`${kref}.refCount`)).toBe('1'); + + expect(refCountStore.incRefCount(kref)).toBe(2); + expect(kv.get(`${kref}.refCount`)).toBe('2'); + }); + + it('handles incrementing from undefined', () => { + const kref: KRef = 'ko2'; + expect(Number.isNaN(refCountStore.incRefCount(kref))).toBe(true); + expect(kv.get(refCountStore.refCountKey(kref))).toBe('NaN'); + }); + + it('increments existing reference counts', () => { + const kref: KRef = 'kp42'; + kv.set(`${kref}.refCount`, '10'); + + expect(refCountStore.incRefCount(kref)).toBe(11); + expect(kv.get(`${kref}.refCount`)).toBe('11'); + }); + }); + + describe('decRefCount', () => { + it('decrements reference counts correctly', () => { + const kref: KRef = 'ko1'; + kv.set(`${kref}.refCount`, '5'); + + expect(refCountStore.decRefCount(kref)).toBe(4); + expect(kv.get(`${kref}.refCount`)).toBe('4'); + + expect(refCountStore.decRefCount(kref)).toBe(3); + expect(kv.get(`${kref}.refCount`)).toBe('3'); + }); + + it('can decrement to zero', () => { + const kref: KRef = 'ko1'; + kv.set(`${kref}.refCount`, '1'); + + expect(refCountStore.decRefCount(kref)).toBe(0); + expect(kv.get(`${kref}.refCount`)).toBe('0'); + }); + + it('can decrement below zero (though this should be avoided in practice)', () => { + const kref: KRef = 'ko1'; + kv.set(`${kref}.refCount`, '0'); + + expect(refCountStore.decRefCount(kref)).toBe(-1); + expect(kv.get(`${kref}.refCount`)).toBe('-1'); + }); + }); + + describe('kernelRefExists', () => { + it('returns false for non-existent references', () => { + expect(refCountStore.kernelRefExists('ko99')).toBe(false); + }); + + it('returns true for existing references with non-zero count', () => { + const kref: KRef = 'ko1'; + kv.set(`${kref}.refCount`, '5'); + expect(refCountStore.kernelRefExists(kref)).toBe(true); + }); + + it('returns true for existing references with zero count', () => { + const kref: KRef = 'ko1'; + kv.set(`${kref}.refCount`, '0'); + expect(refCountStore.kernelRefExists(kref)).toBe(true); + }); + }); + + describe('integration', () => { + it('supports full reference counting lifecycle', () => { + const kref: KRef = 'ko42'; + + expect(refCountStore.kernelRefExists(kref)).toBe(false); + + kv.set(refCountStore.refCountKey(kref), '0'); + expect(refCountStore.getRefCount(kref)).toBe(0); + + expect(refCountStore.kernelRefExists(kref)).toBe(true); + + expect(refCountStore.incRefCount(kref)).toBe(1); + + refCountStore.incRefCount(kref); + refCountStore.incRefCount(kref); + expect(refCountStore.getRefCount(kref)).toBe(3); + + refCountStore.decRefCount(kref); + expect(refCountStore.getRefCount(kref)).toBe(2); + + refCountStore.decRefCount(kref); + refCountStore.decRefCount(kref); + expect(refCountStore.getRefCount(kref)).toBe(0); + + expect(refCountStore.kernelRefExists(kref)).toBe(true); + }); + + it('works with multiple references simultaneously', () => { + const kref1: KRef = 'ko1'; + const kref2: KRef = 'kp2'; + + kv.set(refCountStore.refCountKey(kref1), '0'); + kv.set(refCountStore.refCountKey(kref2), '0'); + + refCountStore.incRefCount(kref1); + refCountStore.incRefCount(kref2); + refCountStore.incRefCount(kref2); + + expect(refCountStore.getRefCount(kref1)).toBe(1); + expect(refCountStore.getRefCount(kref2)).toBe(2); + + refCountStore.decRefCount(kref1); + expect(refCountStore.getRefCount(kref1)).toBe(0); + expect(refCountStore.getRefCount(kref2)).toBe(2); + }); + }); +}); From 0f7d1f0059c26b32df408e70a899069202e51499 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Wed, 12 Mar 2025 15:10:47 -0300 Subject: [PATCH 12/21] Add GC tests --- .../src/services/garbage-collection.test.ts | 257 ++++++++++++++++++ 1 file changed, 257 insertions(+) create mode 100644 packages/kernel/src/services/garbage-collection.test.ts diff --git a/packages/kernel/src/services/garbage-collection.test.ts b/packages/kernel/src/services/garbage-collection.test.ts new file mode 100644 index 000000000..78acd415f --- /dev/null +++ b/packages/kernel/src/services/garbage-collection.test.ts @@ -0,0 +1,257 @@ +import { describe, it, expect, beforeEach } from 'vitest'; + +import { processGCActionSet } from './garbage-collection.ts'; +import { makeMapKVStore } from '../../test/storage.ts'; +import { makeKernelStore } from '../store/index.ts'; +import { RunQueueItemType } from '../types.ts'; + +describe('garbage-collection', () => { + describe('processGCActionSet', () => { + let kernelStore: ReturnType; + + beforeEach(() => { + const mockKVStore = makeMapKVStore(); + kernelStore = makeKernelStore(mockKVStore); + }); + + it('processes dropExport actions', () => { + // Setup: Create object and add GC action + const ko1 = kernelStore.initKernelObject('v1'); + kernelStore.addClistEntry('v1', ko1, 'o+1'); // Export reference + + // Set reachable count to 0 but keep recognizable count + kernelStore.setObjectRefCount(ko1, { reachable: 0, recognizable: 1 }); + + kernelStore.addGCActions([`v1 dropExport ${ko1}`]); + + // Initial state checks + expect(kernelStore.getReachableFlag('v1', ko1)).toBe(true); + expect(kernelStore.getObjectRefCount(ko1)).toStrictEqual({ + reachable: 0, + recognizable: 1, + }); + + // Process GC actions + const result = processGCActionSet(kernelStore); + + // Verify result + expect(result).toStrictEqual({ + type: RunQueueItemType.dropExports, + vatId: 'v1', + krefs: [ko1], + }); + + // Verify actions were removed + expect(kernelStore.getGCActions().size).toBe(0); + }); + + it('processes retireExport actions', () => { + // Setup: Create object with zero refcounts + const ko1 = kernelStore.initKernelObject('v1'); + kernelStore.addClistEntry('v1', ko1, 'o+1'); + kernelStore.setObjectRefCount(ko1, { reachable: 0, recognizable: 0 }); + kernelStore.addGCActions([`v1 retireExport ${ko1}`]); + + // Process GC actions + const result = processGCActionSet(kernelStore); + + // Verify result + expect(result).toStrictEqual({ + type: RunQueueItemType.retireExports, + vatId: 'v1', + krefs: [ko1], + }); + + // Verify actions were removed + expect(kernelStore.getGCActions().size).toBe(0); + }); + + it('processes retireImport actions', () => { + // Setup: Create object and add GC action + const ko1 = kernelStore.initKernelObject('v1'); + kernelStore.addClistEntry('v2', ko1, 'o-1'); // Import reference + kernelStore.addGCActions([`v2 retireImport ${ko1}`]); + + // Process GC actions + const result = processGCActionSet(kernelStore); + + // Verify result + expect(result).toStrictEqual({ + type: RunQueueItemType.retireImports, + vatId: 'v2', + krefs: [ko1], + }); + + // Verify actions were removed + expect(kernelStore.getGCActions().size).toBe(0); + }); + + it('processes actions in priority order', () => { + // Setup: Create objects and add multiple GC actions + const ko1 = kernelStore.initKernelObject('v1'); + const ko2 = kernelStore.initKernelObject('v1'); + + kernelStore.addClistEntry('v1', ko1, 'o+1'); + kernelStore.addClistEntry('v1', ko2, 'o+2'); + + // Set up conditions for dropExport and retireExport + kernelStore.setObjectRefCount(ko1, { reachable: 0, recognizable: 1 }); + kernelStore.setObjectRefCount(ko2, { reachable: 0, recognizable: 0 }); + + // Add actions in reverse priority order + kernelStore.addGCActions([ + `v1 retireExport ${ko2}`, + `v1 dropExport ${ko1}`, + ]); + + // Process first action - should be dropExport + let result = processGCActionSet(kernelStore); + expect(result).toStrictEqual({ + type: RunQueueItemType.dropExports, + vatId: 'v1', + krefs: [ko1], + }); + + // Process second action - should be retireExport + result = processGCActionSet(kernelStore); + expect(result).toStrictEqual({ + type: RunQueueItemType.retireExports, + vatId: 'v1', + krefs: [ko2], + }); + }); + + it('processes actions by vat ID order', () => { + // Setup: Create objects in different vats + const ko1 = kernelStore.initKernelObject('v2'); + const ko2 = kernelStore.initKernelObject('v1'); + + kernelStore.addClistEntry('v2', ko1, 'o+1'); + kernelStore.addClistEntry('v1', ko2, 'o+1'); + + // Set up conditions for dropExport + kernelStore.setObjectRefCount(ko1, { reachable: 0, recognizable: 1 }); + kernelStore.setObjectRefCount(ko2, { reachable: 0, recognizable: 1 }); + + // Add actions in reverse vat order + kernelStore.addGCActions([ + `v2 dropExport ${ko1}`, + `v1 dropExport ${ko2}`, + ]); + + // Process first action - should be v1 + let result = processGCActionSet(kernelStore); + expect(result).toStrictEqual({ + type: RunQueueItemType.dropExports, + vatId: 'v1', + krefs: [ko2], + }); + + // Process second action - should be v2 + result = processGCActionSet(kernelStore); + expect(result).toStrictEqual({ + type: RunQueueItemType.dropExports, + vatId: 'v2', + krefs: [ko1], + }); + }); + + it('skips actions that should not be processed', () => { + const ko1 = kernelStore.initKernelObject('v1'); + kernelStore.addClistEntry('v1', ko1, 'o+1'); + + // Add dropExport action but set reachable to false (should skip) + kernelStore.clearReachableFlag('v1', ko1); + kernelStore.addGCActions([`v1 dropExport ${ko1}`]); + + // Process actions - should return undefined since action should be skipped + const result = processGCActionSet(kernelStore); + expect(result).toBeUndefined(); + + // Verify action was removed from set + expect(kernelStore.getGCActions().size).toBe(0); + }); + + it('returns undefined when no actions to process', () => { + const result = processGCActionSet(kernelStore); + expect(result).toBeUndefined(); + }); + + it('skips dropExport when object does not exist', () => { + const ko1 = kernelStore.initKernelObject('v1'); + kernelStore.addClistEntry('v1', ko1, 'o+1'); + + // Delete the object to simulate non-existence + kernelStore.deleteKernelObject(ko1); + + kernelStore.addGCActions([`v1 dropExport ${ko1}`]); + + const result = processGCActionSet(kernelStore); + expect(result).toBeUndefined(); + expect(kernelStore.getGCActions().size).toBe(0); + }); + + it('skips retireExport when object has non-zero refcounts', () => { + const ko1 = kernelStore.initKernelObject('v1'); + kernelStore.addClistEntry('v1', ko1, 'o+1'); + + // Set non-zero refcounts + kernelStore.setObjectRefCount(ko1, { reachable: 1, recognizable: 1 }); + + kernelStore.addGCActions([`v1 retireExport ${ko1}`]); + + const result = processGCActionSet(kernelStore); + expect(result).toBeUndefined(); + expect(kernelStore.getGCActions().size).toBe(0); + }); + + it('skips retireExport when object does not exist', () => { + const ko1 = kernelStore.initKernelObject('v1'); + kernelStore.addClistEntry('v1', ko1, 'o+1'); + + // Delete the object + kernelStore.deleteKernelObject(ko1); + + kernelStore.addGCActions([`v1 retireExport ${ko1}`]); + + const result = processGCActionSet(kernelStore); + expect(result).toBeUndefined(); + expect(kernelStore.getGCActions().size).toBe(0); + }); + + it('skips retireExport when clist entry does not exist', () => { + const ko1 = kernelStore.initKernelObject('v1'); + kernelStore.setObjectRefCount(ko1, { reachable: 0, recognizable: 0 }); + + kernelStore.addGCActions([`v1 retireExport ${ko1}`]); + + const result = processGCActionSet(kernelStore); + expect(result).toBeUndefined(); + expect(kernelStore.getGCActions().size).toBe(0); + }); + + it('skips retireImport when clist entry does not exist', () => { + const ko1 = kernelStore.initKernelObject('v1'); + + kernelStore.addGCActions([`v2 retireImport ${ko1}`]); + + const result = processGCActionSet(kernelStore); + expect(result).toBeUndefined(); + expect(kernelStore.getGCActions().size).toBe(0); + }); + + it('skips retireExport when object is recognizable', () => { + const ko1 = kernelStore.initKernelObject('v1'); + kernelStore.addClistEntry('v1', ko1, 'o+1'); + + // Set only recognizable count to non-zero + kernelStore.setObjectRefCount(ko1, { reachable: 0, recognizable: 1 }); + + kernelStore.addGCActions([`v1 retireExport ${ko1}`]); + + const result = processGCActionSet(kernelStore); + expect(result).toBeUndefined(); + expect(kernelStore.getGCActions().size).toBe(0); + }); + }); +}); From b170e1adc3969348a6284b97d51686057517a768 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Wed, 12 Mar 2025 17:42:29 -0300 Subject: [PATCH 13/21] add more unit tests --- .../src/services/kernel-marshal.test.ts | 146 +++++++++ .../kernel/src/services/meter-control.test.ts | 199 ++++++++++++ packages/kernel/src/services/syscall.test.ts | 293 ++++++++++++++++++ 3 files changed, 638 insertions(+) create mode 100644 packages/kernel/src/services/kernel-marshal.test.ts create mode 100644 packages/kernel/src/services/meter-control.test.ts create mode 100644 packages/kernel/src/services/syscall.test.ts diff --git a/packages/kernel/src/services/kernel-marshal.test.ts b/packages/kernel/src/services/kernel-marshal.test.ts new file mode 100644 index 000000000..3d5ee6d8e --- /dev/null +++ b/packages/kernel/src/services/kernel-marshal.test.ts @@ -0,0 +1,146 @@ +import { passStyleOf } from '@endo/far'; +import { describe, it, expect } from 'vitest'; + +import { kslot, krefOf, kser, kunser, makeError } from './kernel-marshal.ts'; +import type { SlotValue } from './kernel-marshal.ts'; + +describe('kernel-marshal', () => { + describe('kslot', () => { + it('creates promise standin for promise refs', () => { + const promiseRefs = ['p1', 'kp1', 'rp1']; + + for (const ref of promiseRefs) { + const standin = kslot(ref); + expect(passStyleOf(standin)).toBe('promise'); + expect(krefOf(standin)).toBe(ref); + } + }); + + it('creates remotable standin for object refs', () => { + const ref = 'ko1'; + const iface = 'TestInterface'; + const standin = kslot(ref, iface); + + expect(passStyleOf(standin)).toBe('remotable'); + expect(krefOf(standin)).toBe(ref); + expect((standin as SlotValue & { iface(): string }).iface()).toBe(iface); + }); + + it('strips Alleged: prefix from interface', () => { + const ref = 'ko1'; + const iface = 'Alleged: TestInterface'; + const standin = kslot(ref, iface); + + expect((standin as SlotValue & { iface(): string }).iface()).toBe( + 'TestInterface', + ); + }); + }); + + describe('krefOf', () => { + it('extracts kref from promise standin', () => { + const ref = 'kp1'; + const standin = kslot(ref); + expect(krefOf(standin)).toBe(ref); + }); + + it('extracts kref from remotable standin', () => { + const ref = 'ko1'; + const standin = kslot(ref); + expect(krefOf(standin)).toBe(ref); + }); + + it('throws for invalid input', () => { + expect(() => krefOf(harden({}) as SlotValue)).toThrow( + 'krefOf requires a promise or remotable', + ); + expect(() => krefOf(null as unknown as SlotValue)).toThrow( + 'krefOf requires a promise or remotable', + ); + }); + }); + + describe('kser/kunser', () => { + it('serializes and deserializes primitive values', () => { + const values = [ + 42, + 'hello', + true, + null, + undefined, + ['array', 123], + { key: 'value' }, + ]; + + for (const value of values) { + const serialized = kser(value); + const deserialized = kunser(serialized); + expect(deserialized).toStrictEqual(value); + } + }); + + it('serializes and deserializes objects with krefs', () => { + const ko1 = kslot('ko1', 'TestInterface'); + const kp1 = kslot('kp1'); + + const value = { + obj: ko1, + promise: kp1, + data: 'test', + }; + + const serialized = kser(value); + expect(serialized).toHaveProperty('body'); + expect(serialized).toHaveProperty('slots'); + + const deserialized = kunser(serialized) as { + obj: SlotValue; + promise: SlotValue; + data: string; + }; + expect(deserialized).toHaveProperty('obj'); + expect(deserialized).toHaveProperty('promise'); + expect(deserialized).toHaveProperty('data', 'test'); + + expect(krefOf(deserialized.obj)).toBe('ko1'); + expect(krefOf(deserialized.promise)).toBe('kp1'); + }); + + it('preserves pass-style of serialized values', () => { + const ko1 = kslot('ko1', 'TestInterface'); + const kp1 = kslot('kp1'); + + const serialized = kser({ obj: ko1, promise: kp1 }); + const deserialized = kunser(serialized) as { + obj: SlotValue; + promise: SlotValue; + }; + + expect(passStyleOf(deserialized.obj)).toBe('remotable'); + expect(passStyleOf(deserialized.promise)).toBe('promise'); + }); + }); + + describe('makeError', () => { + it('creates serialized error with message', () => { + const message = 'Test error message'; + const serialized = makeError(message); + const deserialized = kunser(serialized); + + expect(deserialized).toBeInstanceOf(Error); + expect((deserialized as Error).message).toBe(message); + }); + + it('throws for non-string message', () => { + expect(() => makeError(123 as unknown as string)).toThrow( + '123 must be a string', + ); + expect(() => makeError(null as unknown as string)).toThrow( + 'null must be a string', + ); + expect(() => makeError(undefined as unknown as string)).toThrow( + '"[undefined]" must be a string', + ); + }); + }); +}); diff --git a/packages/kernel/src/services/meter-control.test.ts b/packages/kernel/src/services/meter-control.test.ts new file mode 100644 index 000000000..e93305518 --- /dev/null +++ b/packages/kernel/src/services/meter-control.test.ts @@ -0,0 +1,199 @@ +import { describe, it, expect } from 'vitest'; + +import { makeDummyMeterControl } from './meter-control.ts'; + +describe('meter-control', () => { + describe('makeDummyMeterControl', () => { + it('creates a meter control object with expected methods', () => { + const meterControl = makeDummyMeterControl(); + + expect(meterControl).toHaveProperty('isMeteringDisabled'); + expect(meterControl).toHaveProperty('assertIsMetered'); + expect(meterControl).toHaveProperty('assertNotMetered'); + expect(meterControl).toHaveProperty('runWithoutMetering'); + expect(meterControl).toHaveProperty('runWithoutMeteringAsync'); + expect(meterControl).toHaveProperty('unmetered'); + }); + + it('starts with metering enabled', () => { + const meterControl = makeDummyMeterControl(); + expect(meterControl.isMeteringDisabled()).toBe(false); + }); + + describe('assertIsMetered', () => { + it('succeeds when metering is enabled', () => { + const meterControl = makeDummyMeterControl(); + expect(() => meterControl.assertIsMetered('test')).not.toThrow(); + }); + + it('throws when metering is disabled', () => { + const meterControl = makeDummyMeterControl(); + meterControl.runWithoutMetering(() => { + expect(() => meterControl.assertIsMetered('test message')).toThrow( + 'test message', + ); + }); + }); + }); + + describe('assertNotMetered', () => { + it('succeeds when metering is disabled', () => { + const meterControl = makeDummyMeterControl(); + meterControl.runWithoutMetering(() => { + expect(() => meterControl.assertNotMetered('test')).not.toThrow(); + }); + }); + + it('throws when metering is enabled', () => { + const meterControl = makeDummyMeterControl(); + expect(() => meterControl.assertNotMetered('test message')).toThrow( + 'test message', + ); + }); + }); + + describe('runWithoutMetering', () => { + it('disables metering during execution', () => { + const meterControl = makeDummyMeterControl(); + + meterControl.runWithoutMetering(() => { + expect(meterControl.isMeteringDisabled()).toBe(true); + }); + }); + + it('restores metering after execution', () => { + const meterControl = makeDummyMeterControl(); + + meterControl.runWithoutMetering(() => { + // do nothing + }); + expect(meterControl.isMeteringDisabled()).toBe(false); + }); + + it('restores metering even if thunk throws', () => { + const meterControl = makeDummyMeterControl(); + + expect(() => + meterControl.runWithoutMetering(() => { + throw new Error('test error'); + }), + ).toThrow('test error'); + + expect(meterControl.isMeteringDisabled()).toBe(false); + }); + + it('returns thunk result', () => { + const meterControl = makeDummyMeterControl(); + const result = meterControl.runWithoutMetering(() => 'test result'); + expect(result).toBe('test result'); + }); + + it('supports nested calls', () => { + const meterControl = makeDummyMeterControl(); + + meterControl.runWithoutMetering(() => { + expect(meterControl.isMeteringDisabled()).toBe(true); + + meterControl.runWithoutMetering(() => { + expect(meterControl.isMeteringDisabled()).toBe(true); + }); + + expect(meterControl.isMeteringDisabled()).toBe(true); + }); + + expect(meterControl.isMeteringDisabled()).toBe(false); + }); + }); + + describe('runWithoutMeteringAsync', () => { + it('disables metering during execution', async () => { + const meterControl = makeDummyMeterControl(); + + await meterControl.runWithoutMeteringAsync(async () => { + expect(meterControl.isMeteringDisabled()).toBe(true); + }); + }); + + it('restores metering after execution', async () => { + const meterControl = makeDummyMeterControl(); + + await meterControl.runWithoutMeteringAsync(async () => { + // do nothing + }); + expect(meterControl.isMeteringDisabled()).toBe(false); + }); + + it('restores metering even if thunk rejects', async () => { + const meterControl = makeDummyMeterControl(); + + await expect( + meterControl.runWithoutMeteringAsync(async () => { + throw new Error('test error'); + }), + ).rejects.toThrow('test error'); + + expect(meterControl.isMeteringDisabled()).toBe(false); + }); + + it('returns thunk result', async () => { + const meterControl = makeDummyMeterControl(); + const result = await meterControl.runWithoutMeteringAsync( + async () => 'test result', + ); + expect(result).toBe('test result'); + }); + + it('supports nested calls', async () => { + const meterControl = makeDummyMeterControl(); + + await meterControl.runWithoutMeteringAsync(async () => { + expect(meterControl.isMeteringDisabled()).toBe(true); + + await meterControl.runWithoutMeteringAsync(async () => { + expect(meterControl.isMeteringDisabled()).toBe(true); + }); + + expect(meterControl.isMeteringDisabled()).toBe(true); + }); + + expect(meterControl.isMeteringDisabled()).toBe(false); + }); + }); + + describe('unmetered', () => { + it('wraps function to run without metering', () => { + const meterControl = makeDummyMeterControl(); + const fn = meterControl.unmetered(() => { + expect(meterControl.isMeteringDisabled()).toBe(true); + return 'test result'; + }); + + expect(meterControl.isMeteringDisabled()).toBe(false); + const result = fn(); + expect(result).toBe('test result'); + expect(meterControl.isMeteringDisabled()).toBe(false); + }); + + it('preserves function arguments', () => { + const meterControl = makeDummyMeterControl(); + const fn = meterControl.unmetered((a: number, b: string) => { + expect(meterControl.isMeteringDisabled()).toBe(true); + return `${a}-${b}`; + }); + + const result = fn(42, 'test'); + expect(result).toBe('42-test'); + }); + + it('restores metering if function throws', () => { + const meterControl = makeDummyMeterControl(); + const fn = meterControl.unmetered(() => { + throw new Error('test error'); + }); + + expect(() => fn()).toThrow('test error'); + expect(meterControl.isMeteringDisabled()).toBe(false); + }); + }); + }); +}); diff --git a/packages/kernel/src/services/syscall.test.ts b/packages/kernel/src/services/syscall.test.ts new file mode 100644 index 000000000..cd773e9b6 --- /dev/null +++ b/packages/kernel/src/services/syscall.test.ts @@ -0,0 +1,293 @@ +import type { + VatSyscallObject, + VatSyscallResult, + VatOneResolution, +} from '@agoric/swingset-liveslots'; +import type { CapData } from '@endo/marshal'; +import type { KVStore } from '@ocap/store'; +import { describe, it, expect, vi } from 'vitest'; + +import { makeSupervisorSyscall } from './syscall.ts'; +import type { VatSupervisor } from '../VatSupervisor.ts'; + +describe('syscall', () => { + // Mock supervisor that records syscalls and returns predefined results + const createMockSupervisor = (): VatSupervisor => { + const mockSupervisor = { + executeSyscall: vi.fn( + (_vso: VatSyscallObject): VatSyscallResult => ['ok', null], + ), + } as unknown as VatSupervisor; + return mockSupervisor; + }; + + // Mock KV store for testing vatstore operations + const createMockKVStore = (): KVStore => { + const store = new Map(); + const mockKVStore = { + get: vi.fn((key: string) => store.get(key)), + getNextKey: vi.fn((priorKey: string) => { + const keys = Array.from(store.keys()).sort(); + const index = keys.indexOf(priorKey); + return index >= 0 && index < keys.length - 1 + ? keys[index + 1] + : undefined; + }), + set: vi.fn((key: string, value: string) => { + store.set(key, value); + }), + delete: vi.fn((key: string) => { + store.delete(key); + }), + } as unknown as KVStore; + return mockKVStore; + }; + + describe('makeSupervisorSyscall', () => { + it('creates a syscall object with all required methods', () => { + const supervisor = createMockSupervisor(); + const kv = createMockKVStore(); + const syscall = makeSupervisorSyscall(supervisor, kv); + + expect(syscall).toHaveProperty('send'); + expect(syscall).toHaveProperty('subscribe'); + expect(syscall).toHaveProperty('resolve'); + expect(syscall).toHaveProperty('exit'); + expect(syscall).toHaveProperty('dropImports'); + expect(syscall).toHaveProperty('retireImports'); + expect(syscall).toHaveProperty('retireExports'); + expect(syscall).toHaveProperty('abandonExports'); + expect(syscall).toHaveProperty('callNow'); + expect(syscall).toHaveProperty('vatstoreGet'); + expect(syscall).toHaveProperty('vatstoreGetNextKey'); + expect(syscall).toHaveProperty('vatstoreSet'); + expect(syscall).toHaveProperty('vatstoreDelete'); + }); + + describe('syscall methods', () => { + it('handles send syscall', () => { + const supervisor = createMockSupervisor(); + const kv = createMockKVStore(); + const syscall = makeSupervisorSyscall(supervisor, kv); + + const target = 'ko1'; + const methargs: CapData = { body: '[]', slots: [] }; + const result = 'kp1'; + + syscall.send(target, methargs, result); + + expect(supervisor.executeSyscall).toHaveBeenCalledWith([ + 'send', + target, + { methargs, result }, + ]); + }); + + it('handles subscribe syscall', () => { + const supervisor = createMockSupervisor(); + const kv = createMockKVStore(); + const syscall = makeSupervisorSyscall(supervisor, kv); + + const vpid = 'kp1'; + syscall.subscribe(vpid); + + expect(supervisor.executeSyscall).toHaveBeenCalledWith([ + 'subscribe', + vpid, + ]); + }); + + it('handles resolve syscall', () => { + const supervisor = createMockSupervisor(); + const kv = createMockKVStore(); + const syscall = makeSupervisorSyscall(supervisor, kv); + + const resolutions: VatOneResolution[] = [ + ['kp1', false, { body: '[]', slots: [] }], + ]; + syscall.resolve(resolutions); + + expect(supervisor.executeSyscall).toHaveBeenCalledWith([ + 'resolve', + resolutions, + ]); + }); + + it('handles exit syscall', () => { + const supervisor = createMockSupervisor(); + const kv = createMockKVStore(); + const syscall = makeSupervisorSyscall(supervisor, kv); + + const isFailure = true; + const info: CapData = { body: '[]', slots: [] }; + syscall.exit(isFailure, info); + + expect(supervisor.executeSyscall).toHaveBeenCalledWith([ + 'exit', + isFailure, + info, + ]); + }); + + it('handles dropImports syscall', () => { + const supervisor = createMockSupervisor(); + const kv = createMockKVStore(); + const syscall = makeSupervisorSyscall(supervisor, kv); + + const vrefs = ['ko1', 'ko2']; + syscall.dropImports(vrefs); + + expect(supervisor.executeSyscall).toHaveBeenCalledWith([ + 'dropImports', + vrefs, + ]); + }); + + it('handles retireImports syscall', () => { + const supervisor = createMockSupervisor(); + const kv = createMockKVStore(); + const syscall = makeSupervisorSyscall(supervisor, kv); + + const vrefs = ['ko1', 'ko2']; + syscall.retireImports(vrefs); + + expect(supervisor.executeSyscall).toHaveBeenCalledWith([ + 'retireImports', + vrefs, + ]); + }); + + it('handles retireExports syscall', () => { + const supervisor = createMockSupervisor(); + const kv = createMockKVStore(); + const syscall = makeSupervisorSyscall(supervisor, kv); + + const vrefs = ['ko1', 'ko2']; + syscall.retireExports(vrefs); + + expect(supervisor.executeSyscall).toHaveBeenCalledWith([ + 'retireExports', + vrefs, + ]); + }); + + it('handles abandonExports syscall', () => { + const supervisor = createMockSupervisor(); + const kv = createMockKVStore(); + const syscall = makeSupervisorSyscall(supervisor, kv); + + const vrefs = ['ko1', 'ko2']; + syscall.abandonExports(vrefs); + + expect(supervisor.executeSyscall).toHaveBeenCalledWith([ + 'abandonExports', + vrefs, + ]); + }); + + it('throws on callNow syscall', () => { + const supervisor = createMockSupervisor(); + const kv = createMockKVStore(); + const syscall = makeSupervisorSyscall(supervisor, kv); + + expect(() => syscall.callNow('ko1', 'method', [])).toThrow( + 'callNow not supported (we have no devices)', + ); + }); + }); + + describe('vatstore methods', () => { + it('handles vatstoreGet', () => { + const supervisor = createMockSupervisor(); + const kv = createMockKVStore(); + const syscall = makeSupervisorSyscall(supervisor, kv); + + kv.set('test-key', 'test-value'); + const result = syscall.vatstoreGet('test-key'); + + expect(result).toBe('test-value'); + expect(kv.get).toHaveBeenCalledWith('test-key'); + }); + + it('handles vatstoreGetNextKey', () => { + const supervisor = createMockSupervisor(); + const kv = createMockKVStore(); + const syscall = makeSupervisorSyscall(supervisor, kv); + + kv.set('key1', 'value1'); + kv.set('key2', 'value2'); + const result = syscall.vatstoreGetNextKey('key1'); + + expect(result).toBe('key2'); + expect(kv.getNextKey).toHaveBeenCalledWith('key1'); + }); + + it('handles vatstoreSet', () => { + const supervisor = createMockSupervisor(); + const kv = createMockKVStore(); + const syscall = makeSupervisorSyscall(supervisor, kv); + + syscall.vatstoreSet('test-key', 'test-value'); + + expect(kv.set).toHaveBeenCalledWith('test-key', 'test-value'); + expect(kv.get('test-key')).toBe('test-value'); + }); + + it('handles vatstoreDelete', () => { + const supervisor = createMockSupervisor(); + const kv = createMockKVStore(); + const syscall = makeSupervisorSyscall(supervisor, kv); + + kv.set('test-key', 'test-value'); + syscall.vatstoreDelete('test-key'); + + expect(kv.delete).toHaveBeenCalledWith('test-key'); + expect(kv.get('test-key')).toBeUndefined(); + }); + }); + + describe('error handling', () => { + it('throws on supervisor error', () => { + const supervisor = createMockSupervisor(); + const kv = createMockKVStore(); + const syscall = makeSupervisorSyscall(supervisor, kv); + + vi.spyOn(supervisor, 'executeSyscall').mockImplementationOnce(() => { + throw new Error('supervisor error'); + }); + + expect(() => syscall.send('ko1', { body: '[]', slots: [] })).toThrow( + 'supervisor error', + ); + }); + + it('throws on syscall error result', () => { + const supervisor = createMockSupervisor(); + const kv = createMockKVStore(); + const syscall = makeSupervisorSyscall(supervisor, kv); + + vi.spyOn(supervisor, 'executeSyscall').mockImplementationOnce(() => { + return ['error', 'syscall failed']; + }); + + expect(() => syscall.send('ko1', { body: '[]', slots: [] })).toThrow( + 'syscall.send failed: syscall failed', + ); + }); + + it('throws on unknown result type', () => { + const supervisor = createMockSupervisor(); + const kv = createMockKVStore(); + const syscall = makeSupervisorSyscall(supervisor, kv); + + vi.spyOn(supervisor, 'executeSyscall').mockImplementationOnce(() => { + return ['unknown' as 'ok', null]; + }); + + expect(() => syscall.send('ko1', { body: '[]', slots: [] })).toThrow( + 'unknown syscall result type "unknown"', + ); + }); + }); + }); +}); From 629b5f94d5a364b3726573d6e7d3fdc9e50568ae Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Wed, 12 Mar 2025 17:51:52 -0300 Subject: [PATCH 14/21] one more unit test --- .../kernel/src/utils/wait-quiescent.test.ts | 129 ++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 packages/kernel/src/utils/wait-quiescent.test.ts diff --git a/packages/kernel/src/utils/wait-quiescent.test.ts b/packages/kernel/src/utils/wait-quiescent.test.ts new file mode 100644 index 000000000..1b2a2ebab --- /dev/null +++ b/packages/kernel/src/utils/wait-quiescent.test.ts @@ -0,0 +1,129 @@ +import { makePromiseKit } from '@endo/promise-kit'; +import { describe, it, expect } from 'vitest'; + +import { waitUntilQuiescent } from './wait-quiescent.ts'; + +describe('wait-quiescent', () => { + describe('waitUntilQuiescent', () => { + it('resolves after microtask queue is empty', async () => { + const { promise: p1, resolve: r1 } = makePromiseKit(); + const { promise: p2, resolve: r2 } = makePromiseKit(); + + // Start waiting for quiescence first + const quiescentPromise = waitUntilQuiescent(); + + // Create microtasks + Promise.resolve() + .then(() => { + r1(); + return undefined; + }) + .catch(() => undefined); + + Promise.resolve() + .then(() => { + r2(); + return undefined; + }) + .catch(() => undefined); + + // Wait for all promises + await p1; + await p2; + await quiescentPromise; + + expect(true).toBe(true); // If we got here, the test passed + }); + + it('waits for nested promise chains', async () => { + const results: number[] = []; + + // Create nested promise chains + await Promise.resolve().then(async () => { + results.push(1); + await Promise.resolve().then(async () => { + results.push(2); + await Promise.resolve().then(() => { + results.push(3); + return results; + }); + return results; + }); + return results; + }); + + await waitUntilQuiescent(); + + expect(results).toStrictEqual([1, 2, 3]); + }); + + it('waits for concurrent promises', async () => { + const results: number[] = []; + const promises = [ + Promise.resolve().then(() => results.push(1)), + Promise.resolve().then(() => results.push(2)), + Promise.resolve().then(() => results.push(3)), + ]; + + await Promise.all(promises); + await waitUntilQuiescent(); + + expect(results).toHaveLength(3); + expect(results).toContain(1); + expect(results).toContain(2); + expect(results).toContain(3); + }); + + it('handles rejected promises in the queue', async () => { + const results: string[] = []; + + // Create a mix of resolved and rejected promises + await Promise.resolve() + .then(() => { + results.push('success1'); + return results; + }) + .catch(() => { + results.push('caught1'); + return results; + }); + + await Promise.reject(new Error('test error')) + .then(() => { + results.push('success2'); + return results; + }) + .catch(() => { + results.push('caught2'); + return results; + }); + + await waitUntilQuiescent(); + + expect(results).toContain('success1'); + expect(results).toContain('caught2'); + }); + + it('resolves even with setImmediate callbacks', async () => { + const results: string[] = []; + + setImmediate(() => { + results.push('immediate1'); + setImmediate(() => { + results.push('immediate2'); + }); + }); + + await Promise.resolve().then(() => { + results.push('promise'); + return results; + }); + + await waitUntilQuiescent(); + + expect(results).toContain('promise'); + // Note: We don't check for immediate1/2 as they may execute after + // waitUntilQuiescent resolves, since they're lower priority + }); + }); +}); From df652089505df2e635fa59dbb75ef683d18a4cb8 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 13 Mar 2025 16:43:29 -0300 Subject: [PATCH 15/21] Improve garbage collection module structure and type safety --- .../kernel/src/services/garbage-collection.ts | 251 +++++++++--------- 1 file changed, 119 insertions(+), 132 deletions(-) diff --git a/packages/kernel/src/services/garbage-collection.ts b/packages/kernel/src/services/garbage-collection.ts index cef4b7669..d7ea28d9e 100644 --- a/packages/kernel/src/services/garbage-collection.ts +++ b/packages/kernel/src/services/garbage-collection.ts @@ -13,23 +13,98 @@ import { insistVatId, queueTypeFromActionType, } from '../types.ts'; +import { assert } from '../utils/assert.ts'; /** - * Parse a GC action string into a vat id, type, and kref. - * - * @param action - The GC action string to parse. - * @returns The vat id, type, and kref. + * Parsed representation of a GC action. */ -function parseAction(action: GCAction): { +type ParsedGCAction = Readonly<{ vatId: VatId; type: GCActionType; kref: KRef; -} { +}>; + +/** + * Parse a GC action string into a vat id, type, and kref. + * + * @param action - The GC action string to parse. + * @returns The parsed GC action. + */ +function parseAction(action: GCAction): ParsedGCAction { const [vatId, type, kref] = action.split(' '); insistVatId(vatId); insistGCActionType(type); insistKernelType('object', kref); - return { vatId, type, kref }; + return harden({ vatId, type, kref }); +} + +/** + * Determines if a GC action should be processed based on current system state. + * + * @param storage - The kernel storage. + * @param vatId - The vat id of the vat that owns the kref. + * @param type - The type of GC action. + * @param kref - The kref of the object in question. + * @returns True if the action should be processed, false otherwise. + */ +function shouldProcessAction( + storage: KernelStore, + vatId: VatId, + type: GCActionType, + kref: KRef, +): boolean { + const hasCList = storage.hasCListEntry(vatId, kref); + const isReachable = hasCList + ? storage.getReachableFlag(vatId, kref) + : undefined; + const exists = storage.kernelRefExists(kref); + const { reachable, recognizable } = exists + ? storage.getObjectRefCount(kref) + : { reachable: 0, recognizable: 0 }; + + switch (type) { + case 'dropExport': + return exists && reachable === 0 && hasCList && isReachable === true; + + case 'retireExport': + return exists && reachable === 0 && recognizable === 0 && hasCList; + + case 'retireImport': + return hasCList; + + default: + return false; + } +} + +/** + * Filters and processes a group of GC actions for a specific vat and action type. + * + * @param storage - The kernel storage. + * @param vatId - The vat id of the vat that owns the krefs. + * @param actions - The set of GC actions to process. + * @param allActionsSet - The complete set of GC actions. + * @returns Object containing the krefs to process and whether the action set was updated. + */ +function filterActionsForProcessing( + storage: KernelStore, + vatId: VatId, + actions: Set, + allActionsSet: Set, +): { krefs: KRef[]; actionSetUpdated: boolean } { + const krefs: KRef[] = []; + let actionSetUpdated = false; + + for (const action of actions) { + const { type, kref } = parseAction(action); + if (shouldProcessAction(storage, vatId, type, kref)) { + krefs.push(kref); + } + allActionsSet.delete(action); + actionSetUpdated = true; + } + + return harden({ krefs, actionSetUpdated }); } /** @@ -44,148 +119,59 @@ export function processGCActionSet( const allActionsSet = storage.getGCActions(); let actionSetUpdated = false; - // GC actions are each one of 'dropExport', 'retireExport', or - // 'retireImport', aimed at a specific vat and affecting a specific kref. - // They are added to the durable "GC Actions" set (stored in kernelDB) when - // `processRefcounts` notices a refcount sitting at zero, which means some - // vat needs to be told that an object can be freed. Before each crank, the - // kernel calls processGCActionSet to see if there are any GC actions that - // should be taken. All such GC actions are executed before any regular vat - // delivery gets to run. - - // However, things might have changed between the time the action was - // pushed into the durable set and the time the kernel is ready to execute - // it. For example, the kref might have been re-exported: we were all set - // to tell the exporting vat that their object isn't recognizable any more - // (with a `dispatch.retireExport`), but then they sent a brand new copy to - // some importer. We must negate the `retireExport` action, because it's no - // longer the right thing to do. Alternatively, the exporting vat might - // have deleted the object itself (`syscall.retireExport`) before the - // kernel got a chance to deliver the `dispatch.retireExport`, which means - // we must bypass the action as redundant (since it's an error to delete - // the same c-list entry twice). - - /** - * Inspect a queued GC action and decide whether the current state of c-lists - * and reference counts warrants processing it, or if it should instead be - * negated/bypassed. - * - * @param vatId - The vat id of the vat that owns the kref. - * @param type - The type of GC action. - * @param kref - The kref of the object in question. - * @returns True if the action should be processed, false otherwise. - */ - function shouldProcessAction( - vatId: VatId, - type: GCActionType, - kref: KRef, - ): boolean { - const hasCList = storage.hasCListEntry(vatId, kref); - const isReachable = hasCList - ? storage.getReachableFlag(vatId, kref) - : undefined; - const exists = storage.kernelRefExists(kref); - const { reachable, recognizable } = exists - ? storage.getObjectRefCount(kref) - : { reachable: 0, recognizable: 0 }; - - if (type === 'dropExport') { - if (!exists) { - return false; - } // already, shouldn't happen - if (reachable) { - return false; - } // negated - if (!hasCList) { - return false; - } // already, shouldn't happen - if (!isReachable) { - return false; - } // already, shouldn't happen - } - if (type === 'retireExport') { - if (!exists) { - return false; - } // already - if (reachable || recognizable) { - return false; - } // negated - if (!hasCList) { - return false; - } // already - } - if (type === 'retireImport') { - if (!hasCList) { - return false; - } // already - } - return true; - } - - // We process actions in groups (sorted first by vat, then by type), to - // make it deterministic, and to ensure that `dropExport` happens before - // `retireExport`. This examines one group at a time, filtering everything - // in that group, and returning the survivors of the first group that - // wasn't filtered out entirely. Our available dispatch functions take - // multiple krefs (`dispatch.dropExports`, rather than - // `dispatch.dropExport`), so the set of surviving krefs can all be - // delivered to a vat in a single crank. - - // Some day we may consolidate the three GC delivery methods into a single - // one, in which case we'll batch together an entire vat's worth of - // actions, instead of the narrower (vat+type) group. The filtering rules - // may need to change to support that, to ensure that `dropExport` and - // `retireExport` can both be delivered. - - /** - * Process the set of GC actions for a given vat. - * - * @param vatId - The vat id of the vat that owns the krefs. - * @param groupedActions - The set of GC actions to process. - * @returns The krefs to process. - */ - function krefsToProcess(vatId: VatId, groupedActions: Set): KRef[] { - const krefs: KRef[] = []; - for (const action of groupedActions) { - const { type, kref } = parseAction(action); - if (shouldProcessAction(vatId, type, kref)) { - krefs.push(kref); - } - allActionsSet.delete(action); - actionSetUpdated = true; - } - return krefs; - } + // Group actions by vat and type + const actionsByVat = new Map>>(); - const actionsByVat = new Map(); for (const action of allActionsSet) { const { vatId, type } = parseAction(action); + if (!actionsByVat.has(vatId)) { actionsByVat.set(vatId, new Map()); } + const actionsForVatByType = actionsByVat.get(vatId); + assert(actionsForVatByType !== undefined, `No actions for vat: ${vatId}`); + if (!actionsForVatByType.has(type)) { - actionsForVatByType.set(type, []); + actionsForVatByType.set(type, new Set()); } - actionsForVatByType.get(type).push(action); + + const actions = actionsForVatByType.get(type); + assert(actions !== undefined, `No actions for type: ${type}`); + actions.add(action); } - const vatIds = Array.from(actionsByVat.keys()); - vatIds.sort(); + // Process actions in priority order + const vatIds = Array.from(actionsByVat.keys()).sort(); + for (const vatId of vatIds) { const actionsForVatByType = actionsByVat.get(vatId); - // find the highest-priority type of work to do within this vat + assert(actionsForVatByType !== undefined, `No actions for vat: ${vatId}`); + + // Find the highest-priority type of work to do within this vat for (const type of actionTypePriorities) { if (actionsForVatByType.has(type)) { const actions = actionsForVatByType.get(type); - const krefs = krefsToProcess(vatId, actions); - if (krefs.length) { - // at last, we act + assert(actions !== undefined, `No actions for type: ${type}`); + const { krefs, actionSetUpdated: updated } = filterActionsForProcessing( + storage, + vatId, + actions, + allActionsSet, + ); + + actionSetUpdated = actionSetUpdated || updated; + + if (krefs.length > 0) { + // We found actions to process krefs.sort(); - // remove the work we're about to do from the durable set + + // Update the durable set before returning storage.setGCActions(allActionsSet); + const queueType = queueTypeFromActionType.get(type); - assert(queueType, `Unknown action type: ${type}`); + assert(queueType !== undefined, `Unknown action type: ${type}`); + return harden({ type: queueType, vatId, krefs }); } } @@ -193,11 +179,12 @@ export function processGCActionSet( } if (actionSetUpdated) { - // remove negated items from the durable set + // Remove negated items from the durable set storage.setGCActions(allActionsSet); } - // no GC work to do and no DB changes + // No GC work to do return undefined; } + harden(processGCActionSet); From 7cb353100ac87a060fea23dee984a3f44eb62b9b Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Tue, 18 Mar 2025 15:09:41 -0300 Subject: [PATCH 16/21] Rebase --- .github/workflows/lint-build-test.yml | 1 - package.json | 2 +- packages/kernel-test/src/vatstore.test.ts | 2 +- packages/kernel/src/Kernel.ts | 4 ++-- packages/kernel/src/VatHandle.ts | 12 ++++++------ .../kernel/src/services/garbage-collection.test.ts | 5 ++--- packages/kernel/src/store/methods/gc.test.ts | 5 ++--- 7 files changed, 14 insertions(+), 17 deletions(-) diff --git a/.github/workflows/lint-build-test.yml b/.github/workflows/lint-build-test.yml index 03d3272dd..ddfa2170c 100644 --- a/.github/workflows/lint-build-test.yml +++ b/.github/workflows/lint-build-test.yml @@ -149,7 +149,6 @@ jobs: yarn build-release cd - - run: yarn build - - run: yarn build:vats - run: yarn run test - name: Require clean working directory shell: bash diff --git a/package.json b/package.json index c9638d809..75cb9a497 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "build:clean": "yarn clean && yarn build", "build:docs": "yarn workspaces foreach --all --exclude @ocap/monorepo --exclude @ocap/extension --parallel --interlaced --verbose run build:docs", "build:source": "ts-bridge --project tsconfig.build.json --verbose && yarn build:special", - "build:special": "yarn workspace @ocap/shims run build && yarn workspace @ocap/extension run build", + "build:special": "yarn workspace @ocap/shims run build && yarn workspace @ocap/extension run build && yarn workspace @ocap/kernel-test run build:vats", "bundle": "node ./scripts/bundle-vat.js", "changelog:update": "yarn workspaces foreach --all --no-private --parallel --interlaced --verbose run changelog:update", "changelog:validate": "yarn workspaces foreach --all --no-private --parallel --interlaced --verbose run changelog:validate", diff --git a/packages/kernel-test/src/vatstore.test.ts b/packages/kernel-test/src/vatstore.test.ts index b4fba56d6..d1baac376 100644 --- a/packages/kernel-test/src/vatstore.test.ts +++ b/packages/kernel-test/src/vatstore.test.ts @@ -17,7 +17,7 @@ import { } from 'node:worker_threads'; import { describe, vi, expect, it } from 'vitest'; -import { kunser } from '../../kernel/src/kernel-marshal.ts'; +import { kunser } from '../../kernel/src/services/kernel-marshal.ts'; import { NodejsVatWorkerService } from '../../nodejs/src/kernel/VatWorkerService.ts'; /** diff --git a/packages/kernel/src/Kernel.ts b/packages/kernel/src/Kernel.ts index c177005d5..3c5a2270b 100644 --- a/packages/kernel/src/Kernel.ts +++ b/packages/kernel/src/Kernel.ts @@ -192,13 +192,13 @@ export class Kernel { */ async *#runQueueItems(): AsyncGenerator { for (;;) { - const gcAction = processGCActionSet(this.#storage); + const gcAction = processGCActionSet(this.#kernelStore); if (gcAction) { yield gcAction; continue; } - const reapAction = this.#storage.nextReapAction(); + const reapAction = this.#kernelStore.nextReapAction(); if (reapAction) { yield reapAction; continue; diff --git a/packages/kernel/src/VatHandle.ts b/packages/kernel/src/VatHandle.ts index 1570f65ab..2da62e1d7 100644 --- a/packages/kernel/src/VatHandle.ts +++ b/packages/kernel/src/VatHandle.ts @@ -327,7 +327,7 @@ export class VatHandle { `vat ${this.vatId} issued invalid syscall dropImports for ${kref}`, ); } - this.#storage.clearReachableFlag(this.vatId, kref); + this.#kernelStore.clearReachableFlag(this.vatId, kref); } } @@ -345,12 +345,12 @@ export class VatHandle { `vat ${this.vatId} issued invalid syscall retireImports for ${kref}`, ); } - if (this.#storage.getReachableFlag(this.vatId, kref)) { + if (this.#kernelStore.getReachableFlag(this.vatId, kref)) { throw Error(`syscall.retireImports but ${kref} is still reachable`); } // deleting the clist entry will decrement the recognizable count, but // not the reachable count (because it was unreachable, as we asserted) - this.#storage.forgetKref(this.vatId, kref); + this.#kernelStore.forgetKref(this.vatId, kref); } } @@ -372,13 +372,13 @@ export class VatHandle { ); } if (checkReachable) { - if (this.#storage.getReachableFlag(this.vatId, kref)) { + if (this.#kernelStore.getReachableFlag(this.vatId, kref)) { throw Error( `syscall.${action}Exports but ${kref} is still reachable`, ); } } - this.#storage.forgetKref(this.vatId, kref); + this.#kernelStore.forgetKref(this.vatId, kref); this.#logger.debug(`${action}Exports: deleted object ${kref}`); } } @@ -522,7 +522,7 @@ export class VatHandle { await this.sendVatCommand({ method: VatCommandMethod.ping, params: null }); await this.sendVatCommand({ method: VatCommandMethod.initVat, - params: this.config, + params: { vatConfig: this.config, state: this.#vatStore.getKVData() }, }); this.#logger.debug('Created'); } diff --git a/packages/kernel/src/services/garbage-collection.test.ts b/packages/kernel/src/services/garbage-collection.test.ts index 78acd415f..b739efe6f 100644 --- a/packages/kernel/src/services/garbage-collection.test.ts +++ b/packages/kernel/src/services/garbage-collection.test.ts @@ -1,7 +1,7 @@ import { describe, it, expect, beforeEach } from 'vitest'; import { processGCActionSet } from './garbage-collection.ts'; -import { makeMapKVStore } from '../../test/storage.ts'; +import { makeMapKernelDatabase } from '../../test/storage.ts'; import { makeKernelStore } from '../store/index.ts'; import { RunQueueItemType } from '../types.ts'; @@ -10,8 +10,7 @@ describe('garbage-collection', () => { let kernelStore: ReturnType; beforeEach(() => { - const mockKVStore = makeMapKVStore(); - kernelStore = makeKernelStore(mockKVStore); + kernelStore = makeKernelStore(makeMapKernelDatabase()); }); it('processes dropExport actions', () => { diff --git a/packages/kernel/src/store/methods/gc.test.ts b/packages/kernel/src/store/methods/gc.test.ts index 9289172aa..4f5f395e1 100644 --- a/packages/kernel/src/store/methods/gc.test.ts +++ b/packages/kernel/src/store/methods/gc.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect, beforeEach } from 'vitest'; -import { makeMapKVStore } from '../../../test/storage.ts'; +import { makeMapKernelDatabase } from '../../../test/storage.ts'; import { RunQueueItemType } from '../../types.ts'; import type { GCAction } from '../../types.ts'; import { makeKernelStore } from '../index.ts'; @@ -9,8 +9,7 @@ describe('GC methods', () => { let kernelStore: ReturnType; beforeEach(() => { - const mockKVStore = makeMapKVStore(); - kernelStore = makeKernelStore(mockKVStore); + kernelStore = makeKernelStore(makeMapKernelDatabase()); }); describe('GC actions', () => { From 5d108198c3c9608cd2d3b8ab9d53d0a4561d2977 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Tue, 18 Mar 2025 15:13:23 -0300 Subject: [PATCH 17/21] make review easier --- packages/kernel/src/VatHandle.ts | 116 +++++++++++++++---------------- 1 file changed, 58 insertions(+), 58 deletions(-) diff --git a/packages/kernel/src/VatHandle.ts b/packages/kernel/src/VatHandle.ts index 2da62e1d7..f37a20cd6 100644 --- a/packages/kernel/src/VatHandle.ts +++ b/packages/kernel/src/VatHandle.ts @@ -500,6 +500,64 @@ export class VatHandle { } } + /** + * Make a 'notify' delivery to the vat. + * + * @param resolutions - One or more promise resolutions to deliver. + */ + async deliverNotify(resolutions: VatOneResolution[]): Promise { + await this.sendVatCommand({ + method: VatCommandMethod.deliver, + params: ['notify', resolutions], + }); + } + + /** + * Make a 'dropExports' delivery to the vat. + * + * @param krefs - The KRefs of the exports to be dropped. + */ + async deliverDropExports(krefs: KRef[]): Promise { + await this.sendVatCommand({ + method: VatCommandMethod.deliver, + params: ['dropExports', krefs], + }); + } + + /** + * Make a 'retireExports' delivery to the vat. + * + * @param krefs - The KRefs of the exports to be retired. + */ + async deliverRetireExports(krefs: KRef[]): Promise { + await this.sendVatCommand({ + method: VatCommandMethod.deliver, + params: ['retireExports', krefs], + }); + } + + /** + * Make a 'retireImports' delivery to the vat. + * + * @param krefs - The KRefs of the imports to be retired. + */ + async deliverRetireImports(krefs: KRef[]): Promise { + await this.sendVatCommand({ + method: VatCommandMethod.deliver, + params: ['retireImports', krefs], + }); + } + + /** + * Make a 'bringOutYourDead' delivery to the vat. + */ + async deliverBringOutYourDead(): Promise { + await this.sendVatCommand({ + method: VatCommandMethod.deliver, + params: ['bringOutYourDead'], + }); + } + /** * Initializes the vat. * @@ -579,62 +637,4 @@ export class VatHandle { params: ['message', target, message], }); } - - /** - * Make a 'notify' delivery to the vat. - * - * @param resolutions - One or more promise resolutions to deliver. - */ - async deliverNotify(resolutions: VatOneResolution[]): Promise { - await this.sendVatCommand({ - method: VatCommandMethod.deliver, - params: ['notify', resolutions], - }); - } - - /** - * Make a 'dropExports' delivery to the vat. - * - * @param krefs - The KRefs of the exports to be dropped. - */ - async deliverDropExports(krefs: KRef[]): Promise { - await this.sendVatCommand({ - method: VatCommandMethod.deliver, - params: ['dropExports', krefs], - }); - } - - /** - * Make a 'retireExports' delivery to the vat. - * - * @param krefs - The KRefs of the exports to be retired. - */ - async deliverRetireExports(krefs: KRef[]): Promise { - await this.sendVatCommand({ - method: VatCommandMethod.deliver, - params: ['retireExports', krefs], - }); - } - - /** - * Make a 'retireImports' delivery to the vat. - * - * @param krefs - The KRefs of the imports to be retired. - */ - async deliverRetireImports(krefs: KRef[]): Promise { - await this.sendVatCommand({ - method: VatCommandMethod.deliver, - params: ['retireImports', krefs], - }); - } - - /** - * Make a 'bringOutYourDead' delivery to the vat. - */ - async deliverBringOutYourDead(): Promise { - await this.sendVatCommand({ - method: VatCommandMethod.deliver, - params: ['bringOutYourDead'], - }); - } } From ff640b853c06ec76b5063e1fae1aa6639ee5221c Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Tue, 18 Mar 2025 15:14:02 -0300 Subject: [PATCH 18/21] make review easier --- packages/kernel/src/VatHandle.ts | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/packages/kernel/src/VatHandle.ts b/packages/kernel/src/VatHandle.ts index f37a20cd6..e71864173 100644 --- a/packages/kernel/src/VatHandle.ts +++ b/packages/kernel/src/VatHandle.ts @@ -500,6 +500,19 @@ export class VatHandle { } } + /** + * Make a 'message' delivery to the vat. + * + * @param target - The VRef of the object to which the message is addressed. + * @param message - The message to deliver. + */ + async deliverMessage(target: VRef, message: Message): Promise { + await this.sendVatCommand({ + method: VatCommandMethod.deliver, + params: ['message', target, message], + }); + } + /** * Make a 'notify' delivery to the vat. * @@ -624,17 +637,4 @@ export class VatHandle { readonly #nextMessageId = (): VatCommand['id'] => { return `${this.vatId}:${this.#messageCounter()}`; }; - - /** - * Make a 'message' delivery to the vat. - * - * @param target - The VRef of the object to which the message is addressed. - * @param message - The message to deliver. - */ - async deliverMessage(target: VRef, message: Message): Promise { - await this.sendVatCommand({ - method: VatCommandMethod.deliver, - params: ['message', target, message], - }); - } } From 65f94e1ddd7e66b1f005bb6e8320af20ebd07736 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Tue, 18 Mar 2025 15:15:45 -0300 Subject: [PATCH 19/21] remove unused method --- packages/kernel/src/VatHandle.ts | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/packages/kernel/src/VatHandle.ts b/packages/kernel/src/VatHandle.ts index e71864173..330340e47 100644 --- a/packages/kernel/src/VatHandle.ts +++ b/packages/kernel/src/VatHandle.ts @@ -571,33 +571,6 @@ export class VatHandle { }); } - /** - * Initializes the vat. - * - * @returns A promise that resolves when the vat is initialized. - */ - async init(): Promise { - Promise.all([this.#vatStream.drain(this.handleMessage.bind(this))]).catch( - async (error) => { - this.#logger.error(`Unexpected read error`, error); - await this.terminate(new StreamReadError({ vatId: this.vatId }, error)); - }, - ); - - // XXX This initial `ping` was originally put here as a sanity check to make - // sure that the vat was actually running and able to exchange message - // traffic with the kernel, but the addition of the `initVat` message to the - // startup flow has, I'm fairly sure, obviated the need for that as it - // effectively performs the same function. Probably this ping should be - // removed. - await this.sendVatCommand({ method: VatCommandMethod.ping, params: null }); - await this.sendVatCommand({ - method: VatCommandMethod.initVat, - params: { vatConfig: this.config, state: this.#vatStore.getKVData() }, - }); - this.#logger.debug('Created'); - } - /** * Terminates the vat. * From 8a97fce340bb43a5a54e304860460ff00de4d884 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 27 Mar 2025 14:09:24 +0100 Subject: [PATCH 20/21] Revert store refactor :( --- packages/kernel/src/Kernel.ts | 4 +- packages/kernel/src/VatHandle.test.ts | 4 +- packages/kernel/src/VatHandle.ts | 2 +- .../src/services/garbage-collection.test.ts | 2 +- .../kernel/src/services/garbage-collection.ts | 2 +- packages/kernel/src/store/index.ts | 207 --- .../{index.test.ts => kernel-store.test.ts} | 6 +- packages/kernel/src/store/kernel-store.ts | 1105 +++++++++++++++++ .../kernel/src/store/methods/base.test.ts | 160 --- packages/kernel/src/store/methods/base.ts | 103 -- .../kernel/src/store/methods/clist.test.ts | 258 ---- packages/kernel/src/store/methods/clist.ts | 270 ---- packages/kernel/src/store/methods/gc.test.ts | 134 -- packages/kernel/src/store/methods/gc.ts | 157 --- packages/kernel/src/store/methods/id.test.ts | 172 --- packages/kernel/src/store/methods/id.ts | 49 - .../kernel/src/store/methods/object.test.ts | 278 ----- packages/kernel/src/store/methods/object.ts | 119 -- .../kernel/src/store/methods/promise.test.ts | 400 ------ packages/kernel/src/store/methods/promise.ts | 204 --- .../kernel/src/store/methods/queue.test.ts | 276 ---- packages/kernel/src/store/methods/queue.ts | 117 -- .../kernel/src/store/methods/refcount.test.ts | 164 --- packages/kernel/src/store/methods/refcount.ts | 76 -- packages/kernel/src/store/types.ts | 27 - 25 files changed, 1113 insertions(+), 3183 deletions(-) delete mode 100644 packages/kernel/src/store/index.ts rename packages/kernel/src/store/{index.test.ts => kernel-store.test.ts} (98%) create mode 100644 packages/kernel/src/store/kernel-store.ts delete mode 100644 packages/kernel/src/store/methods/base.test.ts delete mode 100644 packages/kernel/src/store/methods/base.ts delete mode 100644 packages/kernel/src/store/methods/clist.test.ts delete mode 100644 packages/kernel/src/store/methods/clist.ts delete mode 100644 packages/kernel/src/store/methods/gc.test.ts delete mode 100644 packages/kernel/src/store/methods/gc.ts delete mode 100644 packages/kernel/src/store/methods/id.test.ts delete mode 100644 packages/kernel/src/store/methods/id.ts delete mode 100644 packages/kernel/src/store/methods/object.test.ts delete mode 100644 packages/kernel/src/store/methods/object.ts delete mode 100644 packages/kernel/src/store/methods/promise.test.ts delete mode 100644 packages/kernel/src/store/methods/promise.ts delete mode 100644 packages/kernel/src/store/methods/queue.test.ts delete mode 100644 packages/kernel/src/store/methods/queue.ts delete mode 100644 packages/kernel/src/store/methods/refcount.test.ts delete mode 100644 packages/kernel/src/store/methods/refcount.ts delete mode 100644 packages/kernel/src/store/types.ts diff --git a/packages/kernel/src/Kernel.ts b/packages/kernel/src/Kernel.ts index 3c5a2270b..eed95cd5b 100644 --- a/packages/kernel/src/Kernel.ts +++ b/packages/kernel/src/Kernel.ts @@ -22,8 +22,8 @@ import type { import { processGCActionSet } from './services/garbage-collection.ts'; import type { SlotValue } from './services/kernel-marshal.ts'; import { kser, kunser, krefOf, kslot } from './services/kernel-marshal.ts'; -import { makeKernelStore } from './store/index.ts'; -import type { KernelStore } from './store/index.ts'; +import { makeKernelStore } from './store/kernel-store.ts'; +import type { KernelStore } from './store/kernel-store.ts'; import { parseRef } from './store/utils/parse-ref.ts'; import { isPromiseRef } from './store/utils/promise-ref.ts'; import type { diff --git a/packages/kernel/src/VatHandle.test.ts b/packages/kernel/src/VatHandle.test.ts index 53671e7ad..9a62e6847 100644 --- a/packages/kernel/src/VatHandle.test.ts +++ b/packages/kernel/src/VatHandle.test.ts @@ -8,8 +8,8 @@ import type { MockInstance } from 'vitest'; import { Kernel } from './Kernel.ts'; import { isVatCommandReply, VatCommandMethod } from './messages/index.ts'; import type { VatCommand, VatCommandReply } from './messages/index.ts'; -import { makeKernelStore } from './store/index.ts'; -import type { KernelStore } from './store/index.ts'; +import { makeKernelStore } from './store/kernel-store.ts'; +import type { KernelStore } from './store/kernel-store.ts'; import { VatHandle } from './VatHandle.ts'; import { makeMapKernelDatabase } from '../test/storage.ts'; diff --git a/packages/kernel/src/VatHandle.ts b/packages/kernel/src/VatHandle.ts index 330340e47..bbc54597f 100644 --- a/packages/kernel/src/VatHandle.ts +++ b/packages/kernel/src/VatHandle.ts @@ -18,7 +18,7 @@ import type { VatCommand, VatCommandReturnType, } from './messages/index.ts'; -import type { KernelStore } from './store'; +import type { KernelStore } from './store/kernel-store.ts'; import { parseRef } from './store/utils/parse-ref.ts'; import type { PromiseCallbacks, diff --git a/packages/kernel/src/services/garbage-collection.test.ts b/packages/kernel/src/services/garbage-collection.test.ts index b739efe6f..2f4a4de56 100644 --- a/packages/kernel/src/services/garbage-collection.test.ts +++ b/packages/kernel/src/services/garbage-collection.test.ts @@ -2,7 +2,7 @@ import { describe, it, expect, beforeEach } from 'vitest'; import { processGCActionSet } from './garbage-collection.ts'; import { makeMapKernelDatabase } from '../../test/storage.ts'; -import { makeKernelStore } from '../store/index.ts'; +import { makeKernelStore } from '../store/kernel-store.ts'; import { RunQueueItemType } from '../types.ts'; describe('garbage-collection', () => { diff --git a/packages/kernel/src/services/garbage-collection.ts b/packages/kernel/src/services/garbage-collection.ts index d7ea28d9e..be599967e 100644 --- a/packages/kernel/src/services/garbage-collection.ts +++ b/packages/kernel/src/services/garbage-collection.ts @@ -1,4 +1,4 @@ -import type { KernelStore } from '../store'; +import type { KernelStore } from '../store/kernel-store.ts'; import { insistKernelType } from '../store/utils/kernel-slots.ts'; import type { GCAction, diff --git a/packages/kernel/src/store/index.ts b/packages/kernel/src/store/index.ts deleted file mode 100644 index f8dddfde0..000000000 --- a/packages/kernel/src/store/index.ts +++ /dev/null @@ -1,207 +0,0 @@ -/* - * Organization of keys in the key value store: - * - * Definitions - * NN ::= some decimal integer - * CAPDATA ::= capdata encoded structure value - * JSON(${xx}) ::= JSON encoding of ${xx} - * - * ${koid} ::= ko${NN} // kernel object ID - * ${kpid} ::= kp${NN} // kernel promise ID - * ${kref} ::= ${koid} | ${kpid} // kernel reference - * ${dir} ::= + | - // direction (for remote and vat references) - * ${roid} ::= ro${dir}${NN} // remote object ID - * ${rpid} ::= rp${dir}${NN} // remote promise ID - * ${rref} ::= ${roid} | ${rpid} // remote reference - * ${void} ::= o${dir}${NN} // vat object ID - * ${vpid} ::= p${dir}${NN} // vat promise ID - * ${vref} ::= ${void} | ${vpid} // vat reference - * ${eref} ::= ${vref} | ${rref} // external reference - * ${vatid} ::= v${NN} // vat ID - * ${remid} ::= r${NN} // remote ID - * ${endid} ::= ${vatid} | ${remid} // endpoint ID - * ${queueName} ::= run | ${kpid} - * - * Queues - * queue.${queueName}.head = NN // queue head index - * queue.${queueName}.tail = NN // queue tail index - * queue.${queueName}.${NN} = JSON(CAPDATA) // queue entry #NN - * - * Kernel objects - * ${koid}.refCount = NN // reference count - * ${koid}.owner = ${vatid} // owner (where the object is) - * - * Kernel promises - * ${kpid}.refCount = NN // reference count - * ${kpid}.state = unresolved | fulfilled | rejected // current state of settlement - * ${kpid}.subscribers = JSON([${endid}]) // array of who is waiting for settlement - * ${kpid}.decider = ${endid} // who decides on settlement - * ${kpid}.value = JSON(CAPDATA) // value settled to, if settled - * - * C-lists - * cle.${endpointId}.${eref} = ${kref} // ERef->KRef mapping - * clk.${endpointId}.${kref} = ${eref} // KRef->ERef mapping - * - * Vat bookkeeping - * e.nextObjectId.${endid} = NN // allocation counter for imported object ERefs - * e.nextPromiseId.${endid} = NN // allocation counter for imported promise ERefs - * - * Kernel bookkeeping - * nextVatId = NN // allocation counter for vat IDs - * nextRemoteId = NN // allocation counter for remote IDs - * k.nextObjectId = NN // allocation counter for object KRefs - * k.nextPromiseId = NN // allocation counter for promise KRefs - */ - -import type { KernelDatabase, KVStore, VatStore } from '@ocap/store'; - -import { getBaseMethods } from './methods/base.ts'; -import { getCListMethods } from './methods/clist.ts'; -import { getGCMethods } from './methods/gc.ts'; -import { getIdMethods } from './methods/id.ts'; -import { getObjectMethods } from './methods/object.ts'; -import { getPromiseMethods } from './methods/promise.ts'; -import { getQueueMethods } from './methods/queue.ts'; -import { getRefCountMethods } from './methods/refcount.ts'; -import type { StoreContext } from './types.ts'; -import type { KRef, RunQueueItem } from '../types.ts'; - -/** - * Create a new KernelStore object wrapped around a raw kernel database. The - * resulting object provides a variety of operations for accessing various - * kernel-relevent persistent data structure abstractions on their own terms, - * without burdening the kernel with the particular details of how they are - * represented in storage. It is our hope that these operations may be later - * reimplemented on top of a more sophisticated database layer that can realize - * them more directly (and thus, one hopes, more efficiently) without requiring - * the kernel itself to be any the wiser. - * - * @param kdb - The kernel database this store is based on. - * @returns A KernelStore object that maps various persistent kernel data - * structures onto `kdb`. - */ -// eslint-disable-next-line @typescript-eslint/explicit-function-return-type -export function makeKernelStore(kdb: KernelDatabase) { - // Initialize core state - - /** KV store in which all the kernel's own state is kept. */ - const kv: KVStore = kdb.kernelKVStore; - - const { provideCachedStoredValue } = getBaseMethods(kv); - const queue = getQueueMethods(kv); - - const context: StoreContext = { - kv, - /** The kernel's run queue. */ - runQueue: queue.createStoredQueue('run', true), - /** Counter for allocating kernel object IDs */ - nextObjectId: provideCachedStoredValue('nextObjectId', '1'), - /** Counter for allocating kernel promise IDs */ - nextPromiseId: provideCachedStoredValue('nextPromiseId', '1'), - /** Counter for allocating VatIDs */ - nextVatId: provideCachedStoredValue('nextVatId', '1'), - /** Counter for allocating RemoteIDs */ - nextRemoteId: provideCachedStoredValue('nextRemoteId', '1'), - // As refcounts are decremented, we accumulate a set of krefs for which - // action might need to be taken: - // * promises which are now resolved and unreferenced can be deleted - // * objects which are no longer reachable: export can be dropped - // * objects which are no longer recognizable: export can be retired - // This set is ephemeral: it lives in RAM, grows as deliveries and syscalls - // cause decrefs, and will be harvested by processRefcounts(). This needs to be - // called in the same transaction window as the syscalls/etc which prompted - // the change, else removals might be lost (not performed during the next - // replay). - maybeFreeKrefs: new Set(), - // Garbage collection - gcActions: provideCachedStoredValue('gcActions', '[]'), - reapQueue: provideCachedStoredValue('reapQueue', '[]'), - }; - - const id = getIdMethods(context); - const refCount = getRefCountMethods(context); - const object = getObjectMethods(context); - const promise = getPromiseMethods(context); - const gc = getGCMethods(context); - const cList = getCListMethods(context); - - /** - * Append a message to the kernel's run queue. - * - * @param message - The message to enqueue. - */ - function enqueueRun(message: RunQueueItem): void { - context.runQueue.enqueue(message); - } - - /** - * Fetch the next message on the kernel's run queue. - * - * @returns The next message on the run queue, or undefined if the queue is - * empty. - */ - function dequeueRun(): RunQueueItem | undefined { - return context.runQueue.dequeue() as RunQueueItem | undefined; - } - - /** - * Obtain the number of entries in the run queue. - * - * @returns the number of items in the run queue. - */ - function runQueueLength(): number { - return queue.getQueueLength('run'); - } - - /** - * Create a new VatStore for a vat. - * - * @param vatID - The vat for which this is being done. - * - * @returns a a VatStore object for the given vat. - */ - function makeVatStore(vatID: string): VatStore { - return kdb.makeVatStore(vatID); - } - - /** - * Reset the kernel's persistent state and reset all counters. - */ - function reset(): void { - kdb.clear(); - context.maybeFreeKrefs.clear(); - context.runQueue = queue.createStoredQueue('run', true); - context.gcActions = provideCachedStoredValue('gcActions', '[]'); - context.reapQueue = provideCachedStoredValue('reapQueue', '[]'); - context.nextObjectId = provideCachedStoredValue('nextObjectId', '1'); - context.nextPromiseId = provideCachedStoredValue('nextPromiseId', '1'); - context.nextVatId = provideCachedStoredValue('nextVatId', '1'); - context.nextRemoteId = provideCachedStoredValue('nextRemoteId', '1'); - } - - /** - * Delete everything from the database. - */ - function clear(): void { - kdb.clear(); - } - - return harden({ - ...id, - ...queue, - ...refCount, - ...object, - ...promise, - ...gc, - ...cList, - enqueueRun, - dequeueRun, - runQueueLength, - makeVatStore, - clear, - reset, - kv, - }); -} - -export type KernelStore = ReturnType; diff --git a/packages/kernel/src/store/index.test.ts b/packages/kernel/src/store/kernel-store.test.ts similarity index 98% rename from packages/kernel/src/store/index.test.ts rename to packages/kernel/src/store/kernel-store.test.ts index c4d04a629..83799c1ed 100644 --- a/packages/kernel/src/store/index.test.ts +++ b/packages/kernel/src/store/kernel-store.test.ts @@ -2,7 +2,7 @@ import type { Message } from '@agoric/swingset-liveslots'; import type { KernelDatabase } from '@ocap/store'; import { describe, it, expect, beforeEach } from 'vitest'; -import { makeKernelStore } from './index.ts'; +import { makeKernelStore } from './kernel-store.ts'; import { makeMapKernelDatabase } from '../../test/storage.ts'; import type { RunQueueItem } from '../types.ts'; @@ -74,8 +74,6 @@ describe('kernel store', () => { 'getGCActions', 'getKernelPromise', 'getKernelPromiseMessageQueue', - 'getNextObjectId', - 'getNextPromiseId', 'getNextRemoteId', 'getNextVatId', 'getObjectRefCount', @@ -94,8 +92,6 @@ describe('kernel store', () => { 'kv', 'makeVatStore', 'nextReapAction', - 'provideStoredQueue', - 'refCountKey', 'reset', 'resolveKernelPromise', 'runQueueLength', diff --git a/packages/kernel/src/store/kernel-store.ts b/packages/kernel/src/store/kernel-store.ts new file mode 100644 index 000000000..2d059fe3c --- /dev/null +++ b/packages/kernel/src/store/kernel-store.ts @@ -0,0 +1,1105 @@ +/* + * Organization of keys in the key value store: + * + * Definitions + * NN ::= some decimal integer + * CAPDATA ::= capdata encoded structure value + * JSON(${xx}) ::= JSON encoding of ${xx} + * + * ${koid} ::= ko${NN} // kernel object ID + * ${kpid} ::= kp${NN} // kernel promise ID + * ${kref} ::= ${koid} | ${kpid} // kernel reference + * ${dir} ::= + | - // direction (for remote and vat references) + * ${roid} ::= ro${dir}${NN} // remote object ID + * ${rpid} ::= rp${dir}${NN} // remote promise ID + * ${rref} ::= ${roid} | ${rpid} // remote reference + * ${void} ::= o${dir}${NN} // vat object ID + * ${vpid} ::= p${dir}${NN} // vat promise ID + * ${vref} ::= ${void} | ${vpid} // vat reference + * ${eref} ::= ${vref} | ${rref} // external reference + * ${vatid} ::= v${NN} // vat ID + * ${remid} ::= r${NN} // remote ID + * ${endid} ::= ${vatid} | ${remid} // endpoint ID + * ${queueName} ::= run | ${kpid} + * + * Queues + * queue.${queueName}.head = NN // queue head index + * queue.${queueName}.tail = NN // queue tail index + * queue.${queueName}.${NN} = JSON(CAPDATA) // queue entry #NN + * + * Kernel objects + * ${koid}.refCount = NN // reference count + * ${koid}.owner = ${vatid} // owner (where the object is) + * + * Kernel promises + * ${kpid}.refCount = NN // reference count + * ${kpid}.state = unresolved | fulfilled | rejected // current state of settlement + * ${kpid}.subscribers = JSON([${endid}]) // array of who is waiting for settlement + * ${kpid}.decider = ${endid} // who decides on settlement + * ${kpid}.value = JSON(CAPDATA) // value settled to, if settled + * + * C-lists + * cle.${endpointId}.${eref} = ${kref} // ERef->KRef mapping + * clk.${endpointId}.${kref} = ${eref} // KRef->ERef mapping + * + * Vat bookkeeping + * e.nextObjectId.${endid} = NN // allocation counter for imported object ERefs + * e.nextPromiseId.${endid} = NN // allocation counter for imported promise ERefs + * + * Kernel bookkeeping + * nextVatId = NN // allocation counter for vat IDs + * nextRemoteId = NN // allocation counter for remote IDs + * k.nextObjectId = NN // allocation counter for object KRefs + * k.nextPromiseId = NN // allocation counter for promise KRefs + */ + +import type { Message } from '@agoric/swingset-liveslots'; +import { Fail } from '@endo/errors'; +import type { CapData } from '@endo/marshal'; +import type { KVStore, VatStore, KernelDatabase } from '@ocap/store'; + +import { insistKernelType } from './utils/kernel-slots.ts'; +import { parseRef } from './utils/parse-ref.ts'; +import { + buildReachableAndVatSlot, + parseReachableAndVatSlot, +} from './utils/reachable.ts'; +import type { + VatId, + RemoteId, + EndpointId, + KRef, + ERef, + RunQueueItem, + PromiseState, + KernelPromise, + RunQueueItemBringOutYourDead, + GCAction, +} from '../types.ts'; +import { insistGCActionType, insistVatId, RunQueueItemType } from '../types.ts'; + +type StoredValue = { + get(): string | undefined; + set(newValue: string): void; + delete(): void; +}; + +type StoredQueue = { + enqueue(item: object): void; + dequeue(): object | undefined; + delete(): void; +}; + +/** + * Test if a KRef designates a promise. + * + * @param kref - The KRef to test. + * + * @returns true iff the given KRef references a promise. + */ +export function isPromiseRef(kref: KRef): boolean { + return kref[1] === 'p'; +} + +/** + * Create a new KernelStore object wrapped around a raw kernel database. The + * resulting object provides a variety of operations for accessing various + * kernel-relevent persistent data structure abstractions on their own terms, + * without burdening the kernel with the particular details of how they are + * represented in storage. It is our hope that these operations may be later + * reimplemented on top of a more sophisticated database layer that can realize + * them more directly (and thus, one hopes, more efficiently) without requiring + * the kernel itself to be any the wiser. + * + * @param kdb - The kernel database this store is based on. + * @returns A KernelStore object that maps various persistent kernel data + * structures onto `kdb`. + */ +// eslint-disable-next-line @typescript-eslint/explicit-function-return-type +export function makeKernelStore(kdb: KernelDatabase) { + // Initialize core state + + /** KV store in which all the kernel's own state is kept. */ + const kv: KVStore = kdb.kernelKVStore; + + /** The kernel's run queue. */ + let runQueue = createStoredQueue('run', true); + /** Counter for allocating VatIDs */ + let nextVatId = provideCachedStoredValue('nextVatId', '1'); + /** Counter for allocating RemoteIDs */ + let nextRemoteId = provideCachedStoredValue('nextRemoteId', '1'); + /** Counter for allocating kernel object IDs */ + let nextObjectId = provideCachedStoredValue('nextObjectId', '1'); + /** Counter for allocating kernel promise IDs */ + let nextPromiseId = provideCachedStoredValue('nextPromiseId', '1'); + + // As refcounts are decremented, we accumulate a set of krefs for which + // action might need to be taken: + // * promises which are now resolved and unreferenced can be deleted + // * objects which are no longer reachable: export can be dropped + // * objects which are no longer recognizable: export can be retired + // This set is ephemeral: it lives in RAM, grows as deliveries and syscalls + // cause decrefs, and will be harvested by processRefcounts(). This needs to be + // called in the same transaction window as the syscalls/etc which prompted + // the change, else removals might be lost (not performed during the next + // replay). + const maybeFreeKrefs = new Set(); + // Garbage collection + let gcActions = provideCachedStoredValue('gcActions', '[]'); + let reapQueue = provideCachedStoredValue('reapQueue', '[]'); + + /** + * Provide a stored value object for which we keep an in-memory cache. We only + * touch persistent storage if the value hasn't ever been read of if it is + * modified; otherwise we can service read requests from memory. + * + * IMPORTANT NOTE: in order for the cache to work, all subsequent accesses to + * the value MUST be made through a common stored value object. + * + * @param key - A key string that identifies the value. + * @param init - If provided, an initial setting if the stored entity does not exist. + * @returns An object for interacting with the value. + */ + function provideCachedStoredValue(key: string, init?: string): StoredValue { + let value: string | undefined = kv.get(key); + if (value === undefined && init !== undefined) { + kv.set(key, init); + value = init; + } + return harden({ + get(): string | undefined { + return value; + }, + set(newValue: string): void { + value = newValue; + kv.set(key, value); + }, + delete(): void { + value = undefined; + kv.delete(key); + }, + }); + } + + /** + * Provide a stored value object that is kept in persistent storage without caching. + * + * @param key - A key string that identifies the value. + * @param init - If provided, an initial setting if the stored entity does not exist. + * @returns An object for interacting with the value. + */ + function provideRawStoredValue(key: string, init?: string): StoredValue { + if (kv.get(key) === undefined && init !== undefined) { + kv.set(key, init); + } + return harden({ + get: () => kv.get(key), + set: (newValue: string) => kv.set(key, newValue), + delete: () => kv.delete(key), + }); + } + + /** + * Increment the value of a persistently stored counter. + * + * Note that the while the value is interpreted as an integer (in order to + * enable it to be incremented), it is stored and returned in the form of a + * string. This is because (a) our persistent storage only stores strings, and + * (b) the sole purpose of one of these counters is simply to provide an + * unending sequence of unique values; we don't actually use them as numbers + * or, indeed, even care at all if this sequence is produced using numbers. + * + * @param value - Reference to the stored value to be incremented. + * @returns The value as it was prior to being incremented. + */ + function incCounter(value: StoredValue): string { + const current = value.get(); + const next = `${Number(current) + 1}`; + value.set(next); + return current as string; + } + + /** + * Create a new (empty) persistently stored queue. + * + * @param queueName - The name for the queue (must be unique among queues). + * @param cached - Optional flag: set to true if the queue should cache its + * @returns An object for interacting with the new queue. + */ + function createStoredQueue( + queueName: string, + cached: boolean = false, + ): StoredQueue { + const qk = `queue.${queueName}`; + kv.set(`${qk}.head`, '1'); + kv.set(`${qk}.tail`, '1'); + return provideStoredQueue(queueName, cached); + } + + /** + * Find out how long some queue is. + * + * @param queueName - The name of the queue of interest. + * + * @returns the number of items in the given queue. + */ + function getQueueLength(queueName: string): number { + const qk = `queue.${queueName}`; + const head = kv.get(`${qk}.head`); + const tail = kv.get(`${qk}.tail`); + if (head === undefined || tail === undefined) { + throw Error(`unknown queue ${queueName}`); + } + return Number(head) - Number(tail); + } + + /** + * Produce an object to access a persistently stored queue. + * + * @param queueName - The name for the queue (must be unique among queues). + * @param cached - Optional flag: set to true if the queue should cache its + * limit indices in memory (only do this if the queue is going to be accessed or + * checked frequently). + * @returns An object for interacting with the queue. + */ + function provideStoredQueue( + queueName: string, + cached: boolean = false, + ): StoredQueue { + const qk = `queue.${queueName}`; + // Note: cached=true ==> caches only the head & tail indices, NOT the queue entries themselves + const provideValue = cached + ? provideCachedStoredValue + : provideRawStoredValue; + const head = provideValue(`${qk}.head`); + const tail = provideValue(`${qk}.tail`); + if (head.get() === undefined || tail.get() === undefined) { + throw Error(`queue ${queueName} not initialized`); + } + return { + enqueue(item: object): void { + if (head.get() === undefined) { + throw Error(`enqueue into deleted queue ${queueName}`); + } + const entryPos = incCounter(head); + kv.set(`${qk}.${entryPos}`, JSON.stringify(item)); + }, + dequeue(): object | undefined { + const headPos = head.get(); + if (headPos === undefined) { + return undefined; + } + const tailPos = tail.get(); + if (tailPos !== headPos) { + const entry = kv.getRequired(`${qk}.${tailPos}`); + kv.delete(`${qk}.${tailPos}`); + incCounter(tail); + return JSON.parse(entry) as object; + } + return undefined; + }, + delete(): void { + const headPos = head.get(); + if (headPos !== undefined) { + let tailPos = tail.get(); + while (tailPos !== headPos) { + kv.delete(`${qk}.${tailPos}`); + tailPos = `${Number(tailPos) + 1}`; + } + head.delete(); + tail.delete(); + } + }, + }; + } + + /** + * Append a message to the kernel's run queue. + * + * @param message - The message to enqueue. + */ + function enqueueRun(message: RunQueueItem): void { + runQueue.enqueue(message); + } + + /** + * Fetch the next message on the kernel's run queue. + * + * @returns The next message on the run queue, or undefined if the queue is + * empty. + */ + function dequeueRun(): RunQueueItem | undefined { + return runQueue.dequeue() as RunQueueItem | undefined; + } + + /** + * Obtain the number of entries in the run queue. + * + * @returns the number of items in the run queue. + */ + function runQueueLength(): number { + return getQueueLength('run'); + } + + /** + * Obtain an ID for a new vat. + * + * @returns The next VatID use. + */ + function getNextVatId(): VatId { + return `v${incCounter(nextVatId)}`; + } + + /** + * Obtain an ID for a new remote connection. + * + * @returns The next remote ID use. + */ + function getNextRemoteId(): RemoteId { + return `r${incCounter(nextRemoteId)}`; + } + + /** + * Initialize persistent state for a new endpoint. + * + * @param endpointId - The ID of the endpoint being added. + */ + function initEndpoint(endpointId: EndpointId): void { + kv.set(`e.nextPromiseId.${endpointId}`, '1'); + kv.set(`e.nextObjectId.${endpointId}`, '1'); + } + + /** + * Generate a new eref for a kernel object or promise being imported into an + * endpoint. + * + * @param endpointId - The endpoint the kref is being imported into. + * @param kref - The kref for the kernel object or promise in question. + * + * @returns A new eref in the scope of the given endpoint for the given kernel entity. + */ + function allocateErefForKref(endpointId: EndpointId, kref: KRef): ERef { + let id; + const refTag = endpointId.startsWith('v') ? '' : endpointId[0]; + let refType; + if (isPromiseRef(kref)) { + id = kv.get(`e.nextPromiseId.${endpointId}`); + kv.set(`e.nextPromiseId.${endpointId}`, `${Number(id) + 1}`); + refType = 'p'; + } else { + id = kv.get(`e.nextObjectId.${endpointId}`); + kv.set(`e.nextObjectId.${endpointId}`, `${Number(id) + 1}`); + refType = 'o'; + } + const eref = `${refTag}${refType}-${id}`; + addClistEntry(endpointId, kref, eref); + return eref; + } + + /** + * Obtain a KRef for the next unallocated kernel object. + * + * @returns The next koId use. + */ + function getNextObjectId(): KRef { + return `ko${incCounter(nextObjectId)}`; + } + + /** + * Generate the storage key for a kernel entity's reference count. + * + * @param kref - The KRef of interest. + * @returns the key to store the indicated reference count at. + */ + function refCountKey(kref: KRef): string { + return `${kref}.refCount`; + } + + /** + * Get a kernel entity's reference count. + * + * @param kref - The KRef of interest. + * @returns the reference count of the indicated kernel entity. + */ + function getRefCount(kref: KRef): number { + return Number(kv.get(refCountKey(kref))); + } + + /** + * Increment a kernel entity's reference count. + * + * @param kref - The KRef of the entity to increment the ref count of. + * @returns the new reference count after incrementing. + */ + function incRefCount(kref: KRef): number { + const key = refCountKey(kref); + const newCount = Number(kv.get(key)) + 1; + kv.set(key, `${newCount}`); + return newCount; + } + + /** + * Decrement a kernel entity's reference count. + * + * @param kref - The KRef of the entity to decrement the ref count of. + * @returns the new reference count after decrementing. + */ + function decRefCount(kref: KRef): number { + const key = refCountKey(kref); + const newCount = Number(kv.get(key)) - 1; + kv.set(key, `${newCount}`); + return newCount; + } + + /** + * Create a new kernel object. The new object will be born with reference and + * recognizability counts of 1, on the assumption that the new object + * corresponds to an object that has just been imported from somewhere. + * + * @param owner - The endpoint that is the owner of the new object. + * @returns The new object's KRef. + */ + function initKernelObject(owner: EndpointId): KRef { + const koId = getNextObjectId(); + kv.set(`${koId}.owner`, owner); + setObjectRefCount(koId, { reachable: 1, recognizable: 1 }); + return koId; + } + + /** + * Get a kernel object's owner. + * + * @param koId - The KRef of the kernel object of interest. + * @returns The identity of the vat or remote that owns the object. + */ + function getOwner(koId: KRef): EndpointId { + const owner = kv.get(`${koId}.owner`); + if (owner === undefined) { + throw Error(`unknown kernel object ${koId}`); + } + return owner; + } + + /** + * Expunge a kernel object from the kernel's persistent state. + * + * @param koId - The KRef of the kernel object to delete. + */ + function deleteKernelObject(koId: KRef): void { + kv.delete(`${koId}.owner`); + kv.delete(refCountKey(koId)); + } + + /** + * Obtain a KRef for the next unallocated kernel promise. + * + * @returns The next kpid use. + */ + function getNextPromiseId(): KRef { + return `kp${incCounter(nextPromiseId)}`; + } + + /** + * Create a new, unresolved kernel promise. The new promise will be born with + * a reference count of 1 on the assumption that the promise has just been + * imported from somewhere. + * + * @returns A tuple of the new promise's KRef and an object describing the + * new promise itself. + */ + function initKernelPromise(): [KRef, KernelPromise] { + const kpr: KernelPromise = { + state: 'unresolved', + subscribers: [], + }; + const kpid = getNextPromiseId(); + createStoredQueue(kpid, false); + kv.set(`${kpid}.state`, 'unresolved'); + kv.set(`${kpid}.subscribers`, '[]'); + kv.set(refCountKey(kpid), '1'); + return [kpid, kpr]; + } + + /** + * Append a message to a promise's message queue. + * + * @param kpid - The KRef of the promise to enqueue on. + * @param message - The message to enqueue. + */ + function enqueuePromiseMessage(kpid: KRef, message: Message): void { + provideStoredQueue(kpid, false).enqueue(message); + } + + /** + * Add a new subscriber to a kernel promise's collection of subscribers. + * + * @param vatId - The vat that is subscribing. + * @param kpid - The KRef of the promise being subscribed to. + */ + function addPromiseSubscriber(vatId: VatId, kpid: KRef): void { + insistVatId(vatId); + const kp = getKernelPromise(kpid); + kp.state === 'unresolved' || + Fail`attempt to add subscriber to resolved promise ${kpid}`; + const tempSet = new Set(kp.subscribers); + tempSet.add(vatId); + const newSubscribers = Array.from(tempSet).sort(); + const key = `${kpid}.subscribers`; + kv.set(key, JSON.stringify(newSubscribers)); + } + + /** + * Assign a kernel promise's decider. + * + * @param kpid - The KRef of promise whose decider is being set. + * @param vatId - The vat which will become the decider. + */ + function setPromiseDecider(kpid: KRef, vatId: VatId): void { + insistVatId(vatId); + if (kpid) { + kv.set(`${kpid}.decider`, vatId); + } + } + + /** + * Fetch the descriptive record for a kernel promise. + * + * @param kpid - The KRef of the kernel promise of interest. + * @returns An object describing the requested kernel promise. + */ + function getKernelPromise(kpid: KRef): KernelPromise { + const { context, isPromise } = parseRef(kpid); + assert(context === 'kernel' && isPromise); + const state = kv.get(`${kpid}.state`) as PromiseState; + if (state === undefined) { + throw Error(`unknown kernel promise ${kpid}`); + } + const result: KernelPromise = { state }; + switch (state as string) { + case 'unresolved': { + const decider = kv.get(`${kpid}.decider`); + if (decider !== '' && decider !== undefined) { + result.decider = decider; + } + const subscribers = kv.getRequired(`${kpid}.subscribers`); + result.subscribers = JSON.parse(subscribers); + break; + } + case 'fulfilled': + case 'rejected': { + result.value = JSON.parse(kv.getRequired(`${kpid}.value`)); + break; + } + default: + throw Error(`unknown state for ${kpid}: ${state}`); + } + return result; + } + + /** + * Fetch the messages in a kernel promise's message queue. + * + * @param kpid - The KRef of the kernel promise of interest. + * @returns An array of all the messages in the given promise's message queue. + */ + function getKernelPromiseMessageQueue(kpid: KRef): Message[] { + const result: Message[] = []; + const queue = provideStoredQueue(kpid, false); + for (;;) { + const message = queue.dequeue() as Message; + if (message) { + result.push(message); + } else { + return result; + } + } + } + + /** + * Record the resolution of a kernel promise. + * + * @param kpid - The ref of the promise being resolved. + * @param rejected - True if the promise is being rejected, false if fulfilled. + * @param value - The value the promise is being fulfilled to or rejected with. + */ + function resolveKernelPromise( + kpid: KRef, + rejected: boolean, + value: CapData, + ): void { + const queue = provideStoredQueue(kpid, false); + for (const message of getKernelPromiseMessageQueue(kpid)) { + queue.enqueue(message); + } + kv.set(`${kpid}.state`, rejected ? 'rejected' : 'fulfilled'); + kv.set(`${kpid}.value`, JSON.stringify(value)); + kv.delete(`${kpid}.decider`); + kv.delete(`${kpid}.subscribers`); + } + + /** + * Expunge a kernel promise from the kernel's persistent state. + * + * @param kpid - The KRef of the kernel promise to delete. + */ + function deleteKernelPromise(kpid: KRef): void { + kv.delete(`${kpid}.state`); + kv.delete(`${kpid}.decider`); + kv.delete(`${kpid}.subscribers`); + kv.delete(`${kpid}.value`); + kv.delete(refCountKey(kpid)); + provideStoredQueue(kpid).delete(); + } + + /** + * Look up the ERef that and endpoint's c-list maps a KRef to. + * + * @param endpointId - The endpoint in question. + * @param eref - The ERef to look up. + * @returns The KRef corresponding to `eref` in the given endpoints c-list, or undefined + * if there is no such mapping. + */ + function erefToKref(endpointId: EndpointId, eref: ERef): KRef | undefined { + return kv.get(getSlotKey(endpointId, eref)); + } + + /** + * Look up the KRef that and endpoint's c-list maps an ERef to. + * + * @param endpointId - The endpoint in question. + * @param kref - The KRef to look up. + * @returns The given endpoint's ERef corresponding to `kref`, or undefined if + * there is no such mapping. + */ + function krefToEref(endpointId: EndpointId, kref: KRef): ERef | undefined { + const key = getSlotKey(endpointId, kref); + const data = kv.get(key); + if (!data) { + return undefined; + } + const { vatSlot } = parseReachableAndVatSlot(data); + return vatSlot; + } + + /** + * Add an entry to an endpoint's c-list, creating a new bidirectional mapping + * between an ERef belonging to the endpoint and a KRef belonging to the + * kernel. + * + * @param endpointId - The endpoint whose c-list is to be added to. + * @param kref - The KRef. + * @param eref - The ERef. + */ + function addClistEntry(endpointId: EndpointId, kref: KRef, eref: ERef): void { + kv.set(getSlotKey(endpointId, kref), buildReachableAndVatSlot(true, eref)); + kv.set(getSlotKey(endpointId, eref), kref); + } + + /** + * Remove an entry from an endpoint's c-list. + * + * @param endpointId - The endpoint whose c-list entry is to be removed. + * @param kref - The KRef. + * @param eref - The ERef. + */ + function deleteClistEntry( + endpointId: EndpointId, + kref: KRef, + eref: ERef, + ): void { + const kernelKey = getSlotKey(endpointId, kref); + const vatKey = getSlotKey(endpointId, eref); + assert(kv.get(kernelKey)); + clearReachableFlag(endpointId, kref); + const { direction } = parseRef(eref); + decrementRefCount(kref, { + isExport: direction === 'export', + onlyRecognizable: true, + }); + kv.delete(kernelKey); + kv.delete(vatKey); + } + + /** + * Remove an entry from an endpoint's c-list given an eref. + * + * @param endpointId - The endpoint whose c-list entry is to be removed. + * @param eref - The ERef. + */ + function forgetEref(endpointId: EndpointId, eref: ERef): void { + const kref = erefToKref(endpointId, eref); + if (kref) { + deleteClistEntry(endpointId, kref, eref); + } + } + + /** + * Remove an entry from an endpoint's c-list given a kref. + * + * @param endpointId - The endpoint whose c-list entry is to be removed. + * @param kref - The Kref. + */ + function forgetKref(endpointId: EndpointId, kref: KRef): void { + const eref = krefToEref(endpointId, kref); + if (eref) { + deleteClistEntry(endpointId, kref, eref); + } + } + + /** + * Delete everything from the database. + */ + function clear(): void { + kdb.clear(); + } + + /** + * Create a new VatStore for a vat. + * + * @param vatID - The vat for which this is being done. + * + * @returns a a VatStore object for the given vat. + */ + function makeVatStore(vatID: string): VatStore { + return kdb.makeVatStore(vatID); + } + + /** + * Reset the kernel's persistent queues and counters. + */ + function reset(): void { + kdb.clear(); + runQueue = createStoredQueue('run', true); + nextVatId = provideCachedStoredValue('nextVatId', '1'); + nextRemoteId = provideCachedStoredValue('nextRemoteId', '1'); + nextObjectId = provideCachedStoredValue('nextObjectId', '1'); + nextPromiseId = provideCachedStoredValue('nextPromiseId', '1'); + maybeFreeKrefs.clear(); + gcActions = provideCachedStoredValue('gcActions', '[]'); + reapQueue = provideCachedStoredValue('reapQueue', '[]'); + } + + /** + * Check if a kernel object exists in the kernel's persistent state. + * + * @param kref - The KRef of the kernel object in question. + * @returns True if the kernel object exists, false otherwise. + */ + function kernelRefExists(kref: KRef): boolean { + return Boolean(kv.get(refCountKey(kref))); + } + + /** + * Get the reference counts for a kernel object + * + * @param kref - The KRef of the object of interest. + * @returns The reference counts for the object. + */ + function getObjectRefCount(kref: KRef): { + reachable: number; + recognizable: number; + } { + const data = kv.get(refCountKey(kref)); + if (!data) { + return { reachable: 0, recognizable: 0 }; + } + const [reachable = 0, recognizable = 0] = data.split(',').map(Number); + reachable <= recognizable || + Fail`refMismatch(get) ${kref} ${reachable},${recognizable}`; + return { reachable, recognizable }; + } + + /** + * Set the reference counts for a kernel object + * + * @param kref - The KRef of the object of interest. + * @param counts - The reference counts to set. + * @param counts.reachable - The reachable reference count. + * @param counts.recognizable - The recognizable reference count. + */ + function setObjectRefCount( + kref: KRef, + counts: { reachable: number; recognizable: number }, + ): void { + const { reachable, recognizable } = counts; + assert.typeof(reachable, 'number'); + assert.typeof(recognizable, 'number'); + (reachable >= 0 && recognizable >= 0) || + Fail`${kref} underflow ${reachable},${recognizable}`; + reachable <= recognizable || + Fail`refMismatch(set) ${kref} ${reachable},${recognizable}`; + kv.set(refCountKey(kref), `${reachable},${recognizable}`); + } + + /** + * Get the set of GC actions to perform. + * + * @returns The set of GC actions to perform. + */ + function getGCActions(): Set { + return new Set(JSON.parse(gcActions.get() ?? '[]')); + } + + /** + * Set the set of GC actions to perform. + * + * @param actions - The set of GC actions to perform. + */ + function setGCActions(actions: Set): void { + const a = Array.from(actions); + a.sort(); + gcActions.set(JSON.stringify(a)); + } + + /** + * Add a new GC action to the set of GC actions to perform. + * + * @param newActions - The new GC action to add. + */ + function addGCActions(newActions: GCAction[]): void { + const actions = getGCActions(); + for (const action of newActions) { + assert.typeof(action, 'string', 'addGCActions given bad action'); + const [vatId, type, kref] = action.split(' '); + insistVatId(vatId); + insistGCActionType(type); + insistKernelType('object', kref); + actions.add(action); + } + setGCActions(actions); + } + + /** + * Check if a kernel object is reachable. + * + * @param endpointId - The endpoint for which the reachable flag is being checked. + * @param kref - The kref. + * @returns True if the kernel object is reachable, false otherwise. + */ + function getReachableFlag(endpointId: EndpointId, kref: KRef): boolean { + const key = getSlotKey(endpointId, kref); + const data = kv.getRequired(key); + const { isReachable } = parseReachableAndVatSlot(data); + return isReachable; + } + + /** + * Clear the reachable flag for a given endpoint and kref. + * + * @param endpointId - The endpoint for which the reachable flag is being cleared. + * @param kref - The kref. + */ + function clearReachableFlag(endpointId: EndpointId, kref: KRef): void { + const key = getSlotKey(endpointId, kref); + const { isReachable, vatSlot } = parseReachableAndVatSlot( + kv.getRequired(key), + ); + kv.set(key, buildReachableAndVatSlot(false, vatSlot)); + const { direction, isPromise } = parseRef(vatSlot); + // decrement 'reachable' part of refcount, but only for object imports + if ( + isReachable && + !isPromise && + direction === 'import' && + kernelRefExists(kref) + ) { + const counts = getObjectRefCount(kref); + counts.reachable -= 1; + setObjectRefCount(kref, counts); + if (counts.reachable === 0) { + maybeFreeKrefs.add(kref); + } + } + } + + /** + * Schedule a vat for reaping. + * + * @param vatId - The vat to schedule for reaping. + */ + function scheduleReap(vatId: VatId): void { + const queue = JSON.parse(reapQueue.get() ?? '[]'); + if (!queue.includes(vatId)) { + queue.push(vatId); + reapQueue.set(JSON.stringify(queue)); + } + } + + /** + * Get the next reap action. + * + * @returns The next reap action, or undefined if the queue is empty. + */ + function nextReapAction(): RunQueueItemBringOutYourDead | undefined { + const queue = JSON.parse(reapQueue.get() ?? '[]'); + if (queue.length > 0) { + const vatId = queue.shift(); + reapQueue.set(JSON.stringify(queue)); + return harden({ type: RunQueueItemType.bringOutYourDead, vatId }); + } + return undefined; + } + + /** + * Get the key for the reachable flag and vatSlot for a given endpoint and kref. + * + * @param endpointId - The endpoint for which the reachable flag is being set. + * @param kref - The kref. + * @returns The key for the reachable flag and vatSlot. + */ + function getSlotKey(endpointId: EndpointId, kref: KRef): string { + return `${endpointId}.c.${kref}`; + } + + /** + * Test if there's a c-list entry for some slot. + * + * @param endpointId - The endpoint of interest + * @param slot - The slot of interest + * @returns true iff this vat has a c-list entry mapping for `slot`. + */ + function hasCListEntry(endpointId: EndpointId, slot: string): boolean { + return kv.get(getSlotKey(endpointId, slot)) !== undefined; + } + + /** + * Increment the reference count associated with some kernel object. + * + * We track references to promises and objects, but not devices. Promises + * have only a "reachable" count, whereas objects track both "reachable" + * and "recognizable" counts. + * + * @param kref - The kernel slot whose refcount is to be incremented. + * @param options - Options for the increment. + * @param options.isExport - True if the reference comes from a clist export, which counts for promises but not objects. + * @param options.onlyRecognizable - True if the reference provides only recognition, not reachability. + */ + function incrementRefCount( + kref: KRef, + { + isExport = false, + onlyRecognizable = false, + }: { isExport?: boolean; onlyRecognizable?: boolean }, + ): void { + kref || Fail`incrementRefCount called with empty kref`; + + const { isPromise } = parseRef(kref); + if (isPromise) { + const refCount = Number(kv.get(refCountKey(kref))) + 1; + kv.set(refCountKey(kref), `${refCount}`); + return; + } + + // If `isExport` the reference comes from a clist export, which counts for promises but not objects + if (isExport) { + return; + } + + const counts = getObjectRefCount(kref); + if (!onlyRecognizable) { + counts.reachable += 1; + } + counts.recognizable += 1; + setObjectRefCount(kref, counts); + } + + /** + * Decrement the reference count associated with some kernel object. + * + * @param kref - The kernel slot whose refcount is to be decremented. + * @param options - Options for the decrement. + * @param options.isExport - True if the reference comes from a clist export, which counts for promises but not objects. + * @param options.onlyRecognizable - True if the reference provides only recognition, not reachability. + * @returns True if the reference count has been decremented to zero, false if it is still non-zero. + * @throws if this tries to decrement the reference count below zero. + */ + function decrementRefCount( + kref: KRef, + { + isExport = false, + onlyRecognizable = false, + }: { isExport?: boolean; onlyRecognizable?: boolean }, + ): boolean { + kref || Fail`decrementRefCount called with empty kref`; + + const { isPromise } = parseRef(kref); + if (isPromise) { + let refCount = Number(kv.get(refCountKey(kref))); + refCount > 0 || Fail`refCount underflow ${kref}`; + refCount -= 1; + kv.set(refCountKey(kref), `${refCount}`); + if (refCount === 0) { + maybeFreeKrefs.add(kref); + return true; + } + return false; + } + + if (isExport || !kernelRefExists(kref)) { + return false; + } + + const counts = getObjectRefCount(kref); + if (!onlyRecognizable) { + counts.reachable -= 1; + } + counts.recognizable -= 1; + if (!counts.reachable || !counts.recognizable) { + maybeFreeKrefs.add(kref); + } + setObjectRefCount(kref, counts); + + return false; + } + + return harden({ + enqueueRun, + dequeueRun, + runQueueLength, + getNextVatId, + getNextRemoteId, + initEndpoint, + getRefCount, + incRefCount, + decRefCount, + initKernelObject, + getOwner, + deleteKernelObject, + initKernelPromise, + getKernelPromise, + enqueuePromiseMessage, + setPromiseDecider, + getKernelPromiseMessageQueue, + resolveKernelPromise, + deleteKernelPromise, + addPromiseSubscriber, + erefToKref, + allocateErefForKref, + krefToEref, + addClistEntry, + forgetEref, + forgetKref, + clear, + makeVatStore, + reset, + kv, + kernelRefExists, + hasCListEntry, + getReachableFlag, + clearReachableFlag, + getObjectRefCount, + setObjectRefCount, + getGCActions, + setGCActions, + addGCActions, + nextReapAction, + scheduleReap, + incrementRefCount, + decrementRefCount, + createStoredQueue, + deleteClistEntry, + getQueueLength, + }); +} + +export type KernelStore = ReturnType; diff --git a/packages/kernel/src/store/methods/base.test.ts b/packages/kernel/src/store/methods/base.test.ts deleted file mode 100644 index 5b09faf27..000000000 --- a/packages/kernel/src/store/methods/base.test.ts +++ /dev/null @@ -1,160 +0,0 @@ -import type { KVStore } from '@ocap/store'; -import { describe, it, expect, beforeEach } from 'vitest'; - -import { getBaseMethods } from './base.ts'; -import { makeMapKVStore } from '../../../test/storage.ts'; - -describe('base-methods', () => { - let kv: KVStore; - let baseStore: ReturnType; - - beforeEach(() => { - kv = makeMapKVStore(); - baseStore = getBaseMethods(kv); - }); - - describe('getSlotKey', () => { - it('generates correct slot keys', () => { - expect(baseStore.getSlotKey('v1', 'ko123')).toBe('v1.c.ko123'); - expect(baseStore.getSlotKey('r2', 'kp456')).toBe('r2.c.kp456'); - }); - }); - - describe('incCounter', () => { - it('increments a stored counter value', () => { - // Create a stored value to increment - const storedValue = baseStore.provideCachedStoredValue( - 'test-counter', - '5', - ); - - // Increment and check return value - expect(baseStore.incCounter(storedValue)).toBe('5'); - expect(storedValue.get()).toBe('6'); - - // Increment again - expect(baseStore.incCounter(storedValue)).toBe('6'); - expect(storedValue.get()).toBe('7'); - }); - }); - - describe('provideCachedStoredValue', () => { - it('creates a new value if it does not exist', () => { - const value = baseStore.provideCachedStoredValue('new-key', 'initial'); - expect(value.get()).toBe('initial'); - expect(kv.get('new-key')).toBe('initial'); - }); - - it('retrieves an existing value', () => { - kv.set('existing-key', 'existing-value'); - const value = baseStore.provideCachedStoredValue('existing-key'); - expect(value.get()).toBe('existing-value'); - }); - - it('caches values in memory', () => { - const value = baseStore.provideCachedStoredValue('cached-key', 'initial'); - - // Change the value through the stored value object - value.set('updated'); - expect(value.get()).toBe('updated'); - expect(kv.get('cached-key')).toBe('updated'); - - // Change the value directly in the KV store - kv.set('cached-key', 'changed-externally'); - - // The cached value should still return the cached value, not the updated KV store value - // This is because the value is cached in memory - expect(value.get()).toBe('updated'); - - // But a new stored value object should see the updated KV store value - const newValue = baseStore.provideCachedStoredValue('cached-key'); - expect(newValue.get()).toBe('changed-externally'); - }); - - it('deletes values correctly', () => { - const value = baseStore.provideCachedStoredValue( - 'delete-key', - 'to-delete', - ); - expect(value.get()).toBe('to-delete'); - - value.delete(); - expect(value.get()).toBeUndefined(); - expect(kv.get('delete-key')).toBeUndefined(); - }); - }); - - describe('provideRawStoredValue', () => { - it('creates a new value if it does not exist', () => { - const value = baseStore.provideRawStoredValue('new-raw-key', 'initial'); - expect(value.get()).toBe('initial'); - expect(kv.get('new-raw-key')).toBe('initial'); - }); - - it('retrieves an existing value', () => { - kv.set('existing-raw-key', 'existing-value'); - const value = baseStore.provideRawStoredValue('existing-raw-key'); - expect(value.get()).toBe('existing-value'); - }); - - it('does not cache values in memory', () => { - const value = baseStore.provideRawStoredValue('raw-key', 'initial'); - - // Change the value through the stored value object - value.set('updated'); - expect(value.get()).toBe('updated'); - expect(kv.get('raw-key')).toBe('updated'); - - // Change the value directly in the KV store - kv.set('raw-key', 'changed-externally'); - - // The raw value should always read from the KV store - expect(value.get()).toBe('changed-externally'); - }); - - it('deletes values correctly', () => { - const value = baseStore.provideRawStoredValue( - 'delete-raw-key', - 'to-delete', - ); - expect(value.get()).toBe('to-delete'); - - value.delete(); - expect(value.get()).toBeUndefined(); - expect(kv.get('delete-raw-key')).toBeUndefined(); - }); - }); - - describe('integration', () => { - it('works with multiple stored values', () => { - const counter1 = baseStore.provideCachedStoredValue('counter1', '1'); - const counter2 = baseStore.provideCachedStoredValue('counter2', '10'); - - expect(baseStore.incCounter(counter1)).toBe('1'); - expect(baseStore.incCounter(counter2)).toBe('10'); - - expect(counter1.get()).toBe('2'); - expect(counter2.get()).toBe('11'); - }); - - it('supports both cached and raw stored values', () => { - const cachedValue = baseStore.provideCachedStoredValue( - 'cached', - 'cached-value', - ); - const rawValue = baseStore.provideRawStoredValue('raw', 'raw-value'); - - expect(cachedValue.get()).toBe('cached-value'); - expect(rawValue.get()).toBe('raw-value'); - - // Modify directly in KV store - kv.set('cached', 'modified-cached'); - kv.set('raw', 'modified-raw'); - - // Cached value should still return the cached value - expect(cachedValue.get()).toBe('cached-value'); - // Raw value should return the updated value - expect(rawValue.get()).toBe('modified-raw'); - }); - }); -}); diff --git a/packages/kernel/src/store/methods/base.ts b/packages/kernel/src/store/methods/base.ts deleted file mode 100644 index 4117bda48..000000000 --- a/packages/kernel/src/store/methods/base.ts +++ /dev/null @@ -1,103 +0,0 @@ -import type { KVStore } from '@ocap/store'; - -import type { EndpointId, KRef } from '../../types.ts'; -import type { StoredValue } from '../types.ts'; - -/** - * Get the base store methods for managing stored values and queues. - * - * @param kv - The key/value store to provide the underlying persistence mechanism. - * @returns An object with methods for managing stored values and queues. - */ -// eslint-disable-next-line @typescript-eslint/explicit-function-return-type -export function getBaseMethods(kv: KVStore) { - /** - * Get the key for the reachable flag and vatSlot for a given endpoint and kref. - * - * @param endpointId - The endpoint for which the reachable flag is being set. - * @param kref - The kref. - * @returns The key for the reachable flag and vatSlot. - */ - function getSlotKey(endpointId: EndpointId, kref: KRef): string { - return `${endpointId}.c.${kref}`; - } - - /** - * Increment the value of a persistently stored counter. - * - * Note that the while the value is interpreted as an integer (in order to - * enable it to be incremented), it is stored and returned in the form of a - * string. This is because (a) our persistent storage only stores strings, and - * (b) the sole purpose of one of these counters is simply to provide an - * unending sequence of unique values; we don't actually use them as numbers - * or, indeed, even care at all if this sequence is produced using numbers. - * - * @param value - Reference to the stored value to be incremented. - * @returns The value as it was prior to being incremented. - */ - function incCounter(value: StoredValue): string { - const current = value.get(); - const next = `${Number(current) + 1}`; - value.set(next); - return current as string; - } - - /** - * Provide a stored value object for which we keep an in-memory cache. We only - * touch persistent storage if the value hasn't ever been read of if it is - * modified; otherwise we can service read requests from memory. - * - * IMPORTANT NOTE: in order for the cache to work, all subsequent accesses to - * the value MUST be made through a common stored value object. - * - * @param key - A key string that identifies the value. - * @param init - If provided, an initial setting if the stored entity does not exist. - * @returns An object for interacting with the value. - */ - function provideCachedStoredValue(key: string, init?: string): StoredValue { - let value: string | undefined = kv.get(key); - if (value === undefined && init !== undefined) { - kv.set(key, init); - value = init; - } - return harden({ - get(): string | undefined { - return value; - }, - set(newValue: string): void { - value = newValue; - kv.set(key, value); - }, - delete(): void { - value = undefined; - kv.delete(key); - }, - }); - } - - /** - * Provide a stored value object that is kept in persistent storage without caching. - * - * @param key - A key string that identifies the value. - * @param init - If provided, an initial setting if the stored entity does not exist. - * @returns An object for interacting with the value. - */ - function provideRawStoredValue(key: string, init?: string): StoredValue { - if (kv.get(key) === undefined && init !== undefined) { - kv.set(key, init); - } - return harden({ - get: () => kv.get(key), - set: (newValue: string) => kv.set(key, newValue), - delete: () => kv.delete(key), - }); - } - - return { - getSlotKey, - incCounter, - // Stored value - provideCachedStoredValue, - provideRawStoredValue, - }; -} diff --git a/packages/kernel/src/store/methods/clist.test.ts b/packages/kernel/src/store/methods/clist.test.ts deleted file mode 100644 index bcb71ce2a..000000000 --- a/packages/kernel/src/store/methods/clist.test.ts +++ /dev/null @@ -1,258 +0,0 @@ -import type { KVStore } from '@ocap/store'; -import { describe, it, expect, beforeEach, vi } from 'vitest'; - -import { getCListMethods } from './clist.ts'; -import { makeMapKVStore } from '../../../test/storage.ts'; -import type { EndpointId, KRef, ERef } from '../../types.ts'; -import type { StoreContext } from '../types.ts'; - -describe('clist-methods', () => { - let kv: KVStore; - let clistMethods: ReturnType; - let maybeFreeKrefs: Set; - - // Mock dependencies - const mockGetObjectRefCount = vi.fn(); - const mockSetObjectRefCount = vi.fn(); - const mockClearReachableFlag = vi.fn(); - - beforeEach(() => { - kv = makeMapKVStore(); - maybeFreeKrefs = new Set(); - - // Initialize endpoint counters - kv.set('e.nextPromiseId.v1', '1'); - kv.set('e.nextObjectId.v1', '1'); - kv.set('e.nextPromiseId.r1', '1'); - kv.set('e.nextObjectId.r1', '1'); - - // Reset mocks - mockGetObjectRefCount.mockReset(); - mockSetObjectRefCount.mockReset(); - mockClearReachableFlag.mockReset(); - - // Default mock implementations - mockGetObjectRefCount.mockImplementation(() => ({ - reachable: 1, - recognizable: 1, - })); - - // Create the store with mocked dependencies - clistMethods = getCListMethods({ - kv, - maybeFreeKrefs, - } as StoreContext); - }); - - describe('addClistEntry', () => { - it('adds a bidirectional mapping between KRef and ERef', () => { - const endpointId: EndpointId = 'v1'; - const kref: KRef = 'ko1'; - const eref: ERef = 'o-1'; - - clistMethods.addClistEntry(endpointId, kref, eref); - - // Check that both mappings are stored - expect(kv.get(`${endpointId}.c.${kref}`)).toBe(`R ${eref}`); - expect(kv.get(`${endpointId}.c.${eref}`)).toBe(kref); - }); - - it('works with promise refs', () => { - const endpointId: EndpointId = 'v1'; - const kref: KRef = 'kp1'; - const eref: ERef = 'p+2'; - - clistMethods.addClistEntry(endpointId, kref, eref); - - expect(kv.get(`${endpointId}.c.${kref}`)).toBe(`R ${eref}`); - expect(kv.get(`${endpointId}.c.${eref}`)).toBe(kref); - }); - - it('works with remote endpoints', () => { - const endpointId: EndpointId = 'r1'; - const kref: KRef = 'ko2'; - const eref: ERef = 'ro+3'; - - clistMethods.addClistEntry(endpointId, kref, eref); - - expect(kv.get(`${endpointId}.c.${kref}`)).toBe(`R ${eref}`); - expect(kv.get(`${endpointId}.c.${eref}`)).toBe(kref); - }); - }); - - describe('hasCListEntry', () => { - it('returns true for existing entries', () => { - const endpointId: EndpointId = 'v1'; - const kref: KRef = 'ko1'; - const eref: ERef = 'o-1'; - - clistMethods.addClistEntry(endpointId, kref, eref); - - expect(clistMethods.hasCListEntry(endpointId, kref)).toBe(true); - expect(clistMethods.hasCListEntry(endpointId, eref)).toBe(true); - }); - - it('returns false for non-existent entries', () => { - const endpointId: EndpointId = 'v1'; - - expect(clistMethods.hasCListEntry(endpointId, 'ko99')).toBe(false); - expect(clistMethods.hasCListEntry(endpointId, 'o-99')).toBe(false); - }); - }); - - describe('allocateErefForKref', () => { - it('allocates a new object ERef for a KRef', () => { - const endpointId: EndpointId = 'v1'; - const kref: KRef = 'ko1'; - - const eref = clistMethods.allocateErefForKref(endpointId, kref); - - // Check the allocated ERef format - expect(eref).toBe('o-1'); - - // Check that the counter was incremented - expect(kv.get(`e.nextObjectId.${endpointId}`)).toBe('2'); - - // Check that the mapping was added - expect(kv.get(`${endpointId}.c.${kref}`)).toBe(`R ${eref}`); - expect(kv.get(`${endpointId}.c.${eref}`)).toBe(kref); - }); - - it('allocates a new promise ERef for a KRef', () => { - const endpointId: EndpointId = 'v1'; - const kref: KRef = 'kp1'; - - const eref = clistMethods.allocateErefForKref(endpointId, kref); - - // Check the allocated ERef format - expect(eref).toBe('p-1'); - - // Check that the counter was incremented - expect(kv.get(`e.nextPromiseId.${endpointId}`)).toBe('2'); - - // Check that the mapping was added - expect(kv.get(`${endpointId}.c.${kref}`)).toBe(`R ${eref}`); - expect(kv.get(`${endpointId}.c.${eref}`)).toBe(kref); - }); - - it('allocates ERefs with remote prefix for remote endpoints', () => { - const endpointId: EndpointId = 'r1'; - const kref: KRef = 'ko1'; - - const eref = clistMethods.allocateErefForKref(endpointId, kref); - - // Check the allocated ERef format (should have 'r' prefix) - expect(eref).toBe('ro-1'); - - // Check that the counter was incremented - expect(kv.get(`e.nextObjectId.${endpointId}`)).toBe('2'); - }); - }); - - describe('erefToKref and krefToEref', () => { - it('converts between ERef and KRef', () => { - const endpointId: EndpointId = 'v1'; - const kref: KRef = 'ko1'; - const eref: ERef = 'o-1'; - - clistMethods.addClistEntry(endpointId, kref, eref); - - expect(clistMethods.erefToKref(endpointId, eref)).toBe(kref); - expect(clistMethods.krefToEref(endpointId, kref)).toBe(eref); - }); - - it('returns undefined for non-existent mappings', () => { - const endpointId: EndpointId = 'v1'; - - expect(clistMethods.erefToKref(endpointId, 'o-99')).toBeUndefined(); - expect(clistMethods.krefToEref(endpointId, 'ko99')).toBeUndefined(); - }); - }); - - describe('incrementRefCount', () => { - it('increments promise reference counts', () => { - const kref: KRef = 'kp1'; - - // Set up initial refCount - kv.set(`${kref}.refCount`, '1'); - - clistMethods.incrementRefCount(kref, {}); - - // Check that the refCount was incremented - expect(kv.get(`${kref}.refCount`)).toBe('2'); - }); - - it('does not increment object counts for exports', () => { - const kref: KRef = 'ko1'; - - clistMethods.incrementRefCount(kref, { isExport: true }); - - // Should not call getObjectRefCount or setObjectRefCount - expect(mockGetObjectRefCount).not.toHaveBeenCalled(); - expect(mockSetObjectRefCount).not.toHaveBeenCalled(); - }); - - it('throws for empty kref', () => { - expect(() => clistMethods.incrementRefCount('' as KRef, {})).toThrow( - 'incrementRefCount called with empty kref', - ); - }); - }); - - describe('decrementRefCount', () => { - it('decrements promise reference counts', () => { - const kref: KRef = 'kp1'; - - // Set up initial refCount - kv.set(`${kref}.refCount`, '2'); - - const result = clistMethods.decrementRefCount(kref, {}); - - // Check that the refCount was decremented - expect(kv.get(`${kref}.refCount`)).toBe('1'); - expect(result).toBe(false); // Not zero yet - }); - - it('adds promise to maybeFreeKrefs when count reaches zero', () => { - const kref: KRef = 'kp1'; - - // Set up initial refCount - kv.set(`${kref}.refCount`, '1'); - - const result = clistMethods.decrementRefCount(kref, {}); - - // Check that the refCount was decremented to zero - expect(kv.get(`${kref}.refCount`)).toBe('0'); - expect(result).toBe(true); // Now zero - expect(maybeFreeKrefs.has(kref)).toBe(true); - }); - - it('does not decrement object counts for exports', () => { - const kref: KRef = 'ko1'; - - const result = clistMethods.decrementRefCount(kref, { isExport: true }); - - // Should not call getObjectRefCount or setObjectRefCount - expect(mockGetObjectRefCount).not.toHaveBeenCalled(); - expect(mockSetObjectRefCount).not.toHaveBeenCalled(); - expect(result).toBe(false); - }); - - it('throws for empty kref', () => { - expect(() => clistMethods.decrementRefCount('' as KRef, {})).toThrow( - 'decrementRefCount called with empty kref', - ); - }); - - it('throws for underflow on promise refCount', () => { - const kref: KRef = 'kp1'; - - // Set up initial refCount at 0 - kv.set(`${kref}.refCount`, '0'); - - expect(() => clistMethods.decrementRefCount(kref, {})).toThrow( - /refCount underflow/u, - ); - }); - }); -}); diff --git a/packages/kernel/src/store/methods/clist.ts b/packages/kernel/src/store/methods/clist.ts deleted file mode 100644 index ef5076e4e..000000000 --- a/packages/kernel/src/store/methods/clist.ts +++ /dev/null @@ -1,270 +0,0 @@ -import { Fail } from '@endo/errors'; - -import { getBaseMethods } from './base.ts'; -import { getGCMethods } from './gc.ts'; -import { getObjectMethods } from './object.ts'; -import { getRefCountMethods } from './refcount.ts'; -import type { EndpointId, KRef, ERef } from '../../types.ts'; -import type { StoreContext } from '../types.ts'; -import { parseRef } from '../utils/parse-ref.ts'; -import { isPromiseRef } from '../utils/promise-ref.ts'; -import { - buildReachableAndVatSlot, - parseReachableAndVatSlot, -} from '../utils/reachable.ts'; - -/** - * Get the c-list methods that provide functionality for managing c-lists. - * - * @param ctx - The store context. - * @returns The c-list store. - */ -// eslint-disable-next-line @typescript-eslint/explicit-function-return-type -export function getCListMethods(ctx: StoreContext) { - const { getSlotKey } = getBaseMethods(ctx.kv); - const { clearReachableFlag } = getGCMethods(ctx); - const { getObjectRefCount, setObjectRefCount } = getObjectMethods(ctx); - const { kernelRefExists, refCountKey } = getRefCountMethods(ctx); - - /** - * Add an entry to an endpoint's c-list, creating a new bidirectional mapping - * between an ERef belonging to the endpoint and a KRef belonging to the - * kernel. - * - * @param endpointId - The endpoint whose c-list is to be added to. - * @param kref - The KRef. - * @param eref - The ERef. - */ - function addClistEntry(endpointId: EndpointId, kref: KRef, eref: ERef): void { - ctx.kv.set( - getSlotKey(endpointId, kref), - buildReachableAndVatSlot(true, eref), - ); - ctx.kv.set(getSlotKey(endpointId, eref), kref); - } - - /** - * Test if there's a c-list entry for some slot. - * - * @param endpointId - The endpoint of interest - * @param slot - The slot of interest - * @returns true iff this vat has a c-list entry mapping for `slot`. - */ - function hasCListEntry(endpointId: EndpointId, slot: string): boolean { - return ctx.kv.get(getSlotKey(endpointId, slot)) !== undefined; - } - - /** - * Remove an entry from an endpoint's c-list. - * - * @param endpointId - The endpoint whose c-list entry is to be removed. - * @param kref - The KRef. - * @param eref - The ERef. - */ - function deleteClistEntry( - endpointId: EndpointId, - kref: KRef, - eref: ERef, - ): void { - const kernelKey = getSlotKey(endpointId, kref); - const vatKey = getSlotKey(endpointId, eref); - assert(ctx.kv.get(kernelKey)); - clearReachableFlag(endpointId, kref); - const { direction } = parseRef(eref); - decrementRefCount(kref, { - isExport: direction === 'export', - onlyRecognizable: true, - }); - ctx.kv.delete(kernelKey); - ctx.kv.delete(vatKey); - } - - /** - * Generate a new eref for a kernel object or promise being imported into an - * endpoint. - * - * @param endpointId - The endpoint the kref is being imported into. - * @param kref - The kref for the kernel object or promise in question. - * - * @returns A new eref in the scope of the given endpoint for the given kernel entity. - */ - function allocateErefForKref(endpointId: EndpointId, kref: KRef): ERef { - let id; - const refTag = endpointId.startsWith('v') ? '' : endpointId[0]; - let refType; - if (isPromiseRef(kref)) { - id = ctx.kv.get(`e.nextPromiseId.${endpointId}`); - ctx.kv.set(`e.nextPromiseId.${endpointId}`, `${Number(id) + 1}`); - refType = 'p'; - } else { - id = ctx.kv.get(`e.nextObjectId.${endpointId}`); - ctx.kv.set(`e.nextObjectId.${endpointId}`, `${Number(id) + 1}`); - refType = 'o'; - } - const eref = `${refTag}${refType}-${id}`; - addClistEntry(endpointId, kref, eref); - return eref; - } - - /** - * Look up the ERef that and endpoint's c-list maps a KRef to. - * - * @param endpointId - The endpoint in question. - * @param eref - The ERef to look up. - * @returns The KRef corresponding to `eref` in the given endpoints c-list, or undefined - * if there is no such mapping. - */ - function erefToKref(endpointId: EndpointId, eref: ERef): KRef | undefined { - return ctx.kv.get(getSlotKey(endpointId, eref)); - } - - /** - * Look up the KRef that and endpoint's c-list maps an ERef to. - * - * @param endpointId - The endpoint in question. - * @param kref - The KRef to look up. - * @returns The given endpoint's ERef corresponding to `kref`, or undefined if - * there is no such mapping. - */ - function krefToEref(endpointId: EndpointId, kref: KRef): ERef | undefined { - const key = getSlotKey(endpointId, kref); - const data = ctx.kv.get(key); - if (!data) { - return undefined; - } - const { vatSlot } = parseReachableAndVatSlot(data); - return vatSlot; - } - - /** - * Remove an entry from an endpoint's c-list given an eref. - * - * @param endpointId - The endpoint whose c-list entry is to be removed. - * @param eref - The ERef. - */ - function forgetEref(endpointId: EndpointId, eref: ERef): void { - const kref = erefToKref(endpointId, eref); - if (kref) { - deleteClistEntry(endpointId, kref, eref); - } - } - - /** - * Remove an entry from an endpoint's c-list given a kref. - * - * @param endpointId - The endpoint whose c-list entry is to be removed. - * @param kref - The Kref. - */ - function forgetKref(endpointId: EndpointId, kref: KRef): void { - const eref = krefToEref(endpointId, kref); - if (eref) { - deleteClistEntry(endpointId, kref, eref); - } - } - - /** - * Increment the reference count associated with some kernel object. - * - * We track references to promises and objects, but not devices. Promises - * have only a "reachable" count, whereas objects track both "reachable" - * and "recognizable" counts. - * - * @param kref - The kernel slot whose refcount is to be incremented. - * @param options - Options for the increment. - * @param options.isExport - True if the reference comes from a clist export, which counts for promises but not objects. - * @param options.onlyRecognizable - True if the reference provides only recognition, not reachability. - */ - function incrementRefCount( - kref: KRef, - { - isExport = false, - onlyRecognizable = false, - }: { isExport?: boolean; onlyRecognizable?: boolean }, - ): void { - kref || Fail`incrementRefCount called with empty kref`; - - const { isPromise } = parseRef(kref); - if (isPromise) { - const refCount = Number(ctx.kv.get(refCountKey(kref))) + 1; - ctx.kv.set(refCountKey(kref), `${refCount}`); - return; - } - - // If `isExport` the reference comes from a clist export, which counts for promises but not objects - if (isExport) { - return; - } - - const counts = getObjectRefCount(kref); - if (!onlyRecognizable) { - counts.reachable += 1; - } - counts.recognizable += 1; - setObjectRefCount(kref, counts); - } - - /** - * Decrement the reference count associated with some kernel object. - * - * @param kref - The kernel slot whose refcount is to be decremented. - * @param options - Options for the decrement. - * @param options.isExport - True if the reference comes from a clist export, which counts for promises but not objects. - * @param options.onlyRecognizable - True if the reference provides only recognition, not reachability. - * @returns True if the reference count has been decremented to zero, false if it is still non-zero. - * @throws if this tries to decrement the reference count below zero. - */ - function decrementRefCount( - kref: KRef, - { - isExport = false, - onlyRecognizable = false, - }: { isExport?: boolean; onlyRecognizable?: boolean }, - ): boolean { - kref || Fail`decrementRefCount called with empty kref`; - - const { isPromise } = parseRef(kref); - if (isPromise) { - let refCount = Number(ctx.kv.get(refCountKey(kref))); - refCount > 0 || Fail`refCount underflow ${kref}`; - refCount -= 1; - ctx.kv.set(refCountKey(kref), `${refCount}`); - if (refCount === 0) { - ctx.maybeFreeKrefs.add(kref); - return true; - } - return false; - } - - if (isExport || !kernelRefExists(kref)) { - return false; - } - - const counts = getObjectRefCount(kref); - if (!onlyRecognizable) { - counts.reachable -= 1; - } - counts.recognizable -= 1; - if (!counts.reachable || !counts.recognizable) { - ctx.maybeFreeKrefs.add(kref); - } - setObjectRefCount(kref, counts); - - return false; - } - - return { - // C-List entries - addClistEntry, - hasCListEntry, - deleteClistEntry, - // Eref allocation - allocateErefForKref, - erefToKref, - krefToEref, - forgetEref, - forgetKref, - // Refcount management - incrementRefCount, - decrementRefCount, - }; -} diff --git a/packages/kernel/src/store/methods/gc.test.ts b/packages/kernel/src/store/methods/gc.test.ts deleted file mode 100644 index 4f5f395e1..000000000 --- a/packages/kernel/src/store/methods/gc.test.ts +++ /dev/null @@ -1,134 +0,0 @@ -import { describe, it, expect, beforeEach } from 'vitest'; - -import { makeMapKernelDatabase } from '../../../test/storage.ts'; -import { RunQueueItemType } from '../../types.ts'; -import type { GCAction } from '../../types.ts'; -import { makeKernelStore } from '../index.ts'; - -describe('GC methods', () => { - let kernelStore: ReturnType; - - beforeEach(() => { - kernelStore = makeKernelStore(makeMapKernelDatabase()); - }); - - describe('GC actions', () => { - it('manages all valid GC action types', () => { - const ko1 = kernelStore.initKernelObject('v1'); - const ko2 = kernelStore.initKernelObject('v1'); - const ko3 = kernelStore.initKernelObject('v2'); - - const validActions: GCAction[] = [ - `v1 dropExport ${ko1}`, - `v1 retireExport ${ko2}`, - `v2 retireImport ${ko3}`, - ]; - - kernelStore.addGCActions(validActions); - - const actions = kernelStore.getGCActions(); - expect(actions.size).toBe(3); - expect(actions).toStrictEqual(new Set(validActions)); - }); - - it('rejects invalid GC actions', () => { - const ko1 = kernelStore.initKernelObject('v1'); - - // Invalid vat ID - expect(() => { - kernelStore.addGCActions(['x1 dropExport ko1']); - }).toThrow('not a valid VatId'); - - // Invalid action type - expect(() => { - kernelStore.addGCActions([`v1 invalidAction ${ko1}`] as GCAction[]); - }).toThrow('not a valid GCActionType "invalidAction"'); - - // Invalid kref (must be kernel object, not promise) - expect(() => { - kernelStore.addGCActions(['v1 dropExport kp1']); - }).toThrow('kernelSlot "kp1" is not of type "object"'); - - // Malformed action string - expect(() => { - kernelStore.addGCActions(['v1 dropExport'] as unknown as GCAction[]); - }).toThrow('kernelSlot is undefined'); - }); - - it('maintains action order when storing', () => { - const ko1 = kernelStore.initKernelObject('v1'); - const ko2 = kernelStore.initKernelObject('v2'); - const ko3 = kernelStore.initKernelObject('v3'); - - const actions = [ - `v3 retireImport ${ko3}`, - `v1 dropExport ${ko1}`, - `v2 retireExport ${ko2}`, - ]; - - kernelStore.setGCActions(new Set(actions) as Set); - - // Actions should be sorted when retrieved - const sortedActions = Array.from(kernelStore.getGCActions()); - expect(sortedActions).toStrictEqual([ - `v1 dropExport ${ko1}`, - `v2 retireExport ${ko2}`, - `v3 retireImport ${ko3}`, - ]); - }); - }); - - describe('reachability tracking', () => { - it('manages reachable flags', () => { - const ko1 = kernelStore.initKernelObject('v1'); - kernelStore.addClistEntry('v1', ko1, 'o-1'); - - expect(kernelStore.getReachableFlag('v1', ko1)).toBe(true); - - kernelStore.clearReachableFlag('v1', ko1); - expect(kernelStore.getReachableFlag('v1', ko1)).toBe(false); - - const refCounts = kernelStore.getObjectRefCount(ko1); - expect(refCounts.reachable).toBe(0); - }); - }); - - describe('reaping', () => { - it('processes reap queue in order', () => { - const vatIds = ['v1', 'v2', 'v3']; - - // Schedule multiple vats for reaping - vatIds.forEach((vatId) => kernelStore.scheduleReap(vatId)); - - // Verify they are processed in order - vatIds.forEach((vatId) => { - expect(kernelStore.nextReapAction()).toStrictEqual({ - type: RunQueueItemType.bringOutYourDead, - vatId, - }); - }); - - // Queue should be empty after processing all items - expect(kernelStore.nextReapAction()).toBeUndefined(); - }); - - it('handles duplicate reap scheduling', () => { - kernelStore.scheduleReap('v1'); - kernelStore.scheduleReap('v1'); // Duplicate scheduling - kernelStore.scheduleReap('v2'); - - // Should only process v1 once - expect(kernelStore.nextReapAction()).toStrictEqual({ - type: RunQueueItemType.bringOutYourDead, - vatId: 'v1', - }); - - expect(kernelStore.nextReapAction()).toStrictEqual({ - type: RunQueueItemType.bringOutYourDead, - vatId: 'v2', - }); - - expect(kernelStore.nextReapAction()).toBeUndefined(); - }); - }); -}); diff --git a/packages/kernel/src/store/methods/gc.ts b/packages/kernel/src/store/methods/gc.ts deleted file mode 100644 index 3f2c6753f..000000000 --- a/packages/kernel/src/store/methods/gc.ts +++ /dev/null @@ -1,157 +0,0 @@ -import { getBaseMethods } from './base.ts'; -import { getObjectMethods } from './object.ts'; -import { getRefCountMethods } from './refcount.ts'; -import type { - VatId, - EndpointId, - KRef, - GCAction, - RunQueueItemBringOutYourDead, -} from '../../types.ts'; -import { - insistGCActionType, - insistVatId, - RunQueueItemType, -} from '../../types.ts'; -import type { StoreContext } from '../types.ts'; -import { insistKernelType } from '../utils/kernel-slots.ts'; -import { parseRef } from '../utils/parse-ref.ts'; -import { - buildReachableAndVatSlot, - parseReachableAndVatSlot, -} from '../utils/reachable.ts'; - -/** - * Create a store for garbage collection. - * - * @param ctx - The store context. - * @returns The GC store. - */ -// eslint-disable-next-line @typescript-eslint/explicit-function-return-type -export function getGCMethods(ctx: StoreContext) { - const { getSlotKey } = getBaseMethods(ctx.kv); - const { getObjectRefCount, setObjectRefCount } = getObjectMethods(ctx); - const { kernelRefExists } = getRefCountMethods(ctx); - - /** - * Get the set of GC actions to perform. - * - * @returns The set of GC actions to perform. - */ - function getGCActions(): Set { - return new Set(JSON.parse(ctx.gcActions.get() ?? '[]')); - } - - /** - * Set the set of GC actions to perform. - * - * @param actions - The set of GC actions to perform. - */ - function setGCActions(actions: Set): void { - const a = Array.from(actions); - a.sort(); - ctx.gcActions.set(JSON.stringify(a)); - } - - /** - * Add a new GC action to the set of GC actions to perform. - * - * @param newActions - The new GC action to add. - */ - function addGCActions(newActions: GCAction[]): void { - const actions = getGCActions(); - for (const action of newActions) { - assert.typeof(action, 'string', 'addGCActions given bad action'); - const [vatId, type, kref] = action.split(' '); - insistVatId(vatId); - insistGCActionType(type); - insistKernelType('object', kref); - actions.add(action); - } - setGCActions(actions); - } - - /** - * Check if a kernel object is reachable. - * - * @param endpointId - The endpoint for which the reachable flag is being checked. - * @param kref - The kref. - * @returns True if the kernel object is reachable, false otherwise. - */ - function getReachableFlag(endpointId: EndpointId, kref: KRef): boolean { - const key = getSlotKey(endpointId, kref); - const data = ctx.kv.getRequired(key); - const { isReachable } = parseReachableAndVatSlot(data); - return isReachable; - } - - /** - * Clear the reachable flag for a given endpoint and kref. - * - * @param endpointId - The endpoint for which the reachable flag is being cleared. - * @param kref - The kref. - */ - function clearReachableFlag(endpointId: EndpointId, kref: KRef): void { - const key = getSlotKey(endpointId, kref); - const { isReachable, vatSlot } = parseReachableAndVatSlot( - ctx.kv.getRequired(key), - ); - ctx.kv.set(key, buildReachableAndVatSlot(false, vatSlot)); - const { direction, isPromise } = parseRef(vatSlot); - // decrement 'reachable' part of refcount, but only for object imports - if ( - isReachable && - !isPromise && - direction === 'import' && - kernelRefExists(kref) - ) { - const counts = getObjectRefCount(kref); - counts.reachable -= 1; - setObjectRefCount(kref, counts); - if (counts.reachable === 0) { - ctx.maybeFreeKrefs.add(kref); - } - } - } - - /** - * Schedule a vat for reaping. - * - * @param vatId - The vat to schedule for reaping. - */ - function scheduleReap(vatId: VatId): void { - const queue = JSON.parse(ctx.reapQueue.get() ?? '[]'); - if (!queue.includes(vatId)) { - queue.push(vatId); - ctx.reapQueue.set(JSON.stringify(queue)); - } - } - - /** - * Get the next reap action. - * - * @returns The next reap action, or undefined if the queue is empty. - */ - function nextReapAction(): RunQueueItemBringOutYourDead | undefined { - const queue = JSON.parse(ctx.reapQueue.get() ?? '[]'); - if (queue.length > 0) { - const vatId = queue.shift(); - ctx.reapQueue.set(JSON.stringify(queue)); - return harden({ type: RunQueueItemType.bringOutYourDead, vatId }); - } - return undefined; - } - - return { - // GC actions - getGCActions, - setGCActions, - addGCActions, - // Reachability tracking - getReachableFlag, - clearReachableFlag, - // Reaping - scheduleReap, - nextReapAction, - }; -} diff --git a/packages/kernel/src/store/methods/id.test.ts b/packages/kernel/src/store/methods/id.test.ts deleted file mode 100644 index 2c507db25..000000000 --- a/packages/kernel/src/store/methods/id.test.ts +++ /dev/null @@ -1,172 +0,0 @@ -import type { KVStore } from '@ocap/store'; -import { describe, it, expect, beforeEach } from 'vitest'; - -import { getIdMethods } from './id.ts'; -import { makeMapKVStore } from '../../../test/storage.ts'; -import type { EndpointId } from '../../types.ts'; -import type { StoreContext } from '../types.ts'; - -describe('id-methods', () => { - let kv: KVStore; - let idStore: ReturnType; - let nextVatId: { get: () => string; set: (value: string) => void }; - let nextRemoteId: { get: () => string; set: (value: string) => void }; - - beforeEach(() => { - kv = makeMapKVStore(); - - // Initialize ID counters - kv.set('nextVatId', '0'); - kv.set('nextRemoteId', '0'); - - nextVatId = { - get: () => kv.get('nextVatId') ?? '0', - set: (value: string) => kv.set('nextVatId', value), - }; - - nextRemoteId = { - get: () => kv.get('nextRemoteId') ?? '0', - set: (value: string) => kv.set('nextRemoteId', value), - }; - - idStore = getIdMethods({ - kv, - nextVatId, - nextRemoteId, - } as StoreContext); - }); - - describe('getNextVatId', () => { - it('returns sequential vat IDs', () => { - expect(idStore.getNextVatId()).toBe('v0'); - expect(idStore.getNextVatId()).toBe('v1'); - expect(idStore.getNextVatId()).toBe('v2'); - }); - - it('increments the vat ID counter', () => { - idStore.getNextVatId(); - expect(kv.get('nextVatId')).toBe('1'); - - idStore.getNextVatId(); - expect(kv.get('nextVatId')).toBe('2'); - }); - - it('continues from existing counter value', () => { - // Set an existing counter value - kv.set('nextVatId', '42'); - - expect(idStore.getNextVatId()).toBe('v42'); - expect(idStore.getNextVatId()).toBe('v43'); - }); - }); - - describe('getNextRemoteId', () => { - it('returns sequential remote IDs', () => { - expect(idStore.getNextRemoteId()).toBe('r0'); - expect(idStore.getNextRemoteId()).toBe('r1'); - expect(idStore.getNextRemoteId()).toBe('r2'); - }); - - it('increments the remote ID counter', () => { - idStore.getNextRemoteId(); - expect(kv.get('nextRemoteId')).toBe('1'); - - idStore.getNextRemoteId(); - expect(kv.get('nextRemoteId')).toBe('2'); - }); - - it('continues from existing counter value', () => { - // Set an existing counter value - kv.set('nextRemoteId', '99'); - - expect(idStore.getNextRemoteId()).toBe('r99'); - expect(idStore.getNextRemoteId()).toBe('r100'); - }); - }); - - describe('initEndpoint', () => { - it('initializes a vat endpoint', () => { - const vatId: EndpointId = 'v1'; - - idStore.initEndpoint(vatId); - - expect(kv.get(`e.nextPromiseId.${vatId}`)).toBe('1'); - expect(kv.get(`e.nextObjectId.${vatId}`)).toBe('1'); - }); - - it('initializes a remote endpoint', () => { - const remoteId: EndpointId = 'r2'; - - idStore.initEndpoint(remoteId); - - expect(kv.get(`e.nextPromiseId.${remoteId}`)).toBe('1'); - expect(kv.get(`e.nextObjectId.${remoteId}`)).toBe('1'); - }); - - it('can initialize multiple endpoints', () => { - idStore.initEndpoint('v1'); - idStore.initEndpoint('v2'); - idStore.initEndpoint('r1'); - - expect(kv.get('e.nextPromiseId.v1')).toBe('1'); - expect(kv.get('e.nextObjectId.v1')).toBe('1'); - - expect(kv.get('e.nextPromiseId.v2')).toBe('1'); - expect(kv.get('e.nextObjectId.v2')).toBe('1'); - - expect(kv.get('e.nextPromiseId.r1')).toBe('1'); - expect(kv.get('e.nextObjectId.r1')).toBe('1'); - }); - - it('can reinitialize an existing endpoint', () => { - const vatId: EndpointId = 'v3'; - - // Initialize with default values - idStore.initEndpoint(vatId); - - // Modify the values - kv.set(`e.nextPromiseId.${vatId}`, '10'); - kv.set(`e.nextObjectId.${vatId}`, '20'); - - // Reinitialize - idStore.initEndpoint(vatId); - - // Values should be reset to 1 - expect(kv.get(`e.nextPromiseId.${vatId}`)).toBe('1'); - expect(kv.get(`e.nextObjectId.${vatId}`)).toBe('1'); - }); - }); - - describe('integration', () => { - it('supports creating and initializing endpoints', () => { - // Get new vat and remote IDs - const vatId = idStore.getNextVatId(); - const remoteId = idStore.getNextRemoteId(); - - // Initialize them as endpoints - idStore.initEndpoint(vatId); - idStore.initEndpoint(remoteId); - - // Check that they're properly initialized - expect(kv.get(`e.nextPromiseId.${vatId}`)).toBe('1'); - expect(kv.get(`e.nextObjectId.${vatId}`)).toBe('1'); - - expect(kv.get(`e.nextPromiseId.${remoteId}`)).toBe('1'); - expect(kv.get(`e.nextObjectId.${remoteId}`)).toBe('1'); - }); - - it('maintains separate counters for vats and remotes', () => { - // Generate multiple IDs of each type - const vatId1 = idStore.getNextVatId(); - const vatId2 = idStore.getNextVatId(); - const remoteId1 = idStore.getNextRemoteId(); - const remoteId2 = idStore.getNextRemoteId(); - - // Check that they follow the expected sequence - expect(vatId1).toBe('v0'); - expect(vatId2).toBe('v1'); - expect(remoteId1).toBe('r0'); - expect(remoteId2).toBe('r1'); - }); - }); -}); diff --git a/packages/kernel/src/store/methods/id.ts b/packages/kernel/src/store/methods/id.ts deleted file mode 100644 index 964839825..000000000 --- a/packages/kernel/src/store/methods/id.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { getBaseMethods } from './base.ts'; -import type { VatId, RemoteId, EndpointId } from '../../types.ts'; -import type { StoreContext } from '../types.ts'; - -/** - * Create a store for allocating IDs. - * - * @param ctx - The store context. - * @returns The ID store. - */ -// eslint-disable-next-line @typescript-eslint/explicit-function-return-type -export function getIdMethods(ctx: StoreContext) { - const { kv } = ctx; - const { incCounter } = getBaseMethods(kv); - - /** - * Obtain an ID for a new vat. - * - * @returns The next VatID use. - */ - function getNextVatId(): VatId { - return `v${incCounter(ctx.nextVatId)}`; - } - - /** - * Obtain an ID for a new remote connection. - * - * @returns The next remote ID use. - */ - function getNextRemoteId(): RemoteId { - return `r${incCounter(ctx.nextRemoteId)}`; - } - - /** - * Initialize persistent state for a new endpoint. - * - * @param endpointId - The ID of the endpoint being added. - */ - function initEndpoint(endpointId: EndpointId): void { - kv.set(`e.nextPromiseId.${endpointId}`, '1'); - kv.set(`e.nextObjectId.${endpointId}`, '1'); - } - - return { - getNextVatId, - getNextRemoteId, - initEndpoint, - }; -} diff --git a/packages/kernel/src/store/methods/object.test.ts b/packages/kernel/src/store/methods/object.test.ts deleted file mode 100644 index 6d81a4145..000000000 --- a/packages/kernel/src/store/methods/object.test.ts +++ /dev/null @@ -1,278 +0,0 @@ -import type { KVStore } from '@ocap/store'; -import { describe, it, expect, beforeEach } from 'vitest'; - -import { getObjectMethods } from './object.ts'; -import { makeMapKVStore } from '../../../test/storage.ts'; -import type { EndpointId } from '../../types.ts'; -import type { StoreContext } from '../types.ts'; - -describe('object-methods', () => { - let kv: KVStore; - let objectStore: ReturnType; - let nextObjectId: { get: () => string; set: (value: string) => void }; - - beforeEach(() => { - kv = makeMapKVStore(); - // Initialize nextObjectId counter - kv.set('nextObjectId', '0'); - nextObjectId = { - get: () => kv.get('nextObjectId') ?? '0', - set: (value: string) => kv.set('nextObjectId', value), - }; - - objectStore = getObjectMethods({ - kv, - nextObjectId, - } as StoreContext); - }); - - describe('initKernelObject', () => { - it('creates a new kernel object with initial reference counts', () => { - const owner: EndpointId = 'v1'; - const koId = objectStore.initKernelObject(owner); - - // Check the object ID format - expect(koId).toBe('ko0'); - - // Check the owner is set correctly - expect(kv.get(`${koId}.owner`)).toBe(owner); - - // Check reference counts are initialized to 1,1 - expect(kv.get(`${koId}.refCount`)).toBe('1,1'); - - // Check via the API - const refCounts = objectStore.getObjectRefCount(koId); - expect(refCounts.reachable).toBe(1); - expect(refCounts.recognizable).toBe(1); - }); - - it('increments the object ID counter', () => { - const koId1 = objectStore.initKernelObject('v1'); - const koId2 = objectStore.initKernelObject('v2'); - const koId3 = objectStore.initKernelObject('r1'); - - expect(koId1).toBe('ko0'); - expect(koId2).toBe('ko1'); - expect(koId3).toBe('ko2'); - }); - }); - - describe('getOwner', () => { - it('returns the owner of a kernel object', () => { - const owner1: EndpointId = 'v1'; - const owner2: EndpointId = 'r2'; - - const koId1 = objectStore.initKernelObject(owner1); - const koId2 = objectStore.initKernelObject(owner2); - - expect(objectStore.getOwner(koId1)).toBe(owner1); - expect(objectStore.getOwner(koId2)).toBe(owner2); - }); - - it('throws for unknown kernel objects', () => { - expect(() => objectStore.getOwner('ko99')).toThrow( - 'unknown kernel object ko99', - ); - }); - }); - - describe('deleteKernelObject', () => { - it('removes a kernel object from storage', () => { - const koId = objectStore.initKernelObject('v1'); - - // Object exists before deletion - expect(kv.get(`${koId}.owner`)).toBeDefined(); - expect(kv.get(`${koId}.refCount`)).toBeDefined(); - - // Delete the object - objectStore.deleteKernelObject(koId); - - // Object should be gone - expect(kv.get(`${koId}.owner`)).toBeUndefined(); - expect(kv.get(`${koId}.refCount`)).toBeUndefined(); - - // getOwner should throw - expect(() => objectStore.getOwner(koId)).toThrow( - `unknown kernel object ${koId}`, - ); - }); - }); - - describe('getNextObjectId', () => { - it('returns sequential object IDs', () => { - expect(objectStore.getNextObjectId()).toBe('ko0'); - expect(objectStore.getNextObjectId()).toBe('ko1'); - expect(objectStore.getNextObjectId()).toBe('ko2'); - }); - }); - - describe('getObjectRefCount', () => { - it('returns reference counts for existing objects', () => { - const koId = objectStore.initKernelObject('v1'); - - const refCounts = objectStore.getObjectRefCount(koId); - expect(refCounts).toStrictEqual({ reachable: 1, recognizable: 1 }); - }); - - it('returns zero counts for non-existent objects', () => { - const refCounts = objectStore.getObjectRefCount('ko99'); - expect(refCounts).toStrictEqual({ reachable: 0, recognizable: 0 }); - }); - - it('parses reference counts correctly', () => { - const koId = objectStore.initKernelObject('v1'); - - // Manually set reference counts - kv.set(`${koId}.refCount`, '5,10'); - - const refCounts = objectStore.getObjectRefCount(koId); - expect(refCounts).toStrictEqual({ reachable: 5, recognizable: 10 }); - }); - }); - - describe('setObjectRefCount', () => { - it('sets reference counts for an object', () => { - const koId = objectStore.initKernelObject('v1'); - - // Set new reference counts - objectStore.setObjectRefCount(koId, { reachable: 3, recognizable: 5 }); - - // Check via direct KV access - expect(kv.get(`${koId}.refCount`)).toBe('3,5'); - - // Check via API - const refCounts = objectStore.getObjectRefCount(koId); - expect(refCounts).toStrictEqual({ reachable: 3, recognizable: 5 }); - }); - - it('allows zero reference counts', () => { - const koId = objectStore.initKernelObject('v1'); - - objectStore.setObjectRefCount(koId, { reachable: 0, recognizable: 0 }); - - const refCounts = objectStore.getObjectRefCount(koId); - expect(refCounts).toStrictEqual({ reachable: 0, recognizable: 0 }); - }); - - it('throws when reachable count exceeds recognizable count', () => { - const koId = objectStore.initKernelObject('v1'); - - expect(() => - objectStore.setObjectRefCount(koId, { reachable: 5, recognizable: 3 }), - ).toThrow(/refMismatch/u); - }); - - it('throws when counts are negative', () => { - const koId = objectStore.initKernelObject('v1'); - - expect(() => - objectStore.setObjectRefCount(koId, { reachable: -1, recognizable: 1 }), - ).toThrow(/underflow/u); - - expect(() => - objectStore.setObjectRefCount(koId, { reachable: 0, recognizable: -1 }), - ).toThrow(/underflow/u); - }); - }); - - describe('integration', () => { - it('supports the full kernel object lifecycle', () => { - // Create an object - const koId = objectStore.initKernelObject('v1'); - - // Check initial state - expect(objectStore.getOwner(koId)).toBe('v1'); - expect(objectStore.getObjectRefCount(koId)).toStrictEqual({ - reachable: 1, - recognizable: 1, - }); - - // Update reference counts - objectStore.setObjectRefCount(koId, { reachable: 2, recognizable: 3 }); - expect(objectStore.getObjectRefCount(koId)).toStrictEqual({ - reachable: 2, - recognizable: 3, - }); - - // Reduce to zero - objectStore.setObjectRefCount(koId, { reachable: 0, recognizable: 0 }); - expect(objectStore.getObjectRefCount(koId)).toStrictEqual({ - reachable: 0, - recognizable: 0, - }); - - // Object still exists with zero counts - expect(objectStore.getOwner(koId)).toBe('v1'); - - // Delete the object - objectStore.deleteKernelObject(koId); - - // Object should be gone - expect(() => objectStore.getOwner(koId)).toThrow( - `unknown kernel object ${koId}`, - ); - expect(objectStore.getObjectRefCount(koId)).toStrictEqual({ - reachable: 0, - recognizable: 0, - }); - }); - - it('handles multiple objects simultaneously', () => { - // Create multiple objects with different owners - const koId1 = objectStore.initKernelObject('v1'); - const koId2 = objectStore.initKernelObject('r2'); - - // Check owners - expect(objectStore.getOwner(koId1)).toBe('v1'); - expect(objectStore.getOwner(koId2)).toBe('r2'); - - // Set different reference counts - objectStore.setObjectRefCount(koId1, { reachable: 2, recognizable: 2 }); - objectStore.setObjectRefCount(koId2, { reachable: 3, recognizable: 5 }); - - // Check counts - expect(objectStore.getObjectRefCount(koId1)).toStrictEqual({ - reachable: 2, - recognizable: 2, - }); - expect(objectStore.getObjectRefCount(koId2)).toStrictEqual({ - reachable: 3, - recognizable: 5, - }); - - // Delete one object - objectStore.deleteKernelObject(koId1); - - // First object should be gone, second still exists - expect(() => objectStore.getOwner(koId1)).toThrow( - `unknown kernel object ${koId1}`, - ); - expect(objectStore.getOwner(koId2)).toBe('r2'); - }); - - it('handles reference count patterns correctly', () => { - const koId = objectStore.initKernelObject('v1'); - - // Equal reachable and recognizable - objectStore.setObjectRefCount(koId, { reachable: 5, recognizable: 5 }); - expect(objectStore.getObjectRefCount(koId)).toStrictEqual({ - reachable: 5, - recognizable: 5, - }); - - // Reachable less than recognizable - objectStore.setObjectRefCount(koId, { reachable: 3, recognizable: 5 }); - expect(objectStore.getObjectRefCount(koId)).toStrictEqual({ - reachable: 3, - recognizable: 5, - }); - - // Both zero - objectStore.setObjectRefCount(koId, { reachable: 0, recognizable: 0 }); - expect(objectStore.getObjectRefCount(koId)).toStrictEqual({ - reachable: 0, - recognizable: 0, - }); - }); - }); -}); diff --git a/packages/kernel/src/store/methods/object.ts b/packages/kernel/src/store/methods/object.ts deleted file mode 100644 index e69a1e107..000000000 --- a/packages/kernel/src/store/methods/object.ts +++ /dev/null @@ -1,119 +0,0 @@ -import { Fail } from '@endo/errors'; - -import { getBaseMethods } from './base.ts'; -import { getRefCountMethods } from './refcount.ts'; -import type { EndpointId, KRef } from '../../types.ts'; -import type { StoreContext } from '../types.ts'; -import { makeKernelSlot } from '../utils/kernel-slots.ts'; - -/** - * Create an object store object that provides functionality for managing kernel objects. - * - * @param ctx - The store context. - * @returns An object store object that maps various persistent kernel data - * structures onto `kv`. - */ -// eslint-disable-next-line @typescript-eslint/explicit-function-return-type -export function getObjectMethods(ctx: StoreContext) { - const { incCounter } = getBaseMethods(ctx.kv); - const { refCountKey } = getRefCountMethods(ctx); - - /** - * Create a new kernel object. The new object will be born with reference and - * recognizability counts of 1, on the assumption that the new object - * corresponds to an object that has just been imported from somewhere. - * - * @param owner - The endpoint that is the owner of the new object. - * @returns The new object's KRef. - */ - function initKernelObject(owner: EndpointId): KRef { - const koId = getNextObjectId(); - ctx.kv.set(`${koId}.owner`, owner); - setObjectRefCount(koId, { reachable: 1, recognizable: 1 }); - return koId; - } - - /** - * Get a kernel object's owner. - * - * @param koId - The KRef of the kernel object of interest. - * @returns The identity of the vat or remote that owns the object. - */ - function getOwner(koId: KRef): EndpointId { - const owner = ctx.kv.get(`${koId}.owner`); - if (owner === undefined) { - throw Error(`unknown kernel object ${koId}`); - } - return owner; - } - - /** - * Expunge a kernel object from the kernel's persistent state. - * - * @param koId - The KRef of the kernel object to delete. - */ - function deleteKernelObject(koId: KRef): void { - ctx.kv.delete(`${koId}.owner`); - ctx.kv.delete(refCountKey(koId)); - } - - /** - * Obtain a KRef for the next unallocated kernel object. - * - * @returns The next koId use. - */ - function getNextObjectId(): KRef { - return makeKernelSlot('object', incCounter(ctx.nextObjectId)); - } - - /** - * Get the reference counts for a kernel object - * - * @param kref - The KRef of the object of interest. - * @returns The reference counts for the object. - */ - function getObjectRefCount(kref: KRef): { - reachable: number; - recognizable: number; - } { - const data = ctx.kv.get(refCountKey(kref)); - if (!data) { - return { reachable: 0, recognizable: 0 }; - } - const [reachable = 0, recognizable = 0] = data.split(',').map(Number); - reachable <= recognizable || - Fail`refMismatch(get) ${kref} ${reachable},${recognizable}`; - return { reachable, recognizable }; - } - - /** - * Set the reference counts for a kernel object - * - * @param kref - The KRef of the object of interest. - * @param counts - The reference counts to set. - * @param counts.reachable - The reachable reference count. - * @param counts.recognizable - The recognizable reference count. - */ - function setObjectRefCount( - kref: KRef, - counts: { reachable: number; recognizable: number }, - ): void { - const { reachable, recognizable } = counts; - assert.typeof(reachable, 'number'); - assert.typeof(recognizable, 'number'); - (reachable >= 0 && recognizable >= 0) || - Fail`${kref} underflow ${reachable},${recognizable}`; - reachable <= recognizable || - Fail`refMismatch(set) ${kref} ${reachable},${recognizable}`; - ctx.kv.set(refCountKey(kref), `${reachable},${recognizable}`); - } - - return { - initKernelObject, - getOwner, - deleteKernelObject, - getNextObjectId, - getObjectRefCount, - setObjectRefCount, - }; -} diff --git a/packages/kernel/src/store/methods/promise.test.ts b/packages/kernel/src/store/methods/promise.test.ts deleted file mode 100644 index 96b34d0af..000000000 --- a/packages/kernel/src/store/methods/promise.test.ts +++ /dev/null @@ -1,400 +0,0 @@ -import type { Message } from '@agoric/swingset-liveslots'; -import type { CapData } from '@endo/marshal'; -import type { KVStore } from '@ocap/store'; -import { describe, it, expect, beforeEach } from 'vitest'; - -import { getPromiseMethods } from './promise.ts'; -import { makeMapKVStore } from '../../../test/storage.ts'; -import type { KRef } from '../../types.ts'; -import type { StoreContext } from '../types.ts'; - -/** - * Mock Message: A helper to allow simple objects to be used as Messages for testing - * - * @param obj - An object to use as a message. - * @returns The same object coerced to type Message. - */ -function mockMessage(obj: object): Message { - return obj as unknown as Message; -} - -describe('promise-methods', () => { - let kv: KVStore; - let promiseStore: ReturnType; - let nextPromiseId: { get: () => string; set: (value: string) => void }; - - beforeEach(() => { - kv = makeMapKVStore(); - // Initialize nextPromiseId counter - kv.set('nextPromiseId', '0'); - nextPromiseId = { - get: () => kv.get('nextPromiseId') ?? '0', - set: (value: string) => kv.set('nextPromiseId', value), - }; - - promiseStore = getPromiseMethods({ - kv, - nextPromiseId, - } as StoreContext); - }); - - describe('initKernelPromise', () => { - it('creates a new unresolved kernel promise', () => { - const [kpid, kp] = promiseStore.initKernelPromise(); - - // Check the returned promise - expect(kpid).toBe('kp0'); - expect(kp).toStrictEqual({ - state: 'unresolved', - subscribers: [], - }); - - // Check the stored promise - expect(kv.get(`${kpid}.state`)).toBe('unresolved'); - expect(kv.get(`${kpid}.subscribers`)).toBe('[]'); - expect(kv.get(`${kpid}.refCount`)).toBe('1'); - }); - - it('increments the promise ID counter', () => { - const [kpid1] = promiseStore.initKernelPromise(); - const [kpid2] = promiseStore.initKernelPromise(); - const [kpid3] = promiseStore.initKernelPromise(); - - expect(kpid1).toBe('kp0'); - expect(kpid2).toBe('kp1'); - expect(kpid3).toBe('kp2'); - }); - }); - - describe('getKernelPromise', () => { - it('retrieves an unresolved promise', () => { - const [kpid] = promiseStore.initKernelPromise(); - const kp = promiseStore.getKernelPromise(kpid); - - expect(kp).toStrictEqual({ - state: 'unresolved', - subscribers: [], - }); - }); - - it('retrieves an unresolved promise with decider', () => { - const [kpid] = promiseStore.initKernelPromise(); - promiseStore.setPromiseDecider(kpid, 'v1'); - - const kp = promiseStore.getKernelPromise(kpid); - expect(kp).toStrictEqual({ - state: 'unresolved', - decider: 'v1', - subscribers: [], - }); - }); - - it('retrieves a fulfilled promise', () => { - const [kpid] = promiseStore.initKernelPromise(); - const value: CapData = { body: 'fulfilled-value', slots: [] }; - - promiseStore.resolveKernelPromise(kpid, false, value); - - const kp = promiseStore.getKernelPromise(kpid); - expect(kp).toStrictEqual({ - state: 'fulfilled', - value, - }); - }); - - it('retrieves a rejected promise', () => { - const [kpid] = promiseStore.initKernelPromise(); - const value: CapData = { body: 'error-message', slots: [] }; - - promiseStore.resolveKernelPromise(kpid, true, value); - - const kp = promiseStore.getKernelPromise(kpid); - expect(kp).toStrictEqual({ - state: 'rejected', - value, - }); - }); - - it('throws for unknown promises', () => { - expect(() => promiseStore.getKernelPromise('kp99')).toThrow( - 'unknown kernel promise kp99', - ); - }); - }); - - describe('deleteKernelPromise', () => { - it('removes a promise from storage', () => { - const [kpid] = promiseStore.initKernelPromise(); - - // Add a message to the promise queue - promiseStore.enqueuePromiseMessage( - kpid, - mockMessage({ test: 'message' }), - ); - - // Delete the promise - promiseStore.deleteKernelPromise(kpid); - - // Check that all promise data is gone - expect(kv.get(`${kpid}.state`)).toBeUndefined(); - expect(kv.get(`${kpid}.subscribers`)).toBeUndefined(); - expect(kv.get(`${kpid}.refCount`)).toBeUndefined(); - - // Check that the promise queue is gone - expect(() => - promiseStore.enqueuePromiseMessage(kpid, mockMessage({})), - ).toThrow(`queue ${kpid} not initialized`); - }); - }); - - describe('getNextPromiseId', () => { - it('returns sequential promise IDs', () => { - expect(promiseStore.getNextPromiseId()).toBe('kp0'); - expect(promiseStore.getNextPromiseId()).toBe('kp1'); - expect(promiseStore.getNextPromiseId()).toBe('kp2'); - }); - }); - - describe('addPromiseSubscriber', () => { - it('adds a subscriber to an unresolved promise', () => { - const [kpid] = promiseStore.initKernelPromise(); - - promiseStore.addPromiseSubscriber('v1', kpid); - - const kp = promiseStore.getKernelPromise(kpid); - expect(kp.subscribers).toStrictEqual(['v1']); - - // Add another subscriber - promiseStore.addPromiseSubscriber('v2', kpid); - - const kpUpdated = promiseStore.getKernelPromise(kpid); - expect(kpUpdated.subscribers).toStrictEqual(['v1', 'v2']); - }); - - it('does not add duplicate subscribers', () => { - const [kpid] = promiseStore.initKernelPromise(); - - promiseStore.addPromiseSubscriber('v1', kpid); - promiseStore.addPromiseSubscriber('v1', kpid); - - const kp = promiseStore.getKernelPromise(kpid); - expect(kp.subscribers).toStrictEqual(['v1']); - }); - - it('throws when adding a subscriber to a resolved promise', () => { - const [kpid] = promiseStore.initKernelPromise(); - - // Resolve the promise - promiseStore.resolveKernelPromise(kpid, false, { - body: 'value', - slots: [], - }); - - // Try to add a subscriber - expect(() => promiseStore.addPromiseSubscriber('v1', kpid)).toThrow( - /attempt to add subscriber to resolved promise/u, - ); - }); - }); - - describe('setPromiseDecider', () => { - it('sets the decider for a promise', () => { - const [kpid] = promiseStore.initKernelPromise(); - - promiseStore.setPromiseDecider(kpid, 'v3'); - - const kp = promiseStore.getKernelPromise(kpid); - expect(kp.decider).toBe('v3'); - }); - - it('updates the decider for a promise', () => { - const [kpid] = promiseStore.initKernelPromise(); - - promiseStore.setPromiseDecider(kpid, 'v1'); - promiseStore.setPromiseDecider(kpid, 'v2'); - - const kp = promiseStore.getKernelPromise(kpid); - expect(kp.decider).toBe('v2'); - }); - }); - - describe('resolveKernelPromise', () => { - it('fulfills a promise', () => { - const [kpid] = promiseStore.initKernelPromise(); - const value: CapData = { body: 'fulfilled-value', slots: ['ko1'] }; - - promiseStore.resolveKernelPromise(kpid, false, value); - - const kp = promiseStore.getKernelPromise(kpid); - expect(kp.state).toBe('fulfilled'); - expect(kp.value).toStrictEqual(value); - }); - - it('rejects a promise', () => { - const [kpid] = promiseStore.initKernelPromise(); - const value: CapData = { body: 'error-message', slots: [] }; - - promiseStore.resolveKernelPromise(kpid, true, value); - - const kp = promiseStore.getKernelPromise(kpid); - expect(kp.state).toBe('rejected'); - expect(kp.value).toStrictEqual(value); - }); - - it('clears decider and subscribers when resolving', () => { - const [kpid] = promiseStore.initKernelPromise(); - - // Add decider and subscribers - promiseStore.setPromiseDecider(kpid, 'v1'); - promiseStore.addPromiseSubscriber('v2', kpid); - - // Resolve the promise - promiseStore.resolveKernelPromise(kpid, false, { - body: 'value', - slots: [], - }); - - // Check that decider and subscribers are gone - expect(kv.get(`${kpid}.decider`)).toBeUndefined(); - expect(kv.get(`${kpid}.subscribers`)).toBeUndefined(); - }); - - it('preserves queued messages when resolving', () => { - const [kpid] = promiseStore.initKernelPromise(); - - // Add messages to the queue - const message1 = mockMessage({ id: 1 }); - const message2 = mockMessage({ id: 2 }); - - promiseStore.enqueuePromiseMessage(kpid, message1); - promiseStore.enqueuePromiseMessage(kpid, message2); - - // Resolve the promise - promiseStore.resolveKernelPromise(kpid, false, { - body: 'value', - slots: [], - }); - - // Check that messages are still in the queue - const messages = promiseStore.getKernelPromiseMessageQueue(kpid); - expect(messages).toHaveLength(2); - expect(messages[0]).toStrictEqual(message1); - expect(messages[1]).toStrictEqual(message2); - }); - }); - - describe('enqueuePromiseMessage and getKernelPromiseMessageQueue', () => { - it('enqueues and retrieves messages', () => { - const [kpid] = promiseStore.initKernelPromise(); - - const message1 = mockMessage({ id: 1, data: 'first' }); - const message2 = mockMessage({ id: 2, data: 'second' }); - - promiseStore.enqueuePromiseMessage(kpid, message1); - promiseStore.enqueuePromiseMessage(kpid, message2); - - const messages = promiseStore.getKernelPromiseMessageQueue(kpid); - - expect(messages).toHaveLength(2); - expect(messages[0]).toStrictEqual(message1); - expect(messages[1]).toStrictEqual(message2); - }); - - it('empties the queue when retrieving messages', () => { - const [kpid] = promiseStore.initKernelPromise(); - - promiseStore.enqueuePromiseMessage(kpid, mockMessage({ id: 1 })); - promiseStore.enqueuePromiseMessage(kpid, mockMessage({ id: 2 })); - - // First call gets all messages - const messages1 = promiseStore.getKernelPromiseMessageQueue(kpid); - expect(messages1).toHaveLength(2); - - // Second call gets empty array - const messages2 = promiseStore.getKernelPromiseMessageQueue(kpid); - expect(messages2).toHaveLength(0); - }); - - it('throws when enqueueing to a non-existent promise', () => { - expect(() => - promiseStore.enqueuePromiseMessage('kp99', mockMessage({})), - ).toThrow('queue kp99 not initialized'); - }); - }); - - describe('integration', () => { - it('supports the full promise lifecycle', () => { - // Create a promise - const [kpid] = promiseStore.initKernelPromise(); - - // Add subscribers and decider - promiseStore.addPromiseSubscriber('v1', kpid); - promiseStore.addPromiseSubscriber('v2', kpid); - promiseStore.setPromiseDecider(kpid, 'v3'); - - // Add messages - promiseStore.enqueuePromiseMessage(kpid, mockMessage({ id: 1 })); - - // Check the promise state - let kp = promiseStore.getKernelPromise(kpid); - expect(kp.state).toBe('unresolved'); - expect(kp.subscribers).toStrictEqual(['v1', 'v2']); - expect(kp.decider).toBe('v3'); - - // Resolve the promise - const value: CapData = { body: 'final-value', slots: ['ko5'] }; - promiseStore.resolveKernelPromise(kpid, false, value); - - // Check the resolved state - kp = promiseStore.getKernelPromise(kpid); - expect(kp.state).toBe('fulfilled'); - expect(kp.value).toStrictEqual(value); - - // Messages should still be available - const messages = promiseStore.getKernelPromiseMessageQueue(kpid); - expect(messages).toHaveLength(1); - - // Delete the promise - promiseStore.deleteKernelPromise(kpid); - - // Promise should be gone - expect(() => promiseStore.getKernelPromise(kpid)).toThrow( - `unknown kernel promise ${kpid}`, - ); - }); - - it('handles multiple promises simultaneously', () => { - // Create two promises - const [kpid1] = promiseStore.initKernelPromise(); - const [kpid2] = promiseStore.initKernelPromise(); - - // Set up different states - promiseStore.addPromiseSubscriber('v1', kpid1); - promiseStore.setPromiseDecider(kpid2, 'v2'); - - // Add messages to both - promiseStore.enqueuePromiseMessage(kpid1, mockMessage({ for: 'kp1' })); - promiseStore.enqueuePromiseMessage(kpid2, mockMessage({ for: 'kp2' })); - - // Resolve one promise - promiseStore.resolveKernelPromise(kpid1, false, { - body: 'resolved', - slots: [], - }); - - // Check states - const kp1 = promiseStore.getKernelPromise(kpid1); - const kp2 = promiseStore.getKernelPromise(kpid2); - - expect(kp1.state).toBe('fulfilled'); - expect(kp2.state).toBe('unresolved'); - - // Messages should be preserved - const messages1 = promiseStore.getKernelPromiseMessageQueue(kpid1); - const messages2 = promiseStore.getKernelPromiseMessageQueue(kpid2); - - expect(messages1).toHaveLength(1); - expect(messages2).toHaveLength(1); - }); - }); -}); diff --git a/packages/kernel/src/store/methods/promise.ts b/packages/kernel/src/store/methods/promise.ts deleted file mode 100644 index 6cbd1c044..000000000 --- a/packages/kernel/src/store/methods/promise.ts +++ /dev/null @@ -1,204 +0,0 @@ -import type { Message } from '@agoric/swingset-liveslots'; -import { Fail } from '@endo/errors'; -import type { CapData } from '@endo/marshal'; - -import { getBaseMethods } from './base.ts'; -import { getQueueMethods } from './queue.ts'; -import { getRefCountMethods } from './refcount.ts'; -import type { KRef, KernelPromise, PromiseState, VatId } from '../../types.ts'; -import { insistVatId } from '../../types.ts'; -import type { StoreContext } from '../types.ts'; -import { makeKernelSlot } from '../utils/kernel-slots.ts'; -import { parseRef } from '../utils/parse-ref.ts'; - -/** - * Create a promise store object that provides functionality for managing kernel promises. - * - * @param ctx - The store context. - * @returns A promise store object that maps various persistent kernel data - * structures onto `kv`. - */ -// eslint-disable-next-line @typescript-eslint/explicit-function-return-type -export function getPromiseMethods(ctx: StoreContext) { - const { incCounter } = getBaseMethods(ctx.kv); - const { createStoredQueue, provideStoredQueue } = getQueueMethods(ctx.kv); - const { refCountKey } = getRefCountMethods(ctx); - - /** - * Create a new, unresolved kernel promise. The new promise will be born with - * a reference count of 1 on the assumption that the promise has just been - * imported from somewhere. - * - * @returns A tuple of the new promise's KRef and an object describing the - * new promise itself. - */ - function initKernelPromise(): [KRef, KernelPromise] { - const kpr: KernelPromise = { - state: 'unresolved', - subscribers: [], - }; - const kpid = getNextPromiseId(); - createStoredQueue(kpid, false); - ctx.kv.set(`${kpid}.state`, 'unresolved'); - ctx.kv.set(`${kpid}.subscribers`, '[]'); - ctx.kv.set(refCountKey(kpid), '1'); - return [kpid, kpr]; - } - - /** - * Fetch the descriptive record for a kernel promise. - * - * @param kpid - The KRef of the kernel promise of interest. - * @returns An object describing the requested kernel promise. - */ - function getKernelPromise(kpid: KRef): KernelPromise { - const { context, isPromise } = parseRef(kpid); - assert(context === 'kernel' && isPromise); - const state = ctx.kv.get(`${kpid}.state`) as PromiseState; - if (state === undefined) { - throw Error(`unknown kernel promise ${kpid}`); - } - const result: KernelPromise = { state }; - switch (state as string) { - case 'unresolved': { - const decider = ctx.kv.get(`${kpid}.decider`); - if (decider !== '' && decider !== undefined) { - result.decider = decider; - } - const subscribers = ctx.kv.getRequired(`${kpid}.subscribers`); - result.subscribers = JSON.parse(subscribers); - break; - } - case 'fulfilled': - case 'rejected': { - result.value = JSON.parse(ctx.kv.getRequired(`${kpid}.value`)); - break; - } - default: - throw Error(`unknown state for ${kpid}: ${state}`); - } - return result; - } - - /** - * Expunge a kernel promise from the kernel's persistent state. - * - * @param kpid - The KRef of the kernel promise to delete. - */ - function deleteKernelPromise(kpid: KRef): void { - ctx.kv.delete(`${kpid}.state`); - ctx.kv.delete(`${kpid}.decider`); - ctx.kv.delete(`${kpid}.subscribers`); - ctx.kv.delete(`${kpid}.value`); - ctx.kv.delete(refCountKey(kpid)); - provideStoredQueue(kpid).delete(); - } - - /** - * Obtain a KRef for the next unallocated kernel promise. - * - * @returns The next kpid use. - */ - function getNextPromiseId(): KRef { - return makeKernelSlot('promise', incCounter(ctx.nextPromiseId)); - } - - /** - * Add a new subscriber to a kernel promise's collection of subscribers. - * - * @param vatId - The vat that is subscribing. - * @param kpid - The KRef of the promise being subscribed to. - */ - function addPromiseSubscriber(vatId: VatId, kpid: KRef): void { - insistVatId(vatId); - const kp = getKernelPromise(kpid); - kp.state === 'unresolved' || - Fail`attempt to add subscriber to resolved promise ${kpid}`; - const tempSet = new Set(kp.subscribers); - tempSet.add(vatId); - const newSubscribers = Array.from(tempSet).sort(); - const key = `${kpid}.subscribers`; - ctx.kv.set(key, JSON.stringify(newSubscribers)); - } - - /** - * Assign a kernel promise's decider. - * - * @param kpid - The KRef of promise whose decider is being set. - * @param vatId - The vat which will become the decider. - */ - function setPromiseDecider(kpid: KRef, vatId: VatId): void { - insistVatId(vatId); - if (kpid) { - ctx.kv.set(`${kpid}.decider`, vatId); - } - } - - /** - * Record the resolution of a kernel promise. - * - * @param kpid - The ref of the promise being resolved. - * @param rejected - True if the promise is being rejected, false if fulfilled. - * @param value - The value the promise is being fulfilled to or rejected with. - */ - function resolveKernelPromise( - kpid: KRef, - rejected: boolean, - value: CapData, - ): void { - const queue = provideStoredQueue(kpid, false); - for (const message of getKernelPromiseMessageQueue(kpid)) { - queue.enqueue(message); - } - ctx.kv.set(`${kpid}.state`, rejected ? 'rejected' : 'fulfilled'); - ctx.kv.set(`${kpid}.value`, JSON.stringify(value)); - ctx.kv.delete(`${kpid}.decider`); - ctx.kv.delete(`${kpid}.subscribers`); - } - - /** - * Append a message to a promise's message queue. - * - * @param kpid - The KRef of the promise to enqueue on. - * @param message - The message to enqueue. - */ - function enqueuePromiseMessage(kpid: KRef, message: Message): void { - provideStoredQueue(kpid, false).enqueue(message); - } - - /** - * Fetch the messages in a kernel promise's message queue. - * - * @param kpid - The KRef of the kernel promise of interest. - * @returns An array of all the messages in the given promise's message queue. - */ - function getKernelPromiseMessageQueue(kpid: KRef): Message[] { - const result: Message[] = []; - const queue = provideStoredQueue(kpid, false); - for (;;) { - const message = queue.dequeue() as Message; - if (message) { - result.push(message); - } else { - return result; - } - } - } - - return { - // Promise lifecycle - initKernelPromise, - getKernelPromise, - deleteKernelPromise, - getNextPromiseId, - - // Promise state management - addPromiseSubscriber, - setPromiseDecider, - resolveKernelPromise, - - // Promise messaging - enqueuePromiseMessage, - getKernelPromiseMessageQueue, - }; -} diff --git a/packages/kernel/src/store/methods/queue.test.ts b/packages/kernel/src/store/methods/queue.test.ts deleted file mode 100644 index aa7a51096..000000000 --- a/packages/kernel/src/store/methods/queue.test.ts +++ /dev/null @@ -1,276 +0,0 @@ -import type { KVStore } from '@ocap/store'; -import { describe, it, expect, beforeEach } from 'vitest'; - -import { getQueueMethods } from './queue.ts'; -import { makeMapKVStore } from '../../../test/storage.ts'; - -describe('queue-methods', () => { - let kv: KVStore; - let queueStore: ReturnType; - - beforeEach(() => { - kv = makeMapKVStore(); - queueStore = getQueueMethods(kv); - }); - - describe('createStoredQueue', () => { - it('creates a new empty queue', () => { - queueStore.createStoredQueue('test-queue'); - expect(kv.get('queue.test-queue.head')).toBe('1'); - expect(kv.get('queue.test-queue.tail')).toBe('1'); - }); - - it('creates a cached queue when specified', () => { - queueStore.createStoredQueue('cached-queue', true); - expect(kv.get('queue.cached-queue.head')).toBe('1'); - expect(kv.get('queue.cached-queue.tail')).toBe('1'); - }); - }); - - describe('provideStoredQueue', () => { - it('throws an error for uninitialized queues', () => { - expect(() => queueStore.provideStoredQueue('nonexistent')).toThrow( - 'queue nonexistent not initialized', - ); - }); - - it('provides access to an existing queue', () => { - // Create a queue first - queueStore.createStoredQueue('existing-queue'); - - // Then access it - const queue = queueStore.provideStoredQueue('existing-queue'); - expect(queue).toBeDefined(); - }); - }); - - describe('queue operations', () => { - it('enqueues and dequeues items correctly', () => { - const queue = queueStore.createStoredQueue('ops-queue'); - - // Enqueue items - queue.enqueue({ id: 1, value: 'first' }); - queue.enqueue({ id: 2, value: 'second' }); - - // Check queue length - expect(queueStore.getQueueLength('ops-queue')).toBe(2); - - // Dequeue items in FIFO order - const item1 = queue.dequeue(); - expect(item1).toStrictEqual({ id: 1, value: 'first' }); - - const item2 = queue.dequeue(); - expect(item2).toStrictEqual({ id: 2, value: 'second' }); - - // Queue should be empty now - expect(queueStore.getQueueLength('ops-queue')).toBe(0); - - // Dequeue from empty queue returns undefined - const emptyResult = queue.dequeue(); - expect(emptyResult).toBeUndefined(); - }); - - it('handles complex objects in the queue', () => { - const queue = queueStore.createStoredQueue('complex-queue'); - - const complexObject = { - id: 123, - nested: { - array: [1, 2, 3], - map: { a: 1, b: 2 }, - }, - date: new Date().toISOString(), - }; - - queue.enqueue(complexObject); - const result = queue.dequeue(); - - expect(result).toStrictEqual(complexObject); - }); - - it('deletes queues correctly', () => { - const queue = queueStore.createStoredQueue('delete-queue'); - - // Add some items - queue.enqueue({ id: 1 }); - queue.enqueue({ id: 2 }); - queue.enqueue({ id: 3 }); - - // Delete the queue - queue.delete(); - - // Queue metadata should be gone - expect(kv.get('queue.delete-queue.head')).toBeUndefined(); - expect(kv.get('queue.delete-queue.tail')).toBeUndefined(); - - // Queue entries should be gone - expect(kv.get('queue.delete-queue.1')).toBeUndefined(); - expect(kv.get('queue.delete-queue.2')).toBeUndefined(); - expect(kv.get('queue.delete-queue.3')).toBeUndefined(); - - // Operations on deleted queue should behave appropriately - expect(queue.dequeue()).toBeUndefined(); - expect(() => queue.enqueue({ id: 4 })).toThrow( - 'enqueue into deleted queue delete-queue', - ); - }); - }); - - describe('getQueueLength', () => { - it('returns the correct queue length', () => { - const queue = queueStore.createStoredQueue('length-queue'); - - expect(queueStore.getQueueLength('length-queue')).toBe(0); - - queue.enqueue({ id: 1 }); - expect(queueStore.getQueueLength('length-queue')).toBe(1); - - queue.enqueue({ id: 2 }); - queue.enqueue({ id: 3 }); - expect(queueStore.getQueueLength('length-queue')).toBe(3); - - queue.dequeue(); - expect(queueStore.getQueueLength('length-queue')).toBe(2); - - queue.dequeue(); - queue.dequeue(); - expect(queueStore.getQueueLength('length-queue')).toBe(0); - }); - - it('throws an error for unknown queues', () => { - expect(() => queueStore.getQueueLength('unknown-queue')).toThrow( - 'unknown queue unknown-queue', - ); - }); - }); - - describe('cached vs uncached queues', () => { - it('both cached and uncached queues work the same way', () => { - const cachedQueue = queueStore.createStoredQueue('cached-queue', true); - const uncachedQueue = queueStore.createStoredQueue( - 'uncached-queue', - false, - ); - - // Add same items to both queues - cachedQueue.enqueue({ id: 1 }); - uncachedQueue.enqueue({ id: 1 }); - - cachedQueue.enqueue({ id: 2 }); - uncachedQueue.enqueue({ id: 2 }); - - // Both should have same length - expect(queueStore.getQueueLength('cached-queue')).toBe(2); - expect(queueStore.getQueueLength('uncached-queue')).toBe(2); - - // Both should dequeue the same items - expect(cachedQueue.dequeue()).toStrictEqual({ id: 1 }); - expect(uncachedQueue.dequeue()).toStrictEqual({ id: 1 }); - - expect(cachedQueue.dequeue()).toStrictEqual({ id: 2 }); - expect(uncachedQueue.dequeue()).toStrictEqual({ id: 2 }); - - // Both should be empty - expect(queueStore.getQueueLength('cached-queue')).toBe(0); - expect(queueStore.getQueueLength('uncached-queue')).toBe(0); - }); - - it('cached vs uncached queues handle external changes differently', () => { - // First, let's create two separate queues - const cachedQueue = queueStore.createStoredQueue('cached-test', true); - const uncachedQueue = queueStore.createStoredQueue( - 'uncached-test', - false, - ); - - // Add an item to each queue to advance the head counter - cachedQueue.enqueue({ test: 'cached' }); - uncachedQueue.enqueue({ test: 'uncached' }); - - // Both heads should now be at 2 - expect(kv.get('queue.cached-test.head')).toBe('2'); - expect(kv.get('queue.uncached-test.head')).toBe('2'); - - // Now let's modify the KV store directly for both queues - kv.set('queue.cached-test.head', '10'); - kv.set('queue.uncached-test.head', '10'); - - // Enqueue new items - cachedQueue.enqueue({ test: 'cached-after-change' }); - uncachedQueue.enqueue({ test: 'uncached-after-change' }); - - // For the cached queue, the cached head value (2) should have been used, - // so the item should be at position 2 - expect(kv.get('queue.cached-test.2')).toBeDefined(); - - // For the uncached queue, the modified head value (10) should have been used, - // so the item should be at position 10 - expect(kv.get('queue.uncached-test.10')).toBeDefined(); - - // The heads should now be at 3 and 11 respectively - expect(kv.get('queue.cached-test.head')).toBe('3'); - expect(kv.get('queue.uncached-test.head')).toBe('11'); - }); - }); - - describe('integration', () => { - it('supports multiple queues simultaneously', () => { - const queue1 = queueStore.createStoredQueue('queue1'); - const queue2 = queueStore.createStoredQueue('queue2'); - - queue1.enqueue({ id: 'q1-1' }); - queue2.enqueue({ id: 'q2-1' }); - queue1.enqueue({ id: 'q1-2' }); - queue2.enqueue({ id: 'q2-2' }); - - expect(queueStore.getQueueLength('queue1')).toBe(2); - expect(queueStore.getQueueLength('queue2')).toBe(2); - - expect(queue1.dequeue()).toStrictEqual({ id: 'q1-1' }); - expect(queue2.dequeue()).toStrictEqual({ id: 'q2-1' }); - - expect(queueStore.getQueueLength('queue1')).toBe(1); - expect(queueStore.getQueueLength('queue2')).toBe(1); - - queue1.delete(); - expect(() => queueStore.getQueueLength('queue1')).toThrow( - 'unknown queue queue1', - ); - expect(queueStore.getQueueLength('queue2')).toBe(1); - }); - - it('handles a large number of queue operations', () => { - const queue = queueStore.createStoredQueue('large-queue'); - - // Enqueue 100 items - for (let i = 0; i < 100; i++) { - queue.enqueue({ index: i }); - } - - expect(queueStore.getQueueLength('large-queue')).toBe(100); - - // Dequeue 50 items - for (let i = 0; i < 50; i++) { - const item = queue.dequeue(); - expect(item).toStrictEqual({ index: i }); - } - - expect(queueStore.getQueueLength('large-queue')).toBe(50); - - // Enqueue 50 more - for (let i = 100; i < 150; i++) { - queue.enqueue({ index: i }); - } - - expect(queueStore.getQueueLength('large-queue')).toBe(100); - - // Dequeue all remaining - for (let i = 50; i < 150; i++) { - const item = queue.dequeue(); - expect(item).toStrictEqual({ index: i }); - } - - expect(queueStore.getQueueLength('large-queue')).toBe(0); - }); - }); -}); diff --git a/packages/kernel/src/store/methods/queue.ts b/packages/kernel/src/store/methods/queue.ts deleted file mode 100644 index 68a2ff5d6..000000000 --- a/packages/kernel/src/store/methods/queue.ts +++ /dev/null @@ -1,117 +0,0 @@ -import type { KVStore } from '@ocap/store'; - -import { getBaseMethods } from './base.ts'; -import type { StoredQueue } from '../types.ts'; - -/** - * Get a queue store object that provides functionality for managing queues. - * - * @param kv - A key/value store to provide the underlying persistence mechanism. - * @returns A queue store object that maps various persistent kernel data - * structures onto `kv`. - */ -// eslint-disable-next-line @typescript-eslint/explicit-function-return-type -export function getQueueMethods(kv: KVStore) { - const { provideCachedStoredValue, provideRawStoredValue, incCounter } = - getBaseMethods(kv); - - /** - * Create a new (empty) persistently stored queue. - * - * @param queueName - The name for the queue (must be unique among queues). - * @param cached - Optional flag: set to true if the queue should cache its - * @returns An object for interacting with the new queue. - */ - function createStoredQueue( - queueName: string, - cached: boolean = false, - ): StoredQueue { - const qk = `queue.${queueName}`; - kv.set(`${qk}.head`, '1'); - kv.set(`${qk}.tail`, '1'); - return provideStoredQueue(queueName, cached); - } - - /** - * Produce an object to access a persistently stored queue. - * - * @param queueName - The name for the queue (must be unique among queues). - * @param cached - Optional flag: set to true if the queue should cache its - * limit indices in memory (only do this if the queue is going to be accessed or - * checked frequently). - * @returns An object for interacting with the queue. - */ - function provideStoredQueue( - queueName: string, - cached: boolean = false, - ): StoredQueue { - const qk = `queue.${queueName}`; - // Note: cached=true ==> caches only the head & tail indices, NOT the queue entries themselves - const provideValue = cached - ? provideCachedStoredValue - : provideRawStoredValue; - const head = provideValue(`${qk}.head`); - const tail = provideValue(`${qk}.tail`); - if (head.get() === undefined || tail.get() === undefined) { - throw Error(`queue ${queueName} not initialized`); - } - return { - enqueue(item: object): void { - if (head.get() === undefined) { - throw Error(`enqueue into deleted queue ${queueName}`); - } - const entryPos = incCounter(head); - kv.set(`${qk}.${entryPos}`, JSON.stringify(item)); - }, - dequeue(): object | undefined { - const headPos = head.get(); - if (headPos === undefined) { - return undefined; - } - const tailPos = tail.get(); - if (tailPos !== headPos) { - const entry = kv.getRequired(`${qk}.${tailPos}`); - kv.delete(`${qk}.${tailPos}`); - incCounter(tail); - return JSON.parse(entry) as object; - } - return undefined; - }, - delete(): void { - const headPos = head.get(); - if (headPos !== undefined) { - let tailPos = tail.get(); - while (tailPos !== headPos) { - kv.delete(`${qk}.${tailPos}`); - tailPos = `${Number(tailPos) + 1}`; - } - head.delete(); - tail.delete(); - } - }, - }; - } - - /** - * Find out how long some queue is. - * - * @param queueName - The name of the queue of interest. - * - * @returns the number of items in the given queue. - */ - function getQueueLength(queueName: string): number { - const qk = `queue.${queueName}`; - const head = kv.get(`${qk}.head`); - const tail = kv.get(`${qk}.tail`); - if (head === undefined || tail === undefined) { - throw Error(`unknown queue ${queueName}`); - } - return Number(head) - Number(tail); - } - - return { - createStoredQueue, - provideStoredQueue, - getQueueLength, - }; -} diff --git a/packages/kernel/src/store/methods/refcount.test.ts b/packages/kernel/src/store/methods/refcount.test.ts deleted file mode 100644 index aa6e5c852..000000000 --- a/packages/kernel/src/store/methods/refcount.test.ts +++ /dev/null @@ -1,164 +0,0 @@ -import type { KVStore } from '@ocap/store'; -import { describe, it, expect, beforeEach } from 'vitest'; - -import { getRefCountMethods } from './refcount.ts'; -import { makeMapKVStore } from '../../../test/storage.ts'; -import type { KRef } from '../../types.ts'; -import type { StoreContext } from '../types.ts'; - -describe('refcount-methods', () => { - let kv: KVStore; - let refCountStore: ReturnType; - - beforeEach(() => { - kv = makeMapKVStore(); - refCountStore = getRefCountMethods({ kv } as StoreContext); - }); - - describe('refCountKey', () => { - it('generates correct reference count keys', () => { - expect(refCountStore.refCountKey('ko1')).toBe('ko1.refCount'); - expect(refCountStore.refCountKey('kp42')).toBe('kp42.refCount'); - expect(refCountStore.refCountKey('v7')).toBe('v7.refCount'); - }); - }); - - describe('getRefCount', () => { - it('returns 0 for non-existent references', () => { - kv.set(refCountStore.refCountKey('ko99'), '0'); - expect(refCountStore.getRefCount('ko99')).toBe(0); - }); - - it('handles undefined values by returning NaN', () => { - expect(Number.isNaN(refCountStore.getRefCount('nonexistent'))).toBe(true); - }); - - it('returns the correct reference count for existing references', () => { - const kref: KRef = 'ko1'; - kv.set(`${kref}.refCount`, '5'); - expect(refCountStore.getRefCount(kref)).toBe(5); - }); - }); - - describe('incRefCount', () => { - it('initializes and increments reference count for new references', () => { - const kref: KRef = 'ko1'; - - kv.set(refCountStore.refCountKey(kref), '0'); - - expect(refCountStore.incRefCount(kref)).toBe(1); - expect(kv.get(`${kref}.refCount`)).toBe('1'); - - expect(refCountStore.incRefCount(kref)).toBe(2); - expect(kv.get(`${kref}.refCount`)).toBe('2'); - }); - - it('handles incrementing from undefined', () => { - const kref: KRef = 'ko2'; - expect(Number.isNaN(refCountStore.incRefCount(kref))).toBe(true); - expect(kv.get(refCountStore.refCountKey(kref))).toBe('NaN'); - }); - - it('increments existing reference counts', () => { - const kref: KRef = 'kp42'; - kv.set(`${kref}.refCount`, '10'); - - expect(refCountStore.incRefCount(kref)).toBe(11); - expect(kv.get(`${kref}.refCount`)).toBe('11'); - }); - }); - - describe('decRefCount', () => { - it('decrements reference counts correctly', () => { - const kref: KRef = 'ko1'; - kv.set(`${kref}.refCount`, '5'); - - expect(refCountStore.decRefCount(kref)).toBe(4); - expect(kv.get(`${kref}.refCount`)).toBe('4'); - - expect(refCountStore.decRefCount(kref)).toBe(3); - expect(kv.get(`${kref}.refCount`)).toBe('3'); - }); - - it('can decrement to zero', () => { - const kref: KRef = 'ko1'; - kv.set(`${kref}.refCount`, '1'); - - expect(refCountStore.decRefCount(kref)).toBe(0); - expect(kv.get(`${kref}.refCount`)).toBe('0'); - }); - - it('can decrement below zero (though this should be avoided in practice)', () => { - const kref: KRef = 'ko1'; - kv.set(`${kref}.refCount`, '0'); - - expect(refCountStore.decRefCount(kref)).toBe(-1); - expect(kv.get(`${kref}.refCount`)).toBe('-1'); - }); - }); - - describe('kernelRefExists', () => { - it('returns false for non-existent references', () => { - expect(refCountStore.kernelRefExists('ko99')).toBe(false); - }); - - it('returns true for existing references with non-zero count', () => { - const kref: KRef = 'ko1'; - kv.set(`${kref}.refCount`, '5'); - expect(refCountStore.kernelRefExists(kref)).toBe(true); - }); - - it('returns true for existing references with zero count', () => { - const kref: KRef = 'ko1'; - kv.set(`${kref}.refCount`, '0'); - expect(refCountStore.kernelRefExists(kref)).toBe(true); - }); - }); - - describe('integration', () => { - it('supports full reference counting lifecycle', () => { - const kref: KRef = 'ko42'; - - expect(refCountStore.kernelRefExists(kref)).toBe(false); - - kv.set(refCountStore.refCountKey(kref), '0'); - expect(refCountStore.getRefCount(kref)).toBe(0); - - expect(refCountStore.kernelRefExists(kref)).toBe(true); - - expect(refCountStore.incRefCount(kref)).toBe(1); - - refCountStore.incRefCount(kref); - refCountStore.incRefCount(kref); - expect(refCountStore.getRefCount(kref)).toBe(3); - - refCountStore.decRefCount(kref); - expect(refCountStore.getRefCount(kref)).toBe(2); - - refCountStore.decRefCount(kref); - refCountStore.decRefCount(kref); - expect(refCountStore.getRefCount(kref)).toBe(0); - - expect(refCountStore.kernelRefExists(kref)).toBe(true); - }); - - it('works with multiple references simultaneously', () => { - const kref1: KRef = 'ko1'; - const kref2: KRef = 'kp2'; - - kv.set(refCountStore.refCountKey(kref1), '0'); - kv.set(refCountStore.refCountKey(kref2), '0'); - - refCountStore.incRefCount(kref1); - refCountStore.incRefCount(kref2); - refCountStore.incRefCount(kref2); - - expect(refCountStore.getRefCount(kref1)).toBe(1); - expect(refCountStore.getRefCount(kref2)).toBe(2); - - refCountStore.decRefCount(kref1); - expect(refCountStore.getRefCount(kref1)).toBe(0); - expect(refCountStore.getRefCount(kref2)).toBe(2); - }); - }); -}); diff --git a/packages/kernel/src/store/methods/refcount.ts b/packages/kernel/src/store/methods/refcount.ts deleted file mode 100644 index 5a8c040d8..000000000 --- a/packages/kernel/src/store/methods/refcount.ts +++ /dev/null @@ -1,76 +0,0 @@ -import type { KRef } from '../../types.ts'; -import type { StoreContext } from '../types.ts'; - -/** - * Create a refcount store object that provides functionality for managing reference counts. - * - * @param ctx - The store context. - * @returns A refcount store object that maps various persistent kernel data - * structures onto `kv`. - */ -// eslint-disable-next-line @typescript-eslint/explicit-function-return-type -export function getRefCountMethods(ctx: StoreContext) { - /** - * Generate the storage key for a kernel entity's reference count. - * - * @param kref - The KRef of interest. - * @returns the key to store the indicated reference count at. - */ - function refCountKey(kref: KRef): string { - return `${kref}.refCount`; - } - - /** - * Get a kernel entity's reference count. - * - * @param kref - The KRef of interest. - * @returns the reference count of the indicated kernel entity. - */ - function getRefCount(kref: KRef): number { - return Number(ctx.kv.get(refCountKey(kref))); - } - - /** - * Increment a kernel entity's reference count. - * - * @param kref - The KRef of the entity to increment the ref count of. - * @returns the new reference count after incrementing. - */ - function incRefCount(kref: KRef): number { - const key = refCountKey(kref); - const newCount = Number(ctx.kv.get(key)) + 1; - ctx.kv.set(key, `${newCount}`); - return newCount; - } - - /** - * Decrement a kernel entity's reference count. - * - * @param kref - The KRef of the entity to decrement the ref count of. - * @returns the new reference count after decrementing. - */ - function decRefCount(kref: KRef): number { - const key = refCountKey(kref); - const newCount = Number(ctx.kv.get(key)) - 1; - ctx.kv.set(key, `${newCount}`); - return newCount; - } - - /** - * Check if a kernel object exists in the kernel's persistent state. - * - * @param kref - The KRef of the kernel object in question. - * @returns True if the kernel object exists, false otherwise. - */ - function kernelRefExists(kref: KRef): boolean { - return Boolean(ctx.kv.get(refCountKey(kref))); - } - - return { - refCountKey, - getRefCount, - incRefCount, - decRefCount, - kernelRefExists, - }; -} diff --git a/packages/kernel/src/store/types.ts b/packages/kernel/src/store/types.ts deleted file mode 100644 index 75aac5bc9..000000000 --- a/packages/kernel/src/store/types.ts +++ /dev/null @@ -1,27 +0,0 @@ -import type { KVStore } from '@ocap/store'; - -import type { KRef } from '../types.ts'; - -export type StoreContext = { - kv: KVStore; - runQueue: StoredQueue; - nextObjectId: StoredValue; - nextPromiseId: StoredValue; - nextVatId: StoredValue; - nextRemoteId: StoredValue; - maybeFreeKrefs: Set; - gcActions: StoredValue; - reapQueue: StoredValue; -}; - -export type StoredValue = { - get(): string | undefined; - set(newValue: string): void; - delete(): void; -}; - -export type StoredQueue = { - enqueue(item: object): void; - dequeue(): object | undefined; - delete(): void; -}; From ca199c6b9c4b955e75ee06969f70c363c1841f63 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 27 Mar 2025 17:33:17 +0100 Subject: [PATCH 21/21] thresholds --- packages/kernel-test/src/supervisor.test.ts | 2 +- vitest.config.ts | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/kernel-test/src/supervisor.test.ts b/packages/kernel-test/src/supervisor.test.ts index 5d3e94c88..89bd18506 100644 --- a/packages/kernel-test/src/supervisor.test.ts +++ b/packages/kernel-test/src/supervisor.test.ts @@ -5,7 +5,7 @@ import { readFile } from 'fs/promises'; import { join } from 'path'; import { describe, it, expect } from 'vitest'; -import { kser } from '../../kernel/src/kernel-marshal.ts'; +import { kser } from '../../kernel/src/services/kernel-marshal.ts'; import { TestDuplexStream } from '../../streams/test/stream-mocks.ts'; const makeVatSupervisor = async ({ diff --git a/vitest.config.ts b/vitest.config.ts index 42f3f247b..ec1c2402b 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -88,10 +88,10 @@ export default defineConfig({ lines: 81.78, }, 'packages/kernel/**': { - statements: 0, - functions: 0, - branches: 0, - lines: 0, + statements: 82.4, + functions: 88.71, + branches: 64.15, + lines: 82.51, }, 'packages/nodejs/**': { statements: 72.91,