From ac3d7d60bff29cc5ca2525c9ceac08492e6333bf Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Tue, 16 Sep 2025 13:38:54 +0100 Subject: [PATCH 01/14] wip --- packages/db/src/collection.ts | 1923 ++--------------- packages/db/src/collection/changes.ts | 221 ++ packages/db/src/collection/indexes.ts | 162 ++ packages/db/src/collection/lifecycle.ts | 172 ++ packages/db/src/collection/mutations.ts | 517 +++++ packages/db/src/collection/state.ts | 720 ++++++ packages/db/src/collection/sync.ts | 222 ++ packages/db/src/transactions.ts | 6 +- packages/db/tests/collection-errors.test.ts | 42 +- .../db/tests/collection-lifecycle.test.ts | 20 +- packages/db/tests/collection-schema.test.ts | 8 +- .../collection-subscribe-changes.test.ts | 14 +- packages/db/tests/collection.test.ts | 40 +- .../tests/electric.test.ts | 8 +- packages/query-db-collection/src/query.ts | 2 +- .../query-db-collection/tests/query.test.ts | 6 +- .../rxdb-db-collection/tests/rxdb.test.ts | 8 +- 17 files changed, 2220 insertions(+), 1871 deletions(-) create mode 100644 packages/db/src/collection/changes.ts create mode 100644 packages/db/src/collection/indexes.ts create mode 100644 packages/db/src/collection/lifecycle.ts create mode 100644 packages/db/src/collection/mutations.ts create mode 100644 packages/db/src/collection/state.ts create mode 100644 packages/db/src/collection/sync.ts diff --git a/packages/db/src/collection.ts b/packages/db/src/collection.ts index dc4cc9f2a..45c8ce882 100644 --- a/packages/db/src/collection.ts +++ b/packages/db/src/collection.ts @@ -1,44 +1,10 @@ -import { withArrayChangeTracking, withChangeTracking } from "./proxy" -import { deepEquals } from "./utils" -import { SortedMap } from "./SortedMap" -import { - createSingleRowRefProxy, - toExpression, -} from "./query/builder/ref-proxy" import { BTreeIndex } from "./indexes/btree-index.js" -import { IndexProxy, LazyIndexWrapper } from "./indexes/lazy-index.js" -import { ensureIndexForExpression } from "./indexes/auto-index.js" -import { createTransaction, getActiveTransaction } from "./transactions" +import type { IndexProxy } from "./indexes/lazy-index.js" import { - CollectionInErrorStateError, - CollectionIsInErrorStateError, CollectionRequiresConfigError, CollectionRequiresSyncConfigError, - DeleteKeyNotFoundError, - DuplicateKeyError, - DuplicateKeySyncError, - InvalidCollectionStatusTransitionError, - InvalidSchemaError, - KeyUpdateNotAllowedError, - MissingDeleteHandlerError, - MissingInsertHandlerError, - MissingUpdateArgumentError, - MissingUpdateHandlerError, - NegativeActiveSubscribersError, - NoKeysPassedToDeleteError, - NoKeysPassedToUpdateError, - NoPendingSyncTransactionCommitError, - NoPendingSyncTransactionWriteError, - SchemaMustBeSynchronousError, - SchemaValidationError, - SyncCleanupError, - SyncTransactionAlreadyCommittedError, - SyncTransactionAlreadyCommittedWriteError, - UndefinedKeyError, - UpdateKeyNotFoundError, } from "./errors" -import { createFilteredCallback, currentStateAsChanges } from "./change-events" -import type { Transaction } from "./transactions" +import { currentStateAsChanges } from "./change-events" import type { StandardSchemaV1 } from "@standard-schema/spec" import type { SingleRowRefProxy } from "./query/builder/ref-proxy" import type { @@ -52,24 +18,20 @@ import type { InferSchemaOutput, InsertConfig, OperationConfig, - OptimisticChangeMessage, - PendingMutation, - StandardSchema, SubscribeChangesOptions, Transaction as TransactionType, - TransactionWithMutations, UtilsRecord, WritableDeep, } from "./types" import type { IndexOptions } from "./indexes/index-options.js" import type { BaseIndex, IndexResolver } from "./indexes/base-index.js" -interface PendingSyncedTransaction> { - committed: boolean - operations: Array> - truncate?: boolean - deletedKeys: Set -} +import { CollectionStateManager } from "./collection/state" +import { CollectionChangesManager } from "./collection/changes" +import { CollectionLifecycleManager } from "./collection/lifecycle.js" +import { CollectionSyncManager } from "./collection/sync" +import { CollectionIndexesManager } from "./collection/indexes" +import { CollectionMutationsManager } from "./collection/mutations" /** * Enhanced Collection interface that includes both data type T and utilities TUtils @@ -212,199 +174,26 @@ export class CollectionImpl< TSchema extends StandardSchemaV1 = StandardSchemaV1, TInput extends object = TOutput, > { + public id: string public config: CollectionConfig - // Core state - make public for testing - public transactions: SortedMap> - public pendingSyncedTransactions: Array> = - [] - public syncedData: Map | SortedMap - public syncedMetadata = new Map() - - // Optimistic state tracking - make public for testing - public optimisticUpserts = new Map() - public optimisticDeletes = new Set() - - // Cached size for performance - private _size = 0 - - // Index storage - private lazyIndexes = new Map>() - private resolvedIndexes = new Map>() - private isIndexesResolved = false - private indexCounter = 0 - - // Event system - private changeListeners = new Set>() - private changeKeyListeners = new Map< - TKey, - Set> - >() - // Utilities namespace // This is populated by createCollection public utils: Record = {} - // State used for computing the change events - private syncedKeys = new Set() - private preSyncVisibleState = new Map() - private recentlySyncedKeys = new Set() - private hasReceivedFirstCommit = false - private isCommittingSyncTransactions = false - - // Array to store one-time ready listeners - private onFirstReadyCallbacks: Array<() => void> = [] - private hasBeenReady = false - - // Event batching for preventing duplicate emissions during transaction flows - private batchedEvents: Array> = [] - private shouldBatchEvents = false - - // Lifecycle management - private _status: CollectionStatus = `idle` - private activeSubscribersCount = 0 - private gcTimeoutId: ReturnType | null = null - private preloadPromise: Promise | null = null - private syncCleanupFn: (() => void) | null = null - - /** - * Register a callback to be executed when the collection first becomes ready - * Useful for preloading collections - * @param callback Function to call when the collection first becomes ready - * @example - * collection.onFirstReady(() => { - * console.log('Collection is ready for the first time') - * // Safe to access collection.state now - * }) - */ - public onFirstReady(callback: () => void): void { - // If already ready, call immediately - if (this.hasBeenReady) { - callback() - return - } - - this.onFirstReadyCallbacks.push(callback) - } - - /** - * Check if the collection is ready for use - * Returns true if the collection has been marked as ready by its sync implementation - * @returns true if the collection is ready, false otherwise - * @example - * if (collection.isReady()) { - * console.log('Collection is ready, data is available') - * // Safe to access collection.state - * } else { - * console.log('Collection is still loading') - * } - */ - public isReady(): boolean { - return this._status === `ready` - } - - /** - * Mark the collection as ready for use - * This is called by sync implementations to explicitly signal that the collection is ready, - * providing a more intuitive alternative to using commits for readiness signaling - * @private - Should only be called by sync implementations - */ - private markReady(): void { - // Can transition to ready from loading or initialCommit states - if (this._status === `loading` || this._status === `initialCommit`) { - this.setStatus(`ready`) - - // Call any registered first ready callbacks (only on first time becoming ready) - if (!this.hasBeenReady) { - this.hasBeenReady = true - - // Also mark as having received first commit for backwards compatibility - if (!this.hasReceivedFirstCommit) { - this.hasReceivedFirstCommit = true - } - - const callbacks = [...this.onFirstReadyCallbacks] - this.onFirstReadyCallbacks = [] - callbacks.forEach((callback) => callback()) - } - } - - // Always notify dependents when markReady is called, after status is set - // This ensures live queries get notified when their dependencies become ready - if (this.changeListeners.size > 0) { - this.emitEmptyReadyEvent() - } - } - - public id = `` - - /** - * Gets the current status of the collection - */ - public get status(): CollectionStatus { - return this._status - } - - /** - * Validates that the collection is in a usable state for data operations - * @private - */ - private validateCollectionUsable(operation: string): void { - switch (this._status) { - case `error`: - throw new CollectionInErrorStateError(operation, this.id) - case `cleaned-up`: - // Automatically restart the collection when operations are called on cleaned-up collections - this.startSync() - break - } - } - - /** - * Validates state transitions to prevent invalid status changes - * @private - */ - private validateStatusTransition( - from: CollectionStatus, - to: CollectionStatus - ): void { - if (from === to) { - // Allow same state transitions - return - } - const validTransitions: Record< - CollectionStatus, - Array - > = { - idle: [`loading`, `error`, `cleaned-up`], - loading: [`initialCommit`, `ready`, `error`, `cleaned-up`], - initialCommit: [`ready`, `error`, `cleaned-up`], - ready: [`cleaned-up`, `error`], - error: [`cleaned-up`, `idle`], - "cleaned-up": [`loading`, `error`], - } - - if (!validTransitions[from].includes(to)) { - throw new InvalidCollectionStatusTransitionError(from, to, this.id) - } - } - - /** - * Safely update the collection status with validation - * @private - */ - private setStatus(newStatus: CollectionStatus): void { - this.validateStatusTransition(this._status, newStatus) - this._status = newStatus - - // Resolve indexes when collection becomes ready - if (newStatus === `ready` && !this.isIndexesResolved) { - // Resolve indexes asynchronously without blocking - this.resolveAllIndexes().catch((error) => { - console.warn(`Failed to resolve indexes:`, error) - }) - } - } + // Managers + public _state: CollectionStateManager + public _changes: CollectionChangesManager + public _lifecycle: CollectionLifecycleManager + public _sync: CollectionSyncManager + public _indexes: CollectionIndexesManager + public _mutations: CollectionMutationsManager< + TOutput, + TKey, + TUtils, + TSchema, + TInput + > /** * Creates a new Collection instance @@ -417,20 +206,17 @@ export class CollectionImpl< if (!config) { throw new CollectionRequiresConfigError() } - if (config.id) { - this.id = config.id - } else { - this.id = crypto.randomUUID() - } // eslint-disable-next-line if (!config.sync) { throw new CollectionRequiresSyncConfigError() } - this.transactions = new SortedMap>((a, b) => - a.compareCreatedAt(b) - ) + if (config.id) { + this.id = config.id + } else { + this.id = crypto.randomUUID() + } // Set default values for optional config properties this.config = { @@ -438,618 +224,160 @@ export class CollectionImpl< autoIndex: config.autoIndex ?? `eager`, } - // Set up data storage with optional comparison function - if (this.config.compare) { - this.syncedData = new SortedMap(this.config.compare) - } else { - this.syncedData = new Map() - } + this._state = new CollectionStateManager( + this + ) + this._changes = new CollectionChangesManager< + TOutput, + TKey, + TSchema, + TInput + >(this) + this._lifecycle = new CollectionLifecycleManager< + TOutput, + TKey, + TSchema, + TInput + >(this) + this._sync = new CollectionSyncManager(this) + this._indexes = new CollectionIndexesManager< + TOutput, + TKey, + TSchema, + TInput + >(this) + this._mutations = new CollectionMutationsManager< + TOutput, + TKey, + TUtils, + TSchema, + TInput + >(this) // Only start sync immediately if explicitly enabled if (config.startSync === true) { - this.startSync() - } - } - - /** - * Start sync immediately - internal method for compiled queries - * This bypasses lazy loading for special cases like live query results - */ - public startSyncImmediate(): void { - this.startSync() - } - - /** - * Start the sync process for this collection - * This is called when the collection is first accessed or preloaded - */ - private startSync(): void { - if (this._status !== `idle` && this._status !== `cleaned-up`) { - return // Already started or in progress - } - - this.setStatus(`loading`) - - try { - const cleanupFn = this.config.sync.sync({ - collection: this, - begin: () => { - this.pendingSyncedTransactions.push({ - committed: false, - operations: [], - deletedKeys: new Set(), - }) - }, - write: (messageWithoutKey: Omit, `key`>) => { - const pendingTransaction = - this.pendingSyncedTransactions[ - this.pendingSyncedTransactions.length - 1 - ] - if (!pendingTransaction) { - throw new NoPendingSyncTransactionWriteError() - } - if (pendingTransaction.committed) { - throw new SyncTransactionAlreadyCommittedWriteError() - } - const key = this.getKeyFromItem(messageWithoutKey.value) - - // Check if an item with this key already exists when inserting - if (messageWithoutKey.type === `insert`) { - const insertingIntoExistingSynced = this.syncedData.has(key) - const hasPendingDeleteForKey = - pendingTransaction.deletedKeys.has(key) - const isTruncateTransaction = pendingTransaction.truncate === true - // Allow insert after truncate in the same transaction even if it existed in syncedData - if ( - insertingIntoExistingSynced && - !hasPendingDeleteForKey && - !isTruncateTransaction - ) { - throw new DuplicateKeySyncError(key, this.id) - } - } - - const message: ChangeMessage = { - ...messageWithoutKey, - key, - } - pendingTransaction.operations.push(message) - - if (messageWithoutKey.type === `delete`) { - pendingTransaction.deletedKeys.add(key) - } - }, - commit: () => { - const pendingTransaction = - this.pendingSyncedTransactions[ - this.pendingSyncedTransactions.length - 1 - ] - if (!pendingTransaction) { - throw new NoPendingSyncTransactionCommitError() - } - if (pendingTransaction.committed) { - throw new SyncTransactionAlreadyCommittedError() - } - - pendingTransaction.committed = true - - // Update status to initialCommit when transitioning from loading - // This indicates we're in the process of committing the first transaction - if (this._status === `loading`) { - this.setStatus(`initialCommit`) - } - - this.commitPendingTransactions() - }, - markReady: () => { - this.markReady() - }, - truncate: () => { - const pendingTransaction = - this.pendingSyncedTransactions[ - this.pendingSyncedTransactions.length - 1 - ] - if (!pendingTransaction) { - throw new NoPendingSyncTransactionWriteError() - } - if (pendingTransaction.committed) { - throw new SyncTransactionAlreadyCommittedWriteError() - } - - // Clear all operations from the current transaction - pendingTransaction.operations = [] - pendingTransaction.deletedKeys.clear() - - // Mark the transaction as a truncate operation. During commit, this triggers: - // - Delete events for all previously synced keys (excluding optimistic-deleted keys) - // - Clearing of syncedData/syncedMetadata - // - Subsequent synced ops applied on the fresh base - // - Finally, optimistic mutations re-applied on top (single batch) - pendingTransaction.truncate = true - }, - }) - - // Store cleanup function if provided - this.syncCleanupFn = typeof cleanupFn === `function` ? cleanupFn : null - } catch (error) { - this.setStatus(`error`) - throw error - } - } - - /** - * Preload the collection data by starting sync if not already started - * Multiple concurrent calls will share the same promise - */ - public preload(): Promise { - if (this.preloadPromise) { - return this.preloadPromise - } - - this.preloadPromise = new Promise((resolve, reject) => { - if (this._status === `ready`) { - resolve() - return - } - - if (this._status === `error`) { - reject(new CollectionIsInErrorStateError()) - return - } - - // Register callback BEFORE starting sync to avoid race condition - this.onFirstReady(() => { - resolve() - }) - - // Start sync if collection hasn't started yet or was cleaned up - if (this._status === `idle` || this._status === `cleaned-up`) { - try { - this.startSync() - } catch (error) { - reject(error) - return - } - } - }) - - return this.preloadPromise - } - - /** - * Clean up the collection by stopping sync and clearing data - * This can be called manually or automatically by garbage collection - */ - public async cleanup(): Promise { - // Clear GC timeout - if (this.gcTimeoutId) { - clearTimeout(this.gcTimeoutId) - this.gcTimeoutId = null - } - - // Stop sync - wrap in try/catch since it's user-provided code - try { - if (this.syncCleanupFn) { - this.syncCleanupFn() - this.syncCleanupFn = null - } - } catch (error) { - // Re-throw in a microtask to surface the error after cleanup completes - queueMicrotask(() => { - if (error instanceof Error) { - // Preserve the original error and stack trace - const wrappedError = new SyncCleanupError(this.id, error) - wrappedError.cause = error - wrappedError.stack = error.stack - throw wrappedError - } else { - throw new SyncCleanupError(this.id, error as Error | string) - } - }) + this._sync.startSync() } - - // Clear data - this.syncedData.clear() - this.syncedMetadata.clear() - this.optimisticUpserts.clear() - this.optimisticDeletes.clear() - this._size = 0 - this.pendingSyncedTransactions = [] - this.syncedKeys.clear() - this.hasReceivedFirstCommit = false - this.hasBeenReady = false - this.onFirstReadyCallbacks = [] - this.preloadPromise = null - this.batchedEvents = [] - this.shouldBatchEvents = false - - // Update status - this.setStatus(`cleaned-up`) - - return Promise.resolve() } /** - * Start the garbage collection timer - * Called when the collection becomes inactive (no subscribers) - */ - private startGCTimer(): void { - if (this.gcTimeoutId) { - clearTimeout(this.gcTimeoutId) - } - - const gcTime = this.config.gcTime ?? 300000 // 5 minutes default - - // If gcTime is 0, GC is disabled - if (gcTime === 0) { - return - } - - this.gcTimeoutId = setTimeout(() => { - if (this.activeSubscribersCount === 0) { - this.cleanup() - } - }, gcTime) - } - - /** - * Cancel the garbage collection timer - * Called when the collection becomes active again - */ - private cancelGCTimer(): void { - if (this.gcTimeoutId) { - clearTimeout(this.gcTimeoutId) - this.gcTimeoutId = null - } - } - - /** - * Increment the active subscribers count and start sync if needed - */ - private addSubscriber(): void { - this.activeSubscribersCount++ - this.cancelGCTimer() - - // Start sync if collection was cleaned up - if (this._status === `cleaned-up` || this._status === `idle`) { - this.startSync() - } - } - - /** - * Decrement the active subscribers count and start GC timer if needed + * Gets the current status of the collection */ - private removeSubscriber(): void { - this.activeSubscribersCount-- - - if (this.activeSubscribersCount === 0) { - this.startGCTimer() - } else if (this.activeSubscribersCount < 0) { - throw new NegativeActiveSubscribersError() - } + public get status(): CollectionStatus { + return this._lifecycle.status } /** - * Recompute optimistic state from active transactions + * Register a callback to be executed when the collection first becomes ready + * Useful for preloading collections + * @param callback Function to call when the collection first becomes ready + * @example + * collection.onFirstReady(() => { + * console.log('Collection is ready for the first time') + * // Safe to access collection.state now + * }) */ - private recomputeOptimisticState( - triggeredByUserAction: boolean = false - ): void { - // Skip redundant recalculations when we're in the middle of committing sync transactions - if (this.isCommittingSyncTransactions) { + public onFirstReady(callback: () => void): void { + // If already ready, call immediately + if (this._lifecycle.hasBeenReady) { + callback() return } - const previousState = new Map(this.optimisticUpserts) - const previousDeletes = new Set(this.optimisticDeletes) - - // Clear current optimistic state - this.optimisticUpserts.clear() - this.optimisticDeletes.clear() - - const activeTransactions: Array> = [] - - for (const transaction of this.transactions.values()) { - if (![`completed`, `failed`].includes(transaction.state)) { - activeTransactions.push(transaction) - } - } - - // Apply active transactions only (completed transactions are handled by sync operations) - for (const transaction of activeTransactions) { - for (const mutation of transaction.mutations) { - if (mutation.collection === this && mutation.optimistic) { - switch (mutation.type) { - case `insert`: - case `update`: - this.optimisticUpserts.set( - mutation.key, - mutation.modified as TOutput - ) - this.optimisticDeletes.delete(mutation.key) - break - case `delete`: - this.optimisticUpserts.delete(mutation.key) - this.optimisticDeletes.add(mutation.key) - break - } - } - } - } - - // Update cached size - this._size = this.calculateSize() - - // Collect events for changes - const events: Array> = [] - this.collectOptimisticChanges(previousState, previousDeletes, events) - - // Filter out events for recently synced keys to prevent duplicates - // BUT: Only filter out events that are actually from sync operations - // New user transactions should NOT be filtered even if the key was recently synced - const filteredEventsBySyncStatus = events.filter((event) => { - if (!this.recentlySyncedKeys.has(event.key)) { - return true // Key not recently synced, allow event through - } - - // Key was recently synced - allow if this is a user-triggered action - if (triggeredByUserAction) { - return true - } - - // Otherwise filter out duplicate sync events - return false - }) - - // Filter out redundant delete events if there are pending sync transactions - // that will immediately restore the same data, but only for completed transactions - // IMPORTANT: Skip complex filtering for user-triggered actions to prevent UI blocking - if (this.pendingSyncedTransactions.length > 0 && !triggeredByUserAction) { - const pendingSyncKeys = new Set() - - // Collect keys from pending sync operations - for (const transaction of this.pendingSyncedTransactions) { - for (const operation of transaction.operations) { - pendingSyncKeys.add(operation.key as TKey) - } - } - - // Only filter out delete events for keys that: - // 1. Have pending sync operations AND - // 2. Are from completed transactions (being cleaned up) - const filteredEvents = filteredEventsBySyncStatus.filter((event) => { - if (event.type === `delete` && pendingSyncKeys.has(event.key)) { - // Check if this delete is from clearing optimistic state of completed transactions - // We can infer this by checking if we have no remaining optimistic mutations for this key - const hasActiveOptimisticMutation = activeTransactions.some((tx) => - tx.mutations.some( - (m) => m.collection === this && m.key === event.key - ) - ) - - if (!hasActiveOptimisticMutation) { - return false // Skip this delete event as sync will restore the data - } - } - return true - }) - - // Update indexes for the filtered events - if (filteredEvents.length > 0) { - this.updateIndexes(filteredEvents) - } - this.emitEvents(filteredEvents, triggeredByUserAction) - } else { - // Update indexes for all events - if (filteredEventsBySyncStatus.length > 0) { - this.updateIndexes(filteredEventsBySyncStatus) - } - // Emit all events if no pending sync transactions - this.emitEvents(filteredEventsBySyncStatus, triggeredByUserAction) - } - } - - /** - * Calculate the current size based on synced data and optimistic changes - */ - private calculateSize(): number { - const syncedSize = this.syncedData.size - const deletesFromSynced = Array.from(this.optimisticDeletes).filter( - (key) => this.syncedData.has(key) && !this.optimisticUpserts.has(key) - ).length - const upsertsNotInSynced = Array.from(this.optimisticUpserts.keys()).filter( - (key) => !this.syncedData.has(key) - ).length - - return syncedSize - deletesFromSynced + upsertsNotInSynced + this._lifecycle.onFirstReadyCallbacks.push(callback) } /** - * Collect events for optimistic changes - */ - private collectOptimisticChanges( - previousUpserts: Map, - previousDeletes: Set, - events: Array> - ): void { - const allKeys = new Set([ - ...previousUpserts.keys(), - ...this.optimisticUpserts.keys(), - ...previousDeletes, - ...this.optimisticDeletes, - ]) - - for (const key of allKeys) { - const currentValue = this.get(key) - const previousValue = this.getPreviousValue( - key, - previousUpserts, - previousDeletes - ) - - if (previousValue !== undefined && currentValue === undefined) { - events.push({ type: `delete`, key, value: previousValue }) - } else if (previousValue === undefined && currentValue !== undefined) { - events.push({ type: `insert`, key, value: currentValue }) - } else if ( - previousValue !== undefined && - currentValue !== undefined && - previousValue !== currentValue - ) { - events.push({ - type: `update`, - key, - value: currentValue, - previousValue, - }) - } - } - } - - /** - * Get the previous value for a key given previous optimistic state + * Check if the collection is ready for use + * Returns true if the collection has been marked as ready by its sync implementation + * @returns true if the collection is ready, false otherwise + * @example + * if (collection.isReady()) { + * console.log('Collection is ready, data is available') + * // Safe to access collection.state + * } else { + * console.log('Collection is still loading') + * } */ - private getPreviousValue( - key: TKey, - previousUpserts: Map, - previousDeletes: Set - ): TOutput | undefined { - if (previousDeletes.has(key)) { - return undefined - } - if (previousUpserts.has(key)) { - return previousUpserts.get(key) - } - return this.syncedData.get(key) + public isReady(): boolean { + return this._lifecycle.status === `ready` } /** - * Emit an empty ready event to notify subscribers that the collection is ready - * This bypasses the normal empty array check in emitEvents + * Start sync immediately - internal method for compiled queries + * This bypasses lazy loading for special cases like live query results */ - private emitEmptyReadyEvent(): void { - // Emit empty array directly to all listeners - for (const listener of this.changeListeners) { - listener([]) - } - // Emit to key-specific listeners - for (const [_key, keyListeners] of this.changeKeyListeners) { - for (const listener of keyListeners) { - listener([]) - } - } + public startSyncImmediate(): void { + this._sync.startSync() } /** - * Emit events either immediately or batch them for later emission + * Preload the collection data by starting sync if not already started + * Multiple concurrent calls will share the same promise */ - private emitEvents( - changes: Array>, - forceEmit = false - ): void { - // Skip batching for user actions (forceEmit=true) to keep UI responsive - if (this.shouldBatchEvents && !forceEmit) { - // Add events to the batch - this.batchedEvents.push(...changes) - return - } - - // Either we're not batching, or we're forcing emission (user action or ending batch cycle) - let eventsToEmit = changes - - // If we have batched events and this is a forced emit, combine them - if (this.batchedEvents.length > 0 && forceEmit) { - eventsToEmit = [...this.batchedEvents, ...changes] - this.batchedEvents = [] - this.shouldBatchEvents = false - } - - if (eventsToEmit.length === 0) return - - // Emit to all listeners - for (const listener of this.changeListeners) { - listener(eventsToEmit) - } - - // Emit to key-specific listeners - if (this.changeKeyListeners.size > 0) { - // Group changes by key, but only for keys that have listeners - const changesByKey = new Map>>() - for (const change of eventsToEmit) { - if (this.changeKeyListeners.has(change.key)) { - if (!changesByKey.has(change.key)) { - changesByKey.set(change.key, []) - } - changesByKey.get(change.key)!.push(change) - } - } - - // Emit batched changes to each key's listeners - for (const [key, keyChanges] of changesByKey) { - const keyListeners = this.changeKeyListeners.get(key)! - for (const listener of keyListeners) { - listener(keyChanges) - } - } - } + public preload(): Promise { + return this._sync.preload() } /** * Get the current value for a key (virtual derived state) */ public get(key: TKey): TOutput | undefined { + const { optimisticDeletes, optimisticUpserts, syncedData } = this._state // Check if optimistically deleted - if (this.optimisticDeletes.has(key)) { + if (optimisticDeletes.has(key)) { return undefined } // Check optimistic upserts first - if (this.optimisticUpserts.has(key)) { - return this.optimisticUpserts.get(key) + if (optimisticDeletes.has(key)) { + return optimisticUpserts.get(key) } // Fall back to synced data - return this.syncedData.get(key) + return syncedData.get(key) } /** * Check if a key exists in the collection (virtual derived state) */ public has(key: TKey): boolean { + const { optimisticDeletes, optimisticUpserts, syncedData } = this._state // Check if optimistically deleted - if (this.optimisticDeletes.has(key)) { + if (optimisticDeletes.has(key)) { return false } // Check optimistic upserts first - if (this.optimisticUpserts.has(key)) { + if (optimisticUpserts.has(key)) { return true } // Fall back to synced data - return this.syncedData.has(key) + return syncedData.has(key) } /** * Get the current size of the collection (cached) */ public get size(): number { - return this._size + return this._state.size } /** * Get all keys (virtual derived state) */ public *keys(): IterableIterator { + const { syncedData, optimisticDeletes, optimisticUpserts } = this._state // Yield keys from synced data, skipping any that are deleted. - for (const key of this.syncedData.keys()) { - if (!this.optimisticDeletes.has(key)) { + for (const key of syncedData.keys()) { + if (!optimisticDeletes.has(key)) { yield key } } // Yield keys from upserts that were not already in synced data. - for (const key of this.optimisticUpserts.keys()) { - if (!this.syncedData.has(key) && !this.optimisticDeletes.has(key)) { + for (const key of optimisticUpserts.keys()) { + if (!syncedData.has(key) && !optimisticDeletes.has(key)) { // The optimisticDeletes check is technically redundant if inserts/updates always remove from deletes, // but it's safer to keep it. yield key @@ -1116,415 +444,10 @@ export class CollectionImpl< return result } - /** - * Attempts to commit pending synced transactions if there are no active transactions - * This method processes operations from pending transactions and applies them to the synced data - */ - commitPendingTransactions = () => { - // Check if there are any persisting transaction - let hasPersistingTransaction = false - for (const transaction of this.transactions.values()) { - if (transaction.state === `persisting`) { - hasPersistingTransaction = true - break - } - } - - // pending synced transactions could be either `committed` or still open. - // we only want to process `committed` transactions here - const { - committedSyncedTransactions, - uncommittedSyncedTransactions, - hasTruncateSync, - } = this.pendingSyncedTransactions.reduce( - (acc, t) => { - if (t.committed) { - acc.committedSyncedTransactions.push(t) - if (t.truncate === true) { - acc.hasTruncateSync = true - } - } else { - acc.uncommittedSyncedTransactions.push(t) - } - return acc - }, - { - committedSyncedTransactions: [] as Array< - PendingSyncedTransaction - >, - uncommittedSyncedTransactions: [] as Array< - PendingSyncedTransaction - >, - hasTruncateSync: false, - } - ) - - if (!hasPersistingTransaction || hasTruncateSync) { - // Set flag to prevent redundant optimistic state recalculations - this.isCommittingSyncTransactions = true - - // First collect all keys that will be affected by sync operations - const changedKeys = new Set() - for (const transaction of committedSyncedTransactions) { - for (const operation of transaction.operations) { - changedKeys.add(operation.key as TKey) - } - } - - // Use pre-captured state if available (from optimistic scenarios), - // otherwise capture current state (for pure sync scenarios) - let currentVisibleState = this.preSyncVisibleState - if (currentVisibleState.size === 0) { - // No pre-captured state, capture it now for pure sync operations - currentVisibleState = new Map() - for (const key of changedKeys) { - const currentValue = this.get(key) - if (currentValue !== undefined) { - currentVisibleState.set(key, currentValue) - } - } - } - - const events: Array> = [] - const rowUpdateMode = this.config.sync.rowUpdateMode || `partial` - - for (const transaction of committedSyncedTransactions) { - // Handle truncate operations first - if (transaction.truncate) { - // TRUNCATE PHASE - // 1) Emit a delete for every currently-synced key so downstream listeners/indexes - // observe a clear-before-rebuild. We intentionally skip keys already in - // optimisticDeletes because their delete was previously emitted by the user. - for (const key of this.syncedData.keys()) { - if (this.optimisticDeletes.has(key)) continue - const previousValue = - this.optimisticUpserts.get(key) || this.syncedData.get(key) - if (previousValue !== undefined) { - events.push({ type: `delete`, key, value: previousValue }) - } - } - - // 2) Clear the authoritative synced base. Subsequent server ops in this - // same commit will rebuild the base atomically. - this.syncedData.clear() - this.syncedMetadata.clear() - this.syncedKeys.clear() - - // 3) Clear currentVisibleState for truncated keys to ensure subsequent operations - // are compared against the post-truncate state (undefined) rather than pre-truncate state - // This ensures that re-inserted keys are emitted as INSERT events, not UPDATE events - for (const key of changedKeys) { - currentVisibleState.delete(key) - } - } - - for (const operation of transaction.operations) { - const key = operation.key as TKey - this.syncedKeys.add(key) - - // Update metadata - switch (operation.type) { - case `insert`: - this.syncedMetadata.set(key, operation.metadata) - break - case `update`: - this.syncedMetadata.set( - key, - Object.assign( - {}, - this.syncedMetadata.get(key), - operation.metadata - ) - ) - break - case `delete`: - this.syncedMetadata.delete(key) - break - } - - // Update synced data - switch (operation.type) { - case `insert`: - this.syncedData.set(key, operation.value) - break - case `update`: { - if (rowUpdateMode === `partial`) { - const updatedValue = Object.assign( - {}, - this.syncedData.get(key), - operation.value - ) - this.syncedData.set(key, updatedValue) - } else { - this.syncedData.set(key, operation.value) - } - break - } - case `delete`: - this.syncedData.delete(key) - break - } - } - } - - // After applying synced operations, if this commit included a truncate, - // re-apply optimistic mutations on top of the fresh synced base. This ensures - // the UI preserves local intent while respecting server rebuild semantics. - // Ordering: deletes (above) -> server ops (just applied) -> optimistic upserts. - if (hasTruncateSync) { - // Avoid duplicating keys that were inserted/updated by synced operations in this commit - const syncedInsertedOrUpdatedKeys = new Set() - for (const t of committedSyncedTransactions) { - for (const op of t.operations) { - if (op.type === `insert` || op.type === `update`) { - syncedInsertedOrUpdatedKeys.add(op.key as TKey) - } - } - } - - // Build re-apply sets from ACTIVE optimistic transactions against the new synced base - // We do not copy maps; we compute intent directly from transactions to avoid drift. - const reapplyUpserts = new Map() - const reapplyDeletes = new Set() - - for (const tx of this.transactions.values()) { - if ([`completed`, `failed`].includes(tx.state)) continue - for (const mutation of tx.mutations) { - if (mutation.collection !== this || !mutation.optimistic) continue - const key = mutation.key as TKey - switch (mutation.type) { - case `insert`: - reapplyUpserts.set(key, mutation.modified as TOutput) - reapplyDeletes.delete(key) - break - case `update`: { - const base = this.syncedData.get(key) - const next = base - ? (Object.assign({}, base, mutation.changes) as TOutput) - : (mutation.modified as TOutput) - reapplyUpserts.set(key, next) - reapplyDeletes.delete(key) - break - } - case `delete`: - reapplyUpserts.delete(key) - reapplyDeletes.add(key) - break - } - } - } - - // Emit inserts for re-applied upserts, skipping any keys that have an optimistic delete. - // If the server also inserted/updated the same key in this batch, override that value - // with the optimistic value to preserve local intent. - for (const [key, value] of reapplyUpserts) { - if (reapplyDeletes.has(key)) continue - if (syncedInsertedOrUpdatedKeys.has(key)) { - let foundInsert = false - for (let i = events.length - 1; i >= 0; i--) { - const evt = events[i]! - if (evt.key === key && evt.type === `insert`) { - evt.value = value - foundInsert = true - break - } - } - if (!foundInsert) { - events.push({ type: `insert`, key, value }) - } - } else { - events.push({ type: `insert`, key, value }) - } - } - - // Finally, ensure we do NOT insert keys that have an outstanding optimistic delete. - if (events.length > 0 && reapplyDeletes.size > 0) { - const filtered: Array> = [] - for (const evt of events) { - if (evt.type === `insert` && reapplyDeletes.has(evt.key)) { - continue - } - filtered.push(evt) - } - events.length = 0 - events.push(...filtered) - } - - // Ensure listeners are active before emitting this critical batch - if (!this.isReady()) { - this.setStatus(`ready`) - } - } - - // Maintain optimistic state appropriately - // Clear optimistic state since sync operations will now provide the authoritative data. - // Any still-active user transactions will be re-applied below in recompute. - this.optimisticUpserts.clear() - this.optimisticDeletes.clear() - - // Reset flag and recompute optimistic state for any remaining active transactions - this.isCommittingSyncTransactions = false - for (const transaction of this.transactions.values()) { - if (![`completed`, `failed`].includes(transaction.state)) { - for (const mutation of transaction.mutations) { - if (mutation.collection === this && mutation.optimistic) { - switch (mutation.type) { - case `insert`: - case `update`: - this.optimisticUpserts.set( - mutation.key, - mutation.modified as TOutput - ) - this.optimisticDeletes.delete(mutation.key) - break - case `delete`: - this.optimisticUpserts.delete(mutation.key) - this.optimisticDeletes.add(mutation.key) - break - } - } - } - } - } - - // Check for redundant sync operations that match completed optimistic operations - const completedOptimisticOps = new Map() - - for (const transaction of this.transactions.values()) { - if (transaction.state === `completed`) { - for (const mutation of transaction.mutations) { - if (mutation.collection === this && changedKeys.has(mutation.key)) { - completedOptimisticOps.set(mutation.key, { - type: mutation.type, - value: mutation.modified, - }) - } - } - } - } - - // Now check what actually changed in the final visible state - for (const key of changedKeys) { - const previousVisibleValue = currentVisibleState.get(key) - const newVisibleValue = this.get(key) // This returns the new derived state - - // Check if this sync operation is redundant with a completed optimistic operation - const completedOp = completedOptimisticOps.get(key) - const isRedundantSync = - completedOp && - newVisibleValue !== undefined && - deepEquals(completedOp.value, newVisibleValue) - - if (!isRedundantSync) { - if ( - previousVisibleValue === undefined && - newVisibleValue !== undefined - ) { - events.push({ - type: `insert`, - key, - value: newVisibleValue, - }) - } else if ( - previousVisibleValue !== undefined && - newVisibleValue === undefined - ) { - events.push({ - type: `delete`, - key, - value: previousVisibleValue, - }) - } else if ( - previousVisibleValue !== undefined && - newVisibleValue !== undefined && - !deepEquals(previousVisibleValue, newVisibleValue) - ) { - events.push({ - type: `update`, - key, - value: newVisibleValue, - previousValue: previousVisibleValue, - }) - } - } - } - - // Update cached size after synced data changes - this._size = this.calculateSize() - - // Update indexes for all events before emitting - if (events.length > 0) { - this.updateIndexes(events) - } - - // End batching and emit all events (combines any batched events with sync events) - this.emitEvents(events, true) - - this.pendingSyncedTransactions = uncommittedSyncedTransactions - - // Clear the pre-sync state since sync operations are complete - this.preSyncVisibleState.clear() - - // Clear recently synced keys after a microtask to allow recomputeOptimisticState to see them - Promise.resolve().then(() => { - this.recentlySyncedKeys.clear() - }) - - // Call any registered one-time commit listeners - if (!this.hasReceivedFirstCommit) { - this.hasReceivedFirstCommit = true - const callbacks = [...this.onFirstReadyCallbacks] - this.onFirstReadyCallbacks = [] - callbacks.forEach((callback) => callback()) - } - } - } - - /** - * Schedule cleanup of a transaction when it completes - * @private - */ - private scheduleTransactionCleanup(transaction: Transaction): void { - // Only schedule cleanup for transactions that aren't already completed - if (transaction.state === `completed`) { - this.transactions.delete(transaction.id) - return - } - - // Schedule cleanup when the transaction completes - transaction.isPersisted.promise - .then(() => { - // Transaction completed successfully, remove it immediately - this.transactions.delete(transaction.id) - }) - .catch(() => { - // Transaction failed, but we want to keep failed transactions for reference - // so don't remove it. - // This empty catch block is necessary to prevent unhandled promise rejections. - }) - } - - private ensureStandardSchema(schema: unknown): StandardSchema { - // If the schema already implements the standard-schema interface, return it - if (schema && `~standard` in (schema as {})) { - return schema as StandardSchema - } - - throw new InvalidSchemaError() - } - public getKeyFromItem(item: TOutput): TKey { return this.config.getKey(item) } - public generateGlobalKey(key: any, item: any): string { - if (typeof key === `undefined`) { - throw new UndefinedKeyError(item) - } - - return `KEY::${this.id}/${key}` - } - /** * Creates an index on a collection for faster queries. * Indexes significantly improve query performance by allowing constant time lookups @@ -1559,196 +482,25 @@ export class CollectionImpl< indexCallback: (row: SingleRowRefProxy) => any, config: IndexOptions = {} ): IndexProxy { - this.validateCollectionUsable(`createIndex`) - - const indexId = ++this.indexCounter - const singleRowRefProxy = createSingleRowRefProxy() - const indexExpression = indexCallback(singleRowRefProxy) - const expression = toExpression(indexExpression) - - // Default to BTreeIndex if no type specified - const resolver = config.indexType ?? (BTreeIndex as unknown as TResolver) - - // Create lazy wrapper - const lazyIndex = new LazyIndexWrapper( - indexId, - expression, - config.name, - resolver, - config.options, - this.entries() - ) - - this.lazyIndexes.set(indexId, lazyIndex) - - // For BTreeIndex, resolve immediately and synchronously - if ((resolver as unknown) === BTreeIndex) { - try { - const resolvedIndex = lazyIndex.getResolved() - this.resolvedIndexes.set(indexId, resolvedIndex) - } catch (error) { - console.warn(`Failed to resolve BTreeIndex:`, error) - } - } else if (typeof resolver === `function` && resolver.prototype) { - // Other synchronous constructors - resolve immediately - try { - const resolvedIndex = lazyIndex.getResolved() - this.resolvedIndexes.set(indexId, resolvedIndex) - } catch { - // Fallback to async resolution - this.resolveSingleIndex(indexId, lazyIndex).catch((error) => { - console.warn(`Failed to resolve single index:`, error) - }) - } - } else if (this.isIndexesResolved) { - // Async loader but indexes are already resolved - resolve this one - this.resolveSingleIndex(indexId, lazyIndex).catch((error) => { - console.warn(`Failed to resolve single index:`, error) - }) - } - - return new IndexProxy(indexId, lazyIndex) - } - - /** - * Resolve all lazy indexes (called when collection first syncs) - * @private - */ - private async resolveAllIndexes(): Promise { - if (this.isIndexesResolved) return - - const resolutionPromises = Array.from(this.lazyIndexes.entries()).map( - async ([indexId, lazyIndex]) => { - const resolvedIndex = await lazyIndex.resolve() - - // Build index with current data - resolvedIndex.build(this.entries()) - - this.resolvedIndexes.set(indexId, resolvedIndex) - return { indexId, resolvedIndex } - } - ) - - await Promise.all(resolutionPromises) - this.isIndexesResolved = true - } - - /** - * Resolve a single index immediately - * @private - */ - private async resolveSingleIndex( - indexId: number, - lazyIndex: LazyIndexWrapper - ): Promise> { - const resolvedIndex = await lazyIndex.resolve() - resolvedIndex.build(this.entries()) - this.resolvedIndexes.set(indexId, resolvedIndex) - return resolvedIndex + return this._indexes.createIndex(indexCallback, config) } /** * Get resolved indexes for query optimization */ get indexes(): Map> { - return this.resolvedIndexes + return this._indexes.indexes } /** - * Updates all indexes when the collection changes - * @private + * Validates the data against the schema */ - private updateIndexes(changes: Array>): void { - for (const index of this.resolvedIndexes.values()) { - for (const change of changes) { - switch (change.type) { - case `insert`: - index.add(change.key, change.value) - break - case `update`: - if (change.previousValue) { - index.update(change.key, change.previousValue, change.value) - } else { - index.add(change.key, change.value) - } - break - case `delete`: - index.remove(change.key, change.value) - break - } - } - } - } - public validateData( data: unknown, type: `insert` | `update`, key?: TKey ): TOutput | never { - if (!this.config.schema) return data as TOutput - - const standardSchema = this.ensureStandardSchema(this.config.schema) - - // For updates, we need to merge with the existing data before validation - if (type === `update` && key) { - // Get the existing data for this key - const existingData = this.get(key) - - if ( - existingData && - data && - typeof data === `object` && - typeof existingData === `object` - ) { - // Merge the update with the existing data - const mergedData = Object.assign({}, existingData, data) - - // Validate the merged data - const result = standardSchema[`~standard`].validate(mergedData) - - // Ensure validation is synchronous - if (result instanceof Promise) { - throw new SchemaMustBeSynchronousError() - } - - // If validation fails, throw a SchemaValidationError with the issues - if (`issues` in result && result.issues) { - const typedIssues = result.issues.map((issue) => ({ - message: issue.message, - path: issue.path?.map((p) => String(p)), - })) - throw new SchemaValidationError(type, typedIssues) - } - - // Extract only the modified keys from the validated result - const validatedMergedData = result.value as TOutput - const modifiedKeys = Object.keys(data) - const extractedChanges = Object.fromEntries( - modifiedKeys.map((k) => [k, validatedMergedData[k as keyof TOutput]]) - ) as TOutput - - return extractedChanges - } - } - - // For inserts or updates without existing data, validate the data directly - const result = standardSchema[`~standard`].validate(data) - - // Ensure validation is synchronous - if (result instanceof Promise) { - throw new SchemaMustBeSynchronousError() - } - - // If validation fails, throw a SchemaValidationError with the issues - if (`issues` in result && result.issues) { - const typedIssues = result.issues.map((issue) => ({ - message: issue.message, - path: issue.path?.map((p) => String(p)), - })) - throw new SchemaValidationError(type, typedIssues) - } - - return result.value as TOutput + return this._mutations.validateData(data, type, key) } /** @@ -1788,92 +540,7 @@ export class CollectionImpl< * } */ insert = (data: TInput | Array, config?: InsertConfig) => { - this.validateCollectionUsable(`insert`) - const ambientTransaction = getActiveTransaction() - - // If no ambient transaction exists, check for an onInsert handler early - if (!ambientTransaction && !this.config.onInsert) { - throw new MissingInsertHandlerError() - } - - const items = Array.isArray(data) ? data : [data] - const mutations: Array> = [] - - // Create mutations for each item - items.forEach((item) => { - // Validate the data against the schema if one exists - const validatedData = this.validateData(item, `insert`) - - // Check if an item with this ID already exists in the collection - const key = this.getKeyFromItem(validatedData) - if (this.has(key)) { - throw new DuplicateKeyError(key) - } - const globalKey = this.generateGlobalKey(key, item) - - const mutation: PendingMutation = { - mutationId: crypto.randomUUID(), - original: {}, - modified: validatedData, - // Pick the values from validatedData based on what's passed in - this is for cases - // where a schema has default values. The validated data has the extra default - // values but for changes, we just want to show the data that was actually passed in. - changes: Object.fromEntries( - Object.keys(item).map((k) => [ - k, - validatedData[k as keyof typeof validatedData], - ]) - ) as TInput, - globalKey, - key, - metadata: config?.metadata as unknown, - syncMetadata: this.config.sync.getSyncMetadata?.() || {}, - optimistic: config?.optimistic ?? true, - type: `insert`, - createdAt: new Date(), - updatedAt: new Date(), - collection: this, - } - - mutations.push(mutation) - }) - - // If an ambient transaction exists, use it - if (ambientTransaction) { - ambientTransaction.applyMutations(mutations) - - this.transactions.set(ambientTransaction.id, ambientTransaction) - this.scheduleTransactionCleanup(ambientTransaction) - this.recomputeOptimisticState(true) - - return ambientTransaction - } else { - // Create a new transaction with a mutation function that calls the onInsert handler - const directOpTransaction = createTransaction({ - mutationFn: async (params) => { - // Call the onInsert handler with the transaction and collection - return await this.config.onInsert!({ - transaction: - params.transaction as unknown as TransactionWithMutations< - TOutput, - `insert` - >, - collection: this as unknown as Collection, - }) - }, - }) - - // Apply mutations to the new transaction - directOpTransaction.applyMutations(mutations) - directOpTransaction.commit() - - // Add the transaction to the collection's transactions store - this.transactions.set(directOpTransaction.id, directOpTransaction) - this.scheduleTransactionCleanup(directOpTransaction) - this.recomputeOptimisticState(true) - - return directOpTransaction - } + return this._mutations.insert(data, config) } /** @@ -1952,171 +619,7 @@ export class CollectionImpl< | ((draft: WritableDeep) => void) | ((drafts: Array>) => void) ) { - if (typeof keys === `undefined`) { - throw new MissingUpdateArgumentError() - } - - this.validateCollectionUsable(`update`) - - const ambientTransaction = getActiveTransaction() - - // If no ambient transaction exists, check for an onUpdate handler early - if (!ambientTransaction && !this.config.onUpdate) { - throw new MissingUpdateHandlerError() - } - - const isArray = Array.isArray(keys) - const keysArray = isArray ? keys : [keys] - - if (isArray && keysArray.length === 0) { - throw new NoKeysPassedToUpdateError() - } - - const callback = - typeof configOrCallback === `function` ? configOrCallback : maybeCallback! - const config = - typeof configOrCallback === `function` ? {} : configOrCallback - - // Get the current objects or empty objects if they don't exist - const currentObjects = keysArray.map((key) => { - const item = this.get(key) - if (!item) { - throw new UpdateKeyNotFoundError(key) - } - - return item - }) as unknown as Array - - let changesArray - if (isArray) { - // Use the proxy to track changes for all objects - changesArray = withArrayChangeTracking( - currentObjects, - callback as (draft: Array) => void - ) - } else { - const result = withChangeTracking( - currentObjects[0]!, - callback as (draft: TInput) => void - ) - changesArray = [result] - } - - // Create mutations for each object that has changes - const mutations: Array> = keysArray - .map((key, index) => { - const itemChanges = changesArray[index] // User-provided changes for this specific item - - // Skip items with no changes - if (!itemChanges || Object.keys(itemChanges).length === 0) { - return null - } - - const originalItem = currentObjects[index] as unknown as TOutput - // Validate the user-provided changes for this item - const validatedUpdatePayload = this.validateData( - itemChanges, - `update`, - key - ) - - // Construct the full modified item by applying the validated update payload to the original item - const modifiedItem = Object.assign( - {}, - originalItem, - validatedUpdatePayload - ) - - // Check if the ID of the item is being changed - const originalItemId = this.getKeyFromItem(originalItem) - const modifiedItemId = this.getKeyFromItem(modifiedItem) - - if (originalItemId !== modifiedItemId) { - throw new KeyUpdateNotAllowedError(originalItemId, modifiedItemId) - } - - const globalKey = this.generateGlobalKey(modifiedItemId, modifiedItem) - - return { - mutationId: crypto.randomUUID(), - original: originalItem, - modified: modifiedItem, - // Pick the values from modifiedItem based on what's passed in - this is for cases - // where a schema has default values or transforms. The modified data has the extra - // default or transformed values but for changes, we just want to show the data that - // was actually passed in. - changes: Object.fromEntries( - Object.keys(itemChanges).map((k) => [ - k, - modifiedItem[k as keyof typeof modifiedItem], - ]) - ) as TInput, - globalKey, - key, - metadata: config.metadata as unknown, - syncMetadata: (this.syncedMetadata.get(key) || {}) as Record< - string, - unknown - >, - optimistic: config.optimistic ?? true, - type: `update`, - createdAt: new Date(), - updatedAt: new Date(), - collection: this, - } - }) - .filter(Boolean) as Array> - - // If no changes were made, return an empty transaction early - if (mutations.length === 0) { - const emptyTransaction = createTransaction({ - mutationFn: async () => {}, - }) - emptyTransaction.commit() - // Schedule cleanup for empty transaction - this.scheduleTransactionCleanup(emptyTransaction) - return emptyTransaction - } - - // If an ambient transaction exists, use it - if (ambientTransaction) { - ambientTransaction.applyMutations(mutations) - - this.transactions.set(ambientTransaction.id, ambientTransaction) - this.scheduleTransactionCleanup(ambientTransaction) - this.recomputeOptimisticState(true) - - return ambientTransaction - } - - // No need to check for onUpdate handler here as we've already checked at the beginning - - // Create a new transaction with a mutation function that calls the onUpdate handler - const directOpTransaction = createTransaction({ - mutationFn: async (params) => { - // Call the onUpdate handler with the transaction and collection - return this.config.onUpdate!({ - transaction: - params.transaction as unknown as TransactionWithMutations< - TOutput, - `update` - >, - collection: this as unknown as Collection, - }) - }, - }) - - // Apply mutations to the new transaction - directOpTransaction.applyMutations(mutations) - directOpTransaction.commit() - - // Add the transaction to the collection's transactions store - - this.transactions.set(directOpTransaction.id, directOpTransaction) - this.scheduleTransactionCleanup(directOpTransaction) - this.recomputeOptimisticState(true) - - return directOpTransaction + return this._mutations.update(keys, configOrCallback, maybeCallback) } /** @@ -2153,85 +656,7 @@ export class CollectionImpl< keys: Array | TKey, config?: OperationConfig ): TransactionType => { - this.validateCollectionUsable(`delete`) - - const ambientTransaction = getActiveTransaction() - - // If no ambient transaction exists, check for an onDelete handler early - if (!ambientTransaction && !this.config.onDelete) { - throw new MissingDeleteHandlerError() - } - - if (Array.isArray(keys) && keys.length === 0) { - throw new NoKeysPassedToDeleteError() - } - - const keysArray = Array.isArray(keys) ? keys : [keys] - const mutations: Array> = [] - - for (const key of keysArray) { - if (!this.has(key)) { - throw new DeleteKeyNotFoundError(key) - } - const globalKey = this.generateGlobalKey(key, this.get(key)!) - const mutation: PendingMutation = { - mutationId: crypto.randomUUID(), - original: this.get(key)!, - modified: this.get(key)!, - changes: this.get(key)!, - globalKey, - key, - metadata: config?.metadata as unknown, - syncMetadata: (this.syncedMetadata.get(key) || {}) as Record< - string, - unknown - >, - optimistic: config?.optimistic ?? true, - type: `delete`, - createdAt: new Date(), - updatedAt: new Date(), - collection: this, - } - - mutations.push(mutation) - } - - // If an ambient transaction exists, use it - if (ambientTransaction) { - ambientTransaction.applyMutations(mutations) - - this.transactions.set(ambientTransaction.id, ambientTransaction) - this.scheduleTransactionCleanup(ambientTransaction) - this.recomputeOptimisticState(true) - - return ambientTransaction - } - - // Create a new transaction with a mutation function that calls the onDelete handler - const directOpTransaction = createTransaction({ - autoCommit: true, - mutationFn: async (params) => { - // Call the onDelete handler with the transaction and collection - return this.config.onDelete!({ - transaction: - params.transaction as unknown as TransactionWithMutations< - TOutput, - `delete` - >, - collection: this as unknown as Collection, - }) - }, - }) - - // Apply mutations to the new transaction - directOpTransaction.applyMutations(mutations) - directOpTransaction.commit() - - this.transactions.set(directOpTransaction.id, directOpTransaction) - this.scheduleTransactionCleanup(directOpTransaction) - this.recomputeOptimisticState(true) - - return directOpTransaction + return this._mutations.delete(keys, config) } /** @@ -2374,36 +799,7 @@ export class CollectionImpl< callback: (changes: Array>) => void, options: SubscribeChangesOptions = {} ): () => void { - // Start sync and track subscriber - this.addSubscriber() - - // Auto-index for where expressions if enabled - if (options.whereExpression) { - ensureIndexForExpression(options.whereExpression, this) - } - - // Create a filtered callback if where clause is provided - const filteredCallback = - options.where || options.whereExpression - ? createFilteredCallback(callback, options) - : callback - - if (options.includeInitialState) { - // First send the current state as changes (filtered if needed) - const initialChanges = this.currentStateAsChanges({ - where: options.where, - whereExpression: options.whereExpression, - }) - filteredCallback(initialChanges) - } - - // Add to batched listeners - this.changeListeners.add(filteredCallback) - - return () => { - this.changeListeners.delete(filteredCallback) - this.removeSubscriber() - } + return this._changes.subscribeChanges(callback, options) } /** @@ -2414,83 +810,22 @@ export class CollectionImpl< listener: ChangeListener, { includeInitialState = false }: { includeInitialState?: boolean } = {} ): () => void { - // Start sync and track subscriber - this.addSubscriber() - - if (!this.changeKeyListeners.has(key)) { - this.changeKeyListeners.set(key, new Set()) - } - - if (includeInitialState) { - // First send the current state as changes - listener([ - { - type: `insert`, - key, - value: this.get(key)!, - }, - ]) - } - - this.changeKeyListeners.get(key)!.add(listener) - - return () => { - const listeners = this.changeKeyListeners.get(key) - if (listeners) { - listeners.delete(listener) - if (listeners.size === 0) { - this.changeKeyListeners.delete(key) - } - } - this.removeSubscriber() - } - } - - /** - * Capture visible state for keys that will be affected by pending sync operations - * This must be called BEFORE onTransactionStateChange clears optimistic state - */ - private capturePreSyncVisibleState(): void { - if (this.pendingSyncedTransactions.length === 0) return - - // Clear any previous capture - this.preSyncVisibleState.clear() - - // Get all keys that will be affected by sync operations - const syncedKeys = new Set() - for (const transaction of this.pendingSyncedTransactions) { - for (const operation of transaction.operations) { - syncedKeys.add(operation.key as TKey) - } - } - - // Mark keys as about to be synced to suppress intermediate events from recomputeOptimisticState - for (const key of syncedKeys) { - this.recentlySyncedKeys.add(key) - } - - // Only capture current visible state for keys that will be affected by sync operations - // This is much more efficient than capturing the entire collection state - for (const key of syncedKeys) { - const currentValue = this.get(key) - if (currentValue !== undefined) { - this.preSyncVisibleState.set(key, currentValue) - } - } + return this._changes.subscribeChangesKey(key, listener, { + includeInitialState, + }) } /** - * Trigger a recomputation when transactions change - * This method should be called by the Transaction class when state changes + * Clean up the collection by stopping sync and clearing data + * This can be called manually or automatically by garbage collection */ - public onTransactionStateChange(): void { - // Check if commitPendingTransactions will be called after this - // by checking if there are pending sync transactions (same logic as in transactions.ts) - this.shouldBatchEvents = this.pendingSyncedTransactions.length > 0 - - // CRITICAL: Capture visible state BEFORE clearing optimistic state - this.capturePreSyncVisibleState() - - this.recomputeOptimisticState(false) + public async cleanup(): Promise { + await Promise.all([ + this._state.cleanup(), + this._changes.cleanup(), + this._lifecycle.cleanup(), + this._sync.cleanup(), + this._indexes.cleanup(), + ]) } } diff --git a/packages/db/src/collection/changes.ts b/packages/db/src/collection/changes.ts new file mode 100644 index 000000000..8de3d76f6 --- /dev/null +++ b/packages/db/src/collection/changes.ts @@ -0,0 +1,221 @@ +import { StandardSchemaV1 } from "@standard-schema/spec" +import { CollectionImpl } from "../collection" +import { + ChangeListener, + ChangeMessage, + SubscribeChangesOptions, +} from "../types" +import { ensureIndexForExpression } from "../indexes/auto-index.js" +import { createFilteredCallback } from "../change-events" +import { NegativeActiveSubscribersError } from "../errors" + +export class CollectionChangesManager< + TOutput extends object = Record, + TKey extends string | number = string | number, + TSchema extends StandardSchemaV1 = StandardSchemaV1, + TInput extends object = TOutput, +> { + public activeSubscribersCount = 0 + public changeListeners = new Set>() + public changeKeyListeners = new Map< + TKey, + Set> + >() + public batchedEvents: Array> = [] + public shouldBatchEvents = false + + /** + * Creates a new CollectionChangesManager instance + */ + constructor( + public collection: CollectionImpl + ) {} + + /** + * Emit an empty ready event to notify subscribers that the collection is ready + * This bypasses the normal empty array check in emitEvents + */ + public emitEmptyReadyEvent(): void { + // Emit empty array directly to all listeners + for (const listener of this.changeListeners) { + listener([]) + } + // Emit to key-specific listeners + for (const [_key, keyListeners] of this.changeKeyListeners) { + for (const listener of keyListeners) { + listener([]) + } + } + } + + /** + * Emit events either immediately or batch them for later emission + */ + public emitEvents( + changes: Array>, + forceEmit = false + ): void { + // Skip batching for user actions (forceEmit=true) to keep UI responsive + if (this.shouldBatchEvents && !forceEmit) { + // Add events to the batch + this.batchedEvents.push(...changes) + return + } + + // Either we're not batching, or we're forcing emission (user action or ending batch cycle) + let eventsToEmit = changes + + // If we have batched events and this is a forced emit, combine them + if (this.batchedEvents.length > 0 && forceEmit) { + eventsToEmit = [...this.batchedEvents, ...changes] + this.batchedEvents = [] + this.shouldBatchEvents = false + } + + if (eventsToEmit.length === 0) return + + // Emit to all listeners + for (const listener of this.changeListeners) { + listener(eventsToEmit) + } + + // Emit to key-specific listeners + if (this.changeKeyListeners.size > 0) { + // Group changes by key, but only for keys that have listeners + const changesByKey = new Map>>() + for (const change of eventsToEmit) { + if (this.changeKeyListeners.has(change.key)) { + if (!changesByKey.has(change.key)) { + changesByKey.set(change.key, []) + } + changesByKey.get(change.key)!.push(change) + } + } + + // Emit batched changes to each key's listeners + for (const [key, keyChanges] of changesByKey) { + const keyListeners = this.changeKeyListeners.get(key)! + for (const listener of keyListeners) { + listener(keyChanges) + } + } + } + } + + /** + * Subscribe to changes in the collection + */ + public subscribeChanges( + callback: (changes: Array>) => void, + options: SubscribeChangesOptions = {} + ): () => void { + // Start sync and track subscriber + this.addSubscriber() + + // Auto-index for where expressions if enabled + if (options.whereExpression) { + ensureIndexForExpression(options.whereExpression, this.collection) + } + + // Create a filtered callback if where clause is provided + const filteredCallback = + options.where || options.whereExpression + ? createFilteredCallback(callback, options) + : callback + + if (options.includeInitialState) { + // First send the current state as changes (filtered if needed) + const initialChanges = this.collection.currentStateAsChanges({ + where: options.where, + whereExpression: options.whereExpression, + }) + filteredCallback(initialChanges) + } + + // Add to batched listeners + this.changeListeners.add(filteredCallback) + + return () => { + this.changeListeners.delete(filteredCallback) + this.removeSubscriber() + } + } + + /** + * Subscribe to changes for a specific key + */ + public subscribeChangesKey( + key: TKey, + listener: ChangeListener, + { includeInitialState = false }: { includeInitialState?: boolean } = {} + ): () => void { + // Start sync and track subscriber + this.addSubscriber() + + if (!this.changeKeyListeners.has(key)) { + this.changeKeyListeners.set(key, new Set()) + } + + if (includeInitialState) { + // First send the current state as changes + listener([ + { + type: `insert`, + key, + value: this.collection.get(key)!, + }, + ]) + } + + this.changeKeyListeners.get(key)!.add(listener) + + return () => { + const listeners = this.changeKeyListeners.get(key) + if (listeners) { + listeners.delete(listener) + if (listeners.size === 0) { + this.changeKeyListeners.delete(key) + } + } + this.removeSubscriber() + } + } + + /** + * Increment the active subscribers count and start sync if needed + */ + private addSubscriber(): void { + this.activeSubscribersCount++ + this.collection._lifecycle.cancelGCTimer() + + // Start sync if collection was cleaned up + if ( + this.collection.status === `cleaned-up` || + this.collection.status === `idle` + ) { + this.collection._sync.startSync() + } + } + + /** + * Decrement the active subscribers count and start GC timer if needed + */ + private removeSubscriber(): void { + this.activeSubscribersCount-- + + if (this.activeSubscribersCount === 0) { + this.collection._lifecycle.startGCTimer() + } else if (this.activeSubscribersCount < 0) { + throw new NegativeActiveSubscribersError() + } + } + + /** + * Clean up the collection by stopping sync and clearing data + * This can be called manually or automatically by garbage collection + */ + public async cleanup(): Promise { + this.batchedEvents = [] + this.shouldBatchEvents = false + } +} diff --git a/packages/db/src/collection/indexes.ts b/packages/db/src/collection/indexes.ts new file mode 100644 index 000000000..6d3264420 --- /dev/null +++ b/packages/db/src/collection/indexes.ts @@ -0,0 +1,162 @@ +import { StandardSchemaV1 } from "@standard-schema/spec" +import { CollectionImpl } from "../collection" +import { IndexProxy, LazyIndexWrapper } from "../indexes/lazy-index" +import { BaseIndex, IndexResolver } from "../indexes/base-index" +import { ChangeMessage } from "../types" +import { IndexOptions } from "../indexes/index-options" +import { + createSingleRowRefProxy, + SingleRowRefProxy, + toExpression, +} from "../query/builder/ref-proxy" +import { BTreeIndex } from "../indexes/btree-index" + +export class CollectionIndexesManager< + TOutput extends object = Record, + TKey extends string | number = string | number, + TSchema extends StandardSchemaV1 = StandardSchemaV1, + TInput extends object = TOutput, +> { + public lazyIndexes = new Map>() + public resolvedIndexes = new Map>() + public isIndexesResolved = false + public indexCounter = 0 + + constructor( + public collection: CollectionImpl + ) {} + + /** + * Creates an index on a collection for faster queries. + */ + public createIndex = typeof BTreeIndex>( + indexCallback: (row: SingleRowRefProxy) => any, + config: IndexOptions = {} + ): IndexProxy { + this.collection._lifecycle.validateCollectionUsable(`createIndex`) + + const indexId = ++this.indexCounter + const singleRowRefProxy = createSingleRowRefProxy() + const indexExpression = indexCallback(singleRowRefProxy) + const expression = toExpression(indexExpression) + + // Default to BTreeIndex if no type specified + const resolver = config.indexType ?? (BTreeIndex as unknown as TResolver) + + // Create lazy wrapper + const lazyIndex = new LazyIndexWrapper( + indexId, + expression, + config.name, + resolver, + config.options, + this.collection.entries() + ) + + this.lazyIndexes.set(indexId, lazyIndex) + + // For BTreeIndex, resolve immediately and synchronously + if ((resolver as unknown) === BTreeIndex) { + try { + const resolvedIndex = lazyIndex.getResolved() + this.resolvedIndexes.set(indexId, resolvedIndex) + } catch (error) { + console.warn(`Failed to resolve BTreeIndex:`, error) + } + } else if (typeof resolver === `function` && resolver.prototype) { + // Other synchronous constructors - resolve immediately + try { + const resolvedIndex = lazyIndex.getResolved() + this.resolvedIndexes.set(indexId, resolvedIndex) + } catch { + // Fallback to async resolution + this.resolveSingleIndex(indexId, lazyIndex).catch((error) => { + console.warn(`Failed to resolve single index:`, error) + }) + } + } else if (this.isIndexesResolved) { + // Async loader but indexes are already resolved - resolve this one + this.resolveSingleIndex(indexId, lazyIndex).catch((error) => { + console.warn(`Failed to resolve single index:`, error) + }) + } + + return new IndexProxy(indexId, lazyIndex) + } + + /** + * Resolve all lazy indexes (called when collection first syncs) + */ + public async resolveAllIndexes(): Promise { + if (this.isIndexesResolved) return + + const resolutionPromises = Array.from(this.lazyIndexes.entries()).map( + async ([indexId, lazyIndex]) => { + const resolvedIndex = await lazyIndex.resolve() + + // Build index with current data + resolvedIndex.build(this.collection.entries()) + + this.resolvedIndexes.set(indexId, resolvedIndex) + return { indexId, resolvedIndex } + } + ) + + await Promise.all(resolutionPromises) + this.isIndexesResolved = true + } + + /** + * Resolve a single index immediately + */ + private async resolveSingleIndex( + indexId: number, + lazyIndex: LazyIndexWrapper + ): Promise> { + const resolvedIndex = await lazyIndex.resolve() + resolvedIndex.build(this.collection.entries()) + this.resolvedIndexes.set(indexId, resolvedIndex) + return resolvedIndex + } + + /** + * Get resolved indexes for query optimization + */ + get indexes(): Map> { + return this.resolvedIndexes + } + + /** + * Updates all indexes when the collection changes + */ + public updateIndexes(changes: Array>): void { + for (const index of this.resolvedIndexes.values()) { + for (const change of changes) { + switch (change.type) { + case `insert`: + index.add(change.key, change.value) + break + case `update`: + if (change.previousValue) { + index.update(change.key, change.previousValue, change.value) + } else { + index.add(change.key, change.value) + } + break + case `delete`: + index.remove(change.key, change.value) + break + } + } + } + } + + /** + * Clean up the collection by stopping sync and clearing data + * This can be called manually or automatically by garbage collection + */ + public async cleanup(): Promise { + this.lazyIndexes.clear() + this.resolvedIndexes.clear() + } +} diff --git a/packages/db/src/collection/lifecycle.ts b/packages/db/src/collection/lifecycle.ts new file mode 100644 index 000000000..13aea1f43 --- /dev/null +++ b/packages/db/src/collection/lifecycle.ts @@ -0,0 +1,172 @@ +import { StandardSchemaV1 } from "@standard-schema/spec" +import { CollectionImpl } from "../collection" +import { + InvalidCollectionStatusTransitionError, + CollectionInErrorStateError, +} from "../errors" +import type { CollectionStatus } from "../types" + +export class CollectionLifecycleManager< + TOutput extends object = Record, + TKey extends string | number = string | number, + TSchema extends StandardSchemaV1 = StandardSchemaV1, + TInput extends object = TOutput, +> { + public status: CollectionStatus = `idle` + public hasBeenReady = false + public hasReceivedFirstCommit = false + public onFirstReadyCallbacks: Array<() => void> = [] + + private gcTimeoutId: ReturnType | null = null + + /** + * Creates a new CollectionLifecycleManager instance + */ + constructor( + public collection: CollectionImpl + ) {} + + /** + * Validates state transitions to prevent invalid status changes + * @private + */ + private validateStatusTransition( + from: CollectionStatus, + to: CollectionStatus + ): void { + if (from === to) { + // Allow same state transitions + return + } + const validTransitions: Record< + CollectionStatus, + Array + > = { + idle: [`loading`, `error`, `cleaned-up`], + loading: [`initialCommit`, `ready`, `error`, `cleaned-up`], + initialCommit: [`ready`, `error`, `cleaned-up`], + ready: [`cleaned-up`, `error`], + error: [`cleaned-up`, `idle`], + "cleaned-up": [`loading`, `error`], + } + + if (!validTransitions[from].includes(to)) { + throw new InvalidCollectionStatusTransitionError( + from, + to, + this.collection.id + ) + } + } + + /** + * Safely update the collection status with validation + * @private + */ + public setStatus(newStatus: CollectionStatus): void { + this.validateStatusTransition(this.status, newStatus) + this.status = newStatus + + // Resolve indexes when collection becomes ready + if (newStatus === `ready` && !this.collection._indexes.isIndexesResolved) { + // Resolve indexes asynchronously without blocking + this.collection._indexes.resolveAllIndexes().catch((error) => { + console.warn(`Failed to resolve indexes:`, error) + }) + } + } + + /** + * Validates that the collection is in a usable state for data operations + * @private + */ + public validateCollectionUsable(operation: string): void { + switch (this.status) { + case `error`: + throw new CollectionInErrorStateError(operation, this.collection.id) + case `cleaned-up`: + // Automatically restart the collection when operations are called on cleaned-up collections + this.collection._sync.startSync() + break + } + } + + /** + * Mark the collection as ready for use + * This is called by sync implementations to explicitly signal that the collection is ready, + * providing a more intuitive alternative to using commits for readiness signaling + * @private - Should only be called by sync implementations + */ + public markReady(): void { + // Can transition to ready from loading or initialCommit states + if (this.status === `loading` || this.status === `initialCommit`) { + this.setStatus(`ready`) + + // Call any registered first ready callbacks (only on first time becoming ready) + if (!this.hasBeenReady) { + this.hasBeenReady = true + + // Also mark as having received first commit for backwards compatibility + if (!this.hasReceivedFirstCommit) { + this.hasReceivedFirstCommit = true + } + + const callbacks = [...this.onFirstReadyCallbacks] + this.onFirstReadyCallbacks = [] + callbacks.forEach((callback) => callback()) + } + } + + // Always notify dependents when markReady is called, after status is set + // This ensures live queries get notified when their dependencies become ready + if (this.collection._changes.changeListeners.size > 0) { + this.collection._changes.emitEmptyReadyEvent() + } + } + + /** + * Start the garbage collection timer + * Called when the collection becomes inactive (no subscribers) + */ + public startGCTimer(): void { + if (this.gcTimeoutId) { + clearTimeout(this.gcTimeoutId) + } + + const gcTime = this.collection.config.gcTime ?? 300000 // 5 minutes default + + // If gcTime is 0, GC is disabled + if (gcTime === 0) { + return + } + + this.gcTimeoutId = setTimeout(() => { + if (this.collection._changes.activeSubscribersCount === 0) { + this.cleanup() + } + }, gcTime) + } + + /** + * Cancel the garbage collection timer + * Called when the collection becomes active again + */ + public cancelGCTimer(): void { + if (this.gcTimeoutId) { + clearTimeout(this.gcTimeoutId) + this.gcTimeoutId = null + } + } + + public cleanup(): Promise { + if (this.gcTimeoutId) { + clearTimeout(this.gcTimeoutId) + this.gcTimeoutId = null + } + + this.hasBeenReady = false + this.hasReceivedFirstCommit = false + + return Promise.resolve() + } +} diff --git a/packages/db/src/collection/mutations.ts b/packages/db/src/collection/mutations.ts new file mode 100644 index 000000000..6c623e452 --- /dev/null +++ b/packages/db/src/collection/mutations.ts @@ -0,0 +1,517 @@ +import { withArrayChangeTracking, withChangeTracking } from "../proxy" +import { createTransaction, getActiveTransaction } from "../transactions" +import { + DeleteKeyNotFoundError, + DuplicateKeyError, + InvalidSchemaError, + KeyUpdateNotAllowedError, + MissingDeleteHandlerError, + MissingInsertHandlerError, + MissingUpdateArgumentError, + MissingUpdateHandlerError, + NoKeysPassedToDeleteError, + NoKeysPassedToUpdateError, + SchemaMustBeSynchronousError, + SchemaValidationError, + UndefinedKeyError, + UpdateKeyNotFoundError, +} from "../errors" +import type { StandardSchemaV1 } from "@standard-schema/spec" +import type { + InsertConfig, + OperationConfig, + PendingMutation, + StandardSchema, + Transaction as TransactionType, + TransactionWithMutations, + UtilsRecord, + WritableDeep, +} from "../types" +import { CollectionImpl, type Collection } from "../collection" + +export class CollectionMutationsManager< + TOutput extends object = Record, + TKey extends string | number = string | number, + TUtils extends UtilsRecord = {}, + TSchema extends StandardSchemaV1 = StandardSchemaV1, + TInput extends object = TOutput, +> { + constructor( + public collection: CollectionImpl + ) {} + + private ensureStandardSchema(schema: unknown): StandardSchema { + // If the schema already implements the standard-schema interface, return it + if (schema && `~standard` in (schema as {})) { + return schema as StandardSchema + } + + throw new InvalidSchemaError() + } + + public validateData( + data: unknown, + type: `insert` | `update`, + key?: TKey + ): TOutput | never { + if (!this.collection.config.schema) return data as TOutput + + const standardSchema = this.ensureStandardSchema( + this.collection.config.schema + ) + + // For updates, we need to merge with the existing data before validation + if (type === `update` && key) { + // Get the existing data for this key + const existingData = this.collection.get(key) + + if ( + existingData && + data && + typeof data === `object` && + typeof existingData === `object` + ) { + // Merge the update with the existing data + const mergedData = Object.assign({}, existingData, data) + + // Validate the merged data + const result = standardSchema[`~standard`].validate(mergedData) + + // Ensure validation is synchronous + if (result instanceof Promise) { + throw new SchemaMustBeSynchronousError() + } + + // If validation fails, throw a SchemaValidationError with the issues + if (`issues` in result && result.issues) { + const typedIssues = result.issues.map((issue) => ({ + message: issue.message, + path: issue.path?.map((p) => String(p)), + })) + throw new SchemaValidationError(type, typedIssues) + } + + // Extract only the modified keys from the validated result + const validatedMergedData = result.value as TOutput + const modifiedKeys = Object.keys(data) + const extractedChanges = Object.fromEntries( + modifiedKeys.map((k) => [k, validatedMergedData[k as keyof TOutput]]) + ) as TOutput + + return extractedChanges + } + } + + // For inserts or updates without existing data, validate the data directly + const result = standardSchema[`~standard`].validate(data) + + // Ensure validation is synchronous + if (result instanceof Promise) { + throw new SchemaMustBeSynchronousError() + } + + // If validation fails, throw a SchemaValidationError with the issues + if (`issues` in result && result.issues) { + const typedIssues = result.issues.map((issue) => ({ + message: issue.message, + path: issue.path?.map((p) => String(p)), + })) + throw new SchemaValidationError(type, typedIssues) + } + + return result.value as TOutput + } + + public generateGlobalKey(key: any, item: any): string { + if (typeof key === `undefined`) { + throw new UndefinedKeyError(item) + } + + return `KEY::${this.collection.id}/${key}` + } + + /** + * Inserts one or more items into the collection + */ + insert = (data: TInput | Array, config?: InsertConfig) => { + this.collection._lifecycle.validateCollectionUsable(`insert`) + const _state = this.collection._state + const ambientTransaction = getActiveTransaction() + + // If no ambient transaction exists, check for an onInsert handler early + if (!ambientTransaction && !this.collection.config.onInsert) { + throw new MissingInsertHandlerError() + } + + const items = Array.isArray(data) ? data : [data] + const mutations: Array> = [] + + // Create mutations for each item + items.forEach((item) => { + // Validate the data against the schema if one exists + const validatedData = this.validateData(item, `insert`) + + // Check if an item with this ID already exists in the collection + const key = this.collection.getKeyFromItem(validatedData) + if (this.collection.has(key)) { + throw new DuplicateKeyError(key) + } + const globalKey = this.generateGlobalKey(key, item) + + const mutation: PendingMutation = { + mutationId: crypto.randomUUID(), + original: {}, + modified: validatedData, + // Pick the values from validatedData based on what's passed in - this is for cases + // where a schema has default values. The validated data has the extra default + // values but for changes, we just want to show the data that was actually passed in. + changes: Object.fromEntries( + Object.keys(item).map((k) => [ + k, + validatedData[k as keyof typeof validatedData], + ]) + ) as TInput, + globalKey, + key, + metadata: config?.metadata as unknown, + syncMetadata: this.collection.config.sync.getSyncMetadata?.() || {}, + optimistic: config?.optimistic ?? true, + type: `insert`, + createdAt: new Date(), + updatedAt: new Date(), + collection: this.collection, + } + + mutations.push(mutation) + }) + + // If an ambient transaction exists, use it + if (ambientTransaction) { + ambientTransaction.applyMutations(mutations) + + _state.transactions.set(ambientTransaction.id, ambientTransaction) + _state.scheduleTransactionCleanup(ambientTransaction) + _state.recomputeOptimisticState(true) + + return ambientTransaction + } else { + // Create a new transaction with a mutation function that calls the onInsert handler + const directOpTransaction = createTransaction({ + mutationFn: async (params) => { + // Call the onInsert handler with the transaction and collection + return await this.collection.config.onInsert!({ + transaction: + params.transaction as unknown as TransactionWithMutations< + TOutput, + `insert` + >, + collection: this.collection as unknown as Collection, + }) + }, + }) + + // Apply mutations to the new transaction + directOpTransaction.applyMutations(mutations) + directOpTransaction.commit() + + // Add the transaction to the collection's transactions store + _state.transactions.set(directOpTransaction.id, directOpTransaction) + _state.scheduleTransactionCleanup(directOpTransaction) + _state.recomputeOptimisticState(true) + + return directOpTransaction + } + } + + /** + * Updates one or more items in the collection using a callback function + */ + update( + keys: (TKey | unknown) | Array, + configOrCallback: + | ((draft: WritableDeep) => void) + | ((drafts: Array>) => void) + | OperationConfig, + maybeCallback?: + | ((draft: WritableDeep) => void) + | ((drafts: Array>) => void) + ) { + if (typeof keys === `undefined`) { + throw new MissingUpdateArgumentError() + } + + const _state = this.collection._state + this.collection._lifecycle.validateCollectionUsable(`update`) + + const ambientTransaction = getActiveTransaction() + + // If no ambient transaction exists, check for an onUpdate handler early + if (!ambientTransaction && !this.collection.config.onUpdate) { + throw new MissingUpdateHandlerError() + } + + const isArray = Array.isArray(keys) + const keysArray = isArray ? keys : [keys] + + if (isArray && keysArray.length === 0) { + throw new NoKeysPassedToUpdateError() + } + + const callback = + typeof configOrCallback === `function` ? configOrCallback : maybeCallback! + const config = + typeof configOrCallback === `function` ? {} : configOrCallback + + // Get the current objects or empty objects if they don't exist + const currentObjects = keysArray.map((key) => { + const item = this.collection.get(key) + if (!item) { + throw new UpdateKeyNotFoundError(key) + } + + return item + }) as unknown as Array + + let changesArray + if (isArray) { + // Use the proxy to track changes for all objects + changesArray = withArrayChangeTracking( + currentObjects, + callback as (draft: Array) => void + ) + } else { + const result = withChangeTracking( + currentObjects[0]!, + callback as (draft: TInput) => void + ) + changesArray = [result] + } + + // Create mutations for each object that has changes + const mutations: Array< + PendingMutation< + TOutput, + `update`, + CollectionImpl + > + > = keysArray + .map((key, index) => { + const itemChanges = changesArray[index] // User-provided changes for this specific item + + // Skip items with no changes + if (!itemChanges || Object.keys(itemChanges).length === 0) { + return null + } + + const originalItem = currentObjects[index] as unknown as TOutput + // Validate the user-provided changes for this item + const validatedUpdatePayload = this.validateData( + itemChanges, + `update`, + key + ) + + // Construct the full modified item by applying the validated update payload to the original item + const modifiedItem = Object.assign( + {}, + originalItem, + validatedUpdatePayload + ) + + // Check if the ID of the item is being changed + const originalItemId = this.collection.getKeyFromItem(originalItem) + const modifiedItemId = this.collection.getKeyFromItem(modifiedItem) + + if (originalItemId !== modifiedItemId) { + throw new KeyUpdateNotAllowedError(originalItemId, modifiedItemId) + } + + const globalKey = this.generateGlobalKey(modifiedItemId, modifiedItem) + + return { + mutationId: crypto.randomUUID(), + original: originalItem, + modified: modifiedItem, + // Pick the values from modifiedItem based on what's passed in - this is for cases + // where a schema has default values or transforms. The modified data has the extra + // default or transformed values but for changes, we just want to show the data that + // was actually passed in. + changes: Object.fromEntries( + Object.keys(itemChanges).map((k) => [ + k, + modifiedItem[k as keyof typeof modifiedItem], + ]) + ) as TInput, + globalKey, + key, + metadata: config.metadata as unknown, + syncMetadata: (_state.syncedMetadata.get(key) || {}) as Record< + string, + unknown + >, + optimistic: config.optimistic ?? true, + type: `update`, + createdAt: new Date(), + updatedAt: new Date(), + collection: this.collection, + } + }) + .filter(Boolean) as Array< + PendingMutation< + TOutput, + `update`, + CollectionImpl + > + > + + // If no changes were made, return an empty transaction early + if (mutations.length === 0) { + const emptyTransaction = createTransaction({ + mutationFn: async () => {}, + }) + emptyTransaction.commit() + // Schedule cleanup for empty transaction + _state.scheduleTransactionCleanup(emptyTransaction) + return emptyTransaction + } + + // If an ambient transaction exists, use it + if (ambientTransaction) { + ambientTransaction.applyMutations(mutations) + + _state.transactions.set(ambientTransaction.id, ambientTransaction) + _state.scheduleTransactionCleanup(ambientTransaction) + _state.recomputeOptimisticState(true) + + return ambientTransaction + } + + // No need to check for onUpdate handler here as we've already checked at the beginning + + // Create a new transaction with a mutation function that calls the onUpdate handler + const directOpTransaction = createTransaction({ + mutationFn: async (params) => { + // Call the onUpdate handler with the transaction and collection + return this.collection.config.onUpdate!({ + transaction: + params.transaction as unknown as TransactionWithMutations< + TOutput, + `update` + >, + collection: this.collection as unknown as Collection, + }) + }, + }) + + // Apply mutations to the new transaction + directOpTransaction.applyMutations(mutations) + directOpTransaction.commit() + + // Add the transaction to the collection's transactions store + + _state.transactions.set(directOpTransaction.id, directOpTransaction) + _state.scheduleTransactionCleanup(directOpTransaction) + _state.recomputeOptimisticState(true) + + return directOpTransaction + } + + /** + * Deletes one or more items from the collection + */ + delete = ( + keys: Array | TKey, + config?: OperationConfig + ): TransactionType => { + const _state = this.collection._state + this.collection._lifecycle.validateCollectionUsable(`delete`) + + const ambientTransaction = getActiveTransaction() + + // If no ambient transaction exists, check for an onDelete handler early + if (!ambientTransaction && !this.collection.config.onDelete) { + throw new MissingDeleteHandlerError() + } + + if (Array.isArray(keys) && keys.length === 0) { + throw new NoKeysPassedToDeleteError() + } + + const keysArray = Array.isArray(keys) ? keys : [keys] + const mutations: Array< + PendingMutation< + TOutput, + `delete`, + CollectionImpl + > + > = [] + + for (const key of keysArray) { + if (!this.collection.has(key)) { + throw new DeleteKeyNotFoundError(key) + } + const globalKey = this.generateGlobalKey(key, this.collection.get(key)!) + const mutation: PendingMutation< + TOutput, + `delete`, + CollectionImpl + > = { + mutationId: crypto.randomUUID(), + original: this.collection.get(key)!, + modified: this.collection.get(key)!, + changes: this.collection.get(key)!, + globalKey, + key, + metadata: config?.metadata as unknown, + syncMetadata: (_state.syncedMetadata.get(key) || {}) as Record< + string, + unknown + >, + optimistic: config?.optimistic ?? true, + type: `delete`, + createdAt: new Date(), + updatedAt: new Date(), + collection: this.collection, + } + + mutations.push(mutation) + } + + // If an ambient transaction exists, use it + if (ambientTransaction) { + ambientTransaction.applyMutations(mutations) + + _state.transactions.set(ambientTransaction.id, ambientTransaction) + _state.scheduleTransactionCleanup(ambientTransaction) + _state.recomputeOptimisticState(true) + + return ambientTransaction + } + + // Create a new transaction with a mutation function that calls the onDelete handler + const directOpTransaction = createTransaction({ + autoCommit: true, + mutationFn: async (params) => { + // Call the onDelete handler with the transaction and collection + return this.collection.config.onDelete!({ + transaction: + params.transaction as unknown as TransactionWithMutations< + TOutput, + `delete` + >, + collection: this.collection as unknown as Collection, + }) + }, + }) + + // Apply mutations to the new transaction + directOpTransaction.applyMutations(mutations) + directOpTransaction.commit() + + _state.transactions.set(directOpTransaction.id, directOpTransaction) + _state.scheduleTransactionCleanup(directOpTransaction) + _state.recomputeOptimisticState(true) + + return directOpTransaction + } +} diff --git a/packages/db/src/collection/state.ts b/packages/db/src/collection/state.ts new file mode 100644 index 000000000..3de8e7987 --- /dev/null +++ b/packages/db/src/collection/state.ts @@ -0,0 +1,720 @@ +import { deepEquals } from "../utils" +import { SortedMap } from "../SortedMap" +import type { Transaction } from "../transactions" +import type { StandardSchemaV1 } from "@standard-schema/spec" +import type { ChangeMessage, OptimisticChangeMessage } from "../types" +import type { CollectionImpl } from "../collection" + +interface PendingSyncedTransaction> { + committed: boolean + operations: Array> + truncate?: boolean + deletedKeys: Set +} + +export class CollectionStateManager< + TOutput extends object = Record, + TKey extends string | number = string | number, + TSchema extends StandardSchemaV1 = StandardSchemaV1, + TInput extends object = TOutput, +> { + public collection: CollectionImpl + + // Core state - make public for testing + public transactions: SortedMap> + public pendingSyncedTransactions: Array> = + [] + public syncedData: Map | SortedMap + public syncedMetadata = new Map() + + // Optimistic state tracking - make public for testing + public optimisticUpserts = new Map() + public optimisticDeletes = new Set() + + // Cached size for performance + public size = 0 + + // State used for computing the change events + private syncedKeys = new Set() + private preSyncVisibleState = new Map() + private recentlySyncedKeys = new Set() + private hasReceivedFirstCommit = false + private isCommittingSyncTransactions = false + + /** + * Creates a new CollectionState manager + */ + constructor(collection: CollectionImpl) { + this.collection = collection + + this.transactions = new SortedMap>((a, b) => + a.compareCreatedAt(b) + ) + + // Set up data storage with optional comparison function + if (collection.config.compare) { + this.syncedData = new SortedMap(collection.config.compare) + } else { + this.syncedData = new Map() + } + } + + /** + * Recompute optimistic state from active transactions + */ + public recomputeOptimisticState( + triggeredByUserAction: boolean = false + ): void { + // Skip redundant recalculations when we're in the middle of committing sync transactions + if (this.isCommittingSyncTransactions) { + return + } + + const previousState = new Map(this.optimisticUpserts) + const previousDeletes = new Set(this.optimisticDeletes) + + // Clear current optimistic state + this.optimisticUpserts.clear() + this.optimisticDeletes.clear() + + const activeTransactions: Array> = [] + + for (const transaction of this.transactions.values()) { + if (![`completed`, `failed`].includes(transaction.state)) { + activeTransactions.push(transaction) + } + } + + // Apply active transactions only (completed transactions are handled by sync operations) + for (const transaction of activeTransactions) { + for (const mutation of transaction.mutations) { + if (mutation.collection === this.collection && mutation.optimistic) { + switch (mutation.type) { + case `insert`: + case `update`: + this.optimisticUpserts.set( + mutation.key, + mutation.modified as TOutput + ) + this.optimisticDeletes.delete(mutation.key) + break + case `delete`: + this.optimisticUpserts.delete(mutation.key) + this.optimisticDeletes.add(mutation.key) + break + } + } + } + } + + // Update cached size + this.size = this.calculateSize() + + // Collect events for changes + const events: Array> = [] + this.collectOptimisticChanges(previousState, previousDeletes, events) + + // Filter out events for recently synced keys to prevent duplicates + // BUT: Only filter out events that are actually from sync operations + // New user transactions should NOT be filtered even if the key was recently synced + const filteredEventsBySyncStatus = events.filter((event) => { + if (!this.recentlySyncedKeys.has(event.key)) { + return true // Key not recently synced, allow event through + } + + // Key was recently synced - allow if this is a user-triggered action + if (triggeredByUserAction) { + return true + } + + // Otherwise filter out duplicate sync events + return false + }) + + // Filter out redundant delete events if there are pending sync transactions + // that will immediately restore the same data, but only for completed transactions + // IMPORTANT: Skip complex filtering for user-triggered actions to prevent UI blocking + if (this.pendingSyncedTransactions.length > 0 && !triggeredByUserAction) { + const pendingSyncKeys = new Set() + + // Collect keys from pending sync operations + for (const transaction of this.pendingSyncedTransactions) { + for (const operation of transaction.operations) { + pendingSyncKeys.add(operation.key as TKey) + } + } + + // Only filter out delete events for keys that: + // 1. Have pending sync operations AND + // 2. Are from completed transactions (being cleaned up) + const filteredEvents = filteredEventsBySyncStatus.filter((event) => { + if (event.type === `delete` && pendingSyncKeys.has(event.key)) { + // Check if this delete is from clearing optimistic state of completed transactions + // We can infer this by checking if we have no remaining optimistic mutations for this key + const hasActiveOptimisticMutation = activeTransactions.some((tx) => + tx.mutations.some( + (m) => m.collection === this.collection && m.key === event.key + ) + ) + + if (!hasActiveOptimisticMutation) { + return false // Skip this delete event as sync will restore the data + } + } + return true + }) + + // Update indexes for the filtered events + if (filteredEvents.length > 0) { + this.collection._indexes.updateIndexes(filteredEvents) + } + this.collection._changes.emitEvents(filteredEvents, triggeredByUserAction) + } else { + // Update indexes for all events + if (filteredEventsBySyncStatus.length > 0) { + this.collection._indexes.updateIndexes(filteredEventsBySyncStatus) + } + // Emit all events if no pending sync transactions + this.collection._changes.emitEvents( + filteredEventsBySyncStatus, + triggeredByUserAction + ) + } + } + + /** + * Calculate the current size based on synced data and optimistic changes + */ + private calculateSize(): number { + const syncedSize = this.syncedData.size + const deletesFromSynced = Array.from(this.optimisticDeletes).filter( + (key) => this.syncedData.has(key) && !this.optimisticUpserts.has(key) + ).length + const upsertsNotInSynced = Array.from(this.optimisticUpserts.keys()).filter( + (key) => !this.syncedData.has(key) + ).length + + return syncedSize - deletesFromSynced + upsertsNotInSynced + } + + /** + * Collect events for optimistic changes + */ + private collectOptimisticChanges( + previousUpserts: Map, + previousDeletes: Set, + events: Array> + ): void { + const allKeys = new Set([ + ...previousUpserts.keys(), + ...this.optimisticUpserts.keys(), + ...previousDeletes, + ...this.optimisticDeletes, + ]) + + for (const key of allKeys) { + const currentValue = this.collection.get(key) + const previousValue = this.getPreviousValue( + key, + previousUpserts, + previousDeletes + ) + + if (previousValue !== undefined && currentValue === undefined) { + events.push({ type: `delete`, key, value: previousValue }) + } else if (previousValue === undefined && currentValue !== undefined) { + events.push({ type: `insert`, key, value: currentValue }) + } else if ( + previousValue !== undefined && + currentValue !== undefined && + previousValue !== currentValue + ) { + events.push({ + type: `update`, + key, + value: currentValue, + previousValue, + }) + } + } + } + + /** + * Get the previous value for a key given previous optimistic state + */ + private getPreviousValue( + key: TKey, + previousUpserts: Map, + previousDeletes: Set + ): TOutput | undefined { + if (previousDeletes.has(key)) { + return undefined + } + if (previousUpserts.has(key)) { + return previousUpserts.get(key) + } + return this.syncedData.get(key) + } + + /** + * Attempts to commit pending synced transactions if there are no active transactions + * This method processes operations from pending transactions and applies them to the synced data + */ + commitPendingTransactions = () => { + // Check if there are any persisting transaction + let hasPersistingTransaction = false + for (const transaction of this.transactions.values()) { + if (transaction.state === `persisting`) { + hasPersistingTransaction = true + break + } + } + + // pending synced transactions could be either `committed` or still open. + // we only want to process `committed` transactions here + const { + committedSyncedTransactions, + uncommittedSyncedTransactions, + hasTruncateSync, + } = this.pendingSyncedTransactions.reduce( + (acc, t) => { + if (t.committed) { + acc.committedSyncedTransactions.push(t) + if (t.truncate === true) { + acc.hasTruncateSync = true + } + } else { + acc.uncommittedSyncedTransactions.push(t) + } + return acc + }, + { + committedSyncedTransactions: [] as Array< + PendingSyncedTransaction + >, + uncommittedSyncedTransactions: [] as Array< + PendingSyncedTransaction + >, + hasTruncateSync: false, + } + ) + + if (!hasPersistingTransaction || hasTruncateSync) { + // Set flag to prevent redundant optimistic state recalculations + this.isCommittingSyncTransactions = true + + // First collect all keys that will be affected by sync operations + const changedKeys = new Set() + for (const transaction of committedSyncedTransactions) { + for (const operation of transaction.operations) { + changedKeys.add(operation.key as TKey) + } + } + + // Use pre-captured state if available (from optimistic scenarios), + // otherwise capture current state (for pure sync scenarios) + let currentVisibleState = this.preSyncVisibleState + if (currentVisibleState.size === 0) { + // No pre-captured state, capture it now for pure sync operations + currentVisibleState = new Map() + for (const key of changedKeys) { + const currentValue = this.collection.get(key) + if (currentValue !== undefined) { + currentVisibleState.set(key, currentValue) + } + } + } + + const events: Array> = [] + const rowUpdateMode = + this.collection.config.sync.rowUpdateMode || `partial` + + for (const transaction of committedSyncedTransactions) { + // Handle truncate operations first + if (transaction.truncate) { + // TRUNCATE PHASE + // 1) Emit a delete for every currently-synced key so downstream listeners/indexes + // observe a clear-before-rebuild. We intentionally skip keys already in + // optimisticDeletes because their delete was previously emitted by the user. + for (const key of this.syncedData.keys()) { + if (this.optimisticDeletes.has(key)) continue + const previousValue = + this.optimisticUpserts.get(key) || this.syncedData.get(key) + if (previousValue !== undefined) { + events.push({ type: `delete`, key, value: previousValue }) + } + } + + // 2) Clear the authoritative synced base. Subsequent server ops in this + // same commit will rebuild the base atomically. + this.syncedData.clear() + this.syncedMetadata.clear() + this.syncedKeys.clear() + + // 3) Clear currentVisibleState for truncated keys to ensure subsequent operations + // are compared against the post-truncate state (undefined) rather than pre-truncate state + // This ensures that re-inserted keys are emitted as INSERT events, not UPDATE events + for (const key of changedKeys) { + currentVisibleState.delete(key) + } + } + + for (const operation of transaction.operations) { + const key = operation.key as TKey + this.syncedKeys.add(key) + + // Update metadata + switch (operation.type) { + case `insert`: + this.syncedMetadata.set(key, operation.metadata) + break + case `update`: + this.syncedMetadata.set( + key, + Object.assign( + {}, + this.syncedMetadata.get(key), + operation.metadata + ) + ) + break + case `delete`: + this.syncedMetadata.delete(key) + break + } + + // Update synced data + switch (operation.type) { + case `insert`: + this.syncedData.set(key, operation.value) + break + case `update`: { + if (rowUpdateMode === `partial`) { + const updatedValue = Object.assign( + {}, + this.syncedData.get(key), + operation.value + ) + this.syncedData.set(key, updatedValue) + } else { + this.syncedData.set(key, operation.value) + } + break + } + case `delete`: + this.syncedData.delete(key) + break + } + } + } + + // After applying synced operations, if this commit included a truncate, + // re-apply optimistic mutations on top of the fresh synced base. This ensures + // the UI preserves local intent while respecting server rebuild semantics. + // Ordering: deletes (above) -> server ops (just applied) -> optimistic upserts. + if (hasTruncateSync) { + // Avoid duplicating keys that were inserted/updated by synced operations in this commit + const syncedInsertedOrUpdatedKeys = new Set() + for (const t of committedSyncedTransactions) { + for (const op of t.operations) { + if (op.type === `insert` || op.type === `update`) { + syncedInsertedOrUpdatedKeys.add(op.key as TKey) + } + } + } + + // Build re-apply sets from ACTIVE optimistic transactions against the new synced base + // We do not copy maps; we compute intent directly from transactions to avoid drift. + const reapplyUpserts = new Map() + const reapplyDeletes = new Set() + + for (const tx of this.transactions.values()) { + if ([`completed`, `failed`].includes(tx.state)) continue + for (const mutation of tx.mutations) { + if (mutation.collection !== this.collection || !mutation.optimistic) + continue + const key = mutation.key as TKey + switch (mutation.type) { + case `insert`: + reapplyUpserts.set(key, mutation.modified as TOutput) + reapplyDeletes.delete(key) + break + case `update`: { + const base = this.syncedData.get(key) + const next = base + ? (Object.assign({}, base, mutation.changes) as TOutput) + : (mutation.modified as TOutput) + reapplyUpserts.set(key, next) + reapplyDeletes.delete(key) + break + } + case `delete`: + reapplyUpserts.delete(key) + reapplyDeletes.add(key) + break + } + } + } + + // Emit inserts for re-applied upserts, skipping any keys that have an optimistic delete. + // If the server also inserted/updated the same key in this batch, override that value + // with the optimistic value to preserve local intent. + for (const [key, value] of reapplyUpserts) { + if (reapplyDeletes.has(key)) continue + if (syncedInsertedOrUpdatedKeys.has(key)) { + let foundInsert = false + for (let i = events.length - 1; i >= 0; i--) { + const evt = events[i]! + if (evt.key === key && evt.type === `insert`) { + evt.value = value + foundInsert = true + break + } + } + if (!foundInsert) { + events.push({ type: `insert`, key, value }) + } + } else { + events.push({ type: `insert`, key, value }) + } + } + + // Finally, ensure we do NOT insert keys that have an outstanding optimistic delete. + if (events.length > 0 && reapplyDeletes.size > 0) { + const filtered: Array> = [] + for (const evt of events) { + if (evt.type === `insert` && reapplyDeletes.has(evt.key)) { + continue + } + filtered.push(evt) + } + events.length = 0 + events.push(...filtered) + } + + // Ensure listeners are active before emitting this critical batch + if (!this.collection.isReady()) { + this.collection._lifecycle.setStatus(`ready`) + } + } + + // Maintain optimistic state appropriately + // Clear optimistic state since sync operations will now provide the authoritative data. + // Any still-active user transactions will be re-applied below in recompute. + this.optimisticUpserts.clear() + this.optimisticDeletes.clear() + + // Reset flag and recompute optimistic state for any remaining active transactions + this.isCommittingSyncTransactions = false + for (const transaction of this.transactions.values()) { + if (![`completed`, `failed`].includes(transaction.state)) { + for (const mutation of transaction.mutations) { + if ( + mutation.collection === this.collection && + mutation.optimistic + ) { + switch (mutation.type) { + case `insert`: + case `update`: + this.optimisticUpserts.set( + mutation.key, + mutation.modified as TOutput + ) + this.optimisticDeletes.delete(mutation.key) + break + case `delete`: + this.optimisticUpserts.delete(mutation.key) + this.optimisticDeletes.add(mutation.key) + break + } + } + } + } + } + + // Check for redundant sync operations that match completed optimistic operations + const completedOptimisticOps = new Map() + + for (const transaction of this.transactions.values()) { + if (transaction.state === `completed`) { + for (const mutation of transaction.mutations) { + if ( + mutation.collection === this.collection && + changedKeys.has(mutation.key) + ) { + completedOptimisticOps.set(mutation.key, { + type: mutation.type, + value: mutation.modified, + }) + } + } + } + } + + // Now check what actually changed in the final visible state + for (const key of changedKeys) { + const previousVisibleValue = currentVisibleState.get(key) + const newVisibleValue = this.collection.get(key) // This returns the new derived state + + // Check if this sync operation is redundant with a completed optimistic operation + const completedOp = completedOptimisticOps.get(key) + const isRedundantSync = + completedOp && + newVisibleValue !== undefined && + deepEquals(completedOp.value, newVisibleValue) + + if (!isRedundantSync) { + if ( + previousVisibleValue === undefined && + newVisibleValue !== undefined + ) { + events.push({ + type: `insert`, + key, + value: newVisibleValue, + }) + } else if ( + previousVisibleValue !== undefined && + newVisibleValue === undefined + ) { + events.push({ + type: `delete`, + key, + value: previousVisibleValue, + }) + } else if ( + previousVisibleValue !== undefined && + newVisibleValue !== undefined && + !deepEquals(previousVisibleValue, newVisibleValue) + ) { + events.push({ + type: `update`, + key, + value: newVisibleValue, + previousValue: previousVisibleValue, + }) + } + } + } + + // Update cached size after synced data changes + this.size = this.calculateSize() + + // Update indexes for all events before emitting + if (events.length > 0) { + this.collection._indexes.updateIndexes(events) + } + + // End batching and emit all events (combines any batched events with sync events) + this.collection._changes.emitEvents(events, true) + + this.pendingSyncedTransactions = uncommittedSyncedTransactions + + // Clear the pre-sync state since sync operations are complete + this.preSyncVisibleState.clear() + + // Clear recently synced keys after a microtask to allow recomputeOptimisticState to see them + Promise.resolve().then(() => { + this.recentlySyncedKeys.clear() + }) + + // Call any registered one-time commit listeners + if (!this.hasReceivedFirstCommit) { + this.hasReceivedFirstCommit = true + const callbacks = [...this.collection._lifecycle.onFirstReadyCallbacks] + this.collection._lifecycle.onFirstReadyCallbacks = [] + callbacks.forEach((callback) => callback()) + } + } + } + + /** + * Schedule cleanup of a transaction when it completes + */ + public scheduleTransactionCleanup(transaction: Transaction): void { + // Only schedule cleanup for transactions that aren't already completed + if (transaction.state === `completed`) { + this.transactions.delete(transaction.id) + return + } + + // Schedule cleanup when the transaction completes + transaction.isPersisted.promise + .then(() => { + // Transaction completed successfully, remove it immediately + this.transactions.delete(transaction.id) + }) + .catch(() => { + // Transaction failed, but we want to keep failed transactions for reference + // so don't remove it. + // This empty catch block is necessary to prevent unhandled promise rejections. + }) + } + + /** + * Capture visible state for keys that will be affected by pending sync operations + * This must be called BEFORE onTransactionStateChange clears optimistic state + */ + public capturePreSyncVisibleState(): void { + if (this.pendingSyncedTransactions.length === 0) return + + // Clear any previous capture + this.preSyncVisibleState.clear() + + // Get all keys that will be affected by sync operations + const syncedKeys = new Set() + for (const transaction of this.pendingSyncedTransactions) { + for (const operation of transaction.operations) { + syncedKeys.add(operation.key as TKey) + } + } + + // Mark keys as about to be synced to suppress intermediate events from recomputeOptimisticState + for (const key of syncedKeys) { + this.recentlySyncedKeys.add(key) + } + + // Only capture current visible state for keys that will be affected by sync operations + // This is much more efficient than capturing the entire collection state + for (const key of syncedKeys) { + const currentValue = this.collection.get(key) + if (currentValue !== undefined) { + this.preSyncVisibleState.set(key, currentValue) + } + } + } + + /** + * Trigger a recomputation when transactions change + * This method should be called by the Transaction class when state changes + */ + public onTransactionStateChange(): void { + // Check if commitPendingTransactions will be called after this + // by checking if there are pending sync transactions (same logic as in transactions.ts) + this.collection._changes.shouldBatchEvents = + this.pendingSyncedTransactions.length > 0 + + // CRITICAL: Capture visible state BEFORE clearing optimistic state + this.capturePreSyncVisibleState() + + this.recomputeOptimisticState(false) + } + + /** + * Clean up the collection by stopping sync and clearing data + * This can be called manually or automatically by garbage collection + */ + public async cleanup(): Promise { + // Clear data + this.syncedData.clear() + this.syncedMetadata.clear() + this.optimisticUpserts.clear() + this.optimisticDeletes.clear() + this.size = 0 + this.pendingSyncedTransactions = [] + this.syncedKeys.clear() + this.hasReceivedFirstCommit = false + + return Promise.resolve() + } +} diff --git a/packages/db/src/collection/sync.ts b/packages/db/src/collection/sync.ts new file mode 100644 index 000000000..e2f1c1930 --- /dev/null +++ b/packages/db/src/collection/sync.ts @@ -0,0 +1,222 @@ +import { + CollectionIsInErrorStateError, + DuplicateKeySyncError, + NoPendingSyncTransactionCommitError, + NoPendingSyncTransactionWriteError, + SyncCleanupError, + SyncTransactionAlreadyCommittedError, + SyncTransactionAlreadyCommittedWriteError, +} from "../errors" +import type { StandardSchemaV1 } from "@standard-schema/spec" +import type { ChangeMessage } from "../types" +import type { CollectionImpl } from "../collection" + +export class CollectionSyncManager< + TOutput extends object = Record, + TKey extends string | number = string | number, + TSchema extends StandardSchemaV1 = StandardSchemaV1, + TInput extends object = TOutput, +> { + private preloadPromise: Promise | null = null + private syncCleanupFn: (() => void) | null = null + + /** + * Creates a new CollectionSyncManager instance + */ + constructor( + public collection: CollectionImpl + ) {} + + /** + * Start the sync process for this collection + * This is called when the collection is first accessed or preloaded + */ + public startSync(): void { + const _state = this.collection._state + if ( + this.collection.status !== `idle` && + this.collection.status !== `cleaned-up` + ) { + return // Already started or in progress + } + + this.collection._lifecycle.setStatus(`loading`) + + try { + const cleanupFn = this.collection.config.sync.sync({ + collection: this.collection, + begin: () => { + _state.pendingSyncedTransactions.push({ + committed: false, + operations: [], + deletedKeys: new Set(), + }) + }, + write: (messageWithoutKey: Omit, `key`>) => { + const pendingTransaction = + _state.pendingSyncedTransactions[ + _state.pendingSyncedTransactions.length - 1 + ] + if (!pendingTransaction) { + throw new NoPendingSyncTransactionWriteError() + } + if (pendingTransaction.committed) { + throw new SyncTransactionAlreadyCommittedWriteError() + } + const key = this.collection.getKeyFromItem(messageWithoutKey.value) + + // Check if an item with this key already exists when inserting + if (messageWithoutKey.type === `insert`) { + const insertingIntoExistingSynced = _state.syncedData.has(key) + const hasPendingDeleteForKey = + pendingTransaction.deletedKeys.has(key) + const isTruncateTransaction = pendingTransaction.truncate === true + // Allow insert after truncate in the same transaction even if it existed in syncedData + if ( + insertingIntoExistingSynced && + !hasPendingDeleteForKey && + !isTruncateTransaction + ) { + throw new DuplicateKeySyncError(key, this.collection.id) + } + } + + const message: ChangeMessage = { + ...messageWithoutKey, + key, + } + pendingTransaction.operations.push(message) + + if (messageWithoutKey.type === `delete`) { + pendingTransaction.deletedKeys.add(key) + } + }, + commit: () => { + const pendingTransaction = + _state.pendingSyncedTransactions[ + _state.pendingSyncedTransactions.length - 1 + ] + if (!pendingTransaction) { + throw new NoPendingSyncTransactionCommitError() + } + if (pendingTransaction.committed) { + throw new SyncTransactionAlreadyCommittedError() + } + + pendingTransaction.committed = true + + // Update status to initialCommit when transitioning from loading + // This indicates we're in the process of committing the first transaction + if (this.collection.status === `loading`) { + this.collection._lifecycle.setStatus(`initialCommit`) + } + + _state.commitPendingTransactions() + }, + markReady: () => { + this.collection._lifecycle.markReady() + }, + truncate: () => { + const pendingTransaction = + _state.pendingSyncedTransactions[ + _state.pendingSyncedTransactions.length - 1 + ] + if (!pendingTransaction) { + throw new NoPendingSyncTransactionWriteError() + } + if (pendingTransaction.committed) { + throw new SyncTransactionAlreadyCommittedWriteError() + } + + // Clear all operations from the current transaction + pendingTransaction.operations = [] + pendingTransaction.deletedKeys.clear() + + // Mark the transaction as a truncate operation. During commit, this triggers: + // - Delete events for all previously synced keys (excluding optimistic-deleted keys) + // - Clearing of syncedData/syncedMetadata + // - Subsequent synced ops applied on the fresh base + // - Finally, optimistic mutations re-applied on top (single batch) + pendingTransaction.truncate = true + }, + }) + + // Store cleanup function if provided + this.syncCleanupFn = typeof cleanupFn === `function` ? cleanupFn : null + } catch (error) { + this.collection._lifecycle.setStatus(`error`) + throw error + } + } + + /** + * Preload the collection data by starting sync if not already started + * Multiple concurrent calls will share the same promise + */ + public preload(): Promise { + if (this.preloadPromise) { + return this.preloadPromise + } + + this.preloadPromise = new Promise((resolve, reject) => { + if (this.collection.status === `ready`) { + resolve() + return + } + + if (this.collection.status === `error`) { + reject(new CollectionIsInErrorStateError()) + return + } + + // Register callback BEFORE starting sync to avoid race condition + this.collection.onFirstReady(() => { + resolve() + }) + + // Start sync if collection hasn't started yet or was cleaned up + if ( + this.collection.status === `idle` || + this.collection.status === `cleaned-up` + ) { + try { + this.startSync() + } catch (error) { + reject(error) + return + } + } + }) + + return this.preloadPromise + } + + public cleanup(): Promise { + // Stop sync - wrap in try/catch since it's user-provided code + try { + if (this.syncCleanupFn) { + this.syncCleanupFn() + this.syncCleanupFn = null + } + } catch (error) { + // Re-throw in a microtask to surface the error after cleanup completes + queueMicrotask(() => { + if (error instanceof Error) { + // Preserve the original error and stack trace + const wrappedError = new SyncCleanupError(this.collection.id, error) + wrappedError.cause = error + wrappedError.stack = error.stack + throw wrappedError + } else { + throw new SyncCleanupError( + this.collection.id, + error as Error | string + ) + } + }) + } + + this.preloadPromise = null + return Promise.resolve() + } +} diff --git a/packages/db/src/transactions.ts b/packages/db/src/transactions.ts index c5a6594d3..1e334bdaf 100644 --- a/packages/db/src/transactions.ts +++ b/packages/db/src/transactions.ts @@ -295,11 +295,11 @@ class Transaction> { const hasCalled = new Set() for (const mutation of this.mutations) { if (!hasCalled.has(mutation.collection.id)) { - mutation.collection.onTransactionStateChange() + mutation.collection._state.onTransactionStateChange() // Only call commitPendingTransactions if there are pending sync transactions - if (mutation.collection.pendingSyncedTransactions.length > 0) { - mutation.collection.commitPendingTransactions() + if (mutation.collection._state.pendingSyncedTransactions.length > 0) { + mutation.collection._state.commitPendingTransactions() } hasCalled.add(mutation.collection.id) diff --git a/packages/db/tests/collection-errors.test.ts b/packages/db/tests/collection-errors.test.ts index d8784760f..97a9561a5 100644 --- a/packages/db/tests/collection-errors.test.ts +++ b/packages/db/tests/collection-errors.test.ts @@ -372,12 +372,12 @@ describe(`Collection Error Handling`, () => { // Test invalid transition expect(() => { - collectionImpl.validateStatusTransition(`ready`, `loading`) + collectionImpl._lifecycle.validateStatusTransition(`ready`, `loading`) }).toThrow(InvalidCollectionStatusTransitionError) // Test valid transition expect(() => { - collectionImpl.validateStatusTransition(`idle`, `loading`) + collectionImpl._lifecycle.validateStatusTransition(`idle`, `loading`) }).not.toThrow() }) @@ -397,76 +397,76 @@ describe(`Collection Error Handling`, () => { // Valid transitions from idle expect(() => - collectionImpl.validateStatusTransition(`idle`, `loading`) + collectionImpl._lifecycle.validateStatusTransition(`idle`, `loading`) ).not.toThrow() expect(() => - collectionImpl.validateStatusTransition(`idle`, `error`) + collectionImpl._lifecycle.validateStatusTransition(`idle`, `error`) ).not.toThrow() expect(() => - collectionImpl.validateStatusTransition(`idle`, `cleaned-up`) + collectionImpl._lifecycle.validateStatusTransition(`idle`, `cleaned-up`) ).not.toThrow() // Valid transitions from loading expect(() => - collectionImpl.validateStatusTransition(`loading`, `initialCommit`) + collectionImpl._lifecycle.validateStatusTransition(`loading`, `initialCommit`) ).not.toThrow() expect(() => - collectionImpl.validateStatusTransition(`loading`, `error`) + collectionImpl._lifecycle.validateStatusTransition(`loading`, `error`) ).not.toThrow() expect(() => - collectionImpl.validateStatusTransition(`loading`, `cleaned-up`) + collectionImpl._lifecycle.validateStatusTransition(`loading`, `cleaned-up`) ).not.toThrow() // Valid transitions from initialCommit expect(() => - collectionImpl.validateStatusTransition(`initialCommit`, `ready`) + collectionImpl._lifecycle.validateStatusTransition(`initialCommit`, `ready`) ).not.toThrow() expect(() => - collectionImpl.validateStatusTransition(`initialCommit`, `error`) + collectionImpl._lifecycle.validateStatusTransition(`initialCommit`, `error`) ).not.toThrow() expect(() => - collectionImpl.validateStatusTransition(`initialCommit`, `cleaned-up`) + collectionImpl._lifecycle.validateStatusTransition(`initialCommit`, `cleaned-up`) ).not.toThrow() // Valid transitions from ready expect(() => - collectionImpl.validateStatusTransition(`ready`, `cleaned-up`) + collectionImpl._lifecycle.validateStatusTransition(`ready`, `cleaned-up`) ).not.toThrow() expect(() => - collectionImpl.validateStatusTransition(`ready`, `error`) + collectionImpl._lifecycle.validateStatusTransition(`ready`, `error`) ).not.toThrow() // Valid transitions from error (allow recovery) expect(() => - collectionImpl.validateStatusTransition(`error`, `cleaned-up`) + collectionImpl._lifecycle.validateStatusTransition(`error`, `cleaned-up`) ).not.toThrow() expect(() => - collectionImpl.validateStatusTransition(`error`, `idle`) + collectionImpl._lifecycle.validateStatusTransition(`error`, `idle`) ).not.toThrow() // Valid transitions from cleaned-up (allow restart) expect(() => - collectionImpl.validateStatusTransition(`cleaned-up`, `loading`) + collectionImpl._lifecycle.validateStatusTransition(`cleaned-up`, `loading`) ).not.toThrow() expect(() => - collectionImpl.validateStatusTransition(`cleaned-up`, `error`) + collectionImpl._lifecycle.validateStatusTransition(`cleaned-up`, `error`) ).not.toThrow() // Allow same-state transitions (idempotent operations) expect(() => - collectionImpl.validateStatusTransition(`idle`, `idle`) + collectionImpl._lifecycle.validateStatusTransition(`idle`, `idle`) ).not.toThrow() expect(() => - collectionImpl.validateStatusTransition( + collectionImpl._lifecycle.validateStatusTransition( `initialCommit`, `initialCommit` ) ).not.toThrow() expect(() => - collectionImpl.validateStatusTransition(`ready`, `ready`) + collectionImpl._lifecycle.validateStatusTransition(`ready`, `ready`) ).not.toThrow() expect(() => - collectionImpl.validateStatusTransition(`cleaned-up`, `cleaned-up`) + collectionImpl._lifecycle.validateStatusTransition(`cleaned-up`, `cleaned-up`) ).not.toThrow() }) }) diff --git a/packages/db/tests/collection-lifecycle.test.ts b/packages/db/tests/collection-lifecycle.test.ts index 62c32dfc7..35206f61b 100644 --- a/packages/db/tests/collection-lifecycle.test.ts +++ b/packages/db/tests/collection-lifecycle.test.ts @@ -200,21 +200,21 @@ describe(`Collection Lifecycle Management`, () => { }) // No subscribers initially - expect((collection as any).activeSubscribersCount).toBe(0) + expect(collection._changes.activeSubscribersCount).toBe(0) // Subscribe to changes const unsubscribe1 = collection.subscribeChanges(() => {}) - expect((collection as any).activeSubscribersCount).toBe(1) + expect(collection._changes.activeSubscribersCount).toBe(1) const unsubscribe2 = collection.subscribeChanges(() => {}) - expect((collection as any).activeSubscribersCount).toBe(2) + expect(collection._changes.activeSubscribersCount).toBe(2) // Unsubscribe unsubscribe1() - expect((collection as any).activeSubscribersCount).toBe(1) + expect(collection._changes.activeSubscribersCount).toBe(1) unsubscribe2() - expect((collection as any).activeSubscribersCount).toBe(0) + expect(collection._changes.activeSubscribersCount).toBe(0) }) it(`should track key-specific subscribers`, () => { @@ -230,14 +230,14 @@ describe(`Collection Lifecycle Management`, () => { const unsubscribe2 = collection.subscribeChangesKey(`key2`, () => {}) const unsubscribe3 = collection.subscribeChangesKey(`key1`, () => {}) - expect((collection as any).activeSubscribersCount).toBe(3) + expect(collection._changes.activeSubscribersCount).toBe(3) unsubscribe1() - expect((collection as any).activeSubscribersCount).toBe(2) + expect(collection._changes.activeSubscribersCount).toBe(2) unsubscribe2() unsubscribe3() - expect((collection as any).activeSubscribersCount).toBe(0) + expect(collection._changes.activeSubscribersCount).toBe(0) }) it(`should handle rapid subscribe/unsubscribe correctly`, () => { @@ -253,9 +253,9 @@ describe(`Collection Lifecycle Management`, () => { // Subscribe and immediately unsubscribe multiple times for (let i = 0; i < 5; i++) { const unsubscribe = collection.subscribeChanges(() => {}) - expect((collection as any).activeSubscribersCount).toBe(1) + expect(collection._changes.activeSubscribersCount).toBe(1) unsubscribe() - expect((collection as any).activeSubscribersCount).toBe(0) + expect(collection._changes.activeSubscribersCount).toBe(0) // Should start GC timer each time expect(mockSetTimeout).toHaveBeenCalledWith(expect.any(Function), 1000) diff --git a/packages/db/tests/collection-schema.test.ts b/packages/db/tests/collection-schema.test.ts index 3dc5e7293..93498fc26 100644 --- a/packages/db/tests/collection-schema.test.ts +++ b/packages/db/tests/collection-schema.test.ts @@ -64,7 +64,7 @@ describe(`Collection Schema Validation`, () => { }) // Add the validated insert data to the update collection - ;(updateCollection as any).syncedData.set(`1`, validatedInsert) + updateCollection._state.syncedData.set(`1`, validatedInsert) const updateData = { name: `Jane Doe`, @@ -116,7 +116,7 @@ describe(`Collection Schema Validation`, () => { const validatedInsert = collection.validateData(insertData, `insert`) // Manually add the item to the collection's synced data for testing - ;(collection as any).syncedData.set(`1`, validatedInsert) + collection._state.syncedData.set(`1`, validatedInsert) // Test update validation with only modified fields const updateData = { @@ -176,7 +176,7 @@ describe(`Collection Schema Validation`, () => { expect(validatedInsert.email).toBe(`john@example.com`) // Manually add the item to the collection's synced data for testing - ;(collection as any).syncedData.set(`1`, validatedInsert) + collection._state.syncedData.set(`1`, validatedInsert) // Test update validation without providing defaulted fields const updateData = { @@ -248,7 +248,7 @@ describe(`Collection Schema Validation`, () => { expect(typeof validatedInsert.age).toBe(`number`) // Add to collection for update testing - ;(collection as any).syncedData.set(`1`, validatedInsert) + collection._state.syncedData.set(`1`, validatedInsert) // Test that update validation accepts input type for new fields const updateData = { diff --git a/packages/db/tests/collection-subscribe-changes.test.ts b/packages/db/tests/collection-subscribe-changes.test.ts index 6b3ea1d76..9516bab1c 100644 --- a/packages/db/tests/collection-subscribe-changes.test.ts +++ b/packages/db/tests/collection-subscribe-changes.test.ts @@ -678,15 +678,15 @@ describe(`Collection.subscribeChanges`, () => { syncCollection.config.sync.sync({ collection: syncCollection, begin: () => { - syncCollection.pendingSyncedTransactions.push({ + syncCollection._state.pendingSyncedTransactions.push({ committed: false, operations: [], }) }, write: (messageWithoutKey: any) => { const pendingTransaction = - syncCollection.pendingSyncedTransactions[ - syncCollection.pendingSyncedTransactions.length - 1 + syncCollection._state.pendingSyncedTransactions[ + syncCollection._state.pendingSyncedTransactions.length - 1 ] const key = syncCollection.getKeyFromItem(messageWithoutKey.value) const message = { ...messageWithoutKey, key } @@ -694,8 +694,8 @@ describe(`Collection.subscribeChanges`, () => { }, commit: () => { const pendingTransaction = - syncCollection.pendingSyncedTransactions[ - syncCollection.pendingSyncedTransactions.length - 1 + syncCollection._state.pendingSyncedTransactions[ + syncCollection._state.pendingSyncedTransactions.length - 1 ] pendingTransaction.committed = true syncCollection.commitPendingTransactions() @@ -924,8 +924,8 @@ describe(`Collection.subscribeChanges`, () => { const tx2 = collection.insert({ id: 3, value: `optimistic insert` }) // Verify optimistic state exists - expect(collection.optimisticUpserts.has(1)).toBe(true) - expect(collection.optimisticUpserts.has(3)).toBe(true) + expect(collection._state.optimisticUpserts.has(1)).toBe(true) + expect(collection._state.optimisticUpserts.has(3)).toBe(true) expect(collection.state.get(1)?.value).toBe(`optimistic update 1`) expect(collection.state.get(3)?.value).toBe(`optimistic insert`) diff --git a/packages/db/tests/collection.test.ts b/packages/db/tests/collection.test.ts index 74cd66077..c1694f445 100644 --- a/packages/db/tests/collection.test.ts +++ b/packages/db/tests/collection.test.ts @@ -227,8 +227,8 @@ describe(`Collection`, () => { // Check the optimistic operation is there const insertKey = 1 - expect(collection.optimisticUpserts.has(insertKey)).toBe(true) - expect(collection.optimisticUpserts.get(insertKey)).toEqual({ + expect(collection._state.optimisticUpserts.has(insertKey)).toBe(true) + expect(collection._state.optimisticUpserts.get(insertKey)).toEqual({ id: 1, value: `bar`, }) @@ -262,11 +262,11 @@ describe(`Collection`, () => { // after mutationFn returns, check that the transaction is cleaned up, // optimistic update is gone & synced data & combined state are all updated. - expect(collection.transactions.size).toEqual(0) // Transaction should be cleaned up + expect(collection._state.transactions.size).toEqual(0) // Transaction should be cleaned up expect(collection.state).toEqual( new Map([[insertedKey, { id: 1, value: `bar` }]]) ) - expect(collection.optimisticUpserts.size).toEqual(0) + expect(collection._state.optimisticUpserts.size).toEqual(0) // Test insert with provided key const tx2 = createTransaction({ mutationFn }) @@ -480,7 +480,7 @@ describe(`Collection`, () => { // check there's a transaction in peristing state expect( // @ts-expect-error possibly undefined is ok in test - Array.from(collection.transactions.values())[0].mutations[0].changes + Array.from(collection._state.transactions.values())[0].mutations[0].changes ).toEqual({ id: 1, value: `bar`, @@ -488,8 +488,8 @@ describe(`Collection`, () => { // Check the optimistic operation is there const insertKey = 1 - expect(collection.optimisticUpserts.has(insertKey)).toBe(true) - expect(collection.optimisticUpserts.get(insertKey)).toEqual({ + expect(collection._state.optimisticUpserts.has(insertKey)).toBe(true) + expect(collection._state.optimisticUpserts.get(insertKey)).toEqual({ id: 1, value: `bar`, }) @@ -789,7 +789,7 @@ describe(`Collection`, () => { // The item should NOT appear in the collection state immediately expect(collection.state.has(2)).toBe(false) - expect(collection.optimisticUpserts.has(2)).toBe(false) + expect(collection._state.optimisticUpserts.has(2)).toBe(false) expect(collection.state.size).toBe(1) // Only the initial item // Now resolve the mutation and wait for completion @@ -814,7 +814,7 @@ describe(`Collection`, () => { // The original value should still be there immediately expect(collection.state.get(1)?.value).toBe(`initial value`) - expect(collection.optimisticUpserts.has(1)).toBe(false) + expect(collection._state.optimisticUpserts.has(1)).toBe(false) // Now resolve the update mutation and wait for completion pendingMutations[1]?.() @@ -828,7 +828,7 @@ describe(`Collection`, () => { // The item should still be there immediately expect(collection.state.has(2)).toBe(true) - expect(collection.optimisticDeletes.has(2)).toBe(false) + expect(collection._state.optimisticDeletes.has(2)).toBe(false) // Now resolve the delete mutation and wait for completion pendingMutations[2]?.() @@ -889,7 +889,7 @@ describe(`Collection`, () => { // The item should appear immediately expect(collection.state.has(2)).toBe(true) - expect(collection.optimisticUpserts.has(2)).toBe(true) + expect(collection._state.optimisticUpserts.has(2)).toBe(true) expect(collection.state.get(2)).toEqual({ id: 2, value: `default optimistic`, @@ -905,7 +905,7 @@ describe(`Collection`, () => { // The item should appear immediately expect(collection.state.has(3)).toBe(true) - expect(collection.optimisticUpserts.has(3)).toBe(true) + expect(collection._state.optimisticUpserts.has(3)).toBe(true) expect(collection.state.get(3)).toEqual({ id: 3, value: `explicit optimistic`, @@ -924,7 +924,7 @@ describe(`Collection`, () => { // The update should be reflected immediately expect(collection.state.get(1)?.value).toBe(`optimistic update`) - expect(collection.optimisticUpserts.has(1)).toBe(true) + expect(collection._state.optimisticUpserts.has(1)).toBe(true) await optimisticUpdateTx.isPersisted.promise @@ -933,7 +933,7 @@ describe(`Collection`, () => { // The item should be gone immediately expect(collection.state.has(3)).toBe(false) - expect(collection.optimisticDeletes.has(3)).toBe(true) + expect(collection._state.optimisticDeletes.has(3)).toBe(true) await optimisticDeleteTx.isPersisted.promise }) @@ -1097,8 +1097,8 @@ describe(`Collection`, () => { // Verify collection is cleared expect(collection.state.size).toBe(0) - expect(collection.syncedData.size).toBe(0) - expect(collection.syncedMetadata.size).toBe(0) + expect(collection._state.syncedData.size).toBe(0) + expect(collection._state.syncedMetadata.size).toBe(0) }) it(`should keep operations written after truncate in the same transaction`, async () => { @@ -1163,8 +1163,8 @@ describe(`Collection`, () => { id: 3, value: `should not be cleared`, }) - expect(collection.syncedData.size).toBe(1) - expect(collection.syncedMetadata.size).toBe(1) + expect(collection._state.syncedData.size).toBe(1) + expect(collection._state.syncedMetadata.size).toBe(1) }) it(`should handle truncate with empty collection`, async () => { @@ -1200,8 +1200,8 @@ describe(`Collection`, () => { // Verify collection remains empty expect(collection.state.size).toBe(0) - expect(collection.syncedData.size).toBe(0) - expect(collection.syncedMetadata.size).toBe(0) + expect(collection._state.syncedData.size).toBe(0) + expect(collection._state.syncedMetadata.size).toBe(0) }) it(`open sync transaction isn't applied when optimistic mutation is resolved/rejected`, async () => { diff --git a/packages/electric-db-collection/tests/electric.test.ts b/packages/electric-db-collection/tests/electric.test.ts index fdaf56f2b..72812acd2 100644 --- a/packages/electric-db-collection/tests/electric.test.ts +++ b/packages/electric-db-collection/tests/electric.test.ts @@ -466,7 +466,7 @@ describe(`Electric Integration`, () => { await transaction.isPersisted.promise - transaction = collection.transactions.get(transaction.id)! + transaction = collection._state.transactions.get(transaction.id)! // Verify the mutation function was called correctly expect(testMutationFn).toHaveBeenCalledTimes(1) @@ -628,7 +628,7 @@ describe(`Electric Integration`, () => { }) // If awaitTxId wasn't called automatically, this wouldn't be true. - expect(testCollection.syncedData.size).toEqual(0) + expect(testcollection._state.syncedData.size).toEqual(0) // Verify that our onInsert handler was called expect(onInsert).toHaveBeenCalled() @@ -641,7 +641,7 @@ describe(`Electric Integration`, () => { id: 1, name: `Direct Persistence User`, }) - expect(testCollection.syncedData.size).toEqual(1) + expect(testcollection._state.syncedData.size).toEqual(1) }) }) @@ -854,7 +854,7 @@ describe(`Electric Integration`, () => { }).not.toThrow() // Should have processed the valid message without issues - expect(testCollection.syncedData.size).toBe(0) // Still pending until up-to-date + expect(testcollection._state.syncedData.size).toBe(0) // Still pending until up-to-date // Send up-to-date to commit expect(() => { diff --git a/packages/query-db-collection/src/query.ts b/packages/query-db-collection/src/query.ts index b96e2946f..bd3d6c47d 100644 --- a/packages/query-db-collection/src/query.ts +++ b/packages/query-db-collection/src/query.ts @@ -400,7 +400,7 @@ export function queryCollectionOptions( return } - const currentSyncedItems = new Map(collection.syncedData) + const currentSyncedItems = new Map(collection._state.syncedData) const newItemsMap = new Map() newItemsArray.forEach((item) => { const key = getKey(item) diff --git a/packages/query-db-collection/tests/query.test.ts b/packages/query-db-collection/tests/query.test.ts index b14fa591a..3f09fc22b 100644 --- a/packages/query-db-collection/tests/query.test.ts +++ b/packages/query-db-collection/tests/query.test.ts @@ -86,9 +86,9 @@ describe(`QueryCollection`, () => { expect(collection.get(`2`)).toEqual(initialItems[1]) // Verify the synced data - expect(collection.syncedData.size).toBe(initialItems.length) - expect(collection.syncedData.get(`1`)).toEqual(initialItems[0]) - expect(collection.syncedData.get(`2`)).toEqual(initialItems[1]) + expect(collection._state.syncedData.size).toBe(initialItems.length) + expect(collection._state.syncedData.get(`1`)).toEqual(initialItems[0]) + expect(collection._state.syncedData.get(`2`)).toEqual(initialItems[1]) }) it(`should update collection when query data changes`, async () => { diff --git a/packages/rxdb-db-collection/tests/rxdb.test.ts b/packages/rxdb-db-collection/tests/rxdb.test.ts index d06e52cbb..9522d5b1b 100644 --- a/packages/rxdb-db-collection/tests/rxdb.test.ts +++ b/packages/rxdb-db-collection/tests/rxdb.test.ts @@ -113,9 +113,9 @@ describe(`RxDB Integration`, () => { expect(collection.get(`2`)).toEqual(initialItems[1]) // Verify the synced data - expect(collection.syncedData.size).toBe(initialItems.length) - expect(collection.syncedData.get(`1`)).toEqual(initialItems[0]) - expect(collection.syncedData.get(`2`)).toEqual(initialItems[1]) + expect(collection._state.syncedData.size).toBe(initialItems.length) + expect(collection._state.syncedData.get(`1`)).toEqual(initialItems[0]) + expect(collection._state.syncedData.get(`2`)).toEqual(initialItems[1]) await db.remove() }) @@ -127,7 +127,7 @@ describe(`RxDB Integration`, () => { // All docs should be present after initial sync expect(collection.size).toBe(docsAmount) - expect(collection.syncedData.size).toBe(docsAmount) + expect(collection._state.syncedData.size).toBe(docsAmount) // Spot-check a few positions expect(collection.get(`1`)).toEqual({ id: `1`, name: `Item 1` }) From b00ec21c27a6a341cfb692d5773af217ca9d091f Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Tue, 16 Sep 2025 16:00:33 +0100 Subject: [PATCH 02/14] done? --- packages/db/src/change-events.ts | 2 +- packages/db/src/collection/changes.ts | 14 ++--- .../{collection.ts => collection/index.ts} | 47 ++++++++--------- packages/db/src/collection/indexes.ts | 14 ++--- packages/db/src/collection/lifecycle.ts | 20 +++---- packages/db/src/collection/mutations.ts | 2 +- packages/db/src/collection/state.ts | 17 +++--- packages/db/src/collection/sync.ts | 11 ++-- packages/db/src/index.ts | 4 +- packages/db/src/indexes/auto-index.ts | 2 +- packages/db/src/query/builder/index.ts | 2 +- packages/db/src/query/builder/types.ts | 2 +- packages/db/src/query/compiler/index.ts | 2 +- packages/db/src/query/compiler/joins.ts | 2 +- packages/db/src/query/compiler/order-by.ts | 2 +- packages/db/src/query/ir.ts | 2 +- .../db/src/query/live-query-collection.ts | 4 +- .../query/live/collection-config-builder.ts | 2 +- .../src/query/live/collection-subscriber.ts | 2 +- packages/db/src/types.ts | 2 +- .../db/tests/collection-auto-index.test.ts | 2 +- packages/db/tests/collection-errors.test.ts | 52 +++++++++++++++---- packages/db/tests/collection-getters.test.ts | 4 +- packages/db/tests/collection-indexes.test.ts | 4 +- .../db/tests/collection-lifecycle.test.ts | 2 +- packages/db/tests/collection-schema.test.ts | 2 +- .../collection-subscribe-changes.test.ts | 10 ++-- packages/db/tests/collection.test-d.ts | 2 +- packages/db/tests/collection.test.ts | 5 +- packages/db/tests/query/basic.test-d.ts | 2 +- packages/db/tests/query/basic.test.ts | 2 +- .../db/tests/query/builder/buildQuery.test.ts | 2 +- .../query/builder/callback-types.test-d.ts | 2 +- packages/db/tests/query/builder/from.test.ts | 2 +- .../query/builder/functional-variants.test.ts | 2 +- .../db/tests/query/builder/functions.test.ts | 2 +- .../db/tests/query/builder/group-by.test.ts | 2 +- packages/db/tests/query/builder/join.test.ts | 2 +- .../db/tests/query/builder/order-by.test.ts | 2 +- .../db/tests/query/builder/select.test.ts | 2 +- .../tests/query/builder/subqueries.test-d.ts | 2 +- packages/db/tests/query/builder/where.test.ts | 2 +- .../db/tests/query/compiler/basic.test.ts | 2 +- .../tests/query/compiler/subqueries.test.ts | 2 +- .../query/compiler/subquery-caching.test.ts | 2 +- packages/db/tests/query/composables.test.ts | 2 +- packages/db/tests/query/distinct.test.ts | 2 +- .../tests/query/functional-variants.test-d.ts | 2 +- .../tests/query/functional-variants.test.ts | 2 +- packages/db/tests/query/group-by.test-d.ts | 2 +- packages/db/tests/query/group-by.test.ts | 2 +- packages/db/tests/query/indexes.test.ts | 2 +- .../db/tests/query/join-subquery.test-d.ts | 2 +- packages/db/tests/query/join-subquery.test.ts | 2 +- packages/db/tests/query/join.test-d.ts | 2 +- packages/db/tests/query/join.test.ts | 2 +- .../tests/query/live-query-collection.test.ts | 2 +- .../db/tests/query/nested-props.test-d.ts | 2 +- .../query/optional-fields-negative.test-d.ts | 2 +- .../query/optional-fields-runtime.test.ts | 2 +- packages/db/tests/query/order-by.test.ts | 2 +- .../db/tests/query/select-spread.test-d.ts | 2 +- packages/db/tests/query/select-spread.test.ts | 2 +- packages/db/tests/query/select.test-d.ts | 2 +- packages/db/tests/query/select.test.ts | 2 +- packages/db/tests/query/subquery.test-d.ts | 2 +- packages/db/tests/query/subquery.test.ts | 2 +- packages/db/tests/query/where.test.ts | 2 +- packages/db/tests/transaction-types.test.ts | 2 +- packages/db/tests/transactions.test.ts | 2 +- packages/db/tests/utility-exposure.test.ts | 2 +- 71 files changed, 174 insertions(+), 148 deletions(-) rename packages/db/src/{collection.ts => collection/index.ts} (95%) diff --git a/packages/db/src/change-events.ts b/packages/db/src/change-events.ts index d06b8a933..5bf123867 100644 --- a/packages/db/src/change-events.ts +++ b/packages/db/src/change-events.ts @@ -9,7 +9,7 @@ import type { CurrentStateAsChangesOptions, SubscribeChangesOptions, } from "./types" -import type { Collection } from "./collection" +import type { Collection } from "./collection/index.js" import type { SingleRowRefProxy } from "./query/builder/ref-proxy" import type { BasicExpression } from "./query/ir.js" diff --git a/packages/db/src/collection/changes.ts b/packages/db/src/collection/changes.ts index 8de3d76f6..b6d4a3b58 100644 --- a/packages/db/src/collection/changes.ts +++ b/packages/db/src/collection/changes.ts @@ -1,13 +1,13 @@ -import { StandardSchemaV1 } from "@standard-schema/spec" -import { CollectionImpl } from "../collection" -import { +import { ensureIndexForExpression } from "../indexes/auto-index.js" +import { createFilteredCallback } from "../change-events" +import { NegativeActiveSubscribersError } from "../errors" +import type { StandardSchemaV1 } from "@standard-schema/spec" +import type { CollectionImpl } from "../collection/index.js" +import type { ChangeListener, ChangeMessage, SubscribeChangesOptions, } from "../types" -import { ensureIndexForExpression } from "../indexes/auto-index.js" -import { createFilteredCallback } from "../change-events" -import { NegativeActiveSubscribersError } from "../errors" export class CollectionChangesManager< TOutput extends object = Record, @@ -214,7 +214,7 @@ export class CollectionChangesManager< * Clean up the collection by stopping sync and clearing data * This can be called manually or automatically by garbage collection */ - public async cleanup(): Promise { + public cleanup(): void { this.batchedEvents = [] this.shouldBatchEvents = false } diff --git a/packages/db/src/collection.ts b/packages/db/src/collection/index.ts similarity index 95% rename from packages/db/src/collection.ts rename to packages/db/src/collection/index.ts index 45c8ce882..2a782dd50 100644 --- a/packages/db/src/collection.ts +++ b/packages/db/src/collection/index.ts @@ -1,12 +1,17 @@ -import { BTreeIndex } from "./indexes/btree-index.js" -import type { IndexProxy } from "./indexes/lazy-index.js" import { CollectionRequiresConfigError, CollectionRequiresSyncConfigError, -} from "./errors" -import { currentStateAsChanges } from "./change-events" -import type { StandardSchemaV1 } from "@standard-schema/spec" -import type { SingleRowRefProxy } from "./query/builder/ref-proxy" +} from "../errors" +import { currentStateAsChanges } from "../change-events" + +import { CollectionStateManager } from "../collection/state" +import { CollectionChangesManager } from "../collection/changes" +import { CollectionLifecycleManager } from "../collection/lifecycle.js" +import { CollectionSyncManager } from "../collection/sync" +import { CollectionIndexesManager } from "../collection/indexes" +import { CollectionMutationsManager } from "../collection/mutations" +import type { BaseIndex, IndexResolver } from "../indexes/base-index.js" +import type { IndexOptions } from "../indexes/index-options.js" import type { ChangeListener, ChangeMessage, @@ -22,16 +27,11 @@ import type { Transaction as TransactionType, UtilsRecord, WritableDeep, -} from "./types" -import type { IndexOptions } from "./indexes/index-options.js" -import type { BaseIndex, IndexResolver } from "./indexes/base-index.js" - -import { CollectionStateManager } from "./collection/state" -import { CollectionChangesManager } from "./collection/changes" -import { CollectionLifecycleManager } from "./collection/lifecycle.js" -import { CollectionSyncManager } from "./collection/sync" -import { CollectionIndexesManager } from "./collection/indexes" -import { CollectionMutationsManager } from "./collection/mutations" +} from "../types" +import type { SingleRowRefProxy } from "../query/builder/ref-proxy" +import type { StandardSchemaV1 } from "@standard-schema/spec" +import type { BTreeIndex } from "../indexes/btree-index.js" +import type { IndexProxy } from "../indexes/lazy-index.js" /** * Enhanced Collection interface that includes both data type T and utilities TUtils @@ -330,7 +330,7 @@ export class CollectionImpl< } // Check optimistic upserts first - if (optimisticDeletes.has(key)) { + if (optimisticUpserts.has(key)) { return optimisticUpserts.get(key) } @@ -820,12 +820,11 @@ export class CollectionImpl< * This can be called manually or automatically by garbage collection */ public async cleanup(): Promise { - await Promise.all([ - this._state.cleanup(), - this._changes.cleanup(), - this._lifecycle.cleanup(), - this._sync.cleanup(), - this._indexes.cleanup(), - ]) + this._sync.cleanup() + this._state.cleanup() + this._changes.cleanup() + this._indexes.cleanup() + this._lifecycle.cleanup() + return Promise.resolve() } } diff --git a/packages/db/src/collection/indexes.ts b/packages/db/src/collection/indexes.ts index 6d3264420..b9b1e0326 100644 --- a/packages/db/src/collection/indexes.ts +++ b/packages/db/src/collection/indexes.ts @@ -1,15 +1,15 @@ -import { StandardSchemaV1 } from "@standard-schema/spec" -import { CollectionImpl } from "../collection" import { IndexProxy, LazyIndexWrapper } from "../indexes/lazy-index" -import { BaseIndex, IndexResolver } from "../indexes/base-index" -import { ChangeMessage } from "../types" -import { IndexOptions } from "../indexes/index-options" import { createSingleRowRefProxy, - SingleRowRefProxy, toExpression, } from "../query/builder/ref-proxy" import { BTreeIndex } from "../indexes/btree-index" +import type { StandardSchemaV1 } from "@standard-schema/spec" +import type { CollectionImpl } from "../collection/index.js" +import type { BaseIndex, IndexResolver } from "../indexes/base-index" +import type { ChangeMessage } from "../types" +import type { IndexOptions } from "../indexes/index-options" +import type { SingleRowRefProxy } from "../query/builder/ref-proxy" export class CollectionIndexesManager< TOutput extends object = Record, @@ -155,7 +155,7 @@ export class CollectionIndexesManager< * Clean up the collection by stopping sync and clearing data * This can be called manually or automatically by garbage collection */ - public async cleanup(): Promise { + public cleanup(): void { this.lazyIndexes.clear() this.resolvedIndexes.clear() } diff --git a/packages/db/src/collection/lifecycle.ts b/packages/db/src/collection/lifecycle.ts index 13aea1f43..eaba5028f 100644 --- a/packages/db/src/collection/lifecycle.ts +++ b/packages/db/src/collection/lifecycle.ts @@ -1,9 +1,9 @@ -import { StandardSchemaV1 } from "@standard-schema/spec" -import { CollectionImpl } from "../collection" import { - InvalidCollectionStatusTransitionError, CollectionInErrorStateError, + InvalidCollectionStatusTransitionError, } from "../errors" +import type { StandardSchemaV1 } from "@standard-schema/spec" +import type { CollectionImpl } from "../collection/index.js" import type { CollectionStatus } from "../types" export class CollectionLifecycleManager< @@ -16,8 +16,7 @@ export class CollectionLifecycleManager< public hasBeenReady = false public hasReceivedFirstCommit = false public onFirstReadyCallbacks: Array<() => void> = [] - - private gcTimeoutId: ReturnType | null = null + public gcTimeoutId: ReturnType | null = null /** * Creates a new CollectionLifecycleManager instance @@ -142,7 +141,9 @@ export class CollectionLifecycleManager< this.gcTimeoutId = setTimeout(() => { if (this.collection._changes.activeSubscribersCount === 0) { - this.cleanup() + // We call the main collection cleanup, not just the one for the + // lifecycle manager + this.collection.cleanup() } }, gcTime) } @@ -158,15 +159,16 @@ export class CollectionLifecycleManager< } } - public cleanup(): Promise { + public cleanup(): void { if (this.gcTimeoutId) { clearTimeout(this.gcTimeoutId) this.gcTimeoutId = null } this.hasBeenReady = false - this.hasReceivedFirstCommit = false + this.onFirstReadyCallbacks = [] - return Promise.resolve() + // Set status to cleaned-up + this.setStatus(`cleaned-up`) } } diff --git a/packages/db/src/collection/mutations.ts b/packages/db/src/collection/mutations.ts index 6c623e452..bad3b8e73 100644 --- a/packages/db/src/collection/mutations.ts +++ b/packages/db/src/collection/mutations.ts @@ -16,6 +16,7 @@ import { UndefinedKeyError, UpdateKeyNotFoundError, } from "../errors" +import type { Collection, CollectionImpl } from "../collection/index.js" import type { StandardSchemaV1 } from "@standard-schema/spec" import type { InsertConfig, @@ -27,7 +28,6 @@ import type { UtilsRecord, WritableDeep, } from "../types" -import { CollectionImpl, type Collection } from "../collection" export class CollectionMutationsManager< TOutput extends object = Record, diff --git a/packages/db/src/collection/state.ts b/packages/db/src/collection/state.ts index 3de8e7987..2363beeb8 100644 --- a/packages/db/src/collection/state.ts +++ b/packages/db/src/collection/state.ts @@ -3,7 +3,7 @@ import { SortedMap } from "../SortedMap" import type { Transaction } from "../transactions" import type { StandardSchemaV1 } from "@standard-schema/spec" import type { ChangeMessage, OptimisticChangeMessage } from "../types" -import type { CollectionImpl } from "../collection" +import type { CollectionImpl } from "../collection/index.js" interface PendingSyncedTransaction> { committed: boolean @@ -35,11 +35,11 @@ export class CollectionStateManager< public size = 0 // State used for computing the change events - private syncedKeys = new Set() - private preSyncVisibleState = new Map() - private recentlySyncedKeys = new Set() - private hasReceivedFirstCommit = false - private isCommittingSyncTransactions = false + public syncedKeys = new Set() + public preSyncVisibleState = new Map() + public recentlySyncedKeys = new Set() + public hasReceivedFirstCommit = false + public isCommittingSyncTransactions = false /** * Creates a new CollectionState manager @@ -704,8 +704,7 @@ export class CollectionStateManager< * Clean up the collection by stopping sync and clearing data * This can be called manually or automatically by garbage collection */ - public async cleanup(): Promise { - // Clear data + public cleanup(): void { this.syncedData.clear() this.syncedMetadata.clear() this.optimisticUpserts.clear() @@ -714,7 +713,5 @@ export class CollectionStateManager< this.pendingSyncedTransactions = [] this.syncedKeys.clear() this.hasReceivedFirstCommit = false - - return Promise.resolve() } } diff --git a/packages/db/src/collection/sync.ts b/packages/db/src/collection/sync.ts index e2f1c1930..018fa15e3 100644 --- a/packages/db/src/collection/sync.ts +++ b/packages/db/src/collection/sync.ts @@ -9,7 +9,7 @@ import { } from "../errors" import type { StandardSchemaV1 } from "@standard-schema/spec" import type { ChangeMessage } from "../types" -import type { CollectionImpl } from "../collection" +import type { CollectionImpl } from "../collection/index.js" export class CollectionSyncManager< TOutput extends object = Record, @@ -17,8 +17,8 @@ export class CollectionSyncManager< TSchema extends StandardSchemaV1 = StandardSchemaV1, TInput extends object = TOutput, > { - private preloadPromise: Promise | null = null - private syncCleanupFn: (() => void) | null = null + public preloadPromise: Promise | null = null + public syncCleanupFn: (() => void) | null = null /** * Creates a new CollectionSyncManager instance @@ -191,8 +191,7 @@ export class CollectionSyncManager< return this.preloadPromise } - public cleanup(): Promise { - // Stop sync - wrap in try/catch since it's user-provided code + public cleanup(): void { try { if (this.syncCleanupFn) { this.syncCleanupFn() @@ -215,8 +214,6 @@ export class CollectionSyncManager< } }) } - this.preloadPromise = null - return Promise.resolve() } } diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index e7ccebed5..66f15ff10 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -1,5 +1,5 @@ // Re-export all public APIs -export * from "./collection" +export * from "./collection/index.js" export * from "./SortedMap" export * from "./transactions" export * from "./types" @@ -17,4 +17,4 @@ export * from "./indexes/lazy-index.js" export { type IndexOptions } from "./indexes/index-options.js" // Re-export some stuff explicitly to ensure the type & value is exported -export type { Collection } from "./collection" +export type { Collection } from "./collection/index.js" diff --git a/packages/db/src/indexes/auto-index.ts b/packages/db/src/indexes/auto-index.ts index 4280dcb62..b7caef579 100644 --- a/packages/db/src/indexes/auto-index.ts +++ b/packages/db/src/indexes/auto-index.ts @@ -1,6 +1,6 @@ import { BTreeIndex } from "./btree-index" import type { BasicExpression } from "../query/ir" -import type { CollectionImpl } from "../collection" +import type { CollectionImpl } from "../collection/index.js" export interface AutoIndexConfig { autoIndex?: `off` | `eager` diff --git a/packages/db/src/query/builder/index.ts b/packages/db/src/query/builder/index.ts index 6fe5fce65..423f5be05 100644 --- a/packages/db/src/query/builder/index.ts +++ b/packages/db/src/query/builder/index.ts @@ -1,4 +1,4 @@ -import { CollectionImpl } from "../../collection.js" +import { CollectionImpl } from "../../collection/index.js" import { Aggregate as AggregateExpr, CollectionRef, diff --git a/packages/db/src/query/builder/types.ts b/packages/db/src/query/builder/types.ts index ca0dc97a4..bef1c2bed 100644 --- a/packages/db/src/query/builder/types.ts +++ b/packages/db/src/query/builder/types.ts @@ -1,4 +1,4 @@ -import type { CollectionImpl } from "../../collection.js" +import type { CollectionImpl } from "../../collection/index.js" import type { Aggregate, BasicExpression, diff --git a/packages/db/src/query/compiler/index.ts b/packages/db/src/query/compiler/index.ts index 9ea827d61..14b1504e1 100644 --- a/packages/db/src/query/compiler/index.ts +++ b/packages/db/src/query/compiler/index.ts @@ -21,7 +21,7 @@ import type { QueryRef, } from "../ir.js" import type { LazyCollectionCallbacks } from "./joins.js" -import type { Collection } from "../../collection.js" +import type { Collection } from "../../collection/index.js" import type { KeyedStream, NamespacedAndKeyedStream, diff --git a/packages/db/src/query/compiler/joins.ts b/packages/db/src/query/compiler/joins.ts index 961937ecd..f79fa1fe4 100644 --- a/packages/db/src/query/compiler/joins.ts +++ b/packages/db/src/query/compiler/joins.ts @@ -30,7 +30,7 @@ import type { QueryRef, } from "../ir.js" import type { IStreamBuilder, JoinType } from "@tanstack/db-ivm" -import type { Collection } from "../../collection.js" +import type { Collection } from "../../collection/index.js" import type { KeyedStream, NamespacedAndKeyedStream, diff --git a/packages/db/src/query/compiler/order-by.ts b/packages/db/src/query/compiler/order-by.ts index ddf4703ab..d2d27bb26 100644 --- a/packages/db/src/query/compiler/order-by.ts +++ b/packages/db/src/query/compiler/order-by.ts @@ -11,7 +11,7 @@ import type { OrderByClause, QueryIR, Select } from "../ir.js" import type { NamespacedAndKeyedStream, NamespacedRow } from "../../types.js" import type { IStreamBuilder, KeyValue } from "@tanstack/db-ivm" import type { BaseIndex } from "../../indexes/base-index.js" -import type { Collection } from "../../collection.js" +import type { Collection } from "../../collection/index.js" export type OrderByOptimizationInfo = { offset: number diff --git a/packages/db/src/query/ir.ts b/packages/db/src/query/ir.ts index ee0647beb..add2b1cdc 100644 --- a/packages/db/src/query/ir.ts +++ b/packages/db/src/query/ir.ts @@ -3,7 +3,7 @@ This is the intermediate representation of the query. */ import type { CompareOptions } from "./builder/types" -import type { CollectionImpl } from "../collection" +import type { CollectionImpl } from "../collection/index.js" import type { NamespacedRow } from "../types" export interface QueryIR { diff --git a/packages/db/src/query/live-query-collection.ts b/packages/db/src/query/live-query-collection.ts index b3fc5d793..ec631cd8f 100644 --- a/packages/db/src/query/live-query-collection.ts +++ b/packages/db/src/query/live-query-collection.ts @@ -1,8 +1,8 @@ -import { createCollection } from "../collection.js" +import { createCollection } from "../collection/index.js" import { CollectionConfigBuilder } from "./live/collection-config-builder.js" import type { LiveQueryCollectionConfig } from "./live/types.js" import type { InitialQueryBuilder, QueryBuilder } from "./builder/index.js" -import type { Collection } from "../collection.js" +import type { Collection } from "../collection/index.js" import type { CollectionConfig, UtilsRecord } from "../types.js" import type { Context, GetResult } from "./builder/types.js" diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index 26bfd6c2f..40c680d58 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -4,7 +4,7 @@ import { buildQuery, getQueryIR } from "../builder/index.js" import { CollectionSubscriber } from "./collection-subscriber.js" import type { RootStreamBuilder } from "@tanstack/db-ivm" import type { OrderByOptimizationInfo } from "../compiler/order-by.js" -import type { Collection } from "../../collection.js" +import type { Collection } from "../../collection/index.js" import type { CollectionConfig, KeyedStream, diff --git a/packages/db/src/query/live/collection-subscriber.ts b/packages/db/src/query/live/collection-subscriber.ts index bb45a0f35..9d7c4f6b2 100644 --- a/packages/db/src/query/live/collection-subscriber.ts +++ b/packages/db/src/query/live/collection-subscriber.ts @@ -3,7 +3,7 @@ import { createFilterFunctionFromExpression } from "../../change-events.js" import { convertToBasicExpression } from "../compiler/expressions.js" import type { FullSyncState } from "./types.js" import type { MultiSetArray, RootStreamBuilder } from "@tanstack/db-ivm" -import type { Collection } from "../../collection.js" +import type { Collection } from "../../collection/index.js" import type { ChangeMessage, SyncConfig } from "../../types.js" import type { Context, GetResult } from "../builder/types.js" import type { BasicExpression } from "../ir.js" diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index 52e67dc29..865a9fb3c 100644 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -1,5 +1,5 @@ import type { IStreamBuilder } from "@tanstack/db-ivm" -import type { Collection } from "./collection" +import type { Collection } from "./collection/index.js" import type { StandardSchemaV1 } from "@standard-schema/spec" import type { Transaction } from "./transactions" diff --git a/packages/db/tests/collection-auto-index.test.ts b/packages/db/tests/collection-auto-index.test.ts index a4d464682..a00341a59 100644 --- a/packages/db/tests/collection-auto-index.test.ts +++ b/packages/db/tests/collection-auto-index.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest" -import { createCollection } from "../src/collection" +import { createCollection } from "../src/collection/index.js" import { and, eq, diff --git a/packages/db/tests/collection-errors.test.ts b/packages/db/tests/collection-errors.test.ts index 97a9561a5..69237e84c 100644 --- a/packages/db/tests/collection-errors.test.ts +++ b/packages/db/tests/collection-errors.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" -import { createCollection } from "../src/collection" +import { createCollection } from "../src/collection/index.js" import { CollectionInErrorStateError, InvalidCollectionStatusTransitionError, @@ -408,29 +408,47 @@ describe(`Collection Error Handling`, () => { // Valid transitions from loading expect(() => - collectionImpl._lifecycle.validateStatusTransition(`loading`, `initialCommit`) + collectionImpl._lifecycle.validateStatusTransition( + `loading`, + `initialCommit` + ) ).not.toThrow() expect(() => collectionImpl._lifecycle.validateStatusTransition(`loading`, `error`) ).not.toThrow() expect(() => - collectionImpl._lifecycle.validateStatusTransition(`loading`, `cleaned-up`) + collectionImpl._lifecycle.validateStatusTransition( + `loading`, + `cleaned-up` + ) ).not.toThrow() // Valid transitions from initialCommit expect(() => - collectionImpl._lifecycle.validateStatusTransition(`initialCommit`, `ready`) + collectionImpl._lifecycle.validateStatusTransition( + `initialCommit`, + `ready` + ) ).not.toThrow() expect(() => - collectionImpl._lifecycle.validateStatusTransition(`initialCommit`, `error`) + collectionImpl._lifecycle.validateStatusTransition( + `initialCommit`, + `error` + ) ).not.toThrow() expect(() => - collectionImpl._lifecycle.validateStatusTransition(`initialCommit`, `cleaned-up`) + collectionImpl._lifecycle.validateStatusTransition( + `initialCommit`, + `cleaned-up` + ) ).not.toThrow() // Valid transitions from ready expect(() => - collectionImpl._lifecycle.validateStatusTransition(`ready`, `cleaned-up`) + collectionImpl._lifecycle.validateStatusTransition( + `ready`, + `cleaned-up` + ) ).not.toThrow() expect(() => collectionImpl._lifecycle.validateStatusTransition(`ready`, `error`) @@ -438,7 +456,10 @@ describe(`Collection Error Handling`, () => { // Valid transitions from error (allow recovery) expect(() => - collectionImpl._lifecycle.validateStatusTransition(`error`, `cleaned-up`) + collectionImpl._lifecycle.validateStatusTransition( + `error`, + `cleaned-up` + ) ).not.toThrow() expect(() => collectionImpl._lifecycle.validateStatusTransition(`error`, `idle`) @@ -446,10 +467,16 @@ describe(`Collection Error Handling`, () => { // Valid transitions from cleaned-up (allow restart) expect(() => - collectionImpl._lifecycle.validateStatusTransition(`cleaned-up`, `loading`) + collectionImpl._lifecycle.validateStatusTransition( + `cleaned-up`, + `loading` + ) ).not.toThrow() expect(() => - collectionImpl._lifecycle.validateStatusTransition(`cleaned-up`, `error`) + collectionImpl._lifecycle.validateStatusTransition( + `cleaned-up`, + `error` + ) ).not.toThrow() // Allow same-state transitions (idempotent operations) @@ -466,7 +493,10 @@ describe(`Collection Error Handling`, () => { collectionImpl._lifecycle.validateStatusTransition(`ready`, `ready`) ).not.toThrow() expect(() => - collectionImpl._lifecycle.validateStatusTransition(`cleaned-up`, `cleaned-up`) + collectionImpl._lifecycle.validateStatusTransition( + `cleaned-up`, + `cleaned-up` + ) ).not.toThrow() }) }) diff --git a/packages/db/tests/collection-getters.test.ts b/packages/db/tests/collection-getters.test.ts index fa3c5682c..c7dc58770 100644 --- a/packages/db/tests/collection-getters.test.ts +++ b/packages/db/tests/collection-getters.test.ts @@ -1,7 +1,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest" import { createTransaction } from "../src/transactions" -import { createCollection } from "../src/collection" -import type { CollectionImpl } from "../src/collection" +import { createCollection } from "../src/collection/index.js" +import type { CollectionImpl } from "../src/collection/index.js" import type { SyncConfig } from "../src/types" type Item = { id: string; name: string } diff --git a/packages/db/tests/collection-indexes.test.ts b/packages/db/tests/collection-indexes.test.ts index 2c1db86fa..7c172a725 100644 --- a/packages/db/tests/collection-indexes.test.ts +++ b/packages/db/tests/collection-indexes.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, it } from "vitest" import mitt from "mitt" -import { createCollection } from "../src/collection" +import { createCollection } from "../src/collection/index.js" import { createTransaction } from "../src/transactions" import { and, @@ -14,7 +14,7 @@ import { or, } from "../src/query/builder/functions" import { expectIndexUsage, withIndexTracking } from "./utils" -import type { Collection } from "../src/collection" +import type { Collection } from "../src/collection/index.js" import type { MutationFn, PendingMutation } from "../src/types" interface TestItem { diff --git a/packages/db/tests/collection-lifecycle.test.ts b/packages/db/tests/collection-lifecycle.test.ts index 35206f61b..d1891216e 100644 --- a/packages/db/tests/collection-lifecycle.test.ts +++ b/packages/db/tests/collection-lifecycle.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" -import { createCollection } from "../src/collection.js" +import { createCollection } from "../src/collection/index.js" // Mock setTimeout and clearTimeout for testing GC behavior const originalSetTimeout = global.setTimeout diff --git a/packages/db/tests/collection-schema.test.ts b/packages/db/tests/collection-schema.test.ts index 93498fc26..0db0bddfc 100644 --- a/packages/db/tests/collection-schema.test.ts +++ b/packages/db/tests/collection-schema.test.ts @@ -1,7 +1,7 @@ import { type } from "arktype" import { describe, expect, expectTypeOf, it } from "vitest" import { z } from "zod" -import { createCollection } from "../src/collection" +import { createCollection } from "../src/collection/index.js" import { SchemaValidationError } from "../src/errors" import { createTransaction } from "../src/transactions" import type { diff --git a/packages/db/tests/collection-subscribe-changes.test.ts b/packages/db/tests/collection-subscribe-changes.test.ts index 9516bab1c..000159aac 100644 --- a/packages/db/tests/collection-subscribe-changes.test.ts +++ b/packages/db/tests/collection-subscribe-changes.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from "vitest" import mitt from "mitt" -import { createCollection } from "../src/collection" +import { createCollection } from "../src/collection/index.js" import { createTransaction } from "../src/transactions" import { eq } from "../src/query/builder/functions" import type { @@ -685,8 +685,8 @@ describe(`Collection.subscribeChanges`, () => { }, write: (messageWithoutKey: any) => { const pendingTransaction = - syncCollection._state.pendingSyncedTransactions[ - syncCollection._state.pendingSyncedTransactions.length - 1 + syncCollection._state.pendingSyncedTransactions[ + syncCollection._state.pendingSyncedTransactions.length - 1 ] const key = syncCollection.getKeyFromItem(messageWithoutKey.value) const message = { ...messageWithoutKey, key } @@ -694,8 +694,8 @@ describe(`Collection.subscribeChanges`, () => { }, commit: () => { const pendingTransaction = - syncCollection._state.pendingSyncedTransactions[ - syncCollection._state.pendingSyncedTransactions.length - 1 + syncCollection._state.pendingSyncedTransactions[ + syncCollection._state.pendingSyncedTransactions.length - 1 ] pendingTransaction.committed = true syncCollection.commitPendingTransactions() diff --git a/packages/db/tests/collection.test-d.ts b/packages/db/tests/collection.test-d.ts index b513ff750..d70d53e19 100644 --- a/packages/db/tests/collection.test-d.ts +++ b/packages/db/tests/collection.test-d.ts @@ -1,6 +1,6 @@ import { assertType, describe, expectTypeOf, it } from "vitest" import { z } from "zod" -import { createCollection } from "../src/collection" +import { createCollection } from "../src/collection/index.js" import type { OperationConfig } from "../src/types" import type { StandardSchemaV1 } from "@standard-schema/spec" diff --git a/packages/db/tests/collection.test.ts b/packages/db/tests/collection.test.ts index c1694f445..9abfbb2ac 100644 --- a/packages/db/tests/collection.test.ts +++ b/packages/db/tests/collection.test.ts @@ -1,6 +1,6 @@ import mitt from "mitt" import { describe, expect, it, vi } from "vitest" -import { createCollection } from "../src/collection" +import { createCollection } from "../src/collection/index.js" import { CollectionRequiresConfigError, DuplicateKeyError, @@ -480,7 +480,8 @@ describe(`Collection`, () => { // check there's a transaction in peristing state expect( // @ts-expect-error possibly undefined is ok in test - Array.from(collection._state.transactions.values())[0].mutations[0].changes + Array.from(collection._state.transactions.values())[0].mutations[0] + .changes ).toEqual({ id: 1, value: `bar`, diff --git a/packages/db/tests/query/basic.test-d.ts b/packages/db/tests/query/basic.test-d.ts index d3b325f54..1817cd124 100644 --- a/packages/db/tests/query/basic.test-d.ts +++ b/packages/db/tests/query/basic.test-d.ts @@ -2,7 +2,7 @@ import { describe, expectTypeOf, test } from "vitest" import { z } from "zod" import { type } from "arktype" import { createLiveQueryCollection, eq, gt } from "../../src/query/index.js" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { mockSyncCollectionOptions } from "../utils.js" // Sample user type for tests diff --git a/packages/db/tests/query/basic.test.ts b/packages/db/tests/query/basic.test.ts index db26a28df..dccb0eab0 100644 --- a/packages/db/tests/query/basic.test.ts +++ b/packages/db/tests/query/basic.test.ts @@ -6,7 +6,7 @@ import { gt, upper, } from "../../src/query/index.js" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { mockSyncCollectionOptions } from "../utils.js" // Sample user type for tests diff --git a/packages/db/tests/query/builder/buildQuery.test.ts b/packages/db/tests/query/builder/buildQuery.test.ts index fc2590fbb..9450c0656 100644 --- a/packages/db/tests/query/builder/buildQuery.test.ts +++ b/packages/db/tests/query/builder/buildQuery.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest" -import { CollectionImpl } from "../../../src/collection.js" +import { CollectionImpl } from "../../../src/collection/index.js" import { buildQuery } from "../../../src/query/builder/index.js" import { and, eq, gt, or } from "../../../src/query/builder/functions.js" diff --git a/packages/db/tests/query/builder/callback-types.test-d.ts b/packages/db/tests/query/builder/callback-types.test-d.ts index 20c577188..e45b76bfa 100644 --- a/packages/db/tests/query/builder/callback-types.test-d.ts +++ b/packages/db/tests/query/builder/callback-types.test-d.ts @@ -1,5 +1,5 @@ import { describe, expectTypeOf, test } from "vitest" -import { createCollection } from "../../../src/collection.js" +import { createCollection } from "../../../src/collection/index.js" import { mockSyncCollectionOptions } from "../../utils.js" import { Query } from "../../../src/query/builder/index.js" import { diff --git a/packages/db/tests/query/builder/from.test.ts b/packages/db/tests/query/builder/from.test.ts index 340a396bc..62946b094 100644 --- a/packages/db/tests/query/builder/from.test.ts +++ b/packages/db/tests/query/builder/from.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest" -import { CollectionImpl } from "../../../src/collection.js" +import { CollectionImpl } from "../../../src/collection/index.js" import { Query, getQueryIR } from "../../../src/query/builder/index.js" import { eq } from "../../../src/query/builder/functions.js" import { diff --git a/packages/db/tests/query/builder/functional-variants.test.ts b/packages/db/tests/query/builder/functional-variants.test.ts index eb87ac277..0ce18cf86 100644 --- a/packages/db/tests/query/builder/functional-variants.test.ts +++ b/packages/db/tests/query/builder/functional-variants.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest" -import { CollectionImpl } from "../../../src/collection.js" +import { CollectionImpl } from "../../../src/collection/index.js" import { Query, getQueryIR } from "../../../src/query/builder/index.js" import { eq, gt } from "../../../src/query/builder/functions.js" diff --git a/packages/db/tests/query/builder/functions.test.ts b/packages/db/tests/query/builder/functions.test.ts index 6648cf62a..383771c7e 100644 --- a/packages/db/tests/query/builder/functions.test.ts +++ b/packages/db/tests/query/builder/functions.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest" -import { CollectionImpl } from "../../../src/collection.js" +import { CollectionImpl } from "../../../src/collection/index.js" import { Query, getQueryIR } from "../../../src/query/builder/index.js" import { add, diff --git a/packages/db/tests/query/builder/group-by.test.ts b/packages/db/tests/query/builder/group-by.test.ts index d12b25254..3cbb9632f 100644 --- a/packages/db/tests/query/builder/group-by.test.ts +++ b/packages/db/tests/query/builder/group-by.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest" -import { CollectionImpl } from "../../../src/collection.js" +import { CollectionImpl } from "../../../src/collection/index.js" import { Query, getQueryIR } from "../../../src/query/builder/index.js" import { avg, count, eq, sum } from "../../../src/query/builder/functions.js" diff --git a/packages/db/tests/query/builder/join.test.ts b/packages/db/tests/query/builder/join.test.ts index cbd09acf2..470ee3853 100644 --- a/packages/db/tests/query/builder/join.test.ts +++ b/packages/db/tests/query/builder/join.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest" -import { CollectionImpl } from "../../../src/collection.js" +import { CollectionImpl } from "../../../src/collection/index.js" import { Query, getQueryIR } from "../../../src/query/builder/index.js" import { and, eq, gt } from "../../../src/query/builder/functions.js" diff --git a/packages/db/tests/query/builder/order-by.test.ts b/packages/db/tests/query/builder/order-by.test.ts index 5f5270eaa..378371d9e 100644 --- a/packages/db/tests/query/builder/order-by.test.ts +++ b/packages/db/tests/query/builder/order-by.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest" -import { CollectionImpl } from "../../../src/collection.js" +import { CollectionImpl } from "../../../src/collection/index.js" import { Query, getQueryIR } from "../../../src/query/builder/index.js" import { eq, upper } from "../../../src/query/builder/functions.js" diff --git a/packages/db/tests/query/builder/select.test.ts b/packages/db/tests/query/builder/select.test.ts index c9bb10c72..3c8ae4352 100644 --- a/packages/db/tests/query/builder/select.test.ts +++ b/packages/db/tests/query/builder/select.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest" -import { CollectionImpl } from "../../../src/collection.js" +import { CollectionImpl } from "../../../src/collection/index.js" import { Query, getQueryIR } from "../../../src/query/builder/index.js" import { avg, diff --git a/packages/db/tests/query/builder/subqueries.test-d.ts b/packages/db/tests/query/builder/subqueries.test-d.ts index ee2b38508..88fdedd47 100644 --- a/packages/db/tests/query/builder/subqueries.test-d.ts +++ b/packages/db/tests/query/builder/subqueries.test-d.ts @@ -1,6 +1,6 @@ import { describe, expectTypeOf, test } from "vitest" import { Query } from "../../../src/query/builder/index.js" -import { CollectionImpl } from "../../../src/collection.js" +import { CollectionImpl } from "../../../src/collection/index.js" import { avg, count, eq } from "../../../src/query/builder/functions.js" import type { ExtractContext } from "../../../src/query/builder/index.js" import type { GetResult } from "../../../src/query/builder/types.js" diff --git a/packages/db/tests/query/builder/where.test.ts b/packages/db/tests/query/builder/where.test.ts index 5a9ce7fbe..7021cc790 100644 --- a/packages/db/tests/query/builder/where.test.ts +++ b/packages/db/tests/query/builder/where.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest" -import { CollectionImpl } from "../../../src/collection.js" +import { CollectionImpl } from "../../../src/collection/index.js" import { Query, getQueryIR } from "../../../src/query/builder/index.js" import { and, diff --git a/packages/db/tests/query/compiler/basic.test.ts b/packages/db/tests/query/compiler/basic.test.ts index 655f1601b..127b4151b 100644 --- a/packages/db/tests/query/compiler/basic.test.ts +++ b/packages/db/tests/query/compiler/basic.test.ts @@ -3,7 +3,7 @@ import { D2, MultiSet, output } from "@tanstack/db-ivm" import { compileQuery } from "../../../src/query/compiler/index.js" import { CollectionRef, Func, PropRef, Value } from "../../../src/query/ir.js" import type { QueryIR } from "../../../src/query/ir.js" -import type { CollectionImpl } from "../../../src/collection.js" +import type { CollectionImpl } from "../../../src/collection/index.js" // Sample user type for tests type User = { diff --git a/packages/db/tests/query/compiler/subqueries.test.ts b/packages/db/tests/query/compiler/subqueries.test.ts index e7bab73eb..79edb6403 100644 --- a/packages/db/tests/query/compiler/subqueries.test.ts +++ b/packages/db/tests/query/compiler/subqueries.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from "vitest" import { D2, MultiSet, output } from "@tanstack/db-ivm" import { Query, getQueryIR } from "../../../src/query/builder/index.js" import { compileQuery } from "../../../src/query/compiler/index.js" -import { CollectionImpl } from "../../../src/collection.js" +import { CollectionImpl } from "../../../src/collection/index.js" import { avg, count, eq } from "../../../src/query/builder/functions.js" // Test schema types diff --git a/packages/db/tests/query/compiler/subquery-caching.test.ts b/packages/db/tests/query/compiler/subquery-caching.test.ts index 97bc639c7..4070288da 100644 --- a/packages/db/tests/query/compiler/subquery-caching.test.ts +++ b/packages/db/tests/query/compiler/subquery-caching.test.ts @@ -3,7 +3,7 @@ import { D2 } from "@tanstack/db-ivm" import { compileQuery } from "../../../src/query/compiler/index.js" import { CollectionRef, PropRef, QueryRef } from "../../../src/query/ir.js" import type { QueryIR } from "../../../src/query/ir.js" -import type { CollectionImpl } from "../../../src/collection.js" +import type { CollectionImpl } from "../../../src/collection/index.js" describe(`Subquery Caching`, () => { it(`should cache compiled subqueries and avoid duplicate compilation`, () => { diff --git a/packages/db/tests/query/composables.test.ts b/packages/db/tests/query/composables.test.ts index 423c9613c..a6bd43d10 100644 --- a/packages/db/tests/query/composables.test.ts +++ b/packages/db/tests/query/composables.test.ts @@ -10,7 +10,7 @@ import { lte, upper, } from "../../src/query/index.js" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { mockSyncCollectionOptions } from "../utils.js" import type { Ref } from "../../src/query/index.js" diff --git a/packages/db/tests/query/distinct.test.ts b/packages/db/tests/query/distinct.test.ts index 101444294..3c17c8a1c 100644 --- a/packages/db/tests/query/distinct.test.ts +++ b/packages/db/tests/query/distinct.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, test } from "vitest" import { concat, createLiveQueryCollection } from "../../src/query/index.js" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { mockSyncCollectionOptions } from "../utils.js" import { DistinctRequiresSelectError } from "../../src/errors" import { count, eq, gte, not } from "../../src/query/builder/functions.js" diff --git a/packages/db/tests/query/functional-variants.test-d.ts b/packages/db/tests/query/functional-variants.test-d.ts index 136bd202d..51e6628d8 100644 --- a/packages/db/tests/query/functional-variants.test-d.ts +++ b/packages/db/tests/query/functional-variants.test-d.ts @@ -5,7 +5,7 @@ import { eq, gt, } from "../../src/query/index.js" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { mockSyncCollectionOptions } from "../utils.js" // Sample user type for tests diff --git a/packages/db/tests/query/functional-variants.test.ts b/packages/db/tests/query/functional-variants.test.ts index e006f7c19..2c12e020e 100644 --- a/packages/db/tests/query/functional-variants.test.ts +++ b/packages/db/tests/query/functional-variants.test.ts @@ -5,7 +5,7 @@ import { eq, gt, } from "../../src/query/index.js" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { mockSyncCollectionOptions } from "../utils.js" // Sample user type for tests diff --git a/packages/db/tests/query/group-by.test-d.ts b/packages/db/tests/query/group-by.test-d.ts index 1efad2209..f25ee92e2 100644 --- a/packages/db/tests/query/group-by.test-d.ts +++ b/packages/db/tests/query/group-by.test-d.ts @@ -1,6 +1,6 @@ import { describe, expectTypeOf, test } from "vitest" import { createLiveQueryCollection } from "../../src/query/index.js" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { mockSyncCollectionOptions } from "../utils.js" import { and, diff --git a/packages/db/tests/query/group-by.test.ts b/packages/db/tests/query/group-by.test.ts index 2fa6b428a..452f98f33 100644 --- a/packages/db/tests/query/group-by.test.ts +++ b/packages/db/tests/query/group-by.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, test } from "vitest" import { createLiveQueryCollection } from "../../src/query/index.js" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { mockSyncCollectionOptions } from "../utils.js" import { and, diff --git a/packages/db/tests/query/indexes.test.ts b/packages/db/tests/query/indexes.test.ts index 24798fd77..2ba3b24d0 100644 --- a/packages/db/tests/query/indexes.test.ts +++ b/packages/db/tests/query/indexes.test.ts @@ -1,5 +1,5 @@ import { beforeEach, describe, expect, it } from "vitest" -import { createCollection } from "../../src/collection" +import { createCollection } from "../../src/collection/index.js" import { createLiveQueryCollection } from "../../src/query/live-query-collection" import { diff --git a/packages/db/tests/query/join-subquery.test-d.ts b/packages/db/tests/query/join-subquery.test-d.ts index c339c4a82..f529d0e99 100644 --- a/packages/db/tests/query/join-subquery.test-d.ts +++ b/packages/db/tests/query/join-subquery.test-d.ts @@ -1,6 +1,6 @@ import { describe, expectTypeOf, test } from "vitest" import { createLiveQueryCollection, eq, gt } from "../../src/query/index.js" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { mockSyncCollectionOptions } from "../utils.js" // Sample data types for join-subquery testing diff --git a/packages/db/tests/query/join-subquery.test.ts b/packages/db/tests/query/join-subquery.test.ts index a9388ea88..e2fd684e8 100644 --- a/packages/db/tests/query/join-subquery.test.ts +++ b/packages/db/tests/query/join-subquery.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, test } from "vitest" import { createLiveQueryCollection, eq, gt } from "../../src/query/index.js" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { mockSyncCollectionOptions } from "../utils.js" // Sample data types for join-subquery testing diff --git a/packages/db/tests/query/join.test-d.ts b/packages/db/tests/query/join.test-d.ts index b5f2ffbb0..f453465c9 100644 --- a/packages/db/tests/query/join.test-d.ts +++ b/packages/db/tests/query/join.test-d.ts @@ -2,7 +2,7 @@ import { describe, expectTypeOf, test } from "vitest" import { z } from "zod" import { type } from "arktype" import { createLiveQueryCollection, eq } from "../../src/query/index.js" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { mockSyncCollectionOptions } from "../utils.js" // Sample data types for join type testing diff --git a/packages/db/tests/query/join.test.ts b/packages/db/tests/query/join.test.ts index 765c5e51d..9b2d3ef24 100644 --- a/packages/db/tests/query/join.test.ts +++ b/packages/db/tests/query/join.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, test } from "vitest" import { createLiveQueryCollection, eq } from "../../src/query/index.js" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { mockSyncCollectionOptions } from "../utils.js" // Sample data types for join testing diff --git a/packages/db/tests/query/live-query-collection.test.ts b/packages/db/tests/query/live-query-collection.test.ts index edf7c6127..01741cfce 100644 --- a/packages/db/tests/query/live-query-collection.test.ts +++ b/packages/db/tests/query/live-query-collection.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, it } from "vitest" import { Temporal } from "temporal-polyfill" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { createLiveQueryCollection, eq, diff --git a/packages/db/tests/query/nested-props.test-d.ts b/packages/db/tests/query/nested-props.test-d.ts index ced2f054a..801b099c1 100644 --- a/packages/db/tests/query/nested-props.test-d.ts +++ b/packages/db/tests/query/nested-props.test-d.ts @@ -6,7 +6,7 @@ import { gt, or, } from "../../src/query/index.js" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { mockSyncCollectionOptions } from "../utils.js" // Complex nested type for testing with optional properties diff --git a/packages/db/tests/query/optional-fields-negative.test-d.ts b/packages/db/tests/query/optional-fields-negative.test-d.ts index 688e1e1f3..153361b68 100644 --- a/packages/db/tests/query/optional-fields-negative.test-d.ts +++ b/packages/db/tests/query/optional-fields-negative.test-d.ts @@ -1,6 +1,6 @@ import { describe, expectTypeOf, test } from "vitest" import { createLiveQueryCollection, eq, gt } from "../../src/query/index.js" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { mockSyncCollectionOptions } from "../utils.js" // Test types with optional fields diff --git a/packages/db/tests/query/optional-fields-runtime.test.ts b/packages/db/tests/query/optional-fields-runtime.test.ts index b79d5c7a5..41fa4888d 100644 --- a/packages/db/tests/query/optional-fields-runtime.test.ts +++ b/packages/db/tests/query/optional-fields-runtime.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest" import { createLiveQueryCollection, eq } from "../../src/query/index.js" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { mockSyncCollectionOptions } from "../utils.js" // Test types with optional fields diff --git a/packages/db/tests/query/order-by.test.ts b/packages/db/tests/query/order-by.test.ts index 4b336dc44..b26603c6a 100644 --- a/packages/db/tests/query/order-by.test.ts +++ b/packages/db/tests/query/order-by.test.ts @@ -1,5 +1,5 @@ import { beforeEach, describe, expect, it } from "vitest" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { mockSyncCollectionOptions } from "../utils.js" import { createLiveQueryCollection } from "../../src/query/live-query-collection.js" import { diff --git a/packages/db/tests/query/select-spread.test-d.ts b/packages/db/tests/query/select-spread.test-d.ts index c7a9a344a..a578a6401 100644 --- a/packages/db/tests/query/select-spread.test-d.ts +++ b/packages/db/tests/query/select-spread.test-d.ts @@ -1,5 +1,5 @@ import { describe, expectTypeOf, test } from "vitest" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { createLiveQueryCollection } from "../../src/query/index.js" import { mockSyncCollectionOptions } from "../utils.js" import { add, length, upper } from "../../src/query/builder/functions.js" diff --git a/packages/db/tests/query/select-spread.test.ts b/packages/db/tests/query/select-spread.test.ts index 988461841..ee713c667 100644 --- a/packages/db/tests/query/select-spread.test.ts +++ b/packages/db/tests/query/select-spread.test.ts @@ -1,5 +1,5 @@ import { beforeEach, describe, expect, it } from "vitest" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { createLiveQueryCollection } from "../../src/query/index.js" import { mockSyncCollectionOptions } from "../utils.js" import { add, eq, upper } from "../../src/query/builder/functions.js" diff --git a/packages/db/tests/query/select.test-d.ts b/packages/db/tests/query/select.test-d.ts index fd472061b..b41bb60a6 100644 --- a/packages/db/tests/query/select.test-d.ts +++ b/packages/db/tests/query/select.test-d.ts @@ -1,5 +1,5 @@ import { describe, expectTypeOf, test } from "vitest" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { createLiveQueryCollection } from "../../src/query/index.js" import { mockSyncCollectionOptions } from "../utils.js" import { upper } from "../../src/query/builder/functions.js" diff --git a/packages/db/tests/query/select.test.ts b/packages/db/tests/query/select.test.ts index 86cf1333d..0a14e1b18 100644 --- a/packages/db/tests/query/select.test.ts +++ b/packages/db/tests/query/select.test.ts @@ -1,5 +1,5 @@ import { beforeEach, describe, expect, it } from "vitest" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { createLiveQueryCollection } from "../../src/query/index.js" import { mockSyncCollectionOptions } from "../utils.js" import { upper } from "../../src/query/builder/functions.js" diff --git a/packages/db/tests/query/subquery.test-d.ts b/packages/db/tests/query/subquery.test-d.ts index c2b306e70..5c169eeb4 100644 --- a/packages/db/tests/query/subquery.test-d.ts +++ b/packages/db/tests/query/subquery.test-d.ts @@ -1,6 +1,6 @@ import { describe, expectTypeOf, test } from "vitest" import { createLiveQueryCollection, eq, gt } from "../../src/query/index.js" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { mockSyncCollectionOptions } from "../utils.js" // Sample types for subquery testing diff --git a/packages/db/tests/query/subquery.test.ts b/packages/db/tests/query/subquery.test.ts index fa9fd0831..80b5bdd15 100644 --- a/packages/db/tests/query/subquery.test.ts +++ b/packages/db/tests/query/subquery.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, test } from "vitest" import { createLiveQueryCollection, eq, gt } from "../../src/query/index.js" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { mockSyncCollectionOptions } from "../utils.js" // Sample types for subquery testing diff --git a/packages/db/tests/query/where.test.ts b/packages/db/tests/query/where.test.ts index 39d7db763..a769f71a6 100644 --- a/packages/db/tests/query/where.test.ts +++ b/packages/db/tests/query/where.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, test } from "vitest" import { createLiveQueryCollection } from "../../src/query/index.js" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { mockSyncCollectionOptions } from "../utils.js" import { add, diff --git a/packages/db/tests/transaction-types.test.ts b/packages/db/tests/transaction-types.test.ts index c759c7c90..d69e80f73 100644 --- a/packages/db/tests/transaction-types.test.ts +++ b/packages/db/tests/transaction-types.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest" -import type { Collection } from "../src/collection" +import type { Collection } from "../src/collection/index.js" import type { MutationFn, PendingMutation, diff --git a/packages/db/tests/transactions.test.ts b/packages/db/tests/transactions.test.ts index 7bc58a9e2..d44d0d974 100644 --- a/packages/db/tests/transactions.test.ts +++ b/packages/db/tests/transactions.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest" import { createTransaction } from "../src/transactions" -import { createCollection } from "../src/collection" +import { createCollection } from "../src/collection/index.js" import { MissingMutationFunctionError, TransactionAlreadyCompletedRollbackError, diff --git a/packages/db/tests/utility-exposure.test.ts b/packages/db/tests/utility-exposure.test.ts index 8317ec7b7..08a67bbf2 100644 --- a/packages/db/tests/utility-exposure.test.ts +++ b/packages/db/tests/utility-exposure.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "vitest" -import { createCollection } from "../src/collection" +import { createCollection } from "../src/collection/index.js" import type { CollectionConfig, SyncConfig, UtilsRecord } from "../src/types" // Mock utility functions for testing From 718798ef79c268483246eb054edda9d0590749fd Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Tue, 16 Sep 2025 18:08:51 +0100 Subject: [PATCH 03/14] fix types --- packages/query-db-collection/src/query.ts | 4 +++- pnpm-lock.yaml | 22 ++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/packages/query-db-collection/src/query.ts b/packages/query-db-collection/src/query.ts index bd3d6c47d..88b4df333 100644 --- a/packages/query-db-collection/src/query.ts +++ b/packages/query-db-collection/src/query.ts @@ -400,7 +400,9 @@ export function queryCollectionOptions( return } - const currentSyncedItems = new Map(collection._state.syncedData) + const currentSyncedItems: Map = new Map( + collection._state.syncedData.entries() + ) const newItemsMap = new Map() newItemsArray.forEach((item) => { const key = getKey(item) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ab88dd09b..6777a5b9a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -578,6 +578,28 @@ importers: specifier: ^0.14.0 version: 0.14.10 + packages/benchmarks: + dependencies: + '@tanstack/db': + specifier: workspace:* + version: link:../db + '@tanstack/db-ivm': + specifier: workspace:* + version: link:../db-ivm + tsx: + specifier: ^4.0.0 + version: 4.20.5 + zod: + specifier: ^3.22.0 + version: 3.25.76 + devDependencies: + '@types/node': + specifier: ^22.0.0 + version: 22.18.1 + typescript: + specifier: ^5.0.0 + version: 5.9.2 + packages/db: dependencies: '@standard-schema/spec': From 0beb4241f54b4cc39ab1012f3e8cd56a615651a5 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Tue, 16 Sep 2025 18:15:17 +0100 Subject: [PATCH 04/14] fix --- packages/electric-db-collection/tests/electric.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/electric-db-collection/tests/electric.test.ts b/packages/electric-db-collection/tests/electric.test.ts index 72812acd2..a4f29cfe6 100644 --- a/packages/electric-db-collection/tests/electric.test.ts +++ b/packages/electric-db-collection/tests/electric.test.ts @@ -628,7 +628,7 @@ describe(`Electric Integration`, () => { }) // If awaitTxId wasn't called automatically, this wouldn't be true. - expect(testcollection._state.syncedData.size).toEqual(0) + expect(testCollection._state.syncedData.size).toEqual(0) // Verify that our onInsert handler was called expect(onInsert).toHaveBeenCalled() @@ -641,7 +641,7 @@ describe(`Electric Integration`, () => { id: 1, name: `Direct Persistence User`, }) - expect(testcollection._state.syncedData.size).toEqual(1) + expect(testCollection._state.syncedData.size).toEqual(1) }) }) @@ -854,7 +854,7 @@ describe(`Electric Integration`, () => { }).not.toThrow() // Should have processed the valid message without issues - expect(testcollection._state.syncedData.size).toBe(0) // Still pending until up-to-date + expect(testCollection._state.syncedData.size).toBe(0) // Still pending until up-to-date // Send up-to-date to commit expect(() => { From b03fd2f227a9bc63ce94b62a06df5af75528d5cb Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Tue, 16 Sep 2025 18:18:21 +0100 Subject: [PATCH 05/14] changeset --- .changeset/little-hats-build.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .changeset/little-hats-build.md diff --git a/.changeset/little-hats-build.md b/.changeset/little-hats-build.md new file mode 100644 index 000000000..eed10ccda --- /dev/null +++ b/.changeset/little-hats-build.md @@ -0,0 +1,8 @@ +--- +"@tanstack/electric-db-collection": patch +"@tanstack/query-db-collection": patch +"@tanstack/rxdb-db-collection": patch +"@tanstack/db": patch +--- + +Refactor the main Collection class into smaller classes to make it easier to maintain. From 05238d6026e851e7a37be688eab3df026850b8f7 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Tue, 16 Sep 2025 18:46:46 +0100 Subject: [PATCH 06/14] tweak after copilot review --- packages/db/src/collection/lifecycle.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/db/src/collection/lifecycle.ts b/packages/db/src/collection/lifecycle.ts index eaba5028f..0f08222ab 100644 --- a/packages/db/src/collection/lifecycle.ts +++ b/packages/db/src/collection/lifecycle.ts @@ -27,9 +27,8 @@ export class CollectionLifecycleManager< /** * Validates state transitions to prevent invalid status changes - * @private */ - private validateStatusTransition( + public validateStatusTransition( from: CollectionStatus, to: CollectionStatus ): void { From edc44b08d209d64c2e36a71af6ff988a97c09286 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Wed, 24 Sep 2025 12:07:35 +0100 Subject: [PATCH 07/14] late binding --- packages/db/src/collection/changes.ts | 48 +++++++++++----- packages/db/src/collection/events.ts | 8 ++- packages/db/src/collection/index.ts | 76 +++++++++++++++---------- packages/db/src/collection/indexes.ts | 18 ++++-- packages/db/src/collection/lifecycle.ts | 42 ++++++++++---- packages/db/src/collection/mutations.ts | 74 ++++++++++++++---------- packages/db/src/collection/state.ts | 58 ++++++++++++------- packages/db/src/collection/sync.ts | 55 +++++++++++------- 8 files changed, 247 insertions(+), 132 deletions(-) diff --git a/packages/db/src/collection/changes.ts b/packages/db/src/collection/changes.ts index 9c32f4c33..ad645c4eb 100644 --- a/packages/db/src/collection/changes.ts +++ b/packages/db/src/collection/changes.ts @@ -1,11 +1,16 @@ import { CollectionSubscription } from "../collection-subscription.js" import { NegativeActiveSubscribersError } from "../errors" import type { StandardSchemaV1 } from "@standard-schema/spec" +import type { ChangeMessage, SubscribeChangesOptions } from "../types" +import type { CollectionLifecycleManager } from "./lifecycle.js" +import type { CollectionSyncManager } from "./sync.js" +import type { CollectionEventsManager } from "./events.js" import type { CollectionImpl } from "./index.js" -import type { - ChangeMessage, - SubscribeChangesOptions, -} from "../types" + +// depends on: +// - lifecycle +// - sync +// - events export class CollectionChangesManager< TOutput extends object = Record, @@ -13,6 +18,11 @@ export class CollectionChangesManager< TSchema extends StandardSchemaV1 = StandardSchemaV1, TInput extends object = TOutput, > { + private lifecycle!: CollectionLifecycleManager + private sync!: CollectionSyncManager + private events!: CollectionEventsManager + private collection!: CollectionImpl + public activeSubscribersCount = 0 public changeSubscriptions = new Set() public batchedEvents: Array> = [] @@ -21,9 +31,19 @@ export class CollectionChangesManager< /** * Creates a new CollectionChangesManager instance */ - constructor( - public collection: CollectionImpl - ) {} + constructor() {} + + public bind(deps: { + lifecycle: CollectionLifecycleManager + sync: CollectionSyncManager + events: CollectionEventsManager + collection: CollectionImpl + }) { + this.lifecycle = deps.lifecycle + this.sync = deps.sync + this.events = deps.events + this.collection = deps.collection + } /** * Emit an empty ready event to notify subscribers that the collection is ready @@ -102,17 +122,17 @@ export class CollectionChangesManager< private addSubscriber(): void { const previousSubscriberCount = this.activeSubscribersCount this.activeSubscribersCount++ - this.collection._lifecycle.cancelGCTimer() + this.lifecycle.cancelGCTimer() // Start sync if collection was cleaned up if ( - this.collection.status === `cleaned-up` || - this.collection.status === `idle` + this.lifecycle.status === `cleaned-up` || + this.lifecycle.status === `idle` ) { - this.collection._sync.startSync() + this.sync.startSync() } - this.collection._events.emitSubscribersChange( + this.events.emitSubscribersChange( this.activeSubscribersCount, previousSubscriberCount ) @@ -126,12 +146,12 @@ export class CollectionChangesManager< this.activeSubscribersCount-- if (this.activeSubscribersCount === 0) { - this.collection._lifecycle.startGCTimer() + this.lifecycle.startGCTimer() } else if (this.activeSubscribersCount < 0) { throw new NegativeActiveSubscribersError() } - this.collection._events.emitSubscribersChange( + this.events.emitSubscribersChange( this.activeSubscribersCount, previousSubscriberCount ) diff --git a/packages/db/src/collection/events.ts b/packages/db/src/collection/events.ts index bf0ff8aab..dc1b751b2 100644 --- a/packages/db/src/collection/events.ts +++ b/packages/db/src/collection/events.ts @@ -48,14 +48,16 @@ export type CollectionEventHandler = ( ) => void export class CollectionEventsManager { - private collection: Collection + private collection!: Collection private listeners = new Map< keyof AllCollectionEvents, Set> >() - constructor(collection: Collection) { - this.collection = collection + constructor() {} + + bind(deps: { collection: Collection }) { + this.collection = deps.collection } on( diff --git a/packages/db/src/collection/index.ts b/packages/db/src/collection/index.ts index 53184bd38..5398478cf 100644 --- a/packages/db/src/collection/index.ts +++ b/packages/db/src/collection/index.ts @@ -11,6 +11,7 @@ import { CollectionSyncManager } from "./sync" import { CollectionIndexesManager } from "./indexes" import { CollectionMutationsManager } from "./mutations" import { CollectionEventsManager } from "./events.js" +import type { CollectionSubscription } from "../collection-subscription" import type { AllCollectionEvents, CollectionEventHandler } from "./events.js" import type { BaseIndex, IndexResolver } from "../indexes/base-index.js" import type { IndexOptions } from "../indexes/index-options.js" @@ -33,7 +34,6 @@ import type { SingleRowRefProxy } from "../query/builder/ref-proxy" import type { StandardSchemaV1 } from "@standard-schema/spec" import type { BTreeIndex } from "../indexes/btree-index.js" import type { IndexProxy } from "../indexes/lazy-index.js" -import { CollectionSubscription } from "../collection-subscription" /** * Enhanced Collection interface that includes both data type T and utilities TUtils @@ -227,36 +227,50 @@ export class CollectionImpl< autoIndex: config.autoIndex ?? `eager`, } - this._events = new CollectionEventsManager(this) - this._state = new CollectionStateManager( - this - ) - this._changes = new CollectionChangesManager< - TOutput, - TKey, - TSchema, - TInput - >(this) - this._lifecycle = new CollectionLifecycleManager< - TOutput, - TKey, - TSchema, - TInput - >(this) - this._sync = new CollectionSyncManager(this) - this._indexes = new CollectionIndexesManager< - TOutput, - TKey, - TSchema, - TInput - >(this) - this._mutations = new CollectionMutationsManager< - TOutput, - TKey, - TUtils, - TSchema, - TInput - >(this) + this._changes = new CollectionChangesManager() + this._events = new CollectionEventsManager() + this._indexes = new CollectionIndexesManager() + this._lifecycle = new CollectionLifecycleManager() + this._mutations = new CollectionMutationsManager() + this._state = new CollectionStateManager(config) + this._sync = new CollectionSyncManager(config) + + this._changes.bind({ + lifecycle: this._lifecycle, + sync: this._sync, + events: this._events, + collection: this, + }) + this._events.bind({ + collection: this, + }) + this._indexes.bind({ + collection: this, + lifecycle: this._lifecycle, + }) + this._lifecycle.bind({ + collection: this, + indexes: this._indexes, + events: this._events, + changes: this._changes, + sync: this._sync, + }) + this._mutations.bind({ + lifecycle: this._lifecycle, + state: this._state, + collection: this, + }) + this._state.bind({ + collection: this, + lifecycle: this._lifecycle, + changes: this._changes, + indexes: this._indexes, + }) + this._sync.bind({ + collection: this, + state: this._state, + lifecycle: this._lifecycle, + }) // Only start sync immediately if explicitly enabled if (config.startSync === true) { diff --git a/packages/db/src/collection/indexes.ts b/packages/db/src/collection/indexes.ts index 3518c63c6..b95054d82 100644 --- a/packages/db/src/collection/indexes.ts +++ b/packages/db/src/collection/indexes.ts @@ -10,6 +10,7 @@ import type { BaseIndex, IndexResolver } from "../indexes/base-index" import type { ChangeMessage } from "../types" import type { IndexOptions } from "../indexes/index-options" import type { SingleRowRefProxy } from "../query/builder/ref-proxy" +import type { CollectionLifecycleManager } from "./lifecycle" export class CollectionIndexesManager< TOutput extends object = Record, @@ -17,14 +18,23 @@ export class CollectionIndexesManager< TSchema extends StandardSchemaV1 = StandardSchemaV1, TInput extends object = TOutput, > { + private lifecycle!: CollectionLifecycleManager + private collection!: CollectionImpl + public lazyIndexes = new Map>() public resolvedIndexes = new Map>() public isIndexesResolved = false public indexCounter = 0 - constructor( - public collection: CollectionImpl - ) {} + constructor() {} + + bind(deps: { + collection: CollectionImpl + lifecycle: CollectionLifecycleManager + }) { + this.collection = deps.collection + this.lifecycle = deps.lifecycle + } /** * Creates an index on a collection for faster queries. @@ -33,7 +43,7 @@ export class CollectionIndexesManager< indexCallback: (row: SingleRowRefProxy) => any, config: IndexOptions = {} ): IndexProxy { - this.collection._lifecycle.validateCollectionUsable(`createIndex`) + this.lifecycle.validateCollectionUsable(`createIndex`) const indexId = ++this.indexCounter const singleRowRefProxy = createSingleRowRefProxy() diff --git a/packages/db/src/collection/lifecycle.ts b/packages/db/src/collection/lifecycle.ts index 6412c298a..3c4369eea 100644 --- a/packages/db/src/collection/lifecycle.ts +++ b/packages/db/src/collection/lifecycle.ts @@ -5,6 +5,10 @@ import { import type { StandardSchemaV1 } from "@standard-schema/spec" import type { CollectionImpl } from "./index.js" import type { CollectionStatus } from "../types" +import type { CollectionEventsManager } from "./events" +import type { CollectionIndexesManager } from "./indexes" +import type { CollectionChangesManager } from "./changes" +import type { CollectionSyncManager } from "./sync" export class CollectionLifecycleManager< TOutput extends object = Record, @@ -12,6 +16,12 @@ export class CollectionLifecycleManager< TSchema extends StandardSchemaV1 = StandardSchemaV1, TInput extends object = TOutput, > { + private collection!: CollectionImpl + private indexes!: CollectionIndexesManager + private events!: CollectionEventsManager + private changes!: CollectionChangesManager + private sync!: CollectionSyncManager + public status: CollectionStatus = `idle` public hasBeenReady = false public hasReceivedFirstCommit = false @@ -21,9 +31,21 @@ export class CollectionLifecycleManager< /** * Creates a new CollectionLifecycleManager instance */ - constructor( - public collection: CollectionImpl - ) {} + constructor() {} + + bind(deps: { + collection: CollectionImpl + indexes: CollectionIndexesManager + events: CollectionEventsManager + changes: CollectionChangesManager + sync: CollectionSyncManager + }) { + this.collection = deps.collection + this.indexes = deps.indexes + this.events = deps.events + this.changes = deps.changes + this.sync = deps.sync + } /** * Validates state transitions to prevent invalid status changes @@ -67,15 +89,15 @@ export class CollectionLifecycleManager< this.status = newStatus // Resolve indexes when collection becomes ready - if (newStatus === `ready` && !this.collection._indexes.isIndexesResolved) { + if (newStatus === `ready` && !this.indexes.isIndexesResolved) { // Resolve indexes asynchronously without blocking - this.collection._indexes.resolveAllIndexes().catch((error) => { + this.indexes.resolveAllIndexes().catch((error) => { console.warn(`Failed to resolve indexes:`, error) }) } // Emit event - this.collection._events.emitStatusChange(newStatus, previousStatus) + this.events.emitStatusChange(newStatus, previousStatus) } /** @@ -88,7 +110,7 @@ export class CollectionLifecycleManager< throw new CollectionInErrorStateError(operation, this.collection.id) case `cleaned-up`: // Automatically restart the collection when operations are called on cleaned-up collections - this.collection._sync.startSync() + this.sync.startSync() break } } @@ -121,8 +143,8 @@ export class CollectionLifecycleManager< // Always notify dependents when markReady is called, after status is set // This ensures live queries get notified when their dependencies become ready - if (this.collection._changes.changeSubscriptions.size > 0) { - this.collection._changes.emitEmptyReadyEvent() + if (this.changes.changeSubscriptions.size > 0) { + this.changes.emitEmptyReadyEvent() } } @@ -143,7 +165,7 @@ export class CollectionLifecycleManager< } this.gcTimeoutId = setTimeout(() => { - if (this.collection._changes.activeSubscribersCount === 0) { + if (this.changes.activeSubscribersCount === 0) { // We call the main collection cleanup, not just the one for the // lifecycle manager this.collection.cleanup() diff --git a/packages/db/src/collection/mutations.ts b/packages/db/src/collection/mutations.ts index c9de332c9..63ecc97d5 100644 --- a/packages/db/src/collection/mutations.ts +++ b/packages/db/src/collection/mutations.ts @@ -28,6 +28,8 @@ import type { UtilsRecord, WritableDeep, } from "../types" +import type { CollectionLifecycleManager } from "./lifecycle" +import type { CollectionStateManager } from "./state" export class CollectionMutationsManager< TOutput extends object = Record, @@ -36,9 +38,21 @@ export class CollectionMutationsManager< TSchema extends StandardSchemaV1 = StandardSchemaV1, TInput extends object = TOutput, > { - constructor( - public collection: CollectionImpl - ) {} + private lifecycle!: CollectionLifecycleManager + private state!: CollectionStateManager + private collection!: CollectionImpl + + constructor() {} + + bind(deps: { + lifecycle: CollectionLifecycleManager + state: CollectionStateManager + collection: CollectionImpl + }) { + this.lifecycle = deps.lifecycle + this.state = deps.state + this.collection = deps.collection + } private ensureStandardSchema(schema: unknown): StandardSchema { // If the schema already implements the standard-schema interface, return it @@ -134,8 +148,8 @@ export class CollectionMutationsManager< * Inserts one or more items into the collection */ insert = (data: TInput | Array, config?: InsertConfig) => { - this.collection._lifecycle.validateCollectionUsable(`insert`) - const _state = this.collection._state + this.lifecycle.validateCollectionUsable(`insert`) + const state = this.state const ambientTransaction = getActiveTransaction() // If no ambient transaction exists, check for an onInsert handler early @@ -189,9 +203,9 @@ export class CollectionMutationsManager< if (ambientTransaction) { ambientTransaction.applyMutations(mutations) - _state.transactions.set(ambientTransaction.id, ambientTransaction) - _state.scheduleTransactionCleanup(ambientTransaction) - _state.recomputeOptimisticState(true) + state.transactions.set(ambientTransaction.id, ambientTransaction) + state.scheduleTransactionCleanup(ambientTransaction) + state.recomputeOptimisticState(true) return ambientTransaction } else { @@ -215,9 +229,9 @@ export class CollectionMutationsManager< directOpTransaction.commit() // Add the transaction to the collection's transactions store - _state.transactions.set(directOpTransaction.id, directOpTransaction) - _state.scheduleTransactionCleanup(directOpTransaction) - _state.recomputeOptimisticState(true) + state.transactions.set(directOpTransaction.id, directOpTransaction) + state.scheduleTransactionCleanup(directOpTransaction) + state.recomputeOptimisticState(true) return directOpTransaction } @@ -240,8 +254,8 @@ export class CollectionMutationsManager< throw new MissingUpdateArgumentError() } - const _state = this.collection._state - this.collection._lifecycle.validateCollectionUsable(`update`) + const state = this.state + this.lifecycle.validateCollectionUsable(`update`) const ambientTransaction = getActiveTransaction() @@ -345,7 +359,7 @@ export class CollectionMutationsManager< globalKey, key, metadata: config.metadata as unknown, - syncMetadata: (_state.syncedMetadata.get(key) || {}) as Record< + syncMetadata: (state.syncedMetadata.get(key) || {}) as Record< string, unknown >, @@ -371,7 +385,7 @@ export class CollectionMutationsManager< }) emptyTransaction.commit() // Schedule cleanup for empty transaction - _state.scheduleTransactionCleanup(emptyTransaction) + state.scheduleTransactionCleanup(emptyTransaction) return emptyTransaction } @@ -379,9 +393,9 @@ export class CollectionMutationsManager< if (ambientTransaction) { ambientTransaction.applyMutations(mutations) - _state.transactions.set(ambientTransaction.id, ambientTransaction) - _state.scheduleTransactionCleanup(ambientTransaction) - _state.recomputeOptimisticState(true) + state.transactions.set(ambientTransaction.id, ambientTransaction) + state.scheduleTransactionCleanup(ambientTransaction) + state.recomputeOptimisticState(true) return ambientTransaction } @@ -409,9 +423,9 @@ export class CollectionMutationsManager< // Add the transaction to the collection's transactions store - _state.transactions.set(directOpTransaction.id, directOpTransaction) - _state.scheduleTransactionCleanup(directOpTransaction) - _state.recomputeOptimisticState(true) + state.transactions.set(directOpTransaction.id, directOpTransaction) + state.scheduleTransactionCleanup(directOpTransaction) + state.recomputeOptimisticState(true) return directOpTransaction } @@ -423,8 +437,8 @@ export class CollectionMutationsManager< keys: Array | TKey, config?: OperationConfig ): TransactionType => { - const _state = this.collection._state - this.collection._lifecycle.validateCollectionUsable(`delete`) + const state = this.state + this.lifecycle.validateCollectionUsable(`delete`) const ambientTransaction = getActiveTransaction() @@ -463,7 +477,7 @@ export class CollectionMutationsManager< globalKey, key, metadata: config?.metadata as unknown, - syncMetadata: (_state.syncedMetadata.get(key) || {}) as Record< + syncMetadata: (state.syncedMetadata.get(key) || {}) as Record< string, unknown >, @@ -481,9 +495,9 @@ export class CollectionMutationsManager< if (ambientTransaction) { ambientTransaction.applyMutations(mutations) - _state.transactions.set(ambientTransaction.id, ambientTransaction) - _state.scheduleTransactionCleanup(ambientTransaction) - _state.recomputeOptimisticState(true) + state.transactions.set(ambientTransaction.id, ambientTransaction) + state.scheduleTransactionCleanup(ambientTransaction) + state.recomputeOptimisticState(true) return ambientTransaction } @@ -508,9 +522,9 @@ export class CollectionMutationsManager< directOpTransaction.applyMutations(mutations) directOpTransaction.commit() - _state.transactions.set(directOpTransaction.id, directOpTransaction) - _state.scheduleTransactionCleanup(directOpTransaction) - _state.recomputeOptimisticState(true) + state.transactions.set(directOpTransaction.id, directOpTransaction) + state.scheduleTransactionCleanup(directOpTransaction) + state.recomputeOptimisticState(true) return directOpTransaction } diff --git a/packages/db/src/collection/state.ts b/packages/db/src/collection/state.ts index 3a06cea60..0ae588d7b 100644 --- a/packages/db/src/collection/state.ts +++ b/packages/db/src/collection/state.ts @@ -2,8 +2,15 @@ import { deepEquals } from "../utils" import { SortedMap } from "../SortedMap" import type { Transaction } from "../transactions" import type { StandardSchemaV1 } from "@standard-schema/spec" -import type { ChangeMessage, OptimisticChangeMessage } from "../types" +import type { + ChangeMessage, + CollectionConfig, + OptimisticChangeMessage, +} from "../types" import type { CollectionImpl } from "./index.js" +import type { CollectionLifecycleManager } from "./lifecycle" +import type { CollectionChangesManager } from "./changes" +import type { CollectionIndexesManager } from "./indexes" interface PendingSyncedTransaction> { committed: boolean @@ -18,7 +25,10 @@ export class CollectionStateManager< TSchema extends StandardSchemaV1 = StandardSchemaV1, TInput extends object = TOutput, > { - public collection: CollectionImpl + public collection!: CollectionImpl + public lifecycle!: CollectionLifecycleManager + public changes!: CollectionChangesManager + public indexes!: CollectionIndexesManager // Core state - make public for testing public transactions: SortedMap> @@ -44,21 +54,31 @@ export class CollectionStateManager< /** * Creates a new CollectionState manager */ - constructor(collection: CollectionImpl) { - this.collection = collection - + constructor(config: CollectionConfig) { this.transactions = new SortedMap>((a, b) => a.compareCreatedAt(b) ) // Set up data storage with optional comparison function - if (collection.config.compare) { - this.syncedData = new SortedMap(collection.config.compare) + if (config.compare) { + this.syncedData = new SortedMap(config.compare) } else { this.syncedData = new Map() } } + bind(deps: { + collection: CollectionImpl + lifecycle: CollectionLifecycleManager + changes: CollectionChangesManager + indexes: CollectionIndexesManager + }) { + this.collection = deps.collection + this.lifecycle = deps.lifecycle + this.changes = deps.changes + this.indexes = deps.indexes + } + /** * Recompute optimistic state from active transactions */ @@ -166,19 +186,16 @@ export class CollectionStateManager< // Update indexes for the filtered events if (filteredEvents.length > 0) { - this.collection._indexes.updateIndexes(filteredEvents) + this.indexes.updateIndexes(filteredEvents) } - this.collection._changes.emitEvents(filteredEvents, triggeredByUserAction) + this.changes.emitEvents(filteredEvents, triggeredByUserAction) } else { // Update indexes for all events if (filteredEventsBySyncStatus.length > 0) { - this.collection._indexes.updateIndexes(filteredEventsBySyncStatus) + this.indexes.updateIndexes(filteredEventsBySyncStatus) } // Emit all events if no pending sync transactions - this.collection._changes.emitEvents( - filteredEventsBySyncStatus, - triggeredByUserAction - ) + this.changes.emitEvents(filteredEventsBySyncStatus, triggeredByUserAction) } } @@ -494,7 +511,7 @@ export class CollectionStateManager< // Ensure listeners are active before emitting this critical batch if (!this.collection.isReady()) { - this.collection._lifecycle.setStatus(`ready`) + this.lifecycle.setStatus(`ready`) } } @@ -602,11 +619,11 @@ export class CollectionStateManager< // Update indexes for all events before emitting if (events.length > 0) { - this.collection._indexes.updateIndexes(events) + this.indexes.updateIndexes(events) } // End batching and emit all events (combines any batched events with sync events) - this.collection._changes.emitEvents(events, true) + this.changes.emitEvents(events, true) this.pendingSyncedTransactions = uncommittedSyncedTransactions @@ -621,8 +638,8 @@ export class CollectionStateManager< // Call any registered one-time commit listeners if (!this.hasReceivedFirstCommit) { this.hasReceivedFirstCommit = true - const callbacks = [...this.collection._lifecycle.onFirstReadyCallbacks] - this.collection._lifecycle.onFirstReadyCallbacks = [] + const callbacks = [...this.lifecycle.onFirstReadyCallbacks] + this.lifecycle.onFirstReadyCallbacks = [] callbacks.forEach((callback) => callback()) } } @@ -691,8 +708,7 @@ export class CollectionStateManager< public onTransactionStateChange(): void { // Check if commitPendingTransactions will be called after this // by checking if there are pending sync transactions (same logic as in transactions.ts) - this.collection._changes.shouldBatchEvents = - this.pendingSyncedTransactions.length > 0 + this.changes.shouldBatchEvents = this.pendingSyncedTransactions.length > 0 // CRITICAL: Capture visible state BEFORE clearing optimistic state this.capturePreSyncVisibleState() diff --git a/packages/db/src/collection/sync.ts b/packages/db/src/collection/sync.ts index a15c716b6..6a1d6b6c5 100644 --- a/packages/db/src/collection/sync.ts +++ b/packages/db/src/collection/sync.ts @@ -8,8 +8,10 @@ import { SyncTransactionAlreadyCommittedWriteError, } from "../errors" import type { StandardSchemaV1 } from "@standard-schema/spec" -import type { ChangeMessage } from "../types" +import type { ChangeMessage, CollectionConfig } from "../types" import type { CollectionImpl } from "./index.js" +import type { CollectionStateManager } from "./state" +import type { CollectionLifecycleManager } from "./lifecycle" export class CollectionSyncManager< TOutput extends object = Record, @@ -17,22 +19,37 @@ export class CollectionSyncManager< TSchema extends StandardSchemaV1 = StandardSchemaV1, TInput extends object = TOutput, > { + private collection!: CollectionImpl + private state!: CollectionStateManager + private lifecycle!: CollectionLifecycleManager + private config!: CollectionConfig + public preloadPromise: Promise | null = null public syncCleanupFn: (() => void) | null = null /** * Creates a new CollectionSyncManager instance */ - constructor( - public collection: CollectionImpl - ) {} + constructor(config: CollectionConfig) { + this.config = config + } + + bind(deps: { + collection: CollectionImpl + state: CollectionStateManager + lifecycle: CollectionLifecycleManager + }) { + this.collection = deps.collection + this.state = deps.state + this.lifecycle = deps.lifecycle + } /** * Start the sync process for this collection * This is called when the collection is first accessed or preloaded */ public startSync(): void { - const _state = this.collection._state + const state = this.state if ( this.collection.status !== `idle` && this.collection.status !== `cleaned-up` @@ -40,13 +57,13 @@ export class CollectionSyncManager< return // Already started or in progress } - this.collection._lifecycle.setStatus(`loading`) + this.lifecycle.setStatus(`loading`) try { - const cleanupFn = this.collection.config.sync.sync({ + const cleanupFn = this.config.sync.sync({ collection: this.collection, begin: () => { - _state.pendingSyncedTransactions.push({ + state.pendingSyncedTransactions.push({ committed: false, operations: [], deletedKeys: new Set(), @@ -54,8 +71,8 @@ export class CollectionSyncManager< }, write: (messageWithoutKey: Omit, `key`>) => { const pendingTransaction = - _state.pendingSyncedTransactions[ - _state.pendingSyncedTransactions.length - 1 + state.pendingSyncedTransactions[ + state.pendingSyncedTransactions.length - 1 ] if (!pendingTransaction) { throw new NoPendingSyncTransactionWriteError() @@ -67,7 +84,7 @@ export class CollectionSyncManager< // Check if an item with this key already exists when inserting if (messageWithoutKey.type === `insert`) { - const insertingIntoExistingSynced = _state.syncedData.has(key) + const insertingIntoExistingSynced = state.syncedData.has(key) const hasPendingDeleteForKey = pendingTransaction.deletedKeys.has(key) const isTruncateTransaction = pendingTransaction.truncate === true @@ -93,8 +110,8 @@ export class CollectionSyncManager< }, commit: () => { const pendingTransaction = - _state.pendingSyncedTransactions[ - _state.pendingSyncedTransactions.length - 1 + state.pendingSyncedTransactions[ + state.pendingSyncedTransactions.length - 1 ] if (!pendingTransaction) { throw new NoPendingSyncTransactionCommitError() @@ -108,18 +125,18 @@ export class CollectionSyncManager< // Update status to initialCommit when transitioning from loading // This indicates we're in the process of committing the first transaction if (this.collection.status === `loading`) { - this.collection._lifecycle.setStatus(`initialCommit`) + this.lifecycle.setStatus(`initialCommit`) } - _state.commitPendingTransactions() + state.commitPendingTransactions() }, markReady: () => { - this.collection._lifecycle.markReady() + this.lifecycle.markReady() }, truncate: () => { const pendingTransaction = - _state.pendingSyncedTransactions[ - _state.pendingSyncedTransactions.length - 1 + state.pendingSyncedTransactions[ + state.pendingSyncedTransactions.length - 1 ] if (!pendingTransaction) { throw new NoPendingSyncTransactionWriteError() @@ -144,7 +161,7 @@ export class CollectionSyncManager< // Store cleanup function if provided this.syncCleanupFn = typeof cleanupFn === `function` ? cleanupFn : null } catch (error) { - this.collection._lifecycle.setStatus(`error`) + this.lifecycle.setStatus(`error`) throw error } } From e475d66e1b7f8587d026ea16d025b6979635d1de Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Wed, 24 Sep 2025 12:16:43 +0100 Subject: [PATCH 08/14] make most managers private --- packages/db/src/collection/index.ts | 16 +++++++++------- packages/db/tests/collection-lifecycle.test.ts | 14 +++++++------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/packages/db/src/collection/index.ts b/packages/db/src/collection/index.ts index 5398478cf..ef9ff9ce1 100644 --- a/packages/db/src/collection/index.ts +++ b/packages/db/src/collection/index.ts @@ -184,19 +184,21 @@ export class CollectionImpl< public utils: Record = {} // Managers - public _events: CollectionEventsManager - public _state: CollectionStateManager - public _changes: CollectionChangesManager - public _lifecycle: CollectionLifecycleManager - public _sync: CollectionSyncManager - public _indexes: CollectionIndexesManager - public _mutations: CollectionMutationsManager< + private _events: CollectionEventsManager + private _changes: CollectionChangesManager + private _lifecycle: CollectionLifecycleManager + private _sync: CollectionSyncManager + private _indexes: CollectionIndexesManager + private _mutations: CollectionMutationsManager< TOutput, TKey, TUtils, TSchema, TInput > + // The core state of the collection is "public" so that is accessible in tests + // and for debugging + public _state: CollectionStateManager /** * Creates a new Collection instance diff --git a/packages/db/tests/collection-lifecycle.test.ts b/packages/db/tests/collection-lifecycle.test.ts index 9cb2cb6b1..c794ddce0 100644 --- a/packages/db/tests/collection-lifecycle.test.ts +++ b/packages/db/tests/collection-lifecycle.test.ts @@ -200,21 +200,21 @@ describe(`Collection Lifecycle Management`, () => { }) // No subscribers initially - expect(collection._changes.activeSubscribersCount).toBe(0) + expect(collection.subscriberCount).toBe(0) // Subscribe to changes const subscription1 = collection.subscribeChanges(() => {}) - expect(collection._changes.activeSubscribersCount).toBe(1) + expect(collection.subscriberCount).toBe(1) const subscription2 = collection.subscribeChanges(() => {}) - expect(collection._changes.activeSubscribersCount).toBe(2) + expect(collection.subscriberCount).toBe(2) // Unsubscribe subscription1.unsubscribe() - expect(collection._changes.activeSubscribersCount).toBe(1) + expect(collection.subscriberCount).toBe(1) subscription2.unsubscribe() - expect(collection._changes.activeSubscribersCount).toBe(0) + expect(collection.subscriberCount).toBe(0) }) it(`should handle rapid subscribe/unsubscribe correctly`, () => { @@ -230,9 +230,9 @@ describe(`Collection Lifecycle Management`, () => { // Subscribe and immediately unsubscribe multiple times for (let i = 0; i < 5; i++) { const subscription = collection.subscribeChanges(() => {}) - expect(collection._changes.activeSubscribersCount).toBe(1) + expect(collection.subscriberCount).toBe(1) subscription.unsubscribe() - expect(collection._changes.activeSubscribersCount).toBe(0) + expect(collection.subscriberCount).toBe(0) // Should start GC timer each time expect(mockSetTimeout).toHaveBeenCalledWith(expect.any(Function), 1000) From af86f830e610e29bfc996448d22fac18b29b1a95 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Wed, 24 Sep 2025 12:31:49 +0100 Subject: [PATCH 09/14] remove colleciton dep from lifecycle manager --- packages/db/src/collection/index.ts | 13 +++------ packages/db/src/collection/lifecycle.ts | 35 +++++++++++++++---------- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/packages/db/src/collection/index.ts b/packages/db/src/collection/index.ts index ef9ff9ce1..35a1221e6 100644 --- a/packages/db/src/collection/index.ts +++ b/packages/db/src/collection/index.ts @@ -232,7 +232,7 @@ export class CollectionImpl< this._changes = new CollectionChangesManager() this._events = new CollectionEventsManager() this._indexes = new CollectionIndexesManager() - this._lifecycle = new CollectionLifecycleManager() + this._lifecycle = new CollectionLifecycleManager(config, this.id) this._mutations = new CollectionMutationsManager() this._state = new CollectionStateManager(config) this._sync = new CollectionSyncManager(config) @@ -251,10 +251,10 @@ export class CollectionImpl< lifecycle: this._lifecycle, }) this._lifecycle.bind({ - collection: this, - indexes: this._indexes, - events: this._events, changes: this._changes, + events: this._events, + indexes: this._indexes, + state: this._state, sync: this._sync, }) this._mutations.bind({ @@ -866,11 +866,6 @@ export class CollectionImpl< * This can be called manually or automatically by garbage collection */ public async cleanup(): Promise { - this._events.cleanup() - this._sync.cleanup() - this._state.cleanup() - this._changes.cleanup() - this._indexes.cleanup() this._lifecycle.cleanup() return Promise.resolve() } diff --git a/packages/db/src/collection/lifecycle.ts b/packages/db/src/collection/lifecycle.ts index 3c4369eea..f67e0780c 100644 --- a/packages/db/src/collection/lifecycle.ts +++ b/packages/db/src/collection/lifecycle.ts @@ -3,12 +3,12 @@ import { InvalidCollectionStatusTransitionError, } from "../errors" import type { StandardSchemaV1 } from "@standard-schema/spec" -import type { CollectionImpl } from "./index.js" -import type { CollectionStatus } from "../types" +import type { CollectionConfig, CollectionStatus } from "../types" import type { CollectionEventsManager } from "./events" import type { CollectionIndexesManager } from "./indexes" import type { CollectionChangesManager } from "./changes" import type { CollectionSyncManager } from "./sync" +import type { CollectionStateManager } from "./state" export class CollectionLifecycleManager< TOutput extends object = Record, @@ -16,11 +16,13 @@ export class CollectionLifecycleManager< TSchema extends StandardSchemaV1 = StandardSchemaV1, TInput extends object = TOutput, > { - private collection!: CollectionImpl + private config: CollectionConfig + private id: string private indexes!: CollectionIndexesManager private events!: CollectionEventsManager private changes!: CollectionChangesManager private sync!: CollectionSyncManager + private state!: CollectionStateManager public status: CollectionStatus = `idle` public hasBeenReady = false @@ -31,20 +33,23 @@ export class CollectionLifecycleManager< /** * Creates a new CollectionLifecycleManager instance */ - constructor() {} + constructor(config: CollectionConfig, id: string) { + this.config = config + this.id = id + } bind(deps: { - collection: CollectionImpl indexes: CollectionIndexesManager events: CollectionEventsManager changes: CollectionChangesManager sync: CollectionSyncManager + state: CollectionStateManager }) { - this.collection = deps.collection this.indexes = deps.indexes this.events = deps.events this.changes = deps.changes this.sync = deps.sync + this.state = deps.state } /** @@ -71,11 +76,7 @@ export class CollectionLifecycleManager< } if (!validTransitions[from].includes(to)) { - throw new InvalidCollectionStatusTransitionError( - from, - to, - this.collection.id - ) + throw new InvalidCollectionStatusTransitionError(from, to, this.id) } } @@ -107,7 +108,7 @@ export class CollectionLifecycleManager< public validateCollectionUsable(operation: string): void { switch (this.status) { case `error`: - throw new CollectionInErrorStateError(operation, this.collection.id) + throw new CollectionInErrorStateError(operation, this.id) case `cleaned-up`: // Automatically restart the collection when operations are called on cleaned-up collections this.sync.startSync() @@ -157,7 +158,7 @@ export class CollectionLifecycleManager< clearTimeout(this.gcTimeoutId) } - const gcTime = this.collection.config.gcTime ?? 300000 // 5 minutes default + const gcTime = this.config.gcTime ?? 300000 // 5 minutes default // If gcTime is 0, GC is disabled if (gcTime === 0) { @@ -168,7 +169,7 @@ export class CollectionLifecycleManager< if (this.changes.activeSubscribersCount === 0) { // We call the main collection cleanup, not just the one for the // lifecycle manager - this.collection.cleanup() + this.cleanup() } }, gcTime) } @@ -185,6 +186,12 @@ export class CollectionLifecycleManager< } public cleanup(): void { + this.events.cleanup() + this.sync.cleanup() + this.state.cleanup() + this.changes.cleanup() + this.indexes.cleanup() + if (this.gcTimeoutId) { clearTimeout(this.gcTimeoutId) this.gcTimeoutId = null From 6dcf01a9507c3e513aa87bcdbd18d44d0f0e9d0c Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Wed, 24 Sep 2025 13:26:15 +0100 Subject: [PATCH 10/14] remove colleciton refs from manger classes where possible --- packages/db/src/collection/index.ts | 98 ++--------- packages/db/src/collection/indexes.ts | 14 +- packages/db/src/collection/lifecycle.ts | 15 ++ packages/db/src/collection/mutations.ts | 52 +++--- packages/db/src/collection/state.ts | 157 ++++++++++++++++-- packages/db/src/collection/sync.ts | 31 ++-- packages/db/src/query/compiler/index.ts | 2 +- .../db/tests/collection-auto-index.test.ts | 2 +- 8 files changed, 229 insertions(+), 142 deletions(-) diff --git a/packages/db/src/collection/index.ts b/packages/db/src/collection/index.ts index 35a1221e6..52449a45a 100644 --- a/packages/db/src/collection/index.ts +++ b/packages/db/src/collection/index.ts @@ -233,21 +233,21 @@ export class CollectionImpl< this._events = new CollectionEventsManager() this._indexes = new CollectionIndexesManager() this._lifecycle = new CollectionLifecycleManager(config, this.id) - this._mutations = new CollectionMutationsManager() + this._mutations = new CollectionMutationsManager(config, this.id) this._state = new CollectionStateManager(config) - this._sync = new CollectionSyncManager(config) + this._sync = new CollectionSyncManager(config, this.id) this._changes.bind({ + collection: this, // Required for passing to CollectionSubscription lifecycle: this._lifecycle, sync: this._sync, events: this._events, - collection: this, }) this._events.bind({ - collection: this, + collection: this, // Required for adding to emitted events }) this._indexes.bind({ - collection: this, + state: this._state, lifecycle: this._lifecycle, }) this._lifecycle.bind({ @@ -258,18 +258,18 @@ export class CollectionImpl< sync: this._sync, }) this._mutations.bind({ + collection: this, // Required for passing to config.onInsert/onUpdate/onDelete and annotating mutations lifecycle: this._lifecycle, state: this._state, - collection: this, }) this._state.bind({ - collection: this, + collection: this, // Required for filtering events to only include this collection lifecycle: this._lifecycle, changes: this._changes, indexes: this._indexes, }) this._sync.bind({ - collection: this, + collection: this, // Required for passing to config.sync callback state: this._state, lifecycle: this._lifecycle, }) @@ -305,13 +305,7 @@ export class CollectionImpl< * }) */ public onFirstReady(callback: () => void): void { - // If already ready, call immediately - if (this._lifecycle.hasBeenReady) { - callback() - return - } - - this._lifecycle.onFirstReadyCallbacks.push(callback) + return this._lifecycle.onFirstReady(callback) } /** @@ -350,38 +344,14 @@ export class CollectionImpl< * Get the current value for a key (virtual derived state) */ public get(key: TKey): TOutput | undefined { - const { optimisticDeletes, optimisticUpserts, syncedData } = this._state - // Check if optimistically deleted - if (optimisticDeletes.has(key)) { - return undefined - } - - // Check optimistic upserts first - if (optimisticUpserts.has(key)) { - return optimisticUpserts.get(key) - } - - // Fall back to synced data - return syncedData.get(key) + return this._state.get(key) } /** * Check if a key exists in the collection (virtual derived state) */ public has(key: TKey): boolean { - const { optimisticDeletes, optimisticUpserts, syncedData } = this._state - // Check if optimistically deleted - if (optimisticDeletes.has(key)) { - return false - } - - // Check optimistic upserts first - if (optimisticUpserts.has(key)) { - return true - } - - // Fall back to synced data - return syncedData.has(key) + return this._state.has(key) } /** @@ -395,54 +365,28 @@ export class CollectionImpl< * Get all keys (virtual derived state) */ public *keys(): IterableIterator { - const { syncedData, optimisticDeletes, optimisticUpserts } = this._state - // Yield keys from synced data, skipping any that are deleted. - for (const key of syncedData.keys()) { - if (!optimisticDeletes.has(key)) { - yield key - } - } - // Yield keys from upserts that were not already in synced data. - for (const key of optimisticUpserts.keys()) { - if (!syncedData.has(key) && !optimisticDeletes.has(key)) { - // The optimisticDeletes check is technically redundant if inserts/updates always remove from deletes, - // but it's safer to keep it. - yield key - } - } + yield* this._state.keys() } /** * Get all values (virtual derived state) */ public *values(): IterableIterator { - for (const key of this.keys()) { - const value = this.get(key) - if (value !== undefined) { - yield value - } - } + yield* this._state.values() } /** * Get all entries (virtual derived state) */ public *entries(): IterableIterator<[TKey, TOutput]> { - for (const key of this.keys()) { - const value = this.get(key) - if (value !== undefined) { - yield [key, value] - } - } + yield* this._state.entries() } /** * Get all entries (virtual derived state) */ public *[Symbol.iterator](): IterableIterator<[TKey, TOutput]> { - for (const [key, value] of this.entries()) { - yield [key, value] - } + yield* this._state[Symbol.iterator]() } /** @@ -451,10 +395,7 @@ export class CollectionImpl< public forEach( callbackfn: (value: TOutput, key: TKey, index: number) => void ): void { - let index = 0 - for (const [key, value] of this.entries()) { - callbackfn(value, key, index++) - } + return this._state.forEach(callbackfn) } /** @@ -463,12 +404,7 @@ export class CollectionImpl< public map( callbackfn: (value: TOutput, key: TKey, index: number) => U ): Array { - const result: Array = [] - let index = 0 - for (const [key, value] of this.entries()) { - result.push(callbackfn(value, key, index++)) - } - return result + return this._state.map(callbackfn) } public getKeyFromItem(item: TOutput): TKey { diff --git a/packages/db/src/collection/indexes.ts b/packages/db/src/collection/indexes.ts index b95054d82..d4c08e35f 100644 --- a/packages/db/src/collection/indexes.ts +++ b/packages/db/src/collection/indexes.ts @@ -5,12 +5,12 @@ import { } from "../query/builder/ref-proxy" import { BTreeIndex } from "../indexes/btree-index" import type { StandardSchemaV1 } from "@standard-schema/spec" -import type { CollectionImpl } from "./index.js" import type { BaseIndex, IndexResolver } from "../indexes/base-index" import type { ChangeMessage } from "../types" import type { IndexOptions } from "../indexes/index-options" import type { SingleRowRefProxy } from "../query/builder/ref-proxy" import type { CollectionLifecycleManager } from "./lifecycle" +import type { CollectionStateManager } from "./state" export class CollectionIndexesManager< TOutput extends object = Record, @@ -19,7 +19,7 @@ export class CollectionIndexesManager< TInput extends object = TOutput, > { private lifecycle!: CollectionLifecycleManager - private collection!: CollectionImpl + private state!: CollectionStateManager public lazyIndexes = new Map>() public resolvedIndexes = new Map>() @@ -29,10 +29,10 @@ export class CollectionIndexesManager< constructor() {} bind(deps: { - collection: CollectionImpl + state: CollectionStateManager lifecycle: CollectionLifecycleManager }) { - this.collection = deps.collection + this.state = deps.state this.lifecycle = deps.lifecycle } @@ -60,7 +60,7 @@ export class CollectionIndexesManager< config.name, resolver, config.options, - this.collection.entries() + this.state.entries() ) this.lazyIndexes.set(indexId, lazyIndex) @@ -105,7 +105,7 @@ export class CollectionIndexesManager< const resolvedIndex = await lazyIndex.resolve() // Build index with current data - resolvedIndex.build(this.collection.entries()) + resolvedIndex.build(this.state.entries()) this.resolvedIndexes.set(indexId, resolvedIndex) return { indexId, resolvedIndex } @@ -124,7 +124,7 @@ export class CollectionIndexesManager< lazyIndex: LazyIndexWrapper ): Promise> { const resolvedIndex = await lazyIndex.resolve() - resolvedIndex.build(this.collection.entries()) + resolvedIndex.build(this.state.entries()) this.resolvedIndexes.set(indexId, resolvedIndex) return resolvedIndex } diff --git a/packages/db/src/collection/lifecycle.ts b/packages/db/src/collection/lifecycle.ts index f67e0780c..6ed11e5b9 100644 --- a/packages/db/src/collection/lifecycle.ts +++ b/packages/db/src/collection/lifecycle.ts @@ -185,6 +185,21 @@ export class CollectionLifecycleManager< } } + /** + * Register a callback to be executed when the collection first becomes ready + * Useful for preloading collections + * @param callback Function to call when the collection first becomes ready + */ + public onFirstReady(callback: () => void): void { + // If already ready, call immediately + if (this.hasBeenReady) { + callback() + return + } + + this.onFirstReadyCallbacks.push(callback) + } + public cleanup(): void { this.events.cleanup() this.sync.cleanup() diff --git a/packages/db/src/collection/mutations.ts b/packages/db/src/collection/mutations.ts index 63ecc97d5..e7be3442d 100644 --- a/packages/db/src/collection/mutations.ts +++ b/packages/db/src/collection/mutations.ts @@ -19,6 +19,7 @@ import { import type { Collection, CollectionImpl } from "./index.js" import type { StandardSchemaV1 } from "@standard-schema/spec" import type { + CollectionConfig, InsertConfig, OperationConfig, PendingMutation, @@ -41,8 +42,13 @@ export class CollectionMutationsManager< private lifecycle!: CollectionLifecycleManager private state!: CollectionStateManager private collection!: CollectionImpl + private config!: CollectionConfig + private id: string - constructor() {} + constructor(config: CollectionConfig, id: string) { + this.id = id + this.config = config + } bind(deps: { lifecycle: CollectionLifecycleManager @@ -68,16 +74,14 @@ export class CollectionMutationsManager< type: `insert` | `update`, key?: TKey ): TOutput | never { - if (!this.collection.config.schema) return data as TOutput + if (!this.config.schema) return data as TOutput - const standardSchema = this.ensureStandardSchema( - this.collection.config.schema - ) + const standardSchema = this.ensureStandardSchema(this.config.schema) // For updates, we need to merge with the existing data before validation if (type === `update` && key) { // Get the existing data for this key - const existingData = this.collection.get(key) + const existingData = this.state.get(key) if ( existingData && @@ -141,7 +145,7 @@ export class CollectionMutationsManager< throw new UndefinedKeyError(item) } - return `KEY::${this.collection.id}/${key}` + return `KEY::${this.id}/${key}` } /** @@ -153,7 +157,7 @@ export class CollectionMutationsManager< const ambientTransaction = getActiveTransaction() // If no ambient transaction exists, check for an onInsert handler early - if (!ambientTransaction && !this.collection.config.onInsert) { + if (!ambientTransaction && !this.config.onInsert) { throw new MissingInsertHandlerError() } @@ -166,8 +170,8 @@ export class CollectionMutationsManager< const validatedData = this.validateData(item, `insert`) // Check if an item with this ID already exists in the collection - const key = this.collection.getKeyFromItem(validatedData) - if (this.collection.has(key)) { + const key = this.config.getKey(validatedData) + if (this.state.has(key)) { throw new DuplicateKeyError(key) } const globalKey = this.generateGlobalKey(key, item) @@ -188,7 +192,7 @@ export class CollectionMutationsManager< globalKey, key, metadata: config?.metadata as unknown, - syncMetadata: this.collection.config.sync.getSyncMetadata?.() || {}, + syncMetadata: this.config.sync.getSyncMetadata?.() || {}, optimistic: config?.optimistic ?? true, type: `insert`, createdAt: new Date(), @@ -213,7 +217,7 @@ export class CollectionMutationsManager< const directOpTransaction = createTransaction({ mutationFn: async (params) => { // Call the onInsert handler with the transaction and collection - return await this.collection.config.onInsert!({ + return await this.config.onInsert!({ transaction: params.transaction as unknown as TransactionWithMutations< TOutput, @@ -260,7 +264,7 @@ export class CollectionMutationsManager< const ambientTransaction = getActiveTransaction() // If no ambient transaction exists, check for an onUpdate handler early - if (!ambientTransaction && !this.collection.config.onUpdate) { + if (!ambientTransaction && !this.config.onUpdate) { throw new MissingUpdateHandlerError() } @@ -278,7 +282,7 @@ export class CollectionMutationsManager< // Get the current objects or empty objects if they don't exist const currentObjects = keysArray.map((key) => { - const item = this.collection.get(key) + const item = this.state.get(key) if (!item) { throw new UpdateKeyNotFoundError(key) } @@ -333,8 +337,8 @@ export class CollectionMutationsManager< ) // Check if the ID of the item is being changed - const originalItemId = this.collection.getKeyFromItem(originalItem) - const modifiedItemId = this.collection.getKeyFromItem(modifiedItem) + const originalItemId = this.config.getKey(originalItem) + const modifiedItemId = this.config.getKey(modifiedItem) if (originalItemId !== modifiedItemId) { throw new KeyUpdateNotAllowedError(originalItemId, modifiedItemId) @@ -406,7 +410,7 @@ export class CollectionMutationsManager< const directOpTransaction = createTransaction({ mutationFn: async (params) => { // Call the onUpdate handler with the transaction and collection - return this.collection.config.onUpdate!({ + return this.config.onUpdate!({ transaction: params.transaction as unknown as TransactionWithMutations< TOutput, @@ -443,7 +447,7 @@ export class CollectionMutationsManager< const ambientTransaction = getActiveTransaction() // If no ambient transaction exists, check for an onDelete handler early - if (!ambientTransaction && !this.collection.config.onDelete) { + if (!ambientTransaction && !this.config.onDelete) { throw new MissingDeleteHandlerError() } @@ -461,19 +465,19 @@ export class CollectionMutationsManager< > = [] for (const key of keysArray) { - if (!this.collection.has(key)) { + if (!this.state.has(key)) { throw new DeleteKeyNotFoundError(key) } - const globalKey = this.generateGlobalKey(key, this.collection.get(key)!) + const globalKey = this.generateGlobalKey(key, this.state.get(key)!) const mutation: PendingMutation< TOutput, `delete`, CollectionImpl > = { mutationId: crypto.randomUUID(), - original: this.collection.get(key)!, - modified: this.collection.get(key)!, - changes: this.collection.get(key)!, + original: this.state.get(key)!, + modified: this.state.get(key)!, + changes: this.state.get(key)!, globalKey, key, metadata: config?.metadata as unknown, @@ -507,7 +511,7 @@ export class CollectionMutationsManager< autoCommit: true, mutationFn: async (params) => { // Call the onDelete handler with the transaction and collection - return this.collection.config.onDelete!({ + return this.config.onDelete!({ transaction: params.transaction as unknown as TransactionWithMutations< TOutput, diff --git a/packages/db/src/collection/state.ts b/packages/db/src/collection/state.ts index 0ae588d7b..bd29b1fee 100644 --- a/packages/db/src/collection/state.ts +++ b/packages/db/src/collection/state.ts @@ -25,6 +25,7 @@ export class CollectionStateManager< TSchema extends StandardSchemaV1 = StandardSchemaV1, TInput extends object = TOutput, > { + public config!: CollectionConfig public collection!: CollectionImpl public lifecycle!: CollectionLifecycleManager public changes!: CollectionChangesManager @@ -55,6 +56,7 @@ export class CollectionStateManager< * Creates a new CollectionState manager */ constructor(config: CollectionConfig) { + this.config = config this.transactions = new SortedMap>((a, b) => a.compareCreatedAt(b) ) @@ -79,6 +81,135 @@ export class CollectionStateManager< this.indexes = deps.indexes } + /** + * Get the current value for a key (virtual derived state) + */ + public get(key: TKey): TOutput | undefined { + const { optimisticDeletes, optimisticUpserts, syncedData } = this + // Check if optimistically deleted + if (optimisticDeletes.has(key)) { + return undefined + } + + // Check optimistic upserts first + if (optimisticUpserts.has(key)) { + return optimisticUpserts.get(key) + } + + // Fall back to synced data + return syncedData.get(key) + } + + /** + * Check if a key exists in the collection (virtual derived state) + */ + public has(key: TKey): boolean { + const { optimisticDeletes, optimisticUpserts, syncedData } = this + // Check if optimistically deleted + if (optimisticDeletes.has(key)) { + return false + } + + // Check optimistic upserts first + if (optimisticUpserts.has(key)) { + return true + } + + // Fall back to synced data + return syncedData.has(key) + } + + /** + * Get all keys (virtual derived state) + */ + public *keys(): IterableIterator { + const { syncedData, optimisticDeletes, optimisticUpserts } = this + // Yield keys from synced data, skipping any that are deleted. + for (const key of syncedData.keys()) { + if (!optimisticDeletes.has(key)) { + yield key + } + } + // Yield keys from upserts that were not already in synced data. + for (const key of optimisticUpserts.keys()) { + if (!syncedData.has(key) && !optimisticDeletes.has(key)) { + // The optimisticDeletes check is technically redundant if inserts/updates always remove from deletes, + // but it's safer to keep it. + yield key + } + } + } + + /** + * Get all values (virtual derived state) + */ + public *values(): IterableIterator { + for (const key of this.keys()) { + const value = this.get(key) + if (value !== undefined) { + yield value + } + } + } + + /** + * Get all entries (virtual derived state) + */ + public *entries(): IterableIterator<[TKey, TOutput]> { + for (const key of this.keys()) { + const value = this.get(key) + if (value !== undefined) { + yield [key, value] + } + } + } + + /** + * Get all entries (virtual derived state) + */ + public *[Symbol.iterator](): IterableIterator<[TKey, TOutput]> { + for (const [key, value] of this.entries()) { + yield [key, value] + } + } + + /** + * Execute a callback for each entry in the collection + */ + public forEach( + callbackfn: (value: TOutput, key: TKey, index: number) => void + ): void { + let index = 0 + for (const [key, value] of this.entries()) { + callbackfn(value, key, index++) + } + } + + /** + * Create a new array with the results of calling a function for each entry in the collection + */ + public map( + callbackfn: (value: TOutput, key: TKey, index: number) => U + ): Array { + const result: Array = [] + let index = 0 + for (const [key, value] of this.entries()) { + result.push(callbackfn(value, key, index++)) + } + return result + } + + /** + * Check if the given collection is this collection + * @param collection The collection to check + * @returns True if the given collection is this collection, false otherwise + */ + private isThisCollection( + collection: CollectionImpl + ): boolean { + return collection === this.collection + } + /** * Recompute optimistic state from active transactions */ @@ -108,7 +239,7 @@ export class CollectionStateManager< // Apply active transactions only (completed transactions are handled by sync operations) for (const transaction of activeTransactions) { for (const mutation of transaction.mutations) { - if (mutation.collection === this.collection && mutation.optimistic) { + if (this.isThisCollection(mutation.collection) && mutation.optimistic) { switch (mutation.type) { case `insert`: case `update`: @@ -173,7 +304,7 @@ export class CollectionStateManager< // We can infer this by checking if we have no remaining optimistic mutations for this key const hasActiveOptimisticMutation = activeTransactions.some((tx) => tx.mutations.some( - (m) => m.collection === this.collection && m.key === event.key + (m) => this.isThisCollection(m.collection) && m.key === event.key ) ) @@ -230,7 +361,7 @@ export class CollectionStateManager< ]) for (const key of allKeys) { - const currentValue = this.collection.get(key) + const currentValue = this.get(key) const previousValue = this.getPreviousValue( key, previousUpserts, @@ -335,7 +466,7 @@ export class CollectionStateManager< // No pre-captured state, capture it now for pure sync operations currentVisibleState = new Map() for (const key of changedKeys) { - const currentValue = this.collection.get(key) + const currentValue = this.get(key) if (currentValue !== undefined) { currentVisibleState.set(key, currentValue) } @@ -343,8 +474,7 @@ export class CollectionStateManager< } const events: Array> = [] - const rowUpdateMode = - this.collection.config.sync.rowUpdateMode || `partial` + const rowUpdateMode = this.config.sync.rowUpdateMode || `partial` for (const transaction of committedSyncedTransactions) { // Handle truncate operations first @@ -448,7 +578,10 @@ export class CollectionStateManager< for (const tx of this.transactions.values()) { if ([`completed`, `failed`].includes(tx.state)) continue for (const mutation of tx.mutations) { - if (mutation.collection !== this.collection || !mutation.optimistic) + if ( + !this.isThisCollection(mutation.collection) || + !mutation.optimistic + ) continue const key = mutation.key as TKey switch (mutation.type) { @@ -510,7 +643,7 @@ export class CollectionStateManager< } // Ensure listeners are active before emitting this critical batch - if (!this.collection.isReady()) { + if (this.lifecycle.status !== `ready`) { this.lifecycle.setStatus(`ready`) } } @@ -527,7 +660,7 @@ export class CollectionStateManager< if (![`completed`, `failed`].includes(transaction.state)) { for (const mutation of transaction.mutations) { if ( - mutation.collection === this.collection && + this.isThisCollection(mutation.collection) && mutation.optimistic ) { switch (mutation.type) { @@ -556,7 +689,7 @@ export class CollectionStateManager< if (transaction.state === `completed`) { for (const mutation of transaction.mutations) { if ( - mutation.collection === this.collection && + this.isThisCollection(mutation.collection) && changedKeys.has(mutation.key) ) { completedOptimisticOps.set(mutation.key, { @@ -571,7 +704,7 @@ export class CollectionStateManager< // Now check what actually changed in the final visible state for (const key of changedKeys) { const previousVisibleValue = currentVisibleState.get(key) - const newVisibleValue = this.collection.get(key) // This returns the new derived state + const newVisibleValue = this.get(key) // This returns the new derived state // Check if this sync operation is redundant with a completed optimistic operation const completedOp = completedOptimisticOps.get(key) @@ -694,7 +827,7 @@ export class CollectionStateManager< // Only capture current visible state for keys that will be affected by sync operations // This is much more efficient than capturing the entire collection state for (const key of syncedKeys) { - const currentValue = this.collection.get(key) + const currentValue = this.get(key) if (currentValue !== undefined) { this.preSyncVisibleState.set(key, currentValue) } diff --git a/packages/db/src/collection/sync.ts b/packages/db/src/collection/sync.ts index 6a1d6b6c5..322c30132 100644 --- a/packages/db/src/collection/sync.ts +++ b/packages/db/src/collection/sync.ts @@ -23,6 +23,7 @@ export class CollectionSyncManager< private state!: CollectionStateManager private lifecycle!: CollectionLifecycleManager private config!: CollectionConfig + private id: string public preloadPromise: Promise | null = null public syncCleanupFn: (() => void) | null = null @@ -30,8 +31,9 @@ export class CollectionSyncManager< /** * Creates a new CollectionSyncManager instance */ - constructor(config: CollectionConfig) { + constructor(config: CollectionConfig, id: string) { this.config = config + this.id = id } bind(deps: { @@ -51,8 +53,8 @@ export class CollectionSyncManager< public startSync(): void { const state = this.state if ( - this.collection.status !== `idle` && - this.collection.status !== `cleaned-up` + this.lifecycle.status !== `idle` && + this.lifecycle.status !== `cleaned-up` ) { return // Already started or in progress } @@ -80,7 +82,7 @@ export class CollectionSyncManager< if (pendingTransaction.committed) { throw new SyncTransactionAlreadyCommittedWriteError() } - const key = this.collection.getKeyFromItem(messageWithoutKey.value) + const key = this.config.getKey(messageWithoutKey.value) // Check if an item with this key already exists when inserting if (messageWithoutKey.type === `insert`) { @@ -94,7 +96,7 @@ export class CollectionSyncManager< !hasPendingDeleteForKey && !isTruncateTransaction ) { - throw new DuplicateKeySyncError(key, this.collection.id) + throw new DuplicateKeySyncError(key, this.id) } } @@ -124,7 +126,7 @@ export class CollectionSyncManager< // Update status to initialCommit when transitioning from loading // This indicates we're in the process of committing the first transaction - if (this.collection.status === `loading`) { + if (this.lifecycle.status === `loading`) { this.lifecycle.setStatus(`initialCommit`) } @@ -176,25 +178,25 @@ export class CollectionSyncManager< } this.preloadPromise = new Promise((resolve, reject) => { - if (this.collection.status === `ready`) { + if (this.lifecycle.status === `ready`) { resolve() return } - if (this.collection.status === `error`) { + if (this.lifecycle.status === `error`) { reject(new CollectionIsInErrorStateError()) return } // Register callback BEFORE starting sync to avoid race condition - this.collection.onFirstReady(() => { + this.lifecycle.onFirstReady(() => { resolve() }) // Start sync if collection hasn't started yet or was cleaned up if ( - this.collection.status === `idle` || - this.collection.status === `cleaned-up` + this.lifecycle.status === `idle` || + this.lifecycle.status === `cleaned-up` ) { try { this.startSync() @@ -219,15 +221,12 @@ export class CollectionSyncManager< queueMicrotask(() => { if (error instanceof Error) { // Preserve the original error and stack trace - const wrappedError = new SyncCleanupError(this.collection.id, error) + const wrappedError = new SyncCleanupError(this.id, error) wrappedError.cause = error wrappedError.stack = error.stack throw wrappedError } else { - throw new SyncCleanupError( - this.collection.id, - error as Error | string - ) + throw new SyncCleanupError(this.id, error as Error | string) } }) } diff --git a/packages/db/src/query/compiler/index.ts b/packages/db/src/query/compiler/index.ts index 782530eba..3a68e3235 100644 --- a/packages/db/src/query/compiler/index.ts +++ b/packages/db/src/query/compiler/index.ts @@ -13,6 +13,7 @@ import { processJoins } from "./joins.js" import { processGroupBy } from "./group-by.js" import { processOrderBy } from "./order-by.js" import { processSelect } from "./select.js" +import type { CollectionSubscription } from "../../collection-subscription.js" import type { OrderByOptimizationInfo } from "./order-by.js" import type { BasicExpression, @@ -28,7 +29,6 @@ import type { ResultStream, } from "../../types.js" import type { QueryCache, QueryMapping } from "./types.js" -import { CollectionSubscription } from "../../collection-subscription.js" /** * Result of query compilation including both the pipeline and collection-specific WHERE clauses diff --git a/packages/db/tests/collection-auto-index.test.ts b/packages/db/tests/collection-auto-index.test.ts index 3269331ca..f3821f7cb 100644 --- a/packages/db/tests/collection-auto-index.test.ts +++ b/packages/db/tests/collection-auto-index.test.ts @@ -11,12 +11,12 @@ import { } from "../src/query/builder/functions" import { createSingleRowRefProxy } from "../src/query/builder/ref-proxy" import { createLiveQueryCollection } from "../src" +import { PropRef } from "../src/query/ir" import { createIndexUsageTracker, expectIndexUsage, withIndexTracking, } from "./utils" -import { PropRef } from "../src/query/ir" // Global row proxy for expressions const row = createSingleRowRefProxy() From 56b8c6ab4a6ccb2b5aeeaf70fd279948c4a61cdb Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Wed, 24 Sep 2025 13:44:34 +0100 Subject: [PATCH 11/14] more collection-subscription.ts to collection dir --- packages/db/src/collection/changes.ts | 2 +- packages/db/src/collection/index.ts | 2 +- .../subscription.ts} | 14 +++++++------- packages/db/src/query/compiler/index.ts | 2 +- packages/db/src/query/compiler/joins.ts | 2 +- .../db/src/query/live/collection-config-builder.ts | 2 +- .../db/src/query/live/collection-subscriber.ts | 2 +- 7 files changed, 13 insertions(+), 13 deletions(-) rename packages/db/src/{collection-subscription.ts => collection/subscription.ts} (95%) diff --git a/packages/db/src/collection/changes.ts b/packages/db/src/collection/changes.ts index ad645c4eb..2e6586a5f 100644 --- a/packages/db/src/collection/changes.ts +++ b/packages/db/src/collection/changes.ts @@ -1,5 +1,5 @@ -import { CollectionSubscription } from "../collection-subscription.js" import { NegativeActiveSubscribersError } from "../errors" +import { CollectionSubscription } from "./subscription.js" import type { StandardSchemaV1 } from "@standard-schema/spec" import type { ChangeMessage, SubscribeChangesOptions } from "../types" import type { CollectionLifecycleManager } from "./lifecycle.js" diff --git a/packages/db/src/collection/index.ts b/packages/db/src/collection/index.ts index 52449a45a..d39803b61 100644 --- a/packages/db/src/collection/index.ts +++ b/packages/db/src/collection/index.ts @@ -11,7 +11,7 @@ import { CollectionSyncManager } from "./sync" import { CollectionIndexesManager } from "./indexes" import { CollectionMutationsManager } from "./mutations" import { CollectionEventsManager } from "./events.js" -import type { CollectionSubscription } from "../collection-subscription" +import type { CollectionSubscription } from "./subscription" import type { AllCollectionEvents, CollectionEventHandler } from "./events.js" import type { BaseIndex, IndexResolver } from "../indexes/base-index.js" import type { IndexOptions } from "../indexes/index-options.js" diff --git a/packages/db/src/collection-subscription.ts b/packages/db/src/collection/subscription.ts similarity index 95% rename from packages/db/src/collection-subscription.ts rename to packages/db/src/collection/subscription.ts index 8e68173d6..d286d2dd2 100644 --- a/packages/db/src/collection-subscription.ts +++ b/packages/db/src/collection/subscription.ts @@ -1,13 +1,13 @@ import { createFilterFunctionFromExpression, createFilteredCallback, -} from "./change-events.js" -import { ensureIndexForExpression } from "./indexes/auto-index.js" -import { and } from "./query/index.js" -import type { BasicExpression } from "./query/ir.js" -import type { BaseIndex } from "./indexes/base-index.js" -import type { ChangeMessage } from "./types.js" -import type { CollectionImpl } from "./collection/index.js" +} from "../change-events.js" +import { ensureIndexForExpression } from "../indexes/auto-index.js" +import { and } from "../query/index.js" +import type { BasicExpression } from "../query/ir.js" +import type { BaseIndex } from "../indexes/base-index.js" +import type { ChangeMessage } from "../types.js" +import type { CollectionImpl } from "./index.js" type RequestSnapshotOptions = { where?: BasicExpression diff --git a/packages/db/src/query/compiler/index.ts b/packages/db/src/query/compiler/index.ts index 3a68e3235..a79ace809 100644 --- a/packages/db/src/query/compiler/index.ts +++ b/packages/db/src/query/compiler/index.ts @@ -13,7 +13,7 @@ import { processJoins } from "./joins.js" import { processGroupBy } from "./group-by.js" import { processOrderBy } from "./order-by.js" import { processSelect } from "./select.js" -import type { CollectionSubscription } from "../../collection-subscription.js" +import type { CollectionSubscription } from "../../collection/subscription.js" import type { OrderByOptimizationInfo } from "./order-by.js" import type { BasicExpression, diff --git a/packages/db/src/query/compiler/joins.ts b/packages/db/src/query/compiler/joins.ts index 77b6f183a..42687e446 100644 --- a/packages/db/src/query/compiler/joins.ts +++ b/packages/db/src/query/compiler/joins.ts @@ -37,7 +37,7 @@ import type { NamespacedRow, } from "../../types.js" import type { QueryCache, QueryMapping } from "./types.js" -import type { CollectionSubscription } from "../../collection-subscription.js" +import type { CollectionSubscription } from "../../collection/subscription.js" export type LoadKeysFn = (key: Set) => void export type LazyCollectionCallbacks = { diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index 1fbe745eb..72d90f905 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -2,7 +2,7 @@ import { D2, output } from "@tanstack/db-ivm" import { compileQuery } from "../compiler/index.js" import { buildQuery, getQueryIR } from "../builder/index.js" import { CollectionSubscriber } from "./collection-subscriber.js" -import type { CollectionSubscription } from "../../collection-subscription.js" +import type { CollectionSubscription } from "../../collection/subscription.js" import type { RootStreamBuilder } from "@tanstack/db-ivm" import type { OrderByOptimizationInfo } from "../compiler/order-by.js" import type { Collection } from "../../collection/index.js" diff --git a/packages/db/src/query/live/collection-subscriber.ts b/packages/db/src/query/live/collection-subscriber.ts index e24a4567d..c1fb88b3a 100644 --- a/packages/db/src/query/live/collection-subscriber.ts +++ b/packages/db/src/query/live/collection-subscriber.ts @@ -7,7 +7,7 @@ import type { ChangeMessage, SyncConfig } from "../../types.js" import type { Context, GetResult } from "../builder/types.js" import type { BasicExpression } from "../ir.js" import type { CollectionConfigBuilder } from "./collection-config-builder.js" -import type { CollectionSubscription } from "../../collection-subscription.js" +import type { CollectionSubscription } from "../../collection/subscription.js" export class CollectionSubscriber< TContext extends Context, From c964f75cca9664f2ac4dd3405e8e4d71163e9682 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Wed, 24 Sep 2025 13:46:10 +0100 Subject: [PATCH 12/14] rename bind to setDeps --- packages/db/src/collection/changes.ts | 2 +- packages/db/src/collection/events.ts | 2 +- packages/db/src/collection/index.ts | 14 +++++++------- packages/db/src/collection/indexes.ts | 2 +- packages/db/src/collection/lifecycle.ts | 2 +- packages/db/src/collection/mutations.ts | 2 +- packages/db/src/collection/state.ts | 2 +- packages/db/src/collection/sync.ts | 2 +- 8 files changed, 14 insertions(+), 14 deletions(-) diff --git a/packages/db/src/collection/changes.ts b/packages/db/src/collection/changes.ts index 2e6586a5f..f02fc4ebd 100644 --- a/packages/db/src/collection/changes.ts +++ b/packages/db/src/collection/changes.ts @@ -33,7 +33,7 @@ export class CollectionChangesManager< */ constructor() {} - public bind(deps: { + public setDeps(deps: { lifecycle: CollectionLifecycleManager sync: CollectionSyncManager events: CollectionEventsManager diff --git a/packages/db/src/collection/events.ts b/packages/db/src/collection/events.ts index dc1b751b2..1b6c31be5 100644 --- a/packages/db/src/collection/events.ts +++ b/packages/db/src/collection/events.ts @@ -56,7 +56,7 @@ export class CollectionEventsManager { constructor() {} - bind(deps: { collection: Collection }) { + setDeps(deps: { collection: Collection }) { this.collection = deps.collection } diff --git a/packages/db/src/collection/index.ts b/packages/db/src/collection/index.ts index d39803b61..c0ca83184 100644 --- a/packages/db/src/collection/index.ts +++ b/packages/db/src/collection/index.ts @@ -237,38 +237,38 @@ export class CollectionImpl< this._state = new CollectionStateManager(config) this._sync = new CollectionSyncManager(config, this.id) - this._changes.bind({ + this._changes.setDeps({ collection: this, // Required for passing to CollectionSubscription lifecycle: this._lifecycle, sync: this._sync, events: this._events, }) - this._events.bind({ + this._events.setDeps({ collection: this, // Required for adding to emitted events }) - this._indexes.bind({ + this._indexes.setDeps({ state: this._state, lifecycle: this._lifecycle, }) - this._lifecycle.bind({ + this._lifecycle.setDeps({ changes: this._changes, events: this._events, indexes: this._indexes, state: this._state, sync: this._sync, }) - this._mutations.bind({ + this._mutations.setDeps({ collection: this, // Required for passing to config.onInsert/onUpdate/onDelete and annotating mutations lifecycle: this._lifecycle, state: this._state, }) - this._state.bind({ + this._state.setDeps({ collection: this, // Required for filtering events to only include this collection lifecycle: this._lifecycle, changes: this._changes, indexes: this._indexes, }) - this._sync.bind({ + this._sync.setDeps({ collection: this, // Required for passing to config.sync callback state: this._state, lifecycle: this._lifecycle, diff --git a/packages/db/src/collection/indexes.ts b/packages/db/src/collection/indexes.ts index d4c08e35f..96eba97ce 100644 --- a/packages/db/src/collection/indexes.ts +++ b/packages/db/src/collection/indexes.ts @@ -28,7 +28,7 @@ export class CollectionIndexesManager< constructor() {} - bind(deps: { + setDeps(deps: { state: CollectionStateManager lifecycle: CollectionLifecycleManager }) { diff --git a/packages/db/src/collection/lifecycle.ts b/packages/db/src/collection/lifecycle.ts index 6ed11e5b9..0179786fa 100644 --- a/packages/db/src/collection/lifecycle.ts +++ b/packages/db/src/collection/lifecycle.ts @@ -38,7 +38,7 @@ export class CollectionLifecycleManager< this.id = id } - bind(deps: { + setDeps(deps: { indexes: CollectionIndexesManager events: CollectionEventsManager changes: CollectionChangesManager diff --git a/packages/db/src/collection/mutations.ts b/packages/db/src/collection/mutations.ts index e7be3442d..0e8cbf03a 100644 --- a/packages/db/src/collection/mutations.ts +++ b/packages/db/src/collection/mutations.ts @@ -50,7 +50,7 @@ export class CollectionMutationsManager< this.config = config } - bind(deps: { + setDeps(deps: { lifecycle: CollectionLifecycleManager state: CollectionStateManager collection: CollectionImpl diff --git a/packages/db/src/collection/state.ts b/packages/db/src/collection/state.ts index bd29b1fee..10a7c0491 100644 --- a/packages/db/src/collection/state.ts +++ b/packages/db/src/collection/state.ts @@ -69,7 +69,7 @@ export class CollectionStateManager< } } - bind(deps: { + setDeps(deps: { collection: CollectionImpl lifecycle: CollectionLifecycleManager changes: CollectionChangesManager diff --git a/packages/db/src/collection/sync.ts b/packages/db/src/collection/sync.ts index 322c30132..22a14f72d 100644 --- a/packages/db/src/collection/sync.ts +++ b/packages/db/src/collection/sync.ts @@ -36,7 +36,7 @@ export class CollectionSyncManager< this.id = id } - bind(deps: { + setDeps(deps: { collection: CollectionImpl state: CollectionStateManager lifecycle: CollectionLifecycleManager From 06e5bf708c13fa7ba2d334aac88fc76c981c930b Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Wed, 24 Sep 2025 13:56:32 +0100 Subject: [PATCH 13/14] more change-events.ts --- packages/db/src/{ => collection}/change-events.ts | 14 +++++++------- packages/db/src/collection/index.ts | 2 +- packages/db/src/collection/subscription.ts | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) rename packages/db/src/{ => collection}/change-events.ts (95%) diff --git a/packages/db/src/change-events.ts b/packages/db/src/collection/change-events.ts similarity index 95% rename from packages/db/src/change-events.ts rename to packages/db/src/collection/change-events.ts index cfa48ad8e..505f4d97e 100644 --- a/packages/db/src/change-events.ts +++ b/packages/db/src/collection/change-events.ts @@ -1,17 +1,17 @@ import { createSingleRowRefProxy, toExpression, -} from "./query/builder/ref-proxy" -import { compileSingleRowExpression } from "./query/compiler/evaluators.js" -import { optimizeExpressionWithIndexes } from "./utils/index-optimization.js" +} from "../query/builder/ref-proxy" +import { compileSingleRowExpression } from "../query/compiler/evaluators.js" +import { optimizeExpressionWithIndexes } from "../utils/index-optimization.js" import type { ChangeMessage, CurrentStateAsChangesOptions, SubscribeChangesOptions, -} from "./types" -import type { Collection } from "./collection/index.js" -import type { SingleRowRefProxy } from "./query/builder/ref-proxy" -import type { BasicExpression } from "./query/ir.js" +} from "../types" +import type { Collection } from "./index.js" +import type { SingleRowRefProxy } from "../query/builder/ref-proxy" +import type { BasicExpression } from "../query/ir.js" /** * Interface for a collection-like object that provides the necessary methods diff --git a/packages/db/src/collection/index.ts b/packages/db/src/collection/index.ts index c0ca83184..63e818799 100644 --- a/packages/db/src/collection/index.ts +++ b/packages/db/src/collection/index.ts @@ -2,7 +2,7 @@ import { CollectionRequiresConfigError, CollectionRequiresSyncConfigError, } from "../errors" -import { currentStateAsChanges } from "../change-events" +import { currentStateAsChanges } from "./change-events" import { CollectionStateManager } from "./state" import { CollectionChangesManager } from "./changes" diff --git a/packages/db/src/collection/subscription.ts b/packages/db/src/collection/subscription.ts index d286d2dd2..3d65db137 100644 --- a/packages/db/src/collection/subscription.ts +++ b/packages/db/src/collection/subscription.ts @@ -1,9 +1,9 @@ +import { ensureIndexForExpression } from "../indexes/auto-index.js" +import { and } from "../query/index.js" import { createFilterFunctionFromExpression, createFilteredCallback, -} from "../change-events.js" -import { ensureIndexForExpression } from "../indexes/auto-index.js" -import { and } from "../query/index.js" +} from "./change-events.js" import type { BasicExpression } from "../query/ir.js" import type { BaseIndex } from "../indexes/base-index.js" import type { ChangeMessage } from "../types.js" From 77e8303ae11357c6864fa7552dc5bb9ae2e094aa Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Wed, 24 Sep 2025 15:19:13 +0100 Subject: [PATCH 14/14] remove comment --- packages/db/src/collection/changes.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/db/src/collection/changes.ts b/packages/db/src/collection/changes.ts index f02fc4ebd..ae1ea64f7 100644 --- a/packages/db/src/collection/changes.ts +++ b/packages/db/src/collection/changes.ts @@ -7,11 +7,6 @@ import type { CollectionSyncManager } from "./sync.js" import type { CollectionEventsManager } from "./events.js" import type { CollectionImpl } from "./index.js" -// depends on: -// - lifecycle -// - sync -// - events - export class CollectionChangesManager< TOutput extends object = Record, TKey extends string | number = string | number,