diff --git a/.changeset/little-hats-build.md b/.changeset/little-hats-build.md new file mode 100644 index 000000000..eed10ccda --- /dev/null +++ b/.changeset/little-hats-build.md @@ -0,0 +1,8 @@ +--- +"@tanstack/electric-db-collection": patch +"@tanstack/query-db-collection": patch +"@tanstack/rxdb-db-collection": patch +"@tanstack/db": patch +--- + +Refactor the main Collection class into smaller classes to make it easier to maintain. diff --git a/packages/db/src/collection.ts b/packages/db/src/collection.ts deleted file mode 100644 index 9996c1710..000000000 --- a/packages/db/src/collection.ts +++ /dev/null @@ -1,2480 +0,0 @@ -import { withArrayChangeTracking, withChangeTracking } from "./proxy" -import { deepEquals } from "./utils" -import { SortedMap } from "./SortedMap" -import { - createSingleRowRefProxy, - toExpression, -} from "./query/builder/ref-proxy" -import { BTreeIndex } from "./indexes/btree-index.js" -import { IndexProxy, LazyIndexWrapper } from "./indexes/lazy-index.js" -import { createTransaction, getActiveTransaction } from "./transactions" -import { - CollectionInErrorStateError, - CollectionIsInErrorStateError, - CollectionRequiresConfigError, - CollectionRequiresSyncConfigError, - DeleteKeyNotFoundError, - DuplicateKeyError, - DuplicateKeySyncError, - InvalidCollectionStatusTransitionError, - InvalidSchemaError, - KeyUpdateNotAllowedError, - MissingDeleteHandlerError, - MissingInsertHandlerError, - MissingUpdateArgumentError, - MissingUpdateHandlerError, - NegativeActiveSubscribersError, - NoKeysPassedToDeleteError, - NoKeysPassedToUpdateError, - NoPendingSyncTransactionCommitError, - NoPendingSyncTransactionWriteError, - SchemaMustBeSynchronousError, - SchemaValidationError, - SyncCleanupError, - SyncTransactionAlreadyCommittedError, - SyncTransactionAlreadyCommittedWriteError, - UndefinedKeyError, - UpdateKeyNotFoundError, -} from "./errors" -import { CollectionEvents } from "./collection-events.js" -import type { - AllCollectionEvents, - CollectionEventHandler, -} from "./collection-events.js" -import { currentStateAsChanges } from "./change-events" -import { CollectionSubscription } from "./collection-subscription.js" -import type { Transaction } from "./transactions" -import type { StandardSchemaV1 } from "@standard-schema/spec" -import type { SingleRowRefProxy } from "./query/builder/ref-proxy" -import type { - ChangeMessage, - CollectionConfig, - CollectionStatus, - CurrentStateAsChangesOptions, - Fn, - InferSchemaInput, - InferSchemaOutput, - InsertConfig, - OperationConfig, - OptimisticChangeMessage, - PendingMutation, - StandardSchema, - SubscribeChangesOptions, - Transaction as TransactionType, - TransactionWithMutations, - UtilsRecord, - WritableDeep, -} from "./types" -import type { IndexOptions } from "./indexes/index-options.js" -import type { BaseIndex, IndexResolver } from "./indexes/base-index.js" - -interface PendingSyncedTransaction> { - committed: boolean - operations: Array> - truncate?: boolean - deletedKeys: Set -} - -/** - * Enhanced Collection interface that includes both data type T and utilities TUtils - * @template T - The type of items in the collection - * @template TKey - The type of the key for the collection - * @template TUtils - The utilities record type - * @template TInsertInput - The type for insert operations (can be different from T for schemas with defaults) - */ -export interface Collection< - T extends object = Record, - TKey extends string | number = string | number, - TUtils extends UtilsRecord = {}, - TSchema extends StandardSchemaV1 = StandardSchemaV1, - TInsertInput extends object = T, -> extends CollectionImpl { - readonly utils: TUtils -} - -/** - * Creates a new Collection instance with the given configuration - * - * @template T - The schema type if a schema is provided, otherwise the type of items in the collection - * @template TKey - The type of the key for the collection - * @template TUtils - The utilities record type - * @param options - Collection options with optional utilities - * @returns A new Collection with utilities exposed both at top level and under .utils - * - * @example - * // Pattern 1: With operation handlers (direct collection calls) - * const todos = createCollection({ - * id: "todos", - * getKey: (todo) => todo.id, - * schema, - * onInsert: async ({ transaction, collection }) => { - * // Send to API - * await api.createTodo(transaction.mutations[0].modified) - * }, - * onUpdate: async ({ transaction, collection }) => { - * await api.updateTodo(transaction.mutations[0].modified) - * }, - * onDelete: async ({ transaction, collection }) => { - * await api.deleteTodo(transaction.mutations[0].key) - * }, - * sync: { sync: () => {} } - * }) - * - * // Direct usage (handlers manage transactions) - * const tx = todos.insert({ id: "1", text: "Buy milk", completed: false }) - * await tx.isPersisted.promise - * - * @example - * // Pattern 2: Manual transaction management - * const todos = createCollection({ - * getKey: (todo) => todo.id, - * schema: todoSchema, - * sync: { sync: () => {} } - * }) - * - * // Explicit transaction usage - * const tx = createTransaction({ - * mutationFn: async ({ transaction }) => { - * // Handle all mutations in transaction - * await api.saveChanges(transaction.mutations) - * } - * }) - * - * tx.mutate(() => { - * todos.insert({ id: "1", text: "Buy milk" }) - * todos.update("2", draft => { draft.completed = true }) - * }) - * - * await tx.isPersisted.promise - * - * @example - * // Using schema for type inference (preferred as it also gives you client side validation) - * const todoSchema = z.object({ - * id: z.string(), - * title: z.string(), - * completed: z.boolean() - * }) - * - * const todos = createCollection({ - * schema: todoSchema, - * getKey: (todo) => todo.id, - * sync: { sync: () => {} } - * }) - * - */ - -// Overload for when schema is provided -export function createCollection< - T extends StandardSchemaV1, - TKey extends string | number = string | number, - TUtils extends UtilsRecord = {}, ->( - options: CollectionConfig, TKey, T> & { - schema: T - utils?: TUtils - } -): Collection, TKey, TUtils, T, InferSchemaInput> - -// Overload for when no schema is provided -// the type T needs to be passed explicitly unless it can be inferred from the getKey function in the config -export function createCollection< - T extends object, - TKey extends string | number = string | number, - TUtils extends UtilsRecord = {}, ->( - options: CollectionConfig & { - schema?: never // prohibit schema if an explicit type is provided - utils?: TUtils - } -): Collection - -// Implementation -export function createCollection( - options: CollectionConfig & { - schema?: StandardSchemaV1 - utils?: UtilsRecord - } -): Collection { - const collection = new CollectionImpl( - options - ) - - // Copy utils to both top level and .utils namespace - if (options.utils) { - collection.utils = { ...options.utils } - } else { - collection.utils = {} - } - - return collection -} - -export class CollectionImpl< - TOutput extends object = Record, - TKey extends string | number = string | number, - TUtils extends UtilsRecord = {}, - TSchema extends StandardSchemaV1 = StandardSchemaV1, - TInput extends object = TOutput, -> { - public config: CollectionConfig - - // Core state - make public for testing - public transactions: SortedMap> - public pendingSyncedTransactions: Array> = - [] - public syncedData: Map | SortedMap - public syncedMetadata = new Map() - - // Optimistic state tracking - make public for testing - public optimisticUpserts = new Map() - public optimisticDeletes = new Set() - - // Cached size for performance - private _size = 0 - - // Index storage - private lazyIndexes = new Map>() - private resolvedIndexes = new Map>() - private isIndexesResolved = false - private indexCounter = 0 - - // Event system - private changeSubscriptions = new Set() - - // Utilities namespace - // This is populated by createCollection - public utils: Record = {} - - // State used for computing the change events - private syncedKeys = new Set() - private preSyncVisibleState = new Map() - private recentlySyncedKeys = new Set() - private hasReceivedFirstCommit = false - private isCommittingSyncTransactions = false - - // Array to store one-time ready listeners - private onFirstReadyCallbacks: Array<() => void> = [] - private hasBeenReady = false - - // Event batching for preventing duplicate emissions during transaction flows - private batchedEvents: Array> = [] - private shouldBatchEvents = false - - // Lifecycle management - private _status: CollectionStatus = `idle` - private activeSubscribersCount = 0 - private gcTimeoutId: ReturnType | null = null - private preloadPromise: Promise | null = null - private syncCleanupFn: (() => void) | null = null - - // Event system - private events: CollectionEvents - - /** - * Register a callback to be executed when the collection first becomes ready - * Useful for preloading collections - * @param callback Function to call when the collection first becomes ready - * @example - * collection.onFirstReady(() => { - * console.log('Collection is ready for the first time') - * // Safe to access collection.state now - * }) - */ - public onFirstReady(callback: () => void): void { - // If already ready, call immediately - if (this.hasBeenReady) { - callback() - return - } - - this.onFirstReadyCallbacks.push(callback) - } - - /** - * Check if the collection is ready for use - * Returns true if the collection has been marked as ready by its sync implementation - * @returns true if the collection is ready, false otherwise - * @example - * if (collection.isReady()) { - * console.log('Collection is ready, data is available') - * // Safe to access collection.state - * } else { - * console.log('Collection is still loading') - * } - */ - public isReady(): boolean { - return this._status === `ready` - } - - /** - * Mark the collection as ready for use - * This is called by sync implementations to explicitly signal that the collection is ready, - * providing a more intuitive alternative to using commits for readiness signaling - * @private - Should only be called by sync implementations - */ - private markReady(): void { - // Can transition to ready from loading or initialCommit states - if (this._status === `loading` || this._status === `initialCommit`) { - this.setStatus(`ready`) - - // Call any registered first ready callbacks (only on first time becoming ready) - if (!this.hasBeenReady) { - this.hasBeenReady = true - - // Also mark as having received first commit for backwards compatibility - if (!this.hasReceivedFirstCommit) { - this.hasReceivedFirstCommit = true - } - - const callbacks = [...this.onFirstReadyCallbacks] - this.onFirstReadyCallbacks = [] - callbacks.forEach((callback) => callback()) - } - } - - // Always notify dependents when markReady is called, after status is set - // This ensures live queries get notified when their dependencies become ready - if (this.changeSubscriptions.size > 0) { - this.emitEmptyReadyEvent() - } - } - - public id = `` - - /** - * Gets the current status of the collection - */ - public get status(): CollectionStatus { - return this._status - } - - /** - * Get the number of subscribers to the collection - */ - public get subscriberCount(): number { - return this.activeSubscribersCount - } - - /** - * Validates that the collection is in a usable state for data operations - * @private - */ - private validateCollectionUsable(operation: string): void { - switch (this._status) { - case `error`: - throw new CollectionInErrorStateError(operation, this.id) - case `cleaned-up`: - // Automatically restart the collection when operations are called on cleaned-up collections - this.startSync() - break - } - } - - /** - * Validates state transitions to prevent invalid status changes - * @private - */ - private validateStatusTransition( - from: CollectionStatus, - to: CollectionStatus - ): void { - if (from === to) { - // Allow same state transitions - return - } - const validTransitions: Record< - CollectionStatus, - Array - > = { - idle: [`loading`, `error`, `cleaned-up`], - loading: [`initialCommit`, `ready`, `error`, `cleaned-up`], - initialCommit: [`ready`, `error`, `cleaned-up`], - ready: [`cleaned-up`, `error`], - error: [`cleaned-up`, `idle`], - "cleaned-up": [`loading`, `error`], - } - - if (!validTransitions[from].includes(to)) { - throw new InvalidCollectionStatusTransitionError(from, to, this.id) - } - } - - /** - * Safely update the collection status with validation - * @private - */ - private setStatus(newStatus: CollectionStatus): void { - this.validateStatusTransition(this._status, newStatus) - const previousStatus = this._status - this._status = newStatus - - // Resolve indexes when collection becomes ready - if (newStatus === `ready` && !this.isIndexesResolved) { - // Resolve indexes asynchronously without blocking - this.resolveAllIndexes().catch((error) => { - console.warn(`Failed to resolve indexes:`, error) - }) - } - - // Emit event - this.events.emitStatusChange(newStatus, previousStatus) - } - - /** - * Creates a new Collection instance - * - * @param config - Configuration object for the collection - * @throws Error if sync config is missing - */ - constructor(config: CollectionConfig) { - // eslint-disable-next-line - if (!config) { - throw new CollectionRequiresConfigError() - } - if (config.id) { - this.id = config.id - } else { - this.id = crypto.randomUUID() - } - - // eslint-disable-next-line - if (!config.sync) { - throw new CollectionRequiresSyncConfigError() - } - - this.transactions = new SortedMap>((a, b) => - a.compareCreatedAt(b) - ) - - // Set default values for optional config properties - this.config = { - ...config, - autoIndex: config.autoIndex ?? `eager`, - } - - // Set up data storage with optional comparison function - if (this.config.compare) { - this.syncedData = new SortedMap(this.config.compare) - } else { - this.syncedData = new Map() - } - - // Set up event system - this.events = new CollectionEvents(this) - - // Only start sync immediately if explicitly enabled - if (config.startSync === true) { - this.startSync() - } - } - - /** - * Start sync immediately - internal method for compiled queries - * This bypasses lazy loading for special cases like live query results - */ - public startSyncImmediate(): void { - this.startSync() - } - - /** - * Start the sync process for this collection - * This is called when the collection is first accessed or preloaded - */ - private startSync(): void { - if (this._status !== `idle` && this._status !== `cleaned-up`) { - return // Already started or in progress - } - - this.setStatus(`loading`) - - try { - const cleanupFn = this.config.sync.sync({ - collection: this, - begin: () => { - this.pendingSyncedTransactions.push({ - committed: false, - operations: [], - deletedKeys: new Set(), - }) - }, - write: (messageWithoutKey: Omit, `key`>) => { - const pendingTransaction = - this.pendingSyncedTransactions[ - this.pendingSyncedTransactions.length - 1 - ] - if (!pendingTransaction) { - throw new NoPendingSyncTransactionWriteError() - } - if (pendingTransaction.committed) { - throw new SyncTransactionAlreadyCommittedWriteError() - } - const key = this.getKeyFromItem(messageWithoutKey.value) - - // Check if an item with this key already exists when inserting - if (messageWithoutKey.type === `insert`) { - const insertingIntoExistingSynced = this.syncedData.has(key) - const hasPendingDeleteForKey = - pendingTransaction.deletedKeys.has(key) - const isTruncateTransaction = pendingTransaction.truncate === true - // Allow insert after truncate in the same transaction even if it existed in syncedData - if ( - insertingIntoExistingSynced && - !hasPendingDeleteForKey && - !isTruncateTransaction - ) { - throw new DuplicateKeySyncError(key, this.id) - } - } - - const message: ChangeMessage = { - ...messageWithoutKey, - key, - } - pendingTransaction.operations.push(message) - - if (messageWithoutKey.type === `delete`) { - pendingTransaction.deletedKeys.add(key) - } - }, - commit: () => { - const pendingTransaction = - this.pendingSyncedTransactions[ - this.pendingSyncedTransactions.length - 1 - ] - if (!pendingTransaction) { - throw new NoPendingSyncTransactionCommitError() - } - if (pendingTransaction.committed) { - throw new SyncTransactionAlreadyCommittedError() - } - - pendingTransaction.committed = true - - // Update status to initialCommit when transitioning from loading - // This indicates we're in the process of committing the first transaction - if (this._status === `loading`) { - this.setStatus(`initialCommit`) - } - - this.commitPendingTransactions() - }, - markReady: () => { - this.markReady() - }, - truncate: () => { - const pendingTransaction = - this.pendingSyncedTransactions[ - this.pendingSyncedTransactions.length - 1 - ] - if (!pendingTransaction) { - throw new NoPendingSyncTransactionWriteError() - } - if (pendingTransaction.committed) { - throw new SyncTransactionAlreadyCommittedWriteError() - } - - // Clear all operations from the current transaction - pendingTransaction.operations = [] - pendingTransaction.deletedKeys.clear() - - // Mark the transaction as a truncate operation. During commit, this triggers: - // - Delete events for all previously synced keys (excluding optimistic-deleted keys) - // - Clearing of syncedData/syncedMetadata - // - Subsequent synced ops applied on the fresh base - // - Finally, optimistic mutations re-applied on top (single batch) - pendingTransaction.truncate = true - }, - }) - - // Store cleanup function if provided - this.syncCleanupFn = typeof cleanupFn === `function` ? cleanupFn : null - } catch (error) { - this.setStatus(`error`) - throw error - } - } - - /** - * Preload the collection data by starting sync if not already started - * Multiple concurrent calls will share the same promise - */ - public preload(): Promise { - if (this.preloadPromise) { - return this.preloadPromise - } - - this.preloadPromise = new Promise((resolve, reject) => { - if (this._status === `ready`) { - resolve() - return - } - - if (this._status === `error`) { - reject(new CollectionIsInErrorStateError()) - return - } - - // Register callback BEFORE starting sync to avoid race condition - this.onFirstReady(() => { - resolve() - }) - - // Start sync if collection hasn't started yet or was cleaned up - if (this._status === `idle` || this._status === `cleaned-up`) { - try { - this.startSync() - } catch (error) { - reject(error) - return - } - } - }) - - return this.preloadPromise - } - - /** - * Clean up the collection by stopping sync and clearing data - * This can be called manually or automatically by garbage collection - */ - public async cleanup(): Promise { - // Clear GC timeout - if (this.gcTimeoutId) { - clearTimeout(this.gcTimeoutId) - this.gcTimeoutId = null - } - - // Stop sync - wrap in try/catch since it's user-provided code - try { - if (this.syncCleanupFn) { - this.syncCleanupFn() - this.syncCleanupFn = null - } - } catch (error) { - // Re-throw in a microtask to surface the error after cleanup completes - queueMicrotask(() => { - if (error instanceof Error) { - // Preserve the original error and stack trace - const wrappedError = new SyncCleanupError(this.id, error) - wrappedError.cause = error - wrappedError.stack = error.stack - throw wrappedError - } else { - throw new SyncCleanupError(this.id, error as Error | string) - } - }) - } - - // Clear data - this.syncedData.clear() - this.syncedMetadata.clear() - this.optimisticUpserts.clear() - this.optimisticDeletes.clear() - this._size = 0 - this.pendingSyncedTransactions = [] - this.syncedKeys.clear() - this.hasReceivedFirstCommit = false - this.hasBeenReady = false - this.onFirstReadyCallbacks = [] - this.preloadPromise = null - this.batchedEvents = [] - this.shouldBatchEvents = false - - this.events.cleanup() - - // Update status - this.setStatus(`cleaned-up`) - - return Promise.resolve() - } - - /** - * Start the garbage collection timer - * Called when the collection becomes inactive (no subscribers) - */ - private startGCTimer(): void { - if (this.gcTimeoutId) { - clearTimeout(this.gcTimeoutId) - } - - const gcTime = this.config.gcTime ?? 300000 // 5 minutes default - - // If gcTime is 0, GC is disabled - if (gcTime === 0) { - return - } - - this.gcTimeoutId = setTimeout(() => { - if (this.activeSubscribersCount === 0) { - this.cleanup() - } - }, gcTime) - } - - /** - * Cancel the garbage collection timer - * Called when the collection becomes active again - */ - private cancelGCTimer(): void { - if (this.gcTimeoutId) { - clearTimeout(this.gcTimeoutId) - this.gcTimeoutId = null - } - } - - /** - * Increment the active subscribers count and start sync if needed - */ - private addSubscriber(): void { - const previousSubscriberCount = this.activeSubscribersCount - this.activeSubscribersCount++ - this.cancelGCTimer() - - // Start sync if collection was cleaned up - if (this._status === `cleaned-up` || this._status === `idle`) { - this.startSync() - } - - this.events.emitSubscribersChange( - this.activeSubscribersCount, - previousSubscriberCount - ) - } - - /** - * Decrement the active subscribers count and start GC timer if needed - */ - private removeSubscriber(): void { - const previousSubscriberCount = this.activeSubscribersCount - this.activeSubscribersCount-- - - if (this.activeSubscribersCount === 0) { - this.startGCTimer() - } else if (this.activeSubscribersCount < 0) { - throw new NegativeActiveSubscribersError() - } - - this.events.emitSubscribersChange( - this.activeSubscribersCount, - previousSubscriberCount - ) - } - - /** - * Recompute optimistic state from active transactions - */ - private recomputeOptimisticState( - triggeredByUserAction: boolean = false - ): void { - // Skip redundant recalculations when we're in the middle of committing sync transactions - if (this.isCommittingSyncTransactions) { - return - } - - const previousState = new Map(this.optimisticUpserts) - const previousDeletes = new Set(this.optimisticDeletes) - - // Clear current optimistic state - this.optimisticUpserts.clear() - this.optimisticDeletes.clear() - - const activeTransactions: Array> = [] - - for (const transaction of this.transactions.values()) { - if (![`completed`, `failed`].includes(transaction.state)) { - activeTransactions.push(transaction) - } - } - - // Apply active transactions only (completed transactions are handled by sync operations) - for (const transaction of activeTransactions) { - for (const mutation of transaction.mutations) { - if (mutation.collection === this && mutation.optimistic) { - switch (mutation.type) { - case `insert`: - case `update`: - this.optimisticUpserts.set( - mutation.key, - mutation.modified as TOutput - ) - this.optimisticDeletes.delete(mutation.key) - break - case `delete`: - this.optimisticUpserts.delete(mutation.key) - this.optimisticDeletes.add(mutation.key) - break - } - } - } - } - - // Update cached size - this._size = this.calculateSize() - - // Collect events for changes - const events: Array> = [] - this.collectOptimisticChanges(previousState, previousDeletes, events) - - // Filter out events for recently synced keys to prevent duplicates - // BUT: Only filter out events that are actually from sync operations - // New user transactions should NOT be filtered even if the key was recently synced - const filteredEventsBySyncStatus = events.filter((event) => { - if (!this.recentlySyncedKeys.has(event.key)) { - return true // Key not recently synced, allow event through - } - - // Key was recently synced - allow if this is a user-triggered action - if (triggeredByUserAction) { - return true - } - - // Otherwise filter out duplicate sync events - return false - }) - - // Filter out redundant delete events if there are pending sync transactions - // that will immediately restore the same data, but only for completed transactions - // IMPORTANT: Skip complex filtering for user-triggered actions to prevent UI blocking - if (this.pendingSyncedTransactions.length > 0 && !triggeredByUserAction) { - const pendingSyncKeys = new Set() - - // Collect keys from pending sync operations - for (const transaction of this.pendingSyncedTransactions) { - for (const operation of transaction.operations) { - pendingSyncKeys.add(operation.key as TKey) - } - } - - // Only filter out delete events for keys that: - // 1. Have pending sync operations AND - // 2. Are from completed transactions (being cleaned up) - const filteredEvents = filteredEventsBySyncStatus.filter((event) => { - if (event.type === `delete` && pendingSyncKeys.has(event.key)) { - // Check if this delete is from clearing optimistic state of completed transactions - // We can infer this by checking if we have no remaining optimistic mutations for this key - const hasActiveOptimisticMutation = activeTransactions.some((tx) => - tx.mutations.some( - (m) => m.collection === this && m.key === event.key - ) - ) - - if (!hasActiveOptimisticMutation) { - return false // Skip this delete event as sync will restore the data - } - } - return true - }) - - // Update indexes for the filtered events - if (filteredEvents.length > 0) { - this.updateIndexes(filteredEvents) - } - this.emitEvents(filteredEvents, triggeredByUserAction) - } else { - // Update indexes for all events - if (filteredEventsBySyncStatus.length > 0) { - this.updateIndexes(filteredEventsBySyncStatus) - } - // Emit all events if no pending sync transactions - this.emitEvents(filteredEventsBySyncStatus, triggeredByUserAction) - } - } - - /** - * Calculate the current size based on synced data and optimistic changes - */ - private calculateSize(): number { - const syncedSize = this.syncedData.size - const deletesFromSynced = Array.from(this.optimisticDeletes).filter( - (key) => this.syncedData.has(key) && !this.optimisticUpserts.has(key) - ).length - const upsertsNotInSynced = Array.from(this.optimisticUpserts.keys()).filter( - (key) => !this.syncedData.has(key) - ).length - - return syncedSize - deletesFromSynced + upsertsNotInSynced - } - - /** - * Collect events for optimistic changes - */ - private collectOptimisticChanges( - previousUpserts: Map, - previousDeletes: Set, - events: Array> - ): void { - const allKeys = new Set([ - ...previousUpserts.keys(), - ...this.optimisticUpserts.keys(), - ...previousDeletes, - ...this.optimisticDeletes, - ]) - - for (const key of allKeys) { - const currentValue = this.get(key) - const previousValue = this.getPreviousValue( - key, - previousUpserts, - previousDeletes - ) - - if (previousValue !== undefined && currentValue === undefined) { - events.push({ type: `delete`, key, value: previousValue }) - } else if (previousValue === undefined && currentValue !== undefined) { - events.push({ type: `insert`, key, value: currentValue }) - } else if ( - previousValue !== undefined && - currentValue !== undefined && - previousValue !== currentValue - ) { - events.push({ - type: `update`, - key, - value: currentValue, - previousValue, - }) - } - } - } - - /** - * Get the previous value for a key given previous optimistic state - */ - private getPreviousValue( - key: TKey, - previousUpserts: Map, - previousDeletes: Set - ): TOutput | undefined { - if (previousDeletes.has(key)) { - return undefined - } - if (previousUpserts.has(key)) { - return previousUpserts.get(key) - } - return this.syncedData.get(key) - } - - /** - * Emit an empty ready event to notify subscribers that the collection is ready - * This bypasses the normal empty array check in emitEvents - */ - private emitEmptyReadyEvent(): void { - // Emit empty array directly to all subscribers - for (const subscription of this.changeSubscriptions) { - subscription.emitEvents([]) - } - } - - /** - * Emit events either immediately or batch them for later emission - */ - private emitEvents( - changes: Array>, - forceEmit = false - ): void { - // Skip batching for user actions (forceEmit=true) to keep UI responsive - if (this.shouldBatchEvents && !forceEmit) { - // Add events to the batch - this.batchedEvents.push(...changes) - return - } - - // Either we're not batching, or we're forcing emission (user action or ending batch cycle) - let eventsToEmit = changes - - // If we have batched events and this is a forced emit, combine them - if (this.batchedEvents.length > 0 && forceEmit) { - eventsToEmit = [...this.batchedEvents, ...changes] - this.batchedEvents = [] - this.shouldBatchEvents = false - } - - if (eventsToEmit.length === 0) return - - // Emit to all listeners - for (const subscription of this.changeSubscriptions) { - subscription.emitEvents(eventsToEmit) - } - } - - /** - * Get the current value for a key (virtual derived state) - */ - public get(key: TKey): TOutput | undefined { - // Check if optimistically deleted - if (this.optimisticDeletes.has(key)) { - return undefined - } - - // Check optimistic upserts first - if (this.optimisticUpserts.has(key)) { - return this.optimisticUpserts.get(key) - } - - // Fall back to synced data - return this.syncedData.get(key) - } - - /** - * Check if a key exists in the collection (virtual derived state) - */ - public has(key: TKey): boolean { - // Check if optimistically deleted - if (this.optimisticDeletes.has(key)) { - return false - } - - // Check optimistic upserts first - if (this.optimisticUpserts.has(key)) { - return true - } - - // Fall back to synced data - return this.syncedData.has(key) - } - - /** - * Get the current size of the collection (cached) - */ - public get size(): number { - return this._size - } - - /** - * Get all keys (virtual derived state) - */ - public *keys(): IterableIterator { - // Yield keys from synced data, skipping any that are deleted. - for (const key of this.syncedData.keys()) { - if (!this.optimisticDeletes.has(key)) { - yield key - } - } - // Yield keys from upserts that were not already in synced data. - for (const key of this.optimisticUpserts.keys()) { - if (!this.syncedData.has(key) && !this.optimisticDeletes.has(key)) { - // The optimisticDeletes check is technically redundant if inserts/updates always remove from deletes, - // but it's safer to keep it. - yield key - } - } - } - - /** - * Get all values (virtual derived state) - */ - public *values(): IterableIterator { - for (const key of this.keys()) { - const value = this.get(key) - if (value !== undefined) { - yield value - } - } - } - - /** - * Get all entries (virtual derived state) - */ - public *entries(): IterableIterator<[TKey, TOutput]> { - for (const key of this.keys()) { - const value = this.get(key) - if (value !== undefined) { - yield [key, value] - } - } - } - - /** - * Get all entries (virtual derived state) - */ - public *[Symbol.iterator](): IterableIterator<[TKey, TOutput]> { - for (const [key, value] of this.entries()) { - yield [key, value] - } - } - - /** - * Execute a callback for each entry in the collection - */ - public forEach( - callbackfn: (value: TOutput, key: TKey, index: number) => void - ): void { - let index = 0 - for (const [key, value] of this.entries()) { - callbackfn(value, key, index++) - } - } - - /** - * Create a new array with the results of calling a function for each entry in the collection - */ - public map( - callbackfn: (value: TOutput, key: TKey, index: number) => U - ): Array { - const result: Array = [] - let index = 0 - for (const [key, value] of this.entries()) { - result.push(callbackfn(value, key, index++)) - } - return result - } - - /** - * Attempts to commit pending synced transactions if there are no active transactions - * This method processes operations from pending transactions and applies them to the synced data - */ - commitPendingTransactions = () => { - // Check if there are any persisting transaction - let hasPersistingTransaction = false - for (const transaction of this.transactions.values()) { - if (transaction.state === `persisting`) { - hasPersistingTransaction = true - break - } - } - - // pending synced transactions could be either `committed` or still open. - // we only want to process `committed` transactions here - const { - committedSyncedTransactions, - uncommittedSyncedTransactions, - hasTruncateSync, - } = this.pendingSyncedTransactions.reduce( - (acc, t) => { - if (t.committed) { - acc.committedSyncedTransactions.push(t) - if (t.truncate === true) { - acc.hasTruncateSync = true - } - } else { - acc.uncommittedSyncedTransactions.push(t) - } - return acc - }, - { - committedSyncedTransactions: [] as Array< - PendingSyncedTransaction - >, - uncommittedSyncedTransactions: [] as Array< - PendingSyncedTransaction - >, - hasTruncateSync: false, - } - ) - - if (!hasPersistingTransaction || hasTruncateSync) { - // Set flag to prevent redundant optimistic state recalculations - this.isCommittingSyncTransactions = true - - // First collect all keys that will be affected by sync operations - const changedKeys = new Set() - for (const transaction of committedSyncedTransactions) { - for (const operation of transaction.operations) { - changedKeys.add(operation.key as TKey) - } - } - - // Use pre-captured state if available (from optimistic scenarios), - // otherwise capture current state (for pure sync scenarios) - let currentVisibleState = this.preSyncVisibleState - if (currentVisibleState.size === 0) { - // No pre-captured state, capture it now for pure sync operations - currentVisibleState = new Map() - for (const key of changedKeys) { - const currentValue = this.get(key) - if (currentValue !== undefined) { - currentVisibleState.set(key, currentValue) - } - } - } - - const events: Array> = [] - const rowUpdateMode = this.config.sync.rowUpdateMode || `partial` - - for (const transaction of committedSyncedTransactions) { - // Handle truncate operations first - if (transaction.truncate) { - // TRUNCATE PHASE - // 1) Emit a delete for every currently-synced key so downstream listeners/indexes - // observe a clear-before-rebuild. We intentionally skip keys already in - // optimisticDeletes because their delete was previously emitted by the user. - for (const key of this.syncedData.keys()) { - if (this.optimisticDeletes.has(key)) continue - const previousValue = - this.optimisticUpserts.get(key) || this.syncedData.get(key) - if (previousValue !== undefined) { - events.push({ type: `delete`, key, value: previousValue }) - } - } - - // 2) Clear the authoritative synced base. Subsequent server ops in this - // same commit will rebuild the base atomically. - this.syncedData.clear() - this.syncedMetadata.clear() - this.syncedKeys.clear() - - // 3) Clear currentVisibleState for truncated keys to ensure subsequent operations - // are compared against the post-truncate state (undefined) rather than pre-truncate state - // This ensures that re-inserted keys are emitted as INSERT events, not UPDATE events - for (const key of changedKeys) { - currentVisibleState.delete(key) - } - } - - for (const operation of transaction.operations) { - const key = operation.key as TKey - this.syncedKeys.add(key) - - // Update metadata - switch (operation.type) { - case `insert`: - this.syncedMetadata.set(key, operation.metadata) - break - case `update`: - this.syncedMetadata.set( - key, - Object.assign( - {}, - this.syncedMetadata.get(key), - operation.metadata - ) - ) - break - case `delete`: - this.syncedMetadata.delete(key) - break - } - - // Update synced data - switch (operation.type) { - case `insert`: - this.syncedData.set(key, operation.value) - break - case `update`: { - if (rowUpdateMode === `partial`) { - const updatedValue = Object.assign( - {}, - this.syncedData.get(key), - operation.value - ) - this.syncedData.set(key, updatedValue) - } else { - this.syncedData.set(key, operation.value) - } - break - } - case `delete`: - this.syncedData.delete(key) - break - } - } - } - - // After applying synced operations, if this commit included a truncate, - // re-apply optimistic mutations on top of the fresh synced base. This ensures - // the UI preserves local intent while respecting server rebuild semantics. - // Ordering: deletes (above) -> server ops (just applied) -> optimistic upserts. - if (hasTruncateSync) { - // Avoid duplicating keys that were inserted/updated by synced operations in this commit - const syncedInsertedOrUpdatedKeys = new Set() - for (const t of committedSyncedTransactions) { - for (const op of t.operations) { - if (op.type === `insert` || op.type === `update`) { - syncedInsertedOrUpdatedKeys.add(op.key as TKey) - } - } - } - - // Build re-apply sets from ACTIVE optimistic transactions against the new synced base - // We do not copy maps; we compute intent directly from transactions to avoid drift. - const reapplyUpserts = new Map() - const reapplyDeletes = new Set() - - for (const tx of this.transactions.values()) { - if ([`completed`, `failed`].includes(tx.state)) continue - for (const mutation of tx.mutations) { - if (mutation.collection !== this || !mutation.optimistic) continue - const key = mutation.key as TKey - switch (mutation.type) { - case `insert`: - reapplyUpserts.set(key, mutation.modified as TOutput) - reapplyDeletes.delete(key) - break - case `update`: { - const base = this.syncedData.get(key) - const next = base - ? (Object.assign({}, base, mutation.changes) as TOutput) - : (mutation.modified as TOutput) - reapplyUpserts.set(key, next) - reapplyDeletes.delete(key) - break - } - case `delete`: - reapplyUpserts.delete(key) - reapplyDeletes.add(key) - break - } - } - } - - // Emit inserts for re-applied upserts, skipping any keys that have an optimistic delete. - // If the server also inserted/updated the same key in this batch, override that value - // with the optimistic value to preserve local intent. - for (const [key, value] of reapplyUpserts) { - if (reapplyDeletes.has(key)) continue - if (syncedInsertedOrUpdatedKeys.has(key)) { - let foundInsert = false - for (let i = events.length - 1; i >= 0; i--) { - const evt = events[i]! - if (evt.key === key && evt.type === `insert`) { - evt.value = value - foundInsert = true - break - } - } - if (!foundInsert) { - events.push({ type: `insert`, key, value }) - } - } else { - events.push({ type: `insert`, key, value }) - } - } - - // Finally, ensure we do NOT insert keys that have an outstanding optimistic delete. - if (events.length > 0 && reapplyDeletes.size > 0) { - const filtered: Array> = [] - for (const evt of events) { - if (evt.type === `insert` && reapplyDeletes.has(evt.key)) { - continue - } - filtered.push(evt) - } - events.length = 0 - events.push(...filtered) - } - - // Ensure listeners are active before emitting this critical batch - if (!this.isReady()) { - this.setStatus(`ready`) - } - } - - // Maintain optimistic state appropriately - // Clear optimistic state since sync operations will now provide the authoritative data. - // Any still-active user transactions will be re-applied below in recompute. - this.optimisticUpserts.clear() - this.optimisticDeletes.clear() - - // Reset flag and recompute optimistic state for any remaining active transactions - this.isCommittingSyncTransactions = false - for (const transaction of this.transactions.values()) { - if (![`completed`, `failed`].includes(transaction.state)) { - for (const mutation of transaction.mutations) { - if (mutation.collection === this && mutation.optimistic) { - switch (mutation.type) { - case `insert`: - case `update`: - this.optimisticUpserts.set( - mutation.key, - mutation.modified as TOutput - ) - this.optimisticDeletes.delete(mutation.key) - break - case `delete`: - this.optimisticUpserts.delete(mutation.key) - this.optimisticDeletes.add(mutation.key) - break - } - } - } - } - } - - // Check for redundant sync operations that match completed optimistic operations - const completedOptimisticOps = new Map() - - for (const transaction of this.transactions.values()) { - if (transaction.state === `completed`) { - for (const mutation of transaction.mutations) { - if (mutation.collection === this && changedKeys.has(mutation.key)) { - completedOptimisticOps.set(mutation.key, { - type: mutation.type, - value: mutation.modified, - }) - } - } - } - } - - // Now check what actually changed in the final visible state - for (const key of changedKeys) { - const previousVisibleValue = currentVisibleState.get(key) - const newVisibleValue = this.get(key) // This returns the new derived state - - // Check if this sync operation is redundant with a completed optimistic operation - const completedOp = completedOptimisticOps.get(key) - const isRedundantSync = - completedOp && - newVisibleValue !== undefined && - deepEquals(completedOp.value, newVisibleValue) - - if (!isRedundantSync) { - if ( - previousVisibleValue === undefined && - newVisibleValue !== undefined - ) { - events.push({ - type: `insert`, - key, - value: newVisibleValue, - }) - } else if ( - previousVisibleValue !== undefined && - newVisibleValue === undefined - ) { - events.push({ - type: `delete`, - key, - value: previousVisibleValue, - }) - } else if ( - previousVisibleValue !== undefined && - newVisibleValue !== undefined && - !deepEquals(previousVisibleValue, newVisibleValue) - ) { - events.push({ - type: `update`, - key, - value: newVisibleValue, - previousValue: previousVisibleValue, - }) - } - } - } - - // Update cached size after synced data changes - this._size = this.calculateSize() - - // Update indexes for all events before emitting - if (events.length > 0) { - this.updateIndexes(events) - } - - // End batching and emit all events (combines any batched events with sync events) - this.emitEvents(events, true) - - this.pendingSyncedTransactions = uncommittedSyncedTransactions - - // Clear the pre-sync state since sync operations are complete - this.preSyncVisibleState.clear() - - // Clear recently synced keys after a microtask to allow recomputeOptimisticState to see them - Promise.resolve().then(() => { - this.recentlySyncedKeys.clear() - }) - - // Call any registered one-time commit listeners - if (!this.hasReceivedFirstCommit) { - this.hasReceivedFirstCommit = true - const callbacks = [...this.onFirstReadyCallbacks] - this.onFirstReadyCallbacks = [] - callbacks.forEach((callback) => callback()) - } - } - } - - /** - * Schedule cleanup of a transaction when it completes - * @private - */ - private scheduleTransactionCleanup(transaction: Transaction): void { - // Only schedule cleanup for transactions that aren't already completed - if (transaction.state === `completed`) { - this.transactions.delete(transaction.id) - return - } - - // Schedule cleanup when the transaction completes - transaction.isPersisted.promise - .then(() => { - // Transaction completed successfully, remove it immediately - this.transactions.delete(transaction.id) - }) - .catch(() => { - // Transaction failed, but we want to keep failed transactions for reference - // so don't remove it. - // This empty catch block is necessary to prevent unhandled promise rejections. - }) - } - - private ensureStandardSchema(schema: unknown): StandardSchema { - // If the schema already implements the standard-schema interface, return it - if (schema && `~standard` in (schema as {})) { - return schema as StandardSchema - } - - throw new InvalidSchemaError() - } - - public getKeyFromItem(item: TOutput): TKey { - return this.config.getKey(item) - } - - public generateGlobalKey(key: any, item: any): string { - if (typeof key === `undefined`) { - throw new UndefinedKeyError(item) - } - - return `KEY::${this.id}/${key}` - } - - /** - * Creates an index on a collection for faster queries. - * Indexes significantly improve query performance by allowing constant time lookups - * and logarithmic time range queries instead of full scans. - * - * @template TResolver - The type of the index resolver (constructor or async loader) - * @param indexCallback - Function that extracts the indexed value from each item - * @param config - Configuration including index type and type-specific options - * @returns An index proxy that provides access to the index when ready - * - * @example - * // Create a default B+ tree index - * const ageIndex = collection.createIndex((row) => row.age) - * - * // Create a ordered index with custom options - * const ageIndex = collection.createIndex((row) => row.age, { - * indexType: BTreeIndex, - * options: { compareFn: customComparator }, - * name: 'age_btree' - * }) - * - * // Create an async-loaded index - * const textIndex = collection.createIndex((row) => row.content, { - * indexType: async () => { - * const { FullTextIndex } = await import('./indexes/fulltext.js') - * return FullTextIndex - * }, - * options: { language: 'en' } - * }) - */ - public createIndex = typeof BTreeIndex>( - indexCallback: (row: SingleRowRefProxy) => any, - config: IndexOptions = {} - ): IndexProxy { - this.validateCollectionUsable(`createIndex`) - - const indexId = ++this.indexCounter - const singleRowRefProxy = createSingleRowRefProxy() - const indexExpression = indexCallback(singleRowRefProxy) - const expression = toExpression(indexExpression) - - // Default to BTreeIndex if no type specified - const resolver = config.indexType ?? (BTreeIndex as unknown as TResolver) - - // Create lazy wrapper - const lazyIndex = new LazyIndexWrapper( - indexId, - expression, - config.name, - resolver, - config.options, - this.entries() - ) - - this.lazyIndexes.set(indexId, lazyIndex) - - // For BTreeIndex, resolve immediately and synchronously - if ((resolver as unknown) === BTreeIndex) { - try { - const resolvedIndex = lazyIndex.getResolved() - this.resolvedIndexes.set(indexId, resolvedIndex) - } catch (error) { - console.warn(`Failed to resolve BTreeIndex:`, error) - } - } else if (typeof resolver === `function` && resolver.prototype) { - // Other synchronous constructors - resolve immediately - try { - const resolvedIndex = lazyIndex.getResolved() - this.resolvedIndexes.set(indexId, resolvedIndex) - } catch { - // Fallback to async resolution - this.resolveSingleIndex(indexId, lazyIndex).catch((error) => { - console.warn(`Failed to resolve single index:`, error) - }) - } - } else if (this.isIndexesResolved) { - // Async loader but indexes are already resolved - resolve this one - this.resolveSingleIndex(indexId, lazyIndex).catch((error) => { - console.warn(`Failed to resolve single index:`, error) - }) - } - - return new IndexProxy(indexId, lazyIndex) - } - - /** - * Resolve all lazy indexes (called when collection first syncs) - * @private - */ - private async resolveAllIndexes(): Promise { - if (this.isIndexesResolved) return - - const resolutionPromises = Array.from(this.lazyIndexes.entries()).map( - async ([indexId, lazyIndex]) => { - const resolvedIndex = await lazyIndex.resolve() - - // Build index with current data - resolvedIndex.build(this.entries()) - - this.resolvedIndexes.set(indexId, resolvedIndex) - return { indexId, resolvedIndex } - } - ) - - await Promise.all(resolutionPromises) - this.isIndexesResolved = true - } - - /** - * Resolve a single index immediately - * @private - */ - private async resolveSingleIndex( - indexId: number, - lazyIndex: LazyIndexWrapper - ): Promise> { - const resolvedIndex = await lazyIndex.resolve() - resolvedIndex.build(this.entries()) - this.resolvedIndexes.set(indexId, resolvedIndex) - return resolvedIndex - } - - /** - * Get resolved indexes for query optimization - */ - get indexes(): Map> { - return this.resolvedIndexes - } - - /** - * Updates all indexes when the collection changes - * @private - */ - private updateIndexes(changes: Array>): void { - for (const index of this.resolvedIndexes.values()) { - for (const change of changes) { - switch (change.type) { - case `insert`: - index.add(change.key, change.value) - break - case `update`: - if (change.previousValue) { - index.update(change.key, change.previousValue, change.value) - } else { - index.add(change.key, change.value) - } - break - case `delete`: - index.remove(change.key, change.value) - break - } - } - } - } - - public validateData( - data: unknown, - type: `insert` | `update`, - key?: TKey - ): TOutput | never { - if (!this.config.schema) return data as TOutput - - const standardSchema = this.ensureStandardSchema(this.config.schema) - - // For updates, we need to merge with the existing data before validation - if (type === `update` && key) { - // Get the existing data for this key - const existingData = this.get(key) - - if ( - existingData && - data && - typeof data === `object` && - typeof existingData === `object` - ) { - // Merge the update with the existing data - const mergedData = Object.assign({}, existingData, data) - - // Validate the merged data - const result = standardSchema[`~standard`].validate(mergedData) - - // Ensure validation is synchronous - if (result instanceof Promise) { - throw new SchemaMustBeSynchronousError() - } - - // If validation fails, throw a SchemaValidationError with the issues - if (`issues` in result && result.issues) { - const typedIssues = result.issues.map((issue) => ({ - message: issue.message, - path: issue.path?.map((p) => String(p)), - })) - throw new SchemaValidationError(type, typedIssues) - } - - // Extract only the modified keys from the validated result - const validatedMergedData = result.value as TOutput - const modifiedKeys = Object.keys(data) - const extractedChanges = Object.fromEntries( - modifiedKeys.map((k) => [k, validatedMergedData[k as keyof TOutput]]) - ) as TOutput - - return extractedChanges - } - } - - // For inserts or updates without existing data, validate the data directly - const result = standardSchema[`~standard`].validate(data) - - // Ensure validation is synchronous - if (result instanceof Promise) { - throw new SchemaMustBeSynchronousError() - } - - // If validation fails, throw a SchemaValidationError with the issues - if (`issues` in result && result.issues) { - const typedIssues = result.issues.map((issue) => ({ - message: issue.message, - path: issue.path?.map((p) => String(p)), - })) - throw new SchemaValidationError(type, typedIssues) - } - - return result.value as TOutput - } - - /** - * Inserts one or more items into the collection - * @param items - Single item or array of items to insert - * @param config - Optional configuration including metadata - * @returns A Transaction object representing the insert operation(s) - * @throws {SchemaValidationError} If the data fails schema validation - * @example - * // Insert a single todo (requires onInsert handler) - * const tx = collection.insert({ id: "1", text: "Buy milk", completed: false }) - * await tx.isPersisted.promise - * - * @example - * // Insert multiple todos at once - * const tx = collection.insert([ - * { id: "1", text: "Buy milk", completed: false }, - * { id: "2", text: "Walk dog", completed: true } - * ]) - * await tx.isPersisted.promise - * - * @example - * // Insert with metadata - * const tx = collection.insert({ id: "1", text: "Buy groceries" }, - * { metadata: { source: "mobile-app" } } - * ) - * await tx.isPersisted.promise - * - * @example - * // Handle errors - * try { - * const tx = collection.insert({ id: "1", text: "New item" }) - * await tx.isPersisted.promise - * console.log('Insert successful') - * } catch (error) { - * console.log('Insert failed:', error) - * } - */ - insert = (data: TInput | Array, config?: InsertConfig) => { - this.validateCollectionUsable(`insert`) - const ambientTransaction = getActiveTransaction() - - // If no ambient transaction exists, check for an onInsert handler early - if (!ambientTransaction && !this.config.onInsert) { - throw new MissingInsertHandlerError() - } - - const items = Array.isArray(data) ? data : [data] - const mutations: Array> = [] - - // Create mutations for each item - items.forEach((item) => { - // Validate the data against the schema if one exists - const validatedData = this.validateData(item, `insert`) - - // Check if an item with this ID already exists in the collection - const key = this.getKeyFromItem(validatedData) - if (this.has(key)) { - throw new DuplicateKeyError(key) - } - const globalKey = this.generateGlobalKey(key, item) - - const mutation: PendingMutation = { - mutationId: crypto.randomUUID(), - original: {}, - modified: validatedData, - // Pick the values from validatedData based on what's passed in - this is for cases - // where a schema has default values. The validated data has the extra default - // values but for changes, we just want to show the data that was actually passed in. - changes: Object.fromEntries( - Object.keys(item).map((k) => [ - k, - validatedData[k as keyof typeof validatedData], - ]) - ) as TInput, - globalKey, - key, - metadata: config?.metadata as unknown, - syncMetadata: this.config.sync.getSyncMetadata?.() || {}, - optimistic: config?.optimistic ?? true, - type: `insert`, - createdAt: new Date(), - updatedAt: new Date(), - collection: this, - } - - mutations.push(mutation) - }) - - // If an ambient transaction exists, use it - if (ambientTransaction) { - ambientTransaction.applyMutations(mutations) - - this.transactions.set(ambientTransaction.id, ambientTransaction) - this.scheduleTransactionCleanup(ambientTransaction) - this.recomputeOptimisticState(true) - - return ambientTransaction - } else { - // Create a new transaction with a mutation function that calls the onInsert handler - const directOpTransaction = createTransaction({ - mutationFn: async (params) => { - // Call the onInsert handler with the transaction and collection - return await this.config.onInsert!({ - transaction: - params.transaction as unknown as TransactionWithMutations< - TOutput, - `insert` - >, - collection: this as unknown as Collection, - }) - }, - }) - - // Apply mutations to the new transaction - directOpTransaction.applyMutations(mutations) - directOpTransaction.commit() - - // Add the transaction to the collection's transactions store - this.transactions.set(directOpTransaction.id, directOpTransaction) - this.scheduleTransactionCleanup(directOpTransaction) - this.recomputeOptimisticState(true) - - return directOpTransaction - } - } - - /** - * Updates one or more items in the collection using a callback function - * @param keys - Single key or array of keys to update - * @param configOrCallback - Either update configuration or update callback - * @param maybeCallback - Update callback if config was provided - * @returns A Transaction object representing the update operation(s) - * @throws {SchemaValidationError} If the updated data fails schema validation - * @example - * // Update single item by key - * const tx = collection.update("todo-1", (draft) => { - * draft.completed = true - * }) - * await tx.isPersisted.promise - * - * @example - * // Update multiple items - * const tx = collection.update(["todo-1", "todo-2"], (drafts) => { - * drafts.forEach(draft => { draft.completed = true }) - * }) - * await tx.isPersisted.promise - * - * @example - * // Update with metadata - * const tx = collection.update("todo-1", - * { metadata: { reason: "user update" } }, - * (draft) => { draft.text = "Updated text" } - * ) - * await tx.isPersisted.promise - * - * @example - * // Handle errors - * try { - * const tx = collection.update("item-1", draft => { draft.value = "new" }) - * await tx.isPersisted.promise - * console.log('Update successful') - * } catch (error) { - * console.log('Update failed:', error) - * } - */ - - // Overload 1: Update multiple items with a callback - update( - key: Array, - callback: (drafts: Array>) => void - ): TransactionType - - // Overload 2: Update multiple items with config and a callback - update( - keys: Array, - config: OperationConfig, - callback: (drafts: Array>) => void - ): TransactionType - - // Overload 3: Update a single item with a callback - update( - id: TKey | unknown, - callback: (draft: WritableDeep) => void - ): TransactionType - - // Overload 4: Update a single item with config and a callback - update( - id: TKey | unknown, - config: OperationConfig, - callback: (draft: WritableDeep) => void - ): TransactionType - - update( - keys: (TKey | unknown) | Array, - configOrCallback: - | ((draft: WritableDeep) => void) - | ((drafts: Array>) => void) - | OperationConfig, - maybeCallback?: - | ((draft: WritableDeep) => void) - | ((drafts: Array>) => void) - ) { - if (typeof keys === `undefined`) { - throw new MissingUpdateArgumentError() - } - - this.validateCollectionUsable(`update`) - - const ambientTransaction = getActiveTransaction() - - // If no ambient transaction exists, check for an onUpdate handler early - if (!ambientTransaction && !this.config.onUpdate) { - throw new MissingUpdateHandlerError() - } - - const isArray = Array.isArray(keys) - const keysArray = isArray ? keys : [keys] - - if (isArray && keysArray.length === 0) { - throw new NoKeysPassedToUpdateError() - } - - const callback = - typeof configOrCallback === `function` ? configOrCallback : maybeCallback! - const config = - typeof configOrCallback === `function` ? {} : configOrCallback - - // Get the current objects or empty objects if they don't exist - const currentObjects = keysArray.map((key) => { - const item = this.get(key) - if (!item) { - throw new UpdateKeyNotFoundError(key) - } - - return item - }) as unknown as Array - - let changesArray - if (isArray) { - // Use the proxy to track changes for all objects - changesArray = withArrayChangeTracking( - currentObjects, - callback as (draft: Array) => void - ) - } else { - const result = withChangeTracking( - currentObjects[0]!, - callback as (draft: TInput) => void - ) - changesArray = [result] - } - - // Create mutations for each object that has changes - const mutations: Array> = keysArray - .map((key, index) => { - const itemChanges = changesArray[index] // User-provided changes for this specific item - - // Skip items with no changes - if (!itemChanges || Object.keys(itemChanges).length === 0) { - return null - } - - const originalItem = currentObjects[index] as unknown as TOutput - // Validate the user-provided changes for this item - const validatedUpdatePayload = this.validateData( - itemChanges, - `update`, - key - ) - - // Construct the full modified item by applying the validated update payload to the original item - const modifiedItem = Object.assign( - {}, - originalItem, - validatedUpdatePayload - ) - - // Check if the ID of the item is being changed - const originalItemId = this.getKeyFromItem(originalItem) - const modifiedItemId = this.getKeyFromItem(modifiedItem) - - if (originalItemId !== modifiedItemId) { - throw new KeyUpdateNotAllowedError(originalItemId, modifiedItemId) - } - - const globalKey = this.generateGlobalKey(modifiedItemId, modifiedItem) - - return { - mutationId: crypto.randomUUID(), - original: originalItem, - modified: modifiedItem, - // Pick the values from modifiedItem based on what's passed in - this is for cases - // where a schema has default values or transforms. The modified data has the extra - // default or transformed values but for changes, we just want to show the data that - // was actually passed in. - changes: Object.fromEntries( - Object.keys(itemChanges).map((k) => [ - k, - modifiedItem[k as keyof typeof modifiedItem], - ]) - ) as TInput, - globalKey, - key, - metadata: config.metadata as unknown, - syncMetadata: (this.syncedMetadata.get(key) || {}) as Record< - string, - unknown - >, - optimistic: config.optimistic ?? true, - type: `update`, - createdAt: new Date(), - updatedAt: new Date(), - collection: this, - } - }) - .filter(Boolean) as Array> - - // If no changes were made, return an empty transaction early - if (mutations.length === 0) { - const emptyTransaction = createTransaction({ - mutationFn: async () => {}, - }) - emptyTransaction.commit() - // Schedule cleanup for empty transaction - this.scheduleTransactionCleanup(emptyTransaction) - return emptyTransaction - } - - // If an ambient transaction exists, use it - if (ambientTransaction) { - ambientTransaction.applyMutations(mutations) - - this.transactions.set(ambientTransaction.id, ambientTransaction) - this.scheduleTransactionCleanup(ambientTransaction) - this.recomputeOptimisticState(true) - - return ambientTransaction - } - - // No need to check for onUpdate handler here as we've already checked at the beginning - - // Create a new transaction with a mutation function that calls the onUpdate handler - const directOpTransaction = createTransaction({ - mutationFn: async (params) => { - // Call the onUpdate handler with the transaction and collection - return this.config.onUpdate!({ - transaction: - params.transaction as unknown as TransactionWithMutations< - TOutput, - `update` - >, - collection: this as unknown as Collection, - }) - }, - }) - - // Apply mutations to the new transaction - directOpTransaction.applyMutations(mutations) - directOpTransaction.commit() - - // Add the transaction to the collection's transactions store - - this.transactions.set(directOpTransaction.id, directOpTransaction) - this.scheduleTransactionCleanup(directOpTransaction) - this.recomputeOptimisticState(true) - - return directOpTransaction - } - - /** - * Deletes one or more items from the collection - * @param keys - Single key or array of keys to delete - * @param config - Optional configuration including metadata - * @returns A Transaction object representing the delete operation(s) - * @example - * // Delete a single item - * const tx = collection.delete("todo-1") - * await tx.isPersisted.promise - * - * @example - * // Delete multiple items - * const tx = collection.delete(["todo-1", "todo-2"]) - * await tx.isPersisted.promise - * - * @example - * // Delete with metadata - * const tx = collection.delete("todo-1", { metadata: { reason: "completed" } }) - * await tx.isPersisted.promise - * - * @example - * // Handle errors - * try { - * const tx = collection.delete("item-1") - * await tx.isPersisted.promise - * console.log('Delete successful') - * } catch (error) { - * console.log('Delete failed:', error) - * } - */ - delete = ( - keys: Array | TKey, - config?: OperationConfig - ): TransactionType => { - this.validateCollectionUsable(`delete`) - - const ambientTransaction = getActiveTransaction() - - // If no ambient transaction exists, check for an onDelete handler early - if (!ambientTransaction && !this.config.onDelete) { - throw new MissingDeleteHandlerError() - } - - if (Array.isArray(keys) && keys.length === 0) { - throw new NoKeysPassedToDeleteError() - } - - const keysArray = Array.isArray(keys) ? keys : [keys] - const mutations: Array> = [] - - for (const key of keysArray) { - if (!this.has(key)) { - throw new DeleteKeyNotFoundError(key) - } - const globalKey = this.generateGlobalKey(key, this.get(key)!) - const mutation: PendingMutation = { - mutationId: crypto.randomUUID(), - original: this.get(key)!, - modified: this.get(key)!, - changes: this.get(key)!, - globalKey, - key, - metadata: config?.metadata as unknown, - syncMetadata: (this.syncedMetadata.get(key) || {}) as Record< - string, - unknown - >, - optimistic: config?.optimistic ?? true, - type: `delete`, - createdAt: new Date(), - updatedAt: new Date(), - collection: this, - } - - mutations.push(mutation) - } - - // If an ambient transaction exists, use it - if (ambientTransaction) { - ambientTransaction.applyMutations(mutations) - - this.transactions.set(ambientTransaction.id, ambientTransaction) - this.scheduleTransactionCleanup(ambientTransaction) - this.recomputeOptimisticState(true) - - return ambientTransaction - } - - // Create a new transaction with a mutation function that calls the onDelete handler - const directOpTransaction = createTransaction({ - autoCommit: true, - mutationFn: async (params) => { - // Call the onDelete handler with the transaction and collection - return this.config.onDelete!({ - transaction: - params.transaction as unknown as TransactionWithMutations< - TOutput, - `delete` - >, - collection: this as unknown as Collection, - }) - }, - }) - - // Apply mutations to the new transaction - directOpTransaction.applyMutations(mutations) - directOpTransaction.commit() - - this.transactions.set(directOpTransaction.id, directOpTransaction) - this.scheduleTransactionCleanup(directOpTransaction) - this.recomputeOptimisticState(true) - - return directOpTransaction - } - - /** - * Gets the current state of the collection as a Map - * @returns Map containing all items in the collection, with keys as identifiers - * @example - * const itemsMap = collection.state - * console.log(`Collection has ${itemsMap.size} items`) - * - * for (const [key, item] of itemsMap) { - * console.log(`${key}: ${item.title}`) - * } - * - * // Check if specific item exists - * if (itemsMap.has("todo-1")) { - * console.log("Todo 1 exists:", itemsMap.get("todo-1")) - * } - */ - get state() { - const result = new Map() - for (const [key, value] of this.entries()) { - result.set(key, value) - } - return result - } - - /** - * Gets the current state of the collection as a Map, but only resolves when data is available - * Waits for the first sync commit to complete before resolving - * - * @returns Promise that resolves to a Map containing all items in the collection - */ - stateWhenReady(): Promise> { - // If we already have data or collection is ready, resolve immediately - if (this.size > 0 || this.isReady()) { - return Promise.resolve(this.state) - } - - // Use preload to ensure the collection starts loading, then return the state - return this.preload().then(() => this.state) - } - - /** - * Gets the current state of the collection as an Array - * - * @returns An Array containing all items in the collection - */ - get toArray() { - return Array.from(this.values()) - } - - /** - * Gets the current state of the collection as an Array, but only resolves when data is available - * Waits for the first sync commit to complete before resolving - * - * @returns Promise that resolves to an Array containing all items in the collection - */ - toArrayWhenReady(): Promise> { - // If we already have data or collection is ready, resolve immediately - if (this.size > 0 || this.isReady()) { - return Promise.resolve(this.toArray) - } - - // Use preload to ensure the collection starts loading, then return the array - return this.preload().then(() => this.toArray) - } - - /** - * Returns the current state of the collection as an array of changes - * @param options - Options including optional where filter - * @returns An array of changes - * @example - * // Get all items as changes - * const allChanges = collection.currentStateAsChanges() - * - * // Get only items matching a condition - * const activeChanges = collection.currentStateAsChanges({ - * where: (row) => row.status === 'active' - * }) - * - * // Get only items using a pre-compiled expression - * const activeChanges = collection.currentStateAsChanges({ - * whereExpression: eq(row.status, 'active') - * }) - */ - public currentStateAsChanges( - options: CurrentStateAsChangesOptions = {} - ): Array> | void { - return currentStateAsChanges(this, options) - } - - /** - * Subscribe to changes in the collection - * @param callback - Function called when items change - * @param options - Subscription options including includeInitialState and where filter - * @returns Unsubscribe function - Call this to stop listening for changes - * @example - * // Basic subscription - * const subscription = collection.subscribeChanges((changes) => { - * changes.forEach(change => { - * console.log(`${change.type}: ${change.key}`, change.value) - * }) - * }) - * - * // Later: subscription.unsubscribe() - * - * @example - * // Include current state immediately - * const subscription = collection.subscribeChanges((changes) => { - * updateUI(changes) - * }, { includeInitialState: true }) - * - * @example - * // Subscribe only to changes matching a condition - * const subscription = collection.subscribeChanges((changes) => { - * updateUI(changes) - * }, { - * includeInitialState: true, - * where: (row) => row.status === 'active' - * }) - * - * @example - * // Subscribe using a pre-compiled expression - * const subscription = collection.subscribeChanges((changes) => { - * updateUI(changes) - * }, { - * includeInitialState: true, - * whereExpression: eq(row.status, 'active') - * }) - */ - public subscribeChanges( - callback: (changes: Array>) => void, - options: SubscribeChangesOptions = {} - ): CollectionSubscription { - // Start sync and track subscriber - this.addSubscriber() - - const subscription = new CollectionSubscription(this, callback, { - ...options, - onUnsubscribe: () => { - this.removeSubscriber() - this.changeSubscriptions.delete(subscription) - }, - }) - - if (options.includeInitialState) { - subscription.requestSnapshot() - } - - // Add to batched listeners - this.changeSubscriptions.add(subscription) - - return subscription - } - - /** - * Capture visible state for keys that will be affected by pending sync operations - * This must be called BEFORE onTransactionStateChange clears optimistic state - */ - private capturePreSyncVisibleState(): void { - if (this.pendingSyncedTransactions.length === 0) return - - // Clear any previous capture - this.preSyncVisibleState.clear() - - // Get all keys that will be affected by sync operations - const syncedKeys = new Set() - for (const transaction of this.pendingSyncedTransactions) { - for (const operation of transaction.operations) { - syncedKeys.add(operation.key as TKey) - } - } - - // Mark keys as about to be synced to suppress intermediate events from recomputeOptimisticState - for (const key of syncedKeys) { - this.recentlySyncedKeys.add(key) - } - - // Only capture current visible state for keys that will be affected by sync operations - // This is much more efficient than capturing the entire collection state - for (const key of syncedKeys) { - const currentValue = this.get(key) - if (currentValue !== undefined) { - this.preSyncVisibleState.set(key, currentValue) - } - } - } - - /** - * Trigger a recomputation when transactions change - * This method should be called by the Transaction class when state changes - */ - public onTransactionStateChange(): void { - // Check if commitPendingTransactions will be called after this - // by checking if there are pending sync transactions (same logic as in transactions.ts) - this.shouldBatchEvents = this.pendingSyncedTransactions.length > 0 - - // CRITICAL: Capture visible state BEFORE clearing optimistic state - this.capturePreSyncVisibleState() - - this.recomputeOptimisticState(false) - } - - /** - * Subscribe to a collection event - */ - public on( - event: T, - callback: CollectionEventHandler - ) { - return this.events.on(event, callback) - } - - /** - * Subscribe to a collection event once - */ - public once( - event: T, - callback: CollectionEventHandler - ) { - return this.events.once(event, callback) - } - - /** - * Unsubscribe from a collection event - */ - public off( - event: T, - callback: CollectionEventHandler - ) { - this.events.off(event, callback) - } - - /** - * Wait for a collection event - */ - public waitFor( - event: T, - timeout?: number - ) { - return this.events.waitFor(event, timeout) - } -} diff --git a/packages/db/src/change-events.ts b/packages/db/src/collection/change-events.ts similarity index 95% rename from packages/db/src/change-events.ts rename to packages/db/src/collection/change-events.ts index d11fa70ba..505f4d97e 100644 --- a/packages/db/src/change-events.ts +++ b/packages/db/src/collection/change-events.ts @@ -1,17 +1,17 @@ import { createSingleRowRefProxy, toExpression, -} from "./query/builder/ref-proxy" -import { compileSingleRowExpression } from "./query/compiler/evaluators.js" -import { optimizeExpressionWithIndexes } from "./utils/index-optimization.js" +} from "../query/builder/ref-proxy" +import { compileSingleRowExpression } from "../query/compiler/evaluators.js" +import { optimizeExpressionWithIndexes } from "../utils/index-optimization.js" import type { ChangeMessage, CurrentStateAsChangesOptions, SubscribeChangesOptions, -} from "./types" -import type { Collection } from "./collection" -import type { SingleRowRefProxy } from "./query/builder/ref-proxy" -import type { BasicExpression } from "./query/ir.js" +} from "../types" +import type { Collection } from "./index.js" +import type { SingleRowRefProxy } from "../query/builder/ref-proxy" +import type { BasicExpression } from "../query/ir.js" /** * Interface for a collection-like object that provides the necessary methods diff --git a/packages/db/src/collection/changes.ts b/packages/db/src/collection/changes.ts new file mode 100644 index 000000000..ae1ea64f7 --- /dev/null +++ b/packages/db/src/collection/changes.ts @@ -0,0 +1,163 @@ +import { NegativeActiveSubscribersError } from "../errors" +import { CollectionSubscription } from "./subscription.js" +import type { StandardSchemaV1 } from "@standard-schema/spec" +import type { ChangeMessage, SubscribeChangesOptions } from "../types" +import type { CollectionLifecycleManager } from "./lifecycle.js" +import type { CollectionSyncManager } from "./sync.js" +import type { CollectionEventsManager } from "./events.js" +import type { CollectionImpl } from "./index.js" + +export class CollectionChangesManager< + TOutput extends object = Record, + TKey extends string | number = string | number, + TSchema extends StandardSchemaV1 = StandardSchemaV1, + TInput extends object = TOutput, +> { + private lifecycle!: CollectionLifecycleManager + private sync!: CollectionSyncManager + private events!: CollectionEventsManager + private collection!: CollectionImpl + + public activeSubscribersCount = 0 + public changeSubscriptions = new Set() + public batchedEvents: Array> = [] + public shouldBatchEvents = false + + /** + * Creates a new CollectionChangesManager instance + */ + constructor() {} + + public setDeps(deps: { + lifecycle: CollectionLifecycleManager + sync: CollectionSyncManager + events: CollectionEventsManager + collection: CollectionImpl + }) { + this.lifecycle = deps.lifecycle + this.sync = deps.sync + this.events = deps.events + this.collection = deps.collection + } + + /** + * Emit an empty ready event to notify subscribers that the collection is ready + * This bypasses the normal empty array check in emitEvents + */ + public emitEmptyReadyEvent(): void { + // Emit empty array directly to all subscribers + for (const subscription of this.changeSubscriptions) { + subscription.emitEvents([]) + } + } + + /** + * Emit events either immediately or batch them for later emission + */ + public emitEvents( + changes: Array>, + forceEmit = false + ): void { + // Skip batching for user actions (forceEmit=true) to keep UI responsive + if (this.shouldBatchEvents && !forceEmit) { + // Add events to the batch + this.batchedEvents.push(...changes) + return + } + + // Either we're not batching, or we're forcing emission (user action or ending batch cycle) + let eventsToEmit = changes + + // If we have batched events and this is a forced emit, combine them + if (this.batchedEvents.length > 0 && forceEmit) { + eventsToEmit = [...this.batchedEvents, ...changes] + this.batchedEvents = [] + this.shouldBatchEvents = false + } + + if (eventsToEmit.length === 0) return + + // Emit to all listeners + for (const subscription of this.changeSubscriptions) { + subscription.emitEvents(eventsToEmit) + } + } + + /** + * Subscribe to changes in the collection + */ + public subscribeChanges( + callback: (changes: Array>) => void, + options: SubscribeChangesOptions = {} + ): CollectionSubscription { + // Start sync and track subscriber + this.addSubscriber() + + const subscription = new CollectionSubscription(this.collection, callback, { + ...options, + onUnsubscribe: () => { + this.removeSubscriber() + this.changeSubscriptions.delete(subscription) + }, + }) + + if (options.includeInitialState) { + subscription.requestSnapshot() + } + + // Add to batched listeners + this.changeSubscriptions.add(subscription) + + return subscription + } + + /** + * Increment the active subscribers count and start sync if needed + */ + private addSubscriber(): void { + const previousSubscriberCount = this.activeSubscribersCount + this.activeSubscribersCount++ + this.lifecycle.cancelGCTimer() + + // Start sync if collection was cleaned up + if ( + this.lifecycle.status === `cleaned-up` || + this.lifecycle.status === `idle` + ) { + this.sync.startSync() + } + + this.events.emitSubscribersChange( + this.activeSubscribersCount, + previousSubscriberCount + ) + } + + /** + * Decrement the active subscribers count and start GC timer if needed + */ + private removeSubscriber(): void { + const previousSubscriberCount = this.activeSubscribersCount + this.activeSubscribersCount-- + + if (this.activeSubscribersCount === 0) { + this.lifecycle.startGCTimer() + } else if (this.activeSubscribersCount < 0) { + throw new NegativeActiveSubscribersError() + } + + this.events.emitSubscribersChange( + this.activeSubscribersCount, + previousSubscriberCount + ) + } + + /** + * Clean up the collection by stopping sync and clearing data + * This can be called manually or automatically by garbage collection + */ + public cleanup(): void { + this.batchedEvents = [] + this.shouldBatchEvents = false + } +} diff --git a/packages/db/src/collection-events.ts b/packages/db/src/collection/events.ts similarity index 92% rename from packages/db/src/collection-events.ts rename to packages/db/src/collection/events.ts index e0f4ddcec..1b6c31be5 100644 --- a/packages/db/src/collection-events.ts +++ b/packages/db/src/collection/events.ts @@ -1,5 +1,5 @@ -import type { Collection } from "./collection" -import type { CollectionStatus } from "./types" +import type { Collection } from "./index.js" +import type { CollectionStatus } from "../types.js" /** * Event emitted when the collection status changes @@ -47,15 +47,17 @@ export type CollectionEventHandler = ( event: AllCollectionEvents[T] ) => void -export class CollectionEvents { - private collection: Collection +export class CollectionEventsManager { + private collection!: Collection private listeners = new Map< keyof AllCollectionEvents, Set> >() - constructor(collection: Collection) { - this.collection = collection + constructor() {} + + setDeps(deps: { collection: Collection }) { + this.collection = deps.collection } on( diff --git a/packages/db/src/collection/index.ts b/packages/db/src/collection/index.ts new file mode 100644 index 000000000..63e818799 --- /dev/null +++ b/packages/db/src/collection/index.ts @@ -0,0 +1,808 @@ +import { + CollectionRequiresConfigError, + CollectionRequiresSyncConfigError, +} from "../errors" +import { currentStateAsChanges } from "./change-events" + +import { CollectionStateManager } from "./state" +import { CollectionChangesManager } from "./changes" +import { CollectionLifecycleManager } from "./lifecycle.js" +import { CollectionSyncManager } from "./sync" +import { CollectionIndexesManager } from "./indexes" +import { CollectionMutationsManager } from "./mutations" +import { CollectionEventsManager } from "./events.js" +import type { CollectionSubscription } from "./subscription" +import type { AllCollectionEvents, CollectionEventHandler } from "./events.js" +import type { BaseIndex, IndexResolver } from "../indexes/base-index.js" +import type { IndexOptions } from "../indexes/index-options.js" +import type { + ChangeMessage, + CollectionConfig, + CollectionStatus, + CurrentStateAsChangesOptions, + Fn, + InferSchemaInput, + InferSchemaOutput, + InsertConfig, + OperationConfig, + SubscribeChangesOptions, + Transaction as TransactionType, + UtilsRecord, + WritableDeep, +} from "../types" +import type { SingleRowRefProxy } from "../query/builder/ref-proxy" +import type { StandardSchemaV1 } from "@standard-schema/spec" +import type { BTreeIndex } from "../indexes/btree-index.js" +import type { IndexProxy } from "../indexes/lazy-index.js" + +/** + * Enhanced Collection interface that includes both data type T and utilities TUtils + * @template T - The type of items in the collection + * @template TKey - The type of the key for the collection + * @template TUtils - The utilities record type + * @template TInsertInput - The type for insert operations (can be different from T for schemas with defaults) + */ +export interface Collection< + T extends object = Record, + TKey extends string | number = string | number, + TUtils extends UtilsRecord = {}, + TSchema extends StandardSchemaV1 = StandardSchemaV1, + TInsertInput extends object = T, +> extends CollectionImpl { + readonly utils: TUtils +} + +/** + * Creates a new Collection instance with the given configuration + * + * @template T - The schema type if a schema is provided, otherwise the type of items in the collection + * @template TKey - The type of the key for the collection + * @template TUtils - The utilities record type + * @param options - Collection options with optional utilities + * @returns A new Collection with utilities exposed both at top level and under .utils + * + * @example + * // Pattern 1: With operation handlers (direct collection calls) + * const todos = createCollection({ + * id: "todos", + * getKey: (todo) => todo.id, + * schema, + * onInsert: async ({ transaction, collection }) => { + * // Send to API + * await api.createTodo(transaction.mutations[0].modified) + * }, + * onUpdate: async ({ transaction, collection }) => { + * await api.updateTodo(transaction.mutations[0].modified) + * }, + * onDelete: async ({ transaction, collection }) => { + * await api.deleteTodo(transaction.mutations[0].key) + * }, + * sync: { sync: () => {} } + * }) + * + * // Direct usage (handlers manage transactions) + * const tx = todos.insert({ id: "1", text: "Buy milk", completed: false }) + * await tx.isPersisted.promise + * + * @example + * // Pattern 2: Manual transaction management + * const todos = createCollection({ + * getKey: (todo) => todo.id, + * schema: todoSchema, + * sync: { sync: () => {} } + * }) + * + * // Explicit transaction usage + * const tx = createTransaction({ + * mutationFn: async ({ transaction }) => { + * // Handle all mutations in transaction + * await api.saveChanges(transaction.mutations) + * } + * }) + * + * tx.mutate(() => { + * todos.insert({ id: "1", text: "Buy milk" }) + * todos.update("2", draft => { draft.completed = true }) + * }) + * + * await tx.isPersisted.promise + * + * @example + * // Using schema for type inference (preferred as it also gives you client side validation) + * const todoSchema = z.object({ + * id: z.string(), + * title: z.string(), + * completed: z.boolean() + * }) + * + * const todos = createCollection({ + * schema: todoSchema, + * getKey: (todo) => todo.id, + * sync: { sync: () => {} } + * }) + * + */ + +// Overload for when schema is provided +export function createCollection< + T extends StandardSchemaV1, + TKey extends string | number = string | number, + TUtils extends UtilsRecord = {}, +>( + options: CollectionConfig, TKey, T> & { + schema: T + utils?: TUtils + } +): Collection, TKey, TUtils, T, InferSchemaInput> + +// Overload for when no schema is provided +// the type T needs to be passed explicitly unless it can be inferred from the getKey function in the config +export function createCollection< + T extends object, + TKey extends string | number = string | number, + TUtils extends UtilsRecord = {}, +>( + options: CollectionConfig & { + schema?: never // prohibit schema if an explicit type is provided + utils?: TUtils + } +): Collection + +// Implementation +export function createCollection( + options: CollectionConfig & { + schema?: StandardSchemaV1 + utils?: UtilsRecord + } +): Collection { + const collection = new CollectionImpl( + options + ) + + // Copy utils to both top level and .utils namespace + if (options.utils) { + collection.utils = { ...options.utils } + } else { + collection.utils = {} + } + + return collection +} + +export class CollectionImpl< + TOutput extends object = Record, + TKey extends string | number = string | number, + TUtils extends UtilsRecord = {}, + TSchema extends StandardSchemaV1 = StandardSchemaV1, + TInput extends object = TOutput, +> { + public id: string + public config: CollectionConfig + + // Utilities namespace + // This is populated by createCollection + public utils: Record = {} + + // Managers + private _events: CollectionEventsManager + private _changes: CollectionChangesManager + private _lifecycle: CollectionLifecycleManager + private _sync: CollectionSyncManager + private _indexes: CollectionIndexesManager + private _mutations: CollectionMutationsManager< + TOutput, + TKey, + TUtils, + TSchema, + TInput + > + // The core state of the collection is "public" so that is accessible in tests + // and for debugging + public _state: CollectionStateManager + + /** + * Creates a new Collection instance + * + * @param config - Configuration object for the collection + * @throws Error if sync config is missing + */ + constructor(config: CollectionConfig) { + // eslint-disable-next-line + if (!config) { + throw new CollectionRequiresConfigError() + } + + // eslint-disable-next-line + if (!config.sync) { + throw new CollectionRequiresSyncConfigError() + } + + if (config.id) { + this.id = config.id + } else { + this.id = crypto.randomUUID() + } + + // Set default values for optional config properties + this.config = { + ...config, + autoIndex: config.autoIndex ?? `eager`, + } + + this._changes = new CollectionChangesManager() + this._events = new CollectionEventsManager() + this._indexes = new CollectionIndexesManager() + this._lifecycle = new CollectionLifecycleManager(config, this.id) + this._mutations = new CollectionMutationsManager(config, this.id) + this._state = new CollectionStateManager(config) + this._sync = new CollectionSyncManager(config, this.id) + + this._changes.setDeps({ + collection: this, // Required for passing to CollectionSubscription + lifecycle: this._lifecycle, + sync: this._sync, + events: this._events, + }) + this._events.setDeps({ + collection: this, // Required for adding to emitted events + }) + this._indexes.setDeps({ + state: this._state, + lifecycle: this._lifecycle, + }) + this._lifecycle.setDeps({ + changes: this._changes, + events: this._events, + indexes: this._indexes, + state: this._state, + sync: this._sync, + }) + this._mutations.setDeps({ + collection: this, // Required for passing to config.onInsert/onUpdate/onDelete and annotating mutations + lifecycle: this._lifecycle, + state: this._state, + }) + this._state.setDeps({ + collection: this, // Required for filtering events to only include this collection + lifecycle: this._lifecycle, + changes: this._changes, + indexes: this._indexes, + }) + this._sync.setDeps({ + collection: this, // Required for passing to config.sync callback + state: this._state, + lifecycle: this._lifecycle, + }) + + // Only start sync immediately if explicitly enabled + if (config.startSync === true) { + this._sync.startSync() + } + } + + /** + * Gets the current status of the collection + */ + public get status(): CollectionStatus { + return this._lifecycle.status + } + + /** + * Get the number of subscribers to the collection + */ + public get subscriberCount(): number { + return this._changes.activeSubscribersCount + } + + /** + * Register a callback to be executed when the collection first becomes ready + * Useful for preloading collections + * @param callback Function to call when the collection first becomes ready + * @example + * collection.onFirstReady(() => { + * console.log('Collection is ready for the first time') + * // Safe to access collection.state now + * }) + */ + public onFirstReady(callback: () => void): void { + return this._lifecycle.onFirstReady(callback) + } + + /** + * Check if the collection is ready for use + * Returns true if the collection has been marked as ready by its sync implementation + * @returns true if the collection is ready, false otherwise + * @example + * if (collection.isReady()) { + * console.log('Collection is ready, data is available') + * // Safe to access collection.state + * } else { + * console.log('Collection is still loading') + * } + */ + public isReady(): boolean { + return this._lifecycle.status === `ready` + } + + /** + * Start sync immediately - internal method for compiled queries + * This bypasses lazy loading for special cases like live query results + */ + public startSyncImmediate(): void { + this._sync.startSync() + } + + /** + * Preload the collection data by starting sync if not already started + * Multiple concurrent calls will share the same promise + */ + public preload(): Promise { + return this._sync.preload() + } + + /** + * Get the current value for a key (virtual derived state) + */ + public get(key: TKey): TOutput | undefined { + return this._state.get(key) + } + + /** + * Check if a key exists in the collection (virtual derived state) + */ + public has(key: TKey): boolean { + return this._state.has(key) + } + + /** + * Get the current size of the collection (cached) + */ + public get size(): number { + return this._state.size + } + + /** + * Get all keys (virtual derived state) + */ + public *keys(): IterableIterator { + yield* this._state.keys() + } + + /** + * Get all values (virtual derived state) + */ + public *values(): IterableIterator { + yield* this._state.values() + } + + /** + * Get all entries (virtual derived state) + */ + public *entries(): IterableIterator<[TKey, TOutput]> { + yield* this._state.entries() + } + + /** + * Get all entries (virtual derived state) + */ + public *[Symbol.iterator](): IterableIterator<[TKey, TOutput]> { + yield* this._state[Symbol.iterator]() + } + + /** + * Execute a callback for each entry in the collection + */ + public forEach( + callbackfn: (value: TOutput, key: TKey, index: number) => void + ): void { + return this._state.forEach(callbackfn) + } + + /** + * Create a new array with the results of calling a function for each entry in the collection + */ + public map( + callbackfn: (value: TOutput, key: TKey, index: number) => U + ): Array { + return this._state.map(callbackfn) + } + + public getKeyFromItem(item: TOutput): TKey { + return this.config.getKey(item) + } + + /** + * Creates an index on a collection for faster queries. + * Indexes significantly improve query performance by allowing constant time lookups + * and logarithmic time range queries instead of full scans. + * + * @template TResolver - The type of the index resolver (constructor or async loader) + * @param indexCallback - Function that extracts the indexed value from each item + * @param config - Configuration including index type and type-specific options + * @returns An index proxy that provides access to the index when ready + * + * @example + * // Create a default B+ tree index + * const ageIndex = collection.createIndex((row) => row.age) + * + * // Create a ordered index with custom options + * const ageIndex = collection.createIndex((row) => row.age, { + * indexType: BTreeIndex, + * options: { compareFn: customComparator }, + * name: 'age_btree' + * }) + * + * // Create an async-loaded index + * const textIndex = collection.createIndex((row) => row.content, { + * indexType: async () => { + * const { FullTextIndex } = await import('./indexes/fulltext.js') + * return FullTextIndex + * }, + * options: { language: 'en' } + * }) + */ + public createIndex = typeof BTreeIndex>( + indexCallback: (row: SingleRowRefProxy) => any, + config: IndexOptions = {} + ): IndexProxy { + return this._indexes.createIndex(indexCallback, config) + } + + /** + * Get resolved indexes for query optimization + */ + get indexes(): Map> { + return this._indexes.indexes + } + + /** + * Validates the data against the schema + */ + public validateData( + data: unknown, + type: `insert` | `update`, + key?: TKey + ): TOutput | never { + return this._mutations.validateData(data, type, key) + } + + /** + * Inserts one or more items into the collection + * @param items - Single item or array of items to insert + * @param config - Optional configuration including metadata + * @returns A Transaction object representing the insert operation(s) + * @throws {SchemaValidationError} If the data fails schema validation + * @example + * // Insert a single todo (requires onInsert handler) + * const tx = collection.insert({ id: "1", text: "Buy milk", completed: false }) + * await tx.isPersisted.promise + * + * @example + * // Insert multiple todos at once + * const tx = collection.insert([ + * { id: "1", text: "Buy milk", completed: false }, + * { id: "2", text: "Walk dog", completed: true } + * ]) + * await tx.isPersisted.promise + * + * @example + * // Insert with metadata + * const tx = collection.insert({ id: "1", text: "Buy groceries" }, + * { metadata: { source: "mobile-app" } } + * ) + * await tx.isPersisted.promise + * + * @example + * // Handle errors + * try { + * const tx = collection.insert({ id: "1", text: "New item" }) + * await tx.isPersisted.promise + * console.log('Insert successful') + * } catch (error) { + * console.log('Insert failed:', error) + * } + */ + insert = (data: TInput | Array, config?: InsertConfig) => { + return this._mutations.insert(data, config) + } + + /** + * Updates one or more items in the collection using a callback function + * @param keys - Single key or array of keys to update + * @param configOrCallback - Either update configuration or update callback + * @param maybeCallback - Update callback if config was provided + * @returns A Transaction object representing the update operation(s) + * @throws {SchemaValidationError} If the updated data fails schema validation + * @example + * // Update single item by key + * const tx = collection.update("todo-1", (draft) => { + * draft.completed = true + * }) + * await tx.isPersisted.promise + * + * @example + * // Update multiple items + * const tx = collection.update(["todo-1", "todo-2"], (drafts) => { + * drafts.forEach(draft => { draft.completed = true }) + * }) + * await tx.isPersisted.promise + * + * @example + * // Update with metadata + * const tx = collection.update("todo-1", + * { metadata: { reason: "user update" } }, + * (draft) => { draft.text = "Updated text" } + * ) + * await tx.isPersisted.promise + * + * @example + * // Handle errors + * try { + * const tx = collection.update("item-1", draft => { draft.value = "new" }) + * await tx.isPersisted.promise + * console.log('Update successful') + * } catch (error) { + * console.log('Update failed:', error) + * } + */ + + // Overload 1: Update multiple items with a callback + update( + key: Array, + callback: (drafts: Array>) => void + ): TransactionType + + // Overload 2: Update multiple items with config and a callback + update( + keys: Array, + config: OperationConfig, + callback: (drafts: Array>) => void + ): TransactionType + + // Overload 3: Update a single item with a callback + update( + id: TKey | unknown, + callback: (draft: WritableDeep) => void + ): TransactionType + + // Overload 4: Update a single item with config and a callback + update( + id: TKey | unknown, + config: OperationConfig, + callback: (draft: WritableDeep) => void + ): TransactionType + + update( + keys: (TKey | unknown) | Array, + configOrCallback: + | ((draft: WritableDeep) => void) + | ((drafts: Array>) => void) + | OperationConfig, + maybeCallback?: + | ((draft: WritableDeep) => void) + | ((drafts: Array>) => void) + ) { + return this._mutations.update(keys, configOrCallback, maybeCallback) + } + + /** + * Deletes one or more items from the collection + * @param keys - Single key or array of keys to delete + * @param config - Optional configuration including metadata + * @returns A Transaction object representing the delete operation(s) + * @example + * // Delete a single item + * const tx = collection.delete("todo-1") + * await tx.isPersisted.promise + * + * @example + * // Delete multiple items + * const tx = collection.delete(["todo-1", "todo-2"]) + * await tx.isPersisted.promise + * + * @example + * // Delete with metadata + * const tx = collection.delete("todo-1", { metadata: { reason: "completed" } }) + * await tx.isPersisted.promise + * + * @example + * // Handle errors + * try { + * const tx = collection.delete("item-1") + * await tx.isPersisted.promise + * console.log('Delete successful') + * } catch (error) { + * console.log('Delete failed:', error) + * } + */ + delete = ( + keys: Array | TKey, + config?: OperationConfig + ): TransactionType => { + return this._mutations.delete(keys, config) + } + + /** + * Gets the current state of the collection as a Map + * @returns Map containing all items in the collection, with keys as identifiers + * @example + * const itemsMap = collection.state + * console.log(`Collection has ${itemsMap.size} items`) + * + * for (const [key, item] of itemsMap) { + * console.log(`${key}: ${item.title}`) + * } + * + * // Check if specific item exists + * if (itemsMap.has("todo-1")) { + * console.log("Todo 1 exists:", itemsMap.get("todo-1")) + * } + */ + get state() { + const result = new Map() + for (const [key, value] of this.entries()) { + result.set(key, value) + } + return result + } + + /** + * Gets the current state of the collection as a Map, but only resolves when data is available + * Waits for the first sync commit to complete before resolving + * + * @returns Promise that resolves to a Map containing all items in the collection + */ + stateWhenReady(): Promise> { + // If we already have data or collection is ready, resolve immediately + if (this.size > 0 || this.isReady()) { + return Promise.resolve(this.state) + } + + // Use preload to ensure the collection starts loading, then return the state + return this.preload().then(() => this.state) + } + + /** + * Gets the current state of the collection as an Array + * + * @returns An Array containing all items in the collection + */ + get toArray() { + return Array.from(this.values()) + } + + /** + * Gets the current state of the collection as an Array, but only resolves when data is available + * Waits for the first sync commit to complete before resolving + * + * @returns Promise that resolves to an Array containing all items in the collection + */ + toArrayWhenReady(): Promise> { + // If we already have data or collection is ready, resolve immediately + if (this.size > 0 || this.isReady()) { + return Promise.resolve(this.toArray) + } + + // Use preload to ensure the collection starts loading, then return the array + return this.preload().then(() => this.toArray) + } + + /** + * Returns the current state of the collection as an array of changes + * @param options - Options including optional where filter + * @returns An array of changes + * @example + * // Get all items as changes + * const allChanges = collection.currentStateAsChanges() + * + * // Get only items matching a condition + * const activeChanges = collection.currentStateAsChanges({ + * where: (row) => row.status === 'active' + * }) + * + * // Get only items using a pre-compiled expression + * const activeChanges = collection.currentStateAsChanges({ + * whereExpression: eq(row.status, 'active') + * }) + */ + public currentStateAsChanges( + options: CurrentStateAsChangesOptions = {} + ): Array> | void { + return currentStateAsChanges(this, options) + } + + /** + * Subscribe to changes in the collection + * @param callback - Function called when items change + * @param options - Subscription options including includeInitialState and where filter + * @returns Unsubscribe function - Call this to stop listening for changes + * @example + * // Basic subscription + * const subscription = collection.subscribeChanges((changes) => { + * changes.forEach(change => { + * console.log(`${change.type}: ${change.key}`, change.value) + * }) + * }) + * + * // Later: subscription.unsubscribe() + * + * @example + * // Include current state immediately + * const subscription = collection.subscribeChanges((changes) => { + * updateUI(changes) + * }, { includeInitialState: true }) + * + * @example + * // Subscribe only to changes matching a condition + * const subscription = collection.subscribeChanges((changes) => { + * updateUI(changes) + * }, { + * includeInitialState: true, + * where: (row) => row.status === 'active' + * }) + * + * @example + * // Subscribe using a pre-compiled expression + * const subscription = collection.subscribeChanges((changes) => { + * updateUI(changes) + * }, { + * includeInitialState: true, + * whereExpression: eq(row.status, 'active') + * }) + */ + public subscribeChanges( + callback: (changes: Array>) => void, + options: SubscribeChangesOptions = {} + ): CollectionSubscription { + return this._changes.subscribeChanges(callback, options) + } + + /** + * Subscribe to a collection event + */ + public on( + event: T, + callback: CollectionEventHandler + ) { + return this._events.on(event, callback) + } + + /** + * Subscribe to a collection event once + */ + public once( + event: T, + callback: CollectionEventHandler + ) { + return this._events.once(event, callback) + } + + /** + * Unsubscribe from a collection event + */ + public off( + event: T, + callback: CollectionEventHandler + ) { + this._events.off(event, callback) + } + + /** + * Wait for a collection event + */ + public waitFor( + event: T, + timeout?: number + ) { + return this._events.waitFor(event, timeout) + } + + /** + * Clean up the collection by stopping sync and clearing data + * This can be called manually or automatically by garbage collection + */ + public async cleanup(): Promise { + this._lifecycle.cleanup() + return Promise.resolve() + } +} diff --git a/packages/db/src/collection/indexes.ts b/packages/db/src/collection/indexes.ts new file mode 100644 index 000000000..96eba97ce --- /dev/null +++ b/packages/db/src/collection/indexes.ts @@ -0,0 +1,172 @@ +import { IndexProxy, LazyIndexWrapper } from "../indexes/lazy-index" +import { + createSingleRowRefProxy, + toExpression, +} from "../query/builder/ref-proxy" +import { BTreeIndex } from "../indexes/btree-index" +import type { StandardSchemaV1 } from "@standard-schema/spec" +import type { BaseIndex, IndexResolver } from "../indexes/base-index" +import type { ChangeMessage } from "../types" +import type { IndexOptions } from "../indexes/index-options" +import type { SingleRowRefProxy } from "../query/builder/ref-proxy" +import type { CollectionLifecycleManager } from "./lifecycle" +import type { CollectionStateManager } from "./state" + +export class CollectionIndexesManager< + TOutput extends object = Record, + TKey extends string | number = string | number, + TSchema extends StandardSchemaV1 = StandardSchemaV1, + TInput extends object = TOutput, +> { + private lifecycle!: CollectionLifecycleManager + private state!: CollectionStateManager + + public lazyIndexes = new Map>() + public resolvedIndexes = new Map>() + public isIndexesResolved = false + public indexCounter = 0 + + constructor() {} + + setDeps(deps: { + state: CollectionStateManager + lifecycle: CollectionLifecycleManager + }) { + this.state = deps.state + this.lifecycle = deps.lifecycle + } + + /** + * Creates an index on a collection for faster queries. + */ + public createIndex = typeof BTreeIndex>( + indexCallback: (row: SingleRowRefProxy) => any, + config: IndexOptions = {} + ): IndexProxy { + this.lifecycle.validateCollectionUsable(`createIndex`) + + const indexId = ++this.indexCounter + const singleRowRefProxy = createSingleRowRefProxy() + const indexExpression = indexCallback(singleRowRefProxy) + const expression = toExpression(indexExpression) + + // Default to BTreeIndex if no type specified + const resolver = config.indexType ?? (BTreeIndex as unknown as TResolver) + + // Create lazy wrapper + const lazyIndex = new LazyIndexWrapper( + indexId, + expression, + config.name, + resolver, + config.options, + this.state.entries() + ) + + this.lazyIndexes.set(indexId, lazyIndex) + + // For BTreeIndex, resolve immediately and synchronously + if ((resolver as unknown) === BTreeIndex) { + try { + const resolvedIndex = lazyIndex.getResolved() + this.resolvedIndexes.set(indexId, resolvedIndex) + } catch (error) { + console.warn(`Failed to resolve BTreeIndex:`, error) + } + } else if (typeof resolver === `function` && resolver.prototype) { + // Other synchronous constructors - resolve immediately + try { + const resolvedIndex = lazyIndex.getResolved() + this.resolvedIndexes.set(indexId, resolvedIndex) + } catch { + // Fallback to async resolution + this.resolveSingleIndex(indexId, lazyIndex).catch((error) => { + console.warn(`Failed to resolve single index:`, error) + }) + } + } else if (this.isIndexesResolved) { + // Async loader but indexes are already resolved - resolve this one + this.resolveSingleIndex(indexId, lazyIndex).catch((error) => { + console.warn(`Failed to resolve single index:`, error) + }) + } + + return new IndexProxy(indexId, lazyIndex) + } + + /** + * Resolve all lazy indexes (called when collection first syncs) + */ + public async resolveAllIndexes(): Promise { + if (this.isIndexesResolved) return + + const resolutionPromises = Array.from(this.lazyIndexes.entries()).map( + async ([indexId, lazyIndex]) => { + const resolvedIndex = await lazyIndex.resolve() + + // Build index with current data + resolvedIndex.build(this.state.entries()) + + this.resolvedIndexes.set(indexId, resolvedIndex) + return { indexId, resolvedIndex } + } + ) + + await Promise.all(resolutionPromises) + this.isIndexesResolved = true + } + + /** + * Resolve a single index immediately + */ + private async resolveSingleIndex( + indexId: number, + lazyIndex: LazyIndexWrapper + ): Promise> { + const resolvedIndex = await lazyIndex.resolve() + resolvedIndex.build(this.state.entries()) + this.resolvedIndexes.set(indexId, resolvedIndex) + return resolvedIndex + } + + /** + * Get resolved indexes for query optimization + */ + get indexes(): Map> { + return this.resolvedIndexes + } + + /** + * Updates all indexes when the collection changes + */ + public updateIndexes(changes: Array>): void { + for (const index of this.resolvedIndexes.values()) { + for (const change of changes) { + switch (change.type) { + case `insert`: + index.add(change.key, change.value) + break + case `update`: + if (change.previousValue) { + index.update(change.key, change.previousValue, change.value) + } else { + index.add(change.key, change.value) + } + break + case `delete`: + index.remove(change.key, change.value) + break + } + } + } + } + + /** + * Clean up the collection by stopping sync and clearing data + * This can be called manually or automatically by garbage collection + */ + public cleanup(): void { + this.lazyIndexes.clear() + this.resolvedIndexes.clear() + } +} diff --git a/packages/db/src/collection/lifecycle.ts b/packages/db/src/collection/lifecycle.ts new file mode 100644 index 000000000..0179786fa --- /dev/null +++ b/packages/db/src/collection/lifecycle.ts @@ -0,0 +1,221 @@ +import { + CollectionInErrorStateError, + InvalidCollectionStatusTransitionError, +} from "../errors" +import type { StandardSchemaV1 } from "@standard-schema/spec" +import type { CollectionConfig, CollectionStatus } from "../types" +import type { CollectionEventsManager } from "./events" +import type { CollectionIndexesManager } from "./indexes" +import type { CollectionChangesManager } from "./changes" +import type { CollectionSyncManager } from "./sync" +import type { CollectionStateManager } from "./state" + +export class CollectionLifecycleManager< + TOutput extends object = Record, + TKey extends string | number = string | number, + TSchema extends StandardSchemaV1 = StandardSchemaV1, + TInput extends object = TOutput, +> { + private config: CollectionConfig + private id: string + private indexes!: CollectionIndexesManager + private events!: CollectionEventsManager + private changes!: CollectionChangesManager + private sync!: CollectionSyncManager + private state!: CollectionStateManager + + public status: CollectionStatus = `idle` + public hasBeenReady = false + public hasReceivedFirstCommit = false + public onFirstReadyCallbacks: Array<() => void> = [] + public gcTimeoutId: ReturnType | null = null + + /** + * Creates a new CollectionLifecycleManager instance + */ + constructor(config: CollectionConfig, id: string) { + this.config = config + this.id = id + } + + setDeps(deps: { + indexes: CollectionIndexesManager + events: CollectionEventsManager + changes: CollectionChangesManager + sync: CollectionSyncManager + state: CollectionStateManager + }) { + this.indexes = deps.indexes + this.events = deps.events + this.changes = deps.changes + this.sync = deps.sync + this.state = deps.state + } + + /** + * Validates state transitions to prevent invalid status changes + */ + public validateStatusTransition( + from: CollectionStatus, + to: CollectionStatus + ): void { + if (from === to) { + // Allow same state transitions + return + } + const validTransitions: Record< + CollectionStatus, + Array + > = { + idle: [`loading`, `error`, `cleaned-up`], + loading: [`initialCommit`, `ready`, `error`, `cleaned-up`], + initialCommit: [`ready`, `error`, `cleaned-up`], + ready: [`cleaned-up`, `error`], + error: [`cleaned-up`, `idle`], + "cleaned-up": [`loading`, `error`], + } + + if (!validTransitions[from].includes(to)) { + throw new InvalidCollectionStatusTransitionError(from, to, this.id) + } + } + + /** + * Safely update the collection status with validation + * @private + */ + public setStatus(newStatus: CollectionStatus): void { + this.validateStatusTransition(this.status, newStatus) + const previousStatus = this.status + this.status = newStatus + + // Resolve indexes when collection becomes ready + if (newStatus === `ready` && !this.indexes.isIndexesResolved) { + // Resolve indexes asynchronously without blocking + this.indexes.resolveAllIndexes().catch((error) => { + console.warn(`Failed to resolve indexes:`, error) + }) + } + + // Emit event + this.events.emitStatusChange(newStatus, previousStatus) + } + + /** + * Validates that the collection is in a usable state for data operations + * @private + */ + public validateCollectionUsable(operation: string): void { + switch (this.status) { + case `error`: + throw new CollectionInErrorStateError(operation, this.id) + case `cleaned-up`: + // Automatically restart the collection when operations are called on cleaned-up collections + this.sync.startSync() + break + } + } + + /** + * Mark the collection as ready for use + * This is called by sync implementations to explicitly signal that the collection is ready, + * providing a more intuitive alternative to using commits for readiness signaling + * @private - Should only be called by sync implementations + */ + public markReady(): void { + // Can transition to ready from loading or initialCommit states + if (this.status === `loading` || this.status === `initialCommit`) { + this.setStatus(`ready`) + + // Call any registered first ready callbacks (only on first time becoming ready) + if (!this.hasBeenReady) { + this.hasBeenReady = true + + // Also mark as having received first commit for backwards compatibility + if (!this.hasReceivedFirstCommit) { + this.hasReceivedFirstCommit = true + } + + const callbacks = [...this.onFirstReadyCallbacks] + this.onFirstReadyCallbacks = [] + callbacks.forEach((callback) => callback()) + } + } + + // Always notify dependents when markReady is called, after status is set + // This ensures live queries get notified when their dependencies become ready + if (this.changes.changeSubscriptions.size > 0) { + this.changes.emitEmptyReadyEvent() + } + } + + /** + * Start the garbage collection timer + * Called when the collection becomes inactive (no subscribers) + */ + public startGCTimer(): void { + if (this.gcTimeoutId) { + clearTimeout(this.gcTimeoutId) + } + + const gcTime = this.config.gcTime ?? 300000 // 5 minutes default + + // If gcTime is 0, GC is disabled + if (gcTime === 0) { + return + } + + this.gcTimeoutId = setTimeout(() => { + if (this.changes.activeSubscribersCount === 0) { + // We call the main collection cleanup, not just the one for the + // lifecycle manager + this.cleanup() + } + }, gcTime) + } + + /** + * Cancel the garbage collection timer + * Called when the collection becomes active again + */ + public cancelGCTimer(): void { + if (this.gcTimeoutId) { + clearTimeout(this.gcTimeoutId) + this.gcTimeoutId = null + } + } + + /** + * Register a callback to be executed when the collection first becomes ready + * Useful for preloading collections + * @param callback Function to call when the collection first becomes ready + */ + public onFirstReady(callback: () => void): void { + // If already ready, call immediately + if (this.hasBeenReady) { + callback() + return + } + + this.onFirstReadyCallbacks.push(callback) + } + + public cleanup(): void { + this.events.cleanup() + this.sync.cleanup() + this.state.cleanup() + this.changes.cleanup() + this.indexes.cleanup() + + if (this.gcTimeoutId) { + clearTimeout(this.gcTimeoutId) + this.gcTimeoutId = null + } + + this.hasBeenReady = false + this.onFirstReadyCallbacks = [] + + // Set status to cleaned-up + this.setStatus(`cleaned-up`) + } +} diff --git a/packages/db/src/collection/mutations.ts b/packages/db/src/collection/mutations.ts new file mode 100644 index 000000000..0e8cbf03a --- /dev/null +++ b/packages/db/src/collection/mutations.ts @@ -0,0 +1,535 @@ +import { withArrayChangeTracking, withChangeTracking } from "../proxy" +import { createTransaction, getActiveTransaction } from "../transactions" +import { + DeleteKeyNotFoundError, + DuplicateKeyError, + InvalidSchemaError, + KeyUpdateNotAllowedError, + MissingDeleteHandlerError, + MissingInsertHandlerError, + MissingUpdateArgumentError, + MissingUpdateHandlerError, + NoKeysPassedToDeleteError, + NoKeysPassedToUpdateError, + SchemaMustBeSynchronousError, + SchemaValidationError, + UndefinedKeyError, + UpdateKeyNotFoundError, +} from "../errors" +import type { Collection, CollectionImpl } from "./index.js" +import type { StandardSchemaV1 } from "@standard-schema/spec" +import type { + CollectionConfig, + InsertConfig, + OperationConfig, + PendingMutation, + StandardSchema, + Transaction as TransactionType, + TransactionWithMutations, + UtilsRecord, + WritableDeep, +} from "../types" +import type { CollectionLifecycleManager } from "./lifecycle" +import type { CollectionStateManager } from "./state" + +export class CollectionMutationsManager< + TOutput extends object = Record, + TKey extends string | number = string | number, + TUtils extends UtilsRecord = {}, + TSchema extends StandardSchemaV1 = StandardSchemaV1, + TInput extends object = TOutput, +> { + private lifecycle!: CollectionLifecycleManager + private state!: CollectionStateManager + private collection!: CollectionImpl + private config!: CollectionConfig + private id: string + + constructor(config: CollectionConfig, id: string) { + this.id = id + this.config = config + } + + setDeps(deps: { + lifecycle: CollectionLifecycleManager + state: CollectionStateManager + collection: CollectionImpl + }) { + this.lifecycle = deps.lifecycle + this.state = deps.state + this.collection = deps.collection + } + + private ensureStandardSchema(schema: unknown): StandardSchema { + // If the schema already implements the standard-schema interface, return it + if (schema && `~standard` in (schema as {})) { + return schema as StandardSchema + } + + throw new InvalidSchemaError() + } + + public validateData( + data: unknown, + type: `insert` | `update`, + key?: TKey + ): TOutput | never { + if (!this.config.schema) return data as TOutput + + const standardSchema = this.ensureStandardSchema(this.config.schema) + + // For updates, we need to merge with the existing data before validation + if (type === `update` && key) { + // Get the existing data for this key + const existingData = this.state.get(key) + + if ( + existingData && + data && + typeof data === `object` && + typeof existingData === `object` + ) { + // Merge the update with the existing data + const mergedData = Object.assign({}, existingData, data) + + // Validate the merged data + const result = standardSchema[`~standard`].validate(mergedData) + + // Ensure validation is synchronous + if (result instanceof Promise) { + throw new SchemaMustBeSynchronousError() + } + + // If validation fails, throw a SchemaValidationError with the issues + if (`issues` in result && result.issues) { + const typedIssues = result.issues.map((issue) => ({ + message: issue.message, + path: issue.path?.map((p) => String(p)), + })) + throw new SchemaValidationError(type, typedIssues) + } + + // Extract only the modified keys from the validated result + const validatedMergedData = result.value as TOutput + const modifiedKeys = Object.keys(data) + const extractedChanges = Object.fromEntries( + modifiedKeys.map((k) => [k, validatedMergedData[k as keyof TOutput]]) + ) as TOutput + + return extractedChanges + } + } + + // For inserts or updates without existing data, validate the data directly + const result = standardSchema[`~standard`].validate(data) + + // Ensure validation is synchronous + if (result instanceof Promise) { + throw new SchemaMustBeSynchronousError() + } + + // If validation fails, throw a SchemaValidationError with the issues + if (`issues` in result && result.issues) { + const typedIssues = result.issues.map((issue) => ({ + message: issue.message, + path: issue.path?.map((p) => String(p)), + })) + throw new SchemaValidationError(type, typedIssues) + } + + return result.value as TOutput + } + + public generateGlobalKey(key: any, item: any): string { + if (typeof key === `undefined`) { + throw new UndefinedKeyError(item) + } + + return `KEY::${this.id}/${key}` + } + + /** + * Inserts one or more items into the collection + */ + insert = (data: TInput | Array, config?: InsertConfig) => { + this.lifecycle.validateCollectionUsable(`insert`) + const state = this.state + const ambientTransaction = getActiveTransaction() + + // If no ambient transaction exists, check for an onInsert handler early + if (!ambientTransaction && !this.config.onInsert) { + throw new MissingInsertHandlerError() + } + + const items = Array.isArray(data) ? data : [data] + const mutations: Array> = [] + + // Create mutations for each item + items.forEach((item) => { + // Validate the data against the schema if one exists + const validatedData = this.validateData(item, `insert`) + + // Check if an item with this ID already exists in the collection + const key = this.config.getKey(validatedData) + if (this.state.has(key)) { + throw new DuplicateKeyError(key) + } + const globalKey = this.generateGlobalKey(key, item) + + const mutation: PendingMutation = { + mutationId: crypto.randomUUID(), + original: {}, + modified: validatedData, + // Pick the values from validatedData based on what's passed in - this is for cases + // where a schema has default values. The validated data has the extra default + // values but for changes, we just want to show the data that was actually passed in. + changes: Object.fromEntries( + Object.keys(item).map((k) => [ + k, + validatedData[k as keyof typeof validatedData], + ]) + ) as TInput, + globalKey, + key, + metadata: config?.metadata as unknown, + syncMetadata: this.config.sync.getSyncMetadata?.() || {}, + optimistic: config?.optimistic ?? true, + type: `insert`, + createdAt: new Date(), + updatedAt: new Date(), + collection: this.collection, + } + + mutations.push(mutation) + }) + + // If an ambient transaction exists, use it + if (ambientTransaction) { + ambientTransaction.applyMutations(mutations) + + state.transactions.set(ambientTransaction.id, ambientTransaction) + state.scheduleTransactionCleanup(ambientTransaction) + state.recomputeOptimisticState(true) + + return ambientTransaction + } else { + // Create a new transaction with a mutation function that calls the onInsert handler + const directOpTransaction = createTransaction({ + mutationFn: async (params) => { + // Call the onInsert handler with the transaction and collection + return await this.config.onInsert!({ + transaction: + params.transaction as unknown as TransactionWithMutations< + TOutput, + `insert` + >, + collection: this.collection as unknown as Collection, + }) + }, + }) + + // Apply mutations to the new transaction + directOpTransaction.applyMutations(mutations) + directOpTransaction.commit() + + // Add the transaction to the collection's transactions store + state.transactions.set(directOpTransaction.id, directOpTransaction) + state.scheduleTransactionCleanup(directOpTransaction) + state.recomputeOptimisticState(true) + + return directOpTransaction + } + } + + /** + * Updates one or more items in the collection using a callback function + */ + update( + keys: (TKey | unknown) | Array, + configOrCallback: + | ((draft: WritableDeep) => void) + | ((drafts: Array>) => void) + | OperationConfig, + maybeCallback?: + | ((draft: WritableDeep) => void) + | ((drafts: Array>) => void) + ) { + if (typeof keys === `undefined`) { + throw new MissingUpdateArgumentError() + } + + const state = this.state + this.lifecycle.validateCollectionUsable(`update`) + + const ambientTransaction = getActiveTransaction() + + // If no ambient transaction exists, check for an onUpdate handler early + if (!ambientTransaction && !this.config.onUpdate) { + throw new MissingUpdateHandlerError() + } + + const isArray = Array.isArray(keys) + const keysArray = isArray ? keys : [keys] + + if (isArray && keysArray.length === 0) { + throw new NoKeysPassedToUpdateError() + } + + const callback = + typeof configOrCallback === `function` ? configOrCallback : maybeCallback! + const config = + typeof configOrCallback === `function` ? {} : configOrCallback + + // Get the current objects or empty objects if they don't exist + const currentObjects = keysArray.map((key) => { + const item = this.state.get(key) + if (!item) { + throw new UpdateKeyNotFoundError(key) + } + + return item + }) as unknown as Array + + let changesArray + if (isArray) { + // Use the proxy to track changes for all objects + changesArray = withArrayChangeTracking( + currentObjects, + callback as (draft: Array) => void + ) + } else { + const result = withChangeTracking( + currentObjects[0]!, + callback as (draft: TInput) => void + ) + changesArray = [result] + } + + // Create mutations for each object that has changes + const mutations: Array< + PendingMutation< + TOutput, + `update`, + CollectionImpl + > + > = keysArray + .map((key, index) => { + const itemChanges = changesArray[index] // User-provided changes for this specific item + + // Skip items with no changes + if (!itemChanges || Object.keys(itemChanges).length === 0) { + return null + } + + const originalItem = currentObjects[index] as unknown as TOutput + // Validate the user-provided changes for this item + const validatedUpdatePayload = this.validateData( + itemChanges, + `update`, + key + ) + + // Construct the full modified item by applying the validated update payload to the original item + const modifiedItem = Object.assign( + {}, + originalItem, + validatedUpdatePayload + ) + + // Check if the ID of the item is being changed + const originalItemId = this.config.getKey(originalItem) + const modifiedItemId = this.config.getKey(modifiedItem) + + if (originalItemId !== modifiedItemId) { + throw new KeyUpdateNotAllowedError(originalItemId, modifiedItemId) + } + + const globalKey = this.generateGlobalKey(modifiedItemId, modifiedItem) + + return { + mutationId: crypto.randomUUID(), + original: originalItem, + modified: modifiedItem, + // Pick the values from modifiedItem based on what's passed in - this is for cases + // where a schema has default values or transforms. The modified data has the extra + // default or transformed values but for changes, we just want to show the data that + // was actually passed in. + changes: Object.fromEntries( + Object.keys(itemChanges).map((k) => [ + k, + modifiedItem[k as keyof typeof modifiedItem], + ]) + ) as TInput, + globalKey, + key, + metadata: config.metadata as unknown, + syncMetadata: (state.syncedMetadata.get(key) || {}) as Record< + string, + unknown + >, + optimistic: config.optimistic ?? true, + type: `update`, + createdAt: new Date(), + updatedAt: new Date(), + collection: this.collection, + } + }) + .filter(Boolean) as Array< + PendingMutation< + TOutput, + `update`, + CollectionImpl + > + > + + // If no changes were made, return an empty transaction early + if (mutations.length === 0) { + const emptyTransaction = createTransaction({ + mutationFn: async () => {}, + }) + emptyTransaction.commit() + // Schedule cleanup for empty transaction + state.scheduleTransactionCleanup(emptyTransaction) + return emptyTransaction + } + + // If an ambient transaction exists, use it + if (ambientTransaction) { + ambientTransaction.applyMutations(mutations) + + state.transactions.set(ambientTransaction.id, ambientTransaction) + state.scheduleTransactionCleanup(ambientTransaction) + state.recomputeOptimisticState(true) + + return ambientTransaction + } + + // No need to check for onUpdate handler here as we've already checked at the beginning + + // Create a new transaction with a mutation function that calls the onUpdate handler + const directOpTransaction = createTransaction({ + mutationFn: async (params) => { + // Call the onUpdate handler with the transaction and collection + return this.config.onUpdate!({ + transaction: + params.transaction as unknown as TransactionWithMutations< + TOutput, + `update` + >, + collection: this.collection as unknown as Collection, + }) + }, + }) + + // Apply mutations to the new transaction + directOpTransaction.applyMutations(mutations) + directOpTransaction.commit() + + // Add the transaction to the collection's transactions store + + state.transactions.set(directOpTransaction.id, directOpTransaction) + state.scheduleTransactionCleanup(directOpTransaction) + state.recomputeOptimisticState(true) + + return directOpTransaction + } + + /** + * Deletes one or more items from the collection + */ + delete = ( + keys: Array | TKey, + config?: OperationConfig + ): TransactionType => { + const state = this.state + this.lifecycle.validateCollectionUsable(`delete`) + + const ambientTransaction = getActiveTransaction() + + // If no ambient transaction exists, check for an onDelete handler early + if (!ambientTransaction && !this.config.onDelete) { + throw new MissingDeleteHandlerError() + } + + if (Array.isArray(keys) && keys.length === 0) { + throw new NoKeysPassedToDeleteError() + } + + const keysArray = Array.isArray(keys) ? keys : [keys] + const mutations: Array< + PendingMutation< + TOutput, + `delete`, + CollectionImpl + > + > = [] + + for (const key of keysArray) { + if (!this.state.has(key)) { + throw new DeleteKeyNotFoundError(key) + } + const globalKey = this.generateGlobalKey(key, this.state.get(key)!) + const mutation: PendingMutation< + TOutput, + `delete`, + CollectionImpl + > = { + mutationId: crypto.randomUUID(), + original: this.state.get(key)!, + modified: this.state.get(key)!, + changes: this.state.get(key)!, + globalKey, + key, + metadata: config?.metadata as unknown, + syncMetadata: (state.syncedMetadata.get(key) || {}) as Record< + string, + unknown + >, + optimistic: config?.optimistic ?? true, + type: `delete`, + createdAt: new Date(), + updatedAt: new Date(), + collection: this.collection, + } + + mutations.push(mutation) + } + + // If an ambient transaction exists, use it + if (ambientTransaction) { + ambientTransaction.applyMutations(mutations) + + state.transactions.set(ambientTransaction.id, ambientTransaction) + state.scheduleTransactionCleanup(ambientTransaction) + state.recomputeOptimisticState(true) + + return ambientTransaction + } + + // Create a new transaction with a mutation function that calls the onDelete handler + const directOpTransaction = createTransaction({ + autoCommit: true, + mutationFn: async (params) => { + // Call the onDelete handler with the transaction and collection + return this.config.onDelete!({ + transaction: + params.transaction as unknown as TransactionWithMutations< + TOutput, + `delete` + >, + collection: this.collection as unknown as Collection, + }) + }, + }) + + // Apply mutations to the new transaction + directOpTransaction.applyMutations(mutations) + directOpTransaction.commit() + + state.transactions.set(directOpTransaction.id, directOpTransaction) + state.scheduleTransactionCleanup(directOpTransaction) + state.recomputeOptimisticState(true) + + return directOpTransaction + } +} diff --git a/packages/db/src/collection/state.ts b/packages/db/src/collection/state.ts new file mode 100644 index 000000000..10a7c0491 --- /dev/null +++ b/packages/db/src/collection/state.ts @@ -0,0 +1,866 @@ +import { deepEquals } from "../utils" +import { SortedMap } from "../SortedMap" +import type { Transaction } from "../transactions" +import type { StandardSchemaV1 } from "@standard-schema/spec" +import type { + ChangeMessage, + CollectionConfig, + OptimisticChangeMessage, +} from "../types" +import type { CollectionImpl } from "./index.js" +import type { CollectionLifecycleManager } from "./lifecycle" +import type { CollectionChangesManager } from "./changes" +import type { CollectionIndexesManager } from "./indexes" + +interface PendingSyncedTransaction> { + committed: boolean + operations: Array> + truncate?: boolean + deletedKeys: Set +} + +export class CollectionStateManager< + TOutput extends object = Record, + TKey extends string | number = string | number, + TSchema extends StandardSchemaV1 = StandardSchemaV1, + TInput extends object = TOutput, +> { + public config!: CollectionConfig + public collection!: CollectionImpl + public lifecycle!: CollectionLifecycleManager + public changes!: CollectionChangesManager + public indexes!: CollectionIndexesManager + + // Core state - make public for testing + public transactions: SortedMap> + public pendingSyncedTransactions: Array> = + [] + public syncedData: Map | SortedMap + public syncedMetadata = new Map() + + // Optimistic state tracking - make public for testing + public optimisticUpserts = new Map() + public optimisticDeletes = new Set() + + // Cached size for performance + public size = 0 + + // State used for computing the change events + public syncedKeys = new Set() + public preSyncVisibleState = new Map() + public recentlySyncedKeys = new Set() + public hasReceivedFirstCommit = false + public isCommittingSyncTransactions = false + + /** + * Creates a new CollectionState manager + */ + constructor(config: CollectionConfig) { + this.config = config + this.transactions = new SortedMap>((a, b) => + a.compareCreatedAt(b) + ) + + // Set up data storage with optional comparison function + if (config.compare) { + this.syncedData = new SortedMap(config.compare) + } else { + this.syncedData = new Map() + } + } + + setDeps(deps: { + collection: CollectionImpl + lifecycle: CollectionLifecycleManager + changes: CollectionChangesManager + indexes: CollectionIndexesManager + }) { + this.collection = deps.collection + this.lifecycle = deps.lifecycle + this.changes = deps.changes + this.indexes = deps.indexes + } + + /** + * Get the current value for a key (virtual derived state) + */ + public get(key: TKey): TOutput | undefined { + const { optimisticDeletes, optimisticUpserts, syncedData } = this + // Check if optimistically deleted + if (optimisticDeletes.has(key)) { + return undefined + } + + // Check optimistic upserts first + if (optimisticUpserts.has(key)) { + return optimisticUpserts.get(key) + } + + // Fall back to synced data + return syncedData.get(key) + } + + /** + * Check if a key exists in the collection (virtual derived state) + */ + public has(key: TKey): boolean { + const { optimisticDeletes, optimisticUpserts, syncedData } = this + // Check if optimistically deleted + if (optimisticDeletes.has(key)) { + return false + } + + // Check optimistic upserts first + if (optimisticUpserts.has(key)) { + return true + } + + // Fall back to synced data + return syncedData.has(key) + } + + /** + * Get all keys (virtual derived state) + */ + public *keys(): IterableIterator { + const { syncedData, optimisticDeletes, optimisticUpserts } = this + // Yield keys from synced data, skipping any that are deleted. + for (const key of syncedData.keys()) { + if (!optimisticDeletes.has(key)) { + yield key + } + } + // Yield keys from upserts that were not already in synced data. + for (const key of optimisticUpserts.keys()) { + if (!syncedData.has(key) && !optimisticDeletes.has(key)) { + // The optimisticDeletes check is technically redundant if inserts/updates always remove from deletes, + // but it's safer to keep it. + yield key + } + } + } + + /** + * Get all values (virtual derived state) + */ + public *values(): IterableIterator { + for (const key of this.keys()) { + const value = this.get(key) + if (value !== undefined) { + yield value + } + } + } + + /** + * Get all entries (virtual derived state) + */ + public *entries(): IterableIterator<[TKey, TOutput]> { + for (const key of this.keys()) { + const value = this.get(key) + if (value !== undefined) { + yield [key, value] + } + } + } + + /** + * Get all entries (virtual derived state) + */ + public *[Symbol.iterator](): IterableIterator<[TKey, TOutput]> { + for (const [key, value] of this.entries()) { + yield [key, value] + } + } + + /** + * Execute a callback for each entry in the collection + */ + public forEach( + callbackfn: (value: TOutput, key: TKey, index: number) => void + ): void { + let index = 0 + for (const [key, value] of this.entries()) { + callbackfn(value, key, index++) + } + } + + /** + * Create a new array with the results of calling a function for each entry in the collection + */ + public map( + callbackfn: (value: TOutput, key: TKey, index: number) => U + ): Array { + const result: Array = [] + let index = 0 + for (const [key, value] of this.entries()) { + result.push(callbackfn(value, key, index++)) + } + return result + } + + /** + * Check if the given collection is this collection + * @param collection The collection to check + * @returns True if the given collection is this collection, false otherwise + */ + private isThisCollection( + collection: CollectionImpl + ): boolean { + return collection === this.collection + } + + /** + * Recompute optimistic state from active transactions + */ + public recomputeOptimisticState( + triggeredByUserAction: boolean = false + ): void { + // Skip redundant recalculations when we're in the middle of committing sync transactions + if (this.isCommittingSyncTransactions) { + return + } + + const previousState = new Map(this.optimisticUpserts) + const previousDeletes = new Set(this.optimisticDeletes) + + // Clear current optimistic state + this.optimisticUpserts.clear() + this.optimisticDeletes.clear() + + const activeTransactions: Array> = [] + + for (const transaction of this.transactions.values()) { + if (![`completed`, `failed`].includes(transaction.state)) { + activeTransactions.push(transaction) + } + } + + // Apply active transactions only (completed transactions are handled by sync operations) + for (const transaction of activeTransactions) { + for (const mutation of transaction.mutations) { + if (this.isThisCollection(mutation.collection) && mutation.optimistic) { + switch (mutation.type) { + case `insert`: + case `update`: + this.optimisticUpserts.set( + mutation.key, + mutation.modified as TOutput + ) + this.optimisticDeletes.delete(mutation.key) + break + case `delete`: + this.optimisticUpserts.delete(mutation.key) + this.optimisticDeletes.add(mutation.key) + break + } + } + } + } + + // Update cached size + this.size = this.calculateSize() + + // Collect events for changes + const events: Array> = [] + this.collectOptimisticChanges(previousState, previousDeletes, events) + + // Filter out events for recently synced keys to prevent duplicates + // BUT: Only filter out events that are actually from sync operations + // New user transactions should NOT be filtered even if the key was recently synced + const filteredEventsBySyncStatus = events.filter((event) => { + if (!this.recentlySyncedKeys.has(event.key)) { + return true // Key not recently synced, allow event through + } + + // Key was recently synced - allow if this is a user-triggered action + if (triggeredByUserAction) { + return true + } + + // Otherwise filter out duplicate sync events + return false + }) + + // Filter out redundant delete events if there are pending sync transactions + // that will immediately restore the same data, but only for completed transactions + // IMPORTANT: Skip complex filtering for user-triggered actions to prevent UI blocking + if (this.pendingSyncedTransactions.length > 0 && !triggeredByUserAction) { + const pendingSyncKeys = new Set() + + // Collect keys from pending sync operations + for (const transaction of this.pendingSyncedTransactions) { + for (const operation of transaction.operations) { + pendingSyncKeys.add(operation.key as TKey) + } + } + + // Only filter out delete events for keys that: + // 1. Have pending sync operations AND + // 2. Are from completed transactions (being cleaned up) + const filteredEvents = filteredEventsBySyncStatus.filter((event) => { + if (event.type === `delete` && pendingSyncKeys.has(event.key)) { + // Check if this delete is from clearing optimistic state of completed transactions + // We can infer this by checking if we have no remaining optimistic mutations for this key + const hasActiveOptimisticMutation = activeTransactions.some((tx) => + tx.mutations.some( + (m) => this.isThisCollection(m.collection) && m.key === event.key + ) + ) + + if (!hasActiveOptimisticMutation) { + return false // Skip this delete event as sync will restore the data + } + } + return true + }) + + // Update indexes for the filtered events + if (filteredEvents.length > 0) { + this.indexes.updateIndexes(filteredEvents) + } + this.changes.emitEvents(filteredEvents, triggeredByUserAction) + } else { + // Update indexes for all events + if (filteredEventsBySyncStatus.length > 0) { + this.indexes.updateIndexes(filteredEventsBySyncStatus) + } + // Emit all events if no pending sync transactions + this.changes.emitEvents(filteredEventsBySyncStatus, triggeredByUserAction) + } + } + + /** + * Calculate the current size based on synced data and optimistic changes + */ + private calculateSize(): number { + const syncedSize = this.syncedData.size + const deletesFromSynced = Array.from(this.optimisticDeletes).filter( + (key) => this.syncedData.has(key) && !this.optimisticUpserts.has(key) + ).length + const upsertsNotInSynced = Array.from(this.optimisticUpserts.keys()).filter( + (key) => !this.syncedData.has(key) + ).length + + return syncedSize - deletesFromSynced + upsertsNotInSynced + } + + /** + * Collect events for optimistic changes + */ + private collectOptimisticChanges( + previousUpserts: Map, + previousDeletes: Set, + events: Array> + ): void { + const allKeys = new Set([ + ...previousUpserts.keys(), + ...this.optimisticUpserts.keys(), + ...previousDeletes, + ...this.optimisticDeletes, + ]) + + for (const key of allKeys) { + const currentValue = this.get(key) + const previousValue = this.getPreviousValue( + key, + previousUpserts, + previousDeletes + ) + + if (previousValue !== undefined && currentValue === undefined) { + events.push({ type: `delete`, key, value: previousValue }) + } else if (previousValue === undefined && currentValue !== undefined) { + events.push({ type: `insert`, key, value: currentValue }) + } else if ( + previousValue !== undefined && + currentValue !== undefined && + previousValue !== currentValue + ) { + events.push({ + type: `update`, + key, + value: currentValue, + previousValue, + }) + } + } + } + + /** + * Get the previous value for a key given previous optimistic state + */ + private getPreviousValue( + key: TKey, + previousUpserts: Map, + previousDeletes: Set + ): TOutput | undefined { + if (previousDeletes.has(key)) { + return undefined + } + if (previousUpserts.has(key)) { + return previousUpserts.get(key) + } + return this.syncedData.get(key) + } + + /** + * Attempts to commit pending synced transactions if there are no active transactions + * This method processes operations from pending transactions and applies them to the synced data + */ + commitPendingTransactions = () => { + // Check if there are any persisting transaction + let hasPersistingTransaction = false + for (const transaction of this.transactions.values()) { + if (transaction.state === `persisting`) { + hasPersistingTransaction = true + break + } + } + + // pending synced transactions could be either `committed` or still open. + // we only want to process `committed` transactions here + const { + committedSyncedTransactions, + uncommittedSyncedTransactions, + hasTruncateSync, + } = this.pendingSyncedTransactions.reduce( + (acc, t) => { + if (t.committed) { + acc.committedSyncedTransactions.push(t) + if (t.truncate === true) { + acc.hasTruncateSync = true + } + } else { + acc.uncommittedSyncedTransactions.push(t) + } + return acc + }, + { + committedSyncedTransactions: [] as Array< + PendingSyncedTransaction + >, + uncommittedSyncedTransactions: [] as Array< + PendingSyncedTransaction + >, + hasTruncateSync: false, + } + ) + + if (!hasPersistingTransaction || hasTruncateSync) { + // Set flag to prevent redundant optimistic state recalculations + this.isCommittingSyncTransactions = true + + // First collect all keys that will be affected by sync operations + const changedKeys = new Set() + for (const transaction of committedSyncedTransactions) { + for (const operation of transaction.operations) { + changedKeys.add(operation.key as TKey) + } + } + + // Use pre-captured state if available (from optimistic scenarios), + // otherwise capture current state (for pure sync scenarios) + let currentVisibleState = this.preSyncVisibleState + if (currentVisibleState.size === 0) { + // No pre-captured state, capture it now for pure sync operations + currentVisibleState = new Map() + for (const key of changedKeys) { + const currentValue = this.get(key) + if (currentValue !== undefined) { + currentVisibleState.set(key, currentValue) + } + } + } + + const events: Array> = [] + const rowUpdateMode = this.config.sync.rowUpdateMode || `partial` + + for (const transaction of committedSyncedTransactions) { + // Handle truncate operations first + if (transaction.truncate) { + // TRUNCATE PHASE + // 1) Emit a delete for every currently-synced key so downstream listeners/indexes + // observe a clear-before-rebuild. We intentionally skip keys already in + // optimisticDeletes because their delete was previously emitted by the user. + for (const key of this.syncedData.keys()) { + if (this.optimisticDeletes.has(key)) continue + const previousValue = + this.optimisticUpserts.get(key) || this.syncedData.get(key) + if (previousValue !== undefined) { + events.push({ type: `delete`, key, value: previousValue }) + } + } + + // 2) Clear the authoritative synced base. Subsequent server ops in this + // same commit will rebuild the base atomically. + this.syncedData.clear() + this.syncedMetadata.clear() + this.syncedKeys.clear() + + // 3) Clear currentVisibleState for truncated keys to ensure subsequent operations + // are compared against the post-truncate state (undefined) rather than pre-truncate state + // This ensures that re-inserted keys are emitted as INSERT events, not UPDATE events + for (const key of changedKeys) { + currentVisibleState.delete(key) + } + } + + for (const operation of transaction.operations) { + const key = operation.key as TKey + this.syncedKeys.add(key) + + // Update metadata + switch (operation.type) { + case `insert`: + this.syncedMetadata.set(key, operation.metadata) + break + case `update`: + this.syncedMetadata.set( + key, + Object.assign( + {}, + this.syncedMetadata.get(key), + operation.metadata + ) + ) + break + case `delete`: + this.syncedMetadata.delete(key) + break + } + + // Update synced data + switch (operation.type) { + case `insert`: + this.syncedData.set(key, operation.value) + break + case `update`: { + if (rowUpdateMode === `partial`) { + const updatedValue = Object.assign( + {}, + this.syncedData.get(key), + operation.value + ) + this.syncedData.set(key, updatedValue) + } else { + this.syncedData.set(key, operation.value) + } + break + } + case `delete`: + this.syncedData.delete(key) + break + } + } + } + + // After applying synced operations, if this commit included a truncate, + // re-apply optimistic mutations on top of the fresh synced base. This ensures + // the UI preserves local intent while respecting server rebuild semantics. + // Ordering: deletes (above) -> server ops (just applied) -> optimistic upserts. + if (hasTruncateSync) { + // Avoid duplicating keys that were inserted/updated by synced operations in this commit + const syncedInsertedOrUpdatedKeys = new Set() + for (const t of committedSyncedTransactions) { + for (const op of t.operations) { + if (op.type === `insert` || op.type === `update`) { + syncedInsertedOrUpdatedKeys.add(op.key as TKey) + } + } + } + + // Build re-apply sets from ACTIVE optimistic transactions against the new synced base + // We do not copy maps; we compute intent directly from transactions to avoid drift. + const reapplyUpserts = new Map() + const reapplyDeletes = new Set() + + for (const tx of this.transactions.values()) { + if ([`completed`, `failed`].includes(tx.state)) continue + for (const mutation of tx.mutations) { + if ( + !this.isThisCollection(mutation.collection) || + !mutation.optimistic + ) + continue + const key = mutation.key as TKey + switch (mutation.type) { + case `insert`: + reapplyUpserts.set(key, mutation.modified as TOutput) + reapplyDeletes.delete(key) + break + case `update`: { + const base = this.syncedData.get(key) + const next = base + ? (Object.assign({}, base, mutation.changes) as TOutput) + : (mutation.modified as TOutput) + reapplyUpserts.set(key, next) + reapplyDeletes.delete(key) + break + } + case `delete`: + reapplyUpserts.delete(key) + reapplyDeletes.add(key) + break + } + } + } + + // Emit inserts for re-applied upserts, skipping any keys that have an optimistic delete. + // If the server also inserted/updated the same key in this batch, override that value + // with the optimistic value to preserve local intent. + for (const [key, value] of reapplyUpserts) { + if (reapplyDeletes.has(key)) continue + if (syncedInsertedOrUpdatedKeys.has(key)) { + let foundInsert = false + for (let i = events.length - 1; i >= 0; i--) { + const evt = events[i]! + if (evt.key === key && evt.type === `insert`) { + evt.value = value + foundInsert = true + break + } + } + if (!foundInsert) { + events.push({ type: `insert`, key, value }) + } + } else { + events.push({ type: `insert`, key, value }) + } + } + + // Finally, ensure we do NOT insert keys that have an outstanding optimistic delete. + if (events.length > 0 && reapplyDeletes.size > 0) { + const filtered: Array> = [] + for (const evt of events) { + if (evt.type === `insert` && reapplyDeletes.has(evt.key)) { + continue + } + filtered.push(evt) + } + events.length = 0 + events.push(...filtered) + } + + // Ensure listeners are active before emitting this critical batch + if (this.lifecycle.status !== `ready`) { + this.lifecycle.setStatus(`ready`) + } + } + + // Maintain optimistic state appropriately + // Clear optimistic state since sync operations will now provide the authoritative data. + // Any still-active user transactions will be re-applied below in recompute. + this.optimisticUpserts.clear() + this.optimisticDeletes.clear() + + // Reset flag and recompute optimistic state for any remaining active transactions + this.isCommittingSyncTransactions = false + for (const transaction of this.transactions.values()) { + if (![`completed`, `failed`].includes(transaction.state)) { + for (const mutation of transaction.mutations) { + if ( + this.isThisCollection(mutation.collection) && + mutation.optimistic + ) { + switch (mutation.type) { + case `insert`: + case `update`: + this.optimisticUpserts.set( + mutation.key, + mutation.modified as TOutput + ) + this.optimisticDeletes.delete(mutation.key) + break + case `delete`: + this.optimisticUpserts.delete(mutation.key) + this.optimisticDeletes.add(mutation.key) + break + } + } + } + } + } + + // Check for redundant sync operations that match completed optimistic operations + const completedOptimisticOps = new Map() + + for (const transaction of this.transactions.values()) { + if (transaction.state === `completed`) { + for (const mutation of transaction.mutations) { + if ( + this.isThisCollection(mutation.collection) && + changedKeys.has(mutation.key) + ) { + completedOptimisticOps.set(mutation.key, { + type: mutation.type, + value: mutation.modified, + }) + } + } + } + } + + // Now check what actually changed in the final visible state + for (const key of changedKeys) { + const previousVisibleValue = currentVisibleState.get(key) + const newVisibleValue = this.get(key) // This returns the new derived state + + // Check if this sync operation is redundant with a completed optimistic operation + const completedOp = completedOptimisticOps.get(key) + const isRedundantSync = + completedOp && + newVisibleValue !== undefined && + deepEquals(completedOp.value, newVisibleValue) + + if (!isRedundantSync) { + if ( + previousVisibleValue === undefined && + newVisibleValue !== undefined + ) { + events.push({ + type: `insert`, + key, + value: newVisibleValue, + }) + } else if ( + previousVisibleValue !== undefined && + newVisibleValue === undefined + ) { + events.push({ + type: `delete`, + key, + value: previousVisibleValue, + }) + } else if ( + previousVisibleValue !== undefined && + newVisibleValue !== undefined && + !deepEquals(previousVisibleValue, newVisibleValue) + ) { + events.push({ + type: `update`, + key, + value: newVisibleValue, + previousValue: previousVisibleValue, + }) + } + } + } + + // Update cached size after synced data changes + this.size = this.calculateSize() + + // Update indexes for all events before emitting + if (events.length > 0) { + this.indexes.updateIndexes(events) + } + + // End batching and emit all events (combines any batched events with sync events) + this.changes.emitEvents(events, true) + + this.pendingSyncedTransactions = uncommittedSyncedTransactions + + // Clear the pre-sync state since sync operations are complete + this.preSyncVisibleState.clear() + + // Clear recently synced keys after a microtask to allow recomputeOptimisticState to see them + Promise.resolve().then(() => { + this.recentlySyncedKeys.clear() + }) + + // Call any registered one-time commit listeners + if (!this.hasReceivedFirstCommit) { + this.hasReceivedFirstCommit = true + const callbacks = [...this.lifecycle.onFirstReadyCallbacks] + this.lifecycle.onFirstReadyCallbacks = [] + callbacks.forEach((callback) => callback()) + } + } + } + + /** + * Schedule cleanup of a transaction when it completes + */ + public scheduleTransactionCleanup(transaction: Transaction): void { + // Only schedule cleanup for transactions that aren't already completed + if (transaction.state === `completed`) { + this.transactions.delete(transaction.id) + return + } + + // Schedule cleanup when the transaction completes + transaction.isPersisted.promise + .then(() => { + // Transaction completed successfully, remove it immediately + this.transactions.delete(transaction.id) + }) + .catch(() => { + // Transaction failed, but we want to keep failed transactions for reference + // so don't remove it. + // This empty catch block is necessary to prevent unhandled promise rejections. + }) + } + + /** + * Capture visible state for keys that will be affected by pending sync operations + * This must be called BEFORE onTransactionStateChange clears optimistic state + */ + public capturePreSyncVisibleState(): void { + if (this.pendingSyncedTransactions.length === 0) return + + // Clear any previous capture + this.preSyncVisibleState.clear() + + // Get all keys that will be affected by sync operations + const syncedKeys = new Set() + for (const transaction of this.pendingSyncedTransactions) { + for (const operation of transaction.operations) { + syncedKeys.add(operation.key as TKey) + } + } + + // Mark keys as about to be synced to suppress intermediate events from recomputeOptimisticState + for (const key of syncedKeys) { + this.recentlySyncedKeys.add(key) + } + + // Only capture current visible state for keys that will be affected by sync operations + // This is much more efficient than capturing the entire collection state + for (const key of syncedKeys) { + const currentValue = this.get(key) + if (currentValue !== undefined) { + this.preSyncVisibleState.set(key, currentValue) + } + } + } + + /** + * Trigger a recomputation when transactions change + * This method should be called by the Transaction class when state changes + */ + public onTransactionStateChange(): void { + // Check if commitPendingTransactions will be called after this + // by checking if there are pending sync transactions (same logic as in transactions.ts) + this.changes.shouldBatchEvents = this.pendingSyncedTransactions.length > 0 + + // CRITICAL: Capture visible state BEFORE clearing optimistic state + this.capturePreSyncVisibleState() + + this.recomputeOptimisticState(false) + } + + /** + * Clean up the collection by stopping sync and clearing data + * This can be called manually or automatically by garbage collection + */ + public cleanup(): void { + this.syncedData.clear() + this.syncedMetadata.clear() + this.optimisticUpserts.clear() + this.optimisticDeletes.clear() + this.size = 0 + this.pendingSyncedTransactions = [] + this.syncedKeys.clear() + this.hasReceivedFirstCommit = false + } +} diff --git a/packages/db/src/collection-subscription.ts b/packages/db/src/collection/subscription.ts similarity index 94% rename from packages/db/src/collection-subscription.ts rename to packages/db/src/collection/subscription.ts index dd2e92835..3d65db137 100644 --- a/packages/db/src/collection-subscription.ts +++ b/packages/db/src/collection/subscription.ts @@ -1,13 +1,13 @@ +import { ensureIndexForExpression } from "../indexes/auto-index.js" +import { and } from "../query/index.js" import { createFilterFunctionFromExpression, createFilteredCallback, } from "./change-events.js" -import { ensureIndexForExpression } from "./indexes/auto-index.js" -import { and } from "./query/index.js" -import type { BasicExpression } from "./query/ir.js" -import type { BaseIndex } from "./indexes/base-index.js" -import type { ChangeMessage } from "./types.js" -import type { Collection } from "./collection.js" +import type { BasicExpression } from "../query/ir.js" +import type { BaseIndex } from "../indexes/base-index.js" +import type { ChangeMessage } from "../types.js" +import type { CollectionImpl } from "./index.js" type RequestSnapshotOptions = { where?: BasicExpression @@ -41,7 +41,7 @@ export class CollectionSubscription { private orderByIndex: BaseIndex | undefined constructor( - private collection: Collection, + private collection: CollectionImpl, private callback: (changes: Array>) => void, private options: CollectionSubscriptionOptions ) { diff --git a/packages/db/src/collection/sync.ts b/packages/db/src/collection/sync.ts new file mode 100644 index 000000000..22a14f72d --- /dev/null +++ b/packages/db/src/collection/sync.ts @@ -0,0 +1,235 @@ +import { + CollectionIsInErrorStateError, + DuplicateKeySyncError, + NoPendingSyncTransactionCommitError, + NoPendingSyncTransactionWriteError, + SyncCleanupError, + SyncTransactionAlreadyCommittedError, + SyncTransactionAlreadyCommittedWriteError, +} from "../errors" +import type { StandardSchemaV1 } from "@standard-schema/spec" +import type { ChangeMessage, CollectionConfig } from "../types" +import type { CollectionImpl } from "./index.js" +import type { CollectionStateManager } from "./state" +import type { CollectionLifecycleManager } from "./lifecycle" + +export class CollectionSyncManager< + TOutput extends object = Record, + TKey extends string | number = string | number, + TSchema extends StandardSchemaV1 = StandardSchemaV1, + TInput extends object = TOutput, +> { + private collection!: CollectionImpl + private state!: CollectionStateManager + private lifecycle!: CollectionLifecycleManager + private config!: CollectionConfig + private id: string + + public preloadPromise: Promise | null = null + public syncCleanupFn: (() => void) | null = null + + /** + * Creates a new CollectionSyncManager instance + */ + constructor(config: CollectionConfig, id: string) { + this.config = config + this.id = id + } + + setDeps(deps: { + collection: CollectionImpl + state: CollectionStateManager + lifecycle: CollectionLifecycleManager + }) { + this.collection = deps.collection + this.state = deps.state + this.lifecycle = deps.lifecycle + } + + /** + * Start the sync process for this collection + * This is called when the collection is first accessed or preloaded + */ + public startSync(): void { + const state = this.state + if ( + this.lifecycle.status !== `idle` && + this.lifecycle.status !== `cleaned-up` + ) { + return // Already started or in progress + } + + this.lifecycle.setStatus(`loading`) + + try { + const cleanupFn = this.config.sync.sync({ + collection: this.collection, + begin: () => { + state.pendingSyncedTransactions.push({ + committed: false, + operations: [], + deletedKeys: new Set(), + }) + }, + write: (messageWithoutKey: Omit, `key`>) => { + const pendingTransaction = + state.pendingSyncedTransactions[ + state.pendingSyncedTransactions.length - 1 + ] + if (!pendingTransaction) { + throw new NoPendingSyncTransactionWriteError() + } + if (pendingTransaction.committed) { + throw new SyncTransactionAlreadyCommittedWriteError() + } + const key = this.config.getKey(messageWithoutKey.value) + + // Check if an item with this key already exists when inserting + if (messageWithoutKey.type === `insert`) { + const insertingIntoExistingSynced = state.syncedData.has(key) + const hasPendingDeleteForKey = + pendingTransaction.deletedKeys.has(key) + const isTruncateTransaction = pendingTransaction.truncate === true + // Allow insert after truncate in the same transaction even if it existed in syncedData + if ( + insertingIntoExistingSynced && + !hasPendingDeleteForKey && + !isTruncateTransaction + ) { + throw new DuplicateKeySyncError(key, this.id) + } + } + + const message: ChangeMessage = { + ...messageWithoutKey, + key, + } + pendingTransaction.operations.push(message) + + if (messageWithoutKey.type === `delete`) { + pendingTransaction.deletedKeys.add(key) + } + }, + commit: () => { + const pendingTransaction = + state.pendingSyncedTransactions[ + state.pendingSyncedTransactions.length - 1 + ] + if (!pendingTransaction) { + throw new NoPendingSyncTransactionCommitError() + } + if (pendingTransaction.committed) { + throw new SyncTransactionAlreadyCommittedError() + } + + pendingTransaction.committed = true + + // Update status to initialCommit when transitioning from loading + // This indicates we're in the process of committing the first transaction + if (this.lifecycle.status === `loading`) { + this.lifecycle.setStatus(`initialCommit`) + } + + state.commitPendingTransactions() + }, + markReady: () => { + this.lifecycle.markReady() + }, + truncate: () => { + const pendingTransaction = + state.pendingSyncedTransactions[ + state.pendingSyncedTransactions.length - 1 + ] + if (!pendingTransaction) { + throw new NoPendingSyncTransactionWriteError() + } + if (pendingTransaction.committed) { + throw new SyncTransactionAlreadyCommittedWriteError() + } + + // Clear all operations from the current transaction + pendingTransaction.operations = [] + pendingTransaction.deletedKeys.clear() + + // Mark the transaction as a truncate operation. During commit, this triggers: + // - Delete events for all previously synced keys (excluding optimistic-deleted keys) + // - Clearing of syncedData/syncedMetadata + // - Subsequent synced ops applied on the fresh base + // - Finally, optimistic mutations re-applied on top (single batch) + pendingTransaction.truncate = true + }, + }) + + // Store cleanup function if provided + this.syncCleanupFn = typeof cleanupFn === `function` ? cleanupFn : null + } catch (error) { + this.lifecycle.setStatus(`error`) + throw error + } + } + + /** + * Preload the collection data by starting sync if not already started + * Multiple concurrent calls will share the same promise + */ + public preload(): Promise { + if (this.preloadPromise) { + return this.preloadPromise + } + + this.preloadPromise = new Promise((resolve, reject) => { + if (this.lifecycle.status === `ready`) { + resolve() + return + } + + if (this.lifecycle.status === `error`) { + reject(new CollectionIsInErrorStateError()) + return + } + + // Register callback BEFORE starting sync to avoid race condition + this.lifecycle.onFirstReady(() => { + resolve() + }) + + // Start sync if collection hasn't started yet or was cleaned up + if ( + this.lifecycle.status === `idle` || + this.lifecycle.status === `cleaned-up` + ) { + try { + this.startSync() + } catch (error) { + reject(error) + return + } + } + }) + + return this.preloadPromise + } + + public cleanup(): void { + try { + if (this.syncCleanupFn) { + this.syncCleanupFn() + this.syncCleanupFn = null + } + } catch (error) { + // Re-throw in a microtask to surface the error after cleanup completes + queueMicrotask(() => { + if (error instanceof Error) { + // Preserve the original error and stack trace + const wrappedError = new SyncCleanupError(this.id, error) + wrappedError.cause = error + wrappedError.stack = error.stack + throw wrappedError + } else { + throw new SyncCleanupError(this.id, error as Error | string) + } + }) + } + this.preloadPromise = null + } +} diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index e7ccebed5..66f15ff10 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -1,5 +1,5 @@ // Re-export all public APIs -export * from "./collection" +export * from "./collection/index.js" export * from "./SortedMap" export * from "./transactions" export * from "./types" @@ -17,4 +17,4 @@ export * from "./indexes/lazy-index.js" export { type IndexOptions } from "./indexes/index-options.js" // Re-export some stuff explicitly to ensure the type & value is exported -export type { Collection } from "./collection" +export type { Collection } from "./collection/index.js" diff --git a/packages/db/src/indexes/auto-index.ts b/packages/db/src/indexes/auto-index.ts index 4280dcb62..b7caef579 100644 --- a/packages/db/src/indexes/auto-index.ts +++ b/packages/db/src/indexes/auto-index.ts @@ -1,6 +1,6 @@ import { BTreeIndex } from "./btree-index" import type { BasicExpression } from "../query/ir" -import type { CollectionImpl } from "../collection" +import type { CollectionImpl } from "../collection/index.js" export interface AutoIndexConfig { autoIndex?: `off` | `eager` diff --git a/packages/db/src/query/builder/index.ts b/packages/db/src/query/builder/index.ts index 6fe5fce65..423f5be05 100644 --- a/packages/db/src/query/builder/index.ts +++ b/packages/db/src/query/builder/index.ts @@ -1,4 +1,4 @@ -import { CollectionImpl } from "../../collection.js" +import { CollectionImpl } from "../../collection/index.js" import { Aggregate as AggregateExpr, CollectionRef, diff --git a/packages/db/src/query/builder/types.ts b/packages/db/src/query/builder/types.ts index ca0dc97a4..bef1c2bed 100644 --- a/packages/db/src/query/builder/types.ts +++ b/packages/db/src/query/builder/types.ts @@ -1,4 +1,4 @@ -import type { CollectionImpl } from "../../collection.js" +import type { CollectionImpl } from "../../collection/index.js" import type { Aggregate, BasicExpression, diff --git a/packages/db/src/query/compiler/index.ts b/packages/db/src/query/compiler/index.ts index 257408fe8..a79ace809 100644 --- a/packages/db/src/query/compiler/index.ts +++ b/packages/db/src/query/compiler/index.ts @@ -13,6 +13,7 @@ import { processJoins } from "./joins.js" import { processGroupBy } from "./group-by.js" import { processOrderBy } from "./order-by.js" import { processSelect } from "./select.js" +import type { CollectionSubscription } from "../../collection/subscription.js" import type { OrderByOptimizationInfo } from "./order-by.js" import type { BasicExpression, @@ -21,14 +22,13 @@ import type { QueryRef, } from "../ir.js" import type { LazyCollectionCallbacks } from "./joins.js" -import type { Collection } from "../../collection.js" +import type { Collection } from "../../collection/index.js" import type { KeyedStream, NamespacedAndKeyedStream, ResultStream, } from "../../types.js" import type { QueryCache, QueryMapping } from "./types.js" -import { CollectionSubscription } from "../../collection-subscription.js" /** * Result of query compilation including both the pipeline and collection-specific WHERE clauses diff --git a/packages/db/src/query/compiler/joins.ts b/packages/db/src/query/compiler/joins.ts index 7cabe8450..42687e446 100644 --- a/packages/db/src/query/compiler/joins.ts +++ b/packages/db/src/query/compiler/joins.ts @@ -30,14 +30,14 @@ import type { QueryRef, } from "../ir.js" import type { IStreamBuilder, JoinType } from "@tanstack/db-ivm" -import type { Collection } from "../../collection.js" +import type { Collection } from "../../collection/index.js" import type { KeyedStream, NamespacedAndKeyedStream, NamespacedRow, } from "../../types.js" import type { QueryCache, QueryMapping } from "./types.js" -import type { CollectionSubscription } from "../../collection-subscription.js" +import type { CollectionSubscription } from "../../collection/subscription.js" export type LoadKeysFn = (key: Set) => void export type LazyCollectionCallbacks = { diff --git a/packages/db/src/query/compiler/order-by.ts b/packages/db/src/query/compiler/order-by.ts index ddf4703ab..d2d27bb26 100644 --- a/packages/db/src/query/compiler/order-by.ts +++ b/packages/db/src/query/compiler/order-by.ts @@ -11,7 +11,7 @@ import type { OrderByClause, QueryIR, Select } from "../ir.js" import type { NamespacedAndKeyedStream, NamespacedRow } from "../../types.js" import type { IStreamBuilder, KeyValue } from "@tanstack/db-ivm" import type { BaseIndex } from "../../indexes/base-index.js" -import type { Collection } from "../../collection.js" +import type { Collection } from "../../collection/index.js" export type OrderByOptimizationInfo = { offset: number diff --git a/packages/db/src/query/ir.ts b/packages/db/src/query/ir.ts index ee0647beb..add2b1cdc 100644 --- a/packages/db/src/query/ir.ts +++ b/packages/db/src/query/ir.ts @@ -3,7 +3,7 @@ This is the intermediate representation of the query. */ import type { CompareOptions } from "./builder/types" -import type { CollectionImpl } from "../collection" +import type { CollectionImpl } from "../collection/index.js" import type { NamespacedRow } from "../types" export interface QueryIR { diff --git a/packages/db/src/query/live-query-collection.ts b/packages/db/src/query/live-query-collection.ts index b3fc5d793..ec631cd8f 100644 --- a/packages/db/src/query/live-query-collection.ts +++ b/packages/db/src/query/live-query-collection.ts @@ -1,8 +1,8 @@ -import { createCollection } from "../collection.js" +import { createCollection } from "../collection/index.js" import { CollectionConfigBuilder } from "./live/collection-config-builder.js" import type { LiveQueryCollectionConfig } from "./live/types.js" import type { InitialQueryBuilder, QueryBuilder } from "./builder/index.js" -import type { Collection } from "../collection.js" +import type { Collection } from "../collection/index.js" import type { CollectionConfig, UtilsRecord } from "../types.js" import type { Context, GetResult } from "./builder/types.js" diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index 8f67cb644..72d90f905 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -2,10 +2,10 @@ import { D2, output } from "@tanstack/db-ivm" import { compileQuery } from "../compiler/index.js" import { buildQuery, getQueryIR } from "../builder/index.js" import { CollectionSubscriber } from "./collection-subscriber.js" -import type { CollectionSubscription } from "../../collection-subscription.js" +import type { CollectionSubscription } from "../../collection/subscription.js" import type { RootStreamBuilder } from "@tanstack/db-ivm" import type { OrderByOptimizationInfo } from "../compiler/order-by.js" -import type { Collection } from "../../collection.js" +import type { Collection } from "../../collection/index.js" import type { CollectionConfig, KeyedStream, diff --git a/packages/db/src/query/live/collection-subscriber.ts b/packages/db/src/query/live/collection-subscriber.ts index 319b8e09e..c1fb88b3a 100644 --- a/packages/db/src/query/live/collection-subscriber.ts +++ b/packages/db/src/query/live/collection-subscriber.ts @@ -2,12 +2,12 @@ import { MultiSet } from "@tanstack/db-ivm" import { convertToBasicExpression } from "../compiler/expressions.js" import type { FullSyncState } from "./types.js" import type { MultiSetArray, RootStreamBuilder } from "@tanstack/db-ivm" -import type { Collection } from "../../collection.js" +import type { Collection } from "../../collection/index.js" import type { ChangeMessage, SyncConfig } from "../../types.js" import type { Context, GetResult } from "../builder/types.js" import type { BasicExpression } from "../ir.js" import type { CollectionConfigBuilder } from "./collection-config-builder.js" -import type { CollectionSubscription } from "../../collection-subscription.js" +import type { CollectionSubscription } from "../../collection/subscription.js" export class CollectionSubscriber< TContext extends Context, diff --git a/packages/db/src/transactions.ts b/packages/db/src/transactions.ts index 66edbfcfe..8d4500c25 100644 --- a/packages/db/src/transactions.ts +++ b/packages/db/src/transactions.ts @@ -403,11 +403,11 @@ class Transaction> { const hasCalled = new Set() for (const mutation of this.mutations) { if (!hasCalled.has(mutation.collection.id)) { - mutation.collection.onTransactionStateChange() + mutation.collection._state.onTransactionStateChange() // Only call commitPendingTransactions if there are pending sync transactions - if (mutation.collection.pendingSyncedTransactions.length > 0) { - mutation.collection.commitPendingTransactions() + if (mutation.collection._state.pendingSyncedTransactions.length > 0) { + mutation.collection._state.commitPendingTransactions() } hasCalled.add(mutation.collection.id) diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index 6cdb41f88..38ae672ce 100644 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -1,5 +1,5 @@ import type { IStreamBuilder } from "@tanstack/db-ivm" -import type { Collection } from "./collection" +import type { Collection } from "./collection/index.js" import type { StandardSchemaV1 } from "@standard-schema/spec" import type { Transaction } from "./transactions" import type { BasicExpression, OrderBy } from "./query/ir.js" diff --git a/packages/db/tests/collection-auto-index.test.ts b/packages/db/tests/collection-auto-index.test.ts index 118b4a84d..f3821f7cb 100644 --- a/packages/db/tests/collection-auto-index.test.ts +++ b/packages/db/tests/collection-auto-index.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest" -import { createCollection } from "../src/collection" +import { createCollection } from "../src/collection/index.js" import { and, eq, @@ -11,12 +11,12 @@ import { } from "../src/query/builder/functions" import { createSingleRowRefProxy } from "../src/query/builder/ref-proxy" import { createLiveQueryCollection } from "../src" +import { PropRef } from "../src/query/ir" import { createIndexUsageTracker, expectIndexUsage, withIndexTracking, } from "./utils" -import { PropRef } from "../src/query/ir" // Global row proxy for expressions const row = createSingleRowRefProxy() diff --git a/packages/db/tests/collection-errors.test.ts b/packages/db/tests/collection-errors.test.ts index d8784760f..69237e84c 100644 --- a/packages/db/tests/collection-errors.test.ts +++ b/packages/db/tests/collection-errors.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" -import { createCollection } from "../src/collection" +import { createCollection } from "../src/collection/index.js" import { CollectionInErrorStateError, InvalidCollectionStatusTransitionError, @@ -372,12 +372,12 @@ describe(`Collection Error Handling`, () => { // Test invalid transition expect(() => { - collectionImpl.validateStatusTransition(`ready`, `loading`) + collectionImpl._lifecycle.validateStatusTransition(`ready`, `loading`) }).toThrow(InvalidCollectionStatusTransitionError) // Test valid transition expect(() => { - collectionImpl.validateStatusTransition(`idle`, `loading`) + collectionImpl._lifecycle.validateStatusTransition(`idle`, `loading`) }).not.toThrow() }) @@ -397,76 +397,106 @@ describe(`Collection Error Handling`, () => { // Valid transitions from idle expect(() => - collectionImpl.validateStatusTransition(`idle`, `loading`) + collectionImpl._lifecycle.validateStatusTransition(`idle`, `loading`) ).not.toThrow() expect(() => - collectionImpl.validateStatusTransition(`idle`, `error`) + collectionImpl._lifecycle.validateStatusTransition(`idle`, `error`) ).not.toThrow() expect(() => - collectionImpl.validateStatusTransition(`idle`, `cleaned-up`) + collectionImpl._lifecycle.validateStatusTransition(`idle`, `cleaned-up`) ).not.toThrow() // Valid transitions from loading expect(() => - collectionImpl.validateStatusTransition(`loading`, `initialCommit`) + collectionImpl._lifecycle.validateStatusTransition( + `loading`, + `initialCommit` + ) ).not.toThrow() expect(() => - collectionImpl.validateStatusTransition(`loading`, `error`) + collectionImpl._lifecycle.validateStatusTransition(`loading`, `error`) ).not.toThrow() expect(() => - collectionImpl.validateStatusTransition(`loading`, `cleaned-up`) + collectionImpl._lifecycle.validateStatusTransition( + `loading`, + `cleaned-up` + ) ).not.toThrow() // Valid transitions from initialCommit expect(() => - collectionImpl.validateStatusTransition(`initialCommit`, `ready`) + collectionImpl._lifecycle.validateStatusTransition( + `initialCommit`, + `ready` + ) ).not.toThrow() expect(() => - collectionImpl.validateStatusTransition(`initialCommit`, `error`) + collectionImpl._lifecycle.validateStatusTransition( + `initialCommit`, + `error` + ) ).not.toThrow() expect(() => - collectionImpl.validateStatusTransition(`initialCommit`, `cleaned-up`) + collectionImpl._lifecycle.validateStatusTransition( + `initialCommit`, + `cleaned-up` + ) ).not.toThrow() // Valid transitions from ready expect(() => - collectionImpl.validateStatusTransition(`ready`, `cleaned-up`) + collectionImpl._lifecycle.validateStatusTransition( + `ready`, + `cleaned-up` + ) ).not.toThrow() expect(() => - collectionImpl.validateStatusTransition(`ready`, `error`) + collectionImpl._lifecycle.validateStatusTransition(`ready`, `error`) ).not.toThrow() // Valid transitions from error (allow recovery) expect(() => - collectionImpl.validateStatusTransition(`error`, `cleaned-up`) + collectionImpl._lifecycle.validateStatusTransition( + `error`, + `cleaned-up` + ) ).not.toThrow() expect(() => - collectionImpl.validateStatusTransition(`error`, `idle`) + collectionImpl._lifecycle.validateStatusTransition(`error`, `idle`) ).not.toThrow() // Valid transitions from cleaned-up (allow restart) expect(() => - collectionImpl.validateStatusTransition(`cleaned-up`, `loading`) + collectionImpl._lifecycle.validateStatusTransition( + `cleaned-up`, + `loading` + ) ).not.toThrow() expect(() => - collectionImpl.validateStatusTransition(`cleaned-up`, `error`) + collectionImpl._lifecycle.validateStatusTransition( + `cleaned-up`, + `error` + ) ).not.toThrow() // Allow same-state transitions (idempotent operations) expect(() => - collectionImpl.validateStatusTransition(`idle`, `idle`) + collectionImpl._lifecycle.validateStatusTransition(`idle`, `idle`) ).not.toThrow() expect(() => - collectionImpl.validateStatusTransition( + collectionImpl._lifecycle.validateStatusTransition( `initialCommit`, `initialCommit` ) ).not.toThrow() expect(() => - collectionImpl.validateStatusTransition(`ready`, `ready`) + collectionImpl._lifecycle.validateStatusTransition(`ready`, `ready`) ).not.toThrow() expect(() => - collectionImpl.validateStatusTransition(`cleaned-up`, `cleaned-up`) + collectionImpl._lifecycle.validateStatusTransition( + `cleaned-up`, + `cleaned-up` + ) ).not.toThrow() }) }) diff --git a/packages/db/tests/collection-events.test.ts b/packages/db/tests/collection-events.test.ts index 518f3b7f3..04a31b621 100644 --- a/packages/db/tests/collection-events.test.ts +++ b/packages/db/tests/collection-events.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" -import { createCollection } from "../src/collection.js" +import { createCollection } from "../src/collection/index.js" describe(`Collection Events System`, () => { let collection: ReturnType diff --git a/packages/db/tests/collection-getters.test.ts b/packages/db/tests/collection-getters.test.ts index fa3c5682c..c7dc58770 100644 --- a/packages/db/tests/collection-getters.test.ts +++ b/packages/db/tests/collection-getters.test.ts @@ -1,7 +1,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest" import { createTransaction } from "../src/transactions" -import { createCollection } from "../src/collection" -import type { CollectionImpl } from "../src/collection" +import { createCollection } from "../src/collection/index.js" +import type { CollectionImpl } from "../src/collection/index.js" import type { SyncConfig } from "../src/types" type Item = { id: string; name: string } diff --git a/packages/db/tests/collection-indexes.test.ts b/packages/db/tests/collection-indexes.test.ts index 913cfae3f..d3537263a 100644 --- a/packages/db/tests/collection-indexes.test.ts +++ b/packages/db/tests/collection-indexes.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, it } from "vitest" import mitt from "mitt" -import { createCollection } from "../src/collection" +import { createCollection } from "../src/collection/index.js" import { createTransaction } from "../src/transactions" import { and, @@ -15,7 +15,7 @@ import { } from "../src/query/builder/functions" import { PropRef } from "../src/query/ir" import { expectIndexUsage, withIndexTracking } from "./utils" -import type { Collection } from "../src/collection" +import type { Collection } from "../src/collection/index.js" import type { MutationFn, PendingMutation } from "../src/types" interface TestItem { diff --git a/packages/db/tests/collection-lifecycle.test.ts b/packages/db/tests/collection-lifecycle.test.ts index 22ad48817..c794ddce0 100644 --- a/packages/db/tests/collection-lifecycle.test.ts +++ b/packages/db/tests/collection-lifecycle.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" -import { createCollection } from "../src/collection.js" +import { createCollection } from "../src/collection/index.js" // Mock setTimeout and clearTimeout for testing GC behavior const originalSetTimeout = global.setTimeout @@ -200,21 +200,21 @@ describe(`Collection Lifecycle Management`, () => { }) // No subscribers initially - expect((collection as any).activeSubscribersCount).toBe(0) + expect(collection.subscriberCount).toBe(0) // Subscribe to changes const subscription1 = collection.subscribeChanges(() => {}) - expect((collection as any).activeSubscribersCount).toBe(1) + expect(collection.subscriberCount).toBe(1) const subscription2 = collection.subscribeChanges(() => {}) - expect((collection as any).activeSubscribersCount).toBe(2) + expect(collection.subscriberCount).toBe(2) // Unsubscribe subscription1.unsubscribe() - expect((collection as any).activeSubscribersCount).toBe(1) + expect(collection.subscriberCount).toBe(1) subscription2.unsubscribe() - expect((collection as any).activeSubscribersCount).toBe(0) + expect(collection.subscriberCount).toBe(0) }) it(`should handle rapid subscribe/unsubscribe correctly`, () => { @@ -230,9 +230,9 @@ describe(`Collection Lifecycle Management`, () => { // Subscribe and immediately unsubscribe multiple times for (let i = 0; i < 5; i++) { const subscription = collection.subscribeChanges(() => {}) - expect((collection as any).activeSubscribersCount).toBe(1) + expect(collection.subscriberCount).toBe(1) subscription.unsubscribe() - expect((collection as any).activeSubscribersCount).toBe(0) + expect(collection.subscriberCount).toBe(0) // Should start GC timer each time expect(mockSetTimeout).toHaveBeenCalledWith(expect.any(Function), 1000) diff --git a/packages/db/tests/collection-schema.test.ts b/packages/db/tests/collection-schema.test.ts index 3dc5e7293..0db0bddfc 100644 --- a/packages/db/tests/collection-schema.test.ts +++ b/packages/db/tests/collection-schema.test.ts @@ -1,7 +1,7 @@ import { type } from "arktype" import { describe, expect, expectTypeOf, it } from "vitest" import { z } from "zod" -import { createCollection } from "../src/collection" +import { createCollection } from "../src/collection/index.js" import { SchemaValidationError } from "../src/errors" import { createTransaction } from "../src/transactions" import type { @@ -64,7 +64,7 @@ describe(`Collection Schema Validation`, () => { }) // Add the validated insert data to the update collection - ;(updateCollection as any).syncedData.set(`1`, validatedInsert) + updateCollection._state.syncedData.set(`1`, validatedInsert) const updateData = { name: `Jane Doe`, @@ -116,7 +116,7 @@ describe(`Collection Schema Validation`, () => { const validatedInsert = collection.validateData(insertData, `insert`) // Manually add the item to the collection's synced data for testing - ;(collection as any).syncedData.set(`1`, validatedInsert) + collection._state.syncedData.set(`1`, validatedInsert) // Test update validation with only modified fields const updateData = { @@ -176,7 +176,7 @@ describe(`Collection Schema Validation`, () => { expect(validatedInsert.email).toBe(`john@example.com`) // Manually add the item to the collection's synced data for testing - ;(collection as any).syncedData.set(`1`, validatedInsert) + collection._state.syncedData.set(`1`, validatedInsert) // Test update validation without providing defaulted fields const updateData = { @@ -248,7 +248,7 @@ describe(`Collection Schema Validation`, () => { expect(typeof validatedInsert.age).toBe(`number`) // Add to collection for update testing - ;(collection as any).syncedData.set(`1`, validatedInsert) + collection._state.syncedData.set(`1`, validatedInsert) // Test that update validation accepts input type for new fields const updateData = { diff --git a/packages/db/tests/collection-subscribe-changes.test.ts b/packages/db/tests/collection-subscribe-changes.test.ts index 4abcc37e7..31ca3bf7d 100644 --- a/packages/db/tests/collection-subscribe-changes.test.ts +++ b/packages/db/tests/collection-subscribe-changes.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from "vitest" import mitt from "mitt" -import { createCollection } from "../src/collection" +import { createCollection } from "../src/collection/index.js" import { createTransaction } from "../src/transactions" import { eq } from "../src/query/builder/functions" import { PropRef } from "../src/query/ir" @@ -679,15 +679,15 @@ describe(`Collection.subscribeChanges`, () => { syncCollection.config.sync.sync({ collection: syncCollection, begin: () => { - syncCollection.pendingSyncedTransactions.push({ + syncCollection._state.pendingSyncedTransactions.push({ committed: false, operations: [], }) }, write: (messageWithoutKey: any) => { const pendingTransaction = - syncCollection.pendingSyncedTransactions[ - syncCollection.pendingSyncedTransactions.length - 1 + syncCollection._state.pendingSyncedTransactions[ + syncCollection._state.pendingSyncedTransactions.length - 1 ] const key = syncCollection.getKeyFromItem(messageWithoutKey.value) const message = { ...messageWithoutKey, key } @@ -695,8 +695,8 @@ describe(`Collection.subscribeChanges`, () => { }, commit: () => { const pendingTransaction = - syncCollection.pendingSyncedTransactions[ - syncCollection.pendingSyncedTransactions.length - 1 + syncCollection._state.pendingSyncedTransactions[ + syncCollection._state.pendingSyncedTransactions.length - 1 ] pendingTransaction.committed = true syncCollection.commitPendingTransactions() @@ -937,8 +937,8 @@ describe(`Collection.subscribeChanges`, () => { const tx2 = collection.insert({ id: 3, value: `optimistic insert` }) // Verify optimistic state exists - expect(collection.optimisticUpserts.has(1)).toBe(true) - expect(collection.optimisticUpserts.has(3)).toBe(true) + expect(collection._state.optimisticUpserts.has(1)).toBe(true) + expect(collection._state.optimisticUpserts.has(3)).toBe(true) expect(collection.state.get(1)?.value).toBe(`optimistic update 1`) expect(collection.state.get(3)?.value).toBe(`optimistic insert`) diff --git a/packages/db/tests/collection.test-d.ts b/packages/db/tests/collection.test-d.ts index b513ff750..d70d53e19 100644 --- a/packages/db/tests/collection.test-d.ts +++ b/packages/db/tests/collection.test-d.ts @@ -1,6 +1,6 @@ import { assertType, describe, expectTypeOf, it } from "vitest" import { z } from "zod" -import { createCollection } from "../src/collection" +import { createCollection } from "../src/collection/index.js" import type { OperationConfig } from "../src/types" import type { StandardSchemaV1 } from "@standard-schema/spec" diff --git a/packages/db/tests/collection.test.ts b/packages/db/tests/collection.test.ts index 74cd66077..9abfbb2ac 100644 --- a/packages/db/tests/collection.test.ts +++ b/packages/db/tests/collection.test.ts @@ -1,6 +1,6 @@ import mitt from "mitt" import { describe, expect, it, vi } from "vitest" -import { createCollection } from "../src/collection" +import { createCollection } from "../src/collection/index.js" import { CollectionRequiresConfigError, DuplicateKeyError, @@ -227,8 +227,8 @@ describe(`Collection`, () => { // Check the optimistic operation is there const insertKey = 1 - expect(collection.optimisticUpserts.has(insertKey)).toBe(true) - expect(collection.optimisticUpserts.get(insertKey)).toEqual({ + expect(collection._state.optimisticUpserts.has(insertKey)).toBe(true) + expect(collection._state.optimisticUpserts.get(insertKey)).toEqual({ id: 1, value: `bar`, }) @@ -262,11 +262,11 @@ describe(`Collection`, () => { // after mutationFn returns, check that the transaction is cleaned up, // optimistic update is gone & synced data & combined state are all updated. - expect(collection.transactions.size).toEqual(0) // Transaction should be cleaned up + expect(collection._state.transactions.size).toEqual(0) // Transaction should be cleaned up expect(collection.state).toEqual( new Map([[insertedKey, { id: 1, value: `bar` }]]) ) - expect(collection.optimisticUpserts.size).toEqual(0) + expect(collection._state.optimisticUpserts.size).toEqual(0) // Test insert with provided key const tx2 = createTransaction({ mutationFn }) @@ -480,7 +480,8 @@ describe(`Collection`, () => { // check there's a transaction in peristing state expect( // @ts-expect-error possibly undefined is ok in test - Array.from(collection.transactions.values())[0].mutations[0].changes + Array.from(collection._state.transactions.values())[0].mutations[0] + .changes ).toEqual({ id: 1, value: `bar`, @@ -488,8 +489,8 @@ describe(`Collection`, () => { // Check the optimistic operation is there const insertKey = 1 - expect(collection.optimisticUpserts.has(insertKey)).toBe(true) - expect(collection.optimisticUpserts.get(insertKey)).toEqual({ + expect(collection._state.optimisticUpserts.has(insertKey)).toBe(true) + expect(collection._state.optimisticUpserts.get(insertKey)).toEqual({ id: 1, value: `bar`, }) @@ -789,7 +790,7 @@ describe(`Collection`, () => { // The item should NOT appear in the collection state immediately expect(collection.state.has(2)).toBe(false) - expect(collection.optimisticUpserts.has(2)).toBe(false) + expect(collection._state.optimisticUpserts.has(2)).toBe(false) expect(collection.state.size).toBe(1) // Only the initial item // Now resolve the mutation and wait for completion @@ -814,7 +815,7 @@ describe(`Collection`, () => { // The original value should still be there immediately expect(collection.state.get(1)?.value).toBe(`initial value`) - expect(collection.optimisticUpserts.has(1)).toBe(false) + expect(collection._state.optimisticUpserts.has(1)).toBe(false) // Now resolve the update mutation and wait for completion pendingMutations[1]?.() @@ -828,7 +829,7 @@ describe(`Collection`, () => { // The item should still be there immediately expect(collection.state.has(2)).toBe(true) - expect(collection.optimisticDeletes.has(2)).toBe(false) + expect(collection._state.optimisticDeletes.has(2)).toBe(false) // Now resolve the delete mutation and wait for completion pendingMutations[2]?.() @@ -889,7 +890,7 @@ describe(`Collection`, () => { // The item should appear immediately expect(collection.state.has(2)).toBe(true) - expect(collection.optimisticUpserts.has(2)).toBe(true) + expect(collection._state.optimisticUpserts.has(2)).toBe(true) expect(collection.state.get(2)).toEqual({ id: 2, value: `default optimistic`, @@ -905,7 +906,7 @@ describe(`Collection`, () => { // The item should appear immediately expect(collection.state.has(3)).toBe(true) - expect(collection.optimisticUpserts.has(3)).toBe(true) + expect(collection._state.optimisticUpserts.has(3)).toBe(true) expect(collection.state.get(3)).toEqual({ id: 3, value: `explicit optimistic`, @@ -924,7 +925,7 @@ describe(`Collection`, () => { // The update should be reflected immediately expect(collection.state.get(1)?.value).toBe(`optimistic update`) - expect(collection.optimisticUpserts.has(1)).toBe(true) + expect(collection._state.optimisticUpserts.has(1)).toBe(true) await optimisticUpdateTx.isPersisted.promise @@ -933,7 +934,7 @@ describe(`Collection`, () => { // The item should be gone immediately expect(collection.state.has(3)).toBe(false) - expect(collection.optimisticDeletes.has(3)).toBe(true) + expect(collection._state.optimisticDeletes.has(3)).toBe(true) await optimisticDeleteTx.isPersisted.promise }) @@ -1097,8 +1098,8 @@ describe(`Collection`, () => { // Verify collection is cleared expect(collection.state.size).toBe(0) - expect(collection.syncedData.size).toBe(0) - expect(collection.syncedMetadata.size).toBe(0) + expect(collection._state.syncedData.size).toBe(0) + expect(collection._state.syncedMetadata.size).toBe(0) }) it(`should keep operations written after truncate in the same transaction`, async () => { @@ -1163,8 +1164,8 @@ describe(`Collection`, () => { id: 3, value: `should not be cleared`, }) - expect(collection.syncedData.size).toBe(1) - expect(collection.syncedMetadata.size).toBe(1) + expect(collection._state.syncedData.size).toBe(1) + expect(collection._state.syncedMetadata.size).toBe(1) }) it(`should handle truncate with empty collection`, async () => { @@ -1200,8 +1201,8 @@ describe(`Collection`, () => { // Verify collection remains empty expect(collection.state.size).toBe(0) - expect(collection.syncedData.size).toBe(0) - expect(collection.syncedMetadata.size).toBe(0) + expect(collection._state.syncedData.size).toBe(0) + expect(collection._state.syncedMetadata.size).toBe(0) }) it(`open sync transaction isn't applied when optimistic mutation is resolved/rejected`, async () => { diff --git a/packages/db/tests/query/basic.test-d.ts b/packages/db/tests/query/basic.test-d.ts index d3b325f54..1817cd124 100644 --- a/packages/db/tests/query/basic.test-d.ts +++ b/packages/db/tests/query/basic.test-d.ts @@ -2,7 +2,7 @@ import { describe, expectTypeOf, test } from "vitest" import { z } from "zod" import { type } from "arktype" import { createLiveQueryCollection, eq, gt } from "../../src/query/index.js" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { mockSyncCollectionOptions } from "../utils.js" // Sample user type for tests diff --git a/packages/db/tests/query/basic.test.ts b/packages/db/tests/query/basic.test.ts index db26a28df..dccb0eab0 100644 --- a/packages/db/tests/query/basic.test.ts +++ b/packages/db/tests/query/basic.test.ts @@ -6,7 +6,7 @@ import { gt, upper, } from "../../src/query/index.js" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { mockSyncCollectionOptions } from "../utils.js" // Sample user type for tests diff --git a/packages/db/tests/query/builder/buildQuery.test.ts b/packages/db/tests/query/builder/buildQuery.test.ts index fc2590fbb..9450c0656 100644 --- a/packages/db/tests/query/builder/buildQuery.test.ts +++ b/packages/db/tests/query/builder/buildQuery.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest" -import { CollectionImpl } from "../../../src/collection.js" +import { CollectionImpl } from "../../../src/collection/index.js" import { buildQuery } from "../../../src/query/builder/index.js" import { and, eq, gt, or } from "../../../src/query/builder/functions.js" diff --git a/packages/db/tests/query/builder/callback-types.test-d.ts b/packages/db/tests/query/builder/callback-types.test-d.ts index 20c577188..e45b76bfa 100644 --- a/packages/db/tests/query/builder/callback-types.test-d.ts +++ b/packages/db/tests/query/builder/callback-types.test-d.ts @@ -1,5 +1,5 @@ import { describe, expectTypeOf, test } from "vitest" -import { createCollection } from "../../../src/collection.js" +import { createCollection } from "../../../src/collection/index.js" import { mockSyncCollectionOptions } from "../../utils.js" import { Query } from "../../../src/query/builder/index.js" import { diff --git a/packages/db/tests/query/builder/from.test.ts b/packages/db/tests/query/builder/from.test.ts index 340a396bc..62946b094 100644 --- a/packages/db/tests/query/builder/from.test.ts +++ b/packages/db/tests/query/builder/from.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest" -import { CollectionImpl } from "../../../src/collection.js" +import { CollectionImpl } from "../../../src/collection/index.js" import { Query, getQueryIR } from "../../../src/query/builder/index.js" import { eq } from "../../../src/query/builder/functions.js" import { diff --git a/packages/db/tests/query/builder/functional-variants.test.ts b/packages/db/tests/query/builder/functional-variants.test.ts index eb87ac277..0ce18cf86 100644 --- a/packages/db/tests/query/builder/functional-variants.test.ts +++ b/packages/db/tests/query/builder/functional-variants.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest" -import { CollectionImpl } from "../../../src/collection.js" +import { CollectionImpl } from "../../../src/collection/index.js" import { Query, getQueryIR } from "../../../src/query/builder/index.js" import { eq, gt } from "../../../src/query/builder/functions.js" diff --git a/packages/db/tests/query/builder/functions.test.ts b/packages/db/tests/query/builder/functions.test.ts index 6648cf62a..383771c7e 100644 --- a/packages/db/tests/query/builder/functions.test.ts +++ b/packages/db/tests/query/builder/functions.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest" -import { CollectionImpl } from "../../../src/collection.js" +import { CollectionImpl } from "../../../src/collection/index.js" import { Query, getQueryIR } from "../../../src/query/builder/index.js" import { add, diff --git a/packages/db/tests/query/builder/group-by.test.ts b/packages/db/tests/query/builder/group-by.test.ts index d12b25254..3cbb9632f 100644 --- a/packages/db/tests/query/builder/group-by.test.ts +++ b/packages/db/tests/query/builder/group-by.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest" -import { CollectionImpl } from "../../../src/collection.js" +import { CollectionImpl } from "../../../src/collection/index.js" import { Query, getQueryIR } from "../../../src/query/builder/index.js" import { avg, count, eq, sum } from "../../../src/query/builder/functions.js" diff --git a/packages/db/tests/query/builder/join.test.ts b/packages/db/tests/query/builder/join.test.ts index cbd09acf2..470ee3853 100644 --- a/packages/db/tests/query/builder/join.test.ts +++ b/packages/db/tests/query/builder/join.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest" -import { CollectionImpl } from "../../../src/collection.js" +import { CollectionImpl } from "../../../src/collection/index.js" import { Query, getQueryIR } from "../../../src/query/builder/index.js" import { and, eq, gt } from "../../../src/query/builder/functions.js" diff --git a/packages/db/tests/query/builder/order-by.test.ts b/packages/db/tests/query/builder/order-by.test.ts index 5f5270eaa..378371d9e 100644 --- a/packages/db/tests/query/builder/order-by.test.ts +++ b/packages/db/tests/query/builder/order-by.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest" -import { CollectionImpl } from "../../../src/collection.js" +import { CollectionImpl } from "../../../src/collection/index.js" import { Query, getQueryIR } from "../../../src/query/builder/index.js" import { eq, upper } from "../../../src/query/builder/functions.js" diff --git a/packages/db/tests/query/builder/select.test.ts b/packages/db/tests/query/builder/select.test.ts index c9bb10c72..3c8ae4352 100644 --- a/packages/db/tests/query/builder/select.test.ts +++ b/packages/db/tests/query/builder/select.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest" -import { CollectionImpl } from "../../../src/collection.js" +import { CollectionImpl } from "../../../src/collection/index.js" import { Query, getQueryIR } from "../../../src/query/builder/index.js" import { avg, diff --git a/packages/db/tests/query/builder/subqueries.test-d.ts b/packages/db/tests/query/builder/subqueries.test-d.ts index ee2b38508..88fdedd47 100644 --- a/packages/db/tests/query/builder/subqueries.test-d.ts +++ b/packages/db/tests/query/builder/subqueries.test-d.ts @@ -1,6 +1,6 @@ import { describe, expectTypeOf, test } from "vitest" import { Query } from "../../../src/query/builder/index.js" -import { CollectionImpl } from "../../../src/collection.js" +import { CollectionImpl } from "../../../src/collection/index.js" import { avg, count, eq } from "../../../src/query/builder/functions.js" import type { ExtractContext } from "../../../src/query/builder/index.js" import type { GetResult } from "../../../src/query/builder/types.js" diff --git a/packages/db/tests/query/builder/where.test.ts b/packages/db/tests/query/builder/where.test.ts index 5a9ce7fbe..7021cc790 100644 --- a/packages/db/tests/query/builder/where.test.ts +++ b/packages/db/tests/query/builder/where.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest" -import { CollectionImpl } from "../../../src/collection.js" +import { CollectionImpl } from "../../../src/collection/index.js" import { Query, getQueryIR } from "../../../src/query/builder/index.js" import { and, diff --git a/packages/db/tests/query/compiler/basic.test.ts b/packages/db/tests/query/compiler/basic.test.ts index 74cfef707..d4e8afcd0 100644 --- a/packages/db/tests/query/compiler/basic.test.ts +++ b/packages/db/tests/query/compiler/basic.test.ts @@ -3,7 +3,7 @@ import { D2, MultiSet, output } from "@tanstack/db-ivm" import { compileQuery } from "../../../src/query/compiler/index.js" import { CollectionRef, Func, PropRef, Value } from "../../../src/query/ir.js" import type { QueryIR } from "../../../src/query/ir.js" -import type { CollectionImpl } from "../../../src/collection.js" +import type { CollectionImpl } from "../../../src/collection/index.js" // Sample user type for tests type User = { diff --git a/packages/db/tests/query/compiler/subqueries.test.ts b/packages/db/tests/query/compiler/subqueries.test.ts index 1cef89cd4..9903f7a6c 100644 --- a/packages/db/tests/query/compiler/subqueries.test.ts +++ b/packages/db/tests/query/compiler/subqueries.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from "vitest" import { D2, MultiSet, output } from "@tanstack/db-ivm" import { Query, getQueryIR } from "../../../src/query/builder/index.js" import { compileQuery } from "../../../src/query/compiler/index.js" -import { CollectionImpl } from "../../../src/collection.js" +import { CollectionImpl } from "../../../src/collection/index.js" import { avg, count, eq } from "../../../src/query/builder/functions.js" // Test schema types diff --git a/packages/db/tests/query/compiler/subquery-caching.test.ts b/packages/db/tests/query/compiler/subquery-caching.test.ts index ad1108c0a..211452336 100644 --- a/packages/db/tests/query/compiler/subquery-caching.test.ts +++ b/packages/db/tests/query/compiler/subquery-caching.test.ts @@ -3,7 +3,7 @@ import { D2 } from "@tanstack/db-ivm" import { compileQuery } from "../../../src/query/compiler/index.js" import { CollectionRef, PropRef, QueryRef } from "../../../src/query/ir.js" import type { QueryIR } from "../../../src/query/ir.js" -import type { CollectionImpl } from "../../../src/collection.js" +import type { CollectionImpl } from "../../../src/collection/index.js" describe(`Subquery Caching`, () => { it(`should cache compiled subqueries and avoid duplicate compilation`, () => { diff --git a/packages/db/tests/query/composables.test.ts b/packages/db/tests/query/composables.test.ts index 423c9613c..a6bd43d10 100644 --- a/packages/db/tests/query/composables.test.ts +++ b/packages/db/tests/query/composables.test.ts @@ -10,7 +10,7 @@ import { lte, upper, } from "../../src/query/index.js" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { mockSyncCollectionOptions } from "../utils.js" import type { Ref } from "../../src/query/index.js" diff --git a/packages/db/tests/query/distinct.test.ts b/packages/db/tests/query/distinct.test.ts index 1f2b0414f..0d3b480fc 100644 --- a/packages/db/tests/query/distinct.test.ts +++ b/packages/db/tests/query/distinct.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, test } from "vitest" import { concat, createLiveQueryCollection } from "../../src/query/index.js" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { mockSyncCollectionOptions } from "../utils.js" import { DistinctRequiresSelectError } from "../../src/errors" import { count, eq, gte, not } from "../../src/query/builder/functions.js" diff --git a/packages/db/tests/query/functional-variants.test-d.ts b/packages/db/tests/query/functional-variants.test-d.ts index 136bd202d..51e6628d8 100644 --- a/packages/db/tests/query/functional-variants.test-d.ts +++ b/packages/db/tests/query/functional-variants.test-d.ts @@ -5,7 +5,7 @@ import { eq, gt, } from "../../src/query/index.js" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { mockSyncCollectionOptions } from "../utils.js" // Sample user type for tests diff --git a/packages/db/tests/query/functional-variants.test.ts b/packages/db/tests/query/functional-variants.test.ts index e006f7c19..2c12e020e 100644 --- a/packages/db/tests/query/functional-variants.test.ts +++ b/packages/db/tests/query/functional-variants.test.ts @@ -5,7 +5,7 @@ import { eq, gt, } from "../../src/query/index.js" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { mockSyncCollectionOptions } from "../utils.js" // Sample user type for tests diff --git a/packages/db/tests/query/group-by.test-d.ts b/packages/db/tests/query/group-by.test-d.ts index 1efad2209..f25ee92e2 100644 --- a/packages/db/tests/query/group-by.test-d.ts +++ b/packages/db/tests/query/group-by.test-d.ts @@ -1,6 +1,6 @@ import { describe, expectTypeOf, test } from "vitest" import { createLiveQueryCollection } from "../../src/query/index.js" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { mockSyncCollectionOptions } from "../utils.js" import { and, diff --git a/packages/db/tests/query/group-by.test.ts b/packages/db/tests/query/group-by.test.ts index 2fa6b428a..452f98f33 100644 --- a/packages/db/tests/query/group-by.test.ts +++ b/packages/db/tests/query/group-by.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, test } from "vitest" import { createLiveQueryCollection } from "../../src/query/index.js" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { mockSyncCollectionOptions } from "../utils.js" import { and, diff --git a/packages/db/tests/query/indexes.test.ts b/packages/db/tests/query/indexes.test.ts index 6750afd90..b10c6fd17 100644 --- a/packages/db/tests/query/indexes.test.ts +++ b/packages/db/tests/query/indexes.test.ts @@ -1,5 +1,5 @@ import { beforeEach, describe, expect, it } from "vitest" -import { createCollection } from "../../src/collection" +import { createCollection } from "../../src/collection/index.js" import { createLiveQueryCollection } from "../../src/query/live-query-collection" import { diff --git a/packages/db/tests/query/join-subquery.test-d.ts b/packages/db/tests/query/join-subquery.test-d.ts index c339c4a82..f529d0e99 100644 --- a/packages/db/tests/query/join-subquery.test-d.ts +++ b/packages/db/tests/query/join-subquery.test-d.ts @@ -1,6 +1,6 @@ import { describe, expectTypeOf, test } from "vitest" import { createLiveQueryCollection, eq, gt } from "../../src/query/index.js" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { mockSyncCollectionOptions } from "../utils.js" // Sample data types for join-subquery testing diff --git a/packages/db/tests/query/join-subquery.test.ts b/packages/db/tests/query/join-subquery.test.ts index a9388ea88..e2fd684e8 100644 --- a/packages/db/tests/query/join-subquery.test.ts +++ b/packages/db/tests/query/join-subquery.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, test } from "vitest" import { createLiveQueryCollection, eq, gt } from "../../src/query/index.js" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { mockSyncCollectionOptions } from "../utils.js" // Sample data types for join-subquery testing diff --git a/packages/db/tests/query/join.test-d.ts b/packages/db/tests/query/join.test-d.ts index b5f2ffbb0..f453465c9 100644 --- a/packages/db/tests/query/join.test-d.ts +++ b/packages/db/tests/query/join.test-d.ts @@ -2,7 +2,7 @@ import { describe, expectTypeOf, test } from "vitest" import { z } from "zod" import { type } from "arktype" import { createLiveQueryCollection, eq } from "../../src/query/index.js" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { mockSyncCollectionOptions } from "../utils.js" // Sample data types for join type testing diff --git a/packages/db/tests/query/join.test.ts b/packages/db/tests/query/join.test.ts index 765c5e51d..9b2d3ef24 100644 --- a/packages/db/tests/query/join.test.ts +++ b/packages/db/tests/query/join.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, test } from "vitest" import { createLiveQueryCollection, eq } from "../../src/query/index.js" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { mockSyncCollectionOptions } from "../utils.js" // Sample data types for join testing diff --git a/packages/db/tests/query/live-query-collection.test.ts b/packages/db/tests/query/live-query-collection.test.ts index 6b7dbe0b8..e16a8dfef 100644 --- a/packages/db/tests/query/live-query-collection.test.ts +++ b/packages/db/tests/query/live-query-collection.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, it } from "vitest" import { Temporal } from "temporal-polyfill" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { createLiveQueryCollection, eq, diff --git a/packages/db/tests/query/nested-props.test-d.ts b/packages/db/tests/query/nested-props.test-d.ts index ced2f054a..801b099c1 100644 --- a/packages/db/tests/query/nested-props.test-d.ts +++ b/packages/db/tests/query/nested-props.test-d.ts @@ -6,7 +6,7 @@ import { gt, or, } from "../../src/query/index.js" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { mockSyncCollectionOptions } from "../utils.js" // Complex nested type for testing with optional properties diff --git a/packages/db/tests/query/optional-fields-negative.test-d.ts b/packages/db/tests/query/optional-fields-negative.test-d.ts index 688e1e1f3..153361b68 100644 --- a/packages/db/tests/query/optional-fields-negative.test-d.ts +++ b/packages/db/tests/query/optional-fields-negative.test-d.ts @@ -1,6 +1,6 @@ import { describe, expectTypeOf, test } from "vitest" import { createLiveQueryCollection, eq, gt } from "../../src/query/index.js" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { mockSyncCollectionOptions } from "../utils.js" // Test types with optional fields diff --git a/packages/db/tests/query/optional-fields-runtime.test.ts b/packages/db/tests/query/optional-fields-runtime.test.ts index b79d5c7a5..41fa4888d 100644 --- a/packages/db/tests/query/optional-fields-runtime.test.ts +++ b/packages/db/tests/query/optional-fields-runtime.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest" import { createLiveQueryCollection, eq } from "../../src/query/index.js" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { mockSyncCollectionOptions } from "../utils.js" // Test types with optional fields diff --git a/packages/db/tests/query/order-by.test.ts b/packages/db/tests/query/order-by.test.ts index 4b336dc44..b26603c6a 100644 --- a/packages/db/tests/query/order-by.test.ts +++ b/packages/db/tests/query/order-by.test.ts @@ -1,5 +1,5 @@ import { beforeEach, describe, expect, it } from "vitest" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { mockSyncCollectionOptions } from "../utils.js" import { createLiveQueryCollection } from "../../src/query/live-query-collection.js" import { diff --git a/packages/db/tests/query/select-spread.test-d.ts b/packages/db/tests/query/select-spread.test-d.ts index c7a9a344a..a578a6401 100644 --- a/packages/db/tests/query/select-spread.test-d.ts +++ b/packages/db/tests/query/select-spread.test-d.ts @@ -1,5 +1,5 @@ import { describe, expectTypeOf, test } from "vitest" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { createLiveQueryCollection } from "../../src/query/index.js" import { mockSyncCollectionOptions } from "../utils.js" import { add, length, upper } from "../../src/query/builder/functions.js" diff --git a/packages/db/tests/query/select-spread.test.ts b/packages/db/tests/query/select-spread.test.ts index 988461841..ee713c667 100644 --- a/packages/db/tests/query/select-spread.test.ts +++ b/packages/db/tests/query/select-spread.test.ts @@ -1,5 +1,5 @@ import { beforeEach, describe, expect, it } from "vitest" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { createLiveQueryCollection } from "../../src/query/index.js" import { mockSyncCollectionOptions } from "../utils.js" import { add, eq, upper } from "../../src/query/builder/functions.js" diff --git a/packages/db/tests/query/select.test-d.ts b/packages/db/tests/query/select.test-d.ts index fd472061b..b41bb60a6 100644 --- a/packages/db/tests/query/select.test-d.ts +++ b/packages/db/tests/query/select.test-d.ts @@ -1,5 +1,5 @@ import { describe, expectTypeOf, test } from "vitest" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { createLiveQueryCollection } from "../../src/query/index.js" import { mockSyncCollectionOptions } from "../utils.js" import { upper } from "../../src/query/builder/functions.js" diff --git a/packages/db/tests/query/select.test.ts b/packages/db/tests/query/select.test.ts index 86cf1333d..0a14e1b18 100644 --- a/packages/db/tests/query/select.test.ts +++ b/packages/db/tests/query/select.test.ts @@ -1,5 +1,5 @@ import { beforeEach, describe, expect, it } from "vitest" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { createLiveQueryCollection } from "../../src/query/index.js" import { mockSyncCollectionOptions } from "../utils.js" import { upper } from "../../src/query/builder/functions.js" diff --git a/packages/db/tests/query/subquery.test-d.ts b/packages/db/tests/query/subquery.test-d.ts index c2b306e70..5c169eeb4 100644 --- a/packages/db/tests/query/subquery.test-d.ts +++ b/packages/db/tests/query/subquery.test-d.ts @@ -1,6 +1,6 @@ import { describe, expectTypeOf, test } from "vitest" import { createLiveQueryCollection, eq, gt } from "../../src/query/index.js" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { mockSyncCollectionOptions } from "../utils.js" // Sample types for subquery testing diff --git a/packages/db/tests/query/subquery.test.ts b/packages/db/tests/query/subquery.test.ts index fa9fd0831..80b5bdd15 100644 --- a/packages/db/tests/query/subquery.test.ts +++ b/packages/db/tests/query/subquery.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, test } from "vitest" import { createLiveQueryCollection, eq, gt } from "../../src/query/index.js" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { mockSyncCollectionOptions } from "../utils.js" // Sample types for subquery testing diff --git a/packages/db/tests/query/where.test.ts b/packages/db/tests/query/where.test.ts index 39d7db763..a769f71a6 100644 --- a/packages/db/tests/query/where.test.ts +++ b/packages/db/tests/query/where.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, test } from "vitest" import { createLiveQueryCollection } from "../../src/query/index.js" -import { createCollection } from "../../src/collection.js" +import { createCollection } from "../../src/collection/index.js" import { mockSyncCollectionOptions } from "../utils.js" import { add, diff --git a/packages/db/tests/transaction-types.test.ts b/packages/db/tests/transaction-types.test.ts index c759c7c90..d69e80f73 100644 --- a/packages/db/tests/transaction-types.test.ts +++ b/packages/db/tests/transaction-types.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest" -import type { Collection } from "../src/collection" +import type { Collection } from "../src/collection/index.js" import type { MutationFn, PendingMutation, diff --git a/packages/db/tests/transactions.test.ts b/packages/db/tests/transactions.test.ts index bc4639436..538310c31 100644 --- a/packages/db/tests/transactions.test.ts +++ b/packages/db/tests/transactions.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest" import { createTransaction } from "../src/transactions" -import { createCollection } from "../src/collection" +import { createCollection } from "../src/collection/index.js" import { MissingMutationFunctionError, TransactionAlreadyCompletedRollbackError, diff --git a/packages/db/tests/utility-exposure.test.ts b/packages/db/tests/utility-exposure.test.ts index 8317ec7b7..08a67bbf2 100644 --- a/packages/db/tests/utility-exposure.test.ts +++ b/packages/db/tests/utility-exposure.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "vitest" -import { createCollection } from "../src/collection" +import { createCollection } from "../src/collection/index.js" import type { CollectionConfig, SyncConfig, UtilsRecord } from "../src/types" // Mock utility functions for testing diff --git a/packages/electric-db-collection/tests/electric.test.ts b/packages/electric-db-collection/tests/electric.test.ts index 0d6b4796e..617416759 100644 --- a/packages/electric-db-collection/tests/electric.test.ts +++ b/packages/electric-db-collection/tests/electric.test.ts @@ -466,7 +466,7 @@ describe(`Electric Integration`, () => { await transaction.isPersisted.promise - transaction = collection.transactions.get(transaction.id)! + transaction = collection._state.transactions.get(transaction.id)! // Verify the mutation function was called correctly expect(testMutationFn).toHaveBeenCalledTimes(1) @@ -628,7 +628,7 @@ describe(`Electric Integration`, () => { }) // If awaitTxId wasn't called automatically, this wouldn't be true. - expect(testCollection.syncedData.size).toEqual(0) + expect(testCollection._state.syncedData.size).toEqual(0) // Verify that our onInsert handler was called expect(onInsert).toHaveBeenCalled() @@ -641,7 +641,7 @@ describe(`Electric Integration`, () => { id: 1, name: `Direct Persistence User`, }) - expect(testCollection.syncedData.size).toEqual(1) + expect(testCollection._state.syncedData.size).toEqual(1) }) }) @@ -854,7 +854,7 @@ describe(`Electric Integration`, () => { }).not.toThrow() // Should have processed the valid message without issues - expect(testCollection.syncedData.size).toBe(0) // Still pending until up-to-date + expect(testCollection._state.syncedData.size).toBe(0) // Still pending until up-to-date // Send up-to-date to commit expect(() => { diff --git a/packages/query-db-collection/src/query.ts b/packages/query-db-collection/src/query.ts index da1ef286c..09380cd28 100644 --- a/packages/query-db-collection/src/query.ts +++ b/packages/query-db-collection/src/query.ts @@ -399,7 +399,9 @@ export function queryCollectionOptions( return } - const currentSyncedItems = new Map(collection.syncedData) + const currentSyncedItems: Map = new Map( + collection._state.syncedData.entries() + ) const newItemsMap = new Map() newItemsArray.forEach((item) => { const key = getKey(item) diff --git a/packages/query-db-collection/tests/query.test.ts b/packages/query-db-collection/tests/query.test.ts index 4bba0e59a..ce6dc45ad 100644 --- a/packages/query-db-collection/tests/query.test.ts +++ b/packages/query-db-collection/tests/query.test.ts @@ -86,9 +86,9 @@ describe(`QueryCollection`, () => { expect(collection.get(`2`)).toEqual(initialItems[1]) // Verify the synced data - expect(collection.syncedData.size).toBe(initialItems.length) - expect(collection.syncedData.get(`1`)).toEqual(initialItems[0]) - expect(collection.syncedData.get(`2`)).toEqual(initialItems[1]) + expect(collection._state.syncedData.size).toBe(initialItems.length) + expect(collection._state.syncedData.get(`1`)).toEqual(initialItems[0]) + expect(collection._state.syncedData.get(`2`)).toEqual(initialItems[1]) }) it(`should update collection when query data changes`, async () => { diff --git a/packages/rxdb-db-collection/tests/rxdb.test.ts b/packages/rxdb-db-collection/tests/rxdb.test.ts index 08105066d..87c1b9dff 100644 --- a/packages/rxdb-db-collection/tests/rxdb.test.ts +++ b/packages/rxdb-db-collection/tests/rxdb.test.ts @@ -113,9 +113,9 @@ describe(`RxDB Integration`, () => { expect(collection.get(`2`)).toEqual(initialItems[1]) // Verify the synced data - expect(collection.syncedData.size).toBe(initialItems.length) - expect(collection.syncedData.get(`1`)).toEqual(initialItems[0]) - expect(collection.syncedData.get(`2`)).toEqual(initialItems[1]) + expect(collection._state.syncedData.size).toBe(initialItems.length) + expect(collection._state.syncedData.get(`1`)).toEqual(initialItems[0]) + expect(collection._state.syncedData.get(`2`)).toEqual(initialItems[1]) await db.remove() }) @@ -127,7 +127,7 @@ describe(`RxDB Integration`, () => { // All docs should be present after initial sync expect(collection.size).toBe(docsAmount) - expect(collection.syncedData.size).toBe(docsAmount) + expect(collection._state.syncedData.size).toBe(docsAmount) // Spot-check a few positions expect(collection.get(`1`)).toEqual({ id: `1`, name: `Item 1` }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 41c05ca02..93d693e21 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -581,6 +581,28 @@ importers: specifier: ^0.15.1 version: 0.15.1 + packages/benchmarks: + dependencies: + '@tanstack/db': + specifier: workspace:* + version: link:../db + '@tanstack/db-ivm': + specifier: workspace:* + version: link:../db-ivm + tsx: + specifier: ^4.0.0 + version: 4.20.5 + zod: + specifier: ^3.22.0 + version: 3.25.76 + devDependencies: + '@types/node': + specifier: ^22.0.0 + version: 22.18.1 + typescript: + specifier: ^5.0.0 + version: 5.9.2 + packages/db: dependencies: '@standard-schema/spec':