From 43ac7e763d1bcf1974ded45542a973106de52d21 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Mon, 11 Aug 2025 17:09:47 +0100 Subject: [PATCH 01/15] improve schema types --- packages/db/src/local-only.ts | 44 +++++-- packages/db/src/local-storage.ts | 116 +++++++++++------ packages/db/tests/collection.test-d.ts | 118 +++++++++++++++++- .../electric-db-collection/src/electric.ts | 9 +- packages/query-db-collection/package.json | 3 +- packages/query-db-collection/src/query.ts | 101 ++++++++++----- .../query-db-collection/tests/query.test-d.ts | 29 +++++ .../trailbase-db-collection/src/trailbase.ts | 38 +++--- pnpm-lock.yaml | 3 + 9 files changed, 355 insertions(+), 106 deletions(-) diff --git a/packages/db/src/local-only.ts b/packages/db/src/local-only.ts index d6590c610..19725bf21 100644 --- a/packages/db/src/local-only.ts +++ b/packages/db/src/local-only.ts @@ -3,6 +3,7 @@ import type { DeleteMutationFnParams, InsertMutationFnParams, OperationType, + ResolveInsertInput, ResolveType, SyncConfig, UpdateMutationFnParams, @@ -26,7 +27,7 @@ import type { StandardSchemaV1 } from "@standard-schema/spec" * You should provide EITHER an explicit type OR a schema, but not both, as they would conflict. */ export interface LocalOnlyCollectionConfig< - TExplicit = unknown, + TExplicit extends object = Record, TSchema extends StandardSchemaV1 = never, TFallback extends Record = Record, TKey extends string | number = string | number, @@ -51,7 +52,7 @@ export interface LocalOnlyCollectionConfig< */ onInsert?: ( params: InsertMutationFnParams< - ResolveType, + ResolveInsertInput, TKey, LocalOnlyCollectionUtils > @@ -136,33 +137,56 @@ export interface LocalOnlyCollectionUtils extends UtilsRecord {} * ) */ export function localOnlyCollectionOptions< - TExplicit = unknown, + TExplicit extends object = Record, TSchema extends StandardSchemaV1 = never, TFallback extends Record = Record, TKey extends string | number = string | number, >( config: LocalOnlyCollectionConfig -): CollectionConfig, TKey> & { +): CollectionConfig< + ResolveType, + TKey, + TSchema, + ResolveInsertInput +> & { utils: LocalOnlyCollectionUtils } { - type ResolvedType = ResolveType + type TItem = ResolveType + type TInsertInput = ResolveInsertInput const { initialData, onInsert, onUpdate, onDelete, ...restConfig } = config // Create the sync configuration with transaction confirmation capability - const syncResult = createLocalOnlySync(initialData) + const syncResult = createLocalOnlySync( + initialData as Array | undefined + ) /** * Create wrapper handlers that call user handlers first, then confirm transactions * Wraps the user's onInsert handler to also confirm the transaction immediately */ const wrappedOnInsert = async ( - params: InsertMutationFnParams + params: InsertMutationFnParams ) => { // Call user handler first if provided let handlerResult if (onInsert) { - handlerResult = (await onInsert(params)) ?? {} + handlerResult = + (await ( + onInsert as ( + p: InsertMutationFnParams< + TInsertInput, + TKey, + LocalOnlyCollectionUtils + > + ) => Promise + )( + params as unknown as InsertMutationFnParams< + TInsertInput, + TKey, + LocalOnlyCollectionUtils + > + )) ?? {} } // Then synchronously confirm the transaction by looping through mutations @@ -175,7 +199,7 @@ export function localOnlyCollectionOptions< * Wrapper for onUpdate handler that also confirms the transaction immediately */ const wrappedOnUpdate = async ( - params: UpdateMutationFnParams + params: UpdateMutationFnParams ) => { // Call user handler first if provided let handlerResult @@ -193,7 +217,7 @@ export function localOnlyCollectionOptions< * Wrapper for onDelete handler that also confirms the transaction immediately */ const wrappedOnDelete = async ( - params: DeleteMutationFnParams + params: DeleteMutationFnParams ) => { // Call user handler first if provided let handlerResult diff --git a/packages/db/src/local-storage.ts b/packages/db/src/local-storage.ts index 43dfc5afa..c1e63818d 100644 --- a/packages/db/src/local-storage.ts +++ b/packages/db/src/local-storage.ts @@ -9,7 +9,9 @@ import { import type { CollectionConfig, DeleteMutationFnParams, + InsertMutationFn, InsertMutationFnParams, + ResolveInsertInput, ResolveType, SyncConfig, UpdateMutationFnParams, @@ -62,6 +64,7 @@ export interface LocalStorageCollectionConfig< TExplicit = unknown, TSchema extends StandardSchemaV1 = never, TFallback extends object = Record, + TKey extends string | number = string | number, > { /** * The key to use for storing the collection data in localStorage/sessionStorage @@ -85,8 +88,18 @@ export interface LocalStorageCollectionConfig< */ id?: string schema?: TSchema - getKey: CollectionConfig>[`getKey`] - sync?: CollectionConfig>[`sync`] + getKey: CollectionConfig< + ResolveType, + TKey, + TSchema, + ResolveInsertInput + >[`getKey`] + sync?: CollectionConfig< + ResolveType, + TKey, + TSchema, + ResolveInsertInput + >[`sync`] /** * Optional asynchronous handler function called before an insert operation @@ -94,7 +107,10 @@ export interface LocalStorageCollectionConfig< * @returns Promise resolving to any value */ onInsert?: ( - params: InsertMutationFnParams> + params: InsertMutationFnParams< + ResolveInsertInput, + TKey + > ) => Promise /** @@ -103,7 +119,10 @@ export interface LocalStorageCollectionConfig< * @returns Promise resolving to any value */ onUpdate?: ( - params: UpdateMutationFnParams> + params: UpdateMutationFnParams< + ResolveType, + TKey + > ) => Promise /** @@ -112,7 +131,10 @@ export interface LocalStorageCollectionConfig< * @returns Promise resolving to any value */ onDelete?: ( - params: DeleteMutationFnParams> + params: DeleteMutationFnParams< + ResolveType, + TKey + > ) => Promise } @@ -206,13 +228,23 @@ export function localStorageCollectionOptions< TExplicit = unknown, TSchema extends StandardSchemaV1 = never, TFallback extends object = Record, + TKey extends string | number = string | number, >( - config: LocalStorageCollectionConfig -): Omit>, `id`> & { + config: LocalStorageCollectionConfig +): Omit< + CollectionConfig< + ResolveType, + TKey, + TSchema, + ResolveInsertInput + >, + `id` +> & { id: string utils: LocalStorageCollectionUtils } { - type ResolvedType = ResolveType + type TItem = ResolveType + type TInsertInput = ResolveInsertInput // Validate required parameters if (!config.storageKey) { @@ -237,14 +269,14 @@ export function localStorageCollectionOptions< } // Track the last known state to detect changes - const lastKnownData = new Map>() + const lastKnownData = new Map>() // Create the sync configuration - const sync = createLocalStorageSync( + const sync = createLocalStorageSync( config.storageKey, storage, storageEventApi, - config.getKey, + config.getKey as (item: TItem) => TKey, lastKnownData ) @@ -263,11 +295,11 @@ export function localStorageCollectionOptions< * @param dataMap - Map of items with version tracking to save to storage */ const saveToStorage = ( - dataMap: Map> + dataMap: Map> ): void => { try { // Convert Map to object format for storage - const objectData: Record> = {} + const objectData: Record> = {} dataMap.forEach((storedItem, key) => { objectData[String(key)] = storedItem }) @@ -303,7 +335,7 @@ export function localStorageCollectionOptions< * Wraps the user's onInsert handler to also save changes to localStorage */ const wrappedOnInsert = async ( - params: InsertMutationFnParams + params: InsertMutationFnParams ) => { // Validate that all values in the transaction can be JSON serialized params.transaction.mutations.forEach((mutation) => { @@ -313,20 +345,20 @@ export function localStorageCollectionOptions< // Call the user handler BEFORE persisting changes (if provided) let handlerResult: any = {} if (config.onInsert) { - handlerResult = (await config.onInsert(params)) ?? {} + handlerResult = + (await (config.onInsert as InsertMutationFn)( + params as unknown as InsertMutationFnParams + )) ?? {} } // Always persist to storage // Load current data from storage - const currentData = loadFromStorage( - config.storageKey, - storage - ) + const currentData = loadFromStorage(config.storageKey, storage) // Add new items with version keys params.transaction.mutations.forEach((mutation) => { - const key = config.getKey(mutation.modified) - const storedItem: StoredItem = { + const key = (config.getKey as (item: TItem) => TKey)(mutation.modified) + const storedItem: StoredItem = { versionKey: generateUuid(), data: mutation.modified, } @@ -343,7 +375,7 @@ export function localStorageCollectionOptions< } const wrappedOnUpdate = async ( - params: UpdateMutationFnParams + params: UpdateMutationFnParams ) => { // Validate that all values in the transaction can be JSON serialized params.transaction.mutations.forEach((mutation) => { @@ -358,15 +390,12 @@ export function localStorageCollectionOptions< // Always persist to storage // Load current data from storage - const currentData = loadFromStorage( - config.storageKey, - storage - ) + const currentData = loadFromStorage(config.storageKey, storage) // Update items with new version keys params.transaction.mutations.forEach((mutation) => { - const key = config.getKey(mutation.modified) - const storedItem: StoredItem = { + const key = (config.getKey as (item: TItem) => TKey)(mutation.modified) + const storedItem: StoredItem = { versionKey: generateUuid(), data: mutation.modified, } @@ -383,7 +412,7 @@ export function localStorageCollectionOptions< } const wrappedOnDelete = async ( - params: DeleteMutationFnParams + params: DeleteMutationFnParams ) => { // Call the user handler BEFORE persisting changes (if provided) let handlerResult: any = {} @@ -393,15 +422,14 @@ export function localStorageCollectionOptions< // Always persist to storage // Load current data from storage - const currentData = loadFromStorage( - config.storageKey, - storage - ) + const currentData = loadFromStorage(config.storageKey, storage) // Remove items params.transaction.mutations.forEach((mutation) => { // For delete operations, mutation.original contains the full object - const key = config.getKey(mutation.original as ResolvedType) + const key = (config.getKey as (item: TItem) => TKey)( + mutation.original as TItem + ) currentData.delete(key) }) @@ -433,7 +461,10 @@ export function localStorageCollectionOptions< ...restConfig, id: collectionId, sync, - onInsert: wrappedOnInsert, + onInsert: wrappedOnInsert as unknown as InsertMutationFn< + TInsertInput, + TKey + >, onUpdate: wrappedOnUpdate, onDelete: wrappedOnDelete, utils: { @@ -506,14 +537,17 @@ function loadFromStorage( * @param lastKnownData - Map tracking the last known state for change detection * @returns Sync configuration with manual trigger capability */ -function createLocalStorageSync( +function createLocalStorageSync< + T extends object, + TKey extends string | number = string | number, +>( storageKey: string, storage: StorageApi, storageEventApi: StorageEventApi, - _getKey: (item: T) => string | number, + _getKey: (item: T) => TKey, lastKnownData: Map> -): SyncConfig & { manualTrigger?: () => void } { - let syncParams: Parameters[`sync`]>[0] | null = null +): SyncConfig & { manualTrigger?: () => void } { + let syncParams: Parameters[`sync`]>[0] | null = null /** * Compare two Maps to find differences using version keys @@ -588,8 +622,8 @@ function createLocalStorageSync( } } - const syncConfig: SyncConfig & { manualTrigger?: () => void } = { - sync: (params: Parameters[`sync`]>[0]) => { + const syncConfig: SyncConfig & { manualTrigger?: () => void } = { + sync: (params: Parameters[`sync`]>[0]) => { const { begin, write, commit, markReady } = params // Store sync params for later use diff --git a/packages/db/tests/collection.test-d.ts b/packages/db/tests/collection.test-d.ts index 0faa68346..01cca41ab 100644 --- a/packages/db/tests/collection.test-d.ts +++ b/packages/db/tests/collection.test-d.ts @@ -2,7 +2,11 @@ import { assertType, describe, expectTypeOf, it } from "vitest" import { z } from "zod" import { createCollection } from "../src/collection" import type { CollectionImpl } from "../src/collection" -import type { OperationConfig, ResolveType } from "../src/types" +import type { + OperationConfig, + ResolveInsertInput, + ResolveType, +} from "../src/types" import type { StandardSchemaV1 } from "@standard-schema/spec" describe(`Collection.update type tests`, () => { @@ -195,3 +199,115 @@ describe(`Collection type resolution tests`, () => { expectTypeOf>().toEqualTypeOf() }) }) + +describe(`Schema Input/Output Type Distinction`, () => { + // Define schema with different input/output types + const userSchemaWithDefaults = z.object({ + id: z.string(), + name: z.string(), + email: z.string().email(), + created_at: z.date().default(() => new Date()), + updated_at: z.date().default(() => new Date()), + }) + + // Define schema with transformations + const userSchemaTransform = z.object({ + id: z.string(), + name: z.string(), + email: z.string().email(), + created_at: z.string().transform((val) => new Date(val)), + updated_at: z.string().transform((val) => new Date(val)), + }) + + it(`should handle schema with default values correctly`, () => { + const collection = createCollection({ + getKey: (item) => item.id, + sync: { sync: () => {} }, + schema: userSchemaWithDefaults, + }) + + type ExpectedOutputType = ResolveType< + unknown, + typeof userSchemaWithDefaults, + Record + > + type ExpectedInputType = ResolveInsertInput< + unknown, + typeof userSchemaWithDefaults, + Record + > + type InsertArg = Parameters[0] + + // Input type should not include defaulted fields + expectTypeOf().toEqualTypeOf<{ + id: string + name: string + email: string + created_at?: Date + updated_at?: Date + }>() + + // Output type should include all fields + expectTypeOf().toEqualTypeOf<{ + id: string + name: string + email: string + created_at: Date + updated_at: Date + }>() + + // Insert should accept ExpectedInputType or array thereof + expectTypeOf().toEqualTypeOf< + ExpectedInputType | Array + >() + + // Collection items should be ExpectedOutputType + expectTypeOf(collection.toArray).toEqualTypeOf>() + }) + + it(`should handle schema with transformations correctly`, () => { + const collection = createCollection({ + getKey: (item) => item.id, + sync: { sync: () => {} }, + schema: userSchemaTransform, + }) + + type ExpectedInputType = ResolveInsertInput< + unknown, + typeof userSchemaTransform, + Record + > + type ExpectedOutputType = ResolveType< + unknown, + typeof userSchemaTransform, + Record + > + type InsertArg = Parameters[0] + + // Input type should be the raw input (before transformation) + expectTypeOf().toEqualTypeOf<{ + id: string + name: string + email: string + created_at: string + updated_at: string + }>() + + // Output type should be the transformed output + expectTypeOf().toEqualTypeOf<{ + id: string + name: string + email: string + created_at: Date + updated_at: Date + }>() + + // Insert should accept ExpectedInputType or array thereof + expectTypeOf().toEqualTypeOf< + ExpectedInputType | Array + >() + + // Collection items should be ExpectedOutputType + expectTypeOf(collection.toArray).toEqualTypeOf>() + }) +}) diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index d80f46bcd..1ddfbad10 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -286,7 +286,14 @@ export function electricCollectionOptions< TExplicit extends Row = Row, TSchema extends StandardSchemaV1 = never, TFallback extends Row = Row, ->(config: ElectricCollectionConfig) { +>( + config: ElectricCollectionConfig +): CollectionConfig< + ResolveType, + string | number, + TSchema, + ResolveType +> & { utils: ElectricCollectionUtils } { const seenTxids = new Store>(new Set([])) const sync = createElectricSync>( config.shapeOptions, diff --git a/packages/query-db-collection/package.json b/packages/query-db-collection/package.json index e886f0d29..38bf316a8 100644 --- a/packages/query-db-collection/package.json +++ b/packages/query-db-collection/package.json @@ -3,7 +3,8 @@ "description": "TanStack Query collection for TanStack DB", "version": "0.2.1", "dependencies": { - "@tanstack/db": "workspace:*" + "@tanstack/db": "workspace:*", + "@standard-schema/spec": "^1.0.0" }, "devDependencies": { "@tanstack/query-core": "^5.0.5", diff --git a/packages/query-db-collection/src/query.ts b/packages/query-db-collection/src/query.ts index 1a3074fc9..3c1d165d6 100644 --- a/packages/query-db-collection/src/query.ts +++ b/packages/query-db-collection/src/query.ts @@ -19,11 +19,14 @@ import type { DeleteMutationFnParams, InsertMutationFn, InsertMutationFnParams, + ResolveInsertInput, + ResolveType, SyncConfig, UpdateMutationFn, UpdateMutationFnParams, UtilsRecord, } from "@tanstack/db" +import type { StandardSchemaV1 } from "@standard-schema/spec" // Re-export for external use export type { SyncOperation } from "./manual-sync" @@ -35,14 +38,19 @@ export type { SyncOperation } from "./manual-sync" * @template TQueryKey - The type of the query key */ export interface QueryCollectionConfig< - TItem extends object, + TExplicit extends object = Record, + TSchema extends StandardSchemaV1 = never, + TFallback extends object = Record, TError = unknown, TQueryKey extends QueryKey = QueryKey, + TKey extends string | number = string | number, > { /** The query key used by TanStack Query to identify this query */ queryKey: TQueryKey /** Function that fetches data from the server. Must return the complete collection state */ - queryFn: (context: QueryFunctionContext) => Promise> + queryFn: ( + context: QueryFunctionContext + ) => Promise>> /** The TanStack Query client instance */ queryClient: QueryClient @@ -50,31 +58,31 @@ export interface QueryCollectionConfig< /** Whether the query should automatically run (default: true) */ enabled?: boolean refetchInterval?: QueryObserverOptions< - Array, + Array>, TError, - Array, - Array, + Array>, + Array>, TQueryKey >[`refetchInterval`] retry?: QueryObserverOptions< - Array, + Array>, TError, - Array, - Array, + Array>, + Array>, TQueryKey >[`retry`] retryDelay?: QueryObserverOptions< - Array, + Array>, TError, - Array, - Array, + Array>, + Array>, TQueryKey >[`retryDelay`] staleTime?: QueryObserverOptions< - Array, + Array>, TError, - Array, - Array, + Array>, + Array>, TQueryKey >[`staleTime`] @@ -82,11 +90,16 @@ export interface QueryCollectionConfig< /** Unique identifier for the collection */ id?: string /** Function to extract the unique key from an item */ - getKey: CollectionConfig[`getKey`] + getKey: CollectionConfig< + ResolveType, + TKey + >[`getKey`] /** Schema for validating items */ - schema?: CollectionConfig[`schema`] - sync?: CollectionConfig[`sync`] - startSync?: CollectionConfig[`startSync`] + schema?: TSchema + sync?: CollectionConfig>[`sync`] + startSync?: CollectionConfig< + ResolveType + >[`startSync`] // Direct persistence handlers /** @@ -129,7 +142,10 @@ export interface QueryCollectionConfig< * } * } */ - onInsert?: InsertMutationFn + onInsert?: InsertMutationFn< + ResolveInsertInput, + TKey + > /** * Optional asynchronous handler function called before an update operation @@ -182,7 +198,7 @@ export interface QueryCollectionConfig< * return { refetch: false } // Skip automatic refetch since we handled it manually * } */ - onUpdate?: UpdateMutationFn + onUpdate?: UpdateMutationFn, TKey> /** * Optional asynchronous handler function called before a delete operation @@ -228,7 +244,7 @@ export interface QueryCollectionConfig< * return { refetch: false } // Skip automatic refetch since we handled it manually * } */ - onDelete?: DeleteMutationFn + onDelete?: DeleteMutationFn, TKey> // TODO type returning { refetch: boolean } /** @@ -324,16 +340,35 @@ export interface QueryCollectionUtils< * ) */ export function queryCollectionOptions< - TItem extends object, + TExplicit extends object = Record, + TSchema extends StandardSchemaV1 = never, + TFallback extends object = Record, TError = unknown, TQueryKey extends QueryKey = QueryKey, TKey extends string | number = string | number, - TInsertInput extends object = TItem, >( - config: QueryCollectionConfig -): CollectionConfig & { - utils: QueryCollectionUtils + config: QueryCollectionConfig< + TExplicit, + TSchema, + TFallback, + TError, + TQueryKey, + TKey + > +): CollectionConfig< + ResolveType, + TKey, + TSchema, + ResolveInsertInput +> & { + utils: QueryCollectionUtils< + ResolveType, + TKey, + ResolveInsertInput + > } { + type TItem = ResolveType + type TInsertInput = ResolveInsertInput const { queryKey, queryFn, @@ -372,7 +407,7 @@ export function queryCollectionOptions< throw new GetKeyRequiredError() } - const internalSync: SyncConfig[`sync`] = (params) => { + const internalSync: SyncConfig[`sync`] = (params) => { const { begin, write, commit, markReady, collection } = params const observerOptions: QueryObserverOptions< @@ -418,7 +453,7 @@ export function queryCollectionOptions< } const currentSyncedItems = new Map(collection.syncedData) - const newItemsMap = new Map() + const newItemsMap = new Map() newItemsArray.forEach((item) => { const key = getKey(item) newItemsMap.set(key, item) @@ -512,12 +547,12 @@ export function queryCollectionOptions< queryKey: Array getKey: (item: TItem) => TKey begin: () => void - write: (message: Omit, `key`>) => void + write: (message: Omit, `key`>) => void commit: () => void } | null = null // Enhanced internalSync that captures write functions for manual use - const enhancedInternalSync: SyncConfig[`sync`] = (params) => { + const enhancedInternalSync: SyncConfig[`sync`] = (params) => { const { begin, write, commit, collection } = params // Store references for manual write operations @@ -542,7 +577,7 @@ export function queryCollectionOptions< // Create wrapper handlers for direct persistence operations that handle refetching const wrappedOnInsert = onInsert - ? async (params: InsertMutationFnParams) => { + ? async (params: InsertMutationFnParams) => { const handlerResult = (await onInsert(params)) ?? {} const shouldRefetch = (handlerResult as { refetch?: boolean }).refetch !== false @@ -556,7 +591,7 @@ export function queryCollectionOptions< : undefined const wrappedOnUpdate = onUpdate - ? async (params: UpdateMutationFnParams) => { + ? async (params: UpdateMutationFnParams) => { const handlerResult = (await onUpdate(params)) ?? {} const shouldRefetch = (handlerResult as { refetch?: boolean }).refetch !== false @@ -570,7 +605,7 @@ export function queryCollectionOptions< : undefined const wrappedOnDelete = onDelete - ? async (params: DeleteMutationFnParams) => { + ? async (params: DeleteMutationFnParams) => { const handlerResult = (await onDelete(params)) ?? {} const shouldRefetch = (handlerResult as { refetch?: boolean }).refetch !== false diff --git a/packages/query-db-collection/tests/query.test-d.ts b/packages/query-db-collection/tests/query.test-d.ts index eb5d09db3..18819bb96 100644 --- a/packages/query-db-collection/tests/query.test-d.ts +++ b/packages/query-db-collection/tests/query.test-d.ts @@ -186,4 +186,33 @@ describe(`Query collection type resolution tests`, () => { // Test that the getKey function has the correct parameter type expectTypeOf(queryOptions.getKey).parameters.toEqualTypeOf<[UserType]>() }) + it(`types getKey param as schema OUTPUT and onInsert as schema INPUT`, () => { + const schema = z.object({ + id: z.string(), + created_at: z.string().transform((s) => new Date(s)), + updated_at: z.string().transform((s) => new Date(s)), + }) + + type Output = z.output + type Input = z.input + + const queryClient = new QueryClient() + + const options = queryCollectionOptions({ + queryClient, + queryKey: [`users`], + queryFn: () => Promise.resolve([] as Array), + schema, + getKey: (item) => item.id, + onInsert: async () => {}, + onUpdate: async () => {}, + }) + + expectTypeOf(options.getKey).parameters.toEqualTypeOf<[Output]>() + // utils.writeInsert should accept schema INPUT + type WriteInsertParam0 = Parameters< + (typeof options.utils)[`writeInsert`] + >[0] + expectTypeOf().toEqualTypeOf>() + }) }) diff --git a/packages/trailbase-db-collection/src/trailbase.ts b/packages/trailbase-db-collection/src/trailbase.ts index 88519ec1c..a9c9f66e0 100644 --- a/packages/trailbase-db-collection/src/trailbase.ts +++ b/packages/trailbase-db-collection/src/trailbase.ts @@ -84,11 +84,11 @@ function convertPartial< * Configuration interface for Trailbase Collection */ export interface TrailBaseCollectionConfig< - TItem extends ShapeOf, - TRecord extends ShapeOf = TItem, + TExplicit extends ShapeOf, + TRecord extends ShapeOf = TExplicit, TKey extends string | number = string | number, > extends Omit< - CollectionConfig, + CollectionConfig, `sync` | `onInsert` | `onUpdate` | `onDelete` > { /** @@ -96,8 +96,8 @@ export interface TrailBaseCollectionConfig< */ recordApi: RecordApi - parse: Conversions - serialize: Conversions + parse: Conversions + serialize: Conversions } export type AwaitTxIdFn = (txId: string, timeout?: number) => Promise @@ -107,20 +107,20 @@ export interface TrailBaseCollectionUtils extends UtilsRecord { } export function trailBaseCollectionOptions< - TItem extends ShapeOf, - TRecord extends ShapeOf = TItem, + TExplicit extends ShapeOf, + TRecord extends ShapeOf = TExplicit, TKey extends string | number = string | number, >( - config: TrailBaseCollectionConfig -): CollectionConfig & { utils: TrailBaseCollectionUtils } { + config: TrailBaseCollectionConfig +): CollectionConfig & { utils: TrailBaseCollectionUtils } { const getKey = config.getKey const parse = (record: TRecord) => - convert(config.parse, record) - const serialUpd = (item: Partial) => - convertPartial(config.serialize, item) - const serialIns = (item: TItem) => - convert(config.serialize, item) + convert(config.parse, record) + const serialUpd = (item: Partial) => + convertPartial(config.serialize, item) + const serialIns = (item: TExplicit) => + convert(config.serialize, item) const seenIds = new Store(new Map()) @@ -159,7 +159,7 @@ export function trailBaseCollectionOptions< } } - type SyncParams = Parameters[`sync`]>[0] + type SyncParams = Parameters[`sync`]>[0] const sync = { sync: (params: SyncParams) => { const { begin, write, commit, markReady } = params @@ -216,7 +216,7 @@ export function trailBaseCollectionOptions< } begin() - let value: TItem | undefined + let value: TExplicit | undefined if (`Insert` in event) { value = parse(event.Insert as TRecord) write({ type: `insert`, value }) @@ -294,7 +294,7 @@ export function trailBaseCollectionOptions< sync, getKey, onInsert: async ( - params: InsertMutationFnParams + params: InsertMutationFnParams ): Promise> => { const ids = await config.recordApi.createBulk( params.transaction.mutations.map((tx) => { @@ -313,7 +313,7 @@ export function trailBaseCollectionOptions< return ids }, - onUpdate: async (params: UpdateMutationFnParams) => { + onUpdate: async (params: UpdateMutationFnParams) => { const ids: Array = await Promise.all( params.transaction.mutations.map(async (tx) => { const { type, changes, key } = tx @@ -332,7 +332,7 @@ export function trailBaseCollectionOptions< // DB by the subscription. await awaitIds(ids) }, - onDelete: async (params: DeleteMutationFnParams) => { + onDelete: async (params: DeleteMutationFnParams) => { const ids: Array = await Promise.all( params.transaction.mutations.map(async (tx) => { const { type, key } = tx diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b0ea21e23..d6af0bd6c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -539,6 +539,9 @@ importers: packages/query-db-collection: dependencies: + '@standard-schema/spec': + specifier: ^1.0.0 + version: 1.0.0 '@tanstack/db': specifier: workspace:* version: link:../db From 1cbc57dbd57917755458823c6d856e216eb775da Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Tue, 12 Aug 2025 12:17:44 +0100 Subject: [PATCH 02/15] change update to use the schema --- packages/db/src/collection.ts | 12 +- packages/db/tests/collection-schema.test.ts | 263 ++++++++++++++++++++ 2 files changed, 272 insertions(+), 3 deletions(-) create mode 100644 packages/db/tests/collection-schema.test.ts diff --git a/packages/db/src/collection.ts b/packages/db/src/collection.ts index eea7905cd..f89b6432c 100644 --- a/packages/db/src/collection.ts +++ b/packages/db/src/collection.ts @@ -72,6 +72,7 @@ interface PendingSyncedTransaction> { * @template T - The type of items in the collection * @template TKey - The type of the key for the collection * @template TUtils - The utilities record type + * @template TSchema - The schema type for validation and type inference * @template TInsertInput - The type for insert operations (can be different from T for schemas with defaults) */ export interface Collection< @@ -1613,9 +1614,14 @@ export class CollectionImpl< throw new SchemaValidationError(type, typedIssues) } - // Return the original update data, not the merged data - // We only used the merged data for validation - return data as T + // Extract only the modified keys from the validated result + const validatedMergedData = result.value as T + const modifiedKeys = Object.keys(data) + const extractedChanges = Object.fromEntries( + modifiedKeys.map((k) => [k, validatedMergedData[k as keyof T]]) + ) as T + + return extractedChanges } } diff --git a/packages/db/tests/collection-schema.test.ts b/packages/db/tests/collection-schema.test.ts new file mode 100644 index 000000000..2e53c7772 --- /dev/null +++ b/packages/db/tests/collection-schema.test.ts @@ -0,0 +1,263 @@ +import { describe, expect, it } from "vitest" +import { z } from "zod" +import { createCollection } from "../src/collection" + +describe(`Collection Schema Validation`, () => { + it(`should apply transformations for both insert and update operations`, () => { + // Create a schema with transformations + const userSchema = z.object({ + id: z.string(), + name: z.string(), + email: z.string().email(), + created_at: z.string().transform((val) => new Date(val)), + updated_at: z.string().transform((val) => new Date(val)), + }) + + const collection = createCollection({ + getKey: (item) => item.id, + schema: userSchema, + sync: { sync: () => {} }, + }) + + // Test insert validation + const insertData = { + id: `1`, + name: `John Doe`, + email: `john@example.com`, + created_at: `2023-01-01T00:00:00.000Z`, + updated_at: `2023-01-01T00:00:00.000Z`, + } + + const validatedInsert = collection.validateData(insertData, `insert`) + + // Verify that the inserted data has been transformed + expect(validatedInsert.created_at).toBeInstanceOf(Date) + expect(validatedInsert.updated_at).toBeInstanceOf(Date) + expect(validatedInsert.name).toBe(`John Doe`) + expect(validatedInsert.email).toBe(`john@example.com`) + + // Test update validation - use a schema that accepts both string and Date for existing data + const updateSchema = z.object({ + id: z.string(), + name: z.string(), + email: z.string().email(), + created_at: z + .union([z.string(), z.date()]) + .transform((val) => (typeof val === `string` ? new Date(val) : val)), + updated_at: z + .union([z.string(), z.date()]) + .transform((val) => (typeof val === `string` ? new Date(val) : val)), + }) + + const updateCollection = createCollection({ + getKey: (item) => item.id, + schema: updateSchema, + sync: { sync: () => {} }, + }) + + // Add the validated insert data to the update collection + ;(updateCollection as any).syncedData.set(`1`, validatedInsert) + + const updateData = { + name: `Jane Doe`, + email: `jane@example.com`, + updated_at: `2023-01-02T00:00:00.000Z`, + } + + const validatedUpdate = updateCollection.validateData( + updateData, + `update`, + `1` + ) + + // Verify that the updated data has been transformed + expect(validatedUpdate.updated_at).toBeInstanceOf(Date) + expect(validatedUpdate.name).toBe(`Jane Doe`) + expect(validatedUpdate.email).toBe(`jane@example.com`) + }) + + it(`should extract only modified keys from validated update result`, () => { + // Create a schema with transformations that can handle both string and Date inputs + const userSchema = z.object({ + id: z.string(), + name: z.string(), + email: z.string().email(), + created_at: z + .union([z.string(), z.date()]) + .transform((val) => (typeof val === `string` ? new Date(val) : val)), + updated_at: z + .union([z.string(), z.date()]) + .transform((val) => (typeof val === `string` ? new Date(val) : val)), + }) + + const collection = createCollection({ + getKey: (item) => item.id, + schema: userSchema, + sync: { sync: () => {} }, + }) + + // First, we need to add an item to the collection for update validation + const insertData = { + id: `1`, + name: `John Doe`, + email: `john@example.com`, + created_at: `2023-01-01T00:00:00.000Z`, + updated_at: `2023-01-01T00:00:00.000Z`, + } + + const validatedInsert = collection.validateData(insertData, `insert`) + + // Manually add the item to the collection's synced data for testing + ;(collection as any).syncedData.set(`1`, validatedInsert) + + // Test update validation with only modified fields + const updateData = { + name: `Jane Doe`, + updated_at: `2023-01-02T00:00:00.000Z`, + } + + const validatedUpdate = collection.validateData(updateData, `update`, `1`) + + // Verify that only the modified fields are returned + expect(validatedUpdate).toHaveProperty(`name`) + expect(validatedUpdate).toHaveProperty(`updated_at`) + expect(validatedUpdate).not.toHaveProperty(`id`) + expect(validatedUpdate).not.toHaveProperty(`email`) + expect(validatedUpdate).not.toHaveProperty(`created_at`) + + // Verify the changes contain the transformed values + expect(validatedUpdate.name).toBe(`Jane Doe`) + expect(validatedUpdate.updated_at).toBeInstanceOf(Date) + }) + + it(`should handle schemas with default values correctly`, () => { + // Create a schema with default values that can handle both existing Date objects and new string inputs + const userSchema = z.object({ + id: z.string(), + name: z.string(), + email: z.string().email(), + created_at: z + .union([z.date(), z.string()]) + .transform((val) => (typeof val === `string` ? new Date(val) : val)) + .default(() => new Date()), + updated_at: z + .union([z.date(), z.string()]) + .transform((val) => (typeof val === `string` ? new Date(val) : val)) + .default(() => new Date()), + }) + + const collection = createCollection({ + getKey: (item) => item.id, + schema: userSchema, + sync: { sync: () => {} }, + }) + + // Test insert validation without providing defaulted fields + const insertData = { + id: `1`, + name: `John Doe`, + email: `john@example.com`, + } + + const validatedInsert = collection.validateData(insertData, `insert`) + + // Verify that default values are applied + expect(validatedInsert.created_at).toBeInstanceOf(Date) + expect(validatedInsert.updated_at).toBeInstanceOf(Date) + expect(validatedInsert.name).toBe(`John Doe`) + expect(validatedInsert.email).toBe(`john@example.com`) + + // Manually add the item to the collection's synced data for testing + ;(collection as any).syncedData.set(`1`, validatedInsert) + + // Test update validation without providing defaulted fields + const updateData = { + name: `Jane Doe`, + } + + const validatedUpdate = collection.validateData(updateData, `update`, `1`) + + // Verify that only the modified field is returned + expect(validatedUpdate).toHaveProperty(`name`) + expect(validatedUpdate).not.toHaveProperty(`updated_at`) + expect(validatedUpdate.name).toBe(`Jane Doe`) + + // Test update validation with explicit updated_at field + const updateDataWithTimestamp = { + name: `Jane Smith`, + updated_at: `2023-01-02T00:00:00.000Z`, + } + + const validatedUpdateWithTimestamp = collection.validateData( + updateDataWithTimestamp, + `update`, + `1` + ) + + // Verify that both modified fields are returned with transformations applied + expect(validatedUpdateWithTimestamp).toHaveProperty(`name`) + expect(validatedUpdateWithTimestamp).toHaveProperty(`updated_at`) + expect(validatedUpdateWithTimestamp.name).toBe(`Jane Smith`) + expect(validatedUpdateWithTimestamp.updated_at).toBeInstanceOf(Date) + }) + + it(`should validate schema input types for both insert and update`, () => { + // Create a schema with different input and output types that can handle both string and Date inputs + const userSchema = z.object({ + id: z.string(), + name: z.string(), + email: z.string().email(), + age: z.number().int().positive(), + created_at: z + .union([z.string(), z.date()]) + .transform((val) => (typeof val === `string` ? new Date(val) : val)), + updated_at: z + .union([z.string(), z.date()]) + .transform((val) => (typeof val === `string` ? new Date(val) : val)), + }) + + const collection = createCollection({ + getKey: (item) => item.id, + schema: userSchema, + sync: { sync: () => {} }, + }) + + // Test that insert validation accepts input type (with string dates) + const insertData = { + id: `1`, + name: `John Doe`, + email: `john@example.com`, + age: 30, + created_at: `2023-01-01T00:00:00.000Z`, + updated_at: `2023-01-01T00:00:00.000Z`, + } + + const validatedInsert = collection.validateData(insertData, `insert`) + + // Verify that the output type has Date objects + expect(validatedInsert.created_at).toBeInstanceOf(Date) + expect(validatedInsert.updated_at).toBeInstanceOf(Date) + expect(typeof validatedInsert.age).toBe(`number`) + + // Add to collection for update testing + ;(collection as any).syncedData.set(`1`, validatedInsert) + + // Test that update validation accepts input type for new fields + const updateData = { + name: `Jane Doe`, + age: 31, + updated_at: `2023-01-02T00:00:00.000Z`, + } + + const validatedUpdate = collection.validateData(updateData, `update`, `1`) + + // Verify that the output type has Date objects and only modified fields + expect(validatedUpdate).toHaveProperty(`name`) + expect(validatedUpdate).toHaveProperty(`age`) + expect(validatedUpdate).toHaveProperty(`updated_at`) + expect(validatedUpdate.updated_at).toBeInstanceOf(Date) + expect(typeof validatedUpdate.age).toBe(`number`) + expect(validatedUpdate.name).toBe(`Jane Doe`) + expect(validatedUpdate.age).toBe(31) + }) +}) From 380f181a883fe996871d05c9b78fd08d7e630bd8 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Tue, 12 Aug 2025 12:21:00 +0100 Subject: [PATCH 03/15] move runtime schema tests --- packages/db/tests/collection-schema.test.ts | 428 +++++++++++++++++++- packages/db/tests/collection.test.ts | 314 +------------- 2 files changed, 429 insertions(+), 313 deletions(-) diff --git a/packages/db/tests/collection-schema.test.ts b/packages/db/tests/collection-schema.test.ts index 2e53c7772..5301b7751 100644 --- a/packages/db/tests/collection-schema.test.ts +++ b/packages/db/tests/collection-schema.test.ts @@ -1,6 +1,15 @@ -import { describe, expect, it } from "vitest" +import { type } from "arktype" +import mitt from "mitt" +import { describe, expect, expectTypeOf, it, vi } from "vitest" import { z } from "zod" import { createCollection } from "../src/collection" +import { SchemaValidationError } from "../src/errors" +import { createTransaction } from "../src/transactions" +import type { + OperationType, + PendingMutation, + ResolveTransactionChanges, +} from "../src/types" describe(`Collection Schema Validation`, () => { it(`should apply transformations for both insert and update operations`, () => { @@ -261,3 +270,420 @@ describe(`Collection Schema Validation`, () => { expect(validatedUpdate.age).toBe(31) }) }) + +describe(`Collection with schema validation`, () => { + it(`should validate data against arktype schema on insert`, () => { + // Create a Zod schema for a user + const userSchema = type({ + name: `string > 0`, + age: `number.integer > 0`, + "email?": `string.email`, + }) + + // Create a collection with the schema + const collection = createCollection({ + id: `test`, + getKey: (item) => item.name, + startSync: true, + sync: { + sync: ({ begin, commit }) => { + begin() + commit() + }, + }, + schema: userSchema, + }) + const mutationFn = async () => {} + + // Valid data should work + const validUser = { + name: `Alice`, + age: 30, + email: `alice@example.com`, + } + + const tx1 = createTransaction({ mutationFn }) + tx1.mutate(() => collection.insert(validUser)) + + // Invalid data should throw SchemaValidationError + const invalidUser = { + name: ``, // Empty name (fails min length) + age: -5, // Negative age (fails positive) + email: `not-an-email`, // Invalid email + } + + try { + const tx2 = createTransaction({ mutationFn }) + tx2.mutate(() => collection.insert(invalidUser)) + // Should not reach here + expect(true).toBe(false) + } catch (error) { + expect(error).toBeInstanceOf(SchemaValidationError) + if (error instanceof SchemaValidationError) { + expect(error.type).toBe(`insert`) + expect(error.issues.length).toBeGreaterThan(0) + // Check that we have validation errors for each invalid field + expect(error.issues.some((issue) => issue.path?.includes(`name`))).toBe( + true + ) + expect(error.issues.some((issue) => issue.path?.includes(`age`))).toBe( + true + ) + expect( + error.issues.some((issue) => issue.path?.includes(`email`)) + ).toBe(true) + } + } + + // Partial updates should work with valid data + const tx3 = createTransaction({ mutationFn }) + tx3.mutate(() => + collection.update(`Alice`, (draft) => { + draft.age = 31 + }) + ) + + // Partial updates should fail with invalid data + try { + const tx4 = createTransaction({ mutationFn }) + tx4.mutate(() => + collection.update(`Alice`, (draft) => { + draft.age = -1 + }) + ) + // Should not reach here + expect(true).toBe(false) + } catch (error) { + expect(error).toBeInstanceOf(SchemaValidationError) + if (error instanceof SchemaValidationError) { + expect(error.type).toBe(`update`) + expect(error.issues.length).toBeGreaterThan(0) + expect(error.issues.some((issue) => issue.path?.includes(`age`))).toBe( + true + ) + } + } + }) + + it(`should validate data against schema on insert`, () => { + // Create a Zod schema for a user + const userSchema = z.object({ + name: z.string().min(1), + age: z.number().int().positive(), + email: z.string().email().optional(), + }) + + // Create a collection with the schema + const collection = createCollection>({ + id: `test`, + getKey: (item) => item.name, + startSync: true, + sync: { + sync: ({ begin, commit }) => { + begin() + commit() + }, + }, + schema: userSchema, + }) + const mutationFn = async () => {} + + // Valid data should work + const validUser = { + name: `Alice`, + age: 30, + email: `alice@example.com`, + } + + const tx1 = createTransaction({ mutationFn }) + tx1.mutate(() => collection.insert(validUser)) + + // Invalid data should throw SchemaValidationError + const invalidUser = { + name: ``, // Empty name (fails min length) + age: -5, // Negative age (fails positive) + email: `not-an-email`, // Invalid email + } + + try { + const tx2 = createTransaction({ mutationFn }) + tx2.mutate(() => collection.insert(invalidUser)) + // Should not reach here + expect(true).toBe(false) + } catch (error) { + expect(error).toBeInstanceOf(SchemaValidationError) + if (error instanceof SchemaValidationError) { + expect(error.type).toBe(`insert`) + expect(error.issues.length).toBeGreaterThan(0) + // Check that we have validation errors for each invalid field + expect(error.issues.some((issue) => issue.path?.includes(`name`))).toBe( + true + ) + expect(error.issues.some((issue) => issue.path?.includes(`age`))).toBe( + true + ) + expect( + error.issues.some((issue) => issue.path?.includes(`email`)) + ).toBe(true) + } + } + + // Partial updates should work with valid data + const tx3 = createTransaction({ mutationFn }) + tx3.mutate(() => + collection.update(`Alice`, (draft) => { + draft.age = 31 + }) + ) + + // Partial updates should fail with invalid data + try { + const tx4 = createTransaction({ mutationFn }) + tx4.mutate(() => + collection.update(`Alice`, (draft) => { + draft.age = -1 + }) + ) + // Should not reach here + expect(true).toBe(false) + } catch (error) { + expect(error).toBeInstanceOf(SchemaValidationError) + if (error instanceof SchemaValidationError) { + expect(error.type).toBe(`update`) + expect(error.issues.length).toBeGreaterThan(0) + expect(error.issues.some((issue) => issue.path?.includes(`age`))).toBe( + true + ) + } + } + }) + + it(`should apply schema defaults on insert`, () => { + const todoSchema = z.object({ + id: z + .string() + .default(() => `todo-${Math.random().toString(36).substr(2, 9)}`), + text: z.string(), + completed: z.boolean().default(false), + createdAt: z.coerce.date().default(() => new Date()), + updatedAt: z.coerce.date().default(() => new Date()), + }) + + // Define inferred types for clarity and use in assertions + type Todo = z.infer + type TodoInput = z.input + + // NOTE: `createCollection` breaks the schema type inference. + // We have to use only the schema, and not the type generic, like so: + const collection = createCollection({ + id: `defaults-test`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, commit }) => { + begin() + commit() + }, + }, + schema: todoSchema, + }) + + // Type test: should allow inserting input type (with missing fields that have defaults) + // Important: Input type is different from the output type (which is inferred using z.infer) + // For more details, @see https://github.com/colinhacks/zod/issues/4179#issuecomment-2811669261 + type InsertParam = Parameters[0] + expectTypeOf().toEqualTypeOf>() + + const mutationFn = async () => {} + + // Minimal data + const tx1 = createTransaction({ mutationFn }) + tx1.mutate(() => collection.insert({ text: `task-1` })) + + // Type assertions on the mutation structure + expect(tx1.mutations).toHaveLength(1) + const mutation = tx1.mutations[0]! + + // Test the mutation type structure + expectTypeOf(mutation).toExtend>() + expectTypeOf(mutation.type).toEqualTypeOf() + expectTypeOf(mutation.changes).toEqualTypeOf< + ResolveTransactionChanges + >() + expectTypeOf(mutation.modified).toEqualTypeOf() + + // Runtime assertions for actual values + expect(mutation.type).toBe(`insert`) + expect(mutation.changes).toEqual({ text: `task-1` }) + expect(mutation.modified.text).toBe(`task-1`) + expect(mutation.modified.completed).toBe(false) + expect(mutation.modified.id).toBeDefined() + expect(mutation.modified.createdAt).toBeInstanceOf(Date) + expect(mutation.modified.updatedAt).toBeInstanceOf(Date) + + let insertedItems = Array.from(collection.state.values()) + expect(insertedItems).toHaveLength(1) + const insertedItem = insertedItems[0]! + expect(insertedItem.text).toBe(`task-1`) + expect(insertedItem.completed).toBe(false) + expect(insertedItem.id).toBeDefined() + expect(typeof insertedItem.id).toBe(`string`) + expect(insertedItem.createdAt).toBeInstanceOf(Date) + expect(insertedItem.updatedAt).toBeInstanceOf(Date) + + // Partial data + const tx2 = createTransaction({ mutationFn }) + tx2.mutate(() => collection.insert({ text: `task-2`, completed: true })) + + insertedItems = Array.from(collection.state.values()) + expect(insertedItems).toHaveLength(2) + + const secondItem = insertedItems.find((item) => item.text === `task-2`)! + expect(secondItem).toBeDefined() + expect(secondItem.text).toBe(`task-2`) + expect(secondItem.completed).toBe(true) + expect(secondItem.id).toBeDefined() + expect(typeof secondItem.id).toBe(`string`) + expect(secondItem.createdAt).toBeInstanceOf(Date) + expect(secondItem.updatedAt).toBeInstanceOf(Date) + + // All fields provided + const tx3 = createTransaction({ mutationFn }) + + tx3.mutate(() => + collection.insert({ + id: `task-id-3`, + text: `task-3`, + completed: true, + createdAt: new Date(`2023-01-01T00:00:00Z`), + updatedAt: new Date(`2023-01-01T00:00:00Z`), + }) + ) + insertedItems = Array.from(collection.state.values()) + expect(insertedItems).toHaveLength(3) + + // using insertedItems[2] was finding wrong item for some reason. + const thirdItem = insertedItems.find((item) => item.text === `task-3`) + expect(thirdItem).toBeDefined() + expect(thirdItem!.text).toBe(`task-3`) + expect(thirdItem!.completed).toBe(true) + expect(thirdItem!.createdAt).toEqual(new Date(`2023-01-01T00:00:00Z`)) + expect(thirdItem!.updatedAt).toEqual(new Date(`2023-01-01T00:00:00Z`)) + expect(thirdItem!.id).toBe(`task-id-3`) + }) + + it(`should not block user actions when keys are recently synced`, async () => { + // This test reproduces the ACTUAL issue where rapid user actions get blocked + // when optimistic updates back up with slow sync responses + const txResolvers: Array<() => void> = [] + const emitter = mitt() + const changeEvents: Array = [] + + const mutationFn = vi.fn().mockImplementation(async ({ transaction }) => { + // Simulate SLOW server operation - this is key to reproducing the issue + return new Promise((resolve) => { + txResolvers.push(() => { + emitter.emit(`sync`, transaction.mutations) + resolve(null) + }) + }) + }) + + const collection = createCollection<{ id: number; checked: boolean }>({ + id: `user-action-blocking-test`, + getKey: (item) => item.id, + startSync: true, + sync: { + sync: ({ begin, write, commit, markReady }) => { + // Initialize with checkboxes + begin() + for (let i = 1; i <= 3; i++) { + write({ + type: `insert`, + value: { id: i, checked: false }, + }) + } + commit() + markReady() + + // Listen for sync events - this triggers the problematic batching + // @ts-expect-error don't trust mitt's typing + emitter.on(`*`, (_, changes: Array) => { + begin() + changes.forEach((change) => { + write({ + type: change.type, + // @ts-expect-error TODO type changes + value: change.modified, + }) + }) + commit() + }) + }, + }, + onUpdate: mutationFn, + }) + + // Listen to change events to verify they're emitted (this was the actual problem) + collection.subscribeChanges((changes) => { + changeEvents.push(...changes) + }) + + await collection.stateWhenReady() + + // CRITICAL: Simulate rapid clicking WITHOUT waiting for transactions to complete + // This is what actually triggers the bug - multiple pending transactions + + // Step 1: First click + const tx1 = collection.update(1, (draft) => { + draft.checked = true + }) + expect(collection.state.get(1)?.checked).toBe(true) + const initialEventCount = changeEvents.length + + // Step 2: Second click immediately (before first completes) + const tx2 = collection.update(1, (draft) => { + draft.checked = false + }) + expect(collection.state.get(1)?.checked).toBe(false) + + // Step 3: Third click immediately (before others complete) + const tx3 = collection.update(1, (draft) => { + draft.checked = true + }) + expect(collection.state.get(1)?.checked).toBe(true) + + // CRITICAL TEST: Verify events are still being emitted for rapid user actions + // Before the fix, these would be batched and UI would freeze + expect(changeEvents.length).toBeGreaterThan(initialEventCount) + expect(mutationFn).toHaveBeenCalledTimes(3) + + // Now complete the first transaction to trigger sync and batching + txResolvers[0]?.() + await tx1.isPersisted.promise + + // Step 4: More rapid clicks after sync starts (this is where the bug occurred) + const eventCountBeforeRapidClicks = changeEvents.length + + const tx4 = collection.update(1, (draft) => { + draft.checked = false + }) + const tx5 = collection.update(1, (draft) => { + draft.checked = true + }) + + // CRITICAL: Verify that even after sync/batching starts, user actions still emit events + expect(changeEvents.length).toBeGreaterThan(eventCountBeforeRapidClicks) + expect(collection.state.get(1)?.checked).toBe(true) // Last action should win + + // Clean up remaining transactions + for (let i = 1; i < txResolvers.length; i++) { + txResolvers[i]?.() + } + await Promise.all([ + tx2.isPersisted.promise, + tx3.isPersisted.promise, + tx4.isPersisted.promise, + tx5.isPersisted.promise, + ]) + }) +}) diff --git a/packages/db/tests/collection.test.ts b/packages/db/tests/collection.test.ts index 2a15688a2..e837b294b 100644 --- a/packages/db/tests/collection.test.ts +++ b/packages/db/tests/collection.test.ts @@ -1,7 +1,5 @@ -import { type } from "arktype" import mitt from "mitt" -import { describe, expect, expectTypeOf, it, vi } from "vitest" -import { z } from "zod" +import { describe, expect, it, vi } from "vitest" import { createCollection } from "../src/collection" import { CollectionRequiresConfigError, @@ -10,16 +8,9 @@ import { MissingDeleteHandlerError, MissingInsertHandlerError, MissingUpdateHandlerError, - SchemaValidationError, } from "../src/errors" import { createTransaction } from "../src/transactions" -import type { - ChangeMessage, - MutationFn, - OperationType, - PendingMutation, - ResolveTransactionChanges, -} from "../src/types" +import type { ChangeMessage, MutationFn, PendingMutation } from "../src/types" describe(`Collection`, () => { it(`should throw if there's no sync config`, () => { @@ -941,307 +932,6 @@ describe(`Collection`, () => { await optimisticDeleteTx.isPersisted.promise }) -}) - -describe(`Collection with schema validation`, () => { - it(`should validate data against arktype schema on insert`, () => { - // Create a Zod schema for a user - const userSchema = type({ - name: `string > 0`, - age: `number.integer > 0`, - "email?": `string.email`, - }) - - // Create a collection with the schema - const collection = createCollection({ - id: `test`, - getKey: (item) => item.name, - startSync: true, - sync: { - sync: ({ begin, commit }) => { - begin() - commit() - }, - }, - schema: userSchema, - }) - const mutationFn = async () => {} - - // Valid data should work - const validUser = { - name: `Alice`, - age: 30, - email: `alice@example.com`, - } - - const tx1 = createTransaction({ mutationFn }) - tx1.mutate(() => collection.insert(validUser)) - - // Invalid data should throw SchemaValidationError - const invalidUser = { - name: ``, // Empty name (fails min length) - age: -5, // Negative age (fails positive) - email: `not-an-email`, // Invalid email - } - - try { - const tx2 = createTransaction({ mutationFn }) - tx2.mutate(() => collection.insert(invalidUser)) - // Should not reach here - expect(true).toBe(false) - } catch (error) { - expect(error).toBeInstanceOf(SchemaValidationError) - if (error instanceof SchemaValidationError) { - expect(error.type).toBe(`insert`) - expect(error.issues.length).toBeGreaterThan(0) - // Check that we have validation errors for each invalid field - expect(error.issues.some((issue) => issue.path?.includes(`name`))).toBe( - true - ) - expect(error.issues.some((issue) => issue.path?.includes(`age`))).toBe( - true - ) - expect( - error.issues.some((issue) => issue.path?.includes(`email`)) - ).toBe(true) - } - } - - // Partial updates should work with valid data - const tx3 = createTransaction({ mutationFn }) - tx3.mutate(() => - collection.update(`Alice`, (draft) => { - draft.age = 31 - }) - ) - - // Partial updates should fail with invalid data - try { - const tx4 = createTransaction({ mutationFn }) - tx4.mutate(() => - collection.update(`Alice`, (draft) => { - draft.age = -1 - }) - ) - // Should not reach here - expect(true).toBe(false) - } catch (error) { - expect(error).toBeInstanceOf(SchemaValidationError) - if (error instanceof SchemaValidationError) { - expect(error.type).toBe(`update`) - expect(error.issues.length).toBeGreaterThan(0) - expect(error.issues.some((issue) => issue.path?.includes(`age`))).toBe( - true - ) - } - } - }) - - it(`should validate data against schema on insert`, () => { - // Create a Zod schema for a user - const userSchema = z.object({ - name: z.string().min(1), - age: z.number().int().positive(), - email: z.string().email().optional(), - }) - - // Create a collection with the schema - const collection = createCollection>({ - id: `test`, - getKey: (item) => item.name, - startSync: true, - sync: { - sync: ({ begin, commit }) => { - begin() - commit() - }, - }, - schema: userSchema, - }) - const mutationFn = async () => {} - - // Valid data should work - const validUser = { - name: `Alice`, - age: 30, - email: `alice@example.com`, - } - - const tx1 = createTransaction({ mutationFn }) - tx1.mutate(() => collection.insert(validUser)) - - // Invalid data should throw SchemaValidationError - const invalidUser = { - name: ``, // Empty name (fails min length) - age: -5, // Negative age (fails positive) - email: `not-an-email`, // Invalid email - } - - try { - const tx2 = createTransaction({ mutationFn }) - tx2.mutate(() => collection.insert(invalidUser)) - // Should not reach here - expect(true).toBe(false) - } catch (error) { - expect(error).toBeInstanceOf(SchemaValidationError) - if (error instanceof SchemaValidationError) { - expect(error.type).toBe(`insert`) - expect(error.issues.length).toBeGreaterThan(0) - // Check that we have validation errors for each invalid field - expect(error.issues.some((issue) => issue.path?.includes(`name`))).toBe( - true - ) - expect(error.issues.some((issue) => issue.path?.includes(`age`))).toBe( - true - ) - expect( - error.issues.some((issue) => issue.path?.includes(`email`)) - ).toBe(true) - } - } - - // Partial updates should work with valid data - const tx3 = createTransaction({ mutationFn }) - tx3.mutate(() => - collection.update(`Alice`, (draft) => { - draft.age = 31 - }) - ) - - // Partial updates should fail with invalid data - try { - const tx4 = createTransaction({ mutationFn }) - tx4.mutate(() => - collection.update(`Alice`, (draft) => { - draft.age = -1 - }) - ) - // Should not reach here - expect(true).toBe(false) - } catch (error) { - expect(error).toBeInstanceOf(SchemaValidationError) - if (error instanceof SchemaValidationError) { - expect(error.type).toBe(`update`) - expect(error.issues.length).toBeGreaterThan(0) - expect(error.issues.some((issue) => issue.path?.includes(`age`))).toBe( - true - ) - } - } - }) - - it(`should apply schema defaults on insert`, () => { - const todoSchema = z.object({ - id: z - .string() - .default(() => `todo-${Math.random().toString(36).substr(2, 9)}`), - text: z.string(), - completed: z.boolean().default(false), - createdAt: z.coerce.date().default(() => new Date()), - updatedAt: z.coerce.date().default(() => new Date()), - }) - - // Define inferred types for clarity and use in assertions - type Todo = z.infer - type TodoInput = z.input - - // NOTE: `createCollection` breaks the schema type inference. - // We have to use only the schema, and not the type generic, like so: - const collection = createCollection({ - id: `defaults-test`, - getKey: (item) => item.id, - sync: { - sync: ({ begin, commit }) => { - begin() - commit() - }, - }, - schema: todoSchema, - }) - - // Type test: should allow inserting input type (with missing fields that have defaults) - // Important: Input type is different from the output type (which is inferred using z.infer) - // For more details, @see https://github.com/colinhacks/zod/issues/4179#issuecomment-2811669261 - type InsertParam = Parameters[0] - expectTypeOf().toEqualTypeOf>() - - const mutationFn = async () => {} - - // Minimal data - const tx1 = createTransaction({ mutationFn }) - tx1.mutate(() => collection.insert({ text: `task-1` })) - - // Type assertions on the mutation structure - expect(tx1.mutations).toHaveLength(1) - const mutation = tx1.mutations[0]! - - // Test the mutation type structure - expectTypeOf(mutation).toExtend>() - expectTypeOf(mutation.type).toEqualTypeOf() - expectTypeOf(mutation.changes).toEqualTypeOf< - ResolveTransactionChanges - >() - expectTypeOf(mutation.modified).toEqualTypeOf() - - // Runtime assertions for actual values - expect(mutation.type).toBe(`insert`) - expect(mutation.changes).toEqual({ text: `task-1` }) - expect(mutation.modified.text).toBe(`task-1`) - expect(mutation.modified.completed).toBe(false) - expect(mutation.modified.id).toBeDefined() - expect(mutation.modified.createdAt).toBeInstanceOf(Date) - expect(mutation.modified.updatedAt).toBeInstanceOf(Date) - - let insertedItems = Array.from(collection.state.values()) - expect(insertedItems).toHaveLength(1) - const insertedItem = insertedItems[0]! - expect(insertedItem.text).toBe(`task-1`) - expect(insertedItem.completed).toBe(false) - expect(insertedItem.id).toBeDefined() - expect(typeof insertedItem.id).toBe(`string`) - expect(insertedItem.createdAt).toBeInstanceOf(Date) - expect(insertedItem.updatedAt).toBeInstanceOf(Date) - - // Partial data - const tx2 = createTransaction({ mutationFn }) - tx2.mutate(() => collection.insert({ text: `task-2`, completed: true })) - - insertedItems = Array.from(collection.state.values()) - expect(insertedItems).toHaveLength(2) - - const secondItem = insertedItems.find((item) => item.text === `task-2`)! - expect(secondItem).toBeDefined() - expect(secondItem.text).toBe(`task-2`) - expect(secondItem.completed).toBe(true) - expect(secondItem.id).toBeDefined() - expect(typeof secondItem.id).toBe(`string`) - expect(secondItem.createdAt).toBeInstanceOf(Date) - expect(secondItem.updatedAt).toBeInstanceOf(Date) - - // All fields provided - const tx3 = createTransaction({ mutationFn }) - - tx3.mutate(() => - collection.insert({ - id: `task-id-3`, - text: `task-3`, - completed: true, - createdAt: new Date(`2023-01-01T00:00:00Z`), - updatedAt: new Date(`2023-01-01T00:00:00Z`), - }) - ) - insertedItems = Array.from(collection.state.values()) - expect(insertedItems).toHaveLength(3) - - // using insertedItems[2] was finding wrong item for some reason. - const thirdItem = insertedItems.find((item) => item.text === `task-3`) - expect(thirdItem).toBeDefined() - expect(thirdItem!.text).toBe(`task-3`) - expect(thirdItem!.completed).toBe(true) - expect(thirdItem!.createdAt).toEqual(new Date(`2023-01-01T00:00:00Z`)) - expect(thirdItem!.updatedAt).toEqual(new Date(`2023-01-01T00:00:00Z`)) - expect(thirdItem!.id).toBe(`task-id-3`) - }) it(`should not block user actions when keys are recently synced`, async () => { // This test reproduces the ACTUAL issue where rapid user actions get blocked From 5b7945525ed575a67cd21325300d6189dfe894b8 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Tue, 19 Aug 2025 10:23:31 +0100 Subject: [PATCH 04/15] Change to use the input type for update --- packages/db/src/collection.ts | 89 ++--- packages/db/src/local-only.ts | 8 +- packages/db/src/local-storage.ts | 12 +- packages/db/src/types.ts | 10 +- packages/db/tests/collection-schema.test.ts | 369 +++++++++++++++----- packages/db/tests/collection.test-d.ts | 114 +++++- packages/query-db-collection/src/query.ts | 13 +- 7 files changed, 456 insertions(+), 159 deletions(-) diff --git a/packages/db/src/collection.ts b/packages/db/src/collection.ts index f89b6432c..34a0adccf 100644 --- a/packages/db/src/collection.ts +++ b/packages/db/src/collection.ts @@ -51,7 +51,7 @@ import type { OperationConfig, OptimisticChangeMessage, PendingMutation, - ResolveInsertInput, + ResolveInput, ResolveType, StandardSchema, SubscribeChangesOptions, @@ -73,15 +73,15 @@ interface PendingSyncedTransaction> { * @template TKey - The type of the key for the collection * @template TUtils - The utilities record type * @template TSchema - The schema type for validation and type inference - * @template TInsertInput - The type for insert operations (can be different from T for schemas with defaults) + * @template TInput - The type for insert and update operations (can be different from T for schemas with defaults) */ export interface Collection< T extends object = Record, TKey extends string | number = string | number, TUtils extends UtilsRecord = {}, TSchema extends StandardSchemaV1 = StandardSchemaV1, - TInsertInput extends object = T, -> extends CollectionImpl { + TInput extends object = T, +> extends CollectionImpl { readonly utils: TUtils } @@ -170,7 +170,7 @@ export function createCollection< ResolveType, TKey, TSchema, - ResolveInsertInput + ResolveInput > & { schema: TSchema utils?: TUtils @@ -180,7 +180,7 @@ export function createCollection< TKey, TUtils, TSchema, - ResolveInsertInput + ResolveInput > // Overload for when explicit type is provided with schema - explicit type takes precedence @@ -195,7 +195,7 @@ export function createCollection< ResolveType, TKey, TSchema, - ResolveInsertInput + ResolveInput > & { schema: TSchema utils?: TUtils @@ -205,7 +205,7 @@ export function createCollection< TKey, TUtils, TSchema, - ResolveInsertInput + ResolveInput > // Overload for when explicit type is provided or no schema @@ -220,14 +220,14 @@ export function createCollection< ResolveType, TKey, TSchema, - ResolveInsertInput + ResolveInput > & { utils?: TUtils } ): Collection< ResolveType, TKey, TUtils, TSchema, - ResolveInsertInput + ResolveInput > // Implementation @@ -242,21 +242,21 @@ export function createCollection< ResolveType, TKey, TSchema, - ResolveInsertInput + ResolveInput > & { utils?: TUtils } ): Collection< ResolveType, TKey, TUtils, TSchema, - ResolveInsertInput + ResolveInput > { const collection = new CollectionImpl< ResolveType, TKey, TUtils, TSchema, - ResolveInsertInput + ResolveInput >(options) // Copy utils to both top level and .utils namespace @@ -271,7 +271,7 @@ export function createCollection< TKey, TUtils, TSchema, - ResolveInsertInput + ResolveInput > } @@ -280,9 +280,9 @@ export class CollectionImpl< TKey extends string | number = string | number, TUtils extends UtilsRecord = {}, TSchema extends StandardSchemaV1 = StandardSchemaV1, - TInsertInput extends object = T, + TInput extends object = T, > { - public config: CollectionConfig + public config: CollectionConfig // Core state - make public for testing public transactions: SortedMap> @@ -478,7 +478,7 @@ export class CollectionImpl< * @param config - Configuration object for the collection * @throws Error if sync config is missing */ - constructor(config: CollectionConfig) { + constructor(config: CollectionConfig) { // eslint-disable-next-line if (!config) { throw new CollectionRequiresConfigError() @@ -1681,10 +1681,7 @@ export class CollectionImpl< * console.log('Insert failed:', error) * } */ - insert = ( - data: TInsertInput | Array, - config?: InsertConfig - ) => { + insert = (data: TInput | Array, config?: InsertConfig) => { this.validateCollectionUsable(`insert`) const ambientTransaction = getActiveTransaction() @@ -1713,14 +1710,15 @@ export class CollectionImpl< original: {}, modified: validatedData, // Pick the values from validatedData based on what's passed in - this is for cases - // where a schema has default values. The validated data has the extra default - // values but for changes, we just want to show the data that was actually passed in. + // where a schema has default values or transforms. The validated data has the extra + // default or transformed values but for changes, we just want to show the data that + // was actually passed in. changes: Object.fromEntries( Object.keys(item).map((k) => [ k, validatedData[k as keyof typeof validatedData], ]) - ) as TInsertInput, + ) as TInput, globalKey, key, metadata: config?.metadata as unknown, @@ -1752,7 +1750,7 @@ export class CollectionImpl< return await this.config.onInsert!({ transaction: params.transaction as unknown as TransactionWithMutations< - TInsertInput, + T, `insert` >, collection: this as unknown as Collection, @@ -1814,35 +1812,37 @@ export class CollectionImpl< */ // Overload 1: Update multiple items with a callback - update( + update( key: Array, - callback: (drafts: Array) => void + callback: (drafts: Array) => void ): TransactionType // Overload 2: Update multiple items with config and a callback - update( + update( keys: Array, config: OperationConfig, - callback: (drafts: Array) => void + callback: (drafts: Array) => void ): TransactionType // Overload 3: Update a single item with a callback - update( + update( id: TKey | unknown, - callback: (draft: TItem) => void + callback: (draft: TDraft) => void ): TransactionType // Overload 4: Update a single item with config and a callback - update( + update( id: TKey | unknown, config: OperationConfig, - callback: (draft: TItem) => void + callback: (draft: TDraft) => void ): TransactionType - update( + update( keys: (TKey | unknown) | Array, - configOrCallback: ((draft: TItem | Array) => void) | OperationConfig, - maybeCallback?: (draft: TItem | Array) => void + configOrCallback: + | ((draft: TDraft | Array) => void) + | OperationConfig, + maybeCallback?: (draft: TDraft | Array) => void ) { if (typeof keys === `undefined`) { throw new MissingUpdateArgumentError() @@ -1877,19 +1877,19 @@ export class CollectionImpl< } return item - }) as unknown as Array + }) as unknown as Array let changesArray if (isArray) { // Use the proxy to track changes for all objects changesArray = withArrayChangeTracking( currentObjects, - callback as (draft: Array) => void + callback as (draft: Array) => void ) } else { const result = withChangeTracking( currentObjects[0]!, - callback as (draft: TItem) => void + callback as (draft: TDraft) => void ) changesArray = [result] } @@ -1933,7 +1933,16 @@ export class CollectionImpl< mutationId: crypto.randomUUID(), original: originalItem, modified: modifiedItem, - changes: validatedUpdatePayload as Partial, + // Pick the values from modifiedItem based on what's passed in - this is for cases + // where a schema has default values or transforms. The modified data has the extra + // default or transformed values but for changes, we just want to show the data that + // was actually passed in. + changes: Object.fromEntries( + Object.keys(itemChanges).map((k) => [ + k, + modifiedItem[k as keyof typeof modifiedItem], + ]) + ) as TInput, globalKey, key, metadata: config.metadata as unknown, diff --git a/packages/db/src/local-only.ts b/packages/db/src/local-only.ts index 19725bf21..5e3065d34 100644 --- a/packages/db/src/local-only.ts +++ b/packages/db/src/local-only.ts @@ -3,7 +3,7 @@ import type { DeleteMutationFnParams, InsertMutationFnParams, OperationType, - ResolveInsertInput, + ResolveInput, ResolveType, SyncConfig, UpdateMutationFnParams, @@ -52,7 +52,7 @@ export interface LocalOnlyCollectionConfig< */ onInsert?: ( params: InsertMutationFnParams< - ResolveInsertInput, + ResolveInput, TKey, LocalOnlyCollectionUtils > @@ -147,12 +147,12 @@ export function localOnlyCollectionOptions< ResolveType, TKey, TSchema, - ResolveInsertInput + ResolveInput > & { utils: LocalOnlyCollectionUtils } { type TItem = ResolveType - type TInsertInput = ResolveInsertInput + type TInsertInput = ResolveInput const { initialData, onInsert, onUpdate, onDelete, ...restConfig } = config diff --git a/packages/db/src/local-storage.ts b/packages/db/src/local-storage.ts index c1e63818d..1cbaceb0b 100644 --- a/packages/db/src/local-storage.ts +++ b/packages/db/src/local-storage.ts @@ -11,7 +11,7 @@ import type { DeleteMutationFnParams, InsertMutationFn, InsertMutationFnParams, - ResolveInsertInput, + ResolveInput, ResolveType, SyncConfig, UpdateMutationFnParams, @@ -92,13 +92,13 @@ export interface LocalStorageCollectionConfig< ResolveType, TKey, TSchema, - ResolveInsertInput + ResolveInput >[`getKey`] sync?: CollectionConfig< ResolveType, TKey, TSchema, - ResolveInsertInput + ResolveInput >[`sync`] /** @@ -108,7 +108,7 @@ export interface LocalStorageCollectionConfig< */ onInsert?: ( params: InsertMutationFnParams< - ResolveInsertInput, + ResolveInput, TKey > ) => Promise @@ -236,7 +236,7 @@ export function localStorageCollectionOptions< ResolveType, TKey, TSchema, - ResolveInsertInput + ResolveInput >, `id` > & { @@ -244,7 +244,7 @@ export function localStorageCollectionOptions< utils: LocalStorageCollectionUtils } { type TItem = ResolveType - type TInsertInput = ResolveInsertInput + type TInsertInput = ResolveInput // Validate required parameters if (!config.storageKey) { diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index 28b25e0ef..dc0650b54 100644 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -29,7 +29,7 @@ export type InferSchemaInput = T extends StandardSchemaV1 : Record /** - * Helper type to determine the insert input type + * Helper type to determine the input type for insert and update operations * This takes the raw generics (TExplicit, TSchema, TFallback) instead of the resolved T. * * Priority: @@ -37,9 +37,9 @@ export type InferSchemaInput = T extends StandardSchemaV1 * 2. Schema input type (if schema provided) * 3. Fallback type TFallback * - * @internal This is used for collection insert type inference + * @internal This is used for collection insert and update type inference */ -export type ResolveInsertInput< +export type ResolveInput< TExplicit = unknown, TSchema extends StandardSchemaV1 = never, TFallback extends object = Record, @@ -350,7 +350,7 @@ export interface CollectionConfig< T extends object = Record, TKey extends string | number = string | number, TSchema extends StandardSchemaV1 = StandardSchemaV1, - TInsertInput extends object = T, + _TInput extends object = T, > { // If an id isn't passed in, a UUID will be // generated for it. @@ -438,7 +438,7 @@ export interface CollectionConfig< * }) * } */ - onInsert?: InsertMutationFn + onInsert?: InsertMutationFn /** * Optional asynchronous handler function called before an update operation diff --git a/packages/db/tests/collection-schema.test.ts b/packages/db/tests/collection-schema.test.ts index 5301b7751..159d6c70e 100644 --- a/packages/db/tests/collection-schema.test.ts +++ b/packages/db/tests/collection-schema.test.ts @@ -1,6 +1,5 @@ import { type } from "arktype" -import mitt from "mitt" -import { describe, expect, expectTypeOf, it, vi } from "vitest" +import { describe, expect, expectTypeOf, it } from "vitest" import { z } from "zod" import { createCollection } from "../src/collection" import { SchemaValidationError } from "../src/errors" @@ -571,119 +570,315 @@ describe(`Collection with schema validation`, () => { expect(thirdItem!.id).toBe(`task-id-3`) }) - it(`should not block user actions when keys are recently synced`, async () => { - // This test reproduces the ACTUAL issue where rapid user actions get blocked - // when optimistic updates back up with slow sync responses - const txResolvers: Array<() => void> = [] - const emitter = mitt() - const changeEvents: Array = [] - - const mutationFn = vi.fn().mockImplementation(async ({ transaction }) => { - // Simulate SLOW server operation - this is key to reproducing the issue - return new Promise((resolve) => { - txResolvers.push(() => { - emitter.emit(`sync`, transaction.mutations) - resolve(null) - }) - }) + it(`should apply schema transformations on insert operations`, () => { + // Create a schema with transformations + const userSchema = z.object({ + id: z.string(), + name: z.string(), + email: z.string().email(), + age: z.number().int().positive(), + created_at: z.string().transform((val) => new Date(val)), + updated_at: z.string().transform((val) => new Date(val)), + tags: z + .array(z.string()) + .transform((val) => val.map((tag) => tag.toLowerCase())), + metadata: z + .record(z.string()) + .transform((val) => ({ ...val, processed: true })), }) - const collection = createCollection<{ id: number; checked: boolean }>({ - id: `user-action-blocking-test`, + const collection = createCollection({ getKey: (item) => item.id, + schema: userSchema, startSync: true, sync: { - sync: ({ begin, write, commit, markReady }) => { - // Initialize with checkboxes + sync: ({ begin, commit }) => { begin() - for (let i = 1; i <= 3; i++) { - write({ - type: `insert`, - value: { id: i, checked: false }, - }) - } commit() - markReady() - - // Listen for sync events - this triggers the problematic batching - // @ts-expect-error don't trust mitt's typing - emitter.on(`*`, (_, changes: Array) => { - begin() - changes.forEach((change) => { - write({ - type: change.type, - // @ts-expect-error TODO type changes - value: change.modified, - }) - }) - commit() - }) }, }, - onUpdate: mutationFn, }) - // Listen to change events to verify they're emitted (this was the actual problem) - collection.subscribeChanges((changes) => { - changeEvents.push(...changes) + const mutationFn = async () => {} + + // Test insert with data that should be transformed + const insertData = { + id: `1`, + name: `John Doe`, + email: `john@example.com`, + age: 30, + created_at: `2023-01-01T00:00:00.000Z`, + updated_at: `2023-01-01T00:00:00.000Z`, + tags: [`IMPORTANT`, `USER`], + metadata: { source: `manual` } as any, + } + + const tx = createTransaction({ mutationFn }) + tx.mutate(() => collection.insert(insertData)) + + // Verify that transformations were applied + expect(tx.mutations).toHaveLength(1) + const mutation = tx.mutations[0]! + + expect(mutation.type).toBe(`insert`) + expect(mutation.modified.created_at).toBeInstanceOf(Date) + expect(mutation.modified.updated_at).toBeInstanceOf(Date) + expect(mutation.modified.tags).toEqual([`important`, `user`]) + expect(mutation.modified.metadata).toEqual({ + source: `manual`, + processed: true, + }) + expect(mutation.modified.name).toBe(`John Doe`) + expect(mutation.modified.age).toBe(30) + }) + + it(`should apply schema transformations on update operations`, async () => { + // Create a schema with transformations that can handle both input and existing data + const userSchema = z.object({ + id: z.string(), + name: z.string(), + email: z.string().email(), + age: z.number().int().positive(), + created_at: z + .union([z.string(), z.date()]) + .transform((val) => (typeof val === `string` ? new Date(val) : val)), + updated_at: z + .union([z.string(), z.date()]) + .transform((val) => (typeof val === `string` ? new Date(val) : val)), + tags: z + .union([ + z.array(z.string()), + z + .array(z.string()) + .transform((val) => val.map((tag) => tag.toLowerCase())), + ]) + .transform((val) => val.map((tag) => tag.toLowerCase())), + metadata: z + .union([ + z.record(z.string()), + z + .record(z.string()) + .transform((val) => ({ ...val, processed: true })), + ]) + .transform((val) => ({ ...val, processed: true })), + }) + + const collection = createCollection({ + getKey: (item) => item.id, + schema: userSchema, + startSync: true, + sync: { + sync: ({ begin, write, commit }) => { + begin() + // Insert initial data + write({ + type: `insert`, + value: { + id: `1`, + name: `John Doe`, + email: `john@example.com`, + age: 30, + created_at: new Date(`2023-01-01T00:00:00.000Z`), + updated_at: new Date(`2023-01-01T00:00:00.000Z`), + tags: [`user`], + metadata: { source: `manual` } as any, + }, + }) + commit() + }, + }, }) await collection.stateWhenReady() - // CRITICAL: Simulate rapid clicking WITHOUT waiting for transactions to complete - // This is what actually triggers the bug - multiple pending transactions + const mutationFn = async () => {} + + // Test update with data that should be transformed + const tx = createTransaction({ mutationFn }) + tx.mutate(() => + collection.update(`1`, (draft) => { + draft.name = `Jane Doe` + draft.age = 31 + draft.updated_at = `2023-01-02T00:00:00.000Z` + draft.tags = [`IMPORTANT`, `ADMIN`] + draft.metadata = { role: `admin` } as Record + }) + ) - // Step 1: First click - const tx1 = collection.update(1, (draft) => { - draft.checked = true + // Verify that transformations were applied and only modified fields are returned + expect(tx.mutations).toHaveLength(1) + const mutation = tx.mutations[0]! + + expect(mutation.type).toBe(`update`) + expect(mutation.changes).toHaveProperty(`name`) + expect(mutation.changes).toHaveProperty(`age`) + expect(mutation.changes).toHaveProperty(`updated_at`) + expect(mutation.changes).toHaveProperty(`tags`) + expect(mutation.changes).toHaveProperty(`metadata`) + + // Verify transformations + expect(mutation.changes.updated_at).toBeInstanceOf(Date) + expect(mutation.changes.tags).toEqual([`important`, `admin`]) + expect(mutation.changes.metadata).toEqual({ + role: `admin`, + processed: true, }) - expect(collection.state.get(1)?.checked).toBe(true) - const initialEventCount = changeEvents.length + expect(mutation.changes.name).toBe(`Jane Doe`) + expect(mutation.changes.age).toBe(31) + }) - // Step 2: Second click immediately (before first completes) - const tx2 = collection.update(1, (draft) => { - draft.checked = false + it(`should handle complex nested transformations on insert and update`, async () => { + // Create a schema with complex nested transformations + const addressSchema = z.object({ + street: z.string(), + city: z.string(), + country: z.string().transform((val) => val.toUpperCase()), + }) + + const userSchema = z.object({ + id: z.string(), + name: z.string(), + email: z.string().email(), + addresses: z + .array(addressSchema) + .transform((val) => val.map((addr) => ({ ...addr, normalized: true }))), + preferences: z + .object({ + theme: z.string().transform((val) => val.toLowerCase()), + notifications: z.boolean(), + }) + .transform((val) => ({ ...val, version: `1.0` })), + created_at: z.string().transform((val) => new Date(val)), }) - expect(collection.state.get(1)?.checked).toBe(false) - // Step 3: Third click immediately (before others complete) - const tx3 = collection.update(1, (draft) => { - draft.checked = true + const collection = createCollection({ + getKey: (item) => item.id, + schema: userSchema, + startSync: true, + sync: { + sync: ({ begin, commit }) => { + begin() + commit() + }, + }, }) - expect(collection.state.get(1)?.checked).toBe(true) - // CRITICAL TEST: Verify events are still being emitted for rapid user actions - // Before the fix, these would be batched and UI would freeze - expect(changeEvents.length).toBeGreaterThan(initialEventCount) - expect(mutationFn).toHaveBeenCalledTimes(3) + const mutationFn = async () => {} - // Now complete the first transaction to trigger sync and batching - txResolvers[0]?.() - await tx1.isPersisted.promise + // Test insert with complex nested data + const insertData = { + id: `1`, + name: `John Doe`, + email: `john@example.com`, + addresses: [ + { street: `123 Main St`, city: `New York`, country: `usa` }, + { street: `456 Oak Ave`, city: `Los Angeles`, country: `usa` }, + ], + preferences: { + theme: `DARK`, + notifications: true, + }, + created_at: `2023-01-01T00:00:00.000Z`, + } - // Step 4: More rapid clicks after sync starts (this is where the bug occurred) - const eventCountBeforeRapidClicks = changeEvents.length + const insertTx = createTransaction({ mutationFn }) + insertTx.mutate(() => collection.insert(insertData)) + + // Verify complex transformations were applied + expect(insertTx.mutations).toHaveLength(1) + const insertMutation = insertTx.mutations[0]! + + expect(insertMutation.type).toBe(`insert`) + expect(insertMutation.modified.created_at).toBeInstanceOf(Date) + expect((insertMutation.modified as any).addresses).toHaveLength(2) + expect((insertMutation.modified as any).addresses[0].country).toBe(`USA`) + expect((insertMutation.modified as any).addresses[0].normalized).toBe(true) + expect((insertMutation.modified as any).addresses[1].country).toBe(`USA`) + expect((insertMutation.modified as any).addresses[1].normalized).toBe(true) + expect((insertMutation.modified as any).preferences.theme).toBe(`dark`) + expect((insertMutation.modified as any).preferences.version).toBe(`1.0`) + expect((insertMutation.modified as any).preferences.notifications).toBe( + true + ) - const tx4 = collection.update(1, (draft) => { - draft.checked = false + // Now test update with the same schema that can handle existing transformed data + const updateSchema = z.object({ + id: z.string(), + name: z.string(), + email: z.string().email(), + addresses: z + .array( + z.object({ + street: z.string(), + city: z.string(), + country: z.string().transform((val) => val.toUpperCase()), + normalized: z.boolean().optional(), + }) + ) + .transform((val) => val.map((addr) => ({ ...addr, normalized: true }))), + preferences: z + .object({ + theme: z.string().transform((val) => val.toLowerCase()), + notifications: z.boolean(), + version: z.string().optional(), + }) + .transform((val) => ({ ...val, version: `1.0` })), + created_at: z + .union([z.string(), z.date()]) + .transform((val) => (typeof val === `string` ? new Date(val) : val)), }) - const tx5 = collection.update(1, (draft) => { - draft.checked = true + + const updateCollection = createCollection({ + getKey: (item) => item.id, + schema: updateSchema, + startSync: true, + sync: { + sync: ({ begin, write, commit }) => { + begin() + // Add the transformed insert data + write({ + type: `insert`, + value: insertMutation.modified as any, + }) + commit() + }, + }, }) - // CRITICAL: Verify that even after sync/batching starts, user actions still emit events - expect(changeEvents.length).toBeGreaterThan(eventCountBeforeRapidClicks) - expect(collection.state.get(1)?.checked).toBe(true) // Last action should win + await updateCollection.stateWhenReady() + + // Test update with new nested data + const updateTx = createTransaction({ mutationFn }) + updateTx.mutate(() => + updateCollection.update(`1`, (draft) => { + draft.name = `Jane Doe` + draft.addresses = [ + { street: `789 Pine St`, city: `Chicago`, country: `usa` }, + ] + draft.preferences = { + theme: `LIGHT`, + notifications: false, + } + }) + ) - // Clean up remaining transactions - for (let i = 1; i < txResolvers.length; i++) { - txResolvers[i]?.() - } - await Promise.all([ - tx2.isPersisted.promise, - tx3.isPersisted.promise, - tx4.isPersisted.promise, - tx5.isPersisted.promise, - ]) + // Verify update transformations + expect(updateTx.mutations).toHaveLength(1) + const updateMutation = updateTx.mutations[0]! + + expect(updateMutation.type).toBe(`update`) + expect(updateMutation.changes).toHaveProperty(`name`) + expect(updateMutation.changes).toHaveProperty(`addresses`) + expect(updateMutation.changes).toHaveProperty(`preferences`) + + expect(updateMutation.changes.name).toBe(`Jane Doe`) + expect((updateMutation.changes as any).addresses).toHaveLength(1) + expect((updateMutation.changes as any).addresses[0].country).toBe(`USA`) + expect((updateMutation.changes as any).addresses[0].normalized).toBe(true) + expect((updateMutation.changes as any).preferences.theme).toBe(`light`) + expect((updateMutation.changes as any).preferences.version).toBe(`1.0`) + expect((updateMutation.changes as any).preferences.notifications).toBe( + false + ) }) }) diff --git a/packages/db/tests/collection.test-d.ts b/packages/db/tests/collection.test-d.ts index 01cca41ab..7e317d8ec 100644 --- a/packages/db/tests/collection.test-d.ts +++ b/packages/db/tests/collection.test-d.ts @@ -2,11 +2,7 @@ import { assertType, describe, expectTypeOf, it } from "vitest" import { z } from "zod" import { createCollection } from "../src/collection" import type { CollectionImpl } from "../src/collection" -import type { - OperationConfig, - ResolveInsertInput, - ResolveType, -} from "../src/types" +import type { OperationConfig, ResolveInput, ResolveType } from "../src/types" import type { StandardSchemaV1 } from "@standard-schema/spec" describe(`Collection.update type tests`, () => { @@ -219,7 +215,7 @@ describe(`Schema Input/Output Type Distinction`, () => { updated_at: z.string().transform((val) => new Date(val)), }) - it(`should handle schema with default values correctly`, () => { + it(`should handle schema with default values correctly for insert`, () => { const collection = createCollection({ getKey: (item) => item.id, sync: { sync: () => {} }, @@ -231,7 +227,7 @@ describe(`Schema Input/Output Type Distinction`, () => { typeof userSchemaWithDefaults, Record > - type ExpectedInputType = ResolveInsertInput< + type ExpectedInputType = ResolveInput< unknown, typeof userSchemaWithDefaults, Record @@ -265,14 +261,14 @@ describe(`Schema Input/Output Type Distinction`, () => { expectTypeOf(collection.toArray).toEqualTypeOf>() }) - it(`should handle schema with transformations correctly`, () => { + it(`should handle schema with transformations correctly for insert`, () => { const collection = createCollection({ getKey: (item) => item.id, sync: { sync: () => {} }, schema: userSchemaTransform, }) - type ExpectedInputType = ResolveInsertInput< + type ExpectedInputType = ResolveInput< unknown, typeof userSchemaTransform, Record @@ -310,4 +306,104 @@ describe(`Schema Input/Output Type Distinction`, () => { // Collection items should be ExpectedOutputType expectTypeOf(collection.toArray).toEqualTypeOf>() }) + + it(`should handle schema with default values correctly for update method`, () => { + const collection = createCollection({ + getKey: (item) => item.id, + sync: { sync: () => {} }, + schema: userSchemaWithDefaults, + }) + + type ExpectedOutputType = ResolveType< + unknown, + typeof userSchemaWithDefaults, + Record + > + type ExpectedInputType = ResolveInput< + unknown, + typeof userSchemaWithDefaults, + Record + > + + // Input type should not include defaulted fields + expectTypeOf().toEqualTypeOf<{ + id: string + name: string + email: string + created_at?: Date + updated_at?: Date + }>() + + // Output type should include all fields + expectTypeOf().toEqualTypeOf<{ + id: string + name: string + email: string + created_at: Date + updated_at: Date + }>() + + // Test update method with schema types + const updateMethod: typeof collection.update = (() => {}) as any + updateMethod(`test-id`, (draft) => { + expectTypeOf(draft).toEqualTypeOf() + }) + + updateMethod([`test-id1`, `test-id2`], (drafts) => { + expectTypeOf(drafts).toEqualTypeOf>() + }) + + // Collection items should be ExpectedOutputType + expectTypeOf(collection.toArray).toEqualTypeOf>() + }) + + it(`should handle schema with transformations correctly for update method`, () => { + const collection = createCollection({ + getKey: (item) => item.id, + sync: { sync: () => {} }, + schema: userSchemaTransform, + }) + + type ExpectedInputType = ResolveInput< + unknown, + typeof userSchemaTransform, + Record + > + type ExpectedOutputType = ResolveType< + unknown, + typeof userSchemaTransform, + Record + > + + // Input type should be the raw input (before transformation) + expectTypeOf().toEqualTypeOf<{ + id: string + name: string + email: string + created_at: string + updated_at: string + }>() + + // Output type should be the transformed output + expectTypeOf().toEqualTypeOf<{ + id: string + name: string + email: string + created_at: Date + updated_at: Date + }>() + + // Test update method with schema types + const updateMethod: typeof collection.update = (() => {}) as any + updateMethod(`test-id`, (draft) => { + expectTypeOf(draft).toEqualTypeOf() + }) + + updateMethod([`test-id1`, `test-id2`], (drafts) => { + expectTypeOf(drafts).toEqualTypeOf>() + }) + + // Collection items should be ExpectedOutputType + expectTypeOf(collection.toArray).toEqualTypeOf>() + }) }) diff --git a/packages/query-db-collection/src/query.ts b/packages/query-db-collection/src/query.ts index 3c1d165d6..a88142b22 100644 --- a/packages/query-db-collection/src/query.ts +++ b/packages/query-db-collection/src/query.ts @@ -19,7 +19,7 @@ import type { DeleteMutationFnParams, InsertMutationFn, InsertMutationFnParams, - ResolveInsertInput, + ResolveInput, ResolveType, SyncConfig, UpdateMutationFn, @@ -142,10 +142,7 @@ export interface QueryCollectionConfig< * } * } */ - onInsert?: InsertMutationFn< - ResolveInsertInput, - TKey - > + onInsert?: InsertMutationFn, TKey> /** * Optional asynchronous handler function called before an update operation @@ -359,16 +356,16 @@ export function queryCollectionOptions< ResolveType, TKey, TSchema, - ResolveInsertInput + ResolveInput > & { utils: QueryCollectionUtils< ResolveType, TKey, - ResolveInsertInput + ResolveInput > } { type TItem = ResolveType - type TInsertInput = ResolveInsertInput + type TInsertInput = ResolveInput const { queryKey, queryFn, From a200778dd97c4b0cd1f2b7ec6126b6178cfbfa5a Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Wed, 20 Aug 2025 08:41:25 +0100 Subject: [PATCH 05/15] fix collection type types --- packages/db/src/local-only.ts | 20 +------ packages/db/src/local-storage.ts | 12 +--- .../electric-db-collection/src/electric.ts | 27 +++++++-- .../query-db-collection/src/manual-sync.ts | 30 +++++----- packages/query-db-collection/src/query.ts | 57 ++++++++++++------- .../query-db-collection/tests/query.test-d.ts | 26 ++++----- 6 files changed, 90 insertions(+), 82 deletions(-) diff --git a/packages/db/src/local-only.ts b/packages/db/src/local-only.ts index 5e3065d34..e2a9ba02e 100644 --- a/packages/db/src/local-only.ts +++ b/packages/db/src/local-only.ts @@ -52,7 +52,7 @@ export interface LocalOnlyCollectionConfig< */ onInsert?: ( params: InsertMutationFnParams< - ResolveInput, + ResolveType, TKey, LocalOnlyCollectionUtils > @@ -152,7 +152,6 @@ export function localOnlyCollectionOptions< utils: LocalOnlyCollectionUtils } { type TItem = ResolveType - type TInsertInput = ResolveInput const { initialData, onInsert, onUpdate, onDelete, ...restConfig } = config @@ -171,22 +170,7 @@ export function localOnlyCollectionOptions< // Call user handler first if provided let handlerResult if (onInsert) { - handlerResult = - (await ( - onInsert as ( - p: InsertMutationFnParams< - TInsertInput, - TKey, - LocalOnlyCollectionUtils - > - ) => Promise - )( - params as unknown as InsertMutationFnParams< - TInsertInput, - TKey, - LocalOnlyCollectionUtils - > - )) ?? {} + handlerResult = (await onInsert(params)) ?? {} } // Then synchronously confirm the transaction by looping through mutations diff --git a/packages/db/src/local-storage.ts b/packages/db/src/local-storage.ts index 1cbaceb0b..d400a5a67 100644 --- a/packages/db/src/local-storage.ts +++ b/packages/db/src/local-storage.ts @@ -9,7 +9,6 @@ import { import type { CollectionConfig, DeleteMutationFnParams, - InsertMutationFn, InsertMutationFnParams, ResolveInput, ResolveType, @@ -244,7 +243,6 @@ export function localStorageCollectionOptions< utils: LocalStorageCollectionUtils } { type TItem = ResolveType - type TInsertInput = ResolveInput // Validate required parameters if (!config.storageKey) { @@ -345,10 +343,7 @@ export function localStorageCollectionOptions< // Call the user handler BEFORE persisting changes (if provided) let handlerResult: any = {} if (config.onInsert) { - handlerResult = - (await (config.onInsert as InsertMutationFn)( - params as unknown as InsertMutationFnParams - )) ?? {} + handlerResult = (await config.onInsert(params)) ?? {} } // Always persist to storage @@ -461,10 +456,7 @@ export function localStorageCollectionOptions< ...restConfig, id: collectionId, sync, - onInsert: wrappedOnInsert as unknown as InsertMutationFn< - TInsertInput, - TKey - >, + onInsert: wrappedOnInsert, onUpdate: wrappedOnUpdate, onDelete: wrappedOnDelete, utils: { diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index b0ed8b287..a001658c6 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -36,15 +36,34 @@ const debug = DebugModule.debug(`ts/db:electric`) */ export type Txid = number -// The `InferSchemaOutput` and `ResolveType` are copied from the `@tanstack/db` package -// but we modified `InferSchemaOutput` slightly to restrict the schema output to `Row` -// This is needed in order for `GetExtensions` to be able to infer the parser extensions type from the schema +// The `InferSchemaOutput`, `InferSchemaInput`, `ResolveType` and `ResolveInput` are +// copied from the `@tanstack/db` package but we modified `InferSchemaOutput` +// and `InferSchemaInput` slightly to restrict the schema output to `Row` +// This is needed in order for `GetExtensions` to be able to infer the parser +// extensions type from the schema. + type InferSchemaOutput = T extends StandardSchemaV1 ? StandardSchemaV1.InferOutput extends Row ? StandardSchemaV1.InferOutput : Record : Record +type InferSchemaInput = T extends StandardSchemaV1 + ? StandardSchemaV1.InferInput extends Row + ? StandardSchemaV1.InferInput + : Record + : Record + +type ResolveInput< + TExplicit extends Row = Row, + TSchema extends StandardSchemaV1 = never, + TFallback extends object = Record, +> = unknown extends TExplicit + ? [TSchema] extends [never] + ? TFallback + : InferSchemaInput + : TExplicit + type ResolveType< TExplicit extends Row = Row, TSchema extends StandardSchemaV1 = never, @@ -298,7 +317,7 @@ export function electricCollectionOptions< ResolveType, string | number, TSchema, - ResolveType + ResolveInput > & { utils: ElectricCollectionUtils } { const seenTxids = new Store>(new Set([])) const sync = createElectricSync>( diff --git a/packages/query-db-collection/src/manual-sync.ts b/packages/query-db-collection/src/manual-sync.ts index f783eb119..7600afdde 100644 --- a/packages/query-db-collection/src/manual-sync.ts +++ b/packages/query-db-collection/src/manual-sync.ts @@ -20,9 +20,9 @@ const activeBatchContexts = new WeakMap< export type SyncOperation< TRow extends object, TKey extends string | number = string | number, - TInsertInput extends object = TRow, + TInput extends object = TRow, > = - | { type: `insert`; data: TInsertInput | Array } + | { type: `insert`; data: TInput | Array } | { type: `update`; data: Partial | Array> } | { type: `delete`; key: TKey | Array } | { type: `upsert`; data: Partial | Array> } @@ -53,11 +53,11 @@ interface NormalizedOperation< function normalizeOperations< TRow extends object, TKey extends string | number = string | number, - TInsertInput extends object = TRow, + TInput extends object = TRow, >( ops: - | SyncOperation - | Array>, + | SyncOperation + | Array>, ctx: SyncContext ): Array> { const operations = Array.isArray(ops) ? ops : [ops] @@ -126,11 +126,11 @@ function validateOperations< export function performWriteOperations< TRow extends object, TKey extends string | number = string | number, - TInsertInput extends object = TRow, + TInput extends object = TRow, >( operations: - | SyncOperation - | Array>, + | SyncOperation + | Array>, ctx: SyncContext ): void { const normalized = normalizeOperations(operations, ctx) @@ -206,7 +206,7 @@ export function performWriteOperations< export function createWriteUtils< TRow extends object, TKey extends string | number = string | number, - TInsertInput extends object = TRow, + TInput extends object = TRow, >(getContext: () => SyncContext | null) { function ensureContext(): SyncContext { const context = getContext() @@ -217,8 +217,8 @@ export function createWriteUtils< } return { - writeInsert(data: TInsertInput | Array) { - const operation: SyncOperation = { + writeInsert(data: TInput | Array) { + const operation: SyncOperation = { type: `insert`, data, } @@ -237,7 +237,7 @@ export function createWriteUtils< }, writeUpdate(data: Partial | Array>) { - const operation: SyncOperation = { + const operation: SyncOperation = { type: `update`, data, } @@ -254,7 +254,7 @@ export function createWriteUtils< }, writeDelete(key: TKey | Array) { - const operation: SyncOperation = { + const operation: SyncOperation = { type: `delete`, key, } @@ -271,7 +271,7 @@ export function createWriteUtils< }, writeUpsert(data: Partial | Array>) { - const operation: SyncOperation = { + const operation: SyncOperation = { type: `upsert`, data, } @@ -300,7 +300,7 @@ export function createWriteUtils< // Set up the batch context for this specific collection const batchContext = { - operations: [] as Array>, + operations: [] as Array>, isActive: true, } activeBatchContexts.set(ctx, batchContext) diff --git a/packages/query-db-collection/src/query.ts b/packages/query-db-collection/src/query.ts index 1fd968838..c9f461776 100644 --- a/packages/query-db-collection/src/query.ts +++ b/packages/query-db-collection/src/query.ts @@ -17,10 +17,10 @@ import type { CollectionConfig, DeleteMutationFn, DeleteMutationFnParams, + InferSchemaInput, + InferSchemaOutput, InsertMutationFn, InsertMutationFnParams, - ResolveInput, - // ResolveType, SyncConfig, UpdateMutationFn, UpdateMutationFnParams, @@ -31,15 +31,8 @@ import type { StandardSchemaV1 } from "@standard-schema/spec" // Re-export for external use export type { SyncOperation } from "./manual-sync" -// Schema output type inference helper (matches electric.ts pattern) -type InferSchemaOutput = T extends StandardSchemaV1 - ? StandardSchemaV1.InferOutput extends object - ? StandardSchemaV1.InferOutput - : Record - : Record - // QueryFn return type inference helper -type InferQueryFnOutput = TQueryFn extends ( +type InferQueryFn = TQueryFn extends ( context: QueryFunctionContext ) => Promise> ? TItem extends object @@ -47,6 +40,16 @@ type InferQueryFnOutput = TQueryFn extends ( : Record : Record +type ResolveInput< + TExplicit extends object | unknown = unknown, + TSchema extends StandardSchemaV1 = never, + TQueryFn = unknown, +> = unknown extends TExplicit + ? [TSchema] extends [never] + ? InferQueryFn + : InferSchemaInput + : TExplicit + // Type resolution system with priority order (matches electric.ts pattern) type ResolveType< TExplicit extends object | unknown = unknown, @@ -54,7 +57,7 @@ type ResolveType< TQueryFn = unknown, > = unknown extends TExplicit ? [TSchema] extends [never] - ? InferQueryFnOutput + ? InferQueryFn : InferSchemaOutput : TExplicit @@ -128,7 +131,7 @@ export interface QueryCollectionConfig< /** Unique identifier for the collection */ id?: string /** Function to extract the unique key from an item */ - getKey: CollectionConfig>[`getKey`] + getKey: (item: ResolveType) => TKey /** Schema for validating items */ schema?: TSchema sync?: CollectionConfig>[`sync`] @@ -177,7 +180,7 @@ export interface QueryCollectionConfig< * } * } */ - onInsert?: InsertMutationFn> + onInsert?: InsertMutationFn, TKey> /** * Optional asynchronous handler function called before an update operation @@ -230,7 +233,7 @@ export interface QueryCollectionConfig< * return { refetch: false } // Skip automatic refetch since we handled it manually * } */ - onUpdate?: UpdateMutationFn> + onUpdate?: UpdateMutationFn, TKey> /** * Optional asynchronous handler function called before a delete operation @@ -276,7 +279,7 @@ export interface QueryCollectionConfig< * return { refetch: false } // Skip automatic refetch since we handled it manually * } */ - onDelete?: DeleteMutationFn> + onDelete?: DeleteMutationFn, TKey> /** * Metadata to pass to the query. @@ -422,8 +425,20 @@ export function queryCollectionOptions< TKey extends string | number = string | number, TInput extends object = ResolveInput, >( - config: QueryCollectionConfig -): CollectionConfig> & { + config: QueryCollectionConfig< + TExplicit, + TSchema, + TQueryFn, + TError, + TQueryKey, + TKey + > +): CollectionConfig< + ResolveType, + TKey, + TSchema, + TInput +> & { utils: QueryCollectionUtils< ResolveType, TKey, @@ -518,7 +533,7 @@ export function queryCollectionOptions< const currentSyncedItems = new Map(collection.syncedData) const newItemsMap = new Map() newItemsArray.forEach((item) => { - const key = getKey(item) as TKey + const key = getKey(item) newItemsMap.set(key, item) }) @@ -634,13 +649,11 @@ export function queryCollectionOptions< } // Create write utils using the manual-sync module - const writeUtils = createWriteUtils( - () => writeContext - ) + const writeUtils = createWriteUtils(() => writeContext) // Create wrapper handlers for direct persistence operations that handle refetching const wrappedOnInsert = onInsert - ? async (params: InsertMutationFnParams) => { + ? async (params: InsertMutationFnParams) => { const handlerResult = (await onInsert(params)) ?? {} const shouldRefetch = (handlerResult as { refetch?: boolean }).refetch !== false diff --git a/packages/query-db-collection/tests/query.test-d.ts b/packages/query-db-collection/tests/query.test-d.ts index ec898288b..6db812f58 100644 --- a/packages/query-db-collection/tests/query.test-d.ts +++ b/packages/query-db-collection/tests/query.test-d.ts @@ -197,16 +197,16 @@ describe(`Query collection type resolution tests`, () => { type Output = z.output type Input = z.input - const queryClient = new QueryClient() + const testQueryClient = new QueryClient() const options = queryCollectionOptions({ - queryClient, + queryClient: testQueryClient, queryKey: [`users`], queryFn: () => Promise.resolve([] as Array), schema, getKey: (item) => item.id, - onInsert: async () => {}, - onUpdate: async () => {}, + onInsert: () => Promise.resolve(), + onUpdate: () => Promise.resolve(), }) expectTypeOf(options.getKey).parameters.toEqualTypeOf<[Output]>() @@ -228,8 +228,8 @@ describe(`Query collection type resolution tests`, () => { const options = queryCollectionOptions({ queryClient, queryKey: [`queryfn-inference`], - queryFn: async (): Promise> => { - return [] as Array + queryFn: (): Promise> => { + return Promise.resolve([] as Array) }, getKey: (item) => item.id, }) @@ -247,8 +247,8 @@ describe(`Query collection type resolution tests`, () => { const options = queryCollectionOptions({ queryClient, queryKey: [`explicit-priority`], - queryFn: async (): Promise> => { - return [] as Array + queryFn: (): Promise> => { + return Promise.resolve([] as Array) }, getKey: (item) => item.id, }) @@ -267,8 +267,8 @@ describe(`Query collection type resolution tests`, () => { const options = queryCollectionOptions({ queryClient, queryKey: [`schema-priority`], - queryFn: async (): Promise>> => { - return [] as Array> + queryFn: (): Promise>> => { + return Promise.resolve([] as Array>) }, schema: userSchema, getKey: (item) => item.id, @@ -283,7 +283,7 @@ describe(`Query collection type resolution tests`, () => { const options = queryCollectionOptions({ queryClient, queryKey: [`backward-compat`], - queryFn: async () => [] as Array, + queryFn: () => Promise.resolve([] as Array), getKey: (item) => item.id, }) @@ -294,8 +294,8 @@ describe(`Query collection type resolution tests`, () => { const options = queryCollectionOptions({ queryClient, queryKey: [`collection-test`], - queryFn: async (): Promise> => { - return [] as Array + queryFn: (): Promise> => { + return Promise.resolve([] as Array) }, getKey: (item) => item.id, }) From 4103464997d2ba064a27675d3243a97573e420b1 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Wed, 20 Aug 2025 13:40:54 +0100 Subject: [PATCH 06/15] wip --- packages/db/src/local-only.ts | 23 ++-- packages/db/tests/collection.test-d.ts | 78 ++++++++++--- packages/db/tests/local-only.test-d.ts | 156 +++++++------------------ 3 files changed, 120 insertions(+), 137 deletions(-) diff --git a/packages/db/src/local-only.ts b/packages/db/src/local-only.ts index e2a9ba02e..e04e42eae 100644 --- a/packages/db/src/local-only.ts +++ b/packages/db/src/local-only.ts @@ -27,7 +27,7 @@ import type { StandardSchemaV1 } from "@standard-schema/spec" * You should provide EITHER an explicit type OR a schema, but not both, as they would conflict. */ export interface LocalOnlyCollectionConfig< - TExplicit extends object = Record, + TExplicit = unknown, TSchema extends StandardSchemaV1 = never, TFallback extends Record = Record, TKey extends string | number = string | number, @@ -137,18 +137,25 @@ export interface LocalOnlyCollectionUtils extends UtilsRecord {} * ) */ export function localOnlyCollectionOptions< - TExplicit extends object = Record, + TExplicit = unknown, TSchema extends StandardSchemaV1 = never, TFallback extends Record = Record, TKey extends string | number = string | number, >( config: LocalOnlyCollectionConfig -): CollectionConfig< - ResolveType, - TKey, - TSchema, - ResolveInput -> & { +): (TSchema extends never + ? CollectionConfig< + ResolveType, + TKey, + TSchema, + ResolveInput + > + : CollectionConfig< + ResolveType, + TKey, + TSchema, + ResolveInput + > & { schema: TSchema }) & { utils: LocalOnlyCollectionUtils } { type TItem = ResolveType diff --git a/packages/db/tests/collection.test-d.ts b/packages/db/tests/collection.test-d.ts index 7e317d8ec..cd37ee4e6 100644 --- a/packages/db/tests/collection.test-d.ts +++ b/packages/db/tests/collection.test-d.ts @@ -59,11 +59,19 @@ describe(`Collection type resolution tests`, () => { type ItemOf = T extends Array ? U : T it(`should use explicit type when provided without schema`, () => { - const _collection = createCollection({ - getKey: (item) => item.id, + const _collection = createCollection({ + getKey: (item) => { + expectTypeOf(item).toEqualTypeOf() + return item.id + }, sync: { sync: () => {} }, }) + expectTypeOf(_collection.toArray).toEqualTypeOf>() + + type Key = Parameters[0] + expectTypeOf().toEqualTypeOf() + type Param = Parameters[0] expectTypeOf>().toEqualTypeOf() }) @@ -75,14 +83,23 @@ describe(`Collection type resolution tests`, () => { {}, typeof testSchema >({ - getKey: (item) => item.id, + getKey: (item) => { + expectTypeOf(item).toEqualTypeOf() + return item.id + }, sync: { sync: () => {} }, schema: testSchema, }) - type ExpectedType = ResolveType + expectTypeOf(_collection.toArray).toEqualTypeOf>() + + type Key = Parameters[0] + expectTypeOf().toEqualTypeOf() + type Param = Parameters[0] expectTypeOf>().toEqualTypeOf() + + type ExpectedType = ResolveType expectTypeOf().toEqualTypeOf() }) @@ -94,13 +111,22 @@ describe(`Collection type resolution tests`, () => { never, FallbackType >({ - getKey: (item) => item.id, + getKey: (item) => { + expectTypeOf(item).toEqualTypeOf() + return item.id + }, sync: { sync: () => {} }, }) - type ExpectedType = ResolveType + expectTypeOf(_collection.toArray).toEqualTypeOf>() + type Param = Parameters[0] expectTypeOf>().toEqualTypeOf() + + type Key = Parameters[0] + expectTypeOf().toEqualTypeOf() + + type ExpectedType = ResolveType expectTypeOf().toEqualTypeOf() }) @@ -113,31 +139,45 @@ describe(`Collection type resolution tests`, () => { typeof testSchema, FallbackType >({ - getKey: (item) => item.id, + getKey: (item) => { + expectTypeOf(item).toEqualTypeOf() + return item.id + }, sync: { sync: () => {} }, schema: testSchema, }) + type Param = Parameters[0] + expectTypeOf>().toEqualTypeOf() + + type Key = Parameters[0] + expectTypeOf().toEqualTypeOf() + type ExpectedType = ResolveType< ExplicitType, typeof testSchema, FallbackType > - type Param = Parameters[0] - expectTypeOf>().toEqualTypeOf() expectTypeOf().toEqualTypeOf() }) it(`should automatically infer type from schema without generic arguments`, () => { // This is the key test case that was missing - no generic arguments at all const _collection = createCollection({ - getKey: (item) => item.id, + getKey: (item) => { + expectTypeOf(item).toEqualTypeOf() + return item.id + }, sync: { sync: () => {} }, schema: testSchema, }) + expectTypeOf(_collection.toArray).toEqualTypeOf>() + + type Key = Parameters[0] + expectTypeOf().toEqualTypeOf() + type Param = Parameters[0] - // Should infer the schema type automatically expectTypeOf>().toEqualTypeOf() }) @@ -216,12 +256,6 @@ describe(`Schema Input/Output Type Distinction`, () => { }) it(`should handle schema with default values correctly for insert`, () => { - const collection = createCollection({ - getKey: (item) => item.id, - sync: { sync: () => {} }, - schema: userSchemaWithDefaults, - }) - type ExpectedOutputType = ResolveType< unknown, typeof userSchemaWithDefaults, @@ -232,6 +266,16 @@ describe(`Schema Input/Output Type Distinction`, () => { typeof userSchemaWithDefaults, Record > + + const collection = createCollection({ + getKey: (item) => { + expectTypeOf(item).toEqualTypeOf() + return item.id + }, + sync: { sync: () => {} }, + schema: userSchemaWithDefaults, + }) + type InsertArg = Parameters[0] // Input type should not include defaulted fields diff --git a/packages/db/tests/local-only.test-d.ts b/packages/db/tests/local-only.test-d.ts index 0dc4f2ff1..58109595c 100644 --- a/packages/db/tests/local-only.test-d.ts +++ b/packages/db/tests/local-only.test-d.ts @@ -5,6 +5,7 @@ import { localOnlyCollectionOptions } from "../src/local-only" import type { LocalOnlyCollectionUtils } from "../src/local-only" import type { Collection } from "../src/index" import type { Query } from "../src/query/builder" +import type { InsertConfig } from "../src/types" interface TestItem extends Record { id: number @@ -14,17 +15,10 @@ interface TestItem extends Record { describe(`LocalOnly Collection Types`, () => { it(`should have correct return type from localOnlyCollectionOptions`, () => { - const config = { + const options = localOnlyCollectionOptions({ id: `test-local-only`, getKey: (item: TestItem) => item.id, - } - - const options = localOnlyCollectionOptions< - TestItem, - never, - TestItem, - number - >(config) + }) // Test that options has the expected structure expectTypeOf(options).toHaveProperty(`sync`) @@ -35,7 +29,8 @@ describe(`LocalOnly Collection Types`, () => { expectTypeOf(options).toHaveProperty(`getKey`) // Test that getKey returns the correct type - expectTypeOf(options.getKey).toExtend<(item: TestItem) => number>() + expectTypeOf(options.getKey).parameter(0).toEqualTypeOf() + expectTypeOf(options.getKey).returns.toEqualTypeOf() }) it(`should be compatible with createCollection`, () => { @@ -142,119 +137,56 @@ describe(`LocalOnly Collection Types`, () => { id: z.string(), entityId: z.string(), value: z.string(), + createdAt: z.date().optional().default(new Date()), }) - const config = { - id: `test-with-schema`, - getKey: (item: any) => item.id, - schema: testSchema, - } + // We can trust that zod infers the correct types for the schema + type ExpectedType = z.infer + type ExpectedInput = z.input - const options = localOnlyCollectionOptions(config) - const collection = createCollection(options) - - // Test that the collection has the correct inferred type from schema - expectTypeOf(collection).toExtend< - Collection< - { - id: string - entityId: string - value: string + const collection = createCollection( + localOnlyCollectionOptions({ + id: `test-with-schema`, + getKey: (item: any) => item.id, + schema: testSchema, + onInsert: (params) => { + expectTypeOf( + params.transaction.mutations[0].modified + ).toEqualTypeOf() + return Promise.resolve() }, - string, - LocalOnlyCollectionUtils - > - >() - }) + onUpdate: (params) => { + expectTypeOf( + params.transaction.mutations[0].modified + ).toEqualTypeOf() + return Promise.resolve() + }, + onDelete: (params) => { + expectTypeOf( + params.transaction.mutations[0].modified + ).toEqualTypeOf() + return Promise.resolve() + }, + }) + ) - it(`should work with schema and query builder type inference (bug report reproduction)`, () => { - const testSchema = z.object({ - id: z.string(), - entityId: z.string(), - value: z.string(), - createdAt: z.date(), + collection.insert({ + id: '1', + entityId: '1', + value: '1', }) - const config = { - id: `test-with-schema-query`, - getKey: (item: any) => item.id, - schema: testSchema, - } - - const options = localOnlyCollectionOptions(config) - const collection = createCollection(options) - - // This should work without type errors - the query builder should infer the correct type - const query = (q: InstanceType) => - q - .from({ bookmark: collection }) - .orderBy(({ bookmark }) => bookmark.createdAt, `desc`) - - // Test that the collection has the correct inferred type from schema - expectTypeOf(collection).toExtend< - Collection< - { - id: string - entityId: string - value: string - createdAt: Date - }, - string, - LocalOnlyCollectionUtils - > + // Check that the insert method accepts the expected input type + expectTypeOf(collection.insert).parameters.toExtend< + [ExpectedInput | Array, InsertConfig?] >() - // Test that the query builder can access the createdAt property - expectTypeOf(query).toBeFunction() - }) - - it(`should reproduce exact bug report scenario`, () => { - // This reproduces the exact scenario from the bug report - const selectUrlSchema = z.object({ - id: z.string(), - url: z.string(), - title: z.string(), - createdAt: z.date(), + // Check that the update method accepts the expected input type + collection.update('1' ,(draft) => { + expectTypeOf(draft).toExtend() }) - const initialData = [ - { - id: `1`, - url: `https://example.com`, - title: `Example`, - createdAt: new Date(), - }, - ] - - const bookmarkCollection = createCollection( - localOnlyCollectionOptions({ - initialData, - getKey: (url: any) => url.id, - schema: selectUrlSchema, - }) - ) - - // This should work without type errors - the query builder should infer the correct type - const query = (q: InstanceType) => - q - .from({ bookmark: bookmarkCollection }) - .orderBy(({ bookmark }) => bookmark.createdAt, `desc`) - // Test that the collection has the correct inferred type from schema - expectTypeOf(bookmarkCollection).toExtend< - Collection< - { - id: string - url: string - title: string - createdAt: Date - }, - string, - LocalOnlyCollectionUtils - > - >() - - // Test that the query builder can access the createdAt property - expectTypeOf(query).toBeFunction() + expectTypeOf(collection.toArray).toExtend>() }) }) From df206c225c97c9b92355ef419ce9c50ee097da9c Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Thu, 21 Aug 2025 21:04:48 +0100 Subject: [PATCH 07/15] checkpoint --- packages/db/src/collection.ts | 159 +++++++------------------ packages/db/src/local-only.ts | 98 ++++----------- packages/db/src/types.ts | 57 +++++---- packages/db/tests/collection.test-d.ts | 131 +++----------------- packages/db/tests/local-only.test-d.ts | 61 ++-------- 5 files changed, 130 insertions(+), 376 deletions(-) diff --git a/packages/db/src/collection.ts b/packages/db/src/collection.ts index 0261ab6d9..9cc0e2660 100644 --- a/packages/db/src/collection.ts +++ b/packages/db/src/collection.ts @@ -38,7 +38,7 @@ import { } from "./errors" import { createFilteredCallback, currentStateAsChanges } from "./change-events" import type { Transaction } from "./transactions" -import type { StandardSchemaV1 } from "@standard-schema/spec" + import type { SingleRowRefProxy } from "./query/builder/ref-proxy" import type { ChangeListener, @@ -80,7 +80,7 @@ export interface Collection< T extends object = Record, TKey extends string | number = string | number, TUtils extends UtilsRecord = {}, - TSchema extends StandardSchemaV1 = StandardSchemaV1, + TSchema = never, TInput extends object = T, > extends CollectionImpl { readonly utils: TUtils @@ -160,105 +160,33 @@ export interface Collection< * // Note: You can provide an explicit type, a schema, or both. When both are provided, the explicit type takes precedence. */ -// Overload for when schema is provided - infers schema type -export function createCollection< - TSchema extends StandardSchemaV1, - TKey extends string | number = string | number, - TUtils extends UtilsRecord = {}, - TFallback extends object = Record, ->( - options: CollectionConfig< - ResolveType, - TKey, - TSchema, - ResolveInput - > & { - schema: TSchema - utils?: TUtils - } -): Collection< - ResolveType, - TKey, - TUtils, - TSchema, - ResolveInput -> - -// Overload for when explicit type is provided with schema - explicit type takes precedence -export function createCollection< - TExplicit extends object, - TKey extends string | number = string | number, - TUtils extends UtilsRecord = {}, - TSchema extends StandardSchemaV1 = StandardSchemaV1, - TFallback extends object = Record, ->( - options: CollectionConfig< - ResolveType, - TKey, - TSchema, - ResolveInput - > & { - schema: TSchema - utils?: TUtils - } -): Collection< - ResolveType, - TKey, - TUtils, - TSchema, - ResolveInput -> - -// Overload for when explicit type is provided or no schema -export function createCollection< - TExplicit = unknown, - TKey extends string | number = string | number, - TUtils extends UtilsRecord = {}, - TSchema extends StandardSchemaV1 = StandardSchemaV1, - TFallback extends object = Record, ->( - options: CollectionConfig< - ResolveType, - TKey, - TSchema, - ResolveInput - > & { utils?: TUtils } -): Collection< - ResolveType, - TKey, - TUtils, - TSchema, - ResolveInput -> - -// Implementation export function createCollection< - TExplicit = unknown, + TExplicit extends object = Record, TKey extends string | number = string | number, TUtils extends UtilsRecord = {}, - TSchema extends StandardSchemaV1 = StandardSchemaV1, - TFallback extends object = Record, + TSchema = never, >( options: CollectionConfig< - ResolveType, + TExplicit, TKey, TSchema, - ResolveInput + ResolveInput, + ResolveType > & { utils?: TUtils } ): Collection< - ResolveType, + ResolveType, TKey, TUtils, TSchema, - ResolveInput + ResolveInput > { const collection = new CollectionImpl< - ResolveType, + ResolveType, TKey, TUtils, TSchema, - ResolveInput - >(options) + ResolveInput + >(options as any) // Copy utils to both top level and .utils namespace if (options.utils) { @@ -268,11 +196,11 @@ export function createCollection< } return collection as Collection< - ResolveType, + ResolveType, TKey, TUtils, TSchema, - ResolveInput + ResolveInput > } @@ -280,10 +208,10 @@ export class CollectionImpl< T extends object = Record, TKey extends string | number = string | number, TUtils extends UtilsRecord = {}, - TSchema extends StandardSchemaV1 = StandardSchemaV1, - TInput extends object = T, + TSchema = never, + TInput extends object = Record, > { - public config: CollectionConfig + public config: CollectionConfig // Core state - make public for testing public transactions: SortedMap> @@ -502,7 +430,7 @@ export class CollectionImpl< this.config = { ...config, autoIndex: config.autoIndex ?? `eager`, - } + } as unknown as CollectionConfig // Set up data storage with optional comparison function if (this.config.compare) { @@ -1965,38 +1893,41 @@ export class CollectionImpl< */ // Overload 1: Update multiple items with a callback - update( - key: Array, - callback: (drafts: Array) => void - ): TransactionType + update( + keys: Array, + callback: (drafts: Array) => void + ): TransactionType // Overload 2: Update multiple items with config and a callback - update( + update( keys: Array, config: OperationConfig, - callback: (drafts: Array) => void - ): TransactionType + callback: (drafts: Array) => void + ): TransactionType // Overload 3: Update a single item with a callback - update( - id: TKey | unknown, - callback: (draft: TDraft) => void - ): TransactionType + update( + keys: TKey | unknown, + callback: (draft: TInput) => void + ): TransactionType // Overload 4: Update a single item with config and a callback - update( - id: TKey | unknown, + update( + keys: TKey | unknown, config: OperationConfig, - callback: (draft: TDraft) => void - ): TransactionType + callback: (draft: TInput) => void + ): TransactionType - update( + update( keys: (TKey | unknown) | Array, configOrCallback: - | ((draft: TDraft | Array) => void) + | ((draft: TInput) => void) + | ((drafts: Array) => void) | OperationConfig, - maybeCallback?: (draft: TDraft | Array) => void - ) { + maybeCallback?: + | ((draft: TInput) => void) + | ((drafts: Array) => void) + ): TransactionType { if (typeof keys === `undefined`) { throw new MissingUpdateArgumentError() } @@ -2030,19 +1961,19 @@ export class CollectionImpl< } return item - }) as unknown as Array + }) as unknown as Array let changesArray if (isArray) { // Use the proxy to track changes for all objects changesArray = withArrayChangeTracking( currentObjects, - callback as (draft: Array) => void + callback as (draft: Array) => void ) } else { const result = withChangeTracking( currentObjects[0]!, - callback as (draft: TDraft) => void + callback as (draft: TInput) => void ) changesArray = [result] } @@ -2114,7 +2045,7 @@ export class CollectionImpl< // If no changes were made, return an empty transaction early if (mutations.length === 0) { - const emptyTransaction = createTransaction({ + const emptyTransaction = createTransaction({ mutationFn: async () => {}, }) emptyTransaction.commit() @@ -2131,7 +2062,7 @@ export class CollectionImpl< this.scheduleTransactionCleanup(ambientTransaction) this.recomputeOptimisticState(true) - return ambientTransaction + return ambientTransaction as TransactionType } // No need to check for onUpdate handler here as we've already checked at the beginning diff --git a/packages/db/src/local-only.ts b/packages/db/src/local-only.ts index e04e42eae..ae183624f 100644 --- a/packages/db/src/local-only.ts +++ b/packages/db/src/local-only.ts @@ -1,4 +1,5 @@ import type { + BaseCollectionConfig, CollectionConfig, DeleteMutationFnParams, InsertMutationFnParams, @@ -27,62 +28,15 @@ import type { StandardSchemaV1 } from "@standard-schema/spec" * You should provide EITHER an explicit type OR a schema, but not both, as they would conflict. */ export interface LocalOnlyCollectionConfig< - TExplicit = unknown, - TSchema extends StandardSchemaV1 = never, - TFallback extends Record = Record, + TExplicit extends object = Record, TKey extends string | number = string | number, -> { - /** - * Standard Collection configuration properties - */ - id?: string - schema?: TSchema - getKey: (item: ResolveType) => TKey - + TSchema extends StandardSchemaV1 = StandardSchemaV1, +> extends BaseCollectionConfig { /** * Optional initial data to populate the collection with on creation * This data will be applied during the initial sync process */ - initialData?: Array> - - /** - * Optional asynchronous handler function called after an insert operation - * @param params Object containing transaction and collection information - * @returns Promise resolving to any value - */ - onInsert?: ( - params: InsertMutationFnParams< - ResolveType, - TKey, - LocalOnlyCollectionUtils - > - ) => Promise - - /** - * Optional asynchronous handler function called after an update operation - * @param params Object containing transaction and collection information - * @returns Promise resolving to any value - */ - onUpdate?: ( - params: UpdateMutationFnParams< - ResolveType, - TKey, - LocalOnlyCollectionUtils - > - ) => Promise - - /** - * Optional asynchronous handler function called after a delete operation - * @param params Object containing transaction and collection information - * @returns Promise resolving to any value - */ - onDelete?: ( - params: DeleteMutationFnParams< - ResolveType, - TKey, - LocalOnlyCollectionUtils - > - ) => Promise + initialData?: Array } /** @@ -137,34 +91,30 @@ export interface LocalOnlyCollectionUtils extends UtilsRecord {} * ) */ export function localOnlyCollectionOptions< - TExplicit = unknown, - TSchema extends StandardSchemaV1 = never, - TFallback extends Record = Record, + TExplicit extends object = Record, TKey extends string | number = string | number, + TSchema extends StandardSchemaV1 = StandardSchemaV1, >( - config: LocalOnlyCollectionConfig -): (TSchema extends never - ? CollectionConfig< - ResolveType, - TKey, - TSchema, - ResolveInput - > - : CollectionConfig< - ResolveType, - TKey, - TSchema, - ResolveInput - > & { schema: TSchema }) & { + config: LocalOnlyCollectionConfig< + ResolveType, + TKey, + TSchema + > +): CollectionConfig< + ResolveType, + TKey, + TSchema, + ResolveInput +> & { utils: LocalOnlyCollectionUtils } { - type TItem = ResolveType + type ResolvedType = ResolveType const { initialData, onInsert, onUpdate, onDelete, ...restConfig } = config // Create the sync configuration with transaction confirmation capability - const syncResult = createLocalOnlySync( - initialData as Array | undefined + const syncResult = createLocalOnlySync( + initialData as Array ) /** @@ -172,7 +122,7 @@ export function localOnlyCollectionOptions< * Wraps the user's onInsert handler to also confirm the transaction immediately */ const wrappedOnInsert = async ( - params: InsertMutationFnParams + params: InsertMutationFnParams ) => { // Call user handler first if provided let handlerResult @@ -190,7 +140,7 @@ export function localOnlyCollectionOptions< * Wrapper for onUpdate handler that also confirms the transaction immediately */ const wrappedOnUpdate = async ( - params: UpdateMutationFnParams + params: UpdateMutationFnParams ) => { // Call user handler first if provided let handlerResult @@ -208,7 +158,7 @@ export function localOnlyCollectionOptions< * Wrapper for onDelete handler that also confirms the transaction immediately */ const wrappedOnDelete = async ( - params: DeleteMutationFnParams + params: DeleteMutationFnParams ) => { // Call user handler first if provided let handlerResult diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index 10620a2f2..989ed3948 100644 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -40,16 +40,17 @@ export type InferSchemaInput = T extends StandardSchemaV1 * @internal This is used for collection insert and update type inference */ export type ResolveInput< - TExplicit = unknown, - TSchema extends StandardSchemaV1 = never, - TFallback extends object = Record, -> = unknown extends TExplicit - ? [TSchema] extends [never] - ? TFallback - : InferSchemaInput - : TExplicit extends object + TExplicit = Record, + TSchema = never, +> = [TSchema] extends [never] + ? TExplicit extends object ? TExplicit : Record + : TSchema extends StandardSchemaV1 + ? InferSchemaInput + : TExplicit extends object + ? TExplicit + : Record /** * Helper type to determine the final type based on priority: @@ -62,16 +63,17 @@ export type ResolveInput< * Users should not need to use this type directly, but understanding the priority order helps when defining collections. */ export type ResolveType< - TExplicit, - TSchema extends StandardSchemaV1 = never, - TFallback extends object = Record, -> = unknown extends TExplicit - ? [TSchema] extends [never] - ? TFallback - : InferSchemaOutput - : TExplicit extends object + TExplicit = Record, + TSchema = never, +> = [TSchema] extends [never] + ? TExplicit extends object ? TExplicit : Record + : TSchema extends StandardSchemaV1 + ? InferSchemaOutput + : TExplicit extends object + ? TExplicit + : Record export type TransactionState = `pending` | `persisting` | `completed` | `failed` @@ -350,13 +352,14 @@ export type CollectionStatus = export interface CollectionConfig< T extends object = Record, TKey extends string | number = string | number, - TSchema extends StandardSchemaV1 = StandardSchemaV1, - _TInput extends object = T, + TSchema = never, + _TInput extends object = ResolveInput, + TResolvedType extends object = ResolveType, > { // If an id isn't passed in, a UUID will be // generated for it. id?: string - sync: SyncConfig + sync: SyncConfig schema?: TSchema /** * Function to extract the ID from an object @@ -367,7 +370,7 @@ export interface CollectionConfig< * // For a collection with a 'uuid' field as the primary key * getKey: (item) => item.uuid */ - getKey: (item: T) => TKey + getKey: (item: TResolvedType) => TKey /** * Time in milliseconds after which the collection will be garbage collected * when it has no active subscribers. Defaults to 5 minutes (300000ms). @@ -397,7 +400,7 @@ export interface CollectionConfig< * // For a collection with a 'createdAt' field * compare: (x, y) => x.createdAt.getTime() - y.createdAt.getTime() */ - compare?: (x: T, y: T) => number + compare?: (x: TResolvedType, y: TResolvedType) => number /** * Optional asynchronous handler function called before an insert operation * @param params Object containing transaction and collection information @@ -439,7 +442,7 @@ export interface CollectionConfig< * }) * } */ - onInsert?: InsertMutationFn + onInsert?: InsertMutationFn /** * Optional asynchronous handler function called before an update operation @@ -483,7 +486,7 @@ export interface CollectionConfig< * } * } */ - onUpdate?: UpdateMutationFn + onUpdate?: UpdateMutationFn /** * Optional asynchronous handler function called before a delete operation * @param params Object containing transaction and collection information @@ -526,9 +529,15 @@ export interface CollectionConfig< * } * } */ - onDelete?: DeleteMutationFn + onDelete?: DeleteMutationFn } +export type BaseCollectionConfig< + T extends object = Record, + TKey extends string | number = string | number, + TSchema extends StandardSchemaV1 = StandardSchemaV1, +> = Omit, `sync`> + export type ChangesPayload> = Array< ChangeMessage > diff --git a/packages/db/tests/collection.test-d.ts b/packages/db/tests/collection.test-d.ts index cd37ee4e6..b4d9e92aa 100644 --- a/packages/db/tests/collection.test-d.ts +++ b/packages/db/tests/collection.test-d.ts @@ -1,14 +1,17 @@ import { assertType, describe, expectTypeOf, it } from "vitest" import { z } from "zod" import { createCollection } from "../src/collection" -import type { CollectionImpl } from "../src/collection" import type { OperationConfig, ResolveInput, ResolveType } from "../src/types" import type { StandardSchemaV1 } from "@standard-schema/spec" describe(`Collection.update type tests`, () => { type TypeTestItem = { id: string; value: number; optional?: boolean } - const updateMethod: CollectionImpl[`update`] = (() => {}) as any // Dummy assignment for type checking + const testCollection = createCollection({ + getKey: (item) => item.id, + sync: { sync: () => {} }, + }) + const updateMethod = testCollection.update it(`should correctly type drafts for multi-item update with callback (Overload 1)`, () => { updateMethod([`id1`, `id2`], (drafts) => { @@ -48,7 +51,6 @@ describe(`Collection.update type tests`, () => { describe(`Collection type resolution tests`, () => { // Define test types type ExplicitType = { id: string; explicit: boolean } - type FallbackType = { id: string; fallback: boolean } const testSchema = z.object({ id: z.string(), @@ -77,12 +79,7 @@ describe(`Collection type resolution tests`, () => { }) it(`should use schema type when explicit type is not provided`, () => { - const _collection = createCollection< - unknown, - string, - {}, - typeof testSchema - >({ + const _collection = createCollection({ getKey: (item) => { expectTypeOf(item).toEqualTypeOf() return item.id @@ -99,88 +96,10 @@ describe(`Collection type resolution tests`, () => { type Param = Parameters[0] expectTypeOf>().toEqualTypeOf() - type ExpectedType = ResolveType + type ExpectedType = ResolveType expectTypeOf().toEqualTypeOf() }) - it(`should use fallback type when neither explicit nor schema type is provided`, () => { - const _collection = createCollection< - unknown, - string, - {}, - never, - FallbackType - >({ - getKey: (item) => { - expectTypeOf(item).toEqualTypeOf() - return item.id - }, - sync: { sync: () => {} }, - }) - - expectTypeOf(_collection.toArray).toEqualTypeOf>() - - type Param = Parameters[0] - expectTypeOf>().toEqualTypeOf() - - type Key = Parameters[0] - expectTypeOf().toEqualTypeOf() - - type ExpectedType = ResolveType - expectTypeOf().toEqualTypeOf() - }) - - it(`should correctly resolve type with all three types provided`, () => { - // Explicit type should win - const _collection = createCollection< - ExplicitType, - string, - {}, - typeof testSchema, - FallbackType - >({ - getKey: (item) => { - expectTypeOf(item).toEqualTypeOf() - return item.id - }, - sync: { sync: () => {} }, - schema: testSchema, - }) - - type Param = Parameters[0] - expectTypeOf>().toEqualTypeOf() - - type Key = Parameters[0] - expectTypeOf().toEqualTypeOf() - - type ExpectedType = ResolveType< - ExplicitType, - typeof testSchema, - FallbackType - > - expectTypeOf().toEqualTypeOf() - }) - - it(`should automatically infer type from schema without generic arguments`, () => { - // This is the key test case that was missing - no generic arguments at all - const _collection = createCollection({ - getKey: (item) => { - expectTypeOf(item).toEqualTypeOf() - return item.id - }, - sync: { sync: () => {} }, - schema: testSchema, - }) - - expectTypeOf(_collection.toArray).toEqualTypeOf>() - - type Key = Parameters[0] - expectTypeOf().toEqualTypeOf() - - type Param = Parameters[0] - expectTypeOf>().toEqualTypeOf() - }) - it(`should automatically infer type from Zod schema with optional fields`, () => { // Test with a Zod schema that has optional fields const userSchema = z.object({ @@ -258,13 +177,11 @@ describe(`Schema Input/Output Type Distinction`, () => { it(`should handle schema with default values correctly for insert`, () => { type ExpectedOutputType = ResolveType< unknown, - typeof userSchemaWithDefaults, - Record + typeof userSchemaWithDefaults > type ExpectedInputType = ResolveInput< unknown, - typeof userSchemaWithDefaults, - Record + typeof userSchemaWithDefaults > const collection = createCollection({ @@ -312,16 +229,8 @@ describe(`Schema Input/Output Type Distinction`, () => { schema: userSchemaTransform, }) - type ExpectedInputType = ResolveInput< - unknown, - typeof userSchemaTransform, - Record - > - type ExpectedOutputType = ResolveType< - unknown, - typeof userSchemaTransform, - Record - > + type ExpectedInputType = ResolveInput + type ExpectedOutputType = ResolveType type InsertArg = Parameters[0] // Input type should be the raw input (before transformation) @@ -360,13 +269,11 @@ describe(`Schema Input/Output Type Distinction`, () => { type ExpectedOutputType = ResolveType< unknown, - typeof userSchemaWithDefaults, - Record + typeof userSchemaWithDefaults > type ExpectedInputType = ResolveInput< unknown, - typeof userSchemaWithDefaults, - Record + typeof userSchemaWithDefaults > // Input type should not include defaulted fields @@ -408,16 +315,8 @@ describe(`Schema Input/Output Type Distinction`, () => { schema: userSchemaTransform, }) - type ExpectedInputType = ResolveInput< - unknown, - typeof userSchemaTransform, - Record - > - type ExpectedOutputType = ResolveType< - unknown, - typeof userSchemaTransform, - Record - > + type ExpectedInputType = ResolveInput + type ExpectedOutputType = ResolveType // Input type should be the raw input (before transformation) expectTypeOf().toEqualTypeOf<{ diff --git a/packages/db/tests/local-only.test-d.ts b/packages/db/tests/local-only.test-d.ts index 58109595c..c4d1a80fe 100644 --- a/packages/db/tests/local-only.test-d.ts +++ b/packages/db/tests/local-only.test-d.ts @@ -4,7 +4,6 @@ import { createCollection } from "../src/index" import { localOnlyCollectionOptions } from "../src/local-only" import type { LocalOnlyCollectionUtils } from "../src/local-only" import type { Collection } from "../src/index" -import type { Query } from "../src/query/builder" import type { InsertConfig } from "../src/types" interface TestItem extends Record { @@ -34,23 +33,12 @@ describe(`LocalOnly Collection Types`, () => { }) it(`should be compatible with createCollection`, () => { - const config = { + const options = localOnlyCollectionOptions({ id: `test-local-only`, - getKey: (item: TestItem) => item.id, - } - - const options = localOnlyCollectionOptions< - TestItem, - never, - TestItem, - number - >(config) + getKey: (item) => item.id, + }) - const collection = createCollection< - TestItem, - number, - LocalOnlyCollectionUtils - >(options) + const collection = createCollection(options) // Test that the collection has the expected type expectTypeOf(collection).toExtend< @@ -67,12 +55,7 @@ describe(`LocalOnly Collection Types`, () => { onDelete: () => Promise.resolve({}), } - const options = localOnlyCollectionOptions< - TestItem, - never, - TestItem, - number - >(configWithCallbacks) + const options = localOnlyCollectionOptions(configWithCallbacks) const collection = createCollection< TestItem, number, @@ -91,17 +74,8 @@ describe(`LocalOnly Collection Types`, () => { initialData: [{ id: 1, name: `Test` }] as Array, } - const options = localOnlyCollectionOptions< - TestItem, - never, - TestItem, - number - >(configWithInitialData) - const collection = createCollection< - TestItem, - number, - LocalOnlyCollectionUtils - >(options) + const options = localOnlyCollectionOptions(configWithInitialData) + const collection = createCollection(options) expectTypeOf(collection).toExtend< Collection @@ -114,17 +88,8 @@ describe(`LocalOnly Collection Types`, () => { getKey: (item: TestItem) => `item-${item.id}`, } - const options = localOnlyCollectionOptions< - TestItem, - never, - TestItem, - string - >(config) - const collection = createCollection< - TestItem, - string, - LocalOnlyCollectionUtils - >(options) + const options = localOnlyCollectionOptions(config) + const collection = createCollection(options) expectTypeOf(collection).toExtend< Collection @@ -171,9 +136,9 @@ describe(`LocalOnly Collection Types`, () => { ) collection.insert({ - id: '1', - entityId: '1', - value: '1', + id: `1`, + entityId: `1`, + value: `1`, }) // Check that the insert method accepts the expected input type @@ -182,7 +147,7 @@ describe(`LocalOnly Collection Types`, () => { >() // Check that the update method accepts the expected input type - collection.update('1' ,(draft) => { + collection.update(`1`, (draft) => { expectTypeOf(draft).toExtend() }) From 0a06c55e4b5dc223cbe283972fe47a604c637709 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Thu, 21 Aug 2025 21:42:24 +0100 Subject: [PATCH 08/15] checkpoint --- packages/db/src/local-only.ts | 37 ++-- packages/db/src/local-storage.ts | 92 ++------ packages/db/src/types.ts | 2 +- packages/db/tests/local-only.test-d.ts | 73 +++++-- packages/db/tests/local-storage.test-d.ts | 247 +++++++++++++--------- 5 files changed, 239 insertions(+), 212 deletions(-) diff --git a/packages/db/src/local-only.ts b/packages/db/src/local-only.ts index ae183624f..9b7a83e5b 100644 --- a/packages/db/src/local-only.ts +++ b/packages/db/src/local-only.ts @@ -10,7 +10,7 @@ import type { UpdateMutationFnParams, UtilsRecord, } from "./types" -import type { StandardSchemaV1 } from "@standard-schema/spec" + /** * Configuration interface for Local-only collection options @@ -30,13 +30,13 @@ import type { StandardSchemaV1 } from "@standard-schema/spec" export interface LocalOnlyCollectionConfig< TExplicit extends object = Record, TKey extends string | number = string | number, - TSchema extends StandardSchemaV1 = StandardSchemaV1, + TSchema = never, > extends BaseCollectionConfig { /** * Optional initial data to populate the collection with on creation * This data will be applied during the initial sync process */ - initialData?: Array + initialData?: Array> } /** @@ -93,18 +93,15 @@ export interface LocalOnlyCollectionUtils extends UtilsRecord {} export function localOnlyCollectionOptions< TExplicit extends object = Record, TKey extends string | number = string | number, - TSchema extends StandardSchemaV1 = StandardSchemaV1, + TSchema = never, >( - config: LocalOnlyCollectionConfig< - ResolveType, - TKey, - TSchema - > + config: LocalOnlyCollectionConfig ): CollectionConfig< - ResolveType, + TExplicit, TKey, TSchema, - ResolveInput + ResolveInput, + ResolveType > & { utils: LocalOnlyCollectionUtils } { @@ -113,9 +110,7 @@ export function localOnlyCollectionOptions< const { initialData, onInsert, onUpdate, onDelete, ...restConfig } = config // Create the sync configuration with transaction confirmation capability - const syncResult = createLocalOnlySync( - initialData as Array - ) + const syncResult = createLocalOnlySync(initialData) /** * Create wrapper handlers that call user handlers first, then confirm transactions @@ -127,7 +122,7 @@ export function localOnlyCollectionOptions< // Call user handler first if provided let handlerResult if (onInsert) { - handlerResult = (await onInsert(params)) ?? {} + handlerResult = (await onInsert(params as any)) ?? {} } // Then synchronously confirm the transaction by looping through mutations @@ -145,7 +140,7 @@ export function localOnlyCollectionOptions< // Call user handler first if provided let handlerResult if (onUpdate) { - handlerResult = (await onUpdate(params)) ?? {} + handlerResult = (await onUpdate(params as any)) ?? {} } // Then synchronously confirm the transaction by looping through mutations @@ -163,7 +158,7 @@ export function localOnlyCollectionOptions< // Call user handler first if provided let handlerResult if (onDelete) { - handlerResult = (await onDelete(params)) ?? {} + handlerResult = (await onDelete(params as any)) ?? {} } // Then synchronously confirm the transaction by looping through mutations @@ -174,10 +169,10 @@ export function localOnlyCollectionOptions< return { ...restConfig, - sync: syncResult.sync, - onInsert: wrappedOnInsert, - onUpdate: wrappedOnUpdate, - onDelete: wrappedOnDelete, + sync: syncResult.sync as any, + onInsert: wrappedOnInsert as any, + onUpdate: wrappedOnUpdate as any, + onDelete: wrappedOnDelete as any, utils: {} as LocalOnlyCollectionUtils, startSync: true, gcTime: 0, diff --git a/packages/db/src/local-storage.ts b/packages/db/src/local-storage.ts index d400a5a67..c2e0b1cff 100644 --- a/packages/db/src/local-storage.ts +++ b/packages/db/src/local-storage.ts @@ -7,6 +7,7 @@ import { StorageKeyRequiredError, } from "./errors" import type { + BaseCollectionConfig, CollectionConfig, DeleteMutationFnParams, InsertMutationFnParams, @@ -16,7 +17,7 @@ import type { UpdateMutationFnParams, UtilsRecord, } from "./types" -import type { StandardSchemaV1 } from "@standard-schema/spec" + /** * Storage API interface - subset of DOM Storage that we need @@ -60,11 +61,10 @@ interface StoredItem { * You should provide EITHER an explicit type OR a schema, but not both, as they would conflict. */ export interface LocalStorageCollectionConfig< - TExplicit = unknown, - TSchema extends StandardSchemaV1 = never, - TFallback extends object = Record, + TExplicit extends object = Record, TKey extends string | number = string | number, -> { + TSchema = never, +> extends BaseCollectionConfig { /** * The key to use for storing the collection data in localStorage/sessionStorage */ @@ -81,60 +81,6 @@ export interface LocalStorageCollectionConfig< * Can be any object that implements addEventListener/removeEventListener for storage events */ storageEventApi?: StorageEventApi - - /** - * Collection identifier (defaults to "local-collection:{storageKey}" if not provided) - */ - id?: string - schema?: TSchema - getKey: CollectionConfig< - ResolveType, - TKey, - TSchema, - ResolveInput - >[`getKey`] - sync?: CollectionConfig< - ResolveType, - TKey, - TSchema, - ResolveInput - >[`sync`] - - /** - * Optional asynchronous handler function called before an insert operation - * @param params Object containing transaction and collection information - * @returns Promise resolving to any value - */ - onInsert?: ( - params: InsertMutationFnParams< - ResolveInput, - TKey - > - ) => Promise - - /** - * Optional asynchronous handler function called before an update operation - * @param params Object containing transaction and collection information - * @returns Promise resolving to any value - */ - onUpdate?: ( - params: UpdateMutationFnParams< - ResolveType, - TKey - > - ) => Promise - - /** - * Optional asynchronous handler function called before a delete operation - * @param params Object containing transaction and collection information - * @returns Promise resolving to any value - */ - onDelete?: ( - params: DeleteMutationFnParams< - ResolveType, - TKey - > - ) => Promise } /** @@ -224,25 +170,25 @@ function generateUuid(): string { * ) */ export function localStorageCollectionOptions< - TExplicit = unknown, - TSchema extends StandardSchemaV1 = never, - TFallback extends object = Record, + TExplicit extends object = Record, TKey extends string | number = string | number, + TSchema = never, >( - config: LocalStorageCollectionConfig + config: LocalStorageCollectionConfig ): Omit< CollectionConfig< - ResolveType, + TExplicit, TKey, TSchema, - ResolveInput + ResolveInput, + ResolveType >, `id` > & { id: string utils: LocalStorageCollectionUtils } { - type TItem = ResolveType + type TItem = ResolveType // Validate required parameters if (!config.storageKey) { @@ -343,7 +289,7 @@ export function localStorageCollectionOptions< // Call the user handler BEFORE persisting changes (if provided) let handlerResult: any = {} if (config.onInsert) { - handlerResult = (await config.onInsert(params)) ?? {} + handlerResult = (await config.onInsert(params as any)) ?? {} } // Always persist to storage @@ -380,7 +326,7 @@ export function localStorageCollectionOptions< // Call the user handler BEFORE persisting changes (if provided) let handlerResult: any = {} if (config.onUpdate) { - handlerResult = (await config.onUpdate(params)) ?? {} + handlerResult = (await config.onUpdate(params as any)) ?? {} } // Always persist to storage @@ -412,7 +358,7 @@ export function localStorageCollectionOptions< // Call the user handler BEFORE persisting changes (if provided) let handlerResult: any = {} if (config.onDelete) { - handlerResult = (await config.onDelete(params)) ?? {} + handlerResult = (await config.onDelete(params as any)) ?? {} } // Always persist to storage @@ -455,10 +401,10 @@ export function localStorageCollectionOptions< return { ...restConfig, id: collectionId, - sync, - onInsert: wrappedOnInsert, - onUpdate: wrappedOnUpdate, - onDelete: wrappedOnDelete, + sync: sync as any, + onInsert: wrappedOnInsert as any, + onUpdate: wrappedOnUpdate as any, + onDelete: wrappedOnDelete as any, utils: { clearStorage, getStorageSize, diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index 989ed3948..2c1bc645d 100644 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -535,7 +535,7 @@ export interface CollectionConfig< export type BaseCollectionConfig< T extends object = Record, TKey extends string | number = string | number, - TSchema extends StandardSchemaV1 = StandardSchemaV1, + TSchema = never, > = Omit, `sync`> export type ChangesPayload> = Array< diff --git a/packages/db/tests/local-only.test-d.ts b/packages/db/tests/local-only.test-d.ts index c4d1a80fe..cd42e2b25 100644 --- a/packages/db/tests/local-only.test-d.ts +++ b/packages/db/tests/local-only.test-d.ts @@ -3,8 +3,6 @@ import { z } from "zod" import { createCollection } from "../src/index" import { localOnlyCollectionOptions } from "../src/local-only" import type { LocalOnlyCollectionUtils } from "../src/local-only" -import type { Collection } from "../src/index" -import type { InsertConfig } from "../src/types" interface TestItem extends Record { id: number @@ -12,6 +10,8 @@ interface TestItem extends Record { completed?: boolean } +type ItemOf = T extends Array ? U : T + describe(`LocalOnly Collection Types`, () => { it(`should have correct return type from localOnlyCollectionOptions`, () => { const options = localOnlyCollectionOptions({ @@ -40,10 +40,21 @@ describe(`LocalOnly Collection Types`, () => { const collection = createCollection(options) - // Test that the collection has the expected type - expectTypeOf(collection).toExtend< - Collection - >() + // Test that the collection has the essential methods and properties + expectTypeOf(collection.insert).toBeFunction() + expectTypeOf(collection.update).toBeFunction() + expectTypeOf(collection.delete).toBeFunction() + expectTypeOf(collection.get).returns.toEqualTypeOf() + expectTypeOf(collection.toArray).toEqualTypeOf>() + + // Test insert parameter type + type InsertParam = Parameters[0] + expectTypeOf>().toEqualTypeOf() + + // Test update draft type + collection.update(1, (draft) => { + expectTypeOf(draft).toEqualTypeOf() + }) }) it(`should work with custom callbacks`, () => { @@ -62,9 +73,21 @@ describe(`LocalOnly Collection Types`, () => { LocalOnlyCollectionUtils >(options) - expectTypeOf(collection).toExtend< - Collection - >() + // Test that the collection has the essential methods and properties + expectTypeOf(collection.insert).toBeFunction() + expectTypeOf(collection.update).toBeFunction() + expectTypeOf(collection.delete).toBeFunction() + expectTypeOf(collection.get).returns.toEqualTypeOf() + expectTypeOf(collection.toArray).toEqualTypeOf>() + + // Test insert parameter type + type InsertParam2 = Parameters[0] + expectTypeOf>().toEqualTypeOf() + + // Test update draft type + collection.update(1, (draft) => { + expectTypeOf(draft).toEqualTypeOf() + }) }) it(`should work with initial data`, () => { @@ -77,9 +100,12 @@ describe(`LocalOnly Collection Types`, () => { const options = localOnlyCollectionOptions(configWithInitialData) const collection = createCollection(options) - expectTypeOf(collection).toExtend< - Collection - >() + // Test that the collection has the essential methods and properties + expectTypeOf(collection.insert).toBeFunction() + expectTypeOf(collection.update).toBeFunction() + expectTypeOf(collection.delete).toBeFunction() + expectTypeOf(collection.get).returns.toEqualTypeOf() + expectTypeOf(collection.toArray).toEqualTypeOf>() }) it(`should infer key type from getKey function`, () => { @@ -91,10 +117,13 @@ describe(`LocalOnly Collection Types`, () => { const options = localOnlyCollectionOptions(config) const collection = createCollection(options) - expectTypeOf(collection).toExtend< - Collection - >() - expectTypeOf(options.getKey).toExtend<(item: TestItem) => string>() + // Test that the collection has the essential methods and properties + expectTypeOf(collection.insert).toBeFunction() + expectTypeOf(collection.update).toBeFunction() + expectTypeOf(collection.delete).toBeFunction() + expectTypeOf(collection.get).returns.toEqualTypeOf() + expectTypeOf(collection.toArray).toEqualTypeOf>() + expectTypeOf(options.getKey).toBeFunction() }) it(`should work with schema and infer correct types`, () => { @@ -141,17 +170,17 @@ describe(`LocalOnly Collection Types`, () => { value: `1`, }) - // Check that the insert method accepts the expected input type - expectTypeOf(collection.insert).parameters.toExtend< - [ExpectedInput | Array, InsertConfig?] - >() + // Test insert parameter type + type InsertParam = Parameters[0] + type ItemOf = T extends Array ? U : T + expectTypeOf>().toEqualTypeOf() // Check that the update method accepts the expected input type collection.update(`1`, (draft) => { - expectTypeOf(draft).toExtend() + expectTypeOf(draft).toEqualTypeOf() }) // Test that the collection has the correct inferred type from schema - expectTypeOf(collection.toArray).toExtend>() + expectTypeOf(collection.toArray).toEqualTypeOf>() }) }) diff --git a/packages/db/tests/local-storage.test-d.ts b/packages/db/tests/local-storage.test-d.ts index 82be215da..7393778ad 100644 --- a/packages/db/tests/local-storage.test-d.ts +++ b/packages/db/tests/local-storage.test-d.ts @@ -2,19 +2,14 @@ import { describe, expectTypeOf, it } from "vitest" import { z } from "zod" import { createCollection } from "../src/index" import { localStorageCollectionOptions } from "../src/local-storage" -import type { Query } from "../src/query/builder" import type { LocalStorageCollectionConfig, StorageApi, StorageEventApi, } from "../src/local-storage" -import type { - CollectionConfig, - DeleteMutationFnParams, - InsertMutationFnParams, - ResolveType, - UpdateMutationFnParams, -} from "../src/types" +import type { ResolveType } from "../src/types" + +type ItemOf = T extends Array ? U : T describe(`LocalStorage collection type resolution tests`, () => { // Define test types @@ -60,6 +55,15 @@ describe(`LocalStorage collection type resolution tests`, () => { expectTypeOf(collection.size).toBeNumber() expectTypeOf(collection.utils.clearStorage).toBeFunction() expectTypeOf(collection.utils.getStorageSize).toBeFunction() + + // Test insert parameter type + type InsertParam = Parameters[0] + expectTypeOf>().toEqualTypeOf() + + // Test update draft type + collection.update("test-id", (draft) => { + expectTypeOf(draft).toEqualTypeOf() + }) }) it(`should prioritize explicit type in LocalStorageCollectionConfig`, () => { @@ -70,14 +74,22 @@ describe(`LocalStorage collection type resolution tests`, () => { getKey: (item) => item.id, }) - type ExpectedType = ResolveType< - ExplicitType, - never, - Record - > + type ExpectedType = ResolveType // The getKey function should have the resolved type expectTypeOf(options.getKey).parameters.toEqualTypeOf<[ExplicitType]>() expectTypeOf().toEqualTypeOf() + + // Test that the collection works with the options + const collection = createCollection(options) + + // Test insert parameter type + type InsertParam = Parameters[0] + expectTypeOf>().toEqualTypeOf() + + // Test update draft type + collection.update("test-id", (draft) => { + expectTypeOf(draft).toEqualTypeOf() + }) }) it(`should use schema type when explicit type is not provided`, () => { @@ -89,55 +101,76 @@ describe(`LocalStorage collection type resolution tests`, () => { getKey: (item) => item.id, }) - type ExpectedType = ResolveType< - unknown, - typeof testSchema, - Record - > + type ExpectedType = ResolveType // The getKey function should have the resolved type expectTypeOf(options.getKey).parameters.toEqualTypeOf<[SchemaType]>() expectTypeOf().toEqualTypeOf() + + // Test that the collection works with the options + const collection = createCollection(options) + + // Test insert parameter type + type InsertParam = Parameters[0] + expectTypeOf>().toEqualTypeOf() + + // Test update draft type + collection.update("test-id", (draft) => { + expectTypeOf(draft).toEqualTypeOf() + }) }) - it(`should use fallback type when neither explicit nor schema type is provided`, () => { - const config: LocalStorageCollectionConfig = { + it(`should use explicit type when provided`, () => { + const config: LocalStorageCollectionConfig = { storageKey: `test`, storage: mockStorage, storageEventApi: mockStorageEventApi, getKey: (item) => item.id, } - const options = localStorageCollectionOptions( - config - ) + const options = localStorageCollectionOptions(config) - type ExpectedType = ResolveType + type ExpectedType = ResolveType // The getKey function should have the resolved type expectTypeOf(options.getKey).parameters.toEqualTypeOf<[FallbackType]>() expectTypeOf().toEqualTypeOf() + + // Test that the collection works with the options + const collection = createCollection(options) + + // Test insert parameter type + type InsertParam = Parameters[0] + expectTypeOf>().toEqualTypeOf() + + // Test update draft type + collection.update("test-id", (draft) => { + expectTypeOf(draft).toEqualTypeOf() + }) }) - it(`should correctly resolve type with all three types provided`, () => { - const options = localStorageCollectionOptions< - ExplicitType, - typeof testSchema, - FallbackType - >({ + it(`should correctly resolve type with explicit type provided`, () => { + const options = localStorageCollectionOptions({ storageKey: `test`, storage: mockStorage, storageEventApi: mockStorageEventApi, - schema: testSchema, getKey: (item) => item.id, }) - type ExpectedType = ResolveType< - ExplicitType, - typeof testSchema, - FallbackType - > + type ExpectedType = ResolveType // The getKey function should have the resolved type (explicit type should win) expectTypeOf(options.getKey).parameters.toEqualTypeOf<[ExplicitType]>() expectTypeOf().toEqualTypeOf() + + // Test that the collection works with the options + const collection = createCollection(options) + + // Test insert parameter type + type InsertParam = Parameters[0] + expectTypeOf>().toEqualTypeOf() + + // Test update draft type + collection.update("test-id", (draft) => { + expectTypeOf(draft).toEqualTypeOf() + }) }) it(`should properly type the onInsert, onUpdate, and onDelete handlers`, () => { @@ -169,18 +202,18 @@ describe(`LocalStorage collection type resolution tests`, () => { }, }) - // Verify that the handlers are properly typed - expectTypeOf(options.onInsert).parameters.toEqualTypeOf< - [InsertMutationFnParams] - >() - - expectTypeOf(options.onUpdate).parameters.toEqualTypeOf< - [UpdateMutationFnParams] - >() - - expectTypeOf(options.onDelete).parameters.toEqualTypeOf< - [DeleteMutationFnParams] - >() + // Test that the collection works with the options + const collection = createCollection(options) + + // Test insert parameter type + type InsertParam = Parameters[0] + type ItemOf = T extends Array ? U : T + expectTypeOf>().toEqualTypeOf() + + // Test update draft type + collection.update("test-id", (draft) => { + expectTypeOf(draft).toEqualTypeOf() + }) }) it(`should properly type localStorage-specific configuration options`, () => { @@ -228,9 +261,7 @@ describe(`LocalStorage collection type resolution tests`, () => { }) // Verify sync has the correct type and optional getSyncMetadata - expectTypeOf(options.sync).toExtend< - CollectionConfig[`sync`] - >() + expectTypeOf(options.sync).toBeObject() if (options.sync.getSyncMetadata) { expectTypeOf(options.sync.getSyncMetadata).toBeFunction() @@ -281,68 +312,94 @@ describe(`LocalStorage collection type resolution tests`, () => { expectTypeOf(windowEventApi).toExtend() }) - it(`should work with schema and query builder type inference (bug report reproduction)`, () => { - const queryTestSchema = z.object({ + it(`should work with schema and infer correct types`, () => { + const testSchema = z.object({ id: z.string(), entityId: z.string(), value: z.string(), - createdAt: z.date(), + createdAt: z.date().optional().default(new Date()), }) - const config = { - storageKey: `test-with-schema-query`, - storage: mockStorage, - storageEventApi: mockStorageEventApi, - getKey: (item: any) => item.id, - schema: queryTestSchema, - } + // We can trust that zod infers the correct types for the schema + type ExpectedType = z.infer + type ExpectedInput = z.input + + const collection = createCollection( + localStorageCollectionOptions({ + storageKey: `test-with-schema`, + storage: mockStorage, + storageEventApi: mockStorageEventApi, + getKey: (item: any) => item.id, + schema: testSchema, + onInsert: (params) => { + expectTypeOf( + params.transaction.mutations[0].modified + ).toEqualTypeOf() + return Promise.resolve() + }, + onUpdate: (params) => { + expectTypeOf( + params.transaction.mutations[0].modified + ).toEqualTypeOf() + return Promise.resolve() + }, + onDelete: (params) => { + expectTypeOf( + params.transaction.mutations[0].modified + ).toEqualTypeOf() + return Promise.resolve() + }, + }) + ) - const options = localStorageCollectionOptions(config) - const collection = createCollection(options) + collection.insert({ + id: `1`, + entityId: `1`, + value: `1`, + }) - // This should work without type errors - the query builder should infer the correct type - const query = (q: InstanceType) => - q - .from({ bookmark: collection }) - .orderBy(({ bookmark }) => bookmark.createdAt, `desc`) + // Test insert parameter type + type InsertParam = Parameters[0] + expectTypeOf>().toEqualTypeOf() - // Test that the collection has the correct inferred type from schema - expectTypeOf(collection).toExtend() // Using any here since we don't have the exact Collection type imported + // Check that the update method accepts the expected input type + collection.update(`1`, (draft) => { + expectTypeOf(draft).toEqualTypeOf() + }) - // Test that the query builder can access the createdAt property - expectTypeOf(query).toBeFunction() + // Test that the collection has the correct inferred type from schema + expectTypeOf(collection.toArray).toEqualTypeOf>() }) - it(`should reproduce exact bug report scenario with localStorage`, () => { - // This reproduces the exact scenario from the bug report but with localStorage - const selectUrlSchema = z.object({ - id: z.string(), - url: z.string(), - title: z.string(), - createdAt: z.date(), - }) + it(`should work with explicit type for URL scenario`, () => { + type SelectUrlType = { + id: string + url: string + title: string + createdAt: Date + } - const config = { - storageKey: `test-with-schema`, + const options = localStorageCollectionOptions({ + storageKey: `test-with-url-type`, storage: mockStorage, storageEventApi: mockStorageEventApi, - getKey: (url: any) => url.id, - schema: selectUrlSchema, - } + getKey: (url) => url.id, + }) - const options = localStorageCollectionOptions(config) const collection = createCollection(options) - // This should work without type errors - the query builder should infer the correct type - const query = (q: InstanceType) => - q - .from({ bookmark: collection }) - .orderBy(({ bookmark }) => bookmark.createdAt, `desc`) + // Test that the collection has the expected methods + expectTypeOf(collection.insert).toBeFunction() + expectTypeOf(collection.get).returns.toEqualTypeOf() + expectTypeOf(collection.toArray).toEqualTypeOf>() - // Test that the collection has the correct inferred type from schema - expectTypeOf(collection).toExtend() // Using any here since we don't have the exact Collection type imported + // Test insert parameter type + type InsertParam = Parameters[0] + expectTypeOf>().toEqualTypeOf() - // Test that the query builder can access the createdAt property - expectTypeOf(query).toBeFunction() + // Test update draft type + collection.update("test-id", (draft) => { + expectTypeOf(draft).toEqualTypeOf() + }) }) }) From f19189181d7d35b79e3939b92562f88dce3dc7c8 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Thu, 21 Aug 2025 22:01:33 +0100 Subject: [PATCH 09/15] fix other tests --- packages/db/src/query/builder/types.ts | 6 ++- packages/db/src/query/compiler/index.ts | 9 ++-- packages/db/src/query/compiler/joins.ts | 6 +-- packages/db/src/query/compiler/order-by.ts | 2 +- packages/db/src/query/ir.ts | 2 +- .../db/src/query/live-query-collection.ts | 44 ++++++++++++++++--- packages/db/tests/collection-getters.test.ts | 10 +++-- packages/db/tests/collection-schema.test.ts | 4 +- packages/db/tests/utls.ts | 4 +- 9 files changed, 63 insertions(+), 24 deletions(-) diff --git a/packages/db/src/query/builder/types.ts b/packages/db/src/query/builder/types.ts index 763447d88..0a5b5526a 100644 --- a/packages/db/src/query/builder/types.ts +++ b/packages/db/src/query/builder/types.ts @@ -24,14 +24,16 @@ export interface Context { export type ContextSchema = Record export type Source = { - [alias: string]: CollectionImpl | QueryBuilder + [alias: string]: + | CollectionImpl + | QueryBuilder } // Helper type to infer collection type from CollectionImpl // This uses ResolveType directly to ensure consistency with collection creation logic export type InferCollectionType = T extends CollectionImpl - ? ResolveType + ? ResolveType : never // Helper type to create schema from source diff --git a/packages/db/src/query/compiler/index.ts b/packages/db/src/query/compiler/index.ts index 0ee3bc167..ead714cfb 100644 --- a/packages/db/src/query/compiler/index.ts +++ b/packages/db/src/query/compiler/index.ts @@ -318,7 +318,7 @@ export function compileQuery( function processFrom( from: CollectionRef | QueryRef, allInputs: Record, - collections: Record, + collections: Record>, callbacks: Record, lazyCollections: Set, optimizableOrderByCollections: Record, @@ -445,8 +445,11 @@ function getRefFromAlias( export function followRef( query: QueryIR, ref: PropRef, - collection: Collection -): { collection: Collection; path: Array } | void { + collection: Collection +): { + collection: Collection + path: Array +} | void { if (ref.path.length === 0) { return } diff --git a/packages/db/src/query/compiler/joins.ts b/packages/db/src/query/compiler/joins.ts index c8ab79e1c..2ad9f1ef3 100644 --- a/packages/db/src/query/compiler/joins.ts +++ b/packages/db/src/query/compiler/joins.ts @@ -55,7 +55,7 @@ export function processJoins( allInputs: Record, cache: QueryCache, queryMapping: QueryMapping, - collections: Record, + collections: Record>, callbacks: Record, lazyCollections: Set, optimizableOrderByCollections: Record, @@ -96,7 +96,7 @@ function processJoin( allInputs: Record, cache: QueryCache, queryMapping: QueryMapping, - collections: Record, + collections: Record>, callbacks: Record, lazyCollections: Set, optimizableOrderByCollections: Record, @@ -367,7 +367,7 @@ function getTableAliasFromExpression(expr: BasicExpression): string | null { function processJoinSource( from: CollectionRef | QueryRef, allInputs: Record, - collections: Record, + collections: Record>, callbacks: Record, lazyCollections: Set, optimizableOrderByCollections: Record, diff --git a/packages/db/src/query/compiler/order-by.ts b/packages/db/src/query/compiler/order-by.ts index 5e3faa749..e66ae55d1 100644 --- a/packages/db/src/query/compiler/order-by.ts +++ b/packages/db/src/query/compiler/order-by.ts @@ -33,7 +33,7 @@ export function processOrderBy( rawQuery: QueryIR, pipeline: NamespacedAndKeyedStream, orderByClause: Array, - collection: Collection, + collection: Collection, optimizableOrderByCollections: Record, limit?: number, offset?: number diff --git a/packages/db/src/query/ir.ts b/packages/db/src/query/ir.ts index d015ff4b9..a6505635a 100644 --- a/packages/db/src/query/ir.ts +++ b/packages/db/src/query/ir.ts @@ -69,7 +69,7 @@ abstract class BaseExpression { export class CollectionRef extends BaseExpression { public type = `collectionRef` as const constructor( - public collection: CollectionImpl, + public collection: CollectionImpl, public alias: string ) { super() diff --git a/packages/db/src/query/live-query-collection.ts b/packages/db/src/query/live-query-collection.ts index 8d0337ab9..7acd45e2e 100644 --- a/packages/db/src/query/live-query-collection.ts +++ b/packages/db/src/query/live-query-collection.ts @@ -73,14 +73,38 @@ export interface LiveQueryCollectionConfig< /** * Optional schema for validation */ - schema?: CollectionConfig[`schema`] + schema?: CollectionConfig< + TResult, + string | number, + never, + TResult, + TResult + >[`schema`] /** * Optional mutation handlers */ - onInsert?: CollectionConfig[`onInsert`] - onUpdate?: CollectionConfig[`onUpdate`] - onDelete?: CollectionConfig[`onDelete`] + onInsert?: CollectionConfig< + TResult, + string | number, + never, + TResult, + TResult + >[`onInsert`] + onUpdate?: CollectionConfig< + TResult, + string | number, + never, + TResult, + TResult + >[`onUpdate`] + onDelete?: CollectionConfig< + TResult, + string | number, + never, + TResult, + TResult + >[`onDelete`] /** * Start sync / the query immediately @@ -122,7 +146,7 @@ export function liveQueryCollectionOptions< TResult extends object = GetResult, >( config: LiveQueryCollectionConfig -): CollectionConfig { +): CollectionConfig { // Generate a unique ID if not provided const id = config.id || `live-query-${++liveQueryCollectionCounter}` @@ -232,7 +256,7 @@ export function liveQueryCollectionOptions< compileBasePipeline() // Create the sync configuration - const sync: SyncConfig = { + const sync: SyncConfig = { rowUpdateMode: `full`, sync: ({ begin, write, commit, markReady, collection: theCollection }) => { const { graph, inputs, pipeline } = maybeCompileBasePipeline() @@ -745,7 +769,13 @@ function bridgeToCreateCollection< TResult extends object, TUtils extends UtilsRecord = {}, >( - options: CollectionConfig & { utils?: TUtils } + options: CollectionConfig< + TResult, + string | number, + never, + TResult, + TResult + > & { utils?: TUtils } ): Collection { // This is the only place we need a type assertion, hidden from user API return createCollection(options as any) as unknown as Collection< diff --git a/packages/db/tests/collection-getters.test.ts b/packages/db/tests/collection-getters.test.ts index a0b37ee7e..61b04d268 100644 --- a/packages/db/tests/collection-getters.test.ts +++ b/packages/db/tests/collection-getters.test.ts @@ -5,16 +5,18 @@ import type { CollectionImpl } from "../src/collection" import type { ChangeMessage, CollectionConfig } from "../src/types" describe(`Collection getters`, () => { - let collection: CollectionImpl + let collection: CollectionImpl let mockSync: { sync: (params: { - collection: CollectionImpl + collection: CollectionImpl begin: () => void - write: (message: ChangeMessage) => void + write: (message: Omit) => void commit: () => void + markReady: () => void + truncate: () => void }) => void } - let config: CollectionConfig + let config: CollectionConfig beforeEach(() => { mockSync = { diff --git a/packages/db/tests/collection-schema.test.ts b/packages/db/tests/collection-schema.test.ts index 159d6c70e..9e8ffb397 100644 --- a/packages/db/tests/collection-schema.test.ts +++ b/packages/db/tests/collection-schema.test.ts @@ -280,7 +280,7 @@ describe(`Collection with schema validation`, () => { }) // Create a collection with the schema - const collection = createCollection({ + const collection = createCollection({ id: `test`, getKey: (item) => item.name, startSync: true, @@ -373,7 +373,7 @@ describe(`Collection with schema validation`, () => { }) // Create a collection with the schema - const collection = createCollection>({ + const collection = createCollection({ id: `test`, getKey: (item) => item.name, startSync: true, diff --git a/packages/db/tests/utls.ts b/packages/db/tests/utls.ts index 9ab74d3a5..893e0ce68 100644 --- a/packages/db/tests/utls.ts +++ b/packages/db/tests/utls.ts @@ -218,7 +218,9 @@ export function mockSyncCollectionOptions< }, } - const options: CollectionConfig & { utils: typeof utils } = { + const options: CollectionConfig & { + utils: typeof utils + } = { sync: { sync: (params: Parameters[`sync`]>[0]) => { begin = params.begin From 5450c478ad67dcd8c73e36b63729b68cb614c3da Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Thu, 4 Sep 2025 18:30:58 +0100 Subject: [PATCH 10/15] test passing! --- packages/db/src/query/compiler/index.ts | 4 ++-- packages/db/src/query/compiler/joins.ts | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/db/src/query/compiler/index.ts b/packages/db/src/query/compiler/index.ts index b278f4398..a413c23ca 100644 --- a/packages/db/src/query/compiler/index.ts +++ b/packages/db/src/query/compiler/index.ts @@ -52,7 +52,7 @@ export interface CompilationResult { export function compileQuery( rawQuery: QueryIR, inputs: Record, - collections: Record>, + collections: Record>, callbacks: Record, lazyCollections: Set, optimizableOrderByCollections: Record, @@ -320,7 +320,7 @@ export function compileQuery( function processFrom( from: CollectionRef | QueryRef, allInputs: Record, - collections: Record>, + collections: Record>, callbacks: Record, lazyCollections: Set, optimizableOrderByCollections: Record, diff --git a/packages/db/src/query/compiler/joins.ts b/packages/db/src/query/compiler/joins.ts index 2ad9f1ef3..009fa7620 100644 --- a/packages/db/src/query/compiler/joins.ts +++ b/packages/db/src/query/compiler/joins.ts @@ -55,7 +55,7 @@ export function processJoins( allInputs: Record, cache: QueryCache, queryMapping: QueryMapping, - collections: Record>, + collections: Record>, callbacks: Record, lazyCollections: Set, optimizableOrderByCollections: Record, @@ -96,7 +96,7 @@ function processJoin( allInputs: Record, cache: QueryCache, queryMapping: QueryMapping, - collections: Record>, + collections: Record>, callbacks: Record, lazyCollections: Set, optimizableOrderByCollections: Record, @@ -367,7 +367,7 @@ function getTableAliasFromExpression(expr: BasicExpression): string | null { function processJoinSource( from: CollectionRef | QueryRef, allInputs: Record, - collections: Record>, + collections: Record>, callbacks: Record, lazyCollections: Set, optimizableOrderByCollections: Record, From 1c5f3761496249055d324b851d00d76b0773b6d5 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Thu, 4 Sep 2025 20:44:36 +0100 Subject: [PATCH 11/15] main db colleciton types --- packages/db/src/change-events.ts | 21 +++--- packages/db/src/collection.ts | 4 +- packages/db/src/local-only.ts | 5 +- packages/db/src/local-storage.ts | 7 +- packages/db/src/query/compiler/index.ts | 2 +- packages/db/src/query/compiler/joins.ts | 8 +- .../db/src/query/live-query-collection.ts | 71 ++++++++++++------ .../query/live/collection-config-builder.ts | 75 ++++++++++--------- .../src/query/live/collection-subscriber.ts | 51 +++++++------ packages/db/src/query/live/types.ts | 68 +++-------------- packages/db/tests/local-only.test-d.ts | 1 - packages/db/tests/local-storage.test-d.ts | 37 ++++----- packages/db/tests/query/order-by.test.ts | 21 +++--- 13 files changed, 183 insertions(+), 188 deletions(-) diff --git a/packages/db/src/change-events.ts b/packages/db/src/change-events.ts index d06b8a933..2326d91b6 100644 --- a/packages/db/src/change-events.ts +++ b/packages/db/src/change-events.ts @@ -47,12 +47,12 @@ export function currentStateAsChanges< >( collection: CollectionLike, options: CurrentStateAsChangesOptions = {} -): Array> { +): Array> { // Helper function to collect filtered results const collectFilteredResults = ( filterFn?: (value: T) => boolean - ): Array> => { - const result: Array> = [] + ): Array> => { + const result: Array> = [] for (const [key, value] of collection.entries()) { // If no filter function is provided, include all items if (filterFn?.(value) ?? true) { @@ -100,7 +100,7 @@ export function currentStateAsChanges< if (optimizationResult.canOptimize) { // Use index optimization - const result: Array> = [] + const result: Array> = [] for (const key of optimizationResult.matchingKeys) { const value = collection.get(key) if (value !== undefined) { @@ -199,16 +199,19 @@ export function createFilterFunctionFromExpression( * @param options - The subscription options containing the where clause * @returns A filtered callback function */ -export function createFilteredCallback( - originalCallback: (changes: Array>) => void, +export function createFilteredCallback< + T extends object, + TKey extends string | number, +>( + originalCallback: (changes: Array>) => void, options: SubscribeChangesOptions -): (changes: Array>) => void { +): (changes: Array>) => void { const filterFn = options.whereExpression ? createFilterFunctionFromExpression(options.whereExpression) : createFilterFunction(options.where!) - return (changes: Array>) => { - const filteredChanges: Array> = [] + return (changes: Array>) => { + const filteredChanges: Array> = [] for (const change of changes) { if (change.type === `insert`) { diff --git a/packages/db/src/collection.ts b/packages/db/src/collection.ts index 6c8e0a6ce..531784c1c 100644 --- a/packages/db/src/collection.ts +++ b/packages/db/src/collection.ts @@ -2311,7 +2311,7 @@ export class CollectionImpl< */ public currentStateAsChanges( options: CurrentStateAsChangesOptions = {} - ): Array> { + ): Array> { return currentStateAsChanges(this, options) } @@ -2355,7 +2355,7 @@ export class CollectionImpl< * }) */ public subscribeChanges( - callback: (changes: Array>) => void, + callback: (changes: Array>) => void, options: SubscribeChangesOptions = {} ): () => void { // Start sync and track subscriber diff --git a/packages/db/src/local-only.ts b/packages/db/src/local-only.ts index 9b7a83e5b..f6ee1e510 100644 --- a/packages/db/src/local-only.ts +++ b/packages/db/src/local-only.ts @@ -11,12 +11,10 @@ import type { UtilsRecord, } from "./types" - /** * Configuration interface for Local-only collection options * @template TExplicit - The explicit type of items in the collection (highest priority) * @template TSchema - The schema type for validation and type inference (second priority) - * @template TFallback - The fallback type if no explicit or schema type is provided * @template TKey - The type of the key returned by getKey * * @remarks @@ -53,10 +51,9 @@ export interface LocalOnlyCollectionUtils extends UtilsRecord {} * * @template TExplicit - The explicit type of items in the collection (highest priority) * @template TSchema - The schema type for validation and type inference (second priority) - * @template TFallback - The fallback type if no explicit or schema type is provided * @template TKey - The type of the key returned by getKey * @param config - Configuration options for the Local-only collection - * @returns Collection options with utilities (currently empty but follows the pattern) + * @returns Collection options with utilities * * @example * // Basic local-only collection diff --git a/packages/db/src/local-storage.ts b/packages/db/src/local-storage.ts index c2e0b1cff..7fa1034e7 100644 --- a/packages/db/src/local-storage.ts +++ b/packages/db/src/local-storage.ts @@ -18,7 +18,6 @@ import type { UtilsRecord, } from "./types" - /** * Storage API interface - subset of DOM Storage that we need */ @@ -50,7 +49,7 @@ interface StoredItem { * Configuration interface for localStorage collection options * @template TExplicit - The explicit type of items in the collection (highest priority) * @template TSchema - The schema type for validation and type inference (second priority) - * @template TFallback - The fallback type if no explicit or schema type is provided + * @template TKey - The type of the key returned by getKey * * @remarks * Type resolution follows a priority order: @@ -134,8 +133,8 @@ function generateUuid(): string { * * @template TExplicit - The explicit type of items in the collection (highest priority) * @template TSchema - The schema type for validation and type inference (second priority) - * @template TFallback - The fallback type if no explicit or schema type is provided - * @param config - Configuration options for the localStorage collection + * @template TKey - The type of the key returned by getKey + * @param config - Configuration options for the Local-storage collection * @returns Collection options with utilities including clearStorage and getStorageSize * * @example diff --git a/packages/db/src/query/compiler/index.ts b/packages/db/src/query/compiler/index.ts index a413c23ca..6a6a74e4f 100644 --- a/packages/db/src/query/compiler/index.ts +++ b/packages/db/src/query/compiler/index.ts @@ -53,7 +53,7 @@ export function compileQuery( rawQuery: QueryIR, inputs: Record, collections: Record>, - callbacks: Record, + callbacks: Record>, lazyCollections: Set, optimizableOrderByCollections: Record, cache: QueryCache = new WeakMap(), diff --git a/packages/db/src/query/compiler/joins.ts b/packages/db/src/query/compiler/joins.ts index 009fa7620..ae25beae8 100644 --- a/packages/db/src/query/compiler/joins.ts +++ b/packages/db/src/query/compiler/joins.ts @@ -37,9 +37,11 @@ import type { import type { QueryCache, QueryMapping } from "./types.js" import type { BaseIndex } from "../../indexes/base-index.js" -export type LoadKeysFn = (key: Set) => void -export type LazyCollectionCallbacks = { - loadKeys: LoadKeysFn +export type LoadKeysFn = (key: Set) => void +export type LazyCollectionCallbacks< + TKey extends string | number = string | number, +> = { + loadKeys: LoadKeysFn loadInitialState: () => void } diff --git a/packages/db/src/query/live-query-collection.ts b/packages/db/src/query/live-query-collection.ts index 80e58d1ee..7a9be8f01 100644 --- a/packages/db/src/query/live-query-collection.ts +++ b/packages/db/src/query/live-query-collection.ts @@ -33,12 +33,16 @@ import type { Context, GetResult } from "./builder/types.js" export function liveQueryCollectionOptions< TContext extends Context, TResult extends object = GetResult, + TKey extends string | number = string | number, + TSchema = never, >( - config: LiveQueryCollectionConfig -): CollectionConfig { + config: LiveQueryCollectionConfig +): CollectionConfig { const collectionConfigBuilder = new CollectionConfigBuilder< TContext, - TResult + TResult, + TKey, + TSchema >(config) return collectionConfigBuilder.getConfig() } @@ -89,38 +93,59 @@ export function createLiveQueryCollection< export function createLiveQueryCollection< TContext extends Context, TResult extends object = GetResult, + TKey extends string | number = string | number, + TSchema = never, TUtils extends UtilsRecord = {}, >( - config: LiveQueryCollectionConfig & { utils?: TUtils } + config: LiveQueryCollectionConfig & { + utils?: TUtils + } ): Collection // Implementation export function createLiveQueryCollection< TContext extends Context, TResult extends object = GetResult, + TKey extends string | number = string | number, + TSchema = never, TUtils extends UtilsRecord = {}, >( configOrQuery: - | (LiveQueryCollectionConfig & { utils?: TUtils }) + | (LiveQueryCollectionConfig & { + utils?: TUtils + }) | ((q: InitialQueryBuilder) => QueryBuilder) -): Collection { +): Collection { // Determine if the argument is a function (query) or a config object if (typeof configOrQuery === `function`) { // Simple query function case - const config: LiveQueryCollectionConfig = { - query: configOrQuery as ( - q: InitialQueryBuilder - ) => QueryBuilder, - } - const options = liveQueryCollectionOptions(config) + const config: LiveQueryCollectionConfig = + { + query: configOrQuery as ( + q: InitialQueryBuilder + ) => QueryBuilder, + } + const options = liveQueryCollectionOptions< + TContext, + TResult, + TKey, + TSchema + >(config) return bridgeToCreateCollection(options) } else { // Config object case const config = configOrQuery as LiveQueryCollectionConfig< TContext, - TResult + TResult, + TKey, + TSchema > & { utils?: TUtils } - const options = liveQueryCollectionOptions(config) + const options = liveQueryCollectionOptions< + TContext, + TResult, + TKey, + TSchema + >(config) return bridgeToCreateCollection({ ...options, utils: config.utils, @@ -134,20 +159,18 @@ export function createLiveQueryCollection< */ function bridgeToCreateCollection< TResult extends object, + TKey extends string | number = string | number, + TSchema = never, TUtils extends UtilsRecord = {}, >( - options: CollectionConfig< - TResult, - string | number, - never, - TResult, - TResult - > & { utils?: TUtils } -): Collection { + options: CollectionConfig & { utils?: TUtils } +): Collection { // This is the only place we need a type assertion, hidden from user API return createCollection(options as any) as unknown as Collection< TResult, - string | number, - TUtils + TKey, + TUtils, + TSchema, + TResult > } diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index e9f9ffe32..a6e181a9b 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -8,6 +8,7 @@ import type { Collection } from "../../collection.js" import type { CollectionConfig, KeyedStream, + ResolveType, ResultStream, SyncConfig, } from "../../types.js" @@ -27,6 +28,8 @@ let liveQueryCollectionCounter = 0 export class CollectionConfigBuilder< TContext extends Context, TResult extends object = GetResult, + TKey extends string | number = string | number, + TSchema = never, > { private readonly id: string readonly query: QueryIR @@ -39,7 +42,10 @@ export class CollectionConfigBuilder< // WeakMap to store the orderBy index for each result private readonly orderByIndices = new WeakMap() - private readonly compare?: (val1: TResult, val2: TResult) => number + private readonly compare?: ( + val1: ResolveType, + val2: ResolveType + ) => number private graphCache: D2 | undefined private inputsCache: Record> | undefined @@ -49,14 +55,19 @@ export class CollectionConfigBuilder< | undefined // Map of collection IDs to functions that load keys for that lazy collection - lazyCollectionsCallbacks: Record = {} + lazyCollectionsCallbacks: Record> = {} // Set of collection IDs that are lazy collections readonly lazyCollections = new Set() // Set of collection IDs that include an optimizable ORDER BY clause optimizableOrderByCollections: Record = {} constructor( - private readonly config: LiveQueryCollectionConfig + private readonly config: LiveQueryCollectionConfig< + TContext, + TResult, + TKey, + TSchema + > ) { // Generate a unique ID if not provided this.id = config.id || `live-query-${++liveQueryCollectionCounter}` @@ -66,7 +77,9 @@ export class CollectionConfigBuilder< // Create compare function for ordering if the query has orderBy if (this.query.orderBy && this.query.orderBy.length > 0) { - this.compare = createOrderByComparator(this.orderByIndices) + this.compare = createOrderByComparator>( + this.orderByIndices + ) } // Compile the base pipeline once initially @@ -74,26 +87,18 @@ export class CollectionConfigBuilder< this.compileBasePipeline() } - getConfig(): CollectionConfig< - TResult, - string | number, - never, - TResult, - TResult - > { + getConfig(): CollectionConfig { + const { getKey, ...rest } = this.config return { id: this.id, - getKey: - this.config.getKey || - ((item) => this.resultKeys.get(item) as string | number), - sync: this.getSyncConfig(), + getKey: getKey || ((item) => this.resultKeys.get(item) as TKey), + sync: this.getSyncConfig() as SyncConfig< + ResolveType, + TKey + >, compare: this.compare, - gcTime: this.config.gcTime || 5000, // 5 seconds by default for live queries - schema: this.config.schema, - onInsert: this.config.onInsert, - onUpdate: this.config.onUpdate, - onDelete: this.config.onDelete, - startSync: this.config.startSync, + gcTime: this.config.gcTime ?? 5000, // 5 seconds by default for live queries + ...rest, } } @@ -106,7 +111,7 @@ export class CollectionConfigBuilder< // So this callback would notice that it doesn't have enough rows and load some more. // The callback returns a boolean, when it's true it's done loading data and we can mark the collection as ready. maybeRunGraph( - config: Parameters[`sync`]>[0], + config: Parameters[`sync`]>[0], syncState: FullSyncState, callback?: () => boolean ) { @@ -132,16 +137,14 @@ export class CollectionConfigBuilder< } } - private getSyncConfig(): SyncConfig { + private getSyncConfig(): SyncConfig { return { rowUpdateMode: `full`, sync: this.syncFn.bind(this), } } - private syncFn( - config: Parameters[`sync`]>[0] - ) { + private syncFn(config: Parameters[`sync`]>[0]) { const syncState: SyncState = { messagesCount: 0, subscribedToAllCollections: false, @@ -218,7 +221,7 @@ export class CollectionConfigBuilder< } private extendPipelineWithChangeProcessing( - config: Parameters[`sync`]>[0], + config: Parameters[`sync`]>[0], syncState: SyncState ): FullSyncState { const { begin, commit } = config @@ -251,7 +254,7 @@ export class CollectionConfigBuilder< } private applyChanges( - config: Parameters[`sync`]>[0], + config: Parameters[`sync`]>[0], changes: { deletes: number inserts: number @@ -283,7 +286,7 @@ export class CollectionConfigBuilder< inserts > deletes || // Just update(s) but the item is already in the collection (so // was inserted previously). - (inserts === deletes && collection.has(key as string | number)) + (inserts === deletes && collection.has(key as TKey)) ) { write({ value, @@ -316,17 +319,21 @@ export class CollectionConfigBuilder< } private subscribeToAllCollections( - config: Parameters[`sync`]>[0], + config: Parameters[`sync`]>[0], syncState: FullSyncState ) { const loaders = Object.entries(this.collections).map( ([collectionId, collection]) => { - const collectionSubscriber = new CollectionSubscriber( + const collectionSubscriber = new CollectionSubscriber< + TContext, + TResult, + TKey + >( collectionId, collection, config, syncState, - this + this as unknown as CollectionConfigBuilder ) collectionSubscriber.subscribe() @@ -349,8 +356,8 @@ export class CollectionConfigBuilder< } } -function buildQueryFromConfig( - config: LiveQueryCollectionConfig +function buildQueryFromConfig( + config: LiveQueryCollectionConfig ) { // Build the query using the provided query builder function or instance if (typeof config.query === `function`) { diff --git a/packages/db/src/query/live/collection-subscriber.ts b/packages/db/src/query/live/collection-subscriber.ts index 38e5ceee0..2a69d458f 100644 --- a/packages/db/src/query/live/collection-subscriber.ts +++ b/packages/db/src/query/live/collection-subscriber.ts @@ -12,19 +12,24 @@ import type { CollectionConfigBuilder } from "./collection-config-builder.js" export class CollectionSubscriber< TContext extends Context, TResult extends object = GetResult, + TKey extends string | number = string | number, > { // Keep track of the keys we've sent (needed for join and orderBy optimizations) - private sentKeys = new Set() + private sentKeys = new Set() // Keep track of the biggest value we've sent so far (needed for orderBy optimization) private biggest: any = undefined constructor( private collectionId: string, - private collection: Collection, - private config: Parameters[`sync`]>[0], + private collection: Collection, + private config: Parameters[`sync`]>[0], private syncState: FullSyncState, - private collectionConfigBuilder: CollectionConfigBuilder + private collectionConfigBuilder: CollectionConfigBuilder< + TContext, + TResult, + TKey + > ) {} subscribe() { @@ -106,7 +111,7 @@ export class CollectionSubscriber< // for keys that have not been sent to the pipeline yet // and filter out deletes for keys that have not been sent private sendVisibleChangesToPipeline = ( - changes: Array>, + changes: Array>, loadedInitialState: boolean ) => { if (loadedInitialState) { @@ -134,10 +139,7 @@ export class CollectionSubscriber< return this.sendChangesToPipeline(newChanges) } - private loadKeys( - keys: Iterable, - filterFn: (item: object) => boolean - ) { + private loadKeys(keys: Iterable, filterFn: (item: object) => boolean) { for (const key of keys) { // Only load the key once if (this.sentKeys.has(key)) continue @@ -178,9 +180,7 @@ export class CollectionSubscriber< // Until that point we filter out all changes from subscription to the collection. let sendChanges = false - const sendVisibleChanges = ( - changes: Array> - ) => { + const sendVisibleChanges = (changes: Array>) => { // We filter out changes when sendChanges is false to ensure that we don't send // any changes from the live subscription until the join operator requests either // the initial state or its first key. This is needed otherwise it could receive @@ -201,7 +201,7 @@ export class CollectionSubscriber< const filterFn = whereExpression ? createFilterFunctionFromExpression(whereExpression) : () => true - const loadKs = (keys: Set) => { + const loadKs = (keys: Set) => { sendChanges = true return this.loadKeys(keys, filterFn) } @@ -239,7 +239,7 @@ export class CollectionSubscriber< this.loadNextItems(offset + limit) const sendChangesInRange = ( - changes: Iterable> + changes: Iterable> ) => { // Split live updates into a delete of the old value and an insert of the new value // and filter out changes that are bigger than the biggest value we've sent so far @@ -312,7 +312,7 @@ export class CollectionSubscriber< } private sendChangesToPipelineWithTracking( - changes: Iterable> + changes: Iterable> ) { const { comparator } = this.collectionConfigBuilder.optimizableOrderByCollections[ @@ -335,10 +335,15 @@ export class CollectionSubscriber< : biggestSentRow // Take the `n` items after the biggest sent value const nextOrderedKeys = index.take(n, biggestSentValue) - const nextInserts: Array> = - nextOrderedKeys.map((key) => { - return { type: `insert`, key, value: this.collection.get(key) } - }) + const nextInserts: Array> = nextOrderedKeys.map( + (key) => { + return { + type: `insert`, + key: key as TKey, + value: this.collection.get(key as TKey), + } + } + ) this.sendChangesToPipelineWithTracking(nextInserts) return nextInserts.length } @@ -355,7 +360,7 @@ export class CollectionSubscriber< } private *trackSentValues( - changes: Iterable>, + changes: Iterable>, comparator: (a: any, b: any) => number ) { for (const change of changes) { @@ -405,10 +410,10 @@ function findCollectionAlias( /** * Helper function to send changes to a D2 input stream */ -function sendChangesToInput( +function sendChangesToInput( input: RootStreamBuilder, - changes: Iterable, - getKey: (item: ChangeMessage[`value`]) => any + changes: Iterable>, + getKey: (item: ChangeMessage[`value`]) => TKey ): number { const multiSetArray: MultiSetArray = [] for (const change of changes) { diff --git a/packages/db/src/query/live/types.ts b/packages/db/src/query/live/types.ts index 3368e617f..e5a22a2f7 100644 --- a/packages/db/src/query/live/types.ts +++ b/packages/db/src/query/live/types.ts @@ -1,5 +1,9 @@ import type { D2, RootStreamBuilder } from "@tanstack/db-ivm" -import type { CollectionConfig, ResultStream } from "../../types.js" +import type { + BaseCollectionConfig, + ResolveType, + ResultStream, +} from "../../types.js" import type { InitialQueryBuilder, QueryBuilder } from "../builder/index.js" import type { Context, GetResult } from "../builder/types.js" @@ -49,13 +53,12 @@ export type FullSyncState = Required export interface LiveQueryCollectionConfig< TContext extends Context, TResult extends object = GetResult & object, -> { - /** - * Unique identifier for the collection - * If not provided, defaults to `live-query-${number}` with auto-incrementing number - */ - id?: string - + TKey extends string | number = string | number, + TSchema = never, +> extends Omit< + BaseCollectionConfig, + `compare` | `getKey` + > { /** * Query builder function that defines the live query */ @@ -65,53 +68,6 @@ export interface LiveQueryCollectionConfig< /** * Function to extract the key from result items - * If not provided, defaults to using the key from the D2 stream - */ - getKey?: (item: TResult) => string | number - - /** - * Optional schema for validation - */ - schema?: CollectionConfig< - TResult, - string | number, - never, - TResult, - TResult - >[`schema`] - - /** - * Optional mutation handlers - */ - onInsert?: CollectionConfig< - TResult, - string | number, - never, - TResult, - TResult - >[`onInsert`] - onUpdate?: CollectionConfig< - TResult, - string | number, - never, - TResult, - TResult - >[`onUpdate`] - onDelete?: CollectionConfig< - TResult, - string | number, - never, - TResult, - TResult - >[`onDelete`] - - /** - * Start sync / the query immediately - */ - startSync?: boolean - - /** - * GC time for the collection */ - gcTime?: number + getKey?: (item: ResolveType) => TKey } diff --git a/packages/db/tests/local-only.test-d.ts b/packages/db/tests/local-only.test-d.ts index cd42e2b25..659a31f81 100644 --- a/packages/db/tests/local-only.test-d.ts +++ b/packages/db/tests/local-only.test-d.ts @@ -172,7 +172,6 @@ describe(`LocalOnly Collection Types`, () => { // Test insert parameter type type InsertParam = Parameters[0] - type ItemOf = T extends Array ? U : T expectTypeOf>().toEqualTypeOf() // Check that the update method accepts the expected input type diff --git a/packages/db/tests/local-storage.test-d.ts b/packages/db/tests/local-storage.test-d.ts index 7393778ad..fc64e3648 100644 --- a/packages/db/tests/local-storage.test-d.ts +++ b/packages/db/tests/local-storage.test-d.ts @@ -61,7 +61,7 @@ describe(`LocalStorage collection type resolution tests`, () => { expectTypeOf>().toEqualTypeOf() // Test update draft type - collection.update("test-id", (draft) => { + collection.update(`test-id`, (draft) => { expectTypeOf(draft).toEqualTypeOf() }) }) @@ -81,13 +81,13 @@ describe(`LocalStorage collection type resolution tests`, () => { // Test that the collection works with the options const collection = createCollection(options) - + // Test insert parameter type type InsertParam = Parameters[0] expectTypeOf>().toEqualTypeOf() // Test update draft type - collection.update("test-id", (draft) => { + collection.update(`test-id`, (draft) => { expectTypeOf(draft).toEqualTypeOf() }) }) @@ -108,13 +108,13 @@ describe(`LocalStorage collection type resolution tests`, () => { // Test that the collection works with the options const collection = createCollection(options) - + // Test insert parameter type type InsertParam = Parameters[0] expectTypeOf>().toEqualTypeOf() // Test update draft type - collection.update("test-id", (draft) => { + collection.update(`test-id`, (draft) => { expectTypeOf(draft).toEqualTypeOf() }) }) @@ -136,13 +136,13 @@ describe(`LocalStorage collection type resolution tests`, () => { // Test that the collection works with the options const collection = createCollection(options) - + // Test insert parameter type type InsertParam = Parameters[0] expectTypeOf>().toEqualTypeOf() // Test update draft type - collection.update("test-id", (draft) => { + collection.update(`test-id`, (draft) => { expectTypeOf(draft).toEqualTypeOf() }) }) @@ -162,13 +162,13 @@ describe(`LocalStorage collection type resolution tests`, () => { // Test that the collection works with the options const collection = createCollection(options) - + // Test insert parameter type type InsertParam = Parameters[0] expectTypeOf>().toEqualTypeOf() // Test update draft type - collection.update("test-id", (draft) => { + collection.update(`test-id`, (draft) => { expectTypeOf(draft).toEqualTypeOf() }) }) @@ -204,14 +204,13 @@ describe(`LocalStorage collection type resolution tests`, () => { // Test that the collection works with the options const collection = createCollection(options) - + // Test insert parameter type type InsertParam = Parameters[0] - type ItemOf = T extends Array ? U : T expectTypeOf>().toEqualTypeOf() // Test update draft type - collection.update("test-id", (draft) => { + collection.update(`test-id`, (draft) => { expectTypeOf(draft).toEqualTypeOf() }) }) @@ -313,7 +312,7 @@ describe(`LocalStorage collection type resolution tests`, () => { }) it(`should work with schema and infer correct types`, () => { - const testSchema = z.object({ + const testSchemaWithSchema = z.object({ id: z.string(), entityId: z.string(), value: z.string(), @@ -321,8 +320,8 @@ describe(`LocalStorage collection type resolution tests`, () => { }) // We can trust that zod infers the correct types for the schema - type ExpectedType = z.infer - type ExpectedInput = z.input + type ExpectedType = z.infer + type ExpectedInput = z.input const collection = createCollection( localStorageCollectionOptions({ @@ -330,7 +329,7 @@ describe(`LocalStorage collection type resolution tests`, () => { storage: mockStorage, storageEventApi: mockStorageEventApi, getKey: (item: any) => item.id, - schema: testSchema, + schema: testSchemaWithSchema, onInsert: (params) => { expectTypeOf( params.transaction.mutations[0].modified @@ -390,7 +389,9 @@ describe(`LocalStorage collection type resolution tests`, () => { // Test that the collection has the expected methods expectTypeOf(collection.insert).toBeFunction() - expectTypeOf(collection.get).returns.toEqualTypeOf() + expectTypeOf(collection.get).returns.toEqualTypeOf< + SelectUrlType | undefined + >() expectTypeOf(collection.toArray).toEqualTypeOf>() // Test insert parameter type @@ -398,7 +399,7 @@ describe(`LocalStorage collection type resolution tests`, () => { expectTypeOf>().toEqualTypeOf() // Test update draft type - collection.update("test-id", (draft) => { + collection.update(`test-id`, (draft) => { expectTypeOf(draft).toEqualTypeOf() }) }) diff --git a/packages/db/tests/query/order-by.test.ts b/packages/db/tests/query/order-by.test.ts index a871d059e..abf1e5669 100644 --- a/packages/db/tests/query/order-by.test.ts +++ b/packages/db/tests/query/order-by.test.ts @@ -738,10 +738,10 @@ function createOrderByTests(autoIndex: `off` | `eager`): void { query: (q) => q .from({ vehicleDocuments: vehicleDocumentCollection }) - .groupBy((q) => q.vehicleDocuments.vin) - .orderBy((q) => q.vehicleDocuments.vin, `asc`) - .select((q) => ({ - vin: q.vehicleDocuments.vin, + .groupBy(({ vehicleDocuments }) => vehicleDocuments.vin) + .orderBy(({ vehicleDocuments }) => vehicleDocuments.vin, `asc`) + .select(({ vehicleDocuments }) => ({ + vin: vehicleDocuments.vin, })), startSync: true, }) @@ -794,11 +794,14 @@ function createOrderByTests(autoIndex: `off` | `eager`): void { query: (q) => q .from({ vehicleDocuments: vehicleDocumentCollection }) - .groupBy((q) => q.vehicleDocuments.vin) - .orderBy((q) => max(q.vehicleDocuments.updatedAt), `desc`) - .select((q) => ({ - vin: q.vehicleDocuments.vin, - updatedAt: max(q.vehicleDocuments.updatedAt), + .groupBy(({ vehicleDocuments }) => vehicleDocuments.vin) + .orderBy( + ({ vehicleDocuments }) => max(vehicleDocuments.updatedAt), + `desc` + ) + .select(({ vehicleDocuments }) => ({ + vin: vehicleDocuments.vin, + updatedAt: max(vehicleDocuments.updatedAt), })) .offset(0) .limit(10), From 872147de6aec136ad8d7fa14283b98c415910f7c Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Sat, 6 Sep 2025 12:45:18 +0100 Subject: [PATCH 12/15] electric collection types --- packages/db/src/collection.ts | 5 +- packages/db/src/local-only.ts | 16 +- packages/db/src/local-storage.ts | 8 +- packages/db/src/types.ts | 13 +- .../electric-db-collection/src/electric.ts | 302 ++++-------------- .../tests/electric.test-d.ts | 52 +-- .../tests/electric.test.ts | 5 +- 7 files changed, 99 insertions(+), 302 deletions(-) diff --git a/packages/db/src/collection.ts b/packages/db/src/collection.ts index 531784c1c..bafc552dc 100644 --- a/packages/db/src/collection.ts +++ b/packages/db/src/collection.ts @@ -92,11 +92,10 @@ export interface Collection< /** * Creates a new Collection instance with the given configuration * - * @template TExplicit - The explicit type of items in the collection (highest priority) + * @template TExplicit - The explicit type of items in the collection (second priority) * @template TKey - The type of the key for the collection * @template TUtils - The utilities record type - * @template TSchema - The schema type for validation and type inference (second priority) - * @template TFallback - The fallback type if no explicit or schema type is provided + * @template TSchema - The schema type for validation and type inference (highest priority) * @param options - Collection options with optional utilities * @returns A new Collection with utilities exposed both at top level and under .utils * diff --git a/packages/db/src/local-only.ts b/packages/db/src/local-only.ts index f6ee1e510..3571b7bd4 100644 --- a/packages/db/src/local-only.ts +++ b/packages/db/src/local-only.ts @@ -13,17 +13,9 @@ import type { /** * Configuration interface for Local-only collection options - * @template TExplicit - The explicit type of items in the collection (highest priority) - * @template TSchema - The schema type for validation and type inference (second priority) + * @template TExplicit - The explicit type of items in the collection (second priority) + * @template TSchema - The schema type for validation and type inference (highest priority) * @template TKey - The type of the key returned by getKey - * - * @remarks - * Type resolution follows a priority order: - * 1. If you provide an explicit type via generic parameter, it will be used - * 2. If no explicit type is provided but a schema is, the schema's output type will be inferred - * 3. If neither explicit type nor schema is provided, the fallback type will be used - * - * You should provide EITHER an explicit type OR a schema, but not both, as they would conflict. */ export interface LocalOnlyCollectionConfig< TExplicit extends object = Record, @@ -49,8 +41,8 @@ export interface LocalOnlyCollectionUtils extends UtilsRecord {} * that immediately "syncs" all optimistic changes to the collection, making them permanent. * Perfect for local-only data that doesn't need persistence or external synchronization. * - * @template TExplicit - The explicit type of items in the collection (highest priority) - * @template TSchema - The schema type for validation and type inference (second priority) + * @template TExplicit - The explicit type of items in the collection (second priority) + * @template TSchema - The schema type for validation and type inference (highest priority) * @template TKey - The type of the key returned by getKey * @param config - Configuration options for the Local-only collection * @returns Collection options with utilities diff --git a/packages/db/src/local-storage.ts b/packages/db/src/local-storage.ts index 7fa1034e7..6b6e4a9bd 100644 --- a/packages/db/src/local-storage.ts +++ b/packages/db/src/local-storage.ts @@ -47,8 +47,8 @@ interface StoredItem { /** * Configuration interface for localStorage collection options - * @template TExplicit - The explicit type of items in the collection (highest priority) - * @template TSchema - The schema type for validation and type inference (second priority) + * @template TExplicit - The explicit type of items in the collection (second priority) + * @template TSchema - The schema type for validation and type inference (highest priority) * @template TKey - The type of the key returned by getKey * * @remarks @@ -131,8 +131,8 @@ function generateUuid(): string { * This function creates a collection that persists data to localStorage/sessionStorage * and synchronizes changes across browser tabs using storage events. * - * @template TExplicit - The explicit type of items in the collection (highest priority) - * @template TSchema - The schema type for validation and type inference (second priority) + * @template TExplicit - The explicit type of items in the collection (second priority) + * @template TSchema - The schema type for validation and type inference (highest priority) * @template TKey - The type of the key returned by getKey * @param config - Configuration options for the Local-storage collection * @returns Collection options with utilities including clearStorage and getStorageSize diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index 6779afc4f..261fe6228 100644 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -30,12 +30,10 @@ export type InferSchemaInput = T extends StandardSchemaV1 /** * Helper type to determine the input type for insert and update operations - * This takes the raw generics (TExplicit, TSchema, TFallback) instead of the resolved T. * * Priority: - * 1. Explicit generic TExplicit (if not 'unknown') - * 2. Schema input type (if schema provided) - * 3. Fallback type TFallback + * 1. Schema input type (if schema provided and valid) + * 2. Explicit generic TExplicit (if no schema or schema is invalid) * * @internal This is used for collection insert and update type inference */ @@ -54,9 +52,10 @@ export type ResolveInput< /** * Helper type to determine the final type based on priority: - * 1. Explicit generic TExplicit (if not 'unknown') - * 2. Schema output type (if schema provided) - * 3. Fallback type TFallback + * + * Priority: + * 1. Schema output type (if schema provided and valid) + * 2. Explicit generic TExplicit (if no schema or schema is invalid) * * @remarks * This type is used internally to resolve the collection item type based on the provided generics and schema. diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index 097e68349..57291b348 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -13,7 +13,9 @@ import { TimeoutWaitingForTxIdError, } from "./errors" import type { + BaseCollectionConfig, CollectionConfig, + DeleteMutationFn, DeleteMutationFnParams, InsertMutationFnParams, SyncConfig, @@ -54,217 +56,49 @@ type InferSchemaInput = T extends StandardSchemaV1 : Record : Record -type ResolveInput< +export type ResolveInput< TExplicit extends Row = Row, - TSchema extends StandardSchemaV1 = never, - TFallback extends object = Record, -> = unknown extends TExplicit - ? [TSchema] extends [never] - ? TFallback - : InferSchemaInput - : TExplicit - -type ResolveType< + TSchema = never, +> = [TSchema] extends [never] + ? TExplicit extends object + ? TExplicit + : Record + : TSchema extends StandardSchemaV1 + ? InferSchemaInput + : TExplicit extends object + ? TExplicit + : Record + +export type ResolveType< TExplicit extends Row = Row, - TSchema extends StandardSchemaV1 = never, - TFallback extends object = Record, -> = - unknown extends GetExtensions - ? [TSchema] extends [never] - ? TFallback - : InferSchemaOutput - : TExplicit + TSchema = never, +> = [TSchema] extends [never] + ? TExplicit extends Row + ? TExplicit + : Record + : TSchema extends StandardSchemaV1 + ? InferSchemaOutput + : TExplicit extends Row + ? TExplicit + : Record /** * Configuration interface for Electric collection options - * @template TExplicit - The explicit type of items in the collection (highest priority) - * @template TSchema - The schema type for validation and type inference (second priority) - * @template TFallback - The fallback type if no explicit or schema type is provided - * - * @remarks - * Type resolution follows a priority order: - * 1. If you provide an explicit type via generic parameter, it will be used - * 2. If no explicit type is provided but a schema is, the schema's output type will be inferred - * 3. If neither explicit type nor schema is provided, the fallback type will be used - * - * You should provide EITHER an explicit type OR a schema, but not both, as they would conflict. + * @template TExplicit - The explicit type of items in the collection (second priority) + * @template TSchema - The schema type for validation and type inference (highest priority) + * @template TKey - The type of the key returned by getKey */ export interface ElectricCollectionConfig< TExplicit extends Row = Row, - TSchema extends StandardSchemaV1 = never, - TFallback extends Row = Row, -> { + TKey extends string | number = string | number, + TSchema = never, +> extends BaseCollectionConfig { /** * Configuration options for the ElectricSQL ShapeStream */ shapeOptions: ShapeStreamOptions< - GetExtensions> + GetExtensions> > - - /** - * All standard Collection configuration properties - */ - id?: string - schema?: TSchema - getKey: CollectionConfig>[`getKey`] - sync?: CollectionConfig>[`sync`] - - /** - * Optional asynchronous handler function called before an insert operation - * Must return an object containing a txid number or array of txids - * @param params Object containing transaction and collection information - * @returns Promise resolving to an object with txid or txids - * @example - * // Basic Electric insert handler - MUST return { txid: number } - * onInsert: async ({ transaction }) => { - * const newItem = transaction.mutations[0].modified - * const result = await api.todos.create({ - * data: newItem - * }) - * return { txid: result.txid } // Required for Electric sync matching - * } - * - * @example - * // Insert handler with multiple items - return array of txids - * onInsert: async ({ transaction }) => { - * const items = transaction.mutations.map(m => m.modified) - * const results = await Promise.all( - * items.map(item => api.todos.create({ data: item })) - * ) - * return { txid: results.map(r => r.txid) } // Array of txids - * } - * - * @example - * // Insert handler with error handling - * onInsert: async ({ transaction }) => { - * try { - * const newItem = transaction.mutations[0].modified - * const result = await api.createTodo(newItem) - * return { txid: result.txid } - * } catch (error) { - * console.error('Insert failed:', error) - * throw error // This will cause the transaction to fail - * } - * } - * - * @example - * // Insert handler with batch operation - single txid - * onInsert: async ({ transaction }) => { - * const items = transaction.mutations.map(m => m.modified) - * const result = await api.todos.createMany({ - * data: items - * }) - * return { txid: result.txid } // Single txid for batch operation - * } - */ - onInsert?: ( - params: InsertMutationFnParams> - ) => Promise<{ txid: Txid | Array }> - - /** - * Optional asynchronous handler function called before an update operation - * Must return an object containing a txid number or array of txids - * @param params Object containing transaction and collection information - * @returns Promise resolving to an object with txid or txids - * @example - * // Basic Electric update handler - MUST return { txid: number } - * onUpdate: async ({ transaction }) => { - * const { original, changes } = transaction.mutations[0] - * const result = await api.todos.update({ - * where: { id: original.id }, - * data: changes // Only the changed fields - * }) - * return { txid: result.txid } // Required for Electric sync matching - * } - * - * @example - * // Update handler with multiple items - return array of txids - * onUpdate: async ({ transaction }) => { - * const updates = await Promise.all( - * transaction.mutations.map(m => - * api.todos.update({ - * where: { id: m.original.id }, - * data: m.changes - * }) - * ) - * ) - * return { txid: updates.map(u => u.txid) } // Array of txids - * } - * - * @example - * // Update handler with optimistic rollback - * onUpdate: async ({ transaction }) => { - * const mutation = transaction.mutations[0] - * try { - * const result = await api.updateTodo(mutation.original.id, mutation.changes) - * return { txid: result.txid } - * } catch (error) { - * // Transaction will automatically rollback optimistic changes - * console.error('Update failed, rolling back:', error) - * throw error - * } - * } - */ - onUpdate?: ( - params: UpdateMutationFnParams> - ) => Promise<{ txid: Txid | Array }> - - /** - * Optional asynchronous handler function called before a delete operation - * Must return an object containing a txid number or array of txids - * @param params Object containing transaction and collection information - * @returns Promise resolving to an object with txid or txids - * @example - * // Basic Electric delete handler - MUST return { txid: number } - * onDelete: async ({ transaction }) => { - * const mutation = transaction.mutations[0] - * const result = await api.todos.delete({ - * id: mutation.original.id - * }) - * return { txid: result.txid } // Required for Electric sync matching - * } - * - * @example - * // Delete handler with multiple items - return array of txids - * onDelete: async ({ transaction }) => { - * const deletes = await Promise.all( - * transaction.mutations.map(m => - * api.todos.delete({ - * where: { id: m.key } - * }) - * ) - * ) - * return { txid: deletes.map(d => d.txid) } // Array of txids - * } - * - * @example - * // Delete handler with batch operation - single txid - * onDelete: async ({ transaction }) => { - * const idsToDelete = transaction.mutations.map(m => m.original.id) - * const result = await api.todos.deleteMany({ - * ids: idsToDelete - * }) - * return { txid: result.txid } // Single txid for batch operation - * } - * - * @example - * // Delete handler with optimistic rollback - * onDelete: async ({ transaction }) => { - * const mutation = transaction.mutations[0] - * try { - * const result = await api.deleteTodo(mutation.original.id) - * return { txid: result.txid } - * } catch (error) { - * // Transaction will automatically rollback optimistic changes - * console.error('Delete failed, rolling back:', error) - * throw error - * } - * } - * - */ - onDelete?: ( - params: DeleteMutationFnParams> - ) => Promise<{ txid: Txid | Array }> } function isUpToDateMessage>( @@ -301,26 +135,26 @@ export interface ElectricCollectionUtils extends UtilsRecord { /** * Creates Electric collection options for use with a standard Collection * - * @template TExplicit - The explicit type of items in the collection (highest priority) - * @template TSchema - The schema type for validation and type inference (second priority) - * @template TFallback - The fallback type if no explicit or schema type is provided + * @template TExplicit - The explicit type of items in the collection (second priority) + * @template TSchema - The schema type for validation and type inference (highest priority) + * @template TKey - The type of the key returned by getKey * @param config - Configuration options for the Electric collection * @returns Collection options with utilities */ export function electricCollectionOptions< TExplicit extends Row = Row, - TSchema extends StandardSchemaV1 = never, - TFallback extends Row = Row, + TKey extends string | number = string | number, + TSchema = never, >( - config: ElectricCollectionConfig + config: ElectricCollectionConfig ): CollectionConfig< - ResolveType, - string | number, + ResolveType, + TKey, TSchema, - ResolveInput + ResolveInput > & { utils: ElectricCollectionUtils } { const seenTxids = new Store>(new Set([])) - const sync = createElectricSync>( + const sync = createElectricSync, TKey>( config.shapeOptions, { seenTxids, @@ -365,13 +199,11 @@ export function electricCollectionOptions< // Create wrapper handlers for direct persistence operations that handle txid awaiting const wrappedOnInsert = config.onInsert ? async ( - params: InsertMutationFnParams< - ResolveType - > + params: InsertMutationFnParams> ) => { // Runtime check (that doesn't follow type) - // eslint-disable-next-line - const handlerResult = (await config.onInsert!(params)) ?? {} + + const handlerResult = (await config.onInsert!(params as any)) ?? {} const txid = (handlerResult as { txid?: Txid | Array }).txid if (!txid) { @@ -391,13 +223,11 @@ export function electricCollectionOptions< const wrappedOnUpdate = config.onUpdate ? async ( - params: UpdateMutationFnParams< - ResolveType - > + params: UpdateMutationFnParams> ) => { // Runtime check (that doesn't follow type) - // eslint-disable-next-line - const handlerResult = (await config.onUpdate!(params)) ?? {} + + const handlerResult = (await config.onUpdate!(params as any)) ?? {} const txid = (handlerResult as { txid?: Txid | Array }).txid if (!txid) { @@ -415,22 +245,24 @@ export function electricCollectionOptions< } : undefined - const wrappedOnDelete = config.onDelete + const wrappedOnDelete: + | DeleteMutationFn> + | undefined = config.onDelete ? async ( - params: DeleteMutationFnParams< - ResolveType - > + params: DeleteMutationFnParams> ) => { - const handlerResult = await config.onDelete!(params) - if (!handlerResult.txid) { + const handlerResult = await config.onDelete!(params as any) + const txid = (handlerResult as { txid?: Txid | Array }).txid + + if (!txid) { throw new ElectricDeleteHandlerMustReturnTxIdError() } // Handle both single txid and array of txids - if (Array.isArray(handlerResult.txid)) { - await Promise.all(handlerResult.txid.map((id) => awaitTxId(id))) + if (Array.isArray(txid)) { + await Promise.all(txid.map((id) => awaitTxId(id))) } else { - await awaitTxId(handlerResult.txid) + await awaitTxId(txid) } return handlerResult @@ -455,18 +287,26 @@ export function electricCollectionOptions< utils: { awaitTxId, }, - } + } as CollectionConfig< + ResolveType, + TKey, + TSchema, + ResolveInput + > & { utils: ElectricCollectionUtils } } /** * Internal function to create ElectricSQL sync configuration */ -function createElectricSync>( +function createElectricSync< + T extends Row, + TKey extends string | number, +>( shapeOptions: ShapeStreamOptions>, options: { seenTxids: Store> } -): SyncConfig { +): SyncConfig { const { seenTxids } = options // Store for the relation schema information @@ -501,7 +341,7 @@ function createElectricSync>( let unsubscribeStream: () => void return { - sync: (params: Parameters[`sync`]>[0]) => { + sync: (params: Parameters[`sync`]>[0]) => { const { begin, write, commit, markReady, truncate, collection } = params const stream = new ShapeStream({ ...shapeOptions, diff --git a/packages/electric-db-collection/tests/electric.test-d.ts b/packages/electric-db-collection/tests/electric.test-d.ts index df0f127ba..4d67c3ffe 100644 --- a/packages/electric-db-collection/tests/electric.test-d.ts +++ b/packages/electric-db-collection/tests/electric.test-d.ts @@ -7,19 +7,16 @@ import { gt, } from "@tanstack/db" import { electricCollectionOptions } from "../src/electric" -import type { ElectricCollectionConfig } from "../src/electric" import type { DeleteMutationFnParams, InsertMutationFnParams, ResolveType, UpdateMutationFnParams, } from "@tanstack/db" -import type { Row } from "@electric-sql/client" describe(`Electric collection type resolution tests`, () => { // Define test types type ExplicitType = { id: string; explicit: boolean } - type FallbackType = { id: string; fallback: boolean } // Define a schema const testSchema = z.object({ @@ -29,7 +26,7 @@ describe(`Electric collection type resolution tests`, () => { type SchemaType = z.infer - it(`should prioritize explicit type in ElectricCollectionConfig`, () => { + it(`should use explicit type when provided in ElectricCollectionConfig`, () => { const options = electricCollectionOptions({ shapeOptions: { url: `foo`, @@ -38,13 +35,13 @@ describe(`Electric collection type resolution tests`, () => { getKey: (item) => item.id, }) - type ExpectedType = ResolveType> + type ExpectedType = ResolveType // The getKey function should have the resolved type expectTypeOf(options.getKey).parameters.toEqualTypeOf<[ExplicitType]>() expectTypeOf().toEqualTypeOf() }) - it(`should use schema type when explicit type is not provided`, () => { + it(`should use schema type when provided in ElectricCollectionConfig`, () => { const options = electricCollectionOptions({ shapeOptions: { url: `foo`, @@ -54,42 +51,17 @@ describe(`Electric collection type resolution tests`, () => { getKey: (item) => item.id, }) - type ExpectedType = ResolveType> + type ExpectedType = ResolveType // The getKey function should have the resolved type expectTypeOf(options.getKey).parameters.toEqualTypeOf<[SchemaType]>() expectTypeOf().toEqualTypeOf() }) - it(`should use fallback type when neither explicit nor schema type is provided`, () => { - const config: ElectricCollectionConfig< - Row, - never, - FallbackType - > = { - shapeOptions: { - url: `foo`, - params: { table: `test_table` }, - }, - getKey: (item) => item.id, - } - - const options = electricCollectionOptions< - Row, - never, - FallbackType - >(config) - - type ExpectedType = ResolveType - // The getKey function should have the resolved type - expectTypeOf(options.getKey).parameters.toEqualTypeOf<[FallbackType]>() - expectTypeOf().toEqualTypeOf() - }) - - it(`should correctly resolve type with all three types provided`, () => { + it(`should correctly resolve type when explicit and schema types are provided`, () => { const options = electricCollectionOptions< ExplicitType, - typeof testSchema, - FallbackType + string, + typeof testSchema >({ shapeOptions: { url: `test_shape`, @@ -99,14 +71,10 @@ describe(`Electric collection type resolution tests`, () => { getKey: (item) => item.id, }) - type ExpectedType = ResolveType< - ExplicitType, - typeof testSchema, - FallbackType - > + type ExpectedType = ResolveType // The getKey function should have the resolved type - expectTypeOf(options.getKey).parameters.toEqualTypeOf<[ExplicitType]>() - expectTypeOf().toEqualTypeOf() + expectTypeOf(options.getKey).parameters.toEqualTypeOf<[SchemaType]>() + expectTypeOf().toEqualTypeOf() }) it(`should properly type the onInsert, onUpdate, and onDelete handlers`, () => { diff --git a/packages/electric-db-collection/tests/electric.test.ts b/packages/electric-db-collection/tests/electric.test.ts index fdaf56f2b..6021c07f0 100644 --- a/packages/electric-db-collection/tests/electric.test.ts +++ b/packages/electric-db-collection/tests/electric.test.ts @@ -15,7 +15,6 @@ import type { TransactionWithMutations, } from "@tanstack/db" import type { Message, Row } from "@electric-sql/client" -import type { StandardSchemaV1 } from "@standard-schema/spec" // Mock the ShapeStream module const mockSubscribe = vi.fn() @@ -34,9 +33,9 @@ vi.mock(`@electric-sql/client`, async () => { describe(`Electric Integration`, () => { let collection: Collection< Row, - string | number, + number, ElectricCollectionUtils, - StandardSchemaV1, + undefined, Row > let subscriber: (messages: Array>) => void From 501f3ec1b8d77d3b0b7c0d0e1199a627be8391d6 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Sat, 6 Sep 2025 12:52:32 +0100 Subject: [PATCH 13/15] remove copying of the types to the electric package by givving a third restric param --- packages/db/src/types.ts | 25 +++--- packages/electric-db-collection/package.json | 1 - .../electric-db-collection/src/electric.ts | 83 +++++-------------- pnpm-lock.yaml | 3 - 4 files changed, 38 insertions(+), 74 deletions(-) diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index 261fe6228..d402cb0cf 100644 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -11,8 +11,11 @@ import type { BasicExpression } from "./query/ir.js" * * @internal This is used by the type resolution system */ -export type InferSchemaOutput = T extends StandardSchemaV1 - ? StandardSchemaV1.InferOutput extends object +export type InferSchemaOutput< + T, + TRestrict = object, +> = T extends StandardSchemaV1 + ? StandardSchemaV1.InferOutput extends TRestrict ? StandardSchemaV1.InferOutput : Record : Record @@ -22,8 +25,8 @@ export type InferSchemaOutput = T extends StandardSchemaV1 * * @internal This is used for collection insert type inference */ -export type InferSchemaInput = T extends StandardSchemaV1 - ? StandardSchemaV1.InferInput extends object +export type InferSchemaInput = T extends StandardSchemaV1 + ? StandardSchemaV1.InferInput extends TRestrict ? StandardSchemaV1.InferInput : Record : Record @@ -40,13 +43,14 @@ export type InferSchemaInput = T extends StandardSchemaV1 export type ResolveInput< TExplicit = Record, TSchema = never, + TRestrict = object, > = [TSchema] extends [never] - ? TExplicit extends object + ? TExplicit extends TRestrict ? TExplicit : Record : TSchema extends StandardSchemaV1 - ? InferSchemaInput - : TExplicit extends object + ? InferSchemaInput + : TExplicit extends TRestrict ? TExplicit : Record @@ -64,13 +68,14 @@ export type ResolveInput< export type ResolveType< TExplicit = Record, TSchema = never, + TRestrict = object, > = [TSchema] extends [never] - ? TExplicit extends object + ? TExplicit extends TRestrict ? TExplicit : Record : TSchema extends StandardSchemaV1 - ? InferSchemaOutput - : TExplicit extends object + ? InferSchemaOutput + : TExplicit extends TRestrict ? TExplicit : Record diff --git a/packages/electric-db-collection/package.json b/packages/electric-db-collection/package.json index 92d84a13e..eb1d1fcdb 100644 --- a/packages/electric-db-collection/package.json +++ b/packages/electric-db-collection/package.json @@ -4,7 +4,6 @@ "version": "0.1.12", "dependencies": { "@electric-sql/client": "1.0.9", - "@standard-schema/spec": "^1.0.0", "@tanstack/db": "workspace:*", "@tanstack/store": "^0.7.0", "debug": "^4.4.1" diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index 57291b348..5c5c14949 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -18,11 +18,12 @@ import type { DeleteMutationFn, DeleteMutationFnParams, InsertMutationFnParams, + ResolveInput, + ResolveType, SyncConfig, UpdateMutationFnParams, UtilsRecord, } from "@tanstack/db" -import type { StandardSchemaV1 } from "@standard-schema/spec" import type { ControlMessage, GetExtensions, @@ -38,50 +39,6 @@ const debug = DebugModule.debug(`ts/db:electric`) */ export type Txid = number -// The `InferSchemaOutput`, `InferSchemaInput`, `ResolveType` and `ResolveInput` are -// copied from the `@tanstack/db` package but we modified `InferSchemaOutput` -// and `InferSchemaInput` slightly to restrict the schema output to `Row` -// This is needed in order for `GetExtensions` to be able to infer the parser -// extensions type from the schema. - -type InferSchemaOutput = T extends StandardSchemaV1 - ? StandardSchemaV1.InferOutput extends Row - ? StandardSchemaV1.InferOutput - : Record - : Record - -type InferSchemaInput = T extends StandardSchemaV1 - ? StandardSchemaV1.InferInput extends Row - ? StandardSchemaV1.InferInput - : Record - : Record - -export type ResolveInput< - TExplicit extends Row = Row, - TSchema = never, -> = [TSchema] extends [never] - ? TExplicit extends object - ? TExplicit - : Record - : TSchema extends StandardSchemaV1 - ? InferSchemaInput - : TExplicit extends object - ? TExplicit - : Record - -export type ResolveType< - TExplicit extends Row = Row, - TSchema = never, -> = [TSchema] extends [never] - ? TExplicit extends Row - ? TExplicit - : Record - : TSchema extends StandardSchemaV1 - ? InferSchemaOutput - : TExplicit extends Row - ? TExplicit - : Record - /** * Configuration interface for Electric collection options * @template TExplicit - The explicit type of items in the collection (second priority) @@ -97,7 +54,7 @@ export interface ElectricCollectionConfig< * Configuration options for the ElectricSQL ShapeStream */ shapeOptions: ShapeStreamOptions< - GetExtensions> + GetExtensions>> > } @@ -148,18 +105,18 @@ export function electricCollectionOptions< >( config: ElectricCollectionConfig ): CollectionConfig< - ResolveType, + ResolveType>, TKey, TSchema, - ResolveInput + ResolveInput> > & { utils: ElectricCollectionUtils } { const seenTxids = new Store>(new Set([])) - const sync = createElectricSync, TKey>( - config.shapeOptions, - { - seenTxids, - } - ) + const sync = createElectricSync< + ResolveType>, + TKey + >(config.shapeOptions, { + seenTxids, + }) /** * Wait for a specific transaction ID to be synced @@ -199,7 +156,9 @@ export function electricCollectionOptions< // Create wrapper handlers for direct persistence operations that handle txid awaiting const wrappedOnInsert = config.onInsert ? async ( - params: InsertMutationFnParams> + params: InsertMutationFnParams< + ResolveType> + > ) => { // Runtime check (that doesn't follow type) @@ -223,7 +182,9 @@ export function electricCollectionOptions< const wrappedOnUpdate = config.onUpdate ? async ( - params: UpdateMutationFnParams> + params: UpdateMutationFnParams< + ResolveType> + > ) => { // Runtime check (that doesn't follow type) @@ -246,10 +207,12 @@ export function electricCollectionOptions< : undefined const wrappedOnDelete: - | DeleteMutationFn> + | DeleteMutationFn>> | undefined = config.onDelete ? async ( - params: DeleteMutationFnParams> + params: DeleteMutationFnParams< + ResolveType> + > ) => { const handlerResult = await config.onDelete!(params as any) const txid = (handlerResult as { txid?: Txid | Array }).txid @@ -288,10 +251,10 @@ export function electricCollectionOptions< awaitTxId, }, } as CollectionConfig< - ResolveType, + ResolveType>, TKey, TSchema, - ResolveInput + ResolveInput> > & { utils: ElectricCollectionUtils } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5cb31dcbb..657be7221 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -520,9 +520,6 @@ importers: '@electric-sql/client': specifier: 1.0.9 version: 1.0.9 - '@standard-schema/spec': - specifier: ^1.0.0 - version: 1.0.0 '@tanstack/db': specifier: workspace:* version: link:../db From 6828af47f2034481ff224e2d279228aa1a8fa382 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Sun, 7 Sep 2025 12:33:16 +0100 Subject: [PATCH 14/15] checkpoint - checked back presidence --- packages/db/src/collection.ts | 4 +- packages/db/src/types.ts | 49 ++-- packages/db/tests/collection-schema.test.ts | 118 +++++++++ packages/db/tests/collection.test-d.ts | 78 ++++-- packages/query-db-collection/src/query.ts | 260 ++++---------------- 5 files changed, 246 insertions(+), 263 deletions(-) diff --git a/packages/db/src/collection.ts b/packages/db/src/collection.ts index bafc552dc..a6e907647 100644 --- a/packages/db/src/collection.ts +++ b/packages/db/src/collection.ts @@ -163,7 +163,7 @@ export interface Collection< */ export function createCollection< - TExplicit extends object = Record, + TExplicit extends object = never, TKey extends string | number = string | number, TUtils extends UtilsRecord = {}, TSchema = never, @@ -207,7 +207,7 @@ export function createCollection< } export class CollectionImpl< - T extends object = Record, + T extends object = never, TKey extends string | number = string | number, TUtils extends UtilsRecord = {}, TSchema = never, diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index d402cb0cf..88c68102f 100644 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -32,52 +32,49 @@ export type InferSchemaInput = T extends StandardSchemaV1 : Record /** - * Helper type to determine the input type for insert and update operations + * Helper type to determine the insert input type + * This takes the raw generics (TExplicit, TSchema, TFallback) instead of the resolved T. * * Priority: - * 1. Schema input type (if schema provided and valid) - * 2. Explicit generic TExplicit (if no schema or schema is invalid) + * 1. Explicit generic TExplicit (if not 'unknown') + * 2. Schema input type (if schema provided) + * 3. Fallback type TFallback * - * @internal This is used for collection insert and update type inference + * @internal This is used for collection insert type inference */ export type ResolveInput< - TExplicit = Record, + TExplicit = never, TSchema = never, - TRestrict = object, -> = [TSchema] extends [never] - ? TExplicit extends TRestrict + TFallback extends object = Record, +> = [TExplicit] extends [never] + ? [TSchema] extends [never] + ? TFallback + : InferSchemaInput + : TExplicit extends object ? TExplicit : Record - : TSchema extends StandardSchemaV1 - ? InferSchemaInput - : TExplicit extends TRestrict - ? TExplicit - : Record /** * Helper type to determine the final type based on priority: - * - * Priority: - * 1. Schema output type (if schema provided and valid) - * 2. Explicit generic TExplicit (if no schema or schema is invalid) + * 1. Explicit generic TExplicit (if not 'unknown') + * 2. Schema output type (if schema provided) + * 3. Fallback type TFallback * * @remarks * This type is used internally to resolve the collection item type based on the provided generics and schema. * Users should not need to use this type directly, but understanding the priority order helps when defining collections. */ export type ResolveType< - TExplicit = Record, + TExplicit = never, TSchema = never, - TRestrict = object, -> = [TSchema] extends [never] - ? TExplicit extends TRestrict + TFallback extends object = Record, +> = [TExplicit] extends [never] + ? [TSchema] extends [never] + ? TFallback + : InferSchemaOutput + : TExplicit extends object ? TExplicit : Record - : TSchema extends StandardSchemaV1 - ? InferSchemaOutput - : TExplicit extends TRestrict - ? TExplicit - : Record export type TransactionState = `pending` | `persisting` | `completed` | `failed` diff --git a/packages/db/tests/collection-schema.test.ts b/packages/db/tests/collection-schema.test.ts index 9e8ffb397..c5b40e75f 100644 --- a/packages/db/tests/collection-schema.test.ts +++ b/packages/db/tests/collection-schema.test.ts @@ -882,3 +882,121 @@ describe(`Collection with schema validation`, () => { ) }) }) + +describe(`Collection schema callback type tests`, () => { + it(`should correctly type all callback parameters with schema`, () => { + const userSchema = z.object({ + id: z.string(), + name: z.string(), + email: z.string().email(), + age: z.number().int().positive(), + }) + + type ExpectedType = z.infer + type ExpectedInput = z.input + + createCollection({ + getKey: (item) => item.id, + schema: userSchema, + sync: { sync: () => {} }, + onInsert: (params) => { + expectTypeOf(params.transaction).toHaveProperty(`mutations`) + const mutation = params.transaction.mutations[0] + expectTypeOf(mutation).toHaveProperty(`modified`) + expectTypeOf(mutation.modified).toEqualTypeOf() + return Promise.resolve() + }, + onUpdate: (params) => { + expectTypeOf(params.transaction).toHaveProperty(`mutations`) + const mutation = params.transaction.mutations[0] + expectTypeOf(mutation).toHaveProperty(`modified`) + expectTypeOf(mutation.modified).toEqualTypeOf() + expectTypeOf(mutation).toHaveProperty(`changes`) + expectTypeOf(mutation.changes).toEqualTypeOf>() + return Promise.resolve() + }, + onDelete: (params) => { + expectTypeOf(params.transaction).toHaveProperty(`mutations`) + const mutation = params.transaction.mutations[0] + expectTypeOf(mutation).toHaveProperty(`original`) + expectTypeOf(mutation.original).toEqualTypeOf() + return Promise.resolve() + }, + }) + }) + + it(`should correctly type callbacks with schema transformations`, () => { + const userSchema = z.object({ + id: z.string(), + name: z.string(), + email: z.string().email(), + created_at: z.string().transform((val) => new Date(val)), + updated_at: z.string().transform((val) => new Date(val)), + }) + + type ExpectedType = z.infer + + createCollection({ + getKey: (item) => item.id, + schema: userSchema, + sync: { sync: () => {} }, + onInsert: (params) => { + const mutation = params.transaction.mutations[0] + // Modified should be the output type (with Date objects) + expectTypeOf(mutation.modified).toEqualTypeOf() + return Promise.resolve() + }, + onUpdate: (params) => { + const mutation = params.transaction.mutations[0] + // Modified should be the output type (with Date objects) + expectTypeOf(mutation.modified).toEqualTypeOf() + return Promise.resolve() + }, + onDelete: (params) => { + const mutation = params.transaction.mutations[0] + // Original should be the output type (with Date objects) + expectTypeOf(mutation.original).toEqualTypeOf() + return Promise.resolve() + }, + }) + }) + + it(`should correctly type callbacks with schema defaults`, () => { + const userSchema = z.object({ + id: z.string(), + name: z.string(), + email: z.string().email(), + created_at: z.date().default(() => new Date()), + updated_at: z.date().default(() => new Date()), + }) + + type ExpectedType = z.infer + type ExpectedInput = z.input + + createCollection({ + getKey: (item) => item.id, + schema: userSchema, + sync: { sync: () => {} }, + onInsert: (params) => { + const mutation = params.transaction.mutations[0] + // Modified should be the output type (with all fields including defaults) + expectTypeOf(mutation.modified).toEqualTypeOf() + return Promise.resolve() + }, + onUpdate: (params) => { + const mutation = params.transaction.mutations[0] + // Modified should be the output type (with all fields including defaults) + expectTypeOf(mutation.modified).toEqualTypeOf() + // Changes should be the input type (without defaulted fields) + expectTypeOf(mutation.changes).toEqualTypeOf>() + return Promise.resolve() + }, + onDelete: (params) => { + const mutation = params.transaction.mutations[0] + // Original should be the output type (with all fields including defaults) + expectTypeOf(mutation.original).toEqualTypeOf() + return Promise.resolve() + }, + }) + }) +}) diff --git a/packages/db/tests/collection.test-d.ts b/packages/db/tests/collection.test-d.ts index b4d9e92aa..cf7d6fbd7 100644 --- a/packages/db/tests/collection.test-d.ts +++ b/packages/db/tests/collection.test-d.ts @@ -96,7 +96,7 @@ describe(`Collection type resolution tests`, () => { type Param = Parameters[0] expectTypeOf>().toEqualTypeOf() - type ExpectedType = ResolveType + type ExpectedType = ResolveType expectTypeOf().toEqualTypeOf() }) @@ -175,14 +175,8 @@ describe(`Schema Input/Output Type Distinction`, () => { }) it(`should handle schema with default values correctly for insert`, () => { - type ExpectedOutputType = ResolveType< - unknown, - typeof userSchemaWithDefaults - > - type ExpectedInputType = ResolveInput< - unknown, - typeof userSchemaWithDefaults - > + type ExpectedOutputType = ResolveType + type ExpectedInputType = ResolveInput const collection = createCollection({ getKey: (item) => { @@ -229,8 +223,8 @@ describe(`Schema Input/Output Type Distinction`, () => { schema: userSchemaTransform, }) - type ExpectedInputType = ResolveInput - type ExpectedOutputType = ResolveType + type ExpectedInputType = ResolveInput + type ExpectedOutputType = ResolveType type InsertArg = Parameters[0] // Input type should be the raw input (before transformation) @@ -267,14 +261,8 @@ describe(`Schema Input/Output Type Distinction`, () => { schema: userSchemaWithDefaults, }) - type ExpectedOutputType = ResolveType< - unknown, - typeof userSchemaWithDefaults - > - type ExpectedInputType = ResolveInput< - unknown, - typeof userSchemaWithDefaults - > + type ExpectedOutputType = ResolveType + type ExpectedInputType = ResolveInput // Input type should not include defaulted fields expectTypeOf().toEqualTypeOf<{ @@ -315,8 +303,8 @@ describe(`Schema Input/Output Type Distinction`, () => { schema: userSchemaTransform, }) - type ExpectedInputType = ResolveInput - type ExpectedOutputType = ResolveType + type ExpectedInputType = ResolveInput + type ExpectedOutputType = ResolveType // Input type should be the raw input (before transformation) expectTypeOf().toEqualTypeOf<{ @@ -350,3 +338,51 @@ describe(`Schema Input/Output Type Distinction`, () => { expectTypeOf(collection.toArray).toEqualTypeOf>() }) }) + +describe(`Collection callback type tests`, () => { + type TypeTestItem = { id: string; value: number; optional?: boolean } + + it(`should correctly type onInsert callback parameters`, () => { + createCollection({ + getKey: (item) => item.id, + sync: { sync: () => {} }, + onInsert: (params) => { + expectTypeOf(params.transaction).toHaveProperty(`mutations`) + const mutation = params.transaction.mutations[0] + expectTypeOf(mutation).toHaveProperty(`modified`) + expectTypeOf(mutation.modified).toEqualTypeOf() + return Promise.resolve() + }, + }) + }) + + it(`should correctly type onUpdate callback parameters`, () => { + createCollection({ + getKey: (item) => item.id, + sync: { sync: () => {} }, + onUpdate: (params) => { + expectTypeOf(params.transaction).toHaveProperty(`mutations`) + const mutation = params.transaction.mutations[0] + expectTypeOf(mutation).toHaveProperty(`modified`) + expectTypeOf(mutation.modified).toEqualTypeOf() + expectTypeOf(mutation).toHaveProperty(`changes`) + expectTypeOf(mutation.changes).toEqualTypeOf>() + return Promise.resolve() + }, + }) + }) + + it(`should correctly type onDelete callback parameters`, () => { + createCollection({ + getKey: (item) => item.id, + sync: { sync: () => {} }, + onDelete: (params) => { + expectTypeOf(params.transaction).toHaveProperty(`mutations`) + const mutation = params.transaction.mutations[0] + expectTypeOf(mutation).toHaveProperty(`original`) + expectTypeOf(mutation.original).toEqualTypeOf() + return Promise.resolve() + }, + }) + }) +}) diff --git a/packages/query-db-collection/src/query.ts b/packages/query-db-collection/src/query.ts index 9688cdf59..67aab1d4c 100644 --- a/packages/query-db-collection/src/query.ts +++ b/packages/query-db-collection/src/query.ts @@ -13,65 +13,49 @@ import type { QueryObserverOptions, } from "@tanstack/query-core" import type { + BaseCollectionConfig, ChangeMessage, CollectionConfig, - DeleteMutationFn, DeleteMutationFnParams, - InferSchemaInput, - InferSchemaOutput, - InsertMutationFn, InsertMutationFnParams, + ResolveInput, + ResolveType, SyncConfig, - UpdateMutationFn, UpdateMutationFnParams, UtilsRecord, } from "@tanstack/db" -import type { StandardSchemaV1 } from "@standard-schema/spec" // Re-export for external use export type { SyncOperation } from "./manual-sync" -// QueryFn return type inference helper -type InferQueryFn = TQueryFn extends ( +// Infer the explicit type from the queryFn return type +type InferExplicit = TQueryFn extends ( context: QueryFunctionContext ) => Promise> ? TItem extends object ? TItem - : Record - : Record - -type ResolveInput< - TExplicit extends object | unknown = unknown, - TSchema extends StandardSchemaV1 = never, - TQueryFn = unknown, -> = unknown extends TExplicit - ? [TSchema] extends [never] - ? InferQueryFn - : InferSchemaInput + : TExplicit : TExplicit -// Type resolution system with priority order (matches electric.ts pattern) -type ResolveType< - TExplicit extends object | unknown = unknown, - TSchema extends StandardSchemaV1 = never, - TQueryFn = unknown, -> = unknown extends TExplicit - ? [TSchema] extends [never] - ? InferQueryFn - : InferSchemaOutput - : TExplicit +// Resolve the item type from the queryFn return type +type ResolveItemType = ResolveType< + InferExplicit, + TSchema +> /** * Configuration options for creating a Query Collection - * @template TExplicit - The explicit type of items stored in the collection (highest priority) - * @template TSchema - The schema type for validation and type inference (second priority) + * @template TExplicit - The explicit type of items in the collection (second priority) + * @template TSchema - The schema type for validation and type inference (highest priority) + * @template TKey - The type of the key returned by getKey * @template TQueryFn - The queryFn type for inferring return type (third priority) * @template TError - The type of errors that can occur during queries * @template TQueryKey - The type of the query key */ export interface QueryCollectionConfig< - TExplicit extends object = object, - TSchema extends StandardSchemaV1 = never, + TExplicit extends object = Record, + TKey extends string | number = string | number, + TSchema = never, TQueryFn extends ( context: QueryFunctionContext ) => Promise> = ( @@ -79,8 +63,7 @@ export interface QueryCollectionConfig< ) => Promise>, TError = unknown, TQueryKey extends QueryKey = QueryKey, - TKey extends string | number = string | number, -> { +> extends BaseCollectionConfig { /** The query key used by TanStack Query to identify this query */ queryKey: TQueryKey /** Function that fetches data from the server. Must return the complete collection state */ @@ -90,7 +73,7 @@ export interface QueryCollectionConfig< ? TQueryFn : ( context: QueryFunctionContext - ) => Promise>> + ) => Promise>> /** The TanStack Query client instance */ queryClient: QueryClient @@ -99,188 +82,34 @@ export interface QueryCollectionConfig< /** Whether the query should automatically run (default: true) */ enabled?: boolean refetchInterval?: QueryObserverOptions< - Array>, + Array>, TError, - Array>, - Array>, + Array>, + Array>, TQueryKey >[`refetchInterval`] retry?: QueryObserverOptions< - Array>, + Array>, TError, - Array>, - Array>, + Array>, + Array>, TQueryKey >[`retry`] retryDelay?: QueryObserverOptions< - Array>, + Array>, TError, - Array>, - Array>, + Array>, + Array>, TQueryKey >[`retryDelay`] staleTime?: QueryObserverOptions< - Array>, + Array>, TError, - Array>, - Array>, + Array>, + Array>, TQueryKey >[`staleTime`] - // Standard Collection configuration properties - /** Unique identifier for the collection */ - id?: string - /** Function to extract the unique key from an item */ - getKey: (item: ResolveType) => TKey - /** Schema for validating items */ - schema?: TSchema - sync?: CollectionConfig>[`sync`] - startSync?: CollectionConfig< - ResolveType - >[`startSync`] - - // Direct persistence handlers - /** - * Optional asynchronous handler function called before an insert operation - * @param params Object containing transaction and collection information - * @returns Promise resolving to void or { refetch?: boolean } to control refetching - * @example - * // Basic query collection insert handler - * onInsert: async ({ transaction }) => { - * const newItem = transaction.mutations[0].modified - * await api.createTodo(newItem) - * // Automatically refetches query after insert - * } - * - * @example - * // Insert handler with refetch control - * onInsert: async ({ transaction }) => { - * const newItem = transaction.mutations[0].modified - * await api.createTodo(newItem) - * return { refetch: false } // Skip automatic refetch - * } - * - * @example - * // Insert handler with multiple items - * onInsert: async ({ transaction }) => { - * const items = transaction.mutations.map(m => m.modified) - * await api.createTodos(items) - * // Will refetch query to get updated data - * } - * - * @example - * // Insert handler with error handling - * onInsert: async ({ transaction }) => { - * try { - * const newItem = transaction.mutations[0].modified - * await api.createTodo(newItem) - * } catch (error) { - * console.error('Insert failed:', error) - * throw error // Transaction will rollback optimistic changes - * } - * } - */ - onInsert?: InsertMutationFn, TKey> - - /** - * Optional asynchronous handler function called before an update operation - * @param params Object containing transaction and collection information - * @returns Promise resolving to void or { refetch?: boolean } to control refetching - * @example - * // Basic query collection update handler - * onUpdate: async ({ transaction }) => { - * const mutation = transaction.mutations[0] - * await api.updateTodo(mutation.original.id, mutation.changes) - * // Automatically refetches query after update - * } - * - * @example - * // Update handler with multiple items - * onUpdate: async ({ transaction }) => { - * const updates = transaction.mutations.map(m => ({ - * id: m.key, - * changes: m.changes - * })) - * await api.updateTodos(updates) - * // Will refetch query to get updated data - * } - * - * @example - * // Update handler with manual refetch - * onUpdate: async ({ transaction, collection }) => { - * const mutation = transaction.mutations[0] - * await api.updateTodo(mutation.original.id, mutation.changes) - * - * // Manually trigger refetch - * await collection.utils.refetch() - * - * return { refetch: false } // Skip automatic refetch - * } - * - * @example - * // Update handler with related collection refetch - * onUpdate: async ({ transaction, collection }) => { - * const mutation = transaction.mutations[0] - * await api.updateTodo(mutation.original.id, mutation.changes) - * - * // Refetch related collections when this item changes - * await Promise.all([ - * collection.utils.refetch(), // Refetch this collection - * usersCollection.utils.refetch(), // Refetch users - * tagsCollection.utils.refetch() // Refetch tags - * ]) - * - * return { refetch: false } // Skip automatic refetch since we handled it manually - * } - */ - onUpdate?: UpdateMutationFn, TKey> - - /** - * Optional asynchronous handler function called before a delete operation - * @param params Object containing transaction and collection information - * @returns Promise resolving to void or { refetch?: boolean } to control refetching - * @example - * // Basic query collection delete handler - * onDelete: async ({ transaction }) => { - * const mutation = transaction.mutations[0] - * await api.deleteTodo(mutation.original.id) - * // Automatically refetches query after delete - * } - * - * @example - * // Delete handler with refetch control - * onDelete: async ({ transaction }) => { - * const mutation = transaction.mutations[0] - * await api.deleteTodo(mutation.original.id) - * return { refetch: false } // Skip automatic refetch - * } - * - * @example - * // Delete handler with multiple items - * onDelete: async ({ transaction }) => { - * const keysToDelete = transaction.mutations.map(m => m.key) - * await api.deleteTodos(keysToDelete) - * // Will refetch query to get updated data - * } - * - * @example - * // Delete handler with related collection refetch - * onDelete: async ({ transaction, collection }) => { - * const mutation = transaction.mutations[0] - * await api.deleteTodo(mutation.original.id) - * - * // Refetch related collections when this item is deleted - * await Promise.all([ - * collection.utils.refetch(), // Refetch this collection - * usersCollection.utils.refetch(), // Refetch users - * projectsCollection.utils.refetch() // Refetch projects - * ]) - * - * return { refetch: false } // Skip automatic refetch since we handled it manually - * } - */ - onDelete?: DeleteMutationFn, TKey> - /** * Metadata to pass to the query. * Available in queryFn via context.meta @@ -413,8 +242,9 @@ export interface QueryCollectionUtils< * ) */ export function queryCollectionOptions< - TExplicit extends object = object, - TSchema extends StandardSchemaV1 = never, + TExplicit extends object = Record, + TKey extends string | number = string | number, + TSchema = never, TQueryFn extends ( context: QueryFunctionContext ) => Promise> = ( @@ -422,30 +252,32 @@ export function queryCollectionOptions< ) => Promise>, TError = unknown, TQueryKey extends QueryKey = QueryKey, - TKey extends string | number = string | number, - TInput extends object = ResolveInput, + TInput extends object = ResolveInput< + InferExplicit, + TSchema + >, >( config: QueryCollectionConfig< TExplicit, + TKey, TSchema, TQueryFn, TError, - TQueryKey, - TKey + TQueryKey > ): CollectionConfig< - ResolveType, + InferExplicit, TKey, TSchema, TInput > & { utils: QueryCollectionUtils< - ResolveType, + ResolveItemType, TKey, TInput > } { - type TItem = ResolveType + type TItem = ResolveItemType const { queryKey, @@ -655,7 +487,7 @@ export function queryCollectionOptions< // Create wrapper handlers for direct persistence operations that handle refetching const wrappedOnInsert = onInsert ? async (params: InsertMutationFnParams) => { - const handlerResult = (await onInsert(params)) ?? {} + const handlerResult = (await onInsert(params as any)) ?? {} const shouldRefetch = (handlerResult as { refetch?: boolean }).refetch !== false @@ -669,7 +501,7 @@ export function queryCollectionOptions< const wrappedOnUpdate = onUpdate ? async (params: UpdateMutationFnParams) => { - const handlerResult = (await onUpdate(params)) ?? {} + const handlerResult = (await onUpdate(params as any)) ?? {} const shouldRefetch = (handlerResult as { refetch?: boolean }).refetch !== false @@ -683,7 +515,7 @@ export function queryCollectionOptions< const wrappedOnDelete = onDelete ? async (params: DeleteMutationFnParams) => { - const handlerResult = (await onDelete(params)) ?? {} + const handlerResult = (await onDelete(params as any)) ?? {} const shouldRefetch = (handlerResult as { refetch?: boolean }).refetch !== false From 3f6fc4d4dac8f2cac1c664dbd825204f2533c2e1 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Mon, 8 Sep 2025 09:16:18 +0100 Subject: [PATCH 15/15] trying derived collection with a brand --- packages/db/src/collection.ts | 49 +++++++++++++++--- packages/db/src/local-only.ts | 20 +++++--- packages/db/src/types.ts | 15 +++++- packages/db/tests/local-only.test-d.ts | 71 ++++++++++++++++++++++---- 4 files changed, 128 insertions(+), 27 deletions(-) diff --git a/packages/db/src/collection.ts b/packages/db/src/collection.ts index a6e907647..141ffec43 100644 --- a/packages/db/src/collection.ts +++ b/packages/db/src/collection.ts @@ -39,7 +39,6 @@ import { } from "./errors" import { createFilteredCallback, currentStateAsChanges } from "./change-events" import type { Transaction } from "./transactions" - import type { SingleRowRefProxy } from "./query/builder/ref-proxy" import type { ChangeListener, @@ -47,6 +46,7 @@ import type { CollectionConfig, CollectionStatus, CurrentStateAsChangesOptions, + DerivedCollectionConfig, Fn, InsertConfig, OperationConfig, @@ -168,13 +168,46 @@ export function createCollection< TUtils extends UtilsRecord = {}, TSchema = never, >( - options: CollectionConfig< - TExplicit, - TKey, - TSchema, - ResolveInput, - ResolveType - > & { utils?: TUtils } + options: CollectionConfig & { utils?: TUtils } & { + __derivedConfig?: never + } +): Collection< + ResolveType, + TKey, + TUtils, + TSchema, + ResolveInput +> + +// Overload for when the derived config is used, we infer the types from the config +export function createCollection< + TConfig extends DerivedCollectionConfig, + TUtils extends UtilsRecord = {}, +>( + options: TConfig & { utils?: TUtils } +): TConfig extends DerivedCollectionConfig< + infer TExplicit, + infer TKey, + infer TSchema +> + ? TConfig["__derivedType"] extends [never] + ? Collection< + ResolveType, + TKey, + TUtils, + TSchema, + ResolveInput + > + : Collection + : never + +export function createCollection< + TExplicit extends object = never, + TKey extends string | number = string | number, + TUtils extends UtilsRecord = {}, + TSchema = never, +>( + options: CollectionConfig & { utils?: TUtils } ): Collection< ResolveType, TKey, diff --git a/packages/db/src/local-only.ts b/packages/db/src/local-only.ts index 3571b7bd4..e5a4d76f3 100644 --- a/packages/db/src/local-only.ts +++ b/packages/db/src/local-only.ts @@ -1,6 +1,7 @@ import type { BaseCollectionConfig, CollectionConfig, + DerivedCollectionConfig, DeleteMutationFnParams, InsertMutationFnParams, OperationType, @@ -18,10 +19,12 @@ import type { * @template TKey - The type of the key returned by getKey */ export interface LocalOnlyCollectionConfig< - TExplicit extends object = Record, - TKey extends string | number = string | number, + TExplicit extends object = never, + TKey extends string | number = never, TSchema = never, -> extends BaseCollectionConfig { + TInput extends object = ResolveInput, + TResolvedType extends object = ResolveType, +> extends BaseCollectionConfig { /** * Optional initial data to populate the collection with on creation * This data will be applied during the initial sync process @@ -79,18 +82,17 @@ export interface LocalOnlyCollectionUtils extends UtilsRecord {} * }) * ) */ + export function localOnlyCollectionOptions< - TExplicit extends object = Record, + TExplicit extends object = never, TKey extends string | number = string | number, TSchema = never, >( config: LocalOnlyCollectionConfig -): CollectionConfig< +): DerivedCollectionConfig< TExplicit, TKey, - TSchema, - ResolveInput, - ResolveType + TSchema > & { utils: LocalOnlyCollectionUtils } { @@ -165,6 +167,8 @@ export function localOnlyCollectionOptions< utils: {} as LocalOnlyCollectionUtils, startSync: true, gcTime: 0, + } as DerivedCollectionConfig & { + utils: LocalOnlyCollectionUtils } } diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index 88c68102f..a8d003b73 100644 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -537,12 +537,25 @@ export type BaseCollectionConfig< T extends object = Record, TKey extends string | number = string | number, TSchema = never, -> = Omit, `sync`> + TInput extends object = ResolveInput, + TResolvedType extends object = ResolveType, +> = Omit, `sync`> export type ChangesPayload> = Array< ChangeMessage > +export type DerivedCollectionConfig< + T extends object = any, + TKey extends string | number = any, + TSchema = any, + TInput extends object = ResolveInput, + TResolvedType extends object = ResolveType, +> = CollectionConfig & { + __derivedConfig: true + __derivedType: T +} + /** * An input row from a collection */ diff --git a/packages/db/tests/local-only.test-d.ts b/packages/db/tests/local-only.test-d.ts index 659a31f81..a711c77e5 100644 --- a/packages/db/tests/local-only.test-d.ts +++ b/packages/db/tests/local-only.test-d.ts @@ -2,7 +2,6 @@ import { describe, expectTypeOf, it } from "vitest" import { z } from "zod" import { createCollection } from "../src/index" import { localOnlyCollectionOptions } from "../src/local-only" -import type { LocalOnlyCollectionUtils } from "../src/local-only" interface TestItem extends Record { id: number @@ -14,9 +13,9 @@ type ItemOf = T extends Array ? U : T describe(`LocalOnly Collection Types`, () => { it(`should have correct return type from localOnlyCollectionOptions`, () => { - const options = localOnlyCollectionOptions({ + const options = localOnlyCollectionOptions({ id: `test-local-only`, - getKey: (item: TestItem) => item.id, + getKey: (item) => item.id, }) // Test that options has the expected structure @@ -67,11 +66,7 @@ describe(`LocalOnly Collection Types`, () => { } const options = localOnlyCollectionOptions(configWithCallbacks) - const collection = createCollection< - TestItem, - number, - LocalOnlyCollectionUtils - >(options) + const collection = createCollection(options) // Test that the collection has the essential methods and properties expectTypeOf(collection.insert).toBeFunction() @@ -126,7 +121,63 @@ describe(`LocalOnly Collection Types`, () => { expectTypeOf(options.getKey).toBeFunction() }) - it(`should work with schema and infer correct types`, () => { + it(`should work with schema and infer correct types when saved to a variable`, () => { + const testSchema = z.object({ + id: z.string(), + entityId: z.string(), + value: z.string(), + createdAt: z.date().optional().default(new Date()), + }) + + // We can trust that zod infers the correct types for the schema + type ExpectedType = z.infer + type ExpectedInput = z.input + + const config = localOnlyCollectionOptions({ + id: `test-with-schema`, + getKey: (item) => item.id, + schema: testSchema, + onInsert: (params) => { + expectTypeOf( + params.transaction.mutations[0].modified + ).toEqualTypeOf() + return Promise.resolve() + }, + onUpdate: (params) => { + expectTypeOf( + params.transaction.mutations[0].modified + ).toEqualTypeOf() + return Promise.resolve() + }, + onDelete: (params) => { + expectTypeOf( + params.transaction.mutations[0].modified + ).toEqualTypeOf() + return Promise.resolve() + }, + }) + const collection = createCollection(config) + + collection.insert({ + id: `1`, + entityId: `1`, + value: `1`, + }) + + // Test insert parameter type + type InsertParam = Parameters[0] + expectTypeOf>().toEqualTypeOf() + + // Check that the update method accepts the expected input type + collection.update(`1`, (draft) => { + expectTypeOf(draft).toEqualTypeOf() + }) + + // Test that the collection has the correct inferred type from schema + expectTypeOf(collection.toArray).toEqualTypeOf>() + }) + + it(`should work with schema and infer correct types when nested in createCollection`, () => { const testSchema = z.object({ id: z.string(), entityId: z.string(), @@ -141,7 +192,7 @@ describe(`LocalOnly Collection Types`, () => { const collection = createCollection( localOnlyCollectionOptions({ id: `test-with-schema`, - getKey: (item: any) => item.id, + getKey: (item) => item.id, schema: testSchema, onInsert: (params) => { expectTypeOf(