diff --git a/.changeset/includes-parent-referencing-filters.md b/.changeset/includes-parent-referencing-filters.md new file mode 100644 index 000000000..78d209ad5 --- /dev/null +++ b/.changeset/includes-parent-referencing-filters.md @@ -0,0 +1,5 @@ +--- +'@tanstack/db': patch +--- + +feat: support parent-referencing WHERE filters in includes child queries diff --git a/packages/db/src/query/builder/index.ts b/packages/db/src/query/builder/index.ts index 7b7b6d3e8..d6a448f65 100644 --- a/packages/db/src/query/builder/index.ts +++ b/packages/db/src/query/builder/index.ts @@ -31,6 +31,7 @@ import type { OrderBy, OrderByDirection, QueryIR, + Where, } from '../ir.js' import type { CompareOptions, @@ -871,6 +872,39 @@ function buildNestedSelect(obj: any, parentAliases: Array = []): any { return out } +/** + * Recursively collects all PropRef nodes from an expression tree. + */ +function collectRefsFromExpression(expr: BasicExpression): Array { + const refs: Array = [] + switch (expr.type) { + case `ref`: + refs.push(expr) + break + case `func`: + for (const arg of (expr as any).args ?? []) { + refs.push(...collectRefsFromExpression(arg)) + } + break + default: + break + } + return refs +} + +/** + * Checks whether a WHERE clause references any parent alias. + */ +function referencesParent(where: Where, parentAliases: Array): boolean { + const expr = + typeof where === `object` && `expression` in where + ? where.expression + : where + return collectRefsFromExpression(expr).some( + (ref) => ref.path[0] != null && parentAliases.includes(ref.path[0]), + ) +} + /** * Builds an IncludesSubquery IR node from a child query builder. * Extracts the correlation condition from the child's WHERE clauses by finding @@ -891,10 +925,12 @@ function buildIncludesSubquery( } } - // Walk child's WHERE clauses to find the correlation condition + // Walk child's WHERE clauses to find the correlation condition. + // The correlation eq() may be a standalone WHERE or nested inside a top-level and(). let parentRef: PropRef | undefined let childRef: PropRef | undefined let correlationWhereIndex = -1 + let correlationAndArgIndex = -1 // >= 0 when found inside an and() if (childQuery.where) { for (let i = 0; i < childQuery.where.length; i++) { @@ -904,16 +940,15 @@ function buildIncludesSubquery( ? where.expression : where - // Look for eq(a, b) where one side references parent and other references child + // Try standalone eq() if ( expr.type === `func` && expr.name === `eq` && expr.args.length === 2 ) { - const [argA, argB] = expr.args const result = extractCorrelation( - argA!, - argB!, + expr.args[0]!, + expr.args[1]!, parentAliases, childAliases, ) @@ -924,6 +959,37 @@ function buildIncludesSubquery( break } } + + // Try inside top-level and() + if ( + expr.type === `func` && + expr.name === `and` && + expr.args.length >= 2 + ) { + for (let j = 0; j < expr.args.length; j++) { + const arg = expr.args[j]! + if ( + arg.type === `func` && + arg.name === `eq` && + arg.args.length === 2 + ) { + const result = extractCorrelation( + arg.args[0]!, + arg.args[1]!, + parentAliases, + childAliases, + ) + if (result) { + parentRef = result.parentRef + childRef = result.childRef + correlationWhereIndex = i + correlationAndArgIndex = j + break + } + } + } + if (parentRef) break + } } } @@ -935,15 +1001,82 @@ function buildIncludesSubquery( ) } - // Remove the correlation WHERE from the child query + // Remove the correlation eq() from the child query's WHERE clauses. + // If it was inside an and(), remove just that arg (collapsing the and() if needed). const modifiedWhere = [...childQuery.where!] - modifiedWhere.splice(correlationWhereIndex, 1) + if (correlationAndArgIndex >= 0) { + const where = modifiedWhere[correlationWhereIndex]! + const expr = + typeof where === `object` && `expression` in where + ? where.expression + : where + const remainingArgs = (expr as any).args.filter( + (_: any, idx: number) => idx !== correlationAndArgIndex, + ) + if (remainingArgs.length === 1) { + // Collapse and() with single remaining arg to just that expression + const isResidual = + typeof where === `object` && `expression` in where && where.residual + modifiedWhere[correlationWhereIndex] = isResidual + ? { expression: remainingArgs[0], residual: true } + : remainingArgs[0] + } else { + // Rebuild and() without the extracted arg + const newAnd = new FuncExpr(`and`, remainingArgs) + const isResidual = + typeof where === `object` && `expression` in where && where.residual + modifiedWhere[correlationWhereIndex] = isResidual + ? { expression: newAnd, residual: true } + : newAnd + } + } else { + modifiedWhere.splice(correlationWhereIndex, 1) + } + + // Separate remaining WHEREs into pure-child vs parent-referencing + const pureChildWhere: Array = [] + const parentFilters: Array = [] + for (const w of modifiedWhere) { + if (referencesParent(w, parentAliases)) { + parentFilters.push(w) + } else { + pureChildWhere.push(w) + } + } + + // Collect distinct parent PropRefs from parent-referencing filters + let parentProjection: Array | undefined + if (parentFilters.length > 0) { + const seen = new Set() + parentProjection = [] + for (const w of parentFilters) { + const expr = typeof w === `object` && `expression` in w ? w.expression : w + for (const ref of collectRefsFromExpression(expr)) { + if ( + ref.path[0] != null && + parentAliases.includes(ref.path[0]) && + !seen.has(ref.path.join(`.`)) + ) { + seen.add(ref.path.join(`.`)) + parentProjection.push(ref) + } + } + } + } + const modifiedQuery: QueryIR = { ...childQuery, - where: modifiedWhere.length > 0 ? modifiedWhere : undefined, + where: pureChildWhere.length > 0 ? pureChildWhere : undefined, } - return new IncludesSubquery(modifiedQuery, parentRef, childRef, fieldName) + return new IncludesSubquery( + modifiedQuery, + parentRef, + childRef, + fieldName, + parentFilters.length > 0 ? parentFilters : undefined, + parentProjection, + ) } /** diff --git a/packages/db/src/query/compiler/index.ts b/packages/db/src/query/compiler/index.ts index d749e6e3a..0001798cd 100644 --- a/packages/db/src/query/compiler/index.ts +++ b/packages/db/src/query/compiler/index.ts @@ -200,15 +200,20 @@ export function compileQuery( // Inner join: only children whose correlation key exists in parent keys pass through const joined = childRekeyed.pipe(joinOperator(parentKeyStream, `inner`)) - // Extract: [correlationValue, [[childKey, childRow], null]] → [childKey, childRow] + // Extract: [correlationValue, [[childKey, childRow], parentContext]] → [childKey, childRow] // Tag the row with __correlationKey for output routing + // If parentSide is non-null (parent context projected), attach as __parentContext filteredMainInput = joined.pipe( filter(([_correlationValue, [childSide]]: any) => { return childSide != null }), - map(([correlationValue, [childSide, _parentSide]]: any) => { + map(([correlationValue, [childSide, parentSide]]: any) => { const [childKey, childRow] = childSide - return [childKey, { ...childRow, __correlationKey: correlationValue }] + const tagged: any = { ...childRow, __correlationKey: correlationValue } + if (parentSide != null) { + tagged.__parentContext = parentSide + } + return [childKey, tagged] }), ) @@ -220,10 +225,14 @@ export function compileQuery( let pipeline: NamespacedAndKeyedStream = filteredMainInput.pipe( map(([key, row]) => { // Initialize the record with a nested structure - const ret = [key, { [mainSource]: row }] as [ - string, - Record, - ] + // If __parentContext exists (from parent-referencing includes), merge parent + // aliases into the namespaced row so WHERE can resolve parent refs + const { __parentContext, ...cleanRow } = row as any + const nsRow: Record = { [mainSource]: cleanRow } + if (__parentContext) { + Object.assign(nsRow, __parentContext) + } + const ret = [key, nsRow] as [string, Record] return ret }), ) @@ -285,15 +294,60 @@ export function compileQuery( if (query.select) { const includesEntries = extractIncludesFromSelect(query.select) for (const { key, subquery } of includesEntries) { - // Branch parent pipeline: map to [correlationValue, null] + // Branch parent pipeline: map to [correlationValue, parentContext] + // When parentProjection exists, project referenced parent fields; otherwise null (zero overhead) const compiledCorrelation = compileExpression(subquery.correlationField) - const parentKeys = pipeline.pipe( - map(([_key, nsRow]: any) => [compiledCorrelation(nsRow), null] as any), - ) + let parentKeys: any + if (subquery.parentProjection && subquery.parentProjection.length > 0) { + const compiledProjections = subquery.parentProjection.map((ref) => ({ + alias: ref.path[0]!, + field: ref.path.slice(1), + compiled: compileExpression(ref), + })) + parentKeys = pipeline.pipe( + map(([_key, nsRow]: any) => { + const parentContext: Record> = {} + for (const proj of compiledProjections) { + if (!parentContext[proj.alias]) { + parentContext[proj.alias] = {} + } + const value = proj.compiled(nsRow) + // Set nested field in the alias namespace + let target = parentContext[proj.alias]! + for (let i = 0; i < proj.field.length - 1; i++) { + if (!target[proj.field[i]!]) { + target[proj.field[i]!] = {} + } + target = target[proj.field[i]!] + } + target[proj.field[proj.field.length - 1]!] = value + } + return [compiledCorrelation(nsRow), parentContext] as any + }), + ) + } else { + parentKeys = pipeline.pipe( + map( + ([_key, nsRow]: any) => [compiledCorrelation(nsRow), null] as any, + ), + ) + } + + // If parent filters exist, append them to the child query's WHERE + const childQuery = + subquery.parentFilters && subquery.parentFilters.length > 0 + ? { + ...subquery.query, + where: [ + ...(subquery.query.where || []), + ...subquery.parentFilters, + ], + } + : subquery.query // Recursively compile child query WITH the parent key stream const childResult = compileQuery( - subquery.query, + childQuery, allInputs, collections, subscriptions, diff --git a/packages/db/src/query/ir.ts b/packages/db/src/query/ir.ts index 64ddd22c7..422bf4df5 100644 --- a/packages/db/src/query/ir.ts +++ b/packages/db/src/query/ir.ts @@ -139,6 +139,8 @@ export class IncludesSubquery extends BaseExpression { public correlationField: PropRef, // Parent-side ref (e.g., project.id) public childCorrelationField: PropRef, // Child-side ref (e.g., issue.projectId) public fieldName: string, // Result field name (e.g., "issues") + public parentFilters?: Array, // WHERE clauses referencing parent aliases (applied post-join) + public parentProjection?: Array, // Parent field refs used by parentFilters ) { super() } diff --git a/packages/db/tests/query/includes.test.ts b/packages/db/tests/query/includes.test.ts index 07a76a5c8..a7896f853 100644 --- a/packages/db/tests/query/includes.test.ts +++ b/packages/db/tests/query/includes.test.ts @@ -1,5 +1,5 @@ import { beforeEach, describe, expect, it } from 'vitest' -import { createLiveQueryCollection, eq } from '../../src/query/index.js' +import { and, createLiveQueryCollection, eq } from '../../src/query/index.js' import { createCollection } from '../../src/collection/index.js' import { mockSyncCollectionOptions } from '../utils.js' @@ -565,4 +565,384 @@ describe(`includes subqueries`, () => { ]) }) }) + + describe(`parent-referencing filters`, () => { + type ProjectWithCreator = { + id: number + name: string + createdBy: string + } + + type IssueWithCreator = { + id: number + projectId: number + title: string + createdBy: string + } + + const sampleProjectsWithCreator: Array = [ + { id: 1, name: `Alpha`, createdBy: `alice` }, + { id: 2, name: `Beta`, createdBy: `bob` }, + { id: 3, name: `Gamma`, createdBy: `alice` }, + ] + + const sampleIssuesWithCreator: Array = [ + { id: 10, projectId: 1, title: `Bug in Alpha`, createdBy: `alice` }, + { id: 11, projectId: 1, title: `Feature for Alpha`, createdBy: `bob` }, + { id: 20, projectId: 2, title: `Bug in Beta`, createdBy: `bob` }, + { id: 21, projectId: 2, title: `Feature for Beta`, createdBy: `alice` }, + { id: 30, projectId: 3, title: `Bug in Gamma`, createdBy: `alice` }, + ] + + function createProjectsWC() { + return createCollection( + mockSyncCollectionOptions({ + id: `includes-projects-wc`, + getKey: (p) => p.id, + initialData: sampleProjectsWithCreator, + }), + ) + } + + function createIssuesWC() { + return createCollection( + mockSyncCollectionOptions({ + id: `includes-issues-wc`, + getKey: (i) => i.id, + initialData: sampleIssuesWithCreator, + }), + ) + } + + let projectsWC: ReturnType + let issuesWC: ReturnType + + beforeEach(() => { + projectsWC = createProjectsWC() + issuesWC = createIssuesWC() + }) + + it(`filters children by parent-referencing eq()`, async () => { + const collection = createLiveQueryCollection((q) => + q.from({ p: projectsWC }).select(({ p }) => ({ + id: p.id, + name: p.name, + createdBy: p.createdBy, + issues: q + .from({ i: issuesWC }) + .where(({ i }) => eq(i.projectId, p.id)) + .where(({ i }) => eq(i.createdBy, p.createdBy)) + .select(({ i }) => ({ + id: i.id, + title: i.title, + createdBy: i.createdBy, + })), + })), + ) + + await collection.preload() + + expect(toTree(collection)).toEqual([ + { + id: 1, + name: `Alpha`, + createdBy: `alice`, + issues: [ + // Only issue 10 (createdBy: alice) matches project 1 (createdBy: alice) + { id: 10, title: `Bug in Alpha`, createdBy: `alice` }, + ], + }, + { + id: 2, + name: `Beta`, + createdBy: `bob`, + issues: [ + // Only issue 20 (createdBy: bob) matches project 2 (createdBy: bob) + { id: 20, title: `Bug in Beta`, createdBy: `bob` }, + ], + }, + { + id: 3, + name: `Gamma`, + createdBy: `alice`, + issues: [ + // Only issue 30 (createdBy: alice) matches project 3 (createdBy: alice) + { id: 30, title: `Bug in Gamma`, createdBy: `alice` }, + ], + }, + ]) + }) + + it(`reacts to parent field change`, async () => { + const collection = createLiveQueryCollection((q) => + q.from({ p: projectsWC }).select(({ p }) => ({ + id: p.id, + name: p.name, + createdBy: p.createdBy, + issues: q + .from({ i: issuesWC }) + .where(({ i }) => eq(i.projectId, p.id)) + .where(({ i }) => eq(i.createdBy, p.createdBy)) + .select(({ i }) => ({ + id: i.id, + title: i.title, + createdBy: i.createdBy, + })), + })), + ) + + await collection.preload() + + // Project 1 (createdBy: alice) → only issue 10 (alice) + expect(childItems((collection.get(1) as any).issues)).toEqual([ + { id: 10, title: `Bug in Alpha`, createdBy: `alice` }, + ]) + + // Change project 1 createdBy from alice to bob + projectsWC.utils.begin() + projectsWC.utils.write({ + type: `update`, + value: { id: 1, name: `Alpha`, createdBy: `bob` }, + oldValue: sampleProjectsWithCreator[0]!, + }) + projectsWC.utils.commit() + + // Now issue 11 (createdBy: bob) should match, issue 10 (alice) should not + expect(childItems((collection.get(1) as any).issues)).toEqual([ + { id: 11, title: `Feature for Alpha`, createdBy: `bob` }, + ]) + }) + + it(`reacts to child field change`, async () => { + const collection = createLiveQueryCollection((q) => + q.from({ p: projectsWC }).select(({ p }) => ({ + id: p.id, + name: p.name, + createdBy: p.createdBy, + issues: q + .from({ i: issuesWC }) + .where(({ i }) => eq(i.projectId, p.id)) + .where(({ i }) => eq(i.createdBy, p.createdBy)) + .select(({ i }) => ({ + id: i.id, + title: i.title, + createdBy: i.createdBy, + })), + })), + ) + + await collection.preload() + + // Project 1 (alice) → only issue 10 + expect(childItems((collection.get(1) as any).issues)).toEqual([ + { id: 10, title: `Bug in Alpha`, createdBy: `alice` }, + ]) + + // Change issue 11's createdBy from bob to alice → it should now appear + issuesWC.utils.begin() + issuesWC.utils.write({ + type: `update`, + value: { + id: 11, + projectId: 1, + title: `Feature for Alpha`, + createdBy: `alice`, + }, + oldValue: sampleIssuesWithCreator[1]!, + }) + issuesWC.utils.commit() + + expect(childItems((collection.get(1) as any).issues)).toEqual([ + { id: 10, title: `Bug in Alpha`, createdBy: `alice` }, + { id: 11, title: `Feature for Alpha`, createdBy: `alice` }, + ]) + }) + + it(`mixed filters: parent-referencing + pure-child`, async () => { + const collection = createLiveQueryCollection((q) => + q.from({ p: projectsWC }).select(({ p }) => ({ + id: p.id, + name: p.name, + createdBy: p.createdBy, + issues: q + .from({ i: issuesWC }) + .where(({ i }) => eq(i.projectId, p.id)) + .where(({ i }) => eq(i.createdBy, p.createdBy)) + .where(({ i }) => eq(i.title, `Bug in Alpha`)) + .select(({ i }) => ({ + id: i.id, + title: i.title, + createdBy: i.createdBy, + })), + })), + ) + + await collection.preload() + + // Project 1 (alice): matching createdBy + title = only issue 10 + expect(childItems((collection.get(1) as any).issues)).toEqual([ + { id: 10, title: `Bug in Alpha`, createdBy: `alice` }, + ]) + + // Project 2 (bob): no issues with title "Bug in Alpha" + expect(childItems((collection.get(2) as any).issues)).toEqual([]) + + // Project 3 (alice): no issues with title "Bug in Alpha" + expect(childItems((collection.get(3) as any).issues)).toEqual([]) + }) + + it(`extracts correlation from and() with a pure-child filter`, async () => { + // and(correlation, childFilter) in a single .where() — no parent ref + const collection = createLiveQueryCollection((q) => + q.from({ p: projectsWC }).select(({ p }) => ({ + id: p.id, + name: p.name, + issues: q + .from({ i: issuesWC }) + .where(({ i }) => + and(eq(i.projectId, p.id), eq(i.createdBy, `alice`)), + ) + .select(({ i }) => ({ + id: i.id, + title: i.title, + })), + })), + ) + + await collection.preload() + + expect(toTree(collection)).toEqual([ + { + id: 1, + name: `Alpha`, + issues: [{ id: 10, title: `Bug in Alpha` }], + }, + { + id: 2, + name: `Beta`, + issues: [{ id: 21, title: `Feature for Beta` }], + }, + { + id: 3, + name: `Gamma`, + issues: [{ id: 30, title: `Bug in Gamma` }], + }, + ]) + }) + + it(`reactivity works when correlation is inside and()`, async () => { + const collection = createLiveQueryCollection((q) => + q.from({ p: projectsWC }).select(({ p }) => ({ + id: p.id, + name: p.name, + createdBy: p.createdBy, + issues: q + .from({ i: issuesWC }) + .where(({ i }) => + and(eq(i.projectId, p.id), eq(i.createdBy, p.createdBy)), + ) + .select(({ i }) => ({ + id: i.id, + title: i.title, + createdBy: i.createdBy, + })), + })), + ) + + await collection.preload() + + expect(childItems((collection.get(1) as any).issues)).toEqual([ + { id: 10, title: `Bug in Alpha`, createdBy: `alice` }, + ]) + + // Change project 1 createdBy from alice to bob → issue 11 should match instead + projectsWC.utils.begin() + projectsWC.utils.write({ + type: `update`, + value: { id: 1, name: `Alpha`, createdBy: `bob` }, + oldValue: sampleProjectsWithCreator[0]!, + }) + projectsWC.utils.commit() + + expect(childItems((collection.get(1) as any).issues)).toEqual([ + { id: 11, title: `Feature for Alpha`, createdBy: `bob` }, + ]) + }) + + it(`extracts correlation from inside and()`, async () => { + const collection = createLiveQueryCollection((q) => + q.from({ p: projectsWC }).select(({ p }) => ({ + id: p.id, + name: p.name, + createdBy: p.createdBy, + issues: q + .from({ i: issuesWC }) + .where(({ i }) => + and(eq(i.projectId, p.id), eq(i.createdBy, p.createdBy)), + ) + .select(({ i }) => ({ + id: i.id, + title: i.title, + createdBy: i.createdBy, + })), + })), + ) + + await collection.preload() + + expect(toTree(collection)).toEqual([ + { + id: 1, + name: `Alpha`, + createdBy: `alice`, + issues: [{ id: 10, title: `Bug in Alpha`, createdBy: `alice` }], + }, + { + id: 2, + name: `Beta`, + createdBy: `bob`, + issues: [{ id: 20, title: `Bug in Beta`, createdBy: `bob` }], + }, + { + id: 3, + name: `Gamma`, + createdBy: `alice`, + issues: [{ id: 30, title: `Bug in Gamma`, createdBy: `alice` }], + }, + ]) + }) + + it(`extracts correlation from and() with more than 2 args`, async () => { + const collection = createLiveQueryCollection((q) => + q.from({ p: projectsWC }).select(({ p }) => ({ + id: p.id, + name: p.name, + createdBy: p.createdBy, + issues: q + .from({ i: issuesWC }) + .where(({ i }) => + and( + eq(i.projectId, p.id), + eq(i.createdBy, p.createdBy), + eq(i.title, `Bug in Alpha`), + ), + ) + .select(({ i }) => ({ + id: i.id, + title: i.title, + createdBy: i.createdBy, + })), + })), + ) + + await collection.preload() + + // Only project 1 (alice) has an issue matching all three conditions + expect(childItems((collection.get(1) as any).issues)).toEqual([ + { id: 10, title: `Bug in Alpha`, createdBy: `alice` }, + ]) + expect(childItems((collection.get(2) as any).issues)).toEqual([]) + expect(childItems((collection.get(3) as any).issues)).toEqual([]) + }) + }) })