From 9750eaa6b33f8a7b21d3cb6e19941dbdc65cc2a7 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Thu, 9 Apr 2026 15:16:37 -0700 Subject: [PATCH 1/2] fix(log): log cleanup sql query --- apps/sim/app/api/logs/cleanup/route.ts | 40 +++++++++----------------- 1 file changed, 14 insertions(+), 26 deletions(-) diff --git a/apps/sim/app/api/logs/cleanup/route.ts b/apps/sim/app/api/logs/cleanup/route.ts index 25a0acabf55..1322ca96bce 100644 --- a/apps/sim/app/api/logs/cleanup/route.ts +++ b/apps/sim/app/api/logs/cleanup/route.ts @@ -1,5 +1,5 @@ import { db } from '@sim/db' -import { subscription, user, workflowExecutionLogs, workspace } from '@sim/db/schema' +import { subscription, workflowExecutionLogs, workspace } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { and, eq, inArray, isNull, lt } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' @@ -26,38 +26,26 @@ export async function GET(request: NextRequest) { const retentionDate = new Date() retentionDate.setDate(retentionDate.getDate() - Number(env.FREE_PLAN_LOG_RETENTION_DAYS || '7')) - const freeUsers = await db - .select({ userId: user.id }) - .from(user) + /** + * Subquery: workspace IDs whose billed account user has no active paid + * subscription. Kept as a subquery (not materialized into JS) so the + * generated SQL is `WHERE workspace_id IN (SELECT ...)` — this avoids + * PostgreSQL's 65535 bind-parameter limit that was breaking cleanup once + * the free-user count grew beyond ~65k. + */ + const freeWorkspacesSubquery = db + .select({ id: workspace.id }) + .from(workspace) .leftJoin( subscription, and( - eq(user.id, subscription.referenceId), + eq(subscription.referenceId, workspace.billedAccountUserId), inArray(subscription.status, ENTITLED_SUBSCRIPTION_STATUSES), sqlIsPaid(subscription.plan) ) ) .where(isNull(subscription.id)) - if (freeUsers.length === 0) { - logger.info('No free users found for log cleanup') - return NextResponse.json({ message: 'No free users found for cleanup' }) - } - - const freeUserIds = freeUsers.map((u) => u.userId) - - const workspacesQuery = await db - .select({ id: workspace.id }) - .from(workspace) - .where(inArray(workspace.billedAccountUserId, freeUserIds)) - - if (workspacesQuery.length === 0) { - logger.info('No workspaces found for free users') - return NextResponse.json({ message: 'No workspaces found for cleanup' }) - } - - const workspaceIds = workspacesQuery.map((w) => w.id) - const results = { enhancedLogs: { total: 0, @@ -83,7 +71,7 @@ export async function GET(request: NextRequest) { let batchesProcessed = 0 let hasMoreLogs = true - logger.info(`Starting enhanced logs cleanup for ${workspaceIds.length} workspaces`) + logger.info('Starting enhanced logs cleanup for free-plan workspaces') while (hasMoreLogs && batchesProcessed < MAX_BATCHES) { const oldEnhancedLogs = await db @@ -105,7 +93,7 @@ export async function GET(request: NextRequest) { .from(workflowExecutionLogs) .where( and( - inArray(workflowExecutionLogs.workspaceId, workspaceIds), + inArray(workflowExecutionLogs.workspaceId, freeWorkspacesSubquery), lt(workflowExecutionLogs.createdAt, retentionDate) ) ) From cafe8fe1f9c7e58f4044e10c5354f643ae86a77c Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 9 Apr 2026 17:46:44 -0700 Subject: [PATCH 2/2] perf(log): use startedAt index for cleanup query filter Switch cleanup WHERE clause from createdAt to startedAt to leverage the existing composite index (workspaceId, startedAt), converting a full table scan to an index range scan. Also remove explanatory comment. Co-Authored-By: Claude Opus 4.6 --- apps/sim/app/api/logs/cleanup/route.ts | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/apps/sim/app/api/logs/cleanup/route.ts b/apps/sim/app/api/logs/cleanup/route.ts index 1322ca96bce..85623e7d2a8 100644 --- a/apps/sim/app/api/logs/cleanup/route.ts +++ b/apps/sim/app/api/logs/cleanup/route.ts @@ -26,13 +26,6 @@ export async function GET(request: NextRequest) { const retentionDate = new Date() retentionDate.setDate(retentionDate.getDate() - Number(env.FREE_PLAN_LOG_RETENTION_DAYS || '7')) - /** - * Subquery: workspace IDs whose billed account user has no active paid - * subscription. Kept as a subquery (not materialized into JS) so the - * generated SQL is `WHERE workspace_id IN (SELECT ...)` — this avoids - * PostgreSQL's 65535 bind-parameter limit that was breaking cleanup once - * the free-user count grew beyond ~65k. - */ const freeWorkspacesSubquery = db .select({ id: workspace.id }) .from(workspace) @@ -94,7 +87,7 @@ export async function GET(request: NextRequest) { .where( and( inArray(workflowExecutionLogs.workspaceId, freeWorkspacesSubquery), - lt(workflowExecutionLogs.createdAt, retentionDate) + lt(workflowExecutionLogs.startedAt, retentionDate) ) ) .limit(BATCH_SIZE)