From 3ac6d999dbd02129e657270c7273bf311548f22a Mon Sep 17 00:00:00 2001 From: Daniel Riccio Date: Mon, 28 Jul 2025 09:04:55 -0500 Subject: [PATCH 1/4] fix: handle Qdrant deletion errors gracefully to prevent indexing interruption - Fixed deletePointsByMultipleFilePaths to use pathSegments instead of filePath field (filePath field lacks required index causing HTTP 400 errors) - Added graceful error handling for HTTP 400 errors in deletion operations - Enhanced error logging with status codes and detailed error information - Updated scanner.ts and file-watcher.ts to handle deletion failures without stopping - Ensures indexing process continues even when points don't exist in vector store This resolves the issue where bad requests during point deletion would halt the entire indexing process. --- .../code-index/processors/file-watcher.ts | 46 ++++++++--- src/services/code-index/processors/scanner.ts | 53 +++++++++--- .../code-index/vector-store/qdrant-client.ts | 82 ++++++++++++++++--- 3 files changed, 144 insertions(+), 37 deletions(-) diff --git a/src/services/code-index/processors/file-watcher.ts b/src/services/code-index/processors/file-watcher.ts index 6dc1cd1835d..36c35a1a994 100644 --- a/src/services/code-index/processors/file-watcher.ts +++ b/src/services/code-index/processors/file-watcher.ts @@ -204,23 +204,47 @@ export class FileWatcher implements IFileWatcher { currentFile: path, }) } - } catch (error) { - overallBatchError = error as Error + } catch (error: any) { + const errorStatus = error?.status || error?.response?.status || error?.statusCode + const errorMessage = error instanceof Error ? error.message : String(error) + // Log telemetry for deletion error TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { - error: sanitizeErrorMessage(overallBatchError.message), + error: sanitizeErrorMessage(errorMessage), location: "deletePointsByMultipleFilePaths", errorType: "deletion_error", + errorStatus: errorStatus, }) - for (const path of pathsToExplicitlyDelete) { - batchResults.push({ path, status: "error", error: error as Error }) - processedCountInBatch++ - this._onBatchProgressUpdate.fire({ - processedInBatch: processedCountInBatch, - totalInBatch: totalFilesInBatch, - currentFile: path, - }) + // Check if this is a bad request error that we should handle gracefully + if (errorStatus === 400 || errorMessage.toLowerCase().includes("bad request")) { + console.warn( + `[FileWatcher] Received bad request error during deletion for ${allPathsToClearFromDB.size} files. Treating as successful deletion...`, + ) + // Treat as successful deletion - remove from cache and mark as success + for (const path of pathsToExplicitlyDelete) { + this.cacheManager.deleteHash(path) + batchResults.push({ path, status: "success" }) + processedCountInBatch++ + this._onBatchProgressUpdate.fire({ + processedInBatch: processedCountInBatch, + totalInBatch: totalFilesInBatch, + currentFile: path, + }) + } + } else { + // For other errors, mark as error but don't set overallBatchError + // This allows the rest of the batch to continue processing + overallBatchError = error as Error + for (const path of pathsToExplicitlyDelete) { + batchResults.push({ path, status: "error", error: error as Error }) + processedCountInBatch++ + this._onBatchProgressUpdate.fire({ + processedInBatch: processedCountInBatch, + totalInBatch: totalFilesInBatch, + currentFile: path, + }) + } } } } diff --git a/src/services/code-index/processors/scanner.ts b/src/services/code-index/processors/scanner.ts index 3203076d128..c28617c1529 100644 --- a/src/services/code-index/processors/scanner.ts +++ b/src/services/code-index/processors/scanner.ts @@ -281,17 +281,31 @@ export class DirectoryScanner implements IDirectoryScanner { try { await this.qdrantClient.deletePointsByFilePath(cachedFilePath) await this.cacheManager.deleteHash(cachedFilePath) - } catch (error) { + } catch (error: any) { + const errorStatus = error?.status || error?.response?.status || error?.statusCode + const errorMessage = error instanceof Error ? error.message : String(error) + console.error( `[DirectoryScanner] Failed to delete points for ${cachedFilePath} in workspace ${scanWorkspace}:`, error, ) + TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { - error: sanitizeErrorMessage(error instanceof Error ? error.message : String(error)), + error: sanitizeErrorMessage(errorMessage), stack: error instanceof Error ? sanitizeErrorMessage(error.stack || "") : undefined, location: "scanDirectory:deleteRemovedFiles", + errorStatus: errorStatus, }) - if (onError) { + + // Check if this is a bad request error that we should handle gracefully + if (errorStatus === 400 || errorMessage.toLowerCase().includes("bad request")) { + console.warn( + `[DirectoryScanner] Received bad request error when deleting ${cachedFilePath}. File may not exist in vector store. Removing from cache anyway...`, + ) + // Still remove from cache even if vector store deletion failed + await this.cacheManager.deleteHash(cachedFilePath) + } else if (onError) { + // For other errors, report to error handler but don't throw onError( error instanceof Error ? new Error( @@ -304,7 +318,7 @@ export class DirectoryScanner implements IDirectoryScanner { ), ) } - // Decide if we should re-throw or just log + // Don't re-throw - allow scanning to continue } } } @@ -347,27 +361,40 @@ export class DirectoryScanner implements IDirectoryScanner { if (uniqueFilePaths.length > 0) { try { await this.qdrantClient.deletePointsByMultipleFilePaths(uniqueFilePaths) - } catch (deleteError) { + } catch (deleteError: any) { + const errorStatus = + deleteError?.status || deleteError?.response?.status || deleteError?.statusCode + const errorMessage = deleteError instanceof Error ? deleteError.message : String(deleteError) + console.error( `[DirectoryScanner] Failed to delete points for ${uniqueFilePaths.length} files before upsert in workspace ${scanWorkspace}:`, deleteError, ) + TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { - error: sanitizeErrorMessage( - deleteError instanceof Error ? deleteError.message : String(deleteError), - ), + error: sanitizeErrorMessage(errorMessage), stack: deleteError instanceof Error ? sanitizeErrorMessage(deleteError.stack || "") : undefined, location: "processBatch:deletePointsByMultipleFilePaths", fileCount: uniqueFilePaths.length, + errorStatus: errorStatus, }) - // Re-throw the error with workspace context - throw new Error( - `Failed to delete points for ${uniqueFilePaths.length} files. Workspace: ${scanWorkspace}. ${deleteError instanceof Error ? deleteError.message : String(deleteError)}`, - { cause: deleteError }, - ) + + // Check if this is a bad request error that we should handle gracefully + if (errorStatus === 400 || errorMessage.toLowerCase().includes("bad request")) { + console.warn( + `[DirectoryScanner] Received bad request error during deletion. Continuing with upsert operation...`, + ) + // Don't throw - continue with the upsert operation + } else { + // For other errors, re-throw with workspace context + throw new Error( + `Failed to delete points for ${uniqueFilePaths.length} files. Workspace: ${scanWorkspace}. ${errorMessage}`, + { cause: deleteError }, + ) + } } } // --- End Deletion Step --- diff --git a/src/services/code-index/vector-store/qdrant-client.ts b/src/services/code-index/vector-store/qdrant-client.ts index 5121d65b97c..b8b12655f6c 100644 --- a/src/services/code-index/vector-store/qdrant-client.ts +++ b/src/services/code-index/vector-store/qdrant-client.ts @@ -414,7 +414,15 @@ export class QdrantVectorStore implements IVectorStore { * @param filePath Path of the file to delete points for */ async deletePointsByFilePath(filePath: string): Promise { - return this.deletePointsByMultipleFilePaths([filePath]) + try { + return await this.deletePointsByMultipleFilePaths([filePath]) + } catch (error) { + // Error is already handled in deletePointsByMultipleFilePaths + // This is just for consistency in case the method is called directly + console.error(`[QdrantVectorStore] Error in deletePointsByFilePath for ${filePath}:`, error) + // Re-throw to maintain the interface contract + throw error + } } async deletePointsByMultipleFilePaths(filePaths: string[]): Promise { @@ -423,27 +431,75 @@ export class QdrantVectorStore implements IVectorStore { } try { + // First check if the collection exists + const collectionExists = await this.collectionExists() + if (!collectionExists) { + console.warn( + `[QdrantVectorStore] Skipping deletion - collection "${this.collectionName}" does not exist`, + ) + return + } + const workspaceRoot = getWorkspacePath() - const normalizedPaths = filePaths.map((filePath) => { + + // Build filters using pathSegments to match the indexed fields + const filters = filePaths.map((filePath) => { const absolutePath = path.resolve(workspaceRoot, filePath) - return path.normalize(absolutePath) + const normalizedPath = path.normalize(absolutePath) + + // Split the path into segments like we do in upsertPoints + const segments = normalizedPath.split(path.sep).filter(Boolean) + + // Create a filter that matches all segments of the path + // This ensures we only delete points that match the exact file path + const mustConditions = segments.map((segment, index) => ({ + key: `pathSegments.${index}`, + match: { value: segment }, + })) + + return { must: mustConditions } }) - const filter = { - should: normalizedPaths.map((normalizedPath) => ({ - key: "filePath", - match: { - value: normalizedPath, - }, - })), - } + // Log the paths being deleted for debugging + console.log( + `[QdrantVectorStore] Attempting to delete points for ${filePaths.length} file(s) from collection "${this.collectionName}"`, + ) + + // Use 'should' to match any of the file paths (OR condition) + const filter = filters.length === 1 ? filters[0] : { should: filters } await this.client.delete(this.collectionName, { filter, wait: true, }) - } catch (error) { - console.error("Failed to delete points by file paths:", error) + + console.log(`[QdrantVectorStore] Successfully deleted points for ${filePaths.length} file(s)`) + } catch (error: any) { + // Extract more detailed error information + const errorMessage = error?.message || String(error) + const errorStatus = error?.status || error?.response?.status || error?.statusCode + const errorDetails = error?.response?.data || error?.data || "" + + console.error(`[QdrantVectorStore] Failed to delete points by file paths:`, { + error: errorMessage, + status: errorStatus, + details: errorDetails, + collection: this.collectionName, + fileCount: filePaths.length, + // Include first few file paths for debugging (avoid logging too many) + samplePaths: filePaths.slice(0, 3), + }) + + // Check if this is a "bad request" error that we can handle gracefully + if (errorStatus === 400 || errorMessage.toLowerCase().includes("bad request")) { + console.warn( + `[QdrantVectorStore] Received bad request error during deletion. This might indicate the points don't exist or the filter is invalid. Continuing without throwing...`, + ) + // Don't throw the error - allow the indexing process to continue + return + } + + // For other errors, still throw them throw error } } From 2a49fdb970f11d3b0c7f19f28d3c0f72d2f089fc Mon Sep 17 00:00:00 2001 From: Daniel Riccio Date: Tue, 29 Jul 2025 09:32:39 -0500 Subject: [PATCH 2/4] fix: use relative paths in Qdrant deletion to match stored pathSegments - Fixed critical bug where deletions used absolute paths while insertions used relative paths - This mismatch caused deletion filters to never match any points - Removed error swallowing for 400 errors now that root cause is fixed - Added debug logging to help verify correct behavior - Deletions now actually work as intended --- .../code-index/processors/file-watcher.ts | 39 +++++-------------- src/services/code-index/processors/scanner.ts | 32 +++++---------- .../code-index/vector-store/qdrant-client.ts | 20 ++++------ 3 files changed, 26 insertions(+), 65 deletions(-) diff --git a/src/services/code-index/processors/file-watcher.ts b/src/services/code-index/processors/file-watcher.ts index 36c35a1a994..c59d4714498 100644 --- a/src/services/code-index/processors/file-watcher.ts +++ b/src/services/code-index/processors/file-watcher.ts @@ -216,35 +216,16 @@ export class FileWatcher implements IFileWatcher { errorStatus: errorStatus, }) - // Check if this is a bad request error that we should handle gracefully - if (errorStatus === 400 || errorMessage.toLowerCase().includes("bad request")) { - console.warn( - `[FileWatcher] Received bad request error during deletion for ${allPathsToClearFromDB.size} files. Treating as successful deletion...`, - ) - // Treat as successful deletion - remove from cache and mark as success - for (const path of pathsToExplicitlyDelete) { - this.cacheManager.deleteHash(path) - batchResults.push({ path, status: "success" }) - processedCountInBatch++ - this._onBatchProgressUpdate.fire({ - processedInBatch: processedCountInBatch, - totalInBatch: totalFilesInBatch, - currentFile: path, - }) - } - } else { - // For other errors, mark as error but don't set overallBatchError - // This allows the rest of the batch to continue processing - overallBatchError = error as Error - for (const path of pathsToExplicitlyDelete) { - batchResults.push({ path, status: "error", error: error as Error }) - processedCountInBatch++ - this._onBatchProgressUpdate.fire({ - processedInBatch: processedCountInBatch, - totalInBatch: totalFilesInBatch, - currentFile: path, - }) - } + // Mark all paths as error + overallBatchError = error as Error + for (const path of pathsToExplicitlyDelete) { + batchResults.push({ path, status: "error", error: error as Error }) + processedCountInBatch++ + this._onBatchProgressUpdate.fire({ + processedInBatch: processedCountInBatch, + totalInBatch: totalFilesInBatch, + currentFile: path, + }) } } } diff --git a/src/services/code-index/processors/scanner.ts b/src/services/code-index/processors/scanner.ts index c28617c1529..e151d25dc16 100644 --- a/src/services/code-index/processors/scanner.ts +++ b/src/services/code-index/processors/scanner.ts @@ -297,15 +297,8 @@ export class DirectoryScanner implements IDirectoryScanner { errorStatus: errorStatus, }) - // Check if this is a bad request error that we should handle gracefully - if (errorStatus === 400 || errorMessage.toLowerCase().includes("bad request")) { - console.warn( - `[DirectoryScanner] Received bad request error when deleting ${cachedFilePath}. File may not exist in vector store. Removing from cache anyway...`, - ) - // Still remove from cache even if vector store deletion failed - await this.cacheManager.deleteHash(cachedFilePath) - } else if (onError) { - // For other errors, report to error handler but don't throw + if (onError) { + // Report error to error handler onError( error instanceof Error ? new Error( @@ -318,7 +311,8 @@ export class DirectoryScanner implements IDirectoryScanner { ), ) } - // Don't re-throw - allow scanning to continue + // Re-throw to maintain consistent error handling + throw error } } } @@ -382,19 +376,11 @@ export class DirectoryScanner implements IDirectoryScanner { errorStatus: errorStatus, }) - // Check if this is a bad request error that we should handle gracefully - if (errorStatus === 400 || errorMessage.toLowerCase().includes("bad request")) { - console.warn( - `[DirectoryScanner] Received bad request error during deletion. Continuing with upsert operation...`, - ) - // Don't throw - continue with the upsert operation - } else { - // For other errors, re-throw with workspace context - throw new Error( - `Failed to delete points for ${uniqueFilePaths.length} files. Workspace: ${scanWorkspace}. ${errorMessage}`, - { cause: deleteError }, - ) - } + // Re-throw with workspace context + throw new Error( + `Failed to delete points for ${uniqueFilePaths.length} files. Workspace: ${scanWorkspace}. ${errorMessage}`, + { cause: deleteError }, + ) } } // --- End Deletion Step --- diff --git a/src/services/code-index/vector-store/qdrant-client.ts b/src/services/code-index/vector-store/qdrant-client.ts index b8b12655f6c..73110285e90 100644 --- a/src/services/code-index/vector-store/qdrant-client.ts +++ b/src/services/code-index/vector-store/qdrant-client.ts @@ -444,11 +444,15 @@ export class QdrantVectorStore implements IVectorStore { // Build filters using pathSegments to match the indexed fields const filters = filePaths.map((filePath) => { - const absolutePath = path.resolve(workspaceRoot, filePath) - const normalizedPath = path.normalize(absolutePath) + // IMPORTANT: Use the relative path to match what's stored in upsertPoints + // upsertPoints stores the relative filePath, not the absolute path + const relativePath = path.isAbsolute(filePath) ? path.relative(workspaceRoot, filePath) : filePath + + // Normalize the relative path + const normalizedRelativePath = path.normalize(relativePath) // Split the path into segments like we do in upsertPoints - const segments = normalizedPath.split(path.sep).filter(Boolean) + const segments = normalizedRelativePath.split(path.sep).filter(Boolean) // Create a filter that matches all segments of the path // This ensures we only delete points that match the exact file path @@ -490,16 +494,6 @@ export class QdrantVectorStore implements IVectorStore { samplePaths: filePaths.slice(0, 3), }) - // Check if this is a "bad request" error that we can handle gracefully - if (errorStatus === 400 || errorMessage.toLowerCase().includes("bad request")) { - console.warn( - `[QdrantVectorStore] Received bad request error during deletion. This might indicate the points don't exist or the filter is invalid. Continuing without throwing...`, - ) - // Don't throw the error - allow the indexing process to continue - return - } - - // For other errors, still throw them throw error } } From b2bf880e68b21c62902bd8bce4bddd41df26cf65 Mon Sep 17 00:00:00 2001 From: Daniel Riccio Date: Tue, 29 Jul 2025 10:00:53 -0500 Subject: [PATCH 3/4] chore: remove debug logging and simplify deletePointsByFilePath - Removed unnecessary try-catch wrapper in deletePointsByFilePath - Removed debug console.log statements from deletion methods - Code is cleaner and more production-ready --- .../code-index/vector-store/qdrant-client.ts | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/src/services/code-index/vector-store/qdrant-client.ts b/src/services/code-index/vector-store/qdrant-client.ts index 73110285e90..0218e372959 100644 --- a/src/services/code-index/vector-store/qdrant-client.ts +++ b/src/services/code-index/vector-store/qdrant-client.ts @@ -414,15 +414,7 @@ export class QdrantVectorStore implements IVectorStore { * @param filePath Path of the file to delete points for */ async deletePointsByFilePath(filePath: string): Promise { - try { - return await this.deletePointsByMultipleFilePaths([filePath]) - } catch (error) { - // Error is already handled in deletePointsByMultipleFilePaths - // This is just for consistency in case the method is called directly - console.error(`[QdrantVectorStore] Error in deletePointsByFilePath for ${filePath}:`, error) - // Re-throw to maintain the interface contract - throw error - } + return this.deletePointsByMultipleFilePaths([filePath]) } async deletePointsByMultipleFilePaths(filePaths: string[]): Promise { @@ -464,11 +456,6 @@ export class QdrantVectorStore implements IVectorStore { return { must: mustConditions } }) - // Log the paths being deleted for debugging - console.log( - `[QdrantVectorStore] Attempting to delete points for ${filePaths.length} file(s) from collection "${this.collectionName}"`, - ) - // Use 'should' to match any of the file paths (OR condition) const filter = filters.length === 1 ? filters[0] : { should: filters } @@ -476,8 +463,6 @@ export class QdrantVectorStore implements IVectorStore { filter, wait: true, }) - - console.log(`[QdrantVectorStore] Successfully deleted points for ${filePaths.length} file(s)`) } catch (error: any) { // Extract more detailed error information const errorMessage = error?.message || String(error) From eb76b326aa4d82c5ab9733cfe5d6a6485b29915f Mon Sep 17 00:00:00 2001 From: Daniel Riccio Date: Tue, 29 Jul 2025 11:18:17 -0500 Subject: [PATCH 4/4] fix: log error and continue processing on failure to delete points for removed files --- src/services/code-index/processors/scanner.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/services/code-index/processors/scanner.ts b/src/services/code-index/processors/scanner.ts index e151d25dc16..27362b8b746 100644 --- a/src/services/code-index/processors/scanner.ts +++ b/src/services/code-index/processors/scanner.ts @@ -311,8 +311,8 @@ export class DirectoryScanner implements IDirectoryScanner { ), ) } - // Re-throw to maintain consistent error handling - throw error + // Log error and continue processing instead of re-throwing + console.error(`Failed to delete points for removed file: ${cachedFilePath}`, error) } } }