From 4acd1013296761a20bdb230bd47b3548569d2040 Mon Sep 17 00:00:00 2001 From: rostislav Date: Mon, 5 Jan 2026 15:31:04 +0200 Subject: [PATCH] incremental RAG updates ( fixes ) --- .../analysis/BranchAnalysisProcessor.java | 4 +- .../service/rag/RagOperationsService.java | 2 + .../service/RagOperationsServiceImpl.java | 67 ++++++++++++++++--- .../actions/CommentOnMergeRequestAction.java | 10 ++- 4 files changed, 69 insertions(+), 14 deletions(-) diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessor.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessor.java index 2bf04c17..0da3a5ae 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessor.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessor.java @@ -544,8 +544,8 @@ private void performIncrementalRagUpdate( "message", "Updating RAG index with changed files" )); - // Delegate to RAG operations service - ragOperationsService.triggerIncrementalUpdate(project, branch, commit, consumer); + // Delegate to RAG operations service with raw diff for incremental update + ragOperationsService.triggerIncrementalUpdate(project, branch, commit, rawDiff, consumer); log.info("Incremental RAG update triggered for project={}, branch={}, commit={}", project.getId(), branch, commit); diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/rag/RagOperationsService.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/rag/RagOperationsService.java index 0b24feef..a21b5d34 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/rag/RagOperationsService.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/rag/RagOperationsService.java @@ -34,12 +34,14 @@ public interface RagOperationsService { * @param project The project to update * @param branchName The branch name that was updated * @param commitHash The commit hash of the update + * @param rawDiff The raw diff from the VCS (used to determine which files changed) * @param eventConsumer Consumer to receive status updates during processing */ void triggerIncrementalUpdate( Project project, String branchName, String commitHash, + String rawDiff, Consumer> eventConsumer ); } diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImpl.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImpl.java index da5fac21..b8305efa 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImpl.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImpl.java @@ -5,6 +5,8 @@ import org.rostilos.codecrow.core.model.job.Job; import org.rostilos.codecrow.core.model.job.JobTriggerSource; import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.vcs.VcsConnection; +import org.rostilos.codecrow.core.model.vcs.VcsRepoBinding; import org.rostilos.codecrow.core.service.AnalysisJobService; import org.rostilos.codecrow.analysisengine.service.AnalysisLockService; import org.slf4j.Logger; @@ -14,6 +16,7 @@ import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.function.Consumer; /** @@ -66,6 +69,7 @@ public void triggerIncrementalUpdate( Project project, String branchName, String commitHash, + String rawDiff, Consumer> eventConsumer ) { Job job = null; @@ -75,9 +79,23 @@ public void triggerIncrementalUpdate( return; } + // Parse the diff to find changed files + IncrementalRagUpdateService.DiffResult diffResult = incrementalRagUpdateService.parseDiffForRag(rawDiff); + Set addedOrModifiedFiles = diffResult.addedOrModified(); + Set deletedFiles = diffResult.deleted(); + + if (addedOrModifiedFiles.isEmpty() && deletedFiles.isEmpty()) { + log.debug("Skipping RAG incremental update - no files changed in diff"); + return; + } + + log.info("RAG incremental update: {} files to add/update, {} files to delete", + addedOrModifiedFiles.size(), deletedFiles.size()); + job = analysisJobService.createRagIndexJob(project, false, JobTriggerSource.WEBHOOK); analysisJobService.info(job, "rag_init", - String.format("Starting incremental RAG update for branch '%s' (commit: %s)", branchName, commitHash)); + String.format("Starting incremental RAG update for branch '%s' (commit: %s) - %d files to update, %d to delete", + branchName, commitHash, addedOrModifiedFiles.size(), deletedFiles.size())); Optional ragLockKey = analysisLockService.acquireLock( project, @@ -99,27 +117,51 @@ public void triggerIncrementalUpdate( eventConsumer.accept(Map.of( "type", "status", "state", "rag_update", - "message", "Updating RAG index with changed files" + "message", "Updating RAG index with " + (addedOrModifiedFiles.size() + deletedFiles.size()) + " changed files" )); ragIndexTrackingService.markUpdatingStarted(project, branchName, commitHash); - log.info("Triggering RAG incremental update for project={}, branch={}, commit={}", + log.info("Performing RAG incremental update for project={}, branch={}, commit={}", project.getId(), branchName, commitHash); - // Mark as completed - the actual incremental RAG update is complex and - // is handled by the existing IncrementalRagUpdateService when called directly - // This interface is for simple notification from analysis-engine - ragIndexTrackingService.markUpdatingCompleted(project, branchName, commitHash, 0); + // Get VCS connection info from project + VcsRepoBinding vcsRepoBinding = project.getVcsRepoBinding(); + if (vcsRepoBinding == null) { + throw new IllegalStateException("Project has no VcsRepoBinding configured"); + } + + VcsConnection vcsConnection = vcsRepoBinding.getVcsConnection(); + String workspaceSlug = vcsRepoBinding.getExternalNamespace(); + String repoSlug = vcsRepoBinding.getExternalRepoSlug(); + + // Perform the actual incremental update + Map result = incrementalRagUpdateService.performIncrementalUpdate( + project, + vcsConnection, + workspaceSlug, + repoSlug, + branchName, + commitHash, + addedOrModifiedFiles, + deletedFiles + ); + + int filesUpdated = (Integer) result.getOrDefault("updatedFiles", 0); + int filesDeleted = (Integer) result.getOrDefault("deletedFiles", 0); + + ragIndexTrackingService.markUpdatingCompleted(project, branchName, commitHash, filesUpdated); eventConsumer.accept(Map.of( "type", "status", "state", "rag_complete", - "message", "RAG index update scheduled" + "message", String.format("RAG index updated: %d files updated, %d deleted", filesUpdated, filesDeleted) )); - log.info("RAG incremental update scheduled for project={}", project.getId()); - analysisJobService.info(job, "rag_complete", "RAG incremental update scheduled"); + log.info("RAG incremental update completed for project={}: {} files updated, {} deleted", + project.getId(), filesUpdated, filesDeleted); + analysisJobService.info(job, "rag_complete", + String.format("RAG incremental update completed: %d files updated, %d deleted", filesUpdated, filesDeleted)); analysisJobService.completeJob(job, null); } catch (Exception e) { @@ -129,6 +171,11 @@ public void triggerIncrementalUpdate( analysisJobService.error(job, "rag_error", "RAG incremental update failed: " + e.getMessage()); analysisJobService.failJob(job, "RAG incremental update failed: " + e.getMessage()); } + eventConsumer.accept(Map.of( + "type", "warning", + "state", "rag_error", + "message", "RAG incremental update failed: " + e.getMessage() + )); } finally { analysisLockService.releaseLock(ragLockKey.get()); } diff --git a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/gitlab/actions/CommentOnMergeRequestAction.java b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/gitlab/actions/CommentOnMergeRequestAction.java index f5400d73..943927c6 100644 --- a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/gitlab/actions/CommentOnMergeRequestAction.java +++ b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/gitlab/actions/CommentOnMergeRequestAction.java @@ -37,6 +37,8 @@ public void postComment(String namespace, String project, int mergeRequestIid, S String apiUrl = String.format("%s/projects/%s/merge_requests/%d/notes", GitLabConfig.API_BASE, encodedPath, mergeRequestIid); + log.info("Posting comment to GitLab MR: url={}, bodyLength={}", apiUrl, body != null ? body.length() : 0); + Map payload = new HashMap<>(); payload.put("body", body); @@ -51,9 +53,10 @@ public void postComment(String namespace, String project, int mergeRequestIid, S String respBody = resp.body() != null ? resp.body().string() : ""; String msg = String.format("GitLab returned non-success response %d for URL %s: %s", resp.code(), apiUrl, respBody); - log.warn(msg); + log.error(msg); throw new IOException(msg); } + log.info("Successfully posted comment to GitLab MR {}", mergeRequestIid); } } @@ -129,6 +132,8 @@ public void updateNote(String namespace, String project, int mergeRequestIid, lo String apiUrl = String.format("%s/projects/%s/merge_requests/%d/notes/%d", GitLabConfig.API_BASE, encodedPath, mergeRequestIid, noteId); + log.info("Updating note on GitLab MR: url={}, noteId={}, bodyLength={}", apiUrl, noteId, body != null ? body.length() : 0); + Map payload = new HashMap<>(); payload.put("body", body); @@ -141,9 +146,10 @@ public void updateNote(String namespace, String project, int mergeRequestIid, lo try (Response resp = authorizedOkHttpClient.newCall(req).execute()) { if (!resp.isSuccessful()) { String respBody = resp.body() != null ? resp.body().string() : ""; - log.warn("Failed to update note: {} - {}", resp.code(), respBody); + log.error("Failed to update note {}: {} - {}", noteId, resp.code(), respBody); throw new IOException("Failed to update note: " + resp.code()); } + log.info("Successfully updated note {} on GitLab MR {}", noteId, mergeRequestIid); } }