Skip to content

incremental RAG updates ( fixes ) #34

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rostilos merged 1 commit into from
Jan 5, 2026
Merged

incremental RAG updates ( fixes ) #34

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions ...java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -544,8 +544,8 @@ private void performIncrementalRagUpdate(
"message", "Updating RAG index with changed files"
));

// Delegate to RAG operations service
ragOperationsService.triggerIncrementalUpdate(project, branch, commit, consumer);
// Delegate to RAG operations service with raw diff for incremental update
ragOperationsService.triggerIncrementalUpdate(project, branch, commit, rawDiff, consumer);

log.info("Incremental RAG update triggered for project={}, branch={}, commit={}",
project.getId(), branch, commit);
Expand Down
2 changes: 2 additions & 0 deletions .../src/main/java/org/rostilos/codecrow/analysisengine/service/rag/RagOperationsService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,14 @@ public interface RagOperationsService {
* @param project The project to update
* @param branchName The branch name that was updated
* @param commitHash The commit hash of the update
* @param rawDiff The raw diff from the VCS (used to determine which files changed)
* @param eventConsumer Consumer to receive status updates during processing
*/
void triggerIncrementalUpdate(
Project project,
String branchName,
String commitHash,
String rawDiff,
Consumer<Map<String, Object>> eventConsumer
);
}
67 changes: 57 additions & 10 deletions ...ngine/src/main/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
import org.rostilos.codecrow.core.model.job.Job;
import org.rostilos.codecrow.core.model.job.JobTriggerSource;
import org.rostilos.codecrow.core.model.project.Project;
import org.rostilos.codecrow.core.model.vcs.VcsConnection;
import org.rostilos.codecrow.core.model.vcs.VcsRepoBinding;
import org.rostilos.codecrow.core.service.AnalysisJobService;
import org.rostilos.codecrow.analysisengine.service.AnalysisLockService;
import org.slf4j.Logger;
Expand All @@ -14,6 +16,7 @@

import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Consumer;

/**
Expand Down Expand Up @@ -66,6 +69,7 @@ public void triggerIncrementalUpdate(
Project project,
String branchName,
String commitHash,
String rawDiff,
Consumer<Map<String, Object>> eventConsumer
) {
Job job = null;
Expand All @@ -75,9 +79,23 @@ public void triggerIncrementalUpdate(
return;
}

// Parse the diff to find changed files
IncrementalRagUpdateService.DiffResult diffResult = incrementalRagUpdateService.parseDiffForRag(rawDiff);
Set<String> addedOrModifiedFiles = diffResult.addedOrModified();
Set<String> deletedFiles = diffResult.deleted();

if (addedOrModifiedFiles.isEmpty() && deletedFiles.isEmpty()) {
log.debug("Skipping RAG incremental update - no files changed in diff");
return;
}

log.info("RAG incremental update: {} files to add/update, {} files to delete",
addedOrModifiedFiles.size(), deletedFiles.size());

job = analysisJobService.createRagIndexJob(project, false, JobTriggerSource.WEBHOOK);
analysisJobService.info(job, "rag_init",
String.format("Starting incremental RAG update for branch '%s' (commit: %s)", branchName, commitHash));
String.format("Starting incremental RAG update for branch '%s' (commit: %s) - %d files to update, %d to delete",
branchName, commitHash, addedOrModifiedFiles.size(), deletedFiles.size()));

Optional<String> ragLockKey = analysisLockService.acquireLock(
project,
Expand All @@ -99,27 +117,51 @@ public void triggerIncrementalUpdate(
eventConsumer.accept(Map.of(
"type", "status",
"state", "rag_update",
"message", "Updating RAG index with changed files"
"message", "Updating RAG index with " + (addedOrModifiedFiles.size() + deletedFiles.size()) + " changed files"
));

ragIndexTrackingService.markUpdatingStarted(project, branchName, commitHash);

log.info("Triggering RAG incremental update for project={}, branch={}, commit={}",
log.info("Performing RAG incremental update for project={}, branch={}, commit={}",
project.getId(), branchName, commitHash);

// Mark as completed - the actual incremental RAG update is complex and
// is handled by the existing IncrementalRagUpdateService when called directly
// This interface is for simple notification from analysis-engine
ragIndexTrackingService.markUpdatingCompleted(project, branchName, commitHash, 0);
// Get VCS connection info from project
VcsRepoBinding vcsRepoBinding = project.getVcsRepoBinding();
if (vcsRepoBinding == null) {
throw new IllegalStateException("Project has no VcsRepoBinding configured");
}

VcsConnection vcsConnection = vcsRepoBinding.getVcsConnection();
String workspaceSlug = vcsRepoBinding.getExternalNamespace();
String repoSlug = vcsRepoBinding.getExternalRepoSlug();

// Perform the actual incremental update
Map<String, Object> result = incrementalRagUpdateService.performIncrementalUpdate(
project,
vcsConnection,
workspaceSlug,
repoSlug,
branchName,
commitHash,
addedOrModifiedFiles,
deletedFiles
);

int filesUpdated = (Integer) result.getOrDefault("updatedFiles", 0);
int filesDeleted = (Integer) result.getOrDefault("deletedFiles", 0);

ragIndexTrackingService.markUpdatingCompleted(project, branchName, commitHash, filesUpdated);

eventConsumer.accept(Map.of(
"type", "status",
"state", "rag_complete",
"message", "RAG index update scheduled"
"message", String.format("RAG index updated: %d files updated, %d deleted", filesUpdated, filesDeleted)
));

log.info("RAG incremental update scheduled for project={}", project.getId());
analysisJobService.info(job, "rag_complete", "RAG incremental update scheduled");
log.info("RAG incremental update completed for project={}: {} files updated, {} deleted",
project.getId(), filesUpdated, filesDeleted);
analysisJobService.info(job, "rag_complete",
String.format("RAG incremental update completed: %d files updated, %d deleted", filesUpdated, filesDeleted));
analysisJobService.completeJob(job, null);

} catch (Exception e) {
Expand All @@ -129,6 +171,11 @@ public void triggerIncrementalUpdate(
analysisJobService.error(job, "rag_error", "RAG incremental update failed: " + e.getMessage());
analysisJobService.failJob(job, "RAG incremental update failed: " + e.getMessage());
}
eventConsumer.accept(Map.of(
"type", "warning",
"state", "rag_error",
"message", "RAG incremental update failed: " + e.getMessage()
));
} finally {
analysisLockService.releaseLock(ragLockKey.get());
}
Expand Down
10 changes: 8 additions & 2 deletions ...main/java/org/rostilos/codecrow/vcsclient/gitlab/actions/CommentOnMergeRequestAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ public void postComment(String namespace, String project, int mergeRequestIid, S
String apiUrl = String.format("%s/projects/%s/merge_requests/%d/notes",
GitLabConfig.API_BASE, encodedPath, mergeRequestIid);

log.info("Posting comment to GitLab MR: url={}, bodyLength={}", apiUrl, body != null ? body.length() : 0);

Map<String, String> payload = new HashMap<>();
payload.put("body", body);

Expand All @@ -51,9 +53,10 @@ public void postComment(String namespace, String project, int mergeRequestIid, S
String respBody = resp.body() != null ? resp.body().string() : "";
String msg = String.format("GitLab returned non-success response %d for URL %s: %s",
resp.code(), apiUrl, respBody);
log.warn(msg);
log.error(msg);
throw new IOException(msg);
}
log.info("Successfully posted comment to GitLab MR {}", mergeRequestIid);
}
}

Expand Down Expand Up @@ -129,6 +132,8 @@ public void updateNote(String namespace, String project, int mergeRequestIid, lo
String apiUrl = String.format("%s/projects/%s/merge_requests/%d/notes/%d",
GitLabConfig.API_BASE, encodedPath, mergeRequestIid, noteId);

log.info("Updating note on GitLab MR: url={}, noteId={}, bodyLength={}", apiUrl, noteId, body != null ? body.length() : 0);

Map<String, String> payload = new HashMap<>();
payload.put("body", body);

Expand All @@ -141,9 +146,10 @@ public void updateNote(String namespace, String project, int mergeRequestIid, lo
try (Response resp = authorizedOkHttpClient.newCall(req).execute()) {
if (!resp.isSuccessful()) {
String respBody = resp.body() != null ? resp.body().string() : "";
log.warn("Failed to update note: {} - {}", resp.code(), respBody);
log.error("Failed to update note {}: {} - {}", noteId, resp.code(), respBody);
throw new IOException("Failed to update note: " + resp.code());
}
log.info("Successfully updated note {} on GitLab MR {}", noteId, mergeRequestIid);
}
}

Expand Down