From d43cbe769a515d593e4047f92f3d3a0ad40d56c8 Mon Sep 17 00:00:00 2001 From: Gabriel Gordon-Hall Date: Tue, 19 Dec 2023 13:04:27 +0000 Subject: [PATCH 1/4] use keyword filter for path matches in proc --- server/bleep/src/agent.rs | 14 ++++++-- server/bleep/src/agent/tools/code.rs | 4 +-- server/bleep/src/agent/tools/path.rs | 2 +- server/bleep/src/agent/tools/proc.rs | 2 +- server/bleep/src/semantic.rs | 49 +++++++++++++++++++--------- server/bleep/src/semantic/execute.rs | 1 + 6 files changed, 51 insertions(+), 21 deletions(-) diff --git a/server/bleep/src/agent.rs b/server/bleep/src/agent.rs index fd5a639bf6..ec9d517b8d 100644 --- a/server/bleep/src/agent.rs +++ b/server/bleep/src/agent.rs @@ -340,6 +340,7 @@ impl Agent { Ok(history) } + #[allow(clippy::too_many_arguments)] async fn semantic_search( &self, query: parser::Literal<'_>, @@ -348,6 +349,7 @@ impl Agent { offset: u64, threshold: f32, retrieve_more: bool, + exact: bool, ) -> Result> { let paths_set = paths .into_iter() @@ -394,7 +396,7 @@ impl Agent { debug!(?query, %self.thread_id, "executing semantic query"); self.app .semantic - .search(&query, limit, offset, threshold, retrieve_more) + .search(&query, limit, offset, threshold, retrieve_more, exact) .await } @@ -406,6 +408,7 @@ impl Agent { offset: u64, threshold: f32, retrieve_more: bool, + exact: bool, ) -> Result> { let queries = queries .iter() @@ -421,7 +424,14 @@ impl Agent { debug!(?queries, %self.thread_id, "executing semantic query"); self.app .semantic - .batch_search(queries.as_slice(), limit, offset, threshold, retrieve_more) + .batch_search( + queries.as_slice(), + limit, + offset, + threshold, + retrieve_more, + exact, + ) .await } diff --git a/server/bleep/src/agent/tools/code.rs b/server/bleep/src/agent/tools/code.rs index 8f2017c831..2eec0a9228 100644 --- a/server/bleep/src/agent/tools/code.rs +++ b/server/bleep/src/agent/tools/code.rs @@ -23,7 +23,7 @@ impl Agent { .await?; let mut results = self - .semantic_search(query.into(), vec![], CODE_SEARCH_LIMIT, 0, 0.3, true) + .semantic_search(query.into(), vec![], CODE_SEARCH_LIMIT, 0, 0.3, true, false) .await?; debug!("returned {} results", results.len()); @@ -35,7 +35,7 @@ impl Agent { if !hyde_docs.is_empty() { let hyde_doc = hyde_docs.first().unwrap().into(); let hyde_results = self - .semantic_search(hyde_doc, vec![], CODE_SEARCH_LIMIT, 0, 0.3, true) + .semantic_search(hyde_doc, vec![], CODE_SEARCH_LIMIT, 0, 0.3, true, false) .await?; debug!("returned {} HyDE results", results.len()); diff --git a/server/bleep/src/agent/tools/path.rs b/server/bleep/src/agent/tools/path.rs index d206068ea7..16ec1b105d 100644 --- a/server/bleep/src/agent/tools/path.rs +++ b/server/bleep/src/agent/tools/path.rs @@ -34,7 +34,7 @@ impl Agent { // If there are no lexical results, perform a semantic search. if paths.is_empty() { let semantic_paths = self - .semantic_search(query.into(), vec![], 30, 0, 0.0, true) + .semantic_search(query.into(), vec![], 30, 0, 0.0, true, false) .await? .into_iter() .map(|chunk| chunk.relative_path) diff --git a/server/bleep/src/agent/tools/proc.rs b/server/bleep/src/agent/tools/proc.rs index faeba8735f..8e12e00a9b 100644 --- a/server/bleep/src/agent/tools/proc.rs +++ b/server/bleep/src/agent/tools/proc.rs @@ -31,7 +31,7 @@ impl Agent { .await?; let results = self - .semantic_search(query.into(), paths.clone(), 10, 0, 0.0, true) + .semantic_search(query.into(), paths.clone(), 10, 0, 0.0, true, true) .await?; let mut chunks = results diff --git a/server/bleep/src/semantic.rs b/server/bleep/src/semantic.rs index 9c00ffc68d..24e7085b12 100644 --- a/server/bleep/src/semantic.rs +++ b/server/bleep/src/semantic.rs @@ -329,10 +329,11 @@ impl Semantic { limit: u64, offset: u64, threshold: f32, + exact: bool, ) -> anyhow::Result> { let hybrid_filter = Some(Filter { should: build_conditions_lexical(parsed_query), - must: build_conditions(parsed_query), + must: build_conditions(parsed_query, exact), ..Default::default() }); @@ -361,6 +362,7 @@ impl Semantic { limit: u64, offset: u64, threshold: f32, + exact: bool, ) -> anyhow::Result> { let response = self .qdrant @@ -374,7 +376,7 @@ impl Semantic { selector_options: Some(with_payload_selector::SelectorOptions::Enable(true)), }), filter: Some(Filter { - must: build_conditions(parsed_query), + must: build_conditions(parsed_query, exact), ..Default::default() }), with_vectors: Some(WithVectorsSelector { @@ -398,6 +400,7 @@ impl Semantic { limit: u64, offset: u64, threshold: f32, + exact: bool, ) -> anyhow::Result> { // FIXME: This method uses `search_points` internally, and not `search_batch_points`. It's // not clear why, but it seems that the `batch` variant of the `qdrant` calls leads to @@ -412,7 +415,7 @@ impl Semantic { // Queries should contain the same filters, so we get the first one let parsed_query = parsed_queries.first().unwrap(); - let filters = &build_conditions(parsed_query); + let filters = &build_conditions(parsed_query, exact); let responses = stream::iter(vectors.into_iter()) .map(|vector| async move { @@ -521,6 +524,7 @@ impl Semantic { offset: u64, threshold: f32, retrieve_more: bool, + exact: bool, ) -> anyhow::Result> { let Some(query) = parsed_query.target() else { anyhow::bail!("no search target for query"); @@ -537,6 +541,7 @@ impl Semantic { if retrieve_more { limit * 2 } else { limit }, // Retrieve double `limit` and deduplicate offset, threshold, + exact, ) .await .map(|raw| { @@ -553,6 +558,7 @@ impl Semantic { if retrieve_more { limit * 2 } else { limit }, offset, 0.0, + exact, ) .await .map(|raw| { @@ -579,6 +585,7 @@ impl Semantic { offset: u64, threshold: f32, retrieve_more: bool, + exact: bool, ) -> anyhow::Result> { if parsed_queries.iter().any(|q| q.target().is_none()) { anyhow::bail!("no search target for query"); @@ -602,6 +609,7 @@ impl Semantic { if retrieve_more { limit * 2 } else { limit }, // Retrieve double `limit` and deduplicate offset, threshold, + exact, ) .await; @@ -756,20 +764,23 @@ fn build_conditions_lexical( .collect() } -fn build_conditions(query: &SemanticQuery<'_>) -> Vec { - let repo_filter = { +fn build_conditions( + query: &SemanticQuery<'_>, + exact: bool, +) -> Vec { + let path_filter = { let conditions = query - .repos() + .paths() .map(|r| { - if r.contains('/') && !r.starts_with("github.com/") { - format!("github.com/{r}") - } else { - r.to_string() + { + match exact { + true => make_kv_keyword_filter("relative_path", r.as_ref()), + false => make_kv_text_filter("relative_path", r.as_ref()), + } } + .into() }) - .map(|r| make_kv_keyword_filter("repo_name", r.as_ref()).into()) .collect::>(); - // one of the above repos should match if conditions.is_empty() { None } else { @@ -780,11 +791,19 @@ fn build_conditions(query: &SemanticQuery<'_>) -> Vec>(); + // one of the above repos should match if conditions.is_empty() { None } else { diff --git a/server/bleep/src/semantic/execute.rs b/server/bleep/src/semantic/execute.rs index 0af9a4dd7e..7e91fe7241 100644 --- a/server/bleep/src/semantic/execute.rs +++ b/server/bleep/src/semantic/execute.rs @@ -24,6 +24,7 @@ pub async fn execute( ((params.page + 1) * params.page_size) as u64, 0.0, false, + false, ) .await?; From 256ccc292d3bf285c844fe5224543c00ccb49880 Mon Sep 17 00:00:00 2001 From: Gabriel Gordon-Hall Date: Tue, 19 Dec 2023 17:30:20 +0000 Subject: [PATCH 2/4] extract search params into struct --- server/bleep/src/agent.rs | 26 +++------------ server/bleep/src/agent/tools/code.rs | 23 +++++++++++-- server/bleep/src/agent/tools/path.rs | 12 ++++++- server/bleep/src/agent/tools/proc.rs | 12 ++++++- server/bleep/src/semantic.rs | 50 +++++++++++++++++----------- server/bleep/src/semantic/execute.rs | 12 ++++--- 6 files changed, 85 insertions(+), 50 deletions(-) diff --git a/server/bleep/src/agent.rs b/server/bleep/src/agent.rs index ec9d517b8d..37d531bce3 100644 --- a/server/bleep/src/agent.rs +++ b/server/bleep/src/agent.rs @@ -345,11 +345,7 @@ impl Agent { &self, query: parser::Literal<'_>, paths: Vec, - limit: u64, - offset: u64, - threshold: f32, - retrieve_more: bool, - exact: bool, + params: semantic::SemanticSearchParams, ) -> Result> { let paths_set = paths .into_iter() @@ -394,21 +390,14 @@ impl Agent { }; debug!(?query, %self.thread_id, "executing semantic query"); - self.app - .semantic - .search(&query, limit, offset, threshold, retrieve_more, exact) - .await + self.app.semantic.search(&query, params).await } #[allow(dead_code)] async fn batch_semantic_search( &self, queries: Vec>, - limit: u64, - offset: u64, - threshold: f32, - retrieve_more: bool, - exact: bool, + params: semantic::SemanticSearchParams, ) -> Result> { let queries = queries .iter() @@ -424,14 +413,7 @@ impl Agent { debug!(?queries, %self.thread_id, "executing semantic query"); self.app .semantic - .batch_search( - queries.as_slice(), - limit, - offset, - threshold, - retrieve_more, - exact, - ) + .batch_search(queries.as_slice(), params) .await } diff --git a/server/bleep/src/agent/tools/code.rs b/server/bleep/src/agent/tools/code.rs index 2eec0a9228..3afdca8485 100644 --- a/server/bleep/src/agent/tools/code.rs +++ b/server/bleep/src/agent/tools/code.rs @@ -8,6 +8,7 @@ use crate::{ }, analytics::EventData, llm_gateway, + semantic::SemanticSearchParams, }; impl Agent { @@ -23,7 +24,16 @@ impl Agent { .await?; let mut results = self - .semantic_search(query.into(), vec![], CODE_SEARCH_LIMIT, 0, 0.3, true, false) + .semantic_search( + query.into(), + vec![], + SemanticSearchParams { + limit: CODE_SEARCH_LIMIT, + offset: 0, + threshold: 0.3, + exact_match: true, + }, + ) .await?; debug!("returned {} results", results.len()); @@ -35,7 +45,16 @@ impl Agent { if !hyde_docs.is_empty() { let hyde_doc = hyde_docs.first().unwrap().into(); let hyde_results = self - .semantic_search(hyde_doc, vec![], CODE_SEARCH_LIMIT, 0, 0.3, true, false) + .semantic_search( + hyde_doc, + vec![], + SemanticSearchParams { + limit: CODE_SEARCH_LIMIT, + offset: 0, + threshold: 0.3, + exact_match: true, + }, + ) .await?; debug!("returned {} HyDE results", results.len()); diff --git a/server/bleep/src/agent/tools/path.rs b/server/bleep/src/agent/tools/path.rs index 16ec1b105d..5bbbe6dc02 100644 --- a/server/bleep/src/agent/tools/path.rs +++ b/server/bleep/src/agent/tools/path.rs @@ -9,6 +9,7 @@ use crate::{ Agent, }, analytics::EventData, + semantic::SemanticSearchParams, }; impl Agent { @@ -34,7 +35,16 @@ impl Agent { // If there are no lexical results, perform a semantic search. if paths.is_empty() { let semantic_paths = self - .semantic_search(query.into(), vec![], 30, 0, 0.0, true, false) + .semantic_search( + query.into(), + vec![], + SemanticSearchParams { + limit: 30, + offset: 0, + threshold: 0.0, + exact_match: false, + }, + ) .await? .into_iter() .map(|chunk| chunk.relative_path) diff --git a/server/bleep/src/agent/tools/proc.rs b/server/bleep/src/agent/tools/proc.rs index 8e12e00a9b..e55a701482 100644 --- a/server/bleep/src/agent/tools/proc.rs +++ b/server/bleep/src/agent/tools/proc.rs @@ -7,6 +7,7 @@ use crate::{ Agent, }, analytics::EventData, + semantic::SemanticSearchParams, }; impl Agent { @@ -31,7 +32,16 @@ impl Agent { .await?; let results = self - .semantic_search(query.into(), paths.clone(), 10, 0, 0.0, true, true) + .semantic_search( + query.into(), + paths.clone(), + SemanticSearchParams { + limit: 10, + offset: 0, + threshold: 0.0, + exact_match: true, + }, + ) .await?; let mut chunks = results diff --git a/server/bleep/src/semantic.rs b/server/bleep/src/semantic.rs index 24e7085b12..95be32188b 100644 --- a/server/bleep/src/semantic.rs +++ b/server/bleep/src/semantic.rs @@ -50,6 +50,14 @@ pub enum SemanticError { }, } +#[derive(Debug, Clone)] +pub struct SemanticSearchParams { + pub limit: u64, + pub offset: u64, + pub threshold: f32, + pub exact_match: bool, // keyword match for all filters +} + #[derive(Clone)] pub struct Semantic { qdrant: Arc, @@ -520,16 +528,18 @@ impl Semantic { pub async fn search<'a>( &self, parsed_query: &SemanticQuery<'a>, - limit: u64, - offset: u64, - threshold: f32, - retrieve_more: bool, - exact: bool, + params: SemanticSearchParams, ) -> anyhow::Result> { let Some(query) = parsed_query.target() else { anyhow::bail!("no search target for query"); }; let vector = self.embedder.embed(&query).await?; + let SemanticSearchParams { + limit, + offset, + threshold, + exact_match: exact, + } = params; // TODO: Remove the need for `retrieve_more`. It's here because: // In /q `limit` is the maximum number of results returned (the actual number will often be lower due to deduplication) @@ -538,7 +548,7 @@ impl Semantic { .search_with( parsed_query, vector.clone(), - if retrieve_more { limit * 2 } else { limit }, // Retrieve double `limit` and deduplicate + limit * 2, // Retrieve double `limit` and deduplicate offset, threshold, exact, @@ -555,7 +565,7 @@ impl Semantic { .search_lexical( parsed_query, vector.clone(), - if retrieve_more { limit * 2 } else { limit }, + limit * 2, // Retrieve double `limit` and deduplicate offset, 0.0, exact, @@ -581,11 +591,7 @@ impl Semantic { pub async fn batch_search<'a>( &self, parsed_queries: &[&SemanticQuery<'a>], - limit: u64, - offset: u64, - threshold: f32, - retrieve_more: bool, - exact: bool, + params: SemanticSearchParams, ) -> anyhow::Result> { if parsed_queries.iter().any(|q| q.target().is_none()) { anyhow::bail!("no search target for query"); @@ -600,13 +606,20 @@ impl Semantic { .into_iter() .collect::>>()?; + let SemanticSearchParams { + limit, + offset, + threshold, + exact_match: exact, + } = params; + trace!(?parsed_queries, "performing qdrant batch search"); let result = self .batch_search_with( parsed_queries, vectors.clone(), - if retrieve_more { limit * 2 } else { limit }, // Retrieve double `limit` and deduplicate + limit * 2, // Retrieve double `limit` and deduplicate offset, threshold, exact, @@ -766,17 +779,16 @@ fn build_conditions_lexical( fn build_conditions( query: &SemanticQuery<'_>, - exact: bool, + exact_match: bool, ) -> Vec { let path_filter = { let conditions = query .paths() .map(|r| { - { - match exact { - true => make_kv_keyword_filter("relative_path", r.as_ref()), - false => make_kv_text_filter("relative_path", r.as_ref()), - } + if exact_match { + make_kv_keyword_filter("relative_path", r.as_ref()) + } else { + make_kv_text_filter("relative_path", r.as_ref()) } .into() }) diff --git a/server/bleep/src/semantic/execute.rs b/server/bleep/src/semantic/execute.rs index 7e91fe7241..d93561df10 100644 --- a/server/bleep/src/semantic/execute.rs +++ b/server/bleep/src/semantic/execute.rs @@ -5,6 +5,7 @@ use crate::{ execute::{ApiQuery, PagingMetadata, QueryResponse, QueryResult, ResultStats}, parser::SemanticQuery, }, + semantic::SemanticSearchParams, snippet::Snippet, }; @@ -20,11 +21,12 @@ pub async fn execute( let results = semantic .search( &query, - params.page_size as u64, - ((params.page + 1) * params.page_size) as u64, - 0.0, - false, - false, + SemanticSearchParams { + limit: params.page_size as u64, + offset: ((params.page + 1) * params.page_size) as u64, + threshold: 0.0, + exact_match: false, + }, ) .await?; From d93d3ab86432642780e496550c572691dbbd0bec Mon Sep 17 00:00:00 2001 From: Gabriel Gordon-Hall Date: Tue, 19 Dec 2023 17:32:07 +0000 Subject: [PATCH 3/4] remove clippy allow --- server/bleep/src/agent.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/server/bleep/src/agent.rs b/server/bleep/src/agent.rs index 37d531bce3..93f903f3f1 100644 --- a/server/bleep/src/agent.rs +++ b/server/bleep/src/agent.rs @@ -340,7 +340,6 @@ impl Agent { Ok(history) } - #[allow(clippy::too_many_arguments)] async fn semantic_search( &self, query: parser::Literal<'_>, From c4007235108c86dd90201bb750465073c6c799ac Mon Sep 17 00:00:00 2001 From: Gabriel Gordon-Hall Date: Tue, 19 Dec 2023 17:35:45 +0000 Subject: [PATCH 4/4] set exact_match to false for code search --- server/bleep/src/agent/tools/code.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/bleep/src/agent/tools/code.rs b/server/bleep/src/agent/tools/code.rs index 3afdca8485..d4ad4e5e94 100644 --- a/server/bleep/src/agent/tools/code.rs +++ b/server/bleep/src/agent/tools/code.rs @@ -31,7 +31,7 @@ impl Agent { limit: CODE_SEARCH_LIMIT, offset: 0, threshold: 0.3, - exact_match: true, + exact_match: false, }, ) .await?; @@ -52,7 +52,7 @@ impl Agent { limit: CODE_SEARCH_LIMIT, offset: 0, threshold: 0.3, - exact_match: true, + exact_match: false, }, ) .await?;