diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index e2b229ad4a0..e5f7e5bb71f 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -3257,7 +3257,6 @@ dependencies = [ "anyhow", "colored", "env_logger", - "fastfield_codecs", "home", "itertools", "num_cpus", @@ -3268,7 +3267,6 @@ dependencies = [ "regex", "serde", "serde_json", - "tantivy", "tempfile", "thiserror", "tokio", diff --git a/quickwit/quickwit-common/Cargo.toml b/quickwit/quickwit-common/Cargo.toml index 87407a4a253..9d0e6c5ff9b 100644 --- a/quickwit/quickwit-common/Cargo.toml +++ b/quickwit/quickwit-common/Cargo.toml @@ -13,7 +13,6 @@ documentation = "https://quickwit.io/docs/" anyhow = { workspace = true } colored = { workspace = true } env_logger = { workspace = true } -fastfield_codecs = { workspace = true } home = { workspace = true } itertools = { workspace = true } num_cpus = { workspace = true } @@ -23,7 +22,6 @@ prometheus = { workspace = true } rand = { workspace = true } regex = { workspace = true } serde = { workspace = true } -tantivy = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } diff --git a/quickwit/quickwit-common/src/lib.rs b/quickwit/quickwit-common/src/lib.rs index 092e9c96a74..55dd6c1cfad 100644 --- a/quickwit/quickwit-common/src/lib.rs +++ b/quickwit/quickwit-common/src/lib.rs @@ -20,7 +20,6 @@ mod checklist; mod coolid; -pub mod fast_field_reader; pub mod fs; pub mod metrics; pub mod net; diff --git a/quickwit/quickwit-common/src/fast_field_reader.rs b/quickwit/quickwit-doc-mapper/src/fast_field_reader.rs similarity index 88% rename from quickwit/quickwit-common/src/fast_field_reader.rs rename to quickwit/quickwit-doc-mapper/src/fast_field_reader.rs index e8372004014..5ff491063b8 100644 --- a/quickwit/quickwit-common/src/fast_field_reader.rs +++ b/quickwit/quickwit-doc-mapper/src/fast_field_reader.rs @@ -19,9 +19,9 @@ use std::sync::Arc; -use tantivy::fastfield::{Column, FastFieldReaders}; -use tantivy::schema::{Field, FieldEntry, Type}; -use tantivy::{DateTime, DocId, TantivyError}; +use tantivy::fastfield::Column; +use tantivy::schema::{Field, Type}; +use tantivy::{DateTime, DocId, SegmentReader, TantivyError}; #[derive(Clone)] pub enum GenericFastFieldReader { @@ -60,9 +60,11 @@ impl GenericFastFieldReader { pub fn timestamp_field_reader( timestamp_field: Field, - timestamp_field_entry: &FieldEntry, - fast_field_readers: &FastFieldReaders, + segment_reader: &SegmentReader, ) -> tantivy::Result { + let schema = segment_reader.schema(); + let timestamp_field_entry = schema.get_field_entry(timestamp_field); + let fast_field_readers = segment_reader.fast_fields(); let field_schema_type = timestamp_field_entry.field_type().value_type(); let timestamp_field_reader = match field_schema_type { Type::I64 => GenericFastFieldReader::I64(fast_field_readers.i64(timestamp_field)?), diff --git a/quickwit/quickwit-doc-mapper/src/lib.rs b/quickwit/quickwit-doc-mapper/src/lib.rs index d527c8486db..d44edb9fb15 100644 --- a/quickwit/quickwit-doc-mapper/src/lib.rs +++ b/quickwit/quickwit-doc-mapper/src/lib.rs @@ -27,6 +27,9 @@ mod default_doc_mapper; mod doc_mapper; mod error; + +#[allow(missing_docs)] +pub mod fast_field_reader; mod query_builder; mod routing_expression; mod sort_by; diff --git a/quickwit/quickwit-indexing/src/actors/merge_executor.rs b/quickwit/quickwit-indexing/src/actors/merge_executor.rs index 15c3a33ecd5..0d293409a59 100644 --- a/quickwit/quickwit-indexing/src/actors/merge_executor.rs +++ b/quickwit/quickwit-indexing/src/actors/merge_executor.rs @@ -28,9 +28,9 @@ use async_trait::async_trait; use fail::fail_point; use itertools::Itertools; use quickwit_actors::{Actor, ActorContext, ActorExitStatus, Handler, Mailbox, QueueCapacity}; -use quickwit_common::fast_field_reader::timestamp_field_reader; use quickwit_common::runtimes::RuntimeType; use quickwit_directories::UnionDirectory; +use quickwit_doc_mapper::fast_field_reader::timestamp_field_reader; use quickwit_doc_mapper::{DocMapper, QUICKWIT_TOKENIZER_MANAGER}; use quickwit_metastore::{Metastore, SplitMetadata}; use quickwit_proto::metastore_api::DeleteTask; @@ -452,15 +452,8 @@ impl MergeExecutor { timestamp_field_name )) })?; - let timestamp_field_entry = merged_segment_reader - .schema() - .get_field_entry(timestamp_field); - let reader = timestamp_field_reader( - timestamp_field, - timestamp_field_entry, - merged_segment_reader.fast_fields(), - )?; - Some(RangeInclusive::new(reader.min_value(), reader.max_value())) + let reader = timestamp_field_reader(timestamp_field, &merged_segment_reader)?; + Some(reader.min_value()..=reader.max_value()) } else { None }; diff --git a/quickwit/quickwit-search/src/filters.rs b/quickwit/quickwit-search/src/filters.rs index 682137ac5af..77c5d82c51f 100644 --- a/quickwit/quickwit-search/src/filters.rs +++ b/quickwit/quickwit-search/src/filters.rs @@ -19,7 +19,7 @@ use std::ops::{Bound, RangeBounds}; -use quickwit_common::fast_field_reader::{timestamp_field_reader, GenericFastFieldReader}; +use quickwit_doc_mapper::fast_field_reader::{timestamp_field_reader, GenericFastFieldReader}; use tantivy::schema::Field; use tantivy::{DocId, SegmentReader}; @@ -71,14 +71,7 @@ impl TimestampFilterBuilder { &self, segment_reader: &SegmentReader, ) -> tantivy::Result> { - let timestamp_field_entry = segment_reader - .schema() - .get_field_entry(self.timestamp_field); - let timestamp_field_reader = timestamp_field_reader( - self.timestamp_field, - timestamp_field_entry, - segment_reader.fast_fields(), - )?; + let timestamp_field_reader = timestamp_field_reader(self.timestamp_field, segment_reader)?; let segment_range = ( timestamp_field_reader.min_value(), timestamp_field_reader.max_value(),