From af877d576c1cd6a2a7241a6b0c41f27728a7477e Mon Sep 17 00:00:00 2001 From: Will Jones Date: Wed, 4 Mar 2026 12:26:56 -0800 Subject: [PATCH 01/22] draft vector index details --- protos/table.proto | 49 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/protos/table.proto b/protos/table.proto index e7de867e46e..d20ee6a3ad1 100644 --- a/protos/table.proto +++ b/protos/table.proto @@ -461,7 +461,54 @@ message ExternalFile { } // Empty details messages for older indexes that don't take advantage of the details field. -message VectorIndexDetails {} +message VectorIndexDetails { + enum VectorMetricType { + L2 = 0; + COSINE = 1; + DOT = 2; + HAMMING = 3; + } + + VectorMetricType metric_type = 1; + + uint64 target_partition_size = 2; + + optional HnswIndexDetails hnsw_index_config = 3; + + message Flat {} + enum Bits { + BIT_8 = 0; + BIT_4 = 1; + } + message ProductQuantization { + Bits num_bits = 1; + uint32 num_sub_vectors = 2; + } + message ScalarQuantization { + Bits num_bits = 1; + } + message RabitQuantization { + Bits num_bits = 1; + } + + oneof compression { + Flat flat = 4; + ProductQuantization pq = 5; + ScalarQuantization sq = 6; + RabitQuantization rq = 7; + } +} + +// Hierarchical Navigable Small World (HNSW) index details, used as an optional configuration for IVF indexes. +message HnswIndexDetails { + // The maximum number of outgoing edges per node in the HNSW graph. Higher values + // means more connections, better recall, but more memory and slower builds. + // Referred to as "M" in the HNSW literature. + uint32 max_connections = 1; + // "construction exploration factor": The size of the dynamic list used during + // index construction. + uint32 construction_ef = 2; +} message FragmentReuseIndexDetails { From 080ac461ae7b9a409e5e1f570bfd01021caaa3d7 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Wed, 4 Mar 2026 15:04:37 -0800 Subject: [PATCH 02/22] populate and surface VectorIndexDetails through describe_indices Previously, vector indices returned index_type "Unknown" and empty details in describe_indices(). This populates VectorIndexDetails at creation time from build params, derives a human-readable index type string (e.g. "IVF_PQ"), serializes details as JSON, and infers details from index files on disk as a fallback for legacy indices. Also changes proto num_bits from Bits enum to uint32 to support RQ's default of 1 bit, and adds rotation_type to RabitQuantization. Co-Authored-By: Claude Opus 4.6 --- protos/table.proto | 15 +- python/python/tests/test_vector_index.py | 11 +- rust/lance-index/src/vector/bq.rs | 14 + rust/lance-index/src/vector/hnsw/builder.rs | 9 + rust/lance-index/src/vector/pq/builder.rs | 9 + rust/lance-index/src/vector/sq/builder.rs | 8 + rust/lance/src/index.rs | 79 +++-- rust/lance/src/index/append.rs | 11 +- rust/lance/src/index/create.rs | 6 +- rust/lance/src/index/vector.rs | 5 +- rust/lance/src/index/vector/details.rs | 347 ++++++++++++++++++++ rust/lance/src/index/vector/ivf.rs | 8 +- 12 files changed, 482 insertions(+), 40 deletions(-) create mode 100644 rust/lance/src/index/vector/details.rs diff --git a/protos/table.proto b/protos/table.proto index d20ee6a3ad1..2605d8c88d5 100644 --- a/protos/table.proto +++ b/protos/table.proto @@ -476,19 +476,20 @@ message VectorIndexDetails { optional HnswIndexDetails hnsw_index_config = 3; message Flat {} - enum Bits { - BIT_8 = 0; - BIT_4 = 1; - } message ProductQuantization { - Bits num_bits = 1; + uint32 num_bits = 1; uint32 num_sub_vectors = 2; } message ScalarQuantization { - Bits num_bits = 1; + uint32 num_bits = 1; } message RabitQuantization { - Bits num_bits = 1; + enum RotationType { + FAST = 0; + MATRIX = 1; + } + uint32 num_bits = 1; + RotationType rotation_type = 2; } oneof compression { diff --git a/python/python/tests/test_vector_index.py b/python/python/tests/test_vector_index.py index d5ea831dad5..58c4fa84419 100644 --- a/python/python/tests/test_vector_index.py +++ b/python/python/tests/test_vector_index.py @@ -1565,8 +1565,7 @@ def test_describe_vector_index(indexed_dataset: LanceDataset): assert info.name == "vector_idx" assert info.type_url == "/lance.table.VectorIndexDetails" - # This is currently Unknown because vector indices are not yet handled by plugins - assert info.index_type == "Unknown" + assert info.index_type == "IVF_PQ" assert info.num_rows_indexed == 1000 assert info.fields == [0] assert info.field_names == ["vector"] @@ -1576,6 +1575,14 @@ def test_describe_vector_index(indexed_dataset: LanceDataset): assert info.segments[0].index_version == 1 assert info.segments[0].created_at is not None + import json + + details = json.loads(info.details) + assert details["metric_type"] == "L2" + assert details["compression"]["type"] == "pq" + assert details["compression"]["num_bits"] == 8 + assert details["compression"]["num_sub_vectors"] == 16 + def test_optimize_indices(indexed_dataset): data = create_table() diff --git a/rust/lance-index/src/vector/bq.rs b/rust/lance-index/src/vector/bq.rs index d56bfdcafc6..5b036ee9114 100644 --- a/rust/lance-index/src/vector/bq.rs +++ b/rust/lance-index/src/vector/bq.rs @@ -10,6 +10,7 @@ use std::sync::Arc; use arrow_array::types::Float32Type; use arrow_array::{Array, ArrayRef, UInt8Array, cast::AsArray}; use lance_core::{Error, Result}; +use lance_table::format::pb::vector_index_details::RabitQuantization; use num_traits::Float; use serde::{Deserialize, Serialize}; @@ -121,6 +122,19 @@ impl RQBuildParams { } } +impl From<&RQBuildParams> for RabitQuantization { + fn from(value: &RQBuildParams) -> Self { + use lance_table::format::pb::vector_index_details::rabit_quantization::RotationType; + Self { + num_bits: value.num_bits as u32, + rotation_type: match value.rotation_type { + RQRotationType::Fast => RotationType::Fast as i32, + RQRotationType::Matrix => RotationType::Matrix as i32, + }, + } + } +} + impl QuantizerBuildParams for RQBuildParams { fn sample_size(&self) -> usize { 0 diff --git a/rust/lance-index/src/vector/hnsw/builder.rs b/rust/lance-index/src/vector/hnsw/builder.rs index 06b533b2301..419304a1466 100644 --- a/rust/lance-index/src/vector/hnsw/builder.rs +++ b/rust/lance-index/src/vector/hnsw/builder.rs @@ -59,6 +59,15 @@ pub struct HnswBuildParams { pub prefetch_distance: Option, } +impl From<&HnswBuildParams> for lance_table::format::pb::HnswIndexDetails { + fn from(params: &HnswBuildParams) -> Self { + Self { + max_connections: params.m as u32, + construction_ef: params.ef_construction as u32, + } + } +} + impl Default for HnswBuildParams { fn default() -> Self { Self { diff --git a/rust/lance-index/src/vector/pq/builder.rs b/rust/lance-index/src/vector/pq/builder.rs index 1768e9fe8f0..a61d94b8640 100644 --- a/rust/lance-index/src/vector/pq/builder.rs +++ b/rust/lance-index/src/vector/pq/builder.rs @@ -44,6 +44,15 @@ pub struct PQBuildParams { pub sample_rate: usize, } +impl From<&PQBuildParams> for lance_table::format::pb::vector_index_details::ProductQuantization { + fn from(params: &PQBuildParams) -> Self { + Self { + num_bits: params.num_bits as u32, + num_sub_vectors: params.num_sub_vectors as u32, + } + } +} + impl Default for PQBuildParams { fn default() -> Self { Self { diff --git a/rust/lance-index/src/vector/sq/builder.rs b/rust/lance-index/src/vector/sq/builder.rs index 913751062cf..6fa6672708e 100644 --- a/rust/lance-index/src/vector/sq/builder.rs +++ b/rust/lance-index/src/vector/sq/builder.rs @@ -12,6 +12,14 @@ pub struct SQBuildParams { pub sample_rate: usize, } +impl From<&SQBuildParams> for lance_table::format::pb::vector_index_details::ScalarQuantization { + fn from(params: &SQBuildParams) -> Self { + Self { + num_bits: params.num_bits as u32, + } + } +} + impl Default for SQBuildParams { fn default() -> Self { Self { diff --git a/rust/lance/src/index.rs b/rust/lance/src/index.rs index 7fe7ae22074..52a9facac66 100644 --- a/rust/lance/src/index.rs +++ b/rust/lance/src/index.rs @@ -407,10 +407,10 @@ async fn open_index_proto(reader: &dyn Reader) -> Result { Ok(proto) } -fn vector_index_details() -> prost_types::Any { - let details = lance_table::format::pb::VectorIndexDetails::default(); - prost_types::Any::from_msg(&details).unwrap() -} +use vector::details::{ + derive_vector_index_type, infer_vector_index_details, vector_details_as_json, +}; +pub(crate) use vector::details::{vector_index_details, vector_index_details_default}; struct IndexDescriptionImpl { name: String, @@ -464,10 +464,14 @@ impl IndexDescriptionImpl { let details = IndexDetails(index_details.clone()); let mut rows_indexed = 0; - let index_type = details - .get_plugin() - .map(|p| p.name().to_string()) - .unwrap_or_else(|_| "Unknown".to_string()); + let index_type = if details.is_vector() { + derive_vector_index_type(index_details) + } else { + details + .get_plugin() + .map(|p| p.name().to_string()) + .unwrap_or_else(|_| "Unknown".to_string()) + }; for shard in &segments { let fragment_bitmap = shard @@ -519,10 +523,14 @@ impl IndexDescription for IndexDescriptionImpl { } fn details(&self) -> Result { - let plugin = self.details.get_plugin()?; - plugin - .details_as_json(&self.details.0) - .map(|v| v.to_string()) + if self.details.is_vector() { + vector_details_as_json(&self.details.0) + } else { + let plugin = self.details.get_plugin()?; + plugin + .details_as_json(&self.details.0) + .map(|v| v.to_string()) + } } } @@ -657,16 +665,47 @@ impl DatasetIndexExt for Dataset { }; indices.sort_by_key(|idx| &idx.name); - indices + // Collect groups upfront so we don't hold chunk_by across await points + let groups: Vec> = indices .into_iter() - .chunk_by(|idx| &idx.name) + .chunk_by(|idx| idx.name.clone()) .into_iter() - .map(|(_, segments)| { - let segments = segments.cloned().collect::>(); - let desc = IndexDescriptionImpl::try_new(segments, self)?; - Ok(Arc::new(desc) as Arc) - }) - .collect::>>() + .map(|(_, segments)| segments.cloned().collect::>()) + .collect(); + + let mut results: Vec> = Vec::new(); + for mut segments in groups { + // For vector indices with empty details, try to infer from index files + if let Some(first) = segments.first() { + let is_empty_vector = first + .index_details + .as_ref() + .map(|d| d.type_url.ends_with("VectorIndexDetails") && d.value.is_empty()) + .unwrap_or(false); + + if is_empty_vector { + match infer_vector_index_details(self, first).await { + Ok(inferred) => { + let inferred = Arc::new(inferred); + for seg in &mut segments { + seg.index_details = Some(inferred.clone()); + } + } + Err(err) => { + log::warn!( + "Could not infer vector index details for {}: {}", + first.name, + err + ); + } + } + } + } + + let desc = IndexDescriptionImpl::try_new(segments, self)?; + results.push(Arc::new(desc) as Arc); + } + Ok(results) } async fn load_indices(&self) -> Result>> { diff --git a/rust/lance/src/index/append.rs b/rust/lance/src/index/append.rs index 0d01fb442e8..094489b505f 100644 --- a/rust/lance/src/index/append.rs +++ b/rust/lance/src/index/append.rs @@ -20,7 +20,7 @@ use super::vector::ivf::optimize_vector_indices; use crate::dataset::Dataset; use crate::dataset::index::LanceIndexStoreExt; use crate::index::scalar::load_training_data; -use crate::index::vector_index_details; +use crate::index::vector_index_details_default; #[derive(Debug, Clone)] pub struct IndexMergeResults<'a> { @@ -213,11 +213,18 @@ pub async fn merge_indices_with_unindexed_frags<'a>( frag_bitmap.extend(idx.fragment_bitmap.as_ref().unwrap().iter()); }); + // Carry forward existing index details from the most recent index segment + let index_details = old_indices + .last() + .and_then(|idx| idx.index_details.as_ref()) + .map(|d| d.as_ref().clone()) + .unwrap_or_else(vector_index_details_default); + Ok(( new_uuid, indices_merged, CreatedIndex { - index_details: vector_index_details(), + index_details, index_version: VECTOR_INDEX_VERSION, }, )) diff --git a/rust/lance/src/index/create.rs b/rust/lance/src/index/create.rs index 5e16290060a..e4439d63b9d 100644 --- a/rust/lance/src/index/create.rs +++ b/rust/lance/src/index/create.rs @@ -14,7 +14,7 @@ use crate::{ LANCE_VECTOR_INDEX, VectorIndexParams, build_distributed_vector_index, build_empty_vector_index, build_vector_index, }, - vector_index_details, + vector_index_details, vector_index_details_default, }, }; use futures::future::BoxFuture; @@ -353,7 +353,7 @@ impl<'a> CreateIndexBuilder<'a> { .await?; } CreatedIndex { - index_details: vector_index_details(), + index_details: vector_index_details(vec_params), index_version: VECTOR_INDEX_VERSION, } } @@ -387,7 +387,7 @@ impl<'a> CreateIndexBuilder<'a> { todo!("create empty vector index when train=false"); } CreatedIndex { - index_details: vector_index_details(), + index_details: vector_index_details_default(), index_version: VECTOR_INDEX_VERSION, } } diff --git a/rust/lance/src/index/vector.rs b/rust/lance/src/index/vector.rs index 432f0e5cfe4..550289de6ee 100644 --- a/rust/lance/src/index/vector.rs +++ b/rust/lance/src/index/vector.rs @@ -8,6 +8,7 @@ use std::sync::Arc; use std::{any::Any, collections::HashMap}; pub mod builder; +pub(crate) mod details; pub mod ivf; pub mod pq; pub mod utils; @@ -59,7 +60,7 @@ use tracing::instrument; use utils::get_vector_type; use uuid::Uuid; -use super::{DatasetIndexInternalExt, IndexParams, pb, vector_index_details}; +use super::{DatasetIndexInternalExt, IndexParams, pb}; use crate::dataset::transaction::{Operation, Transaction}; use crate::{Error, Result, dataset::Dataset, index::pb::vector_index_stage::Stage}; @@ -1552,7 +1553,7 @@ pub async fn initialize_vector_index( fields: vec![field.id], dataset_version: target_dataset.manifest.version, fragment_bitmap, - index_details: Some(Arc::new(vector_index_details())), + index_details: source_index.index_details.clone(), index_version: VECTOR_INDEX_VERSION as i32, created_at: Some(chrono::Utc::now()), base_id: None, diff --git a/rust/lance/src/index/vector/details.rs b/rust/lance/src/index/vector/details.rs new file mode 100644 index 00000000000..064302330b0 --- /dev/null +++ b/rust/lance/src/index/vector/details.rs @@ -0,0 +1,347 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Serialization and deserialization of [`VectorIndexDetails`] proto messages. +//! +//! This module handles: +//! - Populating `VectorIndexDetails` from build params at index creation time +//! - Deriving a human-readable index type string (e.g., "IVF_PQ") from details +//! - Serializing details as JSON for `describe_indices()` +//! - Inferring details from index files on disk (fallback for legacy indices) + +use std::sync::Arc; + +use lance_file::reader::FileReaderOptions; +use lance_index::pb::index::Implementation; +use lance_index::{INDEX_FILE_NAME, INDEX_METADATA_SCHEMA_KEY, pb}; +use lance_io::scheduler::{ScanScheduler, SchedulerConfig}; +use lance_io::traits::Reader; +use lance_io::utils::{CachedFileSize, read_last_block, read_version}; +use lance_table::format::IndexMetadata; +use serde_json::json; + +use super::{StageParams, VectorIndexParams}; +use crate::dataset::Dataset; +use crate::index::open_index_proto; +use crate::{Error, Result}; + +/// Build a `VectorIndexDetails` proto from build params at index creation time. +pub fn vector_index_details(params: &VectorIndexParams) -> prost_types::Any { + use lance_table::format::pb::VectorIndexDetails; + use lance_table::format::pb::vector_index_details::*; + + let metric_type = match params.metric_type { + lance_linalg::distance::DistanceType::L2 => VectorMetricType::L2, + lance_linalg::distance::DistanceType::Cosine => VectorMetricType::Cosine, + lance_linalg::distance::DistanceType::Dot => VectorMetricType::Dot, + lance_linalg::distance::DistanceType::Hamming => VectorMetricType::Hamming, + }; + + let mut target_partition_size = 0u64; + let mut hnsw_index_config = None; + let mut compression = None; + + for stage in ¶ms.stages { + match stage { + StageParams::Ivf(ivf) => { + if let Some(tps) = ivf.target_partition_size { + target_partition_size = tps as u64; + } + } + StageParams::Hnsw(hnsw) => { + hnsw_index_config = Some(hnsw.into()); + } + StageParams::PQ(pq) => { + compression = Some(Compression::Pq(pq.into())); + } + StageParams::SQ(sq) => { + compression = Some(Compression::Sq(sq.into())); + } + StageParams::RQ(rq) => { + compression = Some(Compression::Rq(rq.into())); + } + } + } + + let details = VectorIndexDetails { + metric_type: metric_type.into(), + target_partition_size, + hnsw_index_config, + compression, + }; + prost_types::Any::from_msg(&details).unwrap() +} + +pub fn vector_index_details_default() -> prost_types::Any { + let details = lance_table::format::pb::VectorIndexDetails::default(); + prost_types::Any::from_msg(&details).unwrap() +} + +/// Returns true if the proto value represents a "truly empty" VectorIndexDetails +/// (i.e., a legacy index that was created before we populated this field). +fn is_empty_vector_details(details: &prost_types::Any) -> bool { + details.value.is_empty() +} + +/// Derive a human-readable index type string from VectorIndexDetails. +pub fn derive_vector_index_type(details: &prost_types::Any) -> String { + use lance_table::format::pb::VectorIndexDetails; + use lance_table::format::pb::vector_index_details::Compression; + + if is_empty_vector_details(details) { + return "Vector".to_string(); + } + + let Ok(d) = details.to_msg::() else { + return "Vector".to_string(); + }; + let has_hnsw = d.hnsw_index_config.is_some(); + match d.compression { + None | Some(Compression::Flat(_)) => { + if has_hnsw { + "IVF_HNSW_FLAT" + } else { + "IVF_FLAT" + } + } + Some(Compression::Pq(_)) => { + if has_hnsw { + "IVF_HNSW_PQ" + } else { + "IVF_PQ" + } + } + Some(Compression::Sq(_)) => { + if has_hnsw { + "IVF_HNSW_SQ" + } else { + "IVF_SQ" + } + } + Some(Compression::Rq(_)) => "IVF_RQ", + } + .to_string() +} + +/// Serialize VectorIndexDetails as a JSON string. +pub fn vector_details_as_json(details: &prost_types::Any) -> Result { + use lance_table::format::pb::VectorIndexDetails; + use lance_table::format::pb::vector_index_details::*; + + if is_empty_vector_details(details) { + return Ok("{}".to_string()); + } + + let d = details + .to_msg::() + .map_err(|e| Error::index(format!("Failed to deserialize VectorIndexDetails: {}", e)))?; + + let metric_type = match VectorMetricType::try_from(d.metric_type) { + Ok(VectorMetricType::L2) => "L2", + Ok(VectorMetricType::Cosine) => "COSINE", + Ok(VectorMetricType::Dot) => "DOT", + Ok(VectorMetricType::Hamming) => "HAMMING", + Err(_) => "UNKNOWN", + }; + + let mut obj = serde_json::Map::new(); + obj.insert( + "metric_type".to_string(), + serde_json::Value::String(metric_type.to_string()), + ); + + if d.target_partition_size > 0 { + obj.insert( + "target_partition_size".to_string(), + json!(d.target_partition_size), + ); + } + + if let Some(hnsw) = d.hnsw_index_config { + obj.insert( + "hnsw".to_string(), + json!({ + "max_connections": hnsw.max_connections, + "construction_ef": hnsw.construction_ef, + }), + ); + } + + if let Some(compression) = d.compression { + let comp_json = match compression { + Compression::Flat(_) => json!({"type": "flat"}), + Compression::Pq(pq) => json!({ + "type": "pq", + "num_bits": pq.num_bits, + "num_sub_vectors": pq.num_sub_vectors, + }), + Compression::Sq(sq) => json!({ + "type": "sq", + "num_bits": sq.num_bits, + }), + Compression::Rq(rq) => json!({ + "type": "rq", + "num_bits": rq.num_bits, + }), + }; + obj.insert("compression".to_string(), comp_json); + } + + Ok(serde_json::Value::Object(obj).to_string()) +} + +/// Infer VectorIndexDetails from index files on disk. +/// Used as a fallback for legacy indices where the manifest doesn't have populated details. +pub async fn infer_vector_index_details( + dataset: &Dataset, + index: &IndexMetadata, +) -> Result { + let uuid = index.uuid.to_string(); + let index_dir = dataset.indice_files_dir(index)?; + let index_file = index_dir.child(uuid.as_str()).child(INDEX_FILE_NAME); + let reader: Arc = dataset.object_store.open(&index_file).await?.into(); + + let tailing_bytes = read_last_block(reader.as_ref()).await?; + let (major_version, minor_version) = read_version(&tailing_bytes)?; + + match (major_version, minor_version) { + (0, 1) | (0, 0) => { + // Legacy v0.1: read pb::Index, extract VectorIndex stages + let proto = open_index_proto(reader.as_ref()).await?; + convert_legacy_proto_to_details(&proto) + } + _ => { + // v0.2+/v0.3: read lance file schema metadata + convert_v3_metadata_to_details(dataset, &index_file).await + } + } +} + +fn convert_legacy_proto_to_details(proto: &pb::Index) -> Result { + use lance_table::format::pb::VectorIndexDetails; + use lance_table::format::pb::vector_index_details::*; + use pb::vector_index_stage::Stage; + + let Some(Implementation::VectorIndex(vector_index)) = &proto.implementation else { + return Ok(vector_index_details_default()); + }; + + let metric_type = match pb::VectorMetricType::try_from(vector_index.metric_type) { + Ok(pb::VectorMetricType::L2) => VectorMetricType::L2, + Ok(pb::VectorMetricType::Cosine) => VectorMetricType::Cosine, + Ok(pb::VectorMetricType::Dot) => VectorMetricType::Dot, + Ok(pb::VectorMetricType::Hamming) => VectorMetricType::Hamming, + Err(_) => VectorMetricType::L2, + }; + + let mut compression = None; + for stage in &vector_index.stages { + if let Some(Stage::Pq(pq)) = &stage.stage { + compression = Some(Compression::Pq(ProductQuantization { + num_bits: pq.num_bits, + num_sub_vectors: pq.num_sub_vectors, + })); + } + } + + let details = VectorIndexDetails { + metric_type: metric_type.into(), + target_partition_size: 0, + hnsw_index_config: None, + compression, + }; + Ok(prost_types::Any::from_msg(&details).unwrap()) +} + +async fn convert_v3_metadata_to_details( + dataset: &Dataset, + index_file: &object_store::path::Path, +) -> Result { + use lance_index::vector::bq::storage::RABIT_METADATA_KEY; + use lance_index::vector::hnsw::HnswMetadata; + use lance_index::vector::ivf::storage::IVF_PARTITION_KEY; + use lance_index::vector::pq::storage::{PQ_METADATA_KEY, ProductQuantizationMetadata}; + use lance_index::vector::sq::storage::{SQ_METADATA_KEY, ScalarQuantizationMetadata}; + use lance_table::format::pb::vector_index_details::*; + use lance_table::format::pb::{HnswIndexDetails, VectorIndexDetails}; + + let scheduler = ScanScheduler::new( + dataset.object_store.clone(), + SchedulerConfig::max_bandwidth(&dataset.object_store), + ); + let file = scheduler + .open_file(index_file, &CachedFileSize::unknown()) + .await?; + let reader = lance_file::reader::FileReader::try_open( + file, + None, + Default::default(), + &dataset.metadata_cache.file_metadata_cache(index_file), + FileReaderOptions::default(), + ) + .await?; + + let metadata = &reader.schema().metadata; + + // Get distance_type from index metadata + let metric_type = if let Some(idx_meta_str) = metadata.get(INDEX_METADATA_SCHEMA_KEY) { + let idx_meta: lance_index::IndexMetadata = serde_json::from_str(idx_meta_str)?; + match idx_meta.distance_type.to_uppercase().as_str() { + "L2" | "EUCLIDEAN" => VectorMetricType::L2, + "COSINE" => VectorMetricType::Cosine, + "DOT" => VectorMetricType::Dot, + "HAMMING" => VectorMetricType::Hamming, + _ => VectorMetricType::L2, + } + } else { + VectorMetricType::L2 + }; + + // Check for compression + let compression = if let Some(pq_str) = metadata.get(PQ_METADATA_KEY) { + let pq_meta: ProductQuantizationMetadata = serde_json::from_str(pq_str)?; + Some(Compression::Pq(ProductQuantization { + num_bits: pq_meta.nbits as u32, + num_sub_vectors: pq_meta.num_sub_vectors as u32, + })) + } else if let Some(sq_str) = metadata.get(SQ_METADATA_KEY) { + let sq_meta: ScalarQuantizationMetadata = serde_json::from_str(sq_str)?; + Some(Compression::Sq(ScalarQuantization { + num_bits: sq_meta.num_bits as u32, + })) + } else if let Some(rq_str) = metadata.get(RABIT_METADATA_KEY) { + let rq_meta: lance_index::vector::bq::storage::RabitQuantizationMetadata = + serde_json::from_str(rq_str)?; + let rotation_type = match rq_meta.rotation_type { + lance_index::vector::bq::RQRotationType::Fast => rabit_quantization::RotationType::Fast, + lance_index::vector::bq::RQRotationType::Matrix => { + rabit_quantization::RotationType::Matrix + } + }; + Some(Compression::Rq(RabitQuantization { + num_bits: rq_meta.num_bits as u32, + rotation_type: rotation_type.into(), + })) + } else { + None + }; + + // Check for HNSW + let hnsw_index_config = if let Some(partition_str) = metadata.get(IVF_PARTITION_KEY) { + let partitions: Vec = serde_json::from_str(partition_str)?; + partitions.first().map(|hnsw| HnswIndexDetails { + max_connections: hnsw.params.m as u32, + construction_ef: hnsw.params.ef_construction as u32, + }) + } else { + None + }; + + let details = VectorIndexDetails { + metric_type: metric_type.into(), + target_partition_size: 0, + hnsw_index_config, + compression, + }; + Ok(prost_types::Any::from_msg(&details).unwrap()) +} diff --git a/rust/lance/src/index/vector/ivf.rs b/rust/lance/src/index/vector/ivf.rs index 67906bd742b..9b5d9a3cf13 100644 --- a/rust/lance/src/index/vector/ivf.rs +++ b/rust/lance/src/index/vector/ivf.rs @@ -2260,7 +2260,7 @@ mod tests { use crate::dataset::{InsertBuilder, WriteMode, WriteParams}; use crate::index::prefilter::DatasetPreFilter; use crate::index::vector::IndexFileVersion; - use crate::index::vector_index_details; + use crate::index::vector_index_details_default; use crate::index::{DatasetIndexExt, DatasetIndexInternalExt, vector::VectorIndexParams}; const DIM: usize = 32; @@ -2658,7 +2658,7 @@ mod tests { .map(|f| f.id() as u32) .collect(), ), - index_details: Some(Arc::new(vector_index_details())), + index_details: Some(Arc::new(vector_index_details_default())), index_version: VECTOR_INDEX_VERSION as i32, created_at: Some(chrono::Utc::now()), base_id: None, @@ -2696,7 +2696,7 @@ mod tests { fields: Vec::new(), name: INDEX_NAME.to_string(), fragment_bitmap: None, - index_details: Some(Arc::new(vector_index_details())), + index_details: Some(Arc::new(vector_index_details_default())), index_version: VECTOR_INDEX_VERSION as i32, created_at: None, // Test index, not setting timestamp base_id: None, @@ -2761,7 +2761,7 @@ mod tests { .map(|f| f.id() as u32) .collect(), ), - index_details: Some(Arc::new(vector_index_details())), + index_details: Some(Arc::new(vector_index_details_default())), index_version: VECTOR_INDEX_VERSION as i32, created_at: Some(chrono::Utc::now()), base_id: None, From eb9afeedf78089493b3fdd477d1b9e1f38fc7e47 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Wed, 4 Mar 2026 15:30:07 -0800 Subject: [PATCH 03/22] use Serialize structs for vector details JSON, add snapshot tests Replace imperative serde_json::Map construction with #[derive(Serialize)] structs for clearer, more maintainable JSON serialization. This also adds the missing rotation_type field to RQ compression output. Add snapshot-style unit tests that assert exact JSON strings to guard backwards compatibility of the describe_indices() output format. Co-Authored-By: Claude Opus 4.6 --- protos/table.proto | 2 +- rust/lance/src/index/vector/details.rs | 299 +++++++++++++++++++++---- 2 files changed, 256 insertions(+), 45 deletions(-) diff --git a/protos/table.proto b/protos/table.proto index 2605d8c88d5..29e7834aa66 100644 --- a/protos/table.proto +++ b/protos/table.proto @@ -460,7 +460,7 @@ message ExternalFile { uint64 size = 3; } -// Empty details messages for older indexes that don't take advantage of the details field. +// Details for vector indexes. message VectorIndexDetails { enum VectorMetricType { L2 = 0; diff --git a/rust/lance/src/index/vector/details.rs b/rust/lance/src/index/vector/details.rs index 064302330b0..1493c0759b0 100644 --- a/rust/lance/src/index/vector/details.rs +++ b/rust/lance/src/index/vector/details.rs @@ -18,13 +18,51 @@ use lance_io::scheduler::{ScanScheduler, SchedulerConfig}; use lance_io::traits::Reader; use lance_io::utils::{CachedFileSize, read_last_block, read_version}; use lance_table::format::IndexMetadata; -use serde_json::json; +use serde::Serialize; use super::{StageParams, VectorIndexParams}; use crate::dataset::Dataset; use crate::index::open_index_proto; use crate::{Error, Result}; +// Private structs for JSON serialization of VectorIndexDetails. +// Changes to field names or structure are backwards-incompatible for users +// parsing the JSON output of describe_indices(). See snapshot tests below. + +#[derive(Serialize)] +struct VectorDetailsJson { + metric_type: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + target_partition_size: Option, + #[serde(skip_serializing_if = "Option::is_none")] + hnsw: Option, + #[serde(skip_serializing_if = "Option::is_none")] + compression: Option, +} + +#[derive(Serialize)] +struct HnswDetailsJson { + max_connections: u32, + construction_ef: u32, +} + +#[derive(Serialize)] +#[serde(tag = "type", rename_all = "lowercase")] +enum CompressionDetailsJson { + Flat, + Pq { + num_bits: u32, + num_sub_vectors: u32, + }, + Sq { + num_bits: u32, + }, + Rq { + num_bits: u32, + rotation_type: &'static str, + }, +} + /// Build a `VectorIndexDetails` proto from build params at index creation time. pub fn vector_index_details(params: &VectorIndexParams) -> prost_types::Any { use lance_table::format::pb::VectorIndexDetails; @@ -144,50 +182,44 @@ pub fn vector_details_as_json(details: &prost_types::Any) -> Result { Err(_) => "UNKNOWN", }; - let mut obj = serde_json::Map::new(); - obj.insert( - "metric_type".to_string(), - serde_json::Value::String(metric_type.to_string()), - ); - - if d.target_partition_size > 0 { - obj.insert( - "target_partition_size".to_string(), - json!(d.target_partition_size), - ); - } - - if let Some(hnsw) = d.hnsw_index_config { - obj.insert( - "hnsw".to_string(), - json!({ - "max_connections": hnsw.max_connections, - "construction_ef": hnsw.construction_ef, - }), - ); - } - - if let Some(compression) = d.compression { - let comp_json = match compression { - Compression::Flat(_) => json!({"type": "flat"}), - Compression::Pq(pq) => json!({ - "type": "pq", - "num_bits": pq.num_bits, - "num_sub_vectors": pq.num_sub_vectors, - }), - Compression::Sq(sq) => json!({ - "type": "sq", - "num_bits": sq.num_bits, - }), - Compression::Rq(rq) => json!({ - "type": "rq", - "num_bits": rq.num_bits, - }), - }; - obj.insert("compression".to_string(), comp_json); - } + let hnsw = d.hnsw_index_config.map(|h| HnswDetailsJson { + max_connections: h.max_connections, + construction_ef: h.construction_ef, + }); + + let compression = d.compression.map(|c| match c { + Compression::Flat(_) => CompressionDetailsJson::Flat, + Compression::Pq(pq) => CompressionDetailsJson::Pq { + num_bits: pq.num_bits, + num_sub_vectors: pq.num_sub_vectors, + }, + Compression::Sq(sq) => CompressionDetailsJson::Sq { + num_bits: sq.num_bits, + }, + Compression::Rq(rq) => { + let rotation_type = match rabit_quantization::RotationType::try_from(rq.rotation_type) { + Ok(rabit_quantization::RotationType::Matrix) => "matrix", + _ => "fast", + }; + CompressionDetailsJson::Rq { + num_bits: rq.num_bits, + rotation_type, + } + } + }); + + let json = VectorDetailsJson { + metric_type, + target_partition_size: if d.target_partition_size > 0 { + Some(d.target_partition_size) + } else { + None + }, + hnsw, + compression, + }; - Ok(serde_json::Value::Object(obj).to_string()) + serde_json::to_string(&json).map_err(|e| Error::index(format!("Failed to serialize: {}", e))) } /// Infer VectorIndexDetails from index files on disk. @@ -345,3 +377,182 @@ async fn convert_v3_metadata_to_details( }; Ok(prost_types::Any::from_msg(&details).unwrap()) } + +#[cfg(test)] +mod tests { + use super::*; + use lance_table::format::pb::vector_index_details::*; + use lance_table::format::pb::{HnswIndexDetails, VectorIndexDetails}; + + fn make_details( + metric: VectorMetricType, + hnsw: Option, + compression: Option, + ) -> prost_types::Any { + let details = VectorIndexDetails { + metric_type: metric.into(), + target_partition_size: 0, + hnsw_index_config: hnsw, + compression, + }; + prost_types::Any::from_msg(&details).unwrap() + } + + #[test] + fn test_derive_index_type_without_hnsw() { + // Note: (None, "IVF_FLAT") is not testable here because a proto with + // all defaults serializes to empty bytes, which is treated as a legacy index. + let cases: [(Option, &str); 4] = [ + (Some(Compression::Flat(Flat {})), "IVF_FLAT"), + ( + Some(Compression::Pq(ProductQuantization { + num_bits: 8, + num_sub_vectors: 16, + })), + "IVF_PQ", + ), + ( + Some(Compression::Sq(ScalarQuantization { num_bits: 8 })), + "IVF_SQ", + ), + ( + Some(Compression::Rq(RabitQuantization { + num_bits: 1, + rotation_type: 0, + })), + "IVF_RQ", + ), + ]; + for (compression, expected) in cases { + let details = make_details(VectorMetricType::L2, None, compression); + assert_eq!(derive_vector_index_type(&details), expected); + } + } + + #[test] + fn test_derive_index_type_with_hnsw() { + let hnsw = Some(HnswIndexDetails { + max_connections: 20, + construction_ef: 150, + }); + assert_eq!( + derive_vector_index_type(&make_details(VectorMetricType::L2, hnsw, None)), + "IVF_HNSW_FLAT" + ); + assert_eq!( + derive_vector_index_type(&make_details( + VectorMetricType::L2, + hnsw, + Some(Compression::Pq(ProductQuantization { + num_bits: 8, + num_sub_vectors: 16, + })) + )), + "IVF_HNSW_PQ" + ); + assert_eq!( + derive_vector_index_type(&make_details( + VectorMetricType::L2, + hnsw, + Some(Compression::Sq(ScalarQuantization { num_bits: 8 })) + )), + "IVF_HNSW_SQ" + ); + } + + #[test] + fn test_derive_index_type_empty_details() { + let details = vector_index_details_default(); + assert_eq!(derive_vector_index_type(&details), "Vector"); + } + + // Snapshot tests for JSON serialization. These guard backwards compatibility + // of the JSON format returned by describe_indices(). + + #[test] + fn test_json_ivf_pq() { + let details = make_details( + VectorMetricType::L2, + None, + Some(Compression::Pq(ProductQuantization { + num_bits: 8, + num_sub_vectors: 16, + })), + ); + assert_eq!( + vector_details_as_json(&details).unwrap(), + r#"{"metric_type":"L2","compression":{"type":"pq","num_bits":8,"num_sub_vectors":16}}"# + ); + } + + #[test] + fn test_json_ivf_hnsw_sq() { + let details = make_details( + VectorMetricType::Cosine, + Some(HnswIndexDetails { + max_connections: 30, + construction_ef: 200, + }), + Some(Compression::Sq(ScalarQuantization { num_bits: 4 })), + ); + assert_eq!( + vector_details_as_json(&details).unwrap(), + r#"{"metric_type":"COSINE","hnsw":{"max_connections":30,"construction_ef":200},"compression":{"type":"sq","num_bits":4}}"# + ); + } + + #[test] + fn test_json_ivf_rq_with_rotation() { + let details = make_details( + VectorMetricType::Dot, + None, + Some(Compression::Rq(RabitQuantization { + num_bits: 1, + rotation_type: rabit_quantization::RotationType::Matrix as i32, + })), + ); + assert_eq!( + vector_details_as_json(&details).unwrap(), + r#"{"metric_type":"DOT","compression":{"type":"rq","num_bits":1,"rotation_type":"matrix"}}"# + ); + } + + #[test] + fn test_json_ivf_rq_fast_rotation() { + let details = make_details( + VectorMetricType::L2, + None, + Some(Compression::Rq(RabitQuantization { + num_bits: 1, + rotation_type: rabit_quantization::RotationType::Fast as i32, + })), + ); + assert_eq!( + vector_details_as_json(&details).unwrap(), + r#"{"metric_type":"L2","compression":{"type":"rq","num_bits":1,"rotation_type":"fast"}}"# + ); + } + + #[test] + fn test_json_ivf_flat_with_target_partition_size() { + let details = { + let d = VectorIndexDetails { + metric_type: VectorMetricType::L2.into(), + target_partition_size: 5000, + hnsw_index_config: None, + compression: Some(Compression::Flat(Flat {})), + }; + prost_types::Any::from_msg(&d).unwrap() + }; + assert_eq!( + vector_details_as_json(&details).unwrap(), + r#"{"metric_type":"L2","target_partition_size":5000,"compression":{"type":"flat"}}"# + ); + } + + #[test] + fn test_json_empty_details() { + let details = vector_index_details_default(); + assert_eq!(vector_details_as_json(&details).unwrap(), "{}"); + } +} From 36fd15984ea6f76363c9d5ff0ca7fda15060ef2f Mon Sep 17 00:00:00 2001 From: Will Jones Date: Wed, 4 Mar 2026 15:51:32 -0800 Subject: [PATCH 04/22] remove flat variant --- protos/table.proto | 3 +-- rust/lance/src/index/vector/details.rs | 13 +++++-------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/protos/table.proto b/protos/table.proto index 29e7834aa66..2e46e6fcce5 100644 --- a/protos/table.proto +++ b/protos/table.proto @@ -475,7 +475,6 @@ message VectorIndexDetails { optional HnswIndexDetails hnsw_index_config = 3; - message Flat {} message ProductQuantization { uint32 num_bits = 1; uint32 num_sub_vectors = 2; @@ -492,8 +491,8 @@ message VectorIndexDetails { RotationType rotation_type = 2; } + // An unset compression oneof means flat / no quantization. oneof compression { - Flat flat = 4; ProductQuantization pq = 5; ScalarQuantization sq = 6; RabitQuantization rq = 7; diff --git a/rust/lance/src/index/vector/details.rs b/rust/lance/src/index/vector/details.rs index 1493c0759b0..db054bd903d 100644 --- a/rust/lance/src/index/vector/details.rs +++ b/rust/lance/src/index/vector/details.rs @@ -49,7 +49,6 @@ struct HnswDetailsJson { #[derive(Serialize)] #[serde(tag = "type", rename_all = "lowercase")] enum CompressionDetailsJson { - Flat, Pq { num_bits: u32, num_sub_vectors: u32, @@ -135,7 +134,7 @@ pub fn derive_vector_index_type(details: &prost_types::Any) -> String { }; let has_hnsw = d.hnsw_index_config.is_some(); match d.compression { - None | Some(Compression::Flat(_)) => { + None => { if has_hnsw { "IVF_HNSW_FLAT" } else { @@ -188,7 +187,6 @@ pub fn vector_details_as_json(details: &prost_types::Any) -> Result { }); let compression = d.compression.map(|c| match c { - Compression::Flat(_) => CompressionDetailsJson::Flat, Compression::Pq(pq) => CompressionDetailsJson::Pq { num_bits: pq.num_bits, num_sub_vectors: pq.num_sub_vectors, @@ -402,8 +400,7 @@ mod tests { fn test_derive_index_type_without_hnsw() { // Note: (None, "IVF_FLAT") is not testable here because a proto with // all defaults serializes to empty bytes, which is treated as a legacy index. - let cases: [(Option, &str); 4] = [ - (Some(Compression::Flat(Flat {})), "IVF_FLAT"), + let cases: [(Option, &str); 3] = [ ( Some(Compression::Pq(ProductQuantization { num_bits: 8, @@ -534,19 +531,19 @@ mod tests { } #[test] - fn test_json_ivf_flat_with_target_partition_size() { + fn test_json_with_target_partition_size() { let details = { let d = VectorIndexDetails { metric_type: VectorMetricType::L2.into(), target_partition_size: 5000, hnsw_index_config: None, - compression: Some(Compression::Flat(Flat {})), + compression: None, }; prost_types::Any::from_msg(&d).unwrap() }; assert_eq!( vector_details_as_json(&details).unwrap(), - r#"{"metric_type":"L2","target_partition_size":5000,"compression":{"type":"flat"}}"# + r#"{"metric_type":"L2","target_partition_size":5000}"# ); } From 822b6aa567425e3f12678a0ef53c47442a32355f Mon Sep 17 00:00:00 2001 From: Will Jones Date: Wed, 4 Mar 2026 16:14:04 -0800 Subject: [PATCH 05/22] infer vector index details eagerly on load and migration Previously, vector index details for legacy indices were only inferred lazily in describe_indices(). This moves inference to load_indices() and migrate_indices(), so details are populated before caching and persisted into new manifest versions. Inference runs once per index name, concurrently. Co-Authored-By: Claude Opus 4.6 --- rust/lance/src/index.rs | 62 ++++++++++++++------------ rust/lance/src/index/vector/details.rs | 10 +++++ rust/lance/src/io/commit.rs | 30 +++++++++++++ 3 files changed, 73 insertions(+), 29 deletions(-) diff --git a/rust/lance/src/index.rs b/rust/lance/src/index.rs index 52a9facac66..a92263933f5 100644 --- a/rust/lance/src/index.rs +++ b/rust/lance/src/index.rs @@ -408,7 +408,8 @@ async fn open_index_proto(reader: &dyn Reader) -> Result { } use vector::details::{ - derive_vector_index_type, infer_vector_index_details, vector_details_as_json, + derive_vector_index_type, infer_vector_index_details, is_vector_index_with_empty_details, + vector_details_as_json, }; pub(crate) use vector::details::{vector_index_details, vector_index_details_default}; @@ -674,34 +675,7 @@ impl DatasetIndexExt for Dataset { .collect(); let mut results: Vec> = Vec::new(); - for mut segments in groups { - // For vector indices with empty details, try to infer from index files - if let Some(first) = segments.first() { - let is_empty_vector = first - .index_details - .as_ref() - .map(|d| d.type_url.ends_with("VectorIndexDetails") && d.value.is_empty()) - .unwrap_or(false); - - if is_empty_vector { - match infer_vector_index_details(self, first).await { - Ok(inferred) => { - let inferred = Arc::new(inferred); - for seg in &mut segments { - seg.index_details = Some(inferred.clone()); - } - } - Err(err) => { - log::warn!( - "Could not infer vector index details for {}: {}", - first.name, - err - ); - } - } - } - } - + for segments in groups { let desc = IndexDescriptionImpl::try_new(segments, self)?; results.push(Arc::new(desc) as Arc); } @@ -722,6 +696,36 @@ impl DatasetIndexExt for Dataset { ) .await?; retain_supported_indices(&mut loaded_indices); + + // Infer details for legacy vector indices (once per index name, concurrently). + let needs_inference: HashMap<&str, &IndexMetadata> = loaded_indices + .iter() + .filter(|idx| is_vector_index_with_empty_details(idx)) + .map(|idx| (idx.name.as_str(), idx)) + .collect(); + let inferred: HashMap> = futures::future::join_all( + needs_inference + .into_iter() + .map(|(name, representative)| async move { + let result = infer_vector_index_details(self, representative).await; + (name.to_string(), result) + }), + ) + .await + .into_iter() + .filter_map(|(name, result)| match result { + Ok(details) => Some((name, Arc::new(details))), + Err(err) => { + log::warn!("Could not infer vector index details for {}: {}", name, err); + None + } + }) + .collect(); + for index in &mut loaded_indices { + if let Some(details) = inferred.get(&index.name) { + index.index_details = Some(details.clone()); + } + } let loaded_indices = Arc::new(loaded_indices); self.index_cache .insert_with_key(&metadata_key, loaded_indices.clone()) diff --git a/rust/lance/src/index/vector/details.rs b/rust/lance/src/index/vector/details.rs index db054bd903d..db0b6c2ec1a 100644 --- a/rust/lance/src/index/vector/details.rs +++ b/rust/lance/src/index/vector/details.rs @@ -120,6 +120,16 @@ fn is_empty_vector_details(details: &prost_types::Any) -> bool { details.value.is_empty() } +/// Returns true if this index has a VectorIndexDetails type_url but empty value bytes, +/// indicating a legacy index that needs details inferred from disk. +pub fn is_vector_index_with_empty_details(index: &IndexMetadata) -> bool { + index + .index_details + .as_ref() + .map(|d| d.type_url.ends_with("VectorIndexDetails") && d.value.is_empty()) + .unwrap_or(false) +} + /// Derive a human-readable index type string from VectorIndexDetails. pub fn derive_vector_index_type(details: &prost_types::Any) -> String { use lance_table::format::pb::VectorIndexDetails; diff --git a/rust/lance/src/io/commit.rs b/rust/lance/src/io/commit.rs index ebab7eb6b4d..d7e4941efc9 100644 --- a/rust/lance/src/io/commit.rs +++ b/rust/lance/src/io/commit.rs @@ -565,6 +565,36 @@ fn must_recalculate_fragment_bitmap( /// /// Indices might be missing `fragment_bitmap`, so this function will add it. async fn migrate_indices(dataset: &Dataset, indices: &mut [IndexMetadata]) -> Result<()> { + use crate::index::vector::details::{ + infer_vector_index_details, is_vector_index_with_empty_details, + }; + let needs_inference: HashMap<&str, &IndexMetadata> = indices + .iter() + .filter(|idx| is_vector_index_with_empty_details(idx)) + .map(|idx| (idx.name.as_str(), idx)) + .collect(); + let inferred: HashMap> = + futures::future::join_all(needs_inference.into_iter().map( + |(name, representative)| async move { + let result = infer_vector_index_details(dataset, representative).await; + (name.to_string(), result) + }, + )) + .await + .into_iter() + .filter_map(|(name, result)| match result { + Ok(details) => Some((name, Arc::new(details))), + Err(err) => { + log::warn!("Could not infer vector index details for {}: {}", name, err); + None + } + }) + .collect(); + for index in indices.iter_mut() { + if let Some(details) = inferred.get(&index.name) { + index.index_details = Some(details.clone()); + } + } let needs_recalculating = match detect_overlapping_fragments(indices) { Ok(()) => vec![], Err(BadFragmentBitmapError { bad_indices }) => { From 6269205090d0f565198491f326d26c3ef5b98d11 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Wed, 4 Mar 2026 16:47:34 -0800 Subject: [PATCH 06/22] add test for legacy vector index details inference and migration Also handles the case where index_details is None (very old indices) by checking if the indexed field is a vector type. Moves inference outside the cache-miss branch in load_indices so it also runs on indices that were opportunistically cached during Dataset::open. Co-Authored-By: Claude Opus 4.6 --- rust/lance/src/index.rs | 161 +++++++++++++++++++++---- rust/lance/src/index/vector/details.rs | 28 +++-- rust/lance/src/io/commit.rs | 5 +- 3 files changed, 163 insertions(+), 31 deletions(-) diff --git a/rust/lance/src/index.rs b/rust/lance/src/index.rs index a92263933f5..ae41d0935fd 100644 --- a/rust/lance/src/index.rs +++ b/rust/lance/src/index.rs @@ -408,7 +408,7 @@ async fn open_index_proto(reader: &dyn Reader) -> Result { } use vector::details::{ - derive_vector_index_type, infer_vector_index_details, is_vector_index_with_empty_details, + derive_vector_index_type, infer_vector_index_details, needs_vector_details_inference, vector_details_as_json, }; pub(crate) use vector::details::{vector_index_details, vector_index_details_default}; @@ -686,7 +686,7 @@ impl DatasetIndexExt for Dataset { let metadata_key = IndexMetadataKey { version: self.version().version, }; - let indices = match self.index_cache.get_with_key(&metadata_key).await { + let mut indices = match self.index_cache.get_with_key(&metadata_key).await { Some(indices) => indices, None => { let mut loaded_indices = read_manifest_indexes( @@ -696,21 +696,31 @@ impl DatasetIndexExt for Dataset { ) .await?; retain_supported_indices(&mut loaded_indices); + let loaded_indices = Arc::new(loaded_indices); + self.index_cache + .insert_with_key(&metadata_key, loaded_indices.clone()) + .await; + loaded_indices + } + }; - // Infer details for legacy vector indices (once per index name, concurrently). - let needs_inference: HashMap<&str, &IndexMetadata> = loaded_indices - .iter() - .filter(|idx| is_vector_index_with_empty_details(idx)) - .map(|idx| (idx.name.as_str(), idx)) - .collect(); - let inferred: HashMap> = futures::future::join_all( - needs_inference - .into_iter() - .map(|(name, representative)| async move { - let result = infer_vector_index_details(self, representative).await; - (name.to_string(), result) - }), - ) + // Infer details for legacy vector indices (once per index name, concurrently). + // This may run on indices that were opportunistically cached during Dataset::open + // before the full Dataset was available for inference. + let schema = self.schema(); + let needs_inference: HashMap<&str, &IndexMetadata> = indices + .iter() + .filter(|idx| needs_vector_details_inference(idx, schema)) + .map(|idx| (idx.name.as_str(), idx)) + .collect(); + if !needs_inference.is_empty() { + let inferred: HashMap> = + futures::future::join_all(needs_inference.into_iter().map( + |(name, representative)| async move { + let result = infer_vector_index_details(self, representative).await; + (name.to_string(), result) + }, + )) .await .into_iter() .filter_map(|(name, result)| match result { @@ -721,18 +731,19 @@ impl DatasetIndexExt for Dataset { } }) .collect(); - for index in &mut loaded_indices { + if !inferred.is_empty() { + let mut updated = indices.as_ref().clone(); + for index in &mut updated { if let Some(details) = inferred.get(&index.name) { index.index_details = Some(details.clone()); } } - let loaded_indices = Arc::new(loaded_indices); + indices = Arc::new(updated); self.index_cache - .insert_with_key(&metadata_key, loaded_indices.clone()) + .insert_with_key(&metadata_key, indices.clone()) .await; - loaded_indices } - }; + } if let Some(frag_reuse_index_meta) = indices.iter().find(|idx| idx.name == FRAG_REUSE_INDEX_NAME) @@ -3181,6 +3192,114 @@ mod tests { "updated_at_timestamp_ms should be null when no indices have created_at timestamps" ); } + + #[tokio::test] + async fn test_legacy_vector_index_details_inferred_on_load_and_migration() { + use lance_linalg::distance::DistanceType; + + // Create a fresh dataset with IVF_HNSW_SQ so inference produces non-default + // details (HNSW config + SQ compression) that survive proto serialization. + let test_dir = lance_core::utils::tempfile::TempDir::default(); + let test_uri = test_dir.path_str(); + let data = gen_batch() + .col("i", array::step::()) + .col("vec", array::rand_vec::(16.into())) + .into_reader_rows(RowCount::from(1024), BatchCount::from(1)); + let mut dataset = Dataset::write(data, &test_uri, None).await.unwrap(); + + let params = VectorIndexParams::with_ivf_hnsw_sq_params( + DistanceType::Cosine, + IvfBuildParams { + num_partitions: Some(2), + ..Default::default() + }, + HnswBuildParams::default(), + SQBuildParams::default(), + ); + dataset + .create_index(&["vec"], IndexType::Vector, None, ¶ms, true) + .await + .unwrap(); + + // Verify the index has populated details. + let descriptions = dataset.describe_indices(None).await.unwrap(); + assert_eq!(descriptions.len(), 1); + assert_eq!(descriptions[0].index_type(), "IVF_HNSW_SQ"); + + // Simulate a legacy dataset by clearing details from the manifest. + // Write a new manifest with empty VectorIndexDetails value bytes. + let mut indices = dataset.load_indices().await.unwrap().as_ref().clone(); + for idx in &mut indices { + if let Some(details) = idx.index_details.as_ref() + && details.type_url.ends_with("VectorIndexDetails") + { + idx.index_details = Some(Arc::new(vector_index_details_default())); + } + } + // Write back via a no-op commit that carries the cleared indices. + // We commit by doing a delete("false") after replacing the cached indices. + let metadata_key = crate::session::index_caches::IndexMetadataKey { + version: dataset.version().version, + }; + dataset + .index_cache + .insert_with_key(&metadata_key, Arc::new(indices)) + .await; + dataset.delete("false").await.unwrap(); + + // -- Part 1: Inference on load -- + // Open with a fresh session so nothing is cached. + let dataset = DatasetBuilder::from_uri(&test_uri) + .with_session(Arc::new(Session::default())) + .load() + .await + .unwrap(); + + // load_indices should detect empty details and infer from index files. + let indices = dataset.load_indices().await.unwrap(); + assert_eq!(indices.len(), 1); + let details = indices[0].index_details.as_ref().unwrap(); + assert!( + !details.value.is_empty(), + "Details should have been inferred from index files" + ); + + // describe_indices should return a real type (not generic "Vector"). + let descriptions = dataset.describe_indices(None).await.unwrap(); + assert_eq!(descriptions.len(), 1); + assert_ne!( + descriptions[0].index_type(), + "Vector", + "Should have inferred a specific index type" + ); + let inferred_type = descriptions[0].index_type().to_string(); + let details_json: serde_json::Value = + serde_json::from_str(&descriptions[0].details().unwrap()).unwrap(); + assert_eq!(details_json["metric_type"], "COSINE"); + + // -- Part 2: Migration persists inferred details -- + let mut dataset = dataset; + dataset.delete("false").await.unwrap(); + + // Open with yet another fresh session. + let dataset = DatasetBuilder::from_uri(&test_uri) + .with_session(Arc::new(Session::default())) + .load() + .await + .unwrap(); + + // The migrated manifest should have non-empty details without + // needing to read index files again. + let indices = dataset.load_indices().await.unwrap(); + assert_eq!(indices.len(), 1); + assert!( + !indices[0].index_details.as_ref().unwrap().value.is_empty(), + "Migrated manifest should have non-empty details" + ); + let descriptions = dataset.describe_indices(None).await.unwrap(); + assert_eq!(descriptions[0].index_type(), inferred_type); + } + #[rstest] #[case::btree("i", IndexType::BTree, Box::new(ScalarIndexParams::default()))] #[case::bitmap("i", IndexType::Bitmap, Box::new(ScalarIndexParams::default()))] diff --git a/rust/lance/src/index/vector/details.rs b/rust/lance/src/index/vector/details.rs index db0b6c2ec1a..a2b77e9e898 100644 --- a/rust/lance/src/index/vector/details.rs +++ b/rust/lance/src/index/vector/details.rs @@ -120,14 +120,26 @@ fn is_empty_vector_details(details: &prost_types::Any) -> bool { details.value.is_empty() } -/// Returns true if this index has a VectorIndexDetails type_url but empty value bytes, -/// indicating a legacy index that needs details inferred from disk. -pub fn is_vector_index_with_empty_details(index: &IndexMetadata) -> bool { - index - .index_details - .as_ref() - .map(|d| d.type_url.ends_with("VectorIndexDetails") && d.value.is_empty()) - .unwrap_or(false) +/// Returns true if this is a vector index whose details need to be inferred from disk. +/// +/// This covers two legacy cases: +/// - Very old indices (<=0.19.2) where `index_details` is `None` but the indexed +/// field is a vector type +/// - Newer pre-details indices where `index_details` has a VectorIndexDetails +/// type_url but empty value bytes +pub fn needs_vector_details_inference( + index: &IndexMetadata, + schema: &lance_core::datatypes::Schema, +) -> bool { + match &index.index_details { + Some(d) => d.type_url.ends_with("VectorIndexDetails") && d.value.is_empty(), + None => index.fields.iter().any(|&field_id| { + schema + .field_by_id(field_id) + .map(|f| matches!(f.data_type(), arrow_schema::DataType::FixedSizeList(_, _))) + .unwrap_or(false) + }), + } } /// Derive a human-readable index type string from VectorIndexDetails. diff --git a/rust/lance/src/io/commit.rs b/rust/lance/src/io/commit.rs index d7e4941efc9..078406e99a4 100644 --- a/rust/lance/src/io/commit.rs +++ b/rust/lance/src/io/commit.rs @@ -566,11 +566,12 @@ fn must_recalculate_fragment_bitmap( /// Indices might be missing `fragment_bitmap`, so this function will add it. async fn migrate_indices(dataset: &Dataset, indices: &mut [IndexMetadata]) -> Result<()> { use crate::index::vector::details::{ - infer_vector_index_details, is_vector_index_with_empty_details, + infer_vector_index_details, needs_vector_details_inference, }; + let schema = dataset.schema(); let needs_inference: HashMap<&str, &IndexMetadata> = indices .iter() - .filter(|idx| is_vector_index_with_empty_details(idx)) + .filter(|idx| needs_vector_details_inference(idx, schema)) .map(|idx| (idx.name.as_str(), idx)) .collect(); let inferred: HashMap> = From d507027f87eeeffb583a0b7680bd3ef4ebe5acab Mon Sep 17 00:00:00 2001 From: Will Jones Date: Thu, 5 Mar 2026 10:14:24 -0800 Subject: [PATCH 07/22] fix proto field numbering gap, extract inference helper - Fix compression oneof field numbers (5,6,7 -> 4,5,6) to avoid gap - Add comment that target_partition_size = 0 means unset - Extract infer_missing_vector_details helper to deduplicate logic between load_indices and migrate_indices Co-Authored-By: Claude Opus 4.6 --- protos/table.proto | 7 +++-- rust/lance/src/index.rs | 38 ++++-------------------- rust/lance/src/index/vector/details.rs | 40 ++++++++++++++++++++++++++ rust/lance/src/io/commit.rs | 33 ++------------------- 4 files changed, 51 insertions(+), 67 deletions(-) diff --git a/protos/table.proto b/protos/table.proto index 2e46e6fcce5..80c8f57262d 100644 --- a/protos/table.proto +++ b/protos/table.proto @@ -471,6 +471,7 @@ message VectorIndexDetails { VectorMetricType metric_type = 1; + // 0 means unset (unknown or not applicable). uint64 target_partition_size = 2; optional HnswIndexDetails hnsw_index_config = 3; @@ -493,9 +494,9 @@ message VectorIndexDetails { // An unset compression oneof means flat / no quantization. oneof compression { - ProductQuantization pq = 5; - ScalarQuantization sq = 6; - RabitQuantization rq = 7; + ProductQuantization pq = 4; + ScalarQuantization sq = 5; + RabitQuantization rq = 6; } } diff --git a/rust/lance/src/index.rs b/rust/lance/src/index.rs index ae41d0935fd..85935f3b048 100644 --- a/rust/lance/src/index.rs +++ b/rust/lance/src/index.rs @@ -408,8 +408,7 @@ async fn open_index_proto(reader: &dyn Reader) -> Result { } use vector::details::{ - derive_vector_index_type, infer_vector_index_details, needs_vector_details_inference, - vector_details_as_json, + derive_vector_index_type, infer_missing_vector_details, vector_details_as_json, }; pub(crate) use vector::details::{vector_index_details, vector_index_details_default}; @@ -707,37 +706,10 @@ impl DatasetIndexExt for Dataset { // Infer details for legacy vector indices (once per index name, concurrently). // This may run on indices that were opportunistically cached during Dataset::open // before the full Dataset was available for inference. - let schema = self.schema(); - let needs_inference: HashMap<&str, &IndexMetadata> = indices - .iter() - .filter(|idx| needs_vector_details_inference(idx, schema)) - .map(|idx| (idx.name.as_str(), idx)) - .collect(); - if !needs_inference.is_empty() { - let inferred: HashMap> = - futures::future::join_all(needs_inference.into_iter().map( - |(name, representative)| async move { - let result = infer_vector_index_details(self, representative).await; - (name.to_string(), result) - }, - )) - .await - .into_iter() - .filter_map(|(name, result)| match result { - Ok(details) => Some((name, Arc::new(details))), - Err(err) => { - log::warn!("Could not infer vector index details for {}: {}", name, err); - None - } - }) - .collect(); - if !inferred.is_empty() { - let mut updated = indices.as_ref().clone(); - for index in &mut updated { - if let Some(details) = inferred.get(&index.name) { - index.index_details = Some(details.clone()); - } - } + { + let mut updated = indices.as_ref().clone(); + infer_missing_vector_details(self, &mut updated).await; + if updated != *indices { indices = Arc::new(updated); self.index_cache .insert_with_key(&metadata_key, indices.clone()) diff --git a/rust/lance/src/index/vector/details.rs b/rust/lance/src/index/vector/details.rs index a2b77e9e898..9607c458880 100644 --- a/rust/lance/src/index/vector/details.rs +++ b/rust/lance/src/index/vector/details.rs @@ -142,6 +142,46 @@ pub fn needs_vector_details_inference( } } +/// Infer missing vector index details for all indices that need it. +/// +/// Runs inference once per unique index name, concurrently across names. +/// Applies the inferred details back to all matching indices in the slice. +pub async fn infer_missing_vector_details(dataset: &Dataset, indices: &mut [IndexMetadata]) { + use std::collections::HashMap; + + let schema = dataset.schema(); + let needs_inference: HashMap<&str, &IndexMetadata> = indices + .iter() + .filter(|idx| needs_vector_details_inference(idx, schema)) + .map(|idx| (idx.name.as_str(), idx)) + .collect(); + if needs_inference.is_empty() { + return; + } + let inferred: HashMap> = + futures::future::join_all(needs_inference.into_iter().map( + |(name, representative)| async move { + let result = infer_vector_index_details(dataset, representative).await; + (name.to_string(), result) + }, + )) + .await + .into_iter() + .filter_map(|(name, result)| match result { + Ok(details) => Some((name, Arc::new(details))), + Err(err) => { + log::warn!("Could not infer vector index details for {}: {}", name, err); + None + } + }) + .collect(); + for index in indices.iter_mut() { + if let Some(details) = inferred.get(&index.name) { + index.index_details = Some(details.clone()); + } + } +} + /// Derive a human-readable index type string from VectorIndexDetails. pub fn derive_vector_index_type(details: &prost_types::Any) -> String { use lance_table::format::pb::VectorIndexDetails; diff --git a/rust/lance/src/io/commit.rs b/rust/lance/src/io/commit.rs index 078406e99a4..c1cc9a8f380 100644 --- a/rust/lance/src/io/commit.rs +++ b/rust/lance/src/io/commit.rs @@ -565,37 +565,8 @@ fn must_recalculate_fragment_bitmap( /// /// Indices might be missing `fragment_bitmap`, so this function will add it. async fn migrate_indices(dataset: &Dataset, indices: &mut [IndexMetadata]) -> Result<()> { - use crate::index::vector::details::{ - infer_vector_index_details, needs_vector_details_inference, - }; - let schema = dataset.schema(); - let needs_inference: HashMap<&str, &IndexMetadata> = indices - .iter() - .filter(|idx| needs_vector_details_inference(idx, schema)) - .map(|idx| (idx.name.as_str(), idx)) - .collect(); - let inferred: HashMap> = - futures::future::join_all(needs_inference.into_iter().map( - |(name, representative)| async move { - let result = infer_vector_index_details(dataset, representative).await; - (name.to_string(), result) - }, - )) - .await - .into_iter() - .filter_map(|(name, result)| match result { - Ok(details) => Some((name, Arc::new(details))), - Err(err) => { - log::warn!("Could not infer vector index details for {}: {}", name, err); - None - } - }) - .collect(); - for index in indices.iter_mut() { - if let Some(details) = inferred.get(&index.name) { - index.index_details = Some(details.clone()); - } - } + use crate::index::vector::details::infer_missing_vector_details; + infer_missing_vector_details(dataset, indices).await; let needs_recalculating = match detect_overlapping_fragments(indices) { Ok(()) => vec![], Err(BadFragmentBitmapError { bad_indices }) => { From 0a8e39415ab1d78b6165c3b6f4c9ad1526cc24ed Mon Sep 17 00:00:00 2001 From: Will Jones Date: Thu, 5 Mar 2026 10:57:15 -0800 Subject: [PATCH 08/22] wip: cleanup --- protos/table.proto | 4 ++- rust/lance/src/index/vector/details.rs | 49 ++++++++------------------ 2 files changed, 17 insertions(+), 36 deletions(-) diff --git a/protos/table.proto b/protos/table.proto index 80c8f57262d..76573266400 100644 --- a/protos/table.proto +++ b/protos/table.proto @@ -471,9 +471,11 @@ message VectorIndexDetails { VectorMetricType metric_type = 1; - // 0 means unset (unknown or not applicable). + // The target number of vectors per partition. + // 0 means unset. uint64 target_partition_size = 2; + // Optional HNSW index configuration. If present, index does not use HNSW. optional HnswIndexDetails hnsw_index_config = 3; message ProductQuantization { diff --git a/rust/lance/src/index/vector/details.rs b/rust/lance/src/index/vector/details.rs index 9607c458880..bfca68c6750 100644 --- a/rust/lance/src/index/vector/details.rs +++ b/rust/lance/src/index/vector/details.rs @@ -9,6 +9,7 @@ //! - Serializing details as JSON for `describe_indices()` //! - Inferring details from index files on disk (fallback for legacy indices) +use std::collections::HashMap; use std::sync::Arc; use lance_file::reader::FileReaderOptions; @@ -18,6 +19,10 @@ use lance_io::scheduler::{ScanScheduler, SchedulerConfig}; use lance_io::traits::Reader; use lance_io::utils::{CachedFileSize, read_last_block, read_version}; use lance_table::format::IndexMetadata; +use lance_table::format::pb::VectorIndexDetails; +use lance_table::format::pb::vector_index_details::{ + Compression, VectorMetricType, rabit_quantization, +}; use serde::Serialize; use super::{StageParams, VectorIndexParams}; @@ -64,9 +69,6 @@ enum CompressionDetailsJson { /// Build a `VectorIndexDetails` proto from build params at index creation time. pub fn vector_index_details(params: &VectorIndexParams) -> prost_types::Any { - use lance_table::format::pb::VectorIndexDetails; - use lance_table::format::pb::vector_index_details::*; - let metric_type = match params.metric_type { lance_linalg::distance::DistanceType::L2 => VectorMetricType::L2, lance_linalg::distance::DistanceType::Cosine => VectorMetricType::Cosine, @@ -147,8 +149,6 @@ pub fn needs_vector_details_inference( /// Runs inference once per unique index name, concurrently across names. /// Applies the inferred details back to all matching indices in the slice. pub async fn infer_missing_vector_details(dataset: &Dataset, indices: &mut [IndexMetadata]) { - use std::collections::HashMap; - let schema = dataset.schema(); let needs_inference: HashMap<&str, &IndexMetadata> = indices .iter() @@ -184,9 +184,6 @@ pub async fn infer_missing_vector_details(dataset: &Dataset, indices: &mut [Inde /// Derive a human-readable index type string from VectorIndexDetails. pub fn derive_vector_index_type(details: &prost_types::Any) -> String { - use lance_table::format::pb::VectorIndexDetails; - use lance_table::format::pb::vector_index_details::Compression; - if is_empty_vector_details(details) { return "Vector".to_string(); } @@ -194,39 +191,21 @@ pub fn derive_vector_index_type(details: &prost_types::Any) -> String { let Ok(d) = details.to_msg::() else { return "Vector".to_string(); }; - let has_hnsw = d.hnsw_index_config.is_some(); + let mut index_type = "IVF_".to_string(); + if d.hnsw_index_config.is_some() { + index_type.push_str("HNSW_"); + } match d.compression { - None => { - if has_hnsw { - "IVF_HNSW_FLAT" - } else { - "IVF_FLAT" - } - } - Some(Compression::Pq(_)) => { - if has_hnsw { - "IVF_HNSW_PQ" - } else { - "IVF_PQ" - } - } - Some(Compression::Sq(_)) => { - if has_hnsw { - "IVF_HNSW_SQ" - } else { - "IVF_SQ" - } - } - Some(Compression::Rq(_)) => "IVF_RQ", + None => index_type.push_str("FLAT"), + Some(Compression::Pq(_)) => index_type.push_str("PQ"), + Some(Compression::Sq(_)) => index_type.push_str("SQ"), + Some(Compression::Rq(_)) => index_type.push_str("RQ"), } - .to_string() + index_type } /// Serialize VectorIndexDetails as a JSON string. pub fn vector_details_as_json(details: &prost_types::Any) -> Result { - use lance_table::format::pb::VectorIndexDetails; - use lance_table::format::pb::vector_index_details::*; - if is_empty_vector_details(details) { return Ok("{}".to_string()); } From dd70fee8cd111be320d237997f1286331530125a Mon Sep 17 00:00:00 2001 From: Will Jones Date: Thu, 5 Mar 2026 11:14:38 -0800 Subject: [PATCH 09/22] fix python test --- python/python/tests/test_vector_index.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/python/python/tests/test_vector_index.py b/python/python/tests/test_vector_index.py index 58c4fa84419..fdf078ba235 100644 --- a/python/python/tests/test_vector_index.py +++ b/python/python/tests/test_vector_index.py @@ -1575,9 +1575,7 @@ def test_describe_vector_index(indexed_dataset: LanceDataset): assert info.segments[0].index_version == 1 assert info.segments[0].created_at is not None - import json - - details = json.loads(info.details) + details = info.details assert details["metric_type"] == "L2" assert details["compression"]["type"] == "pq" assert details["compression"]["num_bits"] == 8 From 45b999595d2e2b4a483d99c599ac152107addda1 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Thu, 5 Mar 2026 12:17:59 -0800 Subject: [PATCH 10/22] move VectorIndexDetails and HnswIndexDetails from table.proto to index.proto These messages belong with other index-related protos. After the move, VectorIndexDetails reuses the existing top-level VectorMetricType enum instead of defining its own nested copy. Rust imports updated from lance_table::format::pb to lance_index::pb throughout. Co-Authored-By: Claude Opus 4.6 --- protos/index.proto | 46 ++++++++++++++++++ protos/table.proto | 53 +-------------------- rust/lance-index/src/vector/bq.rs | 4 +- rust/lance-index/src/vector/hnsw/builder.rs | 2 +- rust/lance-index/src/vector/pq/builder.rs | 2 +- rust/lance-index/src/vector/sq/builder.rs | 2 +- rust/lance/src/dataset/index.rs | 2 +- rust/lance/src/index.rs | 2 +- rust/lance/src/index/scalar.rs | 2 +- rust/lance/src/index/vector/details.rs | 30 +++++------- 10 files changed, 67 insertions(+), 78 deletions(-) diff --git a/protos/index.proto b/protos/index.proto index 1fb51f3291c..de32a93f473 100644 --- a/protos/index.proto +++ b/protos/index.proto @@ -184,6 +184,52 @@ message VectorIndex { VectorMetricType metric_type = 4; } +// Details for vector indexes, stored in the manifest's index_details field. +message VectorIndexDetails { + VectorMetricType metric_type = 1; + + // The target number of vectors per partition. + // 0 means unset. + uint64 target_partition_size = 2; + + // Optional HNSW index configuration. If present, index does not use HNSW. + optional HnswIndexDetails hnsw_index_config = 3; + + message ProductQuantization { + uint32 num_bits = 1; + uint32 num_sub_vectors = 2; + } + message ScalarQuantization { + uint32 num_bits = 1; + } + message RabitQuantization { + enum RotationType { + FAST = 0; + MATRIX = 1; + } + uint32 num_bits = 1; + RotationType rotation_type = 2; + } + + // An unset compression oneof means flat / no quantization. + oneof compression { + ProductQuantization pq = 4; + ScalarQuantization sq = 5; + RabitQuantization rq = 6; + } +} + +// Hierarchical Navigable Small World (HNSW) index details, used as an optional configuration for IVF indexes. +message HnswIndexDetails { + // The maximum number of outgoing edges per node in the HNSW graph. Higher values + // means more connections, better recall, but more memory and slower builds. + // Referred to as "M" in the HNSW literature. + uint32 max_connections = 1; + // "construction exploration factor": The size of the dynamic list used during + // index construction. + uint32 construction_ef = 2; +} + message JsonIndexDetails { string path = 1; google.protobuf.Any target_details = 2; diff --git a/protos/table.proto b/protos/table.proto index 76573266400..b827f238038 100644 --- a/protos/table.proto +++ b/protos/table.proto @@ -460,58 +460,7 @@ message ExternalFile { uint64 size = 3; } -// Details for vector indexes. -message VectorIndexDetails { - enum VectorMetricType { - L2 = 0; - COSINE = 1; - DOT = 2; - HAMMING = 3; - } - - VectorMetricType metric_type = 1; - - // The target number of vectors per partition. - // 0 means unset. - uint64 target_partition_size = 2; - - // Optional HNSW index configuration. If present, index does not use HNSW. - optional HnswIndexDetails hnsw_index_config = 3; - - message ProductQuantization { - uint32 num_bits = 1; - uint32 num_sub_vectors = 2; - } - message ScalarQuantization { - uint32 num_bits = 1; - } - message RabitQuantization { - enum RotationType { - FAST = 0; - MATRIX = 1; - } - uint32 num_bits = 1; - RotationType rotation_type = 2; - } - - // An unset compression oneof means flat / no quantization. - oneof compression { - ProductQuantization pq = 4; - ScalarQuantization sq = 5; - RabitQuantization rq = 6; - } -} - -// Hierarchical Navigable Small World (HNSW) index details, used as an optional configuration for IVF indexes. -message HnswIndexDetails { - // The maximum number of outgoing edges per node in the HNSW graph. Higher values - // means more connections, better recall, but more memory and slower builds. - // Referred to as "M" in the HNSW literature. - uint32 max_connections = 1; - // "construction exploration factor": The size of the dynamic list used during - // index construction. - uint32 construction_ef = 2; -} +// VectorIndexDetails and HnswIndexDetails moved to index.proto message FragmentReuseIndexDetails { diff --git a/rust/lance-index/src/vector/bq.rs b/rust/lance-index/src/vector/bq.rs index 5b036ee9114..a0a16b22169 100644 --- a/rust/lance-index/src/vector/bq.rs +++ b/rust/lance-index/src/vector/bq.rs @@ -7,10 +7,10 @@ use std::iter::once; use std::str::FromStr; use std::sync::Arc; +use crate::pb::vector_index_details::RabitQuantization; use arrow_array::types::Float32Type; use arrow_array::{Array, ArrayRef, UInt8Array, cast::AsArray}; use lance_core::{Error, Result}; -use lance_table::format::pb::vector_index_details::RabitQuantization; use num_traits::Float; use serde::{Deserialize, Serialize}; @@ -124,7 +124,7 @@ impl RQBuildParams { impl From<&RQBuildParams> for RabitQuantization { fn from(value: &RQBuildParams) -> Self { - use lance_table::format::pb::vector_index_details::rabit_quantization::RotationType; + use crate::pb::vector_index_details::rabit_quantization::RotationType; Self { num_bits: value.num_bits as u32, rotation_type: match value.rotation_type { diff --git a/rust/lance-index/src/vector/hnsw/builder.rs b/rust/lance-index/src/vector/hnsw/builder.rs index 419304a1466..0c206a52dcc 100644 --- a/rust/lance-index/src/vector/hnsw/builder.rs +++ b/rust/lance-index/src/vector/hnsw/builder.rs @@ -59,7 +59,7 @@ pub struct HnswBuildParams { pub prefetch_distance: Option, } -impl From<&HnswBuildParams> for lance_table::format::pb::HnswIndexDetails { +impl From<&HnswBuildParams> for crate::pb::HnswIndexDetails { fn from(params: &HnswBuildParams) -> Self { Self { max_connections: params.m as u32, diff --git a/rust/lance-index/src/vector/pq/builder.rs b/rust/lance-index/src/vector/pq/builder.rs index a61d94b8640..c4dad4a6a3e 100644 --- a/rust/lance-index/src/vector/pq/builder.rs +++ b/rust/lance-index/src/vector/pq/builder.rs @@ -44,7 +44,7 @@ pub struct PQBuildParams { pub sample_rate: usize, } -impl From<&PQBuildParams> for lance_table::format::pb::vector_index_details::ProductQuantization { +impl From<&PQBuildParams> for crate::pb::vector_index_details::ProductQuantization { fn from(params: &PQBuildParams) -> Self { Self { num_bits: params.num_bits as u32, diff --git a/rust/lance-index/src/vector/sq/builder.rs b/rust/lance-index/src/vector/sq/builder.rs index 6fa6672708e..359765040dd 100644 --- a/rust/lance-index/src/vector/sq/builder.rs +++ b/rust/lance-index/src/vector/sq/builder.rs @@ -12,7 +12,7 @@ pub struct SQBuildParams { pub sample_rate: usize, } -impl From<&SQBuildParams> for lance_table::format::pb::vector_index_details::ScalarQuantization { +impl From<&SQBuildParams> for crate::pb::vector_index_details::ScalarQuantization { fn from(params: &SQBuildParams) -> Self { Self { num_bits: params.num_bits as u32, diff --git a/rust/lance/src/dataset/index.rs b/rust/lance/src/dataset/index.rs index 404e53dc0be..48ebfb8d931 100644 --- a/rust/lance/src/dataset/index.rs +++ b/rust/lance/src/dataset/index.rs @@ -16,9 +16,9 @@ use async_trait::async_trait; use lance_core::{Error, Result}; use lance_index::DatasetIndexExt; use lance_index::frag_reuse::FRAG_REUSE_INDEX_NAME; +use lance_index::pb::VectorIndexDetails; use lance_index::scalar::lance_format::LanceIndexStore; use lance_table::format::IndexMetadata; -use lance_table::format::pb::VectorIndexDetails; use serde::{Deserialize, Serialize}; use super::optimize::{IndexRemapper, IndexRemapperOptions}; diff --git a/rust/lance/src/index.rs b/rust/lance/src/index.rs index e64558b1b6a..bb0bc07b5f2 100644 --- a/rust/lance/src/index.rs +++ b/rust/lance/src/index.rs @@ -362,7 +362,7 @@ pub(crate) async fn remap_index( .await?; CreatedIndex { index_details: prost_types::Any::from_msg( - &lance_table::format::pb::VectorIndexDetails::default(), + &lance_index::pb::VectorIndexDetails::default(), ) .unwrap(), index_version, diff --git a/rust/lance/src/index/scalar.rs b/rust/lance/src/index/scalar.rs index 0365f306c9b..ec5922f40c6 100644 --- a/rust/lance/src/index/scalar.rs +++ b/rust/lance/src/index/scalar.rs @@ -558,9 +558,9 @@ mod tests { use lance_core::utils::tempfile::TempStrDir; use lance_core::{datatypes::Field, utils::address::RowAddress}; use lance_datagen::array; + use lance_index::pb::VectorIndexDetails; use lance_index::{IndexType, optimize::OptimizeOptions}; use lance_index::{pbold::NGramIndexDetails, scalar::BuiltinIndexType}; - use lance_table::format::pb::VectorIndexDetails; fn make_index_metadata( name: &str, diff --git a/rust/lance/src/index/vector/details.rs b/rust/lance/src/index/vector/details.rs index bfca68c6750..77ad356414a 100644 --- a/rust/lance/src/index/vector/details.rs +++ b/rust/lance/src/index/vector/details.rs @@ -13,16 +13,15 @@ use std::collections::HashMap; use std::sync::Arc; use lance_file::reader::FileReaderOptions; +use lance_index::pb::VectorIndexDetails; +use lance_index::pb::VectorMetricType; use lance_index::pb::index::Implementation; +use lance_index::pb::vector_index_details::{Compression, rabit_quantization}; use lance_index::{INDEX_FILE_NAME, INDEX_METADATA_SCHEMA_KEY, pb}; use lance_io::scheduler::{ScanScheduler, SchedulerConfig}; use lance_io::traits::Reader; use lance_io::utils::{CachedFileSize, read_last_block, read_version}; use lance_table::format::IndexMetadata; -use lance_table::format::pb::VectorIndexDetails; -use lance_table::format::pb::vector_index_details::{ - Compression, VectorMetricType, rabit_quantization, -}; use serde::Serialize; use super::{StageParams, VectorIndexParams}; @@ -112,7 +111,7 @@ pub fn vector_index_details(params: &VectorIndexParams) -> prost_types::Any { } pub fn vector_index_details_default() -> prost_types::Any { - let details = lance_table::format::pb::VectorIndexDetails::default(); + let details = lance_index::pb::VectorIndexDetails::default(); prost_types::Any::from_msg(&details).unwrap() } @@ -289,21 +288,16 @@ pub async fn infer_vector_index_details( } fn convert_legacy_proto_to_details(proto: &pb::Index) -> Result { - use lance_table::format::pb::VectorIndexDetails; - use lance_table::format::pb::vector_index_details::*; + use lance_index::pb::VectorIndexDetails; + use lance_index::pb::vector_index_details::*; use pb::vector_index_stage::Stage; let Some(Implementation::VectorIndex(vector_index)) = &proto.implementation else { return Ok(vector_index_details_default()); }; - let metric_type = match pb::VectorMetricType::try_from(vector_index.metric_type) { - Ok(pb::VectorMetricType::L2) => VectorMetricType::L2, - Ok(pb::VectorMetricType::Cosine) => VectorMetricType::Cosine, - Ok(pb::VectorMetricType::Dot) => VectorMetricType::Dot, - Ok(pb::VectorMetricType::Hamming) => VectorMetricType::Hamming, - Err(_) => VectorMetricType::L2, - }; + let metric_type = pb::VectorMetricType::try_from(vector_index.metric_type) + .unwrap_or(pb::VectorMetricType::L2); let mut compression = None; for stage in &vector_index.stages { @@ -328,13 +322,13 @@ async fn convert_v3_metadata_to_details( dataset: &Dataset, index_file: &object_store::path::Path, ) -> Result { + use lance_index::pb::vector_index_details::*; + use lance_index::pb::{HnswIndexDetails, VectorIndexDetails}; use lance_index::vector::bq::storage::RABIT_METADATA_KEY; use lance_index::vector::hnsw::HnswMetadata; use lance_index::vector::ivf::storage::IVF_PARTITION_KEY; use lance_index::vector::pq::storage::{PQ_METADATA_KEY, ProductQuantizationMetadata}; use lance_index::vector::sq::storage::{SQ_METADATA_KEY, ScalarQuantizationMetadata}; - use lance_table::format::pb::vector_index_details::*; - use lance_table::format::pb::{HnswIndexDetails, VectorIndexDetails}; let scheduler = ScanScheduler::new( dataset.object_store.clone(), @@ -420,8 +414,8 @@ async fn convert_v3_metadata_to_details( #[cfg(test)] mod tests { use super::*; - use lance_table::format::pb::vector_index_details::*; - use lance_table::format::pb::{HnswIndexDetails, VectorIndexDetails}; + use lance_index::pb::vector_index_details::*; + use lance_index::pb::{HnswIndexDetails, VectorIndexDetails}; fn make_details( metric: VectorMetricType, From aa10e5e59082923aae53bf9f22822fa4a4f39764 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Thu, 5 Mar 2026 13:44:15 -0800 Subject: [PATCH 11/22] fix review issues: inverted comment, log->tracing, append carry-forward, type_url - Fix inverted comment on hnsw_index_config field in index.proto - Use tracing::warn! instead of log::warn! in details.rs - Prefer non-empty index_details when carrying forward in append - Revert describe_indices to original chunk_by pattern - Update Python test type_url to match new proto package Co-Authored-By: Claude Opus 4.6 --- protos/index.proto | 2 +- python/python/tests/test_vector_index.py | 2 +- rust/lance/src/index.rs | 20 ++++++++------------ rust/lance/src/index/append.rs | 9 ++++++--- rust/lance/src/index/vector/details.rs | 2 +- 5 files changed, 17 insertions(+), 18 deletions(-) diff --git a/protos/index.proto b/protos/index.proto index de32a93f473..00b54853cea 100644 --- a/protos/index.proto +++ b/protos/index.proto @@ -192,7 +192,7 @@ message VectorIndexDetails { // 0 means unset. uint64 target_partition_size = 2; - // Optional HNSW index configuration. If present, index does not use HNSW. + // Optional HNSW index configuration. If set, the index has an HNSW layer. optional HnswIndexDetails hnsw_index_config = 3; message ProductQuantization { diff --git a/python/python/tests/test_vector_index.py b/python/python/tests/test_vector_index.py index fdf078ba235..114a59caed0 100644 --- a/python/python/tests/test_vector_index.py +++ b/python/python/tests/test_vector_index.py @@ -1564,7 +1564,7 @@ def test_describe_vector_index(indexed_dataset: LanceDataset): info = indexed_dataset.describe_indices()[0] assert info.name == "vector_idx" - assert info.type_url == "/lance.table.VectorIndexDetails" + assert info.type_url == "/lance.index.pb.VectorIndexDetails" assert info.index_type == "IVF_PQ" assert info.num_rows_indexed == 1000 assert info.fields == [0] diff --git a/rust/lance/src/index.rs b/rust/lance/src/index.rs index bb0bc07b5f2..003e013f84d 100644 --- a/rust/lance/src/index.rs +++ b/rust/lance/src/index.rs @@ -669,20 +669,16 @@ impl DatasetIndexExt for Dataset { }; indices.sort_by_key(|idx| &idx.name); - // Collect groups upfront so we don't hold chunk_by across await points - let groups: Vec> = indices + indices .into_iter() - .chunk_by(|idx| idx.name.clone()) + .chunk_by(|idx| &idx.name) .into_iter() - .map(|(_, segments)| segments.cloned().collect::>()) - .collect(); - - let mut results: Vec> = Vec::new(); - for segments in groups { - let desc = IndexDescriptionImpl::try_new(segments, self)?; - results.push(Arc::new(desc) as Arc); - } - Ok(results) + .map(|(_, segments)| { + let segments = segments.cloned().collect::>(); + let desc = IndexDescriptionImpl::try_new(segments, self)?; + Ok(Arc::new(desc) as Arc) + }) + .collect::>>() } async fn load_indices(&self) -> Result>> { diff --git a/rust/lance/src/index/append.rs b/rust/lance/src/index/append.rs index 80ac7f97448..f26d58a58b7 100644 --- a/rust/lance/src/index/append.rs +++ b/rust/lance/src/index/append.rs @@ -212,10 +212,13 @@ pub async fn merge_indices_with_unindexed_frags<'a>( frag_bitmap.extend(idx.fragment_bitmap.as_ref().unwrap().iter()); }); - // Carry forward existing index details from the most recent index segment + // Carry forward existing index details, preferring the first segment + // that has populated (non-empty) details. let index_details = old_indices - .last() - .and_then(|idx| idx.index_details.as_ref()) + .iter() + .rev() + .filter_map(|idx| idx.index_details.as_ref()) + .find(|d| !d.value.is_empty()) .map(|d| d.as_ref().clone()) .unwrap_or_else(vector_index_details_default); diff --git a/rust/lance/src/index/vector/details.rs b/rust/lance/src/index/vector/details.rs index 77ad356414a..51443496710 100644 --- a/rust/lance/src/index/vector/details.rs +++ b/rust/lance/src/index/vector/details.rs @@ -169,7 +169,7 @@ pub async fn infer_missing_vector_details(dataset: &Dataset, indices: &mut [Inde .filter_map(|(name, result)| match result { Ok(details) => Some((name, Arc::new(details))), Err(err) => { - log::warn!("Could not infer vector index details for {}: {}", name, err); + tracing::warn!("Could not infer vector index details for {}: {}", name, err); None } }) From e88ed2f474dfd59d8e59acc6f5822544545eacc7 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Mon, 16 Mar 2026 10:29:26 -0700 Subject: [PATCH 12/22] enhance tests --- .../tests/compat/test_vector_indices.py | 52 ++++++++++++++++++- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/python/python/tests/compat/test_vector_indices.py b/python/python/tests/compat/test_vector_indices.py index 25d43c2f17b..0a37113e743 100644 --- a/python/python/tests/compat/test_vector_indices.py +++ b/python/python/tests/compat/test_vector_indices.py @@ -66,6 +66,18 @@ def check_read(self): ) assert result.num_rows == 4 + if hasattr(ds, "describe_indices"): + indices = ds.describe_indices() + assert len(indices) >= 1 + name = indices[0].name + else: + indices = ds.list_indices() + assert len(indices) >= 1 + name = indices[0].name + + stats = ds.stats.index_stats(name) + assert stats["num_indexed_rows"] > 0 + def check_write(self): """Verify can insert vectors and rebuild index.""" ds = lance.dataset(self.path) @@ -131,6 +143,18 @@ def check_read(self): ) assert result.num_rows == 4 + if hasattr(ds, "describe_indices"): + indices = ds.describe_indices() + assert len(indices) >= 1 + name = indices[0].name + else: + indices = ds.list_indices() + assert len(indices) >= 1 + name = indices[0].name + + stats = ds.stats.index_stats(name) + assert stats["num_indexed_rows"] > 0 + def check_write(self): """Verify can insert vectors and rebuild index.""" ds = lance.dataset(self.path) @@ -196,6 +220,18 @@ def check_read(self): ) assert result.num_rows == 4 + if hasattr(ds, "describe_indices"): + indices = ds.describe_indices() + assert len(indices) >= 1 + name = indices[0].name + else: + indices = ds.list_indices() + assert len(indices) >= 1 + name = indices[0].name + + stats = ds.stats.index_stats(name) + assert stats["num_indexed_rows"] > 0 + def check_write(self): """Verify can insert vectors and rebuild index.""" ds = lance.dataset(self.path) @@ -213,9 +249,9 @@ def check_write(self): ds.optimize.compact_files() -@compat_test(min_version="0.39.0") +@compat_test(min_version="v4.0.0-beta.8") class IvfRqVectorIndex(UpgradeDowngradeTest): - """Test IVF_RQ vector index compatibility.""" + """Test IVF_RQ vector index compatibility. V2 was introduced in v4.0.0-beta.8""" def __init__(self, path: Path): self.path = path @@ -256,6 +292,18 @@ def check_read(self): ) assert result.num_rows == 4 + if hasattr(ds, "describe_indices"): + indices = ds.describe_indices() + assert len(indices) >= 1 + name = indices[0].name + else: + indices = ds.list_indices() + assert len(indices) >= 1 + name = indices[0].name + + stats = ds.stats.index_stats(name) + assert stats["num_indexed_rows"] > 0 + def check_write(self): """Verify can insert vectors and run optimize workflows.""" ds = lance.dataset(self.path) From 2cdfbd21fee8b2bfdd4b623beed1cddc29ff78b3 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Mon, 16 Mar 2026 11:01:24 -0700 Subject: [PATCH 13/22] feat: read metric type from index metadata without I/O Previously, `scanner.rs:3425` called `open_vector_index` just to get the metric type for metric compatibility checks. This required expensive deserialization of the index file. Now we read the metric type directly from `IndexMetadata.index_details` (a `VectorIndexDetails` proto) added on the feat/vector-index-details branch. This provides a fast path for newer indices without I/O. For legacy indices without populated details (empty proto value bytes), we fall back to the original expensive path. Adds `metric_type_from_index_metadata` helper in `details.rs` that: - Returns `None` for missing or empty details (legacy indices) - Converts `VectorIndexDetails.metric_type` to `DistanceType` for populated details - Uses the existing `From for DistanceType` impl Changes `matching_index` tuple from `(index, idx, index_metric)` to `(index, index_metric)` since `idx` is only used in the fallback path. Fixes #5231 Co-Authored-By: Claude Haiku 4.5 --- rust/lance/src/dataset/scanner.rs | 33 +++--- rust/lance/src/index/vector/details.rs | 138 +++++++++++++++++++++++++ 2 files changed, 157 insertions(+), 14 deletions(-) diff --git a/rust/lance/src/dataset/scanner.rs b/rust/lance/src/dataset/scanner.rs index cdaa9408281..d471255a48d 100644 --- a/rust/lance/src/dataset/scanner.rs +++ b/rust/lance/src/dataset/scanner.rs @@ -3419,18 +3419,23 @@ impl Scanner { let matching_index = if let Some(index) = indices.iter().find(|i| i.fields.contains(&column_id)) { - // TODO: Once we do https://github.com/lance-format/lance/issues/5231, we - // should be able to get the metric type directly from the index metadata, - // at least for newer indexes. - let idx = self - .dataset - .open_vector_index( - q.column.as_str(), - &index.uuid.to_string(), - &NoOpMetricsCollector, - ) - .await?; - let index_metric = idx.metric_type(); + // Try to get metric type from index metadata first (fast path for newer indices) + let index_metric = if let Some(metric) = + crate::index::vector::details::metric_type_from_index_metadata(index) + { + metric + } else { + // Fall back to opening the index for legacy indices without details + let idx = self + .dataset + .open_vector_index( + q.column.as_str(), + &index.uuid.to_string(), + &NoOpMetricsCollector, + ) + .await?; + idx.metric_type() + }; // Check if user's requested metric is compatible with index let use_this_index = match q.metric_type { @@ -3450,7 +3455,7 @@ impl Scanner { }; if use_this_index { - Some((index, idx, index_metric)) + Some((index, index_metric)) } else { None } @@ -3459,7 +3464,7 @@ impl Scanner { }; // Only return index and deltas if there is an index on the column and at least one of the target fragments are indexed - let index_and_deltas = if let Some((index, _idx, index_metric)) = matching_index { + let index_and_deltas = if let Some((index, index_metric)) = matching_index { let deltas = self.dataset.load_indices_by_name(&index.name).await?; let index_frags = self.get_indexed_frags(&deltas); if !index_frags.is_empty() { diff --git a/rust/lance/src/index/vector/details.rs b/rust/lance/src/index/vector/details.rs index 51443496710..a57f0871dcb 100644 --- a/rust/lance/src/index/vector/details.rs +++ b/rust/lance/src/index/vector/details.rs @@ -21,6 +21,7 @@ use lance_index::{INDEX_FILE_NAME, INDEX_METADATA_SCHEMA_KEY, pb}; use lance_io::scheduler::{ScanScheduler, SchedulerConfig}; use lance_io::traits::Reader; use lance_io::utils::{CachedFileSize, read_last_block, read_version}; +use lance_linalg::distance::DistanceType; use lance_table::format::IndexMetadata; use serde::Serialize; @@ -115,6 +116,33 @@ pub fn vector_index_details_default() -> prost_types::Any { prost_types::Any::from_msg(&details).unwrap() } +/// Extract metric type from index metadata without opening the index file. +/// +/// For newer indices with populated `VectorIndexDetails`, returns the metric type directly. +/// For legacy indices without details, returns `None` and caller should fall back to opening the index. +/// +/// # Arguments +/// * `index` - The index metadata containing serialized VectorIndexDetails +/// +/// # Returns +/// * `Some(DistanceType)` if details are present and valid +/// * `None` if details are absent or empty (legacy index without details) +pub fn metric_type_from_index_metadata(index: &IndexMetadata) -> Option { + let index_details = index.index_details.as_ref()?; + + // Empty value bytes indicates legacy index that needs to be opened for details + if index_details.value.is_empty() { + return None; + } + + let details = index_details.to_msg::().ok()?; + + // Try to convert the metric_type field. This works even if metric_type is 0 (L2), + // since L2 is a valid metric type. + let metric_enum = VectorMetricType::try_from(details.metric_type).ok()?; + Some(DistanceType::from(metric_enum)) +} + /// Returns true if the proto value represents a "truly empty" VectorIndexDetails /// (i.e., a legacy index that was created before we populated this field). fn is_empty_vector_details(details: &prost_types::Any) -> bool { @@ -587,4 +615,114 @@ mod tests { let details = vector_index_details_default(); assert_eq!(vector_details_as_json(&details).unwrap(), "{}"); } + + #[test] + fn test_metric_type_from_index_metadata_populated() { + // Test that populated details return the metric type. + // Note: We add a non-default compression field so the proto doesn't serialize to empty bytes. + let details = make_details( + VectorMetricType::L2, + None, + Some(Compression::Pq(ProductQuantization { + num_bits: 8, + num_sub_vectors: 16, + })), + ); + let index_details = Some(std::sync::Arc::new(details)); + let index = IndexMetadata { + uuid: uuid::Uuid::new_v4(), + fields: vec![0], + name: "test_index".to_string(), + dataset_version: 1, + fragment_bitmap: None, + index_details, + index_version: 1, + created_at: None, + base_id: None, + }; + + let metric = metric_type_from_index_metadata(&index); + assert_eq!(metric, Some(DistanceType::L2)); + } + + #[test] + fn test_metric_type_from_index_metadata_empty() { + // Test that empty details return None (legacy index) + let details = vector_index_details_default(); + let index_details = Some(std::sync::Arc::new(details)); + let index = IndexMetadata { + uuid: uuid::Uuid::new_v4(), + fields: vec![0], + name: "test_index".to_string(), + dataset_version: 1, + fragment_bitmap: None, + index_details, + index_version: 1, + created_at: None, + base_id: None, + }; + + let metric = metric_type_from_index_metadata(&index); + assert_eq!(metric, None); + } + + #[test] + fn test_metric_type_from_index_metadata_none() { + // Test that missing details return None + let index = IndexMetadata { + uuid: uuid::Uuid::new_v4(), + fields: vec![0], + name: "test_index".to_string(), + dataset_version: 1, + fragment_bitmap: None, + index_details: None, + index_version: 1, + created_at: None, + base_id: None, + }; + + let metric = metric_type_from_index_metadata(&index); + assert_eq!(metric, None); + } + + #[test] + fn test_metric_type_from_index_metadata_all_metrics() { + // Test all supported metric types. + // Note: We add a non-default compression field so the proto doesn't serialize to empty bytes. + let metrics = [ + VectorMetricType::L2, + VectorMetricType::Cosine, + VectorMetricType::Dot, + VectorMetricType::Hamming, + ]; + let expected = [ + DistanceType::L2, + DistanceType::Cosine, + DistanceType::Dot, + DistanceType::Hamming, + ]; + + for (metric_enum, expected_distance) in metrics.iter().zip(expected.iter()) { + let details = make_details( + *metric_enum, + None, + Some(Compression::Sq(ScalarQuantization { num_bits: 8 })), + ); + let index_details = Some(std::sync::Arc::new(details)); + let index = IndexMetadata { + uuid: uuid::Uuid::new_v4(), + fields: vec![0], + name: "test_index".to_string(), + dataset_version: 1, + fragment_bitmap: None, + index_details, + index_version: 1, + created_at: None, + base_id: None, + }; + + let metric = metric_type_from_index_metadata(&index); + assert_eq!(metric, Some(*expected_distance)); + } + } } From d3618957411d2fc99fe1712319c4e1f7a9e645d8 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Tue, 17 Mar 2026 10:45:33 -0700 Subject: [PATCH 14/22] address PR review feedback for vector index details - Rename HnswIndexDetails -> HnswParameters (westonpace) - Move imports to top of index.rs (westonpace) - Add index_version field to VectorIndexDetails proto (BubbleCal) - Add explicit FlatCompression message instead of using unset oneof (westonpace) Co-Authored-By: Claude Opus 4.6 (1M context) --- protos/index.proto | 16 ++++++-- protos/table.proto | 2 +- rust/lance-index/src/vector/hnsw/builder.rs | 2 +- rust/lance/src/index.rs | 9 ++--- rust/lance/src/index/vector/details.rs | 44 +++++++++++++-------- 5 files changed, 45 insertions(+), 28 deletions(-) diff --git a/protos/index.proto b/protos/index.proto index 00b54853cea..df770e121fc 100644 --- a/protos/index.proto +++ b/protos/index.proto @@ -193,7 +193,7 @@ message VectorIndexDetails { uint64 target_partition_size = 2; // Optional HNSW index configuration. If set, the index has an HNSW layer. - optional HnswIndexDetails hnsw_index_config = 3; + optional HnswParameters hnsw_index_config = 3; message ProductQuantization { uint32 num_bits = 1; @@ -211,16 +211,24 @@ message VectorIndexDetails { RotationType rotation_type = 2; } - // An unset compression oneof means flat / no quantization. + // No quantization; vectors are stored as-is. + message FlatCompression {} + oneof compression { ProductQuantization pq = 4; ScalarQuantization sq = 5; RabitQuantization rq = 6; + FlatCompression flat = 8; } + + // The version of the index file format. Useful for maintaining backwards + // compatibility when introducing breaking changes to the index format. + // 0 means unset (legacy index). + uint32 index_version = 7; } -// Hierarchical Navigable Small World (HNSW) index details, used as an optional configuration for IVF indexes. -message HnswIndexDetails { +// Hierarchical Navigable Small World (HNSW) parameters, used as an optional configuration for IVF indexes. +message HnswParameters { // The maximum number of outgoing edges per node in the HNSW graph. Higher values // means more connections, better recall, but more memory and slower builds. // Referred to as "M" in the HNSW literature. diff --git a/protos/table.proto b/protos/table.proto index b827f238038..10ce54d90c0 100644 --- a/protos/table.proto +++ b/protos/table.proto @@ -460,7 +460,7 @@ message ExternalFile { uint64 size = 3; } -// VectorIndexDetails and HnswIndexDetails moved to index.proto +// VectorIndexDetails and HnswParameters (formerly HnswIndexDetails) moved to index.proto message FragmentReuseIndexDetails { diff --git a/rust/lance-index/src/vector/hnsw/builder.rs b/rust/lance-index/src/vector/hnsw/builder.rs index 0c206a52dcc..56cc4d38524 100644 --- a/rust/lance-index/src/vector/hnsw/builder.rs +++ b/rust/lance-index/src/vector/hnsw/builder.rs @@ -59,7 +59,7 @@ pub struct HnswBuildParams { pub prefetch_distance: Option, } -impl From<&HnswBuildParams> for crate::pb::HnswIndexDetails { +impl From<&HnswBuildParams> for crate::pb::HnswParameters { fn from(params: &HnswBuildParams) -> Self { Self { max_connections: params.m as u32, diff --git a/rust/lance/src/index.rs b/rust/lance/src/index.rs index bafc3bd0758..a8dc42b8022 100644 --- a/rust/lance/src/index.rs +++ b/rust/lance/src/index.rs @@ -62,6 +62,10 @@ use scalar::index_matches_criteria; use serde_json::json; use tracing::{info, instrument}; use uuid::Uuid; +use vector::details::{ + derive_vector_index_type, infer_missing_vector_details, vector_details_as_json, +}; +pub(crate) use vector::details::{vector_index_details, vector_index_details_default}; use vector::ivf::v2::IVFIndex; use vector::utils::get_vector_type; @@ -411,11 +415,6 @@ async fn open_index_proto(reader: &dyn Reader) -> Result { Ok(proto) } -use vector::details::{ - derive_vector_index_type, infer_missing_vector_details, vector_details_as_json, -}; -pub(crate) use vector::details::{vector_index_details, vector_index_details_default}; - struct IndexDescriptionImpl { name: String, field_ids: Vec, diff --git a/rust/lance/src/index/vector/details.rs b/rust/lance/src/index/vector/details.rs index 51443496710..e03c7ed3a06 100644 --- a/rust/lance/src/index/vector/details.rs +++ b/rust/lance/src/index/vector/details.rs @@ -16,7 +16,7 @@ use lance_file::reader::FileReaderOptions; use lance_index::pb::VectorIndexDetails; use lance_index::pb::VectorMetricType; use lance_index::pb::index::Implementation; -use lance_index::pb::vector_index_details::{Compression, rabit_quantization}; +use lance_index::pb::vector_index_details::{Compression, FlatCompression, rabit_quantization}; use lance_index::{INDEX_FILE_NAME, INDEX_METADATA_SCHEMA_KEY, pb}; use lance_io::scheduler::{ScanScheduler, SchedulerConfig}; use lance_io::traits::Reader; @@ -101,11 +101,15 @@ pub fn vector_index_details(params: &VectorIndexParams) -> prost_types::Any { } } + let compression = compression.or(Some(Compression::Flat(FlatCompression {}))); + let index_version = params.index_type().version() as u32; + let details = VectorIndexDetails { metric_type: metric_type.into(), target_partition_size, hnsw_index_config, compression, + index_version, }; prost_types::Any::from_msg(&details).unwrap() } @@ -195,7 +199,7 @@ pub fn derive_vector_index_type(details: &prost_types::Any) -> String { index_type.push_str("HNSW_"); } match d.compression { - None => index_type.push_str("FLAT"), + None | Some(Compression::Flat(_)) => index_type.push_str("FLAT"), Some(Compression::Pq(_)) => index_type.push_str("PQ"), Some(Compression::Sq(_)) => index_type.push_str("SQ"), Some(Compression::Rq(_)) => index_type.push_str("RQ"), @@ -226,23 +230,24 @@ pub fn vector_details_as_json(details: &prost_types::Any) -> Result { construction_ef: h.construction_ef, }); - let compression = d.compression.map(|c| match c { - Compression::Pq(pq) => CompressionDetailsJson::Pq { + let compression = d.compression.and_then(|c| match c { + Compression::Flat(_) => None, + Compression::Pq(pq) => Some(CompressionDetailsJson::Pq { num_bits: pq.num_bits, num_sub_vectors: pq.num_sub_vectors, - }, - Compression::Sq(sq) => CompressionDetailsJson::Sq { + }), + Compression::Sq(sq) => Some(CompressionDetailsJson::Sq { num_bits: sq.num_bits, - }, + }), Compression::Rq(rq) => { let rotation_type = match rabit_quantization::RotationType::try_from(rq.rotation_type) { Ok(rabit_quantization::RotationType::Matrix) => "matrix", _ => "fast", }; - CompressionDetailsJson::Rq { + Some(CompressionDetailsJson::Rq { num_bits: rq.num_bits, rotation_type, - } + }) } }); @@ -299,7 +304,7 @@ fn convert_legacy_proto_to_details(proto: &pb::Index) -> Result = None; for stage in &vector_index.stages { if let Some(Stage::Pq(pq)) = &stage.stage { compression = Some(Compression::Pq(ProductQuantization { @@ -308,12 +313,14 @@ fn convert_legacy_proto_to_details(proto: &pb::Index) -> Result Result { use lance_index::pb::vector_index_details::*; - use lance_index::pb::{HnswIndexDetails, VectorIndexDetails}; + use lance_index::pb::{HnswParameters, VectorIndexDetails}; use lance_index::vector::bq::storage::RABIT_METADATA_KEY; use lance_index::vector::hnsw::HnswMetadata; use lance_index::vector::ivf::storage::IVF_PARTITION_KEY; @@ -388,13 +395,13 @@ async fn convert_v3_metadata_to_details( rotation_type: rotation_type.into(), })) } else { - None + Some(Compression::Flat(FlatCompression {})) }; // Check for HNSW let hnsw_index_config = if let Some(partition_str) = metadata.get(IVF_PARTITION_KEY) { let partitions: Vec = serde_json::from_str(partition_str)?; - partitions.first().map(|hnsw| HnswIndexDetails { + partitions.first().map(|hnsw| HnswParameters { max_connections: hnsw.params.m as u32, construction_ef: hnsw.params.ef_construction as u32, }) @@ -407,6 +414,7 @@ async fn convert_v3_metadata_to_details( target_partition_size: 0, hnsw_index_config, compression, + index_version: 0, }; Ok(prost_types::Any::from_msg(&details).unwrap()) } @@ -415,11 +423,11 @@ async fn convert_v3_metadata_to_details( mod tests { use super::*; use lance_index::pb::vector_index_details::*; - use lance_index::pb::{HnswIndexDetails, VectorIndexDetails}; + use lance_index::pb::{HnswParameters, VectorIndexDetails}; fn make_details( metric: VectorMetricType, - hnsw: Option, + hnsw: Option, compression: Option, ) -> prost_types::Any { let details = VectorIndexDetails { @@ -427,6 +435,7 @@ mod tests { target_partition_size: 0, hnsw_index_config: hnsw, compression, + index_version: 0, }; prost_types::Any::from_msg(&details).unwrap() } @@ -463,7 +472,7 @@ mod tests { #[test] fn test_derive_index_type_with_hnsw() { - let hnsw = Some(HnswIndexDetails { + let hnsw = Some(HnswParameters { max_connections: 20, construction_ef: 150, }); @@ -521,7 +530,7 @@ mod tests { fn test_json_ivf_hnsw_sq() { let details = make_details( VectorMetricType::Cosine, - Some(HnswIndexDetails { + Some(HnswParameters { max_connections: 30, construction_ef: 200, }), @@ -573,6 +582,7 @@ mod tests { target_partition_size: 5000, hnsw_index_config: None, compression: None, + index_version: 0, }; prost_types::Any::from_msg(&d).unwrap() }; From 4104b257fb8cd4b9b6e2bab1ec2a673a3e823646 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Mon, 23 Mar 2026 11:32:50 -0700 Subject: [PATCH 15/22] feat: add max_level to HnswParameters proto Co-Authored-By: Claude Opus 4.6 (1M context) --- protos/index.proto | 2 ++ rust/lance-index/src/vector/hnsw/builder.rs | 1 + rust/lance/src/index/vector/details.rs | 12 +++++++++++- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/protos/index.proto b/protos/index.proto index df770e121fc..c98359f67a6 100644 --- a/protos/index.proto +++ b/protos/index.proto @@ -236,6 +236,8 @@ message HnswParameters { // "construction exploration factor": The size of the dynamic list used during // index construction. uint32 construction_ef = 2; + // The maximum number of levels in the HNSW graph. + uint32 max_level = 3; } message JsonIndexDetails { diff --git a/rust/lance-index/src/vector/hnsw/builder.rs b/rust/lance-index/src/vector/hnsw/builder.rs index 3d812e03671..9ddd110fce1 100644 --- a/rust/lance-index/src/vector/hnsw/builder.rs +++ b/rust/lance-index/src/vector/hnsw/builder.rs @@ -65,6 +65,7 @@ impl From<&HnswBuildParams> for crate::pb::HnswParameters { Self { max_connections: params.m as u32, construction_ef: params.ef_construction as u32, + max_level: params.max_level as u32, } } } diff --git a/rust/lance/src/index/vector/details.rs b/rust/lance/src/index/vector/details.rs index d41cd3f9d93..b189869562b 100644 --- a/rust/lance/src/index/vector/details.rs +++ b/rust/lance/src/index/vector/details.rs @@ -49,6 +49,12 @@ struct VectorDetailsJson { struct HnswDetailsJson { max_connections: u32, construction_ef: u32, + #[serde(skip_serializing_if = "is_zero")] + max_level: u32, +} + +fn is_zero(v: &u32) -> bool { + *v == 0 } #[derive(Serialize)] @@ -256,6 +262,7 @@ pub fn vector_details_as_json(details: &prost_types::Any) -> Result { let hnsw = d.hnsw_index_config.map(|h| HnswDetailsJson { max_connections: h.max_connections, construction_ef: h.construction_ef, + max_level: h.max_level, }); let compression = d.compression.and_then(|c| match c { @@ -432,6 +439,7 @@ async fn convert_v3_metadata_to_details( partitions.first().map(|hnsw| HnswParameters { max_connections: hnsw.params.m as u32, construction_ef: hnsw.params.ef_construction as u32, + max_level: hnsw.params.max_level as u32, }) } else { None @@ -503,6 +511,7 @@ mod tests { let hnsw = Some(HnswParameters { max_connections: 20, construction_ef: 150, + max_level: 7, }); assert_eq!( derive_vector_index_type(&make_details(VectorMetricType::L2, hnsw, None)), @@ -561,12 +570,13 @@ mod tests { Some(HnswParameters { max_connections: 30, construction_ef: 200, + max_level: 8, }), Some(Compression::Sq(ScalarQuantization { num_bits: 4 })), ); assert_eq!( vector_details_as_json(&details).unwrap(), - r#"{"metric_type":"COSINE","hnsw":{"max_connections":30,"construction_ef":200},"compression":{"type":"sq","num_bits":4}}"# + r#"{"metric_type":"COSINE","hnsw":{"max_connections":30,"construction_ef":200,"max_level":8},"compression":{"type":"sq","num_bits":4}}"# ); } From b8a99fbcbf1436fd6bfdeed63f78a38cbcef101c Mon Sep 17 00:00:00 2001 From: Will Jones Date: Mon, 23 Mar 2026 15:34:59 -0700 Subject: [PATCH 16/22] fix: update Python binding to use VectorIndexDetails from lance_index VectorIndexDetails moved from table.proto to index.proto, so the Python binding needs to reference lance_index::pb instead of lance_table::format::pb. Co-Authored-By: Claude Opus 4.6 (1M context) --- python/src/indices.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/src/indices.rs b/python/src/indices.rs index 9589b78ff36..e005f7771ab 100644 --- a/python/src/indices.rs +++ b/python/src/indices.rs @@ -510,7 +510,7 @@ async fn do_load_shuffled_vectors( index_id, ds.fragments().iter().map(|f| f.id as u32), Arc::new( - prost_types::Any::from_msg(&lance_table::format::pb::VectorIndexDetails::default()) + prost_types::Any::from_msg(&lance_index::pb::VectorIndexDetails::default()) .unwrap(), ), IndexType::IvfPq.version(), From 1fa9c173f07c1129fe3f151a8a127160d084674e Mon Sep 17 00:00:00 2001 From: Will Jones Date: Tue, 7 Apr 2026 15:21:56 -0700 Subject: [PATCH 17/22] feat: add runtime_hints map to VectorIndexDetails Adds a `map runtime_hints` field to `VectorIndexDetails` for storing optional build preferences that don't affect index structure (e.g., KMeans iterations, shuffle concurrency, GPU accelerator). These are needed so a background index rebuild process can reproduce the original build configuration. Keys use reverse-DNS namespacing: `lance.*` for core Lance hints, `lancedb.*` for LanceDB-specific hints (e.g., `lancedb.accelerator`). Runtimes that don't recognize a key must silently ignore it. Only non-default values are written to keep the map minimal. Also adds `apply_runtime_hints()` to read hints back into build params. Co-Authored-By: Claude Sonnet 4.6 --- protos/index.proto | 5 + python/src/dataset.rs | 7 + rust/lance/src/dataset/optimize.rs | 1 + rust/lance/src/dataset/scanner.rs | 1 + rust/lance/src/index/vector.rs | 15 ++ rust/lance/src/index/vector/details.rs | 315 +++++++++++++++++++++++++ 6 files changed, 344 insertions(+) diff --git a/protos/index.proto b/protos/index.proto index c98359f67a6..5c095c73a91 100644 --- a/protos/index.proto +++ b/protos/index.proto @@ -225,6 +225,11 @@ message VectorIndexDetails { // compatibility when introducing breaking changes to the index format. // 0 means unset (legacy index). uint32 index_version = 7; + + // Runtime hints: optional build preferences that don't affect index structure. + // Keys use reverse-DNS namespacing (e.g., "lance.ivf.max_iters", "lancedb.accelerator"). + // Unrecognized keys must be silently ignored by all runtimes. + map runtime_hints = 9; } // Hierarchical Navigable Small World (HNSW) parameters, used as an optional configuration for IVF indexes. diff --git a/python/src/dataset.rs b/python/src/dataset.rs index 3b76f3ce043..ed96e572ae6 100644 --- a/python/src/dataset.rs +++ b/python/src/dataset.rs @@ -3504,6 +3504,13 @@ fn prepare_vector_index_params( }?; params.version(index_file_version); params.skip_transpose(skip_transpose); + if let Some(kwargs) = kwargs { + if let Some(acc) = kwargs.get_item("accelerator")? { + params + .runtime_hints + .insert("lancedb.accelerator".to_string(), acc.to_string()); + } + } Ok(params) } diff --git a/rust/lance/src/dataset/optimize.rs b/rust/lance/src/dataset/optimize.rs index 7cec534e171..29a220be098 100644 --- a/rust/lance/src/dataset/optimize.rs +++ b/rust/lance/src/dataset/optimize.rs @@ -4091,6 +4091,7 @@ mod tests { ], version: crate::index::vector::IndexFileVersion::V3, skip_transpose: false, + runtime_hints: Default::default(), }, false, ) diff --git a/rust/lance/src/dataset/scanner.rs b/rust/lance/src/dataset/scanner.rs index 6e2a4102a5f..f6787ecc56f 100644 --- a/rust/lance/src/dataset/scanner.rs +++ b/rust/lance/src/dataset/scanner.rs @@ -9322,6 +9322,7 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") ], version: crate::index::vector::IndexFileVersion::Legacy, skip_transpose: false, + runtime_hints: Default::default(), }, false, ) diff --git a/rust/lance/src/index/vector.rs b/rust/lance/src/index/vector.rs index 4e56e2ce7b1..65711a720b6 100644 --- a/rust/lance/src/index/vector.rs +++ b/rust/lance/src/index/vector.rs @@ -111,6 +111,11 @@ pub struct VectorIndexParams { /// Skip transpose / packing for PQ and RQ storage. pub skip_transpose: bool, + + /// Runtime hints: optional build preferences stored in the index manifest. + /// Keys use reverse-DNS namespacing (e.g., "lance.ivf.max_iters"). + /// Populated by the build path and merged into VectorIndexDetails at creation time. + pub runtime_hints: HashMap, } impl VectorIndexParams { @@ -132,6 +137,7 @@ impl VectorIndexParams { metric_type, version: IndexFileVersion::V3, skip_transpose: false, + runtime_hints: HashMap::new(), } } @@ -142,6 +148,7 @@ impl VectorIndexParams { metric_type, version: IndexFileVersion::V3, skip_transpose: false, + runtime_hints: HashMap::new(), } } @@ -176,6 +183,7 @@ impl VectorIndexParams { metric_type, version: IndexFileVersion::V3, skip_transpose: false, + runtime_hints: HashMap::new(), } } @@ -202,6 +210,7 @@ impl VectorIndexParams { metric_type: distance_type, version: IndexFileVersion::V3, skip_transpose: false, + runtime_hints: HashMap::new(), } } @@ -217,6 +226,7 @@ impl VectorIndexParams { metric_type, version: IndexFileVersion::V3, skip_transpose: false, + runtime_hints: HashMap::new(), } } @@ -231,6 +241,7 @@ impl VectorIndexParams { metric_type, version: IndexFileVersion::V3, skip_transpose: false, + runtime_hints: HashMap::new(), } } @@ -245,6 +256,7 @@ impl VectorIndexParams { metric_type, version: IndexFileVersion::V3, skip_transpose: false, + runtime_hints: HashMap::new(), } } @@ -259,6 +271,7 @@ impl VectorIndexParams { metric_type: distance_type, version: IndexFileVersion::V3, skip_transpose: false, + runtime_hints: HashMap::new(), } } @@ -280,6 +293,7 @@ impl VectorIndexParams { metric_type, version: IndexFileVersion::V3, skip_transpose: false, + runtime_hints: HashMap::new(), } } @@ -301,6 +315,7 @@ impl VectorIndexParams { metric_type, version: IndexFileVersion::V3, skip_transpose: false, + runtime_hints: HashMap::new(), } } diff --git a/rust/lance/src/index/vector/details.rs b/rust/lance/src/index/vector/details.rs index b189869562b..489b3748b77 100644 --- a/rust/lance/src/index/vector/details.rs +++ b/rust/lance/src/index/vector/details.rs @@ -10,6 +10,7 @@ //! - Inferring details from index files on disk (fallback for legacy indices) use std::collections::HashMap; +use std::str::FromStr; use std::sync::Arc; use lance_file::reader::FileReaderOptions; @@ -43,6 +44,8 @@ struct VectorDetailsJson { hnsw: Option, #[serde(skip_serializing_if = "Option::is_none")] compression: Option, + #[serde(skip_serializing_if = "HashMap::is_empty")] + runtime_hints: HashMap, } #[derive(Serialize)] @@ -85,22 +88,77 @@ pub fn vector_index_details(params: &VectorIndexParams) -> prost_types::Any { let mut target_partition_size = 0u64; let mut hnsw_index_config = None; let mut compression = None; + let mut runtime_hints: HashMap = params.runtime_hints.clone(); + // Only write hints that differ from their defaults, keeping the map minimal. + // Absence of a key means "use your default". for stage in ¶ms.stages { match stage { StageParams::Ivf(ivf) => { if let Some(tps) = ivf.target_partition_size { target_partition_size = tps as u64; } + if ivf.max_iters != 50 { + runtime_hints + .insert("lance.ivf.max_iters".to_string(), ivf.max_iters.to_string()); + } + if ivf.sample_rate != 256 { + runtime_hints.insert( + "lance.ivf.sample_rate".to_string(), + ivf.sample_rate.to_string(), + ); + } + if ivf.shuffle_partition_batches != 1024 * 10 { + runtime_hints.insert( + "lance.ivf.shuffle_partition_batches".to_string(), + ivf.shuffle_partition_batches.to_string(), + ); + } + if ivf.shuffle_partition_concurrency != 2 { + runtime_hints.insert( + "lance.ivf.shuffle_partition_concurrency".to_string(), + ivf.shuffle_partition_concurrency.to_string(), + ); + } } StageParams::Hnsw(hnsw) => { hnsw_index_config = Some(hnsw.into()); + let default_prefetch: Option = Some(2); + if hnsw.prefetch_distance != default_prefetch { + let val = match hnsw.prefetch_distance { + Some(v) => v.to_string(), + None => "none".to_string(), + }; + runtime_hints.insert("lance.hnsw.prefetch_distance".to_string(), val); + } } StageParams::PQ(pq) => { compression = Some(Compression::Pq(pq.into())); + if pq.max_iters != 50 { + runtime_hints + .insert("lance.pq.max_iters".to_string(), pq.max_iters.to_string()); + } + if pq.sample_rate != 256 { + runtime_hints.insert( + "lance.pq.sample_rate".to_string(), + pq.sample_rate.to_string(), + ); + } + if pq.kmeans_redos != 1 { + runtime_hints.insert( + "lance.pq.kmeans_redos".to_string(), + pq.kmeans_redos.to_string(), + ); + } } StageParams::SQ(sq) => { compression = Some(Compression::Sq(sq.into())); + if sq.sample_rate != 256 { + runtime_hints.insert( + "lance.sq.sample_rate".to_string(), + sq.sample_rate.to_string(), + ); + } } StageParams::RQ(rq) => { compression = Some(Compression::Rq(rq.into())); @@ -108,6 +166,10 @@ pub fn vector_index_details(params: &VectorIndexParams) -> prost_types::Any { } } + if params.skip_transpose { + runtime_hints.insert("lance.skip_transpose".to_string(), "true".to_string()); + } + let compression = compression.or(Some(Compression::Flat(FlatCompression {}))); let index_version = params.index_type().version() as u32; @@ -117,6 +179,7 @@ pub fn vector_index_details(params: &VectorIndexParams) -> prost_types::Any { hnsw_index_config, compression, index_version, + runtime_hints, }; prost_types::Any::from_msg(&details).unwrap() } @@ -126,6 +189,70 @@ pub fn vector_index_details_default() -> prost_types::Any { prost_types::Any::from_msg(&details).unwrap() } +/// Apply stored runtime hints from `VectorIndexDetails` back into build params. +/// +/// Known `lance.*` keys are parsed and applied to the appropriate stage. Unknown +/// keys (e.g., from other runtimes) are silently ignored. Malformed values are +/// also silently ignored — the stage keeps its existing default. +pub fn apply_runtime_hints(hints: &HashMap, params: &mut VectorIndexParams) { + fn parse(hints: &HashMap, key: &str) -> Option { + hints.get(key)?.parse().ok() + } + + for stage in &mut params.stages { + match stage { + StageParams::Ivf(ivf) => { + if let Some(v) = parse(hints, "lance.ivf.max_iters") { + ivf.max_iters = v; + } + if let Some(v) = parse(hints, "lance.ivf.sample_rate") { + ivf.sample_rate = v; + } + if let Some(v) = parse(hints, "lance.ivf.shuffle_partition_batches") { + ivf.shuffle_partition_batches = v; + } + if let Some(v) = parse(hints, "lance.ivf.shuffle_partition_concurrency") { + ivf.shuffle_partition_concurrency = v; + } + } + StageParams::Hnsw(hnsw) => { + if let Some(raw) = hints.get("lance.hnsw.prefetch_distance") { + hnsw.prefetch_distance = if raw == "none" { + None + } else { + raw.parse().ok() + }; + } + } + StageParams::PQ(pq) => { + if let Some(v) = parse(hints, "lance.pq.max_iters") { + pq.max_iters = v; + } + if let Some(v) = parse(hints, "lance.pq.sample_rate") { + pq.sample_rate = v; + } + if let Some(v) = parse(hints, "lance.pq.kmeans_redos") { + pq.kmeans_redos = v; + } + } + StageParams::SQ(sq) => { + if let Some(v) = parse(hints, "lance.sq.sample_rate") { + sq.sample_rate = v; + } + } + StageParams::RQ(_) => {} + } + } + + if hints + .get("lance.skip_transpose") + .map(|v| v == "true") + .unwrap_or(false) + { + params.skip_transpose = true; + } +} + /// Extract metric type from index metadata without opening the index file. /// /// For newer indices with populated `VectorIndexDetails`, returns the metric type directly. @@ -295,6 +422,7 @@ pub fn vector_details_as_json(details: &prost_types::Any) -> Result { }, hnsw, compression, + runtime_hints: d.runtime_hints, }; serde_json::to_string(&json).map_err(|e| Error::index(format!("Failed to serialize: {}", e))) @@ -356,6 +484,7 @@ fn convert_legacy_proto_to_details(proto: &pb::Index) -> Result().unwrap(); + assert_eq!( + details + .runtime_hints + .get("lance.ivf.max_iters") + .map(|s| s.as_str()), + Some("100") + ); + assert_eq!( + details + .runtime_hints + .get("lance.ivf.sample_rate") + .map(|s| s.as_str()), + Some("512") + ); + assert_eq!( + details + .runtime_hints + .get("lance.ivf.shuffle_partition_batches") + .map(|s| s.as_str()), + Some("2048") + ); + assert_eq!( + details + .runtime_hints + .get("lance.ivf.shuffle_partition_concurrency") + .map(|s| s.as_str()), + Some("4") + ); + assert_eq!( + details + .runtime_hints + .get("lance.pq.max_iters") + .map(|s| s.as_str()), + Some("75") + ); + assert_eq!( + details + .runtime_hints + .get("lance.pq.sample_rate") + .map(|s| s.as_str()), + Some("128") + ); + assert_eq!( + details + .runtime_hints + .get("lance.pq.kmeans_redos") + .map(|s| s.as_str()), + Some("3") + ); + // Default values should not appear in the map + assert!( + !details + .runtime_hints + .contains_key("lance.hnsw.prefetch_distance") + ); + assert!(!details.runtime_hints.contains_key("lance.skip_transpose")); + + // Roundtrip: apply hints back to a fresh params struct + let mut restored = VectorIndexParams::with_ivf_pq_params( + DistanceType::L2, + IvfBuildParams::default(), + PQBuildParams { + num_sub_vectors: 8, + num_bits: 8, + ..Default::default() + }, + ); + apply_runtime_hints(&details.runtime_hints, &mut restored); + let StageParams::Ivf(ivf) = &restored.stages[0] else { + panic!() + }; + assert_eq!(ivf.max_iters, 100); + assert_eq!(ivf.sample_rate, 512); + assert_eq!(ivf.shuffle_partition_batches, 2048); + assert_eq!(ivf.shuffle_partition_concurrency, 4); + let StageParams::PQ(pq) = &restored.stages[1] else { + panic!() + }; + assert_eq!(pq.max_iters, 75); + assert_eq!(pq.sample_rate, 128); + assert_eq!(pq.kmeans_redos, 3); + } + + #[test] + fn test_runtime_hints_defaults_omitted() { + use crate::index::vector::VectorIndexParams; + use lance_index::vector::ivf::builder::IvfBuildParams; + use lance_index::vector::pq::builder::PQBuildParams; + use lance_linalg::distance::DistanceType; + + // All defaults — hints map should be empty + let params = VectorIndexParams::with_ivf_pq_params( + DistanceType::L2, + IvfBuildParams::default(), + PQBuildParams { + num_sub_vectors: 8, + num_bits: 8, + ..Default::default() + }, + ); + let any = vector_index_details(¶ms); + let details = any.to_msg::().unwrap(); + assert!(details.runtime_hints.is_empty()); + } + + #[test] + fn test_runtime_hints_in_json() { + use crate::index::vector::VectorIndexParams; + use lance_index::vector::ivf::builder::IvfBuildParams; + use lance_index::vector::pq::builder::PQBuildParams; + use lance_linalg::distance::DistanceType; + + let params = VectorIndexParams::with_ivf_pq_params( + DistanceType::L2, + IvfBuildParams { + max_iters: 100, + ..Default::default() + }, + PQBuildParams { + num_sub_vectors: 8, + num_bits: 8, + ..Default::default() + }, + ); + let any = vector_index_details(¶ms); + let json = vector_details_as_json(&any).unwrap(); + let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed["runtime_hints"]["lance.ivf.max_iters"], "100"); + } + + #[test] + fn test_apply_runtime_hints_ignores_unknown_keys() { + use crate::index::vector::VectorIndexParams; + use lance_index::vector::ivf::builder::IvfBuildParams; + use lance_linalg::distance::DistanceType; + + let hints: HashMap = [ + ("lancedb.accelerator".to_string(), "cuda".to_string()), + ("unknown.vendor.key".to_string(), "value".to_string()), + ("lance.ivf.max_iters".to_string(), "99".to_string()), + ] + .into(); + + let mut params = + VectorIndexParams::with_ivf_flat_params(DistanceType::L2, IvfBuildParams::default()); + apply_runtime_hints(&hints, &mut params); + + let StageParams::Ivf(ivf) = ¶ms.stages[0] else { + panic!() + }; + assert_eq!(ivf.max_iters, 99); + // Unknown keys silently ignored — no panic + } } From e9bb5a5a0468447ac3446d767aaa40f858f6a368 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Tue, 7 Apr 2026 16:13:33 -0700 Subject: [PATCH 18/22] feat: add vector_params_from_details and wire skip_transpose into rebuild Adds `vector_params_from_details()` to reconstruct a full `VectorIndexParams` from stored `VectorIndexDetails` (core spec fields + runtime hints). This enables future index rebuild logic to reproduce the original build config from the manifest without re-opening index files. Also wires the `lance.skip_transpose` hint into `optimize_vector_indices_v2` so incremental rebuilds honour the original skip_transpose preference rather than silently reverting to false on each append. Adds Python tests validating that non-default build params appear as `runtime_hints` in `describe_indices()` output, and that default values are omitted. Co-Authored-By: Claude Sonnet 4.6 --- python/python/tests/test_vector_index.py | 32 +++++++++ rust/lance/src/index.rs | 4 +- rust/lance/src/index/vector/details.rs | 87 ++++++++++++++++++++++++ rust/lance/src/index/vector/ivf.rs | 25 ++++++- 4 files changed, 146 insertions(+), 2 deletions(-) diff --git a/python/python/tests/test_vector_index.py b/python/python/tests/test_vector_index.py index 13be0ae410a..cb29cd10a33 100644 --- a/python/python/tests/test_vector_index.py +++ b/python/python/tests/test_vector_index.py @@ -1695,6 +1695,38 @@ def test_describe_vector_index(indexed_dataset: LanceDataset): assert details["compression"]["num_sub_vectors"] == 16 +def test_describe_index_runtime_hints_stored(tmp_path): + tbl = create_table(nvec=300, ndim=16) + dataset = lance.write_dataset(tbl, tmp_path) + dataset = dataset.create_index( + "vector", + index_type="IVF_PQ", + num_partitions=4, + num_sub_vectors=4, + max_iters=100, + sample_rate=512, + ) + details = dataset.describe_indices()[0].details + hints = details.get("runtime_hints", {}) + assert hints.get("lance.ivf.max_iters") == "100" + assert hints.get("lance.ivf.sample_rate") == "512" + assert hints.get("lance.pq.max_iters") == "100" + assert hints.get("lance.pq.sample_rate") == "512" + + +def test_describe_index_runtime_hints_defaults_omitted(tmp_path): + tbl = create_table(nvec=300, ndim=16) + dataset = lance.write_dataset(tbl, tmp_path) + dataset = dataset.create_index( + "vector", + index_type="IVF_PQ", + num_partitions=4, + num_sub_vectors=4, + ) + details = dataset.describe_indices()[0].details + assert "runtime_hints" not in details + + def test_optimize_indices(indexed_dataset): data = create_table() indexed_dataset = lance.write_dataset(data, indexed_dataset.uri, mode="append") diff --git a/rust/lance/src/index.rs b/rust/lance/src/index.rs index d80bd45a263..b246476f52e 100644 --- a/rust/lance/src/index.rs +++ b/rust/lance/src/index.rs @@ -66,7 +66,9 @@ use uuid::Uuid; use vector::details::{ derive_vector_index_type, infer_missing_vector_details, vector_details_as_json, }; -pub(crate) use vector::details::{vector_index_details, vector_index_details_default}; +pub(crate) use vector::details::{ + vector_index_details, vector_index_details_default, vector_params_from_details, +}; use vector::ivf::v2::{IVFIndex, IvfStateEntryBox}; use vector::utils::get_vector_type; diff --git a/rust/lance/src/index/vector/details.rs b/rust/lance/src/index/vector/details.rs index 489b3748b77..dba82997d93 100644 --- a/rust/lance/src/index/vector/details.rs +++ b/rust/lance/src/index/vector/details.rs @@ -26,6 +26,12 @@ use lance_linalg::distance::DistanceType; use lance_table::format::IndexMetadata; use serde::Serialize; +use lance_index::vector::bq::{RQBuildParams, RQRotationType}; +use lance_index::vector::hnsw::builder::HnswBuildParams; +use lance_index::vector::ivf::IvfBuildParams; +use lance_index::vector::pq::PQBuildParams; +use lance_index::vector::sq::builder::SQBuildParams; + use super::{StageParams, VectorIndexParams}; use crate::dataset::Dataset; use crate::index::open_index_proto; @@ -253,6 +259,87 @@ pub fn apply_runtime_hints(hints: &HashMap, params: &mut VectorI } } +/// Reconstruct `VectorIndexParams` from a stored `VectorIndexDetails` proto. +/// +/// Returns `None` for legacy indices (empty details) or if the proto is malformed. +/// Runtime hints are applied on top of the reconstructed spec. +pub fn vector_params_from_details(details: &prost_types::Any) -> Option { + if details.value.is_empty() { + return None; + } + let d = details.to_msg::().ok()?; + + let metric = DistanceType::from(VectorMetricType::try_from(d.metric_type).ok()?); + + let mut ivf = IvfBuildParams::default(); + if d.target_partition_size > 0 { + ivf.target_partition_size = Some(d.target_partition_size as usize); + } + + let hnsw = d.hnsw_index_config.map(|h| HnswBuildParams { + m: h.max_connections as usize, + ef_construction: h.construction_ef as usize, + max_level: h.max_level as u16, + ..Default::default() + }); + + let mut params = match (hnsw, d.compression) { + (None, Some(Compression::Pq(pq))) => VectorIndexParams::with_ivf_pq_params( + metric, + ivf, + PQBuildParams { + num_bits: pq.num_bits as usize, + num_sub_vectors: pq.num_sub_vectors as usize, + ..Default::default() + }, + ), + (None, Some(Compression::Sq(sq))) => VectorIndexParams::with_ivf_sq_params( + metric, + ivf, + SQBuildParams { + num_bits: sq.num_bits as u16, + ..Default::default() + }, + ), + (None, Some(Compression::Rq(rq))) => { + let rotation_type = + match rabit_quantization::RotationType::try_from(rq.rotation_type).ok()? { + rabit_quantization::RotationType::Matrix => RQRotationType::Matrix, + rabit_quantization::RotationType::Fast => RQRotationType::Fast, + }; + VectorIndexParams::with_ivf_rq_params( + metric, + ivf, + RQBuildParams::with_rotation_type(rq.num_bits as u8, rotation_type), + ) + } + (Some(hnsw), Some(Compression::Pq(pq))) => VectorIndexParams::with_ivf_hnsw_pq_params( + metric, + ivf, + hnsw, + PQBuildParams { + num_bits: pq.num_bits as usize, + num_sub_vectors: pq.num_sub_vectors as usize, + ..Default::default() + }, + ), + (Some(hnsw), Some(Compression::Sq(sq))) => VectorIndexParams::with_ivf_hnsw_sq_params( + metric, + ivf, + hnsw, + SQBuildParams { + num_bits: sq.num_bits as u16, + ..Default::default() + }, + ), + (Some(hnsw), _) => VectorIndexParams::ivf_hnsw(metric, ivf, hnsw), + _ => VectorIndexParams::with_ivf_flat_params(metric, ivf), + }; + + apply_runtime_hints(&d.runtime_hints, &mut params); + Some(params) +} + /// Extract metric type from index metadata without opening the index file. /// /// For newer indices with populated `VectorIndexDetails`, returns the metric type directly. diff --git a/rust/lance/src/index/vector/ivf.rs b/rust/lance/src/index/vector/ivf.rs index cc91b9a3b27..d845e87f92e 100644 --- a/rust/lance/src/index/vector/ivf.rs +++ b/rust/lance/src/index/vector/ivf.rs @@ -17,7 +17,10 @@ use crate::index::DatasetIndexInternalExt; use crate::index::vector::utils::{get_vector_dim, get_vector_type}; use crate::{ dataset::Dataset, - index::{INDEX_FILE_NAME, pb, prefilter::PreFilter, vector::ivf::io::write_pq_partitions}, + index::{ + INDEX_FILE_NAME, pb, prefilter::PreFilter, vector::ivf::io::write_pq_partitions, + vector_params_from_details, + }, }; use crate::{dataset::builder::DatasetBuilder, index::vector::IndexFileVersion}; use arrow::datatypes::UInt8Type; @@ -385,12 +388,22 @@ pub(crate) async fn optimize_vector_indices( // try cast to v1 IVFIndex, // fallback to v2 IVFIndex if it's not v1 IVFIndex if !existing_indices[0].as_any().is::() { + // Restore skip_transpose from stored details so incremental rebuilds + // honour the original preference rather than silently reverting to false. + let skip_transpose = logical_index + .segments() + .next() + .and_then(|(meta, _)| meta.index_details.as_deref()) + .and_then(|d| vector_params_from_details(d)) + .map(|p| p.skip_transpose) + .unwrap_or(false); return optimize_vector_indices_v2( &dataset, unindexed, vector_column, &existing_indices, options, + skip_transpose, ) .await; } @@ -461,6 +474,7 @@ pub(crate) async fn optimize_vector_indices_v2( vector_column: &str, existing_indices: &[Arc], options: &OptimizeOptions, + skip_transpose: bool, ) -> Result<(Uuid, usize)> { // Sanity check the indices if existing_indices.is_empty() { @@ -503,6 +517,7 @@ pub(crate) async fn optimize_vector_indices_v2( .with_ivf(ivf_model.clone()) .with_quantizer(quantizer.try_into()?) .with_existing_indices(existing_indices.clone()) + .with_transpose(!skip_transpose) .shuffle_data(unindexed) .await? .build() @@ -521,6 +536,7 @@ pub(crate) async fn optimize_vector_indices_v2( .with_ivf(ivf_model.clone()) .with_quantizer(quantizer.try_into()?) .with_existing_indices(existing_indices.clone()) + .with_transpose(!skip_transpose) .shuffle_data(unindexed) .await? .build() @@ -542,6 +558,7 @@ pub(crate) async fn optimize_vector_indices_v2( .with_ivf(ivf_model.clone()) .with_quantizer(quantizer.try_into()?) .with_existing_indices(existing_indices.clone()) + .with_transpose(!skip_transpose) .shuffle_data(unindexed) .await? .build() @@ -562,6 +579,7 @@ pub(crate) async fn optimize_vector_indices_v2( .with_ivf(ivf_model.clone()) .with_quantizer(quantizer.try_into()?) .with_existing_indices(existing_indices.clone()) + .with_transpose(!skip_transpose) .shuffle_data(unindexed) .await? .build() @@ -581,6 +599,7 @@ pub(crate) async fn optimize_vector_indices_v2( .with_ivf(ivf_model.clone()) .with_quantizer(quantizer.try_into()?) .with_existing_indices(existing_indices.clone()) + .with_transpose(!skip_transpose) .shuffle_data(unindexed) .await? .build() @@ -602,6 +621,7 @@ pub(crate) async fn optimize_vector_indices_v2( .with_ivf(ivf_model.clone()) .with_quantizer(quantizer.try_into()?) .with_existing_indices(existing_indices.clone()) + .with_transpose(!skip_transpose) .shuffle_data(unindexed) .await? .build() @@ -620,6 +640,7 @@ pub(crate) async fn optimize_vector_indices_v2( .with_ivf(ivf_model.clone()) .with_quantizer(quantizer.try_into()?) .with_existing_indices(existing_indices.clone()) + .with_transpose(!skip_transpose) .shuffle_data(unindexed) .await? .build() @@ -641,6 +662,7 @@ pub(crate) async fn optimize_vector_indices_v2( .with_ivf(ivf_model.clone()) .with_quantizer(quantizer.try_into()?) .with_existing_indices(existing_indices.clone()) + .with_transpose(!skip_transpose) .shuffle_data(unindexed) .await? .build() @@ -661,6 +683,7 @@ pub(crate) async fn optimize_vector_indices_v2( .with_ivf(ivf_model.clone()) .with_quantizer(quantizer.try_into()?) .with_existing_indices(existing_indices.clone()) + .with_transpose(!skip_transpose) .shuffle_data(unindexed) .await? .build() From 11bc12630be964f9734ea1dfdd2fc90fccc33a25 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Tue, 7 Apr 2026 16:18:40 -0700 Subject: [PATCH 19/22] clippy --- rust/lance/src/index/vector/ivf.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/lance/src/index/vector/ivf.rs b/rust/lance/src/index/vector/ivf.rs index d845e87f92e..ef899274a15 100644 --- a/rust/lance/src/index/vector/ivf.rs +++ b/rust/lance/src/index/vector/ivf.rs @@ -394,7 +394,7 @@ pub(crate) async fn optimize_vector_indices( .segments() .next() .and_then(|(meta, _)| meta.index_details.as_deref()) - .and_then(|d| vector_params_from_details(d)) + .and_then(vector_params_from_details) .map(|p| p.skip_transpose) .unwrap_or(false); return optimize_vector_indices_v2( From da888fb2331c74f092eb3803f83f2b9762a51499 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Tue, 7 Apr 2026 17:43:05 -0700 Subject: [PATCH 20/22] fix: wire max_iters kwarg and fix binding compile errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - `python/src/dataset.rs`: `max_iters` kwarg was not forwarded to `IvfBuildParams`/`PQBuildParams` in `prepare_vector_index_params`, so it was silently ignored and never stored as a runtime hint - `python/src/indices.rs`: wrong proto path for `VectorIndexDetails` (`lance_table::format::pb` → `lance_index::pb`) - `java/lance-jni/src/utils.rs`: missing `runtime_hints` field in `VectorIndexParams` struct literal Co-Authored-By: Claude Sonnet 4.6 --- java/lance-jni/Cargo.lock | 1 + java/lance-jni/src/utils.rs | 1 + python/src/dataset.rs | 6 ++++++ python/src/indices.rs | 3 +-- 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/java/lance-jni/Cargo.lock b/java/lance-jni/Cargo.lock index 09143a6adf0..5bb30513354 100644 --- a/java/lance-jni/Cargo.lock +++ b/java/lance-jni/Cargo.lock @@ -3469,6 +3469,7 @@ dependencies = [ "arrow-buffer", "arrow-cast", "arrow-data", + "arrow-ipc", "arrow-ord", "arrow-schema", "arrow-select", diff --git a/java/lance-jni/src/utils.rs b/java/lance-jni/src/utils.rs index 3d7ef2ebdaf..79f9ba25ffe 100644 --- a/java/lance-jni/src/utils.rs +++ b/java/lance-jni/src/utils.rs @@ -426,6 +426,7 @@ pub fn get_vector_index_params( stages, version: IndexFileVersion::V3, skip_transpose: false, + runtime_hints: Default::default(), }) }, )?; diff --git a/python/src/dataset.rs b/python/src/dataset.rs index f77e710ff78..9e340afbf13 100644 --- a/python/src/dataset.rs +++ b/python/src/dataset.rs @@ -3569,6 +3569,12 @@ fn prepare_vector_index_params( sq_params.sample_rate = sample_rate; } + if let Some(max_iters) = kwargs.get_item("max_iters")? { + let max_iters: usize = max_iters.extract()?; + ivf_params.max_iters = max_iters; + pq_params.max_iters = max_iters; + } + // Parse IVF params if let Some(n) = kwargs.get_item("num_partitions")? { ivf_params.num_partitions = Some(n.extract()?) diff --git a/python/src/indices.rs b/python/src/indices.rs index 62f3c0c64ec..6ff95fbe151 100644 --- a/python/src/indices.rs +++ b/python/src/indices.rs @@ -441,8 +441,7 @@ async fn do_load_shuffled_vectors( dataset_version: ds.manifest.version, fragment_bitmap: Some(ds.fragments().iter().map(|f| f.id as u32).collect()), index_details: Some(Arc::new( - prost_types::Any::from_msg(&lance_table::format::pb::VectorIndexDetails::default()) - .unwrap(), + prost_types::Any::from_msg(&lance_index::pb::VectorIndexDetails::default()).unwrap(), )), index_version: IndexType::IvfPq.version(), created_at: Some(Utc::now()), From 5702f43c557ee2b6a5f8f040aa3c315c7e10cebd Mon Sep 17 00:00:00 2001 From: Will Jones Date: Wed, 8 Apr 2026 10:56:03 -0700 Subject: [PATCH 21/22] clippy --- python/src/dataset.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/python/src/dataset.rs b/python/src/dataset.rs index 9e340afbf13..8614e1f6ba5 100644 --- a/python/src/dataset.rs +++ b/python/src/dataset.rs @@ -3737,12 +3737,12 @@ fn prepare_vector_index_params( }?; params.version(index_file_version); params.skip_transpose(skip_transpose); - if let Some(kwargs) = kwargs { - if let Some(acc) = kwargs.get_item("accelerator")? { - params - .runtime_hints - .insert("lancedb.accelerator".to_string(), acc.to_string()); - } + if let Some(kwargs) = kwargs + && let Some(acc) = kwargs.get_item("accelerator")? + { + params + .runtime_hints + .insert("lancedb.accelerator".to_string(), acc.to_string()); } Ok(params) } From c3860e6fa8068fd919869a211433f5c49f8c42b4 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Wed, 8 Apr 2026 15:21:39 -0700 Subject: [PATCH 22/22] fix compat tests --- python/python/tests/compat/test_vector_indices.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/python/python/tests/compat/test_vector_indices.py b/python/python/tests/compat/test_vector_indices.py index 851d06b85eb..b98ffdf63e3 100644 --- a/python/python/tests/compat/test_vector_indices.py +++ b/python/python/tests/compat/test_vector_indices.py @@ -75,13 +75,12 @@ def check_read(self): indices = ds.describe_indices() assert len(indices) >= 1 name = indices[0].name - else: + elif self.compat_version >= "0.39.0": indices = ds.list_indices() assert len(indices) >= 1 - name = indices[0].name - - stats = ds.stats.index_stats(name) - assert stats["num_indexed_rows"] > 0 + name = indices[0]["name"] + stats = ds.stats.index_stats(name) + assert stats["num_indexed_rows"] > 0 def check_write(self): """Verify can insert vectors and rebuild index.""" @@ -159,7 +158,7 @@ def check_read(self): else: indices = ds.list_indices() assert len(indices) >= 1 - name = indices[0].name + name = indices[0]["name"] stats = ds.stats.index_stats(name) assert stats["num_indexed_rows"] > 0 @@ -240,7 +239,7 @@ def check_read(self): else: indices = ds.list_indices() assert len(indices) >= 1 - name = indices[0].name + name = indices[0]["name"] stats = ds.stats.index_stats(name) assert stats["num_indexed_rows"] > 0 @@ -262,7 +261,7 @@ def check_write(self): ds.optimize.compact_files() -@compat_test(min_version="v4.0.0-beta.8") +@compat_test(min_version="4.0.0-beta.8") class IvfRqVectorIndex(UpgradeDowngradeTest): """Test IVF_RQ vector index compatibility. V2 was introduced in v4.0.0-beta.8"""