From 6644681c246977efcc6ca15c31e2393831d04f5f Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Wed, 10 Dec 2025 20:55:01 -0800 Subject: [PATCH 1/6] feat: upgrade lance-namespace to 0.3.0 and add missing apis --- Cargo.lock | 4 +- Cargo.toml | 2 +- java/pom.xml | 4 +- python/pyproject.toml | 2 +- .../src/object_store/storage_options.rs | 1 + rust/lance-namespace-impls/src/dir.rs | 67 +- .../lance-namespace-impls/src/dir/manifest.rs | 45 +- rust/lance-namespace-impls/src/rest.rs | 288 ++++++++- .../lance-namespace-impls/src/rest_adapter.rs | 584 +++++++++++++++++- rust/lance-namespace/src/namespace.rs | 229 ++++++- rust/lance/src/dataset.rs | 1 + rust/lance/src/dataset/builder.rs | 1 + 12 files changed, 1108 insertions(+), 120 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1c71cdd705f..9ffd5ba10c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4919,9 +4919,9 @@ dependencies = [ [[package]] name = "lance-namespace-reqwest-client" -version = "0.0.18" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ea349999bcda4eea53fc05d334b3775ec314761e6a706555c777d7a29b18d19" +checksum = "28bb5310d4f8f197a60dc2878ae48e4cc9ebd20e1fc519302651ed8dbb62a458" dependencies = [ "reqwest", "serde", diff --git a/Cargo.toml b/Cargo.toml index c5cd509f311..f60d28341fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -63,7 +63,7 @@ lance-io = { version = "=1.1.0-beta.2", path = "./rust/lance-io", default-featur lance-linalg = { version = "=1.1.0-beta.2", path = "./rust/lance-linalg" } lance-namespace = { version = "=1.1.0-beta.2", path = "./rust/lance-namespace" } lance-namespace-impls = { version = "=1.1.0-beta.2", path = "./rust/lance-namespace-impls" } -lance-namespace-reqwest-client = "0.0.18" +lance-namespace-reqwest-client = "0.3.0" lance-table = { version = "=1.1.0-beta.2", path = "./rust/lance-table" } lance-test-macros = { version = "=1.1.0-beta.2", path = "./rust/lance-test-macros" } lance-testing = { version = "=1.1.0-beta.2", path = "./rust/lance-testing" } diff --git a/java/pom.xml b/java/pom.xml index b09d7576a97..2b0f29167e5 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -108,12 +108,12 @@ org.lance lance-namespace-core - 0.2.1 + 0.3.0 org.lance lance-namespace-apache-client - 0.2.1 + 0.3.0 com.fasterxml.jackson.core diff --git a/python/pyproject.toml b/python/pyproject.toml index 7aa500668df..d0b48f96165 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "pylance" dynamic = ["version"] -dependencies = ["pyarrow>=14", "numpy>=1.22", "lance-namespace>=0.2.1"] +dependencies = ["pyarrow>=14", "numpy>=1.22", "lance-namespace>=0.3.0"] description = "python wrapper for Lance columnar format" authors = [{ name = "Lance Devs", email = "dev@lance.org" }] license = { file = "LICENSE" } diff --git a/rust/lance-io/src/object_store/storage_options.rs b/rust/lance-io/src/object_store/storage_options.rs index 9405f95d70c..f809df8d1d3 100644 --- a/rust/lance-io/src/object_store/storage_options.rs +++ b/rust/lance-io/src/object_store/storage_options.rs @@ -114,6 +114,7 @@ impl StorageOptionsProvider for LanceNamespaceStorageOptionsProvider { let request = DescribeTableRequest { id: Some(self.table_id.clone()), version: None, + with_table_uri: None, }; let response = self diff --git a/rust/lance-namespace-impls/src/dir.rs b/rust/lance-namespace-impls/src/dir.rs index fd5a63a0848..b0bd52f7ef4 100644 --- a/rust/lance-namespace-impls/src/dir.rs +++ b/rust/lance-namespace-impls/src/dir.rs @@ -602,6 +602,7 @@ impl LanceNamespace for DirectoryNamespace { Self::validate_root_namespace_id(&request.id)?; Ok(DescribeNamespaceResponse { + id: request.id.unwrap_or_default(), properties: Some(HashMap::new()), }) } @@ -777,11 +778,20 @@ impl LanceNamespace for DirectoryNamespace { let arrow_schema: arrow_schema::Schema = lance_schema.into(); let json_schema = arrow_schema_to_json(&arrow_schema)?; Ok(DescribeTableResponse { + table: Some(table_name), + namespace: request.id.as_ref().map(|id| { + if id.len() > 1 { + id[..id.len() - 1].to_vec() + } else { + vec![] + } + }), version: Some(version as i64), - location: Some(table_uri), + location: Some(table_uri.clone()), + table_uri: Some(table_uri), schema: Some(Box::new(json_schema)), - properties: None, storage_options: self.storage_options.clone(), + stats: None, }) } Err(err) => { @@ -793,11 +803,20 @@ impl LanceNamespace for DirectoryNamespace { .unwrap_or(false) { Ok(DescribeTableResponse { + table: Some(table_name), + namespace: request.id.as_ref().map(|id| { + if id.len() > 1 { + id[..id.len() - 1].to_vec() + } else { + vec![] + } + }), version: None, - location: Some(table_uri), + location: Some(table_uri.clone()), + table_uri: Some(table_uri), schema: None, - properties: None, storage_options: self.storage_options.clone(), + stats: None, }) } else { Err(Error::Namespace { @@ -886,21 +905,6 @@ impl LanceNamespace for DirectoryNamespace { }); } - // Validate location if provided - if let Some(location) = &request.location { - let location = location.trim_end_matches('/'); - if location != table_uri { - return Err(Error::Namespace { - source: format!( - "Cannot create table {} at location {}, must be at location {}", - table_name, location, table_uri - ) - .into(), - location: snafu::location!(), - }); - } - } - // Parse the Arrow IPC stream from request_data let cursor = Cursor::new(request_data.to_vec()); let stream_reader = StreamReader::try_new(cursor, None).map_err(|e| Error::Namespace { @@ -948,9 +952,9 @@ impl LanceNamespace for DirectoryNamespace { })?; Ok(CreateTableResponse { + transaction_id: None, version: Some(1), location: Some(table_uri), - properties: None, storage_options: self.storage_options.clone(), }) } @@ -1007,6 +1011,7 @@ impl LanceNamespace for DirectoryNamespace { })?; Ok(CreateEmptyTableResponse { + transaction_id: None, location: Some(table_uri), properties: None, storage_options: self.storage_options.clone(), @@ -1188,28 +1193,6 @@ mod tests { ); } - #[tokio::test] - async fn test_create_table_with_wrong_location() { - let (namespace, _temp_dir) = create_test_namespace().await; - - // Create test IPC data - let schema = create_test_schema(); - let ipc_data = create_test_ipc_data(&schema); - - let mut request = CreateTableRequest::new(); - request.id = Some(vec!["test_table".to_string()]); - request.location = Some("/wrong/path/table.lance".to_string()); - - let result = namespace - .create_table(request, bytes::Bytes::from(ipc_data)) - .await; - assert!(result.is_err()); - assert!(result - .unwrap_err() - .to_string() - .contains("must be at location")); - } - #[tokio::test] async fn test_list_tables() { let (namespace, _temp_dir) = create_test_namespace().await; diff --git a/rust/lance-namespace-impls/src/dir/manifest.rs b/rust/lance-namespace-impls/src/dir/manifest.rs index ddc934d79ee..e004a3dee14 100644 --- a/rust/lance-namespace-impls/src/dir/manifest.rs +++ b/rust/lance-namespace-impls/src/dir/manifest.rs @@ -1069,6 +1069,14 @@ impl LanceNamespace for ManifestNamespace { let object_id = Self::str_object_id(table_id); let table_info = self.query_manifest_for_table(&object_id).await?; + // Extract table name and namespace from table_id + let table_name = table_id.last().cloned().unwrap_or_default(); + let namespace_id: Vec = if table_id.len() > 1 { + table_id[..table_id.len() - 1].to_vec() + } else { + vec![] + }; + match table_info { Some(info) => { // Construct full URI from relative location @@ -1088,21 +1096,27 @@ impl LanceNamespace for ManifestNamespace { let json_schema = arrow_schema_to_json(&arrow_schema)?; Ok(DescribeTableResponse { + table: Some(table_name.clone()), + namespace: Some(namespace_id.clone()), version: Some(version as i64), - location: Some(table_uri), + location: Some(table_uri.clone()), + table_uri: Some(table_uri), schema: Some(Box::new(json_schema)), - properties: None, storage_options: self.storage_options.clone(), + stats: None, }) } Err(_) => { // If dataset can't be opened (e.g., empty table), return minimal info Ok(DescribeTableResponse { + table: Some(table_name), + namespace: Some(namespace_id), version: None, - location: Some(table_uri), + location: Some(table_uri.clone()), + table_uri: Some(table_uri), schema: None, - properties: None, storage_options: self.storage_options.clone(), + stats: None, }) } } @@ -1188,21 +1202,6 @@ impl LanceNamespace for ManifestNamespace { }); } - // Validate location if provided - if let Some(location) = &request.location { - let location = location.trim_end_matches('/'); - if location != table_uri { - return Err(Error::Namespace { - source: format!( - "Cannot create table {} at location {}, must be at location {}", - table_name, location, table_uri - ) - .into(), - location: location!(), - }); - } - } - // Write the data using Lance Dataset let cursor = Cursor::new(data.to_vec()); let stream_reader = StreamReader::try_new(cursor, None) @@ -1241,9 +1240,9 @@ impl LanceNamespace for ManifestNamespace { .await?; Ok(CreateTableResponse { + transaction_id: None, version: Some(1), location: Some(table_uri), - properties: None, storage_options: self.storage_options.clone(), }) } @@ -1362,6 +1361,7 @@ impl LanceNamespace for ManifestNamespace { // Root namespace always exists if namespace_id.is_empty() { return Ok(DescribeNamespaceResponse { + id: namespace_id.clone(), properties: Some(HashMap::new()), }); } @@ -1372,6 +1372,7 @@ impl LanceNamespace for ManifestNamespace { match namespace_info { Some(info) => Ok(DescribeNamespaceResponse { + id: namespace_id.clone(), properties: info.metadata, }), None => Err(Error::Namespace { @@ -1431,6 +1432,7 @@ impl LanceNamespace for ManifestNamespace { .await?; Ok(CreateNamespaceResponse { + transaction_id: None, properties: request.properties, }) } @@ -1613,6 +1615,7 @@ impl LanceNamespace for ManifestNamespace { ); Ok(CreateEmptyTableResponse { + transaction_id: None, location: Some(table_uri), properties: None, storage_options: self.storage_options.clone(), @@ -1688,6 +1691,7 @@ impl LanceNamespace for ManifestNamespace { .await?; Ok(RegisterTableResponse { + transaction_id: None, location, properties: None, }) @@ -1732,6 +1736,7 @@ impl LanceNamespace for ManifestNamespace { }; Ok(DeregisterTableResponse { + transaction_id: None, id: request.id.clone(), location: Some(table_uri), properties: None, diff --git a/rust/lance-namespace-impls/src/rest.rs b/rust/lance-namespace-impls/src/rest.rs index 1f7ee341d26..770752011c0 100644 --- a/rust/lance-namespace-impls/src/rest.rs +++ b/rust/lance-namespace-impls/src/rest.rs @@ -9,22 +9,32 @@ use async_trait::async_trait; use bytes::Bytes; use lance_namespace::apis::{ - configuration::Configuration, namespace_api, table_api, transaction_api, + configuration::Configuration, namespace_api, table_api, tag_api, transaction_api, }; use lance_namespace::models::{ - AlterTransactionRequest, AlterTransactionResponse, CountTableRowsRequest, - CreateEmptyTableRequest, CreateEmptyTableResponse, CreateNamespaceRequest, - CreateNamespaceResponse, CreateTableIndexRequest, CreateTableIndexResponse, CreateTableRequest, - CreateTableResponse, DeleteFromTableRequest, DeleteFromTableResponse, DeregisterTableRequest, - DeregisterTableResponse, DescribeNamespaceRequest, DescribeNamespaceResponse, - DescribeTableIndexStatsRequest, DescribeTableIndexStatsResponse, DescribeTableRequest, - DescribeTableResponse, DescribeTransactionRequest, DescribeTransactionResponse, - DropNamespaceRequest, DropNamespaceResponse, DropTableRequest, DropTableResponse, - InsertIntoTableRequest, InsertIntoTableResponse, ListNamespacesRequest, ListNamespacesResponse, - ListTableIndicesRequest, ListTableIndicesResponse, ListTablesRequest, ListTablesResponse, + AlterTableAddColumnsRequest, AlterTableAddColumnsResponse, AlterTableAlterColumnsRequest, + AlterTableAlterColumnsResponse, AlterTableDropColumnsRequest, AlterTableDropColumnsResponse, + AlterTransactionRequest, AlterTransactionResponse, AnalyzeTableQueryPlanRequest, + CountTableRowsRequest, CreateEmptyTableRequest, CreateEmptyTableResponse, + CreateNamespaceRequest, CreateNamespaceResponse, CreateTableIndexRequest, + CreateTableIndexResponse, CreateTableRequest, CreateTableResponse, + CreateTableScalarIndexResponse, CreateTableTagRequest, CreateTableTagResponse, + DeleteFromTableRequest, DeleteFromTableResponse, DeleteTableTagRequest, DeleteTableTagResponse, + DeregisterTableRequest, DeregisterTableResponse, DescribeNamespaceRequest, + DescribeNamespaceResponse, DescribeTableIndexStatsRequest, DescribeTableIndexStatsResponse, + DescribeTableRequest, DescribeTableResponse, DescribeTransactionRequest, + DescribeTransactionResponse, DropNamespaceRequest, DropNamespaceResponse, + DropTableIndexRequest, DropTableIndexResponse, DropTableRequest, DropTableResponse, + ExplainTableQueryPlanRequest, GetTableStatsRequest, GetTableStatsResponse, + GetTableTagVersionRequest, GetTableTagVersionResponse, InsertIntoTableRequest, + InsertIntoTableResponse, ListNamespacesRequest, ListNamespacesResponse, + ListTableIndicesRequest, ListTableIndicesResponse, ListTableTagsRequest, ListTableTagsResponse, + ListTableVersionsRequest, ListTableVersionsResponse, ListTablesRequest, ListTablesResponse, MergeInsertIntoTableRequest, MergeInsertIntoTableResponse, NamespaceExistsRequest, - QueryTableRequest, RegisterTableRequest, RegisterTableResponse, TableExistsRequest, - UpdateTableRequest, UpdateTableResponse, + QueryTableRequest, RegisterTableRequest, RegisterTableResponse, RenameTableRequest, + RenameTableResponse, RestoreTableRequest, RestoreTableResponse, TableExistsRequest, + UpdateTableRequest, UpdateTableResponse, UpdateTableSchemaMetadataRequest, + UpdateTableSchemaMetadataResponse, UpdateTableTagRequest, UpdateTableTagResponse, }; use lance_core::{box_error, Error, Result}; @@ -456,9 +466,15 @@ impl LanceNamespace for RestNamespace { async fn describe_table(&self, request: DescribeTableRequest) -> Result { let id = object_id_str(&request.id, &self.delimiter)?; - table_api::describe_table(&self.reqwest_config, &id, request, Some(&self.delimiter)) - .await - .map_err(convert_api_error) + table_api::describe_table( + &self.reqwest_config, + &id, + request.clone(), + Some(&self.delimiter), + request.with_table_uri, + ) + .await + .map_err(convert_api_error) } async fn register_table(&self, request: RegisterTableRequest) -> Result { @@ -480,7 +496,7 @@ impl LanceNamespace for RestNamespace { async fn drop_table(&self, request: DropTableRequest) -> Result { let id = object_id_str(&request.id, &self.delimiter)?; - table_api::drop_table(&self.reqwest_config, &id, request, Some(&self.delimiter)) + table_api::drop_table(&self.reqwest_config, &id, Some(&self.delimiter)) .await .map_err(convert_api_error) } @@ -511,16 +527,11 @@ impl LanceNamespace for RestNamespace { ) -> Result { let id = object_id_str(&request.id, &self.delimiter)?; - let properties_json = request - .properties - .as_ref() - .map(|props| serde_json::to_string(props).unwrap_or_else(|_| "{}".to_string())); - use lance_namespace::models::create_table_request::Mode; let mode = request.mode.as_ref().map(|m| match m { - Mode::Create => "create", - Mode::ExistOk => "exist_ok", - Mode::Overwrite => "overwrite", + Mode::Create => "Create", + Mode::ExistOk => "ExistOk", + Mode::Overwrite => "Overwrite", }); table_api::create_table( @@ -529,8 +540,6 @@ impl LanceNamespace for RestNamespace { request_data.to_vec(), Some(&self.delimiter), mode, - request.location.as_deref(), - properties_json.as_deref(), ) .await .map_err(convert_api_error) @@ -558,6 +567,7 @@ impl LanceNamespace for RestNamespace { let mode = request.mode.as_ref().map(|m| match m { Mode::Append => "append", Mode::Overwrite => "overwrite", + Mode::Create => "create", }); table_api::insert_into_table( @@ -594,6 +604,8 @@ impl LanceNamespace for RestNamespace { request.when_not_matched_insert_all, request.when_not_matched_by_source_delete, request.when_not_matched_by_source_delete_filt.as_deref(), + request.timeout.as_deref(), + request.use_index, ) .await .map_err(convert_api_error) @@ -710,6 +722,228 @@ impl LanceNamespace for RestNamespace { .map_err(convert_api_error) } + async fn create_table_scalar_index( + &self, + request: CreateTableIndexRequest, + ) -> Result { + let id = object_id_str(&request.id, &self.delimiter)?; + + table_api::create_table_scalar_index( + &self.reqwest_config, + &id, + request, + Some(&self.delimiter), + ) + .await + .map_err(convert_api_error) + } + + async fn drop_table_index( + &self, + request: DropTableIndexRequest, + ) -> Result { + let id = object_id_str(&request.id, &self.delimiter)?; + + let index_name = request.index_name.as_deref().unwrap_or(""); + + table_api::drop_table_index(&self.reqwest_config, &id, index_name, Some(&self.delimiter)) + .await + .map_err(convert_api_error) + } + + async fn list_all_tables(&self, request: ListTablesRequest) -> Result { + table_api::list_all_tables( + &self.reqwest_config, + Some(&self.delimiter), + request.page_token.as_deref(), + request.limit, + ) + .await + .map_err(convert_api_error) + } + + async fn restore_table(&self, request: RestoreTableRequest) -> Result { + let id = object_id_str(&request.id, &self.delimiter)?; + + table_api::restore_table(&self.reqwest_config, &id, request, Some(&self.delimiter)) + .await + .map_err(convert_api_error) + } + + async fn rename_table(&self, request: RenameTableRequest) -> Result { + let id = object_id_str(&request.id, &self.delimiter)?; + + table_api::rename_table(&self.reqwest_config, &id, request, Some(&self.delimiter)) + .await + .map_err(convert_api_error) + } + + async fn list_table_versions( + &self, + request: ListTableVersionsRequest, + ) -> Result { + let id = object_id_str(&request.id, &self.delimiter)?; + + table_api::list_table_versions( + &self.reqwest_config, + &id, + Some(&self.delimiter), + request.page_token.as_deref(), + request.limit, + ) + .await + .map_err(convert_api_error) + } + + async fn update_table_schema_metadata( + &self, + request: UpdateTableSchemaMetadataRequest, + ) -> Result { + let id = object_id_str(&request.id, &self.delimiter)?; + + let metadata = request.metadata.unwrap_or_default(); + + let result = table_api::update_table_schema_metadata( + &self.reqwest_config, + &id, + metadata, + Some(&self.delimiter), + ) + .await + .map_err(convert_api_error)?; + + Ok(UpdateTableSchemaMetadataResponse { + metadata: Some(result), + ..Default::default() + }) + } + + async fn get_table_stats(&self, request: GetTableStatsRequest) -> Result { + let id = object_id_str(&request.id, &self.delimiter)?; + + table_api::get_table_stats(&self.reqwest_config, &id, request, Some(&self.delimiter)) + .await + .map_err(convert_api_error) + } + + async fn explain_table_query_plan( + &self, + request: ExplainTableQueryPlanRequest, + ) -> Result { + let id = object_id_str(&request.id, &self.delimiter)?; + + table_api::explain_table_query_plan(&self.reqwest_config, &id, request, Some(&self.delimiter)) + .await + .map_err(convert_api_error) + } + + async fn analyze_table_query_plan( + &self, + request: AnalyzeTableQueryPlanRequest, + ) -> Result { + let id = object_id_str(&request.id, &self.delimiter)?; + + table_api::analyze_table_query_plan(&self.reqwest_config, &id, request, Some(&self.delimiter)) + .await + .map_err(convert_api_error) + } + + async fn alter_table_add_columns( + &self, + request: AlterTableAddColumnsRequest, + ) -> Result { + let id = object_id_str(&request.id, &self.delimiter)?; + + table_api::alter_table_add_columns(&self.reqwest_config, &id, request, Some(&self.delimiter)) + .await + .map_err(convert_api_error) + } + + async fn alter_table_alter_columns( + &self, + request: AlterTableAlterColumnsRequest, + ) -> Result { + let id = object_id_str(&request.id, &self.delimiter)?; + + table_api::alter_table_alter_columns( + &self.reqwest_config, + &id, + request, + Some(&self.delimiter), + ) + .await + .map_err(convert_api_error) + } + + async fn alter_table_drop_columns( + &self, + request: AlterTableDropColumnsRequest, + ) -> Result { + let id = object_id_str(&request.id, &self.delimiter)?; + + table_api::alter_table_drop_columns(&self.reqwest_config, &id, request, Some(&self.delimiter)) + .await + .map_err(convert_api_error) + } + + async fn list_table_tags(&self, request: ListTableTagsRequest) -> Result { + let id = object_id_str(&request.id, &self.delimiter)?; + + tag_api::list_table_tags( + &self.reqwest_config, + &id, + Some(&self.delimiter), + request.page_token.as_deref(), + request.limit, + ) + .await + .map_err(convert_api_error) + } + + async fn get_table_tag_version( + &self, + request: GetTableTagVersionRequest, + ) -> Result { + let id = object_id_str(&request.id, &self.delimiter)?; + + tag_api::get_table_tag_version(&self.reqwest_config, &id, request, Some(&self.delimiter)) + .await + .map_err(convert_api_error) + } + + async fn create_table_tag( + &self, + request: CreateTableTagRequest, + ) -> Result { + let id = object_id_str(&request.id, &self.delimiter)?; + + tag_api::create_table_tag(&self.reqwest_config, &id, request, Some(&self.delimiter)) + .await + .map_err(convert_api_error) + } + + async fn delete_table_tag( + &self, + request: DeleteTableTagRequest, + ) -> Result { + let id = object_id_str(&request.id, &self.delimiter)?; + + tag_api::delete_table_tag(&self.reqwest_config, &id, request, Some(&self.delimiter)) + .await + .map_err(convert_api_error) + } + + async fn update_table_tag( + &self, + request: UpdateTableTagRequest, + ) -> Result { + let id = object_id_str(&request.id, &self.delimiter)?; + + tag_api::update_table_tag(&self.reqwest_config, &id, request, Some(&self.delimiter)) + .await + .map_err(convert_api_error) + } + fn namespace_id(&self) -> String { format!( "RestNamespace {{ endpoint: {:?}, delimiter: {:?} }}", diff --git a/rust/lance-namespace-impls/src/rest_adapter.rs b/rust/lance-namespace-impls/src/rest_adapter.rs index eae81d73ba6..45b836a3740 100644 --- a/rust/lance-namespace-impls/src/rest_adapter.rs +++ b/rust/lance-namespace-impls/src/rest_adapter.rs @@ -65,14 +65,68 @@ impl RestAdapter { .route("/v1/namespace/:id/drop", post(drop_namespace)) .route("/v1/namespace/:id/exists", post(namespace_exists)) .route("/v1/namespace/:id/table/list", get(list_tables)) - // Table operations + // Table metadata operations .route("/v1/table/:id/register", post(register_table)) .route("/v1/table/:id/describe", post(describe_table)) .route("/v1/table/:id/exists", post(table_exists)) .route("/v1/table/:id/drop", post(drop_table)) .route("/v1/table/:id/deregister", post(deregister_table)) + .route("/v1/table/:id/rename", post(rename_table)) + .route("/v1/table/:id/restore", post(restore_table)) + .route("/v1/table/:id/versions", get(list_table_versions)) + .route("/v1/table/:id/stats", get(get_table_stats)) + // Table data operations .route("/v1/table/:id/create", post(create_table)) .route("/v1/table/:id/create-empty", post(create_empty_table)) + .route("/v1/table/:id/insert", post(insert_into_table)) + .route("/v1/table/:id/merge-insert", post(merge_insert_into_table)) + .route("/v1/table/:id/update", post(update_table)) + .route("/v1/table/:id/delete", post(delete_from_table)) + .route("/v1/table/:id/query", post(query_table)) + .route("/v1/table/:id/count", get(count_table_rows)) + // Index operations + .route("/v1/table/:id/index/create", post(create_table_index)) + .route( + "/v1/table/:id/index/create-scalar", + post(create_table_scalar_index), + ) + .route("/v1/table/:id/index/list", get(list_table_indices)) + .route("/v1/table/:id/index/:index_name/stats", get(describe_table_index_stats)) + .route("/v1/table/:id/index/:index_name/drop", post(drop_table_index)) + // Schema operations + .route("/v1/table/:id/columns/add", post(alter_table_add_columns)) + .route( + "/v1/table/:id/columns/alter", + post(alter_table_alter_columns), + ) + .route( + "/v1/table/:id/columns/drop", + post(alter_table_drop_columns), + ) + .route( + "/v1/table/:id/schema-metadata", + post(update_table_schema_metadata), + ) + // Tag operations + .route("/v1/table/:id/tags", get(list_table_tags)) + .route("/v1/table/:id/tag/:tag_name", get(get_table_tag_version)) + .route("/v1/table/:id/tag/create", post(create_table_tag)) + .route("/v1/table/:id/tag/:tag_name/delete", post(delete_table_tag)) + .route("/v1/table/:id/tag/:tag_name/update", post(update_table_tag)) + // Query plan operations + .route("/v1/table/:id/query/explain", post(explain_table_query_plan)) + .route("/v1/table/:id/query/analyze", post(analyze_table_query_plan)) + // Transaction operations + .route( + "/v1/table/:id/transaction/:txn_id/describe", + get(describe_transaction), + ) + .route( + "/v1/table/:id/transaction/:txn_id/alter", + post(alter_transaction), + ) + // Global table operations + .route("/v1/tables", get(list_all_tables)) .layer(TraceLayer::new_for_http()) .with_state(self.backend.clone()) } @@ -427,8 +481,6 @@ async fn deregister_table( struct CreateTableQuery { delimiter: Option, mode: Option, - location: Option, - properties: Option, } async fn create_table( @@ -440,22 +492,15 @@ async fn create_table( use lance_namespace::models::create_table_request::Mode; let mode = params.mode.as_deref().and_then(|m| match m { - "create" => Some(Mode::Create), - "exist_ok" => Some(Mode::ExistOk), - "overwrite" => Some(Mode::Overwrite), + "Create" => Some(Mode::Create), + "ExistOk" => Some(Mode::ExistOk), + "Overwrite" => Some(Mode::Overwrite), _ => None, }); - let properties = params - .properties - .as_ref() - .and_then(|p| serde_json::from_str(p).ok()); - let request = CreateTableRequest { id: Some(parse_id(&id, params.delimiter.as_deref())), - location: params.location, mode, - properties, }; match backend.create_table(request, body).await { @@ -478,6 +523,519 @@ async fn create_empty_table( } } +#[derive(Debug, Deserialize)] +struct InsertQuery { + delimiter: Option, + mode: Option, +} + +async fn insert_into_table( + State(backend): State>, + Path(id): Path, + Query(params): Query, + body: Bytes, +) -> Response { + use lance_namespace::models::insert_into_table_request::Mode; + + let mode = params.mode.as_deref().and_then(|m| match m { + "append" => Some(Mode::Append), + "overwrite" => Some(Mode::Overwrite), + "create" => Some(Mode::Create), + _ => None, + }); + + let request = InsertIntoTableRequest { + id: Some(parse_id(&id, params.delimiter.as_deref())), + mode, + }; + + match backend.insert_into_table(request, body).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +#[derive(Debug, Deserialize)] +struct MergeInsertQuery { + delimiter: Option, + on: Option, + when_matched_update_all: Option, + when_matched_update_all_filt: Option, + when_not_matched_insert_all: Option, + when_not_matched_by_source_delete: Option, + when_not_matched_by_source_delete_filt: Option, + timeout: Option, + use_index: Option, +} + +async fn merge_insert_into_table( + State(backend): State>, + Path(id): Path, + Query(params): Query, + body: Bytes, +) -> Response { + let request = MergeInsertIntoTableRequest { + id: Some(parse_id(&id, params.delimiter.as_deref())), + on: params.on, + when_matched_update_all: params.when_matched_update_all, + when_matched_update_all_filt: params.when_matched_update_all_filt, + when_not_matched_insert_all: params.when_not_matched_insert_all, + when_not_matched_by_source_delete: params.when_not_matched_by_source_delete, + when_not_matched_by_source_delete_filt: params.when_not_matched_by_source_delete_filt, + timeout: params.timeout, + use_index: params.use_index, + }; + + match backend.merge_insert_into_table(request, body).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn update_table( + State(backend): State>, + Path(id): Path, + Query(params): Query, + Json(mut request): Json, +) -> Response { + request.id = Some(parse_id(&id, params.delimiter.as_deref())); + + match backend.update_table(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn delete_from_table( + State(backend): State>, + Path(id): Path, + Query(params): Query, + Json(mut request): Json, +) -> Response { + request.id = Some(parse_id(&id, params.delimiter.as_deref())); + + match backend.delete_from_table(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn query_table( + State(backend): State>, + Path(id): Path, + Query(params): Query, + Json(mut request): Json, +) -> Response { + request.id = Some(parse_id(&id, params.delimiter.as_deref())); + + match backend.query_table(request).await { + Ok(bytes) => (StatusCode::OK, bytes).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn count_table_rows( + State(backend): State>, + Path(id): Path, + Query(params): Query, +) -> Response { + let request = CountTableRowsRequest { + id: Some(parse_id(&id, params.delimiter.as_deref())), + filter: None, + }; + + match backend.count_table_rows(request).await { + Ok(count) => (StatusCode::OK, Json(serde_json::json!({ "count": count }))).into_response(), + Err(e) => error_to_response(e), + } +} + +// ============================================================================ +// Table Management Operation Handlers +// ============================================================================ + +async fn rename_table( + State(backend): State>, + Path(id): Path, + Query(params): Query, + Json(mut request): Json, +) -> Response { + request.id = Some(parse_id(&id, params.delimiter.as_deref())); + + match backend.rename_table(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn restore_table( + State(backend): State>, + Path(id): Path, + Query(params): Query, + Json(mut request): Json, +) -> Response { + request.id = Some(parse_id(&id, params.delimiter.as_deref())); + + match backend.restore_table(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn list_table_versions( + State(backend): State>, + Path(id): Path, + Query(params): Query, +) -> Response { + let request = ListTableVersionsRequest { + id: Some(parse_id(&id, params.delimiter.as_deref())), + page_token: params.page_token, + limit: params.limit, + }; + + match backend.list_table_versions(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn get_table_stats( + State(backend): State>, + Path(id): Path, + Query(params): Query, +) -> Response { + let request = GetTableStatsRequest { + id: Some(parse_id(&id, params.delimiter.as_deref())), + }; + + match backend.get_table_stats(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn list_all_tables( + State(backend): State>, + Query(params): Query, +) -> Response { + let request = ListTablesRequest { + id: None, + page_token: params.page_token, + limit: params.limit, + }; + + match backend.list_all_tables(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +// ============================================================================ +// Index Operation Handlers +// ============================================================================ + +async fn create_table_index( + State(backend): State>, + Path(id): Path, + Query(params): Query, + Json(mut request): Json, +) -> Response { + request.id = Some(parse_id(&id, params.delimiter.as_deref())); + + match backend.create_table_index(request).await { + Ok(response) => (StatusCode::CREATED, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn create_table_scalar_index( + State(backend): State>, + Path(id): Path, + Query(params): Query, + Json(mut request): Json, +) -> Response { + request.id = Some(parse_id(&id, params.delimiter.as_deref())); + + match backend.create_table_scalar_index(request).await { + Ok(response) => (StatusCode::CREATED, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn list_table_indices( + State(backend): State>, + Path(id): Path, + Query(params): Query, +) -> Response { + let request = ListTableIndicesRequest { + id: Some(parse_id(&id, params.delimiter.as_deref())), + }; + + match backend.list_table_indices(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +#[derive(Debug, Deserialize)] +struct IndexPathParams { + id: String, + index_name: String, +} + +async fn describe_table_index_stats( + State(backend): State>, + Path(params): Path, + Query(query): Query, +) -> Response { + let request = DescribeTableIndexStatsRequest { + id: Some(parse_id(¶ms.id, query.delimiter.as_deref())), + index_name: Some(params.index_name), + }; + + match backend.describe_table_index_stats(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn drop_table_index( + State(backend): State>, + Path(params): Path, + Query(query): Query, +) -> Response { + let request = DropTableIndexRequest { + id: Some(parse_id(¶ms.id, query.delimiter.as_deref())), + index_name: Some(params.index_name), + }; + + match backend.drop_table_index(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +// ============================================================================ +// Schema Operation Handlers +// ============================================================================ + +async fn alter_table_add_columns( + State(backend): State>, + Path(id): Path, + Query(params): Query, + Json(mut request): Json, +) -> Response { + request.id = Some(parse_id(&id, params.delimiter.as_deref())); + + match backend.alter_table_add_columns(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn alter_table_alter_columns( + State(backend): State>, + Path(id): Path, + Query(params): Query, + Json(mut request): Json, +) -> Response { + request.id = Some(parse_id(&id, params.delimiter.as_deref())); + + match backend.alter_table_alter_columns(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn alter_table_drop_columns( + State(backend): State>, + Path(id): Path, + Query(params): Query, + Json(mut request): Json, +) -> Response { + request.id = Some(parse_id(&id, params.delimiter.as_deref())); + + match backend.alter_table_drop_columns(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn update_table_schema_metadata( + State(backend): State>, + Path(id): Path, + Query(params): Query, + Json(mut request): Json, +) -> Response { + request.id = Some(parse_id(&id, params.delimiter.as_deref())); + + match backend.update_table_schema_metadata(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +// ============================================================================ +// Tag Operation Handlers +// ============================================================================ + +async fn list_table_tags( + State(backend): State>, + Path(id): Path, + Query(params): Query, +) -> Response { + let request = ListTableTagsRequest { + id: Some(parse_id(&id, params.delimiter.as_deref())), + page_token: params.page_token, + limit: params.limit, + }; + + match backend.list_table_tags(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +#[derive(Debug, Deserialize)] +struct TagPathParams { + id: String, + tag_name: String, +} + +async fn get_table_tag_version( + State(backend): State>, + Path(params): Path, + Query(query): Query, +) -> Response { + let request = GetTableTagVersionRequest { + id: Some(parse_id(¶ms.id, query.delimiter.as_deref())), + tag: Some(params.tag_name), + }; + + match backend.get_table_tag_version(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn create_table_tag( + State(backend): State>, + Path(id): Path, + Query(params): Query, + Json(mut request): Json, +) -> Response { + request.id = Some(parse_id(&id, params.delimiter.as_deref())); + + match backend.create_table_tag(request).await { + Ok(response) => (StatusCode::CREATED, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn delete_table_tag( + State(backend): State>, + Path(params): Path, + Query(query): Query, +) -> Response { + let request = DeleteTableTagRequest { + id: Some(parse_id(¶ms.id, query.delimiter.as_deref())), + tag: Some(params.tag_name), + }; + + match backend.delete_table_tag(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn update_table_tag( + State(backend): State>, + Path(params): Path, + Query(query): Query, + Json(mut request): Json, +) -> Response { + request.id = Some(parse_id(¶ms.id, query.delimiter.as_deref())); + request.tag = Some(params.tag_name); + + match backend.update_table_tag(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +// ============================================================================ +// Query Plan Operation Handlers +// ============================================================================ + +async fn explain_table_query_plan( + State(backend): State>, + Path(id): Path, + Query(params): Query, + Json(mut request): Json, +) -> Response { + request.id = Some(parse_id(&id, params.delimiter.as_deref())); + + match backend.explain_table_query_plan(request).await { + Ok(plan) => (StatusCode::OK, plan).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn analyze_table_query_plan( + State(backend): State>, + Path(id): Path, + Query(params): Query, + Json(mut request): Json, +) -> Response { + request.id = Some(parse_id(&id, params.delimiter.as_deref())); + + match backend.analyze_table_query_plan(request).await { + Ok(plan) => (StatusCode::OK, plan).into_response(), + Err(e) => error_to_response(e), + } +} + +// ============================================================================ +// Transaction Operation Handlers +// ============================================================================ + +#[derive(Debug, Deserialize)] +struct TransactionPathParams { + id: String, + txn_id: String, +} + +async fn describe_transaction( + State(backend): State>, + Path(params): Path, + Query(query): Query, +) -> Response { + let request = DescribeTransactionRequest { + id: Some(parse_id(¶ms.id, query.delimiter.as_deref())), + transaction_id: Some(params.txn_id), + }; + + match backend.describe_transaction(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn alter_transaction( + State(backend): State>, + Path(params): Path, + Query(query): Query, + Json(mut request): Json, +) -> Response { + request.id = Some(parse_id(¶ms.id, query.delimiter.as_deref())); + request.transaction_id = Some(params.txn_id); + + match backend.alter_transaction(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + // ============================================================================ // Helper Functions // ============================================================================ diff --git a/rust/lance-namespace/src/namespace.rs b/rust/lance-namespace/src/namespace.rs index ac2d0c8e176..0a13ae17597 100644 --- a/rust/lance-namespace/src/namespace.rs +++ b/rust/lance-namespace/src/namespace.rs @@ -9,19 +9,29 @@ use lance_core::{Error, Result}; use snafu::Location; use lance_namespace_reqwest_client::models::{ - AlterTransactionRequest, AlterTransactionResponse, CountTableRowsRequest, - CreateEmptyTableRequest, CreateEmptyTableResponse, CreateNamespaceRequest, - CreateNamespaceResponse, CreateTableIndexRequest, CreateTableIndexResponse, CreateTableRequest, - CreateTableResponse, DeleteFromTableRequest, DeleteFromTableResponse, DeregisterTableRequest, - DeregisterTableResponse, DescribeNamespaceRequest, DescribeNamespaceResponse, - DescribeTableIndexStatsRequest, DescribeTableIndexStatsResponse, DescribeTableRequest, - DescribeTableResponse, DescribeTransactionRequest, DescribeTransactionResponse, - DropNamespaceRequest, DropNamespaceResponse, DropTableRequest, DropTableResponse, - InsertIntoTableRequest, InsertIntoTableResponse, ListNamespacesRequest, ListNamespacesResponse, - ListTableIndicesRequest, ListTableIndicesResponse, ListTablesRequest, ListTablesResponse, + AlterTableAddColumnsRequest, AlterTableAddColumnsResponse, AlterTableAlterColumnsRequest, + AlterTableAlterColumnsResponse, AlterTableDropColumnsRequest, AlterTableDropColumnsResponse, + AlterTransactionRequest, AlterTransactionResponse, AnalyzeTableQueryPlanRequest, + CountTableRowsRequest, CreateEmptyTableRequest, CreateEmptyTableResponse, + CreateNamespaceRequest, CreateNamespaceResponse, CreateTableIndexRequest, + CreateTableIndexResponse, CreateTableRequest, CreateTableResponse, + CreateTableScalarIndexResponse, CreateTableTagRequest, CreateTableTagResponse, + DeleteFromTableRequest, DeleteFromTableResponse, DeleteTableTagRequest, DeleteTableTagResponse, + DeregisterTableRequest, DeregisterTableResponse, DescribeNamespaceRequest, + DescribeNamespaceResponse, DescribeTableIndexStatsRequest, DescribeTableIndexStatsResponse, + DescribeTableRequest, DescribeTableResponse, DescribeTransactionRequest, + DescribeTransactionResponse, DropNamespaceRequest, DropNamespaceResponse, + DropTableIndexRequest, DropTableIndexResponse, DropTableRequest, DropTableResponse, + ExplainTableQueryPlanRequest, GetTableStatsRequest, GetTableStatsResponse, + GetTableTagVersionRequest, GetTableTagVersionResponse, InsertIntoTableRequest, + InsertIntoTableResponse, ListNamespacesRequest, ListNamespacesResponse, + ListTableIndicesRequest, ListTableIndicesResponse, ListTableTagsRequest, ListTableTagsResponse, + ListTableVersionsRequest, ListTableVersionsResponse, ListTablesRequest, ListTablesResponse, MergeInsertIntoTableRequest, MergeInsertIntoTableResponse, NamespaceExistsRequest, - QueryTableRequest, RegisterTableRequest, RegisterTableResponse, TableExistsRequest, - UpdateTableRequest, UpdateTableResponse, + QueryTableRequest, RegisterTableRequest, RegisterTableResponse, RenameTableRequest, + RenameTableResponse, RestoreTableRequest, RestoreTableResponse, TableExistsRequest, + UpdateTableRequest, UpdateTableResponse, UpdateTableSchemaMetadataRequest, + UpdateTableSchemaMetadataResponse, UpdateTableTagRequest, UpdateTableTagResponse, }; /// Base trait for Lance Namespace implementations. @@ -277,6 +287,201 @@ pub trait LanceNamespace: Send + Sync + std::fmt::Debug { }) } + /// Create a scalar index on a table. + async fn create_table_scalar_index( + &self, + _request: CreateTableIndexRequest, + ) -> Result { + Err(Error::NotSupported { + source: "create_table_scalar_index not implemented".into(), + location: Location::new(file!(), line!(), column!()), + }) + } + + /// Drop a table index. + async fn drop_table_index( + &self, + _request: DropTableIndexRequest, + ) -> Result { + Err(Error::NotSupported { + source: "drop_table_index not implemented".into(), + location: Location::new(file!(), line!(), column!()), + }) + } + + /// List all tables across all namespaces. + async fn list_all_tables(&self, _request: ListTablesRequest) -> Result { + Err(Error::NotSupported { + source: "list_all_tables not implemented".into(), + location: Location::new(file!(), line!(), column!()), + }) + } + + /// Restore a table to a specific version. + async fn restore_table( + &self, + _request: RestoreTableRequest, + ) -> Result { + Err(Error::NotSupported { + source: "restore_table not implemented".into(), + location: Location::new(file!(), line!(), column!()), + }) + } + + /// Rename a table. + async fn rename_table( + &self, + _request: RenameTableRequest, + ) -> Result { + Err(Error::NotSupported { + source: "rename_table not implemented".into(), + location: Location::new(file!(), line!(), column!()), + }) + } + + /// List all versions of a table. + async fn list_table_versions( + &self, + _request: ListTableVersionsRequest, + ) -> Result { + Err(Error::NotSupported { + source: "list_table_versions not implemented".into(), + location: Location::new(file!(), line!(), column!()), + }) + } + + /// Update table schema metadata. + async fn update_table_schema_metadata( + &self, + _request: UpdateTableSchemaMetadataRequest, + ) -> Result { + Err(Error::NotSupported { + source: "update_table_schema_metadata not implemented".into(), + location: Location::new(file!(), line!(), column!()), + }) + } + + /// Get table statistics. + async fn get_table_stats( + &self, + _request: GetTableStatsRequest, + ) -> Result { + Err(Error::NotSupported { + source: "get_table_stats not implemented".into(), + location: Location::new(file!(), line!(), column!()), + }) + } + + /// Explain a table query plan. + async fn explain_table_query_plan( + &self, + _request: ExplainTableQueryPlanRequest, + ) -> Result { + Err(Error::NotSupported { + source: "explain_table_query_plan not implemented".into(), + location: Location::new(file!(), line!(), column!()), + }) + } + + /// Analyze a table query plan. + async fn analyze_table_query_plan( + &self, + _request: AnalyzeTableQueryPlanRequest, + ) -> Result { + Err(Error::NotSupported { + source: "analyze_table_query_plan not implemented".into(), + location: Location::new(file!(), line!(), column!()), + }) + } + + /// Add columns to a table. + async fn alter_table_add_columns( + &self, + _request: AlterTableAddColumnsRequest, + ) -> Result { + Err(Error::NotSupported { + source: "alter_table_add_columns not implemented".into(), + location: Location::new(file!(), line!(), column!()), + }) + } + + /// Alter columns in a table. + async fn alter_table_alter_columns( + &self, + _request: AlterTableAlterColumnsRequest, + ) -> Result { + Err(Error::NotSupported { + source: "alter_table_alter_columns not implemented".into(), + location: Location::new(file!(), line!(), column!()), + }) + } + + /// Drop columns from a table. + async fn alter_table_drop_columns( + &self, + _request: AlterTableDropColumnsRequest, + ) -> Result { + Err(Error::NotSupported { + source: "alter_table_drop_columns not implemented".into(), + location: Location::new(file!(), line!(), column!()), + }) + } + + /// List all tags for a table. + async fn list_table_tags( + &self, + _request: ListTableTagsRequest, + ) -> Result { + Err(Error::NotSupported { + source: "list_table_tags not implemented".into(), + location: Location::new(file!(), line!(), column!()), + }) + } + + /// Get the version for a specific tag. + async fn get_table_tag_version( + &self, + _request: GetTableTagVersionRequest, + ) -> Result { + Err(Error::NotSupported { + source: "get_table_tag_version not implemented".into(), + location: Location::new(file!(), line!(), column!()), + }) + } + + /// Create a tag for a table. + async fn create_table_tag( + &self, + _request: CreateTableTagRequest, + ) -> Result { + Err(Error::NotSupported { + source: "create_table_tag not implemented".into(), + location: Location::new(file!(), line!(), column!()), + }) + } + + /// Delete a tag from a table. + async fn delete_table_tag( + &self, + _request: DeleteTableTagRequest, + ) -> Result { + Err(Error::NotSupported { + source: "delete_table_tag not implemented".into(), + location: Location::new(file!(), line!(), column!()), + }) + } + + /// Update a tag for a table. + async fn update_table_tag( + &self, + _request: UpdateTableTagRequest, + ) -> Result { + Err(Error::NotSupported { + source: "update_table_tag not implemented".into(), + location: Location::new(file!(), line!(), column!()), + }) + } + /// Return a human-readable unique identifier for this namespace instance. /// /// This is used for equality comparison and hashing when the namespace is diff --git a/rust/lance/src/dataset.rs b/rust/lance/src/dataset.rs index 722ba7c97e1..1079a72d600 100644 --- a/rust/lance/src/dataset.rs +++ b/rust/lance/src/dataset.rs @@ -876,6 +876,7 @@ impl Dataset { let request = DescribeTableRequest { id: Some(table_id.clone()), version: None, + with_table_uri: None, }; let response = namespace diff --git a/rust/lance/src/dataset/builder.rs b/rust/lance/src/dataset/builder.rs index 16326630d23..332ba504cf9 100644 --- a/rust/lance/src/dataset/builder.rs +++ b/rust/lance/src/dataset/builder.rs @@ -137,6 +137,7 @@ impl DatasetBuilder { let request = DescribeTableRequest { id: Some(table_id.clone()), version: None, + with_table_uri: None, }; let response = namespace From 1fecf2e000d00e02a36821d663c768bb053cdbd5 Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Wed, 10 Dec 2025 23:09:50 -0800 Subject: [PATCH 2/6] update again --- Cargo.lock | 4 +- Cargo.toml | 2 +- java/pom.xml | 4 +- python/pyproject.toml | 2 +- rust/lance-namespace-impls/src/dir.rs | 3 +- .../lance-namespace-impls/src/dir/manifest.rs | 4 +- rust/lance-namespace-impls/src/rest.rs | 78 ++++--- .../lance-namespace-impls/src/rest_adapter.rs | 197 +++++++----------- 8 files changed, 125 insertions(+), 169 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9ffd5ba10c4..4ea0a23f554 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4919,9 +4919,9 @@ dependencies = [ [[package]] name = "lance-namespace-reqwest-client" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28bb5310d4f8f197a60dc2878ae48e4cc9ebd20e1fc519302651ed8dbb62a458" +checksum = "b748e89a3a0e5d9fb1b51e4382f783f8aa6b620d755012d4856180968014e619" dependencies = [ "reqwest", "serde", diff --git a/Cargo.toml b/Cargo.toml index f60d28341fc..faa2f6d5f8f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -63,7 +63,7 @@ lance-io = { version = "=1.1.0-beta.2", path = "./rust/lance-io", default-featur lance-linalg = { version = "=1.1.0-beta.2", path = "./rust/lance-linalg" } lance-namespace = { version = "=1.1.0-beta.2", path = "./rust/lance-namespace" } lance-namespace-impls = { version = "=1.1.0-beta.2", path = "./rust/lance-namespace-impls" } -lance-namespace-reqwest-client = "0.3.0" +lance-namespace-reqwest-client = "0.3.1" lance-table = { version = "=1.1.0-beta.2", path = "./rust/lance-table" } lance-test-macros = { version = "=1.1.0-beta.2", path = "./rust/lance-test-macros" } lance-testing = { version = "=1.1.0-beta.2", path = "./rust/lance-testing" } diff --git a/java/pom.xml b/java/pom.xml index 2b0f29167e5..d68a6ca3ee2 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -108,12 +108,12 @@ org.lance lance-namespace-core - 0.3.0 + 0.3.1 org.lance lance-namespace-apache-client - 0.3.0 + 0.3.1 com.fasterxml.jackson.core diff --git a/python/pyproject.toml b/python/pyproject.toml index d0b48f96165..bffb76c33d7 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "pylance" dynamic = ["version"] -dependencies = ["pyarrow>=14", "numpy>=1.22", "lance-namespace>=0.3.0"] +dependencies = ["pyarrow>=14", "numpy>=1.22", "lance-namespace>=0.3.1"] description = "python wrapper for Lance columnar format" authors = [{ name = "Lance Devs", email = "dev@lance.org" }] license = { file = "LICENSE" } diff --git a/rust/lance-namespace-impls/src/dir.rs b/rust/lance-namespace-impls/src/dir.rs index b0bd52f7ef4..fdb4370f6ab 100644 --- a/rust/lance-namespace-impls/src/dir.rs +++ b/rust/lance-namespace-impls/src/dir.rs @@ -602,7 +602,6 @@ impl LanceNamespace for DirectoryNamespace { Self::validate_root_namespace_id(&request.id)?; Ok(DescribeNamespaceResponse { - id: request.id.unwrap_or_default(), properties: Some(HashMap::new()), }) } @@ -2343,7 +2342,7 @@ mod tests { register_req.id = Some(vec!["registered_table".to_string()]); let response = namespace.register_table(register_req).await.unwrap(); - assert_eq!(response.location, "external_table.lance"); + assert_eq!(response.location, Some("external_table.lance".to_string())); // Verify table exists in namespace let mut exists_req = TableExistsRequest::new(); diff --git a/rust/lance-namespace-impls/src/dir/manifest.rs b/rust/lance-namespace-impls/src/dir/manifest.rs index e004a3dee14..bc45c084307 100644 --- a/rust/lance-namespace-impls/src/dir/manifest.rs +++ b/rust/lance-namespace-impls/src/dir/manifest.rs @@ -1361,7 +1361,6 @@ impl LanceNamespace for ManifestNamespace { // Root namespace always exists if namespace_id.is_empty() { return Ok(DescribeNamespaceResponse { - id: namespace_id.clone(), properties: Some(HashMap::new()), }); } @@ -1372,7 +1371,6 @@ impl LanceNamespace for ManifestNamespace { match namespace_info { Some(info) => Ok(DescribeNamespaceResponse { - id: namespace_id.clone(), properties: info.metadata, }), None => Err(Error::Namespace { @@ -1692,7 +1690,7 @@ impl LanceNamespace for ManifestNamespace { Ok(RegisterTableResponse { transaction_id: None, - location, + location: Some(location), properties: None, }) } diff --git a/rust/lance-namespace-impls/src/rest.rs b/rust/lance-namespace-impls/src/rest.rs index 770752011c0..11b05b29eb5 100644 --- a/rust/lance-namespace-impls/src/rest.rs +++ b/rust/lance-namespace-impls/src/rest.rs @@ -527,19 +527,12 @@ impl LanceNamespace for RestNamespace { ) -> Result { let id = object_id_str(&request.id, &self.delimiter)?; - use lance_namespace::models::create_table_request::Mode; - let mode = request.mode.as_ref().map(|m| match m { - Mode::Create => "Create", - Mode::ExistOk => "ExistOk", - Mode::Overwrite => "Overwrite", - }); - table_api::create_table( &self.reqwest_config, &id, request_data.to_vec(), Some(&self.delimiter), - mode, + request.mode.as_deref(), ) .await .map_err(convert_api_error) @@ -563,19 +556,12 @@ impl LanceNamespace for RestNamespace { ) -> Result { let id = object_id_str(&request.id, &self.delimiter)?; - use lance_namespace::models::insert_into_table_request::Mode; - let mode = request.mode.as_ref().map(|m| match m { - Mode::Append => "append", - Mode::Overwrite => "overwrite", - Mode::Create => "create", - }); - table_api::insert_into_table( &self.reqwest_config, &id, request_data.to_vec(), Some(&self.delimiter), - mode, + request.mode.as_deref(), ) .await .map_err(convert_api_error) @@ -818,7 +804,10 @@ impl LanceNamespace for RestNamespace { }) } - async fn get_table_stats(&self, request: GetTableStatsRequest) -> Result { + async fn get_table_stats( + &self, + request: GetTableStatsRequest, + ) -> Result { let id = object_id_str(&request.id, &self.delimiter)?; table_api::get_table_stats(&self.reqwest_config, &id, request, Some(&self.delimiter)) @@ -832,9 +821,14 @@ impl LanceNamespace for RestNamespace { ) -> Result { let id = object_id_str(&request.id, &self.delimiter)?; - table_api::explain_table_query_plan(&self.reqwest_config, &id, request, Some(&self.delimiter)) - .await - .map_err(convert_api_error) + table_api::explain_table_query_plan( + &self.reqwest_config, + &id, + request, + Some(&self.delimiter), + ) + .await + .map_err(convert_api_error) } async fn analyze_table_query_plan( @@ -843,9 +837,14 @@ impl LanceNamespace for RestNamespace { ) -> Result { let id = object_id_str(&request.id, &self.delimiter)?; - table_api::analyze_table_query_plan(&self.reqwest_config, &id, request, Some(&self.delimiter)) - .await - .map_err(convert_api_error) + table_api::analyze_table_query_plan( + &self.reqwest_config, + &id, + request, + Some(&self.delimiter), + ) + .await + .map_err(convert_api_error) } async fn alter_table_add_columns( @@ -854,9 +853,14 @@ impl LanceNamespace for RestNamespace { ) -> Result { let id = object_id_str(&request.id, &self.delimiter)?; - table_api::alter_table_add_columns(&self.reqwest_config, &id, request, Some(&self.delimiter)) - .await - .map_err(convert_api_error) + table_api::alter_table_add_columns( + &self.reqwest_config, + &id, + request, + Some(&self.delimiter), + ) + .await + .map_err(convert_api_error) } async fn alter_table_alter_columns( @@ -881,12 +885,20 @@ impl LanceNamespace for RestNamespace { ) -> Result { let id = object_id_str(&request.id, &self.delimiter)?; - table_api::alter_table_drop_columns(&self.reqwest_config, &id, request, Some(&self.delimiter)) - .await - .map_err(convert_api_error) + table_api::alter_table_drop_columns( + &self.reqwest_config, + &id, + request, + Some(&self.delimiter), + ) + .await + .map_err(convert_api_error) } - async fn list_table_tags(&self, request: ListTableTagsRequest) -> Result { + async fn list_table_tags( + &self, + request: ListTableTagsRequest, + ) -> Result { let id = object_id_str(&request.id, &self.delimiter)?; tag_api::list_table_tags( @@ -1257,9 +1269,7 @@ mod tests { "namespace".to_string(), "table".to_string(), ]), - location: None, - mode: Some(create_table_request::Mode::Create), - properties: None, + mode: Some("Create".to_string()), }; let data = Bytes::from("arrow data here"); @@ -1296,7 +1306,7 @@ mod tests { "namespace".to_string(), "table".to_string(), ]), - mode: Some(insert_into_table_request::Mode::Append), + mode: Some("Append".to_string()), }; let data = Bytes::from("arrow data here"); diff --git a/rust/lance-namespace-impls/src/rest_adapter.rs b/rust/lance-namespace-impls/src/rest_adapter.rs index 45b836a3740..06d104ece96 100644 --- a/rust/lance-namespace-impls/src/rest_adapter.rs +++ b/rust/lance-namespace-impls/src/rest_adapter.rs @@ -73,60 +73,57 @@ impl RestAdapter { .route("/v1/table/:id/deregister", post(deregister_table)) .route("/v1/table/:id/rename", post(rename_table)) .route("/v1/table/:id/restore", post(restore_table)) - .route("/v1/table/:id/versions", get(list_table_versions)) + .route("/v1/table/:id/version/list", get(list_table_versions)) .route("/v1/table/:id/stats", get(get_table_stats)) // Table data operations .route("/v1/table/:id/create", post(create_table)) .route("/v1/table/:id/create-empty", post(create_empty_table)) .route("/v1/table/:id/insert", post(insert_into_table)) - .route("/v1/table/:id/merge-insert", post(merge_insert_into_table)) + .route("/v1/table/:id/merge_insert", post(merge_insert_into_table)) .route("/v1/table/:id/update", post(update_table)) .route("/v1/table/:id/delete", post(delete_from_table)) .route("/v1/table/:id/query", post(query_table)) - .route("/v1/table/:id/count", get(count_table_rows)) + .route("/v1/table/:id/count_rows", get(count_table_rows)) // Index operations - .route("/v1/table/:id/index/create", post(create_table_index)) + .route("/v1/table/:id/create_index", post(create_table_index)) .route( - "/v1/table/:id/index/create-scalar", + "/v1/table/:id/create_scalar_index", post(create_table_scalar_index), ) .route("/v1/table/:id/index/list", get(list_table_indices)) - .route("/v1/table/:id/index/:index_name/stats", get(describe_table_index_stats)) - .route("/v1/table/:id/index/:index_name/drop", post(drop_table_index)) - // Schema operations - .route("/v1/table/:id/columns/add", post(alter_table_add_columns)) .route( - "/v1/table/:id/columns/alter", - post(alter_table_alter_columns), + "/v1/table/:id/index/:index_name/stats", + get(describe_table_index_stats), + ) + .route( + "/v1/table/:id/index/:index_name/drop", + post(drop_table_index), ) + // Schema operations + .route("/v1/table/:id/add_columns", post(alter_table_add_columns)) .route( - "/v1/table/:id/columns/drop", - post(alter_table_drop_columns), + "/v1/table/:id/alter_columns", + post(alter_table_alter_columns), ) + .route("/v1/table/:id/drop_columns", post(alter_table_drop_columns)) .route( - "/v1/table/:id/schema-metadata", + "/v1/table/:id/schema_metadata/update", post(update_table_schema_metadata), ) // Tag operations - .route("/v1/table/:id/tags", get(list_table_tags)) - .route("/v1/table/:id/tag/:tag_name", get(get_table_tag_version)) - .route("/v1/table/:id/tag/create", post(create_table_tag)) - .route("/v1/table/:id/tag/:tag_name/delete", post(delete_table_tag)) - .route("/v1/table/:id/tag/:tag_name/update", post(update_table_tag)) + .route("/v1/table/:id/tags/list", get(list_table_tags)) + .route("/v1/table/:id/tags/version", post(get_table_tag_version)) + .route("/v1/table/:id/tags/create", post(create_table_tag)) + .route("/v1/table/:id/tags/delete", post(delete_table_tag)) + .route("/v1/table/:id/tags/update", post(update_table_tag)) // Query plan operations - .route("/v1/table/:id/query/explain", post(explain_table_query_plan)) - .route("/v1/table/:id/query/analyze", post(analyze_table_query_plan)) + .route("/v1/table/:id/explain_plan", post(explain_table_query_plan)) + .route("/v1/table/:id/analyze_plan", post(analyze_table_query_plan)) // Transaction operations - .route( - "/v1/table/:id/transaction/:txn_id/describe", - get(describe_transaction), - ) - .route( - "/v1/table/:id/transaction/:txn_id/alter", - post(alter_transaction), - ) + .route("/v1/transaction/:id/describe", post(describe_transaction)) + .route("/v1/transaction/:id/alter", post(alter_transaction)) // Global table operations - .route("/v1/tables", get(list_all_tables)) + .route("/v1/table", get(list_all_tables)) .layer(TraceLayer::new_for_http()) .with_state(self.backend.clone()) } @@ -489,18 +486,9 @@ async fn create_table( Query(params): Query, body: Bytes, ) -> Response { - use lance_namespace::models::create_table_request::Mode; - - let mode = params.mode.as_deref().and_then(|m| match m { - "Create" => Some(Mode::Create), - "ExistOk" => Some(Mode::ExistOk), - "Overwrite" => Some(Mode::Overwrite), - _ => None, - }); - let request = CreateTableRequest { id: Some(parse_id(&id, params.delimiter.as_deref())), - mode, + mode: params.mode.clone(), }; match backend.create_table(request, body).await { @@ -535,18 +523,9 @@ async fn insert_into_table( Query(params): Query, body: Bytes, ) -> Response { - use lance_namespace::models::insert_into_table_request::Mode; - - let mode = params.mode.as_deref().and_then(|m| match m { - "append" => Some(Mode::Append), - "overwrite" => Some(Mode::Overwrite), - "create" => Some(Mode::Create), - _ => None, - }); - let request = InsertIntoTableRequest { id: Some(parse_id(&id, params.delimiter.as_deref())), - mode, + mode: params.mode.clone(), }; match backend.insert_into_table(request, body).await { @@ -896,21 +875,13 @@ async fn list_table_tags( } } -#[derive(Debug, Deserialize)] -struct TagPathParams { - id: String, - tag_name: String, -} - async fn get_table_tag_version( State(backend): State>, - Path(params): Path, - Query(query): Query, + Path(id): Path, + Query(params): Query, + Json(mut request): Json, ) -> Response { - let request = GetTableTagVersionRequest { - id: Some(parse_id(¶ms.id, query.delimiter.as_deref())), - tag: Some(params.tag_name), - }; + request.id = Some(parse_id(&id, params.delimiter.as_deref())); match backend.get_table_tag_version(request).await { Ok(response) => (StatusCode::OK, Json(response)).into_response(), @@ -934,13 +905,11 @@ async fn create_table_tag( async fn delete_table_tag( State(backend): State>, - Path(params): Path, - Query(query): Query, + Path(id): Path, + Query(params): Query, + Json(mut request): Json, ) -> Response { - let request = DeleteTableTagRequest { - id: Some(parse_id(¶ms.id, query.delimiter.as_deref())), - tag: Some(params.tag_name), - }; + request.id = Some(parse_id(&id, params.delimiter.as_deref())); match backend.delete_table_tag(request).await { Ok(response) => (StatusCode::OK, Json(response)).into_response(), @@ -950,12 +919,11 @@ async fn delete_table_tag( async fn update_table_tag( State(backend): State>, - Path(params): Path, - Query(query): Query, + Path(id): Path, + Query(params): Query, Json(mut request): Json, ) -> Response { - request.id = Some(parse_id(¶ms.id, query.delimiter.as_deref())); - request.tag = Some(params.tag_name); + request.id = Some(parse_id(&id, params.delimiter.as_deref())); match backend.update_table_tag(request).await { Ok(response) => (StatusCode::OK, Json(response)).into_response(), @@ -999,21 +967,21 @@ async fn analyze_table_query_plan( // Transaction Operation Handlers // ============================================================================ -#[derive(Debug, Deserialize)] -struct TransactionPathParams { - id: String, - txn_id: String, -} - async fn describe_transaction( State(backend): State>, - Path(params): Path, - Query(query): Query, + Path(id): Path, + Query(params): Query, + Json(mut request): Json, ) -> Response { - let request = DescribeTransactionRequest { - id: Some(parse_id(¶ms.id, query.delimiter.as_deref())), - transaction_id: Some(params.txn_id), - }; + // The path id is the transaction identifier + // The request.id in body is the table ID (namespace path) + // For the trait, we set request.id to include both table ID and transaction ID + // by appending the transaction ID to the table ID path + if let Some(ref mut table_id) = request.id { + table_id.push(id); + } else { + request.id = Some(vec![id]); + } match backend.describe_transaction(request).await { Ok(response) => (StatusCode::OK, Json(response)).into_response(), @@ -1023,12 +991,17 @@ async fn describe_transaction( async fn alter_transaction( State(backend): State>, - Path(params): Path, - Query(query): Query, + Path(id): Path, + Query(params): Query, Json(mut request): Json, ) -> Response { - request.id = Some(parse_id(¶ms.id, query.delimiter.as_deref())); - request.transaction_id = Some(params.txn_id); + // The path id is the transaction identifier + // Append it to the table ID path in the request + if let Some(ref mut table_id) = request.id { + table_id.push(id); + } else { + request.id = Some(vec![id]); + } match backend.alter_transaction(request).await { Ok(response) => (StatusCode::OK, Json(response)).into_response(), @@ -1292,9 +1265,7 @@ mod tests { // Create table in child namespace let create_table_req = CreateTableRequest { id: Some(vec!["test_namespace".to_string(), "test_table".to_string()]), - location: None, - mode: Some(create_table_request::Mode::Create), - properties: None, + mode: Some("Create".to_string()), }; let result = fixture @@ -1346,9 +1317,7 @@ mod tests { for i in 1..=3 { let create_table_req = CreateTableRequest { id: Some(vec!["test_namespace".to_string(), format!("table{}", i)]), - location: None, - mode: Some(create_table_request::Mode::Create), - properties: None, + mode: Some("Create".to_string()), }; fixture .namespace @@ -1392,9 +1361,7 @@ mod tests { // Create table let create_table_req = CreateTableRequest { id: Some(vec!["test_namespace".to_string(), "test_table".to_string()]), - location: None, - mode: Some(create_table_request::Mode::Create), - properties: None, + mode: Some("Create".to_string()), }; fixture .namespace @@ -1464,9 +1431,7 @@ mod tests { // Create table let create_table_req = CreateTableRequest { id: Some(vec!["test_namespace".to_string(), "test_table".to_string()]), - location: None, - mode: Some(create_table_request::Mode::Create), - properties: None, + mode: Some("Create".to_string()), }; fixture .namespace @@ -1554,9 +1519,7 @@ mod tests { // Create table let create_table_req = CreateTableRequest { id: Some(vec!["test_namespace".to_string(), "test_table".to_string()]), - location: None, - mode: Some(create_table_request::Mode::Create), - properties: None, + mode: Some("Create".to_string()), }; fixture .namespace @@ -1862,9 +1825,7 @@ mod tests { "level3".to_string(), "deep_table".to_string(), ]), - location: None, - mode: Some(create_table_request::Mode::Create), - properties: None, + mode: Some("Create".to_string()), }; let result = fixture @@ -1923,9 +1884,7 @@ mod tests { // Create table with same name in both namespaces let create_table_req = CreateTableRequest { id: Some(vec!["namespace1".to_string(), "shared_table".to_string()]), - location: None, - mode: Some(create_table_request::Mode::Create), - properties: None, + mode: Some("Create".to_string()), }; fixture .namespace @@ -1935,9 +1894,7 @@ mod tests { let create_table_req = CreateTableRequest { id: Some(vec!["namespace2".to_string(), "shared_table".to_string()]), - location: None, - mode: Some(create_table_request::Mode::Create), - properties: None, + mode: Some("Create".to_string()), }; fixture .namespace @@ -1987,9 +1944,7 @@ mod tests { // Create table in namespace let create_table_req = CreateTableRequest { id: Some(vec!["test_namespace".to_string(), "test_table".to_string()]), - location: None, - mode: Some(create_table_request::Mode::Create), - properties: None, + mode: Some("Create".to_string()), }; fixture .namespace @@ -2141,9 +2096,7 @@ mod tests { "test_namespace".to_string(), "physical_table".to_string(), ]), - location: None, - mode: Some(create_table_request::Mode::Create), - properties: None, + mode: Some("Create".to_string()), }; fixture .namespace @@ -2270,9 +2223,7 @@ mod tests { // Create a table let create_table_req = CreateTableRequest { id: Some(vec!["test_namespace".to_string(), "test_table".to_string()]), - location: None, - mode: Some(create_table_request::Mode::Create), - properties: None, + mode: Some("Create".to_string()), }; fixture .namespace @@ -2350,9 +2301,7 @@ mod tests { "test_namespace".to_string(), "original_table".to_string(), ]), - location: None, - mode: Some(create_table_request::Mode::Create), - properties: None, + mode: Some("Create".to_string()), }; let create_response = fixture .namespace From 9045115ac1376d2ec4f545573a694d7ce2b440d1 Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Wed, 10 Dec 2025 23:36:50 -0800 Subject: [PATCH 3/6] fix clippy --- java/lance-jni/Cargo.lock | 6 ++++-- rust/lance-namespace-impls/src/rest_adapter.rs | 11 ++++++++--- rust/lance-namespace/src/namespace.rs | 10 ++-------- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/java/lance-jni/Cargo.lock b/java/lance-jni/Cargo.lock index 06eb7e045f9..0fb00f0a54d 100644 --- a/java/lance-jni/Cargo.lock +++ b/java/lance-jni/Cargo.lock @@ -3417,6 +3417,7 @@ dependencies = [ "arrow-buffer", "arrow-cast", "arrow-data", + "arrow-ord", "arrow-schema", "arrow-select", "bytes", @@ -3451,6 +3452,7 @@ dependencies = [ "datafusion-sql", "deepsize", "futures", + "itertools 0.13.0", "lance-arrow", "libc", "log", @@ -3793,9 +3795,9 @@ dependencies = [ [[package]] name = "lance-namespace-reqwest-client" -version = "0.0.18" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ea349999bcda4eea53fc05d334b3775ec314761e6a706555c777d7a29b18d19" +checksum = "b748e89a3a0e5d9fb1b51e4382f783f8aa6b620d755012d4856180968014e619" dependencies = [ "reqwest", "serde", diff --git a/rust/lance-namespace-impls/src/rest_adapter.rs b/rust/lance-namespace-impls/src/rest_adapter.rs index 06d104ece96..dfeeeba7596 100644 --- a/rust/lance-namespace-impls/src/rest_adapter.rs +++ b/rust/lance-namespace-impls/src/rest_adapter.rs @@ -620,7 +620,8 @@ async fn count_table_rows( ) -> Response { let request = CountTableRowsRequest { id: Some(parse_id(&id, params.delimiter.as_deref())), - filter: None, + version: None, + predicate: None, }; match backend.count_table_rows(request).await { @@ -748,6 +749,9 @@ async fn list_table_indices( ) -> Response { let request = ListTableIndicesRequest { id: Some(parse_id(&id, params.delimiter.as_deref())), + version: None, + page_token: None, + limit: None, }; match backend.list_table_indices(request).await { @@ -769,6 +773,7 @@ async fn describe_table_index_stats( ) -> Response { let request = DescribeTableIndexStatsRequest { id: Some(parse_id(¶ms.id, query.delimiter.as_deref())), + version: None, index_name: Some(params.index_name), }; @@ -970,7 +975,7 @@ async fn analyze_table_query_plan( async fn describe_transaction( State(backend): State>, Path(id): Path, - Query(params): Query, + Query(_params): Query, Json(mut request): Json, ) -> Response { // The path id is the transaction identifier @@ -992,7 +997,7 @@ async fn describe_transaction( async fn alter_transaction( State(backend): State>, Path(id): Path, - Query(params): Query, + Query(_params): Query, Json(mut request): Json, ) -> Response { // The path id is the transaction identifier diff --git a/rust/lance-namespace/src/namespace.rs b/rust/lance-namespace/src/namespace.rs index 0a13ae17597..60c206530f4 100644 --- a/rust/lance-namespace/src/namespace.rs +++ b/rust/lance-namespace/src/namespace.rs @@ -318,10 +318,7 @@ pub trait LanceNamespace: Send + Sync + std::fmt::Debug { } /// Restore a table to a specific version. - async fn restore_table( - &self, - _request: RestoreTableRequest, - ) -> Result { + async fn restore_table(&self, _request: RestoreTableRequest) -> Result { Err(Error::NotSupported { source: "restore_table not implemented".into(), location: Location::new(file!(), line!(), column!()), @@ -329,10 +326,7 @@ pub trait LanceNamespace: Send + Sync + std::fmt::Debug { } /// Rename a table. - async fn rename_table( - &self, - _request: RenameTableRequest, - ) -> Result { + async fn rename_table(&self, _request: RenameTableRequest) -> Result { Err(Error::NotSupported { source: "rename_table not implemented".into(), location: Location::new(file!(), line!(), column!()), From c9bee017ee8527a067aacd53ed9a8df56e03b2c4 Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Wed, 10 Dec 2025 23:45:50 -0800 Subject: [PATCH 4/6] clippy --- rust/lance-namespace-impls/src/rest.rs | 5 ++--- rust/lance-namespace-impls/src/rest_adapter.rs | 7 +++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/rust/lance-namespace-impls/src/rest.rs b/rust/lance-namespace-impls/src/rest.rs index 11b05b29eb5..3b5d0650659 100644 --- a/rust/lance-namespace-impls/src/rest.rs +++ b/rust/lance-namespace-impls/src/rest.rs @@ -968,7 +968,6 @@ impl LanceNamespace for RestNamespace { mod tests { use super::*; use bytes::Bytes; - use lance_namespace::models::{create_table_request, insert_into_table_request}; use wiremock::matchers::{method, path}; use wiremock::{Mock, MockServer, ResponseTemplate}; @@ -1289,7 +1288,7 @@ mod tests { Mock::given(method("POST")) .and(path(path_str.as_str())) .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ - "version": 2 + "transaction_id": "txn-123" }))) .mount(&mock_server) .await; @@ -1315,6 +1314,6 @@ mod tests { // Should succeed with mock server assert!(result.is_ok()); let response = result.unwrap(); - assert_eq!(response.version, Some(2)); + assert_eq!(response.transaction_id, Some("txn-123".to_string())); } } diff --git a/rust/lance-namespace-impls/src/rest_adapter.rs b/rust/lance-namespace-impls/src/rest_adapter.rs index dfeeeba7596..1a9a97545ac 100644 --- a/rust/lance-namespace-impls/src/rest_adapter.rs +++ b/rust/lance-namespace-impls/src/rest_adapter.rs @@ -2128,7 +2128,10 @@ mod tests { ); let response = result.unwrap(); - assert_eq!(response.location, "test_namespace$physical_table.lance"); + assert_eq!( + response.location, + Some("test_namespace$physical_table.lance".to_string()) + ); // Verify registered table exists let mut exists_req = TableExistsRequest::new(); @@ -2358,7 +2361,7 @@ mod tests { .expect("Failed to re-register table with new name"); // Should return the exact location we registered - assert_eq!(register_response.location, relative_location); + assert_eq!(register_response.location, Some(relative_location)); // Verify new table exists let mut exists_req = TableExistsRequest::new(); From 4b3cf690238fe8e085b2e737c061b17ddb399197 Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Wed, 10 Dec 2025 23:49:49 -0800 Subject: [PATCH 5/6] msrv --- rust/lance-namespace-impls/src/rest_adapter.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/lance-namespace-impls/src/rest_adapter.rs b/rust/lance-namespace-impls/src/rest_adapter.rs index 1a9a97545ac..321531391dd 100644 --- a/rust/lance-namespace-impls/src/rest_adapter.rs +++ b/rust/lance-namespace-impls/src/rest_adapter.rs @@ -2361,7 +2361,7 @@ mod tests { .expect("Failed to re-register table with new name"); // Should return the exact location we registered - assert_eq!(register_response.location, Some(relative_location)); + assert_eq!(register_response.location, Some(relative_location.clone())); // Verify new table exists let mut exists_req = TableExistsRequest::new(); From 6850099eab90fb4eaa1e31a06200df65b33a019e Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Thu, 11 Dec 2025 00:03:55 -0800 Subject: [PATCH 6/6] update lock --- python/Cargo.lock | 4 ++-- rust/lance-namespace-impls/src/rest_adapter.rs | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/python/Cargo.lock b/python/Cargo.lock index 8a0fa929db8..8d226fd11a1 100644 --- a/python/Cargo.lock +++ b/python/Cargo.lock @@ -4277,9 +4277,9 @@ dependencies = [ [[package]] name = "lance-namespace-reqwest-client" -version = "0.0.18" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ea349999bcda4eea53fc05d334b3775ec314761e6a706555c777d7a29b18d19" +checksum = "b748e89a3a0e5d9fb1b51e4382f783f8aa6b620d755012d4856180968014e619" dependencies = [ "reqwest", "serde", diff --git a/rust/lance-namespace-impls/src/rest_adapter.rs b/rust/lance-namespace-impls/src/rest_adapter.rs index 321531391dd..dd94b15e7c4 100644 --- a/rust/lance-namespace-impls/src/rest_adapter.rs +++ b/rust/lance-namespace-impls/src/rest_adapter.rs @@ -446,9 +446,10 @@ async fn drop_table( State(backend): State>, Path(id): Path, Query(params): Query, - Json(mut request): Json, ) -> Response { - request.id = Some(parse_id(&id, params.delimiter.as_deref())); + let request = DropTableRequest { + id: Some(parse_id(&id, params.delimiter.as_deref())), + }; match backend.drop_table(request).await { Ok(response) => (StatusCode::OK, Json(response)).into_response(),