From 7b39f05b9f49151497d2b211b45cc5fcc0dcac4d Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Thu, 27 Nov 2025 12:54:20 -0800 Subject: [PATCH 1/5] refactor: separate out python and java LanceNamespace interface --- java/pom.xml | 7 +- .../org/lance/namespace/LanceNamespace.java | 427 ------------------ python/pyproject.toml | 2 +- python/python/lance/namespace.py | 216 +-------- 4 files changed, 16 insertions(+), 636 deletions(-) delete mode 100644 java/src/main/java/org/lance/namespace/LanceNamespace.java diff --git a/java/pom.xml b/java/pom.xml index ae8de41cd05..1884d4c41a5 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -105,10 +105,15 @@ guava 33.3.1-jre + + org.lance + lance-namespace-core + 0.2.0 + org.lance lance-namespace-apache-client - 0.1.0 + 0.2.0 com.fasterxml.jackson.core diff --git a/java/src/main/java/org/lance/namespace/LanceNamespace.java b/java/src/main/java/org/lance/namespace/LanceNamespace.java deleted file mode 100644 index e0c713532bf..00000000000 --- a/java/src/main/java/org/lance/namespace/LanceNamespace.java +++ /dev/null @@ -1,427 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.lance.namespace; - -import org.lance.namespace.model.*; - -import org.apache.arrow.memory.BufferAllocator; - -import java.lang.reflect.Constructor; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -/** - * Interface for LanceDB namespace operations. - * - *

A namespace provides hierarchical organization for tables and supports various storage - * backends (local filesystem, S3, Azure, GCS) with optional credential vending for cloud providers. - * - *

Implementations of this interface can provide different storage backends: - * - *

- * - *

External libraries can implement this interface to provide integration with catalog systems - * like AWS Glue, Hive Metastore, or Databricks Unity Catalog. - * - *

Most methods have default implementations that throw {@link UnsupportedOperationException}. - * Implementations should override the methods they support. - * - *

Use {@link #connect(String, Map, BufferAllocator)} to create namespace instances, and {@link - * #registerNamespaceImpl(String, String)} to register external implementations. - */ -public interface LanceNamespace { - - // ========== Static Registry and Factory Methods ========== - - /** Native implementations (Rust-backed). */ - Map NATIVE_IMPLS = - Collections.unmodifiableMap( - new HashMap() { - { - put("dir", "org.lance.namespace.DirectoryNamespace"); - put("rest", "org.lance.namespace.RestNamespace"); - } - }); - - /** Plugin registry for external implementations. Thread-safe for concurrent access. */ - Map REGISTERED_IMPLS = new ConcurrentHashMap<>(); - - /** - * Register a namespace implementation with a short name. - * - *

External libraries can use this to register their implementations, allowing users to use - * short names like "glue" instead of full class paths. - * - * @param name Short name for the implementation (e.g., "glue", "hive2", "unity") - * @param className Full class name (e.g., "org.lance.namespace.glue.GlueNamespace") - */ - static void registerNamespaceImpl(String name, String className) { - REGISTERED_IMPLS.put(name, className); - } - - /** - * Unregister a previously registered namespace implementation. - * - * @param name Short name of the implementation to unregister - * @return true if an implementation was removed, false if it wasn't registered - */ - static boolean unregisterNamespaceImpl(String name) { - return REGISTERED_IMPLS.remove(name) != null; - } - - /** - * Check if an implementation is registered with the given name. - * - * @param name Short name or class name to check - * @return true if the implementation is available - */ - static boolean isRegistered(String name) { - return NATIVE_IMPLS.containsKey(name) || REGISTERED_IMPLS.containsKey(name); - } - - /** - * Connect to a Lance namespace implementation. - * - *

This factory method creates namespace instances based on implementation aliases or full - * class names. It provides a unified way to instantiate different namespace backends. - * - * @param impl Implementation alias or full class name. Built-in aliases: "dir" for - * DirectoryNamespace, "rest" for RestNamespace. External libraries can register additional - * aliases using {@link #registerNamespaceImpl(String, String)}. - * @param properties Configuration properties passed to the namespace - * @param allocator Arrow buffer allocator for memory management - * @return The connected namespace instance - * @throws IllegalArgumentException If the implementation class cannot be loaded or does not - * implement LanceNamespace interface - */ - static LanceNamespace connect( - String impl, Map properties, BufferAllocator allocator) { - // Check native impls first, then registered plugins, then treat as full class name - String className = NATIVE_IMPLS.get(impl); - if (className == null) { - className = REGISTERED_IMPLS.get(impl); - } - if (className == null) { - className = impl; - } - - try { - Class clazz = Class.forName(className); - - if (!LanceNamespace.class.isAssignableFrom(clazz)) { - throw new IllegalArgumentException( - "Class " + className + " does not implement LanceNamespace interface"); - } - - @SuppressWarnings("unchecked") - Class namespaceClass = (Class) clazz; - - Constructor constructor = namespaceClass.getConstructor(); - LanceNamespace namespace = constructor.newInstance(); - namespace.initialize(properties, allocator); - - return namespace; - } catch (ClassNotFoundException e) { - throw new IllegalArgumentException("Namespace implementation class not found: " + className); - } catch (NoSuchMethodException e) { - throw new IllegalArgumentException( - "Namespace implementation class " + className + " must have a no-arg constructor"); - } catch (Exception e) { - throw new IllegalArgumentException( - "Failed to construct namespace impl " + className + ": " + e.getMessage(), e); - } - } - - // ========== Instance Methods ========== - - /** - * Initialize the namespace with configuration properties. - * - * @param configProperties Configuration properties (e.g., root path, storage options) - * @param allocator Arrow buffer allocator for memory management - */ - void initialize(Map configProperties, BufferAllocator allocator); - - /** - * Return a human-readable unique identifier for this namespace instance. - * - *

This is used for equality comparison and caching. Two namespace instances with the same ID - * are considered equal and will share cached resources. - * - * @return A human-readable unique identifier string - */ - String namespaceId(); - - // Namespace operations - - /** - * List namespaces. - * - * @param request The list namespaces request - * @return The list namespaces response - */ - default ListNamespacesResponse listNamespaces(ListNamespacesRequest request) { - throw new UnsupportedOperationException("Not supported: listNamespaces"); - } - - /** - * Describe a namespace. - * - * @param request The describe namespace request - * @return The describe namespace response - */ - default DescribeNamespaceResponse describeNamespace(DescribeNamespaceRequest request) { - throw new UnsupportedOperationException("Not supported: describeNamespace"); - } - - /** - * Create a new namespace. - * - * @param request The create namespace request - * @return The create namespace response - */ - default CreateNamespaceResponse createNamespace(CreateNamespaceRequest request) { - throw new UnsupportedOperationException("Not supported: createNamespace"); - } - - /** - * Drop a namespace. - * - * @param request The drop namespace request - * @return The drop namespace response - */ - default DropNamespaceResponse dropNamespace(DropNamespaceRequest request) { - throw new UnsupportedOperationException("Not supported: dropNamespace"); - } - - /** - * Check if a namespace exists. - * - * @param request The namespace exists request - * @throws RuntimeException if the namespace does not exist - */ - default void namespaceExists(NamespaceExistsRequest request) { - throw new UnsupportedOperationException("Not supported: namespaceExists"); - } - - // Table operations - - /** - * List tables in a namespace. - * - * @param request The list tables request - * @return The list tables response - */ - default ListTablesResponse listTables(ListTablesRequest request) { - throw new UnsupportedOperationException("Not supported: listTables"); - } - - /** - * Describe a table. - * - * @param request The describe table request - * @return The describe table response - */ - default DescribeTableResponse describeTable(DescribeTableRequest request) { - throw new UnsupportedOperationException("Not supported: describeTable"); - } - - /** - * Register a table. - * - * @param request The register table request - * @return The register table response - */ - default RegisterTableResponse registerTable(RegisterTableRequest request) { - throw new UnsupportedOperationException("Not supported: registerTable"); - } - - /** - * Check if a table exists. - * - * @param request The table exists request - * @throws RuntimeException if the table does not exist - */ - default void tableExists(TableExistsRequest request) { - throw new UnsupportedOperationException("Not supported: tableExists"); - } - - /** - * Drop a table. - * - * @param request The drop table request - * @return The drop table response - */ - default DropTableResponse dropTable(DropTableRequest request) { - throw new UnsupportedOperationException("Not supported: dropTable"); - } - - /** - * Deregister a table. - * - * @param request The deregister table request - * @return The deregister table response - */ - default DeregisterTableResponse deregisterTable(DeregisterTableRequest request) { - throw new UnsupportedOperationException("Not supported: deregisterTable"); - } - - /** - * Count rows in a table. - * - * @param request The count table rows request - * @return The row count - */ - default Long countTableRows(CountTableRowsRequest request) { - throw new UnsupportedOperationException("Not supported: countTableRows"); - } - - // Data operations - - /** - * Create a new table with data from Arrow IPC stream. - * - * @param request The create table request - * @param requestData Arrow IPC stream data - * @return The create table response - */ - default CreateTableResponse createTable(CreateTableRequest request, byte[] requestData) { - throw new UnsupportedOperationException("Not supported: createTable"); - } - - /** - * Create an empty table (metadata only operation). - * - * @param request The create empty table request - * @return The create empty table response - */ - default CreateEmptyTableResponse createEmptyTable(CreateEmptyTableRequest request) { - throw new UnsupportedOperationException("Not supported: createEmptyTable"); - } - - /** - * Insert data into a table. - * - * @param request The insert into table request - * @param requestData Arrow IPC stream data - * @return The insert into table response - */ - default InsertIntoTableResponse insertIntoTable( - InsertIntoTableRequest request, byte[] requestData) { - throw new UnsupportedOperationException("Not supported: insertIntoTable"); - } - - /** - * Merge insert data into a table. - * - * @param request The merge insert into table request - * @param requestData Arrow IPC stream data - * @return The merge insert into table response - */ - default MergeInsertIntoTableResponse mergeInsertIntoTable( - MergeInsertIntoTableRequest request, byte[] requestData) { - throw new UnsupportedOperationException("Not supported: mergeInsertIntoTable"); - } - - /** - * Update a table. - * - * @param request The update table request - * @return The update table response - */ - default UpdateTableResponse updateTable(UpdateTableRequest request) { - throw new UnsupportedOperationException("Not supported: updateTable"); - } - - /** - * Delete from a table. - * - * @param request The delete from table request - * @return The delete from table response - */ - default DeleteFromTableResponse deleteFromTable(DeleteFromTableRequest request) { - throw new UnsupportedOperationException("Not supported: deleteFromTable"); - } - - /** - * Query a table. - * - * @param request The query table request - * @return Arrow IPC stream data containing query results - */ - default byte[] queryTable(QueryTableRequest request) { - throw new UnsupportedOperationException("Not supported: queryTable"); - } - - // Index operations - - /** - * Create a table index. - * - * @param request The create table index request - * @return The create table index response - */ - default CreateTableIndexResponse createTableIndex(CreateTableIndexRequest request) { - throw new UnsupportedOperationException("Not supported: createTableIndex"); - } - - /** - * List table indices. - * - * @param request The list table indices request - * @return The list table indices response - */ - default ListTableIndicesResponse listTableIndices(ListTableIndicesRequest request) { - throw new UnsupportedOperationException("Not supported: listTableIndices"); - } - - /** - * Describe table index statistics. - * - * @param request The describe table index stats request - * @param indexName The name of the index - * @return The describe table index stats response - */ - default DescribeTableIndexStatsResponse describeTableIndexStats( - DescribeTableIndexStatsRequest request, String indexName) { - throw new UnsupportedOperationException("Not supported: describeTableIndexStats"); - } - - // Transaction operations - - /** - * Describe a transaction. - * - * @param request The describe transaction request - * @return The describe transaction response - */ - default DescribeTransactionResponse describeTransaction(DescribeTransactionRequest request) { - throw new UnsupportedOperationException("Not supported: describeTransaction"); - } - - /** - * Alter a transaction. - * - * @param request The alter transaction request - * @return The alter transaction response - */ - default AlterTransactionResponse alterTransaction(AlterTransactionRequest request) { - throw new UnsupportedOperationException("Not supported: alterTransaction"); - } -} diff --git a/python/pyproject.toml b/python/pyproject.toml index 788d747269f..6009bdb4545 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "pylance" dynamic = ["version"] -dependencies = ["pyarrow>=14", "numpy>=1.22", "lance_namespace_urllib3_client>=0.1.0"] +dependencies = ["pyarrow>=14", "numpy>=1.22", "lance-namespace>=0.2.0"] description = "python wrapper for Lance columnar format" authors = [{ name = "Lance Devs", email = "dev@lance.org" }] license = { file = "LICENSE" } diff --git a/python/python/lance/namespace.py b/python/python/lance/namespace.py index da3f1c4ea04..4db4da6bbc1 100644 --- a/python/python/lance/namespace.py +++ b/python/python/lance/namespace.py @@ -4,17 +4,19 @@ """LanceNamespace storage options integration and implementations. This module provides: -1. LanceNamespace ABC interface for namespace implementations -2. Native Rust-backed namespace implementations (DirectoryNamespace, RestNamespace) -3. Storage options integration with LanceNamespace for automatic credential refresh -4. Plugin registry for external namespace implementations +1. Native Rust-backed namespace implementations (DirectoryNamespace, RestNamespace) +2. Storage options integration with LanceNamespace for automatic credential refresh +3. Plugin registry for external namespace implementations + +The LanceNamespace ABC interface is provided by the lance_namespace package. """ import importlib -from abc import ABC, abstractmethod from typing import Dict, List -from lance_namespace_urllib3_client.models import ( +from lance_namespace import ( + LanceNamespace, + # Request/Response types for internal use (users should import from lance_namespace) AlterTransactionRequest, AlterTransactionResponse, CountTableRowsRequest, @@ -75,7 +77,7 @@ PyRestAdapter = None __all__ = [ - # Interface and factory + # Interface and factory (LanceNamespace from lance_namespace package) "LanceNamespace", "connect", "register_namespace_impl", @@ -84,209 +86,9 @@ "RestNamespace", "RestAdapter", "LanceNamespaceStorageOptionsProvider", - # Request/Response types (re-exported from lance_namespace_urllib3_client) - "AlterTransactionRequest", - "AlterTransactionResponse", - "CountTableRowsRequest", - "CreateEmptyTableRequest", - "CreateEmptyTableResponse", - "CreateNamespaceRequest", - "CreateNamespaceResponse", - "CreateTableIndexRequest", - "CreateTableIndexResponse", - "CreateTableRequest", - "CreateTableResponse", - "DeleteFromTableRequest", - "DeleteFromTableResponse", - "DeregisterTableRequest", - "DeregisterTableResponse", - "DescribeNamespaceRequest", - "DescribeNamespaceResponse", - "DescribeTableIndexStatsRequest", - "DescribeTableIndexStatsResponse", - "DescribeTableRequest", - "DescribeTableResponse", - "DescribeTransactionRequest", - "DescribeTransactionResponse", - "DropNamespaceRequest", - "DropNamespaceResponse", - "DropTableRequest", - "DropTableResponse", - "InsertIntoTableRequest", - "InsertIntoTableResponse", - "ListNamespacesRequest", - "ListNamespacesResponse", - "ListTableIndicesRequest", - "ListTableIndicesResponse", - "ListTablesRequest", - "ListTablesResponse", - "MergeInsertIntoTableRequest", - "MergeInsertIntoTableResponse", - "NamespaceExistsRequest", - "QueryTableRequest", - "RegisterTableRequest", - "RegisterTableResponse", - "TableExistsRequest", - "UpdateTableRequest", - "UpdateTableResponse", ] -class LanceNamespace(ABC): - """Base interface for Lance Namespace implementations. - - This abstract base class defines the contract for namespace implementations - that manage Lance tables. Implementations can provide different storage backends - (directory-based, REST API, cloud catalogs, etc.). - - To create a custom namespace implementation, subclass this ABC and implement - at least the `namespace_id()` method. Other methods have default implementations - that raise `NotImplementedError`. - """ - - @abstractmethod - def namespace_id(self) -> str: - """Return a human-readable unique identifier for this namespace instance. - - This is used for equality comparison and hashing when the namespace is - used as part of a storage options provider. Two namespace instances with - the same ID are considered equal and will share cached resources. - - The ID should be human-readable for debugging and logging purposes. - For example: - - REST namespace: "RestNamespace { uri: 'https://api.example.com' }" - - Directory namespace: "DirectoryNamespace { root: '/path/to/data' }" - - Returns - ------- - str - A human-readable unique identifier string - """ - pass - - def list_namespaces(self, request: ListNamespacesRequest) -> ListNamespacesResponse: - """List namespaces.""" - raise NotImplementedError("Not supported: list_namespaces") - - def describe_namespace( - self, request: DescribeNamespaceRequest - ) -> DescribeNamespaceResponse: - """Describe a namespace.""" - raise NotImplementedError("Not supported: describe_namespace") - - def create_namespace( - self, request: CreateNamespaceRequest - ) -> CreateNamespaceResponse: - """Create a new namespace.""" - raise NotImplementedError("Not supported: create_namespace") - - def drop_namespace(self, request: DropNamespaceRequest) -> DropNamespaceResponse: - """Drop a namespace.""" - raise NotImplementedError("Not supported: drop_namespace") - - def namespace_exists(self, request: NamespaceExistsRequest) -> None: - """Check if a namespace exists.""" - raise NotImplementedError("Not supported: namespace_exists") - - def list_tables(self, request: ListTablesRequest) -> ListTablesResponse: - """List tables in a namespace.""" - raise NotImplementedError("Not supported: list_tables") - - def describe_table(self, request: DescribeTableRequest) -> DescribeTableResponse: - """Describe a table.""" - raise NotImplementedError("Not supported: describe_table") - - def register_table(self, request: RegisterTableRequest) -> RegisterTableResponse: - """Register a table.""" - raise NotImplementedError("Not supported: register_table") - - def table_exists(self, request: TableExistsRequest) -> None: - """Check if a table exists.""" - raise NotImplementedError("Not supported: table_exists") - - def drop_table(self, request: DropTableRequest) -> DropTableResponse: - """Drop a table.""" - raise NotImplementedError("Not supported: drop_table") - - def deregister_table( - self, request: DeregisterTableRequest - ) -> DeregisterTableResponse: - """Deregister a table.""" - raise NotImplementedError("Not supported: deregister_table") - - def count_table_rows(self, request: CountTableRowsRequest) -> int: - """Count rows in a table.""" - raise NotImplementedError("Not supported: count_table_rows") - - def create_table( - self, request: CreateTableRequest, request_data: bytes - ) -> CreateTableResponse: - """Create a new table with data from Arrow IPC stream.""" - raise NotImplementedError("Not supported: create_table") - - def create_empty_table( - self, request: CreateEmptyTableRequest - ) -> CreateEmptyTableResponse: - """Create an empty table (metadata only operation).""" - raise NotImplementedError("Not supported: create_empty_table") - - def insert_into_table( - self, request: InsertIntoTableRequest, request_data: bytes - ) -> InsertIntoTableResponse: - """Insert data into a table.""" - raise NotImplementedError("Not supported: insert_into_table") - - def merge_insert_into_table( - self, request: MergeInsertIntoTableRequest, request_data: bytes - ) -> MergeInsertIntoTableResponse: - """Merge insert data into a table.""" - raise NotImplementedError("Not supported: merge_insert_into_table") - - def update_table(self, request: UpdateTableRequest) -> UpdateTableResponse: - """Update a table.""" - raise NotImplementedError("Not supported: update_table") - - def delete_from_table( - self, request: DeleteFromTableRequest - ) -> DeleteFromTableResponse: - """Delete from a table.""" - raise NotImplementedError("Not supported: delete_from_table") - - def query_table(self, request: QueryTableRequest) -> bytes: - """Query a table.""" - raise NotImplementedError("Not supported: query_table") - - def create_table_index( - self, request: CreateTableIndexRequest - ) -> CreateTableIndexResponse: - """Create a table index.""" - raise NotImplementedError("Not supported: create_table_index") - - def list_table_indices( - self, request: ListTableIndicesRequest - ) -> ListTableIndicesResponse: - """List table indices.""" - raise NotImplementedError("Not supported: list_table_indices") - - def describe_table_index_stats( - self, request: DescribeTableIndexStatsRequest - ) -> DescribeTableIndexStatsResponse: - """Describe table index statistics.""" - raise NotImplementedError("Not supported: describe_table_index_stats") - - def describe_transaction( - self, request: DescribeTransactionRequest - ) -> DescribeTransactionResponse: - """Describe a transaction.""" - raise NotImplementedError("Not supported: describe_transaction") - - def alter_transaction( - self, request: AlterTransactionRequest - ) -> AlterTransactionResponse: - """Alter a transaction.""" - raise NotImplementedError("Not supported: alter_transaction") - - class DirectoryNamespace(LanceNamespace): """Directory-based Lance Namespace implementation backed by Rust. From 728b3c7a7b43d5a881d0875dbea89698081e846e Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Thu, 27 Nov 2025 23:27:28 -0800 Subject: [PATCH 2/5] try fix --- java/pom.xml | 4 ++-- python/pyproject.toml | 2 +- python/python/lance/namespace.py | 23 +---------------------- 3 files changed, 4 insertions(+), 25 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index 1884d4c41a5..7e57fcf17ba 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -108,12 +108,12 @@ org.lance lance-namespace-core - 0.2.0 + 0.2.1 org.lance lance-namespace-apache-client - 0.2.0 + 0.2.1 com.fasterxml.jackson.core diff --git a/python/pyproject.toml b/python/pyproject.toml index 6009bdb4545..7aa500668df 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "pylance" dynamic = ["version"] -dependencies = ["pyarrow>=14", "numpy>=1.22", "lance-namespace>=0.2.0"] +dependencies = ["pyarrow>=14", "numpy>=1.22", "lance-namespace>=0.2.1"] description = "python wrapper for Lance columnar format" authors = [{ name = "Lance Devs", email = "dev@lance.org" }] license = { file = "LICENSE" } diff --git a/python/python/lance/namespace.py b/python/python/lance/namespace.py index 4db4da6bbc1..9251b9ef35a 100644 --- a/python/python/lance/namespace.py +++ b/python/python/lance/namespace.py @@ -15,52 +15,31 @@ from typing import Dict, List from lance_namespace import ( - LanceNamespace, - # Request/Response types for internal use (users should import from lance_namespace) - AlterTransactionRequest, - AlterTransactionResponse, - CountTableRowsRequest, CreateEmptyTableRequest, CreateEmptyTableResponse, CreateNamespaceRequest, CreateNamespaceResponse, - CreateTableIndexRequest, - CreateTableIndexResponse, CreateTableRequest, CreateTableResponse, - DeleteFromTableRequest, - DeleteFromTableResponse, DeregisterTableRequest, DeregisterTableResponse, DescribeNamespaceRequest, DescribeNamespaceResponse, - DescribeTableIndexStatsRequest, - DescribeTableIndexStatsResponse, DescribeTableRequest, DescribeTableResponse, - DescribeTransactionRequest, - DescribeTransactionResponse, DropNamespaceRequest, DropNamespaceResponse, DropTableRequest, DropTableResponse, - InsertIntoTableRequest, - InsertIntoTableResponse, + LanceNamespace, ListNamespacesRequest, ListNamespacesResponse, - ListTableIndicesRequest, - ListTableIndicesResponse, ListTablesRequest, ListTablesResponse, - MergeInsertIntoTableRequest, - MergeInsertIntoTableResponse, NamespaceExistsRequest, - QueryTableRequest, RegisterTableRequest, RegisterTableResponse, TableExistsRequest, - UpdateTableRequest, - UpdateTableResponse, ) from .io import StorageOptionsProvider From 747d2a90219dbfe7445d1c3fb85e6a3c417fd626 Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Thu, 27 Nov 2025 23:31:23 -0800 Subject: [PATCH 3/5] commit --- python/python/lance/namespace.py | 79 -------------------------------- 1 file changed, 79 deletions(-) diff --git a/python/python/lance/namespace.py b/python/python/lance/namespace.py index 9251b9ef35a..81af95c29f3 100644 --- a/python/python/lance/namespace.py +++ b/python/python/lance/namespace.py @@ -11,7 +11,6 @@ The LanceNamespace ABC interface is provided by the lance_namespace package. """ -import importlib from typing import Dict, List from lance_namespace import ( @@ -56,11 +55,6 @@ PyRestAdapter = None __all__ = [ - # Interface and factory (LanceNamespace from lance_namespace package) - "LanceNamespace", - "connect", - "register_namespace_impl", - # Implementations "DirectoryNamespace", "RestNamespace", "RestAdapter", @@ -513,76 +507,3 @@ def provider_id(self) -> str: f"LanceNamespaceStorageOptionsProvider {{ " f"namespace: {namespace_id}, table_id: {self._table_id!r} }}" ) - - -# Native implementations (Rust-backed) -NATIVE_IMPLS = { - "rest": "lance.namespace.RestNamespace", - "dir": "lance.namespace.DirectoryNamespace", -} - -# Plugin registry for external implementations -_REGISTERED_IMPLS: Dict[str, str] = {} - - -def register_namespace_impl(name: str, class_path: str) -> None: - """Register a namespace implementation with a short name. - - External libraries can use this to register their implementations, - allowing users to use short names like "glue" instead of full class paths. - - Parameters - ---------- - name : str - Short name for the implementation (e.g., "glue", "hive2", "unity") - class_path : str - Full class path (e.g., "lance_glue.GlueNamespace") - """ - _REGISTERED_IMPLS[name] = class_path - - -def connect(impl: str, properties: Dict[str, str]) -> LanceNamespace: - """Connect to a Lance namespace implementation. - - This factory function creates namespace instances based on implementation - aliases or full class paths. It provides a unified way to instantiate - different namespace backends. - - Parameters - ---------- - impl : str - Implementation alias or full class path. Built-in aliases: - - "rest": RestNamespace (REST API client) - - "dir": DirectoryNamespace (local/cloud filesystem) - You can also use full class paths like "my.custom.Namespace" - External libraries can register additional aliases using - `register_namespace_impl()`. - properties : Dict[str, str] - Configuration properties passed to the namespace constructor - - Returns - ------- - LanceNamespace - The connected namespace instance - - Raises - ------ - ValueError - If the implementation class cannot be loaded or does not - implement LanceNamespace interface - """ - # Check native impls first, then registered plugins, then treat as full class path - impl_class = NATIVE_IMPLS.get(impl) or _REGISTERED_IMPLS.get(impl) or impl - try: - module_name, class_name = impl_class.rsplit(".", 1) - module = importlib.import_module(module_name) - namespace_class = getattr(module, class_name) - - if not issubclass(namespace_class, LanceNamespace): - raise ValueError( - f"Class {impl_class} does not implement LanceNamespace interface" - ) - - return namespace_class(**properties) - except Exception as e: - raise ValueError(f"Failed to construct namespace impl {impl_class}: {e}") From ad5d8c14e3944b8f30f72b083fe32642a5383fed Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Thu, 27 Nov 2025 23:38:29 -0800 Subject: [PATCH 4/5] fix tests --- python/python/tests/test_namespace_dir.py | 2 +- python/python/tests/test_namespace_rest.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/python/tests/test_namespace_dir.py b/python/python/tests/test_namespace_dir.py index 9158acee54e..74871facf6d 100644 --- a/python/python/tests/test_namespace_dir.py +++ b/python/python/tests/test_namespace_dir.py @@ -17,7 +17,7 @@ import lance.namespace import pyarrow as pa import pytest -from lance.namespace import ( +from lance_namespace import ( CreateEmptyTableRequest, CreateNamespaceRequest, CreateTableRequest, diff --git a/python/python/tests/test_namespace_rest.py b/python/python/tests/test_namespace_rest.py index 9410cf89e4c..6b988d7c476 100644 --- a/python/python/tests/test_namespace_rest.py +++ b/python/python/tests/test_namespace_rest.py @@ -17,7 +17,7 @@ import lance.namespace import pyarrow as pa import pytest -from lance.namespace import ( +from lance_namespace import ( CreateEmptyTableRequest, CreateNamespaceRequest, CreateTableRequest, From 4ef76fe597a1202fd718ebb97f08153fb86f60be Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Thu, 27 Nov 2025 23:49:27 -0800 Subject: [PATCH 5/5] fix doctest --- python/python/lance/namespace.py | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/python/python/lance/namespace.py b/python/python/lance/namespace.py index 81af95c29f3..4619f2ce244 100644 --- a/python/python/lance/namespace.py +++ b/python/python/lance/namespace.py @@ -92,16 +92,9 @@ class DirectoryNamespace(LanceNamespace): >>> # Create with properties dict >>> ns = lance.namespace.DirectoryNamespace(root="memory://test") >>> - >>> # With storage options - >>> ns = lance.namespace.DirectoryNamespace( - ... root="/path/to/data", - ... manifest_enabled="true", - ... **{"storage.region": "us-west-2"} - ... ) - >>> - >>> # Using the connect() factory function - >>> import lance.namespace - >>> ns = lance.namespace.connect("dir", {"root": "memory://test"}) + >>> # Using the connect() factory function from lance_namespace + >>> import lance_namespace + >>> ns = lance_namespace.connect("dir", {"root": "memory://test"}) """ def __init__(self, session=None, **properties): @@ -209,16 +202,9 @@ class RestNamespace(LanceNamespace): >>> # Create with properties dict >>> ns = lance.namespace.RestNamespace(uri="http://localhost:4099") >>> - >>> # With custom delimiter and headers - >>> ns = lance.namespace.RestNamespace( - ... uri="http://localhost:4099", - ... delimiter=".", - ... **{"header.Authorization": "Bearer token"} - ... ) - >>> - >>> # Using the connect() factory function - >>> import lance.namespace - >>> ns = lance.namespace.connect("rest", {"uri": "http://localhost:4099"}) + >>> # Using the connect() factory function from lance_namespace + >>> import lance_namespace + >>> ns = lance_namespace.connect("rest", {"uri": "http://localhost:4099"}) """ def __init__(self, **properties):