Skip to content

fix: open additional storage options provider related apis in lance dataset #5932

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jackye1995 merged 2 commits into from
Feb 11, 2026
Merged

fix: open additional storage options provider related apis in lance dataset #5932

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
14a9145
fix lint
jackye1995 Feb 4, 2026
32cae15
fix lint
jackye1995 Feb 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions python/python/lance/dataset.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -465,6 +465,7 @@ def __init__(
)
self._default_scan_options = default_scan_options
self._read_params = read_params
self._storage_options_provider = storage_options_provider

@classmethod
def __deserialize__(
Expand Down Expand Up @@ -526,11 +527,13 @@ def __setstate__(self, state):
)
self._default_scan_options = default_scan_options
self._read_params = read_params
self._storage_options_provider = None

def __copy__(self):
ds = LanceDataset.__new__(LanceDataset)
ds._uri = self._uri
ds._storage_options = self._storage_options
ds._storage_options_provider = self._storage_options_provider
ds._ds = copy.copy(self._ds)
ds._default_scan_options = self._default_scan_options
ds._read_params = self._read_params.copy() if self._read_params else None
Expand Down Expand Up @@ -625,6 +628,7 @@ def create_branch(
ds._ds = new_ds
ds._uri = new_ds.uri
ds._storage_options = self._storage_options
ds._storage_options_provider = self._storage_options_provider
ds._default_scan_options = self._default_scan_options
ds._read_params = self._read_params
return ds
Expand Down Expand Up @@ -2283,6 +2287,27 @@ def storage_options_accessor(self):
"""
return self._ds.storage_options_accessor()

def new_file_session(self):
"""
Create a new file session for reading and writing files in this dataset.

The file session will use the dataset's storage options and provider
for credential management, enabling automatic credential refresh for
long-running operations.

Returns
-------
LanceFileSession
A file session configured for this dataset's storage location.
"""
from lance.file import LanceFileSession

return LanceFileSession(
base_path=self._uri,
storage_options=self.latest_storage_options(),
storage_options_provider=self._storage_options_provider,
)

def checkout_version(
self, version: int | str | Tuple[Optional[str], Optional[int]]
) -> "LanceDataset":
Expand Down Expand Up @@ -3479,6 +3504,7 @@ def commit(

ds = LanceDataset.__new__(LanceDataset)
ds._storage_options = storage_options
ds._storage_options_provider = storage_options_provider
ds._ds = new_ds
ds._uri = new_ds.uri
ds._default_scan_options = None
Expand Down Expand Up @@ -3577,6 +3603,7 @@ def commit_batch(
ds._ds = new_ds
ds._uri = new_ds.uri
ds._storage_options = storage_options
ds._storage_options_provider = storage_options_provider
ds._default_scan_options = None
ds._read_params = None
return BulkCommitResult(
Expand Down Expand Up @@ -5855,6 +5882,7 @@ def write_dataset(

ds = LanceDataset.__new__(LanceDataset)
ds._storage_options = storage_options
ds._storage_options_provider = None
ds._ds = inner_ds
ds._uri = inner_ds.uri
ds._default_scan_options = None
Expand Down