diff --git a/python/python/tests/test_s3_ddb.py b/python/python/tests/test_s3_ddb.py index 9e006fec60e..c2073aee74e 100644 --- a/python/python/tests/test_s3_ddb.py +++ b/python/python/tests/test_s3_ddb.py @@ -287,3 +287,12 @@ def test_file_writer_reader(s3_bucket: str): bytes(reader.read_global_buffer(global_buffer_pos)).decode() == global_buffer_text ) + + +@pytest.mark.integration +def test_append_fragment(s3_bucket: str): + storage_options = copy.deepcopy(CONFIG) + table = pa.table({"a": [1, 2], "b": ["a", "b"]}) + lance.fragment.LanceFragment.create( + f"s3://{s3_bucket}/test_append.lance", table, storage_options=storage_options + ) diff --git a/rust/lance/src/dataset/fragment/write.rs b/rust/lance/src/dataset/fragment/write.rs index 5d330636613..1f6bd53cdd7 100644 --- a/rust/lance/src/dataset/fragment/write.rs +++ b/rust/lance/src/dataset/fragment/write.rs @@ -214,7 +214,15 @@ impl<'a> FragmentCreateBuilder<'a> { } async fn existing_dataset_schema(&self) -> Result> { - match DatasetBuilder::from_uri(self.dataset_uri).load().await { + let mut builder = DatasetBuilder::from_uri(self.dataset_uri); + let storage_options = self + .write_params + .and_then(|p| p.store_params.as_ref()) + .and_then(|p| p.storage_options.clone()); + if let Some(storage_options) = storage_options { + builder = builder.with_storage_options(storage_options); + } + match builder.load().await { Ok(dataset) => { // Use the schema from the dataset, because it has the correct // field ids.