From 5bfd2aacc040604a8082887815739af6d55e14b6 Mon Sep 17 00:00:00 2001 From: majin1102 Date: Tue, 23 Dec 2025 19:57:21 +0800 Subject: [PATCH 1/2] init --- rust/lance/src/dataset/cleanup.rs | 51 +++++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 10 deletions(-) diff --git a/rust/lance/src/dataset/cleanup.rs b/rust/lance/src/dataset/cleanup.rs index a65cb8a85ff..a07861dda7b 100644 --- a/rust/lance/src/dataset/cleanup.rs +++ b/rust/lance/src/dataset/cleanup.rs @@ -669,7 +669,7 @@ pub async fn auto_cleanup_hook( } }; - if manifest.version % interval != 0 { + if interval != 0 && manifest.version % interval != 0 { return Ok(None); } } else { @@ -1345,6 +1345,15 @@ mod tests { assert_eq!(removed.old_versions, 1); } + // Helper function to check that the number of files is correct. + async fn check_num_files(fixture: &MockDatasetFixture, num_expected_files: usize) { + let file_count = fixture.count_files().await.unwrap(); + + assert_eq!(file_count.num_data_files, num_expected_files); + assert_eq!(file_count.num_manifest_files, num_expected_files); + assert_eq!(file_count.num_tx_files, num_expected_files); + } + #[tokio::test] async fn auto_cleanup_old_versions() { // Every n commits, all versions older than T should be deleted. @@ -1371,15 +1380,6 @@ mod tests { ) .unwrap(); - // Helper function to check that the number of files is correct. - async fn check_num_files(fixture: &MockDatasetFixture, num_expected_files: usize) { - let file_count = fixture.count_files().await.unwrap(); - - assert_eq!(file_count.num_data_files, num_expected_files); - assert_eq!(file_count.num_manifest_files, num_expected_files); - assert_eq!(file_count.num_tx_files, num_expected_files); - } - // First, write many files within the "older_than" window. Check that // no files are automatically cleaned up. for num_expected_files in 2..2 * cleanup_interval { @@ -1441,6 +1441,37 @@ mod tests { } } + #[tokio::test] + async fn test_auto_cleanup_interval_zero() { + let fixture = MockDatasetFixture::try_new().unwrap(); + + fixture.create_some_data().await.unwrap(); + fixture.overwrite_some_data().await.unwrap(); + fixture.overwrite_some_data().await.unwrap(); + check_num_files(&fixture, 3).await; + + let mut dataset = fixture.open().await.unwrap(); + let mut config_updates = HashMap::new(); + config_updates.insert( + "lance.auto_cleanup.interval".to_string(), + Some("0".to_string()), + ); + config_updates.insert( + "lance.auto_cleanup.retain_versions".to_string(), + Some("1".to_string()), + ); + dataset.update_config(config_updates).replace().await.unwrap(); + + fixture.overwrite_some_data().await.unwrap(); + fixture.overwrite_some_data().await.unwrap(); + // The last version before the new commit is retained, means we have 2 versions to assert + check_num_files(&fixture, 2).await; + + fixture.overwrite_some_data().await.unwrap(); + check_num_files(&fixture, 2).await; + } + + #[tokio::test] async fn cleanup_recent_verified_files() { let fixture = MockDatasetFixture::try_new().unwrap(); From 539e3119723759bcb9d9a698dc5e4e508e0e1aab Mon Sep 17 00:00:00 2001 From: majin1102 Date: Wed, 24 Dec 2025 14:55:12 +0800 Subject: [PATCH 2/2] fmt --- rust/lance/src/dataset/cleanup.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/rust/lance/src/dataset/cleanup.rs b/rust/lance/src/dataset/cleanup.rs index a07861dda7b..dc948c251b8 100644 --- a/rust/lance/src/dataset/cleanup.rs +++ b/rust/lance/src/dataset/cleanup.rs @@ -1460,7 +1460,11 @@ mod tests { "lance.auto_cleanup.retain_versions".to_string(), Some("1".to_string()), ); - dataset.update_config(config_updates).replace().await.unwrap(); + dataset + .update_config(config_updates) + .replace() + .await + .unwrap(); fixture.overwrite_some_data().await.unwrap(); fixture.overwrite_some_data().await.unwrap(); @@ -1471,7 +1475,6 @@ mod tests { check_num_files(&fixture, 2).await; } - #[tokio::test] async fn cleanup_recent_verified_files() { let fixture = MockDatasetFixture::try_new().unwrap();