From 7d4c26ad0e12d53d1c609789a66ff33216724c55 Mon Sep 17 00:00:00 2001 From: Shawn Chang Date: Wed, 4 Feb 2026 17:03:35 -0800 Subject: [PATCH 1/3] Add storage module and reorganize code --- crates/iceberg/src/io/file_io.rs | 2 +- crates/iceberg/src/io/mod.rs | 10 +--------- .../iceberg/src/io/{ => storage}/config/azdls.rs | 0 crates/iceberg/src/io/{ => storage}/config/gcs.rs | 2 +- crates/iceberg/src/io/{ => storage}/config/mod.rs | 0 crates/iceberg/src/io/{ => storage}/config/oss.rs | 0 crates/iceberg/src/io/{ => storage}/config/s3.rs | 1 + crates/iceberg/src/io/{ => storage}/local_fs.rs | 14 ++++++++++---- crates/iceberg/src/io/{ => storage}/memory.rs | 14 ++++++++++---- .../iceberg/src/io/{storage.rs => storage/mod.rs} | 13 ++++++++++++- .../iceberg/src/io/{ => storage}/opendal/azdls.rs | 9 ++++++++- crates/iceberg/src/io/{ => storage}/opendal/fs.rs | 0 crates/iceberg/src/io/{ => storage}/opendal/gcs.rs | 4 ++-- .../iceberg/src/io/{ => storage}/opendal/memory.rs | 0 crates/iceberg/src/io/{ => storage}/opendal/mod.rs | 6 ++---- crates/iceberg/src/io/{ => storage}/opendal/oss.rs | 2 +- crates/iceberg/src/io/{ => storage}/opendal/s3.rs | 4 ++-- 17 files changed, 51 insertions(+), 30 deletions(-) rename crates/iceberg/src/io/{ => storage}/config/azdls.rs (100%) rename crates/iceberg/src/io/{ => storage}/config/gcs.rs (100%) rename crates/iceberg/src/io/{ => storage}/config/mod.rs (100%) rename crates/iceberg/src/io/{ => storage}/config/oss.rs (100%) rename crates/iceberg/src/io/{ => storage}/config/s3.rs (99%) rename crates/iceberg/src/io/{ => storage}/local_fs.rs (99%) rename crates/iceberg/src/io/{ => storage}/memory.rs (99%) rename crates/iceberg/src/io/{storage.rs => storage/mod.rs} (93%) rename crates/iceberg/src/io/{ => storage}/opendal/azdls.rs (99%) rename crates/iceberg/src/io/{ => storage}/opendal/fs.rs (100%) rename crates/iceberg/src/io/{ => storage}/opendal/gcs.rs (98%) rename crates/iceberg/src/io/{ => storage}/opendal/memory.rs (100%) rename crates/iceberg/src/io/{ => storage}/opendal/mod.rs (99%) rename crates/iceberg/src/io/{ => storage}/opendal/oss.rs (95%) rename crates/iceberg/src/io/{ => storage}/opendal/s3.rs (99%) diff --git a/crates/iceberg/src/io/file_io.rs b/crates/iceberg/src/io/file_io.rs index 1ad71d8531..d7bd6609e0 100644 --- a/crates/iceberg/src/io/file_io.rs +++ b/crates/iceberg/src/io/file_io.rs @@ -23,7 +23,7 @@ use std::sync::Arc; use bytes::Bytes; use url::Url; -use super::opendal::OpenDalStorage; +use super::storage::OpenDalStorage; use super::storage::Storage; use crate::{Error, ErrorKind, Result}; diff --git a/crates/iceberg/src/io/mod.rs b/crates/iceberg/src/io/mod.rs index 5f65a15fc3..07f40d104e 100644 --- a/crates/iceberg/src/io/mod.rs +++ b/crates/iceberg/src/io/mod.rs @@ -66,19 +66,11 @@ //! - `new_input`: Create input file for reading. //! - `new_output`: Create output file for writing. -mod config; mod file_io; -mod local_fs; -mod memory; -mod opendal; mod storage; -pub use config::*; pub use file_io::*; -#[cfg(feature = "storage-s3")] -pub use opendal::CustomAwsCredentialLoader; -pub use opendal::{OpenDalStorage, OpenDalStorageFactory}; -pub use storage::{Storage, StorageConfig, StorageFactory}; +pub use storage::*; pub(crate) mod object_cache; diff --git a/crates/iceberg/src/io/config/azdls.rs b/crates/iceberg/src/io/storage/config/azdls.rs similarity index 100% rename from crates/iceberg/src/io/config/azdls.rs rename to crates/iceberg/src/io/storage/config/azdls.rs diff --git a/crates/iceberg/src/io/config/gcs.rs b/crates/iceberg/src/io/storage/config/gcs.rs similarity index 100% rename from crates/iceberg/src/io/config/gcs.rs rename to crates/iceberg/src/io/storage/config/gcs.rs index 5b11567f4d..61523b1629 100644 --- a/crates/iceberg/src/io/config/gcs.rs +++ b/crates/iceberg/src/io/storage/config/gcs.rs @@ -24,8 +24,8 @@ use serde::{Deserialize, Serialize}; use typed_builder::TypedBuilder; use super::StorageConfig; -use crate::Result; use crate::io::is_truthy; +use crate::Result; /// Google Cloud Project ID. pub const GCS_PROJECT_ID: &str = "gcs.project-id"; diff --git a/crates/iceberg/src/io/config/mod.rs b/crates/iceberg/src/io/storage/config/mod.rs similarity index 100% rename from crates/iceberg/src/io/config/mod.rs rename to crates/iceberg/src/io/storage/config/mod.rs diff --git a/crates/iceberg/src/io/config/oss.rs b/crates/iceberg/src/io/storage/config/oss.rs similarity index 100% rename from crates/iceberg/src/io/config/oss.rs rename to crates/iceberg/src/io/storage/config/oss.rs diff --git a/crates/iceberg/src/io/config/s3.rs b/crates/iceberg/src/io/storage/config/s3.rs similarity index 99% rename from crates/iceberg/src/io/config/s3.rs rename to crates/iceberg/src/io/storage/config/s3.rs index fae3a14757..6f5060e6f9 100644 --- a/crates/iceberg/src/io/config/s3.rs +++ b/crates/iceberg/src/io/storage/config/s3.rs @@ -125,6 +125,7 @@ pub struct S3Config { pub disable_config_load: bool, } + impl TryFrom<&StorageConfig> for S3Config { type Error = crate::Error; diff --git a/crates/iceberg/src/io/local_fs.rs b/crates/iceberg/src/io/storage/local_fs.rs similarity index 99% rename from crates/iceberg/src/io/local_fs.rs rename to crates/iceberg/src/io/storage/local_fs.rs index 0a55199f70..16848f51a3 100644 --- a/crates/iceberg/src/io/local_fs.rs +++ b/crates/iceberg/src/io/storage/local_fs.rs @@ -31,10 +31,8 @@ use async_trait::async_trait; use bytes::Bytes; use serde::{Deserialize, Serialize}; -use super::{ - FileMetadata, FileRead, FileWrite, InputFile, OutputFile, Storage, StorageConfig, - StorageFactory, -}; +use super::{Storage, StorageConfig, StorageFactory}; +use crate::io::{FileMetadata, FileRead, FileWrite, InputFile, OutputFile}; use crate::{Error, ErrorKind, Result}; /// Local filesystem storage implementation. @@ -52,6 +50,7 @@ use crate::{Error, ErrorKind, Result}; #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct LocalFsStorage; + impl LocalFsStorage { /// Create a new `LocalFsStorage` instance. pub fn new() -> Self { @@ -108,6 +107,7 @@ impl Storage for LocalFsStorage { }) } + async fn read(&self, path: &str) -> Result { let path = Self::normalize_path(path); let content = fs::read(&path).map_err(|e| { @@ -165,6 +165,7 @@ impl Storage for LocalFsStorage { })?; } + let file = fs::File::create(&path).map_err(|e| { Error::new( ErrorKind::Unexpected, @@ -215,6 +216,7 @@ pub struct LocalFsFileRead { file: std::sync::Mutex, } + impl LocalFsFileRead { /// Create a new `LocalFsFileRead` with the given file. pub fn new(file: fs::File) -> Self { @@ -269,6 +271,7 @@ impl LocalFsFileWrite { } } + #[async_trait] impl FileWrite for LocalFsFileWrite { async fn write(&mut self, bs: Bytes) -> Result<()> { @@ -324,6 +327,7 @@ impl StorageFactory for LocalFsStorageFactory { } } + #[cfg(test)] mod tests { use tempfile::TempDir; @@ -373,6 +377,7 @@ mod tests { assert_eq!(read_content, content); } + #[tokio::test] async fn test_local_fs_storage_exists() { let tmp_dir = TempDir::new().unwrap(); @@ -446,6 +451,7 @@ mod tests { assert!(!dir_path.exists()); } + #[tokio::test] async fn test_local_fs_storage_reader() { let tmp_dir = TempDir::new().unwrap(); diff --git a/crates/iceberg/src/io/memory.rs b/crates/iceberg/src/io/storage/memory.rs similarity index 99% rename from crates/iceberg/src/io/memory.rs rename to crates/iceberg/src/io/storage/memory.rs index 39f1f5db9d..f875efd01c 100644 --- a/crates/iceberg/src/io/memory.rs +++ b/crates/iceberg/src/io/storage/memory.rs @@ -30,10 +30,8 @@ use async_trait::async_trait; use bytes::Bytes; use serde::{Deserialize, Serialize}; -use super::{ - FileMetadata, FileRead, FileWrite, InputFile, OutputFile, Storage, StorageConfig, - StorageFactory, -}; +use super::{Storage, StorageConfig, StorageFactory}; +use crate::io::{FileMetadata, FileRead, FileWrite, InputFile, OutputFile}; use crate::{Error, ErrorKind, Result}; /// In-memory storage implementation. @@ -106,6 +104,7 @@ impl Storage for MemoryStorage { Ok(data.contains_key(&normalized)) } + async fn metadata(&self, path: &str) -> Result { let normalized = Self::normalize_path(path); let data = self.data.read().map_err(|e| { @@ -159,6 +158,7 @@ impl Storage for MemoryStorage { } } + async fn write(&self, path: &str, bs: Bytes) -> Result<()> { let normalized = Self::normalize_path(path); let mut data = self.data.write().map_err(|e| { @@ -229,6 +229,7 @@ impl Storage for MemoryStorage { } } + /// Factory for creating `MemoryStorage` instances. /// /// This factory implements `StorageFactory` and creates `MemoryStorage` @@ -279,6 +280,7 @@ impl FileRead for MemoryFileRead { } } + /// File writer for in-memory storage. /// /// This struct implements `FileWrite` for writing to in-memory storage. @@ -338,6 +340,7 @@ impl FileWrite for MemoryFileWrite { } } + #[cfg(test)] mod tests { use super::*; @@ -395,6 +398,7 @@ mod tests { assert_eq!(read_content, content); } + #[tokio::test] async fn test_memory_storage_exists() { let storage = MemoryStorage::new(); @@ -463,6 +467,7 @@ mod tests { assert!(storage.exists("memory://other/file.txt").await.unwrap()); } + #[tokio::test] async fn test_memory_storage_reader() { let storage = MemoryStorage::new(); @@ -534,6 +539,7 @@ mod tests { assert!(result.is_err()); } + #[test] fn test_memory_storage_serialization() { let storage = MemoryStorage::new(); diff --git a/crates/iceberg/src/io/storage.rs b/crates/iceberg/src/io/storage/mod.rs similarity index 93% rename from crates/iceberg/src/io/storage.rs rename to crates/iceberg/src/io/storage/mod.rs index 15cc85ab10..1aa46952e8 100644 --- a/crates/iceberg/src/io/storage.rs +++ b/crates/iceberg/src/io/storage/mod.rs @@ -17,6 +17,11 @@ //! Storage interfaces for Iceberg. +mod config; +mod local_fs; +mod memory; +mod opendal; + use std::fmt::Debug; use std::sync::Arc; @@ -25,7 +30,13 @@ use bytes::Bytes; use super::{FileMetadata, FileRead, FileWrite, InputFile, OutputFile}; use crate::Result; -pub use crate::io::config::StorageConfig; + +pub use config::*; +pub use local_fs::{LocalFsStorage, LocalFsStorageFactory}; +pub use memory::{MemoryStorage, MemoryStorageFactory}; +#[cfg(feature = "storage-s3")] +pub use opendal::CustomAwsCredentialLoader; +pub use opendal::{OpenDalStorage, OpenDalStorageFactory}; /// Trait for storage operations in Iceberg. /// diff --git a/crates/iceberg/src/io/opendal/azdls.rs b/crates/iceberg/src/io/storage/opendal/azdls.rs similarity index 99% rename from crates/iceberg/src/io/opendal/azdls.rs rename to crates/iceberg/src/io/storage/opendal/azdls.rs index c957fd62a3..7a86f4fe1c 100644 --- a/crates/iceberg/src/io/opendal/azdls.rs +++ b/crates/iceberg/src/io/storage/opendal/azdls.rs @@ -24,7 +24,7 @@ use opendal::services::AzdlsConfig; use serde::{Deserialize, Serialize}; use url::Url; -use crate::io::config::{ +use crate::io::storage::config::{ ADLS_ACCOUNT_KEY, ADLS_ACCOUNT_NAME, ADLS_AUTHORITY_HOST, ADLS_CLIENT_ID, ADLS_CLIENT_SECRET, ADLS_CONNECTION_STRING, ADLS_SAS_TOKEN, ADLS_TENANT_ID, }; @@ -49,6 +49,7 @@ pub(crate) fn azdls_config_parse(mut properties: HashMap) -> Res config.account_key = Some(account_key); } + if let Some(sas_token) = properties.remove(ADLS_SAS_TOKEN) { config.sas_token = Some(sas_token); } @@ -109,6 +110,7 @@ pub enum AzureStorageScheme { Wasbs, } + impl AzureStorageScheme { // Returns the respective encrypted or plain-text HTTP scheme. pub fn as_http_scheme(&self) -> &str { @@ -169,6 +171,7 @@ fn match_path_with_config( ); } + if let Some(ref configured_endpoint) = config.endpoint { let passed_http_scheme = path.scheme.as_http_scheme(); ensure_data_valid!( @@ -225,6 +228,7 @@ struct AzureStoragePath { path: String, } + impl AzureStoragePath { /// Converts the AzureStoragePath into a full endpoint URL. /// @@ -282,6 +286,7 @@ fn parse_azure_storage_endpoint(url: &Url) -> Result<(&str, &str, &str)> { )); } + let (storage, endpoint_suffix) = endpoint.split_once('.').ok_or(Error::new( ErrorKind::DataInvalid, "AzureStoragePath: No storage service", @@ -382,6 +387,7 @@ mod tests { } } + #[test] fn test_azdls_create_operator() { let test_cases = vec![ @@ -483,6 +489,7 @@ mod tests { } } + #[test] fn test_azure_storage_path_parse() { let test_cases = vec![ diff --git a/crates/iceberg/src/io/opendal/fs.rs b/crates/iceberg/src/io/storage/opendal/fs.rs similarity index 100% rename from crates/iceberg/src/io/opendal/fs.rs rename to crates/iceberg/src/io/storage/opendal/fs.rs diff --git a/crates/iceberg/src/io/opendal/gcs.rs b/crates/iceberg/src/io/storage/opendal/gcs.rs similarity index 98% rename from crates/iceberg/src/io/opendal/gcs.rs rename to crates/iceberg/src/io/storage/opendal/gcs.rs index 5c6145d32b..a21e89fad9 100644 --- a/crates/iceberg/src/io/opendal/gcs.rs +++ b/crates/iceberg/src/io/storage/opendal/gcs.rs @@ -22,11 +22,11 @@ use opendal::Operator; use opendal::services::GcsConfig; use url::Url; -use crate::io::config::{ +use crate::io::is_truthy; +use crate::io::storage::config::{ GCS_ALLOW_ANONYMOUS, GCS_CREDENTIALS_JSON, GCS_DISABLE_CONFIG_LOAD, GCS_DISABLE_VM_METADATA, GCS_NO_AUTH, GCS_SERVICE_PATH, GCS_TOKEN, }; -use crate::io::is_truthy; use crate::{Error, ErrorKind, Result}; /// Parse iceberg properties to [`GcsConfig`]. diff --git a/crates/iceberg/src/io/opendal/memory.rs b/crates/iceberg/src/io/storage/opendal/memory.rs similarity index 100% rename from crates/iceberg/src/io/opendal/memory.rs rename to crates/iceberg/src/io/storage/opendal/memory.rs diff --git a/crates/iceberg/src/io/opendal/mod.rs b/crates/iceberg/src/io/storage/opendal/mod.rs similarity index 99% rename from crates/iceberg/src/io/opendal/mod.rs rename to crates/iceberg/src/io/storage/opendal/mod.rs index fb49dc9e3f..4f342a73eb 100644 --- a/crates/iceberg/src/io/opendal/mod.rs +++ b/crates/iceberg/src/io/storage/opendal/mod.rs @@ -38,10 +38,8 @@ use opendal::{Operator, Scheme}; pub use s3::CustomAwsCredentialLoader; use serde::{Deserialize, Serialize}; -use super::{ - FileIOBuilder, FileMetadata, FileRead, FileWrite, InputFile, OutputFile, Storage, - StorageConfig, StorageFactory, -}; +use super::{Storage, StorageConfig, StorageFactory}; +use crate::io::{FileIOBuilder, FileMetadata, FileRead, FileWrite, InputFile, OutputFile}; use crate::{Error, ErrorKind, Result}; #[cfg(feature = "storage-azdls")] diff --git a/crates/iceberg/src/io/opendal/oss.rs b/crates/iceberg/src/io/storage/opendal/oss.rs similarity index 95% rename from crates/iceberg/src/io/opendal/oss.rs rename to crates/iceberg/src/io/storage/opendal/oss.rs index 83fc1424aa..91a1c0a667 100644 --- a/crates/iceberg/src/io/opendal/oss.rs +++ b/crates/iceberg/src/io/storage/opendal/oss.rs @@ -21,7 +21,7 @@ use opendal::services::OssConfig; use opendal::{Configurator, Operator}; use url::Url; -use crate::io::config::{OSS_ACCESS_KEY_ID, OSS_ACCESS_KEY_SECRET, OSS_ENDPOINT}; +use crate::io::storage::config::{OSS_ACCESS_KEY_ID, OSS_ACCESS_KEY_SECRET, OSS_ENDPOINT}; use crate::{Error, ErrorKind, Result}; /// Parse iceberg props to oss config. diff --git a/crates/iceberg/src/io/opendal/s3.rs b/crates/iceberg/src/io/storage/opendal/s3.rs similarity index 99% rename from crates/iceberg/src/io/opendal/s3.rs rename to crates/iceberg/src/io/storage/opendal/s3.rs index bf7399e01b..9ee30e06e7 100644 --- a/crates/iceberg/src/io/opendal/s3.rs +++ b/crates/iceberg/src/io/storage/opendal/s3.rs @@ -25,13 +25,13 @@ pub use reqsign::{AwsCredential, AwsCredentialLoad}; use reqwest::Client; use url::Url; -use crate::io::config::{ +use crate::io::is_truthy; +use crate::io::storage::config::{ CLIENT_REGION, S3_ACCESS_KEY_ID, S3_ALLOW_ANONYMOUS, S3_ASSUME_ROLE_ARN, S3_ASSUME_ROLE_EXTERNAL_ID, S3_ASSUME_ROLE_SESSION_NAME, S3_DISABLE_CONFIG_LOAD, S3_DISABLE_EC2_METADATA, S3_ENDPOINT, S3_PATH_STYLE_ACCESS, S3_REGION, S3_SECRET_ACCESS_KEY, S3_SESSION_TOKEN, S3_SSE_KEY, S3_SSE_MD5, S3_SSE_TYPE, }; -use crate::io::is_truthy; use crate::{Error, ErrorKind, Result}; /// Parse iceberg props to s3 config. From d5a6eacf723822d0ef99e5013b79e2347ab69f81 Mon Sep 17 00:00:00 2001 From: Shawn Chang Date: Wed, 4 Feb 2026 17:04:23 -0800 Subject: [PATCH 2/3] fmt --- crates/iceberg/src/io/file_io.rs | 3 +-- crates/iceberg/src/io/storage/config/gcs.rs | 2 +- crates/iceberg/src/io/storage/config/s3.rs | 1 - crates/iceberg/src/io/storage/local_fs.rs | 8 -------- crates/iceberg/src/io/storage/memory.rs | 8 -------- crates/iceberg/src/io/storage/mod.rs | 7 +++---- crates/iceberg/src/io/storage/opendal/azdls.rs | 7 ------- 7 files changed, 5 insertions(+), 31 deletions(-) diff --git a/crates/iceberg/src/io/file_io.rs b/crates/iceberg/src/io/file_io.rs index d7bd6609e0..267f295369 100644 --- a/crates/iceberg/src/io/file_io.rs +++ b/crates/iceberg/src/io/file_io.rs @@ -23,8 +23,7 @@ use std::sync::Arc; use bytes::Bytes; use url::Url; -use super::storage::OpenDalStorage; -use super::storage::Storage; +use super::storage::{OpenDalStorage, Storage}; use crate::{Error, ErrorKind, Result}; /// FileIO implementation, used to manipulate files in underlying storage. diff --git a/crates/iceberg/src/io/storage/config/gcs.rs b/crates/iceberg/src/io/storage/config/gcs.rs index 61523b1629..5b11567f4d 100644 --- a/crates/iceberg/src/io/storage/config/gcs.rs +++ b/crates/iceberg/src/io/storage/config/gcs.rs @@ -24,8 +24,8 @@ use serde::{Deserialize, Serialize}; use typed_builder::TypedBuilder; use super::StorageConfig; -use crate::io::is_truthy; use crate::Result; +use crate::io::is_truthy; /// Google Cloud Project ID. pub const GCS_PROJECT_ID: &str = "gcs.project-id"; diff --git a/crates/iceberg/src/io/storage/config/s3.rs b/crates/iceberg/src/io/storage/config/s3.rs index 6f5060e6f9..fae3a14757 100644 --- a/crates/iceberg/src/io/storage/config/s3.rs +++ b/crates/iceberg/src/io/storage/config/s3.rs @@ -125,7 +125,6 @@ pub struct S3Config { pub disable_config_load: bool, } - impl TryFrom<&StorageConfig> for S3Config { type Error = crate::Error; diff --git a/crates/iceberg/src/io/storage/local_fs.rs b/crates/iceberg/src/io/storage/local_fs.rs index 16848f51a3..134242d031 100644 --- a/crates/iceberg/src/io/storage/local_fs.rs +++ b/crates/iceberg/src/io/storage/local_fs.rs @@ -50,7 +50,6 @@ use crate::{Error, ErrorKind, Result}; #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct LocalFsStorage; - impl LocalFsStorage { /// Create a new `LocalFsStorage` instance. pub fn new() -> Self { @@ -107,7 +106,6 @@ impl Storage for LocalFsStorage { }) } - async fn read(&self, path: &str) -> Result { let path = Self::normalize_path(path); let content = fs::read(&path).map_err(|e| { @@ -165,7 +163,6 @@ impl Storage for LocalFsStorage { })?; } - let file = fs::File::create(&path).map_err(|e| { Error::new( ErrorKind::Unexpected, @@ -216,7 +213,6 @@ pub struct LocalFsFileRead { file: std::sync::Mutex, } - impl LocalFsFileRead { /// Create a new `LocalFsFileRead` with the given file. pub fn new(file: fs::File) -> Self { @@ -271,7 +267,6 @@ impl LocalFsFileWrite { } } - #[async_trait] impl FileWrite for LocalFsFileWrite { async fn write(&mut self, bs: Bytes) -> Result<()> { @@ -327,7 +322,6 @@ impl StorageFactory for LocalFsStorageFactory { } } - #[cfg(test)] mod tests { use tempfile::TempDir; @@ -377,7 +371,6 @@ mod tests { assert_eq!(read_content, content); } - #[tokio::test] async fn test_local_fs_storage_exists() { let tmp_dir = TempDir::new().unwrap(); @@ -451,7 +444,6 @@ mod tests { assert!(!dir_path.exists()); } - #[tokio::test] async fn test_local_fs_storage_reader() { let tmp_dir = TempDir::new().unwrap(); diff --git a/crates/iceberg/src/io/storage/memory.rs b/crates/iceberg/src/io/storage/memory.rs index f875efd01c..ef1168b17a 100644 --- a/crates/iceberg/src/io/storage/memory.rs +++ b/crates/iceberg/src/io/storage/memory.rs @@ -104,7 +104,6 @@ impl Storage for MemoryStorage { Ok(data.contains_key(&normalized)) } - async fn metadata(&self, path: &str) -> Result { let normalized = Self::normalize_path(path); let data = self.data.read().map_err(|e| { @@ -158,7 +157,6 @@ impl Storage for MemoryStorage { } } - async fn write(&self, path: &str, bs: Bytes) -> Result<()> { let normalized = Self::normalize_path(path); let mut data = self.data.write().map_err(|e| { @@ -229,7 +227,6 @@ impl Storage for MemoryStorage { } } - /// Factory for creating `MemoryStorage` instances. /// /// This factory implements `StorageFactory` and creates `MemoryStorage` @@ -280,7 +277,6 @@ impl FileRead for MemoryFileRead { } } - /// File writer for in-memory storage. /// /// This struct implements `FileWrite` for writing to in-memory storage. @@ -340,7 +336,6 @@ impl FileWrite for MemoryFileWrite { } } - #[cfg(test)] mod tests { use super::*; @@ -398,7 +393,6 @@ mod tests { assert_eq!(read_content, content); } - #[tokio::test] async fn test_memory_storage_exists() { let storage = MemoryStorage::new(); @@ -467,7 +461,6 @@ mod tests { assert!(storage.exists("memory://other/file.txt").await.unwrap()); } - #[tokio::test] async fn test_memory_storage_reader() { let storage = MemoryStorage::new(); @@ -539,7 +532,6 @@ mod tests { assert!(result.is_err()); } - #[test] fn test_memory_storage_serialization() { let storage = MemoryStorage::new(); diff --git a/crates/iceberg/src/io/storage/mod.rs b/crates/iceberg/src/io/storage/mod.rs index 1aa46952e8..31ceaf2436 100644 --- a/crates/iceberg/src/io/storage/mod.rs +++ b/crates/iceberg/src/io/storage/mod.rs @@ -27,10 +27,6 @@ use std::sync::Arc; use async_trait::async_trait; use bytes::Bytes; - -use super::{FileMetadata, FileRead, FileWrite, InputFile, OutputFile}; -use crate::Result; - pub use config::*; pub use local_fs::{LocalFsStorage, LocalFsStorageFactory}; pub use memory::{MemoryStorage, MemoryStorageFactory}; @@ -38,6 +34,9 @@ pub use memory::{MemoryStorage, MemoryStorageFactory}; pub use opendal::CustomAwsCredentialLoader; pub use opendal::{OpenDalStorage, OpenDalStorageFactory}; +use super::{FileMetadata, FileRead, FileWrite, InputFile, OutputFile}; +use crate::Result; + /// Trait for storage operations in Iceberg. /// /// The trait supports serialization via `typetag`, allowing storage instances to be diff --git a/crates/iceberg/src/io/storage/opendal/azdls.rs b/crates/iceberg/src/io/storage/opendal/azdls.rs index 7a86f4fe1c..1d85aa3d3f 100644 --- a/crates/iceberg/src/io/storage/opendal/azdls.rs +++ b/crates/iceberg/src/io/storage/opendal/azdls.rs @@ -49,7 +49,6 @@ pub(crate) fn azdls_config_parse(mut properties: HashMap) -> Res config.account_key = Some(account_key); } - if let Some(sas_token) = properties.remove(ADLS_SAS_TOKEN) { config.sas_token = Some(sas_token); } @@ -110,7 +109,6 @@ pub enum AzureStorageScheme { Wasbs, } - impl AzureStorageScheme { // Returns the respective encrypted or plain-text HTTP scheme. pub fn as_http_scheme(&self) -> &str { @@ -171,7 +169,6 @@ fn match_path_with_config( ); } - if let Some(ref configured_endpoint) = config.endpoint { let passed_http_scheme = path.scheme.as_http_scheme(); ensure_data_valid!( @@ -228,7 +225,6 @@ struct AzureStoragePath { path: String, } - impl AzureStoragePath { /// Converts the AzureStoragePath into a full endpoint URL. /// @@ -286,7 +282,6 @@ fn parse_azure_storage_endpoint(url: &Url) -> Result<(&str, &str, &str)> { )); } - let (storage, endpoint_suffix) = endpoint.split_once('.').ok_or(Error::new( ErrorKind::DataInvalid, "AzureStoragePath: No storage service", @@ -387,7 +382,6 @@ mod tests { } } - #[test] fn test_azdls_create_operator() { let test_cases = vec![ @@ -489,7 +483,6 @@ mod tests { } } - #[test] fn test_azure_storage_path_parse() { let test_cases = vec![ From 92fdca51f5dfa9273f93528963cd05d6cce7df43 Mon Sep 17 00:00:00 2001 From: Shawn Chang Date: Wed, 4 Feb 2026 17:09:30 -0800 Subject: [PATCH 3/3] clean up imports --- crates/iceberg/src/io/storage/local_fs.rs | 6 ++++-- crates/iceberg/src/io/storage/memory.rs | 6 ++++-- crates/iceberg/src/io/storage/opendal/azdls.rs | 2 +- crates/iceberg/src/io/storage/opendal/gcs.rs | 5 ++--- crates/iceberg/src/io/storage/opendal/mod.rs | 6 ++++-- crates/iceberg/src/io/storage/opendal/oss.rs | 2 +- crates/iceberg/src/io/storage/opendal/s3.rs | 5 ++--- 7 files changed, 18 insertions(+), 14 deletions(-) diff --git a/crates/iceberg/src/io/storage/local_fs.rs b/crates/iceberg/src/io/storage/local_fs.rs index 134242d031..d6dd5b433b 100644 --- a/crates/iceberg/src/io/storage/local_fs.rs +++ b/crates/iceberg/src/io/storage/local_fs.rs @@ -31,8 +31,10 @@ use async_trait::async_trait; use bytes::Bytes; use serde::{Deserialize, Serialize}; -use super::{Storage, StorageConfig, StorageFactory}; -use crate::io::{FileMetadata, FileRead, FileWrite, InputFile, OutputFile}; +use crate::io::{ + FileMetadata, FileRead, FileWrite, InputFile, OutputFile, Storage, StorageConfig, + StorageFactory, +}; use crate::{Error, ErrorKind, Result}; /// Local filesystem storage implementation. diff --git a/crates/iceberg/src/io/storage/memory.rs b/crates/iceberg/src/io/storage/memory.rs index ef1168b17a..cb01ee4709 100644 --- a/crates/iceberg/src/io/storage/memory.rs +++ b/crates/iceberg/src/io/storage/memory.rs @@ -30,8 +30,10 @@ use async_trait::async_trait; use bytes::Bytes; use serde::{Deserialize, Serialize}; -use super::{Storage, StorageConfig, StorageFactory}; -use crate::io::{FileMetadata, FileRead, FileWrite, InputFile, OutputFile}; +use crate::io::{ + FileMetadata, FileRead, FileWrite, InputFile, OutputFile, Storage, StorageConfig, + StorageFactory, +}; use crate::{Error, ErrorKind, Result}; /// In-memory storage implementation. diff --git a/crates/iceberg/src/io/storage/opendal/azdls.rs b/crates/iceberg/src/io/storage/opendal/azdls.rs index 1d85aa3d3f..759ff301ea 100644 --- a/crates/iceberg/src/io/storage/opendal/azdls.rs +++ b/crates/iceberg/src/io/storage/opendal/azdls.rs @@ -24,7 +24,7 @@ use opendal::services::AzdlsConfig; use serde::{Deserialize, Serialize}; use url::Url; -use crate::io::storage::config::{ +use crate::io::{ ADLS_ACCOUNT_KEY, ADLS_ACCOUNT_NAME, ADLS_AUTHORITY_HOST, ADLS_CLIENT_ID, ADLS_CLIENT_SECRET, ADLS_CONNECTION_STRING, ADLS_SAS_TOKEN, ADLS_TENANT_ID, }; diff --git a/crates/iceberg/src/io/storage/opendal/gcs.rs b/crates/iceberg/src/io/storage/opendal/gcs.rs index a21e89fad9..4cb8aa8591 100644 --- a/crates/iceberg/src/io/storage/opendal/gcs.rs +++ b/crates/iceberg/src/io/storage/opendal/gcs.rs @@ -22,10 +22,9 @@ use opendal::Operator; use opendal::services::GcsConfig; use url::Url; -use crate::io::is_truthy; -use crate::io::storage::config::{ +use crate::io::{ GCS_ALLOW_ANONYMOUS, GCS_CREDENTIALS_JSON, GCS_DISABLE_CONFIG_LOAD, GCS_DISABLE_VM_METADATA, - GCS_NO_AUTH, GCS_SERVICE_PATH, GCS_TOKEN, + GCS_NO_AUTH, GCS_SERVICE_PATH, GCS_TOKEN, is_truthy, }; use crate::{Error, ErrorKind, Result}; diff --git a/crates/iceberg/src/io/storage/opendal/mod.rs b/crates/iceberg/src/io/storage/opendal/mod.rs index 4f342a73eb..ee0df20fce 100644 --- a/crates/iceberg/src/io/storage/opendal/mod.rs +++ b/crates/iceberg/src/io/storage/opendal/mod.rs @@ -38,8 +38,10 @@ use opendal::{Operator, Scheme}; pub use s3::CustomAwsCredentialLoader; use serde::{Deserialize, Serialize}; -use super::{Storage, StorageConfig, StorageFactory}; -use crate::io::{FileIOBuilder, FileMetadata, FileRead, FileWrite, InputFile, OutputFile}; +use crate::io::{ + FileIOBuilder, FileMetadata, FileRead, FileWrite, InputFile, OutputFile, Storage, + StorageConfig, StorageFactory, +}; use crate::{Error, ErrorKind, Result}; #[cfg(feature = "storage-azdls")] diff --git a/crates/iceberg/src/io/storage/opendal/oss.rs b/crates/iceberg/src/io/storage/opendal/oss.rs index 91a1c0a667..98b68f7cde 100644 --- a/crates/iceberg/src/io/storage/opendal/oss.rs +++ b/crates/iceberg/src/io/storage/opendal/oss.rs @@ -21,7 +21,7 @@ use opendal::services::OssConfig; use opendal::{Configurator, Operator}; use url::Url; -use crate::io::storage::config::{OSS_ACCESS_KEY_ID, OSS_ACCESS_KEY_SECRET, OSS_ENDPOINT}; +use crate::io::{OSS_ACCESS_KEY_ID, OSS_ACCESS_KEY_SECRET, OSS_ENDPOINT}; use crate::{Error, ErrorKind, Result}; /// Parse iceberg props to oss config. diff --git a/crates/iceberg/src/io/storage/opendal/s3.rs b/crates/iceberg/src/io/storage/opendal/s3.rs index 9ee30e06e7..6b4d64e3a6 100644 --- a/crates/iceberg/src/io/storage/opendal/s3.rs +++ b/crates/iceberg/src/io/storage/opendal/s3.rs @@ -25,12 +25,11 @@ pub use reqsign::{AwsCredential, AwsCredentialLoad}; use reqwest::Client; use url::Url; -use crate::io::is_truthy; -use crate::io::storage::config::{ +use crate::io::{ CLIENT_REGION, S3_ACCESS_KEY_ID, S3_ALLOW_ANONYMOUS, S3_ASSUME_ROLE_ARN, S3_ASSUME_ROLE_EXTERNAL_ID, S3_ASSUME_ROLE_SESSION_NAME, S3_DISABLE_CONFIG_LOAD, S3_DISABLE_EC2_METADATA, S3_ENDPOINT, S3_PATH_STYLE_ACCESS, S3_REGION, S3_SECRET_ACCESS_KEY, - S3_SESSION_TOKEN, S3_SSE_KEY, S3_SSE_MD5, S3_SSE_TYPE, + S3_SESSION_TOKEN, S3_SSE_KEY, S3_SSE_MD5, S3_SSE_TYPE, is_truthy, }; use crate::{Error, ErrorKind, Result};