From c5238fc95b6ffd8c8880d016b6716459e32c1d4b Mon Sep 17 00:00:00 2001 From: ClSlaid Date: Tue, 26 Sep 2023 15:12:02 +0800 Subject: [PATCH 1/3] porting: making OpenDAL compilable on 32hf platforms This patch should allow OpenDAL to compile on 32 bit platforms (hard float) with minium side effects. changes: - opendal::types::capability::Capability: - write_multi_max_size - write_multi_min_size - write_multi_align_size Option -> Option Signed-off-by: ClSlaid --- core/src/layers/complete.rs | 10 ++++++++-- core/src/types/capability.rs | 6 +++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/core/src/layers/complete.rs b/core/src/layers/complete.rs index 82e8066e4424..4142deb1f243 100644 --- a/core/src/layers/complete.rs +++ b/core/src/layers/complete.rs @@ -406,9 +406,15 @@ impl LayeredAccessor for CompleteReaderAccessor { } // Calculate buffer size. - let buffer_size = args.buffer().map(|mut size| { + let buffer_size = args.buffer().map(|size| { + let mut size = size as u64; if let Some(v) = capability.write_multi_max_size { size = cmp::min(v, size); + + #[cfg(target_pointer_width = "32")] + { + size = cmp::min(size, u32::MAX as u64); + } } if let Some(v) = capability.write_multi_min_size { size = cmp::max(v, size); @@ -427,7 +433,7 @@ impl LayeredAccessor for CompleteReaderAccessor { let w = match buffer_size { None => oio::TwoWaysWriter::One(w), - Some(size) => oio::TwoWaysWriter::Two(oio::ExactBufWriter::new(w, size)), + Some(size) => oio::TwoWaysWriter::Two(oio::ExactBufWriter::new(w, size as usize)), }; Ok((rp, w)) diff --git a/core/src/types/capability.rs b/core/src/types/capability.rs index abe7eb9c44cd..7f77c7c2104a 100644 --- a/core/src/types/capability.rs +++ b/core/src/types/capability.rs @@ -87,15 +87,15 @@ pub struct Capability { /// write_multi_max_size is the max size that services support in write_multi. /// /// For example, AWS S3 supports 5GiB as max in write_multi. - pub write_multi_max_size: Option, + pub write_multi_max_size: Option, /// write_multi_min_size is the min size that services support in write_multi. /// /// For example, AWS S3 requires at least 5MiB in write_multi expect the last one. - pub write_multi_min_size: Option, + pub write_multi_min_size: Option, /// write_multi_align_size is the align size that services required in write_multi. /// /// For example, Google GCS requires align size to 256KiB in write_multi. - pub write_multi_align_size: Option, + pub write_multi_align_size: Option, /// If operator supports create dir. pub create_dir: bool, From 9634e9e01f4c282f64442706950b32ba2453b37a Mon Sep 17 00:00:00 2001 From: ClSlaid Date: Tue, 26 Sep 2023 15:32:32 +0800 Subject: [PATCH 2/3] patch: merely restrict literals. Signed-off-by: ClSlaid --- core/src/layers/complete.rs | 10 ++-------- core/src/services/cos/backend.rs | 6 +++++- core/src/services/obs/backend.rs | 6 +++++- core/src/services/oss/backend.rs | 6 +++++- core/src/services/s3/backend.rs | 6 +++++- core/src/types/capability.rs | 6 +++--- 6 files changed, 25 insertions(+), 15 deletions(-) diff --git a/core/src/layers/complete.rs b/core/src/layers/complete.rs index 4142deb1f243..82e8066e4424 100644 --- a/core/src/layers/complete.rs +++ b/core/src/layers/complete.rs @@ -406,15 +406,9 @@ impl LayeredAccessor for CompleteReaderAccessor { } // Calculate buffer size. - let buffer_size = args.buffer().map(|size| { - let mut size = size as u64; + let buffer_size = args.buffer().map(|mut size| { if let Some(v) = capability.write_multi_max_size { size = cmp::min(v, size); - - #[cfg(target_pointer_width = "32")] - { - size = cmp::min(size, u32::MAX as u64); - } } if let Some(v) = capability.write_multi_min_size { size = cmp::max(v, size); @@ -433,7 +427,7 @@ impl LayeredAccessor for CompleteReaderAccessor { let w = match buffer_size { None => oio::TwoWaysWriter::One(w), - Some(size) => oio::TwoWaysWriter::Two(oio::ExactBufWriter::new(w, size as usize)), + Some(size) => oio::TwoWaysWriter::Two(oio::ExactBufWriter::new(w, size)), }; Ok((rp, w)) diff --git a/core/src/services/cos/backend.rs b/core/src/services/cos/backend.rs index e01da22cbd41..893f064afd29 100644 --- a/core/src/services/cos/backend.rs +++ b/core/src/services/cos/backend.rs @@ -277,7 +277,11 @@ impl Accessor for CosBackend { // The max multipart size of COS is 5 GiB. // // ref: - write_multi_max_size: Some(5 * 1024 * 1024 * 1024), + write_multi_max_size: if cfg!(target_pointer_width = "32") { + Some(usize::MAX) + } else { + Some(5 * 1024 * 1024 * 1024) + }, delete: true, create_dir: true, diff --git a/core/src/services/obs/backend.rs b/core/src/services/obs/backend.rs index 3d6281e863b7..087ef02e61ed 100644 --- a/core/src/services/obs/backend.rs +++ b/core/src/services/obs/backend.rs @@ -283,7 +283,11 @@ impl Accessor for ObsBackend { // The max multipart size of OBS is 5 GiB. // // ref: - write_multi_max_size: Some(5 * 1024 * 1024 * 1024), + write_multi_max_size: if cfg!(target_pointer_width = "32") { + Some(usize::MAX) + } else { + Some(5 * 1024 * 1024 * 1024) + }, delete: true, create_dir: true, diff --git a/core/src/services/oss/backend.rs b/core/src/services/oss/backend.rs index 1e32d8ea04e9..7b0815436968 100644 --- a/core/src/services/oss/backend.rs +++ b/core/src/services/oss/backend.rs @@ -410,7 +410,11 @@ impl Accessor for OssBackend { // The max multipart size of OSS is 5 GiB. // // ref: - write_multi_max_size: Some(5 * 1024 * 1024 * 1024), + write_multi_max_size: if cfg!(target_pointer_width = "32") { + Some(usize::MAX) + } else { + Some(5 * 1024 * 1024 * 1024) + }, delete: true, create_dir: true, diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs index 88dea3af9480..610255e9bf69 100644 --- a/core/src/services/s3/backend.rs +++ b/core/src/services/s3/backend.rs @@ -923,7 +923,11 @@ impl Accessor for S3Backend { // The max multipart size of S3 is 5 GiB. // // ref: - write_multi_max_size: Some(5 * 1024 * 1024 * 1024), + write_multi_max_size: if cfg!(target_pointer_width = "32") { + Some(usize::MAX) + } else { + Some(5 * 1024 * 1024 * 1024) + }, create_dir: true, delete: true, diff --git a/core/src/types/capability.rs b/core/src/types/capability.rs index 7f77c7c2104a..abe7eb9c44cd 100644 --- a/core/src/types/capability.rs +++ b/core/src/types/capability.rs @@ -87,15 +87,15 @@ pub struct Capability { /// write_multi_max_size is the max size that services support in write_multi. /// /// For example, AWS S3 supports 5GiB as max in write_multi. - pub write_multi_max_size: Option, + pub write_multi_max_size: Option, /// write_multi_min_size is the min size that services support in write_multi. /// /// For example, AWS S3 requires at least 5MiB in write_multi expect the last one. - pub write_multi_min_size: Option, + pub write_multi_min_size: Option, /// write_multi_align_size is the align size that services required in write_multi. /// /// For example, Google GCS requires align size to 256KiB in write_multi. - pub write_multi_align_size: Option, + pub write_multi_align_size: Option, /// If operator supports create dir. pub create_dir: bool, From 968531dc38462ec995214ee44298a8d67c578918 Mon Sep 17 00:00:00 2001 From: ClSlaid Date: Tue, 26 Sep 2023 15:48:45 +0800 Subject: [PATCH 3/3] patch: configure pointer_width on 64bit instead this should make code clean Signed-off-by: ClSlaid --- core/src/services/cos/backend.rs | 6 +++--- core/src/services/obs/backend.rs | 6 +++--- core/src/services/oss/backend.rs | 6 +++--- core/src/services/s3/backend.rs | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/core/src/services/cos/backend.rs b/core/src/services/cos/backend.rs index 893f064afd29..be19e08717d8 100644 --- a/core/src/services/cos/backend.rs +++ b/core/src/services/cos/backend.rs @@ -277,10 +277,10 @@ impl Accessor for CosBackend { // The max multipart size of COS is 5 GiB. // // ref: - write_multi_max_size: if cfg!(target_pointer_width = "32") { - Some(usize::MAX) - } else { + write_multi_max_size: if cfg!(target_pointer_width = "64") { Some(5 * 1024 * 1024 * 1024) + } else { + Some(usize::MAX) }, delete: true, diff --git a/core/src/services/obs/backend.rs b/core/src/services/obs/backend.rs index 087ef02e61ed..cff7c254f4e1 100644 --- a/core/src/services/obs/backend.rs +++ b/core/src/services/obs/backend.rs @@ -283,10 +283,10 @@ impl Accessor for ObsBackend { // The max multipart size of OBS is 5 GiB. // // ref: - write_multi_max_size: if cfg!(target_pointer_width = "32") { - Some(usize::MAX) - } else { + write_multi_max_size: if cfg!(target_pointer_width = "64") { Some(5 * 1024 * 1024 * 1024) + } else { + Some(usize::MAX) }, delete: true, diff --git a/core/src/services/oss/backend.rs b/core/src/services/oss/backend.rs index 7b0815436968..cc4dddbaefd1 100644 --- a/core/src/services/oss/backend.rs +++ b/core/src/services/oss/backend.rs @@ -410,10 +410,10 @@ impl Accessor for OssBackend { // The max multipart size of OSS is 5 GiB. // // ref: - write_multi_max_size: if cfg!(target_pointer_width = "32") { - Some(usize::MAX) - } else { + write_multi_max_size: if cfg!(target_pointer_width = "64") { Some(5 * 1024 * 1024 * 1024) + } else { + Some(usize::MAX) }, delete: true, diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs index 610255e9bf69..82e420421df8 100644 --- a/core/src/services/s3/backend.rs +++ b/core/src/services/s3/backend.rs @@ -923,10 +923,10 @@ impl Accessor for S3Backend { // The max multipart size of S3 is 5 GiB. // // ref: - write_multi_max_size: if cfg!(target_pointer_width = "32") { - Some(usize::MAX) - } else { + write_multi_max_size: if cfg!(target_pointer_width = "64") { Some(5 * 1024 * 1024 * 1024) + } else { + Some(usize::MAX) }, create_dir: true,