From 56e9ab1278644cff068984c813d40478bdd64d73 Mon Sep 17 00:00:00 2001 From: Anirban Halder Date: Wed, 10 Apr 2024 00:00:28 +0530 Subject: [PATCH 1/8] First commit --- src/uu/cp/src/platform/linux.rs | 173 ++++++++++++++++++++++++++++++-- 1 file changed, 164 insertions(+), 9 deletions(-) diff --git a/src/uu/cp/src/platform/linux.rs b/src/uu/cp/src/platform/linux.rs index 674e66ea575..9aaa0de7d25 100644 --- a/src/uu/cp/src/platform/linux.rs +++ b/src/uu/cp/src/platform/linux.rs @@ -3,9 +3,12 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. // spell-checker:ignore ficlone reflink ftruncate pwrite fiemap + +use libc::SEEK_DATA; use std::fs::{File, OpenOptions}; use std::io::Read; -use std::os::unix::fs::OpenOptionsExt; +use std::os::unix::fs::MetadataExt; +use std::os::unix::fs::{FileTypeExt, OpenOptionsExt}; use std::os::unix::io::AsRawFd; use std::path::Path; @@ -32,6 +35,20 @@ enum CloneFallback { /// Use [`std::fs::copy`]. FSCopy, + + /// Use sparse_copy + SparseCopy, +} + +/// Type of method used for copying files +#[derive(Clone, Copy)] +enum CopyMethod { + /// Do a sparse copy + SparseCopy, + /// Use [`std::fs::copy`]. + FSCopy, + /// Default (can either be sparse_copy or FSCopy) + Default, } /// Use the Linux `ioctl_ficlone` API to do a copy-on-write clone. @@ -53,7 +70,51 @@ where match fallback { CloneFallback::Error => Err(std::io::Error::last_os_error()), CloneFallback::FSCopy => std::fs::copy(source, dest).map(|_| ()), + CloneFallback::SparseCopy => sparse_copy(source, dest), + } +} + +#[cfg(any(target_os = "linux", target_os = "android"))] +fn check_for_data(source: &Path) -> Result<(bool, u64, u64), std::io::Error> { + let mut src_file = File::open(source)?; + let metadata = src_file.metadata()?; + + let size = metadata.size(); + let blocks = metadata.blocks(); + let blksize = metadata.blksize(); + + if size == 0 { + let mut buf: Vec = vec![0; blksize as usize]; + let _ = src_file.read(&mut buf)?; + if buf.iter().any(|&x| x != 0x0) { + return Ok((true, 0, 0)); + } + return Ok((false, 0, 0)); + } + + let src_fd = src_file.as_raw_fd(); + + let result = unsafe { libc::lseek(src_fd, 0, SEEK_DATA) }; + if result == -1 { + return Ok((false, size, blocks)); + } else if result >= 0 && result < size as i64 { + return Ok((true, size, blocks)); + } else { + return Err(std::io::Error::last_os_error()); + } +} + +#[cfg(any(target_os = "linux", target_os = "android"))] +fn check_sparse_detection(source: &Path) -> Result { + let src_file = File::open(source)?; + let metadata = src_file.metadata()?; + let size = metadata.size(); + let blocks = metadata.blocks(); + + if blocks < size / 512 { + return Ok(true); } + Ok(false) } /// Perform a sparse copy from one file to another. @@ -62,8 +123,6 @@ fn sparse_copy

(source: P, dest: P) -> std::io::Result<()> where P: AsRef, { - use std::os::unix::prelude::MetadataExt; - let mut src_file = File::open(source)?; let dst_file = File::create(dest)?; let dst_fd = dst_file.as_raw_fd(); @@ -97,6 +156,17 @@ where Ok(()) } +#[cfg(any(target_os = "linux", target_os = "android"))] +fn check_dest_is_fifo(dest: &Path) -> bool { + // If our destination file exists and its a fifo , we do a standard copy . + let file_type = std::fs::metadata(dest); + match file_type { + Ok(f) => f.file_type().is_fifo(), + + _ => false, + } +} + /// Copy the contents of the given source FIFO to the given file. fn copy_fifo_contents

(source: P, dest: P) -> std::io::Result where @@ -155,23 +225,106 @@ pub(crate) fn copy_on_write( let result = match (reflink_mode, sparse_mode) { (ReflinkMode::Never, SparseMode::Always) => { copy_debug.sparse_detection = SparseDebug::Zeros; - copy_debug.offload = OffloadReflinkDebug::Avoided; copy_debug.reflink = OffloadReflinkDebug::No; - sparse_copy(source, dest) + + if source_is_fifo { + copy_debug.offload = OffloadReflinkDebug::Avoided; + + copy_fifo_contents(source, dest).map(|_| ()) + } else { + let mut copy_method = CopyMethod::Default; + let (data_flag, size, blocks) = check_for_data(source)?; + let sparse_flag = check_sparse_detection(source)?; + if data_flag == true || size < 512 { + copy_debug.offload = OffloadReflinkDebug::Avoided; + } + match (sparse_flag, data_flag, blocks) { + (true, true, 0) => { + // Handling funny files with 0 block allocation but has data + // in it + copy_method = CopyMethod::FSCopy; + copy_debug.sparse_detection = SparseDebug::SeekHoleZeros; + } + (false, true, 0) => copy_method = CopyMethod::FSCopy, + + (true, false, 0) => copy_debug.sparse_detection = SparseDebug::SeekHole, + (true, true, _) => copy_debug.sparse_detection = SparseDebug::SeekHoleZeros, + + (true, false, _) => { + copy_debug.offload = OffloadReflinkDebug::Unknown; + copy_debug.sparse_detection = SparseDebug::SeekHole; + } + + (_, _, _) => (), + } + match copy_method { + CopyMethod::FSCopy => std::fs::copy(source, dest).map(|_| ()), + _ => sparse_copy(source, dest), + } + } } - (ReflinkMode::Never, _) => { - copy_debug.sparse_detection = SparseDebug::No; + (ReflinkMode::Never, SparseMode::Never) => { copy_debug.reflink = OffloadReflinkDebug::No; - std::fs::copy(source, dest).map(|_| ()) + + if source_is_fifo { + copy_debug.offload = OffloadReflinkDebug::Avoided; + + copy_fifo_contents(source, dest).map(|_| ()) + } else { + let (data_flag, size, blocks) = check_for_data(source)?; + let sparse_flag = check_sparse_detection(source)?; + + if sparse_flag == true { + copy_debug.sparse_detection = SparseDebug::SeekHole; + } + + if data_flag == true || size < 512 { + copy_debug.offload = OffloadReflinkDebug::Avoided; + } + + std::fs::copy(source, dest).map(|_| ()) + } + } + (ReflinkMode::Never, SparseMode::Auto) => { + copy_debug.reflink = OffloadReflinkDebug::No; + if source_is_fifo { + copy_debug.offload = OffloadReflinkDebug::Avoided; + + copy_fifo_contents(source, dest).map(|_| ()) + } else { + let mut copy_method = CopyMethod::Default; + let (data_flag, size, blocks) = check_for_data(source)?; + let sparse_flag = check_sparse_detection(source)?; + + if data_flag == true || size < 512 { + copy_debug.offload = OffloadReflinkDebug::Avoided; + } + + if sparse_flag { + if blocks == 0 && data_flag { + copy_method = CopyMethod::FSCopy; + } else { + copy_method = CopyMethod::SparseCopy; + } + copy_debug.sparse_detection = SparseDebug::SeekHole; + } + match copy_method { + CopyMethod::SparseCopy => sparse_copy(source, dest), + _ => std::fs::copy(source, dest).map(|_| ()), + } + } } (ReflinkMode::Auto, SparseMode::Always) => { copy_debug.offload = OffloadReflinkDebug::Avoided; copy_debug.sparse_detection = SparseDebug::Zeros; copy_debug.reflink = OffloadReflinkDebug::Unsupported; + let (data_flag, size, blocks) = check_for_data(source)?; + let sparse_flag = check_sparse_detection(source)?; + sparse_copy(source, dest) } - (ReflinkMode::Auto, _) => { + (ReflinkMode::Auto, SparseMode::Never) => { copy_debug.sparse_detection = SparseDebug::No; copy_debug.reflink = OffloadReflinkDebug::Unsupported; if source_is_fifo { @@ -180,6 +333,8 @@ pub(crate) fn copy_on_write( clone(source, dest, CloneFallback::FSCopy) } } + (ReflinkMode::Auto, SparseMode::Auto) => Ok(()), + (ReflinkMode::Always, SparseMode::Auto) => { copy_debug.sparse_detection = SparseDebug::No; copy_debug.reflink = OffloadReflinkDebug::Yes; From 2cfefd772be333a0cc1920e369cd64f767d38b84 Mon Sep 17 00:00:00 2001 From: Anirban Halder Date: Wed, 10 Apr 2024 19:18:33 +0530 Subject: [PATCH 2/8] Adding debug behaviour for all tests --- src/uu/cp/src/platform/linux.rs | 85 +++++++++++++++++++++++++++++---- tests/by-util/test_cp.rs | 2 +- 2 files changed, 77 insertions(+), 10 deletions(-) diff --git a/src/uu/cp/src/platform/linux.rs b/src/uu/cp/src/platform/linux.rs index 9aaa0de7d25..5edaa4a1865 100644 --- a/src/uu/cp/src/platform/linux.rs +++ b/src/uu/cp/src/platform/linux.rs @@ -225,6 +225,7 @@ pub(crate) fn copy_on_write( let result = match (reflink_mode, sparse_mode) { (ReflinkMode::Never, SparseMode::Always) => { copy_debug.sparse_detection = SparseDebug::Zeros; + // Default SparseDebug val for SparseMode::Always copy_debug.reflink = OffloadReflinkDebug::No; if source_is_fifo { @@ -315,25 +316,91 @@ pub(crate) fn copy_on_write( } } (ReflinkMode::Auto, SparseMode::Always) => { - copy_debug.offload = OffloadReflinkDebug::Avoided; - copy_debug.sparse_detection = SparseDebug::Zeros; - copy_debug.reflink = OffloadReflinkDebug::Unsupported; - let (data_flag, size, blocks) = check_for_data(source)?; - let sparse_flag = check_sparse_detection(source)?; + copy_debug.sparse_detection = SparseDebug::Zeros; // Default SparseDebug val for + // SparseMode::Always + if source_is_fifo { + copy_debug.offload = OffloadReflinkDebug::Avoided; - sparse_copy(source, dest) + copy_fifo_contents(source, dest).map(|_| ()) + } else { + let mut copy_method = CopyMethod::SparseCopy; + let (data_flag, size, blocks) = check_for_data(source)?; + let sparse_flag = check_sparse_detection(source)?; + if data_flag == true || size < 512 { + copy_debug.offload = OffloadReflinkDebug::Avoided; + } + match (sparse_flag, data_flag, blocks) { + (true, true, 0) => { + // Handling funny files with 0 block allocation but has data + // in it + copy_method = CopyMethod::FSCopy; + copy_debug.sparse_detection = SparseDebug::SeekHoleZeros; + } + (false, true, 0) => copy_method = CopyMethod::FSCopy, + + (true, false, 0) => copy_debug.sparse_detection = SparseDebug::SeekHole, + (true, true, _) => copy_debug.sparse_detection = SparseDebug::SeekHoleZeros, + + (true, false, _) => copy_debug.sparse_detection = SparseDebug::SeekHole, + + (_, _, _) => (), + } + match copy_method { + CopyMethod::FSCopy => clone(source, dest, CloneFallback::FSCopy), + _ => clone(source, dest, CloneFallback::SparseCopy), + } + } } (ReflinkMode::Auto, SparseMode::Never) => { - copy_debug.sparse_detection = SparseDebug::No; - copy_debug.reflink = OffloadReflinkDebug::Unsupported; + copy_debug.reflink = OffloadReflinkDebug::No; if source_is_fifo { + copy_debug.offload = OffloadReflinkDebug::Avoided; copy_fifo_contents(source, dest).map(|_| ()) } else { + let (data_flag, size, blocks) = check_for_data(source)?; + let sparse_flag = check_sparse_detection(source)?; + + if sparse_flag == true { + copy_debug.sparse_detection = SparseDebug::SeekHole; + } + + if data_flag == true || size < 512 { + copy_debug.offload = OffloadReflinkDebug::Avoided; + } + clone(source, dest, CloneFallback::FSCopy) } } - (ReflinkMode::Auto, SparseMode::Auto) => Ok(()), + (ReflinkMode::Auto, SparseMode::Auto) => { + if source_is_fifo { + copy_debug.offload = OffloadReflinkDebug::Unsupported; + copy_fifo_contents(source, dest).map(|_| ()) + } else { + let mut copy_method = CopyMethod::SparseCopy; + let (data_flag, size, blocks) = check_for_data(source)?; + let sparse_flag = check_sparse_detection(source)?; + if data_flag == true || (size > 0 && size < 512) { + copy_debug.offload = OffloadReflinkDebug::Yes; + } + if sparse_flag { + if blocks == 0 && data_flag { + copy_method = CopyMethod::FSCopy; + } else { + copy_method = CopyMethod::SparseCopy; + } + copy_debug.sparse_detection = SparseDebug::SeekHole; + } + + if check_dest_is_fifo(dest) { + copy_method = CopyMethod::FSCopy; + } + match copy_method { + CopyMethod::FSCopy => clone(source, dest, CloneFallback::FSCopy), + _ => clone(source, dest, CloneFallback::SparseCopy), + } + } + } (ReflinkMode::Always, SparseMode::Auto) => { copy_debug.sparse_detection = SparseDebug::No; diff --git a/tests/by-util/test_cp.rs b/tests/by-util/test_cp.rs index e3b373da19d..30464edd46e 100644 --- a/tests/by-util/test_cp.rs +++ b/tests/by-util/test_cp.rs @@ -3550,7 +3550,7 @@ fn test_cp_debug_sparse_never() { .arg("b") .succeeds(); let stdout_str = result.stdout_str(); - if !stdout_str.contains("copy offload: unknown, reflink: unsupported, sparse detection: no") { + if !stdout_str.contains("copy offload: avoided, reflink: no, sparse detection: no") { panic!("Failure: stdout was \n{stdout_str}"); } } From b1e7ee02718b8ab5211c1f2f0daa869fb6723690 Mon Sep 17 00:00:00 2001 From: Anirban Halder Date: Fri, 12 Apr 2024 21:31:00 +0530 Subject: [PATCH 3/8] Added tests and made code more readable --- src/uu/cp/src/platform/linux.rs | 381 +++++++++++++++++++++++--------- tests/by-util/test_cp.rs | 343 ++++++++++++++++++++++++++++ 2 files changed, 623 insertions(+), 101 deletions(-) diff --git a/src/uu/cp/src/platform/linux.rs b/src/uu/cp/src/platform/linux.rs index 5edaa4a1865..fc862d1be25 100644 --- a/src/uu/cp/src/platform/linux.rs +++ b/src/uu/cp/src/platform/linux.rs @@ -4,9 +4,10 @@ // file that was distributed with this source code. // spell-checker:ignore ficlone reflink ftruncate pwrite fiemap -use libc::SEEK_DATA; +use libc::{SEEK_DATA, SEEK_HOLE}; use std::fs::{File, OpenOptions}; use std::io::Read; +use std::os::unix::fs::FileExt; use std::os::unix::fs::MetadataExt; use std::os::unix::fs::{FileTypeExt, OpenOptionsExt}; use std::os::unix::io::AsRawFd; @@ -38,6 +39,9 @@ enum CloneFallback { /// Use sparse_copy SparseCopy, + + /// Use sparse_copy_without_hole + SparseCopyWithoutHole, } /// Type of method used for copying files @@ -49,6 +53,8 @@ enum CopyMethod { FSCopy, /// Default (can either be sparse_copy or FSCopy) Default, + /// Use sparse_copy_without_hole + SparseCopyWithoutHole, } /// Use the Linux `ioctl_ficlone` API to do a copy-on-write clone. @@ -71,6 +77,7 @@ where CloneFallback::Error => Err(std::io::Error::last_os_error()), CloneFallback::FSCopy => std::fs::copy(source, dest).map(|_| ()), CloneFallback::SparseCopy => sparse_copy(source, dest), + CloneFallback::SparseCopyWithoutHole => sparse_copy_without_hole(source, dest), } } @@ -87,18 +94,19 @@ fn check_for_data(source: &Path) -> Result<(bool, u64, u64), std::io::Error> { let mut buf: Vec = vec![0; blksize as usize]; let _ = src_file.read(&mut buf)?; if buf.iter().any(|&x| x != 0x0) { - return Ok((true, 0, 0)); + return Ok((true, size, 0)); } - return Ok((false, 0, 0)); + return Ok((false, size, 0)); } let src_fd = src_file.as_raw_fd(); let result = unsafe { libc::lseek(src_fd, 0, SEEK_DATA) }; + if result == -1 { - return Ok((false, size, blocks)); + Ok((false, size, blocks)) } else if result >= 0 && result < size as i64 { - return Ok((true, size, blocks)); + Ok((true, size, blocks)) } else { return Err(std::io::Error::last_os_error()); } @@ -117,7 +125,51 @@ fn check_sparse_detection(source: &Path) -> Result { Ok(false) } +// Optimized sparse_copy, doesn't create holes for large sequences of zeros in non sparse_files +// Used when --sparse=auto +#[cfg(any(target_os = "linux", target_os = "android"))] +fn sparse_copy_without_hole

(source: P, dest: P) -> std::io::Result<()> +where + P: AsRef, +{ + let src_file = File::open(source)?; + let dst_file = File::create(dest)?; + let dst_fd = dst_file.as_raw_fd(); + + let size: usize = src_file.metadata()?.size().try_into().unwrap(); + if unsafe { libc::ftruncate(dst_fd, size.try_into().unwrap()) } < 0 { + return Err(std::io::Error::last_os_error()); + } + let src_fd = src_file.as_raw_fd(); + let mut current_offset: i64 = 0; + loop { + let result = unsafe { libc::lseek(src_fd, current_offset as i64, SEEK_DATA) }; + + current_offset = result; + let hole = unsafe { libc::lseek(src_fd, current_offset as i64, SEEK_HOLE) }; + if result == -1 || hole == -1 { + break; + } + if result <= -2 || hole <= -2 { + return Err(std::io::Error::last_os_error()); + } + let len = hole - current_offset; + let mut buf: Vec = vec![0x0; len as usize]; + let _ = src_file.read_exact_at(&mut buf, current_offset as u64)?; + unsafe { + libc::pwrite( + dst_fd, + buf.as_ptr() as *const libc::c_void, + len as usize, + current_offset.try_into().unwrap(), + ) + }; + current_offset = hole; + } + Ok(()) +} /// Perform a sparse copy from one file to another. +/// Creates a holes for large sequences of zeros in non_sparse_files used for --sparse=always #[cfg(any(target_os = "linux", target_os = "android"))] fn sparse_copy

(source: P, dest: P) -> std::io::Result<()> where @@ -221,43 +273,23 @@ pub(crate) fn copy_on_write( reflink: OffloadReflinkDebug::Unsupported, sparse_detection: SparseDebug::No, }; - let result = match (reflink_mode, sparse_mode) { (ReflinkMode::Never, SparseMode::Always) => { copy_debug.sparse_detection = SparseDebug::Zeros; // Default SparseDebug val for SparseMode::Always copy_debug.reflink = OffloadReflinkDebug::No; - if source_is_fifo { copy_debug.offload = OffloadReflinkDebug::Avoided; copy_fifo_contents(source, dest).map(|_| ()) } else { let mut copy_method = CopyMethod::Default; - let (data_flag, size, blocks) = check_for_data(source)?; - let sparse_flag = check_sparse_detection(source)?; - if data_flag == true || size < 512 { - copy_debug.offload = OffloadReflinkDebug::Avoided; + let result = handle_reflink_never_sparse_always(source, dest); + if let Ok((debug, method)) = result { + copy_debug = debug; + copy_method = method; } - match (sparse_flag, data_flag, blocks) { - (true, true, 0) => { - // Handling funny files with 0 block allocation but has data - // in it - copy_method = CopyMethod::FSCopy; - copy_debug.sparse_detection = SparseDebug::SeekHoleZeros; - } - (false, true, 0) => copy_method = CopyMethod::FSCopy, - (true, false, 0) => copy_debug.sparse_detection = SparseDebug::SeekHole, - (true, true, _) => copy_debug.sparse_detection = SparseDebug::SeekHoleZeros, - - (true, false, _) => { - copy_debug.offload = OffloadReflinkDebug::Unknown; - copy_debug.sparse_detection = SparseDebug::SeekHole; - } - - (_, _, _) => (), - } match copy_method { CopyMethod::FSCopy => std::fs::copy(source, dest).map(|_| ()), _ => sparse_copy(source, dest), @@ -272,45 +304,29 @@ pub(crate) fn copy_on_write( copy_fifo_contents(source, dest).map(|_| ()) } else { - let (data_flag, size, blocks) = check_for_data(source)?; - let sparse_flag = check_sparse_detection(source)?; - - if sparse_flag == true { - copy_debug.sparse_detection = SparseDebug::SeekHole; - } - - if data_flag == true || size < 512 { - copy_debug.offload = OffloadReflinkDebug::Avoided; + let result = handle_reflink_never_sparse_never(source); + if let Ok(debug) = result { + copy_debug = debug; } - std::fs::copy(source, dest).map(|_| ()) } } (ReflinkMode::Never, SparseMode::Auto) => { copy_debug.reflink = OffloadReflinkDebug::No; + if source_is_fifo { copy_debug.offload = OffloadReflinkDebug::Avoided; - copy_fifo_contents(source, dest).map(|_| ()) } else { let mut copy_method = CopyMethod::Default; - let (data_flag, size, blocks) = check_for_data(source)?; - let sparse_flag = check_sparse_detection(source)?; - - if data_flag == true || size < 512 { - copy_debug.offload = OffloadReflinkDebug::Avoided; + let result = handle_reflink_never_sparse_auto(source, dest); + if let Ok((debug, method)) = result { + copy_debug = debug; + copy_method = method; } - if sparse_flag { - if blocks == 0 && data_flag { - copy_method = CopyMethod::FSCopy; - } else { - copy_method = CopyMethod::SparseCopy; - } - copy_debug.sparse_detection = SparseDebug::SeekHole; - } match copy_method { - CopyMethod::SparseCopy => sparse_copy(source, dest), + CopyMethod::SparseCopyWithoutHole => sparse_copy_without_hole(source, dest), _ => std::fs::copy(source, dest).map(|_| ()), } } @@ -323,28 +339,13 @@ pub(crate) fn copy_on_write( copy_fifo_contents(source, dest).map(|_| ()) } else { - let mut copy_method = CopyMethod::SparseCopy; - let (data_flag, size, blocks) = check_for_data(source)?; - let sparse_flag = check_sparse_detection(source)?; - if data_flag == true || size < 512 { - copy_debug.offload = OffloadReflinkDebug::Avoided; + let mut copy_method = CopyMethod::Default; + let result = handle_reflink_auto_sparse_always(source, dest); + if let Ok((debug, method)) = result { + copy_debug = debug; + copy_method = method; } - match (sparse_flag, data_flag, blocks) { - (true, true, 0) => { - // Handling funny files with 0 block allocation but has data - // in it - copy_method = CopyMethod::FSCopy; - copy_debug.sparse_detection = SparseDebug::SeekHoleZeros; - } - (false, true, 0) => copy_method = CopyMethod::FSCopy, - - (true, false, 0) => copy_debug.sparse_detection = SparseDebug::SeekHole, - (true, true, _) => copy_debug.sparse_detection = SparseDebug::SeekHoleZeros, - (true, false, _) => copy_debug.sparse_detection = SparseDebug::SeekHole, - - (_, _, _) => (), - } match copy_method { CopyMethod::FSCopy => clone(source, dest, CloneFallback::FSCopy), _ => clone(source, dest, CloneFallback::SparseCopy), @@ -358,15 +359,9 @@ pub(crate) fn copy_on_write( copy_debug.offload = OffloadReflinkDebug::Avoided; copy_fifo_contents(source, dest).map(|_| ()) } else { - let (data_flag, size, blocks) = check_for_data(source)?; - let sparse_flag = check_sparse_detection(source)?; - - if sparse_flag == true { - copy_debug.sparse_detection = SparseDebug::SeekHole; - } - - if data_flag == true || size < 512 { - copy_debug.offload = OffloadReflinkDebug::Avoided; + let result = handle_reflink_auto_sparse_never(source); + if let Ok(debug) = result { + copy_debug = debug; } clone(source, dest, CloneFallback::FSCopy) @@ -377,27 +372,18 @@ pub(crate) fn copy_on_write( copy_debug.offload = OffloadReflinkDebug::Unsupported; copy_fifo_contents(source, dest).map(|_| ()) } else { - let mut copy_method = CopyMethod::SparseCopy; - let (data_flag, size, blocks) = check_for_data(source)?; - let sparse_flag = check_sparse_detection(source)?; - if data_flag == true || (size > 0 && size < 512) { - copy_debug.offload = OffloadReflinkDebug::Yes; - } - if sparse_flag { - if blocks == 0 && data_flag { - copy_method = CopyMethod::FSCopy; - } else { - copy_method = CopyMethod::SparseCopy; - } - copy_debug.sparse_detection = SparseDebug::SeekHole; + let mut copy_method = CopyMethod::Default; + let result = handle_reflink_auto_sparse_auto(source, dest); + if let Ok((debug, method)) = result { + copy_debug = debug; + copy_method = method; } - if check_dest_is_fifo(dest) { - copy_method = CopyMethod::FSCopy; - } match copy_method { - CopyMethod::FSCopy => clone(source, dest, CloneFallback::FSCopy), - _ => clone(source, dest, CloneFallback::SparseCopy), + CopyMethod::SparseCopyWithoutHole => { + clone(source, dest, CloneFallback::SparseCopyWithoutHole) + } + _ => clone(source, dest, CloneFallback::FSCopy), } } } @@ -415,3 +401,196 @@ pub(crate) fn copy_on_write( result.context(context)?; Ok(copy_debug) } + +fn handle_reflink_auto_sparse_always( + source: &Path, + dest: &Path, +) -> Result<(CopyDebug, CopyMethod), std::io::Error> { + let mut copy_debug = CopyDebug { + offload: OffloadReflinkDebug::Unknown, + reflink: OffloadReflinkDebug::Unsupported, + sparse_detection: SparseDebug::Zeros, + }; + let mut copy_method = CopyMethod::Default; + let (data_flag, size, blocks) = check_for_data(source)?; + let sparse_flag = check_sparse_detection(source)?; + + if data_flag || size < 512 { + copy_debug.offload = OffloadReflinkDebug::Avoided; + } + match (sparse_flag, data_flag, blocks) { + (true, true, 0) => { + // Handling funny files with 0 block allocation but has data + // in it + copy_method = CopyMethod::FSCopy; + copy_debug.sparse_detection = SparseDebug::SeekHoleZeros; + } + (false, true, 0) => copy_method = CopyMethod::FSCopy, + + (true, false, 0) => copy_debug.sparse_detection = SparseDebug::SeekHole, + (true, true, _) => copy_debug.sparse_detection = SparseDebug::SeekHoleZeros, + + (true, false, _) => copy_debug.sparse_detection = SparseDebug::SeekHole, + + (_, _, _) => (), + } + if check_dest_is_fifo(dest) { + copy_method = CopyMethod::FSCopy; + } + Ok((copy_debug, copy_method)) +} + +fn handle_reflink_never_sparse_never(source: &Path) -> Result { + let mut copy_debug = CopyDebug { + offload: OffloadReflinkDebug::Unknown, + reflink: OffloadReflinkDebug::No, + sparse_detection: SparseDebug::No, + }; + let (data_flag, size, _blocks) = check_for_data(source)?; + let sparse_flag = check_sparse_detection(source)?; + + if sparse_flag { + copy_debug.sparse_detection = SparseDebug::SeekHole; + } + + if data_flag || size < 512 { + copy_debug.offload = OffloadReflinkDebug::Avoided; + } + Ok(copy_debug) +} + +fn handle_reflink_auto_sparse_never(source: &Path) -> Result { + let mut copy_debug = CopyDebug { + offload: OffloadReflinkDebug::Unknown, + reflink: OffloadReflinkDebug::No, + sparse_detection: SparseDebug::No, + }; + + let (data_flag, size, _blocks) = check_for_data(source)?; + let sparse_flag = check_sparse_detection(source)?; + + if sparse_flag { + copy_debug.sparse_detection = SparseDebug::SeekHole; + } + + if data_flag || size < 512 { + copy_debug.offload = OffloadReflinkDebug::Avoided; + } + Ok(copy_debug) +} + +fn handle_reflink_auto_sparse_auto( + source: &Path, + dest: &Path, +) -> Result<(CopyDebug, CopyMethod), std::io::Error> { + let mut copy_debug = CopyDebug { + offload: OffloadReflinkDebug::Unknown, + reflink: OffloadReflinkDebug::Unsupported, + sparse_detection: SparseDebug::No, + }; + + let mut copy_method = CopyMethod::Default; + let (data_flag, size, blocks) = check_for_data(source)?; + let sparse_flag = check_sparse_detection(source)?; + + if (data_flag && size != 0) || (size > 0 && size < 512) { + copy_debug.offload = OffloadReflinkDebug::Yes; + } + + if data_flag && size == 0 { + // Handling /proc/ files + copy_debug.offload = OffloadReflinkDebug::Unsupported; + } + if sparse_flag { + if blocks == 0 && data_flag { + // Handling other "virtual" files + copy_debug.offload = OffloadReflinkDebug::Unsupported; + + copy_method = CopyMethod::FSCopy; + } else { + copy_method = CopyMethod::SparseCopyWithoutHole; + } + copy_debug.sparse_detection = SparseDebug::SeekHole; + } + + if check_dest_is_fifo(dest) { + copy_method = CopyMethod::FSCopy; + } + Ok((copy_debug, copy_method)) +} + +fn handle_reflink_never_sparse_auto( + source: &Path, + dest: &Path, +) -> Result<(CopyDebug, CopyMethod), std::io::Error> { + let mut copy_debug = CopyDebug { + offload: OffloadReflinkDebug::Unknown, + reflink: OffloadReflinkDebug::No, + sparse_detection: SparseDebug::No, + }; + + let (data_flag, size, blocks) = check_for_data(source)?; + let sparse_flag = check_sparse_detection(source)?; + + let mut copy_method = CopyMethod::Default; + if data_flag || size < 512 { + copy_debug.offload = OffloadReflinkDebug::Avoided; + } + + if sparse_flag { + if blocks == 0 && data_flag { + copy_method = CopyMethod::FSCopy; + } else { + copy_method = CopyMethod::SparseCopyWithoutHole; + } + copy_debug.sparse_detection = SparseDebug::SeekHole; + } + + if check_dest_is_fifo(dest) { + copy_method = CopyMethod::FSCopy; + } + Ok((copy_debug, copy_method)) +} + +fn handle_reflink_never_sparse_always( + source: &Path, + dest: &Path, +) -> Result<(CopyDebug, CopyMethod), std::io::Error> { + let mut copy_debug = CopyDebug { + offload: OffloadReflinkDebug::Unknown, + reflink: OffloadReflinkDebug::No, + sparse_detection: SparseDebug::Zeros, + }; + let mut copy_method = CopyMethod::SparseCopy; + + let (data_flag, size, blocks) = check_for_data(source)?; + let sparse_flag = check_sparse_detection(source)?; + + if data_flag || size < 512 { + copy_debug.offload = OffloadReflinkDebug::Avoided; + } + match (sparse_flag, data_flag, blocks) { + (true, true, 0) => { + // Handling funny files with 0 block allocation but has data + // in it + copy_method = CopyMethod::FSCopy; + copy_debug.sparse_detection = SparseDebug::SeekHoleZeros; + } + (false, true, 0) => copy_method = CopyMethod::FSCopy, + + (true, false, 0) => copy_debug.sparse_detection = SparseDebug::SeekHole, + (true, true, _) => copy_debug.sparse_detection = SparseDebug::SeekHoleZeros, + + (true, false, _) => { + copy_debug.offload = OffloadReflinkDebug::Unknown; + copy_debug.sparse_detection = SparseDebug::SeekHole; + } + + (_, _, _) => copy_method = CopyMethod::FSCopy, + } + if check_dest_is_fifo(dest) { + copy_method = CopyMethod::FSCopy; + } + + Ok((copy_debug, copy_method)) +} diff --git a/tests/by-util/test_cp.rs b/tests/by-util/test_cp.rs index 30464edd46e..5f91dc864e6 100644 --- a/tests/by-util/test_cp.rs +++ b/tests/by-util/test_cp.rs @@ -3800,6 +3800,349 @@ fn test_acl_preserve() { assert!(compare_xattrs(&file, &file_target)); } +#[test] +#[cfg(any(target_os = "linux", target_os = "android"))] +fn test_cp_debug_reflink_never_with_hole() { + let ts = TestScenario::new(util_name!()); + let at = &ts.fixtures; + at.touch("a"); + at.write("a", "hello"); + let f = std::fs::OpenOptions::new() + .write(true) + .open(at.plus("a")) + .unwrap(); + f.set_len(10000).unwrap(); + + let result = ts + .ucmd() + .arg("--debug") + .arg("--reflink=never") + .arg("a") + .arg("b") + .succeeds(); + + let stdout_str = result.stdout_str(); + if !stdout_str.contains("copy offload: avoided, reflink: no, sparse detection: SEEK_HOLE") { + panic!("Failure: stdout was \n{stdout_str}"); + } +} +#[test] +#[cfg(any(target_os = "linux", target_os = "android"))] +fn test_cp_debug_reflink_never_empty_file_with_hole() { + let ts = TestScenario::new(util_name!()); + let at = &ts.fixtures; + at.touch("a"); + let f = std::fs::OpenOptions::new() + .write(true) + .open(at.plus("a")) + .unwrap(); + f.set_len(10000).unwrap(); + + let result = ts + .ucmd() + .arg("--debug") + .arg("--reflink=never") + .arg("a") + .arg("b") + .succeeds(); + + let stdout_str = result.stdout_str(); + if !stdout_str.contains("copy offload: unknown, reflink: no, sparse detection: SEEK_HOLE") { + panic!("Failure: stdout was \n{stdout_str}"); + } +} + +#[test] +#[cfg(any(target_os = "linux", target_os = "android"))] +fn test_cp_debug_default_with_hole() { + let ts = TestScenario::new(util_name!()); + let at = &ts.fixtures; + at.touch("a"); + let f = std::fs::OpenOptions::new() + .write(true) + .open(at.plus("a")) + .unwrap(); + f.set_len(10000).unwrap(); + + at.append_bytes("a", "hello".as_bytes()); + let result = ts.ucmd().arg("--debug").arg("a").arg("b").succeeds(); + + let stdout_str = result.stdout_str(); + if !stdout_str.contains("copy offload: yes, reflink: unsupported, sparse detection: SEEK_HOLE") + { + panic!("Failure: stdout was \n{stdout_str}"); + } +} + +#[test] +#[cfg(any(target_os = "linux", target_os = "android"))] +fn test_cp_debug_default_less_than_512_bytes() { + let ts = TestScenario::new(util_name!()); + + let at = &ts.fixtures; + at.touch("a"); + at.append_bytes("a", "hello".as_bytes()); + let f = std::fs::OpenOptions::new() + .write(true) + .open(at.plus("a")) + .unwrap(); + f.set_len(400).unwrap(); + + let result = ts + .ucmd() + .arg("--debug") + .arg("--reflink=auto") + .arg("--sparse=auto") + .arg("a") + .arg("b") + .succeeds(); + + let stdout_str = result.stdout_str(); + if !stdout_str.contains("copy offload: yes, reflink: unsupported, sparse detection: no") { + panic!("Failure: stdout was \n{stdout_str}"); + } +} + +#[test] +#[cfg(any(target_os = "linux", target_os = "android"))] +fn test_cp_debug_default_without_hole() { + let ts = TestScenario::new(util_name!()); + + let at = &ts.fixtures; + at.touch("a"); + at.append_bytes("a", "hello".as_bytes()); + + let filler_bytes = [0 as u8; 10000]; + + at.append_bytes("a", &filler_bytes); + + let result = ts.ucmd().arg("--debug").arg("a").arg("b").succeeds(); + + let stdout_str = result.stdout_str(); + if !stdout_str.contains("copy offload: yes, reflink: unsupported, sparse detection: no") { + panic!("Failure: stdout was \n{stdout_str}"); + } +} +#[test] +#[cfg(any(target_os = "linux", target_os = "android"))] +fn test_cp_debug_default_empty_file_with_hole() { + let ts = TestScenario::new(util_name!()); + let at = &ts.fixtures; + at.touch("a"); + let f = std::fs::OpenOptions::new() + .write(true) + .open(at.plus("a")) + .unwrap(); + f.set_len(10000).unwrap(); + + let result = ts.ucmd().arg("--debug").arg("a").arg("b").succeeds(); + + let stdout_str = result.stdout_str(); + if !stdout_str + .contains("copy offload: unknown, reflink: unsupported, sparse detection: SEEK_HOLE") + { + panic!("Failure: stdout was \n{stdout_str}"); + } +} + +#[test] +#[cfg(any(target_os = "linux", target_os = "android"))] +fn test_cp_debug_reflink_never_sparse_always_with_hole() { + let ts = TestScenario::new(util_name!()); + let at = &ts.fixtures; + at.touch("a"); + at.write("a", "hello"); + let f = std::fs::OpenOptions::new() + .write(true) + .open(at.plus("a")) + .unwrap(); + f.set_len(10000).unwrap(); + + let result = ts + .ucmd() + .arg("--debug") + .arg("--reflink=never") + .arg("--sparse=always") + .arg("a") + .arg("b") + .succeeds(); + + let stdout_str = result.stdout_str(); + if !stdout_str + .contains("copy offload: avoided, reflink: no, sparse detection: SEEK_HOLE + zeros") + { + panic!("Failure: stdout was \n{stdout_str}"); + } +} + +#[test] +#[cfg(any(target_os = "linux", target_os = "android"))] +fn test_cp_debug_reflink_never_sparse_always_without_hole() { + let ts = TestScenario::new(util_name!()); + let empty_bytes = [0 as u8; 10000]; + let at = &ts.fixtures; + at.touch("a"); + at.write("a", "hello"); + at.append_bytes("a", &empty_bytes); + + let result = ts + .ucmd() + .arg("--debug") + .arg("--reflink=never") + .arg("--sparse=always") + .arg("a") + .arg("b") + .succeeds(); + + let stdout_str = result.stdout_str(); + if !stdout_str.contains("copy offload: avoided, reflink: no, sparse detection: zeros") { + panic!("Failure: stdout was \n{stdout_str}"); + } +} + +#[test] +#[cfg(any(target_os = "linux", target_os = "android"))] +fn test_cp_debug_reflink_never_sparse_always_empty_file_with_hole() { + let ts = TestScenario::new(util_name!()); + let at = &ts.fixtures; + at.touch("a"); + let f = std::fs::OpenOptions::new() + .write(true) + .open(at.plus("a")) + .unwrap(); + f.set_len(10000).unwrap(); + + let result = ts + .ucmd() + .arg("--debug") + .arg("--reflink=never") + .arg("--sparse=always") + .arg("a") + .arg("b") + .succeeds(); + + let stdout_str = result.stdout_str(); + if !stdout_str.contains("copy offload: unknown, reflink: no, sparse detection: SEEK_HOLE") { + panic!("Failure: stdout was \n{stdout_str}"); + } +} + +#[test] +#[cfg(any(target_os = "linux", target_os = "android"))] +fn test_cp_default_virtual_file() { + use std::os::unix::prelude::MetadataExt; + let ts = TestScenario::new(util_name!()); + let at = &ts.fixtures; + ts.ucmd() + .arg("/sys/kernel/address_bits") + .arg("b") + .succeeds(); + + let dest_size = std::fs::metadata(at.plus("b")) + .expect("Metadata of copied file cannot be read") + .size(); + if dest_size == 0 { + panic!("Copy unsuccessful"); + } +} + +#[test] +#[cfg(any(target_os = "linux", target_os = "android"))] +fn test_cp_debug_sparse_always_sparse_virtual_file() { + let ts = TestScenario::new(util_name!()); + let result = ts + .ucmd() + .arg("--debug") + .arg("--sparse=always") + .arg("/sys/kernel/address_bits") + .arg("b") + .succeeds(); + + let stdout_str = result.stdout_str(); + if !stdout_str.contains( + "copy offload: avoided, reflink: unsupported, sparse detection: SEEK_HOLE + zeros", + ) { + panic!("Failure: stdout was \n{stdout_str}"); + } +} + +#[test] +#[cfg(any(target_os = "linux", target_os = "android"))] +fn test_cp_debug_default_sparse_virtual_file() { + let ts = TestScenario::new(util_name!()); + let result = ts + .ucmd() + .arg("--debug") + .arg("/sys/kernel/address_bits") + .arg("b") + .succeeds(); + + let stdout_str = result.stdout_str(); + if !stdout_str + .contains("copy offload: unsupported, reflink: unsupported, sparse detection: SEEK_HOLE") + { + panic!("Failure: stdout was \n{stdout_str}"); + } +} + +#[test] +#[cfg(any(target_os = "linux", target_os = "android"))] +fn test_cp_debug_sparse_never_zero_sized_virtual_file() { + let ts = TestScenario::new(util_name!()); + let result = ts + .ucmd() + .arg("--debug") + .arg("--sparse=never") + .arg("/proc/version") + .arg("b") + .succeeds(); + + let stdout_str = result.stdout_str(); + if !stdout_str.contains("copy offload: avoided, reflink: no, sparse detection: no") { + panic!("Failure: stdout was \n{stdout_str}"); + } +} + +#[test] +#[cfg(any(target_os = "linux", target_os = "android"))] +fn test_cp_debug_default_zero_sized_virtual_file() { + let ts = TestScenario::new(util_name!()); + let result = ts + .ucmd() + .arg("--debug") + .arg("/proc/version") + .arg("b") + .succeeds(); + + let stdout_str = result.stdout_str(); + if !stdout_str.contains("copy offload: unsupported, reflink: unsupported, sparse detection: no") + { + panic!("Failure: stdout was \n{stdout_str}"); + } +} + +#[test] +#[cfg(any(target_os = "linux", target_os = "android"))] +fn test_cp_debug_reflink_never_without_hole() { + let ts = TestScenario::new(util_name!()); + let filler_bytes = [0 as u8; 1000]; + let at = &ts.fixtures; + at.touch("a"); + at.write("a", "hello"); + at.append_bytes("a", &filler_bytes); + let result = ts + .ucmd() + .arg("--debug") + .arg("--reflink=never") + .arg("a") + .arg("b") + .succeeds(); + + let stdout_str = result.stdout_str(); + if !stdout_str.contains("copy offload: avoided, reflink: no, sparse detection: no") { + panic!("Failure: stdout was \n{stdout_str}"); + } +} #[test] fn test_cp_force_remove_destination_attributes_only_with_symlink() { From 3e21b1ff1627d514d8487bc82390e50553936d68 Mon Sep 17 00:00:00 2001 From: Anirban Halder Date: Sat, 13 Apr 2024 22:42:06 +0530 Subject: [PATCH 4/8] Added and improved tests, docs and fixed a bug --- src/uu/cp/src/platform/linux.rs | 72 +++++--- tests/by-util/test_cp.rs | 295 +++++++++++++++++++++++++++++++- 2 files changed, 343 insertions(+), 24 deletions(-) diff --git a/src/uu/cp/src/platform/linux.rs b/src/uu/cp/src/platform/linux.rs index fc862d1be25..f7662ae7a33 100644 --- a/src/uu/cp/src/platform/linux.rs +++ b/src/uu/cp/src/platform/linux.rs @@ -81,6 +81,9 @@ where } } +/// Checks whether a file contains any non null bytes i.e. any byte != 0x0 +/// This function returns a tuple of (bool, u64, u64) signifying a tuple of (whether a file has +/// data, its size, no of blocks it has allocated in disk) #[cfg(any(target_os = "linux", target_os = "android"))] fn check_for_data(source: &Path) -> Result<(bool, u64, u64), std::io::Error> { let mut src_file = File::open(source)?; @@ -89,7 +92,7 @@ fn check_for_data(source: &Path) -> Result<(bool, u64, u64), std::io::Error> { let size = metadata.size(); let blocks = metadata.blocks(); let blksize = metadata.blksize(); - + // checks edge case of virtual files in /proc which have a size of zero but contains data if size == 0 { let mut buf: Vec = vec![0; blksize as usize]; let _ = src_file.read(&mut buf)?; @@ -105,7 +108,7 @@ fn check_for_data(source: &Path) -> Result<(bool, u64, u64), std::io::Error> { if result == -1 { Ok((false, size, blocks)) - } else if result >= 0 && result < size as i64 { + } else if result >= 0 { Ok((true, size, blocks)) } else { return Err(std::io::Error::last_os_error()); @@ -113,6 +116,8 @@ fn check_for_data(source: &Path) -> Result<(bool, u64, u64), std::io::Error> { } #[cfg(any(target_os = "linux", target_os = "android"))] +/// Checks whether a file is sparse i.e. it contains holes, uses the crude heurestic blocks < size / 512 +/// Reference: https://doc.rust-lang.org/std/os/unix/fs/trait.MetadataExt.html#tymethod.blocks fn check_sparse_detection(source: &Path) -> Result { let src_file = File::open(source)?; let metadata = src_file.metadata()?; @@ -125,8 +130,8 @@ fn check_sparse_detection(source: &Path) -> Result { Ok(false) } -// Optimized sparse_copy, doesn't create holes for large sequences of zeros in non sparse_files -// Used when --sparse=auto +/// Optimized sparse_copy, doesn't create holes for large sequences of zeros in non sparse_files +/// Used when --sparse=auto #[cfg(any(target_os = "linux", target_os = "android"))] fn sparse_copy_without_hole

(source: P, dest: P) -> std::io::Result<()> where @@ -136,26 +141,31 @@ where let dst_file = File::create(dest)?; let dst_fd = dst_file.as_raw_fd(); - let size: usize = src_file.metadata()?.size().try_into().unwrap(); + let size: u64 = src_file.metadata()?.size(); if unsafe { libc::ftruncate(dst_fd, size.try_into().unwrap()) } < 0 { return Err(std::io::Error::last_os_error()); } let src_fd = src_file.as_raw_fd(); - let mut current_offset: i64 = 0; + let mut current_offset: isize = 0; loop { - let result = unsafe { libc::lseek(src_fd, current_offset as i64, SEEK_DATA) }; + let result = unsafe { libc::lseek(src_fd, current_offset.try_into().unwrap(), SEEK_DATA) } + .try_into() + .unwrap(); current_offset = result; - let hole = unsafe { libc::lseek(src_fd, current_offset as i64, SEEK_HOLE) }; + let hole: isize = + unsafe { libc::lseek(src_fd, current_offset.try_into().unwrap(), SEEK_HOLE) } + .try_into() + .unwrap(); if result == -1 || hole == -1 { break; } if result <= -2 || hole <= -2 { return Err(std::io::Error::last_os_error()); } - let len = hole - current_offset; + let len: isize = hole - current_offset; let mut buf: Vec = vec![0x0; len as usize]; - let _ = src_file.read_exact_at(&mut buf, current_offset as u64)?; + src_file.read_exact_at(&mut buf, current_offset as u64)?; unsafe { libc::pwrite( dst_fd, @@ -169,7 +179,7 @@ where Ok(()) } /// Perform a sparse copy from one file to another. -/// Creates a holes for large sequences of zeros in non_sparse_files used for --sparse=always +/// Creates a holes for large sequences of zeros in non_sparse_files, used for --sparse=always #[cfg(any(target_os = "linux", target_os = "android"))] fn sparse_copy

(source: P, dest: P) -> std::io::Result<()> where @@ -209,6 +219,7 @@ where } #[cfg(any(target_os = "linux", target_os = "android"))] +/// Checks whether an existing destination is a fifo fn check_dest_is_fifo(dest: &Path) -> bool { // If our destination file exists and its a fifo , we do a standard copy . let file_type = std::fs::metadata(dest); @@ -348,7 +359,7 @@ pub(crate) fn copy_on_write( match copy_method { CopyMethod::FSCopy => clone(source, dest, CloneFallback::FSCopy), - _ => clone(source, dest, CloneFallback::SparseCopy), + _ => sparse_copy(source, dest), } } } @@ -402,6 +413,8 @@ pub(crate) fn copy_on_write( Ok(copy_debug) } +/// Handles debug results when flags are "--reflink=auto" and "--sparse=always" and specifies what +/// type of copy should be used fn handle_reflink_auto_sparse_always( source: &Path, dest: &Path, @@ -440,6 +453,8 @@ fn handle_reflink_auto_sparse_always( Ok((copy_debug, copy_method)) } +/// Handles debug results when flags are "--reflink=auto" and "--sparse=auto" and specifies what +/// type of copy should be used fn handle_reflink_never_sparse_never(source: &Path) -> Result { let mut copy_debug = CopyDebug { offload: OffloadReflinkDebug::Unknown, @@ -459,6 +474,8 @@ fn handle_reflink_never_sparse_never(source: &Path) -> Result Result { let mut copy_debug = CopyDebug { offload: OffloadReflinkDebug::Unknown, @@ -479,6 +496,8 @@ fn handle_reflink_auto_sparse_never(source: &Path) -> Result 0 but no + // disk allocation } else { - copy_method = CopyMethod::SparseCopyWithoutHole; + copy_method = CopyMethod::SparseCopyWithoutHole; // Handles regular sparse-files } copy_debug.sparse_detection = SparseDebug::SeekHole; } @@ -552,6 +575,8 @@ fn handle_reflink_never_sparse_auto( Ok((copy_debug, copy_method)) } +/// Handles debug results when flags are "--reflink=never" and "--sparse=always" and specifies what +/// type of copy should be used fn handle_reflink_never_sparse_always( source: &Path, dest: &Path, @@ -572,21 +597,22 @@ fn handle_reflink_never_sparse_always( match (sparse_flag, data_flag, blocks) { (true, true, 0) => { // Handling funny files with 0 block allocation but has data - // in it + // in it, e.g. files in /sys and other virtual files copy_method = CopyMethod::FSCopy; copy_debug.sparse_detection = SparseDebug::SeekHoleZeros; } - (false, true, 0) => copy_method = CopyMethod::FSCopy, - - (true, false, 0) => copy_debug.sparse_detection = SparseDebug::SeekHole, - (true, true, _) => copy_debug.sparse_detection = SparseDebug::SeekHoleZeros, - + (false, true, 0) => copy_method = CopyMethod::FSCopy, // Handling data containing zero sized + // files in /proc + (true, false, 0) => copy_debug.sparse_detection = SparseDebug::SeekHole, // Handles files + // with 0 blocks allocated in disk and + (true, true, _) => copy_debug.sparse_detection = SparseDebug::SeekHoleZeros, // Any + // sparse_files with data in it will display SeekHoleZeros (true, false, _) => { copy_debug.offload = OffloadReflinkDebug::Unknown; copy_debug.sparse_detection = SparseDebug::SeekHole; } - (_, _, _) => copy_method = CopyMethod::FSCopy, + (_, _, _) => (), } if check_dest_is_fifo(dest) { copy_method = CopyMethod::FSCopy; diff --git a/tests/by-util/test_cp.rs b/tests/by-util/test_cp.rs index 5f91dc864e6..11132ff31f5 100644 --- a/tests/by-util/test_cp.rs +++ b/tests/by-util/test_cp.rs @@ -3820,6 +3820,11 @@ fn test_cp_debug_reflink_never_with_hole() { .arg("a") .arg("b") .succeeds(); + let dst_file_metadata = std::fs::metadata(at.plus("b")).unwrap(); + let src_file_metadata = std::fs::metadata(at.plus("a")).unwrap(); + if dst_file_metadata.blocks() != src_file_metadata.blocks() { + panic!("File not sparsely copied"); + } let stdout_str = result.stdout_str(); if !stdout_str.contains("copy offload: avoided, reflink: no, sparse detection: SEEK_HOLE") { @@ -3845,6 +3850,11 @@ fn test_cp_debug_reflink_never_empty_file_with_hole() { .arg("a") .arg("b") .succeeds(); + let dst_file_metadata = std::fs::metadata(at.plus("b")).unwrap(); + let src_file_metadata = std::fs::metadata(at.plus("a")).unwrap(); + if dst_file_metadata.blocks() != src_file_metadata.blocks() { + panic!("File not sparsely copied"); + } let stdout_str = result.stdout_str(); if !stdout_str.contains("copy offload: unknown, reflink: no, sparse detection: SEEK_HOLE") { @@ -3866,6 +3876,11 @@ fn test_cp_debug_default_with_hole() { at.append_bytes("a", "hello".as_bytes()); let result = ts.ucmd().arg("--debug").arg("a").arg("b").succeeds(); + let dst_file_metadata = std::fs::metadata(at.plus("b")).unwrap(); + let src_file_metadata = std::fs::metadata(at.plus("a")).unwrap(); + if dst_file_metadata.blocks() != src_file_metadata.blocks() { + panic!("File not sparsely copied"); + } let stdout_str = result.stdout_str(); if !stdout_str.contains("copy offload: yes, reflink: unsupported, sparse detection: SEEK_HOLE") @@ -3936,6 +3951,11 @@ fn test_cp_debug_default_empty_file_with_hole() { f.set_len(10000).unwrap(); let result = ts.ucmd().arg("--debug").arg("a").arg("b").succeeds(); + let dst_file_metadata = std::fs::metadata(at.plus("b")).unwrap(); + let src_file_metadata = std::fs::metadata(at.plus("a")).unwrap(); + if dst_file_metadata.blocks() != src_file_metadata.blocks() { + panic!("File not sparsely copied"); + } let stdout_str = result.stdout_str(); if !stdout_str @@ -3966,7 +3986,11 @@ fn test_cp_debug_reflink_never_sparse_always_with_hole() { .arg("a") .arg("b") .succeeds(); - + let dst_file_metadata = std::fs::metadata(at.plus("b")).unwrap(); + let src_file_metadata = std::fs::metadata(at.plus("a")).unwrap(); + if dst_file_metadata.blocks() != src_file_metadata.blocks() { + panic!("File not sparsely copied"); + } let stdout_str = result.stdout_str(); if !stdout_str .contains("copy offload: avoided, reflink: no, sparse detection: SEEK_HOLE + zeros") @@ -3993,7 +4017,11 @@ fn test_cp_debug_reflink_never_sparse_always_without_hole() { .arg("a") .arg("b") .succeeds(); + let dst_file_metadata = std::fs::metadata(at.plus("b")).unwrap(); + if dst_file_metadata.blocks() != dst_file_metadata.blksize() / 512 { + panic!("Zero sequenced blocks not removed"); + } let stdout_str = result.stdout_str(); if !stdout_str.contains("copy offload: avoided, reflink: no, sparse detection: zeros") { panic!("Failure: stdout was \n{stdout_str}"); @@ -4045,6 +4073,86 @@ fn test_cp_default_virtual_file() { panic!("Copy unsuccessful"); } } +#[test] +#[cfg(any(target_os = "linux", target_os = "android"))] +fn test_cp_debug_reflink_auto_sparse_always_non_sparse_file_with_long_zero_sequence() { + let ts = TestScenario::new(util_name!()); + + let buf: Vec = vec![0; 4096 * 4]; + let at = &ts.fixtures; + at.touch("a"); + at.append_bytes("a", &buf); + at.append_bytes("a", "hello".as_bytes()); + let result = ts + .ucmd() + .arg("--debug") + .arg("--sparse=always") + .arg("a") + .arg("b") + .succeeds(); + + let dst_file_metadata = std::fs::metadata(at.plus("b")).unwrap(); + + if dst_file_metadata.blocks() != dst_file_metadata.blksize() / 512 { + panic!("Zero sequenced blocks not removed"); + } + + let stdout_str = result.stdout_str(); + if !stdout_str.contains("copy offload: avoided, reflink: unsupported, sparse detection: zeros") + { + panic!("Failure: stdout was \n{stdout_str}"); + } +} + +#[test] +#[cfg(target_os = "linux")] +fn test_cp_debug_sparse_never_empty_sparse_file() { + let ts = TestScenario::new(util_name!()); + let at = &ts.fixtures; + at.touch("a"); + let result = ts + .ucmd() + .arg("--debug") + .arg("--sparse=never") + .arg("a") + .arg("b") + .succeeds(); + let stdout_str = result.stdout_str(); + if !stdout_str.contains("copy offload: avoided, reflink: no, sparse detection: no") { + panic!("Failure: stdout was \n{stdout_str}"); + } +} + +#[test] +#[cfg(any(target_os = "linux", target_os = "android"))] +fn test_cp_debug_reflink_never_sparse_always_non_sparse_file_with_long_zero_sequence() { + let ts = TestScenario::new(util_name!()); + + let buf: Vec = vec![0; 4096 * 4]; + let at = &ts.fixtures; + at.touch("a"); + at.append_bytes("a", &buf); + at.append_bytes("a", "hello".as_bytes()); + let result = ts + .ucmd() + .arg("--debug") + .arg("--reflink=never") + .arg("--sparse=always") + .arg("a") + .arg("b") + .succeeds(); + + let dst_file_metadata = std::fs::metadata(at.plus("b")).unwrap(); + + if dst_file_metadata.blocks() != dst_file_metadata.blksize() / 512 { + panic!("Zero sequenced blocks not removed"); + } + + let stdout_str = result.stdout_str(); + if !stdout_str.contains("copy offload: avoided, reflink: no, sparse detection: zeros") { + panic!("Failure: stdout was \n{stdout_str}"); + } +} #[test] #[cfg(any(target_os = "linux", target_os = "android"))] @@ -4065,6 +4173,191 @@ fn test_cp_debug_sparse_always_sparse_virtual_file() { panic!("Failure: stdout was \n{stdout_str}"); } } +#[test] +#[cfg(any(target_os = "linux", target_os = "android"))] +fn test_cp_debug_reflink_never_less_than_512_bytes() { + let ts = TestScenario::new(util_name!()); + + let at = &ts.fixtures; + at.touch("a"); + at.append_bytes("a", "hello".as_bytes()); + let f = std::fs::OpenOptions::new() + .write(true) + .open(at.plus("a")) + .unwrap(); + f.set_len(400).unwrap(); + + let result = ts + .ucmd() + .arg("--debug") + .arg("--reflink=never") + .arg("a") + .arg("b") + .succeeds(); + + let stdout_str = result.stdout_str(); + if !stdout_str.contains("copy offload: avoided, reflink: no, sparse detection: no") { + panic!("Failure: stdout was \n{stdout_str}"); + } +} + +#[test] +#[cfg(any(target_os = "linux", target_os = "android"))] +fn test_cp_debug_reflink_never_sparse_never_empty_file_with_hole() { + let ts = TestScenario::new(util_name!()); + let at = &ts.fixtures; + at.touch("a"); + let f = std::fs::OpenOptions::new() + .write(true) + .open(at.plus("a")) + .unwrap(); + f.set_len(10000).unwrap(); + + let result = ts + .ucmd() + .arg("--debug") + .arg("--reflink=never") + .arg("--sparse=never") + .arg("a") + .arg("b") + .succeeds(); + let stdout_str = result.stdout_str(); + if !stdout_str.contains("copy offload: unknown, reflink: no, sparse detection: SEEK_HOLE") { + panic!("Failure: stdout was \n{stdout_str}"); + } +} +#[test] +#[cfg(any(target_os = "linux", target_os = "android"))] +fn test_cp_debug_reflink_never_file_with_hole() { + let ts = TestScenario::new(util_name!()); + let at = &ts.fixtures; + at.touch("a"); + let f = std::fs::OpenOptions::new() + .write(true) + .open(at.plus("a")) + .unwrap(); + f.set_len(10000).unwrap(); + at.append_bytes("a", "hello".as_bytes()); + let result = ts + .ucmd() + .arg("--debug") + .arg("--reflink=never") + .arg("--sparse=never") + .arg("a") + .arg("b") + .succeeds(); + let stdout_str = result.stdout_str(); + if !stdout_str.contains("copy offload: avoided, reflink: no, sparse detection: SEEK_HOLE") { + panic!("Failure: stdout was \n{stdout_str}"); + } +} + +#[test] +#[cfg(any(target_os = "linux", target_os = "android"))] +fn test_cp_debug_sparse_never_less_than_512_bytes() { + let ts = TestScenario::new(util_name!()); + + let at = &ts.fixtures; + at.touch("a"); + at.append_bytes("a", "hello".as_bytes()); + let f = std::fs::OpenOptions::new() + .write(true) + .open(at.plus("a")) + .unwrap(); + f.set_len(400).unwrap(); + + let result = ts + .ucmd() + .arg("--debug") + .arg("--reflink=auto") + .arg("--sparse=never") + .arg("a") + .arg("b") + .succeeds(); + + let stdout_str = result.stdout_str(); + if !stdout_str.contains("copy offload: avoided, reflink: no, sparse detection: no") { + panic!("Failure: stdout was \n{stdout_str}"); + } +} + +#[test] +#[cfg(any(target_os = "linux", target_os = "android"))] +fn test_cp_debug_sparse_never_without_hole() { + let ts = TestScenario::new(util_name!()); + + let at = &ts.fixtures; + at.touch("a"); + at.append_bytes("a", "hello".as_bytes()); + + let filler_bytes = [0 as u8; 10000]; + + at.append_bytes("a", &filler_bytes); + + let result = ts + .ucmd() + .arg("--reflink=auto") + .arg("--sparse=never") + .arg("--debug") + .arg("a") + .arg("b") + .succeeds(); + + let stdout_str = result.stdout_str(); + if !stdout_str.contains("copy offload: avoided, reflink: no, sparse detection: no") { + panic!("Failure: stdout was \n{stdout_str}"); + } +} +#[test] +#[cfg(any(target_os = "linux", target_os = "android"))] +fn test_cp_debug_sparse_never_empty_file_with_hole() { + let ts = TestScenario::new(util_name!()); + let at = &ts.fixtures; + at.touch("a"); + let f = std::fs::OpenOptions::new() + .write(true) + .open(at.plus("a")) + .unwrap(); + f.set_len(10000).unwrap(); + + let result = ts + .ucmd() + .arg("--debug") + .arg("--reflink=auto") + .arg("--sparse=never") + .arg("a") + .arg("b") + .succeeds(); + let stdout_str = result.stdout_str(); + if !stdout_str.contains("copy offload: unknown, reflink: no, sparse detection: SEEK_HOLE") { + panic!("Failure: stdout was \n{stdout_str}"); + } +} +#[test] +#[cfg(any(target_os = "linux", target_os = "android"))] +fn test_cp_debug_sparse_never_file_with_hole() { + let ts = TestScenario::new(util_name!()); + let at = &ts.fixtures; + at.touch("a"); + let f = std::fs::OpenOptions::new() + .write(true) + .open(at.plus("a")) + .unwrap(); + f.set_len(10000).unwrap(); + at.append_bytes("a", "hello".as_bytes()); + let result = ts + .ucmd() + .arg("--debug") + .arg("--reflink=auto") + .arg("--sparse=never") + .arg("a") + .arg("b") + .succeeds(); + let stdout_str = result.stdout_str(); + if !stdout_str.contains("copy offload: avoided, reflink: no, sparse detection: SEEK_HOLE") { + panic!("Failure: stdout was \n{stdout_str}"); + } +} #[test] #[cfg(any(target_os = "linux", target_os = "android"))] From 89c6464b15b1f08e0c426720656ddca407be5884 Mon Sep 17 00:00:00 2001 From: Anirban Halder <92542059+AnirbanHalder654322@users.noreply.github.com> Date: Sun, 14 Apr 2024 00:16:57 +0530 Subject: [PATCH 5/8] Apply suggestions from code review Co-authored-by: Sylvestre Ledru --- src/uu/cp/src/platform/linux.rs | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/src/uu/cp/src/platform/linux.rs b/src/uu/cp/src/platform/linux.rs index fc862d1be25..8c286f71b58 100644 --- a/src/uu/cp/src/platform/linux.rs +++ b/src/uu/cp/src/platform/linux.rs @@ -90,25 +90,22 @@ fn check_for_data(source: &Path) -> Result<(bool, u64, u64), std::io::Error> { let blocks = metadata.blocks(); let blksize = metadata.blksize(); + if size == 0 { - let mut buf: Vec = vec![0; blksize as usize]; - let _ = src_file.read(&mut buf)?; - if buf.iter().any(|&x| x != 0x0) { - return Ok((true, size, 0)); - } - return Ok((false, size, 0)); + let mut buf: Vec = vec![0; metadata.blksize() as usize]; // Directly use metadata.blksize() + src_file.read(&mut buf)?; + return Ok((buf.iter().any(|&x| x != 0x0), size, 0)); } let src_fd = src_file.as_raw_fd(); let result = unsafe { libc::lseek(src_fd, 0, SEEK_DATA) }; - if result == -1 { - Ok((false, size, blocks)) - } else if result >= 0 && result < size as i64 { - Ok((true, size, blocks)) - } else { - return Err(std::io::Error::last_os_error()); + + match result { + -1 => Ok((false, size, blocks)), // No data found or end of file + _ if result >= 0 => Ok((true, size, blocks)), // Data found + _ => Err(io::Error::last_os_error()), } } From be75006f84472543a086948d41d96fa93ef640a7 Mon Sep 17 00:00:00 2001 From: Anirban Halder Date: Sun, 14 Apr 2024 20:46:19 +0530 Subject: [PATCH 6/8] Fixed failing android tests --- src/uu/cp/src/platform/linux.rs | 8 ++++---- tests/by-util/test_cp.rs | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/uu/cp/src/platform/linux.rs b/src/uu/cp/src/platform/linux.rs index 54133fabe8c..f3ea92d54a8 100644 --- a/src/uu/cp/src/platform/linux.rs +++ b/src/uu/cp/src/platform/linux.rs @@ -94,7 +94,7 @@ fn check_for_data(source: &Path) -> Result<(bool, u64, u64), std::io::Error> { // checks edge case of virtual files in /proc which have a size of zero but contains data if size == 0 { let mut buf: Vec = vec![0; metadata.blksize() as usize]; // Directly use metadata.blksize() - src_file.read(&mut buf)?; + let _ = src_file.read(&mut buf)?; return Ok((buf.iter().any(|&x| x != 0x0), size, 0)); } @@ -111,7 +111,7 @@ fn check_for_data(source: &Path) -> Result<(bool, u64, u64), std::io::Error> { #[cfg(any(target_os = "linux", target_os = "android"))] /// Checks whether a file is sparse i.e. it contains holes, uses the crude heurestic blocks < size / 512 -/// Reference: https://doc.rust-lang.org/std/os/unix/fs/trait.MetadataExt.html#tymethod.blocks +/// Reference:`` fn check_sparse_detection(source: &Path) -> Result { let src_file = File::open(source)?; let metadata = src_file.metadata()?; @@ -135,7 +135,7 @@ where let dst_file = File::create(dest)?; let dst_fd = dst_file.as_raw_fd(); - let size: u64 = src_file.metadata()?.size(); + let size = src_file.metadata()?.size(); if unsafe { libc::ftruncate(dst_fd, size.try_into().unwrap()) } < 0 { return Err(std::io::Error::last_os_error()); } @@ -353,7 +353,7 @@ pub(crate) fn copy_on_write( match copy_method { CopyMethod::FSCopy => clone(source, dest, CloneFallback::FSCopy), - _ => sparse_copy(source, dest), + _ => clone(source, dest, CloneFallback::SparseCopy), } } } diff --git a/tests/by-util/test_cp.rs b/tests/by-util/test_cp.rs index f180d5e62bf..97c789f6953 100644 --- a/tests/by-util/test_cp.rs +++ b/tests/by-util/test_cp.rs @@ -4053,7 +4053,7 @@ fn test_cp_debug_reflink_never_sparse_always_empty_file_with_hole() { } #[test] -#[cfg(any(target_os = "linux", target_os = "android"))] +#[cfg(target_os = "linux")] fn test_cp_default_virtual_file() { use std::os::unix::prelude::MetadataExt; let ts = TestScenario::new(util_name!()); @@ -4152,7 +4152,7 @@ fn test_cp_debug_reflink_never_sparse_always_non_sparse_file_with_long_zero_sequ } #[test] -#[cfg(any(target_os = "linux", target_os = "android"))] +#[cfg(target_os = "linux")] fn test_cp_debug_sparse_always_sparse_virtual_file() { let ts = TestScenario::new(util_name!()); let result = ts @@ -4357,7 +4357,7 @@ fn test_cp_debug_sparse_never_file_with_hole() { } #[test] -#[cfg(any(target_os = "linux", target_os = "android"))] +#[cfg(target_os = "linux")] fn test_cp_debug_default_sparse_virtual_file() { let ts = TestScenario::new(util_name!()); let result = ts @@ -4376,7 +4376,7 @@ fn test_cp_debug_default_sparse_virtual_file() { } #[test] -#[cfg(any(target_os = "linux", target_os = "android"))] +#[cfg(target_os = "linux")] fn test_cp_debug_sparse_never_zero_sized_virtual_file() { let ts = TestScenario::new(util_name!()); let result = ts @@ -4394,7 +4394,7 @@ fn test_cp_debug_sparse_never_zero_sized_virtual_file() { } #[test] -#[cfg(any(target_os = "linux", target_os = "android"))] +#[cfg(target_os = "linux")] fn test_cp_debug_default_zero_sized_virtual_file() { let ts = TestScenario::new(util_name!()); let result = ts From bf62ae6d84d6d330cdd37302f4dafb8c4dded8b8 Mon Sep 17 00:00:00 2001 From: Anirban Halder Date: Mon, 15 Apr 2024 02:15:32 +0530 Subject: [PATCH 7/8] Fixed spelling and added lseek to spellcheck ignore --- src/uu/cp/src/platform/linux.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/uu/cp/src/platform/linux.rs b/src/uu/cp/src/platform/linux.rs index f3ea92d54a8..637b8969c45 100644 --- a/src/uu/cp/src/platform/linux.rs +++ b/src/uu/cp/src/platform/linux.rs @@ -2,7 +2,7 @@ // // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -// spell-checker:ignore ficlone reflink ftruncate pwrite fiemap +// spell-checker:ignore ficlone reflink ftruncate pwrite fiemap lseek use libc::{SEEK_DATA, SEEK_HOLE}; use std::fs::{File, OpenOptions}; @@ -110,7 +110,7 @@ fn check_for_data(source: &Path) -> Result<(bool, u64, u64), std::io::Error> { } #[cfg(any(target_os = "linux", target_os = "android"))] -/// Checks whether a file is sparse i.e. it contains holes, uses the crude heurestic blocks < size / 512 +/// Checks whether a file is sparse i.e. it contains holes, uses the crude heuristic blocks < size / 512 /// Reference:`` fn check_sparse_detection(source: &Path) -> Result { let src_file = File::open(source)?; From c0c034232122b2501803bfdd0e90e7161b9a8af5 Mon Sep 17 00:00:00 2001 From: Anirban Halder Date: Sun, 21 Apr 2024 19:14:59 +0530 Subject: [PATCH 8/8] Fix clippy errors --- tests/by-util/test_cp.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/by-util/test_cp.rs b/tests/by-util/test_cp.rs index fd6f9e37908..8ab6ccc0ec1 100644 --- a/tests/by-util/test_cp.rs +++ b/tests/by-util/test_cp.rs @@ -3958,7 +3958,7 @@ fn test_cp_debug_default_without_hole() { at.touch("a"); at.append_bytes("a", "hello".as_bytes()); - let filler_bytes = [0 as u8; 10000]; + let filler_bytes = [0_u8; 10000]; at.append_bytes("a", &filler_bytes); @@ -4034,7 +4034,7 @@ fn test_cp_debug_reflink_never_sparse_always_with_hole() { #[cfg(any(target_os = "linux", target_os = "android"))] fn test_cp_debug_reflink_never_sparse_always_without_hole() { let ts = TestScenario::new(util_name!()); - let empty_bytes = [0 as u8; 10000]; + let empty_bytes = [0_u8; 10000]; let at = &ts.fixtures; at.touch("a"); at.write("a", "hello"); @@ -4321,7 +4321,7 @@ fn test_cp_debug_sparse_never_without_hole() { at.touch("a"); at.append_bytes("a", "hello".as_bytes()); - let filler_bytes = [0 as u8; 10000]; + let filler_bytes = [0_u8; 10000]; at.append_bytes("a", &filler_bytes); @@ -4449,7 +4449,7 @@ fn test_cp_debug_default_zero_sized_virtual_file() { #[cfg(any(target_os = "linux", target_os = "android"))] fn test_cp_debug_reflink_never_without_hole() { let ts = TestScenario::new(util_name!()); - let filler_bytes = [0 as u8; 1000]; + let filler_bytes = [0_u8; 1000]; let at = &ts.fixtures; at.touch("a"); at.write("a", "hello");