GH-41973: Expose new S3 option check_directory_existence_before_creation

r/NEWS.md

@@ -22,6 +22,8 @@
 ## Minor improvements and fixes
 
 - Added bindings for atan, sinh, cosh, tanh, asinh, acosh, and tanh, and expm1 (#44953)
+- Expose an option `check_directory_existence_before_creation` in `S3FileSystem` 
+  to reduce I/O calls on cloud storage (@HaochengLIU, #41998)
 
 # arrow 20.0.0
 

r/R/filesystem.R

@@ -156,6 +156,10 @@ FileSelector$create <- function(base_dir, allow_not_found = FALSE, recursive = F
 #'    buckets if `$CreateDir()` is called on the bucket level (default `FALSE`).
 #' - `allow_bucket_deletion`: logical, if TRUE, the filesystem will delete
 #'    buckets if`$DeleteDir()` is called on the bucket level (default `FALSE`).
+#' - `check_directory_existence_before_creation`: logical, check if directory
+#'    already exists or not before creation. Helpful for cloud storage operations
+#'    where object mutation operations are rate limited or existing directories
+#'    are read-only. (default `FALSE`).
 #' - `request_timeout`: Socket read time on Windows and macOS in seconds. If
 #'    negative, the AWS SDK default (typically 3 seconds).
 #' - `connect_timeout`: Socket connection timeout in seconds. If negative, AWS
@@ -411,7 +415,8 @@ S3FileSystem$create <- function(anonymous = FALSE, ...) {
     invalid_args <- intersect(
       c(
         "access_key", "secret_key", "session_token", "role_arn", "session_name",
-        "external_id", "load_frequency", "allow_bucket_creation", "allow_bucket_deletion"
+        "external_id", "load_frequency", "allow_bucket_creation", "allow_bucket_deletion",
+        "check_directory_existence_before_creation"
       ),
       names(args)
     )
@@ -459,6 +464,7 @@ default_s3_options <- list(
   background_writes = TRUE,
   allow_bucket_creation = FALSE,
   allow_bucket_deletion = FALSE,
+  check_directory_existence_before_creation = FALSE,
   connect_timeout = -1,
   request_timeout = -1
 )

r/src/filesystem.cpp

@@ -289,7 +289,8 @@ std::shared_ptr<fs::S3FileSystem> fs___S3FileSystem__create(
     std::string region = "", std::string endpoint_override = "", std::string scheme = "",
     std::string proxy_options = "", bool background_writes = true,
     bool allow_bucket_creation = false, bool allow_bucket_deletion = false,
-    double connect_timeout = -1, double request_timeout = -1) {
+    bool check_directory_existence_before_creation = false, double connect_timeout = -1,
+    double request_timeout = -1) {
   // We need to ensure that S3 is initialized before we start messing with the
   // options
   StopIfNotOk(fs::EnsureS3Initialized());
@@ -330,6 +331,8 @@ std::shared_ptr<fs::S3FileSystem> fs___S3FileSystem__create(
 
   s3_opts.allow_bucket_creation = allow_bucket_creation;
   s3_opts.allow_bucket_deletion = allow_bucket_deletion;
+  s3_opts.check_directory_existence_before_creation =
+      check_directory_existence_before_creation;
 
   s3_opts.request_timeout = request_timeout;
   s3_opts.connect_timeout = connect_timeout;

r/tests/testthat/test-s3-minio.R

@@ -46,15 +46,17 @@ fs <- S3FileSystem$create(
   scheme = "http",
   endpoint_override = paste0("localhost:", minio_port),
   allow_bucket_creation = TRUE,
-  allow_bucket_deletion = TRUE
+  allow_bucket_deletion = TRUE,
+  check_directory_existence_before_creation = TRUE,
 )
 limited_fs <- S3FileSystem$create(
   access_key = minio_key,
   secret_key = minio_secret,
   scheme = "http",
   endpoint_override = paste0("localhost:", minio_port),
   allow_bucket_creation = FALSE,
-  allow_bucket_deletion = FALSE
+  allow_bucket_deletion = FALSE,
+  check_directory_existence_before_creation = FALSE,
 )
 now <- as.character(as.numeric(Sys.time()))
 fs$CreateDir(now)

r/vignettes/fs.Rmd

@@ -190,7 +190,7 @@ Also note that parameters in the URI need to be
 
 For S3, only the following options can be included in the URI as query parameters
 are `region`, `scheme`, `endpoint_override`, `access_key`, `secret_key`, `allow_bucket_creation`,
-and `allow_bucket_deletion`. For GCS, the supported parameters are `scheme`, `endpoint_override`,
+`allow_bucket_deletion` and `check_directory_existence_before_creation`. For GCS, the supported parameters are `scheme`, `endpoint_override`,
 and `retry_limit_seconds`.
 
 In GCS, a useful option is `retry_limit_seconds`, which sets the number of seconds