GH-41973: Expose new S3 option check_directory_existence_before_creation - manual rebase

r/NEWS.md

@@ -22,6 +22,8 @@
 ## Minor improvements and fixes
 
 - Added bindings for atan, sinh, cosh, tanh, asinh, acosh, and tanh, and expm1 (#44953)
+- Expose an option `check_directory_existence_before_creation` in `S3FileSystem` 
+  to reduce I/O calls on cloud storage (@HaochengLIU, #41998)
 
 # arrow 20.0.0
 

r/R/dplyr-funcs-datetime.R

@@ -856,7 +856,8 @@ register_bindings_hms <- function() {
         Expression$create("multiply_checked", days, 86400)
 
       return(numeric_to_time32(total_secs))
-    }
+    },
+    notes = "subsecond times not supported"
   )
 
   register_binding(
@@ -880,6 +881,7 @@ register_bindings_hms <- function() {
         )
         return(datetime_to_time32(as_date_time))
       }
-    }
+    },
+    notes = "subsecond times not supported"
   )
 }

r/R/dplyr-funcs-doc.R

@@ -21,7 +21,7 @@
 #'
 #' The `arrow` package contains methods for 37 `dplyr` table functions, many of
 #' which are "verbs" that do transformations to one or more tables.
-#' The package also has mappings of 221 R functions to the corresponding
+#' The package also has mappings of 222 R functions to the corresponding
 #' functions in the Arrow compute library. These allow you to write code inside
 #' of `dplyr` methods that call R functions, including many in packages like
 #' `stringr` and `lubridate`, and they will get translated to Arrow and run
@@ -83,7 +83,7 @@
 #' Functions can be called either as `pkg::fun()` or just `fun()`, i.e. both
 #' `str_sub()` and `stringr::str_sub()` work.
 #'
-#' In addition to these functions, you can call any of Arrow's 271 compute
+#' In addition to these functions, you can call any of Arrow's 280 compute
 #' functions directly. Arrow has many functions that don't map to an existing R
 #' function. In other cases where there is an R function mapping, you can still
 #' call the Arrow function directly if you don't want the adaptations that the R
@@ -96,7 +96,6 @@
 #'
 #' * [`add_filename()`][arrow::add_filename()]
 #' * [`cast()`][arrow::cast()]
-#' * [`one()`][arrow::one()]
 #'
 #' ## base
 #'
@@ -215,6 +214,11 @@
 #' * [`n()`][dplyr::n()]
 #' * [`n_distinct()`][dplyr::n_distinct()]
 #'
+#' ## hms
+#'
+#' * [`as_hms()`][hms::as_hms()]
+#' * [`hms()`][hms::hms()]
+#'
 #' ## lubridate
 #'
 #' * [`am()`][lubridate::am()]
@@ -297,11 +301,6 @@
 #' * [`ymd_hms()`][lubridate::ymd_hms()]: `locale` argument not supported
 #' * [`yq()`][lubridate::yq()]: `locale` argument not supported
 #'
-#' ## hms
-#'
-#' * [`hms()`][hms::hms()]: subsecond times not supported
-#' * [`hms()`][hms::as_hms()]: subsecond times not supported
-#'
 #' ## methods
 #'
 #' * [`is()`][methods::is()]

r/R/filesystem.R

@@ -156,6 +156,10 @@ FileSelector$create <- function(base_dir, allow_not_found = FALSE, recursive = F
 #'    buckets if `$CreateDir()` is called on the bucket level (default `FALSE`).
 #' - `allow_bucket_deletion`: logical, if TRUE, the filesystem will delete
 #'    buckets if`$DeleteDir()` is called on the bucket level (default `FALSE`).
+#' - `check_directory_existence_before_creation`: logical, check if directory
+#'    already exists or not before creation. Helpful for cloud storage operations
+#'    where object mutation operations are rate limited or existing directories
+#'    are read-only. (default `FALSE`).
 #' - `request_timeout`: Socket read time on Windows and macOS in seconds. If
 #'    negative, the AWS SDK default (typically 3 seconds).
 #' - `connect_timeout`: Socket connection timeout in seconds. If negative, AWS
@@ -411,7 +415,8 @@ S3FileSystem$create <- function(anonymous = FALSE, ...) {
     invalid_args <- intersect(
       c(
         "access_key", "secret_key", "session_token", "role_arn", "session_name",
-        "external_id", "load_frequency", "allow_bucket_creation", "allow_bucket_deletion"
+        "external_id", "load_frequency", "allow_bucket_creation", "allow_bucket_deletion",
+        "check_directory_existence_before_creation"
       ),
       names(args)
     )
@@ -459,6 +464,7 @@ default_s3_options <- list(
   background_writes = TRUE,
   allow_bucket_creation = FALSE,
   allow_bucket_deletion = FALSE,
+  check_directory_existence_before_creation = FALSE,
   connect_timeout = -1,
   request_timeout = -1
 )

r/data-raw/docgen.R

@@ -149,6 +149,10 @@ tidyselect <- grep("^tidyselect::", readLines("R/reexports-tidyselect.R"), value
 # HACK: remove the _random_along UDF we're using (fix in ARROW-17974)
 docs[["_random_along"]] <- NULL
 
+# TODO - update the script to add this back in - will fail CI as tries to link
+# to non-existent function as arrow::one only exists as registered binding
+docs[["arrow::one"]] <- NULL
+
 docs <- c(docs, setNames(rep(list(NULL), length(tidyselect)), tidyselect))
 
 fun_df <- tibble::tibble(

r/src/filesystem.cpp

@@ -289,7 +289,8 @@ std::shared_ptr<fs::S3FileSystem> fs___S3FileSystem__create(
     std::string region = "", std::string endpoint_override = "", std::string scheme = "",
     std::string proxy_options = "", bool background_writes = true,
     bool allow_bucket_creation = false, bool allow_bucket_deletion = false,
-    double connect_timeout = -1, double request_timeout = -1) {
+    bool check_directory_existence_before_creation = false, double connect_timeout = -1,
+    double request_timeout = -1) {
   // We need to ensure that S3 is initialized before we start messing with the
   // options
   StopIfNotOk(fs::EnsureS3Initialized());
@@ -331,6 +332,9 @@ std::shared_ptr<fs::S3FileSystem> fs___S3FileSystem__create(
   s3_opts.allow_bucket_creation = allow_bucket_creation;
   s3_opts.allow_bucket_deletion = allow_bucket_deletion;
 
+  s3_opts.check_directory_existence_before_creation =
+      check_directory_existence_before_creation;
+
   s3_opts.request_timeout = request_timeout;
   s3_opts.connect_timeout = connect_timeout;
 

r/tests/testthat/test-s3-minio.R

@@ -46,15 +46,17 @@ fs <- S3FileSystem$create(
   scheme = "http",
   endpoint_override = paste0("localhost:", minio_port),
   allow_bucket_creation = TRUE,
-  allow_bucket_deletion = TRUE
+  allow_bucket_deletion = TRUE,
+  check_directory_existence_before_creation = TRUE
 )
 limited_fs <- S3FileSystem$create(
   access_key = minio_key,
   secret_key = minio_secret,
   scheme = "http",
   endpoint_override = paste0("localhost:", minio_port),
   allow_bucket_creation = FALSE,
-  allow_bucket_deletion = FALSE
+  allow_bucket_deletion = FALSE,
+  check_directory_existence_before_creation = FALSE
 )
 now <- as.character(as.numeric(Sys.time()))
 fs$CreateDir(now)

r/vignettes/fs.Rmd

@@ -190,7 +190,7 @@ Also note that parameters in the URI need to be
 
 For S3, only the following options can be included in the URI as query parameters
 are `region`, `scheme`, `endpoint_override`, `access_key`, `secret_key`, `allow_bucket_creation`,
-and `allow_bucket_deletion`. For GCS, the supported parameters are `scheme`, `endpoint_override`,
+`allow_bucket_deletion` and `check_directory_existence_before_creation`. For GCS, the supported parameters are `scheme`, `endpoint_override`,
 and `retry_limit_seconds`.
 
 In GCS, a useful option is `retry_limit_seconds`, which sets the number of seconds