ARROW-16977: [R] Update dataset row counting so no integer overflow on large datasets

r/NAMESPACE

@@ -388,6 +388,7 @@ importFrom(purrr,map)
 importFrom(purrr,map2)
 importFrom(purrr,map2_chr)
 importFrom(purrr,map_chr)
+importFrom(purrr,map_dbl)
 importFrom(purrr,map_dfr)
 importFrom(purrr,map_int)
 importFrom(purrr,map_lgl)

r/R/arrow-package.R

@@ -17,7 +17,7 @@
 
 #' @importFrom stats quantile median na.omit na.exclude na.pass na.fail
 #' @importFrom R6 R6Class
-#' @importFrom purrr as_mapper map map2 map_chr map2_chr map_dfr map_int map_lgl keep imap imap_chr flatten
+#' @importFrom purrr as_mapper map map2 map_chr map2_chr map_dbl map_dfr map_int map_lgl keep imap imap_chr flatten
 #' @importFrom assertthat assert_that is.string
 #' @importFrom rlang list2 %||% is_false abort dots_n warn enquo quo_is_null enquos is_integerish quos
 #' @importFrom rlang eval_tidy new_data_mask syms env new_environment env_bind set_names exec

r/R/record-batch.R

@@ -196,8 +196,8 @@ rbind.RecordBatch <- function(...) {
 }
 
 cbind_check_length <- function(inputs, call = caller_env()) {
-  sizes <- map_int(inputs, NROW)
-  ok_lengths <- sizes %in% c(head(sizes, 1), 1L)
+  sizes <- map_dbl(inputs, NROW)
+  ok_lengths <- sizes %in% c(head(sizes, 1), 1)
   if (!all(ok_lengths)) {
     first_bad_one <- which.min(ok_lengths)
     abort(

r/R/util.R

@@ -158,7 +158,7 @@ as_writable_table <- function(x) {
 #' @keywords internal
 recycle_scalars <- function(arrays) {
   # Get lengths of items in arrays
-  arr_lens <- map_int(arrays, NROW)
+  arr_lens <- map_dbl(arrays, NROW)
 
   is_scalar <- arr_lens == 1
 

r/src/array.cpp

@@ -244,15 +244,15 @@ int32_t ListArray__value_length(const std::shared_ptr<arrow::ListArray>& array,
 }
 
 // [[arrow::export]]
-int64_t LargeListArray__value_length(const std::shared_ptr<arrow::LargeListArray>& array,
-                                     int64_t i) {
-  return array->value_length(i);
+r_vec_size LargeListArray__value_length(
+    const std::shared_ptr<arrow::LargeListArray>& array, int64_t i) {
+  return r_vec_size(array->value_length(i));
 }
 
 // [[arrow::export]]
-int64_t FixedSizeListArray__value_length(
+r_vec_size FixedSizeListArray__value_length(
     const std::shared_ptr<arrow::FixedSizeListArray>& array, int64_t i) {
-  return array->value_length(i);
+  return r_vec_size(array->value_length(i));
 }
 
 // [[arrow::export]]
@@ -262,15 +262,15 @@ int32_t ListArray__value_offset(const std::shared_ptr<arrow::ListArray>& array,
 }
 
 // [[arrow::export]]
-int64_t LargeListArray__value_offset(const std::shared_ptr<arrow::LargeListArray>& array,
-                                     int64_t i) {
-  return array->value_offset(i);
+r_vec_size LargeListArray__value_offset(
+    const std::shared_ptr<arrow::LargeListArray>& array, int64_t i) {
+  return r_vec_size(array->value_offset(i));
 }
 
 // [[arrow::export]]
-int64_t FixedSizeListArray__value_offset(
+r_vec_size FixedSizeListArray__value_offset(
     const std::shared_ptr<arrow::FixedSizeListArray>& array, int64_t i) {
-  return array->value_offset(i);
+  return r_vec_size(array->value_offset(i));
 }
 
 // [[arrow::export]]
@@ -319,8 +319,8 @@ bool Array__Same(const std::shared_ptr<arrow::Array>& x,
 }
 
 // [[arrow::export]]
-int64_t Array__ReferencedBufferSize(const std::shared_ptr<arrow::Array>& x) {
-  return ValueOrStop(arrow::util::ReferencedBufferSize(*x));
+r_vec_size Array__ReferencedBufferSize(const std::shared_ptr<arrow::Array>& x) {
+  return r_vec_size(ValueOrStop(arrow::util::ReferencedBufferSize(*x)));
 }
 
 // [[arrow::export]]

r/src/buffer.cpp

@@ -28,13 +28,13 @@ void Buffer__ZeroPadding(const std::shared_ptr<arrow::Buffer>& buffer) {
 }
 
 // [[arrow::export]]
-int64_t Buffer__capacity(const std::shared_ptr<arrow::Buffer>& buffer) {
-  return buffer->capacity();
+r_vec_size Buffer__capacity(const std::shared_ptr<arrow::Buffer>& buffer) {
+  return r_vec_size(buffer->capacity());
 }
 
 // [[arrow::export]]
-int64_t Buffer__size(const std::shared_ptr<arrow::Buffer>& buffer) {
-  return buffer->size();
+r_vec_size Buffer__size(const std::shared_ptr<arrow::Buffer>& buffer) {
+  return r_vec_size(buffer->size());
 }
 
 // [[arrow::export]]

r/src/chunkedarray.cpp

@@ -22,18 +22,21 @@
 #include <arrow/util/byte_size.h>
 
 // [[arrow::export]]
-int ChunkedArray__length(const std::shared_ptr<arrow::ChunkedArray>& chunked_array) {
-  return chunked_array->length();
+r_vec_size ChunkedArray__length(
+    const std::shared_ptr<arrow::ChunkedArray>& chunked_array) {
+  return r_vec_size(chunked_array->length());
 }
 
 // [[arrow::export]]
-int ChunkedArray__null_count(const std::shared_ptr<arrow::ChunkedArray>& chunked_array) {
-  return chunked_array->null_count();
+r_vec_size ChunkedArray__null_count(
+    const std::shared_ptr<arrow::ChunkedArray>& chunked_array) {
+  return r_vec_size(chunked_array->null_count());
 }
 
 // [[arrow::export]]
-int ChunkedArray__num_chunks(const std::shared_ptr<arrow::ChunkedArray>& chunked_array) {
-  return chunked_array->num_chunks();
+r_vec_size ChunkedArray__num_chunks(
+    const std::shared_ptr<arrow::ChunkedArray>& chunked_array) {
+  return r_vec_size(chunked_array->num_chunks());
 }
 
 // [[arrow::export]]
@@ -144,7 +147,7 @@ std::shared_ptr<arrow::ChunkedArray> ChunkedArray__from_list(cpp11::list chunks,
 }
 
 // [[arrow::export]]
-int64_t ChunkedArray__ReferencedBufferSize(
+r_vec_size ChunkedArray__ReferencedBufferSize(
     const std::shared_ptr<arrow::ChunkedArray>& chunked_array) {
-  return ValueOrStop(arrow::util::ReferencedBufferSize(*chunked_array));
+  return r_vec_size(ValueOrStop(arrow::util::ReferencedBufferSize(*chunked_array)));
 }

r/src/dataset.cpp

@@ -511,8 +511,8 @@ std::shared_ptr<arrow::Table> dataset___Scanner__TakeRows(
 }
 
 // [[dataset::export]]
-int64_t dataset___Scanner__CountRows(const std::shared_ptr<ds::Scanner>& scanner) {
-  return ValueOrStop(scanner->CountRows());
+r_vec_size dataset___Scanner__CountRows(const std::shared_ptr<ds::Scanner>& scanner) {
+  return r_vec_size(ValueOrStop(scanner->CountRows()));
 }
 
 #endif