From 00952f10767639d25b0b093dc11a23f2d5825d6c Mon Sep 17 00:00:00 2001 From: Romain Francois Date: Mon, 16 Sep 2019 09:26:28 +0200 Subject: [PATCH 01/13] Expose enum arrow::fs::FileType --- r/R/enums.R | 5 +++++ r/src/arrow_types.h | 2 ++ 2 files changed, 7 insertions(+) diff --git a/r/R/enums.R b/r/R/enums.R index 5a5cdfa3d25..17342d0678e 100644 --- a/r/R/enums.R +++ b/r/R/enums.R @@ -78,3 +78,8 @@ MessageType <- enum("Message::Type", CompressionType <- enum("Compression::type", UNCOMPRESSED = 0L, SNAPPY = 1L, GZIP = 2L, BROTLI = 3L, ZSTD = 4L, LZ4 = 5L, LZO = 6L, BZ2 = 7L ) + +#' @rdname enums +FileType <- enum("FileType", + NonExistent = 0L, Unknown = 1L, File = 2L, Directory = 3L +) diff --git a/r/src/arrow_types.h b/r/src/arrow_types.h index 4259a48cdc5..8a15f53fd6d 100644 --- a/r/src/arrow_types.h +++ b/r/src/arrow_types.h @@ -175,6 +175,7 @@ inline std::shared_ptr extract(SEXP x) { #include #include #include +#include #include #include #include @@ -187,6 +188,7 @@ RCPP_EXPOSED_ENUM_NODECL(arrow::StatusCode) RCPP_EXPOSED_ENUM_NODECL(arrow::io::FileMode::type) RCPP_EXPOSED_ENUM_NODECL(arrow::ipc::Message::Type) RCPP_EXPOSED_ENUM_NODECL(arrow::Compression::type) +RCPP_EXPOSED_ENUM_NODECL(arrow::fs::FileType) SEXP ChunkedArray__as_vector(const std::shared_ptr& chunked_array); SEXP Array__as_vector(const std::shared_ptr& array); From 63a2df81b8826892553cb81933e7ca9102e72d8d Mon Sep 17 00:00:00 2001 From: Romain Francois Date: Mon, 16 Sep 2019 12:14:45 +0200 Subject: [PATCH 02/13] Support for arrow::fs::FileStats --- r/DESCRIPTION | 1 + r/NAMESPACE | 1 + r/R/arrowExports.R | 40 ++++++++++ r/R/filesystem.R | 65 ++++++++++++++++ r/man/FileStats.Rd | 10 +++ r/man/enums.Rd | 3 + r/src/arrowExports.cpp | 168 +++++++++++++++++++++++++++++++++++++++++ r/src/arrow_types.h | 5 ++ r/src/filesystem.cpp | 79 +++++++++++++++++++ r/src/symbols.cpp | 10 +++ 10 files changed, 382 insertions(+) create mode 100644 r/R/filesystem.R create mode 100644 r/man/FileStats.Rd create mode 100644 r/src/filesystem.cpp diff --git a/r/DESCRIPTION b/r/DESCRIPTION index 7b68377d41f..71a891d3ea7 100644 --- a/r/DESCRIPTION +++ b/r/DESCRIPTION @@ -63,6 +63,7 @@ Collate: 'dictionary.R' 'feather.R' 'field.R' + 'filesystem.R' 'install-arrow.R' 'json.R' 'list.R' diff --git a/r/NAMESPACE b/r/NAMESPACE index 1060bddf8ea..cc560e2cfdf 100644 --- a/r/NAMESPACE +++ b/r/NAMESPACE @@ -52,6 +52,7 @@ export(FeatherTableWriter) export(Field) export(FileMode) export(FileOutputStream) +export(FileStats) export(FixedSizeBufferWriter) export(MemoryMappedFile) export(MessageReader) diff --git a/r/R/arrowExports.R b/r/R/arrowExports.R index d98a6b06c07..80dcaca9689 100644 --- a/r/R/arrowExports.R +++ b/r/R/arrowExports.R @@ -540,6 +540,46 @@ Field__type <- function(field){ .Call(`_arrow_Field__type` , field) } +fs___FileStats__type <- function(x){ + .Call(`_arrow_fs___FileStats__type` , x) +} + +fs___FileStats__set_type <- function(x, type){ + invisible(.Call(`_arrow_fs___FileStats__set_type` , x, type)) +} + +fs___FileStats__path <- function(x){ + .Call(`_arrow_fs___FileStats__path` , x) +} + +fs___FileStats__set_path <- function(x, path){ + invisible(.Call(`_arrow_fs___FileStats__set_path` , x, path)) +} + +fs___FileStats__size <- function(x){ + .Call(`_arrow_fs___FileStats__size` , x) +} + +fs___FileStats__set_size <- function(x, size){ + invisible(.Call(`_arrow_fs___FileStats__set_size` , x, size)) +} + +fs___FileStats__base_name <- function(x){ + .Call(`_arrow_fs___FileStats__base_name` , x) +} + +fs___FileStats__extension <- function(x){ + .Call(`_arrow_fs___FileStats__extension` , x) +} + +fs___FileStats__mtime <- function(x){ + .Call(`_arrow_fs___FileStats__mtime` , x) +} + +fs___FileStats__set_mtime <- function(x, time){ + invisible(.Call(`_arrow_fs___FileStats__set_mtime` , x, time)) +} + io___Readable__Read <- function(x, nbytes){ .Call(`_arrow_io___Readable__Read` , x, nbytes) } diff --git a/r/R/filesystem.R b/r/R/filesystem.R new file mode 100644 index 00000000000..25a67244a82 --- /dev/null +++ b/r/R/filesystem.R @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +#' @include arrow-package.R + +#' @title FileStats class +#' @description FileSystem entry stats +#' +#' @usage NULL +#' @format NULL +#' @docType class +#' +#' @rdname FileStats +#' @name FileStats +#' @export +FileStats <- R6Class("FileStats", + inherit = Object, + public = list( + base_name = function() fs___FileStats__base_name(self), + extension = function() fs___FileStats__extension(self) + ), + active = list( + type = function() fs___FileStats__type(self), + path = function(path) { + if (missing(path)) { + fs___FileStats__path(self) + } else { + invisible(fs___FileStats__set_path(self)) + } + }, + + size = function(size) { + if (missing(size)) { + fs___FileStats__size(self) + } else { + invisible(fs___FileStats__set_size(self, size)) + } + }, + + mtime = function(time) { + if (missing(time)) { + fs___FileStats__mtime(self) + } else { + if (!inherits(time, "POSIXct") && length(time) == 1L) { + abort("invalid time") + } + invisible(fs___FileStats__set_mtime(self, time)) + } + } + ) +) diff --git a/r/man/FileStats.Rd b/r/man/FileStats.Rd new file mode 100644 index 00000000000..5a8f7c81261 --- /dev/null +++ b/r/man/FileStats.Rd @@ -0,0 +1,10 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/filesystem.R +\docType{class} +\name{FileStats} +\alias{FileStats} +\title{FileStats class} +\description{ +FileSystem entry stats +} +\keyword{datasets} diff --git a/r/man/enums.Rd b/r/man/enums.Rd index f7f76c3c035..3d841fa0c64 100644 --- a/r/man/enums.Rd +++ b/r/man/enums.Rd @@ -10,6 +10,7 @@ \alias{FileMode} \alias{MessageType} \alias{CompressionType} +\alias{FileType} \title{Arrow enums} \format{An object of class \code{TimeUnit::type} (inherits from \code{arrow-enum}) of length 4.} \usage{ @@ -26,6 +27,8 @@ FileMode MessageType CompressionType + +FileType } \description{ Arrow enums diff --git a/r/src/arrowExports.cpp b/r/src/arrowExports.cpp index 0ebac0cc5a1..60a86d01541 100644 --- a/r/src/arrowExports.cpp +++ b/r/src/arrowExports.cpp @@ -2075,6 +2075,164 @@ RcppExport SEXP _arrow_Field__type(SEXP field_sexp){ } #endif +// filesystem.cpp +#if defined(ARROW_R_WITH_ARROW) +arrow::fs::FileType fs___FileStats__type(const std::shared_ptr& x); +RcppExport SEXP _arrow_fs___FileStats__type(SEXP x_sexp){ +BEGIN_RCPP + Rcpp::traits::input_parameter&>::type x(x_sexp); + return Rcpp::wrap(fs___FileStats__type(x)); +END_RCPP +} +#else +RcppExport SEXP _arrow_fs___FileStats__type(SEXP x_sexp){ + Rf_error("Cannot call fs___FileStats__type(). Please use arrow::install_arrow() to install required runtime libraries. "); +} +#endif + +// filesystem.cpp +#if defined(ARROW_R_WITH_ARROW) +void fs___FileStats__set_type(const std::shared_ptr& x, arrow::fs::FileType type); +RcppExport SEXP _arrow_fs___FileStats__set_type(SEXP x_sexp, SEXP type_sexp){ +BEGIN_RCPP + Rcpp::traits::input_parameter&>::type x(x_sexp); + Rcpp::traits::input_parameter::type type(type_sexp); + fs___FileStats__set_type(x, type); + return R_NilValue; +END_RCPP +} +#else +RcppExport SEXP _arrow_fs___FileStats__set_type(SEXP x_sexp, SEXP type_sexp){ + Rf_error("Cannot call fs___FileStats__set_type(). Please use arrow::install_arrow() to install required runtime libraries. "); +} +#endif + +// filesystem.cpp +#if defined(ARROW_R_WITH_ARROW) +std::string fs___FileStats__path(const std::shared_ptr& x); +RcppExport SEXP _arrow_fs___FileStats__path(SEXP x_sexp){ +BEGIN_RCPP + Rcpp::traits::input_parameter&>::type x(x_sexp); + return Rcpp::wrap(fs___FileStats__path(x)); +END_RCPP +} +#else +RcppExport SEXP _arrow_fs___FileStats__path(SEXP x_sexp){ + Rf_error("Cannot call fs___FileStats__path(). Please use arrow::install_arrow() to install required runtime libraries. "); +} +#endif + +// filesystem.cpp +#if defined(ARROW_R_WITH_ARROW) +void fs___FileStats__set_path(const std::shared_ptr& x, const std::string& path); +RcppExport SEXP _arrow_fs___FileStats__set_path(SEXP x_sexp, SEXP path_sexp){ +BEGIN_RCPP + Rcpp::traits::input_parameter&>::type x(x_sexp); + Rcpp::traits::input_parameter::type path(path_sexp); + fs___FileStats__set_path(x, path); + return R_NilValue; +END_RCPP +} +#else +RcppExport SEXP _arrow_fs___FileStats__set_path(SEXP x_sexp, SEXP path_sexp){ + Rf_error("Cannot call fs___FileStats__set_path(). Please use arrow::install_arrow() to install required runtime libraries. "); +} +#endif + +// filesystem.cpp +#if defined(ARROW_R_WITH_ARROW) +int64_t fs___FileStats__size(const std::shared_ptr& x); +RcppExport SEXP _arrow_fs___FileStats__size(SEXP x_sexp){ +BEGIN_RCPP + Rcpp::traits::input_parameter&>::type x(x_sexp); + return Rcpp::wrap(fs___FileStats__size(x)); +END_RCPP +} +#else +RcppExport SEXP _arrow_fs___FileStats__size(SEXP x_sexp){ + Rf_error("Cannot call fs___FileStats__size(). Please use arrow::install_arrow() to install required runtime libraries. "); +} +#endif + +// filesystem.cpp +#if defined(ARROW_R_WITH_ARROW) +void fs___FileStats__set_size(const std::shared_ptr& x, int64_t size); +RcppExport SEXP _arrow_fs___FileStats__set_size(SEXP x_sexp, SEXP size_sexp){ +BEGIN_RCPP + Rcpp::traits::input_parameter&>::type x(x_sexp); + Rcpp::traits::input_parameter::type size(size_sexp); + fs___FileStats__set_size(x, size); + return R_NilValue; +END_RCPP +} +#else +RcppExport SEXP _arrow_fs___FileStats__set_size(SEXP x_sexp, SEXP size_sexp){ + Rf_error("Cannot call fs___FileStats__set_size(). Please use arrow::install_arrow() to install required runtime libraries. "); +} +#endif + +// filesystem.cpp +#if defined(ARROW_R_WITH_ARROW) +std::string fs___FileStats__base_name(const std::shared_ptr& x); +RcppExport SEXP _arrow_fs___FileStats__base_name(SEXP x_sexp){ +BEGIN_RCPP + Rcpp::traits::input_parameter&>::type x(x_sexp); + return Rcpp::wrap(fs___FileStats__base_name(x)); +END_RCPP +} +#else +RcppExport SEXP _arrow_fs___FileStats__base_name(SEXP x_sexp){ + Rf_error("Cannot call fs___FileStats__base_name(). Please use arrow::install_arrow() to install required runtime libraries. "); +} +#endif + +// filesystem.cpp +#if defined(ARROW_R_WITH_ARROW) +std::string fs___FileStats__extension(const std::shared_ptr& x); +RcppExport SEXP _arrow_fs___FileStats__extension(SEXP x_sexp){ +BEGIN_RCPP + Rcpp::traits::input_parameter&>::type x(x_sexp); + return Rcpp::wrap(fs___FileStats__extension(x)); +END_RCPP +} +#else +RcppExport SEXP _arrow_fs___FileStats__extension(SEXP x_sexp){ + Rf_error("Cannot call fs___FileStats__extension(). Please use arrow::install_arrow() to install required runtime libraries. "); +} +#endif + +// filesystem.cpp +#if defined(ARROW_R_WITH_ARROW) +SEXP fs___FileStats__mtime(const std::shared_ptr& x); +RcppExport SEXP _arrow_fs___FileStats__mtime(SEXP x_sexp){ +BEGIN_RCPP + Rcpp::traits::input_parameter&>::type x(x_sexp); + return Rcpp::wrap(fs___FileStats__mtime(x)); +END_RCPP +} +#else +RcppExport SEXP _arrow_fs___FileStats__mtime(SEXP x_sexp){ + Rf_error("Cannot call fs___FileStats__mtime(). Please use arrow::install_arrow() to install required runtime libraries. "); +} +#endif + +// filesystem.cpp +#if defined(ARROW_R_WITH_ARROW) +void fs___FileStats__set_mtime(const std::shared_ptr& x, SEXP time); +RcppExport SEXP _arrow_fs___FileStats__set_mtime(SEXP x_sexp, SEXP time_sexp){ +BEGIN_RCPP + Rcpp::traits::input_parameter&>::type x(x_sexp); + Rcpp::traits::input_parameter::type time(time_sexp); + fs___FileStats__set_mtime(x, time); + return R_NilValue; +END_RCPP +} +#else +RcppExport SEXP _arrow_fs___FileStats__set_mtime(SEXP x_sexp, SEXP time_sexp){ + Rf_error("Cannot call fs___FileStats__set_mtime(). Please use arrow::install_arrow() to install required runtime libraries. "); +} +#endif + // io.cpp #if defined(ARROW_R_WITH_ARROW) std::shared_ptr io___Readable__Read(const std::shared_ptr& x, int64_t nbytes); @@ -3835,6 +3993,16 @@ static const R_CallMethodDef CallEntries[] = { { "_arrow_Field__Equals", (DL_FUNC) &_arrow_Field__Equals, 2}, { "_arrow_Field__nullable", (DL_FUNC) &_arrow_Field__nullable, 1}, { "_arrow_Field__type", (DL_FUNC) &_arrow_Field__type, 1}, + { "_arrow_fs___FileStats__type", (DL_FUNC) &_arrow_fs___FileStats__type, 1}, + { "_arrow_fs___FileStats__set_type", (DL_FUNC) &_arrow_fs___FileStats__set_type, 2}, + { "_arrow_fs___FileStats__path", (DL_FUNC) &_arrow_fs___FileStats__path, 1}, + { "_arrow_fs___FileStats__set_path", (DL_FUNC) &_arrow_fs___FileStats__set_path, 2}, + { "_arrow_fs___FileStats__size", (DL_FUNC) &_arrow_fs___FileStats__size, 1}, + { "_arrow_fs___FileStats__set_size", (DL_FUNC) &_arrow_fs___FileStats__set_size, 2}, + { "_arrow_fs___FileStats__base_name", (DL_FUNC) &_arrow_fs___FileStats__base_name, 1}, + { "_arrow_fs___FileStats__extension", (DL_FUNC) &_arrow_fs___FileStats__extension, 1}, + { "_arrow_fs___FileStats__mtime", (DL_FUNC) &_arrow_fs___FileStats__mtime, 1}, + { "_arrow_fs___FileStats__set_mtime", (DL_FUNC) &_arrow_fs___FileStats__set_mtime, 2}, { "_arrow_io___Readable__Read", (DL_FUNC) &_arrow_io___Readable__Read, 2}, { "_arrow_io___InputStream__Close", (DL_FUNC) &_arrow_io___InputStream__Close, 1}, { "_arrow_io___OutputStream__Close", (DL_FUNC) &_arrow_io___OutputStream__Close, 1}, diff --git a/r/src/arrow_types.h b/r/src/arrow_types.h index 8a15f53fd6d..309a0ee4938 100644 --- a/r/src/arrow_types.h +++ b/r/src/arrow_types.h @@ -33,6 +33,11 @@ struct symbols { static SEXP inspect; static SEXP row_names; }; + +struct data { + static SEXP classes_POSIXct; +}; + } // namespace r } // namespace arrow diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp new file mode 100644 index 00000000000..1cf37ef8036 --- /dev/null +++ b/r/src/filesystem.cpp @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "./arrow_types.h" +#if defined(ARROW_R_WITH_ARROW) + +// FileStype + +// [[arrow::export]] +arrow::fs::FileType fs___FileStats__type(const std::shared_ptr& x) { + return x->type(); +} + +// [[arrow::export]] +void fs___FileStats__set_type(const std::shared_ptr& x, arrow::fs::FileType type) { + x->set_type(type); +} + +// [[arrow::export]] +std::string fs___FileStats__path(const std::shared_ptr& x) { + return x->path(); +} + +// [[arrow::export]] +void fs___FileStats__set_path(const std::shared_ptr& x, const std::string& path) { + x->set_path(path); +} + +// [[arrow::export]] +int64_t fs___FileStats__size(const std::shared_ptr& x) { + return x->size(); +} + +// [[arrow::export]] +void fs___FileStats__set_size(const std::shared_ptr& x, int64_t size) { + x->set_size(size); +} + +// [[arrow::export]] +std::string fs___FileStats__base_name(const std::shared_ptr& x) { + return x->base_name(); +} + +// [[arrow::export]] +std::string fs___FileStats__extension(const std::shared_ptr& x) { + return x->extension(); +} + +// [[arrow::export]] +SEXP fs___FileStats__mtime(const std::shared_ptr& x) { + SEXP res = PROTECT(Rf_allocVector(REALSXP, 1)); + // .mtime() gets us nanoseconds since epoch, POSIXct is seconds since epoch as a double + REAL(res)[0] = static_cast(x->mtime().time_since_epoch().count()) / 1000000; + Rf_classgets(res, arrow::r::data::classes_POSIXct); + UNPROTECT(1); + return res; +} + +// [[arrow::export]] +void fs___FileStats__set_mtime(const std::shared_ptr& x, SEXP time) { + auto secs = std::chrono::seconds(static_cast(REAL(time)[0] * 1000000)); + x->set_mtime(arrow::fs::TimePoint(secs)); +} + +#endif diff --git a/r/src/symbols.cpp b/r/src/symbols.cpp index 828033bf82d..f7b65731959 100644 --- a/r/src/symbols.cpp +++ b/r/src/symbols.cpp @@ -25,6 +25,16 @@ SEXP symbols::dot_Internal = Rf_install(".Internal"); SEXP symbols::inspect = Rf_install("inspect"); SEXP symbols::row_names = Rf_install("row.names"); +SEXP get_classes_POSIXct() { + SEXP classes = Rf_allocVector(STRSXP, 2); + R_PreserveObject(classes); + SET_STRING_ELT(classes, 0, Rf_mkChar("POSIXct")); + SET_STRING_ELT(classes, 0, Rf_mkChar("POSIXt")); + return classes; +} + +SEXP data::classes_POSIXct = get_classes_POSIXct(); + void inspect(SEXP obj) { Rcpp::Shield call_inspect(Rf_lang2(symbols::inspect, obj)); Rcpp::Shield call_internal(Rf_lang2(symbols::dot_Internal, call_inspect)); From b4ffd017ca51b57afd1e3f4f3bb8b570e5138568 Mon Sep 17 00:00:00 2001 From: Romain Francois Date: Mon, 16 Sep 2019 13:10:23 +0200 Subject: [PATCH 03/13] Support for arrow::fs::Selector --- r/NAMESPACE | 1 + r/R/arrowExports.R | 12 +++++++++++ r/R/filesystem.R | 20 ++++++++++++++++++ r/man/Selector.Rd | 10 +++++++++ r/src/arrowExports.cpp | 48 ++++++++++++++++++++++++++++++++++++++++++ r/src/filesystem.cpp | 19 +++++++++++++++++ 6 files changed, 110 insertions(+) create mode 100644 r/man/Selector.Rd diff --git a/r/NAMESPACE b/r/NAMESPACE index cc560e2cfdf..e77cf605488 100644 --- a/r/NAMESPACE +++ b/r/NAMESPACE @@ -67,6 +67,7 @@ export(RecordBatchFileWriter) export(RecordBatchStreamReader) export(RecordBatchStreamWriter) export(Schema) +export(Selector) export(StatusCode) export(Table) export(TimeUnit) diff --git a/r/R/arrowExports.R b/r/R/arrowExports.R index 80dcaca9689..3f056fc853b 100644 --- a/r/R/arrowExports.R +++ b/r/R/arrowExports.R @@ -580,6 +580,18 @@ fs___FileStats__set_mtime <- function(x, time){ invisible(.Call(`_arrow_fs___FileStats__set_mtime` , x, time)) } +fs___Selector__base_dir <- function(selector){ + .Call(`_arrow_fs___Selector__base_dir` , selector) +} + +fs___Selector__allow_non_existent <- function(selector){ + .Call(`_arrow_fs___Selector__allow_non_existent` , selector) +} + +fs___Selector__recursive <- function(selector){ + .Call(`_arrow_fs___Selector__recursive` , selector) +} + io___Readable__Read <- function(x, nbytes){ .Call(`_arrow_io___Readable__Read` , x, nbytes) } diff --git a/r/R/filesystem.R b/r/R/filesystem.R index 25a67244a82..9a6af8ea21c 100644 --- a/r/R/filesystem.R +++ b/r/R/filesystem.R @@ -63,3 +63,23 @@ FileStats <- R6Class("FileStats", } ) ) + +#' @title Selector class +#' @description EXPERIMENTAL: file selector +#' +#' @usage NULL +#' @format NULL +#' @docType class +#' +#' @rdname Selector +#' @name Selector +#' @export +Selector <- R6Class("Selector", + inherit = Object, + active = list( + base_dir = function() fs___Selector__base_dir(self), + allow_non_existent = function() fs___Selector__allow_non_existent(self), + recursive = function() fs___Selector__recursive(self) + ) +) + diff --git a/r/man/Selector.Rd b/r/man/Selector.Rd new file mode 100644 index 00000000000..7d64fae1a10 --- /dev/null +++ b/r/man/Selector.Rd @@ -0,0 +1,10 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/filesystem.R +\docType{class} +\name{Selector} +\alias{Selector} +\title{Selector class} +\description{ +EXPERIMENTAL: file selector +} +\keyword{datasets} diff --git a/r/src/arrowExports.cpp b/r/src/arrowExports.cpp index 60a86d01541..1c79e9decfa 100644 --- a/r/src/arrowExports.cpp +++ b/r/src/arrowExports.cpp @@ -2233,6 +2233,51 @@ RcppExport SEXP _arrow_fs___FileStats__set_mtime(SEXP x_sexp, SEXP time_sexp){ } #endif +// filesystem.cpp +#if defined(ARROW_R_WITH_ARROW) +std::string fs___Selector__base_dir(const std::shared_ptr& selector); +RcppExport SEXP _arrow_fs___Selector__base_dir(SEXP selector_sexp){ +BEGIN_RCPP + Rcpp::traits::input_parameter&>::type selector(selector_sexp); + return Rcpp::wrap(fs___Selector__base_dir(selector)); +END_RCPP +} +#else +RcppExport SEXP _arrow_fs___Selector__base_dir(SEXP selector_sexp){ + Rf_error("Cannot call fs___Selector__base_dir(). Please use arrow::install_arrow() to install required runtime libraries. "); +} +#endif + +// filesystem.cpp +#if defined(ARROW_R_WITH_ARROW) +bool fs___Selector__allow_non_existent(const std::shared_ptr& selector); +RcppExport SEXP _arrow_fs___Selector__allow_non_existent(SEXP selector_sexp){ +BEGIN_RCPP + Rcpp::traits::input_parameter&>::type selector(selector_sexp); + return Rcpp::wrap(fs___Selector__allow_non_existent(selector)); +END_RCPP +} +#else +RcppExport SEXP _arrow_fs___Selector__allow_non_existent(SEXP selector_sexp){ + Rf_error("Cannot call fs___Selector__allow_non_existent(). Please use arrow::install_arrow() to install required runtime libraries. "); +} +#endif + +// filesystem.cpp +#if defined(ARROW_R_WITH_ARROW) +bool fs___Selector__recursive(const std::shared_ptr& selector); +RcppExport SEXP _arrow_fs___Selector__recursive(SEXP selector_sexp){ +BEGIN_RCPP + Rcpp::traits::input_parameter&>::type selector(selector_sexp); + return Rcpp::wrap(fs___Selector__recursive(selector)); +END_RCPP +} +#else +RcppExport SEXP _arrow_fs___Selector__recursive(SEXP selector_sexp){ + Rf_error("Cannot call fs___Selector__recursive(). Please use arrow::install_arrow() to install required runtime libraries. "); +} +#endif + // io.cpp #if defined(ARROW_R_WITH_ARROW) std::shared_ptr io___Readable__Read(const std::shared_ptr& x, int64_t nbytes); @@ -4003,6 +4048,9 @@ static const R_CallMethodDef CallEntries[] = { { "_arrow_fs___FileStats__extension", (DL_FUNC) &_arrow_fs___FileStats__extension, 1}, { "_arrow_fs___FileStats__mtime", (DL_FUNC) &_arrow_fs___FileStats__mtime, 1}, { "_arrow_fs___FileStats__set_mtime", (DL_FUNC) &_arrow_fs___FileStats__set_mtime, 2}, + { "_arrow_fs___Selector__base_dir", (DL_FUNC) &_arrow_fs___Selector__base_dir, 1}, + { "_arrow_fs___Selector__allow_non_existent", (DL_FUNC) &_arrow_fs___Selector__allow_non_existent, 1}, + { "_arrow_fs___Selector__recursive", (DL_FUNC) &_arrow_fs___Selector__recursive, 1}, { "_arrow_io___Readable__Read", (DL_FUNC) &_arrow_io___Readable__Read, 2}, { "_arrow_io___InputStream__Close", (DL_FUNC) &_arrow_io___InputStream__Close, 1}, { "_arrow_io___OutputStream__Close", (DL_FUNC) &_arrow_io___OutputStream__Close, 1}, diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp index 1cf37ef8036..44971d94b98 100644 --- a/r/src/filesystem.cpp +++ b/r/src/filesystem.cpp @@ -76,4 +76,23 @@ void fs___FileStats__set_mtime(const std::shared_ptr& x, S x->set_mtime(arrow::fs::TimePoint(secs)); } + +// Selector + +// [[arrow::export]] +std::string fs___Selector__base_dir(const std::shared_ptr& selector) { + return selector->base_dir; +} + +// [[arrow::export]] +bool fs___Selector__allow_non_existent(const std::shared_ptr& selector) { + return selector->allow_non_existent; +} + +// [[arrow::export]] +bool fs___Selector__recursive(const std::shared_ptr& selector) { + return selector->recursive; +} + + #endif From 2666f8d4c670d59da9c3c8130632bc2563e4ea7f Mon Sep 17 00:00:00 2001 From: Romain Francois Date: Mon, 16 Sep 2019 15:51:14 +0200 Subject: [PATCH 04/13] Classes FileSystem and SubTreeFileSystem --- r/NAMESPACE | 2 + r/R/arrowExports.R | 52 +++++++++ r/R/filesystem.R | 79 +++++++++++++ r/man/SubTreeFileSystem.Rd | 10 ++ r/src/arrowExports.cpp | 231 +++++++++++++++++++++++++++++++++++++ r/src/filesystem.cpp | 80 +++++++++++++ 6 files changed, 454 insertions(+) create mode 100644 r/man/SubTreeFileSystem.Rd diff --git a/r/NAMESPACE b/r/NAMESPACE index e77cf605488..e30c569a3d2 100644 --- a/r/NAMESPACE +++ b/r/NAMESPACE @@ -53,6 +53,7 @@ export(Field) export(FileMode) export(FileOutputStream) export(FileStats) +export(FileSystem) export(FixedSizeBufferWriter) export(MemoryMappedFile) export(MessageReader) @@ -69,6 +70,7 @@ export(RecordBatchStreamWriter) export(Schema) export(Selector) export(StatusCode) +export(SubTreeFileSystem) export(Table) export(TimeUnit) export(Type) diff --git a/r/R/arrowExports.R b/r/R/arrowExports.R index 3f056fc853b..475399c198a 100644 --- a/r/R/arrowExports.R +++ b/r/R/arrowExports.R @@ -592,6 +592,58 @@ fs___Selector__recursive <- function(selector){ .Call(`_arrow_fs___Selector__recursive` , selector) } +fs___FileSystem__GetTargetStats_1 <- function(file_system, paths){ + .Call(`_arrow_fs___FileSystem__GetTargetStats_1` , file_system, paths) +} + +fs___FileSystem__CreateDir <- function(file_system, path, recursive){ + invisible(.Call(`_arrow_fs___FileSystem__CreateDir` , file_system, path, recursive)) +} + +fs___FileSystem__DeleteDir <- function(file_system, path){ + invisible(.Call(`_arrow_fs___FileSystem__DeleteDir` , file_system, path)) +} + +fs___FileSystem__DeleteDirContents <- function(file_system, path){ + invisible(.Call(`_arrow_fs___FileSystem__DeleteDirContents` , file_system, path)) +} + +fs___FileSystem__DeleteFile <- function(file_system, path){ + invisible(.Call(`_arrow_fs___FileSystem__DeleteFile` , file_system, path)) +} + +fs___FileSystem__DeleteFiles <- function(file_system, paths){ + invisible(.Call(`_arrow_fs___FileSystem__DeleteFiles` , file_system, paths)) +} + +fs___FileSystem__Move <- function(file_system, src, dest){ + invisible(.Call(`_arrow_fs___FileSystem__Move` , file_system, src, dest)) +} + +fs___FileSystem__CopyFile <- function(file_system, src, dest){ + invisible(.Call(`_arrow_fs___FileSystem__CopyFile` , file_system, src, dest)) +} + +fs___FileSystem__OpenInputStream <- function(file_system, path){ + .Call(`_arrow_fs___FileSystem__OpenInputStream` , file_system, path) +} + +fs___FileSystem__OpenInputFile <- function(file_system, path){ + .Call(`_arrow_fs___FileSystem__OpenInputFile` , file_system, path) +} + +fs___FileSystem__OpenOutputStream <- function(file_system, path){ + .Call(`_arrow_fs___FileSystem__OpenOutputStream` , file_system, path) +} + +fs___FileSystem__OpenAppendStream <- function(file_system, path){ + .Call(`_arrow_fs___FileSystem__OpenAppendStream` , file_system, path) +} + +fs___SubTreeFileSystem__create <- function(base_fs, base_path){ + .Call(`_arrow_fs___SubTreeFileSystem__create` , base_fs, base_path) +} + io___Readable__Read <- function(x, nbytes){ .Call(`_arrow_io___Readable__Read` , x, nbytes) } diff --git a/r/R/filesystem.R b/r/R/filesystem.R index 9a6af8ea21c..1186ddd6107 100644 --- a/r/R/filesystem.R +++ b/r/R/filesystem.R @@ -83,3 +83,82 @@ Selector <- R6Class("Selector", ) ) +#' @title FileSystem class +#' @description EXPERIMENTAL: abstract file system API +#' +#' @usage NULL +#' @format NULL +#' @docType class +#' +#' @rdname FileSystem +#' @name FileSystem +#' @export +FileSystem <- R6Class("FileSystem", inherit = Object, + public = list( + GetTargetStats = function(paths, selector = NULL) { + if (is.null(selector)) { + shared_ptr(FileStats, fs___FileSystem__GetTargetStats_1(paths)) + } else { + shared_ptr(FileStats, fs___FileSystem__GetTargetStats_2(paths, selector)) + } + }, + + CreateDir = function(path, recursive = TRUE) { + fs___FileSystem__CreateDir(self, path, isTRUE(recursive)) + }, + + DeleteDir = function(path) { + fs___FileSystem__DeleteDir(self, path) + }, + + DeleteDirContents = function(path) { + fs___FileSystem__DeleteDirContents(self, path) + }, + + DeleteFile = function(path) { + fs___FileSystem__DeleteFile(self, path) + }, + + DeleteFiles = function(path) { + fs___FileSystem__DeleteFiles(self, paths) + }, + + Move = function(src, dest) { + fs___FileSystem__Move(self, src, dest) + }, + + CopyFile = function(src, dest) { + fs___FileSystem__CopyFile(self, src, dest) + }, + + OpenInputStream = function(path) { + shared_ptr(InputStream, fs___FileSystem__OpenInputStream(self, path)) + }, + OpenInputFile = function(path) { + shared_ptr(InputStream, fs___FileSystem__OpenInputFile(self, path)) + }, + OpenOutputStream = function(path) { + shared_ptr(OutputStream, fs___FileSystem__OpenOutputStream(self, path)) + }, + OpenAppendStream = function(path) { + shared_ptr(OutputStream, fs___FileSystem__OpenAppendStream(self, path)) + } + + ) +) + +#' @title SubTreeFileSystem class +#' @description EXPERIMENTAL: abstract file system API +#' +#' @usage NULL +#' @format NULL +#' @docType class +#' +#' @rdname SubTreeFileSystem +#' @name SubTreeFileSystem +#' @export +SubTreeFileSystem <- R6Class("SubTreeFileSystem", inherit = FileSystem) + +SubTreeFileSystem$create <- function(base_fs, base_path) { + shared_ptr(SubTreeFileSystem, fs___SubTreeFileSystem__create(base_fs, base_path)) +} diff --git a/r/man/SubTreeFileSystem.Rd b/r/man/SubTreeFileSystem.Rd new file mode 100644 index 00000000000..ffe4f4bdca1 --- /dev/null +++ b/r/man/SubTreeFileSystem.Rd @@ -0,0 +1,10 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/filesystem.R +\docType{class} +\name{SubTreeFileSystem} +\alias{SubTreeFileSystem} +\title{SubTreeFileSystem class} +\description{ +EXPERIMENTAL: abstract file system API +} +\keyword{datasets} diff --git a/r/src/arrowExports.cpp b/r/src/arrowExports.cpp index 1c79e9decfa..47094b57172 100644 --- a/r/src/arrowExports.cpp +++ b/r/src/arrowExports.cpp @@ -2278,6 +2278,224 @@ RcppExport SEXP _arrow_fs___Selector__recursive(SEXP selector_sexp){ } #endif +// filesystem.cpp +#if defined(ARROW_R_WITH_ARROW) +std::vector> fs___FileSystem__GetTargetStats_1(const std::shared_ptr& file_system, const std::vector& paths); +RcppExport SEXP _arrow_fs___FileSystem__GetTargetStats_1(SEXP file_system_sexp, SEXP paths_sexp){ +BEGIN_RCPP + Rcpp::traits::input_parameter&>::type file_system(file_system_sexp); + Rcpp::traits::input_parameter&>::type paths(paths_sexp); + return Rcpp::wrap(fs___FileSystem__GetTargetStats_1(file_system, paths)); +END_RCPP +} +#else +RcppExport SEXP _arrow_fs___FileSystem__GetTargetStats_1(SEXP file_system_sexp, SEXP paths_sexp){ + Rf_error("Cannot call fs___FileSystem__GetTargetStats_1(). Please use arrow::install_arrow() to install required runtime libraries. "); +} +#endif + +// filesystem.cpp +#if defined(ARROW_R_WITH_ARROW) +void fs___FileSystem__CreateDir(const std::shared_ptr& file_system, const std::string& path, bool recursive); +RcppExport SEXP _arrow_fs___FileSystem__CreateDir(SEXP file_system_sexp, SEXP path_sexp, SEXP recursive_sexp){ +BEGIN_RCPP + Rcpp::traits::input_parameter&>::type file_system(file_system_sexp); + Rcpp::traits::input_parameter::type path(path_sexp); + Rcpp::traits::input_parameter::type recursive(recursive_sexp); + fs___FileSystem__CreateDir(file_system, path, recursive); + return R_NilValue; +END_RCPP +} +#else +RcppExport SEXP _arrow_fs___FileSystem__CreateDir(SEXP file_system_sexp, SEXP path_sexp, SEXP recursive_sexp){ + Rf_error("Cannot call fs___FileSystem__CreateDir(). Please use arrow::install_arrow() to install required runtime libraries. "); +} +#endif + +// filesystem.cpp +#if defined(ARROW_R_WITH_ARROW) +void fs___FileSystem__DeleteDir(const std::shared_ptr& file_system, const std::string& path); +RcppExport SEXP _arrow_fs___FileSystem__DeleteDir(SEXP file_system_sexp, SEXP path_sexp){ +BEGIN_RCPP + Rcpp::traits::input_parameter&>::type file_system(file_system_sexp); + Rcpp::traits::input_parameter::type path(path_sexp); + fs___FileSystem__DeleteDir(file_system, path); + return R_NilValue; +END_RCPP +} +#else +RcppExport SEXP _arrow_fs___FileSystem__DeleteDir(SEXP file_system_sexp, SEXP path_sexp){ + Rf_error("Cannot call fs___FileSystem__DeleteDir(). Please use arrow::install_arrow() to install required runtime libraries. "); +} +#endif + +// filesystem.cpp +#if defined(ARROW_R_WITH_ARROW) +void fs___FileSystem__DeleteDirContents(const std::shared_ptr& file_system, const std::string& path); +RcppExport SEXP _arrow_fs___FileSystem__DeleteDirContents(SEXP file_system_sexp, SEXP path_sexp){ +BEGIN_RCPP + Rcpp::traits::input_parameter&>::type file_system(file_system_sexp); + Rcpp::traits::input_parameter::type path(path_sexp); + fs___FileSystem__DeleteDirContents(file_system, path); + return R_NilValue; +END_RCPP +} +#else +RcppExport SEXP _arrow_fs___FileSystem__DeleteDirContents(SEXP file_system_sexp, SEXP path_sexp){ + Rf_error("Cannot call fs___FileSystem__DeleteDirContents(). Please use arrow::install_arrow() to install required runtime libraries. "); +} +#endif + +// filesystem.cpp +#if defined(ARROW_R_WITH_ARROW) +void fs___FileSystem__DeleteFile(const std::shared_ptr& file_system, const std::string& path); +RcppExport SEXP _arrow_fs___FileSystem__DeleteFile(SEXP file_system_sexp, SEXP path_sexp){ +BEGIN_RCPP + Rcpp::traits::input_parameter&>::type file_system(file_system_sexp); + Rcpp::traits::input_parameter::type path(path_sexp); + fs___FileSystem__DeleteFile(file_system, path); + return R_NilValue; +END_RCPP +} +#else +RcppExport SEXP _arrow_fs___FileSystem__DeleteFile(SEXP file_system_sexp, SEXP path_sexp){ + Rf_error("Cannot call fs___FileSystem__DeleteFile(). Please use arrow::install_arrow() to install required runtime libraries. "); +} +#endif + +// filesystem.cpp +#if defined(ARROW_R_WITH_ARROW) +void fs___FileSystem__DeleteFiles(const std::shared_ptr& file_system, const std::vector& paths); +RcppExport SEXP _arrow_fs___FileSystem__DeleteFiles(SEXP file_system_sexp, SEXP paths_sexp){ +BEGIN_RCPP + Rcpp::traits::input_parameter&>::type file_system(file_system_sexp); + Rcpp::traits::input_parameter&>::type paths(paths_sexp); + fs___FileSystem__DeleteFiles(file_system, paths); + return R_NilValue; +END_RCPP +} +#else +RcppExport SEXP _arrow_fs___FileSystem__DeleteFiles(SEXP file_system_sexp, SEXP paths_sexp){ + Rf_error("Cannot call fs___FileSystem__DeleteFiles(). Please use arrow::install_arrow() to install required runtime libraries. "); +} +#endif + +// filesystem.cpp +#if defined(ARROW_R_WITH_ARROW) +void fs___FileSystem__Move(const std::shared_ptr& file_system, const std::string& src, const std::string& dest); +RcppExport SEXP _arrow_fs___FileSystem__Move(SEXP file_system_sexp, SEXP src_sexp, SEXP dest_sexp){ +BEGIN_RCPP + Rcpp::traits::input_parameter&>::type file_system(file_system_sexp); + Rcpp::traits::input_parameter::type src(src_sexp); + Rcpp::traits::input_parameter::type dest(dest_sexp); + fs___FileSystem__Move(file_system, src, dest); + return R_NilValue; +END_RCPP +} +#else +RcppExport SEXP _arrow_fs___FileSystem__Move(SEXP file_system_sexp, SEXP src_sexp, SEXP dest_sexp){ + Rf_error("Cannot call fs___FileSystem__Move(). Please use arrow::install_arrow() to install required runtime libraries. "); +} +#endif + +// filesystem.cpp +#if defined(ARROW_R_WITH_ARROW) +void fs___FileSystem__CopyFile(const std::shared_ptr& file_system, const std::string& src, const std::string& dest); +RcppExport SEXP _arrow_fs___FileSystem__CopyFile(SEXP file_system_sexp, SEXP src_sexp, SEXP dest_sexp){ +BEGIN_RCPP + Rcpp::traits::input_parameter&>::type file_system(file_system_sexp); + Rcpp::traits::input_parameter::type src(src_sexp); + Rcpp::traits::input_parameter::type dest(dest_sexp); + fs___FileSystem__CopyFile(file_system, src, dest); + return R_NilValue; +END_RCPP +} +#else +RcppExport SEXP _arrow_fs___FileSystem__CopyFile(SEXP file_system_sexp, SEXP src_sexp, SEXP dest_sexp){ + Rf_error("Cannot call fs___FileSystem__CopyFile(). Please use arrow::install_arrow() to install required runtime libraries. "); +} +#endif + +// filesystem.cpp +#if defined(ARROW_R_WITH_ARROW) +std::shared_ptr fs___FileSystem__OpenInputStream(const std::shared_ptr& file_system, const std::string& path); +RcppExport SEXP _arrow_fs___FileSystem__OpenInputStream(SEXP file_system_sexp, SEXP path_sexp){ +BEGIN_RCPP + Rcpp::traits::input_parameter&>::type file_system(file_system_sexp); + Rcpp::traits::input_parameter::type path(path_sexp); + return Rcpp::wrap(fs___FileSystem__OpenInputStream(file_system, path)); +END_RCPP +} +#else +RcppExport SEXP _arrow_fs___FileSystem__OpenInputStream(SEXP file_system_sexp, SEXP path_sexp){ + Rf_error("Cannot call fs___FileSystem__OpenInputStream(). Please use arrow::install_arrow() to install required runtime libraries. "); +} +#endif + +// filesystem.cpp +#if defined(ARROW_R_WITH_ARROW) +std::shared_ptr fs___FileSystem__OpenInputFile(const std::shared_ptr& file_system, const std::string& path); +RcppExport SEXP _arrow_fs___FileSystem__OpenInputFile(SEXP file_system_sexp, SEXP path_sexp){ +BEGIN_RCPP + Rcpp::traits::input_parameter&>::type file_system(file_system_sexp); + Rcpp::traits::input_parameter::type path(path_sexp); + return Rcpp::wrap(fs___FileSystem__OpenInputFile(file_system, path)); +END_RCPP +} +#else +RcppExport SEXP _arrow_fs___FileSystem__OpenInputFile(SEXP file_system_sexp, SEXP path_sexp){ + Rf_error("Cannot call fs___FileSystem__OpenInputFile(). Please use arrow::install_arrow() to install required runtime libraries. "); +} +#endif + +// filesystem.cpp +#if defined(ARROW_R_WITH_ARROW) +std::shared_ptr fs___FileSystem__OpenOutputStream(const std::shared_ptr& file_system, const std::string& path); +RcppExport SEXP _arrow_fs___FileSystem__OpenOutputStream(SEXP file_system_sexp, SEXP path_sexp){ +BEGIN_RCPP + Rcpp::traits::input_parameter&>::type file_system(file_system_sexp); + Rcpp::traits::input_parameter::type path(path_sexp); + return Rcpp::wrap(fs___FileSystem__OpenOutputStream(file_system, path)); +END_RCPP +} +#else +RcppExport SEXP _arrow_fs___FileSystem__OpenOutputStream(SEXP file_system_sexp, SEXP path_sexp){ + Rf_error("Cannot call fs___FileSystem__OpenOutputStream(). Please use arrow::install_arrow() to install required runtime libraries. "); +} +#endif + +// filesystem.cpp +#if defined(ARROW_R_WITH_ARROW) +std::shared_ptr fs___FileSystem__OpenAppendStream(const std::shared_ptr& file_system, const std::string& path); +RcppExport SEXP _arrow_fs___FileSystem__OpenAppendStream(SEXP file_system_sexp, SEXP path_sexp){ +BEGIN_RCPP + Rcpp::traits::input_parameter&>::type file_system(file_system_sexp); + Rcpp::traits::input_parameter::type path(path_sexp); + return Rcpp::wrap(fs___FileSystem__OpenAppendStream(file_system, path)); +END_RCPP +} +#else +RcppExport SEXP _arrow_fs___FileSystem__OpenAppendStream(SEXP file_system_sexp, SEXP path_sexp){ + Rf_error("Cannot call fs___FileSystem__OpenAppendStream(). Please use arrow::install_arrow() to install required runtime libraries. "); +} +#endif + +// filesystem.cpp +#if defined(ARROW_R_WITH_ARROW) +std::shared_ptr fs___SubTreeFileSystem__create(const std::shared_ptr& base_fs, const std::string& base_path); +RcppExport SEXP _arrow_fs___SubTreeFileSystem__create(SEXP base_fs_sexp, SEXP base_path_sexp){ +BEGIN_RCPP + Rcpp::traits::input_parameter&>::type base_fs(base_fs_sexp); + Rcpp::traits::input_parameter::type base_path(base_path_sexp); + return Rcpp::wrap(fs___SubTreeFileSystem__create(base_fs, base_path)); +END_RCPP +} +#else +RcppExport SEXP _arrow_fs___SubTreeFileSystem__create(SEXP base_fs_sexp, SEXP base_path_sexp){ + Rf_error("Cannot call fs___SubTreeFileSystem__create(). Please use arrow::install_arrow() to install required runtime libraries. "); +} +#endif + // io.cpp #if defined(ARROW_R_WITH_ARROW) std::shared_ptr io___Readable__Read(const std::shared_ptr& x, int64_t nbytes); @@ -4051,6 +4269,19 @@ static const R_CallMethodDef CallEntries[] = { { "_arrow_fs___Selector__base_dir", (DL_FUNC) &_arrow_fs___Selector__base_dir, 1}, { "_arrow_fs___Selector__allow_non_existent", (DL_FUNC) &_arrow_fs___Selector__allow_non_existent, 1}, { "_arrow_fs___Selector__recursive", (DL_FUNC) &_arrow_fs___Selector__recursive, 1}, + { "_arrow_fs___FileSystem__GetTargetStats_1", (DL_FUNC) &_arrow_fs___FileSystem__GetTargetStats_1, 2}, + { "_arrow_fs___FileSystem__CreateDir", (DL_FUNC) &_arrow_fs___FileSystem__CreateDir, 3}, + { "_arrow_fs___FileSystem__DeleteDir", (DL_FUNC) &_arrow_fs___FileSystem__DeleteDir, 2}, + { "_arrow_fs___FileSystem__DeleteDirContents", (DL_FUNC) &_arrow_fs___FileSystem__DeleteDirContents, 2}, + { "_arrow_fs___FileSystem__DeleteFile", (DL_FUNC) &_arrow_fs___FileSystem__DeleteFile, 2}, + { "_arrow_fs___FileSystem__DeleteFiles", (DL_FUNC) &_arrow_fs___FileSystem__DeleteFiles, 2}, + { "_arrow_fs___FileSystem__Move", (DL_FUNC) &_arrow_fs___FileSystem__Move, 3}, + { "_arrow_fs___FileSystem__CopyFile", (DL_FUNC) &_arrow_fs___FileSystem__CopyFile, 3}, + { "_arrow_fs___FileSystem__OpenInputStream", (DL_FUNC) &_arrow_fs___FileSystem__OpenInputStream, 2}, + { "_arrow_fs___FileSystem__OpenInputFile", (DL_FUNC) &_arrow_fs___FileSystem__OpenInputFile, 2}, + { "_arrow_fs___FileSystem__OpenOutputStream", (DL_FUNC) &_arrow_fs___FileSystem__OpenOutputStream, 2}, + { "_arrow_fs___FileSystem__OpenAppendStream", (DL_FUNC) &_arrow_fs___FileSystem__OpenAppendStream, 2}, + { "_arrow_fs___SubTreeFileSystem__create", (DL_FUNC) &_arrow_fs___SubTreeFileSystem__create, 2}, { "_arrow_io___Readable__Read", (DL_FUNC) &_arrow_io___Readable__Read, 2}, { "_arrow_io___InputStream__Close", (DL_FUNC) &_arrow_io___InputStream__Close, 1}, { "_arrow_io___OutputStream__Close", (DL_FUNC) &_arrow_io___OutputStream__Close, 1}, diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp index 44971d94b98..f96997af97f 100644 --- a/r/src/filesystem.cpp +++ b/r/src/filesystem.cpp @@ -94,5 +94,85 @@ bool fs___Selector__recursive(const std::shared_ptr& select return selector->recursive; } +// FileSystem + + +// [[arrow::export]] +std::vector> fs___FileSystem__GetTargetStats_1(const std::shared_ptr& file_system, const std::vector& paths) { + auto n = paths.size(); + std::vector> res(n); + for (decltype(n) i=0; iGetTargetStats(paths[i], res[i].get())); + } + return res; +} + +// [[arrow::export]] +void fs___FileSystem__CreateDir(const std::shared_ptr& file_system, const std::string& path, bool recursive) { + STOP_IF_NOT_OK(file_system->CreateDir(path, recursive)); +} + +// [[arrow::export]] +void fs___FileSystem__DeleteDir(const std::shared_ptr& file_system, const std::string& path) { + STOP_IF_NOT_OK(file_system->DeleteDir(path)); +} + +// [[arrow::export]] +void fs___FileSystem__DeleteDirContents(const std::shared_ptr& file_system, const std::string& path) { + STOP_IF_NOT_OK(file_system->DeleteDirContents(path)); +} + +// [[arrow::export]] +void fs___FileSystem__DeleteFile(const std::shared_ptr& file_system, const std::string& path) { + STOP_IF_NOT_OK(file_system->DeleteFile(path)); +} + +// [[arrow::export]] +void fs___FileSystem__DeleteFiles(const std::shared_ptr& file_system, const std::vector& paths) { + STOP_IF_NOT_OK(file_system->DeleteFiles(paths)); +} + +// [[arrow::export]] +void fs___FileSystem__Move(const std::shared_ptr& file_system, const std::string& src, const std::string& dest) { + STOP_IF_NOT_OK(file_system->Move(src, dest)); +} + +// [[arrow::export]] +void fs___FileSystem__CopyFile(const std::shared_ptr& file_system, const std::string& src, const std::string& dest) { + STOP_IF_NOT_OK(file_system->CopyFile(src, dest)); +} + +// [[arrow::export]] +std::shared_ptr fs___FileSystem__OpenInputStream(const std::shared_ptr& file_system, const std::string& path) { + std::shared_ptr stream; + STOP_IF_NOT_OK(file_system->OpenInputStream(path, &stream)); + return stream; +} + +// [[arrow::export]] +std::shared_ptr fs___FileSystem__OpenInputFile(const std::shared_ptr& file_system, const std::string& path) { + std::shared_ptr file; + STOP_IF_NOT_OK(file_system->OpenInputFile(path, &file)); + return file; +} + +// [[arrow::export]] +std::shared_ptr fs___FileSystem__OpenOutputStream(const std::shared_ptr& file_system, const std::string& path) { + std::shared_ptr stream; + STOP_IF_NOT_OK(file_system->OpenOutputStream(path, &stream)); + return stream; +} + +// [[arrow::export]] +std::shared_ptr fs___FileSystem__OpenAppendStream(const std::shared_ptr& file_system, const std::string& path) { + std::shared_ptr stream; + STOP_IF_NOT_OK(file_system->OpenAppendStream(path, &stream)); + return stream; +} + +// [[arrow::export]] +std::shared_ptr fs___SubTreeFileSystem__create(const std::string& base_path, const std::shared_ptr& base_fs) { + return std::make_shared(base_path, base_fs); +} #endif From d3aba78b1cc967869bbfe6cb8a667a7739927368 Mon Sep 17 00:00:00 2001 From: Romain Francois Date: Mon, 16 Sep 2019 16:57:44 +0200 Subject: [PATCH 05/13] missing SubTreeFileSystem methods --- r/R/arrowExports.R | 4 ++-- r/man/FileSystem.Rd | 10 ++++++++++ r/src/arrowExports.cpp | 10 +++++----- 3 files changed, 17 insertions(+), 7 deletions(-) create mode 100644 r/man/FileSystem.Rd diff --git a/r/R/arrowExports.R b/r/R/arrowExports.R index 475399c198a..f9dc3ba4a68 100644 --- a/r/R/arrowExports.R +++ b/r/R/arrowExports.R @@ -640,8 +640,8 @@ fs___FileSystem__OpenAppendStream <- function(file_system, path){ .Call(`_arrow_fs___FileSystem__OpenAppendStream` , file_system, path) } -fs___SubTreeFileSystem__create <- function(base_fs, base_path){ - .Call(`_arrow_fs___SubTreeFileSystem__create` , base_fs, base_path) +fs___SubTreeFileSystem__create <- function(base_path, base_fs){ + .Call(`_arrow_fs___SubTreeFileSystem__create` , base_path, base_fs) } io___Readable__Read <- function(x, nbytes){ diff --git a/r/man/FileSystem.Rd b/r/man/FileSystem.Rd new file mode 100644 index 00000000000..3e345cfd158 --- /dev/null +++ b/r/man/FileSystem.Rd @@ -0,0 +1,10 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/filesystem.R +\docType{class} +\name{FileSystem} +\alias{FileSystem} +\title{FileSystem class} +\description{ +EXPERIMENTAL: abstract file system API +} +\keyword{datasets} diff --git a/r/src/arrowExports.cpp b/r/src/arrowExports.cpp index 47094b57172..31e4ab359b4 100644 --- a/r/src/arrowExports.cpp +++ b/r/src/arrowExports.cpp @@ -2482,16 +2482,16 @@ RcppExport SEXP _arrow_fs___FileSystem__OpenAppendStream(SEXP file_system_sexp, // filesystem.cpp #if defined(ARROW_R_WITH_ARROW) -std::shared_ptr fs___SubTreeFileSystem__create(const std::shared_ptr& base_fs, const std::string& base_path); -RcppExport SEXP _arrow_fs___SubTreeFileSystem__create(SEXP base_fs_sexp, SEXP base_path_sexp){ +std::shared_ptr fs___SubTreeFileSystem__create(const std::string& base_path, const std::shared_ptr& base_fs); +RcppExport SEXP _arrow_fs___SubTreeFileSystem__create(SEXP base_path_sexp, SEXP base_fs_sexp){ BEGIN_RCPP - Rcpp::traits::input_parameter&>::type base_fs(base_fs_sexp); Rcpp::traits::input_parameter::type base_path(base_path_sexp); - return Rcpp::wrap(fs___SubTreeFileSystem__create(base_fs, base_path)); + Rcpp::traits::input_parameter&>::type base_fs(base_fs_sexp); + return Rcpp::wrap(fs___SubTreeFileSystem__create(base_path, base_fs)); END_RCPP } #else -RcppExport SEXP _arrow_fs___SubTreeFileSystem__create(SEXP base_fs_sexp, SEXP base_path_sexp){ +RcppExport SEXP _arrow_fs___SubTreeFileSystem__create(SEXP base_path_sexp, SEXP base_fs_sexp){ Rf_error("Cannot call fs___SubTreeFileSystem__create(). Please use arrow::install_arrow() to install required runtime libraries. "); } #endif From 168502b48806b59d96f385ad044b837a8b87ebd6 Mon Sep 17 00:00:00 2001 From: Romain Francois Date: Wed, 18 Sep 2019 08:16:57 +0200 Subject: [PATCH 06/13] added tests for LocalFileSystem --- r/NAMESPACE | 2 + r/R/arrowExports.R | 12 +++- r/R/enums.R | 1 + r/R/filesystem.R | 30 ++++++++-- r/man/LocalFileSystem.Rd | 11 ++++ r/src/arrowExports.cpp | 44 ++++++++++++-- r/src/arrow_types.h | 3 +- r/src/filesystem.cpp | 96 +++++++++++++++++++++--------- r/src/parquet.cpp | 13 ++-- r/src/symbols.cpp | 2 +- r/tests/testthat/test-filesystem.R | 74 +++++++++++++++++++++++ 11 files changed, 237 insertions(+), 51 deletions(-) create mode 100644 r/man/LocalFileSystem.Rd create mode 100644 r/tests/testthat/test-filesystem.R diff --git a/r/NAMESPACE b/r/NAMESPACE index e30c569a3d2..9f210667f9e 100644 --- a/r/NAMESPACE +++ b/r/NAMESPACE @@ -54,7 +54,9 @@ export(FileMode) export(FileOutputStream) export(FileStats) export(FileSystem) +export(FileType) export(FixedSizeBufferWriter) +export(LocalFileSystem) export(MemoryMappedFile) export(MessageReader) export(MessageType) diff --git a/r/R/arrowExports.R b/r/R/arrowExports.R index f9dc3ba4a68..6175e676bb2 100644 --- a/r/R/arrowExports.R +++ b/r/R/arrowExports.R @@ -592,8 +592,12 @@ fs___Selector__recursive <- function(selector){ .Call(`_arrow_fs___Selector__recursive` , selector) } -fs___FileSystem__GetTargetStats_1 <- function(file_system, paths){ - .Call(`_arrow_fs___FileSystem__GetTargetStats_1` , file_system, paths) +fs___FileSystem__GetTargetStats_Paths <- function(file_system, paths){ + .Call(`_arrow_fs___FileSystem__GetTargetStats_Paths` , file_system, paths) +} + +fs___FileSystem__GetTargetStats_Selector <- function(file_system, selector){ + .Call(`_arrow_fs___FileSystem__GetTargetStats_Selector` , file_system, selector) } fs___FileSystem__CreateDir <- function(file_system, path, recursive){ @@ -640,6 +644,10 @@ fs___FileSystem__OpenAppendStream <- function(file_system, path){ .Call(`_arrow_fs___FileSystem__OpenAppendStream` , file_system, path) } +fs___LocalFileSystem__create <- function(){ + .Call(`_arrow_fs___LocalFileSystem__create` ) +} + fs___SubTreeFileSystem__create <- function(base_path, base_fs){ .Call(`_arrow_fs___SubTreeFileSystem__create` , base_path, base_fs) } diff --git a/r/R/enums.R b/r/R/enums.R index 17342d0678e..ade6d8e94f3 100644 --- a/r/R/enums.R +++ b/r/R/enums.R @@ -79,6 +79,7 @@ CompressionType <- enum("Compression::type", UNCOMPRESSED = 0L, SNAPPY = 1L, GZIP = 2L, BROTLI = 3L, ZSTD = 4L, LZ4 = 5L, LZO = 6L, BZ2 = 7L ) +#' @export #' @rdname enums FileType <- enum("FileType", NonExistent = 0L, Unknown = 1L, File = 2L, Directory = 3L diff --git a/r/R/filesystem.R b/r/R/filesystem.R index 1186ddd6107..02df110f250 100644 --- a/r/R/filesystem.R +++ b/r/R/filesystem.R @@ -95,11 +95,13 @@ Selector <- R6Class("Selector", #' @export FileSystem <- R6Class("FileSystem", inherit = Object, public = list( - GetTargetStats = function(paths, selector = NULL) { - if (is.null(selector)) { - shared_ptr(FileStats, fs___FileSystem__GetTargetStats_1(paths)) + GetTargetStats = function(x) { + if (inherits(x, "Selector")) { + map(fs___FileSystem__GetTargetStats_Selector(self, x), shared_ptr, class = FileStats) + } else if (is.character(x)){ + map(fs___FileSystem__GetTargetStats_Paths(self, x), shared_ptr, class = FileStats) } else { - shared_ptr(FileStats, fs___FileSystem__GetTargetStats_2(paths, selector)) + abort("incompatible type for FileSystem$GetTargetStarts()") } }, @@ -119,7 +121,7 @@ FileSystem <- R6Class("FileSystem", inherit = Object, fs___FileSystem__DeleteFile(self, path) }, - DeleteFiles = function(path) { + DeleteFiles = function(paths) { fs___FileSystem__DeleteFiles(self, paths) }, @@ -147,6 +149,24 @@ FileSystem <- R6Class("FileSystem", inherit = Object, ) ) +#' @title LocalFileSystem class +#' @description EXPERIMENTAL: a [FileSystem][FileSystem] implementation accessing files +#' on the local machine. +#' +#' @usage NULL +#' @format NULL +#' @docType class +#' +#' @rdname LocalFileSystem +#' @name LocalFileSystem +#' @export +LocalFileSystem <- R6Class("LocalFileSystem", inherit = FileSystem) + +LocalFileSystem$create <- function() { + shared_ptr(LocalFileSystem, fs___LocalFileSystem__create()) +} + + #' @title SubTreeFileSystem class #' @description EXPERIMENTAL: abstract file system API #' diff --git a/r/man/LocalFileSystem.Rd b/r/man/LocalFileSystem.Rd new file mode 100644 index 00000000000..b6903dd2748 --- /dev/null +++ b/r/man/LocalFileSystem.Rd @@ -0,0 +1,11 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/filesystem.R +\docType{class} +\name{LocalFileSystem} +\alias{LocalFileSystem} +\title{LocalFileSystem class} +\description{ +EXPERIMENTAL: a \link{FileSystem} implementation accessing files +on the local machine. +} +\keyword{datasets} diff --git a/r/src/arrowExports.cpp b/r/src/arrowExports.cpp index 31e4ab359b4..e0fc89b5489 100644 --- a/r/src/arrowExports.cpp +++ b/r/src/arrowExports.cpp @@ -2280,17 +2280,33 @@ RcppExport SEXP _arrow_fs___Selector__recursive(SEXP selector_sexp){ // filesystem.cpp #if defined(ARROW_R_WITH_ARROW) -std::vector> fs___FileSystem__GetTargetStats_1(const std::shared_ptr& file_system, const std::vector& paths); -RcppExport SEXP _arrow_fs___FileSystem__GetTargetStats_1(SEXP file_system_sexp, SEXP paths_sexp){ +std::vector> fs___FileSystem__GetTargetStats_Paths(const std::shared_ptr& file_system, const std::vector& paths); +RcppExport SEXP _arrow_fs___FileSystem__GetTargetStats_Paths(SEXP file_system_sexp, SEXP paths_sexp){ BEGIN_RCPP Rcpp::traits::input_parameter&>::type file_system(file_system_sexp); Rcpp::traits::input_parameter&>::type paths(paths_sexp); - return Rcpp::wrap(fs___FileSystem__GetTargetStats_1(file_system, paths)); + return Rcpp::wrap(fs___FileSystem__GetTargetStats_Paths(file_system, paths)); END_RCPP } #else -RcppExport SEXP _arrow_fs___FileSystem__GetTargetStats_1(SEXP file_system_sexp, SEXP paths_sexp){ - Rf_error("Cannot call fs___FileSystem__GetTargetStats_1(). Please use arrow::install_arrow() to install required runtime libraries. "); +RcppExport SEXP _arrow_fs___FileSystem__GetTargetStats_Paths(SEXP file_system_sexp, SEXP paths_sexp){ + Rf_error("Cannot call fs___FileSystem__GetTargetStats_Paths(). Please use arrow::install_arrow() to install required runtime libraries. "); +} +#endif + +// filesystem.cpp +#if defined(ARROW_R_WITH_ARROW) +std::vector> fs___FileSystem__GetTargetStats_Selector(const std::shared_ptr& file_system, const std::shared_ptr& selector); +RcppExport SEXP _arrow_fs___FileSystem__GetTargetStats_Selector(SEXP file_system_sexp, SEXP selector_sexp){ +BEGIN_RCPP + Rcpp::traits::input_parameter&>::type file_system(file_system_sexp); + Rcpp::traits::input_parameter&>::type selector(selector_sexp); + return Rcpp::wrap(fs___FileSystem__GetTargetStats_Selector(file_system, selector)); +END_RCPP +} +#else +RcppExport SEXP _arrow_fs___FileSystem__GetTargetStats_Selector(SEXP file_system_sexp, SEXP selector_sexp){ + Rf_error("Cannot call fs___FileSystem__GetTargetStats_Selector(). Please use arrow::install_arrow() to install required runtime libraries. "); } #endif @@ -2480,6 +2496,20 @@ RcppExport SEXP _arrow_fs___FileSystem__OpenAppendStream(SEXP file_system_sexp, } #endif +// filesystem.cpp +#if defined(ARROW_R_WITH_ARROW) +std::shared_ptr fs___LocalFileSystem__create(); +RcppExport SEXP _arrow_fs___LocalFileSystem__create(){ +BEGIN_RCPP + return Rcpp::wrap(fs___LocalFileSystem__create()); +END_RCPP +} +#else +RcppExport SEXP _arrow_fs___LocalFileSystem__create(){ + Rf_error("Cannot call fs___LocalFileSystem__create(). Please use arrow::install_arrow() to install required runtime libraries. "); +} +#endif + // filesystem.cpp #if defined(ARROW_R_WITH_ARROW) std::shared_ptr fs___SubTreeFileSystem__create(const std::string& base_path, const std::shared_ptr& base_fs); @@ -4269,7 +4299,8 @@ static const R_CallMethodDef CallEntries[] = { { "_arrow_fs___Selector__base_dir", (DL_FUNC) &_arrow_fs___Selector__base_dir, 1}, { "_arrow_fs___Selector__allow_non_existent", (DL_FUNC) &_arrow_fs___Selector__allow_non_existent, 1}, { "_arrow_fs___Selector__recursive", (DL_FUNC) &_arrow_fs___Selector__recursive, 1}, - { "_arrow_fs___FileSystem__GetTargetStats_1", (DL_FUNC) &_arrow_fs___FileSystem__GetTargetStats_1, 2}, + { "_arrow_fs___FileSystem__GetTargetStats_Paths", (DL_FUNC) &_arrow_fs___FileSystem__GetTargetStats_Paths, 2}, + { "_arrow_fs___FileSystem__GetTargetStats_Selector", (DL_FUNC) &_arrow_fs___FileSystem__GetTargetStats_Selector, 2}, { "_arrow_fs___FileSystem__CreateDir", (DL_FUNC) &_arrow_fs___FileSystem__CreateDir, 3}, { "_arrow_fs___FileSystem__DeleteDir", (DL_FUNC) &_arrow_fs___FileSystem__DeleteDir, 2}, { "_arrow_fs___FileSystem__DeleteDirContents", (DL_FUNC) &_arrow_fs___FileSystem__DeleteDirContents, 2}, @@ -4281,6 +4312,7 @@ static const R_CallMethodDef CallEntries[] = { { "_arrow_fs___FileSystem__OpenInputFile", (DL_FUNC) &_arrow_fs___FileSystem__OpenInputFile, 2}, { "_arrow_fs___FileSystem__OpenOutputStream", (DL_FUNC) &_arrow_fs___FileSystem__OpenOutputStream, 2}, { "_arrow_fs___FileSystem__OpenAppendStream", (DL_FUNC) &_arrow_fs___FileSystem__OpenAppendStream, 2}, + { "_arrow_fs___LocalFileSystem__create", (DL_FUNC) &_arrow_fs___LocalFileSystem__create, 0}, { "_arrow_fs___SubTreeFileSystem__create", (DL_FUNC) &_arrow_fs___SubTreeFileSystem__create, 2}, { "_arrow_io___Readable__Read", (DL_FUNC) &_arrow_io___Readable__Read, 2}, { "_arrow_io___InputStream__Close", (DL_FUNC) &_arrow_io___InputStream__Close, 1}, diff --git a/r/src/arrow_types.h b/r/src/arrow_types.h index 309a0ee4938..38f3f83b22f 100644 --- a/r/src/arrow_types.h +++ b/r/src/arrow_types.h @@ -172,6 +172,8 @@ inline std::shared_ptr extract(SEXP x) { #include #include #include +#include +#include #include #include #include @@ -180,7 +182,6 @@ inline std::shared_ptr extract(SEXP x) { #include #include #include -#include #include #include #include diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp index f96997af97f..0ef14f40dcb 100644 --- a/r/src/filesystem.cpp +++ b/r/src/filesystem.cpp @@ -26,7 +26,8 @@ arrow::fs::FileType fs___FileStats__type(const std::shared_ptr& x, arrow::fs::FileType type) { +void fs___FileStats__set_type(const std::shared_ptr& x, + arrow::fs::FileType type) { x->set_type(type); } @@ -36,7 +37,8 @@ std::string fs___FileStats__path(const std::shared_ptr& x) } // [[arrow::export]] -void fs___FileStats__set_path(const std::shared_ptr& x, const std::string& path) { +void fs___FileStats__set_path(const std::shared_ptr& x, + const std::string& path) { x->set_path(path); } @@ -46,7 +48,8 @@ int64_t fs___FileStats__size(const std::shared_ptr& x) { } // [[arrow::export]] -void fs___FileStats__set_size(const std::shared_ptr& x, int64_t size) { +void fs___FileStats__set_size(const std::shared_ptr& x, + int64_t size) { x->set_size(size); } @@ -64,28 +67,30 @@ std::string fs___FileStats__extension(const std::shared_ptr& x) { SEXP res = PROTECT(Rf_allocVector(REALSXP, 1)); // .mtime() gets us nanoseconds since epoch, POSIXct is seconds since epoch as a double - REAL(res)[0] = static_cast(x->mtime().time_since_epoch().count()) / 1000000; + REAL(res)[0] = static_cast(x->mtime().time_since_epoch().count()) / 1000000000; Rf_classgets(res, arrow::r::data::classes_POSIXct); UNPROTECT(1); return res; } // [[arrow::export]] -void fs___FileStats__set_mtime(const std::shared_ptr& x, SEXP time) { - auto secs = std::chrono::seconds(static_cast(REAL(time)[0] * 1000000)); +void fs___FileStats__set_mtime(const std::shared_ptr& x, + SEXP time) { + auto secs = std::chrono::seconds(static_cast(REAL(time)[0] * 1000000000)); x->set_mtime(arrow::fs::TimePoint(secs)); } - // Selector // [[arrow::export]] -std::string fs___Selector__base_dir(const std::shared_ptr& selector) { +std::string fs___Selector__base_dir( + const std::shared_ptr& selector) { return selector->base_dir; } // [[arrow::export]] -bool fs___Selector__allow_non_existent(const std::shared_ptr& selector) { +bool fs___Selector__allow_non_existent( + const std::shared_ptr& selector) { return selector->allow_non_existent; } @@ -96,82 +101,117 @@ bool fs___Selector__recursive(const std::shared_ptr& select // FileSystem +template +std::vector> shared_ptr_vector(const std::vector& vec) { + std::vector> res(vec.size()); + std::transform(vec.begin(), vec.end(), res.begin(), [](const arrow::fs::FileStats& x) { + return std::make_shared(x); + }); + return res; +} // [[arrow::export]] -std::vector> fs___FileSystem__GetTargetStats_1(const std::shared_ptr& file_system, const std::vector& paths) { - auto n = paths.size(); - std::vector> res(n); - for (decltype(n) i=0; iGetTargetStats(paths[i], res[i].get())); - } - return res; +std::vector> fs___FileSystem__GetTargetStats_Paths( + const std::shared_ptr& file_system, + const std::vector& paths) { + std::vector out; + STOP_IF_NOT_OK(file_system->GetTargetStats(paths, &out)); + return shared_ptr_vector(out); +} + +// [[arrow::export]] +std::vector> +fs___FileSystem__GetTargetStats_Selector( + const std::shared_ptr& file_system, + const std::shared_ptr& selector) { + std::vector out; + STOP_IF_NOT_OK(file_system->GetTargetStats(*selector, &out)); + return shared_ptr_vector(out); } // [[arrow::export]] -void fs___FileSystem__CreateDir(const std::shared_ptr& file_system, const std::string& path, bool recursive) { +void fs___FileSystem__CreateDir(const std::shared_ptr& file_system, + const std::string& path, bool recursive) { STOP_IF_NOT_OK(file_system->CreateDir(path, recursive)); } // [[arrow::export]] -void fs___FileSystem__DeleteDir(const std::shared_ptr& file_system, const std::string& path) { +void fs___FileSystem__DeleteDir(const std::shared_ptr& file_system, + const std::string& path) { STOP_IF_NOT_OK(file_system->DeleteDir(path)); } // [[arrow::export]] -void fs___FileSystem__DeleteDirContents(const std::shared_ptr& file_system, const std::string& path) { +void fs___FileSystem__DeleteDirContents( + const std::shared_ptr& file_system, const std::string& path) { STOP_IF_NOT_OK(file_system->DeleteDirContents(path)); } // [[arrow::export]] -void fs___FileSystem__DeleteFile(const std::shared_ptr& file_system, const std::string& path) { +void fs___FileSystem__DeleteFile( + const std::shared_ptr& file_system, const std::string& path) { STOP_IF_NOT_OK(file_system->DeleteFile(path)); } // [[arrow::export]] -void fs___FileSystem__DeleteFiles(const std::shared_ptr& file_system, const std::vector& paths) { +void fs___FileSystem__DeleteFiles( + const std::shared_ptr& file_system, + const std::vector& paths) { STOP_IF_NOT_OK(file_system->DeleteFiles(paths)); } // [[arrow::export]] -void fs___FileSystem__Move(const std::shared_ptr& file_system, const std::string& src, const std::string& dest) { +void fs___FileSystem__Move(const std::shared_ptr& file_system, + const std::string& src, const std::string& dest) { STOP_IF_NOT_OK(file_system->Move(src, dest)); } // [[arrow::export]] -void fs___FileSystem__CopyFile(const std::shared_ptr& file_system, const std::string& src, const std::string& dest) { +void fs___FileSystem__CopyFile(const std::shared_ptr& file_system, + const std::string& src, const std::string& dest) { STOP_IF_NOT_OK(file_system->CopyFile(src, dest)); } // [[arrow::export]] -std::shared_ptr fs___FileSystem__OpenInputStream(const std::shared_ptr& file_system, const std::string& path) { +std::shared_ptr fs___FileSystem__OpenInputStream( + const std::shared_ptr& file_system, const std::string& path) { std::shared_ptr stream; STOP_IF_NOT_OK(file_system->OpenInputStream(path, &stream)); return stream; } // [[arrow::export]] -std::shared_ptr fs___FileSystem__OpenInputFile(const std::shared_ptr& file_system, const std::string& path) { +std::shared_ptr fs___FileSystem__OpenInputFile( + const std::shared_ptr& file_system, const std::string& path) { std::shared_ptr file; STOP_IF_NOT_OK(file_system->OpenInputFile(path, &file)); return file; } // [[arrow::export]] -std::shared_ptr fs___FileSystem__OpenOutputStream(const std::shared_ptr& file_system, const std::string& path) { +std::shared_ptr fs___FileSystem__OpenOutputStream( + const std::shared_ptr& file_system, const std::string& path) { std::shared_ptr stream; STOP_IF_NOT_OK(file_system->OpenOutputStream(path, &stream)); return stream; } // [[arrow::export]] -std::shared_ptr fs___FileSystem__OpenAppendStream(const std::shared_ptr& file_system, const std::string& path) { +std::shared_ptr fs___FileSystem__OpenAppendStream( + const std::shared_ptr& file_system, const std::string& path) { std::shared_ptr stream; STOP_IF_NOT_OK(file_system->OpenAppendStream(path, &stream)); return stream; } // [[arrow::export]] -std::shared_ptr fs___SubTreeFileSystem__create(const std::string& base_path, const std::shared_ptr& base_fs) { +std::shared_ptr fs___LocalFileSystem__create() { + return std::make_shared(); +} + +// [[arrow::export]] +std::shared_ptr fs___SubTreeFileSystem__create( + const std::string& base_path, const std::shared_ptr& base_fs) { return std::make_shared(base_path, base_fs); } diff --git a/r/src/parquet.cpp b/r/src/parquet.cpp index 692916bb963..9f9216a9522 100644 --- a/r/src/parquet.cpp +++ b/r/src/parquet.cpp @@ -31,29 +31,26 @@ parquet___arrow___ArrowReaderProperties__Make(bool use_threads) { // [[arrow::export]] void parquet___arrow___ArrowReaderProperties__set_use_threads( - const std::shared_ptr& properties, - bool use_threads) { + const std::shared_ptr& properties, bool use_threads) { properties->set_use_threads(use_threads); } // [[arrow::export]] bool parquet___arrow___ArrowReaderProperties__get_use_threads( - const std::shared_ptr& properties, - bool use_threads) { + const std::shared_ptr& properties, bool use_threads) { return properties->use_threads(); } // [[arrow::export]] bool parquet___arrow___ArrowReaderProperties__get_read_dictionary( - const std::shared_ptr& properties, - int column_index) { + const std::shared_ptr& properties, int column_index) { return properties->read_dictionary(column_index); } // [[arrow::export]] void parquet___arrow___ArrowReaderProperties__set_read_dictionary( - const std::shared_ptr& properties, - int column_index, bool read_dict) { + const std::shared_ptr& properties, int column_index, + bool read_dict) { properties->set_read_dictionary(column_index, read_dict); } diff --git a/r/src/symbols.cpp b/r/src/symbols.cpp index f7b65731959..6f27eec126d 100644 --- a/r/src/symbols.cpp +++ b/r/src/symbols.cpp @@ -29,7 +29,7 @@ SEXP get_classes_POSIXct() { SEXP classes = Rf_allocVector(STRSXP, 2); R_PreserveObject(classes); SET_STRING_ELT(classes, 0, Rf_mkChar("POSIXct")); - SET_STRING_ELT(classes, 0, Rf_mkChar("POSIXt")); + SET_STRING_ELT(classes, 1, Rf_mkChar("POSIXt")); return classes; } diff --git a/r/tests/testthat/test-filesystem.R b/r/tests/testthat/test-filesystem.R new file mode 100644 index 00000000000..2fc55ee56e1 --- /dev/null +++ b/r/tests/testthat/test-filesystem.R @@ -0,0 +1,74 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +context("test-type") + +test_that("LocalFilesystem", { + fs <- LocalFileSystem$create() + DESCRIPTION <- system.file("DESCRIPTION", package = "arrow") + stat <- fs$GetTargetStats(DESCRIPTION)[[1]] + expect_equal(stat$base_name(), "DESCRIPTION") + expect_equal(stat$extension(), "") + expect_equal(stat$type, FileType$File) + expect_equal(stat$path, DESCRIPTION) + info <- file.info(DESCRIPTION) + + expect_equal(stat$size, info$size) + expect_equal(stat$mtime, info$mtime) + + tf <- tempfile(fileext = ".txt") + fs$CopyFile(DESCRIPTION, tf) + stat <- fs$GetTargetStats(tf)[[1]] + expect_equal(stat$extension(), "txt") + expect_equal(stat$size, info$size) + expect_true(stat$mtime > info$mtime) + expect_equal(readLines(DESCRIPTION), readLines(tf)) + + tf2 <- tempfile(fileext = ".txt") + fs$Move(tf, tf2) + stats <- fs$GetTargetStats(c(tf, tf2, dirname(tf))) + expect_equal(stats[[1]]$type, FileType$NonExistent) + expect_equal(stats[[2]]$type, FileType$File) + expect_equal(stats[[3]]$type, FileType$Directory) + + fs$DeleteFile(tf2) + expect_equal(fs$GetTargetStats(tf2)[[1L]]$type, FileType$NonExistent) + expect_true(!file.exists(tf2)) + + expect_equal(fs$GetTargetStats(tf)[[1L]]$type, FileType$NonExistent) + expect_true(!file.exists(tf)) + + td <- tempfile() + fs$CreateDir(td) + expect_equal(fs$GetTargetStats(td)[[1L]]$type, FileType$Directory) + fs$CopyFile(DESCRIPTION, file.path(td, "DESCRIPTION")) + fs$DeleteDirContents(td) + expect_equal(length(dir(td)), 0L) + fs$DeleteDir(td) + expect_equal(fs$GetTargetStats(td)[[1L]]$type, FileType$NonExistent) + + tf3 <- tempfile() + os <- fs$OpenOutputStream(path = tf3) + bytes <- as.raw(1:40) + os$write(bytes) + os$close() + + is <- fs$OpenInputStream(tf3) + buf <- is$Read(40) + expect_equal(buf$data(), bytes) + is$close() +}) From 891c613910532c9f8638fab4c8943903bf5f92b6 Mon Sep 17 00:00:00 2001 From: Romain Francois Date: Wed, 18 Sep 2019 08:38:14 +0200 Subject: [PATCH 07/13] tests for SubTreeFileSystem --- r/R/filesystem.R | 5 +++-- r/tests/testthat/test-filesystem.R | 26 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/r/R/filesystem.R b/r/R/filesystem.R index 02df110f250..af21d3fe69c 100644 --- a/r/R/filesystem.R +++ b/r/R/filesystem.R @@ -179,6 +179,7 @@ LocalFileSystem$create <- function() { #' @export SubTreeFileSystem <- R6Class("SubTreeFileSystem", inherit = FileSystem) -SubTreeFileSystem$create <- function(base_fs, base_path) { - shared_ptr(SubTreeFileSystem, fs___SubTreeFileSystem__create(base_fs, base_path)) +SubTreeFileSystem$create <- function(base_path, base_fs) { + xp <- fs___SubTreeFileSystem__create(base_path, base_fs) + shared_ptr(SubTreeFileSystem, xp) } diff --git a/r/tests/testthat/test-filesystem.R b/r/tests/testthat/test-filesystem.R index 2fc55ee56e1..c626873a78d 100644 --- a/r/tests/testthat/test-filesystem.R +++ b/r/tests/testthat/test-filesystem.R @@ -72,3 +72,29 @@ test_that("LocalFilesystem", { expect_equal(buf$data(), bytes) is$close() }) + +test_that("SubTreeFilesystem", { + dir.create(td <- tempfile()) + DESCRIPTION <- system.file("DESCRIPTION", package = "arrow") + file.copy(DESCRIPTION, td) + + local_fs <- LocalFileSystem$create() + st_fs <- SubTreeFileSystem$create(td, local_fs) + expect_is(st_fs, "SubTreeFileSystem") + expect_is(st_fs, "FileSystem") + st_fs$CreateDir("test") + st_fs$CopyFile("DESCRIPTION", "DESC.txt") + stats <- st_fs$GetTargetStats(c("DESCRIPTION", "test", "nope", "DESC.txt")) + expect_equal(stats[[1L]]$type, FileType$File) + expect_equal(stats[[2L]]$type, FileType$Directory) + expect_equal(stats[[3L]]$type, FileType$NonExistent) + expect_equal(stats[[4L]]$type, FileType$File) + expect_equal(stats[[4L]]$extension(), "txt") + + local_fs$DeleteDirContents(td) + stats <- st_fs$GetTargetStats(c("DESCRIPTION", "test", "nope", "DESC.txt")) + expect_equal(stats[[1L]]$type, FileType$NonExistent) + expect_equal(stats[[2L]]$type, FileType$NonExistent) + expect_equal(stats[[3L]]$type, FileType$NonExistent) + expect_equal(stats[[4L]]$type, FileType$NonExistent) +}) From fe29e9ad570e9550bb87e40d92dede328043d5ed Mon Sep 17 00:00:00 2001 From: Romain Francois Date: Wed, 18 Sep 2019 09:08:49 +0200 Subject: [PATCH 08/13] Tests for Selector --- r/R/arrowExports.R | 4 ++++ r/R/filesystem.R | 4 ++++ r/src/arrowExports.cpp | 18 ++++++++++++++++++ r/src/filesystem.cpp | 11 +++++++++++ r/tests/testthat/test-filesystem.R | 23 +++++++++++++++++++++++ 5 files changed, 60 insertions(+) diff --git a/r/R/arrowExports.R b/r/R/arrowExports.R index 6175e676bb2..939f7aca30a 100644 --- a/r/R/arrowExports.R +++ b/r/R/arrowExports.R @@ -592,6 +592,10 @@ fs___Selector__recursive <- function(selector){ .Call(`_arrow_fs___Selector__recursive` , selector) } +fs___Selector__create <- function(base_dir, allow_non_existent, recursive){ + .Call(`_arrow_fs___Selector__create` , base_dir, allow_non_existent, recursive) +} + fs___FileSystem__GetTargetStats_Paths <- function(file_system, paths){ .Call(`_arrow_fs___FileSystem__GetTargetStats_Paths` , file_system, paths) } diff --git a/r/R/filesystem.R b/r/R/filesystem.R index af21d3fe69c..a3e3c8aef34 100644 --- a/r/R/filesystem.R +++ b/r/R/filesystem.R @@ -83,6 +83,10 @@ Selector <- R6Class("Selector", ) ) +Selector$create <- function(base_dir, allow_non_existent = FALSE, recursive = FALSE) { + shared_ptr(Selector, fs___Selector__create(base_dir, allow_non_existent, recursive)) +} + #' @title FileSystem class #' @description EXPERIMENTAL: abstract file system API #' diff --git a/r/src/arrowExports.cpp b/r/src/arrowExports.cpp index e0fc89b5489..ded317d5aef 100644 --- a/r/src/arrowExports.cpp +++ b/r/src/arrowExports.cpp @@ -2278,6 +2278,23 @@ RcppExport SEXP _arrow_fs___Selector__recursive(SEXP selector_sexp){ } #endif +// filesystem.cpp +#if defined(ARROW_R_WITH_ARROW) +std::shared_ptr fs___Selector__create(const std::string& base_dir, bool allow_non_existent, bool recursive); +RcppExport SEXP _arrow_fs___Selector__create(SEXP base_dir_sexp, SEXP allow_non_existent_sexp, SEXP recursive_sexp){ +BEGIN_RCPP + Rcpp::traits::input_parameter::type base_dir(base_dir_sexp); + Rcpp::traits::input_parameter::type allow_non_existent(allow_non_existent_sexp); + Rcpp::traits::input_parameter::type recursive(recursive_sexp); + return Rcpp::wrap(fs___Selector__create(base_dir, allow_non_existent, recursive)); +END_RCPP +} +#else +RcppExport SEXP _arrow_fs___Selector__create(SEXP base_dir_sexp, SEXP allow_non_existent_sexp, SEXP recursive_sexp){ + Rf_error("Cannot call fs___Selector__create(). Please use arrow::install_arrow() to install required runtime libraries. "); +} +#endif + // filesystem.cpp #if defined(ARROW_R_WITH_ARROW) std::vector> fs___FileSystem__GetTargetStats_Paths(const std::shared_ptr& file_system, const std::vector& paths); @@ -4299,6 +4316,7 @@ static const R_CallMethodDef CallEntries[] = { { "_arrow_fs___Selector__base_dir", (DL_FUNC) &_arrow_fs___Selector__base_dir, 1}, { "_arrow_fs___Selector__allow_non_existent", (DL_FUNC) &_arrow_fs___Selector__allow_non_existent, 1}, { "_arrow_fs___Selector__recursive", (DL_FUNC) &_arrow_fs___Selector__recursive, 1}, + { "_arrow_fs___Selector__create", (DL_FUNC) &_arrow_fs___Selector__create, 3}, { "_arrow_fs___FileSystem__GetTargetStats_Paths", (DL_FUNC) &_arrow_fs___FileSystem__GetTargetStats_Paths, 2}, { "_arrow_fs___FileSystem__GetTargetStats_Selector", (DL_FUNC) &_arrow_fs___FileSystem__GetTargetStats_Selector, 2}, { "_arrow_fs___FileSystem__CreateDir", (DL_FUNC) &_arrow_fs___FileSystem__CreateDir, 3}, diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp index 0ef14f40dcb..d402b793218 100644 --- a/r/src/filesystem.cpp +++ b/r/src/filesystem.cpp @@ -99,6 +99,17 @@ bool fs___Selector__recursive(const std::shared_ptr& select return selector->recursive; } +// [[arrow::export]] +std::shared_ptr fs___Selector__create(const std::string& base_dir, + bool allow_non_existent, + bool recursive) { + auto selector = std::make_shared(); + selector->base_dir = base_dir; + selector->allow_non_existent = allow_non_existent; + selector->recursive = recursive; + return selector; +} + // FileSystem template diff --git a/r/tests/testthat/test-filesystem.R b/r/tests/testthat/test-filesystem.R index c626873a78d..b8cabf4beff 100644 --- a/r/tests/testthat/test-filesystem.R +++ b/r/tests/testthat/test-filesystem.R @@ -98,3 +98,26 @@ test_that("SubTreeFilesystem", { expect_equal(stats[[3L]]$type, FileType$NonExistent) expect_equal(stats[[4L]]$type, FileType$NonExistent) }) + +test_that("LocalFileSystem + Selector", { + fs <- LocalFileSystem$create() + dir.create(td <- tempfile()) + writeLines("blah blah", file.path(td, "one.txt")) + writeLines("yada yada", file.path(td, "two.txt")) + dir.create(file.path(td, "dir")) + writeLines("...", file.path(td, "dir", "three.txt")) + + selector <- Selector$create(td, recursive = TRUE) + stats <- fs$GetTargetStats(selector) + expect_equal(length(stats), 4L) + types <- sapply(stats, function(.x) .x$type) + expect_equal(sum(types == FileType$File), 3L) + expect_equal(sum(types == FileType$Directory), 1L) + + selector <- Selector$create(td, recursive = FALSE) + stats <- fs$GetTargetStats(selector) + expect_equal(length(stats), 3L) + types <- sapply(stats, function(.x) .x$type) + expect_equal(sum(types == FileType$File), 2L) + expect_equal(sum(types == FileType$Directory), 1L) +}) From a6e786ef16e29f6a4508de416325d77f0607a400 Mon Sep 17 00:00:00 2001 From: Romain Francois Date: Wed, 18 Sep 2019 14:48:25 +0200 Subject: [PATCH 09/13] remove expectation that fails on windows for some reason --- r/tests/testthat/test-filesystem.R | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/r/tests/testthat/test-filesystem.R b/r/tests/testthat/test-filesystem.R index b8cabf4beff..41327c077cf 100644 --- a/r/tests/testthat/test-filesystem.R +++ b/r/tests/testthat/test-filesystem.R @@ -35,7 +35,6 @@ test_that("LocalFilesystem", { stat <- fs$GetTargetStats(tf)[[1]] expect_equal(stat$extension(), "txt") expect_equal(stat$size, info$size) - expect_true(stat$mtime > info$mtime) expect_equal(readLines(DESCRIPTION), readLines(tf)) tf2 <- tempfile(fileext = ".txt") @@ -76,7 +75,7 @@ test_that("LocalFilesystem", { test_that("SubTreeFilesystem", { dir.create(td <- tempfile()) DESCRIPTION <- system.file("DESCRIPTION", package = "arrow") - file.copy(DESCRIPTION, td) + file.copy(DESCRIPTION, file.path(td, "DESCRIPTION")) local_fs <- LocalFileSystem$create() st_fs <- SubTreeFileSystem$create(td, local_fs) From c77f1517f9f195c0818e3952ab9c32c5a268f577 Mon Sep 17 00:00:00 2001 From: Romain Francois Date: Thu, 19 Sep 2019 07:54:44 +0200 Subject: [PATCH 10/13] using std::chrono::nanoseconds --- r/src/filesystem.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp index d402b793218..0a1d101bb2d 100644 --- a/r/src/filesystem.cpp +++ b/r/src/filesystem.cpp @@ -76,8 +76,8 @@ SEXP fs___FileStats__mtime(const std::shared_ptr& x) { // [[arrow::export]] void fs___FileStats__set_mtime(const std::shared_ptr& x, SEXP time) { - auto secs = std::chrono::seconds(static_cast(REAL(time)[0] * 1000000000)); - x->set_mtime(arrow::fs::TimePoint(secs)); + auto nanosecs = std::chrono::nanoseconds(static_cast(REAL(time)[0] * 1000000000)); + x->set_mtime(arrow::fs::TimePoint(nanosecs)); } // Selector From ec53a795baa9ad069ffab0380e49de8c3de2f3b7 Mon Sep 17 00:00:00 2001 From: Romain Francois Date: Thu, 19 Sep 2019 08:00:53 +0200 Subject: [PATCH 11/13] use classes_POSIXct instead of creating new instance of same vector --- r/src/array_to_vector.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/r/src/array_to_vector.cpp b/r/src/array_to_vector.cpp index 7997a1c6442..1925d7e9f04 100644 --- a/r/src/array_to_vector.cpp +++ b/r/src/array_to_vector.cpp @@ -412,7 +412,7 @@ class Converter_Date64 : public Converter { SEXP Allocate(R_xlen_t n) const { Rcpp::NumericVector data(no_init(n)); - data.attr("class") = Rcpp::CharacterVector::create("POSIXct", "POSIXt"); + Rf_classgets(data, arrow::r::data::classes_POSIXct); return data; } @@ -508,7 +508,7 @@ class Converter_Timestamp : public Converter_Time { SEXP Allocate(R_xlen_t n) const { Rcpp::NumericVector data(no_init(n)); - data.attr("class") = Rcpp::CharacterVector::create("POSIXct", "POSIXt"); + Rf_classgets(data, arrow::r::data::classes_POSIXct); return data; } }; From 0f8117d6d73e7582f33191828679550b609e9451 Mon Sep 17 00:00:00 2001 From: Romain Francois Date: Thu, 19 Sep 2019 10:16:44 +0200 Subject: [PATCH 12/13] update R documentation for FileSystem classes --- r/R/filesystem.R | 96 ++++++++++++++++++++++++++------------ r/man/FileStats.Rd | 10 ---- r/man/FileSystem.Rd | 49 +++++++++++++++++-- r/man/LocalFileSystem.Rd | 11 ----- r/man/Selector.Rd | 10 ---- r/man/SubTreeFileSystem.Rd | 10 ---- r/src/filesystem.cpp | 3 +- 7 files changed, 114 insertions(+), 75 deletions(-) delete mode 100644 r/man/FileStats.Rd delete mode 100644 r/man/LocalFileSystem.Rd delete mode 100644 r/man/Selector.Rd delete mode 100644 r/man/SubTreeFileSystem.Rd diff --git a/r/R/filesystem.R b/r/R/filesystem.R index a3e3c8aef34..b13bbfe85eb 100644 --- a/r/R/filesystem.R +++ b/r/R/filesystem.R @@ -17,15 +17,23 @@ #' @include arrow-package.R -#' @title FileStats class -#' @description FileSystem entry stats -#' +#' @title FileSystem entry stats #' @usage NULL #' @format NULL -#' @docType class +#' +#' @section Methods: +#' +#' - `base_name()` : The file base name (component after the last directory separator). +#' - `extension()` : The file extension +#' +#' @section Active bindings: +#' +#' - `$type`: The file type +#' - `$path`: The full file path in the filesystem +#' - `$size`: The size in bytes, if available. Only regular files are guaranteed to have a size. +#' - `$mtime`: The time of last modification, if available. #' #' @rdname FileStats -#' @name FileStats #' @export FileStats <- R6Class("FileStats", inherit = Object, @@ -34,7 +42,13 @@ FileStats <- R6Class("FileStats", extension = function() fs___FileStats__extension(self) ), active = list( - type = function() fs___FileStats__type(self), + type = function(type) { + if (missing(type)) { + fs___FileStats__type(self) + } else { + fs___FileStats__set_type(self, type) + } + }, path = function(path) { if (missing(path)) { fs___FileStats__path(self) @@ -64,15 +78,20 @@ FileStats <- R6Class("FileStats", ) ) -#' @title Selector class -#' @description EXPERIMENTAL: file selector -#' -#' @usage NULL +#' @title file selector #' @format NULL -#' @docType class +#' +#' @section Factory: +#' +#' The `$create()` factory method instantiates a `Selector` given the 3 fields described below. +#' +#' @section Fields: +#' +#' - `base_dir`: The directory in which to select files. If the path exists but doesn't point to a directory, this should be an error. +#' - `allow_non_existent`: The behavior if `base_dir` doesn't exist in the filesystem. If `FALSE`, an error is returned. If `TRUE`, an empty selection is returned +#' - `recursive`: Whether to recurse into subdirectories. #' #' @rdname Selector -#' @name Selector #' @export Selector <- R6Class("Selector", inherit = Object, @@ -87,8 +106,39 @@ Selector$create <- function(base_dir, allow_non_existent = FALSE, recursive = FA shared_ptr(Selector, fs___Selector__create(base_dir, allow_non_existent, recursive)) } -#' @title FileSystem class -#' @description EXPERIMENTAL: abstract file system API +#' @title FileSystem classes +#' @description `FileSystem` is an abstract file system API, +#' `LocalFileSystem` is an implementation accessing files +#' on the local machine. `SubTreeFileSystem` is an implementation that delegates to another +#' implementation after prepending a fixed base path +#' +#' @section Factory: +#' +#' The `$create()` factory methods instantiate the `FileSystem` object and +#' take the following arguments, depending on the subclass: +#' +#' - no argument is needed for instantiating a `LocalFileSystem` +#' - `base_path` and `base_fs` for instantiating a `SubTreeFileSystem` +#' +#' @section Methods: +#' +#' - `$GetTargetStats(x)`: `x` may be a [Selector][Selector] or a character vector of paths. Returns a list of [FileStats][FileStats] +#' - `$CreateDir(path, recursive = TRUE)`: Create a directory and subdirectories. +#' - `$DeleteDir(path)`: Delete a directory and its contents, recursively. +#' - `$DeleteDirContents(path)`: Delete a directory's contents, recursively. Like `$DeleteDir()`, +#' but doesn't delete the directory itself. Passing an empty path (`""`) will wipe the entire filesystem tree. +#' - `$DeleteFile(path)` : Delete a file. +#' - `$DeleteFiles(paths)` : Delete many files. The default implementation issues individual delete operations in sequence. +#' - `$Move(src, dest)`: Move / rename a file or directory. If the destination exists: +#' if it is a non-empty directory, an error is returned +#' otherwise, if it has the same type as the source, it is replaced +#' otherwise, behavior is unspecified (implementation-dependent). +#' - `$CopyFile(src, dest)`: Copy a file. If the destination exists and is a directory, an error is returned. +#' Otherwise, it is replaced. +#' - `$OpenInputStream(path)`: Open an [InputStream][input stream] for sequential reading. +#' - `$OpenInputFile(path)`: On an [RandomAccessFile][input file] for random access reading. +#' - `$OpenOutputStream(path)`: Open an [OutputStream][output stream] for sequential writing. +#' - `$OpenAppendStream(path)`: Open an [OutputStream][output stream] for appending. #' #' @usage NULL #' @format NULL @@ -153,16 +203,9 @@ FileSystem <- R6Class("FileSystem", inherit = Object, ) ) -#' @title LocalFileSystem class -#' @description EXPERIMENTAL: a [FileSystem][FileSystem] implementation accessing files -#' on the local machine. -#' #' @usage NULL #' @format NULL -#' @docType class -#' -#' @rdname LocalFileSystem -#' @name LocalFileSystem +#' @rdname FileSystem #' @export LocalFileSystem <- R6Class("LocalFileSystem", inherit = FileSystem) @@ -171,16 +214,11 @@ LocalFileSystem$create <- function() { } -#' @title SubTreeFileSystem class -#' @description EXPERIMENTAL: abstract file system API -#' #' @usage NULL #' @format NULL -#' @docType class -#' -#' @rdname SubTreeFileSystem -#' @name SubTreeFileSystem +#' @rdname FileSystem #' @export + SubTreeFileSystem <- R6Class("SubTreeFileSystem", inherit = FileSystem) SubTreeFileSystem$create <- function(base_path, base_fs) { diff --git a/r/man/FileStats.Rd b/r/man/FileStats.Rd deleted file mode 100644 index 5a8f7c81261..00000000000 --- a/r/man/FileStats.Rd +++ /dev/null @@ -1,10 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/filesystem.R -\docType{class} -\name{FileStats} -\alias{FileStats} -\title{FileStats class} -\description{ -FileSystem entry stats -} -\keyword{datasets} diff --git a/r/man/FileSystem.Rd b/r/man/FileSystem.Rd index 3e345cfd158..d3a2ab0555d 100644 --- a/r/man/FileSystem.Rd +++ b/r/man/FileSystem.Rd @@ -1,10 +1,51 @@ % Generated by roxygen2: do not edit by hand % Please edit documentation in R/filesystem.R -\docType{class} -\name{FileSystem} +\docType{data} +\name{FileStats} +\alias{FileStats} +\alias{Selector} \alias{FileSystem} -\title{FileSystem class} +\alias{LocalFileSystem} +\alias{SubTreeFileSystem} +\title{FileSystem classes} \description{ -EXPERIMENTAL: abstract file system API +\code{FileSystem} is an abstract file system API, +\code{LocalFileSystem} is an implementation accessing files +on the local machine. \code{SubTreeFileSystem} is an implementation that delegates to another +implementation after prepending a fixed base path } +\section{Factory}{ + + +The \code{$create()} factory methods instantiate the \code{FileSystem} object and +take the following arguments, depending on the subclass: +\itemize{ +\item no argument is needed for instantiating a \code{LocalFileSystem} +\item \code{base_path} and \code{base_fs} for instantiating a \code{SubTreeFileSystem} +} +} + +\section{Methods}{ + +\itemize{ +\item \code{$GetTargetStats(x)}: \code{x} may be a \link{Selector} or a character vector of paths. Returns a list of \link{FileStats} +\item \code{$CreateDir(path, recursive = TRUE)}: Create a directory and subdirectories. +\item \code{$DeleteDir(path)}: Delete a directory and its contents, recursively. +\item \code{$DeleteDirContents(path)}: Delete a directory's contents, recursively. Like \code{$DeleteDir()}, +but doesn't delete the directory itself. Passing an empty path (\code{""}) will wipe the entire filesystem tree. +\item \code{$DeleteFile(path)} : Delete a file. +\item \code{$DeleteFiles(paths)} : Delete many files. The default implementation issues individual delete operations in sequence. +\item \code{$Move(src, dest)}: Move / rename a file or directory. If the destination exists: +if it is a non-empty directory, an error is returned +otherwise, if it has the same type as the source, it is replaced +otherwise, behavior is unspecified (implementation-dependent). +\item \code{$CopyFile(src, dest)}: Copy a file. If the destination exists and is a directory, an error is returned. +Otherwise, it is replaced. +\item \code{$OpenInputStream(path)}: Open an \link[=input stream]{InputStream} for sequential reading. +\item \code{$OpenInputFile(path)}: On an \link[=input file]{RandomAccessFile} for random access reading. +\item \code{$OpenOutputStream(path)}: Open an \link[=output stream]{OutputStream} for sequential writing. +\item \code{$OpenAppendStream(path)}: Open an \link[=output stream]{OutputStream} for appending. +} +} + \keyword{datasets} diff --git a/r/man/LocalFileSystem.Rd b/r/man/LocalFileSystem.Rd deleted file mode 100644 index b6903dd2748..00000000000 --- a/r/man/LocalFileSystem.Rd +++ /dev/null @@ -1,11 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/filesystem.R -\docType{class} -\name{LocalFileSystem} -\alias{LocalFileSystem} -\title{LocalFileSystem class} -\description{ -EXPERIMENTAL: a \link{FileSystem} implementation accessing files -on the local machine. -} -\keyword{datasets} diff --git a/r/man/Selector.Rd b/r/man/Selector.Rd deleted file mode 100644 index 7d64fae1a10..00000000000 --- a/r/man/Selector.Rd +++ /dev/null @@ -1,10 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/filesystem.R -\docType{class} -\name{Selector} -\alias{Selector} -\title{Selector class} -\description{ -EXPERIMENTAL: file selector -} -\keyword{datasets} diff --git a/r/man/SubTreeFileSystem.Rd b/r/man/SubTreeFileSystem.Rd deleted file mode 100644 index ffe4f4bdca1..00000000000 --- a/r/man/SubTreeFileSystem.Rd +++ /dev/null @@ -1,10 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/filesystem.R -\docType{class} -\name{SubTreeFileSystem} -\alias{SubTreeFileSystem} -\title{SubTreeFileSystem class} -\description{ -EXPERIMENTAL: abstract file system API -} -\keyword{datasets} diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp index 0a1d101bb2d..e204645392c 100644 --- a/r/src/filesystem.cpp +++ b/r/src/filesystem.cpp @@ -76,7 +76,8 @@ SEXP fs___FileStats__mtime(const std::shared_ptr& x) { // [[arrow::export]] void fs___FileStats__set_mtime(const std::shared_ptr& x, SEXP time) { - auto nanosecs = std::chrono::nanoseconds(static_cast(REAL(time)[0] * 1000000000)); + auto nanosecs = + std::chrono::nanoseconds(static_cast(REAL(time)[0] * 1000000000)); x->set_mtime(arrow::fs::TimePoint(nanosecs)); } From 5959b76ce56c5844620d6bb8f1da8a26a942a72f Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Thu, 19 Sep 2019 12:17:29 -0700 Subject: [PATCH 13/13] Fix documentation xrefs and skip test that fails on windows --- r/R/filesystem.R | 156 +++++++++++++++-------------- r/man/FileStats.Rd | 30 ++++++ r/man/FileSystem.Rd | 41 ++++---- r/man/Selector.Rd | 32 ++++++ r/tests/testthat/test-filesystem.R | 1 + 5 files changed, 170 insertions(+), 90 deletions(-) create mode 100644 r/man/FileStats.Rd create mode 100644 r/man/Selector.Rd diff --git a/r/R/filesystem.R b/r/R/filesystem.R index b13bbfe85eb..d20edcc6c02 100644 --- a/r/R/filesystem.R +++ b/r/R/filesystem.R @@ -16,21 +16,22 @@ # under the License. #' @include arrow-package.R - #' @title FileSystem entry stats #' @usage NULL #' @format NULL #' #' @section Methods: #' -#' - `base_name()` : The file base name (component after the last directory separator). +#' - `base_name()` : The file base name (component after the last directory +#' separator). #' - `extension()` : The file extension #' #' @section Active bindings: #' #' - `$type`: The file type #' - `$path`: The full file path in the filesystem -#' - `$size`: The size in bytes, if available. Only regular files are guaranteed to have a size. +#' - `$size`: The size in bytes, if available. Only regular files are +#' guaranteed to have a size. #' - `$mtime`: The time of last modification, if available. #' #' @rdname FileStats @@ -83,12 +84,16 @@ FileStats <- R6Class("FileStats", #' #' @section Factory: #' -#' The `$create()` factory method instantiates a `Selector` given the 3 fields described below. +#' The `$create()` factory method instantiates a `Selector` given the 3 fields +#' described below. #' #' @section Fields: #' -#' - `base_dir`: The directory in which to select files. If the path exists but doesn't point to a directory, this should be an error. -#' - `allow_non_existent`: The behavior if `base_dir` doesn't exist in the filesystem. If `FALSE`, an error is returned. If `TRUE`, an empty selection is returned +#' - `base_dir`: The directory in which to select files. If the path exists but +#' doesn't point to a directory, this should be an error. +#' - `allow_non_existent`: The behavior if `base_dir` doesn't exist in the +#' filesystem. If `FALSE`, an error is returned. If `TRUE`, an empty +#' selection is returned #' - `recursive`: Whether to recurse into subdirectories. #' #' @rdname Selector @@ -109,8 +114,8 @@ Selector$create <- function(base_dir, allow_non_existent = FALSE, recursive = FA #' @title FileSystem classes #' @description `FileSystem` is an abstract file system API, #' `LocalFileSystem` is an implementation accessing files -#' on the local machine. `SubTreeFileSystem` is an implementation that delegates to another -#' implementation after prepending a fixed base path +#' on the local machine. `SubTreeFileSystem` is an implementation that delegates +#' to another implementation after prepending a fixed base path #' #' @section Factory: #' @@ -122,23 +127,32 @@ Selector$create <- function(base_dir, allow_non_existent = FALSE, recursive = FA #' #' @section Methods: #' -#' - `$GetTargetStats(x)`: `x` may be a [Selector][Selector] or a character vector of paths. Returns a list of [FileStats][FileStats] +#' - `$GetTargetStats(x)`: `x` may be a [Selector][Selector] or a character +#' vector of paths. Returns a list of [FileStats][FileStats] #' - `$CreateDir(path, recursive = TRUE)`: Create a directory and subdirectories. #' - `$DeleteDir(path)`: Delete a directory and its contents, recursively. -#' - `$DeleteDirContents(path)`: Delete a directory's contents, recursively. Like `$DeleteDir()`, -#' but doesn't delete the directory itself. Passing an empty path (`""`) will wipe the entire filesystem tree. +#' - `$DeleteDirContents(path)`: Delete a directory's contents, recursively. +#' Like `$DeleteDir()`, +#' but doesn't delete the directory itself. Passing an empty path (`""`) will +#' wipe the entire filesystem tree. #' - `$DeleteFile(path)` : Delete a file. -#' - `$DeleteFiles(paths)` : Delete many files. The default implementation issues individual delete operations in sequence. -#' - `$Move(src, dest)`: Move / rename a file or directory. If the destination exists: +#' - `$DeleteFiles(paths)` : Delete many files. The default implementation +#' issues individual delete operations in sequence. +#' - `$Move(src, dest)`: Move / rename a file or directory. If the destination +#' exists: #' if it is a non-empty directory, an error is returned #' otherwise, if it has the same type as the source, it is replaced #' otherwise, behavior is unspecified (implementation-dependent). -#' - `$CopyFile(src, dest)`: Copy a file. If the destination exists and is a directory, an error is returned. -#' Otherwise, it is replaced. -#' - `$OpenInputStream(path)`: Open an [InputStream][input stream] for sequential reading. -#' - `$OpenInputFile(path)`: On an [RandomAccessFile][input file] for random access reading. -#' - `$OpenOutputStream(path)`: Open an [OutputStream][output stream] for sequential writing. -#' - `$OpenAppendStream(path)`: Open an [OutputStream][output stream] for appending. +#' - `$CopyFile(src, dest)`: Copy a file. If the destination exists and is a +#' directory, an error is returned. Otherwise, it is replaced. +#' - `$OpenInputStream(path)`: Open an [input stream][InputStream] for +#' sequential reading. +#' - `$OpenInputFile(path)`: Open an [input file][RandomAccessFile] for random +#' access reading. +#' - `$OpenOutputStream(path)`: Open an [output stream][OutputStream] for +#' sequential writing. +#' - `$OpenAppendStream(path)`: Open an [output stream][OutputStream] for +#' appending. #' #' @usage NULL #' @format NULL @@ -148,59 +162,58 @@ Selector$create <- function(base_dir, allow_non_existent = FALSE, recursive = FA #' @name FileSystem #' @export FileSystem <- R6Class("FileSystem", inherit = Object, - public = list( - GetTargetStats = function(x) { - if (inherits(x, "Selector")) { - map(fs___FileSystem__GetTargetStats_Selector(self, x), shared_ptr, class = FileStats) - } else if (is.character(x)){ - map(fs___FileSystem__GetTargetStats_Paths(self, x), shared_ptr, class = FileStats) - } else { - abort("incompatible type for FileSystem$GetTargetStarts()") - } - }, - - CreateDir = function(path, recursive = TRUE) { - fs___FileSystem__CreateDir(self, path, isTRUE(recursive)) - }, - - DeleteDir = function(path) { - fs___FileSystem__DeleteDir(self, path) - }, - - DeleteDirContents = function(path) { - fs___FileSystem__DeleteDirContents(self, path) - }, - - DeleteFile = function(path) { - fs___FileSystem__DeleteFile(self, path) - }, - - DeleteFiles = function(paths) { - fs___FileSystem__DeleteFiles(self, paths) - }, - - Move = function(src, dest) { - fs___FileSystem__Move(self, src, dest) - }, - - CopyFile = function(src, dest) { - fs___FileSystem__CopyFile(self, src, dest) - }, - - OpenInputStream = function(path) { - shared_ptr(InputStream, fs___FileSystem__OpenInputStream(self, path)) - }, - OpenInputFile = function(path) { - shared_ptr(InputStream, fs___FileSystem__OpenInputFile(self, path)) - }, - OpenOutputStream = function(path) { - shared_ptr(OutputStream, fs___FileSystem__OpenOutputStream(self, path)) - }, - OpenAppendStream = function(path) { - shared_ptr(OutputStream, fs___FileSystem__OpenAppendStream(self, path)) + public = list( + GetTargetStats = function(x) { + if (inherits(x, "Selector")) { + map(fs___FileSystem__GetTargetStats_Selector(self, x), shared_ptr, class = FileStats) + } else if (is.character(x)){ + map(fs___FileSystem__GetTargetStats_Paths(self, x), shared_ptr, class = FileStats) + } else { + abort("incompatible type for FileSystem$GetTargetStarts()") } + }, + + CreateDir = function(path, recursive = TRUE) { + fs___FileSystem__CreateDir(self, path, isTRUE(recursive)) + }, - ) + DeleteDir = function(path) { + fs___FileSystem__DeleteDir(self, path) + }, + + DeleteDirContents = function(path) { + fs___FileSystem__DeleteDirContents(self, path) + }, + + DeleteFile = function(path) { + fs___FileSystem__DeleteFile(self, path) + }, + + DeleteFiles = function(paths) { + fs___FileSystem__DeleteFiles(self, paths) + }, + + Move = function(src, dest) { + fs___FileSystem__Move(self, src, dest) + }, + + CopyFile = function(src, dest) { + fs___FileSystem__CopyFile(self, src, dest) + }, + + OpenInputStream = function(path) { + shared_ptr(InputStream, fs___FileSystem__OpenInputStream(self, path)) + }, + OpenInputFile = function(path) { + shared_ptr(InputStream, fs___FileSystem__OpenInputFile(self, path)) + }, + OpenOutputStream = function(path) { + shared_ptr(OutputStream, fs___FileSystem__OpenOutputStream(self, path)) + }, + OpenAppendStream = function(path) { + shared_ptr(OutputStream, fs___FileSystem__OpenAppendStream(self, path)) + } + ) ) #' @usage NULL @@ -208,7 +221,6 @@ FileSystem <- R6Class("FileSystem", inherit = Object, #' @rdname FileSystem #' @export LocalFileSystem <- R6Class("LocalFileSystem", inherit = FileSystem) - LocalFileSystem$create <- function() { shared_ptr(LocalFileSystem, fs___LocalFileSystem__create()) } @@ -218,9 +230,7 @@ LocalFileSystem$create <- function() { #' @format NULL #' @rdname FileSystem #' @export - SubTreeFileSystem <- R6Class("SubTreeFileSystem", inherit = FileSystem) - SubTreeFileSystem$create <- function(base_path, base_fs) { xp <- fs___SubTreeFileSystem__create(base_path, base_fs) shared_ptr(SubTreeFileSystem, xp) diff --git a/r/man/FileStats.Rd b/r/man/FileStats.Rd new file mode 100644 index 00000000000..04bd92fe2a8 --- /dev/null +++ b/r/man/FileStats.Rd @@ -0,0 +1,30 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/filesystem.R +\docType{data} +\name{FileStats} +\alias{FileStats} +\title{FileSystem entry stats} +\description{ +FileSystem entry stats +} +\section{Methods}{ + +\itemize{ +\item \code{base_name()} : The file base name (component after the last directory +separator). +\item \code{extension()} : The file extension +} +} + +\section{Active bindings}{ + +\itemize{ +\item \code{$type}: The file type +\item \code{$path}: The full file path in the filesystem +\item \code{$size}: The size in bytes, if available. Only regular files are +guaranteed to have a size. +\item \code{$mtime}: The time of last modification, if available. +} +} + +\keyword{datasets} diff --git a/r/man/FileSystem.Rd b/r/man/FileSystem.Rd index d3a2ab0555d..89fa4e6ea94 100644 --- a/r/man/FileSystem.Rd +++ b/r/man/FileSystem.Rd @@ -1,9 +1,7 @@ % Generated by roxygen2: do not edit by hand % Please edit documentation in R/filesystem.R -\docType{data} -\name{FileStats} -\alias{FileStats} -\alias{Selector} +\docType{class} +\name{FileSystem} \alias{FileSystem} \alias{LocalFileSystem} \alias{SubTreeFileSystem} @@ -11,8 +9,8 @@ \description{ \code{FileSystem} is an abstract file system API, \code{LocalFileSystem} is an implementation accessing files -on the local machine. \code{SubTreeFileSystem} is an implementation that delegates to another -implementation after prepending a fixed base path +on the local machine. \code{SubTreeFileSystem} is an implementation that delegates +to another implementation after prepending a fixed base path } \section{Factory}{ @@ -28,23 +26,32 @@ take the following arguments, depending on the subclass: \section{Methods}{ \itemize{ -\item \code{$GetTargetStats(x)}: \code{x} may be a \link{Selector} or a character vector of paths. Returns a list of \link{FileStats} +\item \code{$GetTargetStats(x)}: \code{x} may be a \link{Selector} or a character +vector of paths. Returns a list of \link{FileStats} \item \code{$CreateDir(path, recursive = TRUE)}: Create a directory and subdirectories. \item \code{$DeleteDir(path)}: Delete a directory and its contents, recursively. -\item \code{$DeleteDirContents(path)}: Delete a directory's contents, recursively. Like \code{$DeleteDir()}, -but doesn't delete the directory itself. Passing an empty path (\code{""}) will wipe the entire filesystem tree. +\item \code{$DeleteDirContents(path)}: Delete a directory's contents, recursively. +Like \code{$DeleteDir()}, +but doesn't delete the directory itself. Passing an empty path (\code{""}) will +wipe the entire filesystem tree. \item \code{$DeleteFile(path)} : Delete a file. -\item \code{$DeleteFiles(paths)} : Delete many files. The default implementation issues individual delete operations in sequence. -\item \code{$Move(src, dest)}: Move / rename a file or directory. If the destination exists: +\item \code{$DeleteFiles(paths)} : Delete many files. The default implementation +issues individual delete operations in sequence. +\item \code{$Move(src, dest)}: Move / rename a file or directory. If the destination +exists: if it is a non-empty directory, an error is returned otherwise, if it has the same type as the source, it is replaced otherwise, behavior is unspecified (implementation-dependent). -\item \code{$CopyFile(src, dest)}: Copy a file. If the destination exists and is a directory, an error is returned. -Otherwise, it is replaced. -\item \code{$OpenInputStream(path)}: Open an \link[=input stream]{InputStream} for sequential reading. -\item \code{$OpenInputFile(path)}: On an \link[=input file]{RandomAccessFile} for random access reading. -\item \code{$OpenOutputStream(path)}: Open an \link[=output stream]{OutputStream} for sequential writing. -\item \code{$OpenAppendStream(path)}: Open an \link[=output stream]{OutputStream} for appending. +\item \code{$CopyFile(src, dest)}: Copy a file. If the destination exists and is a +directory, an error is returned. Otherwise, it is replaced. +\item \code{$OpenInputStream(path)}: Open an \link[=InputStream]{input stream} for +sequential reading. +\item \code{$OpenInputFile(path)}: Open an \link[=RandomAccessFile]{input file} for random +access reading. +\item \code{$OpenOutputStream(path)}: Open an \link[=OutputStream]{output stream} for +sequential writing. +\item \code{$OpenAppendStream(path)}: Open an \link[=OutputStream]{output stream} for +appending. } } diff --git a/r/man/Selector.Rd b/r/man/Selector.Rd new file mode 100644 index 00000000000..fe270ed0a61 --- /dev/null +++ b/r/man/Selector.Rd @@ -0,0 +1,32 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/filesystem.R +\docType{data} +\name{Selector} +\alias{Selector} +\title{file selector} +\usage{ +Selector +} +\description{ +file selector +} +\section{Factory}{ + + +The \code{$create()} factory method instantiates a \code{Selector} given the 3 fields +described below. +} + +\section{Fields}{ + +\itemize{ +\item \code{base_dir}: The directory in which to select files. If the path exists but +doesn't point to a directory, this should be an error. +\item \code{allow_non_existent}: The behavior if \code{base_dir} doesn't exist in the +filesystem. If \code{FALSE}, an error is returned. If \code{TRUE}, an empty +selection is returned +\item \code{recursive}: Whether to recurse into subdirectories. +} +} + +\keyword{datasets} diff --git a/r/tests/testthat/test-filesystem.R b/r/tests/testthat/test-filesystem.R index 41327c077cf..1c198383066 100644 --- a/r/tests/testthat/test-filesystem.R +++ b/r/tests/testthat/test-filesystem.R @@ -83,6 +83,7 @@ test_that("SubTreeFilesystem", { expect_is(st_fs, "FileSystem") st_fs$CreateDir("test") st_fs$CopyFile("DESCRIPTION", "DESC.txt") + skip_on_os("windows") # See ARROW-6622 stats <- st_fs$GetTargetStats(c("DESCRIPTION", "test", "nope", "DESC.txt")) expect_equal(stats[[1L]]$type, FileType$File) expect_equal(stats[[2L]]$type, FileType$Directory)