ARROW-5494: [Python] Create FileSystem bindings

cpp/src/arrow/filesystem/api.h

@@ -0,0 +1,26 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_FILESYSTEM_API_H
+#define ARROW_FILESYSTEM_API_H
+
+#include "arrow/filesystem/filesystem.h"  // IWYU pragma: export
+#include "arrow/filesystem/localfs.h"     // IWYU pragma: export
+#include "arrow/filesystem/mockfs.h"      // IWYU pragma: export
+#include "arrow/filesystem/s3fs.h"        // IWYU pragma: export
+
+#endif  // ARROW_FILESYSTEM_API_H

cpp/src/arrow/filesystem/filesystem.h

@@ -57,7 +57,7 @@ using TimePoint =
     std::chrono::time_point<std::chrono::system_clock, std::chrono::nanoseconds>;
 
 /// \brief EXPERIMENTAL: FileSystem entry type
-enum class ARROW_EXPORT FileType {
+enum class ARROW_EXPORT FileType : int8_t {
   // Target does not exist
   NonExistent,
   // Target exists but its type is unknown (could be a special file such

cpp/src/arrow/python/CMakeLists.txt

@@ -32,6 +32,7 @@ set(ARROW_PYTHON_SRCS
     benchmark.cc
     common.cc
     config.cc
+    datetime.cc
     decimal.cc
     deserialize.cc
     extension_type.cc

cpp/src/arrow/python/api.h

@@ -20,6 +20,7 @@
 
 #include "arrow/python/arrow_to_pandas.h"
 #include "arrow/python/common.h"
+#include "arrow/python/datetime.h"
 #include "arrow/python/deserialize.h"
 #include "arrow/python/helpers.h"
 #include "arrow/python/inference.h"

cpp/src/arrow/python/arrow_to_pandas.cc

@@ -46,13 +46,13 @@
 
 #include "arrow/python/common.h"
 #include "arrow/python/config.h"
+#include "arrow/python/datetime.h"
 #include "arrow/python/decimal.h"
 #include "arrow/python/helpers.h"
 #include "arrow/python/numpy_convert.h"
 #include "arrow/python/numpy_internal.h"
 #include "arrow/python/python_to_arrow.h"
 #include "arrow/python/type_traits.h"
-#include "arrow/python/util/datetime.h"
 
 namespace arrow {
 
@@ -654,7 +654,7 @@ static Status ConvertDates(const PandasOptions& options, const ChunkedArray& dat
     PyDateTime_IMPORT;
   }
   auto WrapValue = [](typename Type::c_type value, PyObject** out) {
-    RETURN_NOT_OK(PyDate_from_int(value, Type::UNIT, out));
+    RETURN_NOT_OK(internal::PyDate_from_int(value, Type::UNIT, out));
     RETURN_IF_PYERROR();
     return Status::OK();
   };
@@ -672,7 +672,7 @@ static Status ConvertTimes(const PandasOptions& options, const ChunkedArray& dat
   const TimeUnit::type unit = checked_cast<const Type&>(*data.type()).unit();
 
   auto WrapValue = [unit](typename Type::c_type value, PyObject** out) {
-    RETURN_NOT_OK(PyTime_from_int(value, unit, out));
+    RETURN_NOT_OK(internal::PyTime_from_int(value, unit, out));
     RETURN_IF_PYERROR();
     return Status::OK();
   };

cpp/src/arrow/python/util/datetime.h → cpp/src/arrow/python/datetime.cc

@@ -15,18 +15,19 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#ifndef PYARROW_UTIL_DATETIME_H
-#define PYARROW_UTIL_DATETIME_H
-
 #include <algorithm>
+#include <chrono>
+#include <iostream>
 
-#include <datetime.h>
+#include "arrow/python/datetime.h"
 #include "arrow/python/platform.h"
 #include "arrow/status.h"
+#include "arrow/type.h"
 #include "arrow/util/logging.h"
 
 namespace arrow {
 namespace py {
+namespace internal {
 
 // The following code is adapted from
 // https://github.com/numpy/numpy/blob/master/numpy/core/src/multiarray/datetime.c
@@ -155,25 +156,6 @@ static void get_date_from_days(int64_t days, int64_t* date_year, int64_t* date_m
   return;
 }
 
-static inline int64_t PyTime_to_us(PyObject* pytime) {
-  return (static_cast<int64_t>(PyDateTime_TIME_GET_HOUR(pytime)) * 3600000000LL +
-          static_cast<int64_t>(PyDateTime_TIME_GET_MINUTE(pytime)) * 60000000LL +
-          static_cast<int64_t>(PyDateTime_TIME_GET_SECOND(pytime)) * 1000000LL +
-          PyDateTime_TIME_GET_MICROSECOND(pytime));
-}
-
-static inline int64_t PyTime_to_s(PyObject* pytime) {
-  return PyTime_to_us(pytime) / 1000000;
-}
-
-static inline int64_t PyTime_to_ms(PyObject* pytime) {
-  return PyTime_to_us(pytime) / 1000;
-}
-
-static inline int64_t PyTime_to_ns(PyObject* pytime) {
-  return PyTime_to_us(pytime) * 1000;
-}
-
 // Splitting time quantities, for example splitting total seconds into
 // minutes and remaining seconds. After we run
 // int64_t remaining = split_time(total, quotient, &next)
@@ -233,25 +215,23 @@ static inline Status PyDate_convert_int(int64_t val, const DateUnit unit, int64_
   return Status::OK();
 }
 
-static inline Status PyTime_from_int(int64_t val, const TimeUnit::type unit,
-                                     PyObject** out) {
+Status PyTime_from_int(int64_t val, const TimeUnit::type unit, PyObject** out) {
   int64_t hour = 0, minute = 0, second = 0, microsecond = 0;
   RETURN_NOT_OK(PyTime_convert_int(val, unit, &hour, &minute, &second, &microsecond));
   *out = PyTime_FromTime(static_cast<int32_t>(hour), static_cast<int32_t>(minute),
                          static_cast<int32_t>(second), static_cast<int32_t>(microsecond));
   return Status::OK();
 }
 
-static inline Status PyDate_from_int(int64_t val, const DateUnit unit, PyObject** out) {
+Status PyDate_from_int(int64_t val, const DateUnit unit, PyObject** out) {
   int64_t year = 0, month = 0, day = 0;
   RETURN_NOT_OK(PyDate_convert_int(val, unit, &year, &month, &day));
   *out = PyDate_FromDate(static_cast<int32_t>(year), static_cast<int32_t>(month),
                          static_cast<int32_t>(day));
   return Status::OK();
 }
 
-static inline Status PyDateTime_from_int(int64_t val, const TimeUnit::type unit,
-                                         PyObject** out) {
+Status PyDateTime_from_int(int64_t val, const TimeUnit::type unit, PyObject** out) {
   int64_t hour = 0, minute = 0, second = 0, microsecond = 0;
   RETURN_NOT_OK(PyTime_convert_int(val, unit, &hour, &minute, &second, &microsecond));
   int64_t total_days = 0;
@@ -265,47 +245,19 @@ static inline Status PyDateTime_from_int(int64_t val, const TimeUnit::type unit,
   return Status::OK();
 }
 
-static inline int64_t PyDate_to_days(PyDateTime_Date* pydate) {
-  return get_days_from_date(PyDateTime_GET_YEAR(pydate), PyDateTime_GET_MONTH(pydate),
-                            PyDateTime_GET_DAY(pydate));
-}
-
-static inline int64_t PyDate_to_ms(PyDateTime_Date* pydate) {
-  int64_t total_seconds = 0;
-  int64_t days =
-      get_days_from_date(PyDateTime_GET_YEAR(pydate), PyDateTime_GET_MONTH(pydate),
-                         PyDateTime_GET_DAY(pydate));
-  total_seconds += days * 24 * 3600;
-  return total_seconds * 1000;
-}
-
-static inline int64_t PyDateTime_to_s(PyDateTime_DateTime* pydatetime) {
-  int64_t total_seconds = 0;
-  total_seconds += PyDateTime_DATE_GET_SECOND(pydatetime);
-  total_seconds += PyDateTime_DATE_GET_MINUTE(pydatetime) * 60;
-  total_seconds += PyDateTime_DATE_GET_HOUR(pydatetime) * 3600;
-
-  return total_seconds +
-         (PyDate_to_ms(reinterpret_cast<PyDateTime_Date*>(pydatetime)) / 1000LL);
-}
-
-static inline int64_t PyDateTime_to_ms(PyDateTime_DateTime* pydatetime) {
-  int64_t date_ms = PyDateTime_to_s(pydatetime) * 1000;
-  int ms = PyDateTime_DATE_GET_MICROSECOND(pydatetime) / 1000;
-  return date_ms + ms;
-}
-
-static inline int64_t PyDateTime_to_us(PyDateTime_DateTime* pydatetime) {
-  int64_t ms = PyDateTime_to_s(pydatetime) * 1000;
-  int us = PyDateTime_DATE_GET_MICROSECOND(pydatetime);
-  return ms * 1000 + us;
+Status PyDateTime_from_TimePoint(TimePoint val, PyObject** out) {
+  PyDateTime_IMPORT;
+  auto nanos = val.time_since_epoch();
+  auto micros = std::chrono::duration_cast<std::chrono::microseconds>(nanos);
+  RETURN_NOT_OK(PyDateTime_from_int(micros.count(), TimeUnit::MICRO, out));
+  return Status::OK();
 }
 
-static inline int64_t PyDateTime_to_ns(PyDateTime_DateTime* pydatetime) {
-  return PyDateTime_to_us(pydatetime) * 1000;
+int64_t PyDate_to_days(PyDateTime_Date* pydate) {
+  return get_days_from_date(PyDateTime_GET_YEAR(pydate), PyDateTime_GET_MONTH(pydate),
+                            PyDateTime_GET_DAY(pydate));
 }
 
+}  // namespace internal
 }  // namespace py
 }  // namespace arrow
-
-#endif  // PYARROW_UTIL_DATETIME_H

cpp/src/arrow/python/datetime.h

@@ -0,0 +1,108 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PYARROW_UTIL_DATETIME_H
+#define PYARROW_UTIL_DATETIME_H
+
+#include <algorithm>
+#include <chrono>
+
+#include "arrow/python/platform.h"
+#include "arrow/python/visibility.h"
+#include "arrow/status.h"
+#include "arrow/type.h"
+#include "arrow/util/logging.h"
+
+namespace arrow {
+namespace py {
+namespace internal {
+
+ARROW_PYTHON_EXPORT
+inline int64_t PyTime_to_us(PyObject* pytime) {
+  return (static_cast<int64_t>(PyDateTime_TIME_GET_HOUR(pytime)) * 3600000000LL +
+          static_cast<int64_t>(PyDateTime_TIME_GET_MINUTE(pytime)) * 60000000LL +
+          static_cast<int64_t>(PyDateTime_TIME_GET_SECOND(pytime)) * 1000000LL +
+          PyDateTime_TIME_GET_MICROSECOND(pytime));
+}
+
+ARROW_PYTHON_EXPORT
+inline int64_t PyTime_to_s(PyObject* pytime) { return PyTime_to_us(pytime) / 1000000; }
+
+ARROW_PYTHON_EXPORT
+inline int64_t PyTime_to_ms(PyObject* pytime) { return PyTime_to_us(pytime) / 1000; }
+
+ARROW_PYTHON_EXPORT
+inline int64_t PyTime_to_ns(PyObject* pytime) { return PyTime_to_us(pytime) * 1000; }
+
+ARROW_PYTHON_EXPORT
+Status PyTime_from_int(int64_t val, const TimeUnit::type unit, PyObject** out);
+
+ARROW_PYTHON_EXPORT
+Status PyDate_from_int(int64_t val, const DateUnit unit, PyObject** out);
+
+ARROW_PYTHON_EXPORT
+Status PyDateTime_from_int(int64_t val, const TimeUnit::type unit, PyObject** out);
+
+using TimePoint =
+    std::chrono::time_point<std::chrono::system_clock, std::chrono::nanoseconds>;
+
+ARROW_PYTHON_EXPORT
+Status PyDateTime_from_TimePoint(TimePoint val, PyObject** out);
+
+ARROW_PYTHON_EXPORT
+int64_t PyDate_to_days(PyDateTime_Date* pydate);
+
+ARROW_PYTHON_EXPORT
+inline int64_t PyDate_to_ms(PyDateTime_Date* pydate) {
+  return PyDate_to_days(pydate) * 24 * 3600 * 1000;
+}
+
+ARROW_PYTHON_EXPORT
+inline int64_t PyDateTime_to_s(PyDateTime_DateTime* pydatetime) {
+  int64_t total_seconds = 0;
+  total_seconds += PyDateTime_DATE_GET_SECOND(pydatetime);
+  total_seconds += PyDateTime_DATE_GET_MINUTE(pydatetime) * 60;
+  total_seconds += PyDateTime_DATE_GET_HOUR(pydatetime) * 3600;
+
+  return total_seconds +
+         (PyDate_to_ms(reinterpret_cast<PyDateTime_Date*>(pydatetime)) / 1000LL);
+}
+
+ARROW_PYTHON_EXPORT
+inline int64_t PyDateTime_to_ms(PyDateTime_DateTime* pydatetime) {
+  int64_t date_ms = PyDateTime_to_s(pydatetime) * 1000;
+  int ms = PyDateTime_DATE_GET_MICROSECOND(pydatetime) / 1000;
+  return date_ms + ms;
+}
+
+ARROW_PYTHON_EXPORT
+inline int64_t PyDateTime_to_us(PyDateTime_DateTime* pydatetime) {
+  int64_t ms = PyDateTime_to_s(pydatetime) * 1000;
+  int us = PyDateTime_DATE_GET_MICROSECOND(pydatetime);
+  return ms * 1000 + us;
+}
+
+ARROW_PYTHON_EXPORT
+inline int64_t PyDateTime_to_ns(PyDateTime_DateTime* pydatetime) {
+  return PyDateTime_to_us(pydatetime) * 1000;
+}
+
+}  // namespace internal
+}  // namespace py
+}  // namespace arrow
+
+#endif  // PYARROW_UTIL_DATETIME_H

cpp/src/arrow/python/deserialize.cc

@@ -39,11 +39,11 @@
 #include "arrow/util/parsing.h"
 
 #include "arrow/python/common.h"
+#include "arrow/python/datetime.h"
 #include "arrow/python/helpers.h"
 #include "arrow/python/numpy_convert.h"
 #include "arrow/python/pyarrow.h"
 #include "arrow/python/serialize.h"
-#include "arrow/python/util/datetime.h"
 
 namespace arrow {
 
@@ -154,7 +154,7 @@ Status GetValue(PyObject* context, const Array& arr, int64_t index, int8_t type,
       *result = PyFloat_FromDouble(checked_cast<const DoubleArray&>(arr).Value(index));
       return Status::OK();
     case PythonType::DATE64: {
-      RETURN_NOT_OK(PyDateTime_from_int(
+      RETURN_NOT_OK(internal::PyDateTime_from_int(
           checked_cast<const Date64Array&>(arr).Value(index), TimeUnit::MICRO, result));
       RETURN_IF_PYERROR();
       return Status::OK();

cpp/src/arrow/python/inference.cc

@@ -31,11 +31,11 @@
 #include "arrow/util/decimal.h"
 #include "arrow/util/logging.h"
 
+#include "arrow/python/datetime.h"
 #include "arrow/python/decimal.h"
 #include "arrow/python/helpers.h"
 #include "arrow/python/iterators.h"
 #include "arrow/python/numpy_convert.h"
-#include "arrow/python/util/datetime.h"
 
 namespace arrow {
 namespace py {

cpp/src/arrow/python/numpy_to_arrow.cc

@@ -48,13 +48,13 @@
 
 #include "arrow/python/common.h"
 #include "arrow/python/config.h"
+#include "arrow/python/datetime.h"
 #include "arrow/python/helpers.h"
 #include "arrow/python/iterators.h"
 #include "arrow/python/numpy_convert.h"
 #include "arrow/python/numpy_internal.h"
 #include "arrow/python/python_to_arrow.h"
 #include "arrow/python/type_traits.h"
-#include "arrow/python/util/datetime.h"
 
 namespace arrow {