From b57d8c108af5e75c7d0546adf8b9a4cbca1f9149 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 2 Jun 2022 12:46:41 +0200 Subject: [PATCH 001/138] Move python related files into pyarrow --- .../pyarrow/src_arrow}/ArrowPythonConfig.cmake.in | 0 .../pyarrow/src_arrow}/ArrowPythonFlightConfig.cmake.in | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/CMakeLists.txt | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/api.h | 0 .../python => python/pyarrow/src_arrow}/arrow-python-flight.pc.in | 0 .../arrow/python => python/pyarrow/src_arrow}/arrow-python.pc.in | 0 .../arrow/python => python/pyarrow/src_arrow}/arrow_to_pandas.cc | 0 .../arrow/python => python/pyarrow/src_arrow}/arrow_to_pandas.h | 0 .../pyarrow/src_arrow}/arrow_to_python_internal.h | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/benchmark.cc | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/benchmark.h | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/common.cc | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/common.h | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/csv.cc | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/csv.h | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/datetime.cc | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/datetime.h | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/decimal.cc | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/decimal.h | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/deserialize.cc | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/deserialize.h | 0 .../arrow/python => python/pyarrow/src_arrow}/extension_type.cc | 0 .../arrow/python => python/pyarrow/src_arrow}/extension_type.h | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/filesystem.cc | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/filesystem.h | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/flight.cc | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/flight.h | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/gdb.cc | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/gdb.h | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/helpers.cc | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/helpers.h | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/inference.cc | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/inference.h | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/init.cc | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/init.h | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/io.cc | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/io.h | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/ipc.cc | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/ipc.h | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/iterators.h | 0 .../arrow/python => python/pyarrow/src_arrow}/numpy_convert.cc | 0 .../src/arrow/python => python/pyarrow/src_arrow}/numpy_convert.h | 0 .../arrow/python => python/pyarrow/src_arrow}/numpy_internal.h | 0 .../src/arrow/python => python/pyarrow/src_arrow}/numpy_interop.h | 0 .../arrow/python => python/pyarrow/src_arrow}/numpy_to_arrow.cc | 0 .../arrow/python => python/pyarrow/src_arrow}/numpy_to_arrow.h | 0 .../python => python/pyarrow/src_arrow}/parquet_encryption.cc | 0 .../python => python/pyarrow/src_arrow}/parquet_encryption.h | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/pch.h | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/platform.h | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/pyarrow.cc | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/pyarrow.h | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/pyarrow_api.h | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/pyarrow_lib.h | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/python_test.cc | 0 .../arrow/python => python/pyarrow/src_arrow}/python_to_arrow.cc | 0 .../arrow/python => python/pyarrow/src_arrow}/python_to_arrow.h | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/serialize.cc | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/serialize.h | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/type_traits.h | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/udf.cc | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/udf.h | 0 .../arrow/python => python/pyarrow/src_arrow}/util/CMakeLists.txt | 0 .../arrow/python => python/pyarrow/src_arrow}/util/test_main.cc | 0 {cpp/src/arrow/python => python/pyarrow/src_arrow}/visibility.h | 0 65 files changed, 0 insertions(+), 0 deletions(-) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/ArrowPythonConfig.cmake.in (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/ArrowPythonFlightConfig.cmake.in (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/CMakeLists.txt (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/api.h (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/arrow-python-flight.pc.in (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/arrow-python.pc.in (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/arrow_to_pandas.cc (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/arrow_to_pandas.h (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/arrow_to_python_internal.h (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/benchmark.cc (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/benchmark.h (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/common.cc (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/common.h (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/csv.cc (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/csv.h (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/datetime.cc (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/datetime.h (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/decimal.cc (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/decimal.h (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/deserialize.cc (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/deserialize.h (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/extension_type.cc (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/extension_type.h (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/filesystem.cc (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/filesystem.h (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/flight.cc (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/flight.h (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/gdb.cc (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/gdb.h (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/helpers.cc (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/helpers.h (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/inference.cc (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/inference.h (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/init.cc (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/init.h (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/io.cc (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/io.h (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/ipc.cc (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/ipc.h (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/iterators.h (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/numpy_convert.cc (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/numpy_convert.h (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/numpy_internal.h (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/numpy_interop.h (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/numpy_to_arrow.cc (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/numpy_to_arrow.h (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/parquet_encryption.cc (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/parquet_encryption.h (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/pch.h (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/platform.h (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/pyarrow.cc (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/pyarrow.h (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/pyarrow_api.h (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/pyarrow_lib.h (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/python_test.cc (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/python_to_arrow.cc (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/python_to_arrow.h (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/serialize.cc (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/serialize.h (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/type_traits.h (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/udf.cc (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/udf.h (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/util/CMakeLists.txt (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/util/test_main.cc (100%) rename {cpp/src/arrow/python => python/pyarrow/src_arrow}/visibility.h (100%) diff --git a/cpp/src/arrow/python/ArrowPythonConfig.cmake.in b/python/pyarrow/src_arrow/ArrowPythonConfig.cmake.in similarity index 100% rename from cpp/src/arrow/python/ArrowPythonConfig.cmake.in rename to python/pyarrow/src_arrow/ArrowPythonConfig.cmake.in diff --git a/cpp/src/arrow/python/ArrowPythonFlightConfig.cmake.in b/python/pyarrow/src_arrow/ArrowPythonFlightConfig.cmake.in similarity index 100% rename from cpp/src/arrow/python/ArrowPythonFlightConfig.cmake.in rename to python/pyarrow/src_arrow/ArrowPythonFlightConfig.cmake.in diff --git a/cpp/src/arrow/python/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt similarity index 100% rename from cpp/src/arrow/python/CMakeLists.txt rename to python/pyarrow/src_arrow/CMakeLists.txt diff --git a/cpp/src/arrow/python/api.h b/python/pyarrow/src_arrow/api.h similarity index 100% rename from cpp/src/arrow/python/api.h rename to python/pyarrow/src_arrow/api.h diff --git a/cpp/src/arrow/python/arrow-python-flight.pc.in b/python/pyarrow/src_arrow/arrow-python-flight.pc.in similarity index 100% rename from cpp/src/arrow/python/arrow-python-flight.pc.in rename to python/pyarrow/src_arrow/arrow-python-flight.pc.in diff --git a/cpp/src/arrow/python/arrow-python.pc.in b/python/pyarrow/src_arrow/arrow-python.pc.in similarity index 100% rename from cpp/src/arrow/python/arrow-python.pc.in rename to python/pyarrow/src_arrow/arrow-python.pc.in diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/python/pyarrow/src_arrow/arrow_to_pandas.cc similarity index 100% rename from cpp/src/arrow/python/arrow_to_pandas.cc rename to python/pyarrow/src_arrow/arrow_to_pandas.cc diff --git a/cpp/src/arrow/python/arrow_to_pandas.h b/python/pyarrow/src_arrow/arrow_to_pandas.h similarity index 100% rename from cpp/src/arrow/python/arrow_to_pandas.h rename to python/pyarrow/src_arrow/arrow_to_pandas.h diff --git a/cpp/src/arrow/python/arrow_to_python_internal.h b/python/pyarrow/src_arrow/arrow_to_python_internal.h similarity index 100% rename from cpp/src/arrow/python/arrow_to_python_internal.h rename to python/pyarrow/src_arrow/arrow_to_python_internal.h diff --git a/cpp/src/arrow/python/benchmark.cc b/python/pyarrow/src_arrow/benchmark.cc similarity index 100% rename from cpp/src/arrow/python/benchmark.cc rename to python/pyarrow/src_arrow/benchmark.cc diff --git a/cpp/src/arrow/python/benchmark.h b/python/pyarrow/src_arrow/benchmark.h similarity index 100% rename from cpp/src/arrow/python/benchmark.h rename to python/pyarrow/src_arrow/benchmark.h diff --git a/cpp/src/arrow/python/common.cc b/python/pyarrow/src_arrow/common.cc similarity index 100% rename from cpp/src/arrow/python/common.cc rename to python/pyarrow/src_arrow/common.cc diff --git a/cpp/src/arrow/python/common.h b/python/pyarrow/src_arrow/common.h similarity index 100% rename from cpp/src/arrow/python/common.h rename to python/pyarrow/src_arrow/common.h diff --git a/cpp/src/arrow/python/csv.cc b/python/pyarrow/src_arrow/csv.cc similarity index 100% rename from cpp/src/arrow/python/csv.cc rename to python/pyarrow/src_arrow/csv.cc diff --git a/cpp/src/arrow/python/csv.h b/python/pyarrow/src_arrow/csv.h similarity index 100% rename from cpp/src/arrow/python/csv.h rename to python/pyarrow/src_arrow/csv.h diff --git a/cpp/src/arrow/python/datetime.cc b/python/pyarrow/src_arrow/datetime.cc similarity index 100% rename from cpp/src/arrow/python/datetime.cc rename to python/pyarrow/src_arrow/datetime.cc diff --git a/cpp/src/arrow/python/datetime.h b/python/pyarrow/src_arrow/datetime.h similarity index 100% rename from cpp/src/arrow/python/datetime.h rename to python/pyarrow/src_arrow/datetime.h diff --git a/cpp/src/arrow/python/decimal.cc b/python/pyarrow/src_arrow/decimal.cc similarity index 100% rename from cpp/src/arrow/python/decimal.cc rename to python/pyarrow/src_arrow/decimal.cc diff --git a/cpp/src/arrow/python/decimal.h b/python/pyarrow/src_arrow/decimal.h similarity index 100% rename from cpp/src/arrow/python/decimal.h rename to python/pyarrow/src_arrow/decimal.h diff --git a/cpp/src/arrow/python/deserialize.cc b/python/pyarrow/src_arrow/deserialize.cc similarity index 100% rename from cpp/src/arrow/python/deserialize.cc rename to python/pyarrow/src_arrow/deserialize.cc diff --git a/cpp/src/arrow/python/deserialize.h b/python/pyarrow/src_arrow/deserialize.h similarity index 100% rename from cpp/src/arrow/python/deserialize.h rename to python/pyarrow/src_arrow/deserialize.h diff --git a/cpp/src/arrow/python/extension_type.cc b/python/pyarrow/src_arrow/extension_type.cc similarity index 100% rename from cpp/src/arrow/python/extension_type.cc rename to python/pyarrow/src_arrow/extension_type.cc diff --git a/cpp/src/arrow/python/extension_type.h b/python/pyarrow/src_arrow/extension_type.h similarity index 100% rename from cpp/src/arrow/python/extension_type.h rename to python/pyarrow/src_arrow/extension_type.h diff --git a/cpp/src/arrow/python/filesystem.cc b/python/pyarrow/src_arrow/filesystem.cc similarity index 100% rename from cpp/src/arrow/python/filesystem.cc rename to python/pyarrow/src_arrow/filesystem.cc diff --git a/cpp/src/arrow/python/filesystem.h b/python/pyarrow/src_arrow/filesystem.h similarity index 100% rename from cpp/src/arrow/python/filesystem.h rename to python/pyarrow/src_arrow/filesystem.h diff --git a/cpp/src/arrow/python/flight.cc b/python/pyarrow/src_arrow/flight.cc similarity index 100% rename from cpp/src/arrow/python/flight.cc rename to python/pyarrow/src_arrow/flight.cc diff --git a/cpp/src/arrow/python/flight.h b/python/pyarrow/src_arrow/flight.h similarity index 100% rename from cpp/src/arrow/python/flight.h rename to python/pyarrow/src_arrow/flight.h diff --git a/cpp/src/arrow/python/gdb.cc b/python/pyarrow/src_arrow/gdb.cc similarity index 100% rename from cpp/src/arrow/python/gdb.cc rename to python/pyarrow/src_arrow/gdb.cc diff --git a/cpp/src/arrow/python/gdb.h b/python/pyarrow/src_arrow/gdb.h similarity index 100% rename from cpp/src/arrow/python/gdb.h rename to python/pyarrow/src_arrow/gdb.h diff --git a/cpp/src/arrow/python/helpers.cc b/python/pyarrow/src_arrow/helpers.cc similarity index 100% rename from cpp/src/arrow/python/helpers.cc rename to python/pyarrow/src_arrow/helpers.cc diff --git a/cpp/src/arrow/python/helpers.h b/python/pyarrow/src_arrow/helpers.h similarity index 100% rename from cpp/src/arrow/python/helpers.h rename to python/pyarrow/src_arrow/helpers.h diff --git a/cpp/src/arrow/python/inference.cc b/python/pyarrow/src_arrow/inference.cc similarity index 100% rename from cpp/src/arrow/python/inference.cc rename to python/pyarrow/src_arrow/inference.cc diff --git a/cpp/src/arrow/python/inference.h b/python/pyarrow/src_arrow/inference.h similarity index 100% rename from cpp/src/arrow/python/inference.h rename to python/pyarrow/src_arrow/inference.h diff --git a/cpp/src/arrow/python/init.cc b/python/pyarrow/src_arrow/init.cc similarity index 100% rename from cpp/src/arrow/python/init.cc rename to python/pyarrow/src_arrow/init.cc diff --git a/cpp/src/arrow/python/init.h b/python/pyarrow/src_arrow/init.h similarity index 100% rename from cpp/src/arrow/python/init.h rename to python/pyarrow/src_arrow/init.h diff --git a/cpp/src/arrow/python/io.cc b/python/pyarrow/src_arrow/io.cc similarity index 100% rename from cpp/src/arrow/python/io.cc rename to python/pyarrow/src_arrow/io.cc diff --git a/cpp/src/arrow/python/io.h b/python/pyarrow/src_arrow/io.h similarity index 100% rename from cpp/src/arrow/python/io.h rename to python/pyarrow/src_arrow/io.h diff --git a/cpp/src/arrow/python/ipc.cc b/python/pyarrow/src_arrow/ipc.cc similarity index 100% rename from cpp/src/arrow/python/ipc.cc rename to python/pyarrow/src_arrow/ipc.cc diff --git a/cpp/src/arrow/python/ipc.h b/python/pyarrow/src_arrow/ipc.h similarity index 100% rename from cpp/src/arrow/python/ipc.h rename to python/pyarrow/src_arrow/ipc.h diff --git a/cpp/src/arrow/python/iterators.h b/python/pyarrow/src_arrow/iterators.h similarity index 100% rename from cpp/src/arrow/python/iterators.h rename to python/pyarrow/src_arrow/iterators.h diff --git a/cpp/src/arrow/python/numpy_convert.cc b/python/pyarrow/src_arrow/numpy_convert.cc similarity index 100% rename from cpp/src/arrow/python/numpy_convert.cc rename to python/pyarrow/src_arrow/numpy_convert.cc diff --git a/cpp/src/arrow/python/numpy_convert.h b/python/pyarrow/src_arrow/numpy_convert.h similarity index 100% rename from cpp/src/arrow/python/numpy_convert.h rename to python/pyarrow/src_arrow/numpy_convert.h diff --git a/cpp/src/arrow/python/numpy_internal.h b/python/pyarrow/src_arrow/numpy_internal.h similarity index 100% rename from cpp/src/arrow/python/numpy_internal.h rename to python/pyarrow/src_arrow/numpy_internal.h diff --git a/cpp/src/arrow/python/numpy_interop.h b/python/pyarrow/src_arrow/numpy_interop.h similarity index 100% rename from cpp/src/arrow/python/numpy_interop.h rename to python/pyarrow/src_arrow/numpy_interop.h diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/python/pyarrow/src_arrow/numpy_to_arrow.cc similarity index 100% rename from cpp/src/arrow/python/numpy_to_arrow.cc rename to python/pyarrow/src_arrow/numpy_to_arrow.cc diff --git a/cpp/src/arrow/python/numpy_to_arrow.h b/python/pyarrow/src_arrow/numpy_to_arrow.h similarity index 100% rename from cpp/src/arrow/python/numpy_to_arrow.h rename to python/pyarrow/src_arrow/numpy_to_arrow.h diff --git a/cpp/src/arrow/python/parquet_encryption.cc b/python/pyarrow/src_arrow/parquet_encryption.cc similarity index 100% rename from cpp/src/arrow/python/parquet_encryption.cc rename to python/pyarrow/src_arrow/parquet_encryption.cc diff --git a/cpp/src/arrow/python/parquet_encryption.h b/python/pyarrow/src_arrow/parquet_encryption.h similarity index 100% rename from cpp/src/arrow/python/parquet_encryption.h rename to python/pyarrow/src_arrow/parquet_encryption.h diff --git a/cpp/src/arrow/python/pch.h b/python/pyarrow/src_arrow/pch.h similarity index 100% rename from cpp/src/arrow/python/pch.h rename to python/pyarrow/src_arrow/pch.h diff --git a/cpp/src/arrow/python/platform.h b/python/pyarrow/src_arrow/platform.h similarity index 100% rename from cpp/src/arrow/python/platform.h rename to python/pyarrow/src_arrow/platform.h diff --git a/cpp/src/arrow/python/pyarrow.cc b/python/pyarrow/src_arrow/pyarrow.cc similarity index 100% rename from cpp/src/arrow/python/pyarrow.cc rename to python/pyarrow/src_arrow/pyarrow.cc diff --git a/cpp/src/arrow/python/pyarrow.h b/python/pyarrow/src_arrow/pyarrow.h similarity index 100% rename from cpp/src/arrow/python/pyarrow.h rename to python/pyarrow/src_arrow/pyarrow.h diff --git a/cpp/src/arrow/python/pyarrow_api.h b/python/pyarrow/src_arrow/pyarrow_api.h similarity index 100% rename from cpp/src/arrow/python/pyarrow_api.h rename to python/pyarrow/src_arrow/pyarrow_api.h diff --git a/cpp/src/arrow/python/pyarrow_lib.h b/python/pyarrow/src_arrow/pyarrow_lib.h similarity index 100% rename from cpp/src/arrow/python/pyarrow_lib.h rename to python/pyarrow/src_arrow/pyarrow_lib.h diff --git a/cpp/src/arrow/python/python_test.cc b/python/pyarrow/src_arrow/python_test.cc similarity index 100% rename from cpp/src/arrow/python/python_test.cc rename to python/pyarrow/src_arrow/python_test.cc diff --git a/cpp/src/arrow/python/python_to_arrow.cc b/python/pyarrow/src_arrow/python_to_arrow.cc similarity index 100% rename from cpp/src/arrow/python/python_to_arrow.cc rename to python/pyarrow/src_arrow/python_to_arrow.cc diff --git a/cpp/src/arrow/python/python_to_arrow.h b/python/pyarrow/src_arrow/python_to_arrow.h similarity index 100% rename from cpp/src/arrow/python/python_to_arrow.h rename to python/pyarrow/src_arrow/python_to_arrow.h diff --git a/cpp/src/arrow/python/serialize.cc b/python/pyarrow/src_arrow/serialize.cc similarity index 100% rename from cpp/src/arrow/python/serialize.cc rename to python/pyarrow/src_arrow/serialize.cc diff --git a/cpp/src/arrow/python/serialize.h b/python/pyarrow/src_arrow/serialize.h similarity index 100% rename from cpp/src/arrow/python/serialize.h rename to python/pyarrow/src_arrow/serialize.h diff --git a/cpp/src/arrow/python/type_traits.h b/python/pyarrow/src_arrow/type_traits.h similarity index 100% rename from cpp/src/arrow/python/type_traits.h rename to python/pyarrow/src_arrow/type_traits.h diff --git a/cpp/src/arrow/python/udf.cc b/python/pyarrow/src_arrow/udf.cc similarity index 100% rename from cpp/src/arrow/python/udf.cc rename to python/pyarrow/src_arrow/udf.cc diff --git a/cpp/src/arrow/python/udf.h b/python/pyarrow/src_arrow/udf.h similarity index 100% rename from cpp/src/arrow/python/udf.h rename to python/pyarrow/src_arrow/udf.h diff --git a/cpp/src/arrow/python/util/CMakeLists.txt b/python/pyarrow/src_arrow/util/CMakeLists.txt similarity index 100% rename from cpp/src/arrow/python/util/CMakeLists.txt rename to python/pyarrow/src_arrow/util/CMakeLists.txt diff --git a/cpp/src/arrow/python/util/test_main.cc b/python/pyarrow/src_arrow/util/test_main.cc similarity index 100% rename from cpp/src/arrow/python/util/test_main.cc rename to python/pyarrow/src_arrow/util/test_main.cc diff --git a/cpp/src/arrow/python/visibility.h b/python/pyarrow/src_arrow/visibility.h similarity index 100% rename from cpp/src/arrow/python/visibility.h rename to python/pyarrow/src_arrow/visibility.h From e276ee7706cb6cb3987066e6c47ac2c3a37c1fb3 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Fri, 3 Jun 2022 11:25:50 +0200 Subject: [PATCH 002/138] Add changes to cmake for C PyArrow --- cpp/src/arrow/CMakeLists.txt | 4 - cpp/src/arrow/public_api_test.cc | 2 +- python/pyarrow/src_arrow/CMakeLists.txt | 104 +++++++++++++++++- python/pyarrow/src_arrow/api.h | 22 ++-- python/pyarrow/src_arrow/arrow_to_pandas.cc | 24 ++-- python/pyarrow/src_arrow/arrow_to_pandas.h | 4 +- .../src_arrow/arrow_to_python_internal.h | 2 +- python/pyarrow/src_arrow/benchmark.cc | 4 +- python/pyarrow/src_arrow/benchmark.h | 4 +- python/pyarrow/src_arrow/common.cc | 4 +- python/pyarrow/src_arrow/common.h | 4 +- python/pyarrow/src_arrow/csv.cc | 4 +- python/pyarrow/src_arrow/csv.h | 2 +- python/pyarrow/src_arrow/datetime.cc | 10 +- python/pyarrow/src_arrow/datetime.h | 4 +- python/pyarrow/src_arrow/decimal.cc | 6 +- python/pyarrow/src_arrow/decimal.h | 2 +- python/pyarrow/src_arrow/deserialize.cc | 16 +-- python/pyarrow/src_arrow/deserialize.h | 4 +- python/pyarrow/src_arrow/extension_type.cc | 6 +- python/pyarrow/src_arrow/extension_type.h | 4 +- python/pyarrow/src_arrow/filesystem.cc | 2 +- python/pyarrow/src_arrow/filesystem.h | 4 +- python/pyarrow/src_arrow/flight.h | 2 +- python/pyarrow/src_arrow/gdb.cc | 2 +- python/pyarrow/src_arrow/gdb.h | 2 +- python/pyarrow/src_arrow/helpers.cc | 8 +- python/pyarrow/src_arrow/helpers.h | 6 +- python/pyarrow/src_arrow/inference.cc | 14 +-- python/pyarrow/src_arrow/inference.h | 6 +- python/pyarrow/src_arrow/init.cc | 4 +- python/pyarrow/src_arrow/init.h | 4 +- python/pyarrow/src_arrow/io.cc | 6 +- python/pyarrow/src_arrow/io.h | 4 +- python/pyarrow/src_arrow/ipc.cc | 4 +- python/pyarrow/src_arrow/ipc.h | 4 +- python/pyarrow/src_arrow/iterators.h | 4 +- python/pyarrow/src_arrow/numpy_convert.cc | 10 +- python/pyarrow/src_arrow/numpy_convert.h | 4 +- python/pyarrow/src_arrow/numpy_internal.h | 4 +- python/pyarrow/src_arrow/numpy_interop.h | 2 +- python/pyarrow/src_arrow/numpy_to_arrow.cc | 20 ++-- python/pyarrow/src_arrow/numpy_to_arrow.h | 4 +- .../pyarrow/src_arrow/parquet_encryption.cc | 2 +- python/pyarrow/src_arrow/parquet_encryption.h | 4 +- python/pyarrow/src_arrow/pch.h | 2 +- python/pyarrow/src_arrow/pyarrow.cc | 8 +- python/pyarrow/src_arrow/pyarrow.h | 4 +- python/pyarrow/src_arrow/python_test.cc | 14 +-- python/pyarrow/src_arrow/python_to_arrow.cc | 18 +-- python/pyarrow/src_arrow/python_to_arrow.h | 6 +- python/pyarrow/src_arrow/serialize.cc | 18 +-- python/pyarrow/src_arrow/serialize.h | 2 +- python/pyarrow/src_arrow/type_traits.h | 4 +- python/pyarrow/src_arrow/udf.cc | 4 +- python/pyarrow/src_arrow/udf.h | 8 +- python/pyarrow/src_arrow/util/test_main.cc | 8 +- python/pyarrow/tensorflow/plasma_op.cc | 4 +- 58 files changed, 277 insertions(+), 185 deletions(-) diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index 5070d22fc55..1e30e50a317 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -805,10 +805,6 @@ if(ARROW_ORC) add_subdirectory(adapters/orc) endif() -if(ARROW_PYTHON) - add_subdirectory(python) -endif() - if(ARROW_TENSORFLOW) add_subdirectory(adapters/tensorflow) endif() diff --git a/cpp/src/arrow/public_api_test.cc b/cpp/src/arrow/public_api_test.cc index 45f3313c67f..5a607c53ad9 100644 --- a/cpp/src/arrow/public_api_test.cc +++ b/cpp/src/arrow/public_api_test.cc @@ -51,7 +51,7 @@ #endif #ifdef ARROW_PYTHON -#include "arrow/python/api.h" // IWYU pragma: keep +#include "api.h" // IWYU pragma: keep #endif #ifdef DCHECK diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index c37240a426c..ff34014f412 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -19,8 +19,37 @@ # arrow_python # +cmake_minimum_required(VERSION 3.5) + +# RPATH settings on macOS do not affect install_name. +# https://cmake.org/cmake/help/latest/policy/CMP0068.html +if(POLICY CMP0068) + cmake_policy(SET CMP0068 NEW) +endif() + +# Define +# ARROW_SOURCE_DIR: location of arrow/cpp +# CMAKE_MODULE_PATH: location of cmake_modules in python +get_filename_component(PYARROW_SOURCE_DIR ${CMAKE_SOURCE_DIR} DIRECTORY) +get_filename_component(PYTHON_SOURCE_DIR ${PYARROW_SOURCE_DIR} DIRECTORY) +get_filename_component(ARROW_SOURCE_DIR ${PYTHON_SOURCE_DIR} DIRECTORY) +set(ARROW_SOURCE_DIR "${ARROW_SOURCE_DIR}/cpp") +set(CMAKE_MODULE_PATH "${PYTHON_SOURCE_DIR}/cmake_modules") + +# Need to set to ARRROW_VERSION before finding Arrow package! +project(arrow_python VERSION 9.0.0) + +if(NOT DEFINED CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release) +endif() + +find_package(Arrow REQUIRED) find_package(Python3Alt 3.7 REQUIRED) +# Needed due to +# CMake Error: INSTALL(EXPORT) given unknown export "arrow_python_targets" +option(ARROW_BUILD_SHARED "Link to the Arrow shared library" ON) + add_custom_target(arrow_python-all) add_custom_target(arrow_python) add_custom_target(arrow_python-tests) @@ -50,6 +79,14 @@ set(ARROW_PYTHON_SRCS set_source_files_properties(init.cc PROPERTIES SKIP_PRECOMPILE_HEADERS ON SKIP_UNITY_BUILD_INCLUSION ON) +# Set necessary components on (from arrow/cpp/CMakeLists.txt, end of the file) +set(ARROW_COMPUTE ON) +set(ARROW_CSV ON) +set(ARROW_DATASET ON) +set(ARROW_FILESYSTEM ON) +set(ARROW_HDFS ON) +set(ARROW_JSON ON) + if(ARROW_CSV) list(APPEND ARROW_PYTHON_SRCS csv.cc) endif() @@ -62,7 +99,14 @@ if(PARQUET_REQUIRE_ENCRYPTION) list(APPEND ARROW_PYTHON_SRCS parquet_encryption.cc) endif() -set(ARROW_PYTHON_DEPENDENCIES arrow_dependencies) +# Link to arrow dependecies +if(ARROW_LINK_SHARED) + set(ARROW_PYTHON_DEPENDENCIES arrow_shared) +else() + set(THREADS_PREFER_PTHREAD_FLAG ON) + find_package(Threads REQUIRED) + set(ARROW_PYTHON_DEPENDENCIES arrow_static Threads::Threads) +endif() if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang") set_property(SOURCE pyarrow.cc @@ -70,6 +114,50 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_CXX_COMPILER_ID STREQUAL PROPERTY COMPILE_FLAGS " -Wno-cast-qual ") endif() +# +# Compiler stuff +# + +include(GNUInstallDirs) + +# This ensures that things like gnu++11 get passed correctly +if(NOT DEFINED CMAKE_CXX_STANDARD) + set(CMAKE_CXX_STANDARD 11) +endif() + +# We require a C++11 compliant compiler +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +# +# Linker flags +# + +# Localize thirdparty symbols using a linker version script. This hides them +# from the client application. The OS X linker does not support the +# version-script option. +if(CMAKE_VERSION VERSION_LESS 3.18) + if(APPLE OR WIN32) + set(CXX_LINKER_SUPPORTS_VERSION_SCRIPT FALSE) + else() + set(CXX_LINKER_SUPPORTS_VERSION_SCRIPT TRUE) + endif() +else() + include(CheckLinkerFlag) + check_linker_flag(CXX + "-Wl,--version-script=${ARROW_SOURCE_DIR}/cpp/src/arrow/symbols.map" + CXX_LINKER_SUPPORTS_VERSION_SCRIPT) +endif() + +# Need to set ARROW_VERSION_SCRIPT_FLAGS for add_arrow_lib +if(CXX_LINKER_SUPPORTS_VERSION_SCRIPT) + set(ARROW_VERSION_SCRIPT_FLAGS + "-Wl,--version-script=${ARROW_SOURCE_DIR}/cpp/src/arrow/symbols.map") +endif() + +# +# shred/static link libs +# + set(ARROW_PYTHON_SHARED_LINK_LIBS arrow_shared) set(ARROW_PYTHON_SHARED_PRIVATE_LINK_LIBS) set(ARROW_PYTHON_STATIC_LINK_LIBS ${PYTHON_OTHER_LIBS}) @@ -87,6 +175,14 @@ endif() set(ARROW_PYTHON_INCLUDES ${NUMPY_INCLUDE_DIRS} ${PYTHON_INCLUDE_DIRS}) +# Inlude macros needed to find and use add_arrow_lib function +include(BuildUtils) +include(CMakePackageConfigHelpers) + +# Set the output directory for cmake module +# (CMAKE_INSTALL_PREFIX = python/build/dist! should be set in setup.py!) +set(ARROW_CMAKE_INSTALL_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}") + add_arrow_lib(arrow_python CMAKE_PACKAGE_NAME ArrowPython @@ -95,13 +191,13 @@ add_arrow_lib(arrow_python SOURCES ${ARROW_PYTHON_SRCS} PRECOMPILED_HEADERS - "$<$:arrow/python/pch.h>" + "$<$:pch.h>" OUTPUTS ARROW_PYTHON_LIBRARIES DEPENDENCIES ${ARROW_PYTHON_DEPENDENCIES} SHARED_LINK_FLAGS - ${ARROW_VERSION_SCRIPT_FLAGS} # Defined in cpp/arrow/CMakeLists.txt + ${ARROW_VERSION_SCRIPT_FLAGS} # Defined in line 95 SHARED_LINK_LIBS ${ARROW_PYTHON_SHARED_LINK_LIBS} SHARED_PRIVATE_LINK_LIBS @@ -139,7 +235,7 @@ if(ARROW_FLIGHT AND ARROW_BUILD_SHARED) DEPENDENCIES flight_grpc_gen SHARED_LINK_FLAGS - ${ARROW_VERSION_SCRIPT_FLAGS} # Defined in cpp/arrow/CMakeLists.txt + ${ARROW_VERSION_SCRIPT_FLAGS} # Defined in line 95 SHARED_LINK_LIBS arrow_python_shared arrow_flight_shared diff --git a/python/pyarrow/src_arrow/api.h b/python/pyarrow/src_arrow/api.h index a0b13d6d130..28c46c95e94 100644 --- a/python/pyarrow/src_arrow/api.h +++ b/python/pyarrow/src_arrow/api.h @@ -17,14 +17,14 @@ #pragma once -#include "arrow/python/arrow_to_pandas.h" -#include "arrow/python/common.h" -#include "arrow/python/datetime.h" -#include "arrow/python/deserialize.h" -#include "arrow/python/helpers.h" -#include "arrow/python/inference.h" -#include "arrow/python/io.h" -#include "arrow/python/numpy_convert.h" -#include "arrow/python/numpy_to_arrow.h" -#include "arrow/python/python_to_arrow.h" -#include "arrow/python/serialize.h" +#include "arrow_to_pandas.h" +#include "common.h" +#include "datetime.h" +#include "deserialize.h" +#include "helpers.h" +#include "inference.h" +#include "io.h" +#include "numpy_convert.h" +#include "numpy_to_arrow.h" +#include "python_to_arrow.h" +#include "serialize.h" diff --git a/python/pyarrow/src_arrow/arrow_to_pandas.cc b/python/pyarrow/src_arrow/arrow_to_pandas.cc index 8f9d1cb45b9..437f0f11925 100644 --- a/python/pyarrow/src_arrow/arrow_to_pandas.cc +++ b/python/pyarrow/src_arrow/arrow_to_pandas.cc @@ -17,8 +17,8 @@ // Functions for pandas conversion via NumPy -#include "arrow/python/arrow_to_pandas.h" -#include "arrow/python/numpy_interop.h" // IWYU pragma: expand +#include "arrow_to_pandas.h" +#include "numpy_interop.h" // IWYU pragma: expand #include #include @@ -48,16 +48,16 @@ #include "arrow/compute/api.h" -#include "arrow/python/arrow_to_python_internal.h" -#include "arrow/python/common.h" -#include "arrow/python/datetime.h" -#include "arrow/python/decimal.h" -#include "arrow/python/helpers.h" -#include "arrow/python/numpy_convert.h" -#include "arrow/python/numpy_internal.h" -#include "arrow/python/pyarrow.h" -#include "arrow/python/python_to_arrow.h" -#include "arrow/python/type_traits.h" +#include "arrow_to_python_internal.h" +#include "common.h" +#include "datetime.h" +#include "decimal.h" +#include "helpers.h" +#include "numpy_convert.h" +#include "numpy_internal.h" +#include "pyarrow.h" +#include "python_to_arrow.h" +#include "type_traits.h" namespace arrow { diff --git a/python/pyarrow/src_arrow/arrow_to_pandas.h b/python/pyarrow/src_arrow/arrow_to_pandas.h index 6570364b8d2..33c08b6fe81 100644 --- a/python/pyarrow/src_arrow/arrow_to_pandas.h +++ b/python/pyarrow/src_arrow/arrow_to_pandas.h @@ -20,14 +20,14 @@ #pragma once -#include "arrow/python/platform.h" +#include "platform.h" #include #include #include #include "arrow/memory_pool.h" -#include "arrow/python/visibility.h" +#include "visibility.h" namespace arrow { diff --git a/python/pyarrow/src_arrow/arrow_to_python_internal.h b/python/pyarrow/src_arrow/arrow_to_python_internal.h index 514cda32001..251c2a38ca0 100644 --- a/python/pyarrow/src_arrow/arrow_to_python_internal.h +++ b/python/pyarrow/src_arrow/arrow_to_python_internal.h @@ -18,7 +18,7 @@ #pragma once #include "arrow/array.h" -#include "arrow/python/platform.h" +#include "platform.h" namespace arrow { namespace py { diff --git a/python/pyarrow/src_arrow/benchmark.cc b/python/pyarrow/src_arrow/benchmark.cc index 2d29f69d25b..1e56552ed80 100644 --- a/python/pyarrow/src_arrow/benchmark.cc +++ b/python/pyarrow/src_arrow/benchmark.cc @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. -#include -#include +#include "benchmark.h" +#include "helpers.h" namespace arrow { namespace py { diff --git a/python/pyarrow/src_arrow/benchmark.h b/python/pyarrow/src_arrow/benchmark.h index 8060dd33722..883f02feceb 100644 --- a/python/pyarrow/src_arrow/benchmark.h +++ b/python/pyarrow/src_arrow/benchmark.h @@ -17,9 +17,9 @@ #pragma once -#include "arrow/python/platform.h" +#include "platform.h" -#include "arrow/python/visibility.h" +#include "visibility.h" namespace arrow { namespace py { diff --git a/python/pyarrow/src_arrow/common.cc b/python/pyarrow/src_arrow/common.cc index 6fe2ed4dae3..09dde3e4489 100644 --- a/python/pyarrow/src_arrow/common.cc +++ b/python/pyarrow/src_arrow/common.cc @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "arrow/python/common.h" +#include "common.h" #include #include @@ -26,7 +26,7 @@ #include "arrow/util/checked_cast.h" #include "arrow/util/logging.h" -#include "arrow/python/helpers.h" +#include "helpers.h" namespace arrow { diff --git a/python/pyarrow/src_arrow/common.h b/python/pyarrow/src_arrow/common.h index 5c16106730b..6558dbb1c99 100644 --- a/python/pyarrow/src_arrow/common.h +++ b/python/pyarrow/src_arrow/common.h @@ -21,8 +21,8 @@ #include #include "arrow/buffer.h" -#include "arrow/python/pyarrow.h" -#include "arrow/python/visibility.h" +#include "pyarrow.h" +#include "visibility.h" #include "arrow/result.h" #include "arrow/util/macros.h" diff --git a/python/pyarrow/src_arrow/csv.cc b/python/pyarrow/src_arrow/csv.cc index d96c9400e2b..61ff23a22ed 100644 --- a/python/pyarrow/src_arrow/csv.cc +++ b/python/pyarrow/src_arrow/csv.cc @@ -15,11 +15,11 @@ // specific language governing permissions and limitations // under the License. -#include "arrow/python/csv.h" +#include "csv.h" #include -#include "arrow/python/common.h" +#include "common.h" namespace arrow { diff --git a/python/pyarrow/src_arrow/csv.h b/python/pyarrow/src_arrow/csv.h index 34302e93667..0992ae3c6e5 100644 --- a/python/pyarrow/src_arrow/csv.h +++ b/python/pyarrow/src_arrow/csv.h @@ -23,7 +23,7 @@ #include #include "arrow/csv/options.h" -#include "arrow/python/common.h" +#include "common.h" #include "arrow/util/macros.h" namespace arrow { diff --git a/python/pyarrow/src_arrow/datetime.cc b/python/pyarrow/src_arrow/datetime.cc index 848b0a6bf10..164a2fe1f92 100644 --- a/python/pyarrow/src_arrow/datetime.cc +++ b/python/pyarrow/src_arrow/datetime.cc @@ -14,17 +14,17 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -#include "arrow/python/datetime.h" +#include "datetime.h" #include #include #include #include "arrow/array.h" -#include "arrow/python/arrow_to_python_internal.h" -#include "arrow/python/common.h" -#include "arrow/python/helpers.h" -#include "arrow/python/platform.h" +#include "arrow_to_python_internal.h" +#include "common.h" +#include "helpers.h" +#include "platform.h" #include "arrow/scalar.h" #include "arrow/status.h" #include "arrow/type.h" diff --git a/python/pyarrow/src_arrow/datetime.h b/python/pyarrow/src_arrow/datetime.h index dd07710aaf6..6f9bfbe2dbf 100644 --- a/python/pyarrow/src_arrow/datetime.h +++ b/python/pyarrow/src_arrow/datetime.h @@ -20,8 +20,8 @@ #include #include -#include "arrow/python/platform.h" -#include "arrow/python/visibility.h" +#include "platform.h" +#include "visibility.h" #include "arrow/status.h" #include "arrow/type.h" #include "arrow/type_fwd.h" diff --git a/python/pyarrow/src_arrow/decimal.cc b/python/pyarrow/src_arrow/decimal.cc index 0c00fcfaa8e..a7244f9dcc1 100644 --- a/python/pyarrow/src_arrow/decimal.cc +++ b/python/pyarrow/src_arrow/decimal.cc @@ -18,9 +18,9 @@ #include #include -#include "arrow/python/common.h" -#include "arrow/python/decimal.h" -#include "arrow/python/helpers.h" +#include "common.h" +#include "decimal.h" +#include "helpers.h" #include "arrow/type_fwd.h" #include "arrow/util/decimal.h" #include "arrow/util/logging.h" diff --git a/python/pyarrow/src_arrow/decimal.h b/python/pyarrow/src_arrow/decimal.h index 1187037aed2..5c4a17dcd48 100644 --- a/python/pyarrow/src_arrow/decimal.h +++ b/python/pyarrow/src_arrow/decimal.h @@ -19,7 +19,7 @@ #include -#include "arrow/python/visibility.h" +#include "visibility.h" #include "arrow/type.h" namespace arrow { diff --git a/python/pyarrow/src_arrow/deserialize.cc b/python/pyarrow/src_arrow/deserialize.cc index 961a1686e0a..ad28874460a 100644 --- a/python/pyarrow/src_arrow/deserialize.cc +++ b/python/pyarrow/src_arrow/deserialize.cc @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -#include "arrow/python/deserialize.h" +#include "deserialize.h" -#include "arrow/python/numpy_interop.h" +#include "numpy_interop.h" #include #include @@ -40,12 +40,12 @@ #include "arrow/util/logging.h" #include "arrow/util/value_parsing.h" -#include "arrow/python/common.h" -#include "arrow/python/datetime.h" -#include "arrow/python/helpers.h" -#include "arrow/python/numpy_convert.h" -#include "arrow/python/pyarrow.h" -#include "arrow/python/serialize.h" +#include "common.h" +#include "datetime.h" +#include "helpers.h" +#include "numpy_convert.h" +#include "pyarrow.h" +#include "serialize.h" namespace arrow { diff --git a/python/pyarrow/src_arrow/deserialize.h b/python/pyarrow/src_arrow/deserialize.h index 41b6a13a388..08d0972048d 100644 --- a/python/pyarrow/src_arrow/deserialize.h +++ b/python/pyarrow/src_arrow/deserialize.h @@ -21,8 +21,8 @@ #include #include -#include "arrow/python/serialize.h" -#include "arrow/python/visibility.h" +#include "serialize.h" +#include "visibility.h" #include "arrow/status.h" namespace arrow { diff --git a/python/pyarrow/src_arrow/extension_type.cc b/python/pyarrow/src_arrow/extension_type.cc index 3ccc171c871..9fd2f2e1466 100644 --- a/python/pyarrow/src_arrow/extension_type.cc +++ b/python/pyarrow/src_arrow/extension_type.cc @@ -19,9 +19,9 @@ #include #include -#include "arrow/python/extension_type.h" -#include "arrow/python/helpers.h" -#include "arrow/python/pyarrow.h" +#include "extension_type.h" +#include "helpers.h" +#include "pyarrow.h" #include "arrow/util/checked_cast.h" #include "arrow/util/logging.h" diff --git a/python/pyarrow/src_arrow/extension_type.h b/python/pyarrow/src_arrow/extension_type.h index e433d9aca70..76ddb327733 100644 --- a/python/pyarrow/src_arrow/extension_type.h +++ b/python/pyarrow/src_arrow/extension_type.h @@ -21,8 +21,8 @@ #include #include "arrow/extension_type.h" -#include "arrow/python/common.h" -#include "arrow/python/visibility.h" +#include "common.h" +#include "visibility.h" #include "arrow/util/macros.h" namespace arrow { diff --git a/python/pyarrow/src_arrow/filesystem.cc b/python/pyarrow/src_arrow/filesystem.cc index 5e9b500a4f7..17ca732e073 100644 --- a/python/pyarrow/src_arrow/filesystem.cc +++ b/python/pyarrow/src_arrow/filesystem.cc @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "arrow/python/filesystem.h" +#include "filesystem.h" #include "arrow/util/logging.h" namespace arrow { diff --git a/python/pyarrow/src_arrow/filesystem.h b/python/pyarrow/src_arrow/filesystem.h index 003fd5cb805..993145b5327 100644 --- a/python/pyarrow/src_arrow/filesystem.h +++ b/python/pyarrow/src_arrow/filesystem.h @@ -22,8 +22,8 @@ #include #include "arrow/filesystem/filesystem.h" -#include "arrow/python/common.h" -#include "arrow/python/visibility.h" +#include "common.h" +#include "visibility.h" #include "arrow/util/macros.h" namespace arrow { diff --git a/python/pyarrow/src_arrow/flight.h b/python/pyarrow/src_arrow/flight.h index 5713b2e4b70..7e0136fcc3e 100644 --- a/python/pyarrow/src_arrow/flight.h +++ b/python/pyarrow/src_arrow/flight.h @@ -23,7 +23,7 @@ #include "arrow/flight/api.h" #include "arrow/ipc/dictionary.h" -#include "arrow/python/common.h" +#include "common.h" #if defined(_WIN32) || defined(__CYGWIN__) // Windows #if defined(_MSC_VER) diff --git a/python/pyarrow/src_arrow/gdb.cc b/python/pyarrow/src_arrow/gdb.cc index 944e1e96d71..297bc6dbffc 100644 --- a/python/pyarrow/src_arrow/gdb.cc +++ b/python/pyarrow/src_arrow/gdb.cc @@ -24,7 +24,7 @@ #include "arrow/datum.h" #include "arrow/extension_type.h" #include "arrow/ipc/json_simple.h" -#include "arrow/python/gdb.h" +#include "gdb.h" #include "arrow/record_batch.h" #include "arrow/scalar.h" #include "arrow/table.h" diff --git a/python/pyarrow/src_arrow/gdb.h b/python/pyarrow/src_arrow/gdb.h index 1ddcbb51f6e..b4296abe6dd 100644 --- a/python/pyarrow/src_arrow/gdb.h +++ b/python/pyarrow/src_arrow/gdb.h @@ -17,7 +17,7 @@ #pragma once -#include "arrow/python/visibility.h" +#include "visibility.h" namespace arrow { namespace gdb { diff --git a/python/pyarrow/src_arrow/helpers.cc b/python/pyarrow/src_arrow/helpers.cc index c266abc169d..73d7cd8dcfc 100644 --- a/python/pyarrow/src_arrow/helpers.cc +++ b/python/pyarrow/src_arrow/helpers.cc @@ -16,17 +16,17 @@ // under the License. // helpers.h includes a NumPy header, so we include this first -#include "arrow/python/numpy_interop.h" +#include "numpy_interop.h" -#include "arrow/python/helpers.h" +#include "helpers.h" #include #include #include #include -#include "arrow/python/common.h" -#include "arrow/python/decimal.h" +#include "common.h" +#include "decimal.h" #include "arrow/type_fwd.h" #include "arrow/util/checked_cast.h" #include "arrow/util/logging.h" diff --git a/python/pyarrow/src_arrow/helpers.h b/python/pyarrow/src_arrow/helpers.h index a8e5f80b606..089d1225dd6 100644 --- a/python/pyarrow/src_arrow/helpers.h +++ b/python/pyarrow/src_arrow/helpers.h @@ -17,18 +17,18 @@ #pragma once -#include "arrow/python/platform.h" +#include "platform.h" #include #include #include #include -#include "arrow/python/numpy_interop.h" +#include "numpy_interop.h" #include -#include "arrow/python/visibility.h" +#include "visibility.h" #include "arrow/type.h" #include "arrow/util/macros.h" diff --git a/python/pyarrow/src_arrow/inference.cc b/python/pyarrow/src_arrow/inference.cc index db5f0896a95..513b0bfdbbb 100644 --- a/python/pyarrow/src_arrow/inference.cc +++ b/python/pyarrow/src_arrow/inference.cc @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. -#include "arrow/python/inference.h" -#include "arrow/python/numpy_interop.h" +#include "inference.h" +#include "numpy_interop.h" #include @@ -31,11 +31,11 @@ #include "arrow/util/decimal.h" #include "arrow/util/logging.h" -#include "arrow/python/datetime.h" -#include "arrow/python/decimal.h" -#include "arrow/python/helpers.h" -#include "arrow/python/iterators.h" -#include "arrow/python/numpy_convert.h" +#include "datetime.h" +#include "decimal.h" +#include "helpers.h" +#include "iterators.h" +#include "numpy_convert.h" namespace arrow { namespace py { diff --git a/python/pyarrow/src_arrow/inference.h b/python/pyarrow/src_arrow/inference.h index eff18362934..24005dd96f1 100644 --- a/python/pyarrow/src_arrow/inference.h +++ b/python/pyarrow/src_arrow/inference.h @@ -20,15 +20,15 @@ #pragma once -#include "arrow/python/platform.h" +#include "platform.h" #include -#include "arrow/python/visibility.h" +#include "visibility.h" #include "arrow/type.h" #include "arrow/util/macros.h" -#include "arrow/python/common.h" +#include "common.h" namespace arrow { diff --git a/python/pyarrow/src_arrow/init.cc b/python/pyarrow/src_arrow/init.cc index dba293bbe23..f09c5cd12a5 100644 --- a/python/pyarrow/src_arrow/init.cc +++ b/python/pyarrow/src_arrow/init.cc @@ -18,7 +18,7 @@ // Trigger the array import (inversion of NO_IMPORT_ARRAY) #define NUMPY_IMPORT_ARRAY -#include "arrow/python/init.h" -#include "arrow/python/numpy_interop.h" +#include "init.h" +#include "numpy_interop.h" int arrow_init_numpy() { return arrow::py::import_numpy(); } diff --git a/python/pyarrow/src_arrow/init.h b/python/pyarrow/src_arrow/init.h index 2e6c954862b..eab467b631e 100644 --- a/python/pyarrow/src_arrow/init.h +++ b/python/pyarrow/src_arrow/init.h @@ -17,8 +17,8 @@ #pragma once -#include "arrow/python/platform.h" -#include "arrow/python/visibility.h" +#include "platform.h" +#include "visibility.h" extern "C" { ARROW_PYTHON_EXPORT diff --git a/python/pyarrow/src_arrow/io.cc b/python/pyarrow/src_arrow/io.cc index 73525feed38..173d84ff567 100644 --- a/python/pyarrow/src_arrow/io.cc +++ b/python/pyarrow/src_arrow/io.cc @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "arrow/python/io.h" +#include "io.h" #include #include @@ -28,8 +28,8 @@ #include "arrow/status.h" #include "arrow/util/logging.h" -#include "arrow/python/common.h" -#include "arrow/python/pyarrow.h" +#include "common.h" +#include "pyarrow.h" namespace arrow { diff --git a/python/pyarrow/src_arrow/io.h b/python/pyarrow/src_arrow/io.h index a38d0ca332c..53b15434ea6 100644 --- a/python/pyarrow/src_arrow/io.h +++ b/python/pyarrow/src_arrow/io.h @@ -22,8 +22,8 @@ #include "arrow/io/interfaces.h" #include "arrow/io/transform.h" -#include "arrow/python/common.h" -#include "arrow/python/visibility.h" +#include "common.h" +#include "visibility.h" namespace arrow { namespace py { diff --git a/python/pyarrow/src_arrow/ipc.cc b/python/pyarrow/src_arrow/ipc.cc index 2e6c9d91275..bed3da2d1ac 100644 --- a/python/pyarrow/src_arrow/ipc.cc +++ b/python/pyarrow/src_arrow/ipc.cc @@ -15,11 +15,11 @@ // specific language governing permissions and limitations // under the License. -#include "arrow/python/ipc.h" +#include "ipc.h" #include -#include "arrow/python/pyarrow.h" +#include "pyarrow.h" namespace arrow { namespace py { diff --git a/python/pyarrow/src_arrow/ipc.h b/python/pyarrow/src_arrow/ipc.h index 92232ed8300..38839af82fd 100644 --- a/python/pyarrow/src_arrow/ipc.h +++ b/python/pyarrow/src_arrow/ipc.h @@ -19,8 +19,8 @@ #include -#include "arrow/python/common.h" -#include "arrow/python/visibility.h" +#include "common.h" +#include "visibility.h" #include "arrow/record_batch.h" #include "arrow/result.h" #include "arrow/util/macros.h" diff --git a/python/pyarrow/src_arrow/iterators.h b/python/pyarrow/src_arrow/iterators.h index 7b31962dac5..d581adf52c9 100644 --- a/python/pyarrow/src_arrow/iterators.h +++ b/python/pyarrow/src_arrow/iterators.h @@ -21,8 +21,8 @@ #include "arrow/array/array_primitive.h" -#include "arrow/python/common.h" -#include "arrow/python/numpy_internal.h" +#include "common.h" +#include "numpy_internal.h" namespace arrow { namespace py { diff --git a/python/pyarrow/src_arrow/numpy_convert.cc b/python/pyarrow/src_arrow/numpy_convert.cc index 49706807644..d8e10605daa 100644 --- a/python/pyarrow/src_arrow/numpy_convert.cc +++ b/python/pyarrow/src_arrow/numpy_convert.cc @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -#include "arrow/python/numpy_interop.h" +#include "numpy_interop.h" -#include "arrow/python/numpy_convert.h" +#include "numpy_convert.h" #include #include @@ -30,9 +30,9 @@ #include "arrow/type.h" #include "arrow/util/logging.h" -#include "arrow/python/common.h" -#include "arrow/python/pyarrow.h" -#include "arrow/python/type_traits.h" +#include "common.h" +#include "pyarrow.h" +#include "type_traits.h" namespace arrow { namespace py { diff --git a/python/pyarrow/src_arrow/numpy_convert.h b/python/pyarrow/src_arrow/numpy_convert.h index 10451077a22..d2772dcdcdf 100644 --- a/python/pyarrow/src_arrow/numpy_convert.h +++ b/python/pyarrow/src_arrow/numpy_convert.h @@ -20,14 +20,14 @@ #pragma once -#include "arrow/python/platform.h" +#include "platform.h" #include #include #include #include "arrow/buffer.h" -#include "arrow/python/visibility.h" +#include "visibility.h" #include "arrow/sparse_tensor.h" namespace arrow { diff --git a/python/pyarrow/src_arrow/numpy_internal.h b/python/pyarrow/src_arrow/numpy_internal.h index b9b632f9f9a..d408e908442 100644 --- a/python/pyarrow/src_arrow/numpy_internal.h +++ b/python/pyarrow/src_arrow/numpy_internal.h @@ -19,11 +19,11 @@ #pragma once -#include "arrow/python/numpy_interop.h" +#include "numpy_interop.h" #include "arrow/status.h" -#include "arrow/python/platform.h" +#include "platform.h" #include #include diff --git a/python/pyarrow/src_arrow/numpy_interop.h b/python/pyarrow/src_arrow/numpy_interop.h index ce7baed259f..d212e014ec6 100644 --- a/python/pyarrow/src_arrow/numpy_interop.h +++ b/python/pyarrow/src_arrow/numpy_interop.h @@ -17,7 +17,7 @@ #pragma once -#include "arrow/python/platform.h" // IWYU pragma: export +#include "platform.h" // IWYU pragma: export #include // IWYU pragma: export diff --git a/python/pyarrow/src_arrow/numpy_to_arrow.cc b/python/pyarrow/src_arrow/numpy_to_arrow.cc index 2727ce32f44..cd01577d71d 100644 --- a/python/pyarrow/src_arrow/numpy_to_arrow.cc +++ b/python/pyarrow/src_arrow/numpy_to_arrow.cc @@ -17,8 +17,8 @@ // Functions for pandas conversion via NumPy -#include "arrow/python/numpy_to_arrow.h" -#include "arrow/python/numpy_interop.h" +#include "numpy_to_arrow.h" +#include "numpy_interop.h" #include #include @@ -49,14 +49,14 @@ #include "arrow/compute/api_scalar.h" -#include "arrow/python/common.h" -#include "arrow/python/datetime.h" -#include "arrow/python/helpers.h" -#include "arrow/python/iterators.h" -#include "arrow/python/numpy_convert.h" -#include "arrow/python/numpy_internal.h" -#include "arrow/python/python_to_arrow.h" -#include "arrow/python/type_traits.h" +#include "common.h" +#include "datetime.h" +#include "helpers.h" +#include "iterators.h" +#include "numpy_convert.h" +#include "numpy_internal.h" +#include "python_to_arrow.h" +#include "type_traits.h" namespace arrow { diff --git a/python/pyarrow/src_arrow/numpy_to_arrow.h b/python/pyarrow/src_arrow/numpy_to_arrow.h index b6cd093e554..13924b1ac1c 100644 --- a/python/pyarrow/src_arrow/numpy_to_arrow.h +++ b/python/pyarrow/src_arrow/numpy_to_arrow.h @@ -19,12 +19,12 @@ #pragma once -#include "arrow/python/platform.h" +#include "platform.h" #include #include "arrow/compute/api.h" -#include "arrow/python/visibility.h" +#include "visibility.h" namespace arrow { diff --git a/python/pyarrow/src_arrow/parquet_encryption.cc b/python/pyarrow/src_arrow/parquet_encryption.cc index a5f924bce78..b64c77f2b9f 100644 --- a/python/pyarrow/src_arrow/parquet_encryption.cc +++ b/python/pyarrow/src_arrow/parquet_encryption.cc @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "arrow/python/parquet_encryption.h" +#include "parquet_encryption.h" #include "parquet/exception.h" namespace arrow { diff --git a/python/pyarrow/src_arrow/parquet_encryption.h b/python/pyarrow/src_arrow/parquet_encryption.h index 23ee478348e..0079b6eed56 100644 --- a/python/pyarrow/src_arrow/parquet_encryption.h +++ b/python/pyarrow/src_arrow/parquet_encryption.h @@ -19,8 +19,8 @@ #include -#include "arrow/python/common.h" -#include "arrow/python/visibility.h" +#include "common.h" +#include "visibility.h" #include "arrow/util/macros.h" #include "parquet/encryption/crypto_factory.h" #include "parquet/encryption/kms_client.h" diff --git a/python/pyarrow/src_arrow/pch.h b/python/pyarrow/src_arrow/pch.h index d1d688b4f17..0afcf7938df 100644 --- a/python/pyarrow/src_arrow/pch.h +++ b/python/pyarrow/src_arrow/pch.h @@ -21,4 +21,4 @@ // may incur a slowdown, since it makes the precompiled header heavier to load. #include "arrow/pch.h" -#include "arrow/python/platform.h" +#include "platform.h" diff --git a/python/pyarrow/src_arrow/pyarrow.cc b/python/pyarrow/src_arrow/pyarrow.cc index c3244b74bf5..b567a68df3e 100644 --- a/python/pyarrow/src_arrow/pyarrow.cc +++ b/python/pyarrow/src_arrow/pyarrow.cc @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "arrow/python/pyarrow.h" +#include "pyarrow.h" #include #include @@ -25,10 +25,10 @@ #include "arrow/tensor.h" #include "arrow/type.h" -#include "arrow/python/common.h" -#include "arrow/python/datetime.h" +#include "common.h" +#include "datetime.h" namespace { -#include "arrow/python/pyarrow_api.h" +#include "pyarrow_api.h" } namespace arrow { diff --git a/python/pyarrow/src_arrow/pyarrow.h b/python/pyarrow/src_arrow/pyarrow.h index 4c365081d70..a63e29a33fb 100644 --- a/python/pyarrow/src_arrow/pyarrow.h +++ b/python/pyarrow/src_arrow/pyarrow.h @@ -17,11 +17,11 @@ #pragma once -#include "arrow/python/platform.h" +#include "platform.h" #include -#include "arrow/python/visibility.h" +#include "visibility.h" #include "arrow/sparse_tensor.h" diff --git a/python/pyarrow/src_arrow/python_test.cc b/python/pyarrow/src_arrow/python_test.cc index c465fabc680..54086faa7ca 100644 --- a/python/pyarrow/src_arrow/python_test.cc +++ b/python/pyarrow/src_arrow/python_test.cc @@ -21,7 +21,7 @@ #include #include -#include "arrow/python/platform.h" +#include "platform.h" #include "arrow/array.h" #include "arrow/array/builder_binary.h" @@ -30,12 +30,12 @@ #include "arrow/util/decimal.h" #include "arrow/util/optional.h" -#include "arrow/python/arrow_to_pandas.h" -#include "arrow/python/decimal.h" -#include "arrow/python/helpers.h" -#include "arrow/python/numpy_convert.h" -#include "arrow/python/numpy_interop.h" -#include "arrow/python/python_to_arrow.h" +#include "arrow_to_pandas.h" +#include "decimal.h" +#include "helpers.h" +#include "numpy_convert.h" +#include "numpy_interop.h" +#include "python_to_arrow.h" #include "arrow/util/checked_cast.h" #include "arrow/util/logging.h" diff --git a/python/pyarrow/src_arrow/python_to_arrow.cc b/python/pyarrow/src_arrow/python_to_arrow.cc index 7a94407d2d9..4ca19049a73 100644 --- a/python/pyarrow/src_arrow/python_to_arrow.cc +++ b/python/pyarrow/src_arrow/python_to_arrow.cc @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. -#include "arrow/python/python_to_arrow.h" -#include "arrow/python/numpy_interop.h" +#include "python_to_arrow.h" +#include "numpy_interop.h" #include @@ -44,13 +44,13 @@ #include "arrow/util/int_util_overflow.h" #include "arrow/util/logging.h" -#include "arrow/python/datetime.h" -#include "arrow/python/decimal.h" -#include "arrow/python/helpers.h" -#include "arrow/python/inference.h" -#include "arrow/python/iterators.h" -#include "arrow/python/numpy_convert.h" -#include "arrow/python/type_traits.h" +#include "datetime.h" +#include "decimal.h" +#include "helpers.h" +#include "inference.h" +#include "iterators.h" +#include "numpy_convert.h" +#include "type_traits.h" #include "arrow/visit_type_inline.h" namespace arrow { diff --git a/python/pyarrow/src_arrow/python_to_arrow.h b/python/pyarrow/src_arrow/python_to_arrow.h index d167996ba8d..ca246a87401 100644 --- a/python/pyarrow/src_arrow/python_to_arrow.h +++ b/python/pyarrow/src_arrow/python_to_arrow.h @@ -20,16 +20,16 @@ #pragma once -#include "arrow/python/platform.h" +#include "platform.h" #include #include -#include "arrow/python/visibility.h" +#include "visibility.h" #include "arrow/type.h" #include "arrow/util/macros.h" -#include "arrow/python/common.h" +#include "common.h" namespace arrow { diff --git a/python/pyarrow/src_arrow/serialize.cc b/python/pyarrow/src_arrow/serialize.cc index ad079cbd9c7..c7c925fa198 100644 --- a/python/pyarrow/src_arrow/serialize.cc +++ b/python/pyarrow/src_arrow/serialize.cc @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. -#include "arrow/python/serialize.h" -#include "arrow/python/numpy_interop.h" +#include "serialize.h" +#include "numpy_interop.h" #include #include @@ -42,13 +42,13 @@ #include "arrow/tensor.h" #include "arrow/util/logging.h" -#include "arrow/python/common.h" -#include "arrow/python/datetime.h" -#include "arrow/python/helpers.h" -#include "arrow/python/iterators.h" -#include "arrow/python/numpy_convert.h" -#include "arrow/python/platform.h" -#include "arrow/python/pyarrow.h" +#include "common.h" +#include "datetime.h" +#include "helpers.h" +#include "iterators.h" +#include "numpy_convert.h" +#include "platform.h" +#include "pyarrow.h" constexpr int32_t kMaxRecursionDepth = 100; diff --git a/python/pyarrow/src_arrow/serialize.h b/python/pyarrow/src_arrow/serialize.h index fd207d3e069..fd78c9a309a 100644 --- a/python/pyarrow/src_arrow/serialize.h +++ b/python/pyarrow/src_arrow/serialize.h @@ -21,7 +21,7 @@ #include #include "arrow/ipc/options.h" -#include "arrow/python/visibility.h" +#include "visibility.h" #include "arrow/sparse_tensor.h" #include "arrow/status.h" diff --git a/python/pyarrow/src_arrow/type_traits.h b/python/pyarrow/src_arrow/type_traits.h index a941577f765..4cdfe9d8d62 100644 --- a/python/pyarrow/src_arrow/type_traits.h +++ b/python/pyarrow/src_arrow/type_traits.h @@ -19,12 +19,12 @@ #pragma once -#include "arrow/python/platform.h" +#include "platform.h" #include #include -#include "arrow/python/numpy_interop.h" +#include "numpy_interop.h" #include diff --git a/python/pyarrow/src_arrow/udf.cc b/python/pyarrow/src_arrow/udf.cc index 81bf47c0ade..51cea5e6c64 100644 --- a/python/pyarrow/src_arrow/udf.cc +++ b/python/pyarrow/src_arrow/udf.cc @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -#include "arrow/python/udf.h" +#include "udf.h" #include "arrow/compute/function.h" -#include "arrow/python/common.h" +#include "common.h" namespace arrow { diff --git a/python/pyarrow/src_arrow/udf.h b/python/pyarrow/src_arrow/udf.h index 4ab3e7cc72b..52f22b4cb4f 100644 --- a/python/pyarrow/src_arrow/udf.h +++ b/python/pyarrow/src_arrow/udf.h @@ -20,11 +20,11 @@ #include "arrow/compute/exec.h" #include "arrow/compute/function.h" #include "arrow/compute/registry.h" -#include "arrow/python/platform.h" +#include "platform.h" -#include "arrow/python/common.h" -#include "arrow/python/pyarrow.h" -#include "arrow/python/visibility.h" +#include "common.h" +#include "pyarrow.h" +#include "visibility.h" namespace arrow { diff --git a/python/pyarrow/src_arrow/util/test_main.cc b/python/pyarrow/src_arrow/util/test_main.cc index dd7f379bdd4..5781ca0b671 100644 --- a/python/pyarrow/src_arrow/util/test_main.cc +++ b/python/pyarrow/src_arrow/util/test_main.cc @@ -15,13 +15,13 @@ // specific language governing permissions and limitations // under the License. -#include "arrow/python/platform.h" +#include "platform.h" #include -#include "arrow/python/datetime.h" -#include "arrow/python/init.h" -#include "arrow/python/pyarrow.h" +#include "datetime.h" +#include "init.h" +#include "pyarrow.h" int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); diff --git a/python/pyarrow/tensorflow/plasma_op.cc b/python/pyarrow/tensorflow/plasma_op.cc index bf4eec78912..6445ba75e5e 100644 --- a/python/pyarrow/tensorflow/plasma_op.cc +++ b/python/pyarrow/tensorflow/plasma_op.cc @@ -36,8 +36,8 @@ #include "arrow/util/logging.h" // These headers do not include Python.h -#include "arrow/python/deserialize.h" -#include "arrow/python/serialize.h" +#include "deserialize.h" +#include "serialize.h" #include "plasma/client.h" From dd97026ee2bbc0439f8b09127d7b7cc4f7fac800 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Fri, 3 Jun 2022 15:24:02 +0200 Subject: [PATCH 003/138] Add cmake and build for C PyArrow to setup.py --- cpp/cmake_modules/FindArrowPython.cmake | 5 +-- python/setup.py | 43 +++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/cpp/cmake_modules/FindArrowPython.cmake b/cpp/cmake_modules/FindArrowPython.cmake index b503e6a9e02..15d15def782 100644 --- a/cpp/cmake_modules/FindArrowPython.cmake +++ b/cpp/cmake_modules/FindArrowPython.cmake @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -# - Find Arrow Python (arrow/python/api.h, libarrow_python.a, libarrow_python.so) +# - Find Arrow Python (python/pyarrrow/src_arrow/api.h, libarrow_python.a, libarrow_python.so) # # This module requires Arrow from which it uses # arrow_find_package() @@ -46,8 +46,9 @@ endif() find_package(Arrow ${find_package_arguments}) if(ARROW_FOUND) + message(STATUS "CPYARROW_HOME: ${CPYARROW_HOME}") arrow_find_package(ARROW_PYTHON - "${ARROW_HOME}" + "${CPYARROW_HOME}" arrow_python arrow/python/api.h ArrowPython diff --git a/python/setup.py b/python/setup.py index 9fd7c480088..2da1ce01f7e 100755 --- a/python/setup.py +++ b/python/setup.py @@ -93,6 +93,7 @@ def build_extensions(self): _build_ext.build_extensions(self) def run(self): + self._run_cmake_cpyarrow() self._run_cmake() _build_ext.run(self) @@ -227,6 +228,44 @@ def initialize_options(self): '_hdfsio', 'gandiva'] + def _run_cmake_cpyarrow(self): + # check if build_type is correctly passed / set + if self.build_type.lower() not in ('release', 'debug'): + raise ValueError("--build-type (or PYARROW_BUILD_TYPE) needs to " + "be 'release' or 'debug'") + + # The directory containing this setup.py + source = os.path.dirname(os.path.abspath(__file__)) + # The directory containing this C PyArrow CMakeLists.txt + source_cpyarrow = pjoin(source, "pyarrow/src_arrow") + + # The directory for the module being built + saved_cwd = os.getcwd() + build_lib = pjoin(saved_cwd, 'build/dist/lib') + build_include = pjoin(saved_cwd, 'build/dist/include') + + if not os.path.isdir(build_lib): + self.mkpath(build_lib) + if not os.path.isdir(build_include): + self.mkpath(build_include) + + with changed_dir(build_lib): + # cmake args + cmake_options = [ + '-DCMAKE_INSTALL_PREFIX=' + str(pjoin(saved_cwd, 'build/dist')), + '-DCMAKE_BUILD_TYPE={0}'.format(self.build_type.lower()), + ] + + # run cmake + print("-- Running cmake for C pyarrow") + self.spawn(['cmake'] + cmake_options + [source_cpyarrow]) + print("-- Finished cmake for C pyarrow") + # run make & install + print("-- Running make build and install for C pyarrow") + self.spawn(['make', '-j4']) + self.spawn(['make', 'install']) + print("-- Finished make build and install for C pyarrow") + def _run_cmake(self): # check if build_type is correctly passed / set if self.build_type.lower() not in ('release', 'debug'): @@ -242,6 +281,9 @@ def _run_cmake(self): build_lib = pjoin(os.getcwd(), build_cmd.build_lib) saved_cwd = os.getcwd() + # The directory containing C PyArrow headers and libs + build_dist = pjoin(saved_cwd, 'build/dist') + if not os.path.isdir(build_temp): self.mkpath(build_temp) @@ -266,6 +308,7 @@ def _run_cmake(self): cmake_options = [ '-DPYTHON_EXECUTABLE=%s' % sys.executable, '-DPython3_EXECUTABLE=%s' % sys.executable, + '-DCPYARROW_HOME=' + str(build_dist), static_lib_option, ] From a1127906a428ec0f0cc3fdf8dd4f1c900b04b98a Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Mon, 6 Jun 2022 10:35:23 +0200 Subject: [PATCH 004/138] Fix typos --- python/pyarrow/src_arrow/CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index ff34014f412..6ef9fec0d67 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -100,7 +100,7 @@ if(PARQUET_REQUIRE_ENCRYPTION) endif() # Link to arrow dependecies -if(ARROW_LINK_SHARED) +if(ARROW_BUILD_SHARED) set(ARROW_PYTHON_DEPENDENCIES arrow_shared) else() set(THREADS_PREFER_PTHREAD_FLAG ON) @@ -144,14 +144,14 @@ if(CMAKE_VERSION VERSION_LESS 3.18) else() include(CheckLinkerFlag) check_linker_flag(CXX - "-Wl,--version-script=${ARROW_SOURCE_DIR}/cpp/src/arrow/symbols.map" + "-Wl,--version-script=${ARROW_SOURCE_DIR}/src/arrow/symbols.map" CXX_LINKER_SUPPORTS_VERSION_SCRIPT) endif() # Need to set ARROW_VERSION_SCRIPT_FLAGS for add_arrow_lib if(CXX_LINKER_SUPPORTS_VERSION_SCRIPT) set(ARROW_VERSION_SCRIPT_FLAGS - "-Wl,--version-script=${ARROW_SOURCE_DIR}/cpp/src/arrow/symbols.map") + "-Wl,--version-script=${ARROW_SOURCE_DIR}/src/arrow/symbols.map") endif() # From f02eae2f6925ec9b4a7ac7b688b0412201c202a4 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Mon, 6 Jun 2022 14:33:00 +0200 Subject: [PATCH 005/138] Linter correction --- python/setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/setup.py b/python/setup.py index 2da1ce01f7e..1233908082a 100755 --- a/python/setup.py +++ b/python/setup.py @@ -252,7 +252,8 @@ def _run_cmake_cpyarrow(self): with changed_dir(build_lib): # cmake args cmake_options = [ - '-DCMAKE_INSTALL_PREFIX=' + str(pjoin(saved_cwd, 'build/dist')), + '-DCMAKE_INSTALL_PREFIX=' + + str(pjoin(saved_cwd, 'build/dist')), '-DCMAKE_BUILD_TYPE={0}'.format(self.build_type.lower()), ] From b2df1d51e0930bb94e4e0c3ab5906dc5e2444a53 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Wed, 8 Jun 2022 11:39:59 +0200 Subject: [PATCH 006/138] Change the dir from where C PyArrow is being built --- python/setup.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python/setup.py b/python/setup.py index 1233908082a..38063e13e7c 100755 --- a/python/setup.py +++ b/python/setup.py @@ -241,15 +241,19 @@ def _run_cmake_cpyarrow(self): # The directory for the module being built saved_cwd = os.getcwd() + build_temp = pjoin(saved_cwd, 'build/dist/temp') build_lib = pjoin(saved_cwd, 'build/dist/lib') build_include = pjoin(saved_cwd, 'build/dist/include') + if not os.path.isdir(build_temp): + self.mkpath(build_temp) if not os.path.isdir(build_lib): self.mkpath(build_lib) if not os.path.isdir(build_include): self.mkpath(build_include) - with changed_dir(build_lib): + # Change to the build directory + with changed_dir(build_temp): # cmake args cmake_options = [ '-DCMAKE_INSTALL_PREFIX=' + From fed81a6c389e28bad037955acf35993712444eb4 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Wed, 8 Jun 2022 12:10:33 +0200 Subject: [PATCH 007/138] Comment out ARROW_PYTHON from cpp/src/arrow/public_api_test.cc --- cpp/src/arrow/public_api_test.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/src/arrow/public_api_test.cc b/cpp/src/arrow/public_api_test.cc index 5a607c53ad9..309d2b77f96 100644 --- a/cpp/src/arrow/public_api_test.cc +++ b/cpp/src/arrow/public_api_test.cc @@ -50,9 +50,9 @@ #include "arrow/json/api.h" // IWYU pragma: keep #endif -#ifdef ARROW_PYTHON -#include "api.h" // IWYU pragma: keep -#endif +// #ifdef ARROW_PYTHON +// #include "arrow/python/api.h" // IWYU pragma: keep +// #endif #ifdef DCHECK #error "DCHECK should not be visible from Arrow public headers." From 62bbaab8954d604b35cc2f0e371e1b23e3d57f58 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Tue, 14 Jun 2022 12:58:42 +0200 Subject: [PATCH 008/138] hacky experiment --- python/CMakeLists.txt | 38 ++++++++++++++++++-------------------- python/setup.py | 24 +++++++++++++++++++++++- 2 files changed, 41 insertions(+), 21 deletions(-) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index a657f56bb2d..eed0e9800b1 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -604,29 +604,27 @@ foreach(module ${CYTHON_EXTENSIONS}) ${module_output_directory}) endif() - if(PYARROW_BUNDLE_ARROW_CPP) - # In the event that we are bundling the shared libraries (e.g. in a - # manylinux1 wheel), we need to set the RPATH of the extensions to the - # root of the pyarrow/ package so that libarrow/libarrow_python are able - # to be loaded properly - if(APPLE) - set(module_install_rpath "@loader_path/") - else() - set(module_install_rpath "\$ORIGIN") - endif() + # In the event that we are bundling the shared libraries (e.g. in a + # manylinux1 wheel), we need to set the RPATH of the extensions to the + # root of the pyarrow/ package so that libarrow/libarrow_python are able + # to be loaded properly + if(APPLE) + set(module_install_rpath "@loader_path/") + else() + set(module_install_rpath "\$ORIGIN") + endif() - # XXX(wesm): ARROW-2326 this logic is only needed when we have Cython - # modules in interior directories. Since all of our C extensions and - # bundled libraries are in the same place, we can skip this part + # XXX(wesm): ARROW-2326 this logic is only needed when we have Cython + # modules in interior directories. Since all of our C extensions and + # bundled libraries are in the same place, we can skip this part - # list(LENGTH directories i) - # while(${i} GREATER 0) - # set(module_install_rpath "${module_install_rpath}/..") - # math(EXPR i "${i} - 1" ) - # endwhile(${i} GREATER 0) + # list(LENGTH directories i) + # while(${i} GREATER 0) + # set(module_install_rpath "${module_install_rpath}/..") + # math(EXPR i "${i} - 1" ) + # endwhile(${i} GREATER 0) - set_target_properties(${module_name} PROPERTIES INSTALL_RPATH ${module_install_rpath}) - endif() + set_target_properties(${module_name} PROPERTIES INSTALL_RPATH ${module_install_rpath}) if(PYARROW_GENERATE_COVERAGE) set_target_properties(${module_name} PROPERTIES COMPILE_DEFINITIONS diff --git a/python/setup.py b/python/setup.py index 38063e13e7c..ef004bfde4d 100755 --- a/python/setup.py +++ b/python/setup.py @@ -271,6 +271,24 @@ def _run_cmake_cpyarrow(self): self.spawn(['make', 'install']) print("-- Finished make build and install for C pyarrow") + if self.inplace: + # a bit hacky + build_lib = saved_cwd + + # Move the libraries to the place expected by the Python build + try: + os.makedirs(pjoin(build_lib, 'pyarrow')) + except OSError: + pass + + print(f"moving {build_temp} to {build_lib}") + #shutil.move(build_temp, pjoin(build_lib, "pyarrow")) + # a bit hacky + shutil.move(pjoin(build_temp, "libarrow_python.so"), pjoin(build_lib, "pyarrow")) + shutil.move(pjoin(build_temp, "libarrow_python.so.900"), pjoin(build_lib, "pyarrow")) + shutil.move(pjoin(build_temp, "libarrow_python.so.900.0.0"), pjoin(build_lib, "pyarrow")) + shutil.move(pjoin(build_include, "arrow", "python"), pjoin(build_lib, "pyarrow", "include", "arrow")) + def _run_cmake(self): # check if build_type is correctly passed / set if self.build_type.lower() not in ('release', 'debug'): @@ -292,6 +310,10 @@ def _run_cmake(self): if not os.path.isdir(build_temp): self.mkpath(build_temp) + if self.inplace: + # a bit hacky + build_lib = saved_cwd + # Change to the build directory with changed_dir(build_temp): # Detect if we built elsewhere @@ -313,7 +335,7 @@ def _run_cmake(self): cmake_options = [ '-DPYTHON_EXECUTABLE=%s' % sys.executable, '-DPython3_EXECUTABLE=%s' % sys.executable, - '-DCPYARROW_HOME=' + str(build_dist), + '-DCPYARROW_HOME=' + str(pjoin(build_lib, "pyarrow")), static_lib_option, ] From 23fd4c3e6a78c66bb494f93bfdac2984d0c8ee8f Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Tue, 14 Jun 2022 15:34:16 +0200 Subject: [PATCH 009/138] correct moving include files --- python/setup.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/python/setup.py b/python/setup.py index ef004bfde4d..edfa5144dd6 100755 --- a/python/setup.py +++ b/python/setup.py @@ -284,10 +284,14 @@ def _run_cmake_cpyarrow(self): print(f"moving {build_temp} to {build_lib}") #shutil.move(build_temp, pjoin(build_lib, "pyarrow")) # a bit hacky - shutil.move(pjoin(build_temp, "libarrow_python.so"), pjoin(build_lib, "pyarrow")) - shutil.move(pjoin(build_temp, "libarrow_python.so.900"), pjoin(build_lib, "pyarrow")) - shutil.move(pjoin(build_temp, "libarrow_python.so.900.0.0"), pjoin(build_lib, "pyarrow")) - shutil.move(pjoin(build_include, "arrow", "python"), pjoin(build_lib, "pyarrow", "include", "arrow")) + for libname in ["libarrow_python.so", "libarrow_python.so.900", "libarrow_python.so.900.0.0"]: + libname_path = pjoin(build_lib, "pyarrow", libname) + if os.path.exists(libname_path): + os.remove(libname_path) + shutil.move(pjoin(build_temp, libname), pjoin(build_lib, "pyarrow")) + if not os.path.isdir(pjoin(build_include, "arrow")): + self.mkpath(pjoin(build_include, "arrow")) + shutil.move(pjoin(build_include, "arrow", "python"), pjoin(build_lib, "pyarrow", "include", "arrow", "python")) def _run_cmake(self): # check if build_type is correctly passed / set From 98a95fce5ffa6525daa8aa5f24f9ea9e06bda372 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Wed, 15 Jun 2022 08:15:26 +0200 Subject: [PATCH 010/138] Make hacky experiment work for MacOS --- python/pyarrow/src_arrow/CMakeLists.txt | 3 +++ python/setup.py | 11 ++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index 6ef9fec0d67..08e8f7e9624 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -223,6 +223,9 @@ if(ARROW_FLIGHT AND ARROW_BUILD_SHARED) # fail with weird errors due to multiple copies of global static state (The # other solution is to link gRPC shared everywhere instead of statically only # in Flight) + + + add_arrow_lib(arrow_python_flight CMAKE_PACKAGE_NAME ArrowPythonFlight diff --git a/python/setup.py b/python/setup.py index edfa5144dd6..c2f519edf7d 100755 --- a/python/setup.py +++ b/python/setup.py @@ -284,11 +284,12 @@ def _run_cmake_cpyarrow(self): print(f"moving {build_temp} to {build_lib}") #shutil.move(build_temp, pjoin(build_lib, "pyarrow")) # a bit hacky - for libname in ["libarrow_python.so", "libarrow_python.so.900", "libarrow_python.so.900.0.0"]: - libname_path = pjoin(build_lib, "pyarrow", libname) - if os.path.exists(libname_path): - os.remove(libname_path) - shutil.move(pjoin(build_temp, libname), pjoin(build_lib, "pyarrow")) + for libname in os.listdir(build_temp): + if "libarrow_python" in libname: + libname_path = pjoin(build_lib, "pyarrow", libname) + if os.path.exists(libname_path): + os.remove(libname_path) + shutil.move(pjoin(build_temp, libname), pjoin(build_lib, "pyarrow")) if not os.path.isdir(pjoin(build_include, "arrow")): self.mkpath(pjoin(build_include, "arrow")) shutil.move(pjoin(build_include, "arrow", "python"), pjoin(build_lib, "pyarrow", "include", "arrow", "python")) From ed2b259286c4bdd51716a9c5e9f283923081c9e2 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 16 Jun 2022 15:58:09 +0200 Subject: [PATCH 011/138] Correct setup.py when pyarrow not build inplace --- python/setup.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/python/setup.py b/python/setup.py index c2f519edf7d..810ddf0fe0a 100755 --- a/python/setup.py +++ b/python/setup.py @@ -240,10 +240,15 @@ def _run_cmake_cpyarrow(self): source_cpyarrow = pjoin(source, "pyarrow/src_arrow") # The directory for the module being built + build_cmd = self.get_finalized_command('build') saved_cwd = os.getcwd() build_temp = pjoin(saved_cwd, 'build/dist/temp') - build_lib = pjoin(saved_cwd, 'build/dist/lib') build_include = pjoin(saved_cwd, 'build/dist/include') + build_lib = pjoin(os.getcwd(), build_cmd.build_lib) + + if self.inplace: + # a bit hacky + build_lib = saved_cwd if not os.path.isdir(build_temp): self.mkpath(build_temp) @@ -271,10 +276,6 @@ def _run_cmake_cpyarrow(self): self.spawn(['make', 'install']) print("-- Finished make build and install for C pyarrow") - if self.inplace: - # a bit hacky - build_lib = saved_cwd - # Move the libraries to the place expected by the Python build try: os.makedirs(pjoin(build_lib, 'pyarrow')) From 8ada8ee7ac89aa94bdb2fba6e31755c0aca4def7 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 16 Jun 2022 17:47:14 +0200 Subject: [PATCH 012/138] Use a pattern to define C pyarrow library version --- python/pyarrow/src_arrow/CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index 08e8f7e9624..5439080420a 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -36,8 +36,10 @@ get_filename_component(ARROW_SOURCE_DIR ${PYTHON_SOURCE_DIR} DIRECTORY) set(ARROW_SOURCE_DIR "${ARROW_SOURCE_DIR}/cpp") set(CMAKE_MODULE_PATH "${PYTHON_SOURCE_DIR}/cmake_modules") +set(ARROW_PYTHON_VERSION "9.0.0-SNAPSHOT") +string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" ARROW_PYTHON_BASE_VERSION "${ARROW_PYTHON_VERSION}") # Need to set to ARRROW_VERSION before finding Arrow package! -project(arrow_python VERSION 9.0.0) +project(arrow_python VERSION "${ARROW_PYTHON_BASE_VERSION}") if(NOT DEFINED CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release) From d2695a91c98c3ef678d0b8291793252cc625c70a Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Wed, 22 Jun 2022 10:37:53 +0200 Subject: [PATCH 013/138] Add partial changes to python_flight build --- cpp/cmake_modules/FindArrowPythonFlight.cmake | 4 ++-- python/pyarrow/src_arrow/CMakeLists.txt | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cpp/cmake_modules/FindArrowPythonFlight.cmake b/cpp/cmake_modules/FindArrowPythonFlight.cmake index 3a639928ce5..679f80d54b1 100644 --- a/cpp/cmake_modules/FindArrowPythonFlight.cmake +++ b/cpp/cmake_modules/FindArrowPythonFlight.cmake @@ -16,7 +16,7 @@ # under the License. # - Find Arrow Python Flight -# (arrow/python/flight.h, libarrow_python_flight.a, libarrow_python_flight.so) +# (python/pyarrrow/src_arrow/flight.h, libarrow_python_flight.a, libarrow_python_flight.so) # # This module requires Arrow from which it uses # arrow_find_package() @@ -50,7 +50,7 @@ find_package(ArrowPython ${find_package_arguments}) if(ARROW_PYTHON_FOUND AND ARROW_FLIGHT_FOUND) arrow_find_package(ARROW_PYTHON_FLIGHT - "${ARROW_HOME}" + "${CPYARROW_HOME}" arrow_python_flight arrow/python/flight.h ArrowPythonFlight diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index 5439080420a..3417ae918d3 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -219,6 +219,8 @@ if(ARROW_BUILD_STATIC AND MSVC) target_compile_definitions(arrow_python_static PUBLIC ARROW_STATIC) endif() +set(ARROW_FLIGHT ON) + if(ARROW_FLIGHT AND ARROW_BUILD_SHARED) # Must link to shared libarrow_flight: we don't want to link more than one # copy of gRPC into the eventual Cython shared object, otherwise gRPC calls @@ -226,7 +228,8 @@ if(ARROW_FLIGHT AND ARROW_BUILD_SHARED) # other solution is to link gRPC shared everywhere instead of statically only # in Flight) - + add_custom_target(flight_grpc_gen ALL DEPENDS ${FLIGHT_GENERATED_PROTO_FILES}) + include_directories(${ARROW_SOURCE_DIR}/src) add_arrow_lib(arrow_python_flight CMAKE_PACKAGE_NAME From d27e103120b75d8d3344b507b6468b9de3b6372d Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Wed, 22 Jun 2022 14:09:05 +0200 Subject: [PATCH 014/138] Correct linter and add gRPC setup for python flight --- python/pyarrow/src_arrow/CMakeLists.txt | 41 ++++++++++++++++++++----- python/setup.py | 10 +++--- 2 files changed, 40 insertions(+), 11 deletions(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index 3417ae918d3..32e18bc9fbb 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -32,8 +32,8 @@ endif() # CMAKE_MODULE_PATH: location of cmake_modules in python get_filename_component(PYARROW_SOURCE_DIR ${CMAKE_SOURCE_DIR} DIRECTORY) get_filename_component(PYTHON_SOURCE_DIR ${PYARROW_SOURCE_DIR} DIRECTORY) -get_filename_component(ARROW_SOURCE_DIR ${PYTHON_SOURCE_DIR} DIRECTORY) -set(ARROW_SOURCE_DIR "${ARROW_SOURCE_DIR}/cpp") +get_filename_component(ARROW_SOURCE ${PYTHON_SOURCE_DIR} DIRECTORY) +set(ARROW_SOURCE_DIR "${ARROW_SOURCE}/cpp") set(CMAKE_MODULE_PATH "${PYTHON_SOURCE_DIR}/cmake_modules") set(ARROW_PYTHON_VERSION "9.0.0-SNAPSHOT") @@ -222,11 +222,38 @@ endif() set(ARROW_FLIGHT ON) if(ARROW_FLIGHT AND ARROW_BUILD_SHARED) - # Must link to shared libarrow_flight: we don't want to link more than one - # copy of gRPC into the eventual Cython shared object, otherwise gRPC calls - # fail with weird errors due to multiple copies of global static state (The - # other solution is to link gRPC shared everywhere instead of statically only - # in Flight) + + find_package(gRPC CONFIG REQUIRED) + set(ARROW_FLIGHT_LINK_LIBS arrow_python_flight_static) + + if(APPLE) + set(GRPC_REFLECTION_LINK_LIBS -Wl,-force_load gRPC::grpc++_reflection) + else() + set(GRPC_REFLECTION_LINK_LIBS -Wl,--whole-archive gRPC::grpc++_reflection + -Wl,--no-whole-archive) + endif() + + set(FLIGHT_PROTO_PATH "${ARROW_SOURCE}/format") + set(FLIGHT_PROTO "${ARROW_SOURCE}/format/Flight.proto") + + set(FLIGHT_GENERATED_PROTO_FILES + "${ARROW_SOURCE_DIR}/build/src/arrow/flight/Flight.pb.cc" + "${ARROW_SOURCE_DIR}/build/src/arrow/flight/Flight.pb.h" + "${ARROW_SOURCE_DIR}/build/src/arrow/flight/Flight.grpc.pb.cc" + "${ARROW_SOURCE_DIR}/build/src/arrow/flight/Flight.grpc.pb.h") + + set(PROTO_DEPENDS ${FLIGHT_PROTO} gRPC::grpc_cpp_plugin) + + add_custom_command(OUTPUT ${FLIGHT_GENERATED_PROTO_FILES} + COMMAND ${ARROW_PROTOBUF_PROTOC} "-I${FLIGHT_PROTO_PATH}" + "--cpp_out=${${ARROW_SOURCE_DIR}/build/src/arrow/flight/}" "${FLIGHT_PROTO}" + DEPENDS ${PROTO_DEPENDS} ARGS + COMMAND ${ARROW_PROTOBUF_PROTOC} "-I${FLIGHT_PROTO_PATH}" + "--grpc_out=${${ARROW_SOURCE_DIR}/build/src/arrow/flight/}" + "--plugin=protoc-gen-grpc=$" + "${FLIGHT_PROTO}") + + set_source_files_properties(${FLIGHT_GENERATED_PROTO_FILES} PROPERTIES GENERATED TRUE) add_custom_target(flight_grpc_gen ALL DEPENDS ${FLIGHT_GENERATED_PROTO_FILES}) include_directories(${ARROW_SOURCE_DIR}/src) diff --git a/python/setup.py b/python/setup.py index 810ddf0fe0a..785976d7451 100755 --- a/python/setup.py +++ b/python/setup.py @@ -281,19 +281,21 @@ def _run_cmake_cpyarrow(self): os.makedirs(pjoin(build_lib, 'pyarrow')) except OSError: pass - + print(f"moving {build_temp} to {build_lib}") - #shutil.move(build_temp, pjoin(build_lib, "pyarrow")) + # shutil.move(build_temp, pjoin(build_lib, "pyarrow")) # a bit hacky for libname in os.listdir(build_temp): if "libarrow_python" in libname: libname_path = pjoin(build_lib, "pyarrow", libname) if os.path.exists(libname_path): os.remove(libname_path) - shutil.move(pjoin(build_temp, libname), pjoin(build_lib, "pyarrow")) + shutil.move(pjoin(build_temp, libname), + pjoin(build_lib, "pyarrow")) if not os.path.isdir(pjoin(build_include, "arrow")): self.mkpath(pjoin(build_include, "arrow")) - shutil.move(pjoin(build_include, "arrow", "python"), pjoin(build_lib, "pyarrow", "include", "arrow", "python")) + shutil.move(pjoin(build_include, "arrow", "python"), pjoin( + build_lib, "pyarrow", "include", "arrow", "python")) def _run_cmake(self): # check if build_type is correctly passed / set From 73e2011c199de0c78e3f79c0fac7a795f68951cd Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Wed, 22 Jun 2022 14:45:45 +0200 Subject: [PATCH 015/138] Add more changes for python flight - still not working --- python/pyarrow/src_arrow/CMakeLists.txt | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index 32e18bc9fbb..ded55bc84b9 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -224,14 +224,11 @@ set(ARROW_FLIGHT ON) if(ARROW_FLIGHT AND ARROW_BUILD_SHARED) find_package(gRPC CONFIG REQUIRED) - set(ARROW_FLIGHT_LINK_LIBS arrow_python_flight_static) - if(APPLE) - set(GRPC_REFLECTION_LINK_LIBS -Wl,-force_load gRPC::grpc++_reflection) - else() - set(GRPC_REFLECTION_LINK_LIBS -Wl,--whole-archive gRPC::grpc++_reflection - -Wl,--no-whole-archive) - endif() + # We don't directly use symbols from the reflection library, so + # ensure the linker still links to it + set(GRPC_REFLECTION_LINK_LIBS -Wl,--no-as-needed gRPC::grpc++_reflection + -Wl,--as-needed) set(FLIGHT_PROTO_PATH "${ARROW_SOURCE}/format") set(FLIGHT_PROTO "${ARROW_SOURCE}/format/Flight.proto") @@ -256,7 +253,7 @@ if(ARROW_FLIGHT AND ARROW_BUILD_SHARED) set_source_files_properties(${FLIGHT_GENERATED_PROTO_FILES} PROPERTIES GENERATED TRUE) add_custom_target(flight_grpc_gen ALL DEPENDS ${FLIGHT_GENERATED_PROTO_FILES}) - include_directories(${ARROW_SOURCE_DIR}/src) + include_directories(${ARROW_SOURCE_DIR}/src ${ARROW_SOURCE_DIR}/build/src) add_arrow_lib(arrow_python_flight CMAKE_PACKAGE_NAME From fddf6ac9f79d6db6f0f5082ec03bc9dd2aacf8a3 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Mon, 27 Jun 2022 06:12:27 +0200 Subject: [PATCH 016/138] Add small changes to c pyarrow CMakeLists - pythin flight build now works locally! Huray --- python/pyarrow/src_arrow/CMakeLists.txt | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index ded55bc84b9..914f778b411 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -223,12 +223,15 @@ set(ARROW_FLIGHT ON) if(ARROW_FLIGHT AND ARROW_BUILD_SHARED) + find_package(ArrowFlight REQUIRED) find_package(gRPC CONFIG REQUIRED) + include_directories("${ARROW_SOURCE_DIR}/src" "${ARROW_SOURCE_DIR}/build/src") - # We don't directly use symbols from the reflection library, so - # ensure the linker still links to it + # Using arrow_flight_shared, see example + # https://github.com/apache/arrow/blob/7a0f00c16e084d194ae53d209b33b809cfc8f2d5/cpp/examples/arrow/CMakeLists.txt + set(ARROW_GRPC_USE_SHARED ON) set(GRPC_REFLECTION_LINK_LIBS -Wl,--no-as-needed gRPC::grpc++_reflection - -Wl,--as-needed) + -Wl,--as-needed) set(FLIGHT_PROTO_PATH "${ARROW_SOURCE}/format") set(FLIGHT_PROTO "${ARROW_SOURCE}/format/Flight.proto") @@ -251,9 +254,7 @@ if(ARROW_FLIGHT AND ARROW_BUILD_SHARED) "${FLIGHT_PROTO}") set_source_files_properties(${FLIGHT_GENERATED_PROTO_FILES} PROPERTIES GENERATED TRUE) - add_custom_target(flight_grpc_gen ALL DEPENDS ${FLIGHT_GENERATED_PROTO_FILES}) - include_directories(${ARROW_SOURCE_DIR}/src ${ARROW_SOURCE_DIR}/build/src) add_arrow_lib(arrow_python_flight CMAKE_PACKAGE_NAME From d86e8cb884f1e0e58fab5f606927f8def8deaa5a Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Mon, 27 Jun 2022 09:08:57 +0200 Subject: [PATCH 017/138] Handle options for Arrow and C PyArrow --- python/pyarrow/src_arrow/CMakeLists.txt | 40 +++++++++++++++++-------- python/setup.py | 12 ++++++++ 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index 914f778b411..ce356abb49b 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -34,7 +34,11 @@ get_filename_component(PYARROW_SOURCE_DIR ${CMAKE_SOURCE_DIR} DIRECTORY) get_filename_component(PYTHON_SOURCE_DIR ${PYARROW_SOURCE_DIR} DIRECTORY) get_filename_component(ARROW_SOURCE ${PYTHON_SOURCE_DIR} DIRECTORY) set(ARROW_SOURCE_DIR "${ARROW_SOURCE}/cpp") -set(CMAKE_MODULE_PATH "${PYTHON_SOURCE_DIR}/cmake_modules") +# Add cmake_modules and lib/cmake/arrow folder to CMAKE_MODULE_PATH +# TODO: remove lib/cmake/arrow folder as it should not be necessary as find_package(Arrow) +# should load all the cmake variables (ex ARROW_DATASET) +set(CMAKE_MODULE_PATH "${PYTHON_SOURCE_DIR}/cmake_modules" "${ARROW_SOURCE}/dist/lib/cmake/arrow") +message(STATUS "CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH}") set(ARROW_PYTHON_VERSION "9.0.0-SNAPSHOT") string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" ARROW_PYTHON_BASE_VERSION "${ARROW_PYTHON_VERSION}") @@ -46,6 +50,8 @@ if(NOT DEFINED CMAKE_BUILD_TYPE) endif() find_package(Arrow REQUIRED) +# Loading ArrowOptions, they should be loaded by default with find_package(Arrow REQUIRED) +include(ArrowOptions) find_package(Python3Alt 3.7 REQUIRED) # Needed due to @@ -81,13 +87,14 @@ set(ARROW_PYTHON_SRCS set_source_files_properties(init.cc PROPERTIES SKIP_PRECOMPILE_HEADERS ON SKIP_UNITY_BUILD_INCLUSION ON) -# Set necessary components on (from arrow/cpp/CMakeLists.txt, end of the file) -set(ARROW_COMPUTE ON) -set(ARROW_CSV ON) -set(ARROW_DATASET ON) -set(ARROW_FILESYSTEM ON) -set(ARROW_HDFS ON) -set(ARROW_JSON ON) +# +# Arrow vs C PyArrow options +# + +# Check all the options from Arrow and C PyArrow to be in line +if(PYARROW_WITH_DATASET) + find_package(ArrowDataset REQUIRED) +endif() if(ARROW_CSV) list(APPEND ARROW_PYTHON_SRCS csv.cc) @@ -97,8 +104,18 @@ if(ARROW_FILESYSTEM) list(APPEND ARROW_PYTHON_SRCS filesystem.cc) endif() -if(PARQUET_REQUIRE_ENCRYPTION) - list(APPEND ARROW_PYTHON_SRCS parquet_encryption.cc) +if(PYARROW_WITH_PARQUET_ENCRYPTION) + if(PARQUET_REQUIRE_ENCRYPTION) + list(APPEND ARROW_PYTHON_SRCS parquet_encryption.cc) + else() + message(FATAL_ERROR "You must build Arrow C++ with PARQUET_REQUIRE_ENCRYPTION=ON") + endif() +endif() + +if(PYARROW_WITH_HDFS) + if(NOT ARROW_HDFS) + message(FATAL_ERROR "You must build Arrow C++ with ARROW_HDFS=ON") + endif() endif() # Link to arrow dependecies @@ -219,8 +236,6 @@ if(ARROW_BUILD_STATIC AND MSVC) target_compile_definitions(arrow_python_static PUBLIC ARROW_STATIC) endif() -set(ARROW_FLIGHT ON) - if(ARROW_FLIGHT AND ARROW_BUILD_SHARED) find_package(ArrowFlight REQUIRED) @@ -301,6 +316,7 @@ arrow_install_all_headers("arrow/python") # ---------------------------------------------------------------------- +set(ARROW_BUILD_TESTS off) if(ARROW_BUILD_TESTS) add_library(arrow_python_test_main STATIC util/test_main.cc) diff --git a/python/setup.py b/python/setup.py index 785976d7451..42c16af0d45 100755 --- a/python/setup.py +++ b/python/setup.py @@ -264,8 +264,20 @@ def _run_cmake_cpyarrow(self): '-DCMAKE_INSTALL_PREFIX=' + str(pjoin(saved_cwd, 'build/dist')), '-DCMAKE_BUILD_TYPE={0}'.format(self.build_type.lower()), + '' ] + # Check for specific options + def append_cmake_bool(value, varname): + cmake_options.append('-D{0}={1}'.format( + varname, 'on' if value else 'off')) + + append_cmake_bool(self.with_dataset, 'PYARROW_WITH_DATASET') + append_cmake_bool(self.with_parquet_encryption, + 'PYARROW_WITH_PARQUET_ENCRYPTION') + append_cmake_bool(self.with_hdfs, + 'PYARROW_WITH_HDFS') + # run cmake print("-- Running cmake for C pyarrow") self.spawn(['cmake'] + cmake_options + [source_cpyarrow]) From 865dfbd44d201c0d7b977c59042eb2ab38285b29 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Mon, 27 Jun 2022 10:11:00 +0200 Subject: [PATCH 018/138] Remove build_dist from setup.py _run_cmake() --- python/setup.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/python/setup.py b/python/setup.py index 42c16af0d45..9c217df5c06 100755 --- a/python/setup.py +++ b/python/setup.py @@ -324,9 +324,6 @@ def _run_cmake(self): build_lib = pjoin(os.getcwd(), build_cmd.build_lib) saved_cwd = os.getcwd() - # The directory containing C PyArrow headers and libs - build_dist = pjoin(saved_cwd, 'build/dist') - if not os.path.isdir(build_temp): self.mkpath(build_temp) From 947ac70cbc2dfed14609aa6d5aa565a08798ec81 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 28 Jun 2022 08:33:11 +0200 Subject: [PATCH 019/138] Try to load ArrowOptions in general with the help of ARROW_HOME env var --- python/pyarrow/src_arrow/CMakeLists.txt | 6 +----- python/setup.py | 1 - 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index ce356abb49b..127a306b4d4 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -34,10 +34,7 @@ get_filename_component(PYARROW_SOURCE_DIR ${CMAKE_SOURCE_DIR} DIRECTORY) get_filename_component(PYTHON_SOURCE_DIR ${PYARROW_SOURCE_DIR} DIRECTORY) get_filename_component(ARROW_SOURCE ${PYTHON_SOURCE_DIR} DIRECTORY) set(ARROW_SOURCE_DIR "${ARROW_SOURCE}/cpp") -# Add cmake_modules and lib/cmake/arrow folder to CMAKE_MODULE_PATH -# TODO: remove lib/cmake/arrow folder as it should not be necessary as find_package(Arrow) -# should load all the cmake variables (ex ARROW_DATASET) -set(CMAKE_MODULE_PATH "${PYTHON_SOURCE_DIR}/cmake_modules" "${ARROW_SOURCE}/dist/lib/cmake/arrow") +set(CMAKE_MODULE_PATH "${PYTHON_SOURCE_DIR}/cmake_modules" "$ENV{ARROW_HOME}/lib/cmake/arrow") message(STATUS "CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH}") set(ARROW_PYTHON_VERSION "9.0.0-SNAPSHOT") @@ -50,7 +47,6 @@ if(NOT DEFINED CMAKE_BUILD_TYPE) endif() find_package(Arrow REQUIRED) -# Loading ArrowOptions, they should be loaded by default with find_package(Arrow REQUIRED) include(ArrowOptions) find_package(Python3Alt 3.7 REQUIRED) diff --git a/python/setup.py b/python/setup.py index 9c217df5c06..c975fd60378 100755 --- a/python/setup.py +++ b/python/setup.py @@ -264,7 +264,6 @@ def _run_cmake_cpyarrow(self): '-DCMAKE_INSTALL_PREFIX=' + str(pjoin(saved_cwd, 'build/dist')), '-DCMAKE_BUILD_TYPE={0}'.format(self.build_type.lower()), - '' ] # Check for specific options From 94f285191f5a2adfd46a1a9b839c54c3ddad401e Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Wed, 29 Jun 2022 09:09:42 +0200 Subject: [PATCH 020/138] Add Parquet module if PARQUET_REQUIRE_ENCRYPTION=on --- python/pyarrow/src_arrow/CMakeLists.txt | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index 127a306b4d4..2a86d74b10a 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -92,17 +92,10 @@ if(PYARROW_WITH_DATASET) find_package(ArrowDataset REQUIRED) endif() -if(ARROW_CSV) - list(APPEND ARROW_PYTHON_SRCS csv.cc) -endif() - -if(ARROW_FILESYSTEM) - list(APPEND ARROW_PYTHON_SRCS filesystem.cc) -endif() - if(PYARROW_WITH_PARQUET_ENCRYPTION) if(PARQUET_REQUIRE_ENCRYPTION) list(APPEND ARROW_PYTHON_SRCS parquet_encryption.cc) + find_package(Parquet REQUIRED) else() message(FATAL_ERROR "You must build Arrow C++ with PARQUET_REQUIRE_ENCRYPTION=ON") endif() @@ -114,6 +107,15 @@ if(PYARROW_WITH_HDFS) endif() endif() +# Check for only Arrow C++ options +if(ARROW_CSV) + list(APPEND ARROW_PYTHON_SRCS csv.cc) +endif() + +if(ARROW_FILESYSTEM) + list(APPEND ARROW_PYTHON_SRCS filesystem.cc) +endif() + # Link to arrow dependecies if(ARROW_BUILD_SHARED) set(ARROW_PYTHON_DEPENDENCIES arrow_shared) From 1ea8e207c3b7dd64c3169c656e3d06a9a07b0849 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 30 Jun 2022 11:39:33 +0200 Subject: [PATCH 021/138] Build tests for C PyArrow --- python/pyarrow/src_arrow/CMakeLists.txt | 135 +++++++++++++++++++-- python/pyarrow/src_arrow/util/test_main.cc | 8 +- 2 files changed, 127 insertions(+), 16 deletions(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index 2a86d74b10a..2c339ba8194 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -37,6 +37,10 @@ set(ARROW_SOURCE_DIR "${ARROW_SOURCE}/cpp") set(CMAKE_MODULE_PATH "${PYTHON_SOURCE_DIR}/cmake_modules" "$ENV{ARROW_HOME}/lib/cmake/arrow") message(STATUS "CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH}") +# +# Arrow version +# + set(ARROW_PYTHON_VERSION "9.0.0-SNAPSHOT") string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" ARROW_PYTHON_BASE_VERSION "${ARROW_PYTHON_VERSION}") # Need to set to ARRROW_VERSION before finding Arrow package! @@ -46,6 +50,10 @@ if(NOT DEFINED CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release) endif() +# +# Arrow +# + find_package(Arrow REQUIRED) include(ArrowOptions) find_package(Python3Alt 3.7 REQUIRED) @@ -314,8 +322,68 @@ arrow_install_all_headers("arrow/python") # ---------------------------------------------------------------------- -set(ARROW_BUILD_TESTS off) if(ARROW_BUILD_TESTS) + enable_testing() + + # Set necessary paths for cmake to find GTest + set(GTEST_ROOT ${ARROW_SOURCE_DIR}/build/googletest_ep-prefix) + set(GTEST_INCLUDE_DIR "${GTEST_ROOT}/include") + set(GTEST_LIBRARY ${GTEST_ROOT}/lib) + set(GTEST_MAIN_LIBRARY ${GTEST_ROOT}/lib) + + # + # Taken from Matlab CMakeLists.txt (enable_gtest and build_gtest) + # + + set(ARROW_GTEST_PREFIX "${GTEST_ROOT}") + set(ARROW_GTEST_MAIN_PREFIX "${GTEST_ROOT}") + + if(WIN32) + set(ARROW_GTEST_SHARED_LIB_DIR "${ARROW_GTEST_PREFIX}/bin") + set(ARROW_GTEST_MAIN_SHARED_LIB_DIR "${ARROW_GTEST_MAIN_PREFIX}/bin") + + set(ARROW_GTEST_LINK_LIB_DIR "${ARROW_GTEST_PREFIX}/lib") + set(ARROW_GTEST_LINK_LIB + "${ARROW_GTEST_LINK_LIB_DIR}/${CMAKE_IMPORT_LIBRARY_PREFIX}gtestd${CMAKE_IMPORT_LIBRARY_SUFFIX}" + ) + + set(ARROW_GTEST_MAIN_LINK_LIB_DIR "${ARROW_GTEST_MAIN_PREFIX}/lib") + set(ARROW_GTEST_MAIN_LINK_LIB + "${ARROW_GTEST_MAIN_LINK_LIB_DIR}/${CMAKE_IMPORT_LIBRARY_PREFIX}gtest_maind${CMAKE_IMPORT_LIBRARY_SUFFIX}" + ) + else() + set(ARROW_GTEST_SHARED_LIB_DIR "${ARROW_GTEST_PREFIX}/lib") + set(ARROW_GTEST_MAIN_SHARED_LIB_DIR "${ARROW_GTEST_MAIN_PREFIX}/lib") + endif() + + set(ARROW_GTEST_INCLUDE_DIR "${ARROW_GTEST_PREFIX}/include") + set(ARROW_GTEST_SHARED_LIB + "${ARROW_GTEST_SHARED_LIB_DIR}/${CMAKE_SHARED_LIBRARY_PREFIX}gtestd${CMAKE_SHARED_LIBRARY_SUFFIX}" + ) + + set(ARROW_GTEST_MAIN_INCLUDE_DIR "${ARROW_GTEST_MAIN_PREFIX}/include") + set(ARROW_GTEST_MAIN_SHARED_LIB + "${ARROW_GTEST_MAIN_SHARED_LIB_DIR}/${CMAKE_SHARED_LIBRARY_PREFIX}gtest_maind${CMAKE_SHARED_LIBRARY_SUFFIX}" + ) + + file(MAKE_DIRECTORY "${ARROW_GTEST_INCLUDE_DIR}") + + # Create target GTest::gtest + add_library(GTest::gtest SHARED IMPORTED) + set_target_properties(GTest::gtest + PROPERTIES IMPORTED_LOCATION ${ARROW_GTEST_SHARED_LIB} + INTERFACE_INCLUDE_DIRECTORIES + ${ARROW_GTEST_INCLUDE_DIR}) + if(WIN32) + set_target_properties(GTest::gtest PROPERTIES IMPORTED_IMPLIB ${ARROW_GTEST_LINK_LIB}) + endif() + + # ArrowTesting + # needed to be able to use arrow_testing_shared target + find_package(ArrowTesting REQUIRED) + + add_custom_target(all-tests) + add_library(arrow_python_test_main STATIC util/test_main.cc) target_link_libraries(arrow_python_test_main GTest::gtest) @@ -338,14 +406,57 @@ if(ARROW_BUILD_TESTS) arrow_testing_static arrow_static) endif() - add_arrow_test(python_test - STATIC_LINK_LIBS - "${ARROW_PYTHON_TEST_LINK_LIBS}" - EXTRA_LINK_LIBS - ${PYTHON_LIBRARIES} - EXTRA_INCLUDES - "${ARROW_PYTHON_INCLUDES}" - LABELS - "arrow_python-tests" - NO_VALGRIND) -endif() + # + # Add a test case + # + + set(REL_TEST_NAME "python_test") + get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE) + set(TEST_NAME "arrow-${TEST_NAME}") + set(SOURCES "${REL_TEST_NAME}.cc") + + # # Make sure the executable name contains only hyphens, not underscores + string(REPLACE "_" "-" TEST_NAME ${TEST_NAME}) + + set(TEST_PATH "${CMAKE_BINARY_DIR}/${TEST_NAME}") + add_executable(${TEST_NAME} ${SOURCES}) + + # With OSX and conda, we need to set the correct RPATH so that dependencies + # are found. The installed libraries with conda have an RPATH that matches + # for executables and libraries lying in $ENV{CONDA_PREFIX}/bin or + # $ENV{CONDA_PREFIX}/lib but our test libraries and executables are not + # installed there. + if(NOT "$ENV{CONDA_PREFIX}" STREQUAL "" AND APPLE) + set_target_properties(${TEST_NAME} + PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE + INSTALL_RPATH_USE_LINK_PATH TRUE + INSTALL_RPATH + "${PYTHON_SOURCE_DIR}/pyarrow;$ENV{CONDA_PREFIX}/lib") + endif() + + # Customize link libraries + target_link_libraries(${TEST_NAME} PRIVATE "${ARROW_PYTHON_TEST_LINK_LIBS}") + # Extra link libs + target_link_libraries(${TEST_NAME} PRIVATE ${PYTHON_LIBRARIES}) + # Extra includes + target_include_directories(${TEST_NAME} SYSTEM PUBLIC "${ARROW_PYTHON_INCLUDES}") + + # Add the test + if(ARROW_TEST_MEMCHECK) + add_test(${TEST_NAME} + bash + -c + "cd '${CMAKE_SOURCE_DIR}'; \ + valgrind --suppressions=valgrind.supp --tool=memcheck --gen-suppressions=all \ + --num-callers=500 --leak-check=full --leak-check-heuristics=stdstring \ + --error-exitcode=1 ${TEST_PATH} ${ARG_TEST_ARGUMENTS}") + elseif(WIN32) + add_test(${TEST_NAME} ${TEST_PATH}) + else() + add_test(${TEST_NAME} + ${ARROW_SOURCE_DIR}/build-support/run-test.sh + ${CMAKE_BINARY_DIR} + test + ${TEST_PATH}) + endif() +endif() \ No newline at end of file diff --git a/python/pyarrow/src_arrow/util/test_main.cc b/python/pyarrow/src_arrow/util/test_main.cc index 5781ca0b671..3ee1657e644 100644 --- a/python/pyarrow/src_arrow/util/test_main.cc +++ b/python/pyarrow/src_arrow/util/test_main.cc @@ -15,13 +15,13 @@ // specific language governing permissions and limitations // under the License. -#include "platform.h" +#include "../platform.h" #include -#include "datetime.h" -#include "init.h" -#include "pyarrow.h" +#include "../datetime.h" +#include "../init.h" +#include "../pyarrow.h" int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); From fe804d3bf6e545ebc426e60b429c9c6977522af8 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 30 Jun 2022 12:15:28 +0200 Subject: [PATCH 022/138] Try to fix appveyor CI error --- python/pyarrow/src_arrow/CMakeLists.txt | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index 2c339ba8194..77ea618980f 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -27,15 +27,20 @@ if(POLICY CMP0068) cmake_policy(SET CMP0068 NEW) endif() +# # Define # ARROW_SOURCE_DIR: location of arrow/cpp # CMAKE_MODULE_PATH: location of cmake_modules in python +# + get_filename_component(PYARROW_SOURCE_DIR ${CMAKE_SOURCE_DIR} DIRECTORY) get_filename_component(PYTHON_SOURCE_DIR ${PYARROW_SOURCE_DIR} DIRECTORY) get_filename_component(ARROW_SOURCE ${PYTHON_SOURCE_DIR} DIRECTORY) set(ARROW_SOURCE_DIR "${ARROW_SOURCE}/cpp") -set(CMAKE_MODULE_PATH "${PYTHON_SOURCE_DIR}/cmake_modules" "$ENV{ARROW_HOME}/lib/cmake/arrow") -message(STATUS "CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH}") + +# normalize ARROW_HOME path +cmake_path(CONVERT "$ENV{ARROW_HOME}" TO_CMAKE_PATH_LIST ARROW_HOME) +set(CMAKE_MODULE_PATH "${PYTHON_SOURCE_DIR}/cmake_modules" "${ARROW_HOME}/lib/cmake/arrow") # # Arrow version From 1c3a8a14826bc61d94029211e631eede9f36efe7 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Fri, 1 Jul 2022 09:49:28 +0200 Subject: [PATCH 023/138] Comments and names changes --- cpp/cmake_modules/FindArrowPython.cmake | 2 +- cpp/src/arrow/public_api_test.cc | 4 ---- python/pyarrow/src_arrow/CMakeLists.txt | 25 +++++++++++++------------ 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/cpp/cmake_modules/FindArrowPython.cmake b/cpp/cmake_modules/FindArrowPython.cmake index 15d15def782..1d8c7d3f40e 100644 --- a/cpp/cmake_modules/FindArrowPython.cmake +++ b/cpp/cmake_modules/FindArrowPython.cmake @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -# - Find Arrow Python (python/pyarrrow/src_arrow/api.h, libarrow_python.a, libarrow_python.so) +# - Find Arrow Python (arrow/python/api.h, libarrow_python.a, libarrow_python.so) # # This module requires Arrow from which it uses # arrow_find_package() diff --git a/cpp/src/arrow/public_api_test.cc b/cpp/src/arrow/public_api_test.cc index 309d2b77f96..a2aa624d092 100644 --- a/cpp/src/arrow/public_api_test.cc +++ b/cpp/src/arrow/public_api_test.cc @@ -50,10 +50,6 @@ #include "arrow/json/api.h" // IWYU pragma: keep #endif -// #ifdef ARROW_PYTHON -// #include "arrow/python/api.h" // IWYU pragma: keep -// #endif - #ifdef DCHECK #error "DCHECK should not be visible from Arrow public headers." #endif diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index 77ea618980f..aea05ff7a59 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -36,7 +36,8 @@ endif() get_filename_component(PYARROW_SOURCE_DIR ${CMAKE_SOURCE_DIR} DIRECTORY) get_filename_component(PYTHON_SOURCE_DIR ${PYARROW_SOURCE_DIR} DIRECTORY) get_filename_component(ARROW_SOURCE ${PYTHON_SOURCE_DIR} DIRECTORY) -set(ARROW_SOURCE_DIR "${ARROW_SOURCE}/cpp") +set(ARROW_CPP_SOURCE_DIR "${ARROW_SOURCE}/cpp") +set(ARROW_SOURCE_DIR "${ARROW_CPP_SOURCE_DIR}") # normalize ARROW_HOME path cmake_path(CONVERT "$ENV{ARROW_HOME}" TO_CMAKE_PATH_LIST ARROW_HOME) @@ -174,14 +175,14 @@ if(CMAKE_VERSION VERSION_LESS 3.18) else() include(CheckLinkerFlag) check_linker_flag(CXX - "-Wl,--version-script=${ARROW_SOURCE_DIR}/src/arrow/symbols.map" + "-Wl,--version-script=${ARROW_CPP_SOURCE_DIR}/src/arrow/symbols.map" CXX_LINKER_SUPPORTS_VERSION_SCRIPT) endif() # Need to set ARROW_VERSION_SCRIPT_FLAGS for add_arrow_lib if(CXX_LINKER_SUPPORTS_VERSION_SCRIPT) set(ARROW_VERSION_SCRIPT_FLAGS - "-Wl,--version-script=${ARROW_SOURCE_DIR}/src/arrow/symbols.map") + "-Wl,--version-script=${ARROW_CPP_SOURCE_DIR}/src/arrow/symbols.map") endif() # @@ -251,7 +252,7 @@ if(ARROW_FLIGHT AND ARROW_BUILD_SHARED) find_package(ArrowFlight REQUIRED) find_package(gRPC CONFIG REQUIRED) - include_directories("${ARROW_SOURCE_DIR}/src" "${ARROW_SOURCE_DIR}/build/src") + include_directories("${ARROW_CPP_SOURCE_DIR}/src" "${ARROW_CPP_SOURCE_DIR}/build/src") # Using arrow_flight_shared, see example # https://github.com/apache/arrow/blob/7a0f00c16e084d194ae53d209b33b809cfc8f2d5/cpp/examples/arrow/CMakeLists.txt @@ -263,19 +264,19 @@ if(ARROW_FLIGHT AND ARROW_BUILD_SHARED) set(FLIGHT_PROTO "${ARROW_SOURCE}/format/Flight.proto") set(FLIGHT_GENERATED_PROTO_FILES - "${ARROW_SOURCE_DIR}/build/src/arrow/flight/Flight.pb.cc" - "${ARROW_SOURCE_DIR}/build/src/arrow/flight/Flight.pb.h" - "${ARROW_SOURCE_DIR}/build/src/arrow/flight/Flight.grpc.pb.cc" - "${ARROW_SOURCE_DIR}/build/src/arrow/flight/Flight.grpc.pb.h") + "${ARROW_CPP_SOURCE_DIR}/build/src/arrow/flight/Flight.pb.cc" + "${ARROW_CPP_SOURCE_DIR}/build/src/arrow/flight/Flight.pb.h" + "${ARROW_CPP_SOURCE_DIR}/build/src/arrow/flight/Flight.grpc.pb.cc" + "${ARROW_CPP_SOURCE_DIR}/build/src/arrow/flight/Flight.grpc.pb.h") set(PROTO_DEPENDS ${FLIGHT_PROTO} gRPC::grpc_cpp_plugin) add_custom_command(OUTPUT ${FLIGHT_GENERATED_PROTO_FILES} COMMAND ${ARROW_PROTOBUF_PROTOC} "-I${FLIGHT_PROTO_PATH}" - "--cpp_out=${${ARROW_SOURCE_DIR}/build/src/arrow/flight/}" "${FLIGHT_PROTO}" + "--cpp_out=${ARROW_CPP_SOURCE_DIR}/build/src/arrow/flight/" "${FLIGHT_PROTO}" DEPENDS ${PROTO_DEPENDS} ARGS COMMAND ${ARROW_PROTOBUF_PROTOC} "-I${FLIGHT_PROTO_PATH}" - "--grpc_out=${${ARROW_SOURCE_DIR}/build/src/arrow/flight/}" + "--grpc_out=${ARROW_CPP_SOURCE_DIR}/build/src/arrow/flight/" "--plugin=protoc-gen-grpc=$" "${FLIGHT_PROTO}") @@ -331,7 +332,7 @@ if(ARROW_BUILD_TESTS) enable_testing() # Set necessary paths for cmake to find GTest - set(GTEST_ROOT ${ARROW_SOURCE_DIR}/build/googletest_ep-prefix) + set(GTEST_ROOT ${ARROW_CPP_SOURCE_DIR}/build/googletest_ep-prefix) set(GTEST_INCLUDE_DIR "${GTEST_ROOT}/include") set(GTEST_LIBRARY ${GTEST_ROOT}/lib) set(GTEST_MAIN_LIBRARY ${GTEST_ROOT}/lib) @@ -459,7 +460,7 @@ if(ARROW_BUILD_TESTS) add_test(${TEST_NAME} ${TEST_PATH}) else() add_test(${TEST_NAME} - ${ARROW_SOURCE_DIR}/build-support/run-test.sh + ${ARROW_CPP_SOURCE_DIR}/build-support/run-test.sh ${CMAKE_BINARY_DIR} test ${TEST_PATH}) From fce199ecd9bd2fa73005df5fe571c4a1ee94559e Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Fri, 1 Jul 2022 10:40:40 +0200 Subject: [PATCH 024/138] Remove comment for ARROW_VERSION_SCRIPT_FLAGS --- python/pyarrow/src_arrow/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index aea05ff7a59..bf2b065e09c 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -228,7 +228,7 @@ add_arrow_lib(arrow_python DEPENDENCIES ${ARROW_PYTHON_DEPENDENCIES} SHARED_LINK_FLAGS - ${ARROW_VERSION_SCRIPT_FLAGS} # Defined in line 95 + ${ARROW_VERSION_SCRIPT_FLAGS} SHARED_LINK_LIBS ${ARROW_PYTHON_SHARED_LINK_LIBS} SHARED_PRIVATE_LINK_LIBS @@ -295,7 +295,7 @@ if(ARROW_FLIGHT AND ARROW_BUILD_SHARED) DEPENDENCIES flight_grpc_gen SHARED_LINK_FLAGS - ${ARROW_VERSION_SCRIPT_FLAGS} # Defined in line 95 + ${ARROW_VERSION_SCRIPT_FLAGS} SHARED_LINK_LIBS arrow_python_shared arrow_flight_shared From 7d2a55dfd8c460dac038756069062779181a94f2 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Fri, 1 Jul 2022 10:59:42 +0200 Subject: [PATCH 025/138] Change _run_cmake_cpyarrow() to _run_cmake_arrow_python() --- python/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/setup.py b/python/setup.py index c975fd60378..0f5e891b154 100755 --- a/python/setup.py +++ b/python/setup.py @@ -93,7 +93,7 @@ def build_extensions(self): _build_ext.build_extensions(self) def run(self): - self._run_cmake_cpyarrow() + self._run_cmake_arrow_python() self._run_cmake() _build_ext.run(self) @@ -228,7 +228,7 @@ def initialize_options(self): '_hdfsio', 'gandiva'] - def _run_cmake_cpyarrow(self): + def _run_cmake_arrow_python(self): # check if build_type is correctly passed / set if self.build_type.lower() not in ('release', 'debug'): raise ValueError("--build-type (or PYARROW_BUILD_TYPE) needs to " From 9c3a62ea7d76e5f13a86cb63bf9dcd5f7cdd9dde Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Fri, 1 Jul 2022 11:05:27 +0200 Subject: [PATCH 026/138] Change the use of cmake_path with file --- python/pyarrow/src_arrow/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index bf2b065e09c..3a0d870f4b1 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -40,7 +40,7 @@ set(ARROW_CPP_SOURCE_DIR "${ARROW_SOURCE}/cpp") set(ARROW_SOURCE_DIR "${ARROW_CPP_SOURCE_DIR}") # normalize ARROW_HOME path -cmake_path(CONVERT "$ENV{ARROW_HOME}" TO_CMAKE_PATH_LIST ARROW_HOME) +file(TO_CMAKE_PATH "$ENV{ARROW_HOME}" ARROW_HOME) set(CMAKE_MODULE_PATH "${PYTHON_SOURCE_DIR}/cmake_modules" "${ARROW_HOME}/lib/cmake/arrow") # From ea45ed425d65a1c46469a7fb45bcb29f2f02cacd Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Fri, 1 Jul 2022 11:15:47 +0200 Subject: [PATCH 027/138] Accept ARROW_BUILD_DIR by the users to indicate build dir for Arrow C++ --- python/pyarrow/src_arrow/CMakeLists.txt | 15 ++++++++------- python/setup.py | 5 ++++- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index 3a0d870f4b1..56f1d5b0663 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -264,19 +264,20 @@ if(ARROW_FLIGHT AND ARROW_BUILD_SHARED) set(FLIGHT_PROTO "${ARROW_SOURCE}/format/Flight.proto") set(FLIGHT_GENERATED_PROTO_FILES - "${ARROW_CPP_SOURCE_DIR}/build/src/arrow/flight/Flight.pb.cc" - "${ARROW_CPP_SOURCE_DIR}/build/src/arrow/flight/Flight.pb.h" - "${ARROW_CPP_SOURCE_DIR}/build/src/arrow/flight/Flight.grpc.pb.cc" - "${ARROW_CPP_SOURCE_DIR}/build/src/arrow/flight/Flight.grpc.pb.h") + "${ARROW_CPP_SOURCE_DIR}/${ARROW_BUILD_DIR}/src/arrow/flight/Flight.pb.cc" + "${ARROW_CPP_SOURCE_DIR}/${ARROW_BUILD_DIR}/src/arrow/flight/Flight.pb.h" + "${ARROW_CPP_SOURCE_DIR}/${ARROW_BUILD_DIR}/src/arrow/flight/Flight.grpc.pb.cc" + "${ARROW_CPP_SOURCE_DIR}/${ARROW_BUILD_DIR}/src/arrow/flight/Flight.grpc.pb.h") set(PROTO_DEPENDS ${FLIGHT_PROTO} gRPC::grpc_cpp_plugin) add_custom_command(OUTPUT ${FLIGHT_GENERATED_PROTO_FILES} COMMAND ${ARROW_PROTOBUF_PROTOC} "-I${FLIGHT_PROTO_PATH}" - "--cpp_out=${ARROW_CPP_SOURCE_DIR}/build/src/arrow/flight/" "${FLIGHT_PROTO}" + "--cpp_out=${ARROW_CPP_SOURCE_DIR}/${ARROW_BUILD_DIR}/src/arrow/flight/" + "${FLIGHT_PROTO}" DEPENDS ${PROTO_DEPENDS} ARGS COMMAND ${ARROW_PROTOBUF_PROTOC} "-I${FLIGHT_PROTO_PATH}" - "--grpc_out=${ARROW_CPP_SOURCE_DIR}/build/src/arrow/flight/" + "--grpc_out=${ARROW_CPP_SOURCE_DIR}/${ARROW_BUILD_DIR}/src/arrow/flight/" "--plugin=protoc-gen-grpc=$" "${FLIGHT_PROTO}") @@ -332,7 +333,7 @@ if(ARROW_BUILD_TESTS) enable_testing() # Set necessary paths for cmake to find GTest - set(GTEST_ROOT ${ARROW_CPP_SOURCE_DIR}/build/googletest_ep-prefix) + set(GTEST_ROOT ${ARROW_CPP_SOURCE_DIR}/${ARROW_BUILD_DIR}/googletest_ep-prefix) set(GTEST_INCLUDE_DIR "${GTEST_ROOT}/include") set(GTEST_LIBRARY ${GTEST_ROOT}/lib) set(GTEST_MAIN_LIBRARY ${GTEST_ROOT}/lib) diff --git a/python/setup.py b/python/setup.py index 0f5e891b154..2ca710a54f1 100755 --- a/python/setup.py +++ b/python/setup.py @@ -246,6 +246,9 @@ def _run_cmake_arrow_python(self): build_include = pjoin(saved_cwd, 'build/dist/include') build_lib = pjoin(os.getcwd(), build_cmd.build_lib) + # The directory containing Arrow C++ build + arrow_build_dir = os.environ.get('ARROW_BUILD_DIR', 'build') + if self.inplace: # a bit hacky build_lib = saved_cwd @@ -264,6 +267,7 @@ def _run_cmake_arrow_python(self): '-DCMAKE_INSTALL_PREFIX=' + str(pjoin(saved_cwd, 'build/dist')), '-DCMAKE_BUILD_TYPE={0}'.format(self.build_type.lower()), + '-DARROW_BUILD_DIR=' + str(arrow_build_dir), ] # Check for specific options @@ -294,7 +298,6 @@ def append_cmake_bool(value, varname): pass print(f"moving {build_temp} to {build_lib}") - # shutil.move(build_temp, pjoin(build_lib, "pyarrow")) # a bit hacky for libname in os.listdir(build_temp): if "libarrow_python" in libname: From 55887d98ceebbe13c60f1157b68009ecda197ad3 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Mon, 4 Jul 2022 10:44:53 +0200 Subject: [PATCH 028/138] Remove the need for serialization_internal.h inside python/flight.cc - part 1 --- python/pyarrow/src_arrow/flight.cc | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/python/pyarrow/src_arrow/flight.cc b/python/pyarrow/src_arrow/flight.cc index bf7af27ac72..8082b7d1f9b 100644 --- a/python/pyarrow/src_arrow/flight.cc +++ b/python/pyarrow/src_arrow/flight.cc @@ -18,7 +18,11 @@ #include #include +<<<<<<< HEAD #include "arrow/python/flight.h" +======= +#include "flight.h" +>>>>>>> 04b1707d5 (Remove the need for serialization_internal.h inside python/flight.cc - part 1) #include "arrow/util/io_util.h" #include "arrow/util/logging.h" @@ -370,11 +374,22 @@ Status CreateFlightInfo(const std::shared_ptr& schema, const std::vector& endpoints, int64_t total_records, int64_t total_bytes, std::unique_ptr* out) { +<<<<<<< HEAD ARROW_ASSIGN_OR_RAISE(auto result, arrow::flight::FlightInfo::Make(*schema, descriptor, endpoints, total_records, total_bytes)); *out = std::unique_ptr( new arrow::flight::FlightInfo(std::move(result))); +======= + + auto result = + arrow::flight::FlightInfo::Make(*schema, + descriptor, + endpoints, + total_records, + total_bytes); + *out = std::unique_ptr(&(*result)); +>>>>>>> 04b1707d5 (Remove the need for serialization_internal.h inside python/flight.cc - part 1) return Status::OK(); } From 01de8e750144c36eef023982c0e9ff917f049ef0 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 5 Jul 2022 06:25:48 +0200 Subject: [PATCH 029/138] Remove the need for serialization_internal.h inside python/flight.cc - part 2 --- python/pyarrow/src_arrow/flight.cc | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/python/pyarrow/src_arrow/flight.cc b/python/pyarrow/src_arrow/flight.cc index 8082b7d1f9b..79b8db074a5 100644 --- a/python/pyarrow/src_arrow/flight.cc +++ b/python/pyarrow/src_arrow/flight.cc @@ -18,11 +18,7 @@ #include #include -<<<<<<< HEAD -#include "arrow/python/flight.h" -======= #include "flight.h" ->>>>>>> 04b1707d5 (Remove the need for serialization_internal.h inside python/flight.cc - part 1) #include "arrow/util/io_util.h" #include "arrow/util/logging.h" @@ -374,22 +370,11 @@ Status CreateFlightInfo(const std::shared_ptr& schema, const std::vector& endpoints, int64_t total_records, int64_t total_bytes, std::unique_ptr* out) { -<<<<<<< HEAD ARROW_ASSIGN_OR_RAISE(auto result, arrow::flight::FlightInfo::Make(*schema, descriptor, endpoints, total_records, total_bytes)); *out = std::unique_ptr( new arrow::flight::FlightInfo(std::move(result))); -======= - - auto result = - arrow::flight::FlightInfo::Make(*schema, - descriptor, - endpoints, - total_records, - total_bytes); - *out = std::unique_ptr(&(*result)); ->>>>>>> 04b1707d5 (Remove the need for serialization_internal.h inside python/flight.cc - part 1) return Status::OK(); } From 51156af0ad6e7c5e778afec2b146931d7126b9f2 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 5 Jul 2022 10:08:27 +0200 Subject: [PATCH 030/138] Rename ARROW_CMAKE_INSTALL_DIR to ARROW_CMAKE_DIR --- python/pyarrow/src_arrow/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index 56f1d5b0663..c5e98876437 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -212,7 +212,7 @@ include(CMakePackageConfigHelpers) # Set the output directory for cmake module # (CMAKE_INSTALL_PREFIX = python/build/dist! should be set in setup.py!) -set(ARROW_CMAKE_INSTALL_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}") +set(ARROW_CMAKE_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}") add_arrow_lib(arrow_python CMAKE_PACKAGE_NAME From a9db174adb41f8e56ce4faff75426f41ca1aff4b Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 5 Jul 2022 10:12:08 +0200 Subject: [PATCH 031/138] Add a note for ARROW_BUILD_DIR to the python dev docs --- docs/source/developers/python.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/source/developers/python.rst b/docs/source/developers/python.rst index 98ed93968a9..2dffc4ad1d4 100644 --- a/docs/source/developers/python.rst +++ b/docs/source/developers/python.rst @@ -391,6 +391,12 @@ variable to 1. To set the number of threads used to compile PyArrow's C++/Cython components, set the ``PYARROW_PARALLEL`` environment variable. +.. note:: + + If you used a different directory name for building Arrow C++ (by default it is + named "build"), then also set `ARROW_BUILD_DIR='name_of_build_dir'`. This way + pyarrow can find Arrow C++ built files. + If you wish to delete stale PyArrow build artifacts before rebuilding, navigate to the ``arrow/python`` folder and run ``git clean -Xfd .``. From 1e51b7f15b33802a1c8f5ef5666807ae13f3a3ce Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 5 Jul 2022 12:00:11 +0200 Subject: [PATCH 032/138] Make sure arrow/python include files do not get removed when building pyarrow --- python/setup.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/python/setup.py b/python/setup.py index 2ca710a54f1..577f98a5e9b 100755 --- a/python/setup.py +++ b/python/setup.py @@ -306,9 +306,10 @@ def append_cmake_bool(value, varname): os.remove(libname_path) shutil.move(pjoin(build_temp, libname), pjoin(build_lib, "pyarrow")) - if not os.path.isdir(pjoin(build_include, "arrow")): - self.mkpath(pjoin(build_include, "arrow")) - shutil.move(pjoin(build_include, "arrow", "python"), pjoin( + print(f"copying include folder: {build_include} to {build_lib}") + if not os.path.isdir(pjoin(build_include, "arrow", "python")): + self.mkpath(pjoin(build_include, "arrow", "python")) + shutil.copytree(pjoin(build_include, "arrow", "python"), pjoin( build_lib, "pyarrow", "include", "arrow", "python")) def _run_cmake(self): @@ -443,6 +444,13 @@ def append_cmake_bool(value, varname): shutil.move(pjoin(build_prefix, 'include'), pjoin(build_lib, 'pyarrow')) + # We need to add the C PyArrow include folder, that was also deleted + build_cpyarrow_include = pjoin(saved_cwd, 'build/dist/include') + if not os.path.isdir(pjoin(build_cpyarrow_include, "arrow", "python")): + self.mkpath(pjoin(build_cpyarrow_include, "arrow", "python")) + shutil.move(pjoin(build_cpyarrow_include, "arrow", "python"), pjoin( + build_lib, "pyarrow", "include", "arrow", "python")) + # Move the built C-extension to the place expected by the Python # build self._found_names = [] From 1d1d5b83f36764f2c88a50071efa3bc49e987dd0 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 5 Jul 2022 12:21:41 +0200 Subject: [PATCH 033/138] Try a fix for Could NOT find Python3 CI error --- python/pyarrow/src_arrow/CMakeLists.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index c5e98876437..c63bf27a47e 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -62,7 +62,19 @@ endif() find_package(Arrow REQUIRED) include(ArrowOptions) + +# +# Python +# +# Use the first Python installation on PATH, not the newest one +set(Python3_FIND_STRATEGY "LOCATION") +# On Windows, use registry last, not first +set(Python3_FIND_REGISTRY "LAST") +# On macOS, use framework last, not first +set(Python3_FIND_FRAMEWORK "LAST") + find_package(Python3Alt 3.7 REQUIRED) +include_directories(SYSTEM ${NUMPY_INCLUDE_DIRS} ${PYTHON_INCLUDE_DIRS} src) # Needed due to # CMake Error: INSTALL(EXPORT) given unknown export "arrow_python_targets" From e87433d263ec0eb880a2375eb45939570e726a8d Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 5 Jul 2022 14:17:45 +0200 Subject: [PATCH 034/138] Linter corrections --- python/setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/setup.py b/python/setup.py index 577f98a5e9b..706ac79c81a 100755 --- a/python/setup.py +++ b/python/setup.py @@ -447,7 +447,8 @@ def append_cmake_bool(value, varname): # We need to add the C PyArrow include folder, that was also deleted build_cpyarrow_include = pjoin(saved_cwd, 'build/dist/include') if not os.path.isdir(pjoin(build_cpyarrow_include, "arrow", "python")): - self.mkpath(pjoin(build_cpyarrow_include, "arrow", "python")) + self.mkpath( + pjoin(build_cpyarrow_include, "arrow", "python")) shutil.move(pjoin(build_cpyarrow_include, "arrow", "python"), pjoin( build_lib, "pyarrow", "include", "arrow", "python")) From d8cbea38118117d042aa742f44c9ce4e51d461df Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Wed, 6 Jul 2022 08:33:51 +0200 Subject: [PATCH 035/138] Linter corrections --- python/setup.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/python/setup.py b/python/setup.py index 706ac79c81a..d47127701fc 100755 --- a/python/setup.py +++ b/python/setup.py @@ -444,13 +444,16 @@ def append_cmake_bool(value, varname): shutil.move(pjoin(build_prefix, 'include'), pjoin(build_lib, 'pyarrow')) - # We need to add the C PyArrow include folder, that was also deleted + # We need to, again, add the C PyArrow include folder build_cpyarrow_include = pjoin(saved_cwd, 'build/dist/include') - if not os.path.isdir(pjoin(build_cpyarrow_include, "arrow", "python")): + if not os.path.isdir( + pjoin(build_cpyarrow_include, "arrow", "python")): self.mkpath( pjoin(build_cpyarrow_include, "arrow", "python")) - shutil.move(pjoin(build_cpyarrow_include, "arrow", "python"), pjoin( - build_lib, "pyarrow", "include", "arrow", "python")) + shutil.move(pjoin(build_cpyarrow_include, "arrow", "python"), + pjoin( + build_lib, "pyarrow", "include", + "arrow", "python")) # Move the built C-extension to the place expected by the Python # build From 5e3038d2015c6fddb5a0f5960f788ef45822663c Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Wed, 6 Jul 2022 09:48:13 +0200 Subject: [PATCH 036/138] Try to fix test_gdb.py error --- python/pyarrow/tests/test_gdb.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/python/pyarrow/tests/test_gdb.py b/python/pyarrow/tests/test_gdb.py index 89c42648d24..b014c393868 100644 --- a/python/pyarrow/tests/test_gdb.py +++ b/python/pyarrow/tests/test_gdb.py @@ -17,6 +17,8 @@ from functools import lru_cache import os +import os.path +from os.path import join as pjoin import re import shutil import subprocess @@ -43,7 +45,11 @@ def environment_for_gdb(): env = {} for var in ['PATH', 'LD_LIBRARY_PATH']: try: - env[var] = os.environ[var] + # Add path to libarrow_python + path = os.environ.get(var, '') + path_add = pjoin(os.getcwd(), 'pyarrow') + env[var] = str(path + ':' + path_add) + except KeyError: pass return env From d11f49e1502a13aae0e3bda31714770dbcb832f2 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Wed, 6 Jul 2022 10:03:54 +0200 Subject: [PATCH 037/138] Change 'os.getcwd()' with '__path__' --- python/pyarrow/tests/test_gdb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/pyarrow/tests/test_gdb.py b/python/pyarrow/tests/test_gdb.py index b014c393868..56156f3c4e0 100644 --- a/python/pyarrow/tests/test_gdb.py +++ b/python/pyarrow/tests/test_gdb.py @@ -47,8 +47,8 @@ def environment_for_gdb(): try: # Add path to libarrow_python path = os.environ.get(var, '') - path_add = pjoin(os.getcwd(), 'pyarrow') - env[var] = str(path + ':' + path_add) + path_add = pa.__path__ + env[var] = str(path + ':' + path_add) except KeyError: pass From b10062cc4b137c9d9765a382acbab297d8c94012 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Wed, 6 Jul 2022 10:24:53 +0200 Subject: [PATCH 038/138] Remove path import from test_gdb.py --- python/pyarrow/tests/test_gdb.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/pyarrow/tests/test_gdb.py b/python/pyarrow/tests/test_gdb.py index 56156f3c4e0..9b6d292bdf0 100644 --- a/python/pyarrow/tests/test_gdb.py +++ b/python/pyarrow/tests/test_gdb.py @@ -17,8 +17,6 @@ from functools import lru_cache import os -import os.path -from os.path import join as pjoin import re import shutil import subprocess From 3075e209769fd98b762bf618ca70ec4c8ed01277 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Wed, 6 Jul 2022 10:32:20 +0200 Subject: [PATCH 039/138] Add changes to CMakeLists.txt due to errors on local Linux build --- python/pyarrow/src_arrow/CMakeLists.txt | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index c63bf27a47e..77139d1d965 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -409,6 +409,9 @@ if(ARROW_BUILD_TESTS) target_include_directories(arrow_python_test_main SYSTEM PUBLIC ${ARROW_PYTHON_INCLUDES}) + # Link libraries to avoid include error on Linux + target_link_libraries(arrow_python_test_main arrow_shared) + if(APPLE) target_link_libraries(arrow_python_test_main ${CMAKE_DL_LIBS}) set_target_properties(arrow_python_test_main PROPERTIES LINK_FLAGS @@ -440,18 +443,12 @@ if(ARROW_BUILD_TESTS) set(TEST_PATH "${CMAKE_BINARY_DIR}/${TEST_NAME}") add_executable(${TEST_NAME} ${SOURCES}) - # With OSX and conda, we need to set the correct RPATH so that dependencies - # are found. The installed libraries with conda have an RPATH that matches - # for executables and libraries lying in $ENV{CONDA_PREFIX}/bin or - # $ENV{CONDA_PREFIX}/lib but our test libraries and executables are not - # installed there. - if(NOT "$ENV{CONDA_PREFIX}" STREQUAL "" AND APPLE) - set_target_properties(${TEST_NAME} - PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE - INSTALL_RPATH_USE_LINK_PATH TRUE - INSTALL_RPATH - "${PYTHON_SOURCE_DIR}/pyarrow;$ENV{CONDA_PREFIX}/lib") - endif() + # We need to set the correct RPATH so that dependencies + set_target_properties(${TEST_NAME} + PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE + INSTALL_RPATH_USE_LINK_PATH TRUE + INSTALL_RPATH + "${PYTHON_SOURCE_DIR}/pyarrow;$ENV{CONDA_PREFIX}/lib") # Customize link libraries target_link_libraries(${TEST_NAME} PRIVATE "${ARROW_PYTHON_TEST_LINK_LIBS}") From dcf5d14b2e861bbb016bf068c3543533932693ae Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Wed, 6 Jul 2022 10:33:03 +0200 Subject: [PATCH 040/138] String manipulation error --- python/pyarrow/tests/test_gdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyarrow/tests/test_gdb.py b/python/pyarrow/tests/test_gdb.py index 9b6d292bdf0..a6dfd8886a2 100644 --- a/python/pyarrow/tests/test_gdb.py +++ b/python/pyarrow/tests/test_gdb.py @@ -46,7 +46,7 @@ def environment_for_gdb(): # Add path to libarrow_python path = os.environ.get(var, '') path_add = pa.__path__ - env[var] = str(path + ':' + path_add) + env[var] = str(path) + ':' + (path_add) except KeyError: pass From 7993b7f23c9ce445a8c80d71931b9c28098a0a94 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Wed, 6 Jul 2022 11:06:37 +0200 Subject: [PATCH 041/138] Fix Typo --- python/pyarrow/tests/test_gdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyarrow/tests/test_gdb.py b/python/pyarrow/tests/test_gdb.py index a6dfd8886a2..8ad24f5cea9 100644 --- a/python/pyarrow/tests/test_gdb.py +++ b/python/pyarrow/tests/test_gdb.py @@ -46,7 +46,7 @@ def environment_for_gdb(): # Add path to libarrow_python path = os.environ.get(var, '') path_add = pa.__path__ - env[var] = str(path) + ':' + (path_add) + env[var] = str(path) + ':' + str(path_add) except KeyError: pass From 248d7e1255eec4dbc9e6aaead0ab96c4c3736b2b Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Wed, 6 Jul 2022 15:02:36 +0200 Subject: [PATCH 042/138] Revert change in test_gdb.py --- python/pyarrow/tests/test_gdb.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/python/pyarrow/tests/test_gdb.py b/python/pyarrow/tests/test_gdb.py index 8ad24f5cea9..89c42648d24 100644 --- a/python/pyarrow/tests/test_gdb.py +++ b/python/pyarrow/tests/test_gdb.py @@ -43,11 +43,7 @@ def environment_for_gdb(): env = {} for var in ['PATH', 'LD_LIBRARY_PATH']: try: - # Add path to libarrow_python - path = os.environ.get(var, '') - path_add = pa.__path__ - env[var] = str(path) + ':' + str(path_add) - + env[var] = os.environ[var] except KeyError: pass return env From beec42c5b7cc1414124c6ba0e269bc72df05a1f5 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 7 Jul 2022 09:11:25 +0200 Subject: [PATCH 043/138] Try to fix gRPC error on CI --- python/pyarrow/src_arrow/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index 77139d1d965..d6d1302aa7b 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -263,7 +263,7 @@ endif() if(ARROW_FLIGHT AND ARROW_BUILD_SHARED) find_package(ArrowFlight REQUIRED) - find_package(gRPC CONFIG REQUIRED) + find_package(gRPC REQUIRED) include_directories("${ARROW_CPP_SOURCE_DIR}/src" "${ARROW_CPP_SOURCE_DIR}/build/src") # Using arrow_flight_shared, see example From 817f377bfb1ada681efc4a6533c9f9db748f1271 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 7 Jul 2022 09:12:13 +0200 Subject: [PATCH 044/138] Relax the regex in test_gdb.py to test change on the CI --- python/pyarrow/tests/test_gdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyarrow/tests/test_gdb.py b/python/pyarrow/tests/test_gdb.py index 89c42648d24..a0d2b3fda3b 100644 --- a/python/pyarrow/tests/test_gdb.py +++ b/python/pyarrow/tests/test_gdb.py @@ -154,7 +154,7 @@ def select_frame(self, func_name): # but it's not available on old GDB versions (such as 8.1.1), # so instead parse the stack trace for a matching frame number. out = self.run_command("info stack") - pat = r"(?mi)^#(\d+)\s+.* in " + re.escape(func_name) + " " + pat = r"(?mi)^#(\d+)\s+.* in " + re.escape(func_name) m = re.search(pat, out) if m is None: pytest.fail(f"Could not select frame for function {func_name}") From a97d940a5c6d96125d005c451320a248357ce3a0 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 7 Jul 2022 10:49:22 +0200 Subject: [PATCH 045/138] Test gdb flags --- python/pyarrow/src_arrow/CMakeLists.txt | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index d6d1302aa7b..0dc82257d01 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -171,6 +171,24 @@ endif() # We require a C++11 compliant compiler set(CMAKE_CXX_STANDARD_REQUIRED ON) +if(NOT MSVC) + if(ARROW_GGDB_DEBUG) + set(ARROW_DEBUG_SYMBOL_TYPE "gdb") + set(C_FLAGS_DEBUG "-g${ARROW_DEBUG_SYMBOL_TYPE} -O0") + set(C_FLAGS_FASTDEBUG "-g${ARROW_DEBUG_SYMBOL_TYPE} -O1") + set(CXX_FLAGS_DEBUG "-g${ARROW_DEBUG_SYMBOL_TYPE} -O0") + set(CXX_FLAGS_FASTDEBUG "-g${ARROW_DEBUG_SYMBOL_TYPE} -O1") + else() + set(C_FLAGS_DEBUG "-g -O0") + set(C_FLAGS_FASTDEBUG "-g -O1") + set(CXX_FLAGS_DEBUG "-g -O0") + set(CXX_FLAGS_FASTDEBUG "-g -O1") + endif() + + set(C_FLAGS_RELEASE "-O3 -DNDEBUG") + set(CXX_FLAGS_RELEASE "-O3 -DNDEBUG") +endif() + # # Linker flags # From e25d3b0422e2585052bbeb6260e7fd0ea0d40c08 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 7 Jul 2022 12:02:42 +0200 Subject: [PATCH 046/138] Another try for gdb flags (to pass test_gdb.py) --- python/pyarrow/src_arrow/CMakeLists.txt | 31 ++++++++++++------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index 0dc82257d01..126972b3ba1 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -171,24 +171,23 @@ endif() # We require a C++11 compliant compiler set(CMAKE_CXX_STANDARD_REQUIRED ON) -if(NOT MSVC) - if(ARROW_GGDB_DEBUG) - set(ARROW_DEBUG_SYMBOL_TYPE "gdb") - set(C_FLAGS_DEBUG "-g${ARROW_DEBUG_SYMBOL_TYPE} -O0") - set(C_FLAGS_FASTDEBUG "-g${ARROW_DEBUG_SYMBOL_TYPE} -O1") - set(CXX_FLAGS_DEBUG "-g${ARROW_DEBUG_SYMBOL_TYPE} -O0") - set(CXX_FLAGS_FASTDEBUG "-g${ARROW_DEBUG_SYMBOL_TYPE} -O1") - else() - set(C_FLAGS_DEBUG "-g -O0") - set(C_FLAGS_FASTDEBUG "-g -O1") - set(CXX_FLAGS_DEBUG "-g -O0") - set(CXX_FLAGS_FASTDEBUG "-g -O1") - endif() - - set(C_FLAGS_RELEASE "-O3 -DNDEBUG") - set(CXX_FLAGS_RELEASE "-O3 -DNDEBUG") +option(ARROW_GGDB_DEBUG "Pass -ggdb flag to debug builds" ON) +if(ARROW_GGDB_DEBUG) + set(ARROW_DEBUG_SYMBOL_TYPE "gdb") + set(C_FLAGS_DEBUG "-g${ARROW_DEBUG_SYMBOL_TYPE} -O0") + set(C_FLAGS_FASTDEBUG "-g${ARROW_DEBUG_SYMBOL_TYPE} -O1") + set(CXX_FLAGS_DEBUG "-g${ARROW_DEBUG_SYMBOL_TYPE} -O0") + set(CXX_FLAGS_FASTDEBUG "-g${ARROW_DEBUG_SYMBOL_TYPE} -O1") +else() + set(C_FLAGS_DEBUG "-g -O0") + set(C_FLAGS_FASTDEBUG "-g -O1") + set(CXX_FLAGS_DEBUG "-g -O0") + set(CXX_FLAGS_FASTDEBUG "-g -O1") endif() +set(C_FLAGS_RELEASE "-O3 -DNDEBUG") +set(CXX_FLAGS_RELEASE "-O3 -DNDEBUG") + # # Linker flags # From 7fba6007a8a8cc9cc628ffb3b5119113c1dd0f44 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 7 Jul 2022 12:20:54 +0200 Subject: [PATCH 047/138] Remove pyarrow/include/arrow/python before copying in setup.py --- python/setup.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/python/setup.py b/python/setup.py index d47127701fc..84ea199b271 100755 --- a/python/setup.py +++ b/python/setup.py @@ -298,7 +298,6 @@ def append_cmake_bool(value, varname): pass print(f"moving {build_temp} to {build_lib}") - # a bit hacky for libname in os.listdir(build_temp): if "libarrow_python" in libname: libname_path = pjoin(build_lib, "pyarrow", libname) @@ -306,11 +305,14 @@ def append_cmake_bool(value, varname): os.remove(libname_path) shutil.move(pjoin(build_temp, libname), pjoin(build_lib, "pyarrow")) + print(f"copying include folder: {build_include} to {build_lib}") - if not os.path.isdir(pjoin(build_include, "arrow", "python")): - self.mkpath(pjoin(build_include, "arrow", "python")) - shutil.copytree(pjoin(build_include, "arrow", "python"), pjoin( - build_lib, "pyarrow", "include", "arrow", "python")) + arrow_python_include = pjoin(build_include, "arrow", "python") + pyarrow_include = pjoin( + build_lib, "pyarrow", "include", "arrow", "python") + if os.path.exists(pyarrow_include): + shutil.rmtree(pyarrow_include) + shutil.copytree(arrow_python_include, pyarrow_include) def _run_cmake(self): # check if build_type is correctly passed / set From cbd9aaf9c3005064c088619cf6722fa42814a5ae Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 7 Jul 2022 12:34:49 +0200 Subject: [PATCH 048/138] Third try to set gdb flags --- python/pyarrow/src_arrow/CMakeLists.txt | 35 ++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index 126972b3ba1..ad9b7e57592 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -171,7 +171,7 @@ endif() # We require a C++11 compliant compiler set(CMAKE_CXX_STANDARD_REQUIRED ON) -option(ARROW_GGDB_DEBUG "Pass -ggdb flag to debug builds" ON) +# compiler flags for different build types (run 'cmake -DCMAKE_BUILD_TYPE= .') if(ARROW_GGDB_DEBUG) set(ARROW_DEBUG_SYMBOL_TYPE "gdb") set(C_FLAGS_DEBUG "-g${ARROW_DEBUG_SYMBOL_TYPE} -O0") @@ -188,6 +188,39 @@ endif() set(C_FLAGS_RELEASE "-O3 -DNDEBUG") set(CXX_FLAGS_RELEASE "-O3 -DNDEBUG") +set(C_FLAGS_PROFILE_GEN "${CXX_FLAGS_RELEASE} -fprofile-generate") +set(C_FLAGS_PROFILE_BUILD "${CXX_FLAGS_RELEASE} -fprofile-use") +set(CXX_FLAGS_PROFILE_GEN "${CXX_FLAGS_RELEASE} -fprofile-generate") +set(CXX_FLAGS_PROFILE_BUILD "${CXX_FLAGS_RELEASE} -fprofile-use") + +string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE) +# Set compile flags based on the build type. +message(STATUS "Configured for ${CMAKE_BUILD_TYPE} build (set with cmake -DCMAKE_BUILD_TYPE={release,debug,...})" +) + +if("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${C_FLAGS_DEBUG}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_DEBUG}") +elseif("${CMAKE_BUILD_TYPE}" STREQUAL "RELWITHDEBINFO") + +elseif("${CMAKE_BUILD_TYPE}" STREQUAL "FASTDEBUG") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${C_FLAGS_FASTDEBUG}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_FASTDEBUG}") +elseif("${CMAKE_BUILD_TYPE}" STREQUAL "RELEASE") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${C_FLAGS_RELEASE}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_RELEASE}") +elseif("${CMAKE_BUILD_TYPE}" STREQUAL "PROFILE_GEN") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${C_FLAGS_PROFILE_GEN}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_PROFILE_GEN}") +elseif("${CMAKE_BUILD_TYPE}" STREQUAL "PROFILE_BUILD") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${C_FLAGS_PROFILE_BUILD}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_PROFILE_BUILD}") +else() + message(FATAL_ERROR "Unknown build type: ${CMAKE_BUILD_TYPE}") +endif() + +message(STATUS "Build Type: ${CMAKE_BUILD_TYPE}") + # # Linker flags # From 90eadcbc64b49c9f0393ae1428e9718d352b6b36 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 7 Jul 2022 13:10:04 +0200 Subject: [PATCH 049/138] Correct a check for PYARROW_WITH_PARQUET_ENCRYPTION --- python/pyarrow/src_arrow/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index ad9b7e57592..ec99cf667e3 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -258,9 +258,10 @@ set(ARROW_PYTHON_STATIC_LINK_LIBS ${PYTHON_OTHER_LIBS}) if(WIN32) list(APPEND ARROW_PYTHON_SHARED_LINK_LIBS ${PYTHON_LIBRARIES} ${PYTHON_OTHER_LIBS}) endif() -if(PARQUET_REQUIRE_ENCRYPTION) +if(PARQUET_REQUIRE_ENCRYPTION AND PYARROW_WITH_PARQUET_ENCRYPTION) list(APPEND ARROW_PYTHON_SHARED_LINK_LIBS parquet_shared) endif() + if(ARROW_USE_XSIMD) list(APPEND ARROW_PYTHON_SHARED_PRIVATE_LINK_LIBS xsimd) list(APPEND ARROW_PYTHON_STATIC_LINK_LIBS xsimd) From 122eecc8f8c102b42f1b9b7d006bc954cb5f0fda Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 7 Jul 2022 14:59:40 +0200 Subject: [PATCH 050/138] Make the gdb failing tests solution simpler with include(SetupCxxFlags) --- python/pyarrow/src_arrow/CMakeLists.txt | 51 +------------------------ 1 file changed, 2 insertions(+), 49 deletions(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index ec99cf667e3..2bf00248897 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -171,55 +171,8 @@ endif() # We require a C++11 compliant compiler set(CMAKE_CXX_STANDARD_REQUIRED ON) -# compiler flags for different build types (run 'cmake -DCMAKE_BUILD_TYPE= .') -if(ARROW_GGDB_DEBUG) - set(ARROW_DEBUG_SYMBOL_TYPE "gdb") - set(C_FLAGS_DEBUG "-g${ARROW_DEBUG_SYMBOL_TYPE} -O0") - set(C_FLAGS_FASTDEBUG "-g${ARROW_DEBUG_SYMBOL_TYPE} -O1") - set(CXX_FLAGS_DEBUG "-g${ARROW_DEBUG_SYMBOL_TYPE} -O0") - set(CXX_FLAGS_FASTDEBUG "-g${ARROW_DEBUG_SYMBOL_TYPE} -O1") -else() - set(C_FLAGS_DEBUG "-g -O0") - set(C_FLAGS_FASTDEBUG "-g -O1") - set(CXX_FLAGS_DEBUG "-g -O0") - set(CXX_FLAGS_FASTDEBUG "-g -O1") -endif() - -set(C_FLAGS_RELEASE "-O3 -DNDEBUG") -set(CXX_FLAGS_RELEASE "-O3 -DNDEBUG") - -set(C_FLAGS_PROFILE_GEN "${CXX_FLAGS_RELEASE} -fprofile-generate") -set(C_FLAGS_PROFILE_BUILD "${CXX_FLAGS_RELEASE} -fprofile-use") -set(CXX_FLAGS_PROFILE_GEN "${CXX_FLAGS_RELEASE} -fprofile-generate") -set(CXX_FLAGS_PROFILE_BUILD "${CXX_FLAGS_RELEASE} -fprofile-use") - -string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE) -# Set compile flags based on the build type. -message(STATUS "Configured for ${CMAKE_BUILD_TYPE} build (set with cmake -DCMAKE_BUILD_TYPE={release,debug,...})" -) - -if("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${C_FLAGS_DEBUG}") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_DEBUG}") -elseif("${CMAKE_BUILD_TYPE}" STREQUAL "RELWITHDEBINFO") - -elseif("${CMAKE_BUILD_TYPE}" STREQUAL "FASTDEBUG") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${C_FLAGS_FASTDEBUG}") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_FASTDEBUG}") -elseif("${CMAKE_BUILD_TYPE}" STREQUAL "RELEASE") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${C_FLAGS_RELEASE}") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_RELEASE}") -elseif("${CMAKE_BUILD_TYPE}" STREQUAL "PROFILE_GEN") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${C_FLAGS_PROFILE_GEN}") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_PROFILE_GEN}") -elseif("${CMAKE_BUILD_TYPE}" STREQUAL "PROFILE_BUILD") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${C_FLAGS_PROFILE_BUILD}") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_PROFILE_BUILD}") -else() - message(FATAL_ERROR "Unknown build type: ${CMAKE_BUILD_TYPE}") -endif() - -message(STATUS "Build Type: ${CMAKE_BUILD_TYPE}") +# Needed gdb flags +include(SetupCxxFlags) # # Linker flags From c45754a69d001e78f9d10a7d808506ea1a3e0dbf Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 7 Jul 2022 15:17:15 +0200 Subject: [PATCH 051/138] Revert the try change for gRPC find_package CI error --- python/pyarrow/src_arrow/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index 2bf00248897..208d9bc42fd 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -267,7 +267,7 @@ endif() if(ARROW_FLIGHT AND ARROW_BUILD_SHARED) find_package(ArrowFlight REQUIRED) - find_package(gRPC REQUIRED) + find_package(gRPC CONFIG REQUIRED) include_directories("${ARROW_CPP_SOURCE_DIR}/src" "${ARROW_CPP_SOURCE_DIR}/build/src") # Using arrow_flight_shared, see example From 4f702b22613b1671c3473c8c56a6f3dd97c814f6 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Fri, 8 Jul 2022 06:37:12 +0200 Subject: [PATCH 052/138] Revert back from using include(SetupCxxFlags) due to xsimd error on MacOs --- python/pyarrow/src_arrow/CMakeLists.txt | 51 ++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index 208d9bc42fd..49c99e9658f 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -172,7 +172,56 @@ endif() set(CMAKE_CXX_STANDARD_REQUIRED ON) # Needed gdb flags -include(SetupCxxFlags) + +# compiler flags for different build types (run 'cmake -DCMAKE_BUILD_TYPE= .') +if(ARROW_GGDB_DEBUG) + set(ARROW_DEBUG_SYMBOL_TYPE "gdb") + set(C_FLAGS_DEBUG "-g${ARROW_DEBUG_SYMBOL_TYPE} -O0") + set(C_FLAGS_FASTDEBUG "-g${ARROW_DEBUG_SYMBOL_TYPE} -O1") + set(CXX_FLAGS_DEBUG "-g${ARROW_DEBUG_SYMBOL_TYPE} -O0") + set(CXX_FLAGS_FASTDEBUG "-g${ARROW_DEBUG_SYMBOL_TYPE} -O1") +else() + set(C_FLAGS_DEBUG "-g -O0") + set(C_FLAGS_FASTDEBUG "-g -O1") + set(CXX_FLAGS_DEBUG "-g -O0") + set(CXX_FLAGS_FASTDEBUG "-g -O1") +endif() + +set(C_FLAGS_RELEASE "-O3 -DNDEBUG") +set(CXX_FLAGS_RELEASE "-O3 -DNDEBUG") + +set(C_FLAGS_PROFILE_GEN "${CXX_FLAGS_RELEASE} -fprofile-generate") +set(C_FLAGS_PROFILE_BUILD "${CXX_FLAGS_RELEASE} -fprofile-use") +set(CXX_FLAGS_PROFILE_GEN "${CXX_FLAGS_RELEASE} -fprofile-generate") +set(CXX_FLAGS_PROFILE_BUILD "${CXX_FLAGS_RELEASE} -fprofile-use") + +string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE) +# Set compile flags based on the build type. +message(STATUS "Configured for ${CMAKE_BUILD_TYPE} build (set with cmake -DCMAKE_BUILD_TYPE={release,debug,...})" +) + +if("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${C_FLAGS_DEBUG}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_DEBUG}") +elseif("${CMAKE_BUILD_TYPE}" STREQUAL "RELWITHDEBINFO") + +elseif("${CMAKE_BUILD_TYPE}" STREQUAL "FASTDEBUG") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${C_FLAGS_FASTDEBUG}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_FASTDEBUG}") +elseif("${CMAKE_BUILD_TYPE}" STREQUAL "RELEASE") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${C_FLAGS_RELEASE}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_RELEASE}") +elseif("${CMAKE_BUILD_TYPE}" STREQUAL "PROFILE_GEN") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${C_FLAGS_PROFILE_GEN}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_PROFILE_GEN}") +elseif("${CMAKE_BUILD_TYPE}" STREQUAL "PROFILE_BUILD") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${C_FLAGS_PROFILE_BUILD}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_PROFILE_BUILD}") +else() + message(FATAL_ERROR "Unknown build type: ${CMAKE_BUILD_TYPE}") +endif() + +message(STATUS "Build Type: ${CMAKE_BUILD_TYPE}") # # Linker flags From 9f8b1b835d177a297e85720271b8940cbe14526f Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Fri, 8 Jul 2022 08:27:44 +0200 Subject: [PATCH 053/138] Add xsmid target --- python/pyarrow/src_arrow/CMakeLists.txt | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index 49c99e9658f..6d832b4468f 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -264,7 +264,29 @@ if(PARQUET_REQUIRE_ENCRYPTION AND PYARROW_WITH_PARQUET_ENCRYPTION) list(APPEND ARROW_PYTHON_SHARED_LINK_LIBS parquet_shared) endif() +if((NOT ARROW_SIMD_LEVEL STREQUAL "NONE") OR (NOT ARROW_RUNTIME_SIMD_LEVEL STREQUAL "NONE" + )) + set(ARROW_USE_XSIMD TRUE) +else() + set(ARROW_USE_XSIMD FALSE) +endif() + if(ARROW_USE_XSIMD) + set(XSIMD_PREFIX "${ARROW_CPP_SOURCE_DIR}/${ARROW_BUILD_DIR}/xsimd_ep/src/xsimd_ep-install") + set(XSIMD_INCLUDE_DIR "${XSIMD_PREFIX}/include") + + if(NOT xsimd_INCLUDE_DIRS STREQUAL "") + add_library(xsimd INTERFACE IMPORTED) + if(CMAKE_VERSION VERSION_LESS 3.11) + set_target_properties(xsimd PROPERTIES INTERFACE_INCLUDE_DIRECTORIES + "${XSIMD_INCLUDE_DIR}") + else() + target_include_directories(xsimd INTERFACE "${XSIMD_INCLUDE_DIR}") + endif() + else() + message(STATUS "xsimd found. Headers: ${xsimd_INCLUDE_DIRS}") + endif() + list(APPEND ARROW_PYTHON_SHARED_PRIVATE_LINK_LIBS xsimd) list(APPEND ARROW_PYTHON_STATIC_LINK_LIBS xsimd) endif() From 3342b6f2315abbf237ea59ae1a1a4d7c108fb046 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Fri, 8 Jul 2022 08:28:41 +0200 Subject: [PATCH 054/138] Correct missed change from 'build' to --- python/pyarrow/src_arrow/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index 6d832b4468f..b4b71fa1349 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -339,7 +339,7 @@ if(ARROW_FLIGHT AND ARROW_BUILD_SHARED) find_package(ArrowFlight REQUIRED) find_package(gRPC CONFIG REQUIRED) - include_directories("${ARROW_CPP_SOURCE_DIR}/src" "${ARROW_CPP_SOURCE_DIR}/build/src") + include_directories("${ARROW_CPP_SOURCE_DIR}/src" "${ARROW_CPP_SOURCE_DIR}/}/${ARROW_BUILD_DIR}/src") # Using arrow_flight_shared, see example # https://github.com/apache/arrow/blob/7a0f00c16e084d194ae53d209b33b809cfc8f2d5/cpp/examples/arrow/CMakeLists.txt From da788b8119ee492307225f0dd88c314bf296b8bf Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Fri, 8 Jul 2022 08:29:27 +0200 Subject: [PATCH 055/138] Removing redundant option as ArrowOptions are now loaded --- python/pyarrow/src_arrow/CMakeLists.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index b4b71fa1349..e22fea1f0e7 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -76,10 +76,6 @@ set(Python3_FIND_FRAMEWORK "LAST") find_package(Python3Alt 3.7 REQUIRED) include_directories(SYSTEM ${NUMPY_INCLUDE_DIRS} ${PYTHON_INCLUDE_DIRS} src) -# Needed due to -# CMake Error: INSTALL(EXPORT) given unknown export "arrow_python_targets" -option(ARROW_BUILD_SHARED "Link to the Arrow shared library" ON) - add_custom_target(arrow_python-all) add_custom_target(arrow_python) add_custom_target(arrow_python-tests) From b30af60e8dbdb32632d267d0cb11a32c6de3d508 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Fri, 8 Jul 2022 09:07:21 +0200 Subject: [PATCH 056/138] Add a fix for conda and xsimd --- python/pyarrow/src_arrow/CMakeLists.txt | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index e22fea1f0e7..0c7ed1137ef 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -268,19 +268,20 @@ else() endif() if(ARROW_USE_XSIMD) - set(XSIMD_PREFIX "${ARROW_CPP_SOURCE_DIR}/${ARROW_BUILD_DIR}/xsimd_ep/src/xsimd_ep-install") - set(XSIMD_INCLUDE_DIR "${XSIMD_PREFIX}/include") - - if(NOT xsimd_INCLUDE_DIRS STREQUAL "") - add_library(xsimd INTERFACE IMPORTED) - if(CMAKE_VERSION VERSION_LESS 3.11) - set_target_properties(xsimd PROPERTIES INTERFACE_INCLUDE_DIRECTORIES - "${XSIMD_INCLUDE_DIR}") - else() - target_include_directories(xsimd INTERFACE "${XSIMD_INCLUDE_DIR}") - endif() + if(NOT "$ENV{CONDA_PREFIX}" STREQUAL "") + set(XSIMD_PREFIX "$ENV{CONDA_PREFIX}") + set(XSIMD_INCLUDE_DIR "${XSIMD_PREFIX}/include") else() - message(STATUS "xsimd found. Headers: ${xsimd_INCLUDE_DIRS}") + set(XSIMD_PREFIX "${ARROW_CPP_SOURCE_DIR}/${ARROW_BUILD_DIR}/xsimd_ep/src/xsimd_ep-install") + set(XSIMD_INCLUDE_DIR "${XSIMD_PREFIX}/include") + endif() + + add_library(xsimd INTERFACE IMPORTED) + if(CMAKE_VERSION VERSION_LESS 3.11) + set_target_properties(xsimd PROPERTIES INTERFACE_INCLUDE_DIRECTORIES + "${XSIMD_INCLUDE_DIR}") + else() + target_include_directories(xsimd INTERFACE "${XSIMD_INCLUDE_DIR}") endif() list(APPEND ARROW_PYTHON_SHARED_PRIVATE_LINK_LIBS xsimd) From f85bfb5e44ce6646d8292f538bff672143840b65 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Fri, 8 Jul 2022 09:08:26 +0200 Subject: [PATCH 057/138] Another try for CI error when ARROW_GRPC_USE_SHARED=off --- python/pyarrow/src_arrow/CMakeLists.txt | 27 +++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index 0c7ed1137ef..a95351b4482 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -338,11 +338,30 @@ if(ARROW_FLIGHT AND ARROW_BUILD_SHARED) find_package(gRPC CONFIG REQUIRED) include_directories("${ARROW_CPP_SOURCE_DIR}/src" "${ARROW_CPP_SOURCE_DIR}/}/${ARROW_BUILD_DIR}/src") - # Using arrow_flight_shared, see example - # https://github.com/apache/arrow/blob/7a0f00c16e084d194ae53d209b33b809cfc8f2d5/cpp/examples/arrow/CMakeLists.txt - set(ARROW_GRPC_USE_SHARED ON) - set(GRPC_REFLECTION_LINK_LIBS -Wl,--no-as-needed gRPC::grpc++_reflection + # # Using arrow_flight_shared, see example + # # https://github.com/apache/arrow/blob/7a0f00c16e084d194ae53d209b33b809cfc8f2d5/cpp/examples/arrow/CMakeLists.txt + # set(ARROW_GRPC_USE_SHARED ON) + # set(GRPC_REFLECTION_LINK_LIBS -Wl,--no-as-needed gRPC::grpc++_reflection + # -Wl,--as-needed) + # Static gRPC means we cannot linked to shared Arrow, since then + # we'll violate ODR for gRPC symbols + if(ARROW_GRPC_USE_SHARED) + set(FLIGHT_EXAMPLES_LINK_LIBS arrow_flight_shared) + # We don't directly use symbols from the reflection library, so + # ensure the linker still links to it + set(GRPC_REFLECTION_LINK_LIBS -Wl,--no-as-needed gRPC::grpc++_reflection -Wl,--as-needed) + elseif(NOT ARROW_BUILD_STATIC) + message(FATAL_ERROR "Statically built gRPC requires ARROW_BUILD_STATIC=ON") + else() + set(FLIGHT_EXAMPLES_LINK_LIBS arrow_flight_static) + if(APPLE) + set(GRPC_REFLECTION_LINK_LIBS -Wl,-force_load gRPC::grpc++_reflection) + else() + set(GRPC_REFLECTION_LINK_LIBS -Wl,--whole-archive gRPC::grpc++_reflection + -Wl,--no-whole-archive) + endif() + endif() set(FLIGHT_PROTO_PATH "${ARROW_SOURCE}/format") set(FLIGHT_PROTO "${ARROW_SOURCE}/format/Flight.proto") From 5ddbd84dfb0924aa34f0a09c23631d72c324e2c8 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Fri, 8 Jul 2022 09:54:37 +0200 Subject: [PATCH 058/138] Catch cases when ARROW_GRPC_USE_SHARED=off and ARROW_BUILD_STATIC=off and give FATAL_ERROR --- python/pyarrow/src_arrow/CMakeLists.txt | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index a95351b4482..b0cef50a061 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -334,25 +334,25 @@ endif() if(ARROW_FLIGHT AND ARROW_BUILD_SHARED) + # Static gRPC means we cannot link to shared Arrow, since then + # we'll violate ODR for gRPC symbols + if(NOT ARROW_GRPC_USE_SHARED AND NOT ARROW_BUILD_STATIC) + message(FATAL_ERROR "Statically built gRPC requires ARROW_BUILD_STATIC=ON") + endif() + find_package(ArrowFlight REQUIRED) find_package(gRPC CONFIG REQUIRED) include_directories("${ARROW_CPP_SOURCE_DIR}/src" "${ARROW_CPP_SOURCE_DIR}/}/${ARROW_BUILD_DIR}/src") - # # Using arrow_flight_shared, see example - # # https://github.com/apache/arrow/blob/7a0f00c16e084d194ae53d209b33b809cfc8f2d5/cpp/examples/arrow/CMakeLists.txt - # set(ARROW_GRPC_USE_SHARED ON) - # set(GRPC_REFLECTION_LINK_LIBS -Wl,--no-as-needed gRPC::grpc++_reflection - # -Wl,--as-needed) - # Static gRPC means we cannot linked to shared Arrow, since then - # we'll violate ODR for gRPC symbols + # Using example + # https://github.com/apache/arrow/blob/7a0f00c16e084d194ae53d209b33b809cfc8f2d5/cpp/examples/arrow/CMakeLists.txt + if(ARROW_GRPC_USE_SHARED) set(FLIGHT_EXAMPLES_LINK_LIBS arrow_flight_shared) # We don't directly use symbols from the reflection library, so # ensure the linker still links to it set(GRPC_REFLECTION_LINK_LIBS -Wl,--no-as-needed gRPC::grpc++_reflection -Wl,--as-needed) - elseif(NOT ARROW_BUILD_STATIC) - message(FATAL_ERROR "Statically built gRPC requires ARROW_BUILD_STATIC=ON") else() set(FLIGHT_EXAMPLES_LINK_LIBS arrow_flight_static) if(APPLE) From c327b143f971982efd724ef15357fa47b2368226 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Fri, 8 Jul 2022 10:14:31 +0200 Subject: [PATCH 059/138] Do not check for conda and xsmid but check if xsmid build dir exists in Arrow C++ --- python/pyarrow/src_arrow/CMakeLists.txt | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index b0cef50a061..24f050db13f 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -268,13 +268,12 @@ else() endif() if(ARROW_USE_XSIMD) - if(NOT "$ENV{CONDA_PREFIX}" STREQUAL "") - set(XSIMD_PREFIX "$ENV{CONDA_PREFIX}") - set(XSIMD_INCLUDE_DIR "${XSIMD_PREFIX}/include") - else() + if(EXISTS "${ARROW_CPP_SOURCE_DIR}/${ARROW_BUILD_DIR}/xsimd_ep/src/xsimd_ep-install") set(XSIMD_PREFIX "${ARROW_CPP_SOURCE_DIR}/${ARROW_BUILD_DIR}/xsimd_ep/src/xsimd_ep-install") - set(XSIMD_INCLUDE_DIR "${XSIMD_PREFIX}/include") + else() + set(XSIMD_PREFIX "$ENV{CONDA_PREFIX}") endif() + set(XSIMD_INCLUDE_DIR "${XSIMD_PREFIX}/include") add_library(xsimd INTERFACE IMPORTED) if(CMAKE_VERSION VERSION_LESS 3.11) From 07e2e01248ef016fd85aefd84299fab7bba8db1d Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Fri, 8 Jul 2022 11:39:39 +0200 Subject: [PATCH 060/138] Rearange checks for xsimd source file --- python/pyarrow/src_arrow/CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index 24f050db13f..6ebd46135da 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -268,10 +268,10 @@ else() endif() if(ARROW_USE_XSIMD) - if(EXISTS "${ARROW_CPP_SOURCE_DIR}/${ARROW_BUILD_DIR}/xsimd_ep/src/xsimd_ep-install") - set(XSIMD_PREFIX "${ARROW_CPP_SOURCE_DIR}/${ARROW_BUILD_DIR}/xsimd_ep/src/xsimd_ep-install") - else() + if(NOT EXISTS "${ARROW_CPP_SOURCE_DIR}/${ARROW_BUILD_DIR}/xsimd_ep/src/xsimd_ep-install") set(XSIMD_PREFIX "$ENV{CONDA_PREFIX}") + else() + set(XSIMD_PREFIX "${ARROW_CPP_SOURCE_DIR}/${ARROW_BUILD_DIR}/xsimd_ep/src/xsimd_ep-install") endif() set(XSIMD_INCLUDE_DIR "${XSIMD_PREFIX}/include") From 2a60b4aefc4b4e4c0c6062a59da34f608f20d304 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Fri, 8 Jul 2022 12:08:27 +0200 Subject: [PATCH 061/138] Add to docs info about running ctest for C PyArrow --- docs/source/developers/python.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/source/developers/python.rst b/docs/source/developers/python.rst index 2dffc4ad1d4..4ccb4bcf4c7 100644 --- a/docs/source/developers/python.rst +++ b/docs/source/developers/python.rst @@ -131,6 +131,29 @@ for ``.py`` files or for ``.pyx`` and ``.pxi`` files. In this case you will also need to install the `pytest-cython `_ plugin. +Testing Arrow Python (C++ code) +------------------------------- +If you want to run ctest for the tests that are included in the Arrow Python +module, you will need to build Arrow with ``DARROW_BUILD_TESTS=ON``. + +.. note:: + + Currently building the C++ unit tests does not work with googletest + from conda-forge, so we must use the ``BUNDLED`` source for building that + dependency + + In case you use conda add ``-DGTest_SOURCE=BUNDLED`` to the cmake flags + when building Arrow. + +After Arrow C++ and PyArrow are built, navigate to ``python/build/dist/temp`` +folder and run ctest: + +.. code-block:: + + $ pushd arrow/python/build/dist/temp + $ ctest + $ popd + Benchmarking ------------ From 2d1a0b3d54cdee5b4bcbe235d24f1020d7f03737 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Fri, 8 Jul 2022 12:13:05 +0200 Subject: [PATCH 062/138] Add link for arrow_python_test_main --- python/pyarrow/src_arrow/CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index 6ebd46135da..7386e0fec37 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -500,7 +500,11 @@ if(ARROW_BUILD_TESTS) PUBLIC ${ARROW_PYTHON_INCLUDES}) # Link libraries to avoid include error on Linux - target_link_libraries(arrow_python_test_main arrow_shared) + if(ARROW_TEST_LINKAGE STREQUAL shared) + target_link_libraries(arrow_python_test_main arrow_shared) + else() + target_link_libraries(arrow_python_test_main arrow_static) + endif() if(APPLE) target_link_libraries(arrow_python_test_main ${CMAKE_DL_LIBS}) From dc9b34fe7fbd8d1cbbb754d58770bd204a0cbaa7 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Fri, 8 Jul 2022 13:53:46 +0200 Subject: [PATCH 063/138] One last try to fix xsimd for today ... --- python/pyarrow/src_arrow/CMakeLists.txt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index 7386e0fec37..a4449a1668b 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -268,12 +268,13 @@ else() endif() if(ARROW_USE_XSIMD) - if(NOT EXISTS "${ARROW_CPP_SOURCE_DIR}/${ARROW_BUILD_DIR}/xsimd_ep/src/xsimd_ep-install") - set(XSIMD_PREFIX "$ENV{CONDA_PREFIX}") + set(XSIMD_PREFIX "${ARROW_CPP_SOURCE_DIR}/${ARROW_BUILD_DIR}/xsimd_ep/src/xsimd_ep-install") + if(EXISTS ${XSIMD_PREFIX}/include) + set(XSIMD_INCLUDE_DIR "${XSIMD_PREFIX}/include") else() - set(XSIMD_PREFIX "${ARROW_CPP_SOURCE_DIR}/${ARROW_BUILD_DIR}/xsimd_ep/src/xsimd_ep-install") + set(XSIMD_PREFIX "$ENV{CONDA_PREFIX}") + set(XSIMD_INCLUDE_DIR "${XSIMD_PREFIX}/include") endif() - set(XSIMD_INCLUDE_DIR "${XSIMD_PREFIX}/include") add_library(xsimd INTERFACE IMPORTED) if(CMAKE_VERSION VERSION_LESS 3.11) From 0eaf8e2a1cfcc0f1eeaa39cb97dce85cb68382ae Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Mon, 11 Jul 2022 06:39:25 +0200 Subject: [PATCH 064/138] Simplify pyhton flight build --- python/pyarrow/src_arrow/CMakeLists.txt | 52 +------------------------ 1 file changed, 1 insertion(+), 51 deletions(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index a4449a1668b..c41565d6602 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -334,58 +334,10 @@ endif() if(ARROW_FLIGHT AND ARROW_BUILD_SHARED) - # Static gRPC means we cannot link to shared Arrow, since then - # we'll violate ODR for gRPC symbols - if(NOT ARROW_GRPC_USE_SHARED AND NOT ARROW_BUILD_STATIC) - message(FATAL_ERROR "Statically built gRPC requires ARROW_BUILD_STATIC=ON") - endif() - find_package(ArrowFlight REQUIRED) - find_package(gRPC CONFIG REQUIRED) include_directories("${ARROW_CPP_SOURCE_DIR}/src" "${ARROW_CPP_SOURCE_DIR}/}/${ARROW_BUILD_DIR}/src") - # Using example - # https://github.com/apache/arrow/blob/7a0f00c16e084d194ae53d209b33b809cfc8f2d5/cpp/examples/arrow/CMakeLists.txt - - if(ARROW_GRPC_USE_SHARED) - set(FLIGHT_EXAMPLES_LINK_LIBS arrow_flight_shared) - # We don't directly use symbols from the reflection library, so - # ensure the linker still links to it - set(GRPC_REFLECTION_LINK_LIBS -Wl,--no-as-needed gRPC::grpc++_reflection - -Wl,--as-needed) - else() - set(FLIGHT_EXAMPLES_LINK_LIBS arrow_flight_static) - if(APPLE) - set(GRPC_REFLECTION_LINK_LIBS -Wl,-force_load gRPC::grpc++_reflection) - else() - set(GRPC_REFLECTION_LINK_LIBS -Wl,--whole-archive gRPC::grpc++_reflection - -Wl,--no-whole-archive) - endif() - endif() - - set(FLIGHT_PROTO_PATH "${ARROW_SOURCE}/format") - set(FLIGHT_PROTO "${ARROW_SOURCE}/format/Flight.proto") - - set(FLIGHT_GENERATED_PROTO_FILES - "${ARROW_CPP_SOURCE_DIR}/${ARROW_BUILD_DIR}/src/arrow/flight/Flight.pb.cc" - "${ARROW_CPP_SOURCE_DIR}/${ARROW_BUILD_DIR}/src/arrow/flight/Flight.pb.h" - "${ARROW_CPP_SOURCE_DIR}/${ARROW_BUILD_DIR}/src/arrow/flight/Flight.grpc.pb.cc" - "${ARROW_CPP_SOURCE_DIR}/${ARROW_BUILD_DIR}/src/arrow/flight/Flight.grpc.pb.h") - - set(PROTO_DEPENDS ${FLIGHT_PROTO} gRPC::grpc_cpp_plugin) - - add_custom_command(OUTPUT ${FLIGHT_GENERATED_PROTO_FILES} - COMMAND ${ARROW_PROTOBUF_PROTOC} "-I${FLIGHT_PROTO_PATH}" - "--cpp_out=${ARROW_CPP_SOURCE_DIR}/${ARROW_BUILD_DIR}/src/arrow/flight/" - "${FLIGHT_PROTO}" - DEPENDS ${PROTO_DEPENDS} ARGS - COMMAND ${ARROW_PROTOBUF_PROTOC} "-I${FLIGHT_PROTO_PATH}" - "--grpc_out=${ARROW_CPP_SOURCE_DIR}/${ARROW_BUILD_DIR}/src/arrow/flight/" - "--plugin=protoc-gen-grpc=$" - "${FLIGHT_PROTO}") - - set_source_files_properties(${FLIGHT_GENERATED_PROTO_FILES} PROPERTIES GENERATED TRUE) - add_custom_target(flight_grpc_gen ALL DEPENDS ${FLIGHT_GENERATED_PROTO_FILES}) + set(FLIGHT_LINK_LIBS arrow_flight_shared) add_arrow_lib(arrow_python_flight CMAKE_PACKAGE_NAME @@ -396,8 +348,6 @@ if(ARROW_FLIGHT AND ARROW_BUILD_SHARED) flight.cc OUTPUTS ARROW_PYFLIGHT_LIBRARIES - DEPENDENCIES - flight_grpc_gen SHARED_LINK_FLAGS ${ARROW_VERSION_SCRIPT_FLAGS} SHARED_LINK_LIBS From 15b05f48efe4cfc11962d4be53361638dd48a4ad Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 12 Jul 2022 05:50:24 +0200 Subject: [PATCH 065/138] Add notes to remove xsimd and add SetupCxxFlags when ARROW-17037 is merged --- python/pyarrow/src_arrow/CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index c41565d6602..10cd06673fa 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -168,6 +168,7 @@ endif() set(CMAKE_CXX_STANDARD_REQUIRED ON) # Needed gdb flags +# Remove and change to include(SetupCxxFlags) when ARROW-17037 is merged # compiler flags for different build types (run 'cmake -DCMAKE_BUILD_TYPE= .') if(ARROW_GGDB_DEBUG) @@ -256,10 +257,14 @@ set(ARROW_PYTHON_STATIC_LINK_LIBS ${PYTHON_OTHER_LIBS}) if(WIN32) list(APPEND ARROW_PYTHON_SHARED_LINK_LIBS ${PYTHON_LIBRARIES} ${PYTHON_OTHER_LIBS}) endif() + if(PARQUET_REQUIRE_ENCRYPTION AND PYARROW_WITH_PARQUET_ENCRYPTION) list(APPEND ARROW_PYTHON_SHARED_LINK_LIBS parquet_shared) endif() +# xsimd +# Remove when ARROW-17037 is merged + if((NOT ARROW_SIMD_LEVEL STREQUAL "NONE") OR (NOT ARROW_RUNTIME_SIMD_LEVEL STREQUAL "NONE" )) set(ARROW_USE_XSIMD TRUE) From dd90ec3e6a0520f256d8798fd745924ac052bf02 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 12 Jul 2022 06:13:13 +0200 Subject: [PATCH 066/138] Add PYTHON_EXECUTABLE and Python3_EXECUTABLE to setup.py to try if it fixes AppVeyor 'Could NOT find Python error' --- python/setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/setup.py b/python/setup.py index 84ea199b271..7e654f7f306 100755 --- a/python/setup.py +++ b/python/setup.py @@ -268,6 +268,8 @@ def _run_cmake_arrow_python(self): str(pjoin(saved_cwd, 'build/dist')), '-DCMAKE_BUILD_TYPE={0}'.format(self.build_type.lower()), '-DARROW_BUILD_DIR=' + str(arrow_build_dir), + '-DPYTHON_EXECUTABLE=%s' % sys.executable, + '-DPython3_EXECUTABLE=%s' % sys.executable, ] # Check for specific options From 4c3b0f3117ab12761c15c3e73d3ee426ff1d3056 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 12 Jul 2022 09:04:56 +0200 Subject: [PATCH 067/138] Try to pass AppVeyor makefile issue --- python/setup.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/setup.py b/python/setup.py index 7e654f7f306..5ca9d43ba72 100755 --- a/python/setup.py +++ b/python/setup.py @@ -283,6 +283,10 @@ def append_cmake_bool(value, varname): append_cmake_bool(self.with_hdfs, 'PYARROW_WITH_HDFS') + # Windows + if self.cmake_generator: + cmake_options += ['-G', self.cmake_generator] + # run cmake print("-- Running cmake for C pyarrow") self.spawn(['cmake'] + cmake_options + [source_cpyarrow]) From 348cc3603052e3432662eba12ed476e537f8ce56 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 12 Jul 2022 09:15:58 +0200 Subject: [PATCH 068/138] Delete replicated hacky code and add PYARROW_PARALLEL to C PyArrow build --- python/setup.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/python/setup.py b/python/setup.py index 5ca9d43ba72..3cf8cf0597f 100755 --- a/python/setup.py +++ b/python/setup.py @@ -287,13 +287,19 @@ def append_cmake_bool(value, varname): if self.cmake_generator: cmake_options += ['-G', self.cmake_generator] + # build args + build_tool_args = [] + if os.environ.get('PYARROW_PARALLEL'): + build_tool_args.append( + '-j{0}'.format(os.environ['PYARROW_PARALLEL'])) + # run cmake print("-- Running cmake for C pyarrow") self.spawn(['cmake'] + cmake_options + [source_cpyarrow]) print("-- Finished cmake for C pyarrow") # run make & install print("-- Running make build and install for C pyarrow") - self.spawn(['make', '-j4']) + self.spawn(['make'] + build_tool_args) self.spawn(['make', 'install']) print("-- Finished make build and install for C pyarrow") @@ -430,10 +436,6 @@ def append_cmake_bool(value, varname): build_tool_args) print("-- Finished cmake --build for pyarrow") - if self.inplace: - # a bit hacky - build_lib = saved_cwd - # Move the libraries to the place expected by the Python build try: os.makedirs(pjoin(build_lib, 'pyarrow')) From bd0d96c16b4bce8748cf807ffb7df3769eb9feae Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 12 Jul 2022 09:17:09 +0200 Subject: [PATCH 069/138] Change C pyarrow to arrow python in setup.py --- python/setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/setup.py b/python/setup.py index 3cf8cf0597f..333fd114672 100755 --- a/python/setup.py +++ b/python/setup.py @@ -294,14 +294,14 @@ def append_cmake_bool(value, varname): '-j{0}'.format(os.environ['PYARROW_PARALLEL'])) # run cmake - print("-- Running cmake for C pyarrow") + print("-- Running cmake for arrow python") self.spawn(['cmake'] + cmake_options + [source_cpyarrow]) - print("-- Finished cmake for C pyarrow") + print("-- Finished cmake for arrow python") # run make & install - print("-- Running make build and install for C pyarrow") + print("-- Running make build and install for arrow python") self.spawn(['make'] + build_tool_args) self.spawn(['make', 'install']) - print("-- Finished make build and install for C pyarrow") + print("-- Finished make build and install for arrow python") # Move the libraries to the place expected by the Python build try: From 703d13e110abb98ffb96aa833ba065fa144ef162 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 12 Jul 2022 09:46:54 +0200 Subject: [PATCH 070/138] Try to use 'cmake --build' and 'cmake --install' --- python/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/setup.py b/python/setup.py index 333fd114672..37867e8ec8a 100755 --- a/python/setup.py +++ b/python/setup.py @@ -299,8 +299,8 @@ def append_cmake_bool(value, varname): print("-- Finished cmake for arrow python") # run make & install print("-- Running make build and install for arrow python") - self.spawn(['make'] + build_tool_args) - self.spawn(['make', 'install']) + self.spawn(['cmake', '--build', '.'] + build_tool_args) + self.spawn(['cmake', '--install', '.']) print("-- Finished make build and install for arrow python") # Move the libraries to the place expected by the Python build From dedfb3ef16ccc8be537690aab8b1420ba1052207 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 12 Jul 2022 10:56:40 +0200 Subject: [PATCH 071/138] Add check for Arrow Python tests - GTest must be bundled --- python/pyarrow/src_arrow/CMakeLists.txt | 270 +++++++++++++----------- 1 file changed, 143 insertions(+), 127 deletions(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index 10cd06673fa..5ffb95305e0 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -387,142 +387,158 @@ arrow_install_all_headers("arrow/python") # ---------------------------------------------------------------------- +# +# Tests +# The tests will be moved to Cython and are currently supported for bundled GTest +# Follow-up: https://issues.apache.org/jira/browse/ARROW-17016?filter=-1 +# + if(ARROW_BUILD_TESTS) - enable_testing() - # Set necessary paths for cmake to find GTest + enable_testing() set(GTEST_ROOT ${ARROW_CPP_SOURCE_DIR}/${ARROW_BUILD_DIR}/googletest_ep-prefix) - set(GTEST_INCLUDE_DIR "${GTEST_ROOT}/include") - set(GTEST_LIBRARY ${GTEST_ROOT}/lib) - set(GTEST_MAIN_LIBRARY ${GTEST_ROOT}/lib) - - # - # Taken from Matlab CMakeLists.txt (enable_gtest and build_gtest) - # - set(ARROW_GTEST_PREFIX "${GTEST_ROOT}") - set(ARROW_GTEST_MAIN_PREFIX "${GTEST_ROOT}") - - if(WIN32) - set(ARROW_GTEST_SHARED_LIB_DIR "${ARROW_GTEST_PREFIX}/bin") - set(ARROW_GTEST_MAIN_SHARED_LIB_DIR "${ARROW_GTEST_MAIN_PREFIX}/bin") - - set(ARROW_GTEST_LINK_LIB_DIR "${ARROW_GTEST_PREFIX}/lib") - set(ARROW_GTEST_LINK_LIB - "${ARROW_GTEST_LINK_LIB_DIR}/${CMAKE_IMPORT_LIBRARY_PREFIX}gtestd${CMAKE_IMPORT_LIBRARY_SUFFIX}" + # GTest must be built from source + if(EXISTS ${ARROW_CPP_SOURCE_DIR}/${ARROW_BUILD_DIR}/googletest_ep-prefix) + + # Set necessary paths for cmake to find GTest + set(GTEST_INCLUDE_DIR "${GTEST_ROOT}/include") + set(GTEST_LIBRARY ${GTEST_ROOT}/lib) + set(GTEST_MAIN_LIBRARY ${GTEST_ROOT}/lib) + + # + # Taken from Matlab CMakeLists.txt (enable_gtest and build_gtest) + # + + set(ARROW_GTEST_PREFIX "${GTEST_ROOT}") + set(ARROW_GTEST_MAIN_PREFIX "${GTEST_ROOT}") + + if(WIN32) + set(ARROW_GTEST_SHARED_LIB_DIR "${ARROW_GTEST_PREFIX}/bin") + set(ARROW_GTEST_MAIN_SHARED_LIB_DIR "${ARROW_GTEST_MAIN_PREFIX}/bin") + + set(ARROW_GTEST_LINK_LIB_DIR "${ARROW_GTEST_PREFIX}/lib") + set(ARROW_GTEST_LINK_LIB + "${ARROW_GTEST_LINK_LIB_DIR}/${CMAKE_IMPORT_LIBRARY_PREFIX}gtestd${CMAKE_IMPORT_LIBRARY_SUFFIX}" + ) + + set(ARROW_GTEST_MAIN_LINK_LIB_DIR "${ARROW_GTEST_MAIN_PREFIX}/lib") + set(ARROW_GTEST_MAIN_LINK_LIB + "${ARROW_GTEST_MAIN_LINK_LIB_DIR}/${CMAKE_IMPORT_LIBRARY_PREFIX}gtest_maind${CMAKE_IMPORT_LIBRARY_SUFFIX}" + ) + else() + set(ARROW_GTEST_SHARED_LIB_DIR "${ARROW_GTEST_PREFIX}/lib") + set(ARROW_GTEST_MAIN_SHARED_LIB_DIR "${ARROW_GTEST_MAIN_PREFIX}/lib") + endif() + + set(ARROW_GTEST_INCLUDE_DIR "${ARROW_GTEST_PREFIX}/include") + set(ARROW_GTEST_SHARED_LIB + "${ARROW_GTEST_SHARED_LIB_DIR}/${CMAKE_SHARED_LIBRARY_PREFIX}gtestd${CMAKE_SHARED_LIBRARY_SUFFIX}" ) - set(ARROW_GTEST_MAIN_LINK_LIB_DIR "${ARROW_GTEST_MAIN_PREFIX}/lib") - set(ARROW_GTEST_MAIN_LINK_LIB - "${ARROW_GTEST_MAIN_LINK_LIB_DIR}/${CMAKE_IMPORT_LIBRARY_PREFIX}gtest_maind${CMAKE_IMPORT_LIBRARY_SUFFIX}" + set(ARROW_GTEST_MAIN_INCLUDE_DIR "${ARROW_GTEST_MAIN_PREFIX}/include") + set(ARROW_GTEST_MAIN_SHARED_LIB + "${ARROW_GTEST_MAIN_SHARED_LIB_DIR}/${CMAKE_SHARED_LIBRARY_PREFIX}gtest_maind${CMAKE_SHARED_LIBRARY_SUFFIX}" ) - else() - set(ARROW_GTEST_SHARED_LIB_DIR "${ARROW_GTEST_PREFIX}/lib") - set(ARROW_GTEST_MAIN_SHARED_LIB_DIR "${ARROW_GTEST_MAIN_PREFIX}/lib") - endif() - - set(ARROW_GTEST_INCLUDE_DIR "${ARROW_GTEST_PREFIX}/include") - set(ARROW_GTEST_SHARED_LIB - "${ARROW_GTEST_SHARED_LIB_DIR}/${CMAKE_SHARED_LIBRARY_PREFIX}gtestd${CMAKE_SHARED_LIBRARY_SUFFIX}" - ) - - set(ARROW_GTEST_MAIN_INCLUDE_DIR "${ARROW_GTEST_MAIN_PREFIX}/include") - set(ARROW_GTEST_MAIN_SHARED_LIB - "${ARROW_GTEST_MAIN_SHARED_LIB_DIR}/${CMAKE_SHARED_LIBRARY_PREFIX}gtest_maind${CMAKE_SHARED_LIBRARY_SUFFIX}" - ) - - file(MAKE_DIRECTORY "${ARROW_GTEST_INCLUDE_DIR}") - - # Create target GTest::gtest - add_library(GTest::gtest SHARED IMPORTED) - set_target_properties(GTest::gtest - PROPERTIES IMPORTED_LOCATION ${ARROW_GTEST_SHARED_LIB} - INTERFACE_INCLUDE_DIRECTORIES - ${ARROW_GTEST_INCLUDE_DIR}) - if(WIN32) - set_target_properties(GTest::gtest PROPERTIES IMPORTED_IMPLIB ${ARROW_GTEST_LINK_LIB}) - endif() - # ArrowTesting - # needed to be able to use arrow_testing_shared target - find_package(ArrowTesting REQUIRED) - - add_custom_target(all-tests) - - add_library(arrow_python_test_main STATIC util/test_main.cc) - - target_link_libraries(arrow_python_test_main GTest::gtest) - target_include_directories(arrow_python_test_main SYSTEM - PUBLIC ${ARROW_PYTHON_INCLUDES}) - - # Link libraries to avoid include error on Linux - if(ARROW_TEST_LINKAGE STREQUAL shared) - target_link_libraries(arrow_python_test_main arrow_shared) - else() - target_link_libraries(arrow_python_test_main arrow_static) - endif() - - if(APPLE) - target_link_libraries(arrow_python_test_main ${CMAKE_DL_LIBS}) - set_target_properties(arrow_python_test_main PROPERTIES LINK_FLAGS - "-undefined dynamic_lookup") - elseif(NOT MSVC) - target_link_libraries(arrow_python_test_main pthread ${CMAKE_DL_LIBS}) - endif() - - if(ARROW_TEST_LINKAGE STREQUAL shared) - set(ARROW_PYTHON_TEST_LINK_LIBS arrow_python_test_main arrow_python_shared - arrow_testing_shared arrow_shared) - else() - set(ARROW_PYTHON_TEST_LINK_LIBS arrow_python_test_main arrow_python_static - arrow_testing_static arrow_static) - endif() + file(MAKE_DIRECTORY "${ARROW_GTEST_INCLUDE_DIR}") + + # Create target GTest::gtest + add_library(GTest::gtest SHARED IMPORTED) + set_target_properties(GTest::gtest + PROPERTIES IMPORTED_LOCATION ${ARROW_GTEST_SHARED_LIB} + INTERFACE_INCLUDE_DIRECTORIES + ${ARROW_GTEST_INCLUDE_DIR}) + if(WIN32) + set_target_properties(GTest::gtest PROPERTIES IMPORTED_IMPLIB ${ARROW_GTEST_LINK_LIB}) + endif() + + # ArrowTesting + # needed to be able to use arrow_testing_shared target + find_package(ArrowTesting REQUIRED) + + add_custom_target(all-tests) + + add_library(arrow_python_test_main STATIC util/test_main.cc) + + target_link_libraries(arrow_python_test_main GTest::gtest) + target_include_directories(arrow_python_test_main SYSTEM + PUBLIC ${ARROW_PYTHON_INCLUDES}) + + # Link libraries to avoid include error on Linux + if(ARROW_TEST_LINKAGE STREQUAL shared) + target_link_libraries(arrow_python_test_main arrow_shared) + else() + target_link_libraries(arrow_python_test_main arrow_static) + endif() + + if(APPLE) + target_link_libraries(arrow_python_test_main ${CMAKE_DL_LIBS}) + set_target_properties(arrow_python_test_main PROPERTIES LINK_FLAGS + "-undefined dynamic_lookup") + elseif(NOT MSVC) + target_link_libraries(arrow_python_test_main pthread ${CMAKE_DL_LIBS}) + endif() + + if(ARROW_TEST_LINKAGE STREQUAL shared) + set(ARROW_PYTHON_TEST_LINK_LIBS arrow_python_test_main arrow_python_shared + arrow_testing_shared arrow_shared) + else() + set(ARROW_PYTHON_TEST_LINK_LIBS arrow_python_test_main arrow_python_static + arrow_testing_static arrow_static) + endif() + + # + # Add a test case + # + + set(REL_TEST_NAME "python_test") + get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE) + set(TEST_NAME "arrow-${TEST_NAME}") + set(SOURCES "${REL_TEST_NAME}.cc") + + # # Make sure the executable name contains only hyphens, not underscores + string(REPLACE "_" "-" TEST_NAME ${TEST_NAME}) + + set(TEST_PATH "${CMAKE_BINARY_DIR}/${TEST_NAME}") + add_executable(${TEST_NAME} ${SOURCES}) + + # We need to set the correct RPATH so that dependencies + set_target_properties(${TEST_NAME} + PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE + INSTALL_RPATH_USE_LINK_PATH TRUE + INSTALL_RPATH + "${PYTHON_SOURCE_DIR}/pyarrow;$ENV{CONDA_PREFIX}/lib") + + # Customize link libraries + target_link_libraries(${TEST_NAME} PRIVATE "${ARROW_PYTHON_TEST_LINK_LIBS}") + # Extra link libs + target_link_libraries(${TEST_NAME} PRIVATE ${PYTHON_LIBRARIES}) + # Extra includes + target_include_directories(${TEST_NAME} SYSTEM PUBLIC "${ARROW_PYTHON_INCLUDES}") + + # Add the test + if(ARROW_TEST_MEMCHECK) + add_test(${TEST_NAME} + bash + -c + "cd '${CMAKE_SOURCE_DIR}'; \ + valgrind --suppressions=valgrind.supp --tool=memcheck --gen-suppressions=all \ + --num-callers=500 --leak-check=full --leak-check-heuristics=stdstring \ + --error-exitcode=1 ${TEST_PATH} ${ARG_TEST_ARGUMENTS}") + elseif(WIN32) + add_test(${TEST_NAME} ${TEST_PATH}) + else() + add_test(${TEST_NAME} + ${ARROW_CPP_SOURCE_DIR}/build-support/run-test.sh + ${CMAKE_BINARY_DIR} + test + ${TEST_PATH}) + endif() - # - # Add a test case - # - - set(REL_TEST_NAME "python_test") - get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE) - set(TEST_NAME "arrow-${TEST_NAME}") - set(SOURCES "${REL_TEST_NAME}.cc") - - # # Make sure the executable name contains only hyphens, not underscores - string(REPLACE "_" "-" TEST_NAME ${TEST_NAME}) - - set(TEST_PATH "${CMAKE_BINARY_DIR}/${TEST_NAME}") - add_executable(${TEST_NAME} ${SOURCES}) - - # We need to set the correct RPATH so that dependencies - set_target_properties(${TEST_NAME} - PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE - INSTALL_RPATH_USE_LINK_PATH TRUE - INSTALL_RPATH - "${PYTHON_SOURCE_DIR}/pyarrow;$ENV{CONDA_PREFIX}/lib") - - # Customize link libraries - target_link_libraries(${TEST_NAME} PRIVATE "${ARROW_PYTHON_TEST_LINK_LIBS}") - # Extra link libs - target_link_libraries(${TEST_NAME} PRIVATE ${PYTHON_LIBRARIES}) - # Extra includes - target_include_directories(${TEST_NAME} SYSTEM PUBLIC "${ARROW_PYTHON_INCLUDES}") - - # Add the test - if(ARROW_TEST_MEMCHECK) - add_test(${TEST_NAME} - bash - -c - "cd '${CMAKE_SOURCE_DIR}'; \ - valgrind --suppressions=valgrind.supp --tool=memcheck --gen-suppressions=all \ - --num-callers=500 --leak-check=full --leak-check-heuristics=stdstring \ - --error-exitcode=1 ${TEST_PATH} ${ARG_TEST_ARGUMENTS}") - elseif(WIN32) - add_test(${TEST_NAME} ${TEST_PATH}) else() - add_test(${TEST_NAME} - ${ARROW_CPP_SOURCE_DIR}/build-support/run-test.sh - ${CMAKE_BINARY_DIR} - test - ${TEST_PATH}) + message(STATUS "Tests for Arrow Python not build") + message(STATUS "Set -DGTest_SOURCE=BUNDLED when building Arrow C++ + to enable building tests for Arrow Python") endif() endif() \ No newline at end of file From 0d4d555ed54689b3d1ff65c660ccfba812cd1b18 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 12 Jul 2022 13:54:41 +0200 Subject: [PATCH 072/138] Remove xsimd and add include(SetupCxxFlags) --- python/pyarrow/src_arrow/CMakeLists.txt | 83 +------------------------ 1 file changed, 1 insertion(+), 82 deletions(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index 5ffb95305e0..837a41f8142 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -168,57 +168,7 @@ endif() set(CMAKE_CXX_STANDARD_REQUIRED ON) # Needed gdb flags -# Remove and change to include(SetupCxxFlags) when ARROW-17037 is merged - -# compiler flags for different build types (run 'cmake -DCMAKE_BUILD_TYPE= .') -if(ARROW_GGDB_DEBUG) - set(ARROW_DEBUG_SYMBOL_TYPE "gdb") - set(C_FLAGS_DEBUG "-g${ARROW_DEBUG_SYMBOL_TYPE} -O0") - set(C_FLAGS_FASTDEBUG "-g${ARROW_DEBUG_SYMBOL_TYPE} -O1") - set(CXX_FLAGS_DEBUG "-g${ARROW_DEBUG_SYMBOL_TYPE} -O0") - set(CXX_FLAGS_FASTDEBUG "-g${ARROW_DEBUG_SYMBOL_TYPE} -O1") -else() - set(C_FLAGS_DEBUG "-g -O0") - set(C_FLAGS_FASTDEBUG "-g -O1") - set(CXX_FLAGS_DEBUG "-g -O0") - set(CXX_FLAGS_FASTDEBUG "-g -O1") -endif() - -set(C_FLAGS_RELEASE "-O3 -DNDEBUG") -set(CXX_FLAGS_RELEASE "-O3 -DNDEBUG") - -set(C_FLAGS_PROFILE_GEN "${CXX_FLAGS_RELEASE} -fprofile-generate") -set(C_FLAGS_PROFILE_BUILD "${CXX_FLAGS_RELEASE} -fprofile-use") -set(CXX_FLAGS_PROFILE_GEN "${CXX_FLAGS_RELEASE} -fprofile-generate") -set(CXX_FLAGS_PROFILE_BUILD "${CXX_FLAGS_RELEASE} -fprofile-use") - -string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE) -# Set compile flags based on the build type. -message(STATUS "Configured for ${CMAKE_BUILD_TYPE} build (set with cmake -DCMAKE_BUILD_TYPE={release,debug,...})" -) - -if("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${C_FLAGS_DEBUG}") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_DEBUG}") -elseif("${CMAKE_BUILD_TYPE}" STREQUAL "RELWITHDEBINFO") - -elseif("${CMAKE_BUILD_TYPE}" STREQUAL "FASTDEBUG") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${C_FLAGS_FASTDEBUG}") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_FASTDEBUG}") -elseif("${CMAKE_BUILD_TYPE}" STREQUAL "RELEASE") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${C_FLAGS_RELEASE}") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_RELEASE}") -elseif("${CMAKE_BUILD_TYPE}" STREQUAL "PROFILE_GEN") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${C_FLAGS_PROFILE_GEN}") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_PROFILE_GEN}") -elseif("${CMAKE_BUILD_TYPE}" STREQUAL "PROFILE_BUILD") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${C_FLAGS_PROFILE_BUILD}") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_PROFILE_BUILD}") -else() - message(FATAL_ERROR "Unknown build type: ${CMAKE_BUILD_TYPE}") -endif() - -message(STATUS "Build Type: ${CMAKE_BUILD_TYPE}") +include(SetupCxxFlags) # # Linker flags @@ -262,37 +212,6 @@ if(PARQUET_REQUIRE_ENCRYPTION AND PYARROW_WITH_PARQUET_ENCRYPTION) list(APPEND ARROW_PYTHON_SHARED_LINK_LIBS parquet_shared) endif() -# xsimd -# Remove when ARROW-17037 is merged - -if((NOT ARROW_SIMD_LEVEL STREQUAL "NONE") OR (NOT ARROW_RUNTIME_SIMD_LEVEL STREQUAL "NONE" - )) - set(ARROW_USE_XSIMD TRUE) -else() - set(ARROW_USE_XSIMD FALSE) -endif() - -if(ARROW_USE_XSIMD) - set(XSIMD_PREFIX "${ARROW_CPP_SOURCE_DIR}/${ARROW_BUILD_DIR}/xsimd_ep/src/xsimd_ep-install") - if(EXISTS ${XSIMD_PREFIX}/include) - set(XSIMD_INCLUDE_DIR "${XSIMD_PREFIX}/include") - else() - set(XSIMD_PREFIX "$ENV{CONDA_PREFIX}") - set(XSIMD_INCLUDE_DIR "${XSIMD_PREFIX}/include") - endif() - - add_library(xsimd INTERFACE IMPORTED) - if(CMAKE_VERSION VERSION_LESS 3.11) - set_target_properties(xsimd PROPERTIES INTERFACE_INCLUDE_DIRECTORIES - "${XSIMD_INCLUDE_DIR}") - else() - target_include_directories(xsimd INTERFACE "${XSIMD_INCLUDE_DIR}") - endif() - - list(APPEND ARROW_PYTHON_SHARED_PRIVATE_LINK_LIBS xsimd) - list(APPEND ARROW_PYTHON_STATIC_LINK_LIBS xsimd) -endif() - set(ARROW_PYTHON_INCLUDES ${NUMPY_INCLUDE_DIRS} ${PYTHON_INCLUDE_DIRS}) # Inlude macros needed to find and use add_arrow_lib function From 0636a108bf4b58d86e7f47606daa0355c884a46e Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 12 Jul 2022 15:01:19 +0200 Subject: [PATCH 073/138] Do not use / when making paths in setup.py --- python/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/setup.py b/python/setup.py index 37867e8ec8a..ee081e3e63f 100755 --- a/python/setup.py +++ b/python/setup.py @@ -242,8 +242,8 @@ def _run_cmake_arrow_python(self): # The directory for the module being built build_cmd = self.get_finalized_command('build') saved_cwd = os.getcwd() - build_temp = pjoin(saved_cwd, 'build/dist/temp') - build_include = pjoin(saved_cwd, 'build/dist/include') + build_temp = pjoin(saved_cwd, 'build', 'dist', 'temp') + build_include = pjoin(saved_cwd, 'build', 'dist', 'include') build_lib = pjoin(os.getcwd(), build_cmd.build_lib) # The directory containing Arrow C++ build From eb242217349ce87372aefea0a57b9e1ddc8ad40b Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Wed, 13 Jul 2022 13:44:39 +0200 Subject: [PATCH 074/138] Add C PyArrow tests to CI --- ci/scripts/python_test.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ci/scripts/python_test.sh b/ci/scripts/python_test.sh index 4400eeeae66..0d00e95d41e 100755 --- a/ci/scripts/python_test.sh +++ b/ci/scripts/python_test.sh @@ -20,6 +20,7 @@ set -ex arrow_dir=${1} +test_dir=${1}/python/build/dist/temp export ARROW_SOURCE_DIR=${arrow_dir} export ARROW_TEST_DATA=${arrow_dir}/testing/data @@ -54,4 +55,10 @@ export PYARROW_TEST_ORC export PYARROW_TEST_PARQUET export PYARROW_TEST_S3 +# Testing Arrow Python +pushd ${test_dir} +ctest +popd + +# Testing PyArrow pytest -r s ${PYTEST_ARGS} --pyargs pyarrow From af2f3159cc946ff2105f685b5805a2cb1a861848 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Wed, 13 Jul 2022 14:42:49 +0200 Subject: [PATCH 075/138] Add -- for PYARROW_PARALLEL in setup.py --- python/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/setup.py b/python/setup.py index ee081e3e63f..056edbc8c9a 100755 --- a/python/setup.py +++ b/python/setup.py @@ -290,6 +290,7 @@ def append_cmake_bool(value, varname): # build args build_tool_args = [] if os.environ.get('PYARROW_PARALLEL'): + build_tool_args.append('--') build_tool_args.append( '-j{0}'.format(os.environ['PYARROW_PARALLEL'])) From 00d39736701a7334a0271904ef7d1983c08a68a1 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Wed, 13 Jul 2022 14:45:53 +0200 Subject: [PATCH 076/138] Try out CI without setting ARROW_VERSION_SCRIPT_FLAGS --- python/pyarrow/src_arrow/CMakeLists.txt | 26 ------------------------- 1 file changed, 26 deletions(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index 837a41f8142..74e15810f41 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -170,32 +170,6 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) # Needed gdb flags include(SetupCxxFlags) -# -# Linker flags -# - -# Localize thirdparty symbols using a linker version script. This hides them -# from the client application. The OS X linker does not support the -# version-script option. -if(CMAKE_VERSION VERSION_LESS 3.18) - if(APPLE OR WIN32) - set(CXX_LINKER_SUPPORTS_VERSION_SCRIPT FALSE) - else() - set(CXX_LINKER_SUPPORTS_VERSION_SCRIPT TRUE) - endif() -else() - include(CheckLinkerFlag) - check_linker_flag(CXX - "-Wl,--version-script=${ARROW_CPP_SOURCE_DIR}/src/arrow/symbols.map" - CXX_LINKER_SUPPORTS_VERSION_SCRIPT) -endif() - -# Need to set ARROW_VERSION_SCRIPT_FLAGS for add_arrow_lib -if(CXX_LINKER_SUPPORTS_VERSION_SCRIPT) - set(ARROW_VERSION_SCRIPT_FLAGS - "-Wl,--version-script=${ARROW_CPP_SOURCE_DIR}/src/arrow/symbols.map") -endif() - # # shred/static link libs # From 6d067876f364200682812ab1aa59aad2c0c419c7 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 14 Jul 2022 06:08:01 +0200 Subject: [PATCH 077/138] Add changes to python_test.sh to check if ARROW_BUILD_TESTS=on --- ci/scripts/python_test.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ci/scripts/python_test.sh b/ci/scripts/python_test.sh index 0d00e95d41e..985bff708c8 100755 --- a/ci/scripts/python_test.sh +++ b/ci/scripts/python_test.sh @@ -56,9 +56,10 @@ export PYARROW_TEST_PARQUET export PYARROW_TEST_S3 # Testing Arrow Python -pushd ${test_dir} -ctest -popd - +if [ "${ARROW_BUILD_TESTS}" == "ON" ]; then + pushd ${test_dir} + ctest + popd +fi # Testing PyArrow pytest -r s ${PYTEST_ARGS} --pyargs pyarrow From 0ed8063e77ba919aa0870a19cfb5b4c6d76234c3 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 14 Jul 2022 13:10:22 +0200 Subject: [PATCH 078/138] Change cmake install steps in setup.py --- python/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/setup.py b/python/setup.py index 056edbc8c9a..19e7364e2ed 100755 --- a/python/setup.py +++ b/python/setup.py @@ -300,8 +300,8 @@ def append_cmake_bool(value, varname): print("-- Finished cmake for arrow python") # run make & install print("-- Running make build and install for arrow python") - self.spawn(['cmake', '--build', '.'] + build_tool_args) - self.spawn(['cmake', '--install', '.']) + self.spawn(['cmake', '--build', '.', '--target', 'install'] + + build_tool_args) print("-- Finished make build and install for arrow python") # Move the libraries to the place expected by the Python build From fc6112b0752e05b5007f0bab5d293ac890e9712f Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 14 Jul 2022 13:12:08 +0200 Subject: [PATCH 079/138] Keep arrow flight build comment from previous build procedure --- python/pyarrow/src_arrow/CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index 74e15810f41..823b2f53b7e 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -231,7 +231,11 @@ if(ARROW_BUILD_STATIC AND MSVC) endif() if(ARROW_FLIGHT AND ARROW_BUILD_SHARED) - + # Must link to shared libarrow_flight: we don't want to link more than one + # copy of gRPC into the eventual Cython shared object, otherwise gRPC calls + # fail with weird errors due to multiple copies of global static state (The + # other solution is to link gRPC shared everywhere instead of statically only + # in Flight) find_package(ArrowFlight REQUIRED) include_directories("${ARROW_CPP_SOURCE_DIR}/src" "${ARROW_CPP_SOURCE_DIR}/}/${ARROW_BUILD_DIR}/src") From accaeac9718e6616d55ebcd0a38b22201ab8a8d0 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 14 Jul 2022 16:47:25 +0200 Subject: [PATCH 080/138] Hackish try to fix python-sdist build --- python/pyarrow/src_arrow/CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index 823b2f53b7e..73f377470e7 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -196,6 +196,12 @@ include(CMakePackageConfigHelpers) # (CMAKE_INSTALL_PREFIX = python/build/dist! should be set in setup.py!) set(ARROW_CMAKE_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}") +# Changing ARROW_SOURCE_DIR for sdist build +# In this case cpp/cmake_modules doesn't exist +if(EXIST "${ARROW_SOURCE_DIR}/cmake_modules/Find${MODULE}.cmake") + set(ARROW_SOURCE_DIR ${PYTHON_SOURCE_DIR}) +endif() + add_arrow_lib(arrow_python CMAKE_PACKAGE_NAME ArrowPython From c2a4ca5d94b6778ef197a5bae7788ca817d2af15 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 14 Jul 2022 16:50:22 +0200 Subject: [PATCH 081/138] Change EXIST to NOT EXIST for python-sdist build fix --- python/pyarrow/src_arrow/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index 73f377470e7..918f4e52d7f 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -198,7 +198,7 @@ set(ARROW_CMAKE_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}") # Changing ARROW_SOURCE_DIR for sdist build # In this case cpp/cmake_modules doesn't exist -if(EXIST "${ARROW_SOURCE_DIR}/cmake_modules/Find${MODULE}.cmake") +if(NOT EXIST "${ARROW_SOURCE_DIR}/cmake_modules/Find${MODULE}.cmake") set(ARROW_SOURCE_DIR ${PYTHON_SOURCE_DIR}) endif() From b449fac19db335b00214360f17a60138ea2ffc45 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 14 Jul 2022 17:09:36 +0200 Subject: [PATCH 082/138] Fix typo :/ --- python/pyarrow/src_arrow/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index 918f4e52d7f..3470d584b49 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -198,7 +198,7 @@ set(ARROW_CMAKE_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}") # Changing ARROW_SOURCE_DIR for sdist build # In this case cpp/cmake_modules doesn't exist -if(NOT EXIST "${ARROW_SOURCE_DIR}/cmake_modules/Find${MODULE}.cmake") +if(NOT EXISTS "${ARROW_SOURCE_DIR}/cmake_modules/Find${MODULE}.cmake") set(ARROW_SOURCE_DIR ${PYTHON_SOURCE_DIR}) endif() From f3bc1ee1e91dce4501c0ff7162ba5840ab0835d3 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Fri, 15 Jul 2022 06:29:12 +0200 Subject: [PATCH 083/138] Set ARROW_PYTHON_LIB_DIR to try fix AppVeyor can't find Arrow Python error --- python/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index eed0e9800b1..1f69cb6757e 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -233,6 +233,7 @@ if(PYARROW_BUILD_SUBSTRAIT) endif() # Arrow +set(ARROW_PYTHON_LIB_DIR "${CMAKE_SOURCE_DIR}/pyarrow") find_package(ArrowPython REQUIRED) include_directories(SYSTEM ${ARROW_INCLUDE_DIR}) From 674ef9c121f4974e22e65a3318d99b9e5c042ac0 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Fri, 15 Jul 2022 11:01:49 +0200 Subject: [PATCH 084/138] Make changes to moving/copying of libs in setup.py --- python/CMakeLists.txt | 1 - python/setup.py | 40 ++++++++++++++++++++++++---------------- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 1f69cb6757e..eed0e9800b1 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -233,7 +233,6 @@ if(PYARROW_BUILD_SUBSTRAIT) endif() # Arrow -set(ARROW_PYTHON_LIB_DIR "${CMAKE_SOURCE_DIR}/pyarrow") find_package(ArrowPython REQUIRED) include_directories(SYSTEM ${ARROW_INCLUDE_DIR}) diff --git a/python/setup.py b/python/setup.py index 19e7364e2ed..4effe0d11d1 100755 --- a/python/setup.py +++ b/python/setup.py @@ -242,8 +242,8 @@ def _run_cmake_arrow_python(self): # The directory for the module being built build_cmd = self.get_finalized_command('build') saved_cwd = os.getcwd() - build_temp = pjoin(saved_cwd, 'build', 'dist', 'temp') - build_include = pjoin(saved_cwd, 'build', 'dist', 'include') + build_dir = pjoin(saved_cwd, 'build') + build_include = pjoin(saved_cwd, 'build', 'include') build_lib = pjoin(os.getcwd(), build_cmd.build_lib) # The directory containing Arrow C++ build @@ -253,19 +253,18 @@ def _run_cmake_arrow_python(self): # a bit hacky build_lib = saved_cwd - if not os.path.isdir(build_temp): - self.mkpath(build_temp) - if not os.path.isdir(build_lib): - self.mkpath(build_lib) + if not os.path.isdir(build_dir): + self.mkpath(build_dir) if not os.path.isdir(build_include): self.mkpath(build_include) + if not os.path.isdir(build_lib): + self.mkpath(build_lib) # Change to the build directory - with changed_dir(build_temp): + with changed_dir(build_dir): # cmake args cmake_options = [ - '-DCMAKE_INSTALL_PREFIX=' + - str(pjoin(saved_cwd, 'build/dist')), + '-DCMAKE_INSTALL_PREFIX=' + str(build_dir), '-DCMAKE_BUILD_TYPE={0}'.format(self.build_type.lower()), '-DARROW_BUILD_DIR=' + str(arrow_build_dir), '-DPYTHON_EXECUTABLE=%s' % sys.executable, @@ -310,21 +309,30 @@ def append_cmake_bool(value, varname): except OSError: pass - print(f"moving {build_temp} to {build_lib}") - for libname in os.listdir(build_temp): - if "libarrow_python" in libname: - libname_path = pjoin(build_lib, "pyarrow", libname) + # helper function + def copy_libs(libname, folder_name): + if "python" in libname: + libname_path = pjoin(saved_cwd, "pyarrow", libname) if os.path.exists(libname_path): os.remove(libname_path) - shutil.move(pjoin(build_temp, libname), + print(f"* Copying {pjoin(build_dir,folder_name, libname)} to {pjoin(build_lib, 'pyarrow')}") + shutil.copy(pjoin(build_dir,folder_name, libname), pjoin(build_lib, "pyarrow")) - - print(f"copying include folder: {build_include} to {build_lib}") + # Move libraries to python/pyarrow + for libname in os.listdir(pjoin(build_dir,'lib')): + copy_libs(libname, 'lib') + # For windows builds, move dll from bin + if os.path.isdir(pjoin(build_dir,'bin')): + for libname in os.listdir(pjoin(build_dir,'bin')): + copy_libs(libname, 'bin') + + # Copy headers tp python/pyarrow/include arrow_python_include = pjoin(build_include, "arrow", "python") pyarrow_include = pjoin( build_lib, "pyarrow", "include", "arrow", "python") if os.path.exists(pyarrow_include): shutil.rmtree(pyarrow_include) + print(f"* Copying include folder: {arrow_python_include} to {pyarrow_include}") shutil.copytree(arrow_python_include, pyarrow_include) def _run_cmake(self): From e04282a523227443748ae83fe182830bc9c66ef0 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Fri, 15 Jul 2022 11:41:27 +0200 Subject: [PATCH 085/138] Linter corrections --- python/setup.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/python/setup.py b/python/setup.py index 4effe0d11d1..09d283a6c6a 100755 --- a/python/setup.py +++ b/python/setup.py @@ -315,15 +315,16 @@ def copy_libs(libname, folder_name): libname_path = pjoin(saved_cwd, "pyarrow", libname) if os.path.exists(libname_path): os.remove(libname_path) - print(f"* Copying {pjoin(build_dir,folder_name, libname)} to {pjoin(build_lib, 'pyarrow')}") - shutil.copy(pjoin(build_dir,folder_name, libname), + print( + f"Copying {pjoin(build_dir,folder_name, libname)} to {pjoin(build_lib, 'pyarrow')}") + shutil.copy(pjoin(build_dir, folder_name, libname), pjoin(build_lib, "pyarrow")) # Move libraries to python/pyarrow - for libname in os.listdir(pjoin(build_dir,'lib')): + for libname in os.listdir(pjoin(build_dir, 'lib')): copy_libs(libname, 'lib') # For windows builds, move dll from bin - if os.path.isdir(pjoin(build_dir,'bin')): - for libname in os.listdir(pjoin(build_dir,'bin')): + if os.path.isdir(pjoin(build_dir,' bin')): + for libname in os.listdir(pjoin(build_dir, 'bin')): copy_libs(libname, 'bin') # Copy headers tp python/pyarrow/include @@ -332,7 +333,8 @@ def copy_libs(libname, folder_name): build_lib, "pyarrow", "include", "arrow", "python") if os.path.exists(pyarrow_include): shutil.rmtree(pyarrow_include) - print(f"* Copying include folder: {arrow_python_include} to {pyarrow_include}") + print( + f"Copying include folder: {arrow_python_include} to {pyarrow_include}") shutil.copytree(arrow_python_include, pyarrow_include) def _run_cmake(self): From b9c33a02ea88a0e0116c8853657cdce561f01030 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Fri, 15 Jul 2022 11:53:00 +0200 Subject: [PATCH 086/138] Update setup.py again due to CI errors --- python/setup.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/python/setup.py b/python/setup.py index 09d283a6c6a..9078c6cd346 100755 --- a/python/setup.py +++ b/python/setup.py @@ -239,32 +239,32 @@ def _run_cmake_arrow_python(self): # The directory containing this C PyArrow CMakeLists.txt source_cpyarrow = pjoin(source, "pyarrow/src_arrow") + # The directory for the module being built build_cmd = self.get_finalized_command('build') saved_cwd = os.getcwd() - build_dir = pjoin(saved_cwd, 'build') - build_include = pjoin(saved_cwd, 'build', 'include') + build_dir = pjoin(saved_cwd, 'build', 'dist') + build_include = pjoin(saved_cwd, 'build', 'dist', 'include') build_lib = pjoin(os.getcwd(), build_cmd.build_lib) # The directory containing Arrow C++ build arrow_build_dir = os.environ.get('ARROW_BUILD_DIR', 'build') - if self.inplace: # a bit hacky build_lib = saved_cwd - if not os.path.isdir(build_dir): self.mkpath(build_dir) - if not os.path.isdir(build_include): - self.mkpath(build_include) if not os.path.isdir(build_lib): self.mkpath(build_lib) + if not os.path.isdir(build_include): + self.mkpath(build_include) # Change to the build directory with changed_dir(build_dir): # cmake args cmake_options = [ - '-DCMAKE_INSTALL_PREFIX=' + str(build_dir), + '-DCMAKE_INSTALL_PREFIX=' + + str(pjoin(saved_cwd, 'build/dist')), '-DCMAKE_BUILD_TYPE={0}'.format(self.build_type.lower()), '-DARROW_BUILD_DIR=' + str(arrow_build_dir), '-DPYTHON_EXECUTABLE=%s' % sys.executable, @@ -312,13 +312,14 @@ def append_cmake_bool(value, varname): # helper function def copy_libs(libname, folder_name): if "python" in libname: - libname_path = pjoin(saved_cwd, "pyarrow", libname) + libname_path = pjoin(build_lib, "pyarrow", libname) if os.path.exists(libname_path): os.remove(libname_path) print( f"Copying {pjoin(build_dir,folder_name, libname)} to {pjoin(build_lib, 'pyarrow')}") shutil.copy(pjoin(build_dir, folder_name, libname), pjoin(build_lib, "pyarrow")) + # Move libraries to python/pyarrow for libname in os.listdir(pjoin(build_dir, 'lib')): copy_libs(libname, 'lib') From e120bacf3d4655f7bc79d7fccee51b78f08d58ba Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Fri, 15 Jul 2022 13:34:18 +0200 Subject: [PATCH 087/138] Try to copy files from /bin in setup.py --- python/setup.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/python/setup.py b/python/setup.py index 9078c6cd346..f8b34947be1 100755 --- a/python/setup.py +++ b/python/setup.py @@ -239,7 +239,6 @@ def _run_cmake_arrow_python(self): # The directory containing this C PyArrow CMakeLists.txt source_cpyarrow = pjoin(source, "pyarrow/src_arrow") - # The directory for the module being built build_cmd = self.get_finalized_command('build') saved_cwd = os.getcwd() @@ -316,7 +315,7 @@ def copy_libs(libname, folder_name): if os.path.exists(libname_path): os.remove(libname_path) print( - f"Copying {pjoin(build_dir,folder_name, libname)} to {pjoin(build_lib, 'pyarrow')}") + f"Copying {pjoin(build_dir, folder_name, libname)} to {pjoin(build_lib, 'pyarrow')}") shutil.copy(pjoin(build_dir, folder_name, libname), pjoin(build_lib, "pyarrow")) @@ -324,9 +323,11 @@ def copy_libs(libname, folder_name): for libname in os.listdir(pjoin(build_dir, 'lib')): copy_libs(libname, 'lib') # For windows builds, move dll from bin - if os.path.isdir(pjoin(build_dir,' bin')): + try: for libname in os.listdir(pjoin(build_dir, 'bin')): copy_libs(libname, 'bin') + except OSError: + pass # Copy headers tp python/pyarrow/include arrow_python_include = pjoin(build_include, "arrow", "python") From 9c1e991495445d33e7d75c2376f7fffab7563810 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Fri, 15 Jul 2022 14:25:45 +0200 Subject: [PATCH 088/138] Fix linter errors --- python/setup.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/setup.py b/python/setup.py index f8b34947be1..852e1d5e208 100755 --- a/python/setup.py +++ b/python/setup.py @@ -315,7 +315,8 @@ def copy_libs(libname, folder_name): if os.path.exists(libname_path): os.remove(libname_path) print( - f"Copying {pjoin(build_dir, folder_name, libname)} to {pjoin(build_lib, 'pyarrow')}") + f"Copying {pjoin(build_dir, folder_name, libname)}" + f" to {pjoin(build_lib, 'pyarrow', libname)}") shutil.copy(pjoin(build_dir, folder_name, libname), pjoin(build_lib, "pyarrow")) @@ -336,7 +337,8 @@ def copy_libs(libname, folder_name): if os.path.exists(pyarrow_include): shutil.rmtree(pyarrow_include) print( - f"Copying include folder: {arrow_python_include} to {pyarrow_include}") + f"Copying include folder: {arrow_python_include}" + f" to {pyarrow_include}") shutil.copytree(arrow_python_include, pyarrow_include) def _run_cmake(self): From 89f4233f2b308719805baf868d61790e78b8f507 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Mon, 18 Jul 2022 06:19:37 +0200 Subject: [PATCH 089/138] Rename C PyArrow to PyArrow cpp --- cpp/cmake_modules/FindArrowPython.cmake | 4 +- cpp/cmake_modules/FindArrowPythonFlight.cmake | 2 +- python/setup.py | 41 ++++++++----------- 3 files changed, 21 insertions(+), 26 deletions(-) diff --git a/cpp/cmake_modules/FindArrowPython.cmake b/cpp/cmake_modules/FindArrowPython.cmake index 1d8c7d3f40e..bd470cc6097 100644 --- a/cpp/cmake_modules/FindArrowPython.cmake +++ b/cpp/cmake_modules/FindArrowPython.cmake @@ -46,9 +46,9 @@ endif() find_package(Arrow ${find_package_arguments}) if(ARROW_FOUND) - message(STATUS "CPYARROW_HOME: ${CPYARROW_HOME}") + message(STATUS "PYARROW_CPP_HOME: ${PYARROW_CPP_HOME}") arrow_find_package(ARROW_PYTHON - "${CPYARROW_HOME}" + "${PYARROW_CPP_HOME}" arrow_python arrow/python/api.h ArrowPython diff --git a/cpp/cmake_modules/FindArrowPythonFlight.cmake b/cpp/cmake_modules/FindArrowPythonFlight.cmake index 679f80d54b1..67007e4fad4 100644 --- a/cpp/cmake_modules/FindArrowPythonFlight.cmake +++ b/cpp/cmake_modules/FindArrowPythonFlight.cmake @@ -50,7 +50,7 @@ find_package(ArrowPython ${find_package_arguments}) if(ARROW_PYTHON_FOUND AND ARROW_FLIGHT_FOUND) arrow_find_package(ARROW_PYTHON_FLIGHT - "${CPYARROW_HOME}" + "${PYARROW_CPP_HOME}" arrow_python_flight arrow/python/flight.h ArrowPythonFlight diff --git a/python/setup.py b/python/setup.py index 852e1d5e208..188cbf54d19 100755 --- a/python/setup.py +++ b/python/setup.py @@ -93,7 +93,7 @@ def build_extensions(self): _build_ext.build_extensions(self) def run(self): - self._run_cmake_arrow_python() + self._run_cmake_pyarrow_cpp() self._run_cmake() _build_ext.run(self) @@ -228,7 +228,7 @@ def initialize_options(self): '_hdfsio', 'gandiva'] - def _run_cmake_arrow_python(self): + def _run_cmake_pyarrow_cpp(self): # check if build_type is correctly passed / set if self.build_type.lower() not in ('release', 'debug'): raise ValueError("--build-type (or PYARROW_BUILD_TYPE) needs to " @@ -236,8 +236,8 @@ def _run_cmake_arrow_python(self): # The directory containing this setup.py source = os.path.dirname(os.path.abspath(__file__)) - # The directory containing this C PyArrow CMakeLists.txt - source_cpyarrow = pjoin(source, "pyarrow/src_arrow") + # The directory containing this PyArrow cpp CMakeLists.txt + source_pyarrow_cpp = pjoin(source, "pyarrow/src_arrow") # The directory for the module being built build_cmd = self.get_finalized_command('build') @@ -293,14 +293,14 @@ def append_cmake_bool(value, varname): '-j{0}'.format(os.environ['PYARROW_PARALLEL'])) # run cmake - print("-- Running cmake for arrow python") - self.spawn(['cmake'] + cmake_options + [source_cpyarrow]) - print("-- Finished cmake for arrow python") + print("-- Running cmake for pyarrow cpp") + self.spawn(['cmake'] + cmake_options + [source_pyarrow_cpp]) + print("-- Finished cmake for pyarrow cpp") # run make & install - print("-- Running make build and install for arrow python") + print("-- Running make build and install for pyarrow cpp") self.spawn(['cmake', '--build', '.', '--target', 'install'] + build_tool_args) - print("-- Finished make build and install for arrow python") + print("-- Finished make build and install for pyarrow cpp") # Move the libraries to the place expected by the Python build try: @@ -331,15 +331,15 @@ def copy_libs(libname, folder_name): pass # Copy headers tp python/pyarrow/include - arrow_python_include = pjoin(build_include, "arrow", "python") + pyarrow_cpp_include = pjoin(build_include, "arrow", "python") pyarrow_include = pjoin( build_lib, "pyarrow", "include", "arrow", "python") if os.path.exists(pyarrow_include): shutil.rmtree(pyarrow_include) print( - f"Copying include folder: {arrow_python_include}" + f"Copying include folder: {pyarrow_cpp_include}" f" to {pyarrow_include}") - shutil.copytree(arrow_python_include, pyarrow_include) + shutil.copytree(pyarrow_cpp_include, pyarrow_include) def _run_cmake(self): # check if build_type is correctly passed / set @@ -384,7 +384,7 @@ def _run_cmake(self): cmake_options = [ '-DPYTHON_EXECUTABLE=%s' % sys.executable, '-DPython3_EXECUTABLE=%s' % sys.executable, - '-DCPYARROW_HOME=' + str(pjoin(build_lib, "pyarrow")), + '-DPYARROW_CPP_HOME=' + str(pjoin(build_lib, "pyarrow")), static_lib_option, ] @@ -469,13 +469,13 @@ def append_cmake_bool(value, varname): shutil.move(pjoin(build_prefix, 'include'), pjoin(build_lib, 'pyarrow')) - # We need to, again, add the C PyArrow include folder - build_cpyarrow_include = pjoin(saved_cwd, 'build/dist/include') + # We need to, again, add the PyArrow cpp include folder + build_pyarrow_cpp_include = pjoin(saved_cwd, 'build/dist/include') if not os.path.isdir( - pjoin(build_cpyarrow_include, "arrow", "python")): + pjoin(build_pyarrow_cpp_include, "arrow", "python")): self.mkpath( - pjoin(build_cpyarrow_include, "arrow", "python")) - shutil.move(pjoin(build_cpyarrow_include, "arrow", "python"), + pjoin(build_pyarrow_cpp_include, "arrow", "python")) + shutil.move(pjoin(build_pyarrow_cpp_include, "arrow", "python"), pjoin( build_lib, "pyarrow", "include", "arrow", "python")) @@ -527,15 +527,10 @@ def append_cmake_bool(value, varname): def _bundle_arrow_cpp(self, build_prefix, build_lib): print(pjoin(build_lib, 'pyarrow')) move_shared_libs(build_prefix, build_lib, "arrow") - move_shared_libs(build_prefix, build_lib, "arrow_python") if self.with_cuda: move_shared_libs(build_prefix, build_lib, "arrow_cuda") if self.with_substrait: move_shared_libs(build_prefix, build_lib, "arrow_substrait") - if self.with_flight: - move_shared_libs(build_prefix, build_lib, "arrow_flight") - move_shared_libs(build_prefix, build_lib, - "arrow_python_flight") if self.with_dataset: move_shared_libs(build_prefix, build_lib, "arrow_dataset") if self.with_plasma: From a78a6fb81b44218e673dc00d1e926606d18a7e73 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Mon, 18 Jul 2022 06:34:53 +0200 Subject: [PATCH 090/138] Remove ARROW_SOURCE_DIR and rename ARROW_SOURCE to ARROW_SOURCE_DIR --- python/pyarrow/src_arrow/CMakeLists.txt | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index 3470d584b49..25b2475f99a 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -29,15 +29,13 @@ endif() # # Define -# ARROW_SOURCE_DIR: location of arrow/cpp # CMAKE_MODULE_PATH: location of cmake_modules in python # get_filename_component(PYARROW_SOURCE_DIR ${CMAKE_SOURCE_DIR} DIRECTORY) get_filename_component(PYTHON_SOURCE_DIR ${PYARROW_SOURCE_DIR} DIRECTORY) -get_filename_component(ARROW_SOURCE ${PYTHON_SOURCE_DIR} DIRECTORY) -set(ARROW_CPP_SOURCE_DIR "${ARROW_SOURCE}/cpp") -set(ARROW_SOURCE_DIR "${ARROW_CPP_SOURCE_DIR}") +get_filename_component(ARROW_SOURCE_DIR ${PYTHON_SOURCE_DIR} DIRECTORY) +set(ARROW_CPP_SOURCE_DIR "${ARROW_SOURCE_DIR}/cpp") # normalize ARROW_HOME path file(TO_CMAKE_PATH "$ENV{ARROW_HOME}" ARROW_HOME) @@ -198,7 +196,7 @@ set(ARROW_CMAKE_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}") # Changing ARROW_SOURCE_DIR for sdist build # In this case cpp/cmake_modules doesn't exist -if(NOT EXISTS "${ARROW_SOURCE_DIR}/cmake_modules/Find${MODULE}.cmake") +if(NOT EXISTS "${ARROW_SOURCE_DIR}/cpp/cmake_modules/Find${MODULE}.cmake") set(ARROW_SOURCE_DIR ${PYTHON_SOURCE_DIR}) endif() From f712bd5e08677378aaecccf5da86dbcfeb04351a Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Mon, 18 Jul 2022 07:03:28 +0200 Subject: [PATCH 091/138] Remove include_directories from python flight build --- python/pyarrow/src_arrow/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index 25b2475f99a..3770473ef47 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -241,7 +241,7 @@ if(ARROW_FLIGHT AND ARROW_BUILD_SHARED) # other solution is to link gRPC shared everywhere instead of statically only # in Flight) find_package(ArrowFlight REQUIRED) - include_directories("${ARROW_CPP_SOURCE_DIR}/src" "${ARROW_CPP_SOURCE_DIR}/}/${ARROW_BUILD_DIR}/src") + # include_directories("${ARROW_CPP_SOURCE_DIR}/src" "${ARROW_CPP_SOURCE_DIR}/}/${ARROW_BUILD_DIR}/src") set(FLIGHT_LINK_LIBS arrow_flight_shared) From 00e572c7730140e4286deafa4fdd3d4015a26184 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Mon, 18 Jul 2022 07:08:26 +0200 Subject: [PATCH 092/138] Update python_test.sh --- ci/scripts/python_test.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ci/scripts/python_test.sh b/ci/scripts/python_test.sh index 985bff708c8..208822c65f4 100755 --- a/ci/scripts/python_test.sh +++ b/ci/scripts/python_test.sh @@ -58,7 +58,10 @@ export PYARROW_TEST_S3 # Testing Arrow Python if [ "${ARROW_BUILD_TESTS}" == "ON" ]; then pushd ${test_dir} - ctest + ctest \ + --output-on-failure \ + --parallel ${n_jobs} \ + --timeout 300 popd fi # Testing PyArrow From 4e2635b1bcee448140c8fdedc8397346f0188d05 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Mon, 18 Jul 2022 07:09:59 +0200 Subject: [PATCH 093/138] Apply suggestions from code review - Kou Typo fixes and some renaming Co-authored-by: Sutou Kouhei --- docs/source/developers/python.rst | 4 ++-- python/setup.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source/developers/python.rst b/docs/source/developers/python.rst index 4ccb4bcf4c7..b02fa2ae64c 100644 --- a/docs/source/developers/python.rst +++ b/docs/source/developers/python.rst @@ -134,7 +134,7 @@ install the `pytest-cython `_ plugin. Testing Arrow Python (C++ code) ------------------------------- If you want to run ctest for the tests that are included in the Arrow Python -module, you will need to build Arrow with ``DARROW_BUILD_TESTS=ON``. +module, you will need to build Arrow with ``-DARROW_BUILD_TESTS=ON``. .. note:: @@ -142,7 +142,7 @@ module, you will need to build Arrow with ``DARROW_BUILD_TESTS=ON``. from conda-forge, so we must use the ``BUNDLED`` source for building that dependency - In case you use conda add ``-DGTest_SOURCE=BUNDLED`` to the cmake flags + In case you use conda add ``-DGTest_SOURCE=BUNDLED`` to the CMake flags when building Arrow. After Arrow C++ and PyArrow are built, navigate to ``python/build/dist/temp`` diff --git a/python/setup.py b/python/setup.py index 188cbf54d19..8a09a131981 100755 --- a/python/setup.py +++ b/python/setup.py @@ -297,10 +297,10 @@ def append_cmake_bool(value, varname): self.spawn(['cmake'] + cmake_options + [source_pyarrow_cpp]) print("-- Finished cmake for pyarrow cpp") # run make & install - print("-- Running make build and install for pyarrow cpp") + print("-- Running cmake build and install for pyarrow cpp") self.spawn(['cmake', '--build', '.', '--target', 'install'] + build_tool_args) - print("-- Finished make build and install for pyarrow cpp") + print("-- Finished cmake build and install for pyarrow cpp") # Move the libraries to the place expected by the Python build try: From 2b4909f3d188565f7beb452a811878940b4fcab4 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Mon, 18 Jul 2022 07:14:56 +0200 Subject: [PATCH 094/138] Correct a follow-up link --- python/pyarrow/src_arrow/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index 3770473ef47..fc02870cb97 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -291,7 +291,7 @@ arrow_install_all_headers("arrow/python") # # Tests # The tests will be moved to Cython and are currently supported for bundled GTest -# Follow-up: https://issues.apache.org/jira/browse/ARROW-17016?filter=-1 +# Follow-up: https://issues.apache.org/jira/browse/ARROW-17016 # if(ARROW_BUILD_TESTS) From 74b69daadebce5710dc7e32f6c03d6fd3e282095 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Mon, 18 Jul 2022 07:23:16 +0200 Subject: [PATCH 095/138] Add missing C PyArrow naming changes --- python/pyarrow/src_arrow/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src_arrow/CMakeLists.txt index fc02870cb97..0dac5a8174a 100644 --- a/python/pyarrow/src_arrow/CMakeLists.txt +++ b/python/pyarrow/src_arrow/CMakeLists.txt @@ -104,10 +104,10 @@ set_source_files_properties(init.cc PROPERTIES SKIP_PRECOMPILE_HEADERS ON SKIP_UNITY_BUILD_INCLUSION ON) # -# Arrow vs C PyArrow options +# Arrow vs PyArrow cpp options # -# Check all the options from Arrow and C PyArrow to be in line +# Check all the options from Arrow and PyArrow cpp to be in line if(PYARROW_WITH_DATASET) find_package(ArrowDataset REQUIRED) endif() From 7f70630e63fab4b968b6ac7178cdcc1c781e9a9b Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Mon, 18 Jul 2022 09:06:28 +0200 Subject: [PATCH 096/138] Rename python/pyarrow/src_arrow to python/pyarrow/src and add a string change to utils_prepare.sh for ARROW_PYTHON_VERSION --- dev/release/utils-prepare.sh | 8 ++++++++ .../pyarrow/{src_arrow => src}/ArrowPythonConfig.cmake.in | 0 .../{src_arrow => src}/ArrowPythonFlightConfig.cmake.in | 0 python/pyarrow/{src_arrow => src}/CMakeLists.txt | 0 python/pyarrow/{src_arrow => src}/api.h | 0 .../pyarrow/{src_arrow => src}/arrow-python-flight.pc.in | 0 python/pyarrow/{src_arrow => src}/arrow-python.pc.in | 0 python/pyarrow/{src_arrow => src}/arrow_to_pandas.cc | 0 python/pyarrow/{src_arrow => src}/arrow_to_pandas.h | 0 .../pyarrow/{src_arrow => src}/arrow_to_python_internal.h | 0 python/pyarrow/{src_arrow => src}/benchmark.cc | 0 python/pyarrow/{src_arrow => src}/benchmark.h | 0 python/pyarrow/{src_arrow => src}/common.cc | 0 python/pyarrow/{src_arrow => src}/common.h | 0 python/pyarrow/{src_arrow => src}/csv.cc | 0 python/pyarrow/{src_arrow => src}/csv.h | 0 python/pyarrow/{src_arrow => src}/datetime.cc | 0 python/pyarrow/{src_arrow => src}/datetime.h | 0 python/pyarrow/{src_arrow => src}/decimal.cc | 0 python/pyarrow/{src_arrow => src}/decimal.h | 0 python/pyarrow/{src_arrow => src}/deserialize.cc | 0 python/pyarrow/{src_arrow => src}/deserialize.h | 0 python/pyarrow/{src_arrow => src}/extension_type.cc | 0 python/pyarrow/{src_arrow => src}/extension_type.h | 0 python/pyarrow/{src_arrow => src}/filesystem.cc | 0 python/pyarrow/{src_arrow => src}/filesystem.h | 0 python/pyarrow/{src_arrow => src}/flight.cc | 0 python/pyarrow/{src_arrow => src}/flight.h | 0 python/pyarrow/{src_arrow => src}/gdb.cc | 0 python/pyarrow/{src_arrow => src}/gdb.h | 0 python/pyarrow/{src_arrow => src}/helpers.cc | 0 python/pyarrow/{src_arrow => src}/helpers.h | 0 python/pyarrow/{src_arrow => src}/inference.cc | 0 python/pyarrow/{src_arrow => src}/inference.h | 0 python/pyarrow/{src_arrow => src}/init.cc | 0 python/pyarrow/{src_arrow => src}/init.h | 0 python/pyarrow/{src_arrow => src}/io.cc | 0 python/pyarrow/{src_arrow => src}/io.h | 0 python/pyarrow/{src_arrow => src}/ipc.cc | 0 python/pyarrow/{src_arrow => src}/ipc.h | 0 python/pyarrow/{src_arrow => src}/iterators.h | 0 python/pyarrow/{src_arrow => src}/numpy_convert.cc | 0 python/pyarrow/{src_arrow => src}/numpy_convert.h | 0 python/pyarrow/{src_arrow => src}/numpy_internal.h | 0 python/pyarrow/{src_arrow => src}/numpy_interop.h | 0 python/pyarrow/{src_arrow => src}/numpy_to_arrow.cc | 0 python/pyarrow/{src_arrow => src}/numpy_to_arrow.h | 0 python/pyarrow/{src_arrow => src}/parquet_encryption.cc | 0 python/pyarrow/{src_arrow => src}/parquet_encryption.h | 0 python/pyarrow/{src_arrow => src}/pch.h | 0 python/pyarrow/{src_arrow => src}/platform.h | 0 python/pyarrow/{src_arrow => src}/pyarrow.cc | 0 python/pyarrow/{src_arrow => src}/pyarrow.h | 0 python/pyarrow/{src_arrow => src}/pyarrow_api.h | 0 python/pyarrow/{src_arrow => src}/pyarrow_lib.h | 0 python/pyarrow/{src_arrow => src}/python_test.cc | 0 python/pyarrow/{src_arrow => src}/python_to_arrow.cc | 0 python/pyarrow/{src_arrow => src}/python_to_arrow.h | 0 python/pyarrow/{src_arrow => src}/serialize.cc | 0 python/pyarrow/{src_arrow => src}/serialize.h | 0 python/pyarrow/{src_arrow => src}/type_traits.h | 0 python/pyarrow/{src_arrow => src}/udf.cc | 0 python/pyarrow/{src_arrow => src}/udf.h | 0 python/pyarrow/{src_arrow => src}/util/CMakeLists.txt | 0 python/pyarrow/{src_arrow => src}/util/test_main.cc | 0 python/pyarrow/{src_arrow => src}/visibility.h | 0 66 files changed, 8 insertions(+) rename python/pyarrow/{src_arrow => src}/ArrowPythonConfig.cmake.in (100%) rename python/pyarrow/{src_arrow => src}/ArrowPythonFlightConfig.cmake.in (100%) rename python/pyarrow/{src_arrow => src}/CMakeLists.txt (100%) rename python/pyarrow/{src_arrow => src}/api.h (100%) rename python/pyarrow/{src_arrow => src}/arrow-python-flight.pc.in (100%) rename python/pyarrow/{src_arrow => src}/arrow-python.pc.in (100%) rename python/pyarrow/{src_arrow => src}/arrow_to_pandas.cc (100%) rename python/pyarrow/{src_arrow => src}/arrow_to_pandas.h (100%) rename python/pyarrow/{src_arrow => src}/arrow_to_python_internal.h (100%) rename python/pyarrow/{src_arrow => src}/benchmark.cc (100%) rename python/pyarrow/{src_arrow => src}/benchmark.h (100%) rename python/pyarrow/{src_arrow => src}/common.cc (100%) rename python/pyarrow/{src_arrow => src}/common.h (100%) rename python/pyarrow/{src_arrow => src}/csv.cc (100%) rename python/pyarrow/{src_arrow => src}/csv.h (100%) rename python/pyarrow/{src_arrow => src}/datetime.cc (100%) rename python/pyarrow/{src_arrow => src}/datetime.h (100%) rename python/pyarrow/{src_arrow => src}/decimal.cc (100%) rename python/pyarrow/{src_arrow => src}/decimal.h (100%) rename python/pyarrow/{src_arrow => src}/deserialize.cc (100%) rename python/pyarrow/{src_arrow => src}/deserialize.h (100%) rename python/pyarrow/{src_arrow => src}/extension_type.cc (100%) rename python/pyarrow/{src_arrow => src}/extension_type.h (100%) rename python/pyarrow/{src_arrow => src}/filesystem.cc (100%) rename python/pyarrow/{src_arrow => src}/filesystem.h (100%) rename python/pyarrow/{src_arrow => src}/flight.cc (100%) rename python/pyarrow/{src_arrow => src}/flight.h (100%) rename python/pyarrow/{src_arrow => src}/gdb.cc (100%) rename python/pyarrow/{src_arrow => src}/gdb.h (100%) rename python/pyarrow/{src_arrow => src}/helpers.cc (100%) rename python/pyarrow/{src_arrow => src}/helpers.h (100%) rename python/pyarrow/{src_arrow => src}/inference.cc (100%) rename python/pyarrow/{src_arrow => src}/inference.h (100%) rename python/pyarrow/{src_arrow => src}/init.cc (100%) rename python/pyarrow/{src_arrow => src}/init.h (100%) rename python/pyarrow/{src_arrow => src}/io.cc (100%) rename python/pyarrow/{src_arrow => src}/io.h (100%) rename python/pyarrow/{src_arrow => src}/ipc.cc (100%) rename python/pyarrow/{src_arrow => src}/ipc.h (100%) rename python/pyarrow/{src_arrow => src}/iterators.h (100%) rename python/pyarrow/{src_arrow => src}/numpy_convert.cc (100%) rename python/pyarrow/{src_arrow => src}/numpy_convert.h (100%) rename python/pyarrow/{src_arrow => src}/numpy_internal.h (100%) rename python/pyarrow/{src_arrow => src}/numpy_interop.h (100%) rename python/pyarrow/{src_arrow => src}/numpy_to_arrow.cc (100%) rename python/pyarrow/{src_arrow => src}/numpy_to_arrow.h (100%) rename python/pyarrow/{src_arrow => src}/parquet_encryption.cc (100%) rename python/pyarrow/{src_arrow => src}/parquet_encryption.h (100%) rename python/pyarrow/{src_arrow => src}/pch.h (100%) rename python/pyarrow/{src_arrow => src}/platform.h (100%) rename python/pyarrow/{src_arrow => src}/pyarrow.cc (100%) rename python/pyarrow/{src_arrow => src}/pyarrow.h (100%) rename python/pyarrow/{src_arrow => src}/pyarrow_api.h (100%) rename python/pyarrow/{src_arrow => src}/pyarrow_lib.h (100%) rename python/pyarrow/{src_arrow => src}/python_test.cc (100%) rename python/pyarrow/{src_arrow => src}/python_to_arrow.cc (100%) rename python/pyarrow/{src_arrow => src}/python_to_arrow.h (100%) rename python/pyarrow/{src_arrow => src}/serialize.cc (100%) rename python/pyarrow/{src_arrow => src}/serialize.h (100%) rename python/pyarrow/{src_arrow => src}/type_traits.h (100%) rename python/pyarrow/{src_arrow => src}/udf.cc (100%) rename python/pyarrow/{src_arrow => src}/udf.h (100%) rename python/pyarrow/{src_arrow => src}/util/CMakeLists.txt (100%) rename python/pyarrow/{src_arrow => src}/util/test_main.cc (100%) rename python/pyarrow/{src_arrow => src}/visibility.h (100%) diff --git a/dev/release/utils-prepare.sh b/dev/release/utils-prepare.sh index 7b8bcf062b1..f6ee589f157 100644 --- a/dev/release/utils-prepare.sh +++ b/dev/release/utils-prepare.sh @@ -121,6 +121,14 @@ update_versions() { git add setup.py popd + pushd "${ARROW_DIR}/python/pyarrow/src" + sed -i.bak -E -e \ + "s/^set\(ARROW_PYTHON_VERSION \".+\"\)/set(ARROW_PYTHON_VERSION \"${version}\")/" \ + CMakeLists.txt + rm -f CMakeLists.txt.bak + git add CMakeLists.txt + popd + pushd "${ARROW_DIR}/r" sed -i.bak -E -e \ "s/^Version: .+/Version: ${r_version}/" \ diff --git a/python/pyarrow/src_arrow/ArrowPythonConfig.cmake.in b/python/pyarrow/src/ArrowPythonConfig.cmake.in similarity index 100% rename from python/pyarrow/src_arrow/ArrowPythonConfig.cmake.in rename to python/pyarrow/src/ArrowPythonConfig.cmake.in diff --git a/python/pyarrow/src_arrow/ArrowPythonFlightConfig.cmake.in b/python/pyarrow/src/ArrowPythonFlightConfig.cmake.in similarity index 100% rename from python/pyarrow/src_arrow/ArrowPythonFlightConfig.cmake.in rename to python/pyarrow/src/ArrowPythonFlightConfig.cmake.in diff --git a/python/pyarrow/src_arrow/CMakeLists.txt b/python/pyarrow/src/CMakeLists.txt similarity index 100% rename from python/pyarrow/src_arrow/CMakeLists.txt rename to python/pyarrow/src/CMakeLists.txt diff --git a/python/pyarrow/src_arrow/api.h b/python/pyarrow/src/api.h similarity index 100% rename from python/pyarrow/src_arrow/api.h rename to python/pyarrow/src/api.h diff --git a/python/pyarrow/src_arrow/arrow-python-flight.pc.in b/python/pyarrow/src/arrow-python-flight.pc.in similarity index 100% rename from python/pyarrow/src_arrow/arrow-python-flight.pc.in rename to python/pyarrow/src/arrow-python-flight.pc.in diff --git a/python/pyarrow/src_arrow/arrow-python.pc.in b/python/pyarrow/src/arrow-python.pc.in similarity index 100% rename from python/pyarrow/src_arrow/arrow-python.pc.in rename to python/pyarrow/src/arrow-python.pc.in diff --git a/python/pyarrow/src_arrow/arrow_to_pandas.cc b/python/pyarrow/src/arrow_to_pandas.cc similarity index 100% rename from python/pyarrow/src_arrow/arrow_to_pandas.cc rename to python/pyarrow/src/arrow_to_pandas.cc diff --git a/python/pyarrow/src_arrow/arrow_to_pandas.h b/python/pyarrow/src/arrow_to_pandas.h similarity index 100% rename from python/pyarrow/src_arrow/arrow_to_pandas.h rename to python/pyarrow/src/arrow_to_pandas.h diff --git a/python/pyarrow/src_arrow/arrow_to_python_internal.h b/python/pyarrow/src/arrow_to_python_internal.h similarity index 100% rename from python/pyarrow/src_arrow/arrow_to_python_internal.h rename to python/pyarrow/src/arrow_to_python_internal.h diff --git a/python/pyarrow/src_arrow/benchmark.cc b/python/pyarrow/src/benchmark.cc similarity index 100% rename from python/pyarrow/src_arrow/benchmark.cc rename to python/pyarrow/src/benchmark.cc diff --git a/python/pyarrow/src_arrow/benchmark.h b/python/pyarrow/src/benchmark.h similarity index 100% rename from python/pyarrow/src_arrow/benchmark.h rename to python/pyarrow/src/benchmark.h diff --git a/python/pyarrow/src_arrow/common.cc b/python/pyarrow/src/common.cc similarity index 100% rename from python/pyarrow/src_arrow/common.cc rename to python/pyarrow/src/common.cc diff --git a/python/pyarrow/src_arrow/common.h b/python/pyarrow/src/common.h similarity index 100% rename from python/pyarrow/src_arrow/common.h rename to python/pyarrow/src/common.h diff --git a/python/pyarrow/src_arrow/csv.cc b/python/pyarrow/src/csv.cc similarity index 100% rename from python/pyarrow/src_arrow/csv.cc rename to python/pyarrow/src/csv.cc diff --git a/python/pyarrow/src_arrow/csv.h b/python/pyarrow/src/csv.h similarity index 100% rename from python/pyarrow/src_arrow/csv.h rename to python/pyarrow/src/csv.h diff --git a/python/pyarrow/src_arrow/datetime.cc b/python/pyarrow/src/datetime.cc similarity index 100% rename from python/pyarrow/src_arrow/datetime.cc rename to python/pyarrow/src/datetime.cc diff --git a/python/pyarrow/src_arrow/datetime.h b/python/pyarrow/src/datetime.h similarity index 100% rename from python/pyarrow/src_arrow/datetime.h rename to python/pyarrow/src/datetime.h diff --git a/python/pyarrow/src_arrow/decimal.cc b/python/pyarrow/src/decimal.cc similarity index 100% rename from python/pyarrow/src_arrow/decimal.cc rename to python/pyarrow/src/decimal.cc diff --git a/python/pyarrow/src_arrow/decimal.h b/python/pyarrow/src/decimal.h similarity index 100% rename from python/pyarrow/src_arrow/decimal.h rename to python/pyarrow/src/decimal.h diff --git a/python/pyarrow/src_arrow/deserialize.cc b/python/pyarrow/src/deserialize.cc similarity index 100% rename from python/pyarrow/src_arrow/deserialize.cc rename to python/pyarrow/src/deserialize.cc diff --git a/python/pyarrow/src_arrow/deserialize.h b/python/pyarrow/src/deserialize.h similarity index 100% rename from python/pyarrow/src_arrow/deserialize.h rename to python/pyarrow/src/deserialize.h diff --git a/python/pyarrow/src_arrow/extension_type.cc b/python/pyarrow/src/extension_type.cc similarity index 100% rename from python/pyarrow/src_arrow/extension_type.cc rename to python/pyarrow/src/extension_type.cc diff --git a/python/pyarrow/src_arrow/extension_type.h b/python/pyarrow/src/extension_type.h similarity index 100% rename from python/pyarrow/src_arrow/extension_type.h rename to python/pyarrow/src/extension_type.h diff --git a/python/pyarrow/src_arrow/filesystem.cc b/python/pyarrow/src/filesystem.cc similarity index 100% rename from python/pyarrow/src_arrow/filesystem.cc rename to python/pyarrow/src/filesystem.cc diff --git a/python/pyarrow/src_arrow/filesystem.h b/python/pyarrow/src/filesystem.h similarity index 100% rename from python/pyarrow/src_arrow/filesystem.h rename to python/pyarrow/src/filesystem.h diff --git a/python/pyarrow/src_arrow/flight.cc b/python/pyarrow/src/flight.cc similarity index 100% rename from python/pyarrow/src_arrow/flight.cc rename to python/pyarrow/src/flight.cc diff --git a/python/pyarrow/src_arrow/flight.h b/python/pyarrow/src/flight.h similarity index 100% rename from python/pyarrow/src_arrow/flight.h rename to python/pyarrow/src/flight.h diff --git a/python/pyarrow/src_arrow/gdb.cc b/python/pyarrow/src/gdb.cc similarity index 100% rename from python/pyarrow/src_arrow/gdb.cc rename to python/pyarrow/src/gdb.cc diff --git a/python/pyarrow/src_arrow/gdb.h b/python/pyarrow/src/gdb.h similarity index 100% rename from python/pyarrow/src_arrow/gdb.h rename to python/pyarrow/src/gdb.h diff --git a/python/pyarrow/src_arrow/helpers.cc b/python/pyarrow/src/helpers.cc similarity index 100% rename from python/pyarrow/src_arrow/helpers.cc rename to python/pyarrow/src/helpers.cc diff --git a/python/pyarrow/src_arrow/helpers.h b/python/pyarrow/src/helpers.h similarity index 100% rename from python/pyarrow/src_arrow/helpers.h rename to python/pyarrow/src/helpers.h diff --git a/python/pyarrow/src_arrow/inference.cc b/python/pyarrow/src/inference.cc similarity index 100% rename from python/pyarrow/src_arrow/inference.cc rename to python/pyarrow/src/inference.cc diff --git a/python/pyarrow/src_arrow/inference.h b/python/pyarrow/src/inference.h similarity index 100% rename from python/pyarrow/src_arrow/inference.h rename to python/pyarrow/src/inference.h diff --git a/python/pyarrow/src_arrow/init.cc b/python/pyarrow/src/init.cc similarity index 100% rename from python/pyarrow/src_arrow/init.cc rename to python/pyarrow/src/init.cc diff --git a/python/pyarrow/src_arrow/init.h b/python/pyarrow/src/init.h similarity index 100% rename from python/pyarrow/src_arrow/init.h rename to python/pyarrow/src/init.h diff --git a/python/pyarrow/src_arrow/io.cc b/python/pyarrow/src/io.cc similarity index 100% rename from python/pyarrow/src_arrow/io.cc rename to python/pyarrow/src/io.cc diff --git a/python/pyarrow/src_arrow/io.h b/python/pyarrow/src/io.h similarity index 100% rename from python/pyarrow/src_arrow/io.h rename to python/pyarrow/src/io.h diff --git a/python/pyarrow/src_arrow/ipc.cc b/python/pyarrow/src/ipc.cc similarity index 100% rename from python/pyarrow/src_arrow/ipc.cc rename to python/pyarrow/src/ipc.cc diff --git a/python/pyarrow/src_arrow/ipc.h b/python/pyarrow/src/ipc.h similarity index 100% rename from python/pyarrow/src_arrow/ipc.h rename to python/pyarrow/src/ipc.h diff --git a/python/pyarrow/src_arrow/iterators.h b/python/pyarrow/src/iterators.h similarity index 100% rename from python/pyarrow/src_arrow/iterators.h rename to python/pyarrow/src/iterators.h diff --git a/python/pyarrow/src_arrow/numpy_convert.cc b/python/pyarrow/src/numpy_convert.cc similarity index 100% rename from python/pyarrow/src_arrow/numpy_convert.cc rename to python/pyarrow/src/numpy_convert.cc diff --git a/python/pyarrow/src_arrow/numpy_convert.h b/python/pyarrow/src/numpy_convert.h similarity index 100% rename from python/pyarrow/src_arrow/numpy_convert.h rename to python/pyarrow/src/numpy_convert.h diff --git a/python/pyarrow/src_arrow/numpy_internal.h b/python/pyarrow/src/numpy_internal.h similarity index 100% rename from python/pyarrow/src_arrow/numpy_internal.h rename to python/pyarrow/src/numpy_internal.h diff --git a/python/pyarrow/src_arrow/numpy_interop.h b/python/pyarrow/src/numpy_interop.h similarity index 100% rename from python/pyarrow/src_arrow/numpy_interop.h rename to python/pyarrow/src/numpy_interop.h diff --git a/python/pyarrow/src_arrow/numpy_to_arrow.cc b/python/pyarrow/src/numpy_to_arrow.cc similarity index 100% rename from python/pyarrow/src_arrow/numpy_to_arrow.cc rename to python/pyarrow/src/numpy_to_arrow.cc diff --git a/python/pyarrow/src_arrow/numpy_to_arrow.h b/python/pyarrow/src/numpy_to_arrow.h similarity index 100% rename from python/pyarrow/src_arrow/numpy_to_arrow.h rename to python/pyarrow/src/numpy_to_arrow.h diff --git a/python/pyarrow/src_arrow/parquet_encryption.cc b/python/pyarrow/src/parquet_encryption.cc similarity index 100% rename from python/pyarrow/src_arrow/parquet_encryption.cc rename to python/pyarrow/src/parquet_encryption.cc diff --git a/python/pyarrow/src_arrow/parquet_encryption.h b/python/pyarrow/src/parquet_encryption.h similarity index 100% rename from python/pyarrow/src_arrow/parquet_encryption.h rename to python/pyarrow/src/parquet_encryption.h diff --git a/python/pyarrow/src_arrow/pch.h b/python/pyarrow/src/pch.h similarity index 100% rename from python/pyarrow/src_arrow/pch.h rename to python/pyarrow/src/pch.h diff --git a/python/pyarrow/src_arrow/platform.h b/python/pyarrow/src/platform.h similarity index 100% rename from python/pyarrow/src_arrow/platform.h rename to python/pyarrow/src/platform.h diff --git a/python/pyarrow/src_arrow/pyarrow.cc b/python/pyarrow/src/pyarrow.cc similarity index 100% rename from python/pyarrow/src_arrow/pyarrow.cc rename to python/pyarrow/src/pyarrow.cc diff --git a/python/pyarrow/src_arrow/pyarrow.h b/python/pyarrow/src/pyarrow.h similarity index 100% rename from python/pyarrow/src_arrow/pyarrow.h rename to python/pyarrow/src/pyarrow.h diff --git a/python/pyarrow/src_arrow/pyarrow_api.h b/python/pyarrow/src/pyarrow_api.h similarity index 100% rename from python/pyarrow/src_arrow/pyarrow_api.h rename to python/pyarrow/src/pyarrow_api.h diff --git a/python/pyarrow/src_arrow/pyarrow_lib.h b/python/pyarrow/src/pyarrow_lib.h similarity index 100% rename from python/pyarrow/src_arrow/pyarrow_lib.h rename to python/pyarrow/src/pyarrow_lib.h diff --git a/python/pyarrow/src_arrow/python_test.cc b/python/pyarrow/src/python_test.cc similarity index 100% rename from python/pyarrow/src_arrow/python_test.cc rename to python/pyarrow/src/python_test.cc diff --git a/python/pyarrow/src_arrow/python_to_arrow.cc b/python/pyarrow/src/python_to_arrow.cc similarity index 100% rename from python/pyarrow/src_arrow/python_to_arrow.cc rename to python/pyarrow/src/python_to_arrow.cc diff --git a/python/pyarrow/src_arrow/python_to_arrow.h b/python/pyarrow/src/python_to_arrow.h similarity index 100% rename from python/pyarrow/src_arrow/python_to_arrow.h rename to python/pyarrow/src/python_to_arrow.h diff --git a/python/pyarrow/src_arrow/serialize.cc b/python/pyarrow/src/serialize.cc similarity index 100% rename from python/pyarrow/src_arrow/serialize.cc rename to python/pyarrow/src/serialize.cc diff --git a/python/pyarrow/src_arrow/serialize.h b/python/pyarrow/src/serialize.h similarity index 100% rename from python/pyarrow/src_arrow/serialize.h rename to python/pyarrow/src/serialize.h diff --git a/python/pyarrow/src_arrow/type_traits.h b/python/pyarrow/src/type_traits.h similarity index 100% rename from python/pyarrow/src_arrow/type_traits.h rename to python/pyarrow/src/type_traits.h diff --git a/python/pyarrow/src_arrow/udf.cc b/python/pyarrow/src/udf.cc similarity index 100% rename from python/pyarrow/src_arrow/udf.cc rename to python/pyarrow/src/udf.cc diff --git a/python/pyarrow/src_arrow/udf.h b/python/pyarrow/src/udf.h similarity index 100% rename from python/pyarrow/src_arrow/udf.h rename to python/pyarrow/src/udf.h diff --git a/python/pyarrow/src_arrow/util/CMakeLists.txt b/python/pyarrow/src/util/CMakeLists.txt similarity index 100% rename from python/pyarrow/src_arrow/util/CMakeLists.txt rename to python/pyarrow/src/util/CMakeLists.txt diff --git a/python/pyarrow/src_arrow/util/test_main.cc b/python/pyarrow/src/util/test_main.cc similarity index 100% rename from python/pyarrow/src_arrow/util/test_main.cc rename to python/pyarrow/src/util/test_main.cc diff --git a/python/pyarrow/src_arrow/visibility.h b/python/pyarrow/src/visibility.h similarity index 100% rename from python/pyarrow/src_arrow/visibility.h rename to python/pyarrow/src/visibility.h From 764d47b1722c8f8d86ddec91da6fb9cab5b4afad Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Mon, 18 Jul 2022 09:15:24 +0200 Subject: [PATCH 097/138] Linter corrections --- python/setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/setup.py b/python/setup.py index 8a09a131981..de1725018d0 100755 --- a/python/setup.py +++ b/python/setup.py @@ -470,7 +470,8 @@ def append_cmake_bool(value, varname): pjoin(build_lib, 'pyarrow')) # We need to, again, add the PyArrow cpp include folder - build_pyarrow_cpp_include = pjoin(saved_cwd, 'build/dist/include') + build_pyarrow_cpp_include = pjoin( + saved_cwd, 'build/dist/include') if not os.path.isdir( pjoin(build_pyarrow_cpp_include, "arrow", "python")): self.mkpath( From da0a0cde37cb7baf7ffbd2009ed3701c0ffec3b6 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Mon, 18 Jul 2022 09:49:18 +0200 Subject: [PATCH 098/138] Correct missing changes /src_arrow to /src plus linter correction --- cpp/cmake_modules/FindArrowPythonFlight.cmake | 2 +- python/setup.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cpp/cmake_modules/FindArrowPythonFlight.cmake b/cpp/cmake_modules/FindArrowPythonFlight.cmake index 67007e4fad4..fb670fa7658 100644 --- a/cpp/cmake_modules/FindArrowPythonFlight.cmake +++ b/cpp/cmake_modules/FindArrowPythonFlight.cmake @@ -16,7 +16,7 @@ # under the License. # - Find Arrow Python Flight -# (python/pyarrrow/src_arrow/flight.h, libarrow_python_flight.a, libarrow_python_flight.so) +# (arrow/python/flight.h, libarrow_python_flight.a, libarrow_python_flight.so) # # This module requires Arrow from which it uses # arrow_find_package() diff --git a/python/setup.py b/python/setup.py index de1725018d0..4b926d85c47 100755 --- a/python/setup.py +++ b/python/setup.py @@ -237,7 +237,7 @@ def _run_cmake_pyarrow_cpp(self): # The directory containing this setup.py source = os.path.dirname(os.path.abspath(__file__)) # The directory containing this PyArrow cpp CMakeLists.txt - source_pyarrow_cpp = pjoin(source, "pyarrow/src_arrow") + source_pyarrow_cpp = pjoin(source, "pyarrow/src") # The directory for the module being built build_cmd = self.get_finalized_command('build') @@ -476,7 +476,8 @@ def append_cmake_bool(value, varname): pjoin(build_pyarrow_cpp_include, "arrow", "python")): self.mkpath( pjoin(build_pyarrow_cpp_include, "arrow", "python")) - shutil.move(pjoin(build_pyarrow_cpp_include, "arrow", "python"), + shutil.move(pjoin( + build_pyarrow_cpp_include, "arrow", "python"), pjoin( build_lib, "pyarrow", "include", "arrow", "python")) From 9c537cc443ae71ce3ab559dd626729948150fbce Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Mon, 18 Jul 2022 10:17:06 +0200 Subject: [PATCH 099/138] Linter correction --- python/setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/setup.py b/python/setup.py index 4b926d85c47..1e4e5f17631 100755 --- a/python/setup.py +++ b/python/setup.py @@ -477,10 +477,10 @@ def append_cmake_bool(value, varname): self.mkpath( pjoin(build_pyarrow_cpp_include, "arrow", "python")) shutil.move(pjoin( - build_pyarrow_cpp_include, "arrow", "python"), - pjoin( - build_lib, "pyarrow", "include", - "arrow", "python")) + build_pyarrow_cpp_include, "arrow", "python"), + pjoin( + build_lib, "pyarrow", "include", + "arrow", "python")) # Move the built C-extension to the place expected by the Python # build From 30b424efe7028ed0c4e095bc6ac36e0f72fadf24 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Mon, 18 Jul 2022 10:35:43 +0200 Subject: [PATCH 100/138] Add ARROW_PYTHON_VERSION to dev/release tests --- dev/release/01-prepare-test.rb | 7 +++++++ dev/release/post-11-bump-versions-test.rb | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/dev/release/01-prepare-test.rb b/dev/release/01-prepare-test.rb index fa00d6290c1..808b48d4c4f 100644 --- a/dev/release/01-prepare-test.rb +++ b/dev/release/01-prepare-test.rb @@ -197,6 +197,13 @@ def test_version_pre_tag "+set(MLARROW_VERSION \"#{@release_version}\")"], ], }, + { + path: "python/pyarrow/src/CMakeLists.txt", + hunks: [ + ["-set(ARROW_PYTHON_VERSION \"#{@snapshot_version}\")", + "+set(ARROW_PYTHON_VERSION \"#{@release_version}\")"], + ], + }, { path: "python/setup.py", hunks: [ diff --git a/dev/release/post-11-bump-versions-test.rb b/dev/release/post-11-bump-versions-test.rb index 5bcd5c55ea3..7f7def52800 100644 --- a/dev/release/post-11-bump-versions-test.rb +++ b/dev/release/post-11-bump-versions-test.rb @@ -144,6 +144,13 @@ def test_version_post_tag "+set(MLARROW_VERSION \"#{@next_snapshot_version}\")"], ], }, + { + path: "python/pyarrow/src/CMakeLists.txt", + hunks: [ + ["-set(ARROW_PYTHON_VERSION \"#{@snapshot_version}\")", + "+set(ARROW_PYTHON_VERSION \"#{@next_snapshot_version}\")"], + ], + }, { path: "python/setup.py", hunks: [ From 55a3c90d110687d280ba8bba581048c04d615ef0 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 19 Jul 2022 08:54:24 +0200 Subject: [PATCH 101/138] Change selection of lib folders to be copied in _run_cmake_pyarrow_cpp --- python/setup.py | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/python/setup.py b/python/setup.py index 1e4e5f17631..d08afb799a4 100755 --- a/python/setup.py +++ b/python/setup.py @@ -309,26 +309,25 @@ def append_cmake_bool(value, varname): pass # helper function - def copy_libs(libname, folder_name): - if "python" in libname: - libname_path = pjoin(build_lib, "pyarrow", libname) - if os.path.exists(libname_path): - os.remove(libname_path) - print( - f"Copying {pjoin(build_dir, folder_name, libname)}" - f" to {pjoin(build_lib, 'pyarrow', libname)}") - shutil.copy(pjoin(build_dir, folder_name, libname), - pjoin(build_lib, "pyarrow")) + def copy_libs(folder_name): + for libname in os.listdir(pjoin(build_dir, folder_name)): + if "python" in libname: + libname_path = pjoin(build_lib, "pyarrow", libname) + if os.path.exists(libname_path): + os.remove(libname_path) + print( + f"Copying {pjoin(build_dir, folder_name, libname)}" + f" to {pjoin(build_lib, 'pyarrow', libname)}") + shutil.copy(pjoin(build_dir, folder_name, libname), + pjoin(build_lib, "pyarrow")) # Move libraries to python/pyarrow - for libname in os.listdir(pjoin(build_dir, 'lib')): - copy_libs(libname, 'lib') # For windows builds, move dll from bin - try: - for libname in os.listdir(pjoin(build_dir, 'bin')): - copy_libs(libname, 'bin') - except OSError: - pass + for folder in ['lib', 'lib64', 'bin']: + try: + copy_libs(folder) + except OSError: + pass # Copy headers tp python/pyarrow/include pyarrow_cpp_include = pjoin(build_include, "arrow", "python") From df1fb0f691ad150bfd485d849bd5d2b76668c00f Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 19 Jul 2022 10:57:23 +0200 Subject: [PATCH 102/138] Add back include_directories for python flight to fix verify-rc-source-python-linux-ubuntu-18.04-amd64 failure --- python/pyarrow/src/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyarrow/src/CMakeLists.txt b/python/pyarrow/src/CMakeLists.txt index 0dac5a8174a..7a139127b6f 100644 --- a/python/pyarrow/src/CMakeLists.txt +++ b/python/pyarrow/src/CMakeLists.txt @@ -241,7 +241,7 @@ if(ARROW_FLIGHT AND ARROW_BUILD_SHARED) # other solution is to link gRPC shared everywhere instead of statically only # in Flight) find_package(ArrowFlight REQUIRED) - # include_directories("${ARROW_CPP_SOURCE_DIR}/src" "${ARROW_CPP_SOURCE_DIR}/}/${ARROW_BUILD_DIR}/src") + include_directories("${ARROW_CPP_SOURCE_DIR}/src" "${ARROW_CPP_SOURCE_DIR}/${ARROW_BUILD_DIR}/src") set(FLIGHT_LINK_LIBS arrow_flight_shared) From 2b90050b6e05ade971754d2964a3dcd729602149 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 19 Jul 2022 10:59:40 +0200 Subject: [PATCH 103/138] One more missing change from Arrow Python to PyArrow CPP --- python/pyarrow/src/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/pyarrow/src/CMakeLists.txt b/python/pyarrow/src/CMakeLists.txt index 7a139127b6f..d3b1ba0a976 100644 --- a/python/pyarrow/src/CMakeLists.txt +++ b/python/pyarrow/src/CMakeLists.txt @@ -438,8 +438,8 @@ if(ARROW_BUILD_TESTS) endif() else() - message(STATUS "Tests for Arrow Python not build") + message(STATUS "Tests for PyArrow CPP not build") message(STATUS "Set -DGTest_SOURCE=BUNDLED when building Arrow C++ - to enable building tests for Arrow Python") + to enable building tests for PyArrow CPP") endif() endif() \ No newline at end of file From fbbcb89cfdf0b82bb2d18691270a42d854537748 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 21 Jul 2022 05:57:26 +0200 Subject: [PATCH 104/138] Add changes to try fixing ubuntu extended build failures --- python/pyarrow/src/CMakeLists.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/pyarrow/src/CMakeLists.txt b/python/pyarrow/src/CMakeLists.txt index d3b1ba0a976..edb7c968a25 100644 --- a/python/pyarrow/src/CMakeLists.txt +++ b/python/pyarrow/src/CMakeLists.txt @@ -72,7 +72,7 @@ set(Python3_FIND_REGISTRY "LAST") set(Python3_FIND_FRAMEWORK "LAST") find_package(Python3Alt 3.7 REQUIRED) -include_directories(SYSTEM ${NUMPY_INCLUDE_DIRS} ${PYTHON_INCLUDE_DIRS} src) +include_directories(SYSTEM ${NUMPY_INCLUDE_DIRS} ${PYTHON_INCLUDE_DIRS} ${ARROW_INCLUDE_DIR} src) add_custom_target(arrow_python-all) add_custom_target(arrow_python) @@ -241,7 +241,6 @@ if(ARROW_FLIGHT AND ARROW_BUILD_SHARED) # other solution is to link gRPC shared everywhere instead of statically only # in Flight) find_package(ArrowFlight REQUIRED) - include_directories("${ARROW_CPP_SOURCE_DIR}/src" "${ARROW_CPP_SOURCE_DIR}/${ARROW_BUILD_DIR}/src") set(FLIGHT_LINK_LIBS arrow_flight_shared) From 5834b100021ece42155436a236702e3978bee6df Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 21 Jul 2022 06:11:03 +0200 Subject: [PATCH 105/138] Add print to check AppVeyor failure --- ci/appveyor-cpp-build.bat | 2 +- python/pyarrow/tests/test_cython.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ci/appveyor-cpp-build.bat b/ci/appveyor-cpp-build.bat index 4b90e49257c..7bb3e79366c 100644 --- a/ci/appveyor-cpp-build.bat +++ b/ci/appveyor-cpp-build.bat @@ -125,7 +125,7 @@ python setup.py develop -q || exit /B set PYTHONDEVMODE=1 -py.test -r sxX --durations=15 --pyargs pyarrow.tests || exit /B +py.test -s sxX --durations=15 --pyargs pyarrow.tests || exit /B @rem @rem Wheels are built and tested separately (see ARROW-5142). diff --git a/python/pyarrow/tests/test_cython.py b/python/pyarrow/tests/test_cython.py index 89fca4190ec..8221035b493 100644 --- a/python/pyarrow/tests/test_cython.py +++ b/python/pyarrow/tests/test_cython.py @@ -138,7 +138,11 @@ def test_cython_api(tmpdir): subprocess_env[var] = delim.join( pa.get_library_dirs() + [subprocess_env.get(var, '')] ) - + print("pa.get_library_dirs() = ", pa.get_library_dirs()) + print("pa.get_library_dirs() contents:") + for d in pa.get_library_dirs(): + print(f"* {d!r}: {sorted(os.listdir(d))}") + print() subprocess.check_call([sys.executable, '-c', code], stdout=subprocess.PIPE, env=subprocess_env) From 6bd575271341012d5df28a56b49bb12ad7105c56 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 21 Jul 2022 06:23:29 +0200 Subject: [PATCH 106/138] Change extra missing renames for pyarrow cpp --- ci/scripts/python_test.sh | 2 +- docs/source/developers/python.rst | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ci/scripts/python_test.sh b/ci/scripts/python_test.sh index 208822c65f4..cc788326680 100755 --- a/ci/scripts/python_test.sh +++ b/ci/scripts/python_test.sh @@ -55,7 +55,7 @@ export PYARROW_TEST_ORC export PYARROW_TEST_PARQUET export PYARROW_TEST_S3 -# Testing Arrow Python +# Testing PyArrow CPP if [ "${ARROW_BUILD_TESTS}" == "ON" ]; then pushd ${test_dir} ctest \ diff --git a/docs/source/developers/python.rst b/docs/source/developers/python.rst index b02fa2ae64c..c43a21531fc 100644 --- a/docs/source/developers/python.rst +++ b/docs/source/developers/python.rst @@ -131,9 +131,10 @@ for ``.py`` files or for ``.pyx`` and ``.pxi`` files. In this case you will also need to install the `pytest-cython `_ plugin. -Testing Arrow Python (C++ code) -------------------------------- -If you want to run ctest for the tests that are included in the Arrow Python +Testing PyArrow CPP +------------------- + +If you want to run ctest for the tests that are included in the PyArrow CPP module, you will need to build Arrow with ``-DARROW_BUILD_TESTS=ON``. .. note:: From 03d545b886b9ab19381bac6985db4a9b98af9dbb Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 21 Jul 2022 08:24:12 +0200 Subject: [PATCH 107/138] Changes from code review - Joris --- python/CMakeLists.txt | 4 ++-- python/setup.py | 20 +++++++++----------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index eed0e9800b1..66087fb9795 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -606,8 +606,8 @@ foreach(module ${CYTHON_EXTENSIONS}) # In the event that we are bundling the shared libraries (e.g. in a # manylinux1 wheel), we need to set the RPATH of the extensions to the - # root of the pyarrow/ package so that libarrow/libarrow_python are able - # to be loaded properly + # root of the pyarrow/ package so that libarrow is able to be + # loaded properly if(APPLE) set(module_install_rpath "@loader_path/") else() diff --git a/python/setup.py b/python/setup.py index d08afb799a4..167553dd798 100755 --- a/python/setup.py +++ b/python/setup.py @@ -262,12 +262,11 @@ def _run_cmake_pyarrow_cpp(self): with changed_dir(build_dir): # cmake args cmake_options = [ - '-DCMAKE_INSTALL_PREFIX=' + - str(pjoin(saved_cwd, 'build/dist')), - '-DCMAKE_BUILD_TYPE={0}'.format(self.build_type.lower()), + '-DCMAKE_INSTALL_PREFIX=' + str(build_dir), + '-DCMAKE_BUILD_TYPE=' + str(self.build_type.lower()), '-DARROW_BUILD_DIR=' + str(arrow_build_dir), - '-DPYTHON_EXECUTABLE=%s' % sys.executable, - '-DPython3_EXECUTABLE=%s' % sys.executable, + '-DPYTHON_EXECUTABLE=' + str(sys.executable), + '-DPython3_EXECUTABLE=' + str(sys.executable), ] # Check for specific options @@ -329,7 +328,7 @@ def copy_libs(folder_name): except OSError: pass - # Copy headers tp python/pyarrow/include + # Copy headers to python/pyarrow/include pyarrow_cpp_include = pjoin(build_include, "arrow", "python") pyarrow_include = pjoin( build_lib, "pyarrow", "include", "arrow", "python") @@ -468,13 +467,10 @@ def append_cmake_bool(value, varname): shutil.move(pjoin(build_prefix, 'include'), pjoin(build_lib, 'pyarrow')) - # We need to, again, add the PyArrow cpp include folder + # As the pyarrow/include file has been deleted in the previous step + # we need to, again, add the PyArrow cpp include folder also build_pyarrow_cpp_include = pjoin( saved_cwd, 'build/dist/include') - if not os.path.isdir( - pjoin(build_pyarrow_cpp_include, "arrow", "python")): - self.mkpath( - pjoin(build_pyarrow_cpp_include, "arrow", "python")) shutil.move(pjoin( build_pyarrow_cpp_include, "arrow", "python"), pjoin( @@ -532,6 +528,8 @@ def _bundle_arrow_cpp(self, build_prefix, build_lib): move_shared_libs(build_prefix, build_lib, "arrow_cuda") if self.with_substrait: move_shared_libs(build_prefix, build_lib, "arrow_substrait") + if self.with_flight: + move_shared_libs(build_prefix, build_lib, "arrow_flight") if self.with_dataset: move_shared_libs(build_prefix, build_lib, "arrow_dataset") if self.with_plasma: From d8fd79f1f25317214e763326da91f3cbd11c438d Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 21 Jul 2022 08:26:31 +0200 Subject: [PATCH 108/138] Just to test the CI remove some pytest flags for appveyor build --- ci/appveyor-cpp-build.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/appveyor-cpp-build.bat b/ci/appveyor-cpp-build.bat index 7bb3e79366c..52f6e01a234 100644 --- a/ci/appveyor-cpp-build.bat +++ b/ci/appveyor-cpp-build.bat @@ -125,7 +125,7 @@ python setup.py develop -q || exit /B set PYTHONDEVMODE=1 -py.test -s sxX --durations=15 --pyargs pyarrow.tests || exit /B +py.test -s --durations=15 --pyargs pyarrow.tests || exit /B @rem @rem Wheels are built and tested separately (see ARROW-5142). From 4b7bb714ba1f9016e640efd34d7fdc1ce0cedfc1 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 21 Jul 2022 10:17:42 +0200 Subject: [PATCH 109/138] Add tmpdir to paths for test_cython on win --- python/pyarrow/tests/test_cython.py | 6 +++++- python/setup.py | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/python/pyarrow/tests/test_cython.py b/python/pyarrow/tests/test_cython.py index 8221035b493..52877b689b5 100644 --- a/python/pyarrow/tests/test_cython.py +++ b/python/pyarrow/tests/test_cython.py @@ -136,8 +136,12 @@ def test_cython_api(tmpdir): delim, var = ':', 'LD_LIBRARY_PATH' subprocess_env[var] = delim.join( - pa.get_library_dirs() + [subprocess_env.get(var, '')] + pa.get_library_dirs() + [subprocess_env.get(var, '')] + [str(tmpdir)] ) + print("str(tmpdir) = ", str(tmpdir)) + print("str(tmpdir) = contents:") + print(sorted(os.listdir(str(tmpdir)))) + print() print("pa.get_library_dirs() = ", pa.get_library_dirs()) print("pa.get_library_dirs() contents:") for d in pa.get_library_dirs(): diff --git a/python/setup.py b/python/setup.py index 167553dd798..4dc78869503 100755 --- a/python/setup.py +++ b/python/setup.py @@ -467,8 +467,8 @@ def append_cmake_bool(value, varname): shutil.move(pjoin(build_prefix, 'include'), pjoin(build_lib, 'pyarrow')) - # As the pyarrow/include file has been deleted in the previous step - # we need to, again, add the PyArrow cpp include folder also + # pyarrow/include file is first deleted in the previous step + # so we need to add the PyArrow cpp include folder again build_pyarrow_cpp_include = pjoin( saved_cwd, 'build/dist/include') shutil.move(pjoin( From 4b24cf2c1f373cd81764cc0ba73fb7ea9f426812 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 21 Jul 2022 10:41:13 +0200 Subject: [PATCH 110/138] Linter corrections --- python/pyarrow/tests/test_cython.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/pyarrow/tests/test_cython.py b/python/pyarrow/tests/test_cython.py index 52877b689b5..2a8046af08e 100644 --- a/python/pyarrow/tests/test_cython.py +++ b/python/pyarrow/tests/test_cython.py @@ -136,7 +136,8 @@ def test_cython_api(tmpdir): delim, var = ':', 'LD_LIBRARY_PATH' subprocess_env[var] = delim.join( - pa.get_library_dirs() + [subprocess_env.get(var, '')] + [str(tmpdir)] + pa.get_library_dirs() + + [subprocess_env.get(var, '')] + [str(tmpdir)] ) print("str(tmpdir) = ", str(tmpdir)) print("str(tmpdir) = contents:") From f0ee97ca16ee90188c73da0135ba90cdc8a82145 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 21 Jul 2022 11:01:58 +0200 Subject: [PATCH 111/138] Code review suggestions - pitrou --- cpp/cmake_modules/FindArrowPython.cmake | 1 - python/pyarrow/src/CMakeLists.txt | 14 +++----------- python/pyarrow/src/common.h | 4 ++-- python/pyarrow/src/csv.h | 2 +- python/pyarrow/src/datetime.cc | 8 ++++---- python/pyarrow/tests/test_gdb.py | 2 +- 6 files changed, 11 insertions(+), 20 deletions(-) diff --git a/cpp/cmake_modules/FindArrowPython.cmake b/cpp/cmake_modules/FindArrowPython.cmake index bd470cc6097..5acd3dab1d0 100644 --- a/cpp/cmake_modules/FindArrowPython.cmake +++ b/cpp/cmake_modules/FindArrowPython.cmake @@ -46,7 +46,6 @@ endif() find_package(Arrow ${find_package_arguments}) if(ARROW_FOUND) - message(STATUS "PYARROW_CPP_HOME: ${PYARROW_CPP_HOME}") arrow_find_package(ARROW_PYTHON "${PYARROW_CPP_HOME}" arrow_python diff --git a/python/pyarrow/src/CMakeLists.txt b/python/pyarrow/src/CMakeLists.txt index edb7c968a25..a8e34d202c4 100644 --- a/python/pyarrow/src/CMakeLists.txt +++ b/python/pyarrow/src/CMakeLists.txt @@ -165,11 +165,11 @@ endif() # We require a C++11 compliant compiler set(CMAKE_CXX_STANDARD_REQUIRED ON) -# Needed gdb flags +# Needed compiler flags include(SetupCxxFlags) # -# shred/static link libs +# Shared/static link libs # set(ARROW_PYTHON_SHARED_LINK_LIBS arrow_shared) @@ -418,15 +418,7 @@ if(ARROW_BUILD_TESTS) target_include_directories(${TEST_NAME} SYSTEM PUBLIC "${ARROW_PYTHON_INCLUDES}") # Add the test - if(ARROW_TEST_MEMCHECK) - add_test(${TEST_NAME} - bash - -c - "cd '${CMAKE_SOURCE_DIR}'; \ - valgrind --suppressions=valgrind.supp --tool=memcheck --gen-suppressions=all \ - --num-callers=500 --leak-check=full --leak-check-heuristics=stdstring \ - --error-exitcode=1 ${TEST_PATH} ${ARG_TEST_ARGUMENTS}") - elseif(WIN32) + if(WIN32) add_test(${TEST_NAME} ${TEST_PATH}) else() add_test(${TEST_NAME} diff --git a/python/pyarrow/src/common.h b/python/pyarrow/src/common.h index 6558dbb1c99..768ff8dce44 100644 --- a/python/pyarrow/src/common.h +++ b/python/pyarrow/src/common.h @@ -21,10 +21,10 @@ #include #include "arrow/buffer.h" -#include "pyarrow.h" -#include "visibility.h" #include "arrow/result.h" #include "arrow/util/macros.h" +#include "pyarrow.h" +#include "visibility.h" namespace arrow { diff --git a/python/pyarrow/src/csv.h b/python/pyarrow/src/csv.h index 0992ae3c6e5..e6e53af0f85 100644 --- a/python/pyarrow/src/csv.h +++ b/python/pyarrow/src/csv.h @@ -23,8 +23,8 @@ #include #include "arrow/csv/options.h" -#include "common.h" #include "arrow/util/macros.h" +#include "common.h" namespace arrow { namespace py { diff --git a/python/pyarrow/src/datetime.cc b/python/pyarrow/src/datetime.cc index 164a2fe1f92..9604b529753 100644 --- a/python/pyarrow/src/datetime.cc +++ b/python/pyarrow/src/datetime.cc @@ -21,15 +21,15 @@ #include #include "arrow/array.h" -#include "arrow_to_python_internal.h" -#include "common.h" -#include "helpers.h" -#include "platform.h" #include "arrow/scalar.h" #include "arrow/status.h" #include "arrow/type.h" #include "arrow/util/logging.h" #include "arrow/util/value_parsing.h" +#include "arrow_to_python_internal.h" +#include "common.h" +#include "helpers.h" +#include "platform.h" namespace arrow { namespace py { diff --git a/python/pyarrow/tests/test_gdb.py b/python/pyarrow/tests/test_gdb.py index a0d2b3fda3b..1990198d9f1 100644 --- a/python/pyarrow/tests/test_gdb.py +++ b/python/pyarrow/tests/test_gdb.py @@ -154,7 +154,7 @@ def select_frame(self, func_name): # but it's not available on old GDB versions (such as 8.1.1), # so instead parse the stack trace for a matching frame number. out = self.run_command("info stack") - pat = r"(?mi)^#(\d+)\s+.* in " + re.escape(func_name) + pat = r"(?mi)^#(\d+)\s+.* in " + re.escape(func_name) + r"\b" m = re.search(pat, out) if m is None: pytest.fail(f"Could not select frame for function {func_name}") From 0501d052cb97adb41d00ef25cb0e62cca643ffae Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 21 Jul 2022 13:50:55 +0200 Subject: [PATCH 112/138] Add back arrow_python to move_shared_libs in setup.py --- python/pyarrow/tests/test_cython.py | 10 +--------- python/setup.py | 1 + 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/python/pyarrow/tests/test_cython.py b/python/pyarrow/tests/test_cython.py index 2a8046af08e..4effea22faa 100644 --- a/python/pyarrow/tests/test_cython.py +++ b/python/pyarrow/tests/test_cython.py @@ -139,15 +139,7 @@ def test_cython_api(tmpdir): pa.get_library_dirs() + [subprocess_env.get(var, '')] + [str(tmpdir)] ) - print("str(tmpdir) = ", str(tmpdir)) - print("str(tmpdir) = contents:") - print(sorted(os.listdir(str(tmpdir)))) - print() - print("pa.get_library_dirs() = ", pa.get_library_dirs()) - print("pa.get_library_dirs() contents:") - for d in pa.get_library_dirs(): - print(f"* {d!r}: {sorted(os.listdir(d))}") - print() + subprocess.check_call([sys.executable, '-c', code], stdout=subprocess.PIPE, env=subprocess_env) diff --git a/python/setup.py b/python/setup.py index 4dc78869503..3cea0e6b8c9 100755 --- a/python/setup.py +++ b/python/setup.py @@ -524,6 +524,7 @@ def append_cmake_bool(value, varname): def _bundle_arrow_cpp(self, build_prefix, build_lib): print(pjoin(build_lib, 'pyarrow')) move_shared_libs(build_prefix, build_lib, "arrow") + move_shared_libs(build_prefix, build_lib, "arrow_python") if self.with_cuda: move_shared_libs(build_prefix, build_lib, "arrow_cuda") if self.with_substrait: From 5cbdb40024cde32fff2a26fea4fcd1a7d0bb7210 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 21 Jul 2022 13:53:48 +0200 Subject: [PATCH 113/138] Add a fix for manylinux wheels - thank you Raul --- ci/scripts/python_wheel_manylinux_build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/scripts/python_wheel_manylinux_build.sh b/ci/scripts/python_wheel_manylinux_build.sh index af17606199e..47721e45e3b 100755 --- a/ci/scripts/python_wheel_manylinux_build.sh +++ b/ci/scripts/python_wheel_manylinux_build.sh @@ -151,6 +151,7 @@ export PYARROW_WITH_PARQUET_ENCRYPTION=${PARQUET_REQUIRE_ENCRYPTION} export PYARROW_WITH_PLASMA=${ARROW_PLASMA} export PYARROW_WITH_SUBSTRAIT=${ARROW_SUBSTRAIT} export PYARROW_WITH_S3=${ARROW_S3} +export ARROW_HOME=/tmp/arrow-dist # PyArrow build configuration export PKG_CONFIG_PATH=/usr/lib/pkgconfig:/tmp/arrow-dist/lib/pkgconfig From b1bdf71c254cd798d8b46cb5601572cb37873a76 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 21 Jul 2022 15:22:55 +0200 Subject: [PATCH 114/138] Add a fix for macos wheels --- ci/scripts/python_wheel_macos_build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/scripts/python_wheel_macos_build.sh b/ci/scripts/python_wheel_macos_build.sh index a6e763b6523..4b152b7075d 100755 --- a/ci/scripts/python_wheel_macos_build.sh +++ b/ci/scripts/python_wheel_macos_build.sh @@ -156,6 +156,7 @@ export PYARROW_WITH_PLASMA=${ARROW_PLASMA} export PYARROW_WITH_SUBSTRAIT=${ARROW_SUBSTRAIT} export PYARROW_WITH_S3=${ARROW_S3} export PYARROW_CMAKE_OPTIONS="-DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} -DARROW_SIMD_LEVEL=${ARROW_SIMD_LEVEL}" +export ARROW_HOME=${build_dir}/build # PyArrow build configuration export PKG_CONFIG_PATH=/usr/lib/pkgconfig:${build_dir}/install/lib/pkgconfig # Set PyArrow version explicitly From 1fe44a0681b8fccb7bfcb017b9389d159d9d3419 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 21 Jul 2022 20:58:14 +0200 Subject: [PATCH 115/138] Skip test_cython_api on windows and investigate the fail as a follow up --- python/pyarrow/tests/test_cython.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/pyarrow/tests/test_cython.py b/python/pyarrow/tests/test_cython.py index 4effea22faa..53c1d76e84c 100644 --- a/python/pyarrow/tests/test_cython.py +++ b/python/pyarrow/tests/test_cython.py @@ -81,6 +81,8 @@ def check_cython_example_module(mod): mod.cast_scalar(scal, pa.list_(pa.int64())) +@pytest.mark.skipif(sys.platform == "win32", + reason="ARROW-17172: currently fails on windows") @pytest.mark.cython def test_cython_api(tmpdir): """ @@ -139,7 +141,6 @@ def test_cython_api(tmpdir): pa.get_library_dirs() + [subprocess_env.get(var, '')] + [str(tmpdir)] ) - subprocess.check_call([sys.executable, '-c', code], stdout=subprocess.PIPE, env=subprocess_env) From c844ed948b90611f9922a607d846b39050ad88f4 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Fri, 22 Jul 2022 11:49:05 +0200 Subject: [PATCH 116/138] Correct ARROW_HOME path for macos wheels --- ci/scripts/python_wheel_macos_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/python_wheel_macos_build.sh b/ci/scripts/python_wheel_macos_build.sh index 4b152b7075d..6bd6ea22a35 100755 --- a/ci/scripts/python_wheel_macos_build.sh +++ b/ci/scripts/python_wheel_macos_build.sh @@ -156,7 +156,7 @@ export PYARROW_WITH_PLASMA=${ARROW_PLASMA} export PYARROW_WITH_SUBSTRAIT=${ARROW_SUBSTRAIT} export PYARROW_WITH_S3=${ARROW_S3} export PYARROW_CMAKE_OPTIONS="-DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} -DARROW_SIMD_LEVEL=${ARROW_SIMD_LEVEL}" -export ARROW_HOME=${build_dir}/build +export ARROW_HOME=${build_dir}/install # PyArrow build configuration export PKG_CONFIG_PATH=/usr/lib/pkgconfig:${build_dir}/install/lib/pkgconfig # Set PyArrow version explicitly From 03d63472338ee2be98c6094a8c2751377c7fdfd1 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Sun, 24 Jul 2022 15:34:04 +0200 Subject: [PATCH 117/138] Add a fix for windows wheels --- python/setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/setup.py b/python/setup.py index 3cea0e6b8c9..e1a4d58d0ef 100755 --- a/python/setup.py +++ b/python/setup.py @@ -297,7 +297,8 @@ def append_cmake_bool(value, varname): print("-- Finished cmake for pyarrow cpp") # run make & install print("-- Running cmake build and install for pyarrow cpp") - self.spawn(['cmake', '--build', '.', '--target', 'install'] + + self.spawn(['cmake', '--build', '.', '--config', + self.build_type, '--target', 'install'] + build_tool_args) print("-- Finished cmake build and install for pyarrow cpp") From f3ab721415c9349ba833ad4daa903e917b9c9f42 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Mon, 25 Jul 2022 18:50:56 +0200 Subject: [PATCH 118/138] Revert change in pytest for appveyor build --- ci/appveyor-cpp-build.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/appveyor-cpp-build.bat b/ci/appveyor-cpp-build.bat index 52f6e01a234..4b90e49257c 100644 --- a/ci/appveyor-cpp-build.bat +++ b/ci/appveyor-cpp-build.bat @@ -125,7 +125,7 @@ python setup.py develop -q || exit /B set PYTHONDEVMODE=1 -py.test -s --durations=15 --pyargs pyarrow.tests || exit /B +py.test -r sxX --durations=15 --pyargs pyarrow.tests || exit /B @rem @rem Wheels are built and tested separately (see ARROW-5142). From 1fdf1ed274030e62a6499de720783a8e11f436b6 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 26 Jul 2022 16:58:46 +0200 Subject: [PATCH 119/138] Test folder changed? Why didn't the CI complain :/ --- ci/scripts/python_test.sh | 2 +- docs/source/developers/python.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/scripts/python_test.sh b/ci/scripts/python_test.sh index cc788326680..769a0871e6f 100755 --- a/ci/scripts/python_test.sh +++ b/ci/scripts/python_test.sh @@ -20,7 +20,7 @@ set -ex arrow_dir=${1} -test_dir=${1}/python/build/dist/temp +test_dir=${1}/python/build/dist export ARROW_SOURCE_DIR=${arrow_dir} export ARROW_TEST_DATA=${arrow_dir}/testing/data diff --git a/docs/source/developers/python.rst b/docs/source/developers/python.rst index c43a21531fc..4dbcd67b2cd 100644 --- a/docs/source/developers/python.rst +++ b/docs/source/developers/python.rst @@ -146,12 +146,12 @@ module, you will need to build Arrow with ``-DARROW_BUILD_TESTS=ON``. In case you use conda add ``-DGTest_SOURCE=BUNDLED`` to the CMake flags when building Arrow. -After Arrow C++ and PyArrow are built, navigate to ``python/build/dist/temp`` +After Arrow C++ and PyArrow are built, navigate to ``python/build/dist`` folder and run ctest: .. code-block:: - $ pushd arrow/python/build/dist/temp + $ pushd arrow/python/build/dist $ ctest $ popd From e495e940c30e64232aa653ec4b9e1fe3113afcc3 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 26 Jul 2022 20:52:08 +0200 Subject: [PATCH 120/138] Add test flags that might be missing --- python/pyarrow/src/CMakeLists.txt | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/python/pyarrow/src/CMakeLists.txt b/python/pyarrow/src/CMakeLists.txt index a8e34d202c4..90d85571d20 100644 --- a/python/pyarrow/src/CMakeLists.txt +++ b/python/pyarrow/src/CMakeLists.txt @@ -428,6 +428,27 @@ if(ARROW_BUILD_TESTS) ${TEST_PATH}) endif() + # Add test as dependency of relevant targets + add_dependencies(all-tests ${TEST_NAME}) + add_dependencies(arrow_python-tests ${TEST_NAME}) + + set(LABELS) + list(APPEND LABELS "unittest" arrow_python-tests) + + # ensure there is a cmake target which exercises tests with this LABEL + set(LABEL_TEST_NAME "test-arrow_python-tests") + if(NOT TARGET ${LABEL_TEST_NAME}) + add_custom_target(${LABEL_TEST_NAME} + ctest -L "${LABEL}" --output-on-failure + USES_TERMINAL) + endif() + # ensure the test is (re)built before the LABEL test runs + add_dependencies(${LABEL_TEST_NAME} ${TEST_NAME}) + + set_property(TEST ${TEST_NAME} + APPEND + PROPERTY LABELS ${LABELS}) + else() message(STATUS "Tests for PyArrow CPP not build") message(STATUS "Set -DGTest_SOURCE=BUNDLED when building Arrow C++ From 627219616fb2e20446274deccc4c01b764c76cc8 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 26 Jul 2022 22:21:00 +0200 Subject: [PATCH 121/138] Ensure exactly one copy of the python lib folder is found and copied into pyarrow --- python/setup.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/python/setup.py b/python/setup.py index e1a4d58d0ef..b97e0ee602c 100755 --- a/python/setup.py +++ b/python/setup.py @@ -308,8 +308,11 @@ def append_cmake_bool(value, varname): except OSError: pass - # helper function - def copy_libs(folder_name): + # Move libraries to python/pyarrow + # For windows builds, move dll from bin + try: + folder_name, = (name for name in ["lib", "lib64", "bin"] + if os.path.exists(pjoin(build_dir, name))) for libname in os.listdir(pjoin(build_dir, folder_name)): if "python" in libname: libname_path = pjoin(build_lib, "pyarrow", libname) @@ -320,14 +323,11 @@ def copy_libs(folder_name): f" to {pjoin(build_lib, 'pyarrow', libname)}") shutil.copy(pjoin(build_dir, folder_name, libname), pjoin(build_lib, "pyarrow")) - - # Move libraries to python/pyarrow - # For windows builds, move dll from bin - for folder in ['lib', 'lib64', 'bin']: - try: - copy_libs(folder) - except OSError: - pass + except ValueError: + print("There are multiple or none libraries for PyArrow cpp ") + print("installed in the python/build/dist folder. Check the ") + print("installation process and be sure there is exactly one ") + print("library folder created") # Copy headers to python/pyarrow/include pyarrow_cpp_include = pjoin(build_include, "arrow", "python") From 2d415b7a10141f079c8448bad9595ebb09b4c00b Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Wed, 27 Jul 2022 02:10:26 +0200 Subject: [PATCH 122/138] Make win libs copying separate --- python/setup.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/python/setup.py b/python/setup.py index b97e0ee602c..a02d101002c 100755 --- a/python/setup.py +++ b/python/setup.py @@ -308,11 +308,8 @@ def append_cmake_bool(value, varname): except OSError: pass - # Move libraries to python/pyarrow - # For windows builds, move dll from bin - try: - folder_name, = (name for name in ["lib", "lib64", "bin"] - if os.path.exists(pjoin(build_dir, name))) + # helper function + def copy_libs(folder_name): for libname in os.listdir(pjoin(build_dir, folder_name)): if "python" in libname: libname_path = pjoin(build_lib, "pyarrow", libname) @@ -323,6 +320,17 @@ def append_cmake_bool(value, varname): f" to {pjoin(build_lib, 'pyarrow', libname)}") shutil.copy(pjoin(build_dir, folder_name, libname), pjoin(build_lib, "pyarrow")) + + # Move libraries to python/pyarrow + # For windows builds, move dll from bin + try: + copy_libs("bin") + except OSError: + pass + try: + folder_name, = (name for name in ["lib", "lib64"] + if os.path.exists(pjoin(build_dir, name))) + copy_libs(folder_name) except ValueError: print("There are multiple or none libraries for PyArrow cpp ") print("installed in the python/build/dist folder. Check the ") From 89bc96382fe74f775f5e51127b2915feb23ab70e Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 18 Aug 2022 16:05:32 +0200 Subject: [PATCH 123/138] Apply suggestions from code review - pitrou Co-authored-by: Antoine Pitrou --- ci/scripts/python_test.sh | 2 +- docs/source/developers/python.rst | 27 ++++++++++++++------------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/ci/scripts/python_test.sh b/ci/scripts/python_test.sh index 769a0871e6f..a18bc5b9133 100755 --- a/ci/scripts/python_test.sh +++ b/ci/scripts/python_test.sh @@ -55,7 +55,7 @@ export PYARROW_TEST_ORC export PYARROW_TEST_PARQUET export PYARROW_TEST_S3 -# Testing PyArrow CPP +# Testing PyArrow C++ if [ "${ARROW_BUILD_TESTS}" == "ON" ]; then pushd ${test_dir} ctest \ diff --git a/docs/source/developers/python.rst b/docs/source/developers/python.rst index 4dbcd67b2cd..c30efd2358f 100644 --- a/docs/source/developers/python.rst +++ b/docs/source/developers/python.rst @@ -131,23 +131,23 @@ for ``.py`` files or for ``.pyx`` and ``.pxi`` files. In this case you will also need to install the `pytest-cython `_ plugin. -Testing PyArrow CPP +Testing PyArrow C++ ------------------- -If you want to run ctest for the tests that are included in the PyArrow CPP -module, you will need to build Arrow with ``-DARROW_BUILD_TESTS=ON``. +Most of the tests for PyArrow are part of the ``pytest``-based test suite mentioned above, +but a few low-level tests are written directly in C++ for historical reasons. +Those tests can be run using ``ctest``, but you first will need to build Arrow C++ +with ``-DARROW_BUILD_TESTS=ON``. .. note:: - Currently building the C++ unit tests does not work with googletest - from conda-forge, so we must use the ``BUNDLED`` source for building that - dependency + Currently, building the PyArrow C++ unit tests does not work with the + googletest package from conda-forge. If you are in this situation, please + add ``-DGTest_SOURCE=BUNDLED`` to the CMake flags + when building Arrow C++. - In case you use conda add ``-DGTest_SOURCE=BUNDLED`` to the CMake flags - when building Arrow. - -After Arrow C++ and PyArrow are built, navigate to ``python/build/dist`` -folder and run ctest: +After Arrow C++ and PyArrow are built, you can navigate to the ``python/build/dist`` +folder and run ``ctest``: .. code-block:: @@ -418,8 +418,9 @@ set the ``PYARROW_PARALLEL`` environment variable. .. note:: If you used a different directory name for building Arrow C++ (by default it is - named "build"), then also set `ARROW_BUILD_DIR='name_of_build_dir'`. This way - pyarrow can find Arrow C++ built files. + named "build"), then you should also set the environment variable + ``ARROW_BUILD_DIR='name_of_build_dir'``. This way + PyArrow can find the Arrow C++ built files. If you wish to delete stale PyArrow build artifacts before rebuilding, navigate to the ``arrow/python`` folder and run ``git clean -Xfd .``. From bb7e870c504d53f4fd7b73818c8c9b0d59dc44c3 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 18 Aug 2022 16:07:43 +0200 Subject: [PATCH 124/138] Update arrow_python version --- python/pyarrow/src/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyarrow/src/CMakeLists.txt b/python/pyarrow/src/CMakeLists.txt index 90d85571d20..178b156a5c0 100644 --- a/python/pyarrow/src/CMakeLists.txt +++ b/python/pyarrow/src/CMakeLists.txt @@ -45,7 +45,7 @@ set(CMAKE_MODULE_PATH "${PYTHON_SOURCE_DIR}/cmake_modules" "${ARROW_HOME}/lib/cm # Arrow version # -set(ARROW_PYTHON_VERSION "9.0.0-SNAPSHOT") +set(ARROW_PYTHON_VERSION "10.0.0-SNAPSHOT") string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" ARROW_PYTHON_BASE_VERSION "${ARROW_PYTHON_VERSION}") # Need to set to ARRROW_VERSION before finding Arrow package! project(arrow_python VERSION "${ARROW_PYTHON_BASE_VERSION}") From 0a0bb0fd1101565956e180d43fbc2987f08bb994 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 18 Aug 2022 16:09:31 +0200 Subject: [PATCH 125/138] Remove tmpdir from the subprocess_env in test_cython_api --- python/pyarrow/tests/test_cython.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/pyarrow/tests/test_cython.py b/python/pyarrow/tests/test_cython.py index 53c1d76e84c..85d65383d8b 100644 --- a/python/pyarrow/tests/test_cython.py +++ b/python/pyarrow/tests/test_cython.py @@ -138,8 +138,7 @@ def test_cython_api(tmpdir): delim, var = ':', 'LD_LIBRARY_PATH' subprocess_env[var] = delim.join( - pa.get_library_dirs() + - [subprocess_env.get(var, '')] + [str(tmpdir)] + pa.get_library_dirs() + [subprocess_env.get(var, '')] ) subprocess.check_call([sys.executable, '-c', code], stdout=subprocess.PIPE, From f755b20a6134cb5b96cc30f5938df1301eebff11 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 23 Aug 2022 15:18:13 +0200 Subject: [PATCH 126/138] Remove use_python and use_flight from arrow.spec.in --- .../apache-arrow/yum/arrow.spec.in | 82 +------------------ 1 file changed, 1 insertion(+), 81 deletions(-) diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in index 59b62315627..0fea5a94a15 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in +++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in @@ -288,7 +288,7 @@ Requires: bzip2-devel %if %{use_flight} Requires: c-ares-devel %endif -Requires: curl-devel +Requires: curl-devel %if %{use_gcs} Requires: json-devel %endif @@ -506,86 +506,6 @@ Libraries and header files for Gandiva. %{_libdir}/pkgconfig/gandiva.pc %endif -%if %{use_python} -%package -n %{name}%{major_version}-python-libs -Summary: Python integration library for Apache Arrow -License: Apache-2.0 -Requires: %{name}%{major_version}-libs = %{version}-%{release} -Requires: python%{python_version}-numpy - -%description -n %{name}%{major_version}-python-libs -This package contains the Python integration library for Apache Arrow. - -%files -n %{name}%{major_version}-python-libs -%defattr(-,root,root,-) -%doc README.md -%license LICENSE.txt NOTICE.txt -%{_libdir}/libarrow_python.so.* - -%package python-devel -Summary: Libraries and header files for Python integration library for Apache Arrow -License: Apache-2.0 -Requires: %{name}%{major_version}-python-libs = %{version}-%{release} -Requires: %{name}-devel = %{version}-%{release} -Requires: python%{python_version}-devel - -%description python-devel -Libraries and header files for Python integration library for Apache Arrow. - -%files python-devel -%defattr(-,root,root,-) -%doc README.md -%license LICENSE.txt NOTICE.txt -%{_includedir}/arrow/python/ -%exclude %{_includedir}/arrow/python/flight.h -%{_libdir}/cmake/arrow/ArrowPythonConfig*.cmake -%{_libdir}/cmake/arrow/ArrowPythonTargets*.cmake -%{_libdir}/cmake/arrow/FindArrowPython.cmake -%{_libdir}/libarrow_python.a -%{_libdir}/libarrow_python.so -%{_libdir}/pkgconfig/arrow-python.pc - -%if %{use_flight} -%package -n %{name}%{major_version}-python-flight-libs -Summary: Python integration library for Apache Arrow Flight -License: Apache-2.0 -Requires: %{name}%{major_version}-flight-libs = %{version}-%{release} -Requires: %{name}%{major_version}-python-libs = %{version}-%{release} - -%description -n %{name}%{major_version}-python-flight-libs -This package contains the Python integration library for Apache Arrow Flight. - -%files -n %{name}%{major_version}-python-flight-libs -%defattr(-,root,root,-) -%doc README.md -%license LICENSE.txt NOTICE.txt -%{_libdir}/libarrow_python_flight.so.* - -%package python-flight-devel -Summary: Libraries and header files for Python integration library for Apache Arrow Flight. -License: Apache-2.0 -Requires: %{name}%{major_version}-python-flight-libs = %{version}-%{release} -Requires: %{name}-flight-devel = %{version}-%{release} -Requires: %{name}-python-devel = %{version}-%{release} - -%description python-flight-devel -Libraries and header files for Python integration library for -Apache Arrow Flight. - -%files python-flight-devel -%defattr(-,root,root,-) -%doc README.md -%license LICENSE.txt NOTICE.txt -%{_includedir}/arrow/python/flight.h -%{_libdir}/cmake/arrow/ArrowPythonFlightConfig*.cmake -%{_libdir}/cmake/arrow/ArrowPythonFlightTargets*.cmake -%{_libdir}/cmake/arrow/FindArrowPythonFlight.cmake -%{_libdir}/libarrow_python_flight.a -%{_libdir}/libarrow_python_flight.so -%{_libdir}/pkgconfig/arrow-python-flight.pc -%endif -%endif - %package -n plasma%{major_version}-libs Summary: Runtime libraries for Plasma in-memory object store License: Apache-2.0 From 77df0bb312b72b4fc8504c47b8dc60f3f71c20ac Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 23 Aug 2022 15:20:57 +0200 Subject: [PATCH 127/138] Remove Python related patterns from dev/tasks/tasks.yml --- dev/tasks/tasks.yml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index fa5cbf969cb..40133f287a4 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -627,14 +627,6 @@ tasks: - libarrow-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - libarrow-glib1000-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - libarrow-glib1000_{no_rc_version}-1_[a-z0-9]+.deb - {% if target != "ubuntu-bionic" %} - - libarrow-python-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-python-flight-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-python-flight1000-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow-python-flight1000_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-python1000-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow-python1000_{no_rc_version}-1_[a-z0-9]+.deb - {% endif %} - libarrow1000-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - libarrow1000_{no_rc_version}-1_[a-z0-9]+.deb - libgandiva-dev_{no_rc_version}-1_[a-z0-9]+.deb @@ -748,16 +740,6 @@ tasks: - arrow[0-9]+-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm {% endif %} - arrow[0-9]+-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - {% if not is_rhel7_based %} - - arrow-python-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - {% if architecture == "amd64" %} - - arrow-python-flight-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - - arrow[0-9]+-python-flight-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - - arrow[0-9]+-python-flight-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - {% endif %} - - arrow[0-9]+-python-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - - arrow[0-9]+-python-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - {% endif %} {% if architecture == "amd64" %} - arrow-{no_rc_version}-1.[a-z0-9]+.src.rpm {% endif %} From 543caee78fffe411be43e824f3f8abf61db64c69 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 23 Aug 2022 15:28:49 +0200 Subject: [PATCH 128/138] Remove lines that are started with @ENABLE_PYTHON@ in debian/control.in --- .../apache-arrow/debian/control.in | 58 +------------------ 1 file changed, 1 insertion(+), 57 deletions(-) diff --git a/dev/tasks/linux-packages/apache-arrow/debian/control.in b/dev/tasks/linux-packages/apache-arrow/debian/control.in index 7a7aeb8dce4..1e05faf2828 100644 --- a/dev/tasks/linux-packages/apache-arrow/debian/control.in +++ b/dev/tasks/linux-packages/apache-arrow/debian/control.in @@ -32,8 +32,6 @@ Build-Depends: nvidia-cuda-toolkit [!arm64], pkg-config, @USE_SYSTEM_GRPC@ protobuf-compiler-grpc, -@ENABLE_PYTHON@ python3-dev, -@ENABLE_PYTHON@ python3-numpy, valac, tzdata, zlib1g-dev @@ -119,35 +117,6 @@ Description: Apache Arrow is a data processing library for analysis . This package provides C++ library files for Flight SQL system. -@ENABLE_PYTHON@Package: libarrow-python1000 -@ENABLE_PYTHON@Section: libs -@ENABLE_PYTHON@Architecture: any -@ENABLE_PYTHON@Multi-Arch: same -@ENABLE_PYTHON@Pre-Depends: ${misc:Pre-Depends} -@ENABLE_PYTHON@Depends: -@ENABLE_PYTHON@ ${misc:Depends}, -@ENABLE_PYTHON@ ${shlibs:Depends}, -@ENABLE_PYTHON@ libarrow1000 (= ${binary:Version}), -@ENABLE_PYTHON@ python3, -@ENABLE_PYTHON@ python3-numpy -@ENABLE_PYTHON@Description: Apache Arrow is a data processing library for analysis -@ENABLE_PYTHON@ . -@ENABLE_PYTHON@ This package provides C++ library files for Python support. - -@ENABLE_PYTHON@Package: libarrow-python-flight1000 -@ENABLE_PYTHON@Section: libs -@ENABLE_PYTHON@Architecture: any -@ENABLE_PYTHON@Multi-Arch: same -@ENABLE_PYTHON@Pre-Depends: ${misc:Pre-Depends} -@ENABLE_PYTHON@Depends: -@ENABLE_PYTHON@ ${misc:Depends}, -@ENABLE_PYTHON@ ${shlibs:Depends}, -@ENABLE_PYTHON@ libarrow-flight1000 (= ${binary:Version}), -@ENABLE_PYTHON@ libarrow-python1000 (= ${binary:Version}) -@ENABLE_PYTHON@Description: Apache Arrow is a data processing library for analysis -@ENABLE_PYTHON@ . -@ENABLE_PYTHON@ This package provides C++ library files for Flight and Python support. - Package: libarrow-dev Section: libdevel Architecture: any @@ -158,7 +127,7 @@ Depends: libbrotli-dev, libbz2-dev, @USE_SYSTEM_C_ARES@ libc-ares-dev, - libcurl4-openssl-dev, + libcurl4-openssl-dev, @USE_SYSTEM_GRPC@ libgrpc++-dev, liblz4-dev, libre2-dev, @@ -223,31 +192,6 @@ Description: Apache Arrow is a data processing library for analysis . This package provides C++ header files for Flight SQL system. -@ENABLE_PYTHON@Package: libarrow-python-dev -@ENABLE_PYTHON@Section: libdevel -@ENABLE_PYTHON@Architecture: any -@ENABLE_PYTHON@Multi-Arch: same -@ENABLE_PYTHON@Depends: -@ENABLE_PYTHON@ ${misc:Depends}, -@ENABLE_PYTHON@ libarrow-dev (= ${binary:Version}), -@ENABLE_PYTHON@ libarrow-python1000 (= ${binary:Version}) -@ENABLE_PYTHON@Description: Apache Arrow is a data processing library for analysis -@ENABLE_PYTHON@ . -@ENABLE_PYTHON@ This package provides C++ header files for Python support. - -@ENABLE_PYTHON@Package: libarrow-python-flight-dev -@ENABLE_PYTHON@Section: libdevel -@ENABLE_PYTHON@Architecture: any -@ENABLE_PYTHON@Multi-Arch: same -@ENABLE_PYTHON@Depends: -@ENABLE_PYTHON@ ${misc:Depends}, -@ENABLE_PYTHON@ libarrow-flight-dev (= ${binary:Version}), -@ENABLE_PYTHON@ libarrow-python-dev (= ${binary:Version}), -@ENABLE_PYTHON@ libarrow-python-flight1000 (= ${binary:Version}) -@ENABLE_PYTHON@Description: Apache Arrow is a data processing library for analysis -@ENABLE_PYTHON@ . -@ENABLE_PYTHON@ This package provides C++ header files for Flight and Python support. - Package: libgandiva1000 Section: libs Architecture: any From 333e757ed164626922dee9a5d94bdce09a4b9020 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 23 Aug 2022 15:30:26 +0200 Subject: [PATCH 129/138] Remove *python* in debian folder --- .../apache-arrow/debian/libarrow-python-dev.install | 6 ------ .../apache-arrow/debian/libarrow-python-flight-dev.install | 6 ------ .../apache-arrow/debian/libarrow-python-flight1000.install | 1 - .../apache-arrow/debian/libarrow-python1000.install | 1 - 4 files changed, 14 deletions(-) delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-dev.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight-dev.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight1000.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libarrow-python1000.install diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-dev.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-dev.install deleted file mode 100644 index 807583f9845..00000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-dev.install +++ /dev/null @@ -1,6 +0,0 @@ -usr/lib/*/cmake/arrow/ArrowPythonConfig*.cmake -usr/lib/*/cmake/arrow/ArrowPythonTargets*.cmake -usr/lib/*/cmake/arrow/FindArrowPython.cmake -usr/lib/*/libarrow_python.a -usr/lib/*/libarrow_python.so -usr/lib/*/pkgconfig/arrow-python.pc diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight-dev.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight-dev.install deleted file mode 100644 index 6cf96e227e9..00000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight-dev.install +++ /dev/null @@ -1,6 +0,0 @@ -usr/lib/*/cmake/arrow/ArrowPythonFlightConfig*.cmake -usr/lib/*/cmake/arrow/ArrowPythonFlightTargets*.cmake -usr/lib/*/cmake/arrow/FindArrowPythonFlight.cmake -usr/lib/*/libarrow_python_flight.a -usr/lib/*/libarrow_python_flight.so -usr/lib/*/pkgconfig/arrow-python-flight.pc diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight1000.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight1000.install deleted file mode 100644 index b7cbfec1f05..00000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight1000.install +++ /dev/null @@ -1 +0,0 @@ -usr/lib/*/libarrow_python_flight.so.* diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python1000.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python1000.install deleted file mode 100644 index eef3e664837..00000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python1000.install +++ /dev/null @@ -1 +0,0 @@ -usr/lib/*/libarrow_python.so.* From d7289694c01a835bdf35f39c00644a0ba7032191 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 23 Aug 2022 15:31:36 +0200 Subject: [PATCH 130/138] Remove dev/tasks/linux-packages/apache-arrow/debian/*python* in rat_exclude_files.txt --- dev/release/rat_exclude_files.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index c9d0309425c..752d06fd595 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -98,10 +98,6 @@ dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.doc-base dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.install dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.links dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib1000.install -dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-dev.install -dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight-dev.install -dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight1000.install -dev/tasks/linux-packages/apache-arrow/debian/libarrow-python1000.install dev/tasks/linux-packages/apache-arrow/debian/libarrow1000.install dev/tasks/linux-packages/apache-arrow/debian/libgandiva-dev.install dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-dev.install From bc0e88d42664035f55ba80b24c64db760a49ee19 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 23 Aug 2022 15:38:36 +0200 Subject: [PATCH 131/138] Remove python3-numpy from dev/tasks/linux-packages/apache-arrow/apt/*/Dockerfile --- .../linux-packages/apache-arrow/apt/debian-bookworm/Dockerfile | 1 - .../linux-packages/apache-arrow/apt/debian-bullseye/Dockerfile | 1 - .../linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile | 1 - .../linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile | 1 - 4 files changed, 4 deletions(-) diff --git a/dev/tasks/linux-packages/apache-arrow/apt/debian-bookworm/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/debian-bookworm/Dockerfile index 152ac08a7ca..0b50d887128 100644 --- a/dev/tasks/linux-packages/apache-arrow/apt/debian-bookworm/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/apt/debian-bookworm/Dockerfile @@ -69,7 +69,6 @@ RUN \ pkg-config \ protobuf-compiler-grpc \ python3-dev \ - python3-numpy \ python3-pip \ rapidjson-dev \ tzdata \ diff --git a/dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye/Dockerfile index 440a07f09c2..600ecc6962e 100644 --- a/dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye/Dockerfile @@ -69,7 +69,6 @@ RUN \ pkg-config \ protobuf-compiler-grpc \ python3-dev \ - python3-numpy \ python3-pip \ rapidjson-dev \ tzdata \ diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile index 5abee7c2e09..cea9eeb6fac 100644 --- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile @@ -63,7 +63,6 @@ RUN \ nlohmann-json3-dev \ pkg-config \ python3-dev \ - python3-numpy \ python3-pip \ python3-setuptools \ rapidjson-dev \ diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile index 7e95c37642a..e305479f721 100644 --- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile @@ -67,7 +67,6 @@ RUN \ pkg-config \ protobuf-compiler-grpc \ python3-dev \ - python3-numpy \ python3-pip \ python3-setuptools \ rapidjson-dev \ From 2c8de0c3b8fbce3b0e8800b23cba27f2c2c30d92 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 23 Aug 2022 15:45:27 +0200 Subject: [PATCH 132/138] Remove Python related options in debian/rules --- dev/tasks/linux-packages/apache-arrow/debian/rules | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/dev/tasks/linux-packages/apache-arrow/debian/rules b/dev/tasks/linux-packages/apache-arrow/debian/rules index 3889ddc42ed..0ca195cd5c7 100755 --- a/dev/tasks/linux-packages/apache-arrow/debian/rules +++ b/dev/tasks/linux-packages/apache-arrow/debian/rules @@ -20,12 +20,7 @@ override_dh_auto_configure: else \ ARROW_CUDA=OFF; \ ARROW_PLASMA=OFF; \ - fi; \ - if python3 -c 'import numpy' > /dev/null 2>&1; then \ - ARROW_PYTHON=ON; \ - else \ - ARROW_PYTHON=OFF; \ - fi; \ + fi; \ \ dh_auto_configure \ --sourcedirectory=cpp \ --builddirectory=cpp_build \ @@ -49,7 +44,6 @@ override_dh_auto_configure: -DARROW_PACKAGE_KIND=deb \ -DARROW_PARQUET=ON \ -DARROW_PLASMA=$${ARROW_PLASMA} \ - -DARROW_PYTHON=$${ARROW_PYTHON} \ -DARROW_S3=ON \ -DARROW_USE_CCACHE=OFF \ -DARROW_WITH_BROTLI=ON \ @@ -62,9 +56,7 @@ override_dh_auto_configure: -DCMAKE_UNITY_BUILD=ON \ -DCUDAToolkit_ROOT=/usr \ -DPARQUET_BUILD_EXECUTABLES=ON \ - -DPARQUET_REQUIRE_ENCRYPTION=ON \ - -DPythonInterp_FIND_VERSION=ON \ - -DPythonInterp_FIND_VERSION_MAJOR=3 + -DPARQUET_REQUIRE_ENCRYPTION=ON override_dh_auto_build: dh_auto_build \ From 4ce2e5b894904ff5466a3e727eaa158b2210ba6f Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 23 Aug 2022 16:24:04 +0200 Subject: [PATCH 133/138] Remove *arrow_python* from conda-recipes/arrow-cpp/meta.yaml in the arrow-cpp tests block --- dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index dcf1afe4a1a..8f1ba74e3af 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -123,7 +123,6 @@ outputs: - test -f $PREFIX/lib/libarrow.so # [linux] - test -f $PREFIX/lib/libarrow_dataset.so # [linux] - test -f $PREFIX/lib/libarrow_flight.so # [linux] - - test -f $PREFIX/lib/libarrow_python.so # [linux] - test -f $PREFIX/lib/libparquet.so # [linux] - test -f $PREFIX/lib/libgandiva.so # [linux] - test -f $PREFIX/lib/libplasma.so # [linux] @@ -133,14 +132,12 @@ outputs: - if exist %PREFIX%\\Library\\bin\\arrow_cuda.dll exit 1 # [(cuda_compiler_version == "None") and win] - test -f $PREFIX/lib/libarrow.dylib # [osx] - test -f $PREFIX/lib/libarrow_dataset.dylib # [osx] - - test -f $PREFIX/lib/libarrow_python.dylib # [osx] - test -f $PREFIX/lib/libgandiva.dylib # [osx and not arm64] - test -f $PREFIX/lib/libparquet.dylib # [osx] - test -f $PREFIX/lib/libplasma.dylib # [osx] - if not exist %PREFIX%\\Library\\bin\\arrow.dll exit 1 # [win] - if not exist %PREFIX%\\Library\\bin\\arrow_dataset.dll exit 1 # [win] - if not exist %PREFIX%\\Library\\bin\\arrow_flight.dll exit 1 # [win] - - if not exist %PREFIX%\\Library\\bin\\arrow_python.dll exit 1 # [win] - if not exist %PREFIX%\\Library\\bin\\parquet.dll exit 1 # [win] - if not exist %PREFIX%\\Library\\bin\\gandiva.dll exit 1 # [win] @@ -148,14 +145,12 @@ outputs: - test ! -f $PREFIX/lib/libarrow.a # [unix] - test ! -f $PREFIX/lib/libarrow_dataset.a # [unix] - test ! -f $PREFIX/lib/libarrow_flight.a # [unix] - - test ! -f $PREFIX/lib/libarrow_python.a # [unix] - test ! -f $PREFIX/lib/libplasma.a # [unix] - test ! -f $PREFIX/lib/libparquet.a # [unix] - test ! -f $PREFIX/lib/libgandiva.a # [unix] - if exist %PREFIX%\\Library\\lib\\arrow_static.lib exit 1 # [win] - if exist %PREFIX%\\Library\\lib\\arrow_dataset_static.lib exit 1 # [win] - if exist %PREFIX%\\Library\\lib\\arrow_flight_static.lib exit 1 # [win] - - if exist %PREFIX%\\Library\\lib\\arrow_python_static.lib exit 1 # [win] - if exist %PREFIX%\\Library\\lib\\parquet_static.lib exit 1 # [win] - if exist %PREFIX%\\Library\\lib\\gandiva_static.lib exit 1 # [win] From 6ab3c6f2cd427936829540014728d573801dc694 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Wed, 24 Aug 2022 07:11:20 +0200 Subject: [PATCH 134/138] Remove tests from dev/release/verify-apt.sh --- dev/release/verify-yum.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/dev/release/verify-yum.sh b/dev/release/verify-yum.sh index 93bbcfe1366..6d40bccf2bc 100755 --- a/dev/release/verify-yum.sh +++ b/dev/release/verify-yum.sh @@ -250,12 +250,6 @@ if [ "${have_flight}" = "yes" ]; then echo "::endgroup::" fi -if [ "${have_python}" = "yes" ]; then - echo "::group::Test libarrow-python" - ${install_command} --enablerepo=epel arrow-python-devel-${package_version} - echo "::endgroup::" -fi - echo "::group::Test Plasma" if [ "${have_glib}" = "yes" ]; then ${install_command} --enablerepo=epel plasma-glib-devel-${package_version} From 5219046c6f9da6c48ee20b619ab01750d0fbb1c1 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Wed, 24 Aug 2022 12:36:44 +0200 Subject: [PATCH 135/138] Correct typo in debian/rules --- dev/tasks/linux-packages/apache-arrow/debian/rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/linux-packages/apache-arrow/debian/rules b/dev/tasks/linux-packages/apache-arrow/debian/rules index 0ca195cd5c7..b9664c15a7c 100755 --- a/dev/tasks/linux-packages/apache-arrow/debian/rules +++ b/dev/tasks/linux-packages/apache-arrow/debian/rules @@ -20,7 +20,7 @@ override_dh_auto_configure: else \ ARROW_CUDA=OFF; \ ARROW_PLASMA=OFF; \ - fi; \ \ + fi; \ dh_auto_configure \ --sourcedirectory=cpp \ --builddirectory=cpp_build \ From 1cb4b39518336407f1699f437278b060cab7f918 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Wed, 24 Aug 2022 14:16:21 +0200 Subject: [PATCH 136/138] Remove tests from dev/release/verify-apt.sh - before the test was removed from verify-yum.sh --- dev/release/verify-apt.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/dev/release/verify-apt.sh b/dev/release/verify-apt.sh index 121abdbc6eb..5de5209a55c 100755 --- a/dev/release/verify-apt.sh +++ b/dev/release/verify-apt.sh @@ -198,13 +198,6 @@ ruby -r gi -e "p GI.load('ArrowFlightSQL')" echo "::endgroup::" -if [ "${have_python}" = "yes" ]; then - echo "::group::Test libarrow-python" - ${APT_INSTALL} libarrow-python-dev=${package_version} - echo "::endgroup::" -fi - - if [ "${have_plasma}" = "yes" ]; then echo "::group::Test Plasma" ${APT_INSTALL} libplasma-glib-dev=${package_version} From 767dda9bd900199b68166cc7bca18ad90bb9316d Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 25 Aug 2022 10:02:48 +0200 Subject: [PATCH 137/138] Remove extra python occurances in arrow.spec.in --- .../linux-packages/apache-arrow/yum/arrow.spec.in | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in index 0fea5a94a15..872f461fa4a 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in +++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in @@ -64,7 +64,6 @@ %define use_gflags (!%{is_amazon_linux}) %define use_glog (%{rhel} <= 8) %define use_mimalloc (%{rhel} >= 8) -%define use_python (%{rhel} >= 8) # TODO: Enable this. This works on local but is fragile on GitHub Actions and # Travis CI. # %%define use_s3 (%%{rhel} >= 8) @@ -115,10 +114,6 @@ BuildRequires: lz4-devel %{lz4_requirement} BuildRequires: ninja-build BuildRequires: openssl-devel BuildRequires: pkgconfig -%if %{use_python} -BuildRequires: python%{python_version}-devel -BuildRequires: python%{python_version}-numpy -%endif %if %{have_rapidjson} BuildRequires: rapidjson-devel %endif @@ -175,9 +170,6 @@ cd cpp -DARROW_PACKAGE_KIND=rpm \ -DARROW_PARQUET=ON \ -DARROW_PLASMA=ON \ -%if %{use_python} - -DARROW_PYTHON=ON \ -%endif %if %{use_s3} -DARROW_S3=ON \ %endif @@ -191,10 +183,6 @@ cd cpp -DCMAKE_BUILD_TYPE=$cpp_build_type \ -DPARQUET_BUILD_EXECUTABLES=ON \ -DPARQUET_REQUIRE_ENCRYPTION=ON \ -%if %{use_python} - -DPythonInterp_FIND_VERSION=ON \ - -DPythonInterp_FIND_VERSION_MAJOR=3 \ -%endif -G"Unix Makefiles" %arrow_cmake_build cd - From 0860306fb2d205cc9d36b83f71e09f5b882741cc Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 25 Aug 2022 10:29:23 +0200 Subject: [PATCH 138/138] Remove python*-devel and python*-numpy from dev/tasks/linux-packages/apache-arrow/yum/*/Dockerfile --- .../linux-packages/apache-arrow/yum/almalinux-8/Dockerfile | 2 -- .../linux-packages/apache-arrow/yum/almalinux-9/Dockerfile | 2 -- dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile | 2 -- .../linux-packages/apache-arrow/yum/centos-8-stream/Dockerfile | 2 -- .../linux-packages/apache-arrow/yum/centos-9-stream/Dockerfile | 2 -- 5 files changed, 10 deletions(-) diff --git a/dev/tasks/linux-packages/apache-arrow/yum/almalinux-8/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/almalinux-8/Dockerfile index a75fd022bda..5a5fd903bfc 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/almalinux-8/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/yum/almalinux-8/Dockerfile @@ -52,8 +52,6 @@ RUN \ openssl-devel \ pkg-config \ python39 \ - python39-devel \ - python39-numpy \ python39-pip \ re2-devel \ # rapidjson-devel \ diff --git a/dev/tasks/linux-packages/apache-arrow/yum/almalinux-9/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/almalinux-9/Dockerfile index df63f7a1a7e..dde7930ff5b 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/almalinux-9/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/yum/almalinux-9/Dockerfile @@ -51,8 +51,6 @@ RUN \ openssl-devel \ pkg-config \ python3 \ - python3-devel \ - python3-numpy \ python3-pip \ re2-devel \ rapidjson-devel \ diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile index 9c93e2f2407..04e74012f95 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile @@ -47,8 +47,6 @@ RUN \ openssl-devel \ pkg-config \ python36 \ - python36-devel \ - python36-numpy \ rapidjson-devel \ rpmdevtools \ snappy-devel \ diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-8-stream/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/centos-8-stream/Dockerfile index b29cc4565bd..5dba632628a 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/centos-8-stream/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/yum/centos-8-stream/Dockerfile @@ -52,8 +52,6 @@ RUN \ openssl-devel \ pkg-config \ python39 \ - python39-devel \ - python39-numpy \ python39-pip \ re2-devel \ rapidjson-devel \ diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-9-stream/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/centos-9-stream/Dockerfile index 38e6ae4531e..8bcd97fa6fe 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/centos-9-stream/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/yum/centos-9-stream/Dockerfile @@ -51,8 +51,6 @@ RUN \ openssl-devel \ pkg-config \ python3 \ - python3-devel \ - python3-numpy \ python3-pip \ re2-devel \ rapidjson-devel \