From aa34b96d3674c6109005b2cc6649bd51b952fd40 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Fri, 8 Mar 2024 09:53:20 +0100 Subject: [PATCH 01/11] GH-40376: [Python] Update for NumPy 2.0 ABI change in PyArray_Descr->elsize Co-authored-by: Sebastian Berg --- python/pyarrow/src/arrow/python/arrow_to_pandas.cc | 2 +- python/pyarrow/src/arrow/python/numpy_convert.cc | 2 +- python/pyarrow/src/arrow/python/numpy_interop.h | 5 +++++ python/pyarrow/src/arrow/python/numpy_to_arrow.cc | 4 ++-- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/python/pyarrow/src/arrow/python/arrow_to_pandas.cc b/python/pyarrow/src/arrow/python/arrow_to_pandas.cc index cb9cbe5b930..e29f5b31f2f 100644 --- a/python/pyarrow/src/arrow/python/arrow_to_pandas.cc +++ b/python/pyarrow/src/arrow/python/arrow_to_pandas.cc @@ -276,7 +276,7 @@ Status PyArray_NewFromPool(int nd, npy_intp* dims, PyArray_Descr* descr, MemoryP // // * Track allocations // * Get better performance through custom allocators - int64_t total_size = descr->elsize; + int64_t total_size = PyDataType_ELSIZE(descr); for (int i = 0; i < nd; ++i) { total_size *= dims[i]; } diff --git a/python/pyarrow/src/arrow/python/numpy_convert.cc b/python/pyarrow/src/arrow/python/numpy_convert.cc index dfee88c092e..f46487a579c 100644 --- a/python/pyarrow/src/arrow/python/numpy_convert.cc +++ b/python/pyarrow/src/arrow/python/numpy_convert.cc @@ -46,7 +46,7 @@ NumPyBuffer::NumPyBuffer(PyObject* ao) : Buffer(nullptr, 0) { PyArrayObject* ndarray = reinterpret_cast(ao); auto ptr = reinterpret_cast(PyArray_DATA(ndarray)); data_ = const_cast(ptr); - size_ = PyArray_SIZE(ndarray) * PyArray_DESCR(ndarray)->elsize; + size_ = PyArray_NBYTES(ndarray); capacity_ = size_; is_mutable_ = !!(PyArray_FLAGS(ndarray) & NPY_ARRAY_WRITEABLE); } diff --git a/python/pyarrow/src/arrow/python/numpy_interop.h b/python/pyarrow/src/arrow/python/numpy_interop.h index ce7baed259f..4a3bb4784be 100644 --- a/python/pyarrow/src/arrow/python/numpy_interop.h +++ b/python/pyarrow/src/arrow/python/numpy_interop.h @@ -67,6 +67,11 @@ #define NPY_INT32_IS_INT 0 #endif +// Backported NumPy 2 API (can be removed if numpy 2 is required) +#if NPY_ABI_VERSION < 0x02000000 + #define PyDataType_ELSIZE(descr) ((descr)->elsize) +#endif + namespace arrow { namespace py { diff --git a/python/pyarrow/src/arrow/python/numpy_to_arrow.cc b/python/pyarrow/src/arrow/python/numpy_to_arrow.cc index 8903df31be8..04572536cc8 100644 --- a/python/pyarrow/src/arrow/python/numpy_to_arrow.cc +++ b/python/pyarrow/src/arrow/python/numpy_to_arrow.cc @@ -196,7 +196,7 @@ class NumPyConverter { mask_ = reinterpret_cast(mo); } length_ = static_cast(PyArray_SIZE(arr_)); - itemsize_ = static_cast(PyArray_DESCR(arr_)->elsize); + itemsize_ = static_cast(PyArray_ITEMSIZE(arr_)); stride_ = static_cast(PyArray_STRIDES(arr_)[0]); } @@ -296,7 +296,7 @@ class NumPyConverter { PyArrayObject* mask_; int64_t length_; int64_t stride_; - int itemsize_; + int64_t itemsize_; bool from_pandas_; compute::CastOptions cast_options_; From 27f15abeba7ffc350896439038ff0b9749111610 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Fri, 8 Mar 2024 10:36:42 +0100 Subject: [PATCH 02/11] fix c_metadata and fields as well --- python/pyarrow/src/arrow/python/arrow_to_pandas.cc | 8 +++++--- python/pyarrow/src/arrow/python/numpy_convert.cc | 4 ++-- python/pyarrow/src/arrow/python/numpy_interop.h | 4 +++- python/pyarrow/src/arrow/python/numpy_to_arrow.cc | 11 +++++++---- 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/python/pyarrow/src/arrow/python/arrow_to_pandas.cc b/python/pyarrow/src/arrow/python/arrow_to_pandas.cc index e29f5b31f2f..023ba5585e7 100644 --- a/python/pyarrow/src/arrow/python/arrow_to_pandas.cc +++ b/python/pyarrow/src/arrow/python/arrow_to_pandas.cc @@ -255,7 +255,8 @@ Status SetBufferBase(PyArrayObject* arr, const std::shared_ptr& buffer) } inline void set_numpy_metadata(int type, const DataType* datatype, PyArray_Descr* out) { - auto metadata = reinterpret_cast(out->c_metadata); + auto metadata = + reinterpret_cast(PyDataType_C_METADATA(out)); if (type == NPY_DATETIME) { if (datatype->id() == Type::TIMESTAMP) { const auto& timestamp_type = checked_cast(*datatype); @@ -537,8 +538,9 @@ class PandasWriter { void SetDatetimeUnit(NPY_DATETIMEUNIT unit) { PyAcquireGIL lock; - auto date_dtype = reinterpret_cast( - PyArray_DESCR(reinterpret_cast(block_arr_.obj()))->c_metadata); + auto date_dtype = + reinterpret_cast(PyDataType_C_METADATA( + PyArray_DESCR(reinterpret_cast(block_arr_.obj())))); date_dtype->meta.base = unit; } diff --git a/python/pyarrow/src/arrow/python/numpy_convert.cc b/python/pyarrow/src/arrow/python/numpy_convert.cc index f46487a579c..5fd2cb511ff 100644 --- a/python/pyarrow/src/arrow/python/numpy_convert.cc +++ b/python/pyarrow/src/arrow/python/numpy_convert.cc @@ -150,7 +150,7 @@ Result> NumPyDtypeToArrow(PyArray_Descr* descr) { TO_ARROW_TYPE_CASE(UNICODE, utf8); case NPY_DATETIME: { auto date_dtype = - reinterpret_cast(descr->c_metadata); + reinterpret_cast(PyDataType_C_METADATA(descr)); switch (date_dtype->meta.base) { case NPY_FR_s: return timestamp(TimeUnit::SECOND); @@ -170,7 +170,7 @@ Result> NumPyDtypeToArrow(PyArray_Descr* descr) { } break; case NPY_TIMEDELTA: { auto timedelta_dtype = - reinterpret_cast(descr->c_metadata); + reinterpret_cast(PyDataType_C_METADATA(descr)); switch (timedelta_dtype->meta.base) { case NPY_FR_s: return duration(TimeUnit::SECOND); diff --git a/python/pyarrow/src/arrow/python/numpy_interop.h b/python/pyarrow/src/arrow/python/numpy_interop.h index 4a3bb4784be..7ea7d6e16f5 100644 --- a/python/pyarrow/src/arrow/python/numpy_interop.h +++ b/python/pyarrow/src/arrow/python/numpy_interop.h @@ -69,7 +69,9 @@ // Backported NumPy 2 API (can be removed if numpy 2 is required) #if NPY_ABI_VERSION < 0x02000000 - #define PyDataType_ELSIZE(descr) ((descr)->elsize) +#define PyDataType_ELSIZE(descr) ((descr)->elsize) +#define PyDataType_C_METADATA(descr) ((descr)->c_metadata) +#define PyDataType_FIELDS(descr) ((descr)->fields) #endif namespace arrow { diff --git a/python/pyarrow/src/arrow/python/numpy_to_arrow.cc b/python/pyarrow/src/arrow/python/numpy_to_arrow.cc index 04572536cc8..97fda4cd3cb 100644 --- a/python/pyarrow/src/arrow/python/numpy_to_arrow.cc +++ b/python/pyarrow/src/arrow/python/numpy_to_arrow.cc @@ -478,7 +478,8 @@ inline Status NumPyConverter::ConvertData(std::shared_ptr* d RETURN_NOT_OK(PrepareInputData(data)); - auto date_dtype = reinterpret_cast(dtype_->c_metadata); + auto date_dtype = + reinterpret_cast(PyDataType_C_METADATA(dtype_)); if (dtype_->type_num == NPY_DATETIME) { // If we have inbound datetime64[D] data, this needs to be downcasted // separately here from int64_t to int32_t, because this data is not @@ -514,7 +515,8 @@ inline Status NumPyConverter::ConvertData(std::shared_ptr* d RETURN_NOT_OK(PrepareInputData(data)); - auto date_dtype = reinterpret_cast(dtype_->c_metadata); + auto date_dtype = + reinterpret_cast(PyDataType_C_METADATA(dtype_)); if (dtype_->type_num == NPY_DATETIME) { // If we have inbound datetime64[D] data, this needs to be downcasted // separately here from int64_t to int32_t, because this data is not @@ -750,12 +752,13 @@ Status NumPyConverter::Visit(const StructType& type) { PyAcquireGIL gil_lock; // Create converters for each struct type field - if (dtype_->fields == NULL || !PyDict_Check(dtype_->fields)) { + if (PyDataType_FIELDS(dtype_) == NULL || !PyDict_Check(PyDataType_FIELDS(dtype_))) { return Status::TypeError("Expected struct array"); } for (auto field : type.fields()) { - PyObject* tup = PyDict_GetItemString(dtype_->fields, field->name().c_str()); + PyObject* tup = + PyDict_GetItemString(PyDataType_FIELDS(dtype_), field->name().c_str()); if (tup == NULL) { return Status::Invalid("Missing field '", field->name(), "' in struct array"); } From 269874a25ad29c6f9def6f4e0d03b8394590e652 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Fri, 8 Mar 2024 14:25:44 +0100 Subject: [PATCH 03/11] add static cast --- python/pyarrow/src/arrow/python/numpy_to_arrow.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/pyarrow/src/arrow/python/numpy_to_arrow.cc b/python/pyarrow/src/arrow/python/numpy_to_arrow.cc index 97fda4cd3cb..460b1d0ce3f 100644 --- a/python/pyarrow/src/arrow/python/numpy_to_arrow.cc +++ b/python/pyarrow/src/arrow/python/numpy_to_arrow.cc @@ -630,11 +630,11 @@ namespace { // NumPy unicode is UCS4/UTF32 always constexpr int kNumPyUnicodeSize = 4; -Status AppendUTF32(const char* data, int itemsize, int byteorder, +Status AppendUTF32(const char* data, int64_t itemsize, int byteorder, ::arrow::internal::ChunkedStringBuilder* builder) { // The binary \x00\x00\x00\x00 indicates a nul terminator in NumPy unicode, // so we need to detect that here to truncate if necessary. Yep. - int actual_length = 0; + Py_ssize_t actual_length = 0; for (; actual_length < itemsize / kNumPyUnicodeSize; ++actual_length) { const char* code_point = data + actual_length * kNumPyUnicodeSize; if ((*code_point == '\0') && (*(code_point + 1) == '\0') && @@ -707,7 +707,7 @@ Status NumPyConverter::Visit(const StringType& type) { auto AppendNonNullValue = [&](const uint8_t* data) { if (is_binary_type) { if (ARROW_PREDICT_TRUE(util::ValidateUTF8(data, itemsize_))) { - return builder.Append(data, itemsize_); + return builder.Append(data, static_cast(itemsize_)); } else { return Status::Invalid("Encountered non-UTF8 binary value: ", HexEncode(data, itemsize_)); From 712da0f0e0ea511285a6cf1456a230fe37b916c0 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Mon, 11 Mar 2024 08:42:24 +0100 Subject: [PATCH 04/11] temp disable cache for building wheel --- dev/tasks/python-wheels/github.linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/python-wheels/github.linux.yml b/dev/tasks/python-wheels/github.linux.yml index 41b18684cee..a80729298b4 100644 --- a/dev/tasks/python-wheels/github.linux.yml +++ b/dev/tasks/python-wheels/github.linux.yml @@ -43,7 +43,7 @@ jobs: - name: Build wheel shell: bash - run: archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-manylinux-{{ manylinux_version }} + run: archery docker run --no-cache -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-manylinux-{{ manylinux_version }} - uses: actions/upload-artifact@v3 with: From a778695307079b2159c51e9ed40ef7c209ae62aa Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Mon, 11 Mar 2024 16:07:02 +0100 Subject: [PATCH 05/11] undo linux no-cache, temp disable pull on windows --- dev/tasks/python-wheels/github.linux.yml | 2 +- dev/tasks/python-wheels/github.windows.yml | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/dev/tasks/python-wheels/github.linux.yml b/dev/tasks/python-wheels/github.linux.yml index a80729298b4..41b18684cee 100644 --- a/dev/tasks/python-wheels/github.linux.yml +++ b/dev/tasks/python-wheels/github.linux.yml @@ -43,7 +43,7 @@ jobs: - name: Build wheel shell: bash - run: archery docker run --no-cache -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-manylinux-{{ manylinux_version }} + run: archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-manylinux-{{ manylinux_version }} - uses: actions/upload-artifact@v3 with: diff --git a/dev/tasks/python-wheels/github.windows.yml b/dev/tasks/python-wheels/github.windows.yml index 01f4977a9b0..50875bb10d7 100644 --- a/dev/tasks/python-wheels/github.windows.yml +++ b/dev/tasks/python-wheels/github.windows.yml @@ -52,10 +52,11 @@ jobs: @rem We can remove this workaround once we find a way to use @rem pulled caches when build an image. echo on - archery docker pull --no-ignore-pull-failures python-wheel-windows-vs2019 - if errorlevel 1 ( - archery docker build --no-pull python-wheel-windows-vs2019 || exit /B 1 - ) + @rem archery docker pull --no-ignore-pull-failures python-wheel-windows-vs2019 + @rem if errorlevel 1 ( + @rem archery docker build --no-pull python-wheel-windows-vs2019 || exit /B 1 + @rem ) + archery docker build --no-pull python-wheel-windows-vs2019 || exit /B 1 archery docker run --no-build -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-windows-vs2019 - uses: actions/upload-artifact@v3 From 3eea035c42d87a6adbfa5037a6184c755c157e85 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Tue, 12 Mar 2024 09:33:31 +0100 Subject: [PATCH 06/11] temp debug prints --- python/setup.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/setup.py b/python/setup.py index 798bd6b05fd..14bc01959eb 100755 --- a/python/setup.py +++ b/python/setup.py @@ -24,6 +24,7 @@ import re import shlex import sys +import pathlib if sys.version_info >= (3, 10): import sysconfig @@ -81,6 +82,9 @@ class build_ext(_build_ext): def build_extensions(self): numpy_incl = pkg_resources.resource_filename('numpy', 'core/include') + print("NumPy include dir: ", numpy_incl) + path = pathlib.Path(numpy_incl) + print(list((path / "numpy").iterdir())) self.extensions = [ext for ext in self.extensions if ext.name != '__dummy__'] From 905c3d44c5082795ee95382433f6defd3e9c287f Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Tue, 12 Mar 2024 12:20:08 +0100 Subject: [PATCH 07/11] tweak docker file to ensure rebuild of specific layer --- ci/docker/python-wheel-windows-vs2019.dockerfile | 3 ++- dev/tasks/python-wheels/github.windows.yml | 9 ++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ci/docker/python-wheel-windows-vs2019.dockerfile b/ci/docker/python-wheel-windows-vs2019.dockerfile index b8e8aad952b..9bf7f8db323 100644 --- a/ci/docker/python-wheel-windows-vs2019.dockerfile +++ b/ci/docker/python-wheel-windows-vs2019.dockerfile @@ -89,7 +89,8 @@ RUN python -m pip install -U pip setuptools COPY python/requirements-wheel-build.txt arrow/python/ # TODO(GH-39848) Remove the `--pre --extra-index-url` for numpy nightly again before the 16.0 release -RUN python -m pip install -r arrow/python/requirements-wheel-build.txt --pre --extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" +RUN python -m pip install -r arrow/python/requirements-wheel-build.txt +RUN python -m pip install numpy -U --pre --extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" # ENV CLCACHE_DIR="C:\clcache" # ENV CLCACHE_COMPRESS=1 diff --git a/dev/tasks/python-wheels/github.windows.yml b/dev/tasks/python-wheels/github.windows.yml index 50875bb10d7..01f4977a9b0 100644 --- a/dev/tasks/python-wheels/github.windows.yml +++ b/dev/tasks/python-wheels/github.windows.yml @@ -52,11 +52,10 @@ jobs: @rem We can remove this workaround once we find a way to use @rem pulled caches when build an image. echo on - @rem archery docker pull --no-ignore-pull-failures python-wheel-windows-vs2019 - @rem if errorlevel 1 ( - @rem archery docker build --no-pull python-wheel-windows-vs2019 || exit /B 1 - @rem ) - archery docker build --no-pull python-wheel-windows-vs2019 || exit /B 1 + archery docker pull --no-ignore-pull-failures python-wheel-windows-vs2019 + if errorlevel 1 ( + archery docker build --no-pull python-wheel-windows-vs2019 || exit /B 1 + ) archery docker run --no-build -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-windows-vs2019 - uses: actions/upload-artifact@v3 From 74caa263ae32e654c403822ed2b529e5cc0a5d99 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Tue, 12 Mar 2024 12:37:01 +0100 Subject: [PATCH 08/11] print numpy version and include dir in cmake logs --- python/CMakeLists.txt | 3 +++ python/setup.py | 4 ---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index c3a1c578689..8c98e269d6f 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -260,6 +260,9 @@ set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}") # Python and Numpy libraries find_package(Python3Alt REQUIRED) +message(STATUS "Found NumPy version: ${Python3_NumPy_VERSION}") +message(STATUS "NumPy include dir: ${NUMPY_INCLUDE_DIRS}") + include(UseCython) # PyArrow C++ diff --git a/python/setup.py b/python/setup.py index 14bc01959eb..798bd6b05fd 100755 --- a/python/setup.py +++ b/python/setup.py @@ -24,7 +24,6 @@ import re import shlex import sys -import pathlib if sys.version_info >= (3, 10): import sysconfig @@ -82,9 +81,6 @@ class build_ext(_build_ext): def build_extensions(self): numpy_incl = pkg_resources.resource_filename('numpy', 'core/include') - print("NumPy include dir: ", numpy_incl) - path = pathlib.Path(numpy_incl) - print(list((path / "numpy").iterdir())) self.extensions = [ext for ext in self.extensions if ext.name != '__dummy__'] From 49df2c5812bf311c1949f8547341d1eaed8c0074 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Tue, 12 Mar 2024 13:52:06 +0100 Subject: [PATCH 09/11] temp disable pull on windows again --- dev/tasks/python-wheels/github.windows.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dev/tasks/python-wheels/github.windows.yml b/dev/tasks/python-wheels/github.windows.yml index 01f4977a9b0..50875bb10d7 100644 --- a/dev/tasks/python-wheels/github.windows.yml +++ b/dev/tasks/python-wheels/github.windows.yml @@ -52,10 +52,11 @@ jobs: @rem We can remove this workaround once we find a way to use @rem pulled caches when build an image. echo on - archery docker pull --no-ignore-pull-failures python-wheel-windows-vs2019 - if errorlevel 1 ( - archery docker build --no-pull python-wheel-windows-vs2019 || exit /B 1 - ) + @rem archery docker pull --no-ignore-pull-failures python-wheel-windows-vs2019 + @rem if errorlevel 1 ( + @rem archery docker build --no-pull python-wheel-windows-vs2019 || exit /B 1 + @rem ) + archery docker build --no-pull python-wheel-windows-vs2019 || exit /B 1 archery docker run --no-build -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-windows-vs2019 - uses: actions/upload-artifact@v3 From 0d97bf8a703f586451ebd3759f3fd1f865db5a56 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Tue, 12 Mar 2024 13:55:57 +0100 Subject: [PATCH 10/11] also test with archery docker run --- dev/tasks/python-wheels/github.windows.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dev/tasks/python-wheels/github.windows.yml b/dev/tasks/python-wheels/github.windows.yml index 50875bb10d7..fff68d80346 100644 --- a/dev/tasks/python-wheels/github.windows.yml +++ b/dev/tasks/python-wheels/github.windows.yml @@ -56,8 +56,7 @@ jobs: @rem if errorlevel 1 ( @rem archery docker build --no-pull python-wheel-windows-vs2019 || exit /B 1 @rem ) - archery docker build --no-pull python-wheel-windows-vs2019 || exit /B 1 - archery docker run --no-build -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-windows-vs2019 + archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-windows-vs2019 - uses: actions/upload-artifact@v3 with: From c3e18d12d7ee56585fdbff153484fa7eb266cba2 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Tue, 12 Mar 2024 16:50:36 +0100 Subject: [PATCH 11/11] properly update docker image --- .env | 2 +- ci/docker/python-wheel-windows-vs2019.dockerfile | 3 +-- dev/tasks/python-wheels/github.windows.yml | 10 +++++----- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.env b/.env index 00c238421d3..afed658db19 100644 --- a/.env +++ b/.env @@ -98,7 +98,7 @@ VCPKG="a42af01b72c28a8e1d7b48107b33e4f286a55ef6" # 2023.11.20 Release # ci/docker/python-wheel-windows-vs2019.dockerfile. # This is a workaround for our CI problem that "archery docker build" doesn't # use pulled built images in dev/tasks/python-wheels/github.windows.yml. -PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2024-02-05 +PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2024-03-12 # Use conanio/${CONAN} for "docker-compose run --rm conan". See # https://github.com/conan-io/conan-docker-tools#readme for available diff --git a/ci/docker/python-wheel-windows-vs2019.dockerfile b/ci/docker/python-wheel-windows-vs2019.dockerfile index 9bf7f8db323..b8e8aad952b 100644 --- a/ci/docker/python-wheel-windows-vs2019.dockerfile +++ b/ci/docker/python-wheel-windows-vs2019.dockerfile @@ -89,8 +89,7 @@ RUN python -m pip install -U pip setuptools COPY python/requirements-wheel-build.txt arrow/python/ # TODO(GH-39848) Remove the `--pre --extra-index-url` for numpy nightly again before the 16.0 release -RUN python -m pip install -r arrow/python/requirements-wheel-build.txt -RUN python -m pip install numpy -U --pre --extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" +RUN python -m pip install -r arrow/python/requirements-wheel-build.txt --pre --extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" # ENV CLCACHE_DIR="C:\clcache" # ENV CLCACHE_COMPRESS=1 diff --git a/dev/tasks/python-wheels/github.windows.yml b/dev/tasks/python-wheels/github.windows.yml index fff68d80346..01f4977a9b0 100644 --- a/dev/tasks/python-wheels/github.windows.yml +++ b/dev/tasks/python-wheels/github.windows.yml @@ -52,11 +52,11 @@ jobs: @rem We can remove this workaround once we find a way to use @rem pulled caches when build an image. echo on - @rem archery docker pull --no-ignore-pull-failures python-wheel-windows-vs2019 - @rem if errorlevel 1 ( - @rem archery docker build --no-pull python-wheel-windows-vs2019 || exit /B 1 - @rem ) - archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-windows-vs2019 + archery docker pull --no-ignore-pull-failures python-wheel-windows-vs2019 + if errorlevel 1 ( + archery docker build --no-pull python-wheel-windows-vs2019 || exit /B 1 + ) + archery docker run --no-build -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-windows-vs2019 - uses: actions/upload-artifact@v3 with: