From 411b44d7dbd0665571c9873e38e51c2e2b3032fb Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 19 Dec 2024 02:44:45 +0530 Subject: [PATCH 1/5] Fix include directory for NumPy v2 NumPy 2.0 changed the location of the header files --- python/CMakeLists.txt | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index c39a1129ac1..9198b3fe694 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -166,8 +166,17 @@ if($ENV{PYODIDE}) # modules (at least under Pyodide it does). set(Python3_INCLUDE_DIR $ENV{PYTHONINCLUDE}) set(Python3_LIBRARY $ENV{CPYTHONLIB}) - set(Python3_NumPy_INCLUDE_DIR $ENV{NUMPY_LIB}/core/include) set(Python3_EXECUTABLE) + execute_process(COMMAND ${Python3_EXECUTABLE} -c + "import numpy; print(numpy.__version__)" + OUTPUT_VARIABLE PYODIDE_NUMPY_VERSION + OUTPUT_STRIP_TRAILING_WHITESPACE) + string(REGEX MATCH "^([0-9]+)" PYODIDE_NUMPY_MAJOR_VERSION ${PYODIDE_NUMPY_VERSION}) + if(NUMPY_MAJOR_VERSION GREATER_EQUAL 2) + set(Python3_NumPy_INCLUDE_DIR $ENV{NUMPY_LIB}/_core/include) + else() + set(Python3_NumPy_INCLUDE_DIR $ENV{NUMPY_LIB}/core/include) + endif() set(ENV{_PYTHON_SYSCONFIGDATA_NAME} $ENV{SYSCONFIG_NAME}) # we set the c and cxx compiler manually to bypass pywasmcross # which is pyodide's way of messing with C++ build parameters. From 42743db3d18de3cd17d5bd0d8aaf2da02ec354ab Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 19 Dec 2024 02:45:36 +0530 Subject: [PATCH 2/5] `pyodide-build` was unvendored from Pyodide --- ci/docker/conda-python-emscripten.dockerfile | 6 +++--- docs/source/developers/cpp/emscripten.rst | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/ci/docker/conda-python-emscripten.dockerfile b/ci/docker/conda-python-emscripten.dockerfile index 8ad705c920b..47ff550cd59 100644 --- a/ci/docker/conda-python-emscripten.dockerfile +++ b/ci/docker/conda-python-emscripten.dockerfile @@ -27,14 +27,14 @@ ARG required_python_min="(3,12)" # fail if python version < 3.12 RUN echo "check PYTHON>=${required_python_min}" && python -c "import sys;sys.exit(0 if sys.version_info>=${required_python_min} else 1)" -# install selenium and pyodide-build and recent python +# install selenium and recent pyodide-build and recent python # needs to be a login shell so ~/.profile is read SHELL ["/bin/bash", "--login", "-c", "-o", "pipefail"] RUN python -m pip install --no-cache-dir selenium==${selenium_version} && \ - python -m pip install --no-cache-dir --upgrade pyodide-build==${pyodide_version} - + python -m pip install --no-cache-dir --upgrade pyodide-build>=${pyodide_version} + # install pyodide dist directory to /pyodide RUN pyodide_dist_url="https://github.com/pyodide/pyodide/releases/download/${pyodide_version}/pyodide-${pyodide_version}.tar.bz2" && \ wget -q "${pyodide_dist_url}" -O- | tar -xj -C / diff --git a/docs/source/developers/cpp/emscripten.rst b/docs/source/developers/cpp/emscripten.rst index b4c563aae1a..d291714c869 100644 --- a/docs/source/developers/cpp/emscripten.rst +++ b/docs/source/developers/cpp/emscripten.rst @@ -46,8 +46,8 @@ versions of emsdk tools. .. code:: shell # install Pyodide build tools. - # e.g. for version 0.24 of Pyodide: - pip install pyodide-build==0.24 + # e.g., for version 0.26 of Pyodide, pyodide-build 0.26 and later work + pip install "pyodide-build>=0.26" Then build with the ``ninja-release-emscripten`` CMake preset, like below: @@ -69,8 +69,7 @@ go to ``arrow/python`` and run pyodide build It should make a wheel targeting the currently enabled version of -Pyodide (i.e. the version corresponding to the currently installed -``pyodide-build``) in the ``dist`` subdirectory. +Pyodide in the ``dist`` subdirectory. Manual Build From e1813bc99b697f76bcb06a431a2af115337980ed Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 19 Dec 2024 02:58:45 +0530 Subject: [PATCH 3/5] Document where to find Pyodide-compatible emsdk --- docs/source/developers/cpp/emscripten.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/source/developers/cpp/emscripten.rst b/docs/source/developers/cpp/emscripten.rst index d291714c869..1802a85b136 100644 --- a/docs/source/developers/cpp/emscripten.rst +++ b/docs/source/developers/cpp/emscripten.rst @@ -33,7 +33,9 @@ activate it using the commands below (see https://emscripten.org/docs/getting_st git clone https://github.com/emscripten-core/emsdk.git cd emsdk # replace with the desired EMSDK version. - # e.g. for Pyodide 0.24, you need EMSDK version 3.1.45 + # e.g. for Pyodide 0.26, you need EMSDK version 3.1.58 + # the versions can be found in the Makefile.envs file in the Pyodide repo: + # https://github.com/pyodide/pyodide/blob/10b484cfe427e076c929a55dc35cfff01ea8d3bc/Makefile.envs ./emsdk install ./emsdk activate source ./emsdk_env.sh From 8147a976027c2ad2840762c15fb1dff41cbd94fc Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 19 Dec 2024 02:59:02 +0530 Subject: [PATCH 4/5] `ARROW_ENABLE_THREADING` must be `OFF` --- docs/source/developers/cpp/emscripten.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/source/developers/cpp/emscripten.rst b/docs/source/developers/cpp/emscripten.rst index 1802a85b136..bfa0c5bc350 100644 --- a/docs/source/developers/cpp/emscripten.rst +++ b/docs/source/developers/cpp/emscripten.rst @@ -86,9 +86,8 @@ you will need to override. In particular you will need: #. ``CMAKE_TOOLCHAIN_FILE`` set by using ``emcmake cmake`` instead of just ``cmake``. -#. You will quite likely need to set ``ARROW_ENABLE_THREADING`` to ``OFF`` - for builds targeting single threaded Emscripten environments such as - Pyodide. +#. You will need to set ``ARROW_ENABLE_THREADING`` to ``OFF`` for builds + targeting single-threaded Emscripten environments such as Pyodide. #. ``ARROW_FLIGHT`` and anything else that uses network probably won't work. From bb6ff292ee3916c991729a0b9b89636c156fd3ca Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 19 Dec 2024 05:10:31 +0530 Subject: [PATCH 5/5] Fix NumPy version variable name --- python/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 9198b3fe694..80d1cd31ac2 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -172,7 +172,7 @@ if($ENV{PYODIDE}) OUTPUT_VARIABLE PYODIDE_NUMPY_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE) string(REGEX MATCH "^([0-9]+)" PYODIDE_NUMPY_MAJOR_VERSION ${PYODIDE_NUMPY_VERSION}) - if(NUMPY_MAJOR_VERSION GREATER_EQUAL 2) + if(PYODIDE_NUMPY_MAJOR_VERSION GREATER_EQUAL 2) set(Python3_NumPy_INCLUDE_DIR $ENV{NUMPY_LIB}/_core/include) else() set(Python3_NumPy_INCLUDE_DIR $ENV{NUMPY_LIB}/core/include)