From 6bac8e4c946914967e3408fa6c936e6c2c6a3d5d Mon Sep 17 00:00:00 2001 From: tqchen Date: Mon, 10 Feb 2025 17:42:57 -0500 Subject: [PATCH] [PYTHON] Build cython by default Historically we have been maintaining cython and ctypes for FFI. One reason was cython was not that available then. Cython have quite good availability now and also enjoys great perf benefit. This PR changes the CMake to make cython as required dependency of the python package and build cython by default. It also removes ctypes testcases in favor of all cython tests. If things go well, we can consider remove ctypes and focus on cython support onward. --- CMakeLists.txt | 12 ++++++++ Makefile | 1 - apps/benchmark/adreno/bench.sh | 2 -- python/setup.py | 29 ++++++------------- tests/scripts/setup-pytest-env.sh | 9 +++--- tests/scripts/task_python_adreno.sh | 8 ++--- .../task_python_arm_compute_library.sh | 9 +++--- tests/scripts/task_python_docs.sh | 9 ++++-- tests/scripts/task_python_hexagon.sh | 9 ++++-- tests/scripts/task_python_integration.sh | 17 +++++------ tests/scripts/task_python_nightly.sh | 6 ++-- tests/scripts/task_python_topi.sh | 6 ++-- tests/scripts/task_python_unittest.sh | 13 ++++----- tests/scripts/task_python_unittest_gpuonly.sh | 2 +- tests/scripts/unity/task_python_relax.sh | 3 +- 15 files changed, 68 insertions(+), 67 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 757b0d1a89ff..f43eab264d20 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -819,6 +819,18 @@ endif() # Custom targets add_custom_target(runtime DEPENDS tvm_runtime) +# By default add cython to all build +find_package(Python) +if(NOT DEFINED ENV{CONDA_BUILD}) + message(STATUS ${CMAKE_CURRENT_BINARY_DIR}) + add_custom_target( + tvm_cython ALL + ${Python_EXECUTABLE} setup.py build_ext --inplace + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/python + ) + add_dependencies(tvm_cython tvm) +endif() + # Installation rules install(TARGETS tvm EXPORT ${PROJECT_NAME}Targets DESTINATION lib${LIB_SUFFIX}) install(TARGETS tvm_runtime EXPORT ${PROJECT_NAME}Targets DESTINATION lib${LIB_SUFFIX}) diff --git a/Makefile b/Makefile index 3a880b94a5af..5134c949ea48 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,6 @@ lint pylint cpplint \ cppdoc docs \ web webclean \ - cython cython3 cyclean \ clean .SECONDEXPANSION: diff --git a/apps/benchmark/adreno/bench.sh b/apps/benchmark/adreno/bench.sh index 7f9adeea5251..2c8ff288202f 100755 --- a/apps/benchmark/adreno/bench.sh +++ b/apps/benchmark/adreno/bench.sh @@ -48,8 +48,6 @@ trap "{ kill ${TRACKER_PID}; kill ${DEVICE_PID}; }" 0 # cleanup pycache find . -type f -path "*.pyc" | xargs rm -f -# Test TVM -make cython3 if [ "texture" == $1 ] ; then python3 apps/benchmark/adreno/adreno_gpu_bench_texture.py --host ${TVM_TRACKER_HOST} --port ${TVM_TRACKER_PORT} --rpc-key ${RPC_DEVICE_KEY} diff --git a/python/setup.py b/python/setup.py index 55f29c651af4..6cca9060bd2d 100644 --- a/python/setup.py +++ b/python/setup.py @@ -20,7 +20,6 @@ import pathlib import shutil import sys -import sysconfig from setuptools import find_packages from setuptools.dist import Distribution @@ -133,24 +132,17 @@ def _remove_path(path): def config_cython(): """Try to configure cython and return cython configuration""" - if FFI_MODE not in ("cython"): - if os.name == "nt" and not CONDA_BUILD: - print("WARNING: Cython is not supported on Windows, will compile without cython module") - return [] - sys_cflags = sysconfig.get_config_var("CFLAGS") - if sys_cflags and "i386" in sys_cflags and "x86_64" in sys_cflags: - print("WARNING: Cython library may not be compiled correctly with both i386 and x64") - return [] + # Enforce cython unless FFI_MODE is explicitly set to ctypes + # we might consider fully converge to cython later + if FFI_MODE == "ctypes": + return [] try: from Cython.Build import cythonize - # from setuptools.extension import Extension - if sys.version_info >= (3, 0): - subdir = "_cy3" - else: - subdir = "_cy2" + subdir = "_cy3" + ret = [] - path = "tvm/_ffi/_cython" + cython_source = "tvm/_ffi/_cython" extra_compile_args = ["-std=c++17", "-DDMLC_USE_LOGGING_LIBRARY="] if os.name == "nt": library_dirs = ["tvm", "../build/Release", "../build"] @@ -166,7 +158,7 @@ def config_cython(): library_dirs = None libraries = None - for fn in os.listdir(path): + for fn in os.listdir(cython_source): if not fn.endswith(".pyx"): continue ret.append( @@ -186,10 +178,7 @@ def config_cython(): ) return cythonize(ret, compiler_directives={"language_level": 3}) except ImportError as error: - if FFI_MODE == "cython": - raise error - print("WARNING: Cython is not installed, will compile without cython module") - return [] + raise RuntimeError("Cython is not installed, please pip install cython") class BinaryDistribution(Distribution): diff --git a/tests/scripts/setup-pytest-env.sh b/tests/scripts/setup-pytest-env.sh index 7bdfbdd1df47..171ddbc2d0d6 100755 --- a/tests/scripts/setup-pytest-env.sh +++ b/tests/scripts/setup-pytest-env.sh @@ -47,15 +47,14 @@ function cleanup() { trap cleanup 0 function run_pytest() { + ffi_type="cython" set -e - local ffi_type="$1" - shift local test_suite_name="$1" shift extra_args=( "$@" ) if [ -z "${ffi_type}" -o -z "${test_suite_name}" ]; then - echo "error: run_pytest called incorrectly: run_pytest ${ffi_type} ${test_suite_name}" "${extra_args[@]}" - echo "usage: run_pytest [pytest args...]" + echo "error: run_pytest called incorrectly: run_pytest ${test_suite_name}" "${extra_args[@]}" + echo "usage: run_pytest [pytest args...]" exit 2 fi @@ -85,7 +84,7 @@ function run_pytest() { exit_code=0 set +e - TVM_FFI=${ffi_type} python3 -m pytest \ + python3 -m pytest \ -o "junit_suite_name=${suite_name}" \ "--junit-xml=${TVM_PYTEST_RESULT_DIR}/${suite_name}.xml" \ "--junit-prefix=${ffi_type}" \ diff --git a/tests/scripts/task_python_adreno.sh b/tests/scripts/task_python_adreno.sh index e20d1bea31c5..a435d588aefb 100755 --- a/tests/scripts/task_python_adreno.sh +++ b/tests/scripts/task_python_adreno.sh @@ -57,8 +57,8 @@ trap "{ kill ${TRACKER_PID}; kill ${DEVICE_PID}; cleanup; }" 0 # cleanup pycache find . -type f -path "*.pyc" | xargs rm -f -# Test TVM -make cython3 +# setup cython +cd python; python3 setup.py build_ext --inplace; cd .. # The RPC to remote Android device has issue of hang after few tests with in CI environments. # Lets run them individually on fresh rpc session. @@ -68,7 +68,7 @@ i=0 IFS=$'\n' for node_id in $TEXTURE_TESTS; do echo "$node_id" - run_pytest ctypes "$TVM_INTEGRATION_TESTSUITE_NAME-opencl-texture-$i" "$node_id" --reruns=0 + run_pytest "$TVM_INTEGRATION_TESTSUITE_NAME-opencl-texture-$i" "$node_id" --reruns=0 i=$((i+1)) done @@ -77,7 +77,7 @@ CLML_TESTS=$(./ci/scripts/jenkins/pytest_ids.py --folder tests/python/contrib/te i=0 for node_id in $CLML_TESTS; do echo "$node_id" - CXX=${TVM_NDK_CC} run_pytest ctypes "$TVM_INTEGRATION_TESTSUITE_NAME-openclml-$i" "$node_id" --reruns=0 + CXX=${TVM_NDK_CC} run_pytest "$TVM_INTEGRATION_TESTSUITE_NAME-openclml-$i" "$node_id" --reruns=0 i=$((i+1)) done diff --git a/tests/scripts/task_python_arm_compute_library.sh b/tests/scripts/task_python_arm_compute_library.sh index 51b453e2b862..520abb58cdc2 100755 --- a/tests/scripts/task_python_arm_compute_library.sh +++ b/tests/scripts/task_python_arm_compute_library.sh @@ -21,10 +21,9 @@ set -euxo pipefail source tests/scripts/setup-pytest-env.sh -# Rebuild cython -# TODO(u99127): Enable cython tests. - find . -type f -path "*.pyc" | xargs rm -f -make cython3 -run_pytest ctypes python-arm_compute_lib tests/python/contrib/test_arm_compute_lib +# setup cython +cd python; python3 setup.py build_ext --inplace; cd .. + +run_pytest python-arm_compute_lib tests/python/contrib/test_arm_compute_lib diff --git a/tests/scripts/task_python_docs.sh b/tests/scripts/task_python_docs.sh index a72078d74011..b217f692b4a3 100755 --- a/tests/scripts/task_python_docs.sh +++ b/tests/scripts/task_python_docs.sh @@ -46,7 +46,9 @@ clean_files() { sphinx_precheck() { clean_files echo "PreCheck sphinx doc generation WARNINGS.." - make cython3 + + # setup cython + cd python; python3 setup.py build_ext --inplace; cd .. pushd docs make clean @@ -123,7 +125,10 @@ clean_files # cleanup stale log files find . -type f -path "*.log" | xargs rm -f find . -type f -path "*.pyc" | xargs rm -f -make cython3 + +# setup cython +cd python; python3 setup.py build_ext --inplace; cd .. + cd docs PYTHONPATH=$(pwd)/../python make htmldepoly SPHINXOPTS='-j auto' |& tee /tmp/$$.log.txt diff --git a/tests/scripts/task_python_hexagon.sh b/tests/scripts/task_python_hexagon.sh index ba125b161a56..11e4e1827421 100755 --- a/tests/scripts/task_python_hexagon.sh +++ b/tests/scripts/task_python_hexagon.sh @@ -26,7 +26,10 @@ if [ $# -ge 1 ] && [[ "$1" = "--device" ]]; then fi source tests/scripts/setup-pytest-env.sh -make cython3 + +# setup cython +cd python; python3 setup.py build_ext --inplace; cd .. + if [[ "${device_serial}" == "simulator" ]]; then export TVM_TRACKER_PORT=9190 @@ -49,9 +52,9 @@ fi export ANDROID_SERIAL_NUMBER=${device_serial} if [ "${device_serial}" == "simulator" ]; then - run_pytest ctypes python-contrib-hexagon tests/python/contrib/test_hexagon + run_pytest python-contrib-hexagon tests/python/contrib/test_hexagon else - run_pytest ctypes python-contrib-hexagon tests/python/contrib/test_hexagon -n=$num_of_devices + run_pytest python-contrib-hexagon tests/python/contrib/test_hexagon -n=$num_of_devices fi if [[ "${device_serial}" == "simulator" ]]; then diff --git a/tests/scripts/task_python_integration.sh b/tests/scripts/task_python_integration.sh index da0d0500837e..51ef86d05ec7 100755 --- a/tests/scripts/task_python_integration.sh +++ b/tests/scripts/task_python_integration.sh @@ -33,23 +33,20 @@ fi # cleanup pycache find . -type f -path "*.pyc" | xargs rm -f -# Test TVM -make cython3 +# setup cython +cd python; python3 setup.py build_ext --inplace; cd .. -run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-integration tests/python/integration +run_pytest ${TVM_INTEGRATION_TESTSUITE_NAME}-integration tests/python/integration # forked is needed because the global registry gets contaminated TVM_TEST_TARGETS="${TVM_RELAY_TEST_TARGETS:-llvm;cuda}" \ - run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-relay tests/python/relay --ignore=tests/python/relay/aot + run_pytest ${TVM_INTEGRATION_TESTSUITE_NAME}-relay tests/python/relay --ignore=tests/python/relay/aot # OpenCL texture test. Deselected specific tests that fails in CI TVM_TEST_TARGETS="${TVM_RELAY_OPENCL_TEXTURE_TARGETS:-opencl}" \ - run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-opencl-texture tests/python/relay/opencl_texture + run_pytest ${TVM_INTEGRATION_TESTSUITE_NAME}-opencl-texture tests/python/relay/opencl_texture # Command line driver test -run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-driver tests/python/driver +run_pytest ${TVM_INTEGRATION_TESTSUITE_NAME}-driver tests/python/driver # Target test -run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-target tests/python/target - -# Do not enable OpenGL -# run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-webgl tests/webgl +run_pytest ${TVM_INTEGRATION_TESTSUITE_NAME}-target tests/python/target diff --git a/tests/scripts/task_python_nightly.sh b/tests/scripts/task_python_nightly.sh index f8423602bd4b..42cf343e71ad 100755 --- a/tests/scripts/task_python_nightly.sh +++ b/tests/scripts/task_python_nightly.sh @@ -20,10 +20,10 @@ set -euxo pipefail source tests/scripts/setup-pytest-env.sh -# Rebuild cython -make cython3 +# setup cython +cd python; python3 setup.py build_ext --inplace; cd .. # cleanup pycache find . -type f -path "*.pyc" | xargs rm -f -run_pytest cython python-topi-nightly tests/python/topi/nightly +run_pytest python-topi-nightly tests/python/topi/nightly diff --git a/tests/scripts/task_python_topi.sh b/tests/scripts/task_python_topi.sh index 3916e0a754f3..97030d6948f9 100755 --- a/tests/scripts/task_python_topi.sh +++ b/tests/scripts/task_python_topi.sh @@ -26,10 +26,10 @@ export TVM_TEST_TARGETS="llvm; cuda" export TVM_BIND_THREADS=0 export OMP_NUM_THREADS=1 -# Rebuild cython -make cython3 +# setup cython +cd python; python3 setup.py build_ext --inplace; cd .. # cleanup pycache find . -type f -path "*.pyc" | xargs rm -f -run_pytest cython python-topi tests/python/topi/ +run_pytest python-topi tests/python/topi/ diff --git a/tests/scripts/task_python_unittest.sh b/tests/scripts/task_python_unittest.sh index 4a13c6ce1ed2..b84694cfd5e2 100755 --- a/tests/scripts/task_python_unittest.sh +++ b/tests/scripts/task_python_unittest.sh @@ -22,7 +22,9 @@ source tests/scripts/setup-pytest-env.sh # cleanup pycache find . -type f -path "*.pyc" | xargs rm -f -make cython3 + +# setup cython +cd python; python3 setup.py build_ext --inplace; cd .. # NOTE: also set by task_python_unittest_gpuonly.sh. if [ -z "${TVM_UNITTEST_TESTSUITE_NAME:-}" ]; then @@ -30,8 +32,7 @@ if [ -z "${TVM_UNITTEST_TESTSUITE_NAME:-}" ]; then fi # First run minimal test on both ctypes and cython. -run_pytest ctypes ${TVM_UNITTEST_TESTSUITE_NAME}-platform-minimal-test-0 tests/python/all-platform-minimal-test -run_pytest cython ${TVM_UNITTEST_TESTSUITE_NAME}-platform-minimal-test-1 tests/python/all-platform-minimal-test +run_pytest ${TVM_UNITTEST_TESTSUITE_NAME}-platform-minimal-test tests/python/all-platform-minimal-test # Then run all unittests on both ctypes and cython. TEST_FILES=( @@ -52,11 +53,9 @@ TEST_FILES=( "tir-usmp" "tvmscript" "usmp" + "ci" ) for TEST_FILE in ${TEST_FILES[@]}; do - run_pytest cython ${TEST_FILE}-1, tests/python/${TEST_FILE} + run_pytest ${TEST_FILE}, tests/python/${TEST_FILE} done - -# Then run CI tests -run_pytest ctypes ${TVM_UNITTEST_TESTSUITE_NAME}-ci tests/python/ci diff --git a/tests/scripts/task_python_unittest_gpuonly.sh b/tests/scripts/task_python_unittest_gpuonly.sh index e68fcba25c91..6c4e642b0c6b 100755 --- a/tests/scripts/task_python_unittest_gpuonly.sh +++ b/tests/scripts/task_python_unittest_gpuonly.sh @@ -33,4 +33,4 @@ export TVM_UNITTEST_TESTSUITE_NAME=python-codegen-vulkan source tests/scripts/setup-pytest-env.sh -run_pytest cython ${TVM_UNITTEST_TESTSUITE_NAME}-1 tests/python/codegen/test_target_codegen_vulkan.py +run_pytest ${TVM_UNITTEST_TESTSUITE_NAME}-1 tests/python/codegen/test_target_codegen_vulkan.py diff --git a/tests/scripts/unity/task_python_relax.sh b/tests/scripts/unity/task_python_relax.sh index 86d917c655bb..28dd78bf6bd2 100755 --- a/tests/scripts/unity/task_python_relax.sh +++ b/tests/scripts/unity/task_python_relax.sh @@ -25,7 +25,8 @@ export LD_LIBRARY_PATH="build:${LD_LIBRARY_PATH:-}" export TVM_BIND_THREADS=0 export TVM_NUM_THREADS=2 -make cython3 +# setup cython +cd python; python3 setup.py build_ext --inplace; cd .. # Run Relax tests TVM_TEST_TARGETS="${TVM_RELAY_TEST_TARGETS:-llvm}" pytest tests/python/relax