From 0e5d80d6f8b4c72c31e828cf7186cac42e3267ed Mon Sep 17 00:00:00 2001 From: Mengwei Liu Date: Thu, 17 Aug 2023 11:41:50 -0700 Subject: [PATCH] Fix CI job (#73) Summary: Pull Request resolved: https://github.com/pytorch/executorch/pull/73 Apparently in macos CI, python3 and python are pointing to different python environment. python3 -> /Library/Python/3.9/site-packages/ python -> /Users/ec2-user/runner/_work/_temp/miniconda/lib/python3.9/site-packages This diff is fixing the CI failure: ``` [2023-08-17T02:59:12.739+00:00] Local command: env -- "ASAN_OPTIONS=detect_leaks=0,detect_odr_violation=0" "GEN_DIR=GEN_DIR_DEPRECATED" "OUT=././../out" "SRCDIR=./." "SRCS=././link_torch.sh" /usr/bin/env bash -e buck-out/v2/gen/root/213ed1b7ab869379/third-party/__libtorch_gen__/sh/genrule.sh [2023-08-17T02:59:12.739+00:00] Stdout: Error: /Library/Python/3.9/site-packages/torch/lib/libtorch.dylib doesn't exist ``` Also adding `tomli` as a ci requirements due to the following error: ``` Error: Traceback (most recent call last): File "/Users/ec2-user/runner/_work/executorch/executorch/pytorch/executorch/build/extract_sources.py", line 18, in import tomllib # Standard in 3.11 and later ModuleNotFoundError: No module named 'tomllib' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/Users/ec2-user/runner/_work/executorch/executorch/pytorch/executorch/build/extract_sources.py", line 20, in import tomli as tomllib ModuleNotFoundError: No module named 'tomli' CMake Error at CMakeLists.txt:101 (message): executorch: source list generation failed ``` Reviewed By: huydhn, cccclai Differential Revision: D48423480 fbshipit-source-id: 0a69c721da90878bda24f8d1c5d1ff59e583025b --- .github/workflows/pull.yml | 4 ++-- CMakeLists.txt | 5 ++++- examples/custom_ops/test_custom_ops.sh | 21 ++++++++++++++------- third-party/link_torch.sh | 2 +- 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 26bd4bffd38..14504a853ed 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -51,7 +51,7 @@ jobs: bash .ci/scripts/test.sh # Test custom ops - bash examples/custom_ops/test_custom_ops.sh buck2 + PYTHON_EXECUTABLE=python bash examples/custom_ops/test_custom_ops.sh buck2 popd cmake-build-test-linux: @@ -90,5 +90,5 @@ jobs: bash .ci/scripts/setup-macos.sh # Build and test custom ops - bash examples/custom_ops/test_custom_ops.sh cmake + PYTHON_EXECUTABLE=python bash examples/custom_ops/test_custom_ops.sh cmake popd diff --git a/CMakeLists.txt b/CMakeLists.txt index 23c980e7160..825417de7a0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,6 +62,9 @@ endif() if(NOT BUCK2) set(BUCK2 buck2) endif() +if(NOT PYTHON_EXECUTABLE) + set(PYTHON_EXECUTABLE python3) +endif() # TODO(dbort): Fix these warnings and remove this flag. set(_common_compile_options -Wno-deprecated-declarations) @@ -87,7 +90,7 @@ if(NOT EXECUTORCH_SRCS_FILE) message(STATUS "executorch: Generating source lists") set(EXECUTORCH_SRCS_FILE "${CMAKE_CURRENT_BINARY_DIR}/executorch_srcs.cmake") execute_process( - COMMAND python3 build/extract_sources.py --buck2=${BUCK2} + COMMAND ${PYTHON_EXECUTABLE} build/extract_sources.py --buck2=${BUCK2} --config=build/cmake_deps.toml --out=${EXECUTORCH_SRCS_FILE} OUTPUT_VARIABLE gen_srcs_output ERROR_VARIABLE gen_srcs_error diff --git a/examples/custom_ops/test_custom_ops.sh b/examples/custom_ops/test_custom_ops.sh index 34c7f8c5f55..67859da7174 100644 --- a/examples/custom_ops/test_custom_ops.sh +++ b/examples/custom_ops/test_custom_ops.sh @@ -14,7 +14,7 @@ set -e test_buck2_custom_op_1() { local model_name='custom_ops_1' echo "Exporting ${model_name}.pte" - python -m "examples.custom_ops.${model_name}" + ${PYTHON_EXECUTABLE} -m "examples.custom_ops.${model_name}" # should save file custom_ops_1.pte echo 'Running executor_runner' @@ -29,12 +29,14 @@ test_buck2_custom_op_1() { test_cmake_custom_op_1() { local model_name='custom_ops_1' echo "Exporting ${model_name}.pte" - python -m "examples.custom_ops.${model_name}" + ${PYTHON_EXECUTABLE} -m "examples.custom_ops.${model_name}" # should save file custom_ops_1.pte (rm -rf cmake-out \ && mkdir cmake-out \ && cd cmake-out \ - && cmake -DBUCK2=buck2 -DREGISTER_EXAMPLE_CUSTOM_OP_1=ON ..) + && cmake -DBUCK2=buck2 \ + -DREGISTER_EXAMPLE_CUSTOM_OP_1=ON \ + -DPYTHON_EXECUTABLE="$PYTHON_EXECUTABLE" ..) echo 'Building executor_runner' cmake --build cmake-out -j9 @@ -50,7 +52,7 @@ test_buck2_custom_op_2() { SO_LIB=$(buck2 build //examples/custom_ops:custom_ops_aot_lib_2 --show-output | grep "buck-out" | cut -d" " -f2) echo "Exporting ${model_name}.pte" - python -m "examples.custom_ops.${model_name}" --so_library="$SO_LIB" + ${PYTHON_EXECUTABLE} -m "examples.custom_ops.${model_name}" --so_library="$SO_LIB" # should save file custom_ops_2.pte buck2 run //examples/executor_runner:executor_runner \ @@ -77,7 +79,7 @@ get_shared_lib_ext() { test_cmake_custom_op_2() { local model_name='custom_ops_2' - SITE_PACKAGES="$(python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')" + SITE_PACKAGES="$(${PYTHON_EXECUTABLE} -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')" CMAKE_PREFIX_PATH="${SITE_PACKAGES}/torch" (rm -rf cmake-out \ @@ -85,20 +87,25 @@ test_cmake_custom_op_2() { && cd cmake-out \ && cmake -DBUCK2=buck2 \ -DREGISTER_EXAMPLE_CUSTOM_OP_2=ON \ - -DCMAKE_PREFIX_PATH="$CMAKE_PREFIX_PATH" ..) + -DCMAKE_PREFIX_PATH="$CMAKE_PREFIX_PATH" \ + -DPYTHON_EXECUTABLE="$PYTHON_EXECUTABLE" ..) echo 'Building executor_runner' cmake --build cmake-out -j4 EXT=$(get_shared_lib_ext) echo "Exporting ${model_name}.pte" - python -m "examples.custom_ops.${model_name}" --so_library="cmake-out/examples/custom_ops/libcustom_ops_aot_lib$EXT" + ${PYTHON_EXECUTABLE} -m "examples.custom_ops.${model_name}" --so_library="cmake-out/examples/custom_ops/libcustom_ops_aot_lib$EXT" # should save file custom_ops_2.pte echo 'Running executor_runner' cmake-out/executor_runner "--model_path=./${model_name}.pte" } +if [[ -z $PYTHON_EXECUTABLE ]]; +then + PYTHON_EXECUTABLE=python3 +fi if [[ $1 == "cmake" ]]; then test_cmake_custom_op_1 diff --git a/third-party/link_torch.sh b/third-party/link_torch.sh index 63bd25f2864..d5c740bfa49 100644 --- a/third-party/link_torch.sh +++ b/third-party/link_torch.sh @@ -33,7 +33,7 @@ while getopts ":o:f:" opt; do esac done -LIB=$(python3 -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())') +LIB=$(python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())') # delimiter , export IFS=","