From 1174c8e31e5231053f5e1031527e717cf381876e Mon Sep 17 00:00:00 2001 From: jiaweizhuang Date: Fri, 30 Jan 2026 10:52:41 +0000 Subject: [PATCH 1/7] fix cmake build settings for manylinux --- CMakeLists.txt | 2 +- README.md | 4 +++- lib/Bindings/Python/CMakeLists.txt | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 47477d4e8..ecf3a6da4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -56,7 +56,7 @@ add_subdirectory(tools) option(PTO_ENABLE_PYTHON_BINDING "Enable Python bindings" ON) if(PTO_ENABLE_PYTHON_BINDING) - find_package(Python3 COMPONENTS Interpreter Development REQUIRED) + find_package(Python3 COMPONENTS Interpreter Development.Module REQUIRED) find_package(pybind11 CONFIG REQUIRED) # 设置 Python 扩展的输出目录，方便调试 diff --git a/README.md b/README.md index 6508f0601..6a93b79bc 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ cmake -G Ninja -S llvm -B $LLVM_BUILD_DIR \ -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ -DPython3_EXECUTABLE=$(which python3) \ -DCMAKE_BUILD_TYPE=Release \ - -DLLVM_TARGETS_TO_BUILD="Host" + -DLLVM_TARGETS_TO_BUILD="host" # 4. 编译 LLVM (这一步耗时较长) ninja -C $LLVM_BUILD_DIR @@ -124,6 +124,8 @@ cmake -G Ninja \ -B build \ -DLLVM_DIR=$LLVM_BUILD_DIR/lib/cmake/llvm \ -DMLIR_DIR=$LLVM_BUILD_DIR/lib/cmake/mlir \ + -DPython3_EXECUTABLE=$(which python3) \ + -DPython3_FIND_STRATEGY=LOCATION \ -Dpybind11_DIR="${PYBIND11_CMAKE_DIR}" \ -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ -DMLIR_PYTHON_PACKAGE_DIR=$LLVM_BUILD_DIR/tools/mlir/python_packages/mlir_core \ diff --git a/lib/Bindings/Python/CMakeLists.txt b/lib/Bindings/Python/CMakeLists.txt index 493b99854..9e66c3c93 100644 --- a/lib/Bindings/Python/CMakeLists.txt +++ b/lib/Bindings/Python/CMakeLists.txt @@ -1,6 +1,6 @@ # lib/Bindings/Python/CMakeLists.txt -find_package(Python3 COMPONENTS Interpreter Development REQUIRED) +find_package(Python3 COMPONENTS Interpreter Development.Module REQUIRED) find_package(pybind11 CONFIG REQUIRED) include(TableGen) From dbeec45d1069295d1b3b88c1bf6c42ed624e270d Mon Sep 17 00:00:00 2001 From: jiaweizhuang Date: Fri, 30 Jan 2026 11:00:17 +0000 Subject: [PATCH 2/7] fix ptoas path --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6a93b79bc..389285e67 100644 --- a/README.md +++ b/README.md @@ -176,7 +176,7 @@ export PYTHONPATH=$MLIR_PYTHON_ROOT:$PTO_PYTHON_ROOT:$PYTHONPATH export LD_LIBRARY_PATH=$LLVM_BUILD_DIR/lib:$PTO_INSTALL_DIR/lib:$LD_LIBRARY_PATH # 3. PATH: 将 ptoas 添加到命令行路径 -export PATH=$PTO_INSTALL_DIR/bin:$PATH +export PATH=$PTO_SOURCE_DIR/build/tools/ptoas:$PATH ``` From 8de48e913f4e9d62481e521206fa84618cb84841 Mon Sep 17 00:00:00 2001 From: jiaweizhuang Date: Fri, 30 Jan 2026 11:00:34 +0000 Subject: [PATCH 3/7] dockerfile to compile ptoas in manylinux base img --- docker/Dockerfile | 62 +++++++++++++++++++++++++++++++++++++++++++++++ docker/README.md | 14 +++++++++++ 2 files changed, 76 insertions(+) create mode 100644 docker/Dockerfile create mode 100644 docker/README.md diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 000000000..03c26ec2a --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,62 @@ +FROM quay.io/pypa/manylinux_2_34_x86_64 + +# NOTE: change $PY_VER for different Python versions (3.8 - 3.14 available) +ARG PY_VER=cp312-cp312 +ARG LLVM_TAG=llvmorg-19.1.7 + +## -- usually no need to change below -- + +ENV PY_PATH="/opt/python/${PY_VER}" +ENV PATH="${PY_PATH}/bin:${PATH}" + +# dependency +RUN dnf install -y ninja-build cmake git ccache gcc-c++ lld && dnf clean all +RUN pip install --no-cache-dir numpy pybind11 nanobind + +# setting build directories +ENV WORKSPACE_DIR=/llvm-workspace +ENV LLVM_SOURCE_DIR=$WORKSPACE_DIR/llvm-project +ENV LLVM_BUILD_DIR=$LLVM_SOURCE_DIR/build-shared + +ENV PTO_SOURCE_DIR=$WORKSPACE_DIR/PTOAS +ENV PTO_INSTALL_DIR=$PTO_SOURCE_DIR/install + +# build LLVM +WORKDIR $WORKSPACE_DIR +RUN git clone --depth 1 --branch ${LLVM_TAG} https://github.com/llvm/llvm-project.git + +WORKDIR $LLVM_SOURCE_DIR +RUN cmake -G Ninja -S llvm -B $LLVM_BUILD_DIR \ + -DLLVM_ENABLE_PROJECTS="mlir;clang" \ + -DBUILD_SHARED_LIBS=ON \ + -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ + -DPython3_EXECUTABLE=${PY_PATH}/bin/python \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_TARGETS_TO_BUILD="host" + +RUN ninja -C $LLVM_BUILD_DIR + +# build ptoas +WORKDIR $WORKSPACE_DIR +RUN git clone -b wheel https://github.com/learning-chip/PTOAS.git +# TODO: tag git commit of PTOAS repo + +WORKDIR $PTO_SOURCE_DIR +RUN cmake -G Ninja \ + -S . \ + -B build \ + -DLLVM_DIR=$LLVM_BUILD_DIR/lib/cmake/llvm \ + -DMLIR_DIR=$LLVM_BUILD_DIR/lib/cmake/mlir \ + -DPython3_ROOT_DIR=${PY_PATH} \ + -DPython3_EXECUTABLE=${PY_PATH}/bin/python \ + -DPython3_FIND_STRATEGY=LOCATION \ + -Dpybind11_DIR=$(python -m pybind11 --cmakedir) \ + -DMLIR_PYTHON_PACKAGE_DIR=${LLVM_BUILD_DIR}/tools/mlir/python_packages/mlir_core \ + -DCMAKE_INSTALL_PREFIX=${PTO_INSTALL_DIR} + +RUN ninja -C build && ninja -C build install + +# test ptoas cli +ENV PATH=$PTO_SOURCE_DIR/build/tools/ptoas:$PATH +RUN which ptoas + diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 000000000..c3efbe12e --- /dev/null +++ b/docker/README.md @@ -0,0 +1,14 @@ +Build: + +```bash +docker build . -t ptoas:py3.12 + +# optional, to change python version +docker build . -t ptoas:py3.11 --build-arg PY_VER=cp311-cp311 +``` + +Use: + +```bash +docker run --rm -it ptoas:py3.12 /bin/bash +``` \ No newline at end of file From bde2c740163aa6e3f369e497668da7f95848ec80 Mon Sep 17 00:00:00 2001 From: jiaweizhuang Date: Fri, 30 Jan 2026 11:12:16 +0000 Subject: [PATCH 4/7] minimum wheel setup --- docker/Dockerfile | 24 ++++++++++++++++++++++++ docker/setup.py | 21 +++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 docker/setup.py diff --git a/docker/Dockerfile b/docker/Dockerfile index 03c26ec2a..32f49b240 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -60,3 +60,27 @@ RUN ninja -C build && ninja -C build install ENV PATH=$PTO_SOURCE_DIR/build/tools/ptoas:$PATH RUN which ptoas +# create python wheel +ENV PY_PACKAGE_DIR=$LLVM_BUILD_DIR/tools/mlir/python_packages/mlir_core/ + +# copy pto.py, _pto_ops_gen.py +RUN cp $PTO_INSTALL_DIR/mlir/dialects/*.py $PY_PACKAGE_DIR/mlir/dialects/ +# copy libPTOCAPI.a, libPTOIR.a, libPTOTransforms.a +RUN cp -r $PTO_INSTALL_DIR/lib/* $PY_PACKAGE_DIR/mlir/_mlir_libs/ + +COPY ./setup.py $PY_PACKAGE_DIR/ + +WORKDIR $PY_PACKAGE_DIR +RUN pip install --no-cache-dir setuptools wheel +RUN python setup.py bdist_wheel +RUN pip install ./dist/ptoas*.whl + +# test import without adjusting $PYTHONPATH, in some irrelevant dir +WORKDIR /test +RUN python -c "import mlir.ir" +# RUN python -c "from mlir.dialects import pto" + +# show the built wheel by default +WORKDIR $PY_PACKAGE_DIR/dist + +CMD ["/bin/bash"] diff --git a/docker/setup.py b/docker/setup.py new file mode 100644 index 000000000..bd42dbb65 --- /dev/null +++ b/docker/setup.py @@ -0,0 +1,21 @@ +from setuptools import setup, find_namespace_packages + +setup( + name="ptoas", + version="0.1.0", + description="PTO Assembler & Optimizer", + # NOTE: find_namespace_packages detects folders even without __init__.py + packages=find_namespace_packages(), + # NOTE: The * at the end captures .so.22, .so.22.1, etc. + package_data={ + "mlir": [ + "**/*.so*", + "**/*.pyd", + "**/*.py", + "_mlir_libs/*.so*", + ], + }, + include_package_data=True, + zip_safe=False, + python_requires=">=3.9", +) From 5c6431e259fdb01288b472942bdb2b21fcca20f9 Mon Sep 17 00:00:00 2001 From: jiaweizhuang Date: Fri, 30 Jan 2026 11:19:59 +0000 Subject: [PATCH 5/7] fix missing so by audirwheel tool --- docker/Dockerfile | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 32f49b240..48a325a90 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -65,20 +65,23 @@ ENV PY_PACKAGE_DIR=$LLVM_BUILD_DIR/tools/mlir/python_packages/mlir_core/ # copy pto.py, _pto_ops_gen.py RUN cp $PTO_INSTALL_DIR/mlir/dialects/*.py $PY_PACKAGE_DIR/mlir/dialects/ -# copy libPTOCAPI.a, libPTOIR.a, libPTOTransforms.a -RUN cp -r $PTO_INSTALL_DIR/lib/* $PY_PACKAGE_DIR/mlir/_mlir_libs/ COPY ./setup.py $PY_PACKAGE_DIR/ WORKDIR $PY_PACKAGE_DIR -RUN pip install --no-cache-dir setuptools wheel +RUN pip install --no-cache-dir setuptools wheel auditwheel RUN python setup.py bdist_wheel -RUN pip install ./dist/ptoas*.whl -# test import without adjusting $PYTHONPATH, in some irrelevant dir +# fix missing so +RUN export LD_LIBRARY_PATH=$LLVM_BUILD_DIR/lib:$PTO_INSTALL_DIR/lib:$LD_LIBRARY_PATH \ + && auditwheel repair dist/ptoas*.whl + +RUN pip install wheelhouse/ptoas*.whl + +# test import without adjusting $PYTHONPATH and $LD_LIBRARY_PATH, in some irrelevant dir WORKDIR /test RUN python -c "import mlir.ir" -# RUN python -c "from mlir.dialects import pto" +RUN python -c "from mlir.dialects import pto" # show the built wheel by default WORKDIR $PY_PACKAGE_DIR/dist From 7e9bfb80234a60d5d2ffc1f5268963f9b1025581 Mon Sep 17 00:00:00 2001 From: jiaweizhuang Date: Fri, 30 Jan 2026 11:26:43 +0000 Subject: [PATCH 6/7] e2e test --- docker/Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index 48a325a90..58e37adb4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -83,6 +83,12 @@ WORKDIR /test RUN python -c "import mlir.ir" RUN python -c "from mlir.dialects import pto" +WORKDIR $PTO_SOURCE_DIR/test/samples/MatMul/ +RUN python ./tmatmulk.py > ./tmatmulk.pto +RUN ptoas ./tmatmulk.pto -o ./tmatmulk.cpp +# TODO: test compilation of the cpp file using `bisheng` +# TODO: also test `ptoas --enable-insert-sync` + # show the built wheel by default WORKDIR $PY_PACKAGE_DIR/dist From 5f44992cfeb77fde073f41d02eacf128d1067488 Mon Sep 17 00:00:00 2001 From: jiaweizhuang Date: Fri, 30 Jan 2026 11:34:25 +0000 Subject: [PATCH 7/7] run ptoas in local dir, otherwise get path error --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 389285e67..e463c0390 100644 --- a/README.md +++ b/README.md @@ -215,11 +215,11 @@ with Context() as ctx, Location.unknown(): ```bash # 运行python binding 测试 -python3 ./test/samples/MatMul/tmatmulk.py > ./test/samples/MatMul/tmatmulk.pto +cd ./test/samples/MatMul/ +python3 ./tmatmulk.py > ./tmatmulk.pto # 运行ptoas 测试 -./build/tools/ptoas/ptoas ./test/samples/Matmul/tmatmulk.pto -o ./test/samples/Matmul/tmatmulk.cpp - +./build/tools/ptoas/ptoas ./tmatmulk.pto -o ./tmatmulk.cpp ``` ---