From 6cb65bd2c2775d3cdf8e8053f864055f9004348a Mon Sep 17 00:00:00 2001 From: Jiayu Liu Date: Thu, 3 Jun 2021 09:51:13 +0800 Subject: [PATCH 1/3] use requirements.txt to formalize python deps --- .github/workflows/python_build.yml | 6 +-- .github/workflows/python_test.yaml | 63 ++++++++++++------------ python/README.md | 2 +- python/requirements.in | 3 ++ python/requirements.txt | 78 ++++++++++++++++++++++++++++++ 5 files changed, 117 insertions(+), 35 deletions(-) create mode 100644 python/requirements.in create mode 100644 python/requirements.txt diff --git a/.github/workflows/python_build.yml b/.github/workflows/python_build.yml index c86bb81581a71..eba11b8e3a41f 100644 --- a/.github/workflows/python_build.yml +++ b/.github/workflows/python_build.yml @@ -28,12 +28,12 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.6, 3.7, 3.8] + python-version: ["3.6", "3.7", "3.8", "3.9"] os: [macos-latest, windows-latest] steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} @@ -54,7 +54,7 @@ jobs: run: dir python/target\wheels\ - name: List wheels - if: matrix.os != 'windows-latest' + if: matrix.os != 'windows-latest' run: find ./python/target/wheels/ - name: Archive wheels diff --git a/.github/workflows/python_test.yaml b/.github/workflows/python_test.yaml index 3b2111b59d49d..13516ff699dac 100644 --- a/.github/workflows/python_test.yaml +++ b/.github/workflows/python_test.yaml @@ -22,37 +22,38 @@ jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Setup Rust toolchain - run: | - rustup toolchain install nightly-2021-01-06 - rustup default nightly-2021-01-06 - rustup component add rustfmt - - name: Cache Cargo - uses: actions/cache@v2 - with: - path: /home/runner/.cargo - key: cargo-maturin-cache- - - name: Cache Rust dependencies - uses: actions/cache@v2 - with: - path: /home/runner/target - key: target-maturin-cache- - - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - name: Install Python dependencies - run: python -m pip install --upgrade pip setuptools wheel - - name: Run tests - run: | - cd python/ - export CARGO_HOME="/home/runner/.cargo" - export CARGO_TARGET_DIR="/home/runner/target" + - uses: actions/checkout@v2 + - name: Setup Rust toolchain + run: | + rustup toolchain install nightly-2021-01-06 + rustup default nightly-2021-01-06 + rustup component add rustfmt + - name: Cache Cargo + uses: actions/cache@v2 + with: + path: /home/runner/.cargo + key: cargo-maturin-cache- + - name: Cache Rust dependencies + uses: actions/cache@v2 + with: + path: /home/runner/target + key: target-maturin-cache- + - uses: actions/setup-python@v2 + with: + python-version: "3.9" + - name: Install Python dependencies + run: python -m pip install --upgrade pip setuptools wheel + - name: Run tests + run: | + cd python/ - python -m venv venv - source venv/bin/activate + python -m venv venv + source venv/bin/activate - pip install maturin==0.10.4 toml==0.10.1 pyarrow==4.0.0 - maturin develop + pip install -r requirements.txt + maturin develop - python -m unittest discover tests + python -m unittest discover tests + env: + CARGO_HOME: "/home/runner/.cargo" + CARGO_TARGET_DIR: "/home/runner/target" diff --git a/python/README.md b/python/README.md index 1859fca9811c0..e875b955a2f02 100644 --- a/python/README.md +++ b/python/README.md @@ -135,7 +135,7 @@ cd arrow-datafusion/python # prepare development environment (used to build wheel / install in development) python3 -m venv venv -pip install maturin==0.10.4 toml==0.10.1 pyarrow==1.0.0 +pip install -r requirements.txt ``` Whenever rust code changes (your changes or via git pull): diff --git a/python/requirements.in b/python/requirements.in new file mode 100644 index 0000000000000..42470ca51bab8 --- /dev/null +++ b/python/requirements.in @@ -0,0 +1,3 @@ +maturin +toml +pyarrow diff --git a/python/requirements.txt b/python/requirements.txt new file mode 100644 index 0000000000000..635eb22784824 --- /dev/null +++ b/python/requirements.txt @@ -0,0 +1,78 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --generate-hashes +# +maturin==0.10.6 \ + --hash=sha256:0e81496f70a4805e6ea7dda7b0425246c111ccb119a2e22c64abeff131f4dd21 \ + --hash=sha256:3b5d5429bc05a816824420d99973f0cab39d8e274f6c3647bfd9afd95a030304 \ + --hash=sha256:4177a223727a0ad57bc3f69ca4c3bc04bb3cc4da787cc59a8e25808c85685c67 \ + --hash=sha256:4eb4481b6c7d6cac043b969d2eb993c982523e91bb2709f0b09e231cf4846731 \ + --hash=sha256:532625f312185b06ec196fdb0fc79efafc0e98768153d226fb9417c0ca85e410 \ + --hash=sha256:53ef64a147f8a5241a3e932f2db22b5ae7dc5892dae994da319446c5db89dc94 \ + --hash=sha256:a04589da42f62b1d515f35c81274a56fe0d29216894525e8a37fd1e3c69d87b1 \ + --hash=sha256:b58e9e2ba5a3f651d8885c41370a00bb1d3e4d7313cbb63354077153be7650f4 \ + --hash=sha256:bd39f7e08eb9908d4fe1cd9b3c953fad5b1fb4fec9c82d14c2973a65751e1899 \ + --hash=sha256:d63f2a15f0b8db4e70d9a59766ca240b2c2ee2146ed5e4385a6118d941d68b25 \ + --hash=sha256:fa7e1cea2a768257a33aeb556fdec5fc36011bfe82d96730117433c635629dd8 + # via -r requirements.in +numpy==1.20.3 \ + --hash=sha256:1676b0a292dd3c99e49305a16d7a9f42a4ab60ec522eac0d3dd20cdf362ac010 \ + --hash=sha256:16f221035e8bd19b9dc9a57159e38d2dd060b48e93e1d843c49cb370b0f415fd \ + --hash=sha256:43909c8bb289c382170e0282158a38cf306a8ad2ff6dfadc447e90f9961bef43 \ + --hash=sha256:4e465afc3b96dbc80cf4a5273e5e2b1e3451286361b4af70ce1adb2984d392f9 \ + --hash=sha256:55b745fca0a5ab738647d0e4db099bd0a23279c32b31a783ad2ccea729e632df \ + --hash=sha256:5d050e1e4bc9ddb8656d7b4f414557720ddcca23a5b88dd7cff65e847864c400 \ + --hash=sha256:637d827248f447e63585ca3f4a7d2dfaa882e094df6cfa177cc9cf9cd6cdf6d2 \ + --hash=sha256:6690080810f77485667bfbff4f69d717c3be25e5b11bb2073e76bb3f578d99b4 \ + --hash=sha256:66fbc6fed94a13b9801fb70b96ff30605ab0a123e775a5e7a26938b717c5d71a \ + --hash=sha256:67d44acb72c31a97a3d5d33d103ab06d8ac20770e1c5ad81bdb3f0c086a56cf6 \ + --hash=sha256:6ca2b85a5997dabc38301a22ee43c82adcb53ff660b89ee88dded6b33687e1d8 \ + --hash=sha256:6e51534e78d14b4a009a062641f465cfaba4fdcb046c3ac0b1f61dd97c861b1b \ + --hash=sha256:70eb5808127284c4e5c9e836208e09d685a7978b6a216db85960b1a112eeace8 \ + --hash=sha256:830b044f4e64a76ba71448fce6e604c0fc47a0e54d8f6467be23749ac2cbd2fb \ + --hash=sha256:8b7bb4b9280da3b2856cb1fc425932f46fba609819ee1c62256f61799e6a51d2 \ + --hash=sha256:a9c65473ebc342715cb2d7926ff1e202c26376c0dcaaee85a1fd4b8d8c1d3b2f \ + --hash=sha256:c1c09247ccea742525bdb5f4b5ceeacb34f95731647fe55774aa36557dbb5fa4 \ + --hash=sha256:c5bf0e132acf7557fc9bb8ded8b53bbbbea8892f3c9a1738205878ca9434206a \ + --hash=sha256:db250fd3e90117e0312b611574cd1b3f78bec046783195075cbd7ba9c3d73f16 \ + --hash=sha256:e515c9a93aebe27166ec9593411c58494fa98e5fcc219e47260d9ab8a1cc7f9f \ + --hash=sha256:e55185e51b18d788e49fe8305fd73ef4470596b33fc2c1ceb304566b99c71a69 \ + --hash=sha256:ea9cff01e75a956dbee133fa8e5b68f2f92175233de2f88de3a682dd94deda65 \ + --hash=sha256:f1452578d0516283c87608a5a5548b0cdde15b99650efdfd85182102ef7a7c17 \ + --hash=sha256:f39a995e47cb8649673cfa0579fbdd1cdd33ea497d1728a6cb194d6252268e48 + # via pyarrow +pyarrow==4.0.1 \ + --hash=sha256:04be0f7cb9090bd029b5b53bed628548fef569e5d0b5c6cd7f6d0106dbbc782d \ + --hash=sha256:0fde9c7a3d5d37f3fe5d18c4ed015e8f585b68b26d72a10d7012cad61afe43ff \ + --hash=sha256:11517f0b4f4acbab0c37c674b4d1aad3c3dfea0f6b1bb322e921555258101ab3 \ + --hash=sha256:150db335143edd00d3ec669c7c8167d401c4aa0a290749351c80bbf146892b2e \ + --hash=sha256:24040a20208e9b16ba7b284624ebfe67e40f5c40b5dc8d874da322ac0053f9d3 \ + --hash=sha256:33c457728a1ce825b80aa8c8ed573709f1efe72003d45fa6fdbb444de9cc0b74 \ + --hash=sha256:423cd6a14810f4e40cb76e13d4240040fc1594d69fe1c4f2c70be00ad512ade5 \ + --hash=sha256:5387db80c6a7b5598884bf4df3fc546b3373771ad614548b782e840b71704877 \ + --hash=sha256:5a76ec44af838862b23fb5cfc48765bc7978f7b58a181c96ad92856280de548b \ + --hash=sha256:5f2660f59dfcfd34adac7c08dc7f615920de703f191066ed6277628975f06878 \ + --hash=sha256:6b7bd8f5aa327cc32a1b9b02a76502851575f5edb110f93c59a45c70211a5618 \ + --hash=sha256:72cf3477538bd8504f14d6299a387cc335444f7a188f548096dfea9533551f02 \ + --hash=sha256:76b75a9cfc572e890a1e000fd532bdd2084ec3f1ee94ee51802a477913a21072 \ + --hash=sha256:a81adbfbe2f6528d4593b5a8962b2751838517401d14e9d4cab6787478802693 \ + --hash=sha256:a968375c66e505f72b421f5864a37f51aad5da61b6396fa283f956e9f2b2b923 \ + --hash=sha256:afd4f7c0a225a326d2c0039cdc8631b5e8be30f78f6b7a3e5ce741cf5dd81c72 \ + --hash=sha256:b05bdd513f045d43228247ef4d9269c88139788e2d566f4cb3e855e282ad0330 \ + --hash=sha256:c2733c9bcd00074ce5497dd0a7b8a10c91d3395ddce322d7021c7fdc4ea6f610 \ + --hash=sha256:d0f080b2d9720bec42624cb0df66f60ae66b84a2ccd1fe2c291322df915ac9db \ + --hash=sha256:dcd20ee0240a88772eeb5691102c276f5cdec79527fb3a0679af7f93f93cb4bd \ + --hash=sha256:e1351576877764fb4d5690e4721ce902e987c85f4ab081c70a34e1d24646586e \ + --hash=sha256:e44dfd7e61c9eb6dda59bc49ad69e77945f6d049185a517c130417e3ca0494d8 \ + --hash=sha256:ee3d87615876550fee9a523307dd4b00f0f44cf47a94a32a07793da307df31a0 \ + --hash=sha256:fa7b165cfa97158c1e6d15c68428317b4f4ae786d1dc2dbab43f1328c1eb43aa \ + --hash=sha256:fe976695318560a97c6d31bba828eeca28c44c6f6401005e54ba476a28ac0a10 + # via -r requirements.in +toml==0.10.2 \ + --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \ + --hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f + # via + # -r requirements.in + # maturin From 7d43e98ba0c8486fcba5f8637eecf6940d9b5f23 Mon Sep 17 00:00:00 2001 From: Jiayu Liu Date: Thu, 3 Jun 2021 09:53:36 +0800 Subject: [PATCH 2/3] fix header --- python/requirements.in | 16 ++++++++++++++++ python/requirements.txt | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/python/requirements.in b/python/requirements.in index 42470ca51bab8..3ef9f18966d4b 100644 --- a/python/requirements.in +++ b/python/requirements.in @@ -1,3 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. maturin toml pyarrow diff --git a/python/requirements.txt b/python/requirements.txt index 635eb22784824..ff02b80cf6fc3 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,3 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # # This file is autogenerated by pip-compile # To update, run: From dfef5afc29677a5b25d97dd77ee3d8f525c665e7 Mon Sep 17 00:00:00 2001 From: Jiayu Liu Date: Fri, 4 Jun 2021 00:15:34 +0800 Subject: [PATCH 3/3] update readme --- python/README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/python/README.md b/python/README.md index e875b955a2f02..2f21589aa0131 100644 --- a/python/README.md +++ b/python/README.md @@ -144,3 +144,23 @@ Whenever rust code changes (your changes or via git pull): venv/bin/maturin develop venv/bin/python -m unittest discover tests ``` + +## How to update dependencies + +To change test dependencies, change the `requirements.in` and run + +```bash +# install pip-tools (this can be done only once), also consider running in venv +pip install pip-tools + +# change requirements.in and then run +pip-compile --generate-hashes +``` + +To update dependencies, run + +```bash +pip-compile update +``` + +More details [here](https://github.com/jazzband/pip-tools)