From 33def0aa876fd61593dbf703ae2cb09c6c775304 Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Sun, 10 Oct 2021 19:53:39 -0700 Subject: [PATCH 1/5] Use Github Actions + cibuildwheel for all wheels --- .github/workflows/build.yml | 102 ++++++++++++++++++++++++++++++++++ .travis/build-linux-wheels.sh | 14 ----- MANIFEST.in | 13 +++++ appveyor.yml | 31 ----------- install_python.ps1 | 37 ------------ release_process.md | 37 +++--------- tools/download_typed_ast.py | 78 -------------------------- 7 files changed, 124 insertions(+), 188 deletions(-) create mode 100644 .github/workflows/build.yml delete mode 100755 .travis/build-linux-wheels.sh delete mode 100644 appveyor.yml delete mode 100644 install_python.ps1 delete mode 100755 tools/download_typed_ast.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..27ae1de5 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,102 @@ +name: Build wheels + +on: + push: + branches: [master] + tags: ['*'] + +jobs: + build_wheels: + name: py${{ matrix.python-version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + # cibuildwheel builds linux wheels inside a manylinux container + # it also takes care of procuring the correct python version for us + os: [ubuntu-latest, windows-latest, macos-latest] + python-version: [36, 37, 38, 39, 310] + + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + name: Install Python + with: + python-version: "3.9" + - name: Install cibuildwheel + run: | + python -m pip install "cibuildwheel==2.1.3" + - name: Build wheels + env: + CIBW_BUILD: "cp${{ matrix.python-version }}-*" + CIBW_SKIP: "*-manylinux_i686 *-win32" + CIBW_BUILD_VERBOSITY: 1 + run: | + python -m cibuildwheel --output-dir wheelhouse . + - uses: actions/upload-artifact@v2 + with: + name: dist + path: ./wheelhouse/*.whl + build_sdist_python_wheel: + name: sdist and python wheel + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + name: Install Python + with: + python-version: "3.9" + - name: Run check-manifest + run: | + pip install check-manifest + check-manifest -v + - name: Build sdist and wheel + run: | + pip install --upgrade setuptools pip wheel + python setup.py sdist + - uses: actions/upload-artifact@v2 + with: + name: dist + path: | + dist/*.tar.gz + release: + name: create release + needs: [build_wheels, build_sdist_python_wheel] + runs-on: ubuntu-latest + steps: + - name: Download artifact + uses: actions/download-artifact@v2 + with: + name: dist + path: dist + - name: Release + # https://github.com/actions/upload-release-asset/issues/47 + uses: actions/github-script@v2 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const fs = require('fs').promises; + const { repo: { owner, repo }, sha } = context; + + console.log('environment', process.versions); + console.log({ owner, repo, sha }); + + const release = await github.repos.createRelease({ + owner, repo, + // if GITHUB_REF just appears to be a branch, use tag-{commit} as the tag + tag_name: process.env.GITHUB_REF.includes("refs/heads/") ? "tag-" + sha : process.env.GITHUB_REF.split("/").pop(), + target_commitish: sha + }); + + console.log('created release', { release }); + + for (let file of await fs.readdir('dist')) { + console.log('uploading', file); + + await github.repos.uploadReleaseAsset({ + owner, repo, + release_id: release.data.id, + name: file, + data: await fs.readFile(`./dist/${file}`) + }); + } diff --git a/.travis/build-linux-wheels.sh b/.travis/build-linux-wheels.sh deleted file mode 100755 index 0449e9e2..00000000 --- a/.travis/build-linux-wheels.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -eux - -# Compile wheels -for PYBIN in /opt/python/*/bin; do - if [ $(echo "${PYBIN}" | grep -o '[[:digit:]][[:digit:]]' | head -n 1) -ge 35 ]; then - # typed_ast only builds on Python 3.3 and newer - "${PYBIN}/pip" wheel /io/ -w wheelhouse/ - fi -done - -# Bundle external shared libraries into the wheels -for whl in wheelhouse/*.whl; do - auditwheel repair "$whl" -w /io/wheelhouse/ -done diff --git a/MANIFEST.in b/MANIFEST.in index 8905ef71..db51c306 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,17 @@ +include ast27/Grammar/Grammar +include ast27/Parser/Python.asdl recursive-include ast27 *.h +recursive-include ast27 *.py + +include ast3/Grammar/Grammar +include ast3/Parser/Python.asdl recursive-include ast3 *.h +recursive-include ast3 *.py + recursive-include ast3/tests *.py include LICENSE + +prune tools +exclude CONTRIBUTING.md +exclude release_process.md +exclude update_process.md diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index d2db0aec..00000000 --- a/appveyor.yml +++ /dev/null @@ -1,31 +0,0 @@ -environment: - matrix: - # For Python versions available on Appveyor, see - # https://www.appveyor.com/docs/windows-images-software/#python - - PYTHON: "C:\\Python35" - - PYTHON: "C:\\Python35-x64" - - PYTHON: "C:\\Python36" - - PYTHON: "C:\\Python36-x64" - - PYTHON: "C:\\Python37" - - PYTHON: "C:\\Python37-x64" - - PYTHON: "C:\\Python38" - - PYTHON: "C:\\Python38-x64" - - PYTHON: "C:\\Python39" - - PYTHON: "C:\\Python39-x64" - -install: - # Download and install python if we don't have the version we need - - ps: .\install_python.ps1 - # We need wheel installed to build wheels - - "%PYTHON%\\python.exe -m pip install wheel" - -build: off - -test_script: - - "%PYTHON%\\python.exe setup.py test" - -after_test: - - "%PYTHON%\\python.exe setup.py bdist_wheel" - -artifacts: - - path: dist\* diff --git a/install_python.ps1 b/install_python.ps1 deleted file mode 100644 index 5307eb63..00000000 --- a/install_python.ps1 +++ /dev/null @@ -1,37 +0,0 @@ -# This comes from the multibuild project (https://github.com/matthew-brett/multibuild) -# which is Copyright (c) 2013-2019, Matt Terry and Matthew Brett; all rights reserved. -# and distributed under a 2-clause BSD license. - -# Install specified Python version. -# Install only if: -# Our current matrix entry uses this Python version AND -# Python version is not already available. - -$py_exe = "${env:PYTHON}\Python.exe" -if ( [System.IO.File]::Exists($py_exe) ) { - exit 0 -} -$req_nodot = $env:PYTHON -replace '\D+Python(\d+(?:rc\d+)?)(?:-x64)?','$1' -$req_ver = $req_nodot -replace '(\d)(\d+)','$1.$2.0' -$req_dir = $req_nodot -replace '(\d)(\d+)(.*)','$1.$2.0' - -if ($env:PYTHON -eq "C:\Python${req_nodot}-x64") { - $exe_suffix="-amd64" -} elseif ($env:PYTHON -eq "C:\Python${req_nodot}") { - $exe_suffix="" -} else { - exit 0 -} - -$py_url = "https://www.python.org/ftp/python" -Write-Host "Installing Python ${req_ver}$exe_suffix..." -ForegroundColor Cyan -$exePath = "$env:TEMP\python-${req_ver}${exe_suffix}.exe" -$downloadFile = "$py_url/${req_dir}/python-${req_ver}${exe_suffix}.exe" - -Write-Host "Downloading $downloadFile..." -(New-Object Net.WebClient).DownloadFile($downloadFile, $exePath) -Write-Host "Installing..." -cmd /c start /wait $exePath /quiet TargetDir="$env:PYTHON" Shortcuts=0 Include_launcher=0 InstallLauncherAllUsers=0 -Write-Host "Python ${req_ver} installed to $env:PYTHON" - -echo "$(& $py_exe --version 2> $null)" diff --git a/release_process.md b/release_process.md index e6e7cecc..34ff5636 100644 --- a/release_process.md +++ b/release_process.md @@ -8,31 +8,12 @@ 2. Make a git tag pointing to this commit with the version number as the name of the tag. 3. Push the commit and the tag. -4. Wait for the Travis CI and Appveyor builds to complete. -5. Make sure there's nothing in your `typed_ast/dist` directory. -6. Run `python3 setup.py sdist` (this creates `dist/typed-ast-VERSION.tar.gz`). -7. Download the wheels from Travis-CI and Appveyor. - - Do this using `tools/download_typed_ast.py`. If you run into issues, - download them manually: - - Find the Appveyor build for the tag - [here](https://ci.appveyor.com/project/ddfisher/typed-ast-a4xqu/history) and - download the artifact produced by each job into the `dist` directory. - - Download [the latest manylinux - wheels](https://console.cloud.google.com/storage/browser/typed-ast) into the - `dist` directory. (You will have to sign in with your Google account to - access these wheels, but all Google accounts have access.) - -8. On a Mac with Python 3.6, 3.7, 3.8 and 3.9 installed, run - `python3.6 setup.py bdist_wheel`, `python3.7 setup.py bdist_wheel`, - `python3.8 setup.py bdist_wheel` and `python3.9 setup.py bdist_wheel` - (this creates wheels in `dist`). -9. Confirm that the wheels for MacOS target `macosx_10_9` (and not say, - `macosx_10_15`). You may need `export MACOSX_DEPLOYMENT_TARGET=10.9` and - possibly to recompile Python with that environment variable set. -10. Compare the wheels produced to previous release of typed-ast to make sure - you have the full matrix. -11. If possible, verify the final `typed_ast` wheels work on Windows, MacOS, - and Linux platforms. -12. Upload the sdist and wheels to PyPI with `twine upload dist/*`. -13. Make a commit which bumps the bugfix version and adds back the `.dev0` - suffix. +4. Wait for the Github Actions build to complete. +5. Download all assets from the release made by the Github build. +6. Compare the wheels produced to previous release of typed-ast to make sure + you have the full matrix. +7. If possible, verify the final `typed_ast` wheels work on Windows, MacOS, + and Linux platforms. +8. Upload the sdist and wheels to PyPI with `twine upload dist/*`. +9. Make a commit which bumps the bugfix version and adds back the `.dev0` + suffix. diff --git a/tools/download_typed_ast.py b/tools/download_typed_ast.py deleted file mode 100755 index 55ed312d..00000000 --- a/tools/download_typed_ast.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python3 - -# Hacky script to download linux and windows typed_ast wheels from appveyor and gcloud - -import os -import os.path -import json -import sys -from urllib.request import urlopen - -# Appveyor download for windows wheels -api_url = 'https://ci.appveyor.com/api/' -def get_json(path): - url = api_url + path - f = urlopen(url) - data = f.read() - return json.loads(data) - -def download(url): - print('Downloading', url) - name = os.path.join('dist', os.path.split(url)[1]) - with urlopen(url) as f: - data = f.read() - with open(name, 'wb') as f: - f.write(data) - - -def download_appveyor(version): - project_base = 'projects/ddfisher/typed-ast-a4xqu' - history = get_json(project_base + '/history?recordsNumber=20') - for build in history['builds']: - if build.get('tag') == version: - build_version = build['version'] - build_version = str(build['buildId']) - break - else: - sys.exit("Couldn't find tag") - print(build_version) - - build = get_json(project_base + '/builds/' + build_version) - for job in build['build']['jobs']: - artifact_url = 'buildjobs/{}/artifacts'.format(job['jobId']) - artifacts = get_json(artifact_url) - for artifact in artifacts: - download(api_url + artifact_url + '/' + artifact['fileName']) - -# gcloud downloads for linux wehels -MIN_VER = 5 -MAX_VER = 9 - -GCLOUD_URL = "https://storage.googleapis.com/typed-ast/typed_ast-{version}-cp3{pyver}-cp3{pyver}{abi_tag}-{platform}.whl" - -def download_entries(base_url, version, platform): - entries = "" - for pyver in range(MIN_VER, MAX_VER + 1): - abi_tag = "" if pyver >= 8 else "m" - url = base_url.format( - version=version, - pyver=pyver, - abi_tag=abi_tag, - platform=platform) - download(url) - -def main(argv): - if len(argv) != 2: - sys.exit("Usage: download_typed_ast.py version") - - version = argv[1] - os.makedirs('dist', exist_ok=True) - - download_entries(GCLOUD_URL, version, 'manylinux1_x86_64') - download_entries(GCLOUD_URL, version, 'manylinux1_i686') - download_entries(GCLOUD_URL, version, 'manylinux2014_aarch64') - - download_appveyor(version) - -if __name__ == '__main__': - main(sys.argv) From dea470d42de34ee6d27a120a353b12d47da51d29 Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Sun, 10 Oct 2021 20:25:13 -0700 Subject: [PATCH 2/5] apple silicon --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 27ae1de5..283b0e73 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -30,6 +30,7 @@ jobs: env: CIBW_BUILD: "cp${{ matrix.python-version }}-*" CIBW_SKIP: "*-manylinux_i686 *-win32" + CIBW_ARCHS_MACOS: "x86_64 arm64" CIBW_BUILD_VERBOSITY: 1 run: | python -m cibuildwheel --output-dir wheelhouse . From e09a4d20a5de2d7dd4efe34091d22f4caf7cffd9 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sun, 10 Oct 2021 22:03:54 -0700 Subject: [PATCH 3/5] Update release_process.md Co-authored-by: Hugo van Kemenade --- release_process.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/release_process.md b/release_process.md index 34ff5636..ed2ceb7c 100644 --- a/release_process.md +++ b/release_process.md @@ -8,11 +8,11 @@ 2. Make a git tag pointing to this commit with the version number as the name of the tag. 3. Push the commit and the tag. -4. Wait for the Github Actions build to complete. -5. Download all assets from the release made by the Github build. -6. Compare the wheels produced to previous release of typed-ast to make sure +4. Wait for the GitHub Actions build to complete. +5. Download all assets from the release made by the GitHub build. +6. Compare the wheels produced with the previous release of typed-ast to make sure you have the full matrix. -7. If possible, verify the final `typed_ast` wheels work on Windows, MacOS, +7. If possible, verify the final `typed_ast` wheels work on Windows, macOS, and Linux platforms. 8. Upload the sdist and wheels to PyPI with `twine upload dist/*`. 9. Make a commit which bumps the bugfix version and adds back the `.dev0` From a5650a523c538276c2acf998411cbfc9e672d46c Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Sun, 10 Oct 2021 22:09:27 -0700 Subject: [PATCH 4/5] run tests, remove travis, don't make release --- .github/workflows/build.yml | 48 ++--------------------- .travis.yml | 77 ------------------------------------- 2 files changed, 3 insertions(+), 122 deletions(-) delete mode 100644 .travis.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 283b0e73..fa056a20 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,9 +1,6 @@ name: Build wheels -on: - push: - branches: [master] - tags: ['*'] +on: [push, pull_request] jobs: build_wheels: @@ -32,6 +29,8 @@ jobs: CIBW_SKIP: "*-manylinux_i686 *-win32" CIBW_ARCHS_MACOS: "x86_64 arm64" CIBW_BUILD_VERBOSITY: 1 + CIBW_BEFORE_TEST: pip install pytest + CIBW_TEST_COMMAND: pytest {package} run: | python -m cibuildwheel --output-dir wheelhouse . - uses: actions/upload-artifact@v2 @@ -60,44 +59,3 @@ jobs: name: dist path: | dist/*.tar.gz - release: - name: create release - needs: [build_wheels, build_sdist_python_wheel] - runs-on: ubuntu-latest - steps: - - name: Download artifact - uses: actions/download-artifact@v2 - with: - name: dist - path: dist - - name: Release - # https://github.com/actions/upload-release-asset/issues/47 - uses: actions/github-script@v2 - with: - github-token: ${{secrets.GITHUB_TOKEN}} - script: | - const fs = require('fs').promises; - const { repo: { owner, repo }, sha } = context; - - console.log('environment', process.versions); - console.log({ owner, repo, sha }); - - const release = await github.repos.createRelease({ - owner, repo, - // if GITHUB_REF just appears to be a branch, use tag-{commit} as the tag - tag_name: process.env.GITHUB_REF.includes("refs/heads/") ? "tag-" + sha : process.env.GITHUB_REF.split("/").pop(), - target_commitish: sha - }); - - console.log('created release', { release }); - - for (let file of await fs.readdir('dist')) { - console.log('uploading', file); - - await github.repos.uploadReleaseAsset({ - owner, repo, - release_id: release.data.id, - name: file, - data: await fs.readFile(`./dist/${file}`) - }); - } diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 5fbbe715..00000000 --- a/.travis.yml +++ /dev/null @@ -1,77 +0,0 @@ -# Run unit tests and build Linux wheels with manylinux. -# See: https://github.com/pypa/manylinux - -os: linux - -dist: xenial - -language: python - -install: - - pip3 install pytest - - pip3 install . - -arch: - - amd64 - - arm64 - -python: - - 3.5 - - 3.6 - - 3.7 - - 3.8 - - 3.9-dev - -script: - - pytest - -jobs: - include: - - name: "manylinux x86-64" - services: - - docker - env: DOCKER_IMAGE=quay.io/pypa/manylinux1_x86_64 - DEPLOY=1 - install: - - docker pull $DOCKER_IMAGE - script: - - docker run --rm -v `pwd`:/io $DOCKER_IMAGE $PRE_CMD /io/.travis/build-linux-wheels.sh - - ls wheelhouse/ - - name: "manylinux i686" - services: - - docker - env: DOCKER_IMAGE=quay.io/pypa/manylinux1_i686 - PRE_CMD=linux32 - DEPLOY=1 - install: - - docker pull $DOCKER_IMAGE - script: - - docker run --rm -v `pwd`:/io $DOCKER_IMAGE $PRE_CMD /io/.travis/build-linux-wheels.sh - - ls wheelhouse/ - - name: "manylinux aarch64" - arch: arm64 - services: - - docker - env: DOCKER_IMAGE=quay.io/pypa/manylinux2014_aarch64 - DEPLOY=1 - install: - - docker pull $DOCKER_IMAGE - script: - - docker run --rm -v `pwd`:/io $DOCKER_IMAGE $PRE_CMD /io/.travis/build-linux-wheels.sh - - ls wheelhouse/ - -# Upload build artifacts to Google Cloud Storage for tagged releases. -# For GCS deployment documentation, see: https://docs.travis-ci.com/user/deployment/gcs/ -# The keys are stored in the repository settings. -# See: https://docs.travis-ci.com/user/environment-variables/#Defining-Variables-in-Repository-Settings -deploy: - provider: gcs - access_key_id: "$GOOGLE_ACCESS_KEY" - secret_access_key: "$GOOGLE_SECRET_KEY" - bucket: "$GOOGLE_BUCKET_NAME" - local_dir: wheelhouse - acl: public-read - on: - repo: python/typed_ast - tags: true - condition: $DEPLOY = 1 # Don't deploy on pytest matrix, only on docker jobs From db6386a2273176041fa78606c61bd1cd61de9068 Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Sun, 10 Oct 2021 22:17:57 -0700 Subject: [PATCH 5/5] doc update --- release_process.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release_process.md b/release_process.md index ed2ceb7c..7053de46 100644 --- a/release_process.md +++ b/release_process.md @@ -9,7 +9,7 @@ of the tag. 3. Push the commit and the tag. 4. Wait for the GitHub Actions build to complete. -5. Download all assets from the release made by the GitHub build. +5. Download all artifacts from the relevant GitHub Actions build. 6. Compare the wheels produced with the previous release of typed-ast to make sure you have the full matrix. 7. If possible, verify the final `typed_ast` wheels work on Windows, macOS,