From dae2d8df9821cf9e832b30138d348b94cfd05552 Mon Sep 17 00:00:00 2001 From: mayeut Date: Sun, 9 May 2021 18:43:28 +0200 Subject: [PATCH 1/2] feat: deploy a single py2.py3 wheel per platform Add python 2.7 tests Check exactly 1 final wheel per build exists in ./dist Only rely on auditwheel to add manylinux* platform tags Remove dist-staging --- .circleci/config.yml | 27 +++++++++++--------- .travis.yml | 6 ----- appveyor.yml | 12 --------- scikit-ci.yml | 20 +++++++-------- scripts/convert_to_generic_platform_wheel.py | 27 +++++++++++++++----- 5 files changed, 45 insertions(+), 47 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ed104b20..3ee5ec0e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -21,6 +21,9 @@ references: echo "MANYLINUX_PYTHON [${MANYLINUX_PYTHON}]" /opt/python/${MANYLINUX_PYTHON}/bin/pip install scikit-ci /opt/python/${MANYLINUX_PYTHON}/bin/ci + if [ "${UPLOAD_SDIST}" == "tests-only" ]; then + rm -f ./dist/* + fi - persist_to_workspace: root: ./ paths: @@ -44,17 +47,17 @@ references: jobs: # x64 - manylinux-x64_cp27-cp27mu: - <<: *x64_build_job manylinux-x64_cp37-cp37m_upload-sdist: <<: *x64_build_job # x86 - manylinux-x86_cp27-cp27mu: - <<: *x86_build_job manylinux-x86_cp37-cp37m: <<: *x86_build_job + # python 2.7 tests + manylinux-x64_cp27-cp27mu_tests-only: + <<: *x64_build_job + deploy-master: docker: - image: circleci/python:3.7.0-stretch @@ -87,35 +90,35 @@ workflows: build-test-deploy: jobs: # x64 - - manylinux-x64_cp27-cp27mu: - <<: *no_filters - manylinux-x64_cp37-cp37m_upload-sdist: <<: *no_filters # x86 - - manylinux-x86_cp27-cp27mu: - <<: *no_filters - manylinux-x86_cp37-cp37m: <<: *no_filters + # python 2.7 tests + - manylinux-x64_cp27-cp27mu_tests-only: + <<: *no_filters + - deploy-master: requires: # x64 - - manylinux-x64_cp27-cp27mu - manylinux-x64_cp37-cp37m_upload-sdist # x86 - - manylinux-x86_cp27-cp27mu - manylinux-x86_cp37-cp37m + # python 2.7 test + - manylinux-x64_cp27-cp27mu_tests-only filters: branches: only: master - deploy-release: requires: # x64 - - manylinux-x64_cp27-cp27mu - manylinux-x64_cp37-cp37m_upload-sdist # x86 - - manylinux-x86_cp27-cp27mu - manylinux-x86_cp37-cp37m + # python 2.7 test + - manylinux-x64_cp27-cp27mu_tests-only filters: tags: only: /^[0-9]+(\.[0-9]+)*(\.post[0-9]+)?$/ diff --git a/.travis.yml b/.travis.yml index 8152f976..fe794655 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,12 +17,6 @@ matrix: - PYTHON_VERSION=3.7.9 - MACOSX_DEPLOYMENT_TARGET=10.10 - - os: osx - language: generic - env: - - PYTHON_VERSION=2.7.18 - - MACOSX_DEPLOYMENT_TARGET=10.10 - - arch: arm64-graviton2 virt: vm group: edge diff --git a/appveyor.yml b/appveyor.yml index 95c6fcc1..4c5d4b67 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -8,18 +8,6 @@ version: "0.0.1.{build}" environment: matrix: - - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 - PYTHON_DIR: "C:\\Python27" - PYTHON_VERSION: "2.7.x" - PYTHON_ARCH: "32" - BLOCK: "0" - - - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 - PYTHON_DIR: "C:\\Python27-x64" - PYTHON_VERSION: "2.7.x" - PYTHON_ARCH: "64" - BLOCK: "0" - - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 PYTHON_DIR: "C:\\Python37" PYTHON_VERSION: "3.7.x" diff --git a/scikit-ci.yml b/scikit-ci.yml index 5b29ba60..2f2f4015 100644 --- a/scikit-ci.yml +++ b/scikit-ci.yml @@ -49,7 +49,6 @@ before_install: import os import ci import platform - os.environ["SETUP_BDIST_WHEEL_ARGS"] = "--plat-name %s" % os.environ["AUDITWHEEL_PLAT"] setup_cmake_args = [] setup_cmake_args.append("-DSTRIP_EXECUTABLE:FILEPATH=/opt/rh/devtoolset-9/root/usr/" + "/bin/strip") if platform.machine() in {"aarch64", "ppc64le", "s390x"}: @@ -95,22 +94,19 @@ build: - | # Since there are no external shared libraries to bundle into the wheels # this step will fixup the wheel switching from 'linux' to 'manylinux' tag - for whl in dist/*linux*_$(arch).whl; do - auditwheel repair --plat ${AUDITWHEEL_PLAT} $whl -w ./dist + for whl in dist/*.whl; do + auditwheel repair --plat ${AUDITWHEEL_PLAT} ${whl} -w ./dist/ + rm ${whl} done circle: commands: - | - arch=${AUDITWHEEL_ARCH} - if [[ ${arch} == "aarch64" ]]; then - exit - fi # Since there are no external shared libraries to bundle into the wheels # this step will fixup the wheel switching from 'linux' to 'manylinux' tag - for whl in dist/*linux_${arch}.whl; do - auditwheel repair --plat ${AUDITWHEEL_PLAT} $whl -w ./dist/ - rm $whl + for whl in dist/*.whl; do + auditwheel repair --plat ${AUDITWHEEL_PLAT} ${whl} -w ./dist/ + rm ${whl} done test: @@ -139,7 +135,9 @@ test: additional_platforms.append("macosx_11_0_universal2") for wheel in wheels: - convert_to_generic_platform_wheel(wheel, remove_original=True, additional_platforms=additional_platforms) + convert_to_generic_platform_wheel(wheel, remove_original=True, py2_py3=True, additional_platforms=additional_platforms) + wheels = glob.glob("dist/*.whl") + assert len(wheels) == 1, "There shall be exactly 1 final wheel per build" appveyor: commands: diff --git a/scripts/convert_to_generic_platform_wheel.py b/scripts/convert_to_generic_platform_wheel.py index cc1de1f8..dc75e030 100644 --- a/scripts/convert_to_generic_platform_wheel.py +++ b/scripts/convert_to_generic_platform_wheel.py @@ -41,7 +41,7 @@ def _to_generic_pyver(pyver_tags): return ['py%s' % tag[2] if tag.startswith('cp') else tag for tag in pyver_tags] -def _convert_to_generic_platform_wheel(wheel_ctx, additional_platforms): +def _convert_to_generic_platform_wheel(wheel_ctx, py2_py3, additional_platforms): """Switch to generic python tags and remove ABI tags from a wheel Convert implementation specific python tags to their generic equivalent and @@ -51,6 +51,8 @@ def _convert_to_generic_platform_wheel(wheel_ctx, additional_platforms): ---------- wheel_ctx : InWheelCtx An open wheel context + py2_py3: Bool + Wether the pyver tag shall be py2.py3 or just the one inferred from the wheel name additional_platforms : Optional[Iterable[str]] An optional iterable of additional platform to add to the wheel """ @@ -91,6 +93,10 @@ def _convert_to_generic_platform_wheel(wheel_ctx, additional_platforms): original_pyver_tags = fparts['pyver'].split('.') logger.debug('Previous pyver tags: %s', ', '.join(original_pyver_tags)) pyver_tags = _to_generic_pyver(original_pyver_tags) + if py2_py3: + if len(set(["py2", "py3"]) & set(pyver_tags)) == 0: + raise ValueError("pyver_tags does not contain py2 nor py3") + pyver_tags = list(sorted(set(pyver_tags + ["py2", "py3"]))) if pyver_tags != original_pyver_tags: logger.debug('New pyver tags ....: %s', ', '.join(pyver_tags)) fparts['pyver'] = '.'.join(pyver_tags) @@ -115,8 +121,7 @@ def _convert_to_generic_platform_wheel(wheel_ctx, additional_platforms): # Python version, C-API version combinations pyc_apis = [] - for tag in in_info_tags: - py_ver = '.'.join(_to_generic_pyver(tag.split('-')[0].split('.'))) + for py_ver in pyver_tags: abi = 'none' pyc_apis.append('-'.join([py_ver, abi])) # unique Python version, C-API version combinations @@ -138,7 +143,7 @@ def _convert_to_generic_platform_wheel(wheel_ctx, additional_platforms): def convert_to_generic_platform_wheel(wheel_path, out_dir='./dist/', remove_original=False, verbose=0, - additional_platforms=None): + py2_py3=False, additional_platforms=None): logging.disable(logging.NOTSET) if verbose >= 1: logging.basicConfig(level=logging.DEBUG) @@ -150,7 +155,7 @@ def convert_to_generic_platform_wheel(wheel_path, out_dir='./dist/', remove_orig with InWheelCtx(wheel_path) as ctx: ctx.out_wheel = pjoin(out_dir, wheel_fname) - ctx.out_wheel = _convert_to_generic_platform_wheel(ctx, additional_platforms) + ctx.out_wheel = _convert_to_generic_platform_wheel(ctx, py2_py3, additional_platforms) if remove_original: logger.info('Removed original wheel %s' % wheel_path) @@ -179,13 +184,23 @@ def main(): dest='remove_original', action='store_true', help='Remove original wheel') + p.add_argument("--py2-py3", + dest='py2_py3', + action='store_true', + help='Remove original wheel') + p.add_argument("-p", + "--add-platform", + dest='additional_platforms', + action="append", + help='Add a platform tag') args = p.parse_args() if not isfile(args.WHEEL_FILE): p.error('cannot access %s. No such file' % args.WHEEL_FILE) - convert_to_generic_platform_wheel(args.WHEEL_FILE, args.WHEEL_DIR, args.remove_original, args.verbose) + convert_to_generic_platform_wheel(args.WHEEL_FILE, args.WHEEL_DIR, args.remove_original, args.verbose, + args.py2_py3, args.additional_platforms) if __name__ == '__main__': From ce1464d161f57737ac17297a11792a552c965157 Mon Sep 17 00:00:00 2001 From: mayeut Date: Fri, 14 May 2021 11:56:03 +0200 Subject: [PATCH 2/2] fix: check files in dist/* on all commits Add `long_description_content_type` to metadata Fixes twine check warning --- .circleci/config.yml | 33 ++++++++++++++++++++++++--------- .travis.yml | 6 +++++- appveyor.yml | 5 ++++- setup.py | 1 + 4 files changed, 34 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3ee5ec0e..4a3e6987 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -58,9 +58,25 @@ jobs: manylinux-x64_cp27-cp27mu_tests-only: <<: *x64_build_job + check-dist: + docker: + - image: cimg/python:3.9.5 + steps: + - attach_workspace: + at: ./ + - run: + name: Check dist + command: | + echo "Check dist" + ls dist + python -m venv ../venv + . ../venv/bin/activate + pip install twine + twine check --strict dist/* + deploy-master: docker: - - image: circleci/python:3.7.0-stretch + - image: cimg/python:3.9.5 steps: - attach_workspace: at: ./ @@ -71,7 +87,7 @@ jobs: deploy-release: docker: - - image: circleci/python:3.7.0-stretch + - image: cimg/python:3.9.5 steps: - attach_workspace: at: ./ @@ -100,7 +116,7 @@ workflows: - manylinux-x64_cp27-cp27mu_tests-only: <<: *no_filters - - deploy-master: + - check-dist: requires: # x64 - manylinux-x64_cp37-cp37m_upload-sdist @@ -108,17 +124,16 @@ workflows: - manylinux-x86_cp37-cp37m # python 2.7 test - manylinux-x64_cp27-cp27mu_tests-only + + - deploy-master: + requires: + - check-dist filters: branches: only: master - deploy-release: requires: - # x64 - - manylinux-x64_cp37-cp37m_upload-sdist - # x86 - - manylinux-x86_cp37-cp37m - # python 2.7 test - - manylinux-x64_cp27-cp27mu_tests-only + - check-dist filters: tags: only: /^[0-9]+(\.[0-9]+)*(\.post[0-9]+)?$/ diff --git a/.travis.yml b/.travis.yml index fe794655..6e77dafb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -85,13 +85,17 @@ script: if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then ci test fi + pwd + ls dist + PATH=~/.pyenv/versions/${PYTHON_VERSION}/bin/:$PATH + twine --version + twine check --strict dist/* after_success: - | if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then ci after_test fi - pwd && ls dist; PATH=~/.pyenv/versions/${PYTHON_VERSION}/bin/:$PATH && twine --version deploy: # deploy-release diff --git a/appveyor.yml b/appveyor.yml index 4c5d4b67..051763c2 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -39,6 +39,9 @@ build_script: test_script: - "%PYTHON_DIR%\\python.exe -m ci test" + - ps: | + $env:PATH="$env:PYTHON_DIR/Scripts/;$env:PATH" + twine check --strict dist/* after_test: - "%PYTHON_DIR%\\python.exe -m ci after_test" @@ -48,9 +51,9 @@ on_finish: deploy_script: - ps: | + $env:PATH="$env:PYTHON_DIR/Scripts/;$env:PATH" if ($env:appveyor_repo_tag -eq $true -and $env:appveyor_repo_tag_name –match "^[0-9]+(\.[0-9]+)*(\.post[0-9]+)?$") { Write-Host "deploy release" - $env:PATH="$env:PYTHON_DIR/Scripts/;$env:PATH" twine upload -u $env:PYPI_USER -p $env:PYPI_PASSWORD --skip-existing dist/* } elseif ($env:appveyor_repo_branch -eq "master") { Write-Host "deploy master (not implemented)" diff --git a/setup.py b/setup.py index 3c9a9db4..97f7350d 100755 --- a/setup.py +++ b/setup.py @@ -68,6 +68,7 @@ def parse_requirements(filename): 'tools designed to build, test and package software', long_description=readme + '\n\n' + history, + long_description_content_type='text/x-rst', classifiers=[ 'License :: OSI Approved :: Apache Software License',