From e7f4a47fb4e603b0f60564f8e13a3227f979c392 Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Thu, 5 May 2022 12:16:56 -0400 Subject: [PATCH 1/7] Create dependabot.yml --- .github/dependabot.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000000..509acf47227c --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,19 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "gomod" + directory: "/sdks" # Location of package manifests + schedule: + interval: "daily" + - package-ecosystem: "pip" + directory: "/sdks/python" # Location of package manifests + schedule: + interval: "daily" + - package-ecosystem: "gradle" + directory: "/" # Location of package manifests + schedule: + interval: "daily" From 694a866424fef8e82695cda714521f57c7ad0125 Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Thu, 5 May 2022 12:49:51 -0400 Subject: [PATCH 2/7] Apache header --- .github/dependabot.yml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 509acf47227c..0641e6f267bb 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,7 +1,17 @@ -# To get started with Dependabot version updates, you'll need to specify which -# package ecosystems to update and where the package manifests are located. -# Please see the documentation for all configuration options: -# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. version: 2 updates: From 817d23734d5b96c84ec1d8a095deb4ae13c7a1a5 Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Thu, 5 May 2022 15:24:00 -0400 Subject: [PATCH 3/7] See if dependabot will parse now --- sdks/python/setup.py | 526 ++++++++++++++++++++++++++++--------------- 1 file changed, 342 insertions(+), 184 deletions(-) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index b808c87405cb..e550db2b3eb1 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -120,121 +120,12 @@ def get_version(): except ImportError: cythonize = lambda *args, **kwargs: [] -REQUIRED_PACKAGES = [ - # Avro 1.9.2 for python3 was broken. The issue was fixed in version 1.9.2.1 - 'crcmod>=1.7,<2.0', - # dataclasses backport for python_version<3.7. No version bound because this - # is Python standard since Python 3.7 and each Python version is compatible - # with a specific dataclasses version. - 'dataclasses;python_version<"3.7"', - 'orjson<4.0', - # Dill doesn't have forwards-compatibility guarantees within minor version. - # Pickles created with a new version of dill may not unpickle using older - # version of dill. It is best to use the same version of dill on client and - # server, therefore list of allowed versions is very narrow. - # See: https://github.com/uqfoundation/dill/issues/341. - 'dill>=0.3.1.1,<0.3.2', - 'cloudpickle>=2.0.0,<3', - 'fastavro>=0.23.6,<2', - 'grpcio>=1.29.0,<2', - 'hdfs>=2.1.0,<3.0.0', - 'httplib2>=0.8,<0.20.0', - 'numpy>=1.14.3,<1.23.0', - 'pymongo>=3.8.0,<4.0.0', - 'protobuf>=3.12.2,<4', - 'proto-plus>=1.7.1,<2', - 'pyarrow>=0.15.1,<8.0.0', - 'pydot>=1.2.0,<2', - 'python-dateutil>=2.8.0,<3', - 'pytz>=2018.3', - 'requests>=2.24.0,<3.0.0', - 'typing-extensions>=3.7.0', -] - # [BEAM-8181] pyarrow cannot be installed on 32-bit Windows platforms. if sys.platform == 'win32' and sys.maxsize <= 2**32: REQUIRED_PACKAGES = [ p for p in REQUIRED_PACKAGES if not p.startswith('pyarrow') ] -REQUIRED_TEST_PACKAGES = [ - 'freezegun>=0.3.12', - 'joblib>=1.0.1', - 'mock>=1.0.1,<3.0.0', - 'pandas<2.0.0', - 'parameterized>=0.7.1,<0.8.0', - 'pyhamcrest>=1.9,!=1.10.0,<2.0.0', - 'pyyaml>=3.12,<7.0.0', - 'requests_mock>=1.7,<2.0', - 'tenacity>=5.0.2,<6.0', - 'pytest>=4.4.0,<5.0', - 'pytest-xdist>=1.29.0,<2', - 'pytest-timeout>=1.3.3,<2', - 'scikit-learn>=0.20.0', - 'sqlalchemy>=1.3,<2.0', - 'psycopg2-binary>=2.8.5,<3.0.0', - 'testcontainers[mysql]>=3.0.3,<4.0.0', - 'cryptography>=36.0.0', -] - -GCP_REQUIREMENTS = [ - 'cachetools>=3.1.0,<5', - 'google-apitools>=0.5.31,<0.5.32', - # NOTE: Maintainers, please do not require google-auth>=2.x.x - # Until this issue is closed - # https://github.com/googleapis/google-cloud-python/issues/10566 - 'google-auth>=1.18.0,<3', - 'google-auth-httplib2>=0.1.0,<0.2.0', - 'google-cloud-datastore>=1.8.0,<2', - 'google-cloud-pubsub>=2.1.0,<3', - 'google-cloud-pubsublite>=1.2.0,<2', - # GCP packages required by tests - 'google-cloud-bigquery>=1.6.0,<3', - 'google-cloud-bigquery-storage>=2.6.3', - 'google-cloud-core>=0.28.1,<2', - 'google-cloud-bigtable>=0.31.1,<2', - 'google-cloud-spanner>=1.13.0,<2', - 'grpcio-gcp>=0.2.2,<1', - # GCP Packages required by ML functionality - 'google-cloud-dlp>=3.0.0,<4', - 'google-cloud-language>=1.3.0,<2', - 'google-cloud-videointelligence>=1.8.0,<2', - 'google-cloud-vision>=0.38.0,<2', - 'google-cloud-recommendations-ai>=0.1.0,<=0.2.0' -] - -INTERACTIVE_BEAM = [ - 'facets-overview>=1.0.0,<2', - 'google-cloud-dataproc>=3.0.0,<3.2.0', - # IPython>=8 is not compatible with Python<=3.7 - 'ipython>=7,<8;python_version<="3.7"', - 'ipython>=8,<9;python_version>"3.7"', - 'ipykernel>=6,<7', - 'ipywidgets>=7.6.5,<8', - # Skip version 6.1.13 due to - # https://github.com/jupyter/jupyter_client/issues/637 - 'jupyter-client>=6.1.11,<6.1.13', - 'timeloop>=1.0.2,<2', -] - -INTERACTIVE_BEAM_TEST = [ - # notebok utils - 'nbformat>=5.0.5,<6', - 'nbconvert>=6.2.0,<7', - # headless chrome based integration tests - 'needle>=0.5.0,<1', - 'chromedriver-binary>=100,<101', - # use a fixed major version of PIL for different python versions - 'pillow>=7.1.1,<8', -] - -AWS_REQUIREMENTS = ['boto3 >=1.9'] - -AZURE_REQUIREMENTS = [ - 'azure-storage-blob >=12.3.2', - 'azure-core >=1.7.0', -] - # We must generate protos after setup_requires are installed. def generate_protos_first(): @@ -272,78 +163,345 @@ def get_portability_package_data(): # structure must exist before the call to setuptools.find_packages() # executes below. generate_protos_first() - setuptools.setup( - name=PACKAGE_NAME, - version=PACKAGE_VERSION, - description=PACKAGE_DESCRIPTION, - long_description=PACKAGE_LONG_DESCRIPTION, - url=PACKAGE_URL, - download_url=PACKAGE_DOWNLOAD_URL, - author=PACKAGE_AUTHOR, - author_email=PACKAGE_EMAIL, - packages=setuptools.find_packages(), - package_data={ - 'apache_beam': [ - '*/*.pyx', - '*/*/*.pyx', - '*/*.pxd', - '*/*/*.pxd', - '*/*.h', - '*/*/*.h', - 'testing/data/*.yaml', - *get_portability_package_data() - ] - }, - ext_modules=cythonize([ - # Make sure to use language_level=3 cython directive in files below. - 'apache_beam/**/*.pyx', - 'apache_beam/coders/coder_impl.py', - 'apache_beam/metrics/cells.py', - 'apache_beam/metrics/execution.py', - 'apache_beam/runners/common.py', - 'apache_beam/runners/worker/logger.py', - 'apache_beam/runners/worker/opcounters.py', - 'apache_beam/runners/worker/operations.py', - 'apache_beam/transforms/cy_combiners.py', - 'apache_beam/transforms/stats.py', - 'apache_beam/utils/counters.py', - 'apache_beam/utils/windowed_value.py', - ]), - install_requires=REQUIRED_PACKAGES, - python_requires=python_requires, - # BEAM-8840: Do NOT use tests_require or setup_requires. - extras_require={ - 'docs': [ - 'Sphinx>=1.5.2,<2.0', - # Pinning docutils as a workaround for Sphinx issue: - # https://github.com/sphinx-doc/sphinx/issues/9727 - 'docutils==0.17.1' - ], - 'test': REQUIRED_TEST_PACKAGES, - 'gcp': GCP_REQUIREMENTS, - 'interactive': INTERACTIVE_BEAM, - 'interactive_test': INTERACTIVE_BEAM_TEST, - 'aws': AWS_REQUIREMENTS, - 'azure': AZURE_REQUIREMENTS, - 'dataframe': ['pandas>=1.0,<1.5'] - }, - zip_safe=False, - # PyPI package information. - classifiers=[ - 'Intended Audience :: End Users/Desktop', - 'License :: OSI Approved :: Apache Software License', - 'Operating System :: POSIX :: Linux', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - # When updating version classifiers, also update version warnings - # above and in apache_beam/__init__.py. - 'Topic :: Software Development :: Libraries', - 'Topic :: Software Development :: Libraries :: Python Modules', - ], - license='Apache License, Version 2.0', - keywords=PACKAGE_KEYWORDS, - cmdclass={ - 'mypy': mypy, - }, - ) + # [BEAM-8181] pyarrow cannot be installed on 32-bit Windows platforms. + if sys.platform == 'win32' and sys.maxsize <= 2**32: + setuptools.setup( + name=PACKAGE_NAME, + version=PACKAGE_VERSION, + description=PACKAGE_DESCRIPTION, + long_description=PACKAGE_LONG_DESCRIPTION, + url=PACKAGE_URL, + download_url=PACKAGE_DOWNLOAD_URL, + author=PACKAGE_AUTHOR, + author_email=PACKAGE_EMAIL, + packages=setuptools.find_packages(), + package_data={ + 'apache_beam': [ + '*/*.pyx', + '*/*/*.pyx', + '*/*.pxd', + '*/*/*.pxd', + '*/*.h', + '*/*/*.h', + 'testing/data/*.yaml', + *get_portability_package_data() + ] + }, + ext_modules=cythonize([ + # Make sure to use language_level=3 cython directive in files below. + 'apache_beam/**/*.pyx', + 'apache_beam/coders/coder_impl.py', + 'apache_beam/metrics/cells.py', + 'apache_beam/metrics/execution.py', + 'apache_beam/runners/common.py', + 'apache_beam/runners/worker/logger.py', + 'apache_beam/runners/worker/opcounters.py', + 'apache_beam/runners/worker/operations.py', + 'apache_beam/transforms/cy_combiners.py', + 'apache_beam/transforms/stats.py', + 'apache_beam/utils/counters.py', + 'apache_beam/utils/windowed_value.py', + ]), + install_requires=[ + # Avro 1.9.2 for python3 was broken. The issue was fixed in version 1.9.2.1 + 'crcmod>=1.7,<2.0', + # dataclasses backport for python_version<3.7. No version bound because this + # is Python standard since Python 3.7 and each Python version is compatible + # with a specific dataclasses version. + 'dataclasses;python_version<"3.7"', + 'orjson<4.0', + # Dill doesn't have forwards-compatibility guarantees within minor version. + # Pickles created with a new version of dill may not unpickle using older + # version of dill. It is best to use the same version of dill on client and + # server, therefore list of allowed versions is very narrow. + # See: https://github.com/uqfoundation/dill/issues/341. + 'dill>=0.3.1.1,<0.3.2', + 'cloudpickle>=2.0.0,<3', + 'fastavro>=0.23.6,<2', + 'grpcio>=1.29.0,<2', + 'hdfs>=2.1.0,<3.0.0', + 'httplib2>=0.8,<0.20.0', + 'numpy>=1.14.3,<1.23.0', + 'pymongo>=3.8.0,<4.0.0', + 'protobuf>=3.12.2,<4', + 'proto-plus>=1.7.1,<2', + 'pydot>=1.2.0,<2', + 'python-dateutil>=2.8.0,<3', + 'pytz>=2018.3', + 'requests>=2.24.0,<3.0.0', + 'typing-extensions>=3.7.0', + ], + python_requires=python_requires, + # BEAM-8840: Do NOT use tests_require or setup_requires. + extras_require={ + 'docs': [ + 'Sphinx>=1.5.2,<2.0', + # Pinning docutils as a workaround for Sphinx issue: + # https://github.com/sphinx-doc/sphinx/issues/9727 + 'docutils==0.17.1' + ], + 'test': [ + 'freezegun>=0.3.12', + 'joblib>=1.0.1', + 'mock>=1.0.1,<3.0.0', + 'pandas<2.0.0', + 'parameterized>=0.7.1,<0.8.0', + 'pyhamcrest>=1.9,!=1.10.0,<2.0.0', + 'pyyaml>=3.12,<7.0.0', + 'requests_mock>=1.7,<2.0', + 'tenacity>=5.0.2,<6.0', + 'pytest>=4.4.0,<5.0', + 'pytest-xdist>=1.29.0,<2', + 'pytest-timeout>=1.3.3,<2', + 'scikit-learn>=0.20.0', + 'sqlalchemy>=1.3,<2.0', + 'psycopg2-binary>=2.8.5,<3.0.0', + 'testcontainers[mysql]>=3.0.3,<4.0.0', + 'cryptography>=36.0.0', + ], + 'gcp': [ + 'cachetools>=3.1.0,<5', + 'google-apitools>=0.5.31,<0.5.32', + # NOTE: Maintainers, please do not require google-auth>=2.x.x + # Until this issue is closed + # https://github.com/googleapis/google-cloud-python/issues/10566 + 'google-auth>=1.18.0,<3', + 'google-auth-httplib2>=0.1.0,<0.2.0', + 'google-cloud-datastore>=1.8.0,<2', + 'google-cloud-pubsub>=2.1.0,<3', + 'google-cloud-pubsublite>=1.2.0,<2', + # GCP packages required by tests + 'google-cloud-bigquery>=1.6.0,<3', + 'google-cloud-bigquery-storage>=2.6.3', + 'google-cloud-core>=0.28.1,<2', + 'google-cloud-bigtable>=0.31.1,<2', + 'google-cloud-spanner>=1.13.0,<2', + 'grpcio-gcp>=0.2.2,<1', + # GCP Packages required by ML functionality + 'google-cloud-dlp>=3.0.0,<4', + 'google-cloud-language>=1.3.0,<2', + 'google-cloud-videointelligence>=1.8.0,<2', + 'google-cloud-vision>=0.38.0,<2', + 'google-cloud-recommendations-ai>=0.1.0,<=0.2.0' + ], + 'interactive': [ + 'facets-overview>=1.0.0,<2', + 'google-cloud-dataproc>=3.0.0,<3.2.0', + # IPython>=8 is not compatible with Python<=3.7 + 'ipython>=7,<8;python_version<="3.7"', + 'ipython>=8,<9;python_version>"3.7"', + 'ipykernel>=6,<7', + 'ipywidgets>=7.6.5,<8', + # Skip version 6.1.13 due to + # https://github.com/jupyter/jupyter_client/issues/637 + 'jupyter-client>=6.1.11,<6.1.13', + 'timeloop>=1.0.2,<2', + ], + 'interactive_test': [ + # notebok utils + 'nbformat>=5.0.5,<6', + 'nbconvert>=6.2.0,<7', + # headless chrome based integration tests + 'needle>=0.5.0,<1', + 'chromedriver-binary>=100,<101', + # use a fixed major version of PIL for different python versions + 'pillow>=7.1.1,<8', + ], + 'aws': ['boto3 >=1.9'], + 'azure': [ + 'azure-storage-blob >=12.3.2', + 'azure-core >=1.7.0', + ], + 'dataframe': ['pandas>=1.0,<1.5'] + }, + zip_safe=False, + # PyPI package information. + classifiers=[ + 'Intended Audience :: End Users/Desktop', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + # When updating version classifiers, also update version warnings + # above and in apache_beam/__init__.py. + 'Topic :: Software Development :: Libraries', + 'Topic :: Software Development :: Libraries :: Python Modules', + ], + license='Apache License, Version 2.0', + keywords=PACKAGE_KEYWORDS, + cmdclass={ + 'mypy': mypy, + }, + ) + else: + setuptools.setup( + name=PACKAGE_NAME, + version=PACKAGE_VERSION, + description=PACKAGE_DESCRIPTION, + long_description=PACKAGE_LONG_DESCRIPTION, + url=PACKAGE_URL, + download_url=PACKAGE_DOWNLOAD_URL, + author=PACKAGE_AUTHOR, + author_email=PACKAGE_EMAIL, + packages=setuptools.find_packages(), + package_data={ + 'apache_beam': [ + '*/*.pyx', + '*/*/*.pyx', + '*/*.pxd', + '*/*/*.pxd', + '*/*.h', + '*/*/*.h', + 'testing/data/*.yaml', + *get_portability_package_data() + ] + }, + ext_modules=cythonize([ + # Make sure to use language_level=3 cython directive in files below. + 'apache_beam/**/*.pyx', + 'apache_beam/coders/coder_impl.py', + 'apache_beam/metrics/cells.py', + 'apache_beam/metrics/execution.py', + 'apache_beam/runners/common.py', + 'apache_beam/runners/worker/logger.py', + 'apache_beam/runners/worker/opcounters.py', + 'apache_beam/runners/worker/operations.py', + 'apache_beam/transforms/cy_combiners.py', + 'apache_beam/transforms/stats.py', + 'apache_beam/utils/counters.py', + 'apache_beam/utils/windowed_value.py', + ]), + install_requires=[ + # Avro 1.9.2 for python3 was broken. The issue was fixed in version 1.9.2.1 + 'crcmod>=1.7,<2.0', + # dataclasses backport for python_version<3.7. No version bound because this + # is Python standard since Python 3.7 and each Python version is compatible + # with a specific dataclasses version. + 'dataclasses;python_version<"3.7"', + 'orjson<4.0', + # Dill doesn't have forwards-compatibility guarantees within minor version. + # Pickles created with a new version of dill may not unpickle using older + # version of dill. It is best to use the same version of dill on client and + # server, therefore list of allowed versions is very narrow. + # See: https://github.com/uqfoundation/dill/issues/341. + 'dill>=0.3.1.1,<0.3.2', + 'cloudpickle>=2.0.0,<3', + 'fastavro>=0.23.6,<2', + 'grpcio>=1.29.0,<2', + 'hdfs>=2.1.0,<3.0.0', + 'httplib2>=0.8,<0.20.0', + 'numpy>=1.14.3,<1.23.0', + 'pymongo>=3.8.0,<4.0.0', + 'protobuf>=3.12.2,<4', + 'proto-plus>=1.7.1,<2', + 'pyarrow>=0.15.1,<8.0.0', + 'pydot>=1.2.0,<2', + 'python-dateutil>=2.8.0,<3', + 'pytz>=2018.3', + 'requests>=2.24.0,<3.0.0', + 'typing-extensions>=3.7.0', + ], + python_requires=python_requires, + # BEAM-8840: Do NOT use tests_require or setup_requires. + extras_require={ + 'docs': [ + 'Sphinx>=1.5.2,<2.0', + # Pinning docutils as a workaround for Sphinx issue: + # https://github.com/sphinx-doc/sphinx/issues/9727 + 'docutils==0.17.1' + ], + 'test': [ + 'freezegun>=0.3.12', + 'joblib>=1.0.1', + 'mock>=1.0.1,<3.0.0', + 'pandas<2.0.0', + 'parameterized>=0.7.1,<0.8.0', + 'pyhamcrest>=1.9,!=1.10.0,<2.0.0', + 'pyyaml>=3.12,<7.0.0', + 'requests_mock>=1.7,<2.0', + 'tenacity>=5.0.2,<6.0', + 'pytest>=4.4.0,<5.0', + 'pytest-xdist>=1.29.0,<2', + 'pytest-timeout>=1.3.3,<2', + 'scikit-learn>=0.20.0', + 'sqlalchemy>=1.3,<2.0', + 'psycopg2-binary>=2.8.5,<3.0.0', + 'testcontainers[mysql]>=3.0.3,<4.0.0', + 'cryptography>=36.0.0', + ], + 'gcp': [ + 'cachetools>=3.1.0,<5', + 'google-apitools>=0.5.31,<0.5.32', + # NOTE: Maintainers, please do not require google-auth>=2.x.x + # Until this issue is closed + # https://github.com/googleapis/google-cloud-python/issues/10566 + 'google-auth>=1.18.0,<3', + 'google-auth-httplib2>=0.1.0,<0.2.0', + 'google-cloud-datastore>=1.8.0,<2', + 'google-cloud-pubsub>=2.1.0,<3', + 'google-cloud-pubsublite>=1.2.0,<2', + # GCP packages required by tests + 'google-cloud-bigquery>=1.6.0,<3', + 'google-cloud-bigquery-storage>=2.6.3', + 'google-cloud-core>=0.28.1,<2', + 'google-cloud-bigtable>=0.31.1,<2', + 'google-cloud-spanner>=1.13.0,<2', + 'grpcio-gcp>=0.2.2,<1', + # GCP Packages required by ML functionality + 'google-cloud-dlp>=3.0.0,<4', + 'google-cloud-language>=1.3.0,<2', + 'google-cloud-videointelligence>=1.8.0,<2', + 'google-cloud-vision>=0.38.0,<2', + 'google-cloud-recommendations-ai>=0.1.0,<=0.2.0' + ], + 'interactive': [ + 'facets-overview>=1.0.0,<2', + 'google-cloud-dataproc>=3.0.0,<3.2.0', + # IPython>=8 is not compatible with Python<=3.7 + 'ipython>=7,<8;python_version<="3.7"', + 'ipython>=8,<9;python_version>"3.7"', + 'ipykernel>=6,<7', + 'ipywidgets>=7.6.5,<8', + # Skip version 6.1.13 due to + # https://github.com/jupyter/jupyter_client/issues/637 + 'jupyter-client>=6.1.11,<6.1.13', + 'timeloop>=1.0.2,<2', + ], + 'interactive_test': [ + # notebok utils + 'nbformat>=5.0.5,<6', + 'nbconvert>=6.2.0,<7', + # headless chrome based integration tests + 'needle>=0.5.0,<1', + 'chromedriver-binary>=100,<101', + # use a fixed major version of PIL for different python versions + 'pillow>=7.1.1,<8', + ], + 'aws': ['boto3 >=1.9'], + 'azure': [ + 'azure-storage-blob >=12.3.2', + 'azure-core >=1.7.0', + ], + 'dataframe': ['pandas>=1.0,<1.5'] + }, + zip_safe=False, + # PyPI package information. + classifiers=[ + 'Intended Audience :: End Users/Desktop', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + # When updating version classifiers, also update version warnings + # above and in apache_beam/__init__.py. + 'Topic :: Software Development :: Libraries', + 'Topic :: Software Development :: Libraries :: Python Modules', + ], + license='Apache License, Version 2.0', + keywords=PACKAGE_KEYWORDS, + cmdclass={ + 'mypy': mypy, + }, + ) From be474c399c2d4d75061653a0fbf43a5bea20b813 Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Thu, 5 May 2022 15:34:41 -0400 Subject: [PATCH 4/7] Explanatory comment --- sdks/python/setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index e550db2b3eb1..279052bec0da 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -164,6 +164,9 @@ def get_portability_package_data(): # executes below. generate_protos_first() # [BEAM-8181] pyarrow cannot be installed on 32-bit Windows platforms. + # We can't the dependencies out of the if/else, otherwise dependabot won't + # be able to parse it, so any dependencies added should be added in both the + # if and the else. if sys.platform == 'win32' and sys.maxsize <= 2**32: setuptools.setup( name=PACKAGE_NAME, From 525146a19852c1428aebdded665226d892d25446 Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Thu, 5 May 2022 16:19:58 -0400 Subject: [PATCH 5/7] Reduce duplication --- sdks/python/setup.py | 525 +++++++++++++++---------------------------- 1 file changed, 176 insertions(+), 349 deletions(-) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 279052bec0da..ea4b064b8b81 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -121,11 +121,10 @@ def get_version(): cythonize = lambda *args, **kwargs: [] # [BEAM-8181] pyarrow cannot be installed on 32-bit Windows platforms. +pyarrow_dependency = 'pyarrow>=0.15.1,<8.0.0' +pyarrow_dependency = '' if sys.platform == 'win32' and sys.maxsize <= 2**32: - REQUIRED_PACKAGES = [ - p for p in REQUIRED_PACKAGES if not p.startswith('pyarrow') - ] - + pyarrow_dependency = '' # We must generate protos after setup_requires are installed. def generate_protos_first(): @@ -163,348 +162,176 @@ def get_portability_package_data(): # structure must exist before the call to setuptools.find_packages() # executes below. generate_protos_first() - # [BEAM-8181] pyarrow cannot be installed on 32-bit Windows platforms. - # We can't the dependencies out of the if/else, otherwise dependabot won't - # be able to parse it, so any dependencies added should be added in both the - # if and the else. - if sys.platform == 'win32' and sys.maxsize <= 2**32: - setuptools.setup( - name=PACKAGE_NAME, - version=PACKAGE_VERSION, - description=PACKAGE_DESCRIPTION, - long_description=PACKAGE_LONG_DESCRIPTION, - url=PACKAGE_URL, - download_url=PACKAGE_DOWNLOAD_URL, - author=PACKAGE_AUTHOR, - author_email=PACKAGE_EMAIL, - packages=setuptools.find_packages(), - package_data={ - 'apache_beam': [ - '*/*.pyx', - '*/*/*.pyx', - '*/*.pxd', - '*/*/*.pxd', - '*/*.h', - '*/*/*.h', - 'testing/data/*.yaml', - *get_portability_package_data() - ] - }, - ext_modules=cythonize([ - # Make sure to use language_level=3 cython directive in files below. - 'apache_beam/**/*.pyx', - 'apache_beam/coders/coder_impl.py', - 'apache_beam/metrics/cells.py', - 'apache_beam/metrics/execution.py', - 'apache_beam/runners/common.py', - 'apache_beam/runners/worker/logger.py', - 'apache_beam/runners/worker/opcounters.py', - 'apache_beam/runners/worker/operations.py', - 'apache_beam/transforms/cy_combiners.py', - 'apache_beam/transforms/stats.py', - 'apache_beam/utils/counters.py', - 'apache_beam/utils/windowed_value.py', - ]), - install_requires=[ - # Avro 1.9.2 for python3 was broken. The issue was fixed in version 1.9.2.1 - 'crcmod>=1.7,<2.0', - # dataclasses backport for python_version<3.7. No version bound because this - # is Python standard since Python 3.7 and each Python version is compatible - # with a specific dataclasses version. - 'dataclasses;python_version<"3.7"', - 'orjson<4.0', - # Dill doesn't have forwards-compatibility guarantees within minor version. - # Pickles created with a new version of dill may not unpickle using older - # version of dill. It is best to use the same version of dill on client and - # server, therefore list of allowed versions is very narrow. - # See: https://github.com/uqfoundation/dill/issues/341. - 'dill>=0.3.1.1,<0.3.2', - 'cloudpickle>=2.0.0,<3', - 'fastavro>=0.23.6,<2', - 'grpcio>=1.29.0,<2', - 'hdfs>=2.1.0,<3.0.0', - 'httplib2>=0.8,<0.20.0', - 'numpy>=1.14.3,<1.23.0', - 'pymongo>=3.8.0,<4.0.0', - 'protobuf>=3.12.2,<4', - 'proto-plus>=1.7.1,<2', - 'pydot>=1.2.0,<2', - 'python-dateutil>=2.8.0,<3', - 'pytz>=2018.3', - 'requests>=2.24.0,<3.0.0', - 'typing-extensions>=3.7.0', - ], - python_requires=python_requires, - # BEAM-8840: Do NOT use tests_require or setup_requires. - extras_require={ - 'docs': [ - 'Sphinx>=1.5.2,<2.0', - # Pinning docutils as a workaround for Sphinx issue: - # https://github.com/sphinx-doc/sphinx/issues/9727 - 'docutils==0.17.1' - ], - 'test': [ - 'freezegun>=0.3.12', - 'joblib>=1.0.1', - 'mock>=1.0.1,<3.0.0', - 'pandas<2.0.0', - 'parameterized>=0.7.1,<0.8.0', - 'pyhamcrest>=1.9,!=1.10.0,<2.0.0', - 'pyyaml>=3.12,<7.0.0', - 'requests_mock>=1.7,<2.0', - 'tenacity>=5.0.2,<6.0', - 'pytest>=4.4.0,<5.0', - 'pytest-xdist>=1.29.0,<2', - 'pytest-timeout>=1.3.3,<2', - 'scikit-learn>=0.20.0', - 'sqlalchemy>=1.3,<2.0', - 'psycopg2-binary>=2.8.5,<3.0.0', - 'testcontainers[mysql]>=3.0.3,<4.0.0', - 'cryptography>=36.0.0', - ], - 'gcp': [ - 'cachetools>=3.1.0,<5', - 'google-apitools>=0.5.31,<0.5.32', - # NOTE: Maintainers, please do not require google-auth>=2.x.x - # Until this issue is closed - # https://github.com/googleapis/google-cloud-python/issues/10566 - 'google-auth>=1.18.0,<3', - 'google-auth-httplib2>=0.1.0,<0.2.0', - 'google-cloud-datastore>=1.8.0,<2', - 'google-cloud-pubsub>=2.1.0,<3', - 'google-cloud-pubsublite>=1.2.0,<2', - # GCP packages required by tests - 'google-cloud-bigquery>=1.6.0,<3', - 'google-cloud-bigquery-storage>=2.6.3', - 'google-cloud-core>=0.28.1,<2', - 'google-cloud-bigtable>=0.31.1,<2', - 'google-cloud-spanner>=1.13.0,<2', - 'grpcio-gcp>=0.2.2,<1', - # GCP Packages required by ML functionality - 'google-cloud-dlp>=3.0.0,<4', - 'google-cloud-language>=1.3.0,<2', - 'google-cloud-videointelligence>=1.8.0,<2', - 'google-cloud-vision>=0.38.0,<2', - 'google-cloud-recommendations-ai>=0.1.0,<=0.2.0' - ], - 'interactive': [ - 'facets-overview>=1.0.0,<2', - 'google-cloud-dataproc>=3.0.0,<3.2.0', - # IPython>=8 is not compatible with Python<=3.7 - 'ipython>=7,<8;python_version<="3.7"', - 'ipython>=8,<9;python_version>"3.7"', - 'ipykernel>=6,<7', - 'ipywidgets>=7.6.5,<8', - # Skip version 6.1.13 due to - # https://github.com/jupyter/jupyter_client/issues/637 - 'jupyter-client>=6.1.11,<6.1.13', - 'timeloop>=1.0.2,<2', - ], - 'interactive_test': [ - # notebok utils - 'nbformat>=5.0.5,<6', - 'nbconvert>=6.2.0,<7', - # headless chrome based integration tests - 'needle>=0.5.0,<1', - 'chromedriver-binary>=100,<101', - # use a fixed major version of PIL for different python versions - 'pillow>=7.1.1,<8', - ], - 'aws': ['boto3 >=1.9'], - 'azure': [ - 'azure-storage-blob >=12.3.2', - 'azure-core >=1.7.0', - ], - 'dataframe': ['pandas>=1.0,<1.5'] - }, - zip_safe=False, - # PyPI package information. - classifiers=[ - 'Intended Audience :: End Users/Desktop', - 'License :: OSI Approved :: Apache Software License', - 'Operating System :: POSIX :: Linux', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - # When updating version classifiers, also update version warnings - # above and in apache_beam/__init__.py. - 'Topic :: Software Development :: Libraries', - 'Topic :: Software Development :: Libraries :: Python Modules', - ], - license='Apache License, Version 2.0', - keywords=PACKAGE_KEYWORDS, - cmdclass={ - 'mypy': mypy, - }, - ) - else: - setuptools.setup( - name=PACKAGE_NAME, - version=PACKAGE_VERSION, - description=PACKAGE_DESCRIPTION, - long_description=PACKAGE_LONG_DESCRIPTION, - url=PACKAGE_URL, - download_url=PACKAGE_DOWNLOAD_URL, - author=PACKAGE_AUTHOR, - author_email=PACKAGE_EMAIL, - packages=setuptools.find_packages(), - package_data={ - 'apache_beam': [ - '*/*.pyx', - '*/*/*.pyx', - '*/*.pxd', - '*/*/*.pxd', - '*/*.h', - '*/*/*.h', - 'testing/data/*.yaml', - *get_portability_package_data() - ] - }, - ext_modules=cythonize([ - # Make sure to use language_level=3 cython directive in files below. - 'apache_beam/**/*.pyx', - 'apache_beam/coders/coder_impl.py', - 'apache_beam/metrics/cells.py', - 'apache_beam/metrics/execution.py', - 'apache_beam/runners/common.py', - 'apache_beam/runners/worker/logger.py', - 'apache_beam/runners/worker/opcounters.py', - 'apache_beam/runners/worker/operations.py', - 'apache_beam/transforms/cy_combiners.py', - 'apache_beam/transforms/stats.py', - 'apache_beam/utils/counters.py', - 'apache_beam/utils/windowed_value.py', - ]), - install_requires=[ - # Avro 1.9.2 for python3 was broken. The issue was fixed in version 1.9.2.1 - 'crcmod>=1.7,<2.0', - # dataclasses backport for python_version<3.7. No version bound because this - # is Python standard since Python 3.7 and each Python version is compatible - # with a specific dataclasses version. - 'dataclasses;python_version<"3.7"', - 'orjson<4.0', - # Dill doesn't have forwards-compatibility guarantees within minor version. - # Pickles created with a new version of dill may not unpickle using older - # version of dill. It is best to use the same version of dill on client and - # server, therefore list of allowed versions is very narrow. - # See: https://github.com/uqfoundation/dill/issues/341. - 'dill>=0.3.1.1,<0.3.2', - 'cloudpickle>=2.0.0,<3', - 'fastavro>=0.23.6,<2', - 'grpcio>=1.29.0,<2', - 'hdfs>=2.1.0,<3.0.0', - 'httplib2>=0.8,<0.20.0', - 'numpy>=1.14.3,<1.23.0', - 'pymongo>=3.8.0,<4.0.0', - 'protobuf>=3.12.2,<4', - 'proto-plus>=1.7.1,<2', - 'pyarrow>=0.15.1,<8.0.0', - 'pydot>=1.2.0,<2', - 'python-dateutil>=2.8.0,<3', - 'pytz>=2018.3', - 'requests>=2.24.0,<3.0.0', - 'typing-extensions>=3.7.0', - ], - python_requires=python_requires, - # BEAM-8840: Do NOT use tests_require or setup_requires. - extras_require={ - 'docs': [ - 'Sphinx>=1.5.2,<2.0', - # Pinning docutils as a workaround for Sphinx issue: - # https://github.com/sphinx-doc/sphinx/issues/9727 - 'docutils==0.17.1' - ], - 'test': [ - 'freezegun>=0.3.12', - 'joblib>=1.0.1', - 'mock>=1.0.1,<3.0.0', - 'pandas<2.0.0', - 'parameterized>=0.7.1,<0.8.0', - 'pyhamcrest>=1.9,!=1.10.0,<2.0.0', - 'pyyaml>=3.12,<7.0.0', - 'requests_mock>=1.7,<2.0', - 'tenacity>=5.0.2,<6.0', - 'pytest>=4.4.0,<5.0', - 'pytest-xdist>=1.29.0,<2', - 'pytest-timeout>=1.3.3,<2', - 'scikit-learn>=0.20.0', - 'sqlalchemy>=1.3,<2.0', - 'psycopg2-binary>=2.8.5,<3.0.0', - 'testcontainers[mysql]>=3.0.3,<4.0.0', - 'cryptography>=36.0.0', - ], - 'gcp': [ - 'cachetools>=3.1.0,<5', - 'google-apitools>=0.5.31,<0.5.32', - # NOTE: Maintainers, please do not require google-auth>=2.x.x - # Until this issue is closed - # https://github.com/googleapis/google-cloud-python/issues/10566 - 'google-auth>=1.18.0,<3', - 'google-auth-httplib2>=0.1.0,<0.2.0', - 'google-cloud-datastore>=1.8.0,<2', - 'google-cloud-pubsub>=2.1.0,<3', - 'google-cloud-pubsublite>=1.2.0,<2', - # GCP packages required by tests - 'google-cloud-bigquery>=1.6.0,<3', - 'google-cloud-bigquery-storage>=2.6.3', - 'google-cloud-core>=0.28.1,<2', - 'google-cloud-bigtable>=0.31.1,<2', - 'google-cloud-spanner>=1.13.0,<2', - 'grpcio-gcp>=0.2.2,<1', - # GCP Packages required by ML functionality - 'google-cloud-dlp>=3.0.0,<4', - 'google-cloud-language>=1.3.0,<2', - 'google-cloud-videointelligence>=1.8.0,<2', - 'google-cloud-vision>=0.38.0,<2', - 'google-cloud-recommendations-ai>=0.1.0,<=0.2.0' - ], - 'interactive': [ - 'facets-overview>=1.0.0,<2', - 'google-cloud-dataproc>=3.0.0,<3.2.0', - # IPython>=8 is not compatible with Python<=3.7 - 'ipython>=7,<8;python_version<="3.7"', - 'ipython>=8,<9;python_version>"3.7"', - 'ipykernel>=6,<7', - 'ipywidgets>=7.6.5,<8', - # Skip version 6.1.13 due to - # https://github.com/jupyter/jupyter_client/issues/637 - 'jupyter-client>=6.1.11,<6.1.13', - 'timeloop>=1.0.2,<2', - ], - 'interactive_test': [ - # notebok utils - 'nbformat>=5.0.5,<6', - 'nbconvert>=6.2.0,<7', - # headless chrome based integration tests - 'needle>=0.5.0,<1', - 'chromedriver-binary>=100,<101', - # use a fixed major version of PIL for different python versions - 'pillow>=7.1.1,<8', - ], - 'aws': ['boto3 >=1.9'], - 'azure': [ - 'azure-storage-blob >=12.3.2', - 'azure-core >=1.7.0', - ], - 'dataframe': ['pandas>=1.0,<1.5'] - }, - zip_safe=False, - # PyPI package information. - classifiers=[ - 'Intended Audience :: End Users/Desktop', - 'License :: OSI Approved :: Apache Software License', - 'Operating System :: POSIX :: Linux', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - # When updating version classifiers, also update version warnings - # above and in apache_beam/__init__.py. - 'Topic :: Software Development :: Libraries', - 'Topic :: Software Development :: Libraries :: Python Modules', - ], - license='Apache License, Version 2.0', - keywords=PACKAGE_KEYWORDS, - cmdclass={ - 'mypy': mypy, - }, - ) + # We can't move the dependencies out of the setup call, otherwise + # dependabot won't be able to parse it. + setuptools.setup( + name=PACKAGE_NAME, + version=PACKAGE_VERSION, + description=PACKAGE_DESCRIPTION, + long_description=PACKAGE_LONG_DESCRIPTION, + url=PACKAGE_URL, + download_url=PACKAGE_DOWNLOAD_URL, + author=PACKAGE_AUTHOR, + author_email=PACKAGE_EMAIL, + packages=setuptools.find_packages(), + package_data={ + 'apache_beam': [ + '*/*.pyx', + '*/*/*.pyx', + '*/*.pxd', + '*/*/*.pxd', + '*/*.h', + '*/*/*.h', + 'testing/data/*.yaml', + *get_portability_package_data() + ] + }, + ext_modules=cythonize([ + # Make sure to use language_level=3 cython directive in files below. + 'apache_beam/**/*.pyx', + 'apache_beam/coders/coder_impl.py', + 'apache_beam/metrics/cells.py', + 'apache_beam/metrics/execution.py', + 'apache_beam/runners/common.py', + 'apache_beam/runners/worker/logger.py', + 'apache_beam/runners/worker/opcounters.py', + 'apache_beam/runners/worker/operations.py', + 'apache_beam/transforms/cy_combiners.py', + 'apache_beam/transforms/stats.py', + 'apache_beam/utils/counters.py', + 'apache_beam/utils/windowed_value.py', + ]), + install_requires=[ + # Avro 1.9.2 for python3 was broken. + # The issue was fixed in version 1.9.2.1 + 'crcmod>=1.7,<2.0', + # dataclasses backport for python_version<3.7. No version bound because + # this is Python standard since Python 3.7 and each Python version is + # compatible with a specific dataclasses version. + 'dataclasses;python_version<"3.7"', + 'orjson<4.0', + # Dill doesn't have forwards-compatibility guarantees within minor + # version. Pickles created with a new version of dill may not unpickle + # using older version of dill. It is best to use the same version of + # dill on client and server, therefore list of allowed versions is very + # narrow. See: https://github.com/uqfoundation/dill/issues/341. + 'dill>=0.3.1.1,<0.3.2', + 'cloudpickle>=2.0.0,<3', + 'fastavro>=0.23.6,<2', + 'grpcio>=1.29.0,<2', + 'hdfs>=2.1.0,<3.0.0', + 'httplib2>=0.8,<0.20.0', + 'numpy>=1.14.3,<1.23.0', + 'pymongo>=3.8.0,<4.0.0', + 'protobuf>=3.12.2,<4', + 'proto-plus>=1.7.1,<2', + pyarrow_dependency, + 'pydot>=1.2.0,<2', + 'python-dateutil>=2.8.0,<3', + 'pytz>=2018.3', + 'requests>=2.24.0,<3.0.0', + 'typing-extensions>=3.7.0', + ], + python_requires=python_requires, + # BEAM-8840: Do NOT use tests_require or setup_requires. + extras_require={ + 'docs': [ + 'Sphinx>=1.5.2,<2.0', + # Pinning docutils as a workaround for Sphinx issue: + # https://github.com/sphinx-doc/sphinx/issues/9727 + 'docutils==0.17.1' + ], + 'test': [ + 'freezegun>=0.3.12', + 'joblib>=1.0.1', + 'mock>=1.0.1,<3.0.0', + 'pandas<2.0.0', + 'parameterized>=0.7.1,<0.8.0', + 'pyhamcrest>=1.9,!=1.10.0,<2.0.0', + 'pyyaml>=3.12,<7.0.0', + 'requests_mock>=1.7,<2.0', + 'tenacity>=5.0.2,<6.0', + 'pytest>=4.4.0,<5.0', + 'pytest-xdist>=1.29.0,<2', + 'pytest-timeout>=1.3.3,<2', + 'scikit-learn>=0.20.0', + 'sqlalchemy>=1.3,<2.0', + 'psycopg2-binary>=2.8.5,<3.0.0', + 'testcontainers[mysql]>=3.0.3,<4.0.0', + 'cryptography>=36.0.0', + ], + 'gcp': [ + 'cachetools>=3.1.0,<5', + 'google-apitools>=0.5.31,<0.5.32', + # NOTE: Maintainers, please do not require google-auth>=2.x.x + # Until this issue is closed + # https://github.com/googleapis/google-cloud-python/issues/10566 + 'google-auth>=1.18.0,<3', + 'google-auth-httplib2>=0.1.0,<0.2.0', + 'google-cloud-datastore>=1.8.0,<2', + 'google-cloud-pubsub>=2.1.0,<3', + 'google-cloud-pubsublite>=1.2.0,<2', + # GCP packages required by tests + 'google-cloud-bigquery>=1.6.0,<3', + 'google-cloud-bigquery-storage>=2.6.3', + 'google-cloud-core>=0.28.1,<2', + 'google-cloud-bigtable>=0.31.1,<2', + 'google-cloud-spanner>=1.13.0,<2', + 'grpcio-gcp>=0.2.2,<1', + # GCP Packages required by ML functionality + 'google-cloud-dlp>=3.0.0,<4', + 'google-cloud-language>=1.3.0,<2', + 'google-cloud-videointelligence>=1.8.0,<2', + 'google-cloud-vision>=0.38.0,<2', + 'google-cloud-recommendations-ai>=0.1.0,<=0.2.0' + ], + 'interactive': [ + 'facets-overview>=1.0.0,<2', + 'google-cloud-dataproc>=3.0.0,<3.2.0', + # IPython>=8 is not compatible with Python<=3.7 + 'ipython>=7,<8;python_version<="3.7"', + 'ipython>=8,<9;python_version>"3.7"', + 'ipykernel>=6,<7', + 'ipywidgets>=7.6.5,<8', + # Skip version 6.1.13 due to + # https://github.com/jupyter/jupyter_client/issues/637 + 'jupyter-client>=6.1.11,<6.1.13', + 'timeloop>=1.0.2,<2', + ], + 'interactive_test': [ + # notebok utils + 'nbformat>=5.0.5,<6', + 'nbconvert>=6.2.0,<7', + # headless chrome based integration tests + 'needle>=0.5.0,<1', + 'chromedriver-binary>=100,<101', + # use a fixed major version of PIL for different python versions + 'pillow>=7.1.1,<8', + ], + 'aws': ['boto3 >=1.9'], + 'azure': [ + 'azure-storage-blob >=12.3.2', + 'azure-core >=1.7.0', + ], + 'dataframe': ['pandas>=1.0,<1.5'] + }, + zip_safe=False, + # PyPI package information. + classifiers=[ + 'Intended Audience :: End Users/Desktop', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + # When updating version classifiers, also update version warnings + # above and in apache_beam/__init__.py. + 'Topic :: Software Development :: Libraries', + 'Topic :: Software Development :: Libraries :: Python Modules', + ], + license='Apache License, Version 2.0', + keywords=PACKAGE_KEYWORDS, + cmdclass={ + 'mypy': mypy, + }, + ) From 68658c6b7bcda4fd03b3a737d71b602bdc67bbdf Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Thu, 5 May 2022 16:38:12 -0400 Subject: [PATCH 6/7] Bug + style from feedback --- sdks/python/setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index ea4b064b8b81..4694d83c908a 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -121,10 +121,10 @@ def get_version(): cythonize = lambda *args, **kwargs: [] # [BEAM-8181] pyarrow cannot be installed on 32-bit Windows platforms. -pyarrow_dependency = 'pyarrow>=0.15.1,<8.0.0' -pyarrow_dependency = '' if sys.platform == 'win32' and sys.maxsize <= 2**32: pyarrow_dependency = '' +else: + pyarrow_dependency = 'pyarrow>=0.15.1,<8.0.0' # We must generate protos after setup_requires are installed. def generate_protos_first(): @@ -162,8 +162,8 @@ def get_portability_package_data(): # structure must exist before the call to setuptools.find_packages() # executes below. generate_protos_first() - # We can't move the dependencies out of the setup call, otherwise - # dependabot won't be able to parse it. + # Keep all dependencies inlined in the setup call, otherwise dependabot won't + # be able to parse it. setuptools.setup( name=PACKAGE_NAME, version=PACKAGE_VERSION, From f848ce3cebe07fb7c30bd553c20bfaf27da35113 Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Fri, 6 May 2022 15:56:50 -0400 Subject: [PATCH 7/7] Change format to make dependabot happy --- sdks/python/setup.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 4694d83c908a..6732e06b4421 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -162,7 +162,7 @@ def get_portability_package_data(): # structure must exist before the call to setuptools.find_packages() # executes below. generate_protos_first() - # Keep all dependencies inlined in the setup call, otherwise dependabot won't + # Keep all dependencies inlined in the setup call, otherwise Dependabot won't # be able to parse it. setuptools.setup( name=PACKAGE_NAME, @@ -225,13 +225,15 @@ def get_portability_package_data(): 'pymongo>=3.8.0,<4.0.0', 'protobuf>=3.12.2,<4', 'proto-plus>=1.7.1,<2', - pyarrow_dependency, 'pydot>=1.2.0,<2', 'python-dateutil>=2.8.0,<3', 'pytz>=2018.3', 'requests>=2.24.0,<3.0.0', 'typing-extensions>=3.7.0', - ], + # Dynamic dependencies must be specified in a separate list, otherwise + # Dependabot won't be able to parse the main list. Any dynamic + # dependencies will not receive updates from Dependabot. + ] + [pyarrow_dependency], python_requires=python_requires, # BEAM-8840: Do NOT use tests_require or setup_requires. extras_require={