Skip to content

[BEAM-12526] Add Dependabot #17563

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
TheNeuralBit merged 8 commits into from
May 11, 2022
Merged

[BEAM-12526] Add Dependabot #17563

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
e7f4a47
Create dependabot.yml
damccorm May 5, 2022
694a866
Apache header
damccorm May 5, 2022
817d237
See if dependabot will parse now
damccorm May 5, 2022
be474c3
Explanatory comment
damccorm May 5, 2022
525146a
Reduce duplication
damccorm May 5, 2022
68658c6
Bug + style from feedback
damccorm May 5, 2022
f848ce3
Change format to make dependabot happy
damccorm May 6, 2022
01824d0
Merge in master
damccorm May 11, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 29 additions & 0 deletions .github/dependabot.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

version: 2
updates:
- package-ecosystem: "gomod"
directory: "/sdks" # Location of package manifests
schedule:
interval: "daily"
- package-ecosystem: "pip"
directory: "/sdks/python" # Location of package manifests
schedule:
interval: "daily"
- package-ecosystem: "gradle"
directory: "/" # Location of package manifests
schedule:
interval: "daily"
230 changes: 110 additions & 120 deletions sdks/python/setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,121 +120,11 @@ def get_version():
except ImportError:
cythonize = lambda *args, **kwargs: []

REQUIRED_PACKAGES = [
# Avro 1.9.2 for python3 was broken. The issue was fixed in version 1.9.2.1
'crcmod>=1.7,<2.0',
# dataclasses backport for python_version<3.7. No version bound because this
# is Python standard since Python 3.7 and each Python version is compatible
# with a specific dataclasses version.
'dataclasses;python_version<"3.7"',
'orjson<4.0',
# Dill doesn't have forwards-compatibility guarantees within minor version.
# Pickles created with a new version of dill may not unpickle using older
# version of dill. It is best to use the same version of dill on client and
# server, therefore list of allowed versions is very narrow.
# See: https://github.com/uqfoundation/dill/issues/341.
'dill>=0.3.1.1,<0.3.2',
'cloudpickle>=2.0.0,<3',
'fastavro>=0.23.6,<2',
'grpcio>=1.29.0,<2',
'hdfs>=2.1.0,<3.0.0',
'httplib2>=0.8,<0.21.0',
'numpy>=1.14.3,<1.23.0',
'pymongo>=3.8.0,<4.0.0',
'protobuf>=3.12.2,<4',
'proto-plus>=1.7.1,<2',
'pyarrow>=0.15.1,<8.0.0',
'pydot>=1.2.0,<2',
'python-dateutil>=2.8.0,<3',
'pytz>=2018.3',
'requests>=2.24.0,<3.0.0',
'typing-extensions>=3.7.0',
]

# [BEAM-8181] pyarrow cannot be installed on 32-bit Windows platforms.
if sys.platform == 'win32' and sys.maxsize <= 2**32:
REQUIRED_PACKAGES = [
p for p in REQUIRED_PACKAGES if not p.startswith('pyarrow')
]

REQUIRED_TEST_PACKAGES = [
'freezegun>=0.3.12',
'joblib>=1.0.1',
'mock>=1.0.1,<3.0.0',
'pandas<2.0.0',
'parameterized>=0.7.1,<0.8.0',
'pyhamcrest>=1.9,!=1.10.0,<2.0.0',
'pyyaml>=3.12,<7.0.0',
'requests_mock>=1.7,<2.0',
'tenacity>=5.0.2,<6.0',
'pytest>=4.4.0,<5.0',
'pytest-xdist>=1.29.0,<2',
'pytest-timeout>=1.3.3,<2',
'scikit-learn>=0.20.0',
'sqlalchemy>=1.3,<2.0',
'psycopg2-binary>=2.8.5,<3.0.0',
'testcontainers[mysql]>=3.0.3,<4.0.0',
'cryptography>=36.0.0',
]

GCP_REQUIREMENTS = [
'cachetools>=3.1.0,<5',
'google-apitools>=0.5.31,<0.5.32',
# NOTE: Maintainers, please do not require google-auth>=2.x.x
# Until this issue is closed
# https://github.com/googleapis/google-cloud-python/issues/10566
'google-auth>=1.18.0,<3',
'google-auth-httplib2>=0.1.0,<0.2.0',
'google-cloud-datastore>=1.8.0,<2',
'google-cloud-pubsub>=2.1.0,<3',
'google-cloud-pubsublite>=1.2.0,<2',
# GCP packages required by tests
'google-cloud-bigquery>=1.6.0,<3',
'google-cloud-bigquery-storage>=2.6.3',
'google-cloud-core>=0.28.1,<2',
'google-cloud-bigtable>=0.31.1,<2',
'google-cloud-spanner>=1.13.0,<2',
'grpcio-gcp>=0.2.2,<1',
# GCP Packages required by ML functionality
'google-cloud-dlp>=3.0.0,<4',
'google-cloud-language>=1.3.0,<2',
'google-cloud-videointelligence>=1.8.0,<2',
'google-cloud-vision>=0.38.0,<2',
'google-cloud-recommendations-ai>=0.1.0,<=0.2.0'
]

INTERACTIVE_BEAM = [
'facets-overview>=1.0.0,<2',
'google-cloud-dataproc>=3.0.0,<3.2.0',
# IPython>=8 is not compatible with Python<=3.7
'ipython>=7,<8;python_version<="3.7"',
'ipython>=8,<9;python_version>"3.7"',
'ipykernel>=6,<7',
'ipywidgets>=7.6.5,<8',
# Skip version 6.1.13 due to
# https://github.com/jupyter/jupyter_client/issues/637
'jupyter-client>=6.1.11,<6.1.13',
'timeloop>=1.0.2,<2',
]

INTERACTIVE_BEAM_TEST = [
# notebok utils
'nbformat>=5.0.5,<6',
'nbconvert>=6.2.0,<7',
# headless chrome based integration tests
'needle>=0.5.0,<1',
'chromedriver-binary>=100,<101',
# use a fixed major version of PIL for different python versions
'pillow>=7.1.1,<8',
]

AWS_REQUIREMENTS = ['boto3 >=1.9']

AZURE_REQUIREMENTS = [
'azure-storage-blob >=12.3.2',
'azure-core >=1.7.0',
]

pyarrow_dependency = ''
else:
pyarrow_dependency = 'pyarrow>=0.15.1,<8.0.0'

# We must generate protos after setup_requires are installed.
def generate_protos_first():
Expand Down Expand Up @@ -272,6 +162,8 @@ def get_portability_package_data():
# structure must exist before the call to setuptools.find_packages()
# executes below.
generate_protos_first()
# Keep all dependencies inlined in the setup call, otherwise Dependabot won't
# be able to parse it.
setuptools.setup(
name=PACKAGE_NAME,
version=PACKAGE_VERSION,
Expand Down Expand Up @@ -309,7 +201,39 @@ def get_portability_package_data():
'apache_beam/utils/counters.py',
'apache_beam/utils/windowed_value.py',
]),
install_requires=REQUIRED_PACKAGES,
install_requires=[
# Avro 1.9.2 for python3 was broken.
# The issue was fixed in version 1.9.2.1
'crcmod>=1.7,<2.0',
# dataclasses backport for python_version<3.7. No version bound because
# this is Python standard since Python 3.7 and each Python version is
# compatible with a specific dataclasses version.
'dataclasses;python_version<"3.7"',
'orjson<4.0',
# Dill doesn't have forwards-compatibility guarantees within minor
# version. Pickles created with a new version of dill may not unpickle
# using older version of dill. It is best to use the same version of
# dill on client and server, therefore list of allowed versions is very
# narrow. See: https://github.com/uqfoundation/dill/issues/341.
'dill>=0.3.1.1,<0.3.2',
'cloudpickle>=2.0.0,<3',
'fastavro>=0.23.6,<2',
'grpcio>=1.29.0,<2',
'hdfs>=2.1.0,<3.0.0',
'httplib2>=0.8,<0.21.0',
'numpy>=1.14.3,<1.23.0',
'pymongo>=3.8.0,<4.0.0',
'protobuf>=3.12.2,<4',
'proto-plus>=1.7.1,<2',
'pydot>=1.2.0,<2',
'python-dateutil>=2.8.0,<3',
'pytz>=2018.3',
'requests>=2.24.0,<3.0.0',
'typing-extensions>=3.7.0',
# Dynamic dependencies must be specified in a separate list, otherwise
# Dependabot won't be able to parse the main list. Any dynamic
# dependencies will not receive updates from Dependabot.
] + [pyarrow_dependency],
python_requires=python_requires,
# BEAM-8840: Do NOT use tests_require or setup_requires.
extras_require={
Expand All @@ -319,12 +243,78 @@ def get_portability_package_data():
# https://github.com/sphinx-doc/sphinx/issues/9727
'docutils==0.17.1'
],
'test': REQUIRED_TEST_PACKAGES,
'gcp': GCP_REQUIREMENTS,
'interactive': INTERACTIVE_BEAM,
'interactive_test': INTERACTIVE_BEAM_TEST,
'aws': AWS_REQUIREMENTS,
'azure': AZURE_REQUIREMENTS,
'test': [
'freezegun>=0.3.12',
'joblib>=1.0.1',
'mock>=1.0.1,<3.0.0',
'pandas<2.0.0',
'parameterized>=0.7.1,<0.8.0',
'pyhamcrest>=1.9,!=1.10.0,<2.0.0',
'pyyaml>=3.12,<7.0.0',
'requests_mock>=1.7,<2.0',
'tenacity>=5.0.2,<6.0',
'pytest>=4.4.0,<5.0',
'pytest-xdist>=1.29.0,<2',
'pytest-timeout>=1.3.3,<2',
'scikit-learn>=0.20.0',
'sqlalchemy>=1.3,<2.0',
'psycopg2-binary>=2.8.5,<3.0.0',
'testcontainers[mysql]>=3.0.3,<4.0.0',
'cryptography>=36.0.0',
],
'gcp': [
'cachetools>=3.1.0,<5',
'google-apitools>=0.5.31,<0.5.32',
# NOTE: Maintainers, please do not require google-auth>=2.x.x
# Until this issue is closed
# https://github.com/googleapis/google-cloud-python/issues/10566
'google-auth>=1.18.0,<3',
'google-auth-httplib2>=0.1.0,<0.2.0',
'google-cloud-datastore>=1.8.0,<2',
'google-cloud-pubsub>=2.1.0,<3',
'google-cloud-pubsublite>=1.2.0,<2',
# GCP packages required by tests
'google-cloud-bigquery>=1.6.0,<3',
'google-cloud-bigquery-storage>=2.6.3',
'google-cloud-core>=0.28.1,<2',
'google-cloud-bigtable>=0.31.1,<2',
'google-cloud-spanner>=1.13.0,<2',
'grpcio-gcp>=0.2.2,<1',
# GCP Packages required by ML functionality
'google-cloud-dlp>=3.0.0,<4',
'google-cloud-language>=1.3.0,<2',
'google-cloud-videointelligence>=1.8.0,<2',
'google-cloud-vision>=0.38.0,<2',
'google-cloud-recommendations-ai>=0.1.0,<=0.2.0'
],
'interactive': [
'facets-overview>=1.0.0,<2',
'google-cloud-dataproc>=3.0.0,<3.2.0',
# IPython>=8 is not compatible with Python<=3.7
'ipython>=7,<8;python_version<="3.7"',
'ipython>=8,<9;python_version>"3.7"',
'ipykernel>=6,<7',
'ipywidgets>=7.6.5,<8',
# Skip version 6.1.13 due to
# https://github.com/jupyter/jupyter_client/issues/637
'jupyter-client>=6.1.11,<6.1.13',
'timeloop>=1.0.2,<2',
],
'interactive_test': [
# notebok utils
'nbformat>=5.0.5,<6',
'nbconvert>=6.2.0,<7',
# headless chrome based integration tests
'needle>=0.5.0,<1',
'chromedriver-binary>=100,<101',
# use a fixed major version of PIL for different python versions
'pillow>=7.1.1,<8',
],
'aws': ['boto3 >=1.9'],
'azure': [
'azure-storage-blob >=12.3.2',
'azure-core >=1.7.0',
],
'dataframe': ['pandas>=1.0,<1.5']
},
zip_safe=False,
Expand Down