From e5fd2eb4a1af1be9b0301f9cc2770b4235b1dfcd Mon Sep 17 00:00:00 2001 From: Danny Mccormick Date: Mon, 30 Jun 2025 11:10:08 -0400 Subject: [PATCH 1/5] Add logic for generating ml requirements --- sdks/python/container/common.gradle | 15 ++++++++++++++- .../container/run_generate_requirements.sh | 16 +++++++++++++--- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/sdks/python/container/common.gradle b/sdks/python/container/common.gradle index 0175778a6301..6c3de38d56c1 100644 --- a/sdks/python/container/common.gradle +++ b/sdks/python/container/common.gradle @@ -39,7 +39,20 @@ def generatePythonRequirements = tasks.register("generatePythonRequirements") { executable 'sh' args '-c', "cd ${rootDir} && ${runScriptsPath} " + "${project.ext.pythonVersion} " + - "${files(configurations.sdkSourceTarball.files).singleFile} " + "${pipExtraOptions}" + "${files(configurations.sdkSourceTarball.files).singleFile} " + + "base_image_requirements.txt " + + "[gcp,dataframe,test] " + + "${pipExtraOptions}" + } + // Generate versions for ML dependencies + exec { + executable 'sh' + args '-c', "cd ${rootDir} && ${runScriptsPath} " + + "${project.ext.pythonVersion} " + + "${files(configurations.sdkSourceTarball.files).singleFile} " + + "ml_image_requirements.txt " + + "[gcp,dataframe,test,tensorflow,torch,transformers] " + + "${pipExtraOptions}" } } } diff --git a/sdks/python/container/run_generate_requirements.sh b/sdks/python/container/run_generate_requirements.sh index 6c160bc6ac9e..b54b0a2020c4 100755 --- a/sdks/python/container/run_generate_requirements.sh +++ b/sdks/python/container/run_generate_requirements.sh @@ -38,10 +38,12 @@ fi PY_VERSION=$1 SDK_TARBALL=$2 +REQUIREMENTS_FILE_NAME=$3 +EXTRAS=$4 # Use the PIP_EXTRA_OPTIONS environment variable to pass additional flags to the pip install command. # For example, you can include the --pre flag in $PIP_EXTRA_OPTIONS to download pre-release versions of packages. # Note that you can modify the behavior of the pip install command in this script by passing in your own $PIP_EXTRA_OPTIONS. -PIP_EXTRA_OPTIONS=$3 +PIP_EXTRA_OPTIONS=$5 if ! python"$PY_VERSION" --version > /dev/null 2>&1 ; then echo "Please install a python${PY_VERSION} interpreter. See s.apache.org/beam-python-dev-wiki for Python installation tips." @@ -53,6 +55,14 @@ if ! python"$PY_VERSION" -m venv --help > /dev/null 2>&1 ; then exit 1 fi +if [ -z "$REQUIREMENTS_FILE_NAME" ]; then + REQUIREMENTS_FILE_NAME="base_image_requirements.txt" +fi + +if [ -z "$EXTRAS" ]; then + EXTRAS="[gcp,dataframe,test]" +fi + set -ex ENV_PATH="$PWD/build/python${PY_VERSION/./}_requirements_gen" @@ -65,7 +75,7 @@ pip install --upgrade pip setuptools wheel # Install dataframe deps to add have Dataframe support in released images. # Install test deps since some integration tests need dependencies, # such as pytest, installed in the runner environment. -pip install ${PIP_EXTRA_OPTIONS:+"$PIP_EXTRA_OPTIONS"} --no-cache-dir "$SDK_TARBALL"[gcp,dataframe,test] +pip install ${PIP_EXTRA_OPTIONS:+"$PIP_EXTRA_OPTIONS"} --no-cache-dir "$SDK_TARBALL""$EXTRAS" pip install ${PIP_EXTRA_OPTIONS:+"$PIP_EXTRA_OPTIONS"} --no-cache-dir -r "$PWD"/sdks/python/container/base_image_requirements_manual.txt pip uninstall -y apache-beam @@ -75,7 +85,7 @@ echo "Installed dependencies:" pip freeze PY_IMAGE="py${PY_VERSION//.}" -REQUIREMENTS_FILE=$PWD/sdks/python/container/$PY_IMAGE/base_image_requirements.txt +REQUIREMENTS_FILE=$PWD/sdks/python/container/$PY_IMAGE/$REQUIREMENTS_FILE_NAME cat < "$REQUIREMENTS_FILE" # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with From 16c537993688c9f59d8a5c6b28cae0bd590d7473 Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Mon, 30 Jun 2025 18:36:20 +0000 Subject: [PATCH 2/5] Add some requirements and loosen torch bounds --- .../py310/base_image_requirements.txt | 22 +- .../container/py310/ml_image_requirements.txt | 241 ++++++++++++++++++ .../py311/base_image_requirements.txt | 24 +- .../container/py311/ml_image_requirements.txt | 238 +++++++++++++++++ .../py312/base_image_requirements.txt | 24 +- .../py39/base_image_requirements.txt | 22 +- .../container/py39/ml_image_requirements.txt | 241 ++++++++++++++++++ sdks/python/setup.py | 2 +- .../python/test-suites/tox/py312/build.gradle | 5 + sdks/python/tox.ini | 16 ++ 10 files changed, 788 insertions(+), 47 deletions(-) create mode 100644 sdks/python/container/py310/ml_image_requirements.txt create mode 100644 sdks/python/container/py311/ml_image_requirements.txt create mode 100644 sdks/python/container/py39/ml_image_requirements.txt diff --git a/sdks/python/container/py310/base_image_requirements.txt b/sdks/python/container/py310/base_image_requirements.txt index 5f69f6b11928..1ef758fc1252 100644 --- a/sdks/python/container/py310/base_image_requirements.txt +++ b/sdks/python/container/py310/base_image_requirements.txt @@ -57,17 +57,17 @@ freezegun==1.5.2 frozenlist==1.7.0 future==1.0.0 google-api-core==2.25.1 -google-api-python-client==2.172.0 +google-api-python-client==2.174.0 google-apitools==0.5.31 google-auth==2.40.3 google-auth-httplib2==0.2.0 -google-cloud-aiplatform==1.97.0 +google-cloud-aiplatform==1.100.0 google-cloud-bigquery==3.34.0 google-cloud-bigquery-storage==2.32.0 google-cloud-bigtable==2.31.0 google-cloud-core==2.4.3 google-cloud-datastore==2.21.0 -google-cloud-dlp==3.30.0 +google-cloud-dlp==3.31.0 google-cloud-language==2.17.2 google-cloud-profiler==4.1.0 google-cloud-pubsub==2.30.0 @@ -79,7 +79,7 @@ google-cloud-storage==2.19.0 google-cloud-videointelligence==2.16.2 google-cloud-vision==3.10.2 google-crc32c==1.7.1 -google-genai==1.20.0 +google-genai==1.23.0 google-resumable-media==2.7.2 googleapis-common-protos==1.70.0 greenlet==3.2.3 @@ -93,27 +93,27 @@ hdfs==2.7.3 httpcore==1.0.9 httplib2==0.22.0 httpx==0.28.1 -hypothesis==6.135.10 +hypothesis==6.135.18 idna==3.10 importlib_metadata==8.7.0 iniconfig==2.1.0 jaraco.classes==3.4.0 jaraco.context==6.0.1 -jaraco.functools==4.1.0 +jaraco.functools==4.2.1 jeepney==0.9.0 Jinja2==3.1.6 joblib==1.5.1 jsonpickle==3.4.2 jsonschema==4.24.0 jsonschema-specifications==2025.4.1 -kafka-python==2.2.11 +kafka-python==2.2.14 keyring==25.6.0 keyrings.google-artifactregistry-auth==1.1.2 MarkupSafe==3.0.2 mmh3==5.1.0 mock==5.2.0 more-itertools==10.7.0 -multidict==6.4.4 +multidict==6.6.3 mysql-connector-python==9.3.0 nltk==3.9.1 numpy==2.2.6 @@ -122,7 +122,7 @@ objsize==0.7.1 opentelemetry-api==1.34.1 opentelemetry-sdk==1.34.1 opentelemetry-semantic-conventions==0.55b1 -oracledb==3.1.1 +oracledb==3.2.0 orjson==3.10.18 overrides==7.7.0 packaging==25.0 @@ -144,7 +144,7 @@ pydantic_core==2.33.2 pydot==1.4.2 PyHamcrest==2.1.0 PyJWT==2.9.0 -pymongo==4.13.1 +pymongo==4.13.2 PyMySQL==1.1.1 pyparsing==3.2.3 pyproject_hooks==1.2.0 @@ -183,7 +183,7 @@ typing-inspection==0.4.1 typing_extensions==4.14.0 tzdata==2025.2 uritemplate==4.2.0 -urllib3==2.4.0 +urllib3==2.5.0 virtualenv-clone==0.5.7 websockets==15.0.1 wrapt==1.17.2 diff --git a/sdks/python/container/py310/ml_image_requirements.txt b/sdks/python/container/py310/ml_image_requirements.txt new file mode 100644 index 000000000000..3bfd74fa7797 --- /dev/null +++ b/sdks/python/container/py310/ml_image_requirements.txt @@ -0,0 +1,241 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Autogenerated requirements file for Apache Beam py310 container image. +# Run ./gradlew :sdks:python:container:generatePythonRequirementsAll to update. +# Do not edit manually, adjust ../base_image_requirements_manual.txt or +# Apache Beam's setup.py instead, and regenerate the list. +# You will need Python interpreters for all versions supported by Beam, see: +# https://s.apache.org/beam-python-dev-wiki +# Reach out to a committer if you need help. + +absl-py==2.3.0 +aiofiles==24.1.0 +aiohappyeyeballs==2.6.1 +aiohttp==3.12.13 +aiosignal==1.3.2 +annotated-types==0.7.0 +anyio==4.9.0 +asn1crypto==1.5.1 +astunparse==1.6.3 +async-timeout==5.0.1 +attrs==25.3.0 +backports.tarfile==1.2.0 +beautifulsoup4==4.13.4 +bs4==0.0.2 +build==1.2.2.post1 +cachetools==5.5.2 +certifi==2025.6.15 +cffi==1.17.1 +charset-normalizer==3.4.2 +click==8.2.1 +cloud-sql-python-connector==1.18.2 +cmake==4.0.3 +crcmod==1.7 +cryptography==45.0.4 +Cython==3.1.2 +deprecation==2.1.0 +dill==0.3.1.1 +dnspython==2.7.0 +docker==7.1.0 +docopt==0.6.2 +docstring_parser==0.16 +exceptiongroup==1.3.0 +execnet==2.1.1 +fastavro==1.11.1 +fasteners==0.19 +filelock==3.18.0 +flatbuffers==25.2.10 +freezegun==1.5.2 +frozenlist==1.7.0 +fsspec==2025.5.1 +future==1.0.0 +gast==0.4.0 +google-api-core==2.25.1 +google-api-python-client==2.174.0 +google-apitools==0.5.31 +google-auth==2.40.3 +google-auth-httplib2==0.2.0 +google-auth-oauthlib==1.0.0 +google-cloud-aiplatform==1.100.0 +google-cloud-bigquery==3.34.0 +google-cloud-bigquery-storage==2.32.0 +google-cloud-bigtable==2.31.0 +google-cloud-core==2.4.3 +google-cloud-datastore==2.21.0 +google-cloud-dlp==3.31.0 +google-cloud-language==2.17.2 +google-cloud-profiler==4.1.0 +google-cloud-pubsub==2.30.0 +google-cloud-pubsublite==1.12.0 +google-cloud-recommendations-ai==0.10.18 +google-cloud-resource-manager==1.14.2 +google-cloud-spanner==3.55.0 +google-cloud-storage==2.19.0 +google-cloud-videointelligence==2.16.2 +google-cloud-vision==3.10.2 +google-crc32c==1.7.1 +google-genai==1.23.0 +google-pasta==0.2.0 +google-resumable-media==2.7.2 +googleapis-common-protos==1.70.0 +greenlet==3.2.3 +grpc-google-iam-v1==0.14.2 +grpc-interceptor==0.15.4 +grpcio==1.65.5 +grpcio-status==1.63.0rc1 +guppy3==3.1.5 +h11==0.16.0 +h5py==3.14.0 +hdfs==2.7.3 +hf-xet==1.1.5 +httpcore==1.0.9 +httplib2==0.22.0 +httpx==0.28.1 +huggingface-hub==0.33.1 +hypothesis==6.135.18 +idna==3.10 +importlib_metadata==8.7.0 +iniconfig==2.1.0 +jaraco.classes==3.4.0 +jaraco.context==6.0.1 +jaraco.functools==4.2.1 +jax==0.4.30 +jaxlib==0.4.30 +jeepney==0.9.0 +Jinja2==3.1.6 +joblib==1.5.1 +jsonpickle==3.4.2 +jsonschema==4.24.0 +jsonschema-specifications==2025.4.1 +kafka-python==2.2.14 +keras==2.12.0 +keyring==25.6.0 +keyrings.google-artifactregistry-auth==1.1.2 +libclang==18.1.1 +lit==18.1.8 +Markdown==3.8.2 +MarkupSafe==3.0.2 +ml_dtypes==0.5.1 +mmh3==5.1.0 +mock==5.2.0 +more-itertools==10.7.0 +mpmath==1.3.0 +multidict==6.6.3 +mysql-connector-python==9.3.0 +networkx==3.4.2 +nltk==3.9.1 +numpy==1.23.5 +nvidia-cublas-cu11==11.10.3.66 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +nvidia-cuda-runtime-cu11==11.7.99 +nvidia-cudnn-cu11==8.5.0.96 +nvidia-cufft-cu11==10.9.0.58 +nvidia-curand-cu11==10.2.10.91 +nvidia-cusolver-cu11==11.4.0.1 +nvidia-cusparse-cu11==11.7.4.91 +nvidia-nccl-cu11==2.14.3 +nvidia-nvtx-cu11==11.7.91 +oauth2client==4.1.3 +oauthlib==3.3.1 +objsize==0.7.1 +opentelemetry-api==1.34.1 +opentelemetry-sdk==1.34.1 +opentelemetry-semantic-conventions==0.55b1 +opt_einsum==3.4.0 +oracledb==3.2.0 +orjson==3.10.18 +overrides==7.7.0 +packaging==25.0 +pandas==2.2.3 +parameterized==0.9.0 +pg8000==1.31.2 +pluggy==1.6.0 +propcache==0.3.2 +proto-plus==1.26.1 +protobuf==4.25.8 +psycopg2-binary==2.9.10 +pyarrow==18.1.0 +pyarrow-hotfix==0.7 +pyasn1==0.6.1 +pyasn1_modules==0.4.2 +pycparser==2.22 +pydantic==2.11.7 +pydantic_core==2.33.2 +pydot==1.4.2 +PyHamcrest==2.1.0 +PyJWT==2.9.0 +pymongo==4.13.2 +PyMySQL==1.1.1 +pyparsing==3.2.3 +pyproject_hooks==1.2.0 +pytest==7.4.4 +pytest-timeout==2.4.0 +pytest-xdist==3.7.0 +python-dateutil==2.9.0.post0 +python-tds==1.16.1 +pytz==2025.2 +PyYAML==6.0.2 +redis==5.3.0 +referencing==0.36.2 +regex==2024.11.6 +requests==2.32.4 +requests-mock==1.12.1 +requests-oauthlib==2.0.0 +rpds-py==0.25.1 +rsa==4.9.1 +safetensors==0.5.3 +scikit-learn==1.7.0 +scipy==1.15.3 +scramp==1.4.5 +SecretStorage==3.3.3 +shapely==2.1.1 +six==1.17.0 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soupsieve==2.7 +SQLAlchemy==2.0.41 +sqlalchemy_pytds==1.0.2 +sqlparse==0.5.3 +sympy==1.14.0 +tenacity==8.5.0 +tensorboard==2.12.3 +tensorboard-data-server==0.7.2 +tensorflow==2.12.0 +tensorflow-cpu-aws==2.12.0;platform_machine=="aarch64" +tensorflow-estimator==2.12.0 +tensorflow-io-gcs-filesystem==0.37.1 +termcolor==3.1.0 +testcontainers==3.7.1 +threadpoolctl==3.6.0 +tokenizers==0.21.2 +tomli==2.2.1 +torch==2.0.1 +tqdm==4.67.1 +transformers==4.48.3 +triton==2.0.0 +typing-inspection==0.4.1 +typing_extensions==4.14.0 +tzdata==2025.2 +uritemplate==4.2.0 +urllib3==2.5.0 +virtualenv-clone==0.5.7 +websockets==15.0.1 +Werkzeug==3.1.3 +wrapt==1.14.1 +yarl==1.20.1 +zipp==3.23.0 +zstandard==0.23.0 diff --git a/sdks/python/container/py311/base_image_requirements.txt b/sdks/python/container/py311/base_image_requirements.txt index 10d55d17f409..fae1b1e85f1a 100644 --- a/sdks/python/container/py311/base_image_requirements.txt +++ b/sdks/python/container/py311/base_image_requirements.txt @@ -55,17 +55,17 @@ freezegun==1.5.2 frozenlist==1.7.0 future==1.0.0 google-api-core==2.25.1 -google-api-python-client==2.172.0 +google-api-python-client==2.174.0 google-apitools==0.5.31 google-auth==2.40.3 google-auth-httplib2==0.2.0 -google-cloud-aiplatform==1.97.0 +google-cloud-aiplatform==1.100.0 google-cloud-bigquery==3.34.0 google-cloud-bigquery-storage==2.32.0 google-cloud-bigtable==2.31.0 google-cloud-core==2.4.3 google-cloud-datastore==2.21.0 -google-cloud-dlp==3.30.0 +google-cloud-dlp==3.31.0 google-cloud-language==2.17.2 google-cloud-profiler==4.1.0 google-cloud-pubsub==2.30.0 @@ -77,7 +77,7 @@ google-cloud-storage==2.19.0 google-cloud-videointelligence==2.16.2 google-cloud-vision==3.10.2 google-crc32c==1.7.1 -google-genai==1.20.0 +google-genai==1.23.0 google-resumable-media==2.7.2 googleapis-common-protos==1.70.0 greenlet==3.2.3 @@ -91,27 +91,27 @@ hdfs==2.7.3 httpcore==1.0.9 httplib2==0.22.0 httpx==0.28.1 -hypothesis==6.135.10 +hypothesis==6.135.18 idna==3.10 importlib_metadata==8.7.0 iniconfig==2.1.0 jaraco.classes==3.4.0 jaraco.context==6.0.1 -jaraco.functools==4.1.0 +jaraco.functools==4.2.1 jeepney==0.9.0 Jinja2==3.1.6 joblib==1.5.1 jsonpickle==3.4.2 jsonschema==4.24.0 jsonschema-specifications==2025.4.1 -kafka-python==2.2.11 +kafka-python==2.2.14 keyring==25.6.0 keyrings.google-artifactregistry-auth==1.1.2 MarkupSafe==3.0.2 mmh3==5.1.0 mock==5.2.0 more-itertools==10.7.0 -multidict==6.4.4 +multidict==6.6.3 mysql-connector-python==9.3.0 nltk==3.9.1 numpy==2.2.6 @@ -120,7 +120,7 @@ objsize==0.7.1 opentelemetry-api==1.34.1 opentelemetry-sdk==1.34.1 opentelemetry-semantic-conventions==0.55b1 -oracledb==3.1.1 +oracledb==3.2.0 orjson==3.10.18 overrides==7.7.0 packaging==25.0 @@ -142,7 +142,7 @@ pydantic_core==2.33.2 pydot==1.4.2 PyHamcrest==2.1.0 PyJWT==2.9.0 -pymongo==4.13.1 +pymongo==4.13.2 PyMySQL==1.1.1 pyparsing==3.2.3 pyproject_hooks==1.2.0 @@ -161,7 +161,7 @@ requests-mock==1.12.1 rpds-py==0.25.1 rsa==4.9.1 scikit-learn==1.7.0 -scipy==1.15.3 +scipy==1.16.0 scramp==1.4.5 SecretStorage==3.3.3 shapely==2.1.1 @@ -180,7 +180,7 @@ typing-inspection==0.4.1 typing_extensions==4.14.0 tzdata==2025.2 uritemplate==4.2.0 -urllib3==2.4.0 +urllib3==2.5.0 virtualenv-clone==0.5.7 websockets==15.0.1 wrapt==1.17.2 diff --git a/sdks/python/container/py311/ml_image_requirements.txt b/sdks/python/container/py311/ml_image_requirements.txt new file mode 100644 index 000000000000..e5ca504d751c --- /dev/null +++ b/sdks/python/container/py311/ml_image_requirements.txt @@ -0,0 +1,238 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Autogenerated requirements file for Apache Beam py311 container image. +# Run ./gradlew :sdks:python:container:generatePythonRequirementsAll to update. +# Do not edit manually, adjust ../base_image_requirements_manual.txt or +# Apache Beam's setup.py instead, and regenerate the list. +# You will need Python interpreters for all versions supported by Beam, see: +# https://s.apache.org/beam-python-dev-wiki +# Reach out to a committer if you need help. + +absl-py==2.3.0 +aiofiles==24.1.0 +aiohappyeyeballs==2.6.1 +aiohttp==3.12.13 +aiosignal==1.3.2 +annotated-types==0.7.0 +anyio==4.9.0 +asn1crypto==1.5.1 +astunparse==1.6.3 +attrs==25.3.0 +backports.tarfile==1.2.0 +beautifulsoup4==4.13.4 +bs4==0.0.2 +build==1.2.2.post1 +cachetools==5.5.2 +certifi==2025.6.15 +cffi==1.17.1 +charset-normalizer==3.4.2 +click==8.2.1 +cloud-sql-python-connector==1.18.2 +cmake==4.0.3 +crcmod==1.7 +cryptography==45.0.4 +Cython==3.1.2 +deprecation==2.1.0 +dill==0.3.1.1 +dnspython==2.7.0 +docker==7.1.0 +docopt==0.6.2 +docstring_parser==0.16 +execnet==2.1.1 +fastavro==1.11.1 +fasteners==0.19 +filelock==3.18.0 +flatbuffers==25.2.10 +freezegun==1.5.2 +frozenlist==1.7.0 +fsspec==2025.5.1 +future==1.0.0 +gast==0.4.0 +google-api-core==2.25.1 +google-api-python-client==2.174.0 +google-apitools==0.5.31 +google-auth==2.40.3 +google-auth-httplib2==0.2.0 +google-auth-oauthlib==1.0.0 +google-cloud-aiplatform==1.100.0 +google-cloud-bigquery==3.34.0 +google-cloud-bigquery-storage==2.32.0 +google-cloud-bigtable==2.31.0 +google-cloud-core==2.4.3 +google-cloud-datastore==2.21.0 +google-cloud-dlp==3.31.0 +google-cloud-language==2.17.2 +google-cloud-profiler==4.1.0 +google-cloud-pubsub==2.30.0 +google-cloud-pubsublite==1.12.0 +google-cloud-recommendations-ai==0.10.18 +google-cloud-resource-manager==1.14.2 +google-cloud-spanner==3.55.0 +google-cloud-storage==2.19.0 +google-cloud-videointelligence==2.16.2 +google-cloud-vision==3.10.2 +google-crc32c==1.7.1 +google-genai==1.23.0 +google-pasta==0.2.0 +google-resumable-media==2.7.2 +googleapis-common-protos==1.70.0 +greenlet==3.2.3 +grpc-google-iam-v1==0.14.2 +grpc-interceptor==0.15.4 +grpcio==1.65.5 +grpcio-status==1.63.0rc1 +guppy3==3.1.5 +h11==0.16.0 +h5py==3.14.0 +hdfs==2.7.3 +hf-xet==1.1.5 +httpcore==1.0.9 +httplib2==0.22.0 +httpx==0.28.1 +huggingface-hub==0.33.1 +hypothesis==6.135.18 +idna==3.10 +importlib_metadata==8.7.0 +iniconfig==2.1.0 +jaraco.classes==3.4.0 +jaraco.context==6.0.1 +jaraco.functools==4.2.1 +jax==0.4.30 +jaxlib==0.4.30 +jeepney==0.9.0 +Jinja2==3.1.6 +joblib==1.5.1 +jsonpickle==3.4.2 +jsonschema==4.24.0 +jsonschema-specifications==2025.4.1 +kafka-python==2.2.14 +keras==2.12.0 +keyring==25.6.0 +keyrings.google-artifactregistry-auth==1.1.2 +libclang==18.1.1 +lit==18.1.8 +Markdown==3.8.2 +MarkupSafe==3.0.2 +ml_dtypes==0.5.1 +mmh3==5.1.0 +mock==5.2.0 +more-itertools==10.7.0 +mpmath==1.3.0 +multidict==6.6.3 +mysql-connector-python==9.3.0 +networkx==3.5 +nltk==3.9.1 +numpy==1.23.5 +nvidia-cublas-cu11==11.10.3.66 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +nvidia-cuda-runtime-cu11==11.7.99 +nvidia-cudnn-cu11==8.5.0.96 +nvidia-cufft-cu11==10.9.0.58 +nvidia-curand-cu11==10.2.10.91 +nvidia-cusolver-cu11==11.4.0.1 +nvidia-cusparse-cu11==11.7.4.91 +nvidia-nccl-cu11==2.14.3 +nvidia-nvtx-cu11==11.7.91 +oauth2client==4.1.3 +oauthlib==3.3.1 +objsize==0.7.1 +opentelemetry-api==1.34.1 +opentelemetry-sdk==1.34.1 +opentelemetry-semantic-conventions==0.55b1 +opt_einsum==3.4.0 +oracledb==3.2.0 +orjson==3.10.18 +overrides==7.7.0 +packaging==25.0 +pandas==2.2.3 +parameterized==0.9.0 +pg8000==1.31.2 +pluggy==1.6.0 +propcache==0.3.2 +proto-plus==1.26.1 +protobuf==4.25.8 +psycopg2-binary==2.9.10 +pyarrow==18.1.0 +pyarrow-hotfix==0.7 +pyasn1==0.6.1 +pyasn1_modules==0.4.2 +pycparser==2.22 +pydantic==2.11.7 +pydantic_core==2.33.2 +pydot==1.4.2 +PyHamcrest==2.1.0 +PyJWT==2.9.0 +pymongo==4.13.2 +PyMySQL==1.1.1 +pyparsing==3.2.3 +pyproject_hooks==1.2.0 +pytest==7.4.4 +pytest-timeout==2.4.0 +pytest-xdist==3.7.0 +python-dateutil==2.9.0.post0 +python-tds==1.16.1 +pytz==2025.2 +PyYAML==6.0.2 +redis==5.3.0 +referencing==0.36.2 +regex==2024.11.6 +requests==2.32.4 +requests-mock==1.12.1 +requests-oauthlib==2.0.0 +rpds-py==0.25.1 +rsa==4.9.1 +safetensors==0.5.3 +scikit-learn==1.7.0 +scipy==1.15.3 +scramp==1.4.5 +SecretStorage==3.3.3 +shapely==2.1.1 +six==1.17.0 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soupsieve==2.7 +SQLAlchemy==2.0.41 +sqlalchemy_pytds==1.0.2 +sqlparse==0.5.3 +sympy==1.14.0 +tenacity==8.5.0 +tensorboard==2.12.3 +tensorboard-data-server==0.7.2 +tensorflow==2.12.0 +tensorflow-cpu-aws==2.12.0;platform_machine=="aarch64" +tensorflow-estimator==2.12.0 +tensorflow-io-gcs-filesystem==0.37.1 +termcolor==3.1.0 +testcontainers==3.7.1 +threadpoolctl==3.6.0 +tokenizers==0.21.2 +torch==2.0.1 +tqdm==4.67.1 +transformers==4.48.3 +triton==2.0.0 +typing-inspection==0.4.1 +typing_extensions==4.14.0 +tzdata==2025.2 +uritemplate==4.2.0 +urllib3==2.5.0 +virtualenv-clone==0.5.7 +websockets==15.0.1 +Werkzeug==3.1.3 +wrapt==1.14.1 +yarl==1.20.1 +zipp==3.23.0 +zstandard==0.23.0 diff --git a/sdks/python/container/py312/base_image_requirements.txt b/sdks/python/container/py312/base_image_requirements.txt index d4b9c8751dca..1cbd5e8122ec 100644 --- a/sdks/python/container/py312/base_image_requirements.txt +++ b/sdks/python/container/py312/base_image_requirements.txt @@ -54,17 +54,17 @@ freezegun==1.5.2 frozenlist==1.7.0 future==1.0.0 google-api-core==2.25.1 -google-api-python-client==2.172.0 +google-api-python-client==2.174.0 google-apitools==0.5.31 google-auth==2.40.3 google-auth-httplib2==0.2.0 -google-cloud-aiplatform==1.97.0 +google-cloud-aiplatform==1.100.0 google-cloud-bigquery==3.34.0 google-cloud-bigquery-storage==2.32.0 google-cloud-bigtable==2.31.0 google-cloud-core==2.4.3 google-cloud-datastore==2.21.0 -google-cloud-dlp==3.30.0 +google-cloud-dlp==3.31.0 google-cloud-language==2.17.2 google-cloud-profiler==4.1.0 google-cloud-pubsub==2.30.0 @@ -76,7 +76,7 @@ google-cloud-storage==2.19.0 google-cloud-videointelligence==2.16.2 google-cloud-vision==3.10.2 google-crc32c==1.7.1 -google-genai==1.20.0 +google-genai==1.23.0 google-resumable-media==2.7.2 googleapis-common-protos==1.70.0 greenlet==3.2.3 @@ -90,27 +90,27 @@ hdfs==2.7.3 httpcore==1.0.9 httplib2==0.22.0 httpx==0.28.1 -hypothesis==6.135.10 +hypothesis==6.135.18 idna==3.10 importlib_metadata==8.7.0 iniconfig==2.1.0 jaraco.classes==3.4.0 jaraco.context==6.0.1 -jaraco.functools==4.1.0 +jaraco.functools==4.2.1 jeepney==0.9.0 Jinja2==3.1.6 joblib==1.5.1 jsonpickle==3.4.2 jsonschema==4.24.0 jsonschema-specifications==2025.4.1 -kafka-python==2.2.11 +kafka-python==2.2.14 keyring==25.6.0 keyrings.google-artifactregistry-auth==1.1.2 MarkupSafe==3.0.2 mmh3==5.1.0 mock==5.2.0 more-itertools==10.7.0 -multidict==6.4.4 +multidict==6.6.3 mysql-connector-python==9.3.0 nltk==3.9.1 numpy==2.2.6 @@ -119,7 +119,7 @@ objsize==0.7.1 opentelemetry-api==1.34.1 opentelemetry-sdk==1.34.1 opentelemetry-semantic-conventions==0.55b1 -oracledb==3.1.1 +oracledb==3.2.0 orjson==3.10.18 overrides==7.7.0 packaging==25.0 @@ -141,7 +141,7 @@ pydantic_core==2.33.2 pydot==1.4.2 PyHamcrest==2.1.0 PyJWT==2.9.0 -pymongo==4.13.1 +pymongo==4.13.2 PyMySQL==1.1.1 pyparsing==3.2.3 pyproject_hooks==1.2.0 @@ -160,7 +160,7 @@ requests-mock==1.12.1 rpds-py==0.25.1 rsa==4.9.1 scikit-learn==1.7.0 -scipy==1.15.3 +scipy==1.16.0 scramp==1.4.5 SecretStorage==3.3.3 setuptools==80.9.0 @@ -180,7 +180,7 @@ typing-inspection==0.4.1 typing_extensions==4.14.0 tzdata==2025.2 uritemplate==4.2.0 -urllib3==2.4.0 +urllib3==2.5.0 virtualenv-clone==0.5.7 websockets==15.0.1 wheel==0.45.1 diff --git a/sdks/python/container/py39/base_image_requirements.txt b/sdks/python/container/py39/base_image_requirements.txt index 849786b95756..70e7a2921618 100644 --- a/sdks/python/container/py39/base_image_requirements.txt +++ b/sdks/python/container/py39/base_image_requirements.txt @@ -57,17 +57,17 @@ freezegun==1.5.2 frozenlist==1.7.0 future==1.0.0 google-api-core==2.25.1 -google-api-python-client==2.172.0 +google-api-python-client==2.174.0 google-apitools==0.5.31 google-auth==2.40.3 google-auth-httplib2==0.2.0 -google-cloud-aiplatform==1.97.0 +google-cloud-aiplatform==1.100.0 google-cloud-bigquery==3.34.0 google-cloud-bigquery-storage==2.32.0 google-cloud-bigtable==2.31.0 google-cloud-core==2.4.3 google-cloud-datastore==2.21.0 -google-cloud-dlp==3.30.0 +google-cloud-dlp==3.31.0 google-cloud-language==2.17.2 google-cloud-profiler==4.1.0 google-cloud-pubsub==2.30.0 @@ -79,7 +79,7 @@ google-cloud-storage==2.19.0 google-cloud-videointelligence==2.16.2 google-cloud-vision==3.10.2 google-crc32c==1.7.1 -google-genai==1.20.0 +google-genai==1.23.0 google-resumable-media==2.7.2 googleapis-common-protos==1.70.0 greenlet==3.2.3 @@ -93,27 +93,27 @@ hdfs==2.7.3 httpcore==1.0.9 httplib2==0.22.0 httpx==0.28.1 -hypothesis==6.135.10 +hypothesis==6.135.18 idna==3.10 importlib_metadata==8.7.0 iniconfig==2.1.0 jaraco.classes==3.4.0 jaraco.context==6.0.1 -jaraco.functools==4.1.0 +jaraco.functools==4.2.1 jeepney==0.9.0 Jinja2==3.1.6 joblib==1.5.1 jsonpickle==3.4.2 jsonschema==4.24.0 jsonschema-specifications==2025.4.1 -kafka-python==2.2.11 +kafka-python==2.2.14 keyring==25.6.0 keyrings.google-artifactregistry-auth==1.1.2 MarkupSafe==3.0.2 mmh3==5.1.0 mock==5.2.0 more-itertools==10.7.0 -multidict==6.4.4 +multidict==6.6.3 mysql-connector-python==9.3.0 nltk==3.9.1 numpy==2.0.2 @@ -122,7 +122,7 @@ objsize==0.7.1 opentelemetry-api==1.34.1 opentelemetry-sdk==1.34.1 opentelemetry-semantic-conventions==0.55b1 -oracledb==3.1.1 +oracledb==3.2.0 orjson==3.10.18 overrides==7.7.0 packaging==25.0 @@ -144,7 +144,7 @@ pydantic_core==2.33.2 pydot==1.4.2 PyHamcrest==2.1.0 PyJWT==2.9.0 -pymongo==4.13.1 +pymongo==4.13.2 PyMySQL==1.1.1 pyparsing==3.2.3 pyproject_hooks==1.2.0 @@ -183,7 +183,7 @@ typing-inspection==0.4.1 typing_extensions==4.14.0 tzdata==2025.2 uritemplate==4.2.0 -urllib3==2.4.0 +urllib3==2.5.0 virtualenv-clone==0.5.7 websockets==15.0.1 wrapt==1.17.2 diff --git a/sdks/python/container/py39/ml_image_requirements.txt b/sdks/python/container/py39/ml_image_requirements.txt new file mode 100644 index 000000000000..6a390eb8217c --- /dev/null +++ b/sdks/python/container/py39/ml_image_requirements.txt @@ -0,0 +1,241 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Autogenerated requirements file for Apache Beam py39 container image. +# Run ./gradlew :sdks:python:container:generatePythonRequirementsAll to update. +# Do not edit manually, adjust ../base_image_requirements_manual.txt or +# Apache Beam's setup.py instead, and regenerate the list. +# You will need Python interpreters for all versions supported by Beam, see: +# https://s.apache.org/beam-python-dev-wiki +# Reach out to a committer if you need help. + +absl-py==2.3.0 +aiofiles==24.1.0 +aiohappyeyeballs==2.6.1 +aiohttp==3.12.13 +aiosignal==1.3.2 +annotated-types==0.7.0 +anyio==4.9.0 +asn1crypto==1.5.1 +astunparse==1.6.3 +async-timeout==5.0.1 +attrs==25.3.0 +backports.tarfile==1.2.0 +beautifulsoup4==4.13.4 +bs4==0.0.2 +build==1.2.2.post1 +cachetools==5.5.2 +certifi==2025.6.15 +cffi==1.17.1 +charset-normalizer==3.4.2 +click==8.1.8 +cloud-sql-python-connector==1.18.2 +cmake==4.0.3 +crcmod==1.7 +cryptography==45.0.4 +Cython==3.1.2 +deprecation==2.1.0 +dill==0.3.1.1 +dnspython==2.7.0 +docker==7.1.0 +docopt==0.6.2 +docstring_parser==0.16 +exceptiongroup==1.3.0 +execnet==2.1.1 +fastavro==1.11.1 +fasteners==0.19 +filelock==3.18.0 +flatbuffers==25.2.10 +freezegun==1.5.2 +frozenlist==1.7.0 +fsspec==2025.5.1 +future==1.0.0 +gast==0.4.0 +google-api-core==2.25.1 +google-api-python-client==2.174.0 +google-apitools==0.5.31 +google-auth==2.40.3 +google-auth-httplib2==0.2.0 +google-auth-oauthlib==1.0.0 +google-cloud-aiplatform==1.100.0 +google-cloud-bigquery==3.34.0 +google-cloud-bigquery-storage==2.32.0 +google-cloud-bigtable==2.31.0 +google-cloud-core==2.4.3 +google-cloud-datastore==2.21.0 +google-cloud-dlp==3.31.0 +google-cloud-language==2.17.2 +google-cloud-profiler==4.1.0 +google-cloud-pubsub==2.30.0 +google-cloud-pubsublite==1.12.0 +google-cloud-recommendations-ai==0.10.18 +google-cloud-resource-manager==1.14.2 +google-cloud-spanner==3.55.0 +google-cloud-storage==2.19.0 +google-cloud-videointelligence==2.16.2 +google-cloud-vision==3.10.2 +google-crc32c==1.7.1 +google-genai==1.23.0 +google-pasta==0.2.0 +google-resumable-media==2.7.2 +googleapis-common-protos==1.70.0 +greenlet==3.2.3 +grpc-google-iam-v1==0.14.2 +grpc-interceptor==0.15.4 +grpcio==1.65.5 +grpcio-status==1.63.0rc1 +guppy3==3.1.5 +h11==0.16.0 +h5py==3.14.0 +hdfs==2.7.3 +hf-xet==1.1.5 +httpcore==1.0.9 +httplib2==0.22.0 +httpx==0.28.1 +huggingface-hub==0.33.1 +hypothesis==6.135.18 +idna==3.10 +importlib_metadata==8.7.0 +iniconfig==2.1.0 +jaraco.classes==3.4.0 +jaraco.context==6.0.1 +jaraco.functools==4.2.1 +jax==0.4.30 +jaxlib==0.4.30 +jeepney==0.9.0 +Jinja2==3.1.6 +joblib==1.5.1 +jsonpickle==3.4.2 +jsonschema==4.24.0 +jsonschema-specifications==2025.4.1 +kafka-python==2.2.14 +keras==2.12.0 +keyring==25.6.0 +keyrings.google-artifactregistry-auth==1.1.2 +libclang==18.1.1 +lit==18.1.8 +Markdown==3.8.2 +MarkupSafe==3.0.2 +ml_dtypes==0.5.1 +mmh3==5.1.0 +mock==5.2.0 +more-itertools==10.7.0 +mpmath==1.3.0 +multidict==6.6.3 +mysql-connector-python==9.3.0 +networkx==3.2.1 +nltk==3.9.1 +numpy==1.23.5 +nvidia-cublas-cu11==11.10.3.66 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +nvidia-cuda-runtime-cu11==11.7.99 +nvidia-cudnn-cu11==8.5.0.96 +nvidia-cufft-cu11==10.9.0.58 +nvidia-curand-cu11==10.2.10.91 +nvidia-cusolver-cu11==11.4.0.1 +nvidia-cusparse-cu11==11.7.4.91 +nvidia-nccl-cu11==2.14.3 +nvidia-nvtx-cu11==11.7.91 +oauth2client==4.1.3 +oauthlib==3.3.1 +objsize==0.7.1 +opentelemetry-api==1.34.1 +opentelemetry-sdk==1.34.1 +opentelemetry-semantic-conventions==0.55b1 +opt_einsum==3.4.0 +oracledb==3.2.0 +orjson==3.10.18 +overrides==7.7.0 +packaging==25.0 +pandas==2.2.3 +parameterized==0.9.0 +pg8000==1.31.2 +pluggy==1.6.0 +propcache==0.3.2 +proto-plus==1.26.1 +protobuf==4.25.8 +psycopg2-binary==2.9.9 +pyarrow==18.1.0 +pyarrow-hotfix==0.7 +pyasn1==0.6.1 +pyasn1_modules==0.4.2 +pycparser==2.22 +pydantic==2.11.7 +pydantic_core==2.33.2 +pydot==1.4.2 +PyHamcrest==2.1.0 +PyJWT==2.9.0 +pymongo==4.13.2 +PyMySQL==1.1.1 +pyparsing==3.2.3 +pyproject_hooks==1.2.0 +pytest==7.4.4 +pytest-timeout==2.4.0 +pytest-xdist==3.7.0 +python-dateutil==2.9.0.post0 +python-tds==1.16.1 +pytz==2025.2 +PyYAML==6.0.2 +redis==5.3.0 +referencing==0.36.2 +regex==2024.11.6 +requests==2.32.4 +requests-mock==1.12.1 +requests-oauthlib==2.0.0 +rpds-py==0.25.1 +rsa==4.9.1 +safetensors==0.5.3 +scikit-learn==1.6.1 +scipy==1.13.1 +scramp==1.4.5 +SecretStorage==3.3.3 +shapely==2.0.7 +six==1.17.0 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soupsieve==2.7 +SQLAlchemy==2.0.41 +sqlalchemy_pytds==1.0.2 +sqlparse==0.5.3 +sympy==1.14.0 +tenacity==8.5.0 +tensorboard==2.12.3 +tensorboard-data-server==0.7.2 +tensorflow==2.12.0 +tensorflow-cpu-aws==2.12.0;platform_machine=="aarch64" +tensorflow-estimator==2.12.0 +tensorflow-io-gcs-filesystem==0.37.1 +termcolor==3.1.0 +testcontainers==3.7.1 +threadpoolctl==3.6.0 +tokenizers==0.21.2 +tomli==2.2.1 +torch==2.0.1 +tqdm==4.67.1 +transformers==4.48.3 +triton==2.0.0 +typing-inspection==0.4.1 +typing_extensions==4.14.0 +tzdata==2025.2 +uritemplate==4.2.0 +urllib3==2.5.0 +virtualenv-clone==0.5.7 +websockets==15.0.1 +Werkzeug==3.1.3 +wrapt==1.14.1 +yarl==1.20.1 +zipp==3.23.0 +zstandard==0.23.0 diff --git a/sdks/python/setup.py b/sdks/python/setup.py index a0bbc301435b..1c1a69c6c971 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -571,7 +571,7 @@ def get_portability_package_data(): # in https://github.com/apache/beam/blob/master/sdks/python/tox.ini # For more info, see # https://docs.google.com/document/d/1c84Gc-cZRCfrU8f7kWGsNR2o8oSRjCM-dGHO9KvPWPw/edit?usp=sharing - 'torch': ['torch>=1.9.0,<2.1.0'], + 'torch': ['torch>=1.9.0,<2.8.0'], 'tensorflow': ['tensorflow>=2.12rc1,<2.13'], 'transformers': [ 'transformers>=4.28.0,<4.49.0', diff --git a/sdks/python/test-suites/tox/py312/build.gradle b/sdks/python/test-suites/tox/py312/build.gradle index 17478ec43aa8..6ddebb2e6483 100644 --- a/sdks/python/test-suites/tox/py312/build.gradle +++ b/sdks/python/test-suites/tox/py312/build.gradle @@ -26,5 +26,10 @@ applyPythonNature() // Required to setup a Python 3 virtualenv and task names. pythonVersion = '3.12' +// run on precommit +toxTask "testPy312pytorch-271", "py312-pytorch-271", "${posargs}" +test.dependsOn "testpy312pytorch-271" +postCommitPyDep.dependsOn "testpy312pytorch-271" + apply from: "../common.gradle" diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini index 5131769509d9..25f85a0b4758 100644 --- a/sdks/python/tox.ini +++ b/sdks/python/tox.ini @@ -409,6 +409,22 @@ commands = # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pytorch {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' +[testenv:py{311,312}-pytorch-271] +deps = + 200: + torch>=2.7.1,<2.8.0 + mpmath==1.3.0 + numpy==1.26.4 +extras = test,gcp +# Don't set TMPDIR to avoid "AF_UNIX path too long" errors in certain tests. +setenv = +commands = + # Log torch version for debugging + /bin/sh -c "pip freeze | grep -E torch" + # Run all PyTorch>=2 unit tests + # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. + /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pytorch {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' + # TODO(https://github.com/apache/beam/issues/25796) - uncomment onnx tox task in tox/py39/build.gradle once onnx supports protobuf 4.x.x [testenv:py{39,310}-onnx-113] # TODO(https://github.com/apache/beam/issues/25443) From 8566339da15a72f8a62c96ad85e8840881baeedc Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Mon, 30 Jun 2025 19:12:12 +0000 Subject: [PATCH 3/5] Widen bounds and generate 3.12 containers --- .../container/py312/ml_image_requirements.txt | 239 ++++++++++++++++++ sdks/python/setup.py | 6 +- .../python/test-suites/tox/py312/build.gradle | 10 +- sdks/python/tox.ini | 16 ++ 4 files changed, 265 insertions(+), 6 deletions(-) create mode 100644 sdks/python/container/py312/ml_image_requirements.txt diff --git a/sdks/python/container/py312/ml_image_requirements.txt b/sdks/python/container/py312/ml_image_requirements.txt new file mode 100644 index 000000000000..0f4aab97a066 --- /dev/null +++ b/sdks/python/container/py312/ml_image_requirements.txt @@ -0,0 +1,239 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Autogenerated requirements file for Apache Beam py312 container image. +# Run ./gradlew :sdks:python:container:generatePythonRequirementsAll to update. +# Do not edit manually, adjust ../base_image_requirements_manual.txt or +# Apache Beam's setup.py instead, and regenerate the list. +# You will need Python interpreters for all versions supported by Beam, see: +# https://s.apache.org/beam-python-dev-wiki +# Reach out to a committer if you need help. + +absl-py==2.3.0 +aiofiles==24.1.0 +aiohappyeyeballs==2.6.1 +aiohttp==3.12.13 +aiosignal==1.3.2 +annotated-types==0.7.0 +anyio==4.9.0 +asn1crypto==1.5.1 +astunparse==1.6.3 +attrs==25.3.0 +beautifulsoup4==4.13.4 +bs4==0.0.2 +build==1.2.2.post1 +cachetools==5.5.2 +certifi==2025.6.15 +cffi==1.17.1 +charset-normalizer==3.4.2 +click==8.2.1 +cloud-sql-python-connector==1.18.2 +crcmod==1.7 +cryptography==45.0.4 +Cython==3.1.2 +deprecation==2.1.0 +dill==0.3.1.1 +dnspython==2.7.0 +docker==7.1.0 +docopt==0.6.2 +docstring_parser==0.16 +execnet==2.1.1 +fastavro==1.11.1 +fasteners==0.19 +filelock==3.18.0 +flatbuffers==25.2.10 +freezegun==1.5.2 +frozenlist==1.7.0 +fsspec==2025.5.1 +future==1.0.0 +gast==0.6.0 +google-api-core==2.25.1 +google-api-python-client==2.174.0 +google-apitools==0.5.31 +google-auth==2.40.3 +google-auth-httplib2==0.2.0 +google-cloud-aiplatform==1.100.0 +google-cloud-bigquery==3.34.0 +google-cloud-bigquery-storage==2.32.0 +google-cloud-bigtable==2.31.0 +google-cloud-core==2.4.3 +google-cloud-datastore==2.21.0 +google-cloud-dlp==3.31.0 +google-cloud-language==2.17.2 +google-cloud-profiler==4.1.0 +google-cloud-pubsub==2.30.0 +google-cloud-pubsublite==1.12.0 +google-cloud-recommendations-ai==0.10.18 +google-cloud-resource-manager==1.14.2 +google-cloud-spanner==3.55.0 +google-cloud-storage==2.19.0 +google-cloud-videointelligence==2.16.2 +google-cloud-vision==3.10.2 +google-crc32c==1.7.1 +google-genai==1.23.0 +google-pasta==0.2.0 +google-resumable-media==2.7.2 +googleapis-common-protos==1.70.0 +greenlet==3.2.3 +grpc-google-iam-v1==0.14.2 +grpc-interceptor==0.15.4 +grpcio==1.65.5 +grpcio-status==1.63.0rc1 +guppy3==3.1.5 +h11==0.16.0 +h5py==3.14.0 +hdfs==2.7.3 +hf-xet==1.1.5 +httpcore==1.0.9 +httplib2==0.22.0 +httpx==0.28.1 +huggingface-hub==0.33.1 +hypothesis==6.135.19 +idna==3.10 +importlib_metadata==8.7.0 +iniconfig==2.1.0 +jaraco.classes==3.4.0 +jaraco.context==6.0.1 +jaraco.functools==4.2.1 +jeepney==0.9.0 +Jinja2==3.1.6 +joblib==1.5.1 +jsonpickle==3.4.2 +jsonschema==4.24.0 +jsonschema-specifications==2025.4.1 +kafka-python==2.2.14 +keras==3.10.0 +keyring==25.6.0 +keyrings.google-artifactregistry-auth==1.1.2 +libclang==18.1.1 +Markdown==3.8.2 +markdown-it-py==3.0.0 +MarkupSafe==3.0.2 +mdurl==0.1.2 +ml-dtypes==0.3.2 +mmh3==5.1.0 +mock==5.2.0 +more-itertools==10.7.0 +mpmath==1.3.0 +multidict==6.6.3 +mysql-connector-python==9.3.0 +namex==0.1.0 +networkx==3.5 +nltk==3.9.1 +numpy==1.26.4 +nvidia-cublas-cu12==12.6.4.1 +nvidia-cuda-cupti-cu12==12.6.80 +nvidia-cuda-nvrtc-cu12==12.6.77 +nvidia-cuda-runtime-cu12==12.6.77 +nvidia-cudnn-cu12==9.5.1.17 +nvidia-cufft-cu12==11.3.0.4 +nvidia-cufile-cu12==1.11.1.6 +nvidia-curand-cu12==10.3.7.77 +nvidia-cusolver-cu12==11.7.1.2 +nvidia-cusparse-cu12==12.5.4.2 +nvidia-cusparselt-cu12==0.6.3 +nvidia-nccl-cu12==2.26.2 +nvidia-nvjitlink-cu12==12.6.85 +nvidia-nvtx-cu12==12.6.77 +oauth2client==4.1.3 +objsize==0.7.1 +opentelemetry-api==1.34.1 +opentelemetry-sdk==1.34.1 +opentelemetry-semantic-conventions==0.55b1 +opt_einsum==3.4.0 +optree==0.16.0 +oracledb==3.2.0 +orjson==3.10.18 +overrides==7.7.0 +packaging==25.0 +pandas==2.2.3 +parameterized==0.9.0 +pg8000==1.31.2 +pluggy==1.6.0 +propcache==0.3.2 +proto-plus==1.26.1 +protobuf==4.25.8 +psycopg2-binary==2.9.10 +pyarrow==18.1.0 +pyarrow-hotfix==0.7 +pyasn1==0.6.1 +pyasn1_modules==0.4.2 +pycparser==2.22 +pydantic==2.11.7 +pydantic_core==2.33.2 +pydot==1.4.2 +Pygments==2.19.2 +PyHamcrest==2.1.0 +PyJWT==2.9.0 +pymongo==4.13.2 +PyMySQL==1.1.1 +pyparsing==3.2.3 +pyproject_hooks==1.2.0 +pytest==7.4.4 +pytest-timeout==2.4.0 +pytest-xdist==3.7.0 +python-dateutil==2.9.0.post0 +python-tds==1.16.1 +pytz==2025.2 +PyYAML==6.0.2 +redis==5.3.0 +referencing==0.36.2 +regex==2024.11.6 +requests==2.32.4 +requests-mock==1.12.1 +rich==14.0.0 +rpds-py==0.25.1 +rsa==4.9.1 +safetensors==0.5.3 +scikit-learn==1.7.0 +scipy==1.16.0 +scramp==1.4.5 +SecretStorage==3.3.3 +setuptools==80.9.0 +shapely==2.1.1 +six==1.17.0 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soupsieve==2.7 +SQLAlchemy==2.0.41 +sqlalchemy_pytds==1.0.2 +sqlparse==0.5.3 +sympy==1.14.0 +tenacity==8.5.0 +tensorboard==2.16.2 +tensorboard-data-server==0.7.2 +tensorflow==2.16.2 +tensorflow-cpu-aws==2.16.2;platform_machine=="aarch64" +termcolor==3.1.0 +testcontainers==3.7.1 +threadpoolctl==3.6.0 +tokenizers==0.21.2 +torch==2.7.1 +tqdm==4.67.1 +transformers==4.48.3 +triton==3.3.1 +typing-inspection==0.4.1 +typing_extensions==4.14.0 +tzdata==2025.2 +uritemplate==4.2.0 +urllib3==2.5.0 +virtualenv-clone==0.5.7 +websockets==15.0.1 +Werkzeug==3.1.3 +wheel==0.45.1 +wrapt==1.17.2 +yarl==1.20.1 +zipp==3.23.0 +zstandard==0.23.0 diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 1c1a69c6c971..aa774e34d1f8 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -572,11 +572,11 @@ def get_portability_package_data(): # For more info, see # https://docs.google.com/document/d/1c84Gc-cZRCfrU8f7kWGsNR2o8oSRjCM-dGHO9KvPWPw/edit?usp=sharing 'torch': ['torch>=1.9.0,<2.8.0'], - 'tensorflow': ['tensorflow>=2.12rc1,<2.13'], + 'tensorflow': ['tensorflow>=2.12rc1,<2.17'], 'transformers': [ 'transformers>=4.28.0,<4.49.0', - 'tensorflow==2.12.0', - 'torch>=1.9.0,<2.1.0' + 'tensorflow>=2.12.0', + 'torch>=1.9.0' ], 'tft': ['tensorflow_transform>=1.14.0,<1.15.0'], 'onnx': [ diff --git a/sdks/python/test-suites/tox/py312/build.gradle b/sdks/python/test-suites/tox/py312/build.gradle index 6ddebb2e6483..cf1e93e268c8 100644 --- a/sdks/python/test-suites/tox/py312/build.gradle +++ b/sdks/python/test-suites/tox/py312/build.gradle @@ -27,9 +27,13 @@ applyPythonNature() pythonVersion = '3.12' // run on precommit -toxTask "testPy312pytorch-271", "py312-pytorch-271", "${posargs}" -test.dependsOn "testpy312pytorch-271" -postCommitPyDep.dependsOn "testpy312pytorch-271" +toxTask "testPy312pytensorflow-216", "py312-pytensorflow-216", "${posargs}" +test.dependsOn "testpy312pytensorflow-216" +postCommitPyDep.dependsOn "testpy312pytensorflow-216" + +toxTask "testPy312pytensorflow-216", "py312-pytensorflow-216", "${posargs}" +test.dependsOn "testpy312pytensorflow-216" +postCommitPyDep.dependsOn "testpy312pytensorflow-216" apply from: "../common.gradle" diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini index 25f85a0b4758..856ca46f64ae 100644 --- a/sdks/python/tox.ini +++ b/sdks/python/tox.ini @@ -461,6 +461,22 @@ commands = # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_tf {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' +[testenv:py312-tensorflow-216] +deps = + 216: + tensorflow>=2.16.1,<2.17 + # Help pip resolve conflict with typing-extensions for old version of TF https://github.com/apache/beam/issues/30852 + pydantic<2.7 +extras = test,gcp +commands_pre = + pip install -U 'protobuf==4.25.5' +commands = + # Log tensorflow version for debugging + /bin/sh -c "pip freeze | grep -E tensorflow" + # Run all Tensorflow unit tests + # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. + /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_tf {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' + [testenv:py39-xgboost-{160,170}] deps = 160: From caf536fd0c41d2eff4d93bd1eee7b8c1268da93b Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Mon, 30 Jun 2025 20:28:59 +0000 Subject: [PATCH 4/5] posargs --- sdks/python/test-suites/tox/py312/build.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdks/python/test-suites/tox/py312/build.gradle b/sdks/python/test-suites/tox/py312/build.gradle index cf1e93e268c8..d426c9ea7cc1 100644 --- a/sdks/python/test-suites/tox/py312/build.gradle +++ b/sdks/python/test-suites/tox/py312/build.gradle @@ -23,6 +23,8 @@ plugins { id 'org.apache.beam.module' } applyPythonNature() +def posargs = project.findProperty("posargs") ?: "" + // Required to setup a Python 3 virtualenv and task names. pythonVersion = '3.12' From 1f445223612586c5b5a2963d8a7ed487206e6f96 Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Mon, 30 Jun 2025 21:04:11 +0000 Subject: [PATCH 5/5] Tox config fixes --- .../beam_PostCommit_Python_Dependency.json | 2 +- .../beam_PostCommit_Python_Dependency.yml | 7 ++++--- sdks/python/test-suites/tox/py312/build.gradle | 17 +++++++++-------- sdks/python/test-suites/tox/py39/build.gradle | 2 +- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/.github/trigger_files/beam_PostCommit_Python_Dependency.json b/.github/trigger_files/beam_PostCommit_Python_Dependency.json index 907b485d4d30..a7fc54b3e4bb 100644 --- a/.github/trigger_files/beam_PostCommit_Python_Dependency.json +++ b/.github/trigger_files/beam_PostCommit_Python_Dependency.json @@ -1,4 +1,4 @@ { "comment": "Modify this file in a trivial way to cause this test suite to run", - "modification": 0 + "modification": 1 } \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Python_Dependency.yml b/.github/workflows/beam_PostCommit_Python_Dependency.yml index c1eef980c82a..a73e288171df 100644 --- a/.github/workflows/beam_PostCommit_Python_Dependency.yml +++ b/.github/workflows/beam_PostCommit_Python_Dependency.yml @@ -59,6 +59,7 @@ jobs: matrix: job_name: [beam_PostCommit_Python_Dependency] job_phrase: [Run Python PostCommit Dependency] + python_version: ['39','312'] timeout-minutes: 120 if: | github.event_name == 'workflow_dispatch' || @@ -70,9 +71,9 @@ jobs: - name: Setup repository uses: ./.github/actions/setup-action with: - comment_phrase: ${{ matrix.job_phrase }} + comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }} github_token: ${{ secrets.GITHUB_TOKEN }} - github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) - name: Setup environment uses: ./.github/actions/setup-environment-action with: @@ -81,7 +82,7 @@ jobs: - name: Run postCommitPyDep uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:python:test-suites:tox:py39:postCommitPyDep + gradle-command: :sdks:python:test-suites:tox:py${{ matrix.python_version }}:postCommitPyDep arguments: -PuseWheelDistribution - name: Archive Python Test Results uses: actions/upload-artifact@v4 diff --git a/sdks/python/test-suites/tox/py312/build.gradle b/sdks/python/test-suites/tox/py312/build.gradle index d426c9ea7cc1..a8f2ac7fa5cb 100644 --- a/sdks/python/test-suites/tox/py312/build.gradle +++ b/sdks/python/test-suites/tox/py312/build.gradle @@ -28,14 +28,15 @@ def posargs = project.findProperty("posargs") ?: "" // Required to setup a Python 3 virtualenv and task names. pythonVersion = '3.12' -// run on precommit -toxTask "testPy312pytensorflow-216", "py312-pytensorflow-216", "${posargs}" -test.dependsOn "testpy312pytensorflow-216" -postCommitPyDep.dependsOn "testpy312pytensorflow-216" - -toxTask "testPy312pytensorflow-216", "py312-pytensorflow-216", "${posargs}" -test.dependsOn "testpy312pytensorflow-216" -postCommitPyDep.dependsOn "testpy312pytensorflow-216" +project.tasks.register("postCommitPyDep") {} + +toxTask "testPy312pytorch-271", "py312-pytorch-271", "${posargs}" +test.dependsOn "testPy312pytorch-271" +postCommitPyDep.dependsOn "testPy312pytorch-271" + +toxTask "testPy312tensorflow-216", "py312-tensorflow-216", "${posargs}" +test.dependsOn "testPy312tensorflow-216" +postCommitPyDep.dependsOn "testPy312tensorflow-216" apply from: "../common.gradle" diff --git a/sdks/python/test-suites/tox/py39/build.gradle b/sdks/python/test-suites/tox/py39/build.gradle index d6a5e08fcf32..9740f056e685 100644 --- a/sdks/python/test-suites/tox/py39/build.gradle +++ b/sdks/python/test-suites/tox/py39/build.gradle @@ -37,7 +37,7 @@ project.tasks.register("preCommitPyCoverage") { } // Dep Postcommit runs test suites that evaluate compatibility of particular -// dependencies. It is exercised on a single Python version. +// dependencies. Each suite is exercised on at most one python version. // // Should still leave at least one version in PreCommit unless the marked tests // are also exercised by existing PreCommit