diff --git a/.github/trigger_files/beam_PostCommit_Python_Dependency.json b/.github/trigger_files/beam_PostCommit_Python_Dependency.json index 907b485d4d30..a7fc54b3e4bb 100644 --- a/.github/trigger_files/beam_PostCommit_Python_Dependency.json +++ b/.github/trigger_files/beam_PostCommit_Python_Dependency.json @@ -1,4 +1,4 @@ { "comment": "Modify this file in a trivial way to cause this test suite to run", - "modification": 0 + "modification": 1 } \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Python_Dependency.yml b/.github/workflows/beam_PostCommit_Python_Dependency.yml index c1eef980c82a..a73e288171df 100644 --- a/.github/workflows/beam_PostCommit_Python_Dependency.yml +++ b/.github/workflows/beam_PostCommit_Python_Dependency.yml @@ -59,6 +59,7 @@ jobs: matrix: job_name: [beam_PostCommit_Python_Dependency] job_phrase: [Run Python PostCommit Dependency] + python_version: ['39','312'] timeout-minutes: 120 if: | github.event_name == 'workflow_dispatch' || @@ -70,9 +71,9 @@ jobs: - name: Setup repository uses: ./.github/actions/setup-action with: - comment_phrase: ${{ matrix.job_phrase }} + comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }} github_token: ${{ secrets.GITHUB_TOKEN }} - github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) - name: Setup environment uses: ./.github/actions/setup-environment-action with: @@ -81,7 +82,7 @@ jobs: - name: Run postCommitPyDep uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:python:test-suites:tox:py39:postCommitPyDep + gradle-command: :sdks:python:test-suites:tox:py${{ matrix.python_version }}:postCommitPyDep arguments: -PuseWheelDistribution - name: Archive Python Test Results uses: actions/upload-artifact@v4 diff --git a/sdks/python/container/common.gradle b/sdks/python/container/common.gradle index 0175778a6301..6c3de38d56c1 100644 --- a/sdks/python/container/common.gradle +++ b/sdks/python/container/common.gradle @@ -39,7 +39,20 @@ def generatePythonRequirements = tasks.register("generatePythonRequirements") { executable 'sh' args '-c', "cd ${rootDir} && ${runScriptsPath} " + "${project.ext.pythonVersion} " + - "${files(configurations.sdkSourceTarball.files).singleFile} " + "${pipExtraOptions}" + "${files(configurations.sdkSourceTarball.files).singleFile} " + + "base_image_requirements.txt " + + "[gcp,dataframe,test] " + + "${pipExtraOptions}" + } + // Generate versions for ML dependencies + exec { + executable 'sh' + args '-c', "cd ${rootDir} && ${runScriptsPath} " + + "${project.ext.pythonVersion} " + + "${files(configurations.sdkSourceTarball.files).singleFile} " + + "ml_image_requirements.txt " + + "[gcp,dataframe,test,tensorflow,torch,transformers] " + + "${pipExtraOptions}" } } } diff --git a/sdks/python/container/py310/base_image_requirements.txt b/sdks/python/container/py310/base_image_requirements.txt index e9b4f1905399..81834540267c 100644 --- a/sdks/python/container/py310/base_image_requirements.txt +++ b/sdks/python/container/py310/base_image_requirements.txt @@ -92,7 +92,7 @@ hdfs==2.7.3 httpcore==1.0.9 httplib2==0.22.0 httpx==0.28.1 -hypothesis==6.135.17 +hypothesis==6.135.19 idna==3.10 importlib_metadata==8.7.0 iniconfig==2.1.0 @@ -112,7 +112,7 @@ milvus-lite==2.5.1 mmh3==5.1.0 mock==5.2.0 more-itertools==10.7.0 -multidict==6.6.2 +multidict==6.6.3 mysql-connector-python==9.3.0 nltk==3.9.1 numpy==2.2.6 diff --git a/sdks/python/container/py310/ml_image_requirements.txt b/sdks/python/container/py310/ml_image_requirements.txt new file mode 100644 index 000000000000..2b70da331f53 --- /dev/null +++ b/sdks/python/container/py310/ml_image_requirements.txt @@ -0,0 +1,247 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Autogenerated requirements file for Apache Beam py310 container image. +# Run ./gradlew :sdks:python:container:generatePythonRequirementsAll to update. +# Do not edit manually, adjust ../base_image_requirements_manual.txt or +# Apache Beam's setup.py instead, and regenerate the list. +# You will need Python interpreters for all versions supported by Beam, see: +# https://s.apache.org/beam-python-dev-wiki +# Reach out to a committer if you need help. + +absl-py==2.3.0 +aiofiles==24.1.0 +aiohappyeyeballs==2.6.1 +aiohttp==3.12.13 +aiosignal==1.3.2 +annotated-types==0.7.0 +anyio==4.9.0 +asn1crypto==1.5.1 +astunparse==1.6.3 +async-timeout==5.0.1 +attrs==25.3.0 +backports.tarfile==1.2.0 +beautifulsoup4==4.13.4 +bs4==0.0.2 +build==1.2.2.post1 +cachetools==5.5.2 +certifi==2025.6.15 +cffi==1.17.1 +charset-normalizer==3.4.2 +click==8.2.1 +cloud-sql-python-connector==1.18.2 +crcmod==1.7 +cryptography==45.0.4 +Cython==3.1.2 +dill==0.3.1.1 +dnspython==2.7.0 +docker==7.1.0 +docopt==0.6.2 +docstring_parser==0.16 +exceptiongroup==1.3.0 +execnet==2.1.1 +fastavro==1.11.1 +fasteners==0.19 +filelock==3.18.0 +flatbuffers==25.2.10 +freezegun==1.5.2 +frozenlist==1.7.0 +fsspec==2025.5.1 +future==1.0.0 +gast==0.6.0 +google-api-core==2.25.1 +google-api-python-client==2.174.0 +google-apitools==0.5.31 +google-auth==2.40.3 +google-auth-httplib2==0.2.0 +google-cloud-aiplatform==1.100.0 +google-cloud-bigquery==3.34.0 +google-cloud-bigquery-storage==2.32.0 +google-cloud-bigtable==2.31.0 +google-cloud-core==2.4.3 +google-cloud-datastore==2.21.0 +google-cloud-dlp==3.31.0 +google-cloud-language==2.17.2 +google-cloud-profiler==4.1.0 +google-cloud-pubsub==2.30.0 +google-cloud-pubsublite==1.12.0 +google-cloud-recommendations-ai==0.10.18 +google-cloud-resource-manager==1.14.2 +google-cloud-spanner==3.55.0 +google-cloud-storage==2.19.0 +google-cloud-videointelligence==2.16.2 +google-cloud-vision==3.10.2 +google-crc32c==1.7.1 +google-genai==1.23.0 +google-pasta==0.2.0 +google-resumable-media==2.7.2 +googleapis-common-protos==1.70.0 +greenlet==3.2.3 +grpc-google-iam-v1==0.14.2 +grpc-interceptor==0.15.4 +grpcio==1.65.5 +grpcio-status==1.63.0rc1 +guppy3==3.1.5 +h11==0.16.0 +h5py==3.14.0 +hdfs==2.7.3 +hf-xet==1.1.5 +httpcore==1.0.9 +httplib2==0.22.0 +httpx==0.28.1 +huggingface-hub==0.33.1 +hypothesis==6.135.19 +idna==3.10 +importlib_metadata==8.7.0 +iniconfig==2.1.0 +jaraco.classes==3.4.0 +jaraco.context==6.0.1 +jaraco.functools==4.2.1 +jeepney==0.9.0 +Jinja2==3.1.6 +joblib==1.5.1 +jsonpickle==3.4.2 +jsonschema==4.24.0 +jsonschema-specifications==2025.4.1 +keras==3.10.0 +keyring==25.6.0 +keyrings.google-artifactregistry-auth==1.1.2 +libclang==18.1.1 +Markdown==3.8.2 +markdown-it-py==3.0.0 +MarkupSafe==3.0.2 +mdurl==0.1.2 +milvus-lite==2.5.1 +ml-dtypes==0.3.2 +mmh3==5.1.0 +mock==5.2.0 +more-itertools==10.7.0 +mpmath==1.3.0 +multidict==6.6.3 +mysql-connector-python==9.3.0 +namex==0.1.0 +networkx==3.4.2 +nltk==3.9.1 +numpy==1.26.4 +nvidia-cublas-cu12==12.6.4.1 +nvidia-cuda-cupti-cu12==12.6.80 +nvidia-cuda-nvrtc-cu12==12.6.77 +nvidia-cuda-runtime-cu12==12.6.77 +nvidia-cudnn-cu12==9.5.1.17 +nvidia-cufft-cu12==11.3.0.4 +nvidia-cufile-cu12==1.11.1.6 +nvidia-curand-cu12==10.3.7.77 +nvidia-cusolver-cu12==11.7.1.2 +nvidia-cusparse-cu12==12.5.4.2 +nvidia-cusparselt-cu12==0.6.3 +nvidia-nccl-cu12==2.26.2 +nvidia-nvjitlink-cu12==12.6.85 +nvidia-nvtx-cu12==12.6.77 +oauth2client==4.1.3 +objsize==0.7.1 +opentelemetry-api==1.34.1 +opentelemetry-sdk==1.34.1 +opentelemetry-semantic-conventions==0.55b1 +opt_einsum==3.4.0 +optree==0.16.0 +oracledb==3.2.0 +orjson==3.10.18 +overrides==7.7.0 +packaging==25.0 +pandas==2.2.3 +parameterized==0.9.0 +pg8000==1.31.2 +pip==25.1.1 +pluggy==1.6.0 +propcache==0.3.2 +proto-plus==1.26.1 +protobuf==4.25.8 +psycopg2-binary==2.9.10 +pyarrow==18.1.0 +pyarrow-hotfix==0.7 +pyasn1==0.6.1 +pyasn1_modules==0.4.2 +pycparser==2.22 +pydantic==2.11.7 +pydantic_core==2.33.2 +pydot==1.4.2 +Pygments==2.19.2 +PyHamcrest==2.1.0 +PyJWT==2.9.0 +pymilvus==2.5.11 +pymongo==4.13.2 +PyMySQL==1.1.1 +pyparsing==3.2.3 +pyproject_hooks==1.2.0 +pytest==7.4.4 +pytest-timeout==2.4.0 +pytest-xdist==3.7.0 +python-dateutil==2.9.0.post0 +python-dotenv==1.1.1 +python-tds==1.16.1 +pytz==2025.2 +PyYAML==6.0.2 +redis==5.3.0 +referencing==0.36.2 +regex==2024.11.6 +requests==2.32.4 +requests-mock==1.12.1 +rich==14.0.0 +rpds-py==0.25.1 +rsa==4.9.1 +safetensors==0.5.3 +scikit-learn==1.7.0 +scipy==1.15.3 +scramp==1.4.5 +SecretStorage==3.3.3 +setuptools==80.9.0 +shapely==2.1.1 +six==1.17.0 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soupsieve==2.7 +SQLAlchemy==2.0.41 +sqlalchemy_pytds==1.0.2 +sqlparse==0.5.3 +sympy==1.14.0 +tenacity==8.5.0 +tensorboard==2.16.2 +tensorboard-data-server==0.7.2 +tensorflow==2.16.2 +tensorflow-cpu-aws==2.16.2;platform_machine=="aarch64" +tensorflow-io-gcs-filesystem==0.37.1 +termcolor==3.1.0 +testcontainers==4.10.0 +threadpoolctl==3.6.0 +tokenizers==0.21.2 +tomli==2.2.1 +torch==2.7.1 +tqdm==4.67.1 +transformers==4.48.3 +triton==3.3.1 +typing-inspection==0.4.1 +typing_extensions==4.14.0 +tzdata==2025.2 +ujson==5.10.0 +uritemplate==4.2.0 +urllib3==2.5.0 +virtualenv-clone==0.5.7 +websockets==15.0.1 +Werkzeug==3.1.3 +wheel==0.45.1 +wrapt==1.17.2 +yarl==1.20.1 +zipp==3.23.0 +zstandard==0.23.0 diff --git a/sdks/python/container/py311/base_image_requirements.txt b/sdks/python/container/py311/base_image_requirements.txt index af2e75a54b8f..2f81ea5e79d1 100644 --- a/sdks/python/container/py311/base_image_requirements.txt +++ b/sdks/python/container/py311/base_image_requirements.txt @@ -90,7 +90,7 @@ hdfs==2.7.3 httpcore==1.0.9 httplib2==0.22.0 httpx==0.28.1 -hypothesis==6.135.17 +hypothesis==6.135.19 idna==3.10 importlib_metadata==8.7.0 iniconfig==2.1.0 @@ -110,7 +110,7 @@ milvus-lite==2.5.1 mmh3==5.1.0 mock==5.2.0 more-itertools==10.7.0 -multidict==6.6.2 +multidict==6.6.3 mysql-connector-python==9.3.0 nltk==3.9.1 numpy==2.2.6 diff --git a/sdks/python/container/py311/ml_image_requirements.txt b/sdks/python/container/py311/ml_image_requirements.txt new file mode 100644 index 000000000000..a5d8add176d2 --- /dev/null +++ b/sdks/python/container/py311/ml_image_requirements.txt @@ -0,0 +1,244 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Autogenerated requirements file for Apache Beam py311 container image. +# Run ./gradlew :sdks:python:container:generatePythonRequirementsAll to update. +# Do not edit manually, adjust ../base_image_requirements_manual.txt or +# Apache Beam's setup.py instead, and regenerate the list. +# You will need Python interpreters for all versions supported by Beam, see: +# https://s.apache.org/beam-python-dev-wiki +# Reach out to a committer if you need help. + +absl-py==2.3.0 +aiofiles==24.1.0 +aiohappyeyeballs==2.6.1 +aiohttp==3.12.13 +aiosignal==1.3.2 +annotated-types==0.7.0 +anyio==4.9.0 +asn1crypto==1.5.1 +astunparse==1.6.3 +attrs==25.3.0 +backports.tarfile==1.2.0 +beautifulsoup4==4.13.4 +bs4==0.0.2 +build==1.2.2.post1 +cachetools==5.5.2 +certifi==2025.6.15 +cffi==1.17.1 +charset-normalizer==3.4.2 +click==8.2.1 +cloud-sql-python-connector==1.18.2 +crcmod==1.7 +cryptography==45.0.4 +Cython==3.1.2 +dill==0.3.1.1 +dnspython==2.7.0 +docker==7.1.0 +docopt==0.6.2 +docstring_parser==0.16 +execnet==2.1.1 +fastavro==1.11.1 +fasteners==0.19 +filelock==3.18.0 +flatbuffers==25.2.10 +freezegun==1.5.2 +frozenlist==1.7.0 +fsspec==2025.5.1 +future==1.0.0 +gast==0.6.0 +google-api-core==2.25.1 +google-api-python-client==2.174.0 +google-apitools==0.5.31 +google-auth==2.40.3 +google-auth-httplib2==0.2.0 +google-cloud-aiplatform==1.100.0 +google-cloud-bigquery==3.34.0 +google-cloud-bigquery-storage==2.32.0 +google-cloud-bigtable==2.31.0 +google-cloud-core==2.4.3 +google-cloud-datastore==2.21.0 +google-cloud-dlp==3.31.0 +google-cloud-language==2.17.2 +google-cloud-profiler==4.1.0 +google-cloud-pubsub==2.30.0 +google-cloud-pubsublite==1.12.0 +google-cloud-recommendations-ai==0.10.18 +google-cloud-resource-manager==1.14.2 +google-cloud-spanner==3.55.0 +google-cloud-storage==2.19.0 +google-cloud-videointelligence==2.16.2 +google-cloud-vision==3.10.2 +google-crc32c==1.7.1 +google-genai==1.23.0 +google-pasta==0.2.0 +google-resumable-media==2.7.2 +googleapis-common-protos==1.70.0 +greenlet==3.2.3 +grpc-google-iam-v1==0.14.2 +grpc-interceptor==0.15.4 +grpcio==1.65.5 +grpcio-status==1.63.0rc1 +guppy3==3.1.5 +h11==0.16.0 +h5py==3.14.0 +hdfs==2.7.3 +hf-xet==1.1.5 +httpcore==1.0.9 +httplib2==0.22.0 +httpx==0.28.1 +huggingface-hub==0.33.1 +hypothesis==6.135.19 +idna==3.10 +importlib_metadata==8.7.0 +iniconfig==2.1.0 +jaraco.classes==3.4.0 +jaraco.context==6.0.1 +jaraco.functools==4.2.1 +jeepney==0.9.0 +Jinja2==3.1.6 +joblib==1.5.1 +jsonpickle==3.4.2 +jsonschema==4.24.0 +jsonschema-specifications==2025.4.1 +keras==3.10.0 +keyring==25.6.0 +keyrings.google-artifactregistry-auth==1.1.2 +libclang==18.1.1 +Markdown==3.8.2 +markdown-it-py==3.0.0 +MarkupSafe==3.0.2 +mdurl==0.1.2 +milvus-lite==2.5.1 +ml-dtypes==0.3.2 +mmh3==5.1.0 +mock==5.2.0 +more-itertools==10.7.0 +mpmath==1.3.0 +multidict==6.6.3 +mysql-connector-python==9.3.0 +namex==0.1.0 +networkx==3.5 +nltk==3.9.1 +numpy==1.26.4 +nvidia-cublas-cu12==12.6.4.1 +nvidia-cuda-cupti-cu12==12.6.80 +nvidia-cuda-nvrtc-cu12==12.6.77 +nvidia-cuda-runtime-cu12==12.6.77 +nvidia-cudnn-cu12==9.5.1.17 +nvidia-cufft-cu12==11.3.0.4 +nvidia-cufile-cu12==1.11.1.6 +nvidia-curand-cu12==10.3.7.77 +nvidia-cusolver-cu12==11.7.1.2 +nvidia-cusparse-cu12==12.5.4.2 +nvidia-cusparselt-cu12==0.6.3 +nvidia-nccl-cu12==2.26.2 +nvidia-nvjitlink-cu12==12.6.85 +nvidia-nvtx-cu12==12.6.77 +oauth2client==4.1.3 +objsize==0.7.1 +opentelemetry-api==1.34.1 +opentelemetry-sdk==1.34.1 +opentelemetry-semantic-conventions==0.55b1 +opt_einsum==3.4.0 +optree==0.16.0 +oracledb==3.2.0 +orjson==3.10.18 +overrides==7.7.0 +packaging==25.0 +pandas==2.2.3 +parameterized==0.9.0 +pg8000==1.31.2 +pip==25.1.1 +pluggy==1.6.0 +propcache==0.3.2 +proto-plus==1.26.1 +protobuf==4.25.8 +psycopg2-binary==2.9.10 +pyarrow==18.1.0 +pyarrow-hotfix==0.7 +pyasn1==0.6.1 +pyasn1_modules==0.4.2 +pycparser==2.22 +pydantic==2.11.7 +pydantic_core==2.33.2 +pydot==1.4.2 +Pygments==2.19.2 +PyHamcrest==2.1.0 +PyJWT==2.9.0 +pymilvus==2.5.11 +pymongo==4.13.2 +PyMySQL==1.1.1 +pyparsing==3.2.3 +pyproject_hooks==1.2.0 +pytest==7.4.4 +pytest-timeout==2.4.0 +pytest-xdist==3.7.0 +python-dateutil==2.9.0.post0 +python-dotenv==1.1.1 +python-tds==1.16.1 +pytz==2025.2 +PyYAML==6.0.2 +redis==5.3.0 +referencing==0.36.2 +regex==2024.11.6 +requests==2.32.4 +requests-mock==1.12.1 +rich==14.0.0 +rpds-py==0.25.1 +rsa==4.9.1 +safetensors==0.5.3 +scikit-learn==1.7.0 +scipy==1.16.0 +scramp==1.4.5 +SecretStorage==3.3.3 +setuptools==80.9.0 +shapely==2.1.1 +six==1.17.0 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soupsieve==2.7 +SQLAlchemy==2.0.41 +sqlalchemy_pytds==1.0.2 +sqlparse==0.5.3 +sympy==1.14.0 +tenacity==8.5.0 +tensorboard==2.16.2 +tensorboard-data-server==0.7.2 +tensorflow==2.16.2 +tensorflow-cpu-aws==2.16.2;platform_machine=="aarch64" +tensorflow-io-gcs-filesystem==0.37.1 +termcolor==3.1.0 +testcontainers==4.10.0 +threadpoolctl==3.6.0 +tokenizers==0.21.2 +torch==2.7.1 +tqdm==4.67.1 +transformers==4.48.3 +triton==3.3.1 +typing-inspection==0.4.1 +typing_extensions==4.14.0 +tzdata==2025.2 +ujson==5.10.0 +uritemplate==4.2.0 +urllib3==2.5.0 +virtualenv-clone==0.5.7 +websockets==15.0.1 +Werkzeug==3.1.3 +wheel==0.45.1 +wrapt==1.17.2 +yarl==1.20.1 +zipp==3.23.0 +zstandard==0.23.0 diff --git a/sdks/python/container/py312/base_image_requirements.txt b/sdks/python/container/py312/base_image_requirements.txt index f48d350e01d3..f39f7ab8f7df 100644 --- a/sdks/python/container/py312/base_image_requirements.txt +++ b/sdks/python/container/py312/base_image_requirements.txt @@ -89,7 +89,7 @@ hdfs==2.7.3 httpcore==1.0.9 httplib2==0.22.0 httpx==0.28.1 -hypothesis==6.135.17 +hypothesis==6.135.19 idna==3.10 importlib_metadata==8.7.0 iniconfig==2.1.0 @@ -109,7 +109,7 @@ milvus-lite==2.5.1 mmh3==5.1.0 mock==5.2.0 more-itertools==10.7.0 -multidict==6.6.2 +multidict==6.6.3 mysql-connector-python==9.3.0 nltk==3.9.1 numpy==2.2.6 diff --git a/sdks/python/container/py312/ml_image_requirements.txt b/sdks/python/container/py312/ml_image_requirements.txt new file mode 100644 index 000000000000..e6e9a2930d26 --- /dev/null +++ b/sdks/python/container/py312/ml_image_requirements.txt @@ -0,0 +1,242 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Autogenerated requirements file for Apache Beam py312 container image. +# Run ./gradlew :sdks:python:container:generatePythonRequirementsAll to update. +# Do not edit manually, adjust ../base_image_requirements_manual.txt or +# Apache Beam's setup.py instead, and regenerate the list. +# You will need Python interpreters for all versions supported by Beam, see: +# https://s.apache.org/beam-python-dev-wiki +# Reach out to a committer if you need help. + +absl-py==2.3.0 +aiofiles==24.1.0 +aiohappyeyeballs==2.6.1 +aiohttp==3.12.13 +aiosignal==1.3.2 +annotated-types==0.7.0 +anyio==4.9.0 +asn1crypto==1.5.1 +astunparse==1.6.3 +attrs==25.3.0 +beautifulsoup4==4.13.4 +bs4==0.0.2 +build==1.2.2.post1 +cachetools==5.5.2 +certifi==2025.6.15 +cffi==1.17.1 +charset-normalizer==3.4.2 +click==8.2.1 +cloud-sql-python-connector==1.18.2 +crcmod==1.7 +cryptography==45.0.4 +Cython==3.1.2 +dill==0.3.1.1 +dnspython==2.7.0 +docker==7.1.0 +docopt==0.6.2 +docstring_parser==0.16 +execnet==2.1.1 +fastavro==1.11.1 +fasteners==0.19 +filelock==3.18.0 +flatbuffers==25.2.10 +freezegun==1.5.2 +frozenlist==1.7.0 +fsspec==2025.5.1 +future==1.0.0 +gast==0.6.0 +google-api-core==2.25.1 +google-api-python-client==2.174.0 +google-apitools==0.5.31 +google-auth==2.40.3 +google-auth-httplib2==0.2.0 +google-cloud-aiplatform==1.100.0 +google-cloud-bigquery==3.34.0 +google-cloud-bigquery-storage==2.32.0 +google-cloud-bigtable==2.31.0 +google-cloud-core==2.4.3 +google-cloud-datastore==2.21.0 +google-cloud-dlp==3.31.0 +google-cloud-language==2.17.2 +google-cloud-profiler==4.1.0 +google-cloud-pubsub==2.30.0 +google-cloud-pubsublite==1.12.0 +google-cloud-recommendations-ai==0.10.18 +google-cloud-resource-manager==1.14.2 +google-cloud-spanner==3.55.0 +google-cloud-storage==2.19.0 +google-cloud-videointelligence==2.16.2 +google-cloud-vision==3.10.2 +google-crc32c==1.7.1 +google-genai==1.23.0 +google-pasta==0.2.0 +google-resumable-media==2.7.2 +googleapis-common-protos==1.70.0 +greenlet==3.2.3 +grpc-google-iam-v1==0.14.2 +grpc-interceptor==0.15.4 +grpcio==1.65.5 +grpcio-status==1.63.0rc1 +guppy3==3.1.5 +h11==0.16.0 +h5py==3.14.0 +hdfs==2.7.3 +hf-xet==1.1.5 +httpcore==1.0.9 +httplib2==0.22.0 +httpx==0.28.1 +huggingface-hub==0.33.1 +hypothesis==6.135.20 +idna==3.10 +importlib_metadata==8.7.0 +iniconfig==2.1.0 +jaraco.classes==3.4.0 +jaraco.context==6.0.1 +jaraco.functools==4.2.1 +jeepney==0.9.0 +Jinja2==3.1.6 +joblib==1.5.1 +jsonpickle==3.4.2 +jsonschema==4.24.0 +jsonschema-specifications==2025.4.1 +keras==3.10.0 +keyring==25.6.0 +keyrings.google-artifactregistry-auth==1.1.2 +libclang==18.1.1 +Markdown==3.8.2 +markdown-it-py==3.0.0 +MarkupSafe==3.0.2 +mdurl==0.1.2 +milvus-lite==2.5.1 +ml-dtypes==0.3.2 +mmh3==5.1.0 +mock==5.2.0 +more-itertools==10.7.0 +mpmath==1.3.0 +multidict==6.6.3 +mysql-connector-python==9.3.0 +namex==0.1.0 +networkx==3.5 +nltk==3.9.1 +numpy==1.26.4 +nvidia-cublas-cu12==12.6.4.1 +nvidia-cuda-cupti-cu12==12.6.80 +nvidia-cuda-nvrtc-cu12==12.6.77 +nvidia-cuda-runtime-cu12==12.6.77 +nvidia-cudnn-cu12==9.5.1.17 +nvidia-cufft-cu12==11.3.0.4 +nvidia-cufile-cu12==1.11.1.6 +nvidia-curand-cu12==10.3.7.77 +nvidia-cusolver-cu12==11.7.1.2 +nvidia-cusparse-cu12==12.5.4.2 +nvidia-cusparselt-cu12==0.6.3 +nvidia-nccl-cu12==2.26.2 +nvidia-nvjitlink-cu12==12.6.85 +nvidia-nvtx-cu12==12.6.77 +oauth2client==4.1.3 +objsize==0.7.1 +opentelemetry-api==1.34.1 +opentelemetry-sdk==1.34.1 +opentelemetry-semantic-conventions==0.55b1 +opt_einsum==3.4.0 +optree==0.16.0 +oracledb==3.2.0 +orjson==3.10.18 +overrides==7.7.0 +packaging==25.0 +pandas==2.2.3 +parameterized==0.9.0 +pg8000==1.31.2 +pip==25.1.1 +pluggy==1.6.0 +propcache==0.3.2 +proto-plus==1.26.1 +protobuf==4.25.8 +psycopg2-binary==2.9.10 +pyarrow==18.1.0 +pyarrow-hotfix==0.7 +pyasn1==0.6.1 +pyasn1_modules==0.4.2 +pycparser==2.22 +pydantic==2.11.7 +pydantic_core==2.33.2 +pydot==1.4.2 +Pygments==2.19.2 +PyHamcrest==2.1.0 +PyJWT==2.9.0 +pymilvus==2.5.11 +pymongo==4.13.2 +PyMySQL==1.1.1 +pyparsing==3.2.3 +pyproject_hooks==1.2.0 +pytest==7.4.4 +pytest-timeout==2.4.0 +pytest-xdist==3.7.0 +python-dateutil==2.9.0.post0 +python-dotenv==1.1.1 +python-tds==1.16.1 +pytz==2025.2 +PyYAML==6.0.2 +redis==5.3.0 +referencing==0.36.2 +regex==2024.11.6 +requests==2.32.4 +requests-mock==1.12.1 +rich==14.0.0 +rpds-py==0.25.1 +rsa==4.9.1 +safetensors==0.5.3 +scikit-learn==1.7.0 +scipy==1.16.0 +scramp==1.4.5 +SecretStorage==3.3.3 +setuptools==80.9.0 +shapely==2.1.1 +six==1.17.0 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soupsieve==2.7 +SQLAlchemy==2.0.41 +sqlalchemy_pytds==1.0.2 +sqlparse==0.5.3 +sympy==1.14.0 +tenacity==8.5.0 +tensorboard==2.16.2 +tensorboard-data-server==0.7.2 +tensorflow==2.16.2 +tensorflow-cpu-aws==2.16.2;platform_machine=="aarch64" +termcolor==3.1.0 +testcontainers==4.10.0 +threadpoolctl==3.6.0 +tokenizers==0.21.2 +torch==2.7.1 +tqdm==4.67.1 +transformers==4.48.3 +triton==3.3.1 +typing-inspection==0.4.1 +typing_extensions==4.14.0 +tzdata==2025.2 +ujson==5.10.0 +uritemplate==4.2.0 +urllib3==2.5.0 +virtualenv-clone==0.5.7 +websockets==15.0.1 +Werkzeug==3.1.3 +wheel==0.45.1 +wrapt==1.17.2 +yarl==1.20.1 +zipp==3.23.0 +zstandard==0.23.0 diff --git a/sdks/python/container/py39/base_image_requirements.txt b/sdks/python/container/py39/base_image_requirements.txt index 1c2ebc4c7a4c..db7961186ba3 100644 --- a/sdks/python/container/py39/base_image_requirements.txt +++ b/sdks/python/container/py39/base_image_requirements.txt @@ -92,7 +92,7 @@ hdfs==2.7.3 httpcore==1.0.9 httplib2==0.22.0 httpx==0.28.1 -hypothesis==6.135.17 +hypothesis==6.135.20 idna==3.10 importlib_metadata==8.7.0 iniconfig==2.1.0 @@ -112,7 +112,7 @@ milvus-lite==2.5.1 mmh3==5.1.0 mock==5.2.0 more-itertools==10.7.0 -multidict==6.6.2 +multidict==6.6.3 mysql-connector-python==9.3.0 nltk==3.9.1 numpy==2.0.2 diff --git a/sdks/python/container/py39/ml_image_requirements.txt b/sdks/python/container/py39/ml_image_requirements.txt new file mode 100644 index 000000000000..3dab7e35b6d1 --- /dev/null +++ b/sdks/python/container/py39/ml_image_requirements.txt @@ -0,0 +1,247 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Autogenerated requirements file for Apache Beam py39 container image. +# Run ./gradlew :sdks:python:container:generatePythonRequirementsAll to update. +# Do not edit manually, adjust ../base_image_requirements_manual.txt or +# Apache Beam's setup.py instead, and regenerate the list. +# You will need Python interpreters for all versions supported by Beam, see: +# https://s.apache.org/beam-python-dev-wiki +# Reach out to a committer if you need help. + +absl-py==2.3.0 +aiofiles==24.1.0 +aiohappyeyeballs==2.6.1 +aiohttp==3.12.13 +aiosignal==1.3.2 +annotated-types==0.7.0 +anyio==4.9.0 +asn1crypto==1.5.1 +astunparse==1.6.3 +async-timeout==5.0.1 +attrs==25.3.0 +backports.tarfile==1.2.0 +beautifulsoup4==4.13.4 +bs4==0.0.2 +build==1.2.2.post1 +cachetools==5.5.2 +certifi==2025.6.15 +cffi==1.17.1 +charset-normalizer==3.4.2 +click==8.1.8 +cloud-sql-python-connector==1.18.2 +crcmod==1.7 +cryptography==45.0.4 +Cython==3.1.2 +dill==0.3.1.1 +dnspython==2.7.0 +docker==7.1.0 +docopt==0.6.2 +docstring_parser==0.16 +exceptiongroup==1.3.0 +execnet==2.1.1 +fastavro==1.11.1 +fasteners==0.19 +filelock==3.18.0 +flatbuffers==25.2.10 +freezegun==1.5.2 +frozenlist==1.7.0 +fsspec==2025.5.1 +future==1.0.0 +gast==0.6.0 +google-api-core==2.25.1 +google-api-python-client==2.174.0 +google-apitools==0.5.31 +google-auth==2.40.3 +google-auth-httplib2==0.2.0 +google-cloud-aiplatform==1.100.0 +google-cloud-bigquery==3.34.0 +google-cloud-bigquery-storage==2.32.0 +google-cloud-bigtable==2.31.0 +google-cloud-core==2.4.3 +google-cloud-datastore==2.21.0 +google-cloud-dlp==3.31.0 +google-cloud-language==2.17.2 +google-cloud-profiler==4.1.0 +google-cloud-pubsub==2.30.0 +google-cloud-pubsublite==1.12.0 +google-cloud-recommendations-ai==0.10.18 +google-cloud-resource-manager==1.14.2 +google-cloud-spanner==3.55.0 +google-cloud-storage==2.19.0 +google-cloud-videointelligence==2.16.2 +google-cloud-vision==3.10.2 +google-crc32c==1.7.1 +google-genai==1.23.0 +google-pasta==0.2.0 +google-resumable-media==2.7.2 +googleapis-common-protos==1.70.0 +greenlet==3.2.3 +grpc-google-iam-v1==0.14.2 +grpc-interceptor==0.15.4 +grpcio==1.65.5 +grpcio-status==1.63.0rc1 +guppy3==3.1.5 +h11==0.16.0 +h5py==3.14.0 +hdfs==2.7.3 +hf-xet==1.1.5 +httpcore==1.0.9 +httplib2==0.22.0 +httpx==0.28.1 +huggingface-hub==0.33.1 +hypothesis==6.135.20 +idna==3.10 +importlib_metadata==8.7.0 +iniconfig==2.1.0 +jaraco.classes==3.4.0 +jaraco.context==6.0.1 +jaraco.functools==4.2.1 +jeepney==0.9.0 +Jinja2==3.1.6 +joblib==1.5.1 +jsonpickle==3.4.2 +jsonschema==4.24.0 +jsonschema-specifications==2025.4.1 +keras==3.10.0 +keyring==25.6.0 +keyrings.google-artifactregistry-auth==1.1.2 +libclang==18.1.1 +Markdown==3.8.2 +markdown-it-py==3.0.0 +MarkupSafe==3.0.2 +mdurl==0.1.2 +milvus-lite==2.5.1 +ml-dtypes==0.3.2 +mmh3==5.1.0 +mock==5.2.0 +more-itertools==10.7.0 +mpmath==1.3.0 +multidict==6.6.3 +mysql-connector-python==9.3.0 +namex==0.1.0 +networkx==3.2.1 +nltk==3.9.1 +numpy==1.26.4 +nvidia-cublas-cu12==12.6.4.1 +nvidia-cuda-cupti-cu12==12.6.80 +nvidia-cuda-nvrtc-cu12==12.6.77 +nvidia-cuda-runtime-cu12==12.6.77 +nvidia-cudnn-cu12==9.5.1.17 +nvidia-cufft-cu12==11.3.0.4 +nvidia-cufile-cu12==1.11.1.6 +nvidia-curand-cu12==10.3.7.77 +nvidia-cusolver-cu12==11.7.1.2 +nvidia-cusparse-cu12==12.5.4.2 +nvidia-cusparselt-cu12==0.6.3 +nvidia-nccl-cu12==2.26.2 +nvidia-nvjitlink-cu12==12.6.85 +nvidia-nvtx-cu12==12.6.77 +oauth2client==4.1.3 +objsize==0.7.1 +opentelemetry-api==1.34.1 +opentelemetry-sdk==1.34.1 +opentelemetry-semantic-conventions==0.55b1 +opt_einsum==3.4.0 +optree==0.16.0 +oracledb==3.2.0 +orjson==3.10.18 +overrides==7.7.0 +packaging==25.0 +pandas==2.2.3 +parameterized==0.9.0 +pg8000==1.31.2 +pip==25.1.1 +pluggy==1.6.0 +propcache==0.3.2 +proto-plus==1.26.1 +protobuf==4.25.8 +psycopg2-binary==2.9.9 +pyarrow==18.1.0 +pyarrow-hotfix==0.7 +pyasn1==0.6.1 +pyasn1_modules==0.4.2 +pycparser==2.22 +pydantic==2.11.7 +pydantic_core==2.33.2 +pydot==1.4.2 +Pygments==2.19.2 +PyHamcrest==2.1.0 +PyJWT==2.9.0 +pymilvus==2.5.11 +pymongo==4.13.2 +PyMySQL==1.1.1 +pyparsing==3.2.3 +pyproject_hooks==1.2.0 +pytest==7.4.4 +pytest-timeout==2.4.0 +pytest-xdist==3.7.0 +python-dateutil==2.9.0.post0 +python-dotenv==1.1.1 +python-tds==1.16.1 +pytz==2025.2 +PyYAML==6.0.2 +redis==5.3.0 +referencing==0.36.2 +regex==2024.11.6 +requests==2.32.4 +requests-mock==1.12.1 +rich==14.0.0 +rpds-py==0.25.1 +rsa==4.9.1 +safetensors==0.5.3 +scikit-learn==1.6.1 +scipy==1.13.1 +scramp==1.4.5 +SecretStorage==3.3.3 +setuptools==80.9.0 +shapely==2.0.7 +six==1.17.0 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soupsieve==2.7 +SQLAlchemy==2.0.41 +sqlalchemy_pytds==1.0.2 +sqlparse==0.5.3 +sympy==1.14.0 +tenacity==8.5.0 +tensorboard==2.16.2 +tensorboard-data-server==0.7.2 +tensorflow==2.16.2 +tensorflow-cpu-aws==2.16.2;platform_machine=="aarch64" +tensorflow-io-gcs-filesystem==0.37.1 +termcolor==3.1.0 +testcontainers==4.10.0 +threadpoolctl==3.6.0 +tokenizers==0.21.2 +tomli==2.2.1 +torch==2.7.1 +tqdm==4.67.1 +transformers==4.48.3 +triton==3.3.1 +typing-inspection==0.4.1 +typing_extensions==4.14.0 +tzdata==2025.2 +ujson==5.10.0 +uritemplate==4.2.0 +urllib3==2.5.0 +virtualenv-clone==0.5.7 +websockets==15.0.1 +Werkzeug==3.1.3 +wheel==0.45.1 +wrapt==1.17.2 +yarl==1.20.1 +zipp==3.23.0 +zstandard==0.23.0 diff --git a/sdks/python/container/run_generate_requirements.sh b/sdks/python/container/run_generate_requirements.sh index 23964d10e7b4..02ff9d2ccd6d 100755 --- a/sdks/python/container/run_generate_requirements.sh +++ b/sdks/python/container/run_generate_requirements.sh @@ -38,10 +38,12 @@ fi PY_VERSION=$1 SDK_TARBALL=$2 +REQUIREMENTS_FILE_NAME=$3 +EXTRAS=$4 # Use the PIP_EXTRA_OPTIONS environment variable to pass additional flags to the pip install command. # For example, you can include the --pre flag in $PIP_EXTRA_OPTIONS to download pre-release versions of packages. # Note that you can modify the behavior of the pip install command in this script by passing in your own $PIP_EXTRA_OPTIONS. -PIP_EXTRA_OPTIONS=$3 +PIP_EXTRA_OPTIONS=$5 if ! python"$PY_VERSION" --version > /dev/null 2>&1 ; then echo "Please install a python${PY_VERSION} interpreter. See s.apache.org/beam-python-dev-wiki for Python installation tips." @@ -53,6 +55,14 @@ if ! python"$PY_VERSION" -m venv --help > /dev/null 2>&1 ; then exit 1 fi +if [ -z "$REQUIREMENTS_FILE_NAME" ]; then + REQUIREMENTS_FILE_NAME="base_image_requirements.txt" +fi + +if [ -z "$EXTRAS" ]; then + EXTRAS="[gcp,dataframe,test]" +fi + set -ex ENV_PATH="$PWD/build/python${PY_VERSION/./}_requirements_gen" @@ -65,7 +75,7 @@ pip install --upgrade pip setuptools wheel # Install dataframe deps to add have Dataframe support in released images. # Install test deps since some integration tests need dependencies, # such as pytest, installed in the runner environment. -pip install ${PIP_EXTRA_OPTIONS:+"$PIP_EXTRA_OPTIONS"} --no-cache-dir "$SDK_TARBALL"[gcp,dataframe,test] +pip install ${PIP_EXTRA_OPTIONS:+"$PIP_EXTRA_OPTIONS"} --no-cache-dir "$SDK_TARBALL""$EXTRAS" pip install ${PIP_EXTRA_OPTIONS:+"$PIP_EXTRA_OPTIONS"} --no-cache-dir -r "$PWD"/sdks/python/container/base_image_requirements_manual.txt pip uninstall -y apache-beam @@ -75,7 +85,7 @@ echo "Installed dependencies:" pip freeze --all PY_IMAGE="py${PY_VERSION//.}" -REQUIREMENTS_FILE=$PWD/sdks/python/container/$PY_IMAGE/base_image_requirements.txt +REQUIREMENTS_FILE=$PWD/sdks/python/container/$PY_IMAGE/$REQUIREMENTS_FILE_NAME cat < "$REQUIREMENTS_FILE" # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with diff --git a/sdks/python/setup.py b/sdks/python/setup.py index d309a7ea4a64..fcb64c2d0260 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -571,12 +571,12 @@ def get_portability_package_data(): # in https://github.com/apache/beam/blob/master/sdks/python/tox.ini # For more info, see # https://docs.google.com/document/d/1c84Gc-cZRCfrU8f7kWGsNR2o8oSRjCM-dGHO9KvPWPw/edit?usp=sharing - 'torch': ['torch>=1.9.0,<2.1.0'], - 'tensorflow': ['tensorflow>=2.12rc1,<2.13'], + 'torch': ['torch>=1.9.0,<2.8.0'], + 'tensorflow': ['tensorflow>=2.12rc1,<2.17'], 'transformers': [ 'transformers>=4.28.0,<4.49.0', - 'tensorflow==2.12.0', - 'torch>=1.9.0,<2.1.0' + 'tensorflow>=2.12.0', + 'torch>=1.9.0' ], 'tft': ['tensorflow_transform>=1.14.0,<1.15.0'], 'onnx': [ diff --git a/sdks/python/test-suites/tox/py312/build.gradle b/sdks/python/test-suites/tox/py312/build.gradle index 17478ec43aa8..a8f2ac7fa5cb 100644 --- a/sdks/python/test-suites/tox/py312/build.gradle +++ b/sdks/python/test-suites/tox/py312/build.gradle @@ -23,8 +23,20 @@ plugins { id 'org.apache.beam.module' } applyPythonNature() +def posargs = project.findProperty("posargs") ?: "" + // Required to setup a Python 3 virtualenv and task names. pythonVersion = '3.12' +project.tasks.register("postCommitPyDep") {} + +toxTask "testPy312pytorch-271", "py312-pytorch-271", "${posargs}" +test.dependsOn "testPy312pytorch-271" +postCommitPyDep.dependsOn "testPy312pytorch-271" + +toxTask "testPy312tensorflow-216", "py312-tensorflow-216", "${posargs}" +test.dependsOn "testPy312tensorflow-216" +postCommitPyDep.dependsOn "testPy312tensorflow-216" + apply from: "../common.gradle" diff --git a/sdks/python/test-suites/tox/py39/build.gradle b/sdks/python/test-suites/tox/py39/build.gradle index d6a5e08fcf32..9740f056e685 100644 --- a/sdks/python/test-suites/tox/py39/build.gradle +++ b/sdks/python/test-suites/tox/py39/build.gradle @@ -37,7 +37,7 @@ project.tasks.register("preCommitPyCoverage") { } // Dep Postcommit runs test suites that evaluate compatibility of particular -// dependencies. It is exercised on a single Python version. +// dependencies. Each suite is exercised on at most one python version. // // Should still leave at least one version in PreCommit unless the marked tests // are also exercised by existing PreCommit diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini index 5131769509d9..856ca46f64ae 100644 --- a/sdks/python/tox.ini +++ b/sdks/python/tox.ini @@ -409,6 +409,22 @@ commands = # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pytorch {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' +[testenv:py{311,312}-pytorch-271] +deps = + 200: + torch>=2.7.1,<2.8.0 + mpmath==1.3.0 + numpy==1.26.4 +extras = test,gcp +# Don't set TMPDIR to avoid "AF_UNIX path too long" errors in certain tests. +setenv = +commands = + # Log torch version for debugging + /bin/sh -c "pip freeze | grep -E torch" + # Run all PyTorch>=2 unit tests + # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. + /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pytorch {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' + # TODO(https://github.com/apache/beam/issues/25796) - uncomment onnx tox task in tox/py39/build.gradle once onnx supports protobuf 4.x.x [testenv:py{39,310}-onnx-113] # TODO(https://github.com/apache/beam/issues/25443) @@ -445,6 +461,22 @@ commands = # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_tf {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' +[testenv:py312-tensorflow-216] +deps = + 216: + tensorflow>=2.16.1,<2.17 + # Help pip resolve conflict with typing-extensions for old version of TF https://github.com/apache/beam/issues/30852 + pydantic<2.7 +extras = test,gcp +commands_pre = + pip install -U 'protobuf==4.25.5' +commands = + # Log tensorflow version for debugging + /bin/sh -c "pip freeze | grep -E tensorflow" + # Run all Tensorflow unit tests + # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. + /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_tf {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' + [testenv:py39-xgboost-{160,170}] deps = 160: