[bugFix] sdks/python: properly add Milvus as extra dependency

.github/trigger_files/beam_PostCommit_Python.json

@@ -1,5 +1,5 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run.",
-  "modification": 33
+  "modification": 27
 }
 

.github/workflows/beam_PreCommit_Python_Coverage.yml

@@ -58,35 +58,45 @@ env:
 
 jobs:
   beam_PreCommit_Python_Coverage:
-    name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
-    runs-on: [self-hosted, ubuntu-20.04, highmem]
+    name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) (${{ join(matrix.os, ', ') }})
+    runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
         job_name: [beam_PreCommit_Python_Coverage]
         job_phrase: [Run Python_Coverage PreCommit]
+        python_version: ['3.9']
+        # Run on both self-hosted and GitHub-hosted runners.
+        # Some tests (marked require_docker_in_docker) can't run on Beam's
+        # self-hosted runners due to Docker-in-Docker environment constraint.
+        # These tests will only execute on ubuntu-latest (GitHub-hosted).
+        # Context: https://github.com/apache/beam/pull/35585
+        # Temporary removed the ubuntu-latest env till resolving deps issues.
+        os: [[self-hosted, ubuntu-20.04, highmem]]
     timeout-minutes: 180
     if: |
       github.event_name == 'push' ||
       github.event_name == 'pull_request_target' ||
       (github.event_name == 'schedule' && github.repository == 'apache/beam') ||
       github.event_name == 'workflow_dispatch' ||
-      github.event.comment.body == 'Run Python_Coverage PreCommit'
+      startswith(github.event.comment.body, 'Run Python_Coverage PreCommit 3.')
     steps:
       - uses: actions/checkout@v4
       - name: Setup repository
         uses: ./.github/actions/setup-action
         with:
-          comment_phrase: ${{ matrix.job_phrase }}
+          comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }}
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
+          github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) (${{ join(matrix.os, ', ') }})
       - name: Setup environment
         uses: ./.github/actions/setup-environment-action
         with:
           java-version: default
-          python-version: default
+          python-version: ${{ matrix.python_version }}
       - name: Start DinD
         uses: ./.github/actions/dind-up-action
         id: dind
+        if: contains(matrix.os, 'self-hosted')
         with:
           # Enable all the new features
           cleanup-dind-on-start: "true"
@@ -97,9 +107,9 @@ jobs:
           export-gh-env: "true"
       - name: Run preCommitPyCoverage
         env:
-          DOCKER_HOST: ${{ steps.dind.outputs.docker-host }}
+          DOCKER_HOST: ${{ contains(matrix.os, 'self-hosted') && steps.dind.outputs.docker-host || '' }}
           TOX_TESTENV_PASSENV: "DOCKER_*,TESTCONTAINERS_*,TC_*,BEAM_*,GRPC_*,OMP_*,OPENBLAS_*,PYTHONHASHSEED,PYTEST_*"
-          TESTCONTAINERS_HOST_OVERRIDE: ${{ env.DIND_IP }}
+          TESTCONTAINERS_HOST_OVERRIDE: ${{ contains(matrix.os, 'self-hosted') && env.DIND_IP || '' }}
           TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE: "/var/run/docker.sock"
           TESTCONTAINERS_RYUK_DISABLED: "false"
           TESTCONTAINERS_RYUK_CONTAINER_PRIVILEGED: "true"          
@@ -110,6 +120,12 @@ jobs:
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:
           gradle-command: :sdks:python:test-suites:tox:py39:preCommitPyCoverage
+          arguments: |
+            -Pposargs="${{
+              contains(matrix.os, 'self-hosted') &&
+                '-m (not require_docker_in_docker)' ||
+                '-m require_docker_in_docker'
+            }}"
       - uses: codecov/codecov-action@v3
         with:
           flags: python
@@ -118,7 +134,7 @@ jobs:
         uses: actions/upload-artifact@v4
         if: failure()
         with:
-          name: Python Test Results
+          name: Python ${{ matrix.python_version }} Test Results (${{ join(matrix.os, ', ') }})
           path: '**/pytest*.xml'
       - name: Publish Python Test Results
         env:
@@ -129,4 +145,5 @@ jobs:
           commit: '${{ env.prsha || env.GITHUB_SHA }}'
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/pytest*.xml'
-          large_files: true
+          large_files: true
+          check_name: "Python ${{ matrix.python_version }} Test Results (${{ join(matrix.os, ', ') }})"

sdks/python/apache_beam/ml/rag/enrichment/milvus_search.py

@@ -27,16 +27,16 @@
 from typing import Union
 
 from google.protobuf.json_format import MessageToDict
-
-from apache_beam.ml.rag.types import Chunk
-from apache_beam.ml.rag.types import Embedding
-from apache_beam.transforms.enrichment import EnrichmentSourceHandler
 from pymilvus import AnnSearchRequest
 from pymilvus import Hit
 from pymilvus import Hits
 from pymilvus import MilvusClient
 from pymilvus import SearchResult
 
+from apache_beam.ml.rag.types import Chunk
+from apache_beam.ml.rag.types import Embedding
+from apache_beam.transforms.enrichment import EnrichmentSourceHandler
+
 
 class SearchStrategy(Enum):
   """Search strategies for information retrieval.

sdks/python/apache_beam/ml/rag/enrichment/milvus_search_it_test.py

@@ -481,7 +481,7 @@ class TestMilvusSearchEnrichment(unittest.TestCase):
   """Tests for search functionality across all search strategies"""
 
   _db: MilvusDBContainerInfo
-  _version = "milvusdb/milvus:v2.3.9"
+  _version = "milvusdb/milvus:v2.5.10"
 
   @classmethod
   def setUpClass(cls):

sdks/python/container/ml/py310/base_image_requirements.txt

@@ -67,8 +67,8 @@ google-api-python-client==2.179.0
 google-apitools==0.5.31
 google-auth==2.40.3
 google-auth-httplib2==0.2.0
-google-cloud-aiplatform==1.109.0
-google-cloud-bigquery==3.35.1
+google-cloud-aiplatform==1.110.0
+google-cloud-bigquery==3.36.0
 google-cloud-bigquery-storage==2.32.0
 google-cloud-bigtable==2.32.0
 google-cloud-core==2.4.3
@@ -85,7 +85,7 @@ google-cloud-storage==2.19.0
 google-cloud-videointelligence==2.16.2
 google-cloud-vision==3.10.2
 google-crc32c==1.7.1
-google-genai==1.30.0
+google-genai==1.31.0
 google-pasta==0.2.0
 google-resumable-media==2.7.2
 googleapis-common-protos==1.70.0
@@ -98,35 +98,33 @@ guppy3==3.1.5
 h11==0.16.0
 h5py==3.14.0
 hdfs==2.7.3
-hf-xet==1.1.7
+hf-xet==1.1.8
 httpcore==1.0.9
 httplib2==0.22.0
 httpx==0.28.1
 huggingface-hub==0.34.4
-hypothesis==6.138.2
+hypothesis==6.138.3
 idna==3.10
 importlib_metadata==8.7.0
 iniconfig==2.1.0
 jaraco.classes==3.4.0
 jaraco.context==6.0.1
-jaraco.functools==4.2.1
+jaraco.functools==4.3.0
 jeepney==0.9.0
 Jinja2==3.1.6
 joblib==1.5.1
 jsonpickle==3.4.2
-jsonschema==4.25.0
+jsonschema==4.25.1
 jsonschema-specifications==2025.4.1
-keras==3.11.2
+keras==3.11.3
 keyring==25.6.0
 keyrings.google-artifactregistry-auth==1.1.2
 libclang==18.1.1
 Markdown==3.8.2
 markdown-it-py==4.0.0
 MarkupSafe==3.0.2
 mdurl==0.1.2
-ujson==5.8.0
 milvus-lite==2.5.1
-pymilvus==2.5.10
 ml-dtypes==0.3.2
 mmh3==5.2.0
 mock==5.2.0
@@ -182,8 +180,9 @@ pydot==1.4.2
 Pygments==2.19.2
 PyHamcrest==2.1.0
 PyJWT==2.10.1
-pymongo==4.14.0
-PyMySQL==1.1.1
+pymilvus==2.5.15
+pymongo==4.14.1
+PyMySQL==1.1.2
 pyparsing==3.2.3
 pyproject_hooks==1.2.0
 pytest==7.4.4
@@ -197,7 +196,7 @@ PyYAML==6.0.2
 redis==5.3.1
 referencing==0.36.2
 regex==2025.7.34
-requests==2.32.4
+requests==2.32.5
 requests-mock==1.12.1
 rich==14.1.0
 rpds-py==0.27.0
@@ -230,11 +229,12 @@ tokenizers==0.21.4
 tomli==2.2.1
 torch==2.7.1
 tqdm==4.67.1
-transformers==4.55.2
+transformers==4.55.4
 triton==3.3.1
 typing-inspection==0.4.1
 typing_extensions==4.14.1
 tzdata==2025.2
+ujson==5.11.0
 uritemplate==4.2.0
 urllib3==2.5.0
 virtualenv-clone==0.5.7

sdks/python/container/ml/py311/base_image_requirements.txt

@@ -65,8 +65,8 @@ google-api-python-client==2.179.0
 google-apitools==0.5.31
 google-auth==2.40.3
 google-auth-httplib2==0.2.0
-google-cloud-aiplatform==1.109.0
-google-cloud-bigquery==3.35.1
+google-cloud-aiplatform==1.110.0
+google-cloud-bigquery==3.36.0
 google-cloud-bigquery-storage==2.32.0
 google-cloud-bigtable==2.32.0
 google-cloud-core==2.4.3
@@ -83,7 +83,7 @@ google-cloud-storage==2.19.0
 google-cloud-videointelligence==2.16.2
 google-cloud-vision==3.10.2
 google-crc32c==1.7.1
-google-genai==1.30.0
+google-genai==1.31.0
 google-pasta==0.2.0
 google-resumable-media==2.7.2
 googleapis-common-protos==1.70.0
@@ -96,35 +96,33 @@ guppy3==3.1.5
 h11==0.16.0
 h5py==3.14.0
 hdfs==2.7.3
-hf-xet==1.1.7
+hf-xet==1.1.8
 httpcore==1.0.9
 httplib2==0.22.0
 httpx==0.28.1
 huggingface-hub==0.34.4
-hypothesis==6.138.2
+hypothesis==6.138.3
 idna==3.10
 importlib_metadata==8.7.0
 iniconfig==2.1.0
 jaraco.classes==3.4.0
 jaraco.context==6.0.1
-jaraco.functools==4.2.1
+jaraco.functools==4.3.0
 jeepney==0.9.0
 Jinja2==3.1.6
 joblib==1.5.1
 jsonpickle==3.4.2
-jsonschema==4.25.0
+jsonschema==4.25.1
 jsonschema-specifications==2025.4.1
-keras==3.11.2
+keras==3.11.3
 keyring==25.6.0
 keyrings.google-artifactregistry-auth==1.1.2
 libclang==18.1.1
 Markdown==3.8.2
 markdown-it-py==4.0.0
 MarkupSafe==3.0.2
 mdurl==0.1.2
-ujson==5.8.0
 milvus-lite==2.5.1
-pymilvus==2.5.10
 ml-dtypes==0.3.2
 mmh3==5.2.0
 mock==5.2.0
@@ -180,8 +178,9 @@ pydot==1.4.2
 Pygments==2.19.2
 PyHamcrest==2.1.0
 PyJWT==2.10.1
-pymongo==4.14.0
-PyMySQL==1.1.1
+pymilvus==2.5.15
+pymongo==4.14.1
+PyMySQL==1.1.2
 pyparsing==3.2.3
 pyproject_hooks==1.2.0
 pytest==7.4.4
@@ -195,7 +194,7 @@ PyYAML==6.0.2
 redis==5.3.1
 referencing==0.36.2
 regex==2025.7.34
-requests==2.32.4
+requests==2.32.5
 requests-mock==1.12.1
 rich==14.1.0
 rpds-py==0.27.0
@@ -227,11 +226,12 @@ threadpoolctl==3.6.0
 tokenizers==0.21.4
 torch==2.7.1
 tqdm==4.67.1
-transformers==4.55.2
+transformers==4.55.4
 triton==3.3.1
 typing-inspection==0.4.1
 typing_extensions==4.14.1
 tzdata==2025.2
+ujson==5.11.0
 uritemplate==4.2.0
 urllib3==2.5.0
 virtualenv-clone==0.5.7