From a71dc4f0c8105e14036bcafb0026ef529430574f Mon Sep 17 00:00:00 2001 From: Mohamed Awnallah Date: Tue, 15 Jul 2025 17:49:59 +0000 Subject: [PATCH] workflows: run ML tests on `ubuntu-latest` In this commit, we run ML tests requiring docker-in-docker environment on `ubuntu-latest`. The motivation behind this is Milvus itests was requiring docker-in-docker environment due to setup issues on beam CI self-hosted environment. --- .github/workflows/beam_PostCommit_Python.yml | 13 ++++++++++++- .github/workflows/beam_PreCommit_Python_ML.yml | 14 ++++++++++++-- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/.github/workflows/beam_PostCommit_Python.yml b/.github/workflows/beam_PostCommit_Python.yml index 31ca2ac2aac3..81a0c64080ca 100644 --- a/.github/workflows/beam_PostCommit_Python.yml +++ b/.github/workflows/beam_PostCommit_Python.yml @@ -54,7 +54,7 @@ env: jobs: beam_PostCommit_Python: name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) - runs-on: [self-hosted, ubuntu-20.04, highmem22] + runs-on: ${{ matrix.os }} timeout-minutes: 240 strategy: fail-fast: false @@ -62,6 +62,12 @@ jobs: job_name: [beam_PostCommit_Python] job_phrase: [Run Python PostCommit] python_version: ['3.9', '3.10', '3.11', '3.12'] + # Run on both self-hosted and GitHub-hosted runners. + # Some tests (marked require_docker_in_docker) can't run on Beam's + # self-hosted runners due to Docker-in-Docker environment constraint. + # These tests will only execute on ubuntu-latest (GitHub-hosted). + # Context: https://github.com/apache/beam/pull/35585 + os: [[self-hosted, ubuntu-20.04, highmem22], [ubuntu-latest]] if: | github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request_target' || @@ -99,6 +105,11 @@ jobs: arguments: | -Pjava21Home=$JAVA_HOME_21_X64 \ -PuseWheelDistribution \ + -Pposargs="${{ + contains(matrix.os, 'self-hosted') && + '-m ''not require_docker_in_docker''' || + '-m ''require_docker_in_docker''' + }}" \ -PpythonVersion=${{ matrix.python_version }} \ env: CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} diff --git a/.github/workflows/beam_PreCommit_Python_ML.yml b/.github/workflows/beam_PreCommit_Python_ML.yml index 3eefa3c1376f..4be0ea1f0e7d 100644 --- a/.github/workflows/beam_PreCommit_Python_ML.yml +++ b/.github/workflows/beam_PreCommit_Python_ML.yml @@ -58,7 +58,7 @@ env: jobs: beam_PreCommit_Python_ML: name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) - runs-on: [self-hosted, ubuntu-20.04, main] + runs-on: ${{ matrix.os }} timeout-minutes: 180 strategy: fail-fast: false @@ -66,6 +66,12 @@ jobs: job_name: ['beam_PreCommit_Python_ML'] job_phrase: ['Run Python_ML PreCommit'] python_version: ['3.9','3.10','3.11','3.12'] + # Run on both self-hosted and GitHub-hosted runners. + # Some tests (marked require_docker_in_docker) can't run on Beam's + # self-hosted runners due to Docker-in-Docker environment constraint. + # These tests will only execute on ubuntu-latest (GitHub-hosted). + # Context: https://github.com/apache/beam/pull/35585 + os: [[self-hosted, ubuntu-20.04, main], [ubuntu-latest]] if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || @@ -96,7 +102,11 @@ jobs: with: gradle-command: :sdks:python:test-suites:tox:py${{steps.set_py_ver_clean.outputs.py_ver_clean}}:testPy${{steps.set_py_ver_clean.outputs.py_ver_clean}}ML arguments: | - -Pposargs=apache_beam/ml/ \ + -Pposargs="${{ + contains(matrix.os, 'self-hosted') && + 'apache_beam/ml/ -m ''not require_docker_in_docker''' || + 'apache_beam/ml/ -m ''require_docker_in_docker''' + }}" \ -PpythonVersion=${{ matrix.python_version }} - name: Archive Python Test Results uses: actions/upload-artifact@v4