From f773f440be962a47d2846e2c6ede3e8dd90bbdae Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Wed, 29 Oct 2025 12:05:44 -0400 Subject: [PATCH 01/29] Remove Python 3.9 support from Apache Beam --- .../test-properties.json | 16 +- .../setup-environment-action/action.yml | 8 +- .../beam_CloudML_Benchmarks_Dataflow.yml | 4 +- ..._LoadTests_Python_CoGBK_Dataflow_Batch.yml | 8 +- ...dTests_Python_CoGBK_Dataflow_Streaming.yml | 8 +- ...eam_LoadTests_Python_CoGBK_Flink_Batch.yml | 6 +- ...oadTests_Python_Combine_Dataflow_Batch.yml | 6 +- ...ests_Python_Combine_Dataflow_Streaming.yml | 6 +- ...m_LoadTests_Python_Combine_Flink_Batch.yml | 6 +- ...adTests_Python_Combine_Flink_Streaming.yml | 4 +- ...ests_Python_FnApiRunner_Microbenchmark.yml | 2 +- ...am_LoadTests_Python_GBK_Dataflow_Batch.yml | 10 +- ...oadTests_Python_GBK_Dataflow_Streaming.yml | 2 +- ...ts_Python_GBK_reiterate_Dataflow_Batch.yml | 4 +- ...ython_GBK_reiterate_Dataflow_Streaming.yml | 4 +- ..._LoadTests_Python_ParDo_Dataflow_Batch.yml | 8 +- ...dTests_Python_ParDo_Dataflow_Streaming.yml | 8 +- ...eam_LoadTests_Python_ParDo_Flink_Batch.yml | 6 +- ...LoadTests_Python_ParDo_Flink_Streaming.yml | 10 +- ...dTests_Python_SideInput_Dataflow_Batch.yml | 20 +- .../workflows/beam_LoadTests_Python_Smoke.yml | 4 +- ...erformanceTests_BiqQueryIO_Read_Python.yml | 2 +- ...nceTests_BiqQueryIO_Write_Python_Batch.yml | 2 +- ...manceTests_PubsubIOIT_Python_Streaming.yml | 2 +- ...ormanceTests_SpannerIO_Read_2GB_Python.yml | 2 +- ...Tests_SpannerIO_Write_2GB_Python_Batch.yml | 2 +- .../beam_PerformanceTests_TextIOIT_Python.yml | 2 +- ...rmanceTests_WordCountIT_PythonVersions.yml | 2 +- ..._PerformanceTests_xlang_KafkaIO_Python.yml | 2 +- .../workflows/beam_Playground_Precommit.yml | 2 +- .../beam_PostCommit_PortableJar_Flink.yml | 2 +- .../beam_PostCommit_PortableJar_Spark.yml | 2 +- .github/workflows/beam_PostCommit_Python.yml | 2 +- .../workflows/beam_PostCommit_Python_Arm.yml | 2 +- .../beam_PostCommit_Python_Dependency.yml | 2 +- ...beam_PostCommit_Python_Examples_Direct.yml | 2 +- .../beam_PostCommit_Python_Examples_Flink.yml | 2 +- .../beam_PostCommit_Python_Examples_Spark.yml | 2 +- .../beam_PostCommit_Python_Nexmark_Direct.yml | 2 +- .../beam_PostCommit_Python_Portable_Flink.yml | 4 +- ...mit_Python_ValidatesContainer_Dataflow.yml | 2 +- ...on_ValidatesContainer_Dataflow_With_RC.yml | 2 +- ...Commit_Python_ValidatesRunner_Dataflow.yml | 2 +- ...ostCommit_Python_ValidatesRunner_Flink.yml | 2 +- ...ostCommit_Python_ValidatesRunner_Samza.yml | 2 +- ...ostCommit_Python_ValidatesRunner_Spark.yml | 2 +- ...m_PostCommit_Python_Xlang_Gcp_Dataflow.yml | 2 +- ...eam_PostCommit_Python_Xlang_Gcp_Direct.yml | 2 +- ...am_PostCommit_Python_Xlang_IO_Dataflow.yml | 2 +- ...beam_PostCommit_Python_Xlang_IO_Direct.yml | 2 +- ...eam_PostCommit_TransformService_Direct.yml | 4 +- .../workflows/beam_PostCommit_XVR_Direct.yml | 4 +- .../workflows/beam_PostCommit_XVR_Flink.yml | 4 +- ...ostCommit_XVR_JavaUsingPython_Dataflow.yml | 4 +- ...ostCommit_XVR_PythonUsingJava_Dataflow.yml | 4 +- .../workflows/beam_PostCommit_XVR_Samza.yml | 4 +- .../workflows/beam_PostCommit_XVR_Spark3.yml | 4 +- .../beam_PreCommit_Flink_Container.yml | 2 +- .../beam_PreCommit_Portable_Python.yml | 4 +- .../workflows/beam_PreCommit_Prism_Python.yml | 4 +- .github/workflows/beam_PreCommit_Python.yml | 2 +- .../workflows/beam_PreCommit_PythonDocker.yml | 2 +- .../beam_PreCommit_Python_Coverage.yml | 2 +- .../beam_PreCommit_Python_Dataframes.yml | 2 +- .../beam_PreCommit_Python_Examples.yml | 2 +- .../beam_PreCommit_Python_Integration.yml | 2 +- .../workflows/beam_PreCommit_Python_ML.yml | 6 +- .../beam_PreCommit_Python_Runners.yml | 2 +- .../beam_PreCommit_Python_Transforms.yml | 2 +- ...m_PreCommit_Xlang_Generated_Transforms.yml | 2 +- .../beam_Publish_Beam_SDK_Snapshots.yml | 3 - ...Python_ValidatesContainer_Dataflow_ARM.yml | 2 +- .github/workflows/build_release_candidate.yml | 12 +- .github/workflows/build_wheels.yml | 8 +- .github/workflows/dask_runner_tests.yml | 3 +- ...n_CoGBK_Flink_Batch_100b_Multiple_Keys.txt | 2 +- ...thon_CoGBK_Flink_Batch_100b_Single_Key.txt | 2 +- .../python_CoGBK_Flink_Batch_10kB.txt | 2 +- ...ombine_Flink_Batch_2GB_10_byte_records.txt | 2 +- ...ython_Combine_Flink_Batch_2GB_Fanout_4.txt | 2 +- ...ython_Combine_Flink_Batch_2GB_Fanout_8.txt | 2 +- ...ne_Flink_Streaming_2GB_10_byte_records.txt | 2 +- ...n_Combine_Flink_Streaming_2GB_Fanout_4.txt | 2 +- ...n_Combine_Flink_Streaming_2GB_Fanout_8.txt | 2 +- ...Combine_Flink_Streaming_small_Fanout_1.txt | 2 +- ...Combine_Flink_Streaming_small_Fanout_2.txt | 2 +- ...on_GBK_Flink_Batch_2GB_of_100B_records.txt | 2 +- ...hon_GBK_Flink_Batch_2GB_of_10B_records.txt | 2 +- ...4_times_with_2GB_10-byte_records_total.txt | 2 +- ...8_times_with_2GB_10-byte_records_total.txt | 2 +- ...nk_Batch_reiterate_4_times_10kB_values.txt | 2 +- .../python_ParDo_Flink_Batch_10_Counters.txt | 2 +- ...python_ParDo_Flink_Batch_10_Iterations.txt | 2 +- ...ython_ParDo_Flink_Batch_200_Iterations.txt | 2 +- ...hon_ParDo_Flink_Streaming_100_Counters.txt | 2 +- ...thon_ParDo_Flink_Streaming_10_Counters.txt | 2 +- ...on_ParDo_Flink_Streaming_10_Iterations.txt | 2 +- ...n_ParDo_Flink_Streaming_200_Iterations.txt | 2 +- ...hon_ParDo_Flink_Streaming_5_Iterations.txt | 2 +- .github/workflows/python_dependency_tests.yml | 1 - .github/workflows/python_tests.yml | 3 +- .../republish_released_docker_containers.yml | 4 +- .github/workflows/run_perf_alert_tool.yml | 2 +- ...run_rc_validation_python_mobile_gaming.yml | 2 +- .github/workflows/typescript_tests.yml | 4 +- .../workflows/update_python_dependencies.yml | 1 - .../dockerized-jenkins/seedjobconfig.xml | 2 +- .../jenkins/PythonTestProperties.groovy | 12 +- .test-infra/jenkins/metrics_report/tox.ini | 6 +- .test-infra/metrics/influxdb/Dockerfile | 2 +- .../metrics/influxdb/gsutil/Dockerfile | 2 +- .test-infra/metrics/sync/github/Dockerfile | 2 +- .test-infra/metrics/sync/jenkins/Dockerfile | 2 +- .test-infra/mock-apis/poetry.lock | 6 +- .test-infra/mock-apis/pyproject.toml | 2 +- .test-infra/tools/python_installer.sh | 2 +- build.gradle.kts | 29 +-- .../beam/gradle/BeamModulePlugin.groovy | 4 +- contributor-docs/python-tips.md | 18 +- contributor-docs/release-guide.md | 2 +- dev-support/docker/pkglist | 7 +- gradle.properties | 2 +- local-env-setup.sh | 4 +- .../cloudbuild/playground_cd_examples.sh | 8 +- .../cloudbuild/playground_ci_examples.sh | 8 +- release/src/main/Dockerfile | 3 +- .../python_release_automation.sh | 2 +- sdks/python/apache_beam/__init__.py | 2 +- .../runners/dataflow/internal/apiclient.py | 2 +- .../dataflow/internal/apiclient_test.py | 4 +- .../expansion-service-container/Dockerfile | 4 +- sdks/python/mypy.ini | 2 +- sdks/python/setup.py | 3 +- sdks/python/test-suites/gradle.properties | 26 +-- .../python/test-suites/tox/py310/build.gradle | 206 ++++++++++++++++++ sdks/python/tox.ini | 44 ++-- 136 files changed, 483 insertions(+), 300 deletions(-) diff --git a/.github/actions/setup-default-test-properties/test-properties.json b/.github/actions/setup-default-test-properties/test-properties.json index e877418a20c4..77d8d0d311f0 100644 --- a/.github/actions/setup-default-test-properties/test-properties.json +++ b/.github/actions/setup-default-test-properties/test-properties.json @@ -1,15 +1,15 @@ { "PythonTestProperties": { - "ALL_SUPPORTED_VERSIONS": ["3.9", "3.10", "3.11", "3.12", "3.13"], - "LOWEST_SUPPORTED": ["3.9"], + "ALL_SUPPORTED_VERSIONS": ["3.10", "3.11", "3.12", "3.13"], + "LOWEST_SUPPORTED": ["3.10"], "HIGHEST_SUPPORTED": ["3.13"], - "ESSENTIAL_VERSIONS": ["3.9", "3.12"], - "CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS": ["3.9", "3.12", "3.13"], + "ESSENTIAL_VERSIONS": ["3.10", "3.13"], + "CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS": ["3.10", "3.12", "3.13"], "CROSS_LANGUAGE_VALIDATES_RUNNER_DATAFLOW_USING_SQL_PYTHON_VERSIONS": ["3.11"], - "VALIDATES_CONTAINER_DATAFLOW_PYTHON_VERSIONS": ["3.9", "3.10", "3.11", "3.12", "3.13"], - "LOAD_TEST_PYTHON_VERSION": "3.9", - "CHICAGO_TAXI_EXAMPLE_FLINK_PYTHON_VERSION": "3.9", - "DEFAULT_INTERPRETER": "python3.9", + "VALIDATES_CONTAINER_DATAFLOW_PYTHON_VERSIONS": ["3.10", "3.11", "3.12", "3.13"], + "LOAD_TEST_PYTHON_VERSION": "3.10", + "CHICAGO_TAXI_EXAMPLE_FLINK_PYTHON_VERSION": "3.10", + "DEFAULT_INTERPRETER": "python3.10", "TOX_ENV": ["Cloud", "Cython"] }, "JavaTestProperties": { diff --git a/.github/actions/setup-environment-action/action.yml b/.github/actions/setup-environment-action/action.yml index cddcd4f50443..95b7f288cc69 100644 --- a/.github/actions/setup-environment-action/action.yml +++ b/.github/actions/setup-environment-action/action.yml @@ -50,7 +50,7 @@ runs: if: ${{ inputs.python-version != '' }} uses: actions/setup-python@v5 with: - python-version: ${{ inputs.python-version == 'default' && '3.9' || inputs.python-version }} + python-version: ${{ inputs.python-version == 'default' && '3.10' || inputs.python-version }} cache: ${{ inputs.python-cache && 'pip' || 'none' }} cache-dependency-path: | sdks/python/setup.py @@ -64,10 +64,10 @@ runs: sdks/python/target/.tox !sdks/python/target/.tox/**/log !sdks/python/target/.tox/.package_cache - key: tox-${{ runner.os }}-py${{ inputs.python-version == 'default' && '39' || inputs.python-version }}-${{ hashFiles('sdks/python/tox.ini') }}-${{ hashFiles('sdks/python/setup.py') }} + key: tox-${{ runner.os }}-py${{ inputs.python-version == 'default' && '310' || inputs.python-version }}-${{ hashFiles('sdks/python/tox.ini') }}-${{ hashFiles('sdks/python/setup.py') }} restore-keys: | - tox-${{ runner.os }}-py${{ inputs.python-version == 'default' && '39' || inputs.python-version }}-${{ hashFiles('sdks/python/tox.ini') }}- - tox-${{ runner.os }}-py${{ inputs.python-version == 'default' && '39' || inputs.python-version }}- + tox-${{ runner.os }}-py${{ inputs.python-version == 'default' && '310' || inputs.python-version }}-${{ hashFiles('sdks/python/tox.ini') }}- + tox-${{ runner.os }}-py${{ inputs.python-version == 'default' && '310' || inputs.python-version }}- - name: Install Java if: ${{ inputs.java-version != '' }} diff --git a/.github/workflows/beam_CloudML_Benchmarks_Dataflow.yml b/.github/workflows/beam_CloudML_Benchmarks_Dataflow.yml index 957553bd3168..4d22b935b2b8 100644 --- a/.github/workflows/beam_CloudML_Benchmarks_Dataflow.yml +++ b/.github/workflows/beam_CloudML_Benchmarks_Dataflow.yml @@ -74,8 +74,8 @@ jobs: uses: ./.github/actions/setup-environment-action with: python-version: | - 3.9 3.10 + 3.11 - name: Prepare test arguments uses: ./.github/actions/test-arguments-action with: @@ -89,6 +89,6 @@ jobs: with: gradle-command: :sdks:python:test-suites:dataflow:tftTests arguments: | - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ -Prunner=DataflowRunner \ '-Popts=${{ env.beam_CloudML_Benchmarks_Dataflow_test_arguments_1 }}' diff --git a/.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Batch.yml index 0cc20160fcb2..43ba58bea40b 100644 --- a/.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Batch.yml +++ b/.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Batch.yml @@ -96,7 +96,7 @@ jobs: --info \ -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Batch_test_arguments_1 }} --job_name=load-tests-python-dataflow-batch-cogbk-1-${{ steps.datetime.outputs.datetime }}' \ - name: run CoGBK 2GB of 100B records with multiple keys uses: ./.github/actions/gradle-command-self-hosted-action @@ -105,7 +105,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Batch_test_arguments_2 }} --job_name=load-tests-python-dataflow-batch-cogbk-2-${{ steps.datetime.outputs.datetime }}' \ - name: run CoGBK reiterate 4 times 10kB values uses: ./.github/actions/gradle-command-self-hosted-action @@ -114,7 +114,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Batch_test_arguments_3 }} --job_name=load-tests-python-dataflow-batch-cogbk-3-${{ steps.datetime.outputs.datetime }}' \ - name: run CoGBK reiterate 4 times 2MB values uses: ./.github/actions/gradle-command-self-hosted-action @@ -123,5 +123,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Batch_test_arguments_4 }} --job_name=load-tests-python-dataflow-batch-cogbk-4-${{ steps.datetime.outputs.datetime }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Streaming.yml b/.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Streaming.yml index 2cc53def9021..efd69d3bd213 100644 --- a/.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Streaming.yml +++ b/.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Streaming.yml @@ -95,7 +95,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Streaming_test_arguments_1 }} --job_name=load-tests-python-dataflow-streaming-cogbk-1-${{ steps.datetime.outputs.datetime }}' \ - name: run CoGBK 2GB of 100B records with multiple keys uses: ./.github/actions/gradle-command-self-hosted-action @@ -104,7 +104,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Streaming_test_arguments_2 }} --job_name=load-tests-python-dataflow-streaming-cogbk-2-${{ steps.datetime.outputs.datetime }}' \ - name: run CoGBK reiterate 4 times 10kB values uses: ./.github/actions/gradle-command-self-hosted-action @@ -113,7 +113,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Streaming_test_arguments_3 }} --job_name=load-tests-python-dataflow-streaming-cogbk-3-${{ steps.datetime.outputs.datetime }}' \ - name: run CoGBK reiterate 4 times 2MB values uses: ./.github/actions/gradle-command-self-hosted-action @@ -122,5 +122,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Streaming_test_arguments_4 }} --job_name=load-tests-python-dataflow-streaming-cogbk-4-${{ steps.datetime.outputs.datetime }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_CoGBK_Flink_Batch.yml b/.github/workflows/beam_LoadTests_Python_CoGBK_Flink_Batch.yml index c40dd5678264..f7a686dacf12 100644 --- a/.github/workflows/beam_LoadTests_Python_CoGBK_Flink_Batch.yml +++ b/.github/workflows/beam_LoadTests_Python_CoGBK_Flink_Batch.yml @@ -108,7 +108,7 @@ jobs: gradle-command: :sdks:python:apache_beam:testing:load_tests:run arguments: | --info \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ -Prunner=PortableRunner \ '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Flink_Batch_test_arguments_1 }} --job_name=load-tests-python-flink-batch-cogbk-1-${{ steps.datetime.outputs.datetime }}' \ @@ -118,7 +118,7 @@ jobs: gradle-command: :sdks:python:apache_beam:testing:load_tests:run arguments: | --info \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ -Prunner=PortableRunner \ '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Flink_Batch_test_arguments_2 }} --job_name=load-tests-python-flink-batch-cogbk-2-${{ steps.datetime.outputs.datetime }}' \ @@ -128,7 +128,7 @@ jobs: gradle-command: :sdks:python:apache_beam:testing:load_tests:run arguments: | --info \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ -Prunner=PortableRunner \ '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Flink_Batch_test_arguments_2 }} --job_name=load-tests-python-flink-batch-cogbk-3-${{ steps.datetime.outputs.datetime }}' \ diff --git a/.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Batch.yml index c20091ffcd74..5d6214ac3793 100644 --- a/.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Batch.yml +++ b/.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Batch.yml @@ -92,7 +92,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Dataflow_Batch_test_arguments_1 }} --job_name=load-tests-python-dataflow-batch-combine-1-${{env.NOW_UTC}}' \ - name: run Combine Dataflow Batch Python Load Test 2 (fanout 4) uses: ./.github/actions/gradle-command-self-hosted-action @@ -101,7 +101,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Dataflow_Batch_test_arguments_2 }} --job_name=load-tests-python-dataflow-batch-combine-2-${{env.NOW_UTC}}' \ - name: run Combine Dataflow Batch Python Load Test 3 (fanout 8) uses: ./.github/actions/gradle-command-self-hosted-action @@ -110,5 +110,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Dataflow_Batch_test_arguments_3 }} --job_name=load-tests-python-dataflow-batch-combine-3-${{env.NOW_UTC}}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Streaming.yml b/.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Streaming.yml index 9a8feaa50efe..2a3f14d801e4 100644 --- a/.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Streaming.yml +++ b/.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Streaming.yml @@ -92,7 +92,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Dataflow_Streaming_test_arguments_1 }} --job_name=load-tests-python-dataflow-streaming-combine-1-${{env.NOW_UTC}}' \ - name: run 2GB Fanout 4 test uses: ./.github/actions/gradle-command-self-hosted-action @@ -101,7 +101,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Dataflow_Streaming_test_arguments_2 }} --job_name=load-tests-python-dataflow-streaming-combine-4-${{env.NOW_UTC}}' \ - name: run 2GB Fanout 8 test uses: ./.github/actions/gradle-command-self-hosted-action @@ -110,5 +110,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Dataflow_Streaming_test_arguments_3 }} --job_name=load-tests-python-dataflow-streaming-combine-5-${{env.NOW_UTC}}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_Combine_Flink_Batch.yml b/.github/workflows/beam_LoadTests_Python_Combine_Flink_Batch.yml index f629bc12c7da..038f69dfa29c 100644 --- a/.github/workflows/beam_LoadTests_Python_Combine_Flink_Batch.yml +++ b/.github/workflows/beam_LoadTests_Python_Combine_Flink_Batch.yml @@ -107,7 +107,7 @@ jobs: with: gradle-command: :sdks:python:apache_beam:testing:load_tests:run arguments: | - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ -Prunner=PortableRunner \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Flink_Batch_test_arguments_1 }} --job_name=load-tests-python-flink-batch-combine-1-${{env.NOW_UTC}}' \ @@ -121,7 +121,7 @@ jobs: with: gradle-command: :sdks:python:apache_beam:testing:load_tests:run arguments: | - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ -Prunner=PortableRunner \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Flink_Batch_test_arguments_2 }} --job_name=load-tests-python-flink-batch-combine-4-${{env.NOW_UTC}}' \ @@ -130,7 +130,7 @@ jobs: with: gradle-command: :sdks:python:apache_beam:testing:load_tests:run arguments: | - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ -Prunner=PortableRunner \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Flink_Batch_test_arguments_3 }} --job_name=load-tests-python-flink-batch-combine-5-${{env.NOW_UTC}}' \ diff --git a/.github/workflows/beam_LoadTests_Python_Combine_Flink_Streaming.yml b/.github/workflows/beam_LoadTests_Python_Combine_Flink_Streaming.yml index b630331ae062..767f2eab5bf9 100644 --- a/.github/workflows/beam_LoadTests_Python_Combine_Flink_Streaming.yml +++ b/.github/workflows/beam_LoadTests_Python_Combine_Flink_Streaming.yml @@ -109,7 +109,7 @@ jobs: with: gradle-command: :sdks:python:apache_beam:testing:load_tests:run arguments: | - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ -Prunner=PortableRunner \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Flink_Streaming_test_arguments_1 }} --job_name=load-tests-python-flink-streaming-combine-4-${{env.NOW_UTC}}' \ @@ -118,7 +118,7 @@ jobs: with: gradle-command: :sdks:python:apache_beam:testing:load_tests:run arguments: | - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ -Prunner=PortableRunner \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Flink_Streaming_test_arguments_2 }} --job_name=load-tests-python-flink-streaming-combine-5-${{env.NOW_UTC}}' \ diff --git a/.github/workflows/beam_LoadTests_Python_FnApiRunner_Microbenchmark.yml b/.github/workflows/beam_LoadTests_Python_FnApiRunner_Microbenchmark.yml index c4334039c187..00310c7500e7 100644 --- a/.github/workflows/beam_LoadTests_Python_FnApiRunner_Microbenchmark.yml +++ b/.github/workflows/beam_LoadTests_Python_FnApiRunner_Microbenchmark.yml @@ -87,5 +87,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.microbenchmarks_test \ -Prunner=DirectRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_FnApiRunner_Microbenchmark_test_arguments_1 }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Batch.yml index d1b18b41442f..f2a874c1dc66 100644 --- a/.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Batch.yml +++ b/.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Batch.yml @@ -94,7 +94,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_Dataflow_Batch_test_arguments_1 }} --job_name=load-tests-python-dataflow-batch-gbk-1-${{env.NOW_UTC}}' \ - name: run 2GB of 100B records test uses: ./.github/actions/gradle-command-self-hosted-action @@ -103,7 +103,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_Dataflow_Batch_test_arguments_2 }} --job_name=load-tests-python-dataflow-batch-gbk-2-${{env.NOW_UTC}}' \ - name: run 2GB of 100kB records test uses: ./.github/actions/gradle-command-self-hosted-action @@ -112,7 +112,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_Dataflow_Batch_test_arguments_3 }} --job_name=load-tests-python-dataflow-batch-gbk-3-${{env.NOW_UTC}}' \ - name: run fanout 4 times with 2GB 10-byte records test uses: ./.github/actions/gradle-command-self-hosted-action @@ -121,7 +121,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_Dataflow_Batch_test_arguments_4 }} --job_name=load-tests-python-dataflow-batch-gbk-4-${{env.NOW_UTC}}' \ - name: run fanout 8 times with 2GB 10-byte records total test uses: ./.github/actions/gradle-command-self-hosted-action @@ -130,5 +130,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_Dataflow_Batch_test_arguments_5 }} --job_name=load-tests-python-dataflow-batch-gbk-5-${{env.NOW_UTC}}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Streaming.yml b/.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Streaming.yml index 44d73348c0f7..d7323989c6ef 100644 --- a/.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Streaming.yml +++ b/.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Streaming.yml @@ -90,7 +90,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_Dataflow_Streaming_test_arguments_1 }} --job_name=load-tests-python-dataflow-streaming-gbk-3-${{env.NOW_UTC}}' \ # // TODO(https://github.com/apache/beam/issues/20403). Skipping some cases because they are too slow: diff --git a/.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch.yml index 2765f333025c..602a5789e4b7 100644 --- a/.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch.yml +++ b/.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch.yml @@ -91,7 +91,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch_test_arguments_1 }} --job_name=load-tests-python-dataflow-batch-gbk-6-${{env.NOW_UTC}}' \ - name: run reiterate 4 times 2MB values test uses: ./.github/actions/gradle-command-self-hosted-action @@ -100,5 +100,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch_test_arguments_2 }} --job_name=load-tests-python-dataflow-batch-gbk-7-${{env.NOW_UTC}}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Streaming.yml b/.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Streaming.yml index 0397c855a13a..408020e288bd 100644 --- a/.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Streaming.yml +++ b/.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Streaming.yml @@ -91,7 +91,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_reiterate_Dataflow_Streaming_test_arguments_1 }} --job_name=load-tests-python-dataflow-streaming-gbk-6-${{env.NOW_UTC}}' \ - name: run reiterate 4 times 2MB values test uses: ./.github/actions/gradle-command-self-hosted-action @@ -100,5 +100,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_reiterate_Dataflow_Streaming_test_arguments_2 }} --job_name=load-tests-python-dataflow-streaming-gbk-7-${{env.NOW_UTC}}' \ diff --git a/.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Batch.yml index e4a2d7f2d4c0..753e70aad0a4 100644 --- a/.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Batch.yml +++ b/.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Batch.yml @@ -95,7 +95,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Dataflow_Batch_test_arguments_1 }} --job_name=load-tests-python-dataflow-batch-pardo-1-${{ steps.datetime.outputs.datetime }}' \ - name: run ParDo Dataflow Batch Python Load Test 2 (200 iterations) uses: ./.github/actions/gradle-command-self-hosted-action @@ -104,7 +104,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Dataflow_Batch_test_arguments_2 }} --job_name=load-tests-python-dataflow-batch-pardo-2-${{ steps.datetime.outputs.datetime }}' \ - name: run ParDo Dataflow Batch Python Load Test 3 (10 counters) uses: ./.github/actions/gradle-command-self-hosted-action @@ -113,7 +113,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Dataflow_Batch_test_arguments_3 }} --job_name=load-tests-python-dataflow-batch-pardo-3-${{ steps.datetime.outputs.datetime }}' \ - name: run ParDo Dataflow Batch Python Load Test 4 (100 counters) uses: ./.github/actions/gradle-command-self-hosted-action @@ -122,5 +122,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Dataflow_Batch_test_arguments_4 }} --job_name=load-tests-python-dataflow-batch-pardo-4-${{ steps.datetime.outputs.datetime }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Streaming.yml b/.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Streaming.yml index 42e9edf109a0..6cccda948f81 100644 --- a/.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Streaming.yml +++ b/.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Streaming.yml @@ -95,7 +95,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Dataflow_Streaming_test_arguments_1 }} --job_name=load-tests-python-dataflow-streaming-pardo-1-${{ steps.datetime.outputs.datetime }}' \ - name: run ParDo Dataflow Streaming Python Load Test 2 (200 iterations) uses: ./.github/actions/gradle-command-self-hosted-action @@ -104,7 +104,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Dataflow_Streaming_test_arguments_2 }} --job_name=load-tests-python-dataflow-streaming-pardo-2-${{ steps.datetime.outputs.datetime }}' \ - name: run ParDo Dataflow Streaming Python Load Test 3 (10 counters) uses: ./.github/actions/gradle-command-self-hosted-action @@ -113,7 +113,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Dataflow_Streaming_test_arguments_3 }} --job_name=load-tests-python-dataflow-streaming-pardo-3-${{ steps.datetime.outputs.datetime }}' \ - name: run ParDo Dataflow Streaming Python Load Test 4 (100 counters) uses: ./.github/actions/gradle-command-self-hosted-action @@ -122,5 +122,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Dataflow_Streaming_test_arguments_4 }} --job_name=load-tests-python-dataflow-streaming-pardo-4-${{ steps.datetime.outputs.datetime }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_ParDo_Flink_Batch.yml b/.github/workflows/beam_LoadTests_Python_ParDo_Flink_Batch.yml index 26fcb5593e34..264934a204d9 100644 --- a/.github/workflows/beam_LoadTests_Python_ParDo_Flink_Batch.yml +++ b/.github/workflows/beam_LoadTests_Python_ParDo_Flink_Batch.yml @@ -109,7 +109,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=PortableRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Flink_Batch_test_arguments_1 }} --job_name=load-tests-python-flink-batch-pardo-1-${{ steps.datetime.outputs.datetime }}' \ - name: run ParDo Flink Batch Python Load Test 2 (200 iterations) uses: ./.github/actions/gradle-command-self-hosted-action @@ -118,7 +118,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=PortableRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Flink_Batch_test_arguments_2 }} --job_name=load-tests-python-flink-batch-pardo-3-${{ steps.datetime.outputs.datetime }}' \ - name: run ParDo Flink Batch Python Load Test 3 (10 counters) uses: ./.github/actions/gradle-command-self-hosted-action @@ -127,5 +127,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=PortableRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Flink_Batch_test_arguments_3 }} --job_name=load-tests-python-flink-batch-pardo-4-${{ steps.datetime.outputs.datetime }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_ParDo_Flink_Streaming.yml b/.github/workflows/beam_LoadTests_Python_ParDo_Flink_Streaming.yml index bc2408ec7be6..48d7865cf28b 100644 --- a/.github/workflows/beam_LoadTests_Python_ParDo_Flink_Streaming.yml +++ b/.github/workflows/beam_LoadTests_Python_ParDo_Flink_Streaming.yml @@ -111,7 +111,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=PortableRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Flink_Streaming_test_arguments_1 }} --job_name=load-tests-python-flink-streaming-pardo-1-${{ steps.datetime.outputs.datetime }}' \ - name: run ParDo Flink Streaming Python Load Test 2 (200 iterations) uses: ./.github/actions/gradle-command-self-hosted-action @@ -120,7 +120,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=PortableRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Flink_Streaming_test_arguments_2 }} --job_name=load-tests-python-flink-streaming-pardo-2-${{ steps.datetime.outputs.datetime }}' \ - name: run ParDo Flink Streaming Python Load Test 3 (10 counters) uses: ./.github/actions/gradle-command-self-hosted-action @@ -129,7 +129,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=PortableRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Flink_Streaming_test_arguments_3 }} --job_name=load-tests-python-flink-streaming-pardo-3-${{ steps.datetime.outputs.datetime }}' \ - name: run ParDo Flink Streaming Python Load Test 4 (100 counters) uses: ./.github/actions/gradle-command-self-hosted-action @@ -138,7 +138,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=PortableRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Flink_Streaming_test_arguments_4 }} --job_name=load-tests-python-flink-streaming-pardo-4-${{ steps.datetime.outputs.datetime }}' \ - name: run ParDo Flink Streaming Python Load Test 5 (5 iterations) uses: ./.github/actions/gradle-command-self-hosted-action @@ -147,7 +147,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=PortableRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Flink_Streaming_test_arguments_5 }} --job_name=load-tests-python-flink-streaming-pardo-6-${{ steps.datetime.outputs.datetime }}' \ - name: Teardown Flink if: always() diff --git a/.github/workflows/beam_LoadTests_Python_SideInput_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Python_SideInput_Dataflow_Batch.yml index 52721574da40..625f25625199 100644 --- a/.github/workflows/beam_LoadTests_Python_SideInput_Dataflow_Batch.yml +++ b/.github/workflows/beam_LoadTests_Python_SideInput_Dataflow_Batch.yml @@ -101,7 +101,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_1 }} --job_name=load-tests-python-dataflow-batch-sideinput-1-${{ steps.datetime.outputs.datetime }}' \ - name: run SideInput Dataflow Batch Python Load Test 2 (1gb-1kb-10workers-1window-99key-percent-dict) uses: ./.github/actions/gradle-command-self-hosted-action @@ -110,7 +110,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_2 }} --job_name=load-tests-python-dataflow-batch-sideinput-2-${{ steps.datetime.outputs.datetime }}' \ - name: run SideInput Dataflow Batch Python Load Test 3 (10gb-1kb-10workers-1window-first-iterable) uses: ./.github/actions/gradle-command-self-hosted-action @@ -119,7 +119,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_3 }} --job_name=load-tests-python-dataflow-batch-sideinput-3-${{ steps.datetime.outputs.datetime }}' \ - name: run SideInput Dataflow Batch Python Load Test 4 (10gb-1kb-10workers-1window-iterable) uses: ./.github/actions/gradle-command-self-hosted-action @@ -128,7 +128,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_4 }} --job_name=load-tests-python-dataflow-batch-sideinput-4-${{ steps.datetime.outputs.datetime }}' \ - name: run SideInput Dataflow Batch Python Load Test 5 (1gb-1kb-10workers-1window-first-list) uses: ./.github/actions/gradle-command-self-hosted-action @@ -137,7 +137,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_5 }} --job_name=load-tests-python-dataflow-batch-sideinput-5-${{ steps.datetime.outputs.datetime }}' \ - name: run SideInput Dataflow Batch Python Load Test 6 (1gb-1kb-10workers-1window-list) uses: ./.github/actions/gradle-command-self-hosted-action @@ -146,7 +146,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_6 }} --job_name=load-tests-python-dataflow-batch-sideinput-6-${{ steps.datetime.outputs.datetime }}' \ - name: run SideInput Dataflow Batch Python Load Test 7 (1gb-1kb-10workers-1000window-1key-percent-dict) uses: ./.github/actions/gradle-command-self-hosted-action @@ -155,7 +155,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_7 }} --job_name=load-tests-python-dataflow-batch-sideinput-7-${{ steps.datetime.outputs.datetime }}' \ - name: run SideInput Dataflow Batch Python Load Test 8 (1gb-1kb-10workers-1000window-99key-percent-dict) uses: ./.github/actions/gradle-command-self-hosted-action @@ -164,7 +164,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_8 }} --job_name=load-tests-python-dataflow-batch-sideinput-8-${{ steps.datetime.outputs.datetime }}' \ - name: run SideInput Dataflow Batch Python Load Test 9 (10gb-1kb-10workers-1000window-first-iterable) uses: ./.github/actions/gradle-command-self-hosted-action @@ -173,7 +173,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_9 }} --job_name=load-tests-python-dataflow-batch-sideinput-9-${{ steps.datetime.outputs.datetime }}' \ - name: run SideInput Dataflow Batch Python Load Test 10 (10gb-1kb-10workers-1000window-iterable) uses: ./.github/actions/gradle-command-self-hosted-action @@ -182,5 +182,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_10 }} --job_name=load-tests-python-dataflow-batch-sideinput-10-${{ steps.datetime.outputs.datetime }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_Smoke.yml b/.github/workflows/beam_LoadTests_Python_Smoke.yml index 0483bb70bf10..9ef14eb2ea1e 100644 --- a/.github/workflows/beam_LoadTests_Python_Smoke.yml +++ b/.github/workflows/beam_LoadTests_Python_Smoke.yml @@ -90,7 +90,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ -Prunner=DirectRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Smoke_test_arguments_1 }} --job_name=load-tests-python-direct-batch-gbk-smoke-${{ steps.datetime.outputs.datetime }}' \ - name: run GroupByKey Python load test Dataflow uses: ./.github/actions/gradle-command-self-hosted-action @@ -99,5 +99,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Smoke_test_arguments_2 }} --job_name=load-tests-python-dataflow-batch-gbk-smoke-${{ steps.datetime.outputs.datetime }}' \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_BiqQueryIO_Read_Python.yml b/.github/workflows/beam_PerformanceTests_BiqQueryIO_Read_Python.yml index 0d50ef30f9ab..d7ebbb68dc2c 100644 --- a/.github/workflows/beam_PerformanceTests_BiqQueryIO_Read_Python.yml +++ b/.github/workflows/beam_PerformanceTests_BiqQueryIO_Read_Python.yml @@ -89,6 +89,6 @@ jobs: gradle-command: :sdks:python:apache_beam:testing:load_tests:run arguments: | -PloadTest.mainClass=apache_beam.io.gcp.bigquery_read_perf_test \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ -Prunner=DataflowRunner \ '-PloadTest.args=${{env.beam_PerformanceTests_BiqQueryIO_Read_Python_test_arguments_1}}' \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_BiqQueryIO_Write_Python_Batch.yml b/.github/workflows/beam_PerformanceTests_BiqQueryIO_Write_Python_Batch.yml index 8b0c278185d3..94fca915644a 100644 --- a/.github/workflows/beam_PerformanceTests_BiqQueryIO_Write_Python_Batch.yml +++ b/.github/workflows/beam_PerformanceTests_BiqQueryIO_Write_Python_Batch.yml @@ -89,6 +89,6 @@ jobs: gradle-command: :sdks:python:apache_beam:testing:load_tests:run arguments: | -PloadTest.mainClass=apache_beam.io.gcp.bigquery_write_perf_test \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ -Prunner=DataflowRunner \ '-PloadTest.args=${{env.beam_PerformanceTests_BiqQueryIO_Write_Python_Batch_test_arguments_1}}' \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_PubsubIOIT_Python_Streaming.yml b/.github/workflows/beam_PerformanceTests_PubsubIOIT_Python_Streaming.yml index 6d15bc507940..647125d628f9 100644 --- a/.github/workflows/beam_PerformanceTests_PubsubIOIT_Python_Streaming.yml +++ b/.github/workflows/beam_PerformanceTests_PubsubIOIT_Python_Streaming.yml @@ -90,5 +90,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.io.gcp.pubsub_io_perf_test \ -Prunner=TestDataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_PerformanceTests_PubsubIOIT_Python_Streaming_test_arguments_1 }}' \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_SpannerIO_Read_2GB_Python.yml b/.github/workflows/beam_PerformanceTests_SpannerIO_Read_2GB_Python.yml index 5960bf6ffb9e..d2a9065e6544 100644 --- a/.github/workflows/beam_PerformanceTests_SpannerIO_Read_2GB_Python.yml +++ b/.github/workflows/beam_PerformanceTests_SpannerIO_Read_2GB_Python.yml @@ -90,5 +90,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.io.gcp.experimental.spannerio_read_perf_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ -PloadTest.args='${{env.beam_PerformanceTests_SpannerIO_Read_2GB_Python_test_arguments_1}}' \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch.yml b/.github/workflows/beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch.yml index 5e1e1a7aa3d0..9af7b55a2e18 100644 --- a/.github/workflows/beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch.yml +++ b/.github/workflows/beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch.yml @@ -90,5 +90,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.io.gcp.experimental.spannerio_write_perf_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ -PloadTest.args='${{env.beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch_test_arguments_1}}' \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_TextIOIT_Python.yml b/.github/workflows/beam_PerformanceTests_TextIOIT_Python.yml index 8749ef3591ab..be45a30f4ee9 100644 --- a/.github/workflows/beam_PerformanceTests_TextIOIT_Python.yml +++ b/.github/workflows/beam_PerformanceTests_TextIOIT_Python.yml @@ -88,7 +88,7 @@ jobs: with: gradle-command: :sdks:python:apache_beam:testing:load_tests:run arguments: | - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ -PloadTest.mainClass=apache_beam.io.filebasedio_perf_test \ -Prunner=DataflowRunner \ '-PloadTest.args=${{env.beam_PerformanceTests_TextIOIT_Python_test_arguments_1}}' \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_WordCountIT_PythonVersions.yml b/.github/workflows/beam_PerformanceTests_WordCountIT_PythonVersions.yml index 8087a860d47f..1e46712a945d 100644 --- a/.github/workflows/beam_PerformanceTests_WordCountIT_PythonVersions.yml +++ b/.github/workflows/beam_PerformanceTests_WordCountIT_PythonVersions.yml @@ -64,7 +64,7 @@ jobs: job_name: ["beam_PerformanceTests_WordCountIT_PythonVersions"] job_phrase_1: [Run Python] job_phrase_2: [WordCountIT Performance Test] - python_version: ['3.9'] + python_version: ['3.10'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml b/.github/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml index e31535286b1c..d51589ae9ceb 100644 --- a/.github/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml +++ b/.github/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml @@ -119,5 +119,5 @@ jobs: arguments: | -Prunner=DataflowRunner \ -PloadTest.mainClass=apache_beam.io.external.xlang_kafkaio_perf_test \ - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ '-PloadTest.args=${{ env.beam_PerfTests_xlang_KafkaIO_Python_test_arguments_1 }}' \ No newline at end of file diff --git a/.github/workflows/beam_Playground_Precommit.yml b/.github/workflows/beam_Playground_Precommit.yml index a0fbe7881fe1..b0d34a5c2dbf 100644 --- a/.github/workflows/beam_Playground_Precommit.yml +++ b/.github/workflows/beam_Playground_Precommit.yml @@ -44,7 +44,7 @@ jobs: job_phrase: [Run Playground PreCommit] env: DATASTORE_EMULATOR_VERSION: '423.0.0' - PYTHON_VERSION: '3.9' + PYTHON_VERSION: '3.10' JAVA_VERSION: '11' steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/beam_PostCommit_PortableJar_Flink.yml b/.github/workflows/beam_PostCommit_PortableJar_Flink.yml index 5cb0d5c922bc..e29eedb8cee5 100644 --- a/.github/workflows/beam_PostCommit_PortableJar_Flink.yml +++ b/.github/workflows/beam_PostCommit_PortableJar_Flink.yml @@ -81,7 +81,7 @@ jobs: with: gradle-command: :sdks:python:test-suites:portable:py39:testPipelineJarFlinkRunner arguments: | - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ - name: Archive Python Test Results uses: actions/upload-artifact@v4 if: failure() diff --git a/.github/workflows/beam_PostCommit_PortableJar_Spark.yml b/.github/workflows/beam_PostCommit_PortableJar_Spark.yml index 8fabcde443a1..11fbfa43fde4 100644 --- a/.github/workflows/beam_PostCommit_PortableJar_Spark.yml +++ b/.github/workflows/beam_PostCommit_PortableJar_Spark.yml @@ -81,7 +81,7 @@ jobs: with: gradle-command: :sdks:python:test-suites:portable:py39:testPipelineJarSparkRunner arguments: | - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ - name: Archive Python Test Results uses: actions/upload-artifact@v4 if: failure() diff --git a/.github/workflows/beam_PostCommit_Python.yml b/.github/workflows/beam_PostCommit_Python.yml index 21d93242a131..b13452996126 100644 --- a/.github/workflows/beam_PostCommit_Python.yml +++ b/.github/workflows/beam_PostCommit_Python.yml @@ -61,7 +61,7 @@ jobs: matrix: job_name: ['beam_PostCommit_Python'] job_phrase: ['Run Python PostCommit'] - python_version: ['3.9', '3.10', '3.11', '3.12', '3.13'] + python_version: ['3.10', '3.11', '3.12', '3.13'] os: [[self-hosted, ubuntu-20.04, highmem22]] if: | github.event_name == 'workflow_dispatch' || diff --git a/.github/workflows/beam_PostCommit_Python_Arm.yml b/.github/workflows/beam_PostCommit_Python_Arm.yml index 893507aad823..59124d551a65 100644 --- a/.github/workflows/beam_PostCommit_Python_Arm.yml +++ b/.github/workflows/beam_PostCommit_Python_Arm.yml @@ -60,7 +60,7 @@ jobs: matrix: job_name: [beam_PostCommit_Python_Arm] job_phrase: [Run Python PostCommit Arm] - python_version: ['3.9', '3.13'] + python_version: ['10', '3.13'] if: | github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PostCommit_Python_Dependency.yml b/.github/workflows/beam_PostCommit_Python_Dependency.yml index b92c6d75483b..03e51bbbd95b 100644 --- a/.github/workflows/beam_PostCommit_Python_Dependency.yml +++ b/.github/workflows/beam_PostCommit_Python_Dependency.yml @@ -59,7 +59,7 @@ jobs: matrix: job_name: ['beam_PostCommit_Python_Dependency'] job_phrase: ['Run Python PostCommit Dependency'] - python_version: ['3.9','3.13'] + python_version: ['3.10','3.13'] timeout-minutes: 360 if: | github.event_name == 'workflow_dispatch' || diff --git a/.github/workflows/beam_PostCommit_Python_Examples_Direct.yml b/.github/workflows/beam_PostCommit_Python_Examples_Direct.yml index 8e559d286700..fc4531c705cd 100644 --- a/.github/workflows/beam_PostCommit_Python_Examples_Direct.yml +++ b/.github/workflows/beam_PostCommit_Python_Examples_Direct.yml @@ -63,7 +63,7 @@ jobs: matrix: job_name: ["beam_PostCommit_Python_Examples_Direct"] job_phrase: ["Run Python Examples_Direct"] - python_version: ['3.9','3.10','3.11','3.12', '3.13'] + python_version: ['3.10','3.11','3.12', '3.13'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_Python_Examples_Flink.yml b/.github/workflows/beam_PostCommit_Python_Examples_Flink.yml index 137ba7ef1b18..2be8d99b3dfa 100644 --- a/.github/workflows/beam_PostCommit_Python_Examples_Flink.yml +++ b/.github/workflows/beam_PostCommit_Python_Examples_Flink.yml @@ -63,7 +63,7 @@ jobs: matrix: job_name: ["beam_PostCommit_Python_Examples_Flink"] job_phrase: ["Run Python Examples_Flink"] - python_version: ['3.9', '3.13'] + python_version: ['3.10', '3.13'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_Python_Examples_Spark.yml b/.github/workflows/beam_PostCommit_Python_Examples_Spark.yml index 3e155eca0e3c..bda615c447ee 100644 --- a/.github/workflows/beam_PostCommit_Python_Examples_Spark.yml +++ b/.github/workflows/beam_PostCommit_Python_Examples_Spark.yml @@ -63,7 +63,7 @@ jobs: matrix: job_name: ["beam_PostCommit_Python_Examples_Spark"] job_phrase: ["Run Python Examples_Spark"] - python_version: ['3.9', '3.13'] + python_version: ['3.10', '3.13'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_Python_Nexmark_Direct.yml b/.github/workflows/beam_PostCommit_Python_Nexmark_Direct.yml index f4b95d7a762e..ff88aa78159e 100644 --- a/.github/workflows/beam_PostCommit_Python_Nexmark_Direct.yml +++ b/.github/workflows/beam_PostCommit_Python_Nexmark_Direct.yml @@ -133,7 +133,7 @@ jobs: with: gradle-command: :sdks:python:apache_beam:testing:benchmarks:nexmark:run arguments: | - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ "-Pnexmark.args=${{ env.GRADLE_PYTHON_COMMAND_ARGUMENTS }} \ --query=${{ matrix.query }} \ --input=gs://temp-storage-for-perf-tests/nexmark/eventFiles/beam_PostCommit_Python_Nexmark_Direct/query${{ matrix.query }}-\*" \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Python_Portable_Flink.yml b/.github/workflows/beam_PostCommit_Python_Portable_Flink.yml index 363d4703ef18..35eb4043b56a 100644 --- a/.github/workflows/beam_PostCommit_Python_Portable_Flink.yml +++ b/.github/workflows/beam_PostCommit_Python_Portable_Flink.yml @@ -77,7 +77,7 @@ jobs: uses: ./.github/actions/setup-environment-action with: java-version: default - python-version: '3.9' + python-version: '3.10' - name: Run flinkCompatibilityMatrix${{ matrix.environment_type }} script env: CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} @@ -85,7 +85,7 @@ jobs: with: gradle-command: :sdks:python:test-suites:portable:py39:flinkCompatibilityMatrix${{ matrix.environment_type }} arguments: | - -PpythonVersion=3.9 \ + -PpythonVersion=3.10 \ - name: Archive Python Test Results uses: actions/upload-artifact@v4 if: failure() diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml index bdf2c5da5444..25365bd1b373 100644 --- a/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml @@ -65,7 +65,7 @@ jobs: matrix: job_name: ["beam_PostCommit_Python_ValidatesContainer_Dataflow"] job_phrase: ["Run Python Dataflow ValidatesContainer"] - python_version: ['3.9','3.10','3.11','3.12','3.13'] + python_version: ['3.10','3.11','3.12','3.13'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml b/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml index 482a4d509106..e4fe4cf4ba24 100644 --- a/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml +++ b/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml @@ -63,7 +63,7 @@ jobs: matrix: job_name: ["beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC"] job_phrase: ["Run Python RC Dataflow ValidatesContainer"] - python_version: ['3.9','3.10','3.11','3.12','3.13'] + python_version: ['3.10','3.11','3.12','3.13'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Dataflow.yml index 7988ebdbdae0..07b2a659cd08 100644 --- a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Dataflow.yml @@ -63,7 +63,7 @@ jobs: matrix: job_name: ["beam_PostCommit_Python_ValidatesRunner_Dataflow"] job_phrase: ["Run Python Dataflow ValidatesRunner"] - python_version: ['3.9', '3.13'] + python_version: ['3.10', '3.13'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Flink.yml b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Flink.yml index f9e16ae82366..51006c079b7e 100644 --- a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Flink.yml +++ b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Flink.yml @@ -63,7 +63,7 @@ jobs: matrix: job_name: ["beam_PostCommit_Python_ValidatesRunner_Flink"] job_phrase: ["Run Python Flink ValidatesRunner"] - python_version: ['3.9', '3.13'] + python_version: ['3.10', '3.13'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml index 262965321141..ba965598aa0e 100644 --- a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml +++ b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml @@ -63,7 +63,7 @@ jobs: matrix: job_name: ["beam_PostCommit_Python_ValidatesRunner_Samza"] job_phrase: ["Run Python Samza ValidatesRunner"] - python_version: ['3.9', '3.13'] + python_version: ['3.10', '3.13'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Spark.yml b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Spark.yml index 127d0cd9e48b..c3e5b3cdc014 100644 --- a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Spark.yml +++ b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Spark.yml @@ -63,7 +63,7 @@ jobs: matrix: job_name: ["beam_PostCommit_Python_ValidatesRunner_Spark"] job_phrase: ["Run Python Spark ValidatesRunner"] - python_version: ['3.9', '3.13'] + python_version: ['3.10', '3.13'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml index 7e1f43a3a480..cf2dddc5e140 100644 --- a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml @@ -75,7 +75,7 @@ jobs: uses: ./.github/actions/setup-environment-action with: python-version: | - 3.9 + 3.10 3.13 - name: run PostCommit Python Xlang Gcp Dataflow script uses: ./.github/actions/gradle-command-self-hosted-action diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml index 5a5081888e99..4e939993d983 100644 --- a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml +++ b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml @@ -75,7 +75,7 @@ jobs: uses: ./.github/actions/setup-environment-action with: python-version: | - 3.9 + 3.10 3.13 - name: Install docker compose run: | diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml index 50d3055f790a..de06b49cfdaf 100644 --- a/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml @@ -74,7 +74,7 @@ jobs: uses: ./.github/actions/setup-environment-action with: python-version: | - 3.9 + 3.10 3.13 - name: run PostCommit Python Xlang IO Dataflow script uses: ./.github/actions/gradle-command-self-hosted-action diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_IO_Direct.yml b/.github/workflows/beam_PostCommit_Python_Xlang_IO_Direct.yml index 7a1b3fe031a2..6d112eae4961 100644 --- a/.github/workflows/beam_PostCommit_Python_Xlang_IO_Direct.yml +++ b/.github/workflows/beam_PostCommit_Python_Xlang_IO_Direct.yml @@ -74,7 +74,7 @@ jobs: uses: ./.github/actions/setup-environment-action with: python-version: | - 3.9 + 3.10 3.13 - name: run PostCommit Python Xlang IO Direct script uses: ./.github/actions/gradle-command-self-hosted-action diff --git a/.github/workflows/beam_PostCommit_TransformService_Direct.yml b/.github/workflows/beam_PostCommit_TransformService_Direct.yml index 3880fb935472..44fe474235ae 100644 --- a/.github/workflows/beam_PostCommit_TransformService_Direct.yml +++ b/.github/workflows/beam_PostCommit_TransformService_Direct.yml @@ -62,7 +62,7 @@ jobs: matrix: job_name: ["beam_PostCommit_TransformService_Direct"] job_phrase: ["Run TransformService_Direct PostCommit"] - python_version: ['3.9','3.13'] + python_version: ['3.10','3.13'] steps: - uses: actions/checkout@v4 - name: Setup repository @@ -76,7 +76,7 @@ jobs: with: java-version: 11 python-version: | - 3.9 + 3.10 ${{ matrix.python_version }} - name: run TransformService Direct script uses: ./.github/actions/gradle-command-self-hosted-action diff --git a/.github/workflows/beam_PostCommit_XVR_Direct.yml b/.github/workflows/beam_PostCommit_XVR_Direct.yml index c39617b8957b..cca4898fb011 100644 --- a/.github/workflows/beam_PostCommit_XVR_Direct.yml +++ b/.github/workflows/beam_PostCommit_XVR_Direct.yml @@ -62,7 +62,7 @@ jobs: matrix: job_name: ["beam_PostCommit_XVR_Direct"] job_phrase: ["Run XVR_Direct PostCommit"] - python_version: ['3.9','3.13'] + python_version: ['3.10','3.13'] steps: - uses: actions/checkout@v4 - name: Setup repository @@ -83,7 +83,7 @@ jobs: gradle-command: :sdks:python:test-suites:direct:xlang:validatesCrossLanguageRunner arguments: | -PpythonVersion=${{ matrix.python_version }} \ - -PskipNonPythonTask=${{ (matrix.python_version == '3.9' && true) || false }} \ + -PskipNonPythonTask=${{ (matrix.python_version == '3.10' && true) || false }} \ - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 if: ${{ !success() }} diff --git a/.github/workflows/beam_PostCommit_XVR_Flink.yml b/.github/workflows/beam_PostCommit_XVR_Flink.yml index 92d97f4322f5..53d1fd81546b 100644 --- a/.github/workflows/beam_PostCommit_XVR_Flink.yml +++ b/.github/workflows/beam_PostCommit_XVR_Flink.yml @@ -63,7 +63,7 @@ jobs: matrix: job_name: ["beam_PostCommit_XVR_Flink"] job_phrase: ["Run XVR_Flink PostCommit"] - python_version: ['3.9','3.13'] + python_version: ['3.10','3.13'] steps: - uses: actions/checkout@v4 - name: Setup repository @@ -84,7 +84,7 @@ jobs: gradle-command: :runners:flink:${{ env.FlinkVersion }}:job-server:validatesCrossLanguageRunner arguments: | -PpythonVersion=${{ matrix.python_version }} \ - -PskipNonPythonTask=${{ (matrix.python_version == '3.9' && true) || false }} \ + -PskipNonPythonTask=${{ (matrix.python_version == '3.10' && true) || false }} \ - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 if: ${{ !success() }} diff --git a/.github/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml b/.github/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml index dcc44d740ee7..9a367497c4fd 100644 --- a/.github/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml @@ -62,7 +62,7 @@ jobs: matrix: job_name: ["beam_PostCommit_XVR_JavaUsingPython_Dataflow"] job_phrase: ["Run XVR_JavaUsingPython_Dataflow PostCommit"] - python_version: ['3.9','3.13'] + python_version: ['3.10','3.13'] steps: - uses: actions/checkout@v4 - name: Setup repository @@ -75,7 +75,7 @@ jobs: uses: ./.github/actions/setup-environment-action with: python-version: | - 3.9 + 3.10 ${{ matrix.python_version }} - name: run PostCommit XVR JavaUsingPython Dataflow script uses: ./.github/actions/gradle-command-self-hosted-action diff --git a/.github/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml b/.github/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml index a7dacd739c05..1ef993eb44fa 100644 --- a/.github/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml @@ -62,7 +62,7 @@ jobs: matrix: job_name: ["beam_PostCommit_XVR_PythonUsingJava_Dataflow"] job_phrase: ["Run XVR_PythonUsingJava_Dataflow PostCommit"] - python_version: ['3.9','3.13'] + python_version: ['3.10','3.13'] steps: - uses: actions/checkout@v4 - name: Setup repository @@ -75,7 +75,7 @@ jobs: uses: ./.github/actions/setup-environment-action with: python-version: | - 3.9 + 3.10 ${{ matrix.python_version }} - name: run PostCommit XVR PythonUsingJava Dataflow script uses: ./.github/actions/gradle-command-self-hosted-action diff --git a/.github/workflows/beam_PostCommit_XVR_Samza.yml b/.github/workflows/beam_PostCommit_XVR_Samza.yml index 6e6d6739402a..fe63772400bb 100644 --- a/.github/workflows/beam_PostCommit_XVR_Samza.yml +++ b/.github/workflows/beam_PostCommit_XVR_Samza.yml @@ -62,7 +62,7 @@ jobs: matrix: job_name: ["beam_PostCommit_XVR_Samza"] job_phrase: ["Run XVR_Samza PostCommit"] - python_version: ['3.9','3.13'] + python_version: ['3.10','3.13'] steps: - uses: actions/checkout@v4 - name: Setup repository @@ -90,7 +90,7 @@ jobs: -PpythonVersion=${{ matrix.python_version }} \ -PtestJavaVersion=8 \ -Pjava8Home=$JAVA_HOME_8_X64 \ - -PskipNonPythonTask=${{ (matrix.python_version == '3.9' && true) || false }} \ + -PskipNonPythonTask=${{ (matrix.python_version == '3.10' && true) || false }} \ - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 if: ${{ !success() }} diff --git a/.github/workflows/beam_PostCommit_XVR_Spark3.yml b/.github/workflows/beam_PostCommit_XVR_Spark3.yml index 7d431c7312ca..d465c6e13be7 100644 --- a/.github/workflows/beam_PostCommit_XVR_Spark3.yml +++ b/.github/workflows/beam_PostCommit_XVR_Spark3.yml @@ -62,7 +62,7 @@ jobs: matrix: job_name: ["beam_PostCommit_XVR_Spark3"] job_phrase: ["Run XVR_Spark3 PostCommit"] - python_version: ['3.9','3.13'] + python_version: ['3.10','3.13'] steps: - uses: actions/checkout@v4 - name: Setup repository @@ -83,7 +83,7 @@ jobs: gradle-command: :runners:spark:3:job-server:validatesCrossLanguageRunner arguments: | -PpythonVersion=${{ matrix.python_version }} \ - -PskipNonPythonTask=${{ (matrix.python_version == '3.9' && true) || false }} \ + -PskipNonPythonTask=${{ (matrix.python_version == '3.10' && true) || false }} \ - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 if: ${{ !success() }} diff --git a/.github/workflows/beam_PreCommit_Flink_Container.yml b/.github/workflows/beam_PreCommit_Flink_Container.yml index f21e1639b4a6..5eba6fcc5e2a 100644 --- a/.github/workflows/beam_PreCommit_Flink_Container.yml +++ b/.github/workflows/beam_PreCommit_Flink_Container.yml @@ -79,7 +79,7 @@ env: ARTIFACTS_DIR: gs://beam-flink-cluster/beam-precommit-flink-container-${{ github.run_id }} DOCKER_REGISTRY: gcr.io DOCKER_REPOSITORY_ROOT: ${{ github.event_name == 'pull_request_target' && 'gcr.io/apache-beam-testing/beam-sdk-pr' || 'gcr.io/apache-beam-testing/beam-sdk' }} - PYTHON_VERSION: 3.9 + PYTHON_VERSION: 3.10 PYTHON_SDK_IMAGE_TAG: latest jobs: diff --git a/.github/workflows/beam_PreCommit_Portable_Python.yml b/.github/workflows/beam_PreCommit_Portable_Python.yml index 9052a87e012f..47f393206f77 100644 --- a/.github/workflows/beam_PreCommit_Portable_Python.yml +++ b/.github/workflows/beam_PreCommit_Portable_Python.yml @@ -82,7 +82,7 @@ jobs: matrix: job_name: ['beam_PreCommit_Portable_Python'] job_phrase: ['Run Portable_Python PreCommit'] - python_version: ['3.9', '3.12'] + python_version: ['3.10', '3.13'] if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || @@ -103,7 +103,7 @@ jobs: java-version: default python-version: | ${{ matrix.python_version }} - 3.9 + 3.10 - name: Set PY_VER_CLEAN id: set_py_ver_clean run: | diff --git a/.github/workflows/beam_PreCommit_Prism_Python.yml b/.github/workflows/beam_PreCommit_Prism_Python.yml index 81429b1e515a..5f3f1d9fa477 100644 --- a/.github/workflows/beam_PreCommit_Prism_Python.yml +++ b/.github/workflows/beam_PreCommit_Prism_Python.yml @@ -76,7 +76,7 @@ jobs: matrix: job_name: ['beam_PreCommit_Prism_Python'] job_phrase: ['Run Prism_Python PreCommit'] - python_version: ['3.9', '3.13'] + python_version: ['3.10', '3.13'] if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || @@ -97,7 +97,7 @@ jobs: java-version: default python-version: | ${{ matrix.python_version }} - 3.9 + 3.10 - name: Set PY_VER_CLEAN id: set_py_ver_clean run: | diff --git a/.github/workflows/beam_PreCommit_Python.yml b/.github/workflows/beam_PreCommit_Python.yml index 07ebc40a851e..4115034a8a19 100644 --- a/.github/workflows/beam_PreCommit_Python.yml +++ b/.github/workflows/beam_PreCommit_Python.yml @@ -81,7 +81,7 @@ jobs: matrix: job_name: ['beam_PreCommit_Python'] job_phrase: ['Run Python PreCommit'] - python_version: ['3.9','3.10','3.11','3.12','3.13'] + python_version: ['3.10','3.11','3.12','3.13'] if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PreCommit_PythonDocker.yml b/.github/workflows/beam_PreCommit_PythonDocker.yml index d3e6f90416f3..119939619d59 100644 --- a/.github/workflows/beam_PreCommit_PythonDocker.yml +++ b/.github/workflows/beam_PreCommit_PythonDocker.yml @@ -64,7 +64,7 @@ jobs: matrix: job_name: ["beam_PreCommit_PythonDocker"] job_phrase: ["Run PythonDocker PreCommit"] - python_version: ['3.9','3.10','3.11','3.12','3.13'] + python_version: ['3.10','3.11','3.12','3.13'] if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PreCommit_Python_Coverage.yml b/.github/workflows/beam_PreCommit_Python_Coverage.yml index 08e8d2089d66..1f76fe23390a 100644 --- a/.github/workflows/beam_PreCommit_Python_Coverage.yml +++ b/.github/workflows/beam_PreCommit_Python_Coverage.yml @@ -65,7 +65,7 @@ jobs: matrix: job_name: [beam_PreCommit_Python_Coverage] job_phrase: [Run Python_Coverage PreCommit] - python_version: ['3.9'] + python_version: ['3.10'] # Run on both self-hosted and GitHub-hosted runners. # Some tests (marked require_docker_in_docker) can't run on Beam's # self-hosted runners due to Docker-in-Docker environment constraint. diff --git a/.github/workflows/beam_PreCommit_Python_Dataframes.yml b/.github/workflows/beam_PreCommit_Python_Dataframes.yml index 0b1f39aaded6..0c1ba5dd2ad7 100644 --- a/.github/workflows/beam_PreCommit_Python_Dataframes.yml +++ b/.github/workflows/beam_PreCommit_Python_Dataframes.yml @@ -64,7 +64,7 @@ jobs: matrix: job_name: ['beam_PreCommit_Python_Dataframes'] job_phrase: ['Run Python_Dataframes PreCommit'] - python_version: ['3.9','3.10','3.11','3.12','3.13'] + python_version: ['3.10','3.11','3.12','3.13'] if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PreCommit_Python_Examples.yml b/.github/workflows/beam_PreCommit_Python_Examples.yml index db52c235fb73..b91207557790 100644 --- a/.github/workflows/beam_PreCommit_Python_Examples.yml +++ b/.github/workflows/beam_PreCommit_Python_Examples.yml @@ -65,7 +65,7 @@ jobs: matrix: job_name: ['beam_PreCommit_Python_Examples'] job_phrase: ['Run Python_Examples PreCommit'] - python_version: ['3.9','3.10','3.11','3.12','3.13'] + python_version: ['3.10','3.11','3.12','3.13'] if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PreCommit_Python_Integration.yml b/.github/workflows/beam_PreCommit_Python_Integration.yml index c61a23b828a7..70993f58ce5d 100644 --- a/.github/workflows/beam_PreCommit_Python_Integration.yml +++ b/.github/workflows/beam_PreCommit_Python_Integration.yml @@ -64,7 +64,7 @@ jobs: matrix: job_name: ['beam_PreCommit_Python_Integration'] job_phrase: ['Run Python_Integration PreCommit'] - python_version: ['3.9', '3.13'] + python_version: ['3.10', '3.13'] if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PreCommit_Python_ML.yml b/.github/workflows/beam_PreCommit_Python_ML.yml index f337fd4cdafe..cecb2e65506a 100644 --- a/.github/workflows/beam_PreCommit_Python_ML.yml +++ b/.github/workflows/beam_PreCommit_Python_ML.yml @@ -65,7 +65,7 @@ jobs: matrix: job_name: ['beam_PreCommit_Python_ML'] job_phrase: ['Run Python_ML PreCommit'] - python_version: ['3.9','3.10','3.11','3.12','3.13'] + python_version: ['3.10','3.11','3.12','3.13'] # Run on both self-hosted and GitHub-hosted runners. # Some tests (marked require_docker_in_docker) can't run on Beam's # self-hosted runners due to Docker-in-Docker environment constraint. @@ -73,11 +73,9 @@ jobs: # Context: https://github.com/apache/beam/pull/35585. os: [[self-hosted, ubuntu-20.04, main], [ubuntu-latest]] exclude: - # Temporary exclude Python 3.9, 3.10, 3.11, from ubuntu-latest. + # Temporary exclude Python 3.10, 3.11, from ubuntu-latest. # This results in pip dependency resolution exceeded maximum depth issue. # Context: https://github.com/apache/beam/pull/35816. - - python_version: '3.9' - os: [ubuntu-latest] - python_version: '3.10' os: [ubuntu-latest] - python_version: '3.11' diff --git a/.github/workflows/beam_PreCommit_Python_Runners.yml b/.github/workflows/beam_PreCommit_Python_Runners.yml index cd7bb4a21ba6..a8364265f93e 100644 --- a/.github/workflows/beam_PreCommit_Python_Runners.yml +++ b/.github/workflows/beam_PreCommit_Python_Runners.yml @@ -64,7 +64,7 @@ jobs: matrix: job_name: ['beam_PreCommit_Python_Runners'] job_phrase: ['Run Python_Runners PreCommit'] - python_version: ['3.9','3.10','3.11','3.12','3.13'] + python_version: ['3.10','3.11','3.12','3.13'] if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PreCommit_Python_Transforms.yml b/.github/workflows/beam_PreCommit_Python_Transforms.yml index f3f0a40a480a..88f7db3c5b08 100644 --- a/.github/workflows/beam_PreCommit_Python_Transforms.yml +++ b/.github/workflows/beam_PreCommit_Python_Transforms.yml @@ -65,7 +65,7 @@ jobs: matrix: job_name: ['beam_PreCommit_Python_Transforms'] job_phrase: ['Run Python_Transforms PreCommit'] - python_version: ['3.9','3.10','3.11','3.12','3.13'] + python_version: ['3.10','3.11','3.12','3.13'] if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml b/.github/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml index 6cacce7c0ebf..4a28186e3635 100644 --- a/.github/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml +++ b/.github/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml @@ -84,7 +84,7 @@ jobs: matrix: job_name: ['beam_PreCommit_Xlang_Generated_Transforms'] job_phrase: ['Run Xlang_Generated_Transforms PreCommit'] - python_version: ['3.9'] + python_version: ['3.10'] if: | github.event_name == 'push' || github.event_name == 'workflow_dispatch' || diff --git a/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml b/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml index 885e35bc74ef..652af3909611 100644 --- a/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml +++ b/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml @@ -63,17 +63,14 @@ jobs: container_task: - "go:container:docker" - "java:container:pushAll" - - "python:container:py39:docker" - "python:container:py310:docker" - "python:container:py311:docker" - "python:container:py312:docker" - "python:container:py313:docker" - - "python:container:distroless:py39:docker" - "python:container:distroless:py310:docker" - "python:container:distroless:py311:docker" - "python:container:distroless:py312:docker" - "python:container:distroless:py313:docker" - - "python:container:ml:py39:docker" - "python:container:ml:py310:docker" - "python:container:ml:py311:docker" - "python:container:ml:py312:docker" diff --git a/.github/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml b/.github/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml index 002888c25aca..4094279048a9 100644 --- a/.github/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml +++ b/.github/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml @@ -54,7 +54,7 @@ jobs: matrix: job_name: [beam_Python_ValidatesContainer_Dataflow_ARM] job_phrase: [Run Python ValidatesContainer Dataflow ARM] - python_version: ['3.9','3.10','3.11','3.12','3.13'] + python_version: ['3.10','3.11','3.12','3.13'] if: | github.event_name == 'push' || (github.event_name == 'schedule' && github.repository == 'apache/beam') || diff --git a/.github/workflows/build_release_candidate.yml b/.github/workflows/build_release_candidate.yml index 6e66f3473032..d94776fd33cb 100644 --- a/.github/workflows/build_release_candidate.yml +++ b/.github/workflows/build_release_candidate.yml @@ -281,10 +281,10 @@ jobs: with: distribution: 'temurin' java-version: '11' - - name: Install Python 3.9 + - name: Install Python 3.10 uses: actions/setup-python@v5 with: - python-version: '3.9' + python-version: '3.10' - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Remove default github maven configuration @@ -327,10 +327,10 @@ jobs: path: beam-site token: ${{ github.event.inputs.REPO_TOKEN }} ref: release-docs - - name: Install Python 3.9 + - name: Install Python 3.10 uses: actions/setup-python@v5 with: - python-version: '3.9' + python-version: '3.10' - name: Install node uses: actions/setup-node@v5 with: @@ -566,10 +566,10 @@ jobs: path: beam token: ${{ github.event.inputs.REPO_TOKEN }} persist-credentials: true - - name: Install Python 3.9 + - name: Install Python 3.10 uses: actions/setup-python@v5 with: - python-version: '3.9' + python-version: '3.10' - name: Install Java 11 uses: actions/setup-java@v4 with: diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 8535983e72ea..9d27f042f073 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -49,7 +49,7 @@ jobs: env: EVENT_NAME: ${{ github.event_name }} # Keep in sync with py_version matrix value below - if changed, change that as well. - PY_VERSIONS_FULL: "cp39-* cp310-* cp311-* cp312-* cp313-*" + PY_VERSIONS_FULL: "cp310-* cp311-* cp312-* cp313-*" outputs: gcp-variables-set: ${{ steps.check_gcp_variables.outputs.gcp-variables-set }} py-versions-full: ${{ steps.set-py-versions.outputs.py-versions-full }} @@ -92,7 +92,7 @@ jobs: - name: Install python uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: 3.10 - name: Get tag id: get_tag run: | @@ -230,7 +230,7 @@ jobs: {"os": "ubuntu-20.04", "runner": "ubuntu-22.04", "python": "${{ needs.check_env_variables.outputs.py-versions-test }}", arch: "aarch64" } ] # Keep in sync (remove asterisks) with PY_VERSIONS_FULL env var above - if changed, change that as well. - py_version: ["cp39-", "cp310-", "cp311-", "cp312-", "cp313-"] + py_version: ["cp310-", "cp311-", "cp312-", "cp313-"] steps: - name: Download python source distribution from artifacts uses: actions/download-artifact@v5 @@ -246,7 +246,7 @@ jobs: - name: Install Python uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: 3.10 - uses: docker/setup-qemu-action@v3 if: ${{matrix.os_python.arch == 'aarch64'}} name: Set up QEMU diff --git a/.github/workflows/dask_runner_tests.yml b/.github/workflows/dask_runner_tests.yml index 8faea77acc9b..055ac91220d0 100644 --- a/.github/workflows/dask_runner_tests.yml +++ b/.github/workflows/dask_runner_tests.yml @@ -44,7 +44,7 @@ jobs: - name: Install python uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: 3.10 - name: Build source working-directory: ./sdks/python run: pip install -U build && python -m build --sdist @@ -65,7 +65,6 @@ jobs: matrix: os: [ubuntu-latest, macos-latest, windows-latest] params: [ - {"py_ver": "3.9", "tox_env": "py39"}, {"py_ver": "3.10", "tox_env": "py310" }, ] steps: diff --git a/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_100b_Multiple_Keys.txt b/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_100b_Multiple_Keys.txt index 6e26ee72a77c..dc851c279215 100644 --- a/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_100b_Multiple_Keys.txt +++ b/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_100b_Multiple_Keys.txt @@ -25,4 +25,4 @@ --runner=PortableRunner --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest diff --git a/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_100b_Single_Key.txt b/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_100b_Single_Key.txt index e1df7e3fd5f9..b462794a444e 100644 --- a/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_100b_Single_Key.txt +++ b/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_100b_Single_Key.txt @@ -25,4 +25,4 @@ --runner=PortableRunner --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest diff --git a/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_10kB.txt b/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_10kB.txt index b1f95027c9da..d8154c115405 100644 --- a/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_10kB.txt +++ b/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_10kB.txt @@ -25,4 +25,4 @@ --runner=PortableRunner --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest diff --git a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_10_byte_records.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_10_byte_records.txt index 57b1bbc854b6..dba7cf9c95d6 100644 --- a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_10_byte_records.txt +++ b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_10_byte_records.txt @@ -22,6 +22,6 @@ --parallelism=5 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest --top_count=20 --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_Fanout_4.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_Fanout_4.txt index 4923929301dc..c79db43476ad 100644 --- a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_Fanout_4.txt +++ b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_Fanout_4.txt @@ -22,7 +22,7 @@ --parallelism=16 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest --fanout=4 --top_count=20 --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_Fanout_8.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_Fanout_8.txt index 8a089fee3516..fb3e08de8916 100644 --- a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_Fanout_8.txt +++ b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_Fanout_8.txt @@ -22,7 +22,7 @@ --parallelism=16 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest --fanout=8 --top_count=20 --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_10_byte_records.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_10_byte_records.txt index 5d1a0be9950e..925cfc75d760 100644 --- a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_10_byte_records.txt +++ b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_10_byte_records.txt @@ -22,7 +22,7 @@ --parallelism=5 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest --top_count=20 --streaming --use_stateful_load_generator diff --git a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_4.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_4.txt index 6280e01dccdb..a89a46a3747c 100644 --- a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_4.txt +++ b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_4.txt @@ -22,7 +22,7 @@ --parallelism=16 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest --fanout=4 --top_count=20 --streaming diff --git a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_8.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_8.txt index e1b77d15b95b..9edc487cdf16 100644 --- a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_8.txt +++ b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_8.txt @@ -22,7 +22,7 @@ --parallelism=16 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest --fanout=8 --top_count=20 --streaming diff --git a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_small_Fanout_1.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_small_Fanout_1.txt index f16e9e4b06ef..91420cb34be0 100644 --- a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_small_Fanout_1.txt +++ b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_small_Fanout_1.txt @@ -22,7 +22,7 @@ --parallelism=16 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest --fanout=1 --top_count=20 --streaming diff --git a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_small_Fanout_2.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_small_Fanout_2.txt index 5f66e519c31a..e8054ceec18f 100644 --- a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_small_Fanout_2.txt +++ b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_small_Fanout_2.txt @@ -22,7 +22,7 @@ --parallelism=16 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest --fanout=2 --top_count=20 --streaming diff --git a/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_2GB_of_100B_records.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_2GB_of_100B_records.txt index f4f5e7de8369..ddf7a314c0d8 100644 --- a/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_2GB_of_100B_records.txt +++ b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_2GB_of_100B_records.txt @@ -24,5 +24,5 @@ --parallelism=5 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_2GB_of_10B_records.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_2GB_of_10B_records.txt index 40db0b6d40bc..29c5a085f5f6 100644 --- a/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_2GB_of_10B_records.txt +++ b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_2GB_of_10B_records.txt @@ -24,5 +24,5 @@ --parallelism=5 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt index df27dc7c4470..34695a08f96e 100644 --- a/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt +++ b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt @@ -24,5 +24,5 @@ --parallelism=16 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt index 6b87f61eed8a..7a7db563a769 100644 --- a/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt +++ b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt @@ -24,5 +24,5 @@ --parallelism=16 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_reiterate_4_times_10kB_values.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_reiterate_4_times_10kB_values.txt index 621777663be0..40db954bce78 100644 --- a/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_reiterate_4_times_10kB_values.txt +++ b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_reiterate_4_times_10kB_values.txt @@ -24,5 +24,5 @@ --parallelism=5 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_10_Counters.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_10_Counters.txt index fe451559e625..2ab93cc13f5c 100644 --- a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_10_Counters.txt +++ b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_10_Counters.txt @@ -25,5 +25,5 @@ --parallelism=5 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_10_Iterations.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_10_Iterations.txt index dd5addb65d14..e0b9e1093716 100644 --- a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_10_Iterations.txt +++ b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_10_Iterations.txt @@ -25,5 +25,5 @@ --parallelism=5 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_200_Iterations.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_200_Iterations.txt index fe451559e625..2ab93cc13f5c 100644 --- a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_200_Iterations.txt +++ b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_200_Iterations.txt @@ -25,5 +25,5 @@ --parallelism=5 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_100_Counters.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_100_Counters.txt index 308deb3ecf4d..1265387706b3 100644 --- a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_100_Counters.txt +++ b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_100_Counters.txt @@ -26,6 +26,6 @@ --streaming --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest --use_stateful_load_generator --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_10_Counters.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_10_Counters.txt index 78ecc1fd98dd..d725a7d1b601 100644 --- a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_10_Counters.txt +++ b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_10_Counters.txt @@ -26,6 +26,6 @@ --streaming --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest --use_stateful_load_generator --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_10_Iterations.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_10_Iterations.txt index 04a1213d4039..7f7f428c4081 100644 --- a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_10_Iterations.txt +++ b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_10_Iterations.txt @@ -27,6 +27,6 @@ --stateful --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest --use_stateful_load_generator --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_200_Iterations.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_200_Iterations.txt index a2f7d7600da8..83411793c2b2 100644 --- a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_200_Iterations.txt +++ b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_200_Iterations.txt @@ -26,6 +26,6 @@ --streaming --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest --use_stateful_load_generator --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_5_Iterations.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_5_Iterations.txt index f49be6c70582..14f94d641847 100644 --- a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_5_Iterations.txt +++ b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_5_Iterations.txt @@ -30,6 +30,6 @@ --shutdown_sources_after_idle_ms=300000 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest --use_stateful_load_generator --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/python_dependency_tests.yml b/.github/workflows/python_dependency_tests.yml index 2f95ea4f48f8..02e01c3166be 100644 --- a/.github/workflows/python_dependency_tests.yml +++ b/.github/workflows/python_dependency_tests.yml @@ -26,7 +26,6 @@ jobs: matrix: os: [ubuntu-latest] params: [ - {"py_ver": "3.9", "py_env": "py39"}, {"py_ver": "3.10", "py_env": "py310" }, { "py_ver": "3.11", "py_env": "py311" }, { "py_ver": "3.12", "py_env": "py312" }, diff --git a/.github/workflows/python_tests.yml b/.github/workflows/python_tests.yml index 82275ef9dfc1..a32402717bc4 100644 --- a/.github/workflows/python_tests.yml +++ b/.github/workflows/python_tests.yml @@ -93,7 +93,6 @@ jobs: matrix: os: [macos-latest, windows-latest] params: [ - { "py_ver": "3.9", "tox_env": "py39" }, { "py_ver": "3.10", "tox_env": "py310" }, { "py_ver": "3.11", "tox_env": "py311" }, { "py_ver": "3.12", "tox_env": "py312" }, @@ -134,7 +133,7 @@ jobs: fail-fast: false matrix: os: [[self-hosted, ubuntu-20.04, main], macos-latest, windows-latest] - python: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python: ["3.10", "3.11", "3.12", "3.13"] steps: - name: Checkout code uses: actions/checkout@v4 diff --git a/.github/workflows/republish_released_docker_containers.yml b/.github/workflows/republish_released_docker_containers.yml index 09c9eb7edf16..a0421f358b2c 100644 --- a/.github/workflows/republish_released_docker_containers.yml +++ b/.github/workflows/republish_released_docker_containers.yml @@ -68,10 +68,10 @@ jobs: with: distribution: 'temurin' java-version: '11' - - name: Install Python 3.9 + - name: Install Python 3.10 uses: actions/setup-python@v5 with: - python-version: '3.9' + python-version: '3.10' - name: Authenticate on GCP uses: google-github-actions/auth@v3 with: diff --git a/.github/workflows/run_perf_alert_tool.yml b/.github/workflows/run_perf_alert_tool.yml index a6aae616efec..1e0c1de3c951 100644 --- a/.github/workflows/run_perf_alert_tool.yml +++ b/.github/workflows/run_perf_alert_tool.yml @@ -39,7 +39,7 @@ jobs: - name: Install python uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: 3.10 - name: Install Apache Beam working-directory: ./sdks/python run: pip install -e .[gcp,test] diff --git a/.github/workflows/run_rc_validation_python_mobile_gaming.yml b/.github/workflows/run_rc_validation_python_mobile_gaming.yml index ea6fe1a44683..27118a139268 100644 --- a/.github/workflows/run_rc_validation_python_mobile_gaming.yml +++ b/.github/workflows/run_rc_validation_python_mobile_gaming.yml @@ -79,7 +79,7 @@ jobs: RELEASE_VERSION: ${{ github.event.inputs.RELEASE_VER }} RC_NUM: ${{ github.event.inputs.RC_NUM }} RC_TAG: "v${{github.event.inputs.RELEASE_VER}}-RC${{github.event.inputs.RC_NUM}}" - PYTHON_VERSION: '3.9' + PYTHON_VERSION: '3.10' BEAM_PYTHON_SDK_TAR_GZ: apache_beam-${{ github.event.inputs.RELEASE_VER }}.tar.gz BEAM_SOURCE_ZIP: apache-beam-${{ github.event.inputs.RELEASE_VER }}-source-release.zip APACHE_DIST_URL_BASE: https://dist.apache.org/repos/dist/dev/beam/${{ github.event.inputs.RELEASE_VER }} diff --git a/.github/workflows/typescript_tests.yml b/.github/workflows/typescript_tests.yml index d438b4dd93f9..af46c68c2f17 100644 --- a/.github/workflows/typescript_tests.yml +++ b/.github/workflows/typescript_tests.yml @@ -110,7 +110,7 @@ jobs: - name: Install Python uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: 3.10 - name: Setup Beam Python working-directory: ./sdks/python run: | @@ -173,7 +173,7 @@ jobs: - name: Install python uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: 3.10 - name: Setup Beam Python working-directory: ./sdks/python run: | diff --git a/.github/workflows/update_python_dependencies.yml b/.github/workflows/update_python_dependencies.yml index 3fe87bf639fd..acbcb66fb63d 100644 --- a/.github/workflows/update_python_dependencies.yml +++ b/.github/workflows/update_python_dependencies.yml @@ -56,7 +56,6 @@ jobs: uses: ./.github/actions/setup-environment-action with: python-version: | - 3.9 3.10 3.11 3.12 diff --git a/.test-infra/dockerized-jenkins/seedjobconfig.xml b/.test-infra/dockerized-jenkins/seedjobconfig.xml index 2ede2430b454..bfd57c323c53 100644 --- a/.test-infra/dockerized-jenkins/seedjobconfig.xml +++ b/.test-infra/dockerized-jenkins/seedjobconfig.xml @@ -18,7 +18,7 @@ false - + 2 diff --git a/.test-infra/jenkins/PythonTestProperties.groovy b/.test-infra/jenkins/PythonTestProperties.groovy index 7e8e4ad3d8fd..f050a29ea779 100644 --- a/.test-infra/jenkins/PythonTestProperties.groovy +++ b/.test-infra/jenkins/PythonTestProperties.groovy @@ -20,10 +20,10 @@ class PythonTestProperties { // Indicates all supported Python versions. // This must be sorted in ascending order. final static List ALL_SUPPORTED_VERSIONS = [ - '3.9', '3.10', '3.11', - '3.12' + '3.12', + '3.13' ] final static List SUPPORTED_CONTAINER_TASKS = ALL_SUPPORTED_VERSIONS.collect { "py${it.replace('.', '')}" @@ -37,10 +37,10 @@ class PythonTestProperties { final static List CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS = ESSENTIAL_VERSIONS final static List CROSS_LANGUAGE_VALIDATES_RUNNER_DATAFLOW_USING_SQL_PYTHON_VERSIONS = [HIGHEST_SUPPORTED] final static List VALIDATES_CONTAINER_DATAFLOW_PYTHON_VERSIONS = ALL_SUPPORTED_VERSIONS - final static String LOAD_TEST_PYTHON_VERSION = '3.9' - final static String RUN_INFERENCE_TEST_PYTHON_VERSION = '3.9' - final static String CHICAGO_TAXI_EXAMPLE_FLINK_PYTHON_VERSION = '3.9' + final static String LOAD_TEST_PYTHON_VERSION = '3.10' + final static String RUN_INFERENCE_TEST_PYTHON_VERSION = '3.10' + final static String CHICAGO_TAXI_EXAMPLE_FLINK_PYTHON_VERSION = '3.10' // Use for various shell scripts triggered by Jenkins. // Gradle scripts should use project.ext.pythonVersion defined by PythonNature/BeamModulePlugin. - final static String DEFAULT_INTERPRETER = 'python3.9' + final static String DEFAULT_INTERPRETER = 'python3.10' } diff --git a/.test-infra/jenkins/metrics_report/tox.ini b/.test-infra/jenkins/metrics_report/tox.ini index 5126b337afcc..56fa14e67275 100644 --- a/.test-infra/jenkins/metrics_report/tox.ini +++ b/.test-infra/jenkins/metrics_report/tox.ini @@ -17,7 +17,7 @@ ; TODO(https://github.com/apache/beam/issues/20209): Don't hardcode Py3.8 version. [tox] skipsdist = True -envlist = py39-test,py39-generate-report +envlist = py310-test,py310-generate-report [testenv] commands_pre = @@ -25,12 +25,12 @@ commands_pre = pip --version pip check -[testenv:py39-test] +[testenv:py310-test] deps = -r requirements.txt passenv = WORKSPACE,INFLUXDB_USER,INFLUXDB_USER_PASSWORD commands = python -m unittest dashboards_parser.py -[testenv:py39-generate-report] +[testenv:py310-generate-report] deps = -r requirements.txt passenv = WORKSPACE,INFLUXDB_USER,INFLUXDB_USER_PASSWORD,GITHUB_WORKSPACE commands = python report_generator.py {posargs} diff --git a/.test-infra/metrics/influxdb/Dockerfile b/.test-infra/metrics/influxdb/Dockerfile index 0ec7bd6f2677..7d08940fcb4b 100644 --- a/.test-infra/metrics/influxdb/Dockerfile +++ b/.test-infra/metrics/influxdb/Dockerfile @@ -16,7 +16,7 @@ # limitations under the License. ################################################################################ -FROM python:3.9-slim +FROM python:3.10-slim RUN pip install --no-cache-dir gsutil diff --git a/.test-infra/metrics/influxdb/gsutil/Dockerfile b/.test-infra/metrics/influxdb/gsutil/Dockerfile index ea6621e2cf9d..87a46d4861cc 100644 --- a/.test-infra/metrics/influxdb/gsutil/Dockerfile +++ b/.test-infra/metrics/influxdb/gsutil/Dockerfile @@ -16,7 +16,7 @@ # limitations under the License. ################################################################################ -FROM python:3.9-slim +FROM python:3.10-slim # google-compute-engine package allows to obtain credentials for service # account specified in .boto file. diff --git a/.test-infra/metrics/sync/github/Dockerfile b/.test-infra/metrics/sync/github/Dockerfile index 3116d0f211fa..358f6ba65115 100644 --- a/.test-infra/metrics/sync/github/Dockerfile +++ b/.test-infra/metrics/sync/github/Dockerfile @@ -16,7 +16,7 @@ # limitations under the License. ################################################################################ -FROM python:3.9-slim +FROM python:3.10-slim WORKDIR /usr/src/app diff --git a/.test-infra/metrics/sync/jenkins/Dockerfile b/.test-infra/metrics/sync/jenkins/Dockerfile index 62829ada38ee..160a7fd206e2 100644 --- a/.test-infra/metrics/sync/jenkins/Dockerfile +++ b/.test-infra/metrics/sync/jenkins/Dockerfile @@ -16,7 +16,7 @@ # limitations under the License. ################################################################################ -FROM python:3.9-slim +FROM python:3.10-slim WORKDIR /usr/src/app diff --git a/.test-infra/mock-apis/poetry.lock b/.test-infra/mock-apis/poetry.lock index 5ac83888b96d..a65afd86540a 100644 --- a/.test-infra/mock-apis/poetry.lock +++ b/.test-infra/mock-apis/poetry.lock @@ -196,7 +196,7 @@ name = "setuptools" version = "78.1.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ {file = "setuptools-78.1.1-py3-none-any.whl", hash = "sha256:c3a9c4211ff4c309edb8b8c4f1cbfa7ae324c4ba9f91ff254e3d305b9fd54561"}, @@ -209,7 +209,7 @@ core = ["importlib_metadata (>=6) ; python_version < \"3.10\"", "jaraco.functool cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] enabler = ["pytest-enabler (>=2.2)"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21) ; python_version >= \"3.10\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] type = ["importlib_metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.develop (>=7.21) ; sys_platform != \"cygwin\"", "mypy (==1.14.*)", "pytest-mypy"] [[package]] @@ -226,5 +226,5 @@ files = [ [metadata] lock-version = "2.1" -python-versions = "^3.9" +python-versions = "^3.10" content-hash = "9c0ea7a2921007c3a26d09de1ae342aa7afc61a32445b13b4702fcd4fee5aa0f" diff --git a/.test-infra/mock-apis/pyproject.toml b/.test-infra/mock-apis/pyproject.toml index c98d9152cfb9..b04d106f8a45 100644 --- a/.test-infra/mock-apis/pyproject.toml +++ b/.test-infra/mock-apis/pyproject.toml @@ -27,7 +27,7 @@ packages = [ ] [tool.poetry.dependencies] -python = "^3.9" +python = "^3.10" google = "^3.0.0" grpcio = "^1.53.0" grpcio-tools = "^1.53.0" diff --git a/.test-infra/tools/python_installer.sh b/.test-infra/tools/python_installer.sh index 04e10555243a..a242e1335b01 100644 --- a/.test-infra/tools/python_installer.sh +++ b/.test-infra/tools/python_installer.sh @@ -20,7 +20,7 @@ set -euo pipefail # Variable containing the python versions to install -python_versions_arr=("3.9.16" "3.10.10" "3.11.4", "3.12.6") +python_versions_arr=("3.10.10" "3.11.4" "3.12.6" "3.13.9") # Install pyenv dependencies. pyenv_dep(){ diff --git a/build.gradle.kts b/build.gradle.kts index 3a96f5341dbc..403517f5ed9f 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -485,7 +485,6 @@ tasks.register("playgroundPreCommit") { tasks.register("pythonPreCommit") { dependsOn(":sdks:python:test-suites:tox:pycommon:preCommitPyCommon") - dependsOn(":sdks:python:test-suites:tox:py39:preCommitPy39") dependsOn(":sdks:python:test-suites:tox:py310:preCommitPy310") dependsOn(":sdks:python:test-suites:tox:py311:preCommitPy311") dependsOn(":sdks:python:test-suites:tox:py312:preCommitPy312") @@ -502,7 +501,6 @@ tasks.register("pythonDocsPreCommit") { } tasks.register("pythonDockerBuildPreCommit") { - dependsOn(":sdks:python:container:py39:docker") dependsOn(":sdks:python:container:py310:docker") dependsOn(":sdks:python:container:py311:docker") dependsOn(":sdks:python:container:py312:docker") @@ -782,21 +780,12 @@ tasks.register("validateChanges") { } } -tasks.register("python39PostCommit") { - dependsOn(":sdks:python:test-suites:dataflow:py39:postCommitIT") - dependsOn(":sdks:python:test-suites:direct:py39:postCommitIT") - dependsOn(":sdks:python:test-suites:direct:py39:hdfsIntegrationTest") - dependsOn(":sdks:python:test-suites:direct:py39:azureIntegrationTest") - dependsOn(":sdks:python:test-suites:portable:py39:postCommitPy39") - // TODO (https://github.com/apache/beam/issues/23966) - // Move this to Python 3.10 test suite once tfx-bsl has python 3.10 wheel. - dependsOn(":sdks:python:test-suites:direct:py39:inferencePostCommitIT") -} - tasks.register("python310PostCommit") { dependsOn(":sdks:python:test-suites:dataflow:py310:postCommitIT") dependsOn(":sdks:python:test-suites:direct:py310:postCommitIT") dependsOn(":sdks:python:test-suites:portable:py310:postCommitPy310") + dependsOn(":sdks:python:test-suites:direct:py310:hdfsIntegrationTest") + dependsOn(":sdks:python:test-suites:direct:py310:azureIntegrationTest") // TODO: https://github.com/apache/beam/issues/22651 // The default container uses Python 3.10. The goal here is to // duild Docker images for TensorRT tests during run time for python versions @@ -827,12 +816,12 @@ tasks.register("python313PostCommit") { } tasks.register("portablePythonPreCommit") { - dependsOn(":sdks:python:test-suites:portable:py39:preCommitPy39") + dependsOn(":sdks:python:test-suites:portable:py310:preCommitPy310") dependsOn(":sdks:python:test-suites:portable:py313:preCommitPy313") } tasks.register("pythonSparkPostCommit") { - dependsOn(":sdks:python:test-suites:portable:py39:sparkValidatesRunner") + dependsOn(":sdks:python:test-suites:portable:py310:sparkValidatesRunner") dependsOn(":sdks:python:test-suites:portable:py313:sparkValidatesRunner") } @@ -856,15 +845,15 @@ tasks.register("javaExamplesDataflowPrecommit") { tasks.register("whitespacePreCommit") { // TODO(https://github.com/apache/beam/issues/20209): Find a better way to specify the tasks without hardcoding py version. - dependsOn(":sdks:python:test-suites:tox:py39:archiveFilesToLint") - dependsOn(":sdks:python:test-suites:tox:py39:unpackFilesToLint") - dependsOn(":sdks:python:test-suites:tox:py39:whitespacelint") + dependsOn(":sdks:python:test-suites:tox:py310:archiveFilesToLint") + dependsOn(":sdks:python:test-suites:tox:py310:unpackFilesToLint") + dependsOn(":sdks:python:test-suites:tox:py310:whitespacelint") } tasks.register("typescriptPreCommit") { // TODO(https://github.com/apache/beam/issues/20209): Find a better way to specify the tasks without hardcoding py version. - dependsOn(":sdks:python:test-suites:tox:py39:eslint") - dependsOn(":sdks:python:test-suites:tox:py39:jest") + dependsOn(":sdks:python:test-suites:tox:py310:eslint") + dependsOn(":sdks:python:test-suites:tox:py310:jest") } tasks.register("pushAllRunnersDockerImages") { diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index 6b6cbe4acd19..b9e1aa71bd8e 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -3093,9 +3093,8 @@ class BeamModulePlugin implements Plugin { // Python interpreter version for virtualenv setup and test run. This value can be // set from commandline with -PpythonVersion, or in build script of certain project. // If none of them applied, version set here will be used as default value. - // TODO(BEAM-12000): Move default value to Py3.9. project.ext.pythonVersion = project.hasProperty('pythonVersion') ? - project.pythonVersion : '3.9' + project.pythonVersion : '3.' // Set min/max python versions used for containers and supported versions. project.ext.minPythonVersion = 9 @@ -3310,7 +3309,6 @@ class BeamModulePlugin implements Plugin { mustRunAfter = [ ":runners:flink:${project.ext.latestFlinkVersion}:job-server:shadowJar", ':runners:spark:3:job-server:shadowJar', - ':sdks:python:container:py39:docker', ':sdks:python:container:py310:docker', ':sdks:python:container:py311:docker', ':sdks:python:container:py312:docker', diff --git a/contributor-docs/python-tips.md b/contributor-docs/python-tips.md index 37c0682e8d23..409a5ed76c22 100644 --- a/contributor-docs/python-tips.md +++ b/contributor-docs/python-tips.md @@ -57,22 +57,22 @@ Installation steps may look as follows: 2. Install Python intepreter for each supported Python minor version. Leaving out the patch version will install the latest. ```bash -pyenv install 3.9 pyenv install 3.10 pyenv install 3.11 pyenv install 3.12 +pyenv install 3.13 ``` 3. Make installed interpreters available in your shell by first running: ```bash -pyenv global 3.9 3.10 3.11 3.12 +pyenv global 3.10 3.11 3.12 3.13 ``` 4. (**OPTIONAL**) Pyenv will sometimes [fail to make these interpreters directly available](https://github.com/pyenv/pyenv/issues/34) without a local configuration. If you see errors trying to use `python3.x`, then also run `pyenv local`   ```bash -pyenv local 3.9 3.10 3.11 3.12 +pyenv local 3.10 3.11 3.12 3.13 ``` After these steps, all `python3.x` interpreters should be available in your shell. The first version in the list passed to pyenv global will be used as default `python` / `python3` interpreter if the minor version is not specified. @@ -156,11 +156,11 @@ curl https://pyenv.run | bash # Run the outputted commands to initialize pyenv in .bashrc ``` -#### Example: How to Run Unit Tests with PyCharm Using Python 3.9.4 in a virtualenv -1. Install Python 3.9.4 and create a virtualenv +#### Example: How to Run Unit Tests with PyCharm Using Python 3.10.10 in a virtualenv +1. Install Python 3.10.10 and create a virtualenv ```bash -pyenv install 3.9.4 -pyenv virtualenv 3.9.4 ENV_NAME +pyenv install 3.10.10 +pyenv virtualenv 3.10.10 ENV_NAME pyenv activate ENV_NAME ``` @@ -171,7 +171,7 @@ pip install --upgrade pip setuptools 3. Set up PyCharm * Start by adding a new project interpreter (from the bottom right or in Settings). - * Select Existing environment and the interpreter, which should be under ~/.pyenv/versions/3.9.4/envs/ENV_NAME/bin/python or ~/.pyenv/versions/ENV_NAME/bin/python. + * Select Existing environment and the interpreter, which should be under ~/.pyenv/versions/3.10.10/envs/ENV_NAME/bin/python or ~/.pyenv/versions/ENV_NAME/bin/python. * Switch interpreters at the bottom right. #### Cleaning up environments @@ -509,7 +509,7 @@ NOTE for RELEASE MANAGERS: We should update dependencies at least once per relea You may see that the pip command will lead to segmentation fault as well. If this happens, remove the python version from pyenv, and reinstall the version like this. ```bash -CFLAGS="-O2" pyenv install 3.9.4 +CFLAGS="-O2" pyenv install 3.10.10 ``` There have been issues with older Python versions. See [here](https://github.com/pyenv/pyenv/issues/2046) for details. \ No newline at end of file diff --git a/contributor-docs/release-guide.md b/contributor-docs/release-guide.md index c0209d6071b7..0be9040760fb 100644 --- a/contributor-docs/release-guide.md +++ b/contributor-docs/release-guide.md @@ -582,7 +582,7 @@ with tags: `${RELEASE_VERSION}rc${RC_NUM}` Verify that third party licenses are included in Docker. You can do this with a simple script: RC_TAG=${RELEASE_VERSION}rc${RC_NUM} - for pyver in 3.9 3.10 3.11 3.12; do + for pyver in 3.10 3.11 3.12 3.13; do docker run --rm --entrypoint sh \ apache/beam_python${pyver}_sdk:${RC_TAG} \ -c 'ls -al /opt/apache/beam/third_party_licenses/ | wc -l' diff --git a/dev-support/docker/pkglist b/dev-support/docker/pkglist index f0a46c34d0ae..c8ab6bc6b6bc 100644 --- a/dev-support/docker/pkglist +++ b/dev-support/docker/pkglist @@ -32,10 +32,6 @@ time openjdk-11-jdk python3-setuptools python3-pip -python3.9 -python3.9-dev -python3.9-distutils -python3.9-venv python3.10 python3.10-dev python3.10-distutils @@ -47,5 +43,8 @@ python3.11-venv python3.12 python3.12-dev python3.12-venv +python3.13 +python3.13-dev +python3.13-venv tox docker.io diff --git a/gradle.properties b/gradle.properties index 961ce2492d18..510122c4e7b0 100644 --- a/gradle.properties +++ b/gradle.properties @@ -41,4 +41,4 @@ docker_image_default_repo_prefix=beam_ # supported flink versions flink_versions=1.17,1.18,1.19 # supported python versions -python_versions=3.9,3.10,3.11,3.12,3.13 +python_versions=3.10,3.11,3.12,3.13 diff --git a/local-env-setup.sh b/local-env-setup.sh index 1cefa2990e21..209c1dee2510 100755 --- a/local-env-setup.sh +++ b/local-env-setup.sh @@ -55,7 +55,7 @@ if [ "$kernelname" = "Linux" ]; then exit fi - for ver in 3.9 3.10 3.11 3.12 3.13 3; do + for ver in 3.10 3.11 3.12 3.13 3; do apt install --yes python$ver-venv done @@ -89,7 +89,7 @@ elif [ "$kernelname" = "Darwin" ]; then echo "Installing openjdk@8" brew install openjdk@8 fi - for ver in 3.9 3.10 3.11 3.12 3.13; do + for ver in 3.10 3.11 3.12 3.13; do if brew ls --versions python@$ver > /dev/null; then echo "python@$ver already installed. Skipping" brew info python@$ver diff --git a/playground/infrastructure/cloudbuild/playground_cd_examples.sh b/playground/infrastructure/cloudbuild/playground_cd_examples.sh index e571bc9fc9d9..410aae1249d3 100644 --- a/playground/infrastructure/cloudbuild/playground_cd_examples.sh +++ b/playground/infrastructure/cloudbuild/playground_cd_examples.sh @@ -97,15 +97,15 @@ LogOutput "Installing python and dependencies." export DEBIAN_FRONTEND=noninteractive apt install -y apt-transport-https ca-certificates software-properties-common curl unzip apt-utils > /dev/null 2>&1 add-apt-repository -y ppa:deadsnakes/ppa > /dev/null 2>&1 && apt update > /dev/null 2>&1 -apt install -y python3.9 python3-distutils python3-pip > /dev/null 2>&1 +apt install -y python3.10 python3-distutils python3-pip > /dev/null 2>&1 apt install -y --reinstall python3-distutils > /dev/null 2>&1 apt install -y python3-virtualenv virtualenv play_venv source play_venv/bin/activate pip install --upgrade google-api-python-client > /dev/null 2>&1 -python3.9 -m pip install pip --upgrade > /dev/null 2>&1 -ln -s /usr/bin/python3.9 /usr/bin/python > /dev/null 2>&1 -apt install -y python3.9-venv > /dev/null 2>&1 +python3.10 -m pip install pip --upgrade > /dev/null 2>&1 +ln -s /usr/bin/python3.10 /usr/bin/python > /dev/null 2>&1 +apt install -y python3.10-venv > /dev/null 2>&1 LogOutput "Installing Python packages from beam/playground/infrastructure/requirements.txt" cd $BEAM_ROOT_DIR diff --git a/playground/infrastructure/cloudbuild/playground_ci_examples.sh b/playground/infrastructure/cloudbuild/playground_ci_examples.sh index ed5b70148623..aa5c94f7e452 100755 --- a/playground/infrastructure/cloudbuild/playground_ci_examples.sh +++ b/playground/infrastructure/cloudbuild/playground_ci_examples.sh @@ -94,12 +94,12 @@ export DEBIAN_FRONTEND=noninteractive LogOutput "Installing Python environment" apt-get install -y apt-transport-https ca-certificates software-properties-common curl unzip apt-utils > /dev/null add-apt-repository -y ppa:deadsnakes/ppa > /dev/null && apt update > /dev/null -apt install -y python3.9 python3-distutils python3-pip > /dev/null +apt install -y python3.10 python3-distutils python3-pip > /dev/null apt install --reinstall python3-distutils > /dev/null pip install --upgrade google-api-python-client > /dev/null -python3.9 -m pip install pip --upgrade > /dev/null -ln -s /usr/bin/python3.9 /usr/bin/python > /dev/null -apt install python3.9-venv > /dev/null +python3.10 -m pip install pip --upgrade > /dev/null +ln -s /usr/bin/python3.10 /usr/bin/python > /dev/null +apt install python3.10-venv > /dev/null LogOutput "Installing Python packages from beam/playground/infrastructure/requirements.txt" pip install -r $BEAM_ROOT_DIR/playground/infrastructure/requirements.txt diff --git a/release/src/main/Dockerfile b/release/src/main/Dockerfile index 6503c5c42ba8..36171674d452 100644 --- a/release/src/main/Dockerfile +++ b/release/src/main/Dockerfile @@ -46,7 +46,8 @@ RUN curl https://pyenv.run | bash && \ pyenv install 3.10.7 && \ pyenv install 3.11.3 && \ pyenv install 3.12.3 && \ - pyenv global 3.9.4 3.10.7 3.11.3 3.12.3 + pyenv install 3.13.9 && \ + pyenv global 3.10.7 3.11.3 3.12.3 3.13.9 # Install a Go version >= 1.16 so we can bootstrap higher # Go versions diff --git a/release/src/main/python-release/python_release_automation.sh b/release/src/main/python-release/python_release_automation.sh index 248bdd9b65ac..892e1c36e9a2 100755 --- a/release/src/main/python-release/python_release_automation.sh +++ b/release/src/main/python-release/python_release_automation.sh @@ -19,7 +19,7 @@ source release/src/main/python-release/run_release_candidate_python_quickstart.sh source release/src/main/python-release/run_release_candidate_python_mobile_gaming.sh -for version in 3.9 3.10 3.11 3.12 +for version in 3.10 3.11 3.12 3.13 do run_release_candidate_python_quickstart "tar" "python${version}" run_release_candidate_python_mobile_gaming "tar" "python${version}" diff --git a/sdks/python/apache_beam/__init__.py b/sdks/python/apache_beam/__init__.py index 690c45b08381..542f75d54d7c 100644 --- a/sdks/python/apache_beam/__init__.py +++ b/sdks/python/apache_beam/__init__.py @@ -70,7 +70,7 @@ import warnings if sys.version_info.major == 3: - if sys.version_info.minor <= 8 or sys.version_info.minor >= 14: + if sys.version_info.minor <= 9 or sys.version_info.minor >= 14: warnings.warn( 'This version of Apache Beam has not been sufficiently tested on ' 'Python %s.%s. You may encounter bugs or missing features.' % diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py index 38cdb62ecdbe..164ace532b23 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py @@ -84,7 +84,7 @@ _LOGGER = logging.getLogger(__name__) -_PYTHON_VERSIONS_SUPPORTED_BY_DATAFLOW = ['3.9', '3.10', '3.11', '3.12', '3.13'] +_PYTHON_VERSIONS_SUPPORTED_BY_DATAFLOW = ['3.10', '3.11', '3.12', '3.13'] class Environment(object): diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py b/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py index 94edc507cde7..b767cef86b2e 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py @@ -1055,12 +1055,12 @@ def test_interpreter_version_check_fails_py38(self): @mock.patch( 'apache_beam.runners.dataflow.internal.apiclient.sys.version_info', - (3, 9, 6)) + (3, 10, 10)) @mock.patch( 'apache_beam.runners.dataflow.internal.apiclient.' 'beam_version.__version__', '2.2.0') - def test_interpreter_version_check_passes_py39(self): + def test_interpreter_version_check_passes_py310(self): pipeline_options = PipelineOptions([]) apiclient._verify_interpreter_version_is_supported(pipeline_options) diff --git a/sdks/python/expansion-service-container/Dockerfile b/sdks/python/expansion-service-container/Dockerfile index 4e82165f594c..e825d852479d 100644 --- a/sdks/python/expansion-service-container/Dockerfile +++ b/sdks/python/expansion-service-container/Dockerfile @@ -17,8 +17,8 @@ ############################################################################### # We just need to support one Python version supported by Beam. -# Picking the current default Beam Python version which is Python 3.9. -FROM python:3.9-bookworm as expansion-service +# Picking the current default Beam Python version which is Python 3.10. +FROM python:3.10-bookworm as expansion-service LABEL Author "Apache Beam " ARG TARGETOS ARG TARGETARCH diff --git a/sdks/python/mypy.ini b/sdks/python/mypy.ini index ee76089fec0b..f22258a13953 100644 --- a/sdks/python/mypy.ini +++ b/sdks/python/mypy.ini @@ -16,7 +16,7 @@ # [mypy] -python_version = 3.9 +python_version = 3.10 ignore_missing_imports = true follow_imports = normal warn_no_return = true diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 719d188ed266..1427e598a0f9 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -297,7 +297,7 @@ def get_portability_package_data(): return files -python_requires = '>=3.9' +python_requires = '>=3.10' if sys.version_info.major == 3 and sys.version_info.minor >= 14: warnings.warn( @@ -618,7 +618,6 @@ def get_portability_package_data(): 'Intended Audience :: End Users/Desktop', 'License :: OSI Approved :: Apache Software License', 'Operating System :: POSIX :: Linux', - 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.12', diff --git a/sdks/python/test-suites/gradle.properties b/sdks/python/test-suites/gradle.properties index 99352a61c401..fcdd6f17eeaf 100644 --- a/sdks/python/test-suites/gradle.properties +++ b/sdks/python/test-suites/gradle.properties @@ -23,34 +23,34 @@ # dataflow test-suites # (TODO): https://github.com/apache/beam/issues/21971 # Add python 3.10 to dataflow test-suites -dataflow_precommit_it_task_py_versions=3.9,3.13 -dataflow_mongodbio_it_task_py_versions=3.9 -dataflow_chicago_taxi_example_task_py_versions=3.9 +dataflow_precommit_it_task_py_versions=3.10,3.13 +dataflow_mongodbio_it_task_py_versions=3.10 +dataflow_chicago_taxi_example_task_py_versions=3.10 # TODO: Enable following tests after making sure we have enough capacity. -dataflow_validates_runner_batch_tests=3.9,3.13 -dataflow_validates_runner_streaming_tests=3.9,3.13 +dataflow_validates_runner_batch_tests=3.10,3.13 +dataflow_validates_runner_streaming_tests=3.10,3.13 dataflow_examples_postcommit_py_versions=3.13 # TFX_BSL is not yet supported on Python 3.10. -dataflow_cloudml_benchmark_tests_py_versions=3.9 +dataflow_cloudml_benchmark_tests_py_versions=3.10 # direct runner test-suites direct_mongodbio_it_task_py_versions=3.13 # flink runner test-suites flink_validates_runner_precommit_py_versions=3.13 -flink_validates_runner_postcommit_py_versions=3.9,3.13 -flink_examples_postcommit_py_versions=3.9,3.13 +flink_validates_runner_postcommit_py_versions=3.10,3.13 +flink_examples_postcommit_py_versions=3.10,3.13 # samza runner test-suites -samza_validates_runner_postcommit_py_versions=3.9,3.13 +samza_validates_runner_postcommit_py_versions=3.10,3.13 # spark runner test-suites -spark_examples_postcommit_py_versions=3.9,3.13 +spark_examples_postcommit_py_versions=3.10,3.13 # prism runner test-suites prism_validates_runner_precommit_py_versions=3.13 -prism_validates_runner_postcommit_py_versions=3.9,3.13 -prism_examples_postcommit_py_versions=3.9,3.13 +prism_validates_runner_postcommit_py_versions=3.10,3.13 +prism_examples_postcommit_py_versions=3.10,3.13 # cross language postcommit python test suites -cross_language_validates_py_versions=3.9,3.13 +cross_language_validates_py_versions=3.10,3.13 diff --git a/sdks/python/test-suites/tox/py310/build.gradle b/sdks/python/test-suites/tox/py310/build.gradle index f1e40a17951f..3099a67f980e 100644 --- a/sdks/python/test-suites/tox/py310/build.gradle +++ b/sdks/python/test-suites/tox/py310/build.gradle @@ -26,5 +26,211 @@ applyPythonNature() // Required to setup a Python 3 virtualenv and task names. pythonVersion = '3.10' +def posargs = project.findProperty("posargs") ?: "" + apply from: "../common.gradle" +toxTask "testPy310CloudCoverage", "py310-cloudcoverage", "${posargs}" +test.dependsOn "testPy310CloudCoverage" +project.tasks.register("preCommitPyCoverage") { + dependsOn = ["testPy310CloudCoverage"] +} + +// Dep Postcommit runs test suites that evaluate compatibility of particular +// dependencies. Each suite is exercised on at most one python version. +// +// Should still leave at least one version in PreCommit unless the marked tests +// are also exercised by existing PreCommit +// e.g. pyarrow and pandas also run on PreCommit Dataframe and Coverage +project.tasks.register("postCommitPyDep") {} + +// Create a test task for supported major versions of pyarrow +// We should have a test for the lowest supported version and +// For versions that we would like to prioritize for testing, +// for example versions released in a timeframe of last 1-2 years. + +toxTask "testPy310pyarrow-3", "py310-pyarrow-3", "${posargs}" +test.dependsOn "testPy310pyarrow-3" +postCommitPyDep.dependsOn "testPy310pyarrow-3" + +toxTask "testPy310pyarrow-9", "py310-pyarrow-9", "${posargs}" +test.dependsOn "testPy310pyarrow-9" +postCommitPyDep.dependsOn "testPy310pyarrow-9" + +toxTask "testPy310pyarrow-10", "py310-pyarrow-10", "${posargs}" +test.dependsOn "testPy310pyarrow-10" +postCommitPyDep.dependsOn "testPy310pyarrow-10" + +toxTask "testPy310pyarrow-11", "py310-pyarrow-11", "${posargs}" +test.dependsOn "testPy310pyarrow-11" +postCommitPyDep.dependsOn "testPy310pyarrow-11" + +toxTask "testPy310pyarrow-12", "py310-pyarrow-12", "${posargs}" +test.dependsOn "testPy310pyarrow-12" +postCommitPyDep.dependsOn "testPy310pyarrow-12" + +toxTask "testPy310pyarrow-13", "py310-pyarrow-13", "${posargs}" +test.dependsOn "testPy310pyarrow-13" +postCommitPyDep.dependsOn "testPy310pyarrow-13" + +toxTask "testPy310pyarrow-14", "py310-pyarrow-14", "${posargs}" +test.dependsOn "testPy310pyarrow-14" +postCommitPyDep.dependsOn "testPy310pyarrow-14" + +toxTask "testPy310pyarrow-15", "py310-pyarrow-15", "${posargs}" +test.dependsOn "testPy310pyarrow-15" +postCommitPyDep.dependsOn "testPy310pyarrow-15" + +toxTask "testPy310pyarrow-16", "py310-pyarrow-16", "${posargs}" +test.dependsOn "testPy310pyarrow-16" +postCommitPyDep.dependsOn "testPy310pyarrow-16" + +toxTask "testPy310pyarrow-17", "py310-pyarrow-17", "${posargs}" +test.dependsOn "testPy310pyarrow-17" +postCommitPyDep.dependsOn "testPy310pyarrow-17" + +toxTask "testPy310pyarrow-18", "py310-pyarrow-18", "${posargs}" +test.dependsOn "testPy310pyarrow-18" +postCommitPyDep.dependsOn "testPy310pyarrow-18" + +// Create a test task for each supported minor version of pandas +toxTask "testPy310pandas-14", "py310-pandas-14", "${posargs}" +test.dependsOn "testPy310pandas-14" +postCommitPyDep.dependsOn "testPy310pandas-14" + +toxTask "testPy310pandas-15", "py310-pandas-15", "${posargs}" +test.dependsOn "testPy310pandas-15" +postCommitPyDep.dependsOn "testPy310pandas-15" + +toxTask "testPy310pandas-20", "py310-pandas-20", "${posargs}" +test.dependsOn "testPy310pandas-20" +postCommitPyDep.dependsOn "testPy310pandas-20" + +// TODO(https://github.com/apache/beam/issues/31192): Add below suites +// after dependency compat tests suite switches to Python 3.9 or we add +// Python 2.2 support. + +// toxTask "testPy310pandas-21", "py310-pandas-21", "${posargs}" +// test.dependsOn "testPy310pandas-21" +// postCommitPyDep.dependsOn "testPy310pandas-21" + +// toxTask "testPy310pandas-22", "py310-pandas-22", "${posargs}" +// test.dependsOn "testPy310pandas-22" +// postCommitPyDep.dependsOn "testPy310pandas-22" + +// TODO(https://github.com/apache/beam/issues/30908): Revise what are we testing + +// Create a test task for each minor version of pytorch +toxTask "testPy310pytorch-19", "py310-pytorch-19", "${posargs}" +test.dependsOn "testPy310pytorch-19" +postCommitPyDep.dependsOn "testPy310pytorch-19" + +toxTask "testPy310pytorch-110", "py310-pytorch-110", "${posargs}" +test.dependsOn "testPy310pytorch-110" +postCommitPyDep.dependsOn "testPy310pytorch-110" + +toxTask "testPy310pytorch-111", "py310-pytorch-111", "${posargs}" +test.dependsOn "testPy310pytorch-111" +postCommitPyDep.dependsOn "testPy310pytorch-111" + +toxTask "testPy310pytorch-112", "py310-pytorch-112", "${posargs}" +test.dependsOn "testPy310pytorch-112" +postCommitPyDep.dependsOn "testPy310pytorch-112" + +toxTask "testPy310pytorch-113", "py310-pytorch-113", "${posargs}" +test.dependsOn "testPy310pytorch-113" +postCommitPyDep.dependsOn "testPy310pytorch-113" + +// run on precommit +toxTask "testPy310pytorch-200", "py310-pytorch-200", "${posargs}" +test.dependsOn "testPy310pytorch-200" +postCommitPyDep.dependsOn "testPy310pytorch-200" + +toxTask "testPy310tft-113", "py310-tft-113", "${posargs}" +test.dependsOn "testPy310tft-113" +postCommitPyDep.dependsOn "testPy310tft-113" + +// TODO(https://github.com/apache/beam/issues/25796) - uncomment onnx tox task once onnx supports protobuf 4.x.x +// Create a test task for each minor version of onnx +// toxTask "testPy310onnx-113", "py310-onnx-113", "${posargs}" +// test.dependsOn "testPy310onnx-113" +// postCommitPyDep.dependsOn "testPy310onnx-113" + +// Create a test task for each minor version of tensorflow +toxTask "testPy310tensorflow-212", "py310-tensorflow-212", "${posargs}" +test.dependsOn "testPy310tensorflow-212" +postCommitPyDep.dependsOn "testPy310tensorflow-212" + +// Create a test task for each minor version of transformers +toxTask "testPy310transformers-428", "py310-transformers-428", "${posargs}" +test.dependsOn "testPy310transformers-428" +postCommitPyDep.dependsOn "testPy310transformers-428" + +toxTask "testPy310transformers-447", "py310-transformers-447", "${posargs}" +test.dependsOn "testPy310transformers-447" +postCommitPyDep.dependsOn "testPy310transformers-447" + +toxTask "testPy310transformers-448", "py310-transformers-448", "${posargs}" +test.dependsOn "testPy310transformers-448" +postCommitPyDep.dependsOn "testPy310transformers-448" + +toxTask "testPy310transformers-latest", "py310-transformers-latest", "${posargs}" +test.dependsOn "testPy310transformers-latest" +postCommitPyDep.dependsOn "testPy310transformers-latest" + +toxTask "testPy310embeddingsMLTransform", "py310-embeddings", "${posargs}" +test.dependsOn "testPy310embeddingsMLTransform" +postCommitPyDep.dependsOn "testPy310embeddingsMLTransform" + +// Part of MLTransform embeddings test suite but requires tensorflow hub, which we need to test on +// mutliple versions so keeping this suite separate. +toxTask "testPy310TensorflowHubEmbeddings-014", "py310-TFHubEmbeddings-014", "${posargs}" +test.dependsOn "testPy310TensorflowHubEmbeddings-014" +postCommitPyDep.dependsOn "testPy310TensorflowHubEmbeddings-014" + +toxTask "testPy310TensorflowHubEmbeddings-015", "py310-TFHubEmbeddings-015", "${posargs}" +test.dependsOn "testPy310TensorflowHubEmbeddings-015" +postCommitPyDep.dependsOn "testPy310TensorflowHubEmbeddings-015" + +toxTask "whitespacelint", "whitespacelint", "${posargs}" + +task archiveFilesToLint(type: Zip) { + archiveFileName = "files-to-whitespacelint.zip" + destinationDirectory = file("$buildDir/dist") + + from ("$rootProject.projectDir") { + include "**/*.md" + include "**/build.gradle" + include '**/build.gradle.kts' + exclude '**/build/**' // intermediate build directory + exclude 'website/www/site/themes/docsy/**' // fork to google/docsy + exclude "**/node_modules/*" + exclude "**/.gogradle/*" + } +} + +task unpackFilesToLint(type: Copy) { + from zipTree("$buildDir/dist/files-to-whitespacelint.zip") + into "$buildDir/files-to-whitespacelint" +} + +whitespacelint.dependsOn archiveFilesToLint, unpackFilesToLint +unpackFilesToLint.dependsOn archiveFilesToLint +archiveFilesToLint.dependsOn cleanPython + +toxTask "jest", "jest", "${posargs}" + +toxTask "eslint", "eslint", "${posargs}" + +task copyTsSource(type: Copy) { + from ("$rootProject.projectDir") { + include "sdks/python/apache_beam/runners/interactive/extensions/**/*" + exclude "sdks/python/apache_beam/runners/interactive/extensions/**/lib/*" + exclude "sdks/python/apache_beam/runners/interactive/extensions/**/node_modules/*" + } + into "$buildDir/ts" +} + +jest.dependsOn copyTsSource +eslint.dependsOn copyTsSource +copyTsSource.dependsOn cleanPython diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini index 8ea95ad8fc8b..d5d257fdb7d8 100644 --- a/sdks/python/tox.ini +++ b/sdks/python/tox.ini @@ -17,7 +17,7 @@ [tox] # new environments will be excluded by default unless explicitly added to envlist. -envlist = py39,py310,py311,py312,py313,py39-{cloud,cloudcoverage,dask},py310-{cloud,dask},py311-{cloud,dask},py312-{cloud,dask},py313-{cloud,dask},docs,lint,mypy,whitespacelint +envlist = py310,py311,py312,py313,py310-{cloud,cloudcoverage,dask},py311-{cloud,dask},py312-{cloud,dask},py313-{cloud,dask},docs,lint,mypy,whitespacelint toxworkdir = {toxinidir}/target/{env:ENV_NAME:.tox} [pycodestyle] @@ -67,7 +67,7 @@ commands_post = commands = false {envname} is misconfigured -[testenv:py{39,310,311,312,313}] +[testenv:py{310,311,312,313}] commands_pre = python --version pip --version @@ -79,7 +79,7 @@ commands = python apache_beam/examples/complete/autocomplete_test.py bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" -[testenv:py{39,310,311,312,313}-macos] +[testenv:py{310,311,312,313}-macos] commands_pre = python --version pip --version @@ -89,21 +89,21 @@ commands = python apache_beam/examples/complete/autocomplete_test.py bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" -[testenv:py{39,310,311,312,313}-win] +[testenv:py{310,311,312,313}-win] commands = python apache_beam/examples/complete/autocomplete_test.py bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" install_command = {envbindir}/python.exe {envbindir}/pip.exe install --retries 10 {opts} {packages} list_dependencies_command = {envbindir}/python.exe {envbindir}/pip.exe freeze -[testenv:py{39,310,311,312,313}-cloud] +[testenv:py{310,311,312,313}-cloud] ; extras = test,gcp,interactive,dataframe,aws,azure extras = test,gcp,interactive,dataframe,aws,azure commands = python apache_beam/examples/complete/autocomplete_test.py bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" -[testenv:py{39,310,311}-ml] +[testenv:py{310,311}-ml] # Don't set TMPDIR to avoid "AF_UNIX path too long" errors in certain tests. deps = pip==25.0.1 @@ -140,14 +140,14 @@ commands = /bin/sh -c "pip freeze | grep -E tensorflow" bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" -[testenv:py{39,310,311,31,313}-dask] +[testenv:py{310,311,31,313}-dask] extras = test,dask,dataframes commands_pre = pip install 'distributed>=2024.4.2' 'dask>=2024.4.2' commands = bash {toxinidir}/scripts/run_pytest.sh {envname} {toxinidir}/apache_beam/runners/dask/ -[testenv:py{39,310,311,312,313}-win-dask] +[testenv:py{310,311,312,313}-win-dask] # use the tight range since the latest dask requires cloudpickle 3.0 commands_pre = pip install 'distributed>=2024.4.2,<2024.9.0' 'dask>=2024.4.2,<2024.9.0' @@ -157,7 +157,7 @@ commands = install_command = {envbindir}/python.exe {envbindir}/pip.exe install --retries 10 {opts} {packages} list_dependencies_command = {envbindir}/python.exe {envbindir}/pip.exe freeze -[testenv:py39-cloudcoverage] +[testenv:py310-cloudcoverage] deps = pytest-cov==3.0.0 @@ -342,7 +342,7 @@ extras = test commands = bash {toxinidir}/scripts/pytest_validates_runner.sh {envname} {toxinidir}/apache_beam/runners/portability/prism_runner_test.py {posargs} -[testenv:py{39,310}-pyarrow-{3,9,10,11,12,13,14,15,16,17,18}] +[testenv:py{310,311}-pyarrow-{3,9,10,11,12,13,14,15,16,17,18}] deps = # As a courtesy to users, test against the oldest allowed version of Pyarrow. # We'd have to increase the pyarrow lower bound when Python 3.9 is deprecated. @@ -372,7 +372,7 @@ commands = /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pyarrow {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' -[testenv:py{39,310}-pandas-{14,15,20}] +[testenv:py{310,311}-pandas-{14,15,20}] deps = 14: pandas>=1.4.3,<1.5.0 14: numpy>=1.14.3,<1.27.0 @@ -388,7 +388,7 @@ commands = # Run all DataFrame API unit tests bash {toxinidir}/scripts/run_pytest.sh {envname} 'apache_beam/dataframe' -[testenv:py{39,310}-tft-{113,114}] +[testenv:py{310,311}-tft-{113,114}] deps = # Help pip resolve conflict with typing-extensions due to an old version of tensorflow https://github.com/apache/beam/issues/30852 113: pydantic<2.0 @@ -396,7 +396,7 @@ deps = commands = bash {toxinidir}/scripts/run_pytest.sh {envname} 'apache_beam/ml/transforms apache_beam/examples/snippets/transforms/elementwise/mltransform_test.py' -[testenv:py{39,310}-pytorch-{19,110,111,112,113}] +[testenv:py{310,311}-pytorch-{19,110,111,112,113}] deps = 19: torch>=1.9.0,<1.10.0 110: torch>=1.10.0,<1.11.0 @@ -414,7 +414,7 @@ commands = # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pytorch {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' -[testenv:py{39,310}-pytorch-200] +[testenv:py{310,311}-pytorch-200] deps = 200: torch>=2.0.0,<2.1.0 @@ -446,8 +446,8 @@ commands = # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pytorch {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' -# TODO(https://github.com/apache/beam/issues/25796) - uncomment onnx tox task in tox/py39/build.gradle once onnx supports protobuf 4.x.x -[testenv:py{39,310}-onnx-113] +# TODO(https://github.com/apache/beam/issues/25796) - uncomment onnx tox task in tox/py310/build.gradle once onnx supports protobuf 4.x.x +[testenv:py{310,311}-onnx-113] # TODO(https://github.com/apache/beam/issues/25443) # apparently tox has problem when substitution key has single value. Change back to -onnx-{113,...} # when multiple onnx versions are tested. @@ -466,7 +466,7 @@ commands = # Run all ONNX unit tests pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_onnx {posargs} -[testenv:py39-tensorflow-212] +[testenv:py310-tensorflow-212] deps = 212: tensorflow>=2.12rc1,<2.13 @@ -498,7 +498,7 @@ commands = # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_tf {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' -[testenv:py39-xgboost-{160,170}] +[testenv:py310-xgboost-{160,170}] deps = 160: xgboost>=1.6.0,<1.7.0 @@ -514,7 +514,7 @@ commands = # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_xgboost {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' -[testenv:py{39,310}-transformers-{428,447,448,latest}] +[testenv:py{310,311}-transformers-{428,447,448,latest}] deps = # sentence-transformers 2.2.2 is the latest version that supports transformers 4.28.x 428: sentence-transformers==2.2.2 @@ -540,7 +540,7 @@ commands = # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_transformers {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' -[testenv:py{39,312}-vertex-ai] +[testenv:py{310,313}-vertex-ai] deps = tensorflow==2.12.0 extras = test,gcp @@ -553,7 +553,7 @@ commands = /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_vertex_ai {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' -[testenv:py{39,310}-embeddings] +[testenv:py{310,311}-embeddings] deps = sentence-transformers==3.3.1 accelerate>=1.6.0 @@ -571,7 +571,7 @@ commands = /bin/sh -c 'pytest apache_beam/ml/transforms/embeddings -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' -[testenv:py{39,310}-TFHubEmbeddings-{014,015}] +[testenv:py{310,311}-TFHubEmbeddings-{014,015}] deps = 014: tensorflow-hub>=0.14.0,<0.15.0 # Help pip resolve conflict with typing-extensions due to an old version of tensorboard https://github.com/apache/beam/issues/30852 From d477b91b5d060246443a155d9547107f93c7b7de Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Wed, 29 Oct 2025 12:08:21 -0400 Subject: [PATCH 02/29] add version removal to CHANGES.md --- CHANGES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.md b/CHANGES.md index 5de202a7933f..67ca0b296658 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -83,6 +83,7 @@ ## Deprecations * X behavior is deprecated and will be removed in X versions ([#X](https://github.com/apache/beam/issues/X)). +* Python 3.9 reached EOL in October 2025 and support for the lanugage version has been removed. ([#36665](https://github.com/apache/beam/pull/36665)). ## Bugfixes From b65d2da41e3dad3d39f856a3eba8717db422b546 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Wed, 29 Oct 2025 12:14:55 -0400 Subject: [PATCH 03/29] wrap python version in quotes in build_wheels --- .github/workflows/build_wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 9d27f042f073..99830ca2b0a3 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -92,7 +92,7 @@ jobs: - name: Install python uses: actions/setup-python@v5 with: - python-version: 3.10 + python-version: '3.10' - name: Get tag id: get_tag run: | @@ -246,7 +246,7 @@ jobs: - name: Install Python uses: actions/setup-python@v5 with: - python-version: 3.10 + python-version: '3.10' - uses: docker/setup-qemu-action@v3 if: ${{matrix.os_python.arch == 'aarch64'}} name: Set up QEMU From a82ebffd42aa85916c2e258d3224c24c26bd6f41 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Wed, 29 Oct 2025 12:16:56 -0400 Subject: [PATCH 04/29] wrap remaining bare version sets in quotes --- .github/workflows/dask_runner_tests.yml | 2 +- .github/workflows/run_perf_alert_tool.yml | 2 +- .github/workflows/typescript_tests.yml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dask_runner_tests.yml b/.github/workflows/dask_runner_tests.yml index 055ac91220d0..c2eb1bdcf84c 100644 --- a/.github/workflows/dask_runner_tests.yml +++ b/.github/workflows/dask_runner_tests.yml @@ -44,7 +44,7 @@ jobs: - name: Install python uses: actions/setup-python@v5 with: - python-version: 3.10 + python-version: '3.10' - name: Build source working-directory: ./sdks/python run: pip install -U build && python -m build --sdist diff --git a/.github/workflows/run_perf_alert_tool.yml b/.github/workflows/run_perf_alert_tool.yml index 1e0c1de3c951..5da12f50315f 100644 --- a/.github/workflows/run_perf_alert_tool.yml +++ b/.github/workflows/run_perf_alert_tool.yml @@ -39,7 +39,7 @@ jobs: - name: Install python uses: actions/setup-python@v5 with: - python-version: 3.10 + python-version: '3.10' - name: Install Apache Beam working-directory: ./sdks/python run: pip install -e .[gcp,test] diff --git a/.github/workflows/typescript_tests.yml b/.github/workflows/typescript_tests.yml index af46c68c2f17..5176c1f2c297 100644 --- a/.github/workflows/typescript_tests.yml +++ b/.github/workflows/typescript_tests.yml @@ -110,7 +110,7 @@ jobs: - name: Install Python uses: actions/setup-python@v5 with: - python-version: 3.10 + python-version: '3.10' - name: Setup Beam Python working-directory: ./sdks/python run: | @@ -173,7 +173,7 @@ jobs: - name: Install python uses: actions/setup-python@v5 with: - python-version: 3.10 + python-version: '3.10' - name: Setup Beam Python working-directory: ./sdks/python run: | From e20aaf46ea7b49e737ca0e70a7d63c6f0866e942 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Wed, 29 Oct 2025 12:23:49 -0400 Subject: [PATCH 05/29] fix typo --- .../groovy/org/apache/beam/gradle/BeamModulePlugin.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index c594abe798f4..03d520514dfa 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -3094,10 +3094,10 @@ class BeamModulePlugin implements Plugin { // set from commandline with -PpythonVersion, or in build script of certain project. // If none of them applied, version set here will be used as default value. project.ext.pythonVersion = project.hasProperty('pythonVersion') ? - project.pythonVersion : '3.' + project.pythonVersion : '3.10' // Set min/max python versions used for containers and supported versions. - project.ext.minPythonVersion = 9 + project.ext.minPythonVersion = 10 project.ext.maxPythonVersion = 13 def setupVirtualenv = project.tasks.register('setupVirtualenv') { From eab0cb2d78d26999d70914fd3e99501bb33275b1 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Wed, 29 Oct 2025 12:24:26 -0400 Subject: [PATCH 06/29] change default expansion service container in BeamModulePlugin.groovy --- .../main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index 03d520514dfa..7c9483343626 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -3004,7 +3004,7 @@ class BeamModulePlugin implements Plugin { // Transform service delivers transforms that refer to SDK harness containers with following sufixes. def transformServiceJavaContainerSuffix = 'java11' - def transformServicePythonContainerSuffix = '39' + def transformServicePythonContainerSuffix = '310' def setupTask = project.tasks.register(config.name+"Setup", Exec) { // Containers for main SDKs when running tests. From bf1f46d6062aaf3c762564a200e61d9789ac6bca Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Wed, 29 Oct 2025 12:32:06 -0400 Subject: [PATCH 07/29] bump manual container versions --- .github/workflows/beam_PostCommit_PortableJar_Flink.yml | 2 +- .github/workflows/beam_PostCommit_PortableJar_Spark.yml | 2 +- .github/workflows/beam_PostCommit_Python_Portable_Flink.yml | 2 +- .github/workflows/beam_PreCommit_Flink_Container.yml | 2 +- .github/workflows/beam_PreCommit_Python_Coverage.yml | 2 +- .test-infra/jenkins/build.gradle | 4 ++-- .../www/site/content/en/documentation/runtime/environments.md | 3 +-- 7 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/beam_PostCommit_PortableJar_Flink.yml b/.github/workflows/beam_PostCommit_PortableJar_Flink.yml index e29eedb8cee5..8b3e05dc175f 100644 --- a/.github/workflows/beam_PostCommit_PortableJar_Flink.yml +++ b/.github/workflows/beam_PostCommit_PortableJar_Flink.yml @@ -79,7 +79,7 @@ jobs: CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:python:test-suites:portable:py39:testPipelineJarFlinkRunner + gradle-command: :sdks:python:test-suites:portable:310:testPipelineJarFlinkRunner arguments: | -PpythonVersion=3.10 \ - name: Archive Python Test Results diff --git a/.github/workflows/beam_PostCommit_PortableJar_Spark.yml b/.github/workflows/beam_PostCommit_PortableJar_Spark.yml index 11fbfa43fde4..873c4451c511 100644 --- a/.github/workflows/beam_PostCommit_PortableJar_Spark.yml +++ b/.github/workflows/beam_PostCommit_PortableJar_Spark.yml @@ -79,7 +79,7 @@ jobs: CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:python:test-suites:portable:py39:testPipelineJarSparkRunner + gradle-command: :sdks:python:test-suites:portable:py310:testPipelineJarSparkRunner arguments: | -PpythonVersion=3.10 \ - name: Archive Python Test Results diff --git a/.github/workflows/beam_PostCommit_Python_Portable_Flink.yml b/.github/workflows/beam_PostCommit_Python_Portable_Flink.yml index 35eb4043b56a..f3c032ebffe2 100644 --- a/.github/workflows/beam_PostCommit_Python_Portable_Flink.yml +++ b/.github/workflows/beam_PostCommit_Python_Portable_Flink.yml @@ -83,7 +83,7 @@ jobs: CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:python:test-suites:portable:py39:flinkCompatibilityMatrix${{ matrix.environment_type }} + gradle-command: :sdks:python:test-suites:portable:py310:flinkCompatibilityMatrix${{ matrix.environment_type }} arguments: | -PpythonVersion=3.10 \ - name: Archive Python Test Results diff --git a/.github/workflows/beam_PreCommit_Flink_Container.yml b/.github/workflows/beam_PreCommit_Flink_Container.yml index 5eba6fcc5e2a..0c42fe0d4d87 100644 --- a/.github/workflows/beam_PreCommit_Flink_Container.yml +++ b/.github/workflows/beam_PreCommit_Flink_Container.yml @@ -120,7 +120,7 @@ jobs: if: ${{ github.event_name == 'pull_request_target' }} uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:python:container:py39:docker + gradle-command: :sdks:python:container:py310:docker arguments: | -PpythonVersion=${{ env.PYTHON_VERSION }} \ -Pdocker-repository-root=${{ env.DOCKER_REPOSITORY_ROOT }} \ diff --git a/.github/workflows/beam_PreCommit_Python_Coverage.yml b/.github/workflows/beam_PreCommit_Python_Coverage.yml index 1f76fe23390a..7c8af0f15d98 100644 --- a/.github/workflows/beam_PreCommit_Python_Coverage.yml +++ b/.github/workflows/beam_PreCommit_Python_Coverage.yml @@ -118,7 +118,7 @@ jobs: TC_SLEEP_TIME: "1" uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:python:test-suites:tox:py39:preCommitPyCoverage + gradle-command: :sdks:python:test-suites:tox:py310:preCommitPyCoverage arguments: | -Pposargs="${{ contains(matrix.os, 'self-hosted') && diff --git a/.test-infra/jenkins/build.gradle b/.test-infra/jenkins/build.gradle index 862e8756c6d7..6a77c4b827ed 100644 --- a/.test-infra/jenkins/build.gradle +++ b/.test-infra/jenkins/build.gradle @@ -36,11 +36,11 @@ task generateMetricsReport { doLast { exec { executable 'sh' - args '-c', ". ${envdir}/bin/activate && tox -e py39-test -c ${toxConfigFilePath}" + args '-c', ". ${envdir}/bin/activate && tox -e py310-test -c ${toxConfigFilePath}" } exec { executable 'sh' - args '-c', ". ${envdir}/bin/activate && tox -e py39-generate-report -c ${toxConfigFilePath} -- --influx-db=${influxDb} --influx-host=${influxHost} --influx-port=${influxPort} --output-file=${generateMetricsReportPath}" + args '-c', ". ${envdir}/bin/activate && tox -e py310-generate-report -c ${toxConfigFilePath} -- --influx-db=${influxDb} --influx-host=${influxHost} --influx-port=${influxPort} --output-file=${generateMetricsReportPath}" } logger.info('Create metrics report file {}', generateMetricsReportPath) } diff --git a/website/www/site/content/en/documentation/runtime/environments.md b/website/www/site/content/en/documentation/runtime/environments.md index 82beae16f02b..3ebabf85385d 100644 --- a/website/www/site/content/en/documentation/runtime/environments.md +++ b/website/www/site/content/en/documentation/runtime/environments.md @@ -117,7 +117,6 @@ This method requires building image artifacts from Beam source. For additional i ./gradlew :sdks:java:container:java17:docker ./gradlew :sdks:java:container:java21:docker ./gradlew :sdks:go:container:docker - ./gradlew :sdks:python:container:py39:docker ./gradlew :sdks:python:container:py310:docker ./gradlew :sdks:python:container:py311:docker ./gradlew :sdks:python:container:py312:docker @@ -135,10 +134,10 @@ This method requires building image artifacts from Beam source. For additional i apache/beam_java11_sdk latest sha256:... ... 1 min ago ... apache/beam_java17_sdk latest sha256:... ... 1 min ago ... apache/beam_java21_sdk latest sha256:... ... 1 min ago ... - apache/beam_python3.9_sdk latest sha256:... ... 1 min ago ... apache/beam_python3.10_sdk latest sha256:... ... 1 min ago ... apache/beam_python3.11_sdk latest sha256:... ... 1 min ago ... apache/beam_python3.12_sdk latest sha256:... ... 1 min ago ... + apache/beam_python3.13_sdk latest sha256:... ... 1 min ago ... apache/beam_go_sdk latest sha256:... ... 1 min ago ... ``` From 6ad97107696d590c639ea0414816966c2ece56f3 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Wed, 29 Oct 2025 12:43:23 -0400 Subject: [PATCH 08/29] revert python_requires change to allow 3.9 workflows to execute on PR --- sdks/python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index d8f00324a7fe..e6599a769e8c 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -297,7 +297,7 @@ def get_portability_package_data(): return files -python_requires = '>=3.10' +python_requires = '>=3.9' if sys.version_info.major == 3 and sys.version_info.minor >= 14: warnings.warn( From 5e42a0c816c33143deeaa29842233e117a3a089a Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Wed, 29 Oct 2025 13:38:22 -0400 Subject: [PATCH 09/29] modify linting versions to fix pylint breakage --- .pre-commit-config.yaml | 2 +- CHANGES.md | 2 +- sdks/python/tox.ini | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 92ab38f29107..f84f6b9e7418 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,7 +34,7 @@ repos: - repo: https://github.com/pycqa/pylint # this rev is a release tag in the repo above and corresponds with a pylint # version. make sure this matches the version of pylint in tox.ini. - rev: v2.17.5 + rev: v4.0.2 hooks: - id: pylint args: ["--rcfile=sdks/python/.pylintrc"] diff --git a/CHANGES.md b/CHANGES.md index 84202cf14397..7f60caea7712 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -83,7 +83,7 @@ ## Deprecations * X behavior is deprecated and will be removed in X versions ([#X](https://github.com/apache/beam/issues/X)). -* Python 3.9 reached EOL in October 2025 and support for the lanugage version has been removed. ([#36665](https://github.com/apache/beam/pull/36665)). +* [Python] Python 3.9 reached EOL in October 2025 and support for the lanugage version has been removed. ([#36665](https://github.com/apache/beam/pull/36665)). ## Bugfixes diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini index d5d257fdb7d8..3128f6f77b83 100644 --- a/sdks/python/tox.ini +++ b/sdks/python/tox.ini @@ -182,10 +182,10 @@ commands = setenv = # keep the version of pylint in sync with the 'rev' in .pre-commit-config.yaml deps = - astroid<2.17.0,>=2.15.6 + astroid<4.1.0,>=4.0.1 pycodestyle==2.8.0 - pylint==2.17.5 - isort==4.2.15 + pylint==4.0.2 + isort==7.0.0 flake8==4.0.1 commands = pylint --version From 5140137ab9ab6da5766b3277b003ca43172c4ad7 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Wed, 29 Oct 2025 14:06:22 -0400 Subject: [PATCH 10/29] update pylintrc to remove new checks --- CHANGES.md | 2 +- sdks/python/.pylintrc | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 7f60caea7712..ca5c00e18fd4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -83,7 +83,7 @@ ## Deprecations * X behavior is deprecated and will be removed in X versions ([#X](https://github.com/apache/beam/issues/X)). -* [Python] Python 3.9 reached EOL in October 2025 and support for the lanugage version has been removed. ([#36665](https://github.com/apache/beam/pull/36665)). +* (Python) Python 3.9 reached EOL in October 2025 and support for the lanugage version has been removed. ([#36665](https://github.com/apache/beam/pull/36665)). ## Bugfixes diff --git a/sdks/python/.pylintrc b/sdks/python/.pylintrc index 364513d98844..9910a7e24e95 100644 --- a/sdks/python/.pylintrc +++ b/sdks/python/.pylintrc @@ -94,13 +94,19 @@ disable = consider-using-dict-items, consider-using-enumerate, consider-using-f-string, + consider-using-from-import, consider-using-generator, consider-using-in, + consider-using-max-builtin, + consider-using-min-builtin, consider-using-sys-exit, consider-using-with, cyclic-import, + deprecated-method, + deprecated-module, design, fixme, + function-redefined, global-statement, global-variable-undefined, import-error, @@ -127,6 +133,7 @@ disable = no-value-for-parameter, not-callable, pointless-statement, + possibly-used-before-assignment, protected-access, raise-missing-from, #TODO(https://github.com/apache/beam/issues/21169) Enable and fix warnings raising-format-tuple, @@ -138,6 +145,7 @@ disable = simplifiable-if-statement, stop-iteration-return, super-init-not-called, + super-with-arguments, superfluous-parens, try-except-raise, undefined-variable, @@ -150,12 +158,14 @@ disable = unnecessary-lambda-assignment, unnecessary-pass, unneeded-not, + use-yield-from, used-before-assignment, unsubscriptable-object, unsupported-binary-operation, unspecified-encoding, #TODO(https://github.com/apache/beam/issues/21236) Enable explicit encoding unused-argument, use-dict-literal, + usless-return, unused-wildcard-import, useless-object-inheritance, wildcard-import, From 9071b7ce9cd22e795e68ff1dd9cc8f37685ccb2a Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Wed, 29 Oct 2025 14:26:56 -0400 Subject: [PATCH 11/29] fix typo --- sdks/python/.pylintrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/.pylintrc b/sdks/python/.pylintrc index 9910a7e24e95..b372e25a245a 100644 --- a/sdks/python/.pylintrc +++ b/sdks/python/.pylintrc @@ -165,7 +165,7 @@ disable = unspecified-encoding, #TODO(https://github.com/apache/beam/issues/21236) Enable explicit encoding unused-argument, use-dict-literal, - usless-return, + useless-return, unused-wildcard-import, useless-object-inheritance, wildcard-import, From b0feee1619136f181a522d558200d78e628331dd Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Wed, 29 Oct 2025 14:44:05 -0400 Subject: [PATCH 12/29] fix extra disables --- sdks/python/.pylintrc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdks/python/.pylintrc b/sdks/python/.pylintrc index b372e25a245a..4aaaf29a9c24 100644 --- a/sdks/python/.pylintrc +++ b/sdks/python/.pylintrc @@ -15,7 +15,7 @@ # limitations under the License. # -[MASTER] +[MAIN] # Ignore auto-generated files. ignore=clients load-plugins=pylint.extensions.no_self_use,pylint.extensions.bad_builtin @@ -119,6 +119,7 @@ disable = len-as-condition, locally-disabled, logging-not-lazy, + logging-too-few-args, missing-docstring, modified-iterating-list, multiple-statements, From f5d496952b732190f22f27082373e646d12c0024 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Wed, 29 Oct 2025 15:39:18 -0400 Subject: [PATCH 13/29] try new ignore format --- sdks/python/.pylintrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/.pylintrc b/sdks/python/.pylintrc index 4aaaf29a9c24..3145ae83144c 100644 --- a/sdks/python/.pylintrc +++ b/sdks/python/.pylintrc @@ -17,7 +17,7 @@ [MAIN] # Ignore auto-generated files. -ignore=clients +ignore=('clients') load-plugins=pylint.extensions.no_self_use,pylint.extensions.bad_builtin [BASIC] From f5b8165887a4d94d5929780d55823574ee97c968 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Wed, 29 Oct 2025 15:53:12 -0400 Subject: [PATCH 14/29] more linting changes --- sdks/python/.pylintrc | 2 +- sdks/python/apache_beam/typehints/decorators.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/python/.pylintrc b/sdks/python/.pylintrc index 3145ae83144c..e6a19c3de59f 100644 --- a/sdks/python/.pylintrc +++ b/sdks/python/.pylintrc @@ -17,7 +17,7 @@ [MAIN] # Ignore auto-generated files. -ignore=('clients') +ignore=('clients', 'cloudbuild','s3') load-plugins=pylint.extensions.no_self_use,pylint.extensions.bad_builtin [BASIC] diff --git a/sdks/python/apache_beam/typehints/decorators.py b/sdks/python/apache_beam/typehints/decorators.py index d7bf1ca9248e..6f27676d6466 100644 --- a/sdks/python/apache_beam/typehints/decorators.py +++ b/sdks/python/apache_beam/typehints/decorators.py @@ -123,7 +123,7 @@ def foo((a, b)): _ANY_VAR_POSITIONAL = typehints.Tuple[typehints.Any, ...] _ANY_VAR_KEYWORD = typehints.Dict[typehints.Any, typehints.Any] -_disable_from_callable = False +_disable_from_callable = False # pylint: disable=invalid-name def get_signature(func): From 2f89cb48d0a1880e3bb2be331829ae34c6202152 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Thu, 30 Oct 2025 09:53:46 -0400 Subject: [PATCH 15/29] uncap cores for linting, explicitly cover cloudbuild files in exclusions --- sdks/python/.pylintrc | 3 ++- sdks/python/scripts/run_pylint.sh | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/sdks/python/.pylintrc b/sdks/python/.pylintrc index e6a19c3de59f..a31621748af3 100644 --- a/sdks/python/.pylintrc +++ b/sdks/python/.pylintrc @@ -17,7 +17,8 @@ [MAIN] # Ignore auto-generated files. -ignore=('clients', 'cloudbuild','s3') +jobs=0 +ignore=clients,cloudbuild,s3 load-plugins=pylint.extensions.no_self_use,pylint.extensions.bad_builtin [BASIC] diff --git a/sdks/python/scripts/run_pylint.sh b/sdks/python/scripts/run_pylint.sh index 644b30fba66f..111c433e730d 100755 --- a/sdks/python/scripts/run_pylint.sh +++ b/sdks/python/scripts/run_pylint.sh @@ -57,6 +57,8 @@ EXCLUDED_GENERATED_FILES=( "apache_beam/io/gcp/internal/clients/storage/storage_v1_client.py" "apache_beam/io/gcp/internal/clients/storage/storage_v1_messages.py" "apache_beam/coders/proto2_coder_test_messages_pb2.py" +"apache_beam/runners/dataflow/internal/clients/cloudbuild/cloudbuild_v1_client.py" +"apache_beam/runners/dataflow/internal/clients/cloudbuild/cloudbuild_v1_messages.py" ) # more portable than shopt -s globstar @@ -76,7 +78,7 @@ echo -e "Skipping lint for files:\n${FILES_TO_IGNORE}" echo -e "Linting modules:\n${MODULE}" echo "Running pylint..." -pylint -j8 ${MODULE} --ignore-patterns="$FILES_TO_IGNORE" +pylint ${MODULE} --ignore-patterns="$FILES_TO_IGNORE" echo "Running flake8..." flake8 ${MODULE} --count --select=E9,F821,F822,F823 --show-source --statistics \ --exclude="${FILES_TO_IGNORE}" From dbc6f753745fdca0788c80c089fe0b6da8951801 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Thu, 30 Oct 2025 09:54:24 -0400 Subject: [PATCH 16/29] add s3 client to exclusion list --- sdks/python/scripts/run_pylint.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/sdks/python/scripts/run_pylint.sh b/sdks/python/scripts/run_pylint.sh index 111c433e730d..a5b6204ef7a6 100755 --- a/sdks/python/scripts/run_pylint.sh +++ b/sdks/python/scripts/run_pylint.sh @@ -59,6 +59,7 @@ EXCLUDED_GENERATED_FILES=( "apache_beam/coders/proto2_coder_test_messages_pb2.py" "apache_beam/runners/dataflow/internal/clients/cloudbuild/cloudbuild_v1_client.py" "apache_beam/runners/dataflow/internal/clients/cloudbuild/cloudbuild_v1_messages.py" +"apache_beam/io/aws/clients/s3/boto3_client.py" ) # more portable than shopt -s globstar From 9b02751c0e3ca981894574ae22ba4d5f4d1ff00f Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Thu, 30 Oct 2025 11:55:14 -0400 Subject: [PATCH 17/29] build out isort config --- sdks/python/.isort.cfg | 58 +++++++++++++++++++++++++++++++ sdks/python/.pylintrc | 1 - sdks/python/scripts/run_pylint.sh | 4 +-- sdks/python/setup.cfg | 3 -- 4 files changed, 60 insertions(+), 6 deletions(-) create mode 100644 sdks/python/.isort.cfg diff --git a/sdks/python/.isort.cfg b/sdks/python/.isort.cfg new file mode 100644 index 000000000000..f17e2bb9530e --- /dev/null +++ b/sdks/python/.isort.cfg @@ -0,0 +1,58 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the License); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an AS IS BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +[settings] +py_version=310 +line_length=120 +old_finders=True +order_by_type=True +force_single_line=True +combine_star=true +src_paths=apache_beam +extra_standard_library=dataclasses +known_third_party=yaml +skip=apiclient.py, + avroio_test.py, + cloudpickle.py, + datastore_wordcount.py, + datastoreio_test.py, + doctests_test.py, + fast_coders_test.py, + hadoopfilesystem.py, + iobase_test.py, + main_test.py, + model.py, + preprocess.py, + process_tfma.py, + render_test.py, + slow_coders_test.py, + taxi.py, + tfdv_analyze_and_validate.py, + yaml/main.py, + main_test.py, + yaml_testing_test.py, + bigquery_v2_client.py, + bigquery_v2_messages.py, + dataflow_v1b3_client.py, + dataflow_v1b3_messages.py, + storage_v1_client.py, + storage_v1_messages.py, + proto2_coder_test_messages_pb2.py, + cloudbuild_v1_client.py, + cloudbuild_v1_messages.py, + boto3_client.py +skip_glob=*.pxd,*pb2*.py \ No newline at end of file diff --git a/sdks/python/.pylintrc b/sdks/python/.pylintrc index a31621748af3..a263e168fc2f 100644 --- a/sdks/python/.pylintrc +++ b/sdks/python/.pylintrc @@ -17,7 +17,6 @@ [MAIN] # Ignore auto-generated files. -jobs=0 ignore=clients,cloudbuild,s3 load-plugins=pylint.extensions.no_self_use,pylint.extensions.bad_builtin diff --git a/sdks/python/scripts/run_pylint.sh b/sdks/python/scripts/run_pylint.sh index a5b6204ef7a6..4a6bf4c2ef06 100755 --- a/sdks/python/scripts/run_pylint.sh +++ b/sdks/python/scripts/run_pylint.sh @@ -79,7 +79,7 @@ echo -e "Skipping lint for files:\n${FILES_TO_IGNORE}" echo -e "Linting modules:\n${MODULE}" echo "Running pylint..." -pylint ${MODULE} --ignore-patterns="$FILES_TO_IGNORE" +pylint -j8 ${MODULE} --ignore-patterns="$FILES_TO_IGNORE" echo "Running flake8..." flake8 ${MODULE} --count --select=E9,F821,F822,F823 --show-source --statistics \ --exclude="${FILES_TO_IGNORE}" @@ -116,7 +116,7 @@ for file in "${EXCLUDED_GENERATED_FILES[@]}"; do SKIP_PARAM="$SKIP_PARAM --skip $(basename $file)" done isort ${MODULE} -p apache_beam --line-width 120 --check-only --order-by-type \ - --combine-star --force-single-line-imports --diff --recursive ${SKIP_PARAM} + --combine-star --force-single-line-imports --diff --magic-placement ${SKIP_PARAM} echo "Checking unittest.main..." TESTS_MISSING_MAIN=$( diff --git a/sdks/python/setup.cfg b/sdks/python/setup.cfg index a25ee68d9378..301c1412eeb1 100644 --- a/sdks/python/setup.cfg +++ b/sdks/python/setup.cfg @@ -52,9 +52,6 @@ exclude_lines = [coverage:xml] output = target/site/cobertura/coverage.xml -[isort] -known_standard_library = dataclasses - [yapf] indent_width = 2 continuation_indent_width = 4 From 5796d053a1a657ed4f7362e186c7991f6e25b03d Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Thu, 30 Oct 2025 12:06:45 -0400 Subject: [PATCH 18/29] exclude .pyx files --- sdks/python/.isort.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/.isort.cfg b/sdks/python/.isort.cfg index f17e2bb9530e..30735840789d 100644 --- a/sdks/python/.isort.cfg +++ b/sdks/python/.isort.cfg @@ -55,4 +55,4 @@ skip=apiclient.py, cloudbuild_v1_client.py, cloudbuild_v1_messages.py, boto3_client.py -skip_glob=*.pxd,*pb2*.py \ No newline at end of file +skip_glob=*.pxd,*.pyx,*pb2*.py \ No newline at end of file From 925be5d6bab997aab2f2fff01be0403a3f4862f6 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Thu, 30 Oct 2025 12:07:40 -0400 Subject: [PATCH 19/29] revert errant version bump --- .test-infra/dockerized-jenkins/seedjobconfig.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.test-infra/dockerized-jenkins/seedjobconfig.xml b/.test-infra/dockerized-jenkins/seedjobconfig.xml index bfd57c323c53..2ede2430b454 100644 --- a/.test-infra/dockerized-jenkins/seedjobconfig.xml +++ b/.test-infra/dockerized-jenkins/seedjobconfig.xml @@ -18,7 +18,7 @@ false - + 2 From b7e9635e6cb48288be15ac8610b68c9753117b3e Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Thu, 30 Oct 2025 12:27:52 -0400 Subject: [PATCH 20/29] standardize bools --- sdks/python/.isort.cfg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdks/python/.isort.cfg b/sdks/python/.isort.cfg index 30735840789d..7d8b03b703ec 100644 --- a/sdks/python/.isort.cfg +++ b/sdks/python/.isort.cfg @@ -18,9 +18,9 @@ [settings] py_version=310 line_length=120 -old_finders=True -order_by_type=True -force_single_line=True +old_finders=true +order_by_type=true +force_single_line=true combine_star=true src_paths=apache_beam extra_standard_library=dataclasses From 444f34f3ac6b49d36c5342ac7392af44f370e3cb Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Thu, 30 Oct 2025 13:17:13 -0400 Subject: [PATCH 21/29] initial import ordering changes --- sdks/python/.isort.cfg | 2 +- sdks/python/apache_beam/__init__.py | 3 +- sdks/python/apache_beam/coders/coder_impl.py | 6 ++-- sdks/python/apache_beam/coders/coders.py | 1 + .../coders/coders_property_based_test.py | 2 +- sdks/python/apache_beam/coders/coders_test.py | 2 +- .../apache_beam/coders/coders_test_common.py | 3 +- .../apache_beam/dataframe/expressions.py | 3 +- sdks/python/apache_beam/dataframe/io.py | 1 + sdks/python/apache_beam/internal/gcp/auth.py | 2 +- sdks/python/apache_beam/io/__init__.py | 4 +-- .../io/azure/blobstoragefilesystem_test.py | 2 +- .../apache_beam/io/azure/blobstorageio.py | 6 ++-- .../io/external/xlang_jdbcio_it_test.py | 2 +- .../apache_beam/io/filebasedsource_test.py | 2 +- sdks/python/apache_beam/io/filesystem_test.py | 1 + sdks/python/apache_beam/io/gcp/__init__.py | 1 + sdks/python/apache_beam/io/gcp/bigquery.py | 3 +- .../io/gcp/bigquery_file_loads_test.py | 2 +- .../apache_beam/io/gcp/bigquery_test.py | 9 ++--- .../apache_beam/io/gcp/bigquery_tools_test.py | 6 ++-- sdks/python/apache_beam/io/gcp/bigtableio.py | 3 +- .../apache_beam/io/gcp/bigtableio_it_test.py | 4 ++- .../apache_beam/io/gcp/bigtableio_test.py | 9 +++-- .../io/gcp/datastore/v1new/datastoreio.py | 3 +- .../datastore/v1new/query_splitter_test.py | 3 +- .../io/gcp/datastore/v1new/types_test.py | 1 + .../io/gcp/experimental/spannerio.py | 5 +-- .../experimental/spannerio_read_it_test.py | 5 +-- .../experimental/spannerio_read_perf_test.py | 1 + .../io/gcp/experimental/spannerio_test.py | 9 ++--- .../experimental/spannerio_write_it_test.py | 5 +-- .../experimental/spannerio_write_perf_test.py | 1 + .../io/gcp/gcsio_integration_test.py | 1 + .../apache_beam/io/gcp/gcsio_retry_test.py | 3 +- sdks/python/apache_beam/io/gcp/gcsio_test.py | 4 ++- .../healthcare/dicomio_integration_test.py | 5 +-- .../gcp/internal/clients/bigquery/__init__.py | 1 + .../apache_beam/io/gcp/pubsub_io_perf_test.py | 1 + .../apache_beam/io/gcp/pubsublite/__init__.py | 3 +- sdks/python/apache_beam/io/mongodbio.py | 1 - sdks/python/apache_beam/io/parquetio.py | 1 + .../apache_beam/io/requestresponse_it_test.py | 1 + .../apache_beam/io/requestresponse_test.py | 1 + sdks/python/apache_beam/io/textio_test.py | 4 +-- sdks/python/apache_beam/io/tfrecordio.py | 1 + sdks/python/apache_beam/metrics/metric.py | 2 +- .../ml/anomaly/detectors/__init__.py | 4 +-- .../ml/anomaly/detectors/pyod_adapter.py | 2 +- .../ml/anomaly/detectors/pyod_adapter_test.py | 3 +- .../apache_beam/ml/gcp/cloud_dlp_test.py | 3 +- .../ml/gcp/recommendations_ai_test.py | 1 + .../ml/gcp/recommendations_ai_test_it.py | 1 + .../ml/gcp/videointelligenceml_test.py | 3 +- .../ml/gcp/videointelligenceml_test_it.py | 3 +- .../apache_beam/ml/gcp/visionml_test.py | 3 +- .../apache_beam/ml/gcp/visionml_test_it.py | 3 +- .../ml/inference/gemini_inference.py | 2 +- .../ml/inference/gemini_inference_test.py | 5 +-- .../ml/inference/huggingface_inference.py | 9 ++--- .../inference/huggingface_inference_test.py | 1 + .../ml/inference/onnx_inference.py | 2 +- .../ml/inference/onnx_inference_it_test.py | 1 + .../ml/inference/onnx_inference_test.py | 9 ++--- .../ml/inference/pytorch_inference.py | 1 + .../ml/inference/pytorch_inference_it_test.py | 3 +- .../ml/inference/pytorch_inference_test.py | 7 ++-- .../ml/inference/tensorflow_inference.py | 2 +- .../inference/tensorflow_inference_it_test.py | 1 + .../ml/inference/tensorflow_inference_test.py | 6 ++-- .../ml/inference/tensorrt_inference.py | 2 +- .../ml/inference/tensorrt_inference_test.py | 7 ++-- .../ml/inference/vertex_ai_inference_test.py | 5 +-- .../ml/inference/vllm_inference.py | 5 +-- .../ml/inference/xgboost_inference.py | 4 +-- .../ml/inference/xgboost_inference_it_test.py | 3 +- .../ml/rag/chunking/langchain_test.py | 6 ++-- .../ml/rag/embeddings/vertex_ai_test.py | 1 + .../bigquery_vector_search_it_test.py | 6 ++-- .../ml/rag/enrichment/milvus_search.py | 3 +- .../rag/enrichment/milvus_search_it_test.py | 35 +++++++++---------- .../ml/rag/enrichment/milvus_search_test.py | 21 ++++++----- .../transforms/embeddings/huggingface_test.py | 8 +++-- .../ml/transforms/embeddings/open_ai.py | 7 ++-- .../transforms/embeddings/tensorflow_hub.py | 3 +- .../embeddings/tensorflow_hub_test.py | 4 ++- .../ml/transforms/embeddings/vertex_ai.py | 18 +++++----- .../transforms/embeddings/vertex_ai_test.py | 10 +++--- .../apache_beam/ml/transforms/handlers.py | 18 +++++----- .../ml/transforms/handlers_test.py | 7 ++-- sdks/python/apache_beam/ml/transforms/tft.py | 5 +-- .../python/apache_beam/ml/transforms/utils.py | 2 +- .../options/pipeline_options_test.py | 3 +- sdks/python/apache_beam/pipeline.py | 1 + sdks/python/apache_beam/pipeline_test.py | 2 +- sdks/python/apache_beam/pvalue.py | 6 ++-- sdks/python/apache_beam/runners/__init__.py | 5 ++- sdks/python/apache_beam/runners/common.py | 6 ++-- .../runners/dask/transform_evaluator.py | 3 +- .../runners/dataflow/dataflow_runner.py | 5 ++- .../internal/clients/cloudbuild/__init__.py | 1 + .../internal/clients/dataflow/__init__.py | 3 +- .../runners/dataflow/ptransform_overrides.py | 2 +- .../runners/direct/direct_runner.py | 13 ++++--- .../runners/direct/evaluation_context.py | 3 +- .../runners/direct/transform_evaluator.py | 5 +-- .../runners/interactive/augmented_pipeline.py | 2 +- .../background_caching_job_test.py | 1 + .../runners/interactive/caching/cacheable.py | 1 + .../dataproc/dataproc_cluster_manager.py | 3 +- .../interactive/display/display_manager.py | 1 + .../display/pcoll_visualization.py | 3 +- .../interactive/interactive_environment.py | 3 +- .../runners/interactive/interactive_runner.py | 2 +- .../interactive/pipeline_instrument.py | 2 +- .../interactive/pipeline_instrument_test.py | 4 +-- .../interactive/sql/beam_sql_magics.py | 7 ++-- .../interactive/sql/beam_sql_magics_test.py | 2 +- .../apache_beam/runners/interactive/utils.py | 4 +++ .../runners/interactive/utils_test.py | 3 +- .../fn_api_runner/worker_handlers.py | 9 +++-- sdks/python/apache_beam/runners/runner.py | 4 +-- .../runners/worker/bundle_processor.py | 1 + .../apache_beam/runners/worker/operations.py | 2 +- .../testing/analyzers/perf_analysis_test.py | 4 +-- .../testing/analyzers/perf_analysis_utils.py | 2 +- .../benchmarks/cloudml/pipelines/workflow.py | 5 +-- ...pytorch_image_classification_benchmarks.py | 3 +- .../testing/metric_result_matchers_test.py | 2 +- .../testing/pipeline_verifiers_test.py | 1 + .../tools/coders_microbenchmark.py | 2 +- sdks/python/apache_beam/transforms/core.py | 11 +++--- .../enrichment_handlers/bigquery_it_test.py | 6 ++-- .../enrichment_handlers/bigtable_it_test.py | 1 + .../enrichment_handlers/cloudsql_it_test.py | 31 +++++++++------- .../enrichment_handlers/cloudsql_test.py | 21 +++++------ .../feast_feature_store.py | 3 +- .../feast_feature_store_it_test.py | 6 ++-- .../feast_feature_store_test.py | 6 ++-- .../vertex_ai_feature_store_it_test.py | 6 ++-- .../vertex_ai_feature_store_test.py | 7 ++-- .../apache_beam/transforms/ptransform.py | 7 +++- .../apache_beam/transforms/trigger_test.py | 1 + .../apache_beam/transforms/userstate.py | 1 + sdks/python/apache_beam/typehints/__init__.py | 4 +-- .../typehints/pytorch_type_compatibility.py | 1 + .../pytorch_type_compatibility_test.py | 1 + .../apache_beam/utils/interactive_utils.py | 1 + .../apache_beam/yaml/integration_tests.py | 2 +- sdks/python/apache_beam/yaml/yaml_ml.py | 1 + sdks/python/apache_beam/yaml/yaml_provider.py | 7 ++-- sdks/python/gen_managed_doc.py | 3 +- sdks/python/gen_xlang_wrappers.py | 1 - 153 files changed, 378 insertions(+), 277 deletions(-) diff --git a/sdks/python/.isort.cfg b/sdks/python/.isort.cfg index 7d8b03b703ec..ae37f37a113f 100644 --- a/sdks/python/.isort.cfg +++ b/sdks/python/.isort.cfg @@ -55,4 +55,4 @@ skip=apiclient.py, cloudbuild_v1_client.py, cloudbuild_v1_messages.py, boto3_client.py -skip_glob=*.pxd,*.pyx,*pb2*.py \ No newline at end of file +skip_glob=*.pxd,*.pyx,*pb2*.py,**/examples/**/*.py \ No newline at end of file diff --git a/sdks/python/apache_beam/__init__.py b/sdks/python/apache_beam/__init__.py index 542f75d54d7c..9906c95aee14 100644 --- a/sdks/python/apache_beam/__init__.py +++ b/sdks/python/apache_beam/__init__.py @@ -83,17 +83,16 @@ # pylint: disable=wrong-import-position import apache_beam.internal.pickler - from apache_beam import coders from apache_beam import io from apache_beam import metrics from apache_beam import typehints from apache_beam import version from apache_beam.pipeline import * -from apache_beam.transforms import * from apache_beam.pvalue import PCollection from apache_beam.pvalue import Row from apache_beam.pvalue import TaggedOutput +from apache_beam.transforms import * try: # Add mitigation for CVE-2023-47248 while Beam allows affected versions diff --git a/sdks/python/apache_beam/coders/coder_impl.py b/sdks/python/apache_beam/coders/coder_impl.py index e0b109386b44..35d67258b560 100644 --- a/sdks/python/apache_beam/coders/coder_impl.py +++ b/sdks/python/apache_beam/coders/coder_impl.py @@ -79,6 +79,7 @@ if TYPE_CHECKING: import proto + from apache_beam.transforms import userstate from apache_beam.transforms.window import IntervalWindow @@ -93,9 +94,9 @@ fits_in_64_bits = lambda x: -(1 << 63) <= x <= (1 << 63) - 1 if TYPE_CHECKING or SLOW_STREAM: + from .slow_stream import ByteCountingOutputStream from .slow_stream import InputStream as create_InputStream from .slow_stream import OutputStream as create_OutputStream - from .slow_stream import ByteCountingOutputStream from .slow_stream import get_varint_size try: @@ -106,10 +107,11 @@ else: # pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports + from .stream import ByteCountingOutputStream from .stream import InputStream as create_InputStream from .stream import OutputStream as create_OutputStream - from .stream import ByteCountingOutputStream from .stream import get_varint_size + # Make it possible to import create_InputStream and other cdef-classes # from apache_beam.coders.coder_impl when Cython codepath is used. globals()['create_InputStream'] = create_InputStream diff --git a/sdks/python/apache_beam/coders/coders.py b/sdks/python/apache_beam/coders/coders.py index 2fea9717db18..0f73197f5cb1 100644 --- a/sdks/python/apache_beam/coders/coders.py +++ b/sdks/python/apache_beam/coders/coders.py @@ -933,6 +933,7 @@ def __init__(self, coder, step_label, update_compatibility_version=None): self._use_relative_filepaths = True self._version_tag = "v2_69" from apache_beam.transforms.util import is_v1_prior_to_v2 + # Versions prior to 2.69.0 did not use relative filepaths. if update_compatibility_version and is_v1_prior_to_v2( v1=update_compatibility_version, v2="2.69.0"): diff --git a/sdks/python/apache_beam/coders/coders_property_based_test.py b/sdks/python/apache_beam/coders/coders_property_based_test.py index d8d844975b9b..9b5600d7156d 100644 --- a/sdks/python/apache_beam/coders/coders_property_based_test.py +++ b/sdks/python/apache_beam/coders/coders_property_based_test.py @@ -34,10 +34,10 @@ from string import digits import numpy as np -from hypothesis import strategies as st from hypothesis import assume from hypothesis import given from hypothesis import settings +from hypothesis import strategies as st from pytz import utc from apache_beam.coders import FloatCoder diff --git a/sdks/python/apache_beam/coders/coders_test.py b/sdks/python/apache_beam/coders/coders_test.py index 74e6c55e4188..ccd947457ad7 100644 --- a/sdks/python/apache_beam/coders/coders_test.py +++ b/sdks/python/apache_beam/coders/coders_test.py @@ -26,8 +26,8 @@ import apache_beam as beam from apache_beam import typehints -from apache_beam.coders import proto2_coder_test_messages_pb2 as test_message from apache_beam.coders import coders +from apache_beam.coders import proto2_coder_test_messages_pb2 as test_message from apache_beam.coders.avro_record import AvroRecord from apache_beam.coders.typecoders import registry as coders_registry from apache_beam.testing.test_pipeline import TestPipeline diff --git a/sdks/python/apache_beam/coders/coders_test_common.py b/sdks/python/apache_beam/coders/coders_test_common.py index 6b916adbfcc3..8a57d1e63e2c 100644 --- a/sdks/python/apache_beam/coders/coders_test_common.py +++ b/sdks/python/apache_beam/coders/coders_test_common.py @@ -38,8 +38,8 @@ from parameterized import param from parameterized import parameterized -from apache_beam.coders import proto2_coder_test_messages_pb2 as test_message from apache_beam.coders import coders +from apache_beam.coders import proto2_coder_test_messages_pb2 as test_message from apache_beam.coders import typecoders from apache_beam.internal import pickler from apache_beam.runners import pipeline_context @@ -650,6 +650,7 @@ def test_windowed_value_coder(self): def test_param_windowed_value_coder(self): from apache_beam.transforms.window import IntervalWindow from apache_beam.utils.windowed_value import PaneInfo + # pylint: disable=too-many-function-args wv = windowed_value.create( b'', diff --git a/sdks/python/apache_beam/dataframe/expressions.py b/sdks/python/apache_beam/dataframe/expressions.py index 2ef172b8dad3..2dfc84975e6d 100644 --- a/sdks/python/apache_beam/dataframe/expressions.py +++ b/sdks/python/apache_beam/dataframe/expressions.py @@ -61,9 +61,10 @@ class PartitioningSession(Session): For testing only. """ def evaluate(self, expr): - import pandas as pd import collections + import pandas as pd + def is_scalar(expr): return not isinstance(expr.proxy(), pd.core.generic.NDFrame) diff --git a/sdks/python/apache_beam/dataframe/io.py b/sdks/python/apache_beam/dataframe/io.py index 752df1e68b7c..d16191bc4ccf 100644 --- a/sdks/python/apache_beam/dataframe/io.py +++ b/sdks/python/apache_beam/dataframe/io.py @@ -107,6 +107,7 @@ def read_csv(path, *args, splittable=False, binary=True, **kwargs): def _as_pc(df, label=None): from apache_beam.dataframe import convert # avoid circular import + # TODO(roberwb): Amortize the computation for multiple writes? return convert.to_pcollection(df, yield_elements='pandas', label=label) diff --git a/sdks/python/apache_beam/internal/gcp/auth.py b/sdks/python/apache_beam/internal/gcp/auth.py index 66c08b8344cb..168d6aa26939 100644 --- a/sdks/python/apache_beam/internal/gcp/auth.py +++ b/sdks/python/apache_beam/internal/gcp/auth.py @@ -30,9 +30,9 @@ # google.auth is only available when Beam is installed with the gcp extra. try: - from google.auth import impersonated_credentials import google.auth import google_auth_httplib2 + from google.auth import impersonated_credentials _GOOGLE_AUTH_AVAILABLE = True except ImportError: _GOOGLE_AUTH_AVAILABLE = False diff --git a/sdks/python/apache_beam/io/__init__.py b/sdks/python/apache_beam/io/__init__.py index 83d45d81a5a1..a3e51150d8a1 100644 --- a/sdks/python/apache_beam/io/__init__.py +++ b/sdks/python/apache_beam/io/__init__.py @@ -26,16 +26,16 @@ from apache_beam.io.iobase import Writer from apache_beam.io.mongodbio import * from apache_beam.io.parquetio import * +from apache_beam.io.range_trackers import * from apache_beam.io.textio import * from apache_beam.io.tfrecordio import * -from apache_beam.io.range_trackers import * # Protect against environments where clientslibrary is not available. # pylint: disable=wrong-import-order, wrong-import-position try: + from apache_beam.io.gcp import gcsio from apache_beam.io.gcp.bigquery import * from apache_beam.io.gcp.pubsub import * - from apache_beam.io.gcp import gcsio from apache_beam.transforms.xlang.io import * except ImportError: pass diff --git a/sdks/python/apache_beam/io/azure/blobstoragefilesystem_test.py b/sdks/python/apache_beam/io/azure/blobstoragefilesystem_test.py index c3418e137e87..1bf1fb7b84c7 100644 --- a/sdks/python/apache_beam/io/azure/blobstoragefilesystem_test.py +++ b/sdks/python/apache_beam/io/azure/blobstoragefilesystem_test.py @@ -32,8 +32,8 @@ # Protect against environments where azure library is not available. # pylint: disable=wrong-import-order, wrong-import-position try: - from apache_beam.io.azure import blobstorageio from apache_beam.io.azure import blobstoragefilesystem + from apache_beam.io.azure import blobstorageio except ImportError: blobstoragefilesystem = None # type: ignore[assignment] # pylint: enable=wrong-import-order, wrong-import-position diff --git a/sdks/python/apache_beam/io/azure/blobstorageio.py b/sdks/python/apache_beam/io/azure/blobstorageio.py index cfa4fe7d2916..9b0f595e102f 100644 --- a/sdks/python/apache_beam/io/azure/blobstorageio.py +++ b/sdks/python/apache_beam/io/azure/blobstorageio.py @@ -43,10 +43,8 @@ # pylint: disable=wrong-import-order, wrong-import-position # pylint: disable=ungrouped-imports from azure.core.exceptions import ResourceNotFoundError - from azure.storage.blob import ( - BlobServiceClient, - ContentSettings, - ) + from azure.storage.blob import BlobServiceClient + from azure.storage.blob import ContentSettings AZURE_DEPS_INSTALLED = True except ImportError: AZURE_DEPS_INSTALLED = False diff --git a/sdks/python/apache_beam/io/external/xlang_jdbcio_it_test.py b/sdks/python/apache_beam/io/external/xlang_jdbcio_it_test.py index 26fa2f400d83..069f13e11bfb 100644 --- a/sdks/python/apache_beam/io/external/xlang_jdbcio_it_test.py +++ b/sdks/python/apache_beam/io/external/xlang_jdbcio_it_test.py @@ -47,8 +47,8 @@ # pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports try: - from testcontainers.postgres import PostgresContainer from testcontainers.mysql import MySqlContainer + from testcontainers.postgres import PostgresContainer except ImportError: PostgresContainer = None # pylint: enable=wrong-import-order, wrong-import-position, ungrouped-imports diff --git a/sdks/python/apache_beam/io/filebasedsource_test.py b/sdks/python/apache_beam/io/filebasedsource_test.py index e68d2afbac9d..2728d2f91e0f 100644 --- a/sdks/python/apache_beam/io/filebasedsource_test.py +++ b/sdks/python/apache_beam/io/filebasedsource_test.py @@ -34,8 +34,8 @@ from apache_beam.io import range_trackers # importing following private classes for testing from apache_beam.io.concat_source import ConcatSource -from apache_beam.io.filebasedsource import _SingleFileSource as SingleFileSource from apache_beam.io.filebasedsource import FileBasedSource +from apache_beam.io.filebasedsource import _SingleFileSource as SingleFileSource from apache_beam.io.filesystem import CompressionTypes from apache_beam.options.value_provider import RuntimeValueProvider from apache_beam.options.value_provider import StaticValueProvider diff --git a/sdks/python/apache_beam/io/filesystem_test.py b/sdks/python/apache_beam/io/filesystem_test.py index ff701132bf75..fa68bc5ef6df 100644 --- a/sdks/python/apache_beam/io/filesystem_test.py +++ b/sdks/python/apache_beam/io/filesystem_test.py @@ -518,6 +518,7 @@ def test_concatenated_compressed_file(self): # interface does not allow you to modify the read_size. import random import threading + from six import int2byte num_test_lines = 10 timeout = 30 diff --git a/sdks/python/apache_beam/io/gcp/__init__.py b/sdks/python/apache_beam/io/gcp/__init__.py index f88a0117aa46..861a39f5c75d 100644 --- a/sdks/python/apache_beam/io/gcp/__init__.py +++ b/sdks/python/apache_beam/io/gcp/__init__.py @@ -22,6 +22,7 @@ # pylint: disable=wrong-import-order, wrong-import-position # pylint: disable=ungrouped-imports import email.generator as email_generator + from apitools.base.py import transfer class _WrapperNamespace(object): diff --git a/sdks/python/apache_beam/io/gcp/bigquery.py b/sdks/python/apache_beam/io/gcp/bigquery.py index ede0355b7c53..181c891c1b65 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery.py +++ b/sdks/python/apache_beam/io/gcp/bigquery.py @@ -425,8 +425,8 @@ def chain_after(result): try: from apache_beam.io.gcp.internal.clients.bigquery import DatasetReference - from apache_beam.io.gcp.internal.clients.bigquery import TableReference from apache_beam.io.gcp.internal.clients.bigquery import JobReference + from apache_beam.io.gcp.internal.clients.bigquery import TableReference except ImportError: DatasetReference = None TableReference = None @@ -2343,6 +2343,7 @@ def find_in_nested_dict(schema): find_in_nested_dict(self.schema) from apache_beam.io.gcp.bigquery_file_loads import BigQueryBatchFileLoads + # Only cast to int when a value is given. # We only use an int for BigQueryBatchFileLoads if self.triggering_frequency is not None: diff --git a/sdks/python/apache_beam/io/gcp/bigquery_file_loads_test.py b/sdks/python/apache_beam/io/gcp/bigquery_file_loads_test.py index f690f3477728..30f09ff4f56a 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery_file_loads_test.py +++ b/sdks/python/apache_beam/io/gcp/bigquery_file_loads_test.py @@ -37,8 +37,8 @@ import apache_beam as beam from apache_beam.io.filebasedsink_test import _TestCaseWithTempDirCleanUp -from apache_beam.io.gcp import bigquery_file_loads as bqfl from apache_beam.io.gcp import bigquery +from apache_beam.io.gcp import bigquery_file_loads as bqfl from apache_beam.io.gcp import bigquery_tools from apache_beam.io.gcp.bigquery import BigQueryDisposition from apache_beam.io.gcp.bigquery_tools import BigQueryWrapper diff --git a/sdks/python/apache_beam/io/gcp/bigquery_test.py b/sdks/python/apache_beam/io/gcp/bigquery_test.py index dcb85d60f87f..f516b206d5e3 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery_test.py +++ b/sdks/python/apache_beam/io/gcp/bigquery_test.py @@ -84,12 +84,13 @@ # pylint: disable=wrong-import-order, wrong-import-position try: - from apache_beam.io.gcp.internal.clients.bigquery import bigquery_v2_client from apitools.base.py.exceptions import HttpError from apitools.base.py.exceptions import HttpForbiddenError + from google.api_core import exceptions from google.cloud import bigquery as gcp_bigquery from google.cloud import bigquery_storage_v1 as bq_storage - from google.api_core import exceptions + + from apache_beam.io.gcp.internal.clients.bigquery import bigquery_v2_client except ImportError: gcp_bigquery = None bq_storage = None @@ -1947,8 +1948,8 @@ def store_callback(table, **kwargs): def test_with_batched_input_exceeds_size_limit(self): - from apache_beam.utils.windowed_value import WindowedValue from apache_beam.transforms import window + from apache_beam.utils.windowed_value import WindowedValue client = mock.Mock() client.tables.Get.return_value = bigquery.Table( @@ -2021,8 +2022,8 @@ def test_with_batched_input_exceeds_size_limit(self): def test_with_batched_input_splits_large_batch(self): - from apache_beam.utils.windowed_value import WindowedValue from apache_beam.transforms import window + from apache_beam.utils.windowed_value import WindowedValue client = mock.Mock() client.tables.Get.return_value = bigquery.Table( diff --git a/sdks/python/apache_beam/io/gcp/bigquery_tools_test.py b/sdks/python/apache_beam/io/gcp/bigquery_tools_test.py index 58edba387ebd..2594e6728e0e 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery_tools_test.py +++ b/sdks/python/apache_beam/io/gcp/bigquery_tools_test.py @@ -57,8 +57,10 @@ # Protect against environments where bigquery library is not available. # pylint: disable=wrong-import-order, wrong-import-position try: - from apitools.base.py.exceptions import HttpError, HttpForbiddenError - from google.api_core.exceptions import ClientError, DeadlineExceeded + from apitools.base.py.exceptions import HttpError + from apitools.base.py.exceptions import HttpForbiddenError + from google.api_core.exceptions import ClientError + from google.api_core.exceptions import DeadlineExceeded from google.api_core.exceptions import InternalServerError except ImportError: ClientError = None diff --git a/sdks/python/apache_beam/io/gcp/bigtableio.py b/sdks/python/apache_beam/io/gcp/bigtableio.py index ff140082a1ef..f10039e564d1 100644 --- a/sdks/python/apache_beam/io/gcp/bigtableio.py +++ b/sdks/python/apache_beam/io/gcp/bigtableio.py @@ -60,8 +60,9 @@ try: from google.cloud.bigtable import Client - from google.cloud.bigtable.row import Cell, PartialRowData from google.cloud.bigtable.batcher import MutationsBatcher + from google.cloud.bigtable.row import Cell + from google.cloud.bigtable.row import PartialRowData except ImportError: _LOGGER.warning( diff --git a/sdks/python/apache_beam/io/gcp/bigtableio_it_test.py b/sdks/python/apache_beam/io/gcp/bigtableio_it_test.py index 5e03020e1f74..27b910ad5f08 100644 --- a/sdks/python/apache_beam/io/gcp/bigtableio_it_test.py +++ b/sdks/python/apache_beam/io/gcp/bigtableio_it_test.py @@ -40,8 +40,10 @@ try: from apitools.base.py.exceptions import HttpError from google.cloud.bigtable import client + from google.cloud.bigtable.row import Cell + from google.cloud.bigtable.row import DirectRow + from google.cloud.bigtable.row import PartialRowData from google.cloud.bigtable.row_filters import TimestampRange - from google.cloud.bigtable.row import DirectRow, PartialRowData, Cell from google.cloud.bigtable.table import Table from google.cloud.bigtable_admin_v2.types import instance except ImportError as e: diff --git a/sdks/python/apache_beam/io/gcp/bigtableio_test.py b/sdks/python/apache_beam/io/gcp/bigtableio_test.py index 2b7463f93c13..d9ef12a16592 100644 --- a/sdks/python/apache_beam/io/gcp/bigtableio_test.py +++ b/sdks/python/apache_beam/io/gcp/bigtableio_test.py @@ -44,11 +44,14 @@ try: from google.cloud.bigtable import client from google.cloud.bigtable.batcher import MutationsBatcher - from google.cloud.bigtable.row_filters import TimestampRange from google.cloud.bigtable.instance import Instance - from google.cloud.bigtable.row import DirectRow, PartialRowData, Cell + from google.cloud.bigtable.row import Cell + from google.cloud.bigtable.row import DirectRow + from google.cloud.bigtable.row import PartialRowData + from google.cloud.bigtable.row_filters import TimestampRange from google.cloud.bigtable.table import Table - from google.rpc.code_pb2 import OK, ALREADY_EXISTS + from google.rpc.code_pb2 import ALREADY_EXISTS + from google.rpc.code_pb2 import OK from google.rpc.status_pb2 import Status except ImportError as e: client = None diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1new/datastoreio.py b/sdks/python/apache_beam/io/gcp/datastore/v1new/datastoreio.py index f120234e9740..6f870b7cfeb7 100644 --- a/sdks/python/apache_beam/io/gcp/datastore/v1new/datastoreio.py +++ b/sdks/python/apache_beam/io/gcp/datastore/v1new/datastoreio.py @@ -54,7 +54,8 @@ # pylint: disable=wrong-import-order, wrong-import-position try: from apitools.base.py.exceptions import HttpError - from google.api_core.exceptions import ClientError, GoogleAPICallError + from google.api_core.exceptions import ClientError + from google.api_core.exceptions import GoogleAPICallError except ImportError: pass diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1new/query_splitter_test.py b/sdks/python/apache_beam/io/gcp/datastore/v1new/query_splitter_test.py index b26651e9066e..a6da79ad9832 100644 --- a/sdks/python/apache_beam/io/gcp/datastore/v1new/query_splitter_test.py +++ b/sdks/python/apache_beam/io/gcp/datastore/v1new/query_splitter_test.py @@ -25,11 +25,12 @@ # Protect against environments where datastore library is not available. try: + from google.cloud.datastore import key + from apache_beam.io.gcp.datastore.v1new import helper from apache_beam.io.gcp.datastore.v1new import query_splitter from apache_beam.io.gcp.datastore.v1new import types from apache_beam.io.gcp.datastore.v1new.query_splitter import SplitNotPossibleError - from google.cloud.datastore import key except ImportError: query_splitter = None # type: ignore diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1new/types_test.py b/sdks/python/apache_beam/io/gcp/datastore/v1new/types_test.py index b54f42f5e86e..9a33fa690a54 100644 --- a/sdks/python/apache_beam/io/gcp/datastore/v1new/types_test.py +++ b/sdks/python/apache_beam/io/gcp/datastore/v1new/types_test.py @@ -31,6 +31,7 @@ from google.cloud.datastore import entity from google.cloud.datastore import key from google.cloud.datastore.helpers import GeoPoint + from apache_beam.io.gcp.datastore.v1new.types import Entity from apache_beam.io.gcp.datastore.v1new.types import Key from apache_beam.io.gcp.datastore.v1new.types import Query diff --git a/sdks/python/apache_beam/io/gcp/experimental/spannerio.py b/sdks/python/apache_beam/io/gcp/experimental/spannerio.py index cac66bd2ef54..c94c43a637d4 100644 --- a/sdks/python/apache_beam/io/gcp/experimental/spannerio.py +++ b/sdks/python/apache_beam/io/gcp/experimental/spannerio.py @@ -196,12 +196,13 @@ # pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports # pylint: disable=unused-import try: + from apitools.base.py.exceptions import HttpError + from google.api_core.exceptions import ClientError + from google.api_core.exceptions import GoogleAPICallError from google.cloud.spanner import Client from google.cloud.spanner import KeySet from google.cloud.spanner_v1 import batch from google.cloud.spanner_v1.database import BatchSnapshot - from google.api_core.exceptions import ClientError, GoogleAPICallError - from apitools.base.py.exceptions import HttpError except ImportError: Client = None KeySet = None diff --git a/sdks/python/apache_beam/io/gcp/experimental/spannerio_read_it_test.py b/sdks/python/apache_beam/io/gcp/experimental/spannerio_read_it_test.py index 88db0ad20794..753d9148b334 100644 --- a/sdks/python/apache_beam/io/gcp/experimental/spannerio_read_it_test.py +++ b/sdks/python/apache_beam/io/gcp/experimental/spannerio_read_it_test.py @@ -33,10 +33,11 @@ # pylint: disable=unused-import try: from google.cloud import spanner + from apache_beam.io.gcp import resource_identifiers - from apache_beam.io.gcp.experimental.spannerio import create_transaction - from apache_beam.io.gcp.experimental.spannerio import ReadOperation from apache_beam.io.gcp.experimental.spannerio import ReadFromSpanner + from apache_beam.io.gcp.experimental.spannerio import ReadOperation + from apache_beam.io.gcp.experimental.spannerio import create_transaction from apache_beam.metrics import monitoring_infos from apache_beam.metrics.execution import MetricsEnvironment from apache_beam.metrics.metricbase import MetricName diff --git a/sdks/python/apache_beam/io/gcp/experimental/spannerio_read_perf_test.py b/sdks/python/apache_beam/io/gcp/experimental/spannerio_read_perf_test.py index 18f6c29593e7..0fb97594d824 100644 --- a/sdks/python/apache_beam/io/gcp/experimental/spannerio_read_perf_test.py +++ b/sdks/python/apache_beam/io/gcp/experimental/spannerio_read_perf_test.py @@ -119,6 +119,7 @@ def format_record(record): def make_insert_mutations(element): import uuid + from apache_beam.io.gcp.experimental.spannerio import WriteMutation ins_mutation = WriteMutation.insert( table='test_data', diff --git a/sdks/python/apache_beam/io/gcp/experimental/spannerio_test.py b/sdks/python/apache_beam/io/gcp/experimental/spannerio_test.py index ec508bf9276e..f7922ec1a6e7 100644 --- a/sdks/python/apache_beam/io/gcp/experimental/spannerio_test.py +++ b/sdks/python/apache_beam/io/gcp/experimental/spannerio_test.py @@ -35,14 +35,15 @@ # pylint: disable=unused-import try: from google.cloud import spanner - from apache_beam.io.gcp.experimental.spannerio import create_transaction - from apache_beam.io.gcp.experimental.spannerio import ReadOperation + + from apache_beam.io.gcp import resource_identifiers + from apache_beam.io.gcp.experimental.spannerio import MutationGroup from apache_beam.io.gcp.experimental.spannerio import ReadFromSpanner + from apache_beam.io.gcp.experimental.spannerio import ReadOperation from apache_beam.io.gcp.experimental.spannerio import WriteMutation - from apache_beam.io.gcp.experimental.spannerio import MutationGroup from apache_beam.io.gcp.experimental.spannerio import WriteToSpanner from apache_beam.io.gcp.experimental.spannerio import _BatchFn - from apache_beam.io.gcp import resource_identifiers + from apache_beam.io.gcp.experimental.spannerio import create_transaction from apache_beam.metrics import monitoring_infos from apache_beam.metrics.execution import MetricsEnvironment from apache_beam.metrics.metricbase import MetricName diff --git a/sdks/python/apache_beam/io/gcp/experimental/spannerio_write_it_test.py b/sdks/python/apache_beam/io/gcp/experimental/spannerio_write_it_test.py index 7172e97ba337..2341509bd476 100644 --- a/sdks/python/apache_beam/io/gcp/experimental/spannerio_write_it_test.py +++ b/sdks/python/apache_beam/io/gcp/experimental/spannerio_write_it_test.py @@ -29,11 +29,12 @@ # pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports # pylint: disable=unused-import try: - from google.cloud import spanner from google.api_core.exceptions import NotFound + from google.cloud import spanner + from apache_beam.io.gcp import resource_identifiers - from apache_beam.io.gcp.experimental.spannerio import WriteMutation from apache_beam.io.gcp.experimental.spannerio import MutationGroup + from apache_beam.io.gcp.experimental.spannerio import WriteMutation from apache_beam.io.gcp.experimental.spannerio import WriteToSpanner from apache_beam.metrics import monitoring_infos from apache_beam.metrics.execution import MetricsEnvironment diff --git a/sdks/python/apache_beam/io/gcp/experimental/spannerio_write_perf_test.py b/sdks/python/apache_beam/io/gcp/experimental/spannerio_write_perf_test.py index c61608ff6743..06be99fc6335 100644 --- a/sdks/python/apache_beam/io/gcp/experimental/spannerio_write_perf_test.py +++ b/sdks/python/apache_beam/io/gcp/experimental/spannerio_write_perf_test.py @@ -113,6 +113,7 @@ def format_record(record): def make_insert_mutations(element): import uuid # pylint: disable=reimported + from apache_beam.io.gcp.experimental.spannerio import WriteMutation ins_mutation = WriteMutation.insert( table='test', diff --git a/sdks/python/apache_beam/io/gcp/gcsio_integration_test.py b/sdks/python/apache_beam/io/gcp/gcsio_integration_test.py index 03f12a7ef06c..4616f007bfc5 100644 --- a/sdks/python/apache_beam/io/gcp/gcsio_integration_test.py +++ b/sdks/python/apache_beam/io/gcp/gcsio_integration_test.py @@ -209,6 +209,7 @@ def test_create_default_bucket(self, mock_default_gcs_bucket_name): import random from hashlib import blake2b + # Add a random number to avoid collision if multiple test instances # are run at the same time. To avoid too many dangling buckets if bucket # removal fails, we limit the max number of possible bucket names in this diff --git a/sdks/python/apache_beam/io/gcp/gcsio_retry_test.py b/sdks/python/apache_beam/io/gcp/gcsio_retry_test.py index 750879ae0284..f53ea5244244 100644 --- a/sdks/python/apache_beam/io/gcp/gcsio_retry_test.py +++ b/sdks/python/apache_beam/io/gcp/gcsio_retry_test.py @@ -27,8 +27,9 @@ from apache_beam.utils import counters try: - from apache_beam.io.gcp import gcsio_retry from google.api_core import exceptions as api_exceptions + + from apache_beam.io.gcp import gcsio_retry except ImportError: gcsio_retry = None api_exceptions = None diff --git a/sdks/python/apache_beam/io/gcp/gcsio_test.py b/sdks/python/apache_beam/io/gcp/gcsio_test.py index 4c18647729e3..60ba3bda8eb2 100644 --- a/sdks/python/apache_beam/io/gcp/gcsio_test.py +++ b/sdks/python/apache_beam/io/gcp/gcsio_test.py @@ -37,9 +37,11 @@ # pylint: disable=wrong-import-order, wrong-import-position try: + from google.cloud.exceptions import BadRequest + from google.cloud.exceptions import NotFound + from apache_beam.io.gcp import gcsio from apache_beam.io.gcp.gcsio_retry import DEFAULT_RETRY_WITH_THROTTLING_COUNTER - from google.cloud.exceptions import BadRequest, NotFound except ImportError: NotFound = None # pylint: enable=wrong-import-order, wrong-import-position diff --git a/sdks/python/apache_beam/io/gcp/healthcare/dicomio_integration_test.py b/sdks/python/apache_beam/io/gcp/healthcare/dicomio_integration_test.py index 499649beae46..ca602c79a464 100644 --- a/sdks/python/apache_beam/io/gcp/healthcare/dicomio_integration_test.py +++ b/sdks/python/apache_beam/io/gcp/healthcare/dicomio_integration_test.py @@ -40,11 +40,12 @@ # pylint: disable=wrong-import-order, wrong-import-position try: + from google.auth import default + from google.auth.transport import requests + from apache_beam.io.gcp.healthcare.dicomclient import DicomApiHttpClient from apache_beam.io.gcp.healthcare.dicomio import DicomSearch from apache_beam.io.gcp.healthcare.dicomio import UploadToDicomStore - from google.auth import default - from google.auth.transport import requests except ImportError: DicomSearch = None # pylint: enable=wrong-import-order, wrong-import-position diff --git a/sdks/python/apache_beam/io/gcp/internal/clients/bigquery/__init__.py b/sdks/python/apache_beam/io/gcp/internal/clients/bigquery/__init__.py index 6f7bb4adbb8b..ec7df8aa128f 100644 --- a/sdks/python/apache_beam/io/gcp/internal/clients/bigquery/__init__.py +++ b/sdks/python/apache_beam/io/gcp/internal/clients/bigquery/__init__.py @@ -24,6 +24,7 @@ # pylint: disable=wrong-import-order, wrong-import-position try: from apitools.base.py import * + from apache_beam.io.gcp.internal.clients.bigquery.bigquery_v2_client import * from apache_beam.io.gcp.internal.clients.bigquery.bigquery_v2_messages import * except ImportError: diff --git a/sdks/python/apache_beam/io/gcp/pubsub_io_perf_test.py b/sdks/python/apache_beam/io/gcp/pubsub_io_perf_test.py index aece17a1eaf3..7ca831c980e7 100644 --- a/sdks/python/apache_beam/io/gcp/pubsub_io_perf_test.py +++ b/sdks/python/apache_beam/io/gcp/pubsub_io_perf_test.py @@ -117,6 +117,7 @@ def __init__(self): def test(self): def to_pubsub_message(element): import uuid + from apache_beam.io import PubsubMessage return PubsubMessage( data=element[1], diff --git a/sdks/python/apache_beam/io/gcp/pubsublite/__init__.py b/sdks/python/apache_beam/io/gcp/pubsublite/__init__.py index e0d08c918031..565777e14050 100644 --- a/sdks/python/apache_beam/io/gcp/pubsublite/__init__.py +++ b/sdks/python/apache_beam/io/gcp/pubsublite/__init__.py @@ -15,7 +15,8 @@ # limitations under the License. # -from .proto_api import ReadFromPubSubLite, WriteToPubSubLite +from .proto_api import ReadFromPubSubLite +from .proto_api import WriteToPubSubLite __all__ = [ "ReadFromPubSubLite", diff --git a/sdks/python/apache_beam/io/mongodbio.py b/sdks/python/apache_beam/io/mongodbio.py index 834c051aca5c..e8602edd40cd 100644 --- a/sdks/python/apache_beam/io/mongodbio.py +++ b/sdks/python/apache_beam/io/mongodbio.py @@ -92,7 +92,6 @@ from bson import json_util from bson import objectid from bson.objectid import ObjectId - # pymongo also internally depends on bson. from pymongo import ASCENDING from pymongo import DESCENDING diff --git a/sdks/python/apache_beam/io/parquetio.py b/sdks/python/apache_beam/io/parquetio.py index 0b38c69437c0..e5dce15e71ef 100644 --- a/sdks/python/apache_beam/io/parquetio.py +++ b/sdks/python/apache_beam/io/parquetio.py @@ -54,6 +54,7 @@ import pyarrow as pa paTable = pa.Table import pyarrow.parquet as pq + # pylint: disable=ungrouped-imports from apache_beam.typehints import arrow_type_compatibility except ImportError: diff --git a/sdks/python/apache_beam/io/requestresponse_it_test.py b/sdks/python/apache_beam/io/requestresponse_it_test.py index 8ac7cdb6f5fd..8703653b266e 100644 --- a/sdks/python/apache_beam/io/requestresponse_it_test.py +++ b/sdks/python/apache_beam/io/requestresponse_it_test.py @@ -35,6 +35,7 @@ # pylint: disable=ungrouped-imports try: from testcontainers.redis import RedisContainer + from apache_beam.io.requestresponse import Caller from apache_beam.io.requestresponse import RedisCache from apache_beam.io.requestresponse import RequestResponseIO diff --git a/sdks/python/apache_beam/io/requestresponse_test.py b/sdks/python/apache_beam/io/requestresponse_test.py index 4adf2fc7649c..f88df9657dae 100644 --- a/sdks/python/apache_beam/io/requestresponse_test.py +++ b/sdks/python/apache_beam/io/requestresponse_test.py @@ -28,6 +28,7 @@ # pylint: disable=ungrouped-imports try: from google.api_core.exceptions import TooManyRequests + from apache_beam.io.requestresponse import Caller from apache_beam.io.requestresponse import DefaultThrottler from apache_beam.io.requestresponse import RequestResponseIO diff --git a/sdks/python/apache_beam/io/textio_test.py b/sdks/python/apache_beam/io/textio_test.py index 4f804fa44c44..3854a22640a9 100644 --- a/sdks/python/apache_beam/io/textio_test.py +++ b/sdks/python/apache_beam/io/textio_test.py @@ -39,14 +39,14 @@ from apache_beam.io import source_test_utils from apache_beam.io.filesystem import CompressionTypes from apache_beam.io.filesystems import FileSystems -from apache_beam.io.textio import _TextSink as TextSink -from apache_beam.io.textio import _TextSource as TextSource # Importing following private classes for testing. from apache_beam.io.textio import ReadAllFromText from apache_beam.io.textio import ReadAllFromTextContinuously from apache_beam.io.textio import ReadFromText from apache_beam.io.textio import ReadFromTextWithFilename from apache_beam.io.textio import WriteToText +from apache_beam.io.textio import _TextSink as TextSink +from apache_beam.io.textio import _TextSource as TextSource from apache_beam.options.pipeline_options import PipelineOptions from apache_beam.testing.test_pipeline import TestPipeline from apache_beam.testing.test_stream import TestStream diff --git a/sdks/python/apache_beam/io/tfrecordio.py b/sdks/python/apache_beam/io/tfrecordio.py index e27ea5070b06..c6c59b2c2bed 100644 --- a/sdks/python/apache_beam/io/tfrecordio.py +++ b/sdks/python/apache_beam/io/tfrecordio.py @@ -47,6 +47,7 @@ def _default_crc32c_fn(value): if not _default_crc32c_fn.fn: try: import snappy # pylint: disable=import-error + # Support multiple versions of python-snappy: # https://github.com/andrix/python-snappy/pull/53 if getattr(snappy, '_crc32c', None): diff --git a/sdks/python/apache_beam/metrics/metric.py b/sdks/python/apache_beam/metrics/metric.py index a812ef7f3366..7080dfef009d 100644 --- a/sdks/python/apache_beam/metrics/metric.py +++ b/sdks/python/apache_beam/metrics/metric.py @@ -53,10 +53,10 @@ from apache_beam.metrics.metricbase import StringSet if TYPE_CHECKING: + from apache_beam.internal.metrics.metric import MetricLogger from apache_beam.metrics.execution import MetricKey from apache_beam.metrics.metricbase import Metric from apache_beam.utils.histogram import BucketType - from apache_beam.internal.metrics.metric import MetricLogger __all__ = ['Metrics', 'MetricsFilter', 'Lineage'] diff --git a/sdks/python/apache_beam/ml/anomaly/detectors/__init__.py b/sdks/python/apache_beam/ml/anomaly/detectors/__init__.py index f3268755cf99..45f952c7c2f5 100644 --- a/sdks/python/apache_beam/ml/anomaly/detectors/__init__.py +++ b/sdks/python/apache_beam/ml/anomaly/detectors/__init__.py @@ -15,6 +15,6 @@ # limitations under the License. # -from apache_beam.ml.anomaly.detectors.zscore import ZScore -from apache_beam.ml.anomaly.detectors.robust_zscore import RobustZScore from apache_beam.ml.anomaly.detectors.iqr import IQR +from apache_beam.ml.anomaly.detectors.robust_zscore import RobustZScore +from apache_beam.ml.anomaly.detectors.zscore import ZScore diff --git a/sdks/python/apache_beam/ml/anomaly/detectors/pyod_adapter.py b/sdks/python/apache_beam/ml/anomaly/detectors/pyod_adapter.py index 10bd25514761..0e2d4f2b4a56 100644 --- a/sdks/python/apache_beam/ml/anomaly/detectors/pyod_adapter.py +++ b/sdks/python/apache_beam/ml/anomaly/detectors/pyod_adapter.py @@ -22,6 +22,7 @@ from typing import Optional import numpy as np +from pyod.models.base import BaseDetector as PyODBaseDetector import apache_beam as beam from apache_beam.io.filesystems import FileSystems @@ -33,7 +34,6 @@ from apache_beam.ml.inference.base import PredictionResult from apache_beam.ml.inference.base import _PostProcessingModelHandler from apache_beam.ml.inference.utils import _convert_to_result -from pyod.models.base import BaseDetector as PyODBaseDetector # Turn the used ModelHandler into specifiable, but without lazy init. KeyedModelHandler = specifiable( # type: ignore[misc] diff --git a/sdks/python/apache_beam/ml/anomaly/detectors/pyod_adapter_test.py b/sdks/python/apache_beam/ml/anomaly/detectors/pyod_adapter_test.py index c9acfdbb11d0..7b49ef781e61 100644 --- a/sdks/python/apache_beam/ml/anomaly/detectors/pyod_adapter_test.py +++ b/sdks/python/apache_beam/ml/anomaly/detectors/pyod_adapter_test.py @@ -37,8 +37,9 @@ # Protect against environments where onnx and pytorch library is not available. # pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports try: - from apache_beam.ml.anomaly.detectors.pyod_adapter import PyODFactory from pyod.models.iforest import IForest + + from apache_beam.ml.anomaly.detectors.pyod_adapter import PyODFactory except ImportError: raise unittest.SkipTest('PyOD dependencies are not installed') diff --git a/sdks/python/apache_beam/ml/gcp/cloud_dlp_test.py b/sdks/python/apache_beam/ml/gcp/cloud_dlp_test.py index 51916eaaf6c7..a6677197a0a9 100644 --- a/sdks/python/apache_beam/ml/gcp/cloud_dlp_test.py +++ b/sdks/python/apache_beam/ml/gcp/cloud_dlp_test.py @@ -33,11 +33,12 @@ except ImportError: dlp_v2 = None else: + from google.cloud.dlp_v2.types import dlp + from apache_beam.ml.gcp.cloud_dlp import InspectForDetails from apache_beam.ml.gcp.cloud_dlp import MaskDetectedDetails from apache_beam.ml.gcp.cloud_dlp import _DeidentifyFn from apache_beam.ml.gcp.cloud_dlp import _InspectFn - from google.cloud.dlp_v2.types import dlp # pylint: enable=wrong-import-order, wrong-import-position, ungrouped-imports _LOGGER = logging.getLogger(__name__) diff --git a/sdks/python/apache_beam/ml/gcp/recommendations_ai_test.py b/sdks/python/apache_beam/ml/gcp/recommendations_ai_test.py index 2f688d97a309..d2844f8ac08c 100644 --- a/sdks/python/apache_beam/ml/gcp/recommendations_ai_test.py +++ b/sdks/python/apache_beam/ml/gcp/recommendations_ai_test.py @@ -29,6 +29,7 @@ # pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports try: from google.cloud import recommendationengine + from apache_beam.ml.gcp import recommendations_ai except ImportError: recommendationengine = None diff --git a/sdks/python/apache_beam/ml/gcp/recommendations_ai_test_it.py b/sdks/python/apache_beam/ml/gcp/recommendations_ai_test_it.py index 9f739de7883d..ad2d45a8e539 100644 --- a/sdks/python/apache_beam/ml/gcp/recommendations_ai_test_it.py +++ b/sdks/python/apache_beam/ml/gcp/recommendations_ai_test_it.py @@ -34,6 +34,7 @@ # pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports try: from google.cloud import recommendationengine + from apache_beam.ml.gcp import recommendations_ai except ImportError: recommendationengine = None diff --git a/sdks/python/apache_beam/ml/gcp/videointelligenceml_test.py b/sdks/python/apache_beam/ml/gcp/videointelligenceml_test.py index 79c841938cdb..3ea25965efe9 100644 --- a/sdks/python/apache_beam/ml/gcp/videointelligenceml_test.py +++ b/sdks/python/apache_beam/ml/gcp/videointelligenceml_test.py @@ -31,8 +31,9 @@ # Protect against environments where video intelligence lib is not available. # pylint: disable=ungrouped-imports try: - from google.cloud.videointelligence import VideoIntelligenceServiceClient from google.cloud import videointelligence + from google.cloud.videointelligence import VideoIntelligenceServiceClient + from apache_beam.ml.gcp import videointelligenceml except ImportError: VideoIntelligenceServiceClient = None diff --git a/sdks/python/apache_beam/ml/gcp/videointelligenceml_test_it.py b/sdks/python/apache_beam/ml/gcp/videointelligenceml_test_it.py index 03f79d171597..eac951697d3d 100644 --- a/sdks/python/apache_beam/ml/gcp/videointelligenceml_test_it.py +++ b/sdks/python/apache_beam/ml/gcp/videointelligenceml_test_it.py @@ -32,9 +32,10 @@ # Protect against environments where Google Cloud VideoIntelligence client is # not available. try: - from apache_beam.ml.gcp.videointelligenceml import AnnotateVideoWithContext from google.cloud.videointelligence import enums from google.cloud.videointelligence import types + + from apache_beam.ml.gcp.videointelligenceml import AnnotateVideoWithContext except ImportError: AnnotateVideoWithContext = None diff --git a/sdks/python/apache_beam/ml/gcp/visionml_test.py b/sdks/python/apache_beam/ml/gcp/visionml_test.py index 479b3d80e4de..79b3e47f9cb7 100644 --- a/sdks/python/apache_beam/ml/gcp/visionml_test.py +++ b/sdks/python/apache_beam/ml/gcp/visionml_test.py @@ -31,8 +31,9 @@ # Protect against environments where vision lib is not available. try: - from google.cloud.vision import ImageAnnotatorClient from google.cloud import vision + from google.cloud.vision import ImageAnnotatorClient + from apache_beam.ml.gcp import visionml except ImportError: ImageAnnotatorClient = None diff --git a/sdks/python/apache_beam/ml/gcp/visionml_test_it.py b/sdks/python/apache_beam/ml/gcp/visionml_test_it.py index 00fd38704a02..68b1aeccfb6e 100644 --- a/sdks/python/apache_beam/ml/gcp/visionml_test_it.py +++ b/sdks/python/apache_beam/ml/gcp/visionml_test_it.py @@ -28,8 +28,9 @@ # Protect against environments where Google Cloud Vision client is not # available. try: - from apache_beam.ml.gcp.visionml import AnnotateImage from google.cloud import vision + + from apache_beam.ml.gcp.visionml import AnnotateImage except ImportError: vision = None diff --git a/sdks/python/apache_beam/ml/inference/gemini_inference.py b/sdks/python/apache_beam/ml/inference/gemini_inference.py index a04ee2533a1b..c840efedd8fd 100644 --- a/sdks/python/apache_beam/ml/inference/gemini_inference.py +++ b/sdks/python/apache_beam/ml/inference/gemini_inference.py @@ -26,11 +26,11 @@ from google import genai from google.genai import errors from google.genai.types import Part +from PIL.Image import Image from apache_beam.ml.inference import utils from apache_beam.ml.inference.base import PredictionResult from apache_beam.ml.inference.base import RemoteModelHandler -from PIL.Image import Image LOGGER = logging.getLogger("GeminiModelHandler") diff --git a/sdks/python/apache_beam/ml/inference/gemini_inference_test.py b/sdks/python/apache_beam/ml/inference/gemini_inference_test.py index bb6127a32872..cb73c7de13f4 100644 --- a/sdks/python/apache_beam/ml/inference/gemini_inference_test.py +++ b/sdks/python/apache_beam/ml/inference/gemini_inference_test.py @@ -19,10 +19,11 @@ import unittest try: - from apache_beam.ml.inference.gemini_inference import _retry_on_appropriate_service_error + from google.genai import errors + from apache_beam.ml.inference.gemini_inference import GeminiModelHandler + from apache_beam.ml.inference.gemini_inference import _retry_on_appropriate_service_error from apache_beam.ml.inference.gemini_inference import generate_from_string - from google.genai import errors except ImportError: raise unittest.SkipTest('Gemini dependencies are not installed') diff --git a/sdks/python/apache_beam/ml/inference/huggingface_inference.py b/sdks/python/apache_beam/ml/inference/huggingface_inference.py index 6e22fb3b3d69..501a019c378e 100644 --- a/sdks/python/apache_beam/ml/inference/huggingface_inference.py +++ b/sdks/python/apache_beam/ml/inference/huggingface_inference.py @@ -30,15 +30,16 @@ import tensorflow as tf import torch -from apache_beam.ml.inference import utils -from apache_beam.ml.inference.base import ModelHandler -from apache_beam.ml.inference.base import PredictionResult -from apache_beam.ml.inference.pytorch_inference import _convert_to_device from transformers import AutoModel from transformers import Pipeline from transformers import TFAutoModel from transformers import pipeline +from apache_beam.ml.inference import utils +from apache_beam.ml.inference.base import ModelHandler +from apache_beam.ml.inference.base import PredictionResult +from apache_beam.ml.inference.pytorch_inference import _convert_to_device + _LOGGER = logging.getLogger(__name__) __all__ = [ diff --git a/sdks/python/apache_beam/ml/inference/huggingface_inference_test.py b/sdks/python/apache_beam/ml/inference/huggingface_inference_test.py index 2c45cf64902b..5cd55f1b1f5f 100644 --- a/sdks/python/apache_beam/ml/inference/huggingface_inference_test.py +++ b/sdks/python/apache_beam/ml/inference/huggingface_inference_test.py @@ -39,6 +39,7 @@ import torch from transformers import AutoModel from transformers import TFAutoModel + from apache_beam.ml.inference.huggingface_inference import HuggingFaceModelHandlerTensor except ImportError: raise unittest.SkipTest('Transformers dependencies are not installed.') diff --git a/sdks/python/apache_beam/ml/inference/onnx_inference.py b/sdks/python/apache_beam/ml/inference/onnx_inference.py index 53099a6f3e90..3485866f11c3 100644 --- a/sdks/python/apache_beam/ml/inference/onnx_inference.py +++ b/sdks/python/apache_beam/ml/inference/onnx_inference.py @@ -23,9 +23,9 @@ from typing import Optional import numpy - import onnx import onnxruntime as ort + from apache_beam.io.filesystems import FileSystems from apache_beam.ml.inference import utils from apache_beam.ml.inference.base import ModelHandler diff --git a/sdks/python/apache_beam/ml/inference/onnx_inference_it_test.py b/sdks/python/apache_beam/ml/inference/onnx_inference_it_test.py index 3902a61dc260..cc86be570acf 100644 --- a/sdks/python/apache_beam/ml/inference/onnx_inference_it_test.py +++ b/sdks/python/apache_beam/ml/inference/onnx_inference_it_test.py @@ -30,6 +30,7 @@ # pylint: disable=ungrouped-imports try: import onnx + from apache_beam.examples.inference import onnx_sentiment_classification except ImportError as e: onnx = None diff --git a/sdks/python/apache_beam/ml/inference/onnx_inference_test.py b/sdks/python/apache_beam/ml/inference/onnx_inference_test.py index 2d2de4a388e0..61e8c983c9d3 100644 --- a/sdks/python/apache_beam/ml/inference/onnx_inference_test.py +++ b/sdks/python/apache_beam/ml/inference/onnx_inference_test.py @@ -39,17 +39,18 @@ # pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports try: import onnxruntime as ort - import torch import tensorflow as tf import tf2onnx - from tensorflow.keras import layers - from sklearn import linear_model + import torch from skl2onnx import convert_sklearn from skl2onnx.common.data_types import FloatTensorType + from sklearn import linear_model + from tensorflow.keras import layers + from apache_beam.ml.inference.base import PredictionResult from apache_beam.ml.inference.base import RunInference - from apache_beam.ml.inference.onnx_inference import default_numpy_inference_fn from apache_beam.ml.inference.onnx_inference import OnnxModelHandlerNumpy + from apache_beam.ml.inference.onnx_inference import default_numpy_inference_fn except ImportError: raise unittest.SkipTest('Onnx dependencies are not installed') diff --git a/sdks/python/apache_beam/ml/inference/pytorch_inference.py b/sdks/python/apache_beam/ml/inference/pytorch_inference.py index 80e31f1aac8b..f73eeff808ce 100644 --- a/sdks/python/apache_beam/ml/inference/pytorch_inference.py +++ b/sdks/python/apache_beam/ml/inference/pytorch_inference.py @@ -26,6 +26,7 @@ from typing import Optional import torch + from apache_beam.io.filesystems import FileSystems from apache_beam.ml.inference import utils from apache_beam.ml.inference.base import ModelHandler diff --git a/sdks/python/apache_beam/ml/inference/pytorch_inference_it_test.py b/sdks/python/apache_beam/ml/inference/pytorch_inference_it_test.py index 035047547a77..c9c3d06434a7 100644 --- a/sdks/python/apache_beam/ml/inference/pytorch_inference_it_test.py +++ b/sdks/python/apache_beam/ml/inference/pytorch_inference_it_test.py @@ -30,10 +30,11 @@ # pylint: disable=ungrouped-imports try: import torch + from apache_beam.examples.inference import pytorch_image_classification from apache_beam.examples.inference import pytorch_image_segmentation - from apache_beam.examples.inference import pytorch_model_per_key_image_segmentation from apache_beam.examples.inference import pytorch_language_modeling + from apache_beam.examples.inference import pytorch_model_per_key_image_segmentation except ImportError as e: torch = None diff --git a/sdks/python/apache_beam/ml/inference/pytorch_inference_test.py b/sdks/python/apache_beam/ml/inference/pytorch_inference_test.py index fcc374c06d78..50279820b267 100644 --- a/sdks/python/apache_beam/ml/inference/pytorch_inference_test.py +++ b/sdks/python/apache_beam/ml/inference/pytorch_inference_test.py @@ -35,15 +35,16 @@ # pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports try: import torch + + from apache_beam.ml.inference import pytorch_inference from apache_beam.ml.inference.base import PredictionResult from apache_beam.ml.inference.base import RunInference - from apache_beam.ml.inference import pytorch_inference + from apache_beam.ml.inference.pytorch_inference import PytorchModelHandlerKeyedTensor + from apache_beam.ml.inference.pytorch_inference import PytorchModelHandlerTensor from apache_beam.ml.inference.pytorch_inference import default_keyed_tensor_inference_fn from apache_beam.ml.inference.pytorch_inference import default_tensor_inference_fn from apache_beam.ml.inference.pytorch_inference import make_keyed_tensor_model_fn from apache_beam.ml.inference.pytorch_inference import make_tensor_model_fn - from apache_beam.ml.inference.pytorch_inference import PytorchModelHandlerTensor - from apache_beam.ml.inference.pytorch_inference import PytorchModelHandlerKeyedTensor except ImportError: raise unittest.SkipTest('PyTorch dependencies are not installed') diff --git a/sdks/python/apache_beam/ml/inference/tensorflow_inference.py b/sdks/python/apache_beam/ml/inference/tensorflow_inference.py index 36340aa36b60..d13ea53cf1bc 100644 --- a/sdks/python/apache_beam/ml/inference/tensorflow_inference.py +++ b/sdks/python/apache_beam/ml/inference/tensorflow_inference.py @@ -27,9 +27,9 @@ from typing import Union import numpy - import tensorflow as tf import tensorflow_hub as hub + from apache_beam.ml.inference import utils from apache_beam.ml.inference.base import ModelHandler from apache_beam.ml.inference.base import PredictionResult diff --git a/sdks/python/apache_beam/ml/inference/tensorflow_inference_it_test.py b/sdks/python/apache_beam/ml/inference/tensorflow_inference_it_test.py index 4786b7a03980..679c4d7f74cb 100644 --- a/sdks/python/apache_beam/ml/inference/tensorflow_inference_it_test.py +++ b/sdks/python/apache_beam/ml/inference/tensorflow_inference_it_test.py @@ -31,6 +31,7 @@ try: import tensorflow as tf import tensorflow_hub as hub + from apache_beam.examples.inference import tensorflow_imagenet_segmentation from apache_beam.examples.inference import tensorflow_mnist_classification from apache_beam.examples.inference import tensorflow_mnist_with_weights diff --git a/sdks/python/apache_beam/ml/inference/tensorflow_inference_test.py b/sdks/python/apache_beam/ml/inference/tensorflow_inference_test.py index 7286274e180c..c884ee58b0a0 100644 --- a/sdks/python/apache_beam/ml/inference/tensorflow_inference_test.py +++ b/sdks/python/apache_beam/ml/inference/tensorflow_inference_test.py @@ -43,9 +43,11 @@ # pylint: disable=ungrouped-imports try: import tensorflow as tf - from apache_beam.ml.inference.sklearn_inference_test import _compare_prediction_result - from apache_beam.ml.inference.tensorflow_inference import TFModelHandlerNumpy, TFModelHandlerTensor + from apache_beam.ml.inference import tensorflow_inference + from apache_beam.ml.inference.sklearn_inference_test import _compare_prediction_result + from apache_beam.ml.inference.tensorflow_inference import TFModelHandlerNumpy + from apache_beam.ml.inference.tensorflow_inference import TFModelHandlerTensor except ImportError: raise unittest.SkipTest( 'Tensorflow dependencies are not installed. ' + diff --git a/sdks/python/apache_beam/ml/inference/tensorrt_inference.py b/sdks/python/apache_beam/ml/inference/tensorrt_inference.py index 0f49489a437a..1b11bd9f39e2 100644 --- a/sdks/python/apache_beam/ml/inference/tensorrt_inference.py +++ b/sdks/python/apache_beam/ml/inference/tensorrt_inference.py @@ -110,8 +110,8 @@ def __init__(self, engine: trt.ICudaEngine): Args: engine: trt.ICudaEngine object that contains TensorRT engine """ - from cuda import cuda import tensorrt as trt + from cuda import cuda self.engine = engine self.context = engine.create_execution_context() self.context_lock = threading.RLock() diff --git a/sdks/python/apache_beam/ml/inference/tensorrt_inference_test.py b/sdks/python/apache_beam/ml/inference/tensorrt_inference_test.py index cb010e82cfca..39e46c7f7c0d 100644 --- a/sdks/python/apache_beam/ml/inference/tensorrt_inference_test.py +++ b/sdks/python/apache_beam/ml/inference/tensorrt_inference_test.py @@ -32,10 +32,11 @@ # pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports try: import tensorrt as trt + from apache_beam.ml.inference import utils - from apache_beam.ml.inference.base import PredictionResult, RunInference - from apache_beam.ml.inference.tensorrt_inference import \ - TensorRTEngineHandlerNumPy + from apache_beam.ml.inference.base import PredictionResult + from apache_beam.ml.inference.base import RunInference + from apache_beam.ml.inference.tensorrt_inference import TensorRTEngineHandlerNumPy except ImportError: raise unittest.SkipTest('TensorRT dependencies are not installed') diff --git a/sdks/python/apache_beam/ml/inference/vertex_ai_inference_test.py b/sdks/python/apache_beam/ml/inference/vertex_ai_inference_test.py index 34c7927272d6..91a3b82cf762 100644 --- a/sdks/python/apache_beam/ml/inference/vertex_ai_inference_test.py +++ b/sdks/python/apache_beam/ml/inference/vertex_ai_inference_test.py @@ -19,9 +19,10 @@ import unittest try: - from apache_beam.ml.inference.vertex_ai_inference import _retry_on_appropriate_gcp_error - from apache_beam.ml.inference.vertex_ai_inference import VertexAIModelHandlerJSON from google.api_core.exceptions import TooManyRequests + + from apache_beam.ml.inference.vertex_ai_inference import VertexAIModelHandlerJSON + from apache_beam.ml.inference.vertex_ai_inference import _retry_on_appropriate_gcp_error except ImportError: raise unittest.SkipTest('VertexAI dependencies are not installed') diff --git a/sdks/python/apache_beam/ml/inference/vllm_inference.py b/sdks/python/apache_beam/ml/inference/vllm_inference.py index 0bb6ccd6108e..bdbee9e51fd5 100644 --- a/sdks/python/apache_beam/ml/inference/vllm_inference.py +++ b/sdks/python/apache_beam/ml/inference/vllm_inference.py @@ -31,12 +31,13 @@ from typing import Any from typing import Optional +from openai import AsyncOpenAI +from openai import OpenAI + from apache_beam.io.filesystems import FileSystems from apache_beam.ml.inference.base import ModelHandler from apache_beam.ml.inference.base import PredictionResult from apache_beam.utils import subprocess_server -from openai import AsyncOpenAI -from openai import OpenAI try: # VLLM logging config breaks beam logging. diff --git a/sdks/python/apache_beam/ml/inference/xgboost_inference.py b/sdks/python/apache_beam/ml/inference/xgboost_inference.py index ee4e8a9b6e07..10289b076416 100644 --- a/sdks/python/apache_beam/ml/inference/xgboost_inference.py +++ b/sdks/python/apache_beam/ml/inference/xgboost_inference.py @@ -25,12 +25,12 @@ from typing import Optional from typing import Union +import datatable import numpy import pandas import scipy - -import datatable import xgboost + from apache_beam.io.filesystems import FileSystems from apache_beam.ml.inference.base import ExampleT from apache_beam.ml.inference.base import ModelHandler diff --git a/sdks/python/apache_beam/ml/inference/xgboost_inference_it_test.py b/sdks/python/apache_beam/ml/inference/xgboost_inference_it_test.py index 3db62bcc6a99..73becf4ff104 100644 --- a/sdks/python/apache_beam/ml/inference/xgboost_inference_it_test.py +++ b/sdks/python/apache_beam/ml/inference/xgboost_inference_it_test.py @@ -19,8 +19,9 @@ import uuid try: - import pytest import unittest + + import pytest import xgboost from apache_beam.examples.inference import xgboost_iris_classification diff --git a/sdks/python/apache_beam/ml/rag/chunking/langchain_test.py b/sdks/python/apache_beam/ml/rag/chunking/langchain_test.py index 542d1cd79bc2..580d2b17e25b 100644 --- a/sdks/python/apache_beam/ml/rag/chunking/langchain_test.py +++ b/sdks/python/apache_beam/ml/rag/chunking/langchain_test.py @@ -27,10 +27,10 @@ from apache_beam.testing.util import is_not_empty try: - from apache_beam.ml.rag.chunking.langchain import LangChainChunker + from langchain.text_splitter import CharacterTextSplitter + from langchain.text_splitter import RecursiveCharacterTextSplitter - from langchain.text_splitter import ( - CharacterTextSplitter, RecursiveCharacterTextSplitter) + from apache_beam.ml.rag.chunking.langchain import LangChainChunker LANGCHAIN_AVAILABLE = True except ImportError: LANGCHAIN_AVAILABLE = False diff --git a/sdks/python/apache_beam/ml/rag/embeddings/vertex_ai_test.py b/sdks/python/apache_beam/ml/rag/embeddings/vertex_ai_test.py index 320a562d5009..435475ffb33b 100644 --- a/sdks/python/apache_beam/ml/rag/embeddings/vertex_ai_test.py +++ b/sdks/python/apache_beam/ml/rag/embeddings/vertex_ai_test.py @@ -32,6 +32,7 @@ # pylint: disable=ungrouped-imports try: import vertexai # pylint: disable=unused-import + from apache_beam.ml.rag.embeddings.vertex_ai import VertexAITextEmbeddings VERTEX_AI_AVAILABLE = True except ImportError: diff --git a/sdks/python/apache_beam/ml/rag/enrichment/bigquery_vector_search_it_test.py b/sdks/python/apache_beam/ml/rag/enrichment/bigquery_vector_search_it_test.py index 1d4f7597d625..f626139040cf 100644 --- a/sdks/python/apache_beam/ml/rag/enrichment/bigquery_vector_search_it_test.py +++ b/sdks/python/apache_beam/ml/rag/enrichment/bigquery_vector_search_it_test.py @@ -32,11 +32,9 @@ # pylint: disable=ungrouped-imports try: + from apache_beam.ml.rag.enrichment.bigquery_vector_search import BigQueryVectorSearchEnrichmentHandler + from apache_beam.ml.rag.enrichment.bigquery_vector_search import BigQueryVectorSearchParameters from apache_beam.transforms.enrichment import Enrichment - from apache_beam.ml.rag.enrichment.bigquery_vector_search import \ - BigQueryVectorSearchEnrichmentHandler - from apache_beam.ml.rag.enrichment.bigquery_vector_search import \ - BigQueryVectorSearchParameters except ImportError: raise unittest.SkipTest('BigQuery dependencies not installed') diff --git a/sdks/python/apache_beam/ml/rag/enrichment/milvus_search.py b/sdks/python/apache_beam/ml/rag/enrichment/milvus_search.py index 431c0db3f416..8f631746748b 100644 --- a/sdks/python/apache_beam/ml/rag/enrichment/milvus_search.py +++ b/sdks/python/apache_beam/ml/rag/enrichment/milvus_search.py @@ -411,8 +411,9 @@ def __init__( self.use_custom_types = True def __enter__(self): - import time import logging + import time + from pymilvus.exceptions import MilvusException connection_params = unpack_dataclass_with_kwargs( diff --git a/sdks/python/apache_beam/ml/rag/enrichment/milvus_search_it_test.py b/sdks/python/apache_beam/ml/rag/enrichment/milvus_search_it_test.py index 2df9af2f1144..b3a0dcd55722 100644 --- a/sdks/python/apache_beam/ml/rag/enrichment/milvus_search_it_test.py +++ b/sdks/python/apache_beam/ml/rag/enrichment/milvus_search_it_test.py @@ -44,30 +44,29 @@ # pylint: disable=ungrouped-imports try: - from pymilvus import ( - CollectionSchema, - DataType, - FieldSchema, - Function, - FunctionType, - MilvusClient, - RRFRanker) + from pymilvus import CollectionSchema + from pymilvus import DataType + from pymilvus import FieldSchema + from pymilvus import Function + from pymilvus import FunctionType + from pymilvus import MilvusClient + from pymilvus import RRFRanker from pymilvus.milvus_client import IndexParams from testcontainers.core.config import MAX_TRIES as TC_MAX_TRIES from testcontainers.core.config import testcontainers_config from testcontainers.core.generic import DbContainer from testcontainers.milvus import MilvusContainer + + from apache_beam.ml.rag.enrichment.milvus_search import HybridSearchParameters + from apache_beam.ml.rag.enrichment.milvus_search import KeywordSearchMetrics + from apache_beam.ml.rag.enrichment.milvus_search import KeywordSearchParameters + from apache_beam.ml.rag.enrichment.milvus_search import MilvusCollectionLoadParameters + from apache_beam.ml.rag.enrichment.milvus_search import MilvusConnectionParameters + from apache_beam.ml.rag.enrichment.milvus_search import MilvusSearchEnrichmentHandler + from apache_beam.ml.rag.enrichment.milvus_search import MilvusSearchParameters + from apache_beam.ml.rag.enrichment.milvus_search import VectorSearchMetrics + from apache_beam.ml.rag.enrichment.milvus_search import VectorSearchParameters from apache_beam.transforms.enrichment import Enrichment - from apache_beam.ml.rag.enrichment.milvus_search import ( - MilvusSearchEnrichmentHandler, - MilvusConnectionParameters, - MilvusSearchParameters, - MilvusCollectionLoadParameters, - VectorSearchParameters, - KeywordSearchParameters, - HybridSearchParameters, - VectorSearchMetrics, - KeywordSearchMetrics) except ImportError as e: raise unittest.SkipTest(f'Milvus dependencies not installed: {str(e)}') diff --git a/sdks/python/apache_beam/ml/rag/enrichment/milvus_search_test.py b/sdks/python/apache_beam/ml/rag/enrichment/milvus_search_test.py index e69915cb3e9b..ef5af8ca4940 100644 --- a/sdks/python/apache_beam/ml/rag/enrichment/milvus_search_test.py +++ b/sdks/python/apache_beam/ml/rag/enrichment/milvus_search_test.py @@ -19,19 +19,18 @@ from parameterized import parameterized try: + from apache_beam.ml.rag.enrichment.milvus_search import HybridSearchParameters + from apache_beam.ml.rag.enrichment.milvus_search import KeywordSearchParameters + from apache_beam.ml.rag.enrichment.milvus_search import MilvusBaseRanker + from apache_beam.ml.rag.enrichment.milvus_search import MilvusCollectionLoadParameters + from apache_beam.ml.rag.enrichment.milvus_search import MilvusConnectionParameters + from apache_beam.ml.rag.enrichment.milvus_search import MilvusSearchEnrichmentHandler + from apache_beam.ml.rag.enrichment.milvus_search import MilvusSearchParameters + from apache_beam.ml.rag.enrichment.milvus_search import VectorSearchParameters + from apache_beam.ml.rag.enrichment.milvus_search import unpack_dataclass_with_kwargs from apache_beam.ml.rag.types import Chunk - from apache_beam.ml.rag.types import Embedding from apache_beam.ml.rag.types import Content - from apache_beam.ml.rag.enrichment.milvus_search import ( - MilvusSearchEnrichmentHandler, - MilvusConnectionParameters, - MilvusSearchParameters, - MilvusCollectionLoadParameters, - VectorSearchParameters, - KeywordSearchParameters, - HybridSearchParameters, - MilvusBaseRanker, - unpack_dataclass_with_kwargs) + from apache_beam.ml.rag.types import Embedding except ImportError as e: raise unittest.SkipTest(f'Milvus dependencies not installed: {str(e)}') diff --git a/sdks/python/apache_beam/ml/transforms/embeddings/huggingface_test.py b/sdks/python/apache_beam/ml/transforms/embeddings/huggingface_test.py index a6abe7fbdbc3..a2358c544781 100644 --- a/sdks/python/apache_beam/ml/transforms/embeddings/huggingface_test.py +++ b/sdks/python/apache_beam/ml/transforms/embeddings/huggingface_test.py @@ -33,16 +33,18 @@ # pylint: disable=ungrouped-imports try: - from apache_beam.ml.transforms.embeddings.huggingface import SentenceTransformerEmbeddings - from apache_beam.ml.transforms.embeddings.huggingface import InferenceAPIEmbeddings - from PIL import Image import torch + from PIL import Image + + from apache_beam.ml.transforms.embeddings.huggingface import InferenceAPIEmbeddings + from apache_beam.ml.transforms.embeddings.huggingface import SentenceTransformerEmbeddings except ImportError: SentenceTransformerEmbeddings = None # type: ignore # pylint: disable=ungrouped-imports try: import tensorflow_transform as tft + from apache_beam.ml.transforms.tft import ScaleTo01 except ImportError: tft = None diff --git a/sdks/python/apache_beam/ml/transforms/embeddings/open_ai.py b/sdks/python/apache_beam/ml/transforms/embeddings/open_ai.py index a162c333b199..2092fa5cba9a 100644 --- a/sdks/python/apache_beam/ml/transforms/embeddings/open_ai.py +++ b/sdks/python/apache_beam/ml/transforms/embeddings/open_ai.py @@ -21,16 +21,17 @@ from typing import TypeVar from typing import Union -import apache_beam as beam import openai +from openai import APIError +from openai import RateLimitError + +import apache_beam as beam from apache_beam.ml.inference.base import RemoteModelHandler from apache_beam.ml.inference.base import RunInference from apache_beam.ml.transforms.base import EmbeddingsManager from apache_beam.ml.transforms.base import _TextEmbeddingHandler from apache_beam.pvalue import PCollection from apache_beam.pvalue import Row -from openai import APIError -from openai import RateLimitError __all__ = ["OpenAITextEmbeddings"] diff --git a/sdks/python/apache_beam/ml/transforms/embeddings/tensorflow_hub.py b/sdks/python/apache_beam/ml/transforms/embeddings/tensorflow_hub.py index c14904df7c2c..86ceecc390cc 100644 --- a/sdks/python/apache_beam/ml/transforms/embeddings/tensorflow_hub.py +++ b/sdks/python/apache_beam/ml/transforms/embeddings/tensorflow_hub.py @@ -17,10 +17,11 @@ from collections.abc import Iterable from typing import Optional -import apache_beam as beam import tensorflow as tf import tensorflow_hub as hub import tensorflow_text as text # required to register TF ops. # pylint: disable=unused-import + +import apache_beam as beam from apache_beam.ml.inference import utils from apache_beam.ml.inference.base import ModelHandler from apache_beam.ml.inference.base import PredictionResult diff --git a/sdks/python/apache_beam/ml/transforms/embeddings/tensorflow_hub_test.py b/sdks/python/apache_beam/ml/transforms/embeddings/tensorflow_hub_test.py index 64dc1e95d641..0a4f8c8275c3 100644 --- a/sdks/python/apache_beam/ml/transforms/embeddings/tensorflow_hub_test.py +++ b/sdks/python/apache_beam/ml/transforms/embeddings/tensorflow_hub_test.py @@ -40,14 +40,16 @@ # pylint: disable=ungrouped-imports try: import tensorflow_transform as tft + from apache_beam.ml.transforms.tft import ScaleTo01 except ImportError: tft = None # pylint: disable=ungrouped-imports try: - from apache_beam.ml.transforms.embeddings.tensorflow_hub import TensorflowHubImageEmbeddings from PIL import Image + + from apache_beam.ml.transforms.embeddings.tensorflow_hub import TensorflowHubImageEmbeddings except ImportError: TensorflowHubImageEmbeddings = None # type: ignore Image = None diff --git a/sdks/python/apache_beam/ml/transforms/embeddings/vertex_ai.py b/sdks/python/apache_beam/ml/transforms/embeddings/vertex_ai.py index c7c46d246b93..de3e5b0c6a92 100644 --- a/sdks/python/apache_beam/ml/transforms/embeddings/vertex_ai.py +++ b/sdks/python/apache_beam/ml/transforms/embeddings/vertex_ai.py @@ -28,12 +28,20 @@ from typing import Optional from typing import cast +import vertexai from google.api_core.exceptions import ServerError from google.api_core.exceptions import TooManyRequests from google.auth.credentials import Credentials +from vertexai.language_models import TextEmbeddingInput +from vertexai.language_models import TextEmbeddingModel +from vertexai.vision_models import Image +from vertexai.vision_models import MultiModalEmbeddingModel +from vertexai.vision_models import MultiModalEmbeddingResponse +from vertexai.vision_models import Video +from vertexai.vision_models import VideoEmbedding +from vertexai.vision_models import VideoSegmentConfig import apache_beam as beam -import vertexai from apache_beam.ml.inference.base import ModelHandler from apache_beam.ml.inference.base import RemoteModelHandler from apache_beam.ml.inference.base import RunInference @@ -44,14 +52,6 @@ from apache_beam.ml.transforms.base import _ImageEmbeddingHandler from apache_beam.ml.transforms.base import _MultiModalEmbeddingHandler from apache_beam.ml.transforms.base import _TextEmbeddingHandler -from vertexai.language_models import TextEmbeddingInput -from vertexai.language_models import TextEmbeddingModel -from vertexai.vision_models import Image -from vertexai.vision_models import MultiModalEmbeddingModel -from vertexai.vision_models import MultiModalEmbeddingResponse -from vertexai.vision_models import Video -from vertexai.vision_models import VideoEmbedding -from vertexai.vision_models import VideoSegmentConfig __all__ = [ "VertexAITextEmbeddings", diff --git a/sdks/python/apache_beam/ml/transforms/embeddings/vertex_ai_test.py b/sdks/python/apache_beam/ml/transforms/embeddings/vertex_ai_test.py index ba43ea325089..7f820d4da2f3 100644 --- a/sdks/python/apache_beam/ml/transforms/embeddings/vertex_ai_test.py +++ b/sdks/python/apache_beam/ml/transforms/embeddings/vertex_ai_test.py @@ -26,16 +26,17 @@ from apache_beam.ml.transforms.base import MLTransform try: + from vertexai.vision_models import Image + from vertexai.vision_models import Video + from vertexai.vision_models import VideoSegmentConfig + from apache_beam.ml.rag.types import Chunk from apache_beam.ml.rag.types import Content + from apache_beam.ml.transforms.embeddings.vertex_ai import VertexAIImageEmbeddings from apache_beam.ml.transforms.embeddings.vertex_ai import VertexAIMultiModalEmbeddings from apache_beam.ml.transforms.embeddings.vertex_ai import VertexAITextEmbeddings - from apache_beam.ml.transforms.embeddings.vertex_ai import VertexAIImageEmbeddings from apache_beam.ml.transforms.embeddings.vertex_ai import VertexImage from apache_beam.ml.transforms.embeddings.vertex_ai import VertexVideo - from vertexai.vision_models import Image - from vertexai.vision_models import Video - from vertexai.vision_models import VideoSegmentConfig except ImportError: VertexAIMultiModalEmbeddings = None # type: ignore VertexAITextEmbeddings = None # type: ignore @@ -44,6 +45,7 @@ # pylint: disable=ungrouped-imports try: import tensorflow_transform as tft + from apache_beam.ml.transforms.tft import ScaleTo01 except ImportError: tft = None diff --git a/sdks/python/apache_beam/ml/transforms/handlers.py b/sdks/python/apache_beam/ml/transforms/handlers.py index 1e752049f6e5..5916e0fe21e9 100644 --- a/sdks/python/apache_beam/ml/transforms/handlers.py +++ b/sdks/python/apache_beam/ml/transforms/handlers.py @@ -27,10 +27,17 @@ from typing import Union import numpy as np - -import apache_beam as beam import tensorflow as tf import tensorflow_transform.beam as tft_beam +from tensorflow_metadata.proto.v0 import schema_pb2 +from tensorflow_transform import common_types +from tensorflow_transform.beam.tft_beam_io import beam_metadata_io +from tensorflow_transform.beam.tft_beam_io import transform_fn_io +from tensorflow_transform.tf_metadata import dataset_metadata +from tensorflow_transform.tf_metadata import metadata_io +from tensorflow_transform.tf_metadata import schema_utils + +import apache_beam as beam from apache_beam import coders from apache_beam.io.filesystems import FileSystems from apache_beam.ml.transforms.base import ArtifactMode @@ -39,13 +46,6 @@ from apache_beam.ml.transforms.tft import TFTOperation from apache_beam.typehints import native_type_compatibility from apache_beam.typehints.row_type import RowTypeConstraint -from tensorflow_metadata.proto.v0 import schema_pb2 -from tensorflow_transform import common_types -from tensorflow_transform.beam.tft_beam_io import beam_metadata_io -from tensorflow_transform.beam.tft_beam_io import transform_fn_io -from tensorflow_transform.tf_metadata import dataset_metadata -from tensorflow_transform.tf_metadata import metadata_io -from tensorflow_transform.tf_metadata import schema_utils __all__ = [ 'TFTProcessHandler', diff --git a/sdks/python/apache_beam/ml/transforms/handlers_test.py b/sdks/python/apache_beam/ml/transforms/handlers_test.py index bb5f9b5f0f70..35ffda971003 100644 --- a/sdks/python/apache_beam/ml/transforms/handlers_test.py +++ b/sdks/python/apache_beam/ml/transforms/handlers_test.py @@ -34,14 +34,15 @@ # pylint: disable=wrong-import-position, ungrouped-imports try: + import tensorflow as tf + from tensorflow_transform.tf_metadata import dataset_metadata + from tensorflow_transform.tf_metadata import schema_utils + from apache_beam.ml.transforms import handlers from apache_beam.ml.transforms import tft from apache_beam.ml.transforms.tft import TFTOperation from apache_beam.testing.util import assert_that from apache_beam.testing.util import equal_to - import tensorflow as tf - from tensorflow_transform.tf_metadata import dataset_metadata - from tensorflow_transform.tf_metadata import schema_utils except ImportError: tft = None # type: ignore[assignment] diff --git a/sdks/python/apache_beam/ml/transforms/tft.py b/sdks/python/apache_beam/ml/transforms/tft.py index bfe23757642b..78070211f6f4 100644 --- a/sdks/python/apache_beam/ml/transforms/tft.py +++ b/sdks/python/apache_beam/ml/transforms/tft.py @@ -39,12 +39,13 @@ from typing import Optional from typing import Union -import apache_beam as beam import tensorflow as tf import tensorflow_transform as tft -from apache_beam.ml.transforms.base import BaseOperation from tensorflow_transform import common_types +import apache_beam as beam +from apache_beam.ml.transforms.base import BaseOperation + __all__ = [ 'ComputeAndApplyVocabulary', 'ScaleToZScore', diff --git a/sdks/python/apache_beam/ml/transforms/utils.py b/sdks/python/apache_beam/ml/transforms/utils.py index 023657895686..646ed2e4e247 100644 --- a/sdks/python/apache_beam/ml/transforms/utils.py +++ b/sdks/python/apache_beam/ml/transforms/utils.py @@ -20,10 +20,10 @@ import os import tempfile +import tensorflow_transform as tft from google.cloud.storage import Client from google.cloud.storage import transfer_manager -import tensorflow_transform as tft from apache_beam.ml.transforms import base diff --git a/sdks/python/apache_beam/options/pipeline_options_test.py b/sdks/python/apache_beam/options/pipeline_options_test.py index b9c2061744b8..705e8e1e2c04 100644 --- a/sdks/python/apache_beam/options/pipeline_options_test.py +++ b/sdks/python/apache_beam/options/pipeline_options_test.py @@ -752,8 +752,7 @@ def test_options_store_false_with_different_dest(self): "store_true. It would be confusing " "to the user. Please specify the dest as the " "flag_name instead.")) - from apache_beam.options.pipeline_options import ( - _FLAG_THAT_SETS_FALSE_VALUE) + from apache_beam.options.pipeline_options import _FLAG_THAT_SETS_FALSE_VALUE self.assertDictEqual( _FLAG_THAT_SETS_FALSE_VALUE, diff --git a/sdks/python/apache_beam/pipeline.py b/sdks/python/apache_beam/pipeline.py index 0e03c684153f..87d63d423156 100644 --- a/sdks/python/apache_beam/pipeline.py +++ b/sdks/python/apache_beam/pipeline.py @@ -109,6 +109,7 @@ if TYPE_CHECKING: from types import TracebackType + from apache_beam.runners.pipeline_context import PipelineContext from apache_beam.runners.runner import PipelineResult from apache_beam.transforms import environments diff --git a/sdks/python/apache_beam/pipeline_test.py b/sdks/python/apache_beam/pipeline_test.py index 7c1f40b696e5..3e7d083cb2fb 100644 --- a/sdks/python/apache_beam/pipeline_test.py +++ b/sdks/python/apache_beam/pipeline_test.py @@ -188,8 +188,8 @@ def test_runner_overrides_default_pickler(self, mock_info): pcoll = pipeline | 'label1' >> Create([1, 2, 3]) assert_that(pcoll, equal_to([1, 2, 3])) - from apache_beam.internal import pickler from apache_beam.internal import dill_pickler + from apache_beam.internal import pickler self.assertIs(pickler.desired_pickle_lib, dill_pickler) mock_info.assert_any_call( 'Runner defaulting to pickling library: %s.', 'dill') diff --git a/sdks/python/apache_beam/pvalue.py b/sdks/python/apache_beam/pvalue.py index 3865af184b61..ca9a662d399e 100644 --- a/sdks/python/apache_beam/pvalue.py +++ b/sdks/python/apache_beam/pvalue.py @@ -47,12 +47,12 @@ from apache_beam.portability.api import beam_runner_api_pb2 if TYPE_CHECKING: - from apache_beam.transforms import sideinputs - from apache_beam.transforms.core import ParDo - from apache_beam.transforms.core import Windowing from apache_beam.pipeline import AppliedPTransform from apache_beam.pipeline import Pipeline from apache_beam.runners.pipeline_context import PipelineContext + from apache_beam.transforms import sideinputs + from apache_beam.transforms.core import ParDo + from apache_beam.transforms.core import Windowing __all__ = [ 'PCollection', diff --git a/sdks/python/apache_beam/runners/__init__.py b/sdks/python/apache_beam/runners/__init__.py index f92d95aa4826..b4ead2885bf3 100644 --- a/sdks/python/apache_beam/runners/__init__.py +++ b/sdks/python/apache_beam/runners/__init__.py @@ -20,11 +20,10 @@ This package defines runners, which are used to execute a pipeline. """ +from apache_beam.runners.dataflow.dataflow_runner import DataflowRunner +from apache_beam.runners.dataflow.test_dataflow_runner import TestDataflowRunner from apache_beam.runners.direct.direct_runner import DirectRunner from apache_beam.runners.direct.test_direct_runner import TestDirectRunner from apache_beam.runners.runner import PipelineRunner from apache_beam.runners.runner import PipelineState from apache_beam.runners.runner import create_runner - -from apache_beam.runners.dataflow.dataflow_runner import DataflowRunner -from apache_beam.runners.dataflow.test_dataflow_runner import TestDataflowRunner diff --git a/sdks/python/apache_beam/runners/common.py b/sdks/python/apache_beam/runners/common.py index abe3792b4d8b..034090cf7bdc 100644 --- a/sdks/python/apache_beam/runners/common.py +++ b/sdks/python/apache_beam/runners/common.py @@ -65,12 +65,12 @@ from apache_beam.utils.windowed_value import WindowedValue if TYPE_CHECKING: - from apache_beam.runners.worker.bundle_processor import ExecutionContext - from apache_beam.transforms import sideinputs - from apache_beam.transforms.core import TimerSpec from apache_beam.io.iobase import RestrictionProgress from apache_beam.iobase import RestrictionTracker from apache_beam.iobase import WatermarkEstimator + from apache_beam.runners.worker.bundle_processor import ExecutionContext + from apache_beam.transforms import sideinputs + from apache_beam.transforms.core import TimerSpec IMPULSE_VALUE_CODER_IMPL = coders.WindowedValueCoder( coders.BytesCoder(), coders.GlobalWindowCoder()).get_impl() diff --git a/sdks/python/apache_beam/runners/dask/transform_evaluator.py b/sdks/python/apache_beam/runners/dask/transform_evaluator.py index 7cad1fe40451..6fd216fadb53 100644 --- a/sdks/python/apache_beam/runners/dask/transform_evaluator.py +++ b/sdks/python/apache_beam/runners/dask/transform_evaluator.py @@ -27,8 +27,9 @@ import typing as t from dataclasses import field -import apache_beam import dask.bag as db + +import apache_beam from apache_beam import DoFn from apache_beam import TaggedOutput from apache_beam.pipeline import AppliedPTransform diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py index 57aed7cf9be2..d33c33f84fee 100644 --- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py +++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py @@ -303,8 +303,8 @@ def visit_transform(self, transform_node): @staticmethod def combinefn_visitor(): # Imported here to avoid circular dependencies. - from apache_beam.pipeline import PipelineVisitor from apache_beam import core + from apache_beam.pipeline import PipelineVisitor class CombineFnVisitor(PipelineVisitor): """Checks if `CombineFn` has non-default setup or teardown methods. @@ -380,8 +380,7 @@ def run_pipeline(self, pipeline, options, pipeline_proto=None): # Apply DataflowRunner-specific overrides (e.g., streaming PubSub # optimizations) - from apache_beam.runners.dataflow.ptransform_overrides import ( - get_dataflow_transform_overrides) + from apache_beam.runners.dataflow.ptransform_overrides import get_dataflow_transform_overrides dataflow_overrides = get_dataflow_transform_overrides(options) if dataflow_overrides: pipeline.replace_all(dataflow_overrides) diff --git a/sdks/python/apache_beam/runners/dataflow/internal/clients/cloudbuild/__init__.py b/sdks/python/apache_beam/runners/dataflow/internal/clients/cloudbuild/__init__.py index 767bd4cec605..1dc089bbf700 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/clients/cloudbuild/__init__.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/clients/cloudbuild/__init__.py @@ -25,6 +25,7 @@ # pylint: disable=wrong-import-order, wrong-import-position try: from apitools.base.py import * + from apache_beam.runners.dataflow.internal.clients.cloudbuild.cloudbuild_v1_client import * from apache_beam.runners.dataflow.internal.clients.cloudbuild.cloudbuild_v1_messages import * except ImportError: diff --git a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/__init__.py b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/__init__.py index c0d20c3ec8f9..8e69c725830a 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/__init__.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/__init__.py @@ -24,8 +24,9 @@ # pylint: disable=wrong-import-order, wrong-import-position try: from apitools.base.py import * - from apache_beam.runners.dataflow.internal.clients.dataflow.dataflow_v1b3_messages import * + from apache_beam.runners.dataflow.internal.clients.dataflow.dataflow_v1b3_client import * + from apache_beam.runners.dataflow.internal.clients.dataflow.dataflow_v1b3_messages import * except ImportError: pass # pylint: enable=wrong-import-order, wrong-import-position diff --git a/sdks/python/apache_beam/runners/dataflow/ptransform_overrides.py b/sdks/python/apache_beam/runners/dataflow/ptransform_overrides.py index 4e75f202c098..9862957de115 100644 --- a/sdks/python/apache_beam/runners/dataflow/ptransform_overrides.py +++ b/sdks/python/apache_beam/runners/dataflow/ptransform_overrides.py @@ -30,8 +30,8 @@ class StreamingPubSubWriteDoFnOverride(PTransformOverride): and replaces it with Write(sink) for streaming optimization. """ def matches(self, applied_ptransform): - from apache_beam.transforms import ParDo from apache_beam.io.gcp.pubsub import _PubSubWriteDoFn + from apache_beam.transforms import ParDo if not isinstance(applied_ptransform.transform, ParDo): return False diff --git a/sdks/python/apache_beam/runners/direct/direct_runner.py b/sdks/python/apache_beam/runners/direct/direct_runner.py index 68add6ea3c1a..73b0321b5de4 100644 --- a/sdks/python/apache_beam/runners/direct/direct_runner.py +++ b/sdks/python/apache_beam/runners/direct/direct_runner.py @@ -76,10 +76,10 @@ def is_interactive(self): def run_pipeline(self, pipeline, options): - from apache_beam.pipeline import PipelineVisitor - from apache_beam.testing.test_stream import TestStream from apache_beam.io.gcp.pubsub import ReadFromPubSub from apache_beam.io.gcp.pubsub import WriteToPubSub + from apache_beam.pipeline import PipelineVisitor + from apache_beam.testing.test_stream import TestStream class _FnApiRunnerSupportVisitor(PipelineVisitor): """Visitor determining if a Pipeline can be run on the FnApiRunner.""" @@ -292,6 +292,7 @@ def infer_output_type(self, input_type): def start_bundle(self): # pylint: disable=wrong-import-order, wrong-import-position from apache_beam.transforms.trigger import create_trigger_driver + # pylint: enable=wrong-import-order, wrong-import-position self.driver = create_trigger_driver(self.windowing, True) @@ -398,9 +399,9 @@ def _get_transform_overrides(pipeline_options): # Importing following locally to avoid a circular dependency. from apache_beam.pipeline import PTransformOverride - from apache_beam.transforms.combiners import LiftedCombinePerKey from apache_beam.runners.direct.sdf_direct_runner import ProcessKeyedElementsViaKeyedWorkItemsOverride from apache_beam.runners.direct.sdf_direct_runner import SplittableParDoOverride + from apache_beam.transforms.combiners import LiftedCombinePerKey class CombinePerKeyOverride(PTransformOverride): def matches(self, applied_ptransform): @@ -555,12 +556,10 @@ def run_pipeline(self, pipeline, options): # with resolving imports when they are at top. # pylint: disable=wrong-import-position from apache_beam.pipeline import PipelineVisitor - from apache_beam.runners.direct.consumer_tracking_pipeline_visitor import \ - ConsumerTrackingPipelineVisitor + from apache_beam.runners.direct.consumer_tracking_pipeline_visitor import ConsumerTrackingPipelineVisitor from apache_beam.runners.direct.evaluation_context import EvaluationContext from apache_beam.runners.direct.executor import Executor - from apache_beam.runners.direct.transform_evaluator import \ - TransformEvaluatorRegistry + from apache_beam.runners.direct.transform_evaluator import TransformEvaluatorRegistry from apache_beam.testing.test_stream import TestStream from apache_beam.transforms.external import ExternalTransform diff --git a/sdks/python/apache_beam/runners/direct/evaluation_context.py b/sdks/python/apache_beam/runners/direct/evaluation_context.py index e787eafbc259..6138577bb91d 100644 --- a/sdks/python/apache_beam/runners/direct/evaluation_context.py +++ b/sdks/python/apache_beam/runners/direct/evaluation_context.py @@ -42,7 +42,8 @@ from apache_beam.utils.timestamp import Timestamp if TYPE_CHECKING: - from apache_beam.runners.direct.bundle_factory import BundleFactory, _Bundle + from apache_beam.runners.direct.bundle_factory import BundleFactory + from apache_beam.runners.direct.bundle_factory import _Bundle from apache_beam.runners.direct.util import TimerFiring from apache_beam.runners.direct.util import TransformResult from apache_beam.runners.direct.watermark_manager import _TransformWatermarks diff --git a/sdks/python/apache_beam/runners/direct/transform_evaluator.py b/sdks/python/apache_beam/runners/direct/transform_evaluator.py index 3443a519e54c..49e7d9d02106 100644 --- a/sdks/python/apache_beam/runners/direct/transform_evaluator.py +++ b/sdks/python/apache_beam/runners/direct/transform_evaluator.py @@ -76,8 +76,8 @@ from apache_beam.utils.timestamp import Timestamp if TYPE_CHECKING: - from apache_beam.io.gcp.pubsub import _PubSubSource from apache_beam.io.gcp.pubsub import PubsubMessage + from apache_beam.io.gcp.pubsub import _PubSubSource from apache_beam.runners.direct.evaluation_context import EvaluationContext _LOGGER = logging.getLogger(__name__) @@ -652,9 +652,10 @@ def process_element(self, element): def _read_from_pubsub( self, timestamp_attribute) -> List[Tuple[Timestamp, 'PubsubMessage']]: - from apache_beam.io.gcp.pubsub import PubsubMessage from google.cloud import pubsub + from apache_beam.io.gcp.pubsub import PubsubMessage + def _get_element(message): parsed_message = PubsubMessage._from_message(message) if (timestamp_attribute and diff --git a/sdks/python/apache_beam/runners/interactive/augmented_pipeline.py b/sdks/python/apache_beam/runners/interactive/augmented_pipeline.py index c1adc0c4a4f7..519bf3514c53 100644 --- a/sdks/python/apache_beam/runners/interactive/augmented_pipeline.py +++ b/sdks/python/apache_beam/runners/interactive/augmented_pipeline.py @@ -28,8 +28,8 @@ import apache_beam as beam from apache_beam.portability.api import beam_runner_api_pb2 -from apache_beam.runners.interactive import interactive_environment as ie from apache_beam.runners.interactive import background_caching_job +from apache_beam.runners.interactive import interactive_environment as ie from apache_beam.runners.interactive.caching.cacheable import Cacheable from apache_beam.runners.interactive.caching.read_cache import ReadCache from apache_beam.runners.interactive.caching.write_cache import WriteCache diff --git a/sdks/python/apache_beam/runners/interactive/background_caching_job_test.py b/sdks/python/apache_beam/runners/interactive/background_caching_job_test.py index 5954d436ad28..5c832d595dc8 100644 --- a/sdks/python/apache_beam/runners/interactive/background_caching_job_test.py +++ b/sdks/python/apache_beam/runners/interactive/background_caching_job_test.py @@ -224,6 +224,7 @@ def test_source_to_cache_changed_when_source_is_altered(self, cell): with cell: # Cell 2 from apache_beam.io.gcp.pubsub import _PubSubSource + # Alter the transform. transform._source = _PubSubSource(subscription=_BAR_PUBSUB_SUB) diff --git a/sdks/python/apache_beam/runners/interactive/caching/cacheable.py b/sdks/python/apache_beam/runners/interactive/caching/cacheable.py index f69324e99f9e..230d4e080bdf 100644 --- a/sdks/python/apache_beam/runners/interactive/caching/cacheable.py +++ b/sdks/python/apache_beam/runners/interactive/caching/cacheable.py @@ -68,6 +68,7 @@ class CacheKey: def __post_init__(self): from apache_beam.runners.interactive.utils import obfuscate + # Normalize arbitrary variable name to a fixed length hex str. self.var = obfuscate(self.var)[:10] diff --git a/sdks/python/apache_beam/runners/interactive/dataproc/dataproc_cluster_manager.py b/sdks/python/apache_beam/runners/interactive/dataproc/dataproc_cluster_manager.py index 4d260d4a6a56..f15541d423ac 100644 --- a/sdks/python/apache_beam/runners/interactive/dataproc/dataproc_cluster_manager.py +++ b/sdks/python/apache_beam/runners/interactive/dataproc/dataproc_cluster_manager.py @@ -32,7 +32,8 @@ try: from google.cloud import dataproc_v1 - from apache_beam.io.gcp import gcsfilesystem #pylint: disable=ungrouped-imports + + from apache_beam.io.gcp import gcsfilesystem # pylint: disable=ungrouped-imports except ImportError: class UnimportedDataproc: diff --git a/sdks/python/apache_beam/runners/interactive/display/display_manager.py b/sdks/python/apache_beam/runners/interactive/display/display_manager.py index e1f248304228..b52de19656d7 100644 --- a/sdks/python/apache_beam/runners/interactive/display/display_manager.py +++ b/sdks/python/apache_beam/runners/interactive/display/display_manager.py @@ -33,6 +33,7 @@ import IPython # pylint: disable=import-error from IPython import get_ipython # pylint: disable=import-error from IPython.display import display as ip_display # pylint: disable=import-error + # _display_progress defines how outputs are printed on the frontend. _display_progress = ip_display diff --git a/sdks/python/apache_beam/runners/interactive/display/pcoll_visualization.py b/sdks/python/apache_beam/runners/interactive/display/pcoll_visualization.py index 0bb3d1ba1876..63b6dbd963ac 100644 --- a/sdks/python/apache_beam/runners/interactive/display/pcoll_visualization.py +++ b/sdks/python/apache_beam/runners/interactive/display/pcoll_visualization.py @@ -38,12 +38,13 @@ from apache_beam.transforms.window import IntervalWindow try: + from facets_overview.generic_feature_statistics_generator import \ + GenericFeatureStatisticsGenerator # pylint: disable=import-error from IPython import get_ipython # pylint: disable=import-error from IPython.display import HTML # pylint: disable=import-error from IPython.display import Javascript # pylint: disable=import-error from IPython.display import display # pylint: disable=import-error from IPython.display import display_javascript # pylint: disable=import-error - from facets_overview.generic_feature_statistics_generator import GenericFeatureStatisticsGenerator # pylint: disable=import-error from timeloop import Timeloop # pylint: disable=import-error if get_ipython(): diff --git a/sdks/python/apache_beam/runners/interactive/interactive_environment.py b/sdks/python/apache_beam/runners/interactive/interactive_environment.py index 1f48121016c5..e9ff86c6276f 100644 --- a/sdks/python/apache_beam/runners/interactive/interactive_environment.py +++ b/sdks/python/apache_beam/runners/interactive/interactive_environment.py @@ -181,7 +181,8 @@ def __init__(self): try: import IPython # pylint: disable=unused-import import timeloop # pylint: disable=unused-import - from facets_overview.generic_feature_statistics_generator import GenericFeatureStatisticsGenerator # pylint: disable=unused-import + from facets_overview.generic_feature_statistics_generator import \ + GenericFeatureStatisticsGenerator # pylint: disable=unused-import from google.cloud import dataproc_v1 # pylint: disable=unused-import self._is_interactive_ready = True except ImportError: diff --git a/sdks/python/apache_beam/runners/interactive/interactive_runner.py b/sdks/python/apache_beam/runners/interactive/interactive_runner.py index c8b0be0941d0..241dcf388dd0 100644 --- a/sdks/python/apache_beam/runners/interactive/interactive_runner.py +++ b/sdks/python/apache_beam/runners/interactive/interactive_runner.py @@ -33,9 +33,9 @@ from apache_beam.options.pipeline_options import WorkerOptions from apache_beam.pipeline import PipelineVisitor from apache_beam.runners.direct import direct_runner +from apache_beam.runners.interactive import background_caching_job from apache_beam.runners.interactive import interactive_environment as ie from apache_beam.runners.interactive import pipeline_instrument as inst -from apache_beam.runners.interactive import background_caching_job from apache_beam.runners.interactive.dataproc.types import ClusterMetadata from apache_beam.runners.interactive.display import pipeline_graph from apache_beam.runners.interactive.options import capture_control diff --git a/sdks/python/apache_beam/runners/interactive/pipeline_instrument.py b/sdks/python/apache_beam/runners/interactive/pipeline_instrument.py index 95e30f7cb0fa..07e35f96877c 100644 --- a/sdks/python/apache_beam/runners/interactive/pipeline_instrument.py +++ b/sdks/python/apache_beam/runners/interactive/pipeline_instrument.py @@ -29,9 +29,9 @@ import apache_beam as beam from apache_beam.pipeline import PipelineVisitor from apache_beam.portability.api import beam_runner_api_pb2 +from apache_beam.runners.interactive import background_caching_job from apache_beam.runners.interactive import interactive_environment as ie from apache_beam.runners.interactive import pipeline_fragment as pf -from apache_beam.runners.interactive import background_caching_job from apache_beam.runners.interactive import utils from apache_beam.runners.interactive.caching.cacheable import Cacheable from apache_beam.runners.interactive.caching.cacheable import CacheKey diff --git a/sdks/python/apache_beam/runners/interactive/pipeline_instrument_test.py b/sdks/python/apache_beam/runners/interactive/pipeline_instrument_test.py index 7f5c4f913bd9..3b9244725e54 100644 --- a/sdks/python/apache_beam/runners/interactive/pipeline_instrument_test.py +++ b/sdks/python/apache_beam/runners/interactive/pipeline_instrument_test.py @@ -26,8 +26,8 @@ from apache_beam.runners.interactive import cache_manager as cache from apache_beam.runners.interactive import interactive_beam as ib from apache_beam.runners.interactive import interactive_environment as ie -from apache_beam.runners.interactive import pipeline_instrument as instr from apache_beam.runners.interactive import interactive_runner +from apache_beam.runners.interactive import pipeline_instrument as instr from apache_beam.runners.interactive import utils from apache_beam.runners.interactive.caching.cacheable import Cacheable from apache_beam.runners.interactive.caching.cacheable import CacheKey @@ -36,7 +36,7 @@ from apache_beam.runners.interactive.testing.pipeline_assertion import assert_pipeline_proto_contain_top_level_transform from apache_beam.runners.interactive.testing.pipeline_assertion import assert_pipeline_proto_equal from apache_beam.runners.interactive.testing.pipeline_assertion import \ - assert_pipeline_proto_not_contain_top_level_transform + assert_pipeline_proto_not_contain_top_level_transform from apache_beam.runners.interactive.testing.test_cache_manager import InMemoryCache from apache_beam.testing.test_stream import TestStream diff --git a/sdks/python/apache_beam/runners/interactive/sql/beam_sql_magics.py b/sdks/python/apache_beam/runners/interactive/sql/beam_sql_magics.py index bf4c4c0380e5..3dc866907a40 100644 --- a/sdks/python/apache_beam/runners/interactive/sql/beam_sql_magics.py +++ b/sdks/python/apache_beam/runners/interactive/sql/beam_sql_magics.py @@ -31,6 +31,10 @@ from typing import Tuple from typing import Union +from IPython.core.magic import Magics +from IPython.core.magic import line_cell_magic +from IPython.core.magic import magics_class + import apache_beam as beam from apache_beam.pvalue import PValue from apache_beam.runners.interactive import interactive_environment as ie @@ -54,9 +58,6 @@ from apache_beam.testing.test_stream_service import TestStreamServiceController from apache_beam.transforms.sql import SqlTransform from apache_beam.typehints.native_type_compatibility import match_is_named_tuple -from IPython.core.magic import Magics -from IPython.core.magic import line_cell_magic -from IPython.core.magic import magics_class _LOGGER = logging.getLogger(__name__) diff --git a/sdks/python/apache_beam/runners/interactive/sql/beam_sql_magics_test.py b/sdks/python/apache_beam/runners/interactive/sql/beam_sql_magics_test.py index 9dd74b16a5ce..9be9c6db875f 100644 --- a/sdks/python/apache_beam/runners/interactive/sql/beam_sql_magics_test.py +++ b/sdks/python/apache_beam/runners/interactive/sql/beam_sql_magics_test.py @@ -31,10 +31,10 @@ from apache_beam.runners.interactive.caching.cacheable import CacheKey try: + from apache_beam.runners.interactive.sql.beam_sql_magics import BeamSqlParser from apache_beam.runners.interactive.sql.beam_sql_magics import _build_query_components from apache_beam.runners.interactive.sql.beam_sql_magics import _generate_output_name from apache_beam.runners.interactive.sql.beam_sql_magics import cache_output - from apache_beam.runners.interactive.sql.beam_sql_magics import BeamSqlParser except (ImportError, NameError): pass # The test is to be skipped because [interactive] dep not installed. diff --git a/sdks/python/apache_beam/runners/interactive/utils.py b/sdks/python/apache_beam/runners/interactive/utils.py index 828f23a467c2..136fe372c214 100644 --- a/sdks/python/apache_beam/runners/interactive/utils.py +++ b/sdks/python/apache_beam/runners/interactive/utils.py @@ -191,6 +191,7 @@ class IPythonLogHandler(logging.Handler): def emit(self, record): try: from html import escape + from IPython.display import HTML from IPython.display import display display(HTML(_INTERACTIVE_LOG_STYLE)) @@ -234,6 +235,7 @@ def __enter__(self): try: from IPython.display import HTML from IPython.display import display + from apache_beam.runners.interactive import interactive_environment as ie if ie.current_env().is_in_notebook: display( @@ -252,6 +254,7 @@ def __exit__(self, exc_type, exc_value, traceback): from IPython.display import Javascript from IPython.display import display from IPython.display import display_javascript + from apache_beam.runners.interactive import interactive_environment as ie if ie.current_env().is_in_notebook: script = self.spinner_removal_template.format(id=self._id) @@ -447,6 +450,7 @@ def assert_bucket_exists(bucket_name: str) -> None: try: from google.cloud.exceptions import ClientError from google.cloud.exceptions import NotFound + from apache_beam.io.gcp.gcsio import create_storage_client storage_client = create_storage_client(PipelineOptions()) storage_client.get_bucket(bucket_name) diff --git a/sdks/python/apache_beam/runners/interactive/utils_test.py b/sdks/python/apache_beam/runners/interactive/utils_test.py index f3d7f96b0dbb..5fb41df35862 100644 --- a/sdks/python/apache_beam/runners/interactive/utils_test.py +++ b/sdks/python/apache_beam/runners/interactive/utils_test.py @@ -46,7 +46,8 @@ # Protect against environments where apitools library is not available. try: - from google.cloud.exceptions import BadRequest, NotFound + from google.cloud.exceptions import BadRequest + from google.cloud.exceptions import NotFound except ImportError: _http_error_imported = False else: diff --git a/sdks/python/apache_beam/runners/portability/fn_api_runner/worker_handlers.py b/sdks/python/apache_beam/runners/portability/fn_api_runner/worker_handlers.py index ac346776565f..338f6ece57c0 100644 --- a/sdks/python/apache_beam/runners/portability/fn_api_runner/worker_handlers.py +++ b/sdks/python/apache_beam/runners/portability/fn_api_runner/worker_handlers.py @@ -76,9 +76,11 @@ from apache_beam.utils.sentinel import Sentinel if TYPE_CHECKING: - from grpc import ServicerContext from google.protobuf import message - from apache_beam.runners.portability.fn_api_runner.fn_runner import ExtendedProvisionInfo # pylint: disable=ungrouped-imports + from grpc import ServicerContext + + from apache_beam.runners.portability.fn_api_runner.fn_runner import \ + ExtendedProvisionInfo # pylint: disable=ungrouped-imports # State caching is enabled in the fn_api_runner for testing, except for one # test which runs without state caching (FnApiRunnerTestWithDisabledCaching). @@ -747,6 +749,7 @@ def host_from_worker(self): return 'host.docker.internal' if sys.platform == 'linux' and is_in_notebook(): import socket + # Gets ipv4 address of current host. Note the host is not guaranteed to # be localhost because the python SDK could be running within a container. return socket.gethostbyname(socket.getfqdn()) @@ -764,8 +767,8 @@ def start_worker(self): except ImportError: pass else: - from google.auth import environment_vars from google.auth import _cloud_sdk + from google.auth import environment_vars gcloud_cred_file = os.environ.get( environment_vars.CREDENTIALS, _cloud_sdk.get_application_default_credentials_path()) diff --git a/sdks/python/apache_beam/runners/runner.py b/sdks/python/apache_beam/runners/runner.py index e3b7a9de9483..00ca84bb8e7d 100644 --- a/sdks/python/apache_beam/runners/runner.py +++ b/sdks/python/apache_beam/runners/runner.py @@ -36,8 +36,8 @@ from apache_beam.transforms import environments if TYPE_CHECKING: - from apache_beam import pvalue from apache_beam import PTransform + from apache_beam import pvalue from apache_beam.pipeline import Pipeline __all__ = ['PipelineRunner', 'PipelineState', 'PipelineResult'] @@ -136,8 +136,8 @@ def run_async( # Imported here to avoid circular dependencies. # pylint: disable=wrong-import-order, wrong-import-position from apache_beam import PTransform - from apache_beam.pvalue import PBegin from apache_beam.pipeline import Pipeline + from apache_beam.pvalue import PBegin p = Pipeline(runner=self, options=options) if isinstance(transform, PTransform): p | transform diff --git a/sdks/python/apache_beam/runners/worker/bundle_processor.py b/sdks/python/apache_beam/runners/worker/bundle_processor.py index 85f1e43d6039..4094fd1d8058 100644 --- a/sdks/python/apache_beam/runners/worker/bundle_processor.py +++ b/sdks/python/apache_beam/runners/worker/bundle_processor.py @@ -91,6 +91,7 @@ if TYPE_CHECKING: from google.protobuf import message # pylint: disable=ungrouped-imports + from apache_beam import pvalue from apache_beam.portability.api import metrics_pb2 from apache_beam.runners.sdf_utils import SplitResultPrimary diff --git a/sdks/python/apache_beam/runners/worker/operations.py b/sdks/python/apache_beam/runners/worker/operations.py index 2b20bebe0940..9f490e4ae44f 100644 --- a/sdks/python/apache_beam/runners/worker/operations.py +++ b/sdks/python/apache_beam/runners/worker/operations.py @@ -50,9 +50,9 @@ from apache_beam.runners.worker import operation_specs from apache_beam.runners.worker import sideinputs from apache_beam.runners.worker.data_sampler import DataSampler -from apache_beam.transforms import sideinputs as apache_sideinputs from apache_beam.transforms import combiners from apache_beam.transforms import core +from apache_beam.transforms import sideinputs as apache_sideinputs from apache_beam.transforms import userstate from apache_beam.transforms import window from apache_beam.transforms.combiners import PhasedCombineFnExecutor diff --git a/sdks/python/apache_beam/testing/analyzers/perf_analysis_test.py b/sdks/python/apache_beam/testing/analyzers/perf_analysis_test.py index 5dbeba74b7e9..e32791c98900 100644 --- a/sdks/python/apache_beam/testing/analyzers/perf_analysis_test.py +++ b/sdks/python/apache_beam/testing/analyzers/perf_analysis_test.py @@ -34,12 +34,12 @@ from apache_beam.testing.analyzers.perf_analysis_utils import BigQueryMetricsFetcher from apache_beam.testing.analyzers.perf_analysis_utils import MetricContainer from apache_beam.testing.analyzers.perf_analysis_utils import TestConfigContainer - from apache_beam.testing.analyzers.perf_analysis_utils import is_change_point_in_valid_window - from apache_beam.testing.analyzers.perf_analysis_utils import is_sibling_change_point from apache_beam.testing.analyzers.perf_analysis_utils import e_divisive from apache_beam.testing.analyzers.perf_analysis_utils import filter_change_points_by_median_threshold from apache_beam.testing.analyzers.perf_analysis_utils import find_change_points from apache_beam.testing.analyzers.perf_analysis_utils import find_latest_change_point_index + from apache_beam.testing.analyzers.perf_analysis_utils import is_change_point_in_valid_window + from apache_beam.testing.analyzers.perf_analysis_utils import is_sibling_change_point from apache_beam.testing.analyzers.perf_analysis_utils import validate_config from apache_beam.testing.load_tests import load_test_metrics_utils diff --git a/sdks/python/apache_beam/testing/analyzers/perf_analysis_utils.py b/sdks/python/apache_beam/testing/analyzers/perf_analysis_utils.py index ac3eac0f7641..0ca4514443f6 100644 --- a/sdks/python/apache_beam/testing/analyzers/perf_analysis_utils.py +++ b/sdks/python/apache_beam/testing/analyzers/perf_analysis_utils.py @@ -28,11 +28,11 @@ import pandas as pd import yaml from google.api_core import exceptions +from signal_processing_algorithms.energy_statistics.energy_statistics import e_divisive from apache_beam.testing.analyzers import constants from apache_beam.testing.load_tests import load_test_metrics_utils from apache_beam.testing.load_tests.load_test_metrics_utils import BigQueryMetricsPublisher -from signal_processing_algorithms.energy_statistics.energy_statistics import e_divisive # pylint: disable=ungrouped-imports try: diff --git a/sdks/python/apache_beam/testing/benchmarks/cloudml/pipelines/workflow.py b/sdks/python/apache_beam/testing/benchmarks/cloudml/pipelines/workflow.py index 67d7bcee28be..6c50ffd6f384 100644 --- a/sdks/python/apache_beam/testing/benchmarks/cloudml/pipelines/workflow.py +++ b/sdks/python/apache_beam/testing/benchmarks/cloudml/pipelines/workflow.py @@ -19,15 +19,16 @@ import logging import os -import apache_beam as beam import tensorflow_transform as tft import tensorflow_transform.beam as tft_beam -from apache_beam.testing.benchmarks.cloudml.criteo_tft import criteo from tensorflow_transform import coders from tensorflow_transform.tf_metadata import dataset_metadata from tensorflow_transform.tf_metadata import schema_utils from tfx_bsl.public import tfxio +import apache_beam as beam +from apache_beam.testing.benchmarks.cloudml.criteo_tft import criteo + # Name of the column for the synthetic version of the benchmark. _SYNTHETIC_COLUMN = 'x' diff --git a/sdks/python/apache_beam/testing/benchmarks/inference/pytorch_image_classification_benchmarks.py b/sdks/python/apache_beam/testing/benchmarks/inference/pytorch_image_classification_benchmarks.py index a90c268ed538..6a056bb06463 100644 --- a/sdks/python/apache_beam/testing/benchmarks/inference/pytorch_image_classification_benchmarks.py +++ b/sdks/python/apache_beam/testing/benchmarks/inference/pytorch_image_classification_benchmarks.py @@ -18,9 +18,10 @@ import logging +from torchvision import models + from apache_beam.examples.inference import pytorch_image_classification from apache_beam.testing.load_tests.dataflow_cost_benchmark import DataflowCostBenchmark -from torchvision import models _PERF_TEST_MODELS = ['resnet50', 'resnet101', 'resnet152'] _PRETRAINED_MODEL_MODULE = 'torchvision.models' diff --git a/sdks/python/apache_beam/testing/metric_result_matchers_test.py b/sdks/python/apache_beam/testing/metric_result_matchers_test.py index 3657356a9fe0..f97a3ef30de4 100644 --- a/sdks/python/apache_beam/testing/metric_result_matchers_test.py +++ b/sdks/python/apache_beam/testing/metric_result_matchers_test.py @@ -21,8 +21,8 @@ import unittest -from hamcrest import assert_that as hc_assert_that from hamcrest import anything +from hamcrest import assert_that as hc_assert_that from hamcrest import equal_to from hamcrest.core.core.isnot import is_not from hamcrest.library.number.ordering_comparison import greater_than diff --git a/sdks/python/apache_beam/testing/pipeline_verifiers_test.py b/sdks/python/apache_beam/testing/pipeline_verifiers_test.py index 085339003699..cc286c33aaaa 100644 --- a/sdks/python/apache_beam/testing/pipeline_verifiers_test.py +++ b/sdks/python/apache_beam/testing/pipeline_verifiers_test.py @@ -38,6 +38,7 @@ # pylint: disable=wrong-import-order, wrong-import-position # pylint: disable=ungrouped-imports from apitools.base.py.exceptions import HttpError + from apache_beam.io.gcp.gcsfilesystem import GCSFileSystem except ImportError: HttpError = None diff --git a/sdks/python/apache_beam/tools/coders_microbenchmark.py b/sdks/python/apache_beam/tools/coders_microbenchmark.py index 7a1f9f6dcc1b..a8d4b13b6ae2 100644 --- a/sdks/python/apache_beam/tools/coders_microbenchmark.py +++ b/sdks/python/apache_beam/tools/coders_microbenchmark.py @@ -39,10 +39,10 @@ import sys import apache_beam as beam -from apache_beam.coders import proto2_coder_test_messages_pb2 as test_message from apache_beam.coders import coder_impl from apache_beam.coders import coders from apache_beam.coders import coders_test_common +from apache_beam.coders import proto2_coder_test_messages_pb2 as test_message from apache_beam.coders import row_coder from apache_beam.coders import typecoders from apache_beam.tools import utils diff --git a/sdks/python/apache_beam/transforms/core.py b/sdks/python/apache_beam/transforms/core.py index 7ba8aa128c24..45923916ceae 100644 --- a/sdks/python/apache_beam/transforms/core.py +++ b/sdks/python/apache_beam/transforms/core.py @@ -78,6 +78,7 @@ if typing.TYPE_CHECKING: from google.protobuf import message # pylint: disable=ungrouped-imports + from apache_beam.io import iobase from apache_beam.pipeline import Pipeline from apache_beam.runners.pipeline_context import PipelineContext @@ -2675,7 +2676,8 @@ def process(self, *args, **kwargs): self._pool = concurrent.futures.ThreadPoolExecutor(10) # Import here to avoid circular dependency - from apache_beam.runners.worker.statesampler import get_current_tracker, set_current_tracker + from apache_beam.runners.worker.statesampler import get_current_tracker + from apache_beam.runners.worker.statesampler import set_current_tracker # State sampler/tracker is stored as a thread local variable, and is used # when incrementing counter metrics. @@ -3004,8 +3006,7 @@ def has_side_inputs(): # If the CombineFn has deferred side inputs, the python SDK # doesn't implement it. # Use a ParDo-based CombinePerKey instead. - from apache_beam.transforms.combiners import \ - LiftedCombinePerKey + from apache_beam.transforms.combiners import LiftedCombinePerKey combine_fn, *args = args return LiftedCombinePerKey(combine_fn, args, kwargs) return super(CombinePerKey, cls).__new__(cls) @@ -3760,7 +3761,9 @@ def __init__( """ global AccumulationMode, DefaultTrigger # pylint: disable=global-variable-not-assigned # pylint: disable=wrong-import-order, wrong-import-position - from apache_beam.transforms.trigger import AccumulationMode, DefaultTrigger + from apache_beam.transforms.trigger import AccumulationMode + from apache_beam.transforms.trigger import DefaultTrigger + # pylint: enable=wrong-import-order, wrong-import-position if triggerfn is None: triggerfn = DefaultTrigger() diff --git a/sdks/python/apache_beam/transforms/enrichment_handlers/bigquery_it_test.py b/sdks/python/apache_beam/transforms/enrichment_handlers/bigquery_it_test.py index 1889b0845e6e..ab9375a12e71 100644 --- a/sdks/python/apache_beam/transforms/enrichment_handlers/bigquery_it_test.py +++ b/sdks/python/apache_beam/transforms/enrichment_handlers/bigquery_it_test.py @@ -33,11 +33,11 @@ # pylint: disable=ungrouped-imports try: + from apitools.base.py.exceptions import HttpError from testcontainers.redis import RedisContainer + from apache_beam.transforms.enrichment import Enrichment - from apache_beam.transforms.enrichment_handlers.bigquery import \ - BigQueryEnrichmentHandler - from apitools.base.py.exceptions import HttpError + from apache_beam.transforms.enrichment_handlers.bigquery import BigQueryEnrichmentHandler except ImportError: raise unittest.SkipTest( 'Google Cloud BigQuery dependencies are not installed.') diff --git a/sdks/python/apache_beam/transforms/enrichment_handlers/bigtable_it_test.py b/sdks/python/apache_beam/transforms/enrichment_handlers/bigtable_it_test.py index 09d025b006a2..e8e66fdcc48d 100644 --- a/sdks/python/apache_beam/transforms/enrichment_handlers/bigtable_it_test.py +++ b/sdks/python/apache_beam/transforms/enrichment_handlers/bigtable_it_test.py @@ -34,6 +34,7 @@ from google.cloud.bigtable import Client from google.cloud.bigtable.row_filters import ColumnRangeFilter from testcontainers.redis import RedisContainer + from apache_beam.transforms.enrichment import Enrichment from apache_beam.transforms.enrichment_handlers.bigtable import BigTableEnrichmentHandler from apache_beam.transforms.enrichment_handlers.bigtable import ExceptionLevel diff --git a/sdks/python/apache_beam/transforms/enrichment_handlers/cloudsql_it_test.py b/sdks/python/apache_beam/transforms/enrichment_handlers/cloudsql_it_test.py index b953b67841ac..04db85a75c29 100644 --- a/sdks/python/apache_beam/transforms/enrichment_handlers/cloudsql_it_test.py +++ b/sdks/python/apache_beam/transforms/enrichment_handlers/cloudsql_it_test.py @@ -33,23 +33,28 @@ # pylint: disable=ungrouped-imports try: + from sqlalchemy import VARCHAR + from sqlalchemy import Column + from sqlalchemy import Engine + from sqlalchemy import Integer + from sqlalchemy import MetaData + from sqlalchemy import Table + from sqlalchemy import create_engine from testcontainers.core.generic import DbContainer - from testcontainers.postgres import PostgresContainer - from testcontainers.mysql import MySqlContainer from testcontainers.mssql import SqlServerContainer + from testcontainers.mysql import MySqlContainer + from testcontainers.postgres import PostgresContainer from testcontainers.redis import RedisContainer - from sqlalchemy import ( - create_engine, MetaData, Table, Column, Integer, VARCHAR, Engine) + from apache_beam.transforms.enrichment import Enrichment - from apache_beam.transforms.enrichment_handlers.cloudsql import ( - CloudSQLEnrichmentHandler, - DatabaseTypeAdapter, - CustomQueryConfig, - TableFieldsQueryConfig, - TableFunctionQueryConfig, - CloudSQLConnectionConfig, - ExternalSQLDBConnectionConfig, - ConnectionConfig) + from apache_beam.transforms.enrichment_handlers.cloudsql import CloudSQLConnectionConfig + from apache_beam.transforms.enrichment_handlers.cloudsql import CloudSQLEnrichmentHandler + from apache_beam.transforms.enrichment_handlers.cloudsql import ConnectionConfig + from apache_beam.transforms.enrichment_handlers.cloudsql import CustomQueryConfig + from apache_beam.transforms.enrichment_handlers.cloudsql import DatabaseTypeAdapter + from apache_beam.transforms.enrichment_handlers.cloudsql import ExternalSQLDBConnectionConfig + from apache_beam.transforms.enrichment_handlers.cloudsql import TableFieldsQueryConfig + from apache_beam.transforms.enrichment_handlers.cloudsql import TableFunctionQueryConfig except ImportError as e: raise unittest.SkipTest(f'CloudSQL dependencies not installed: {str(e)}') diff --git a/sdks/python/apache_beam/transforms/enrichment_handlers/cloudsql_test.py b/sdks/python/apache_beam/transforms/enrichment_handlers/cloudsql_test.py index 99823f6d89a6..98f1acfa53cf 100644 --- a/sdks/python/apache_beam/transforms/enrichment_handlers/cloudsql_test.py +++ b/sdks/python/apache_beam/transforms/enrichment_handlers/cloudsql_test.py @@ -20,18 +20,15 @@ # pylint: disable=ungrouped-imports try: - from apache_beam.transforms.enrichment_handlers.cloudsql import ( - CloudSQLEnrichmentHandler, - DatabaseTypeAdapter, - CustomQueryConfig, - TableFieldsQueryConfig, - TableFunctionQueryConfig, - CloudSQLConnectionConfig, - ExternalSQLDBConnectionConfig) - from apache_beam.transforms.enrichment_handlers.cloudsql_it_test import ( - query_fn, - where_clause_value_fn, - ) + from apache_beam.transforms.enrichment_handlers.cloudsql import CloudSQLConnectionConfig + from apache_beam.transforms.enrichment_handlers.cloudsql import CloudSQLEnrichmentHandler + from apache_beam.transforms.enrichment_handlers.cloudsql import CustomQueryConfig + from apache_beam.transforms.enrichment_handlers.cloudsql import DatabaseTypeAdapter + from apache_beam.transforms.enrichment_handlers.cloudsql import ExternalSQLDBConnectionConfig + from apache_beam.transforms.enrichment_handlers.cloudsql import TableFieldsQueryConfig + from apache_beam.transforms.enrichment_handlers.cloudsql import TableFunctionQueryConfig + from apache_beam.transforms.enrichment_handlers.cloudsql_it_test import query_fn + from apache_beam.transforms.enrichment_handlers.cloudsql_it_test import where_clause_value_fn except ImportError as e: raise unittest.SkipTest(f'CloudSQL dependencies not installed: {str(e)}') diff --git a/sdks/python/apache_beam/transforms/enrichment_handlers/feast_feature_store.py b/sdks/python/apache_beam/transforms/enrichment_handlers/feast_feature_store.py index f8e8b4db1d7f..458602457df6 100644 --- a/sdks/python/apache_beam/transforms/enrichment_handlers/feast_feature_store.py +++ b/sdks/python/apache_beam/transforms/enrichment_handlers/feast_feature_store.py @@ -22,11 +22,12 @@ from typing import Any from typing import Optional +from feast import FeatureStore + import apache_beam as beam from apache_beam.io.filesystems import FileSystems from apache_beam.transforms.enrichment import EnrichmentSourceHandler from apache_beam.transforms.enrichment_handlers.utils import ExceptionLevel -from feast import FeatureStore __all__ = [ 'FeastFeatureStoreEnrichmentHandler', diff --git a/sdks/python/apache_beam/transforms/enrichment_handlers/feast_feature_store_it_test.py b/sdks/python/apache_beam/transforms/enrichment_handlers/feast_feature_store_it_test.py index 9c4dab3d68b8..8e3819d71c39 100644 --- a/sdks/python/apache_beam/transforms/enrichment_handlers/feast_feature_store_it_test.py +++ b/sdks/python/apache_beam/transforms/enrichment_handlers/feast_feature_store_it_test.py @@ -33,9 +33,9 @@ # pylint: disable=ungrouped-imports try: from apache_beam.transforms.enrichment import Enrichment - from apache_beam.transforms.enrichment_handlers.feast_feature_store import \ - FeastFeatureStoreEnrichmentHandler - from apache_beam.transforms.enrichment_handlers.vertex_ai_feature_store_it_test import ValidateResponse # pylint: disable=line-too-long + from apache_beam.transforms.enrichment_handlers.feast_feature_store import FeastFeatureStoreEnrichmentHandler + from apache_beam.transforms.enrichment_handlers.vertex_ai_feature_store_it_test import \ + ValidateResponse # pylint: disable=line-too-long except ImportError: raise unittest.SkipTest( 'Feast feature store test dependencies are not installed.') diff --git a/sdks/python/apache_beam/transforms/enrichment_handlers/feast_feature_store_test.py b/sdks/python/apache_beam/transforms/enrichment_handlers/feast_feature_store_test.py index 764086ab2c98..4bad71c83198 100644 --- a/sdks/python/apache_beam/transforms/enrichment_handlers/feast_feature_store_test.py +++ b/sdks/python/apache_beam/transforms/enrichment_handlers/feast_feature_store_test.py @@ -19,10 +19,8 @@ from parameterized import parameterized try: - from apache_beam.transforms.enrichment_handlers.feast_feature_store import \ - FeastFeatureStoreEnrichmentHandler - from apache_beam.transforms.enrichment_handlers.feast_feature_store_it_test \ - import _entity_row_fn + from apache_beam.transforms.enrichment_handlers.feast_feature_store import FeastFeatureStoreEnrichmentHandler + from apache_beam.transforms.enrichment_handlers.feast_feature_store_it_test import _entity_row_fn except ImportError: raise unittest.SkipTest( 'Feast feature store test dependencies are not installed.') diff --git a/sdks/python/apache_beam/transforms/enrichment_handlers/vertex_ai_feature_store_it_test.py b/sdks/python/apache_beam/transforms/enrichment_handlers/vertex_ai_feature_store_it_test.py index d83f1010dd83..dd46db28ecbf 100644 --- a/sdks/python/apache_beam/transforms/enrichment_handlers/vertex_ai_feature_store_it_test.py +++ b/sdks/python/apache_beam/transforms/enrichment_handlers/vertex_ai_feature_store_it_test.py @@ -29,12 +29,12 @@ # pylint: disable=ungrouped-imports try: from testcontainers.redis import RedisContainer + from apache_beam.transforms.enrichment import Enrichment from apache_beam.transforms.enrichment_handlers.utils import ExceptionLevel + from apache_beam.transforms.enrichment_handlers.vertex_ai_feature_store import VertexAIFeatureStoreEnrichmentHandler from apache_beam.transforms.enrichment_handlers.vertex_ai_feature_store import \ - VertexAIFeatureStoreEnrichmentHandler - from apache_beam.transforms.enrichment_handlers.vertex_ai_feature_store import \ - VertexAIFeatureStoreLegacyEnrichmentHandler + VertexAIFeatureStoreLegacyEnrichmentHandler except ImportError: raise unittest.SkipTest( 'VertexAI Feature Store test dependencies ' diff --git a/sdks/python/apache_beam/transforms/enrichment_handlers/vertex_ai_feature_store_test.py b/sdks/python/apache_beam/transforms/enrichment_handlers/vertex_ai_feature_store_test.py index 352146ecc078..211529be4dc9 100644 --- a/sdks/python/apache_beam/transforms/enrichment_handlers/vertex_ai_feature_store_test.py +++ b/sdks/python/apache_beam/transforms/enrichment_handlers/vertex_ai_feature_store_test.py @@ -17,10 +17,9 @@ import unittest try: - from apache_beam.transforms.enrichment_handlers.vertex_ai_feature_store \ - import VertexAIFeatureStoreEnrichmentHandler - from apache_beam.transforms.enrichment_handlers.vertex_ai_feature_store \ - import VertexAIFeatureStoreLegacyEnrichmentHandler + from apache_beam.transforms.enrichment_handlers.vertex_ai_feature_store import VertexAIFeatureStoreEnrichmentHandler + from apache_beam.transforms.enrichment_handlers.vertex_ai_feature_store import \ + VertexAIFeatureStoreLegacyEnrichmentHandler except ImportError: raise unittest.SkipTest( 'VertexAI Feature Store test dependencies ' diff --git a/sdks/python/apache_beam/transforms/ptransform.py b/sdks/python/apache_beam/transforms/ptransform.py index f69677b89a17..9c5306e143ec 100644 --- a/sdks/python/apache_beam/transforms/ptransform.py +++ b/sdks/python/apache_beam/transforms/ptransform.py @@ -88,9 +88,9 @@ class and wrapper class that allows lambda functions to be used as if TYPE_CHECKING: from apache_beam import coders from apache_beam.pipeline import Pipeline + from apache_beam.portability.api import beam_runner_api_pb2 from apache_beam.runners.pipeline_context import PipelineContext from apache_beam.transforms.core import Windowing - from apache_beam.portability.api import beam_runner_api_pb2 __all__ = [ 'PTransform', @@ -567,6 +567,7 @@ def get_windowing(self, inputs): else: from apache_beam.transforms.core import Windowing from apache_beam.transforms.window import GlobalWindows + # TODO(robertwb): Return something compatible with every windowing? return Windowing(GlobalWindows()) @@ -590,6 +591,7 @@ def __ror__(self, left, label=None): # pylint: disable=wrong-import-order, wrong-import-position from apache_beam import pipeline from apache_beam.options.pipeline_options import PipelineOptions + # pylint: enable=wrong-import-order, wrong-import-position p = pipeline.Pipeline('DirectRunner', PipelineOptions(sys.argv)) else: @@ -610,6 +612,7 @@ def __ror__(self, left, label=None): deferred = not getattr(p.runner, 'is_eager', False) # pylint: disable=wrong-import-order, wrong-import-position from apache_beam.transforms.core import Create + # pylint: enable=wrong-import-order, wrong-import-position replacements = { id(v): p | 'CreatePInput%s' % ix >> Create(v, reshuffle=False) @@ -639,6 +642,7 @@ def _extract_input_pvalues(self, pvalueish): """ # pylint: disable=wrong-import-order from apache_beam import pipeline + # pylint: enable=wrong-import-order if isinstance(pvalueish, pipeline.Pipeline): pvalueish = pvalue.PBegin(pvalueish) @@ -747,6 +751,7 @@ def register(constructor): def to_runner_api(self, context, has_parts=False, **extra_kwargs): # type: (PipelineContext, bool, Any) -> beam_runner_api_pb2.FunctionSpec from apache_beam.portability.api import beam_runner_api_pb2 + # typing: only ParDo supports extra_kwargs urn, typed_param = self.to_runner_api_parameter(context, **extra_kwargs) if urn == python_urns.GENERIC_COMPOSITE_TRANSFORM and not has_parts: diff --git a/sdks/python/apache_beam/transforms/trigger_test.py b/sdks/python/apache_beam/transforms/trigger_test.py index 9f9b7fe51a9f..a5ed77be7c6d 100644 --- a/sdks/python/apache_beam/transforms/trigger_test.py +++ b/sdks/python/apache_beam/transforms/trigger_test.py @@ -966,6 +966,7 @@ def parse_fn(s, names): # pylint: disable=wrong-import-order, wrong-import-position from apache_beam.transforms import window as window_module + # pylint: enable=wrong-import-order, wrong-import-position window_fn_names = dict(window_module.__dict__) # yapf: disable diff --git a/sdks/python/apache_beam/transforms/userstate.py b/sdks/python/apache_beam/transforms/userstate.py index 625d4cd0e779..54d66abbb392 100644 --- a/sdks/python/apache_beam/transforms/userstate.py +++ b/sdks/python/apache_beam/transforms/userstate.py @@ -131,6 +131,7 @@ def __init__( """ # Avoid circular import. from apache_beam.transforms.core import CombineFn + # We want the coder to be optional, but unfortunately it comes # before the non-optional combine_fn parameter, which we can't # change for backwards compatibility reasons. diff --git a/sdks/python/apache_beam/typehints/__init__.py b/sdks/python/apache_beam/typehints/__init__.py index 81ffc9f307d9..871fa6d045c7 100644 --- a/sdks/python/apache_beam/typehints/__init__.py +++ b/sdks/python/apache_beam/typehints/__init__.py @@ -18,9 +18,9 @@ """A package defining the syntax and decorator semantics for type-hints.""" # pylint: disable=wildcard-import -from apache_beam.typehints.typehints import * -from apache_beam.typehints.decorators import * from apache_beam.typehints.batch import * +from apache_beam.typehints.decorators import * +from apache_beam.typehints.typehints import * # pylint: disable=ungrouped-imports try: diff --git a/sdks/python/apache_beam/typehints/pytorch_type_compatibility.py b/sdks/python/apache_beam/typehints/pytorch_type_compatibility.py index eab93f54e6b9..95158085b172 100644 --- a/sdks/python/apache_beam/typehints/pytorch_type_compatibility.py +++ b/sdks/python/apache_beam/typehints/pytorch_type_compatibility.py @@ -18,6 +18,7 @@ from typing import Optional import torch + from apache_beam.typehints import typehints from apache_beam.typehints.batch import BatchConverter from apache_beam.typehints.batch import N diff --git a/sdks/python/apache_beam/typehints/pytorch_type_compatibility_test.py b/sdks/python/apache_beam/typehints/pytorch_type_compatibility_test.py index d1f5c0d271ee..609550916bba 100644 --- a/sdks/python/apache_beam/typehints/pytorch_type_compatibility_test.py +++ b/sdks/python/apache_beam/typehints/pytorch_type_compatibility_test.py @@ -32,6 +32,7 @@ # pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports try: import torch + from apache_beam.typehints.pytorch_type_compatibility import PytorchTensor except ImportError: raise unittest.SkipTest('PyTorch dependencies are not installed') diff --git a/sdks/python/apache_beam/utils/interactive_utils.py b/sdks/python/apache_beam/utils/interactive_utils.py index 02d7d0e2d047..222c07a91414 100644 --- a/sdks/python/apache_beam/utils/interactive_utils.py +++ b/sdks/python/apache_beam/utils/interactive_utils.py @@ -69,6 +69,7 @@ def alter_label_if_ipython(transform, pvalueish): """ if is_in_ipython(): from apache_beam.runners.interactive import interactive_environment as ie + # Tracks user defined pipeline instances in watched scopes so that we only # alter labels for any transform to pvalueish belonging to those pipeline # instances, excluding any transform to be applied in other pipeline diff --git a/sdks/python/apache_beam/yaml/integration_tests.py b/sdks/python/apache_beam/yaml/integration_tests.py index 0e25a7fd224c..534082ddab37 100644 --- a/sdks/python/apache_beam/yaml/integration_tests.py +++ b/sdks/python/apache_beam/yaml/integration_tests.py @@ -37,6 +37,7 @@ import pytds import sqlalchemy import yaml +from apitools.base.py.exceptions import HttpError from google.cloud import pubsub_v1 from google.cloud.bigtable import client from google.cloud.bigtable_admin_v2.types import instance @@ -58,7 +59,6 @@ from apache_beam.yaml import yaml_provider from apache_beam.yaml import yaml_transform from apache_beam.yaml.conftest import yaml_test_files_dir -from apitools.base.py.exceptions import HttpError _LOGGER = logging.getLogger(__name__) diff --git a/sdks/python/apache_beam/yaml/yaml_ml.py b/sdks/python/apache_beam/yaml/yaml_ml.py index 1cec67cf3621..e5a88f54eba7 100644 --- a/sdks/python/apache_beam/yaml/yaml_ml.py +++ b/sdks/python/apache_beam/yaml/yaml_ml.py @@ -51,6 +51,7 @@ def _list_submodules(package): _transform_constructors = {} try: from apache_beam.ml.transforms.base import MLTransform + # Load all available ML Transform modules for module_name in _list_submodules(beam.ml.transforms): try: diff --git a/sdks/python/apache_beam/yaml/yaml_provider.py b/sdks/python/apache_beam/yaml/yaml_provider.py index 0b47cbf2e686..3d1d1f5a742c 100755 --- a/sdks/python/apache_beam/yaml/yaml_provider.py +++ b/sdks/python/apache_beam/yaml/yaml_provider.py @@ -506,8 +506,9 @@ def create_transform( yaml_create_transform: Callable[ [Mapping[str, Any], Iterable[beam.PCollection]], beam.PTransform] ) -> beam.PTransform: - from apache_beam.yaml.yaml_transform import expand_jinja, preprocess from apache_beam.yaml.yaml_transform import SafeLineLoader + from apache_beam.yaml.yaml_transform import expand_jinja + from apache_beam.yaml.yaml_transform import preprocess spec = self._transforms[type] try: import jsonschema @@ -1629,9 +1630,9 @@ def merge_providers(*provider_sets) -> Mapping[str, Iterable[Provider]]: @functools.cache def standard_providers(): from apache_beam.yaml.yaml_combine import create_combine_providers - from apache_beam.yaml.yaml_mapping import create_mapping_providers - from apache_beam.yaml.yaml_join import create_join_providers from apache_beam.yaml.yaml_io import io_providers + from apache_beam.yaml.yaml_join import create_join_providers + from apache_beam.yaml.yaml_mapping import create_mapping_providers from apache_beam.yaml.yaml_specifiable import create_spec_providers return merge_providers( diff --git a/sdks/python/gen_managed_doc.py b/sdks/python/gen_managed_doc.py index fa467d1ccf04..75301d6a7bb5 100644 --- a/sdks/python/gen_managed_doc.py +++ b/sdks/python/gen_managed_doc.py @@ -25,7 +25,6 @@ from typing import Dict import yaml - from gen_protos import PROJECT_ROOT from gen_protos import PYTHON_SDK_ROOT from gen_xlang_wrappers import pretty_type @@ -104,11 +103,11 @@ def generate_managed_doc(output_location): + from apache_beam.transforms import managed from apache_beam.transforms.external import MANAGED_TRANSFORM_URN_TO_JAR_TARGET_MAPPING from apache_beam.transforms.external import BeamJarExpansionService from apache_beam.transforms.external_transform_provider import ExternalTransform from apache_beam.transforms.external_transform_provider import ExternalTransformProvider - from apache_beam.transforms import managed with open(_DOCUMENTED_MANAGED_CONFIGS) as f: available_configs: dict = yaml.safe_load(f) diff --git a/sdks/python/gen_xlang_wrappers.py b/sdks/python/gen_xlang_wrappers.py index e3b408f9eeb7..335fc2ee395b 100644 --- a/sdks/python/gen_xlang_wrappers.py +++ b/sdks/python/gen_xlang_wrappers.py @@ -34,7 +34,6 @@ from typing import Union import yaml - from gen_protos import LICENSE_HEADER from gen_protos import PROJECT_ROOT from gen_protos import PYTHON_SDK_ROOT From aa962e67d9c4cc602def87143e304fce84c78664 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Thu, 30 Oct 2025 13:57:10 -0400 Subject: [PATCH 22/29] fix circulat dependencies, clear ungrouped imports --- sdks/python/apache_beam/io/gcp/bigquery_test.py | 4 ++-- sdks/python/apache_beam/io/gcp/gcsio_retry_test.py | 3 +++ sdks/python/apache_beam/io/gcp/gcsio_test.py | 4 ++-- .../io/gcp/healthcare/dicomio_integration_test.py | 4 ++-- sdks/python/apache_beam/runners/__init__.py | 8 +++++--- sdks/python/apache_beam/typehints/decorators.py | 2 +- 6 files changed, 15 insertions(+), 10 deletions(-) diff --git a/sdks/python/apache_beam/io/gcp/bigquery_test.py b/sdks/python/apache_beam/io/gcp/bigquery_test.py index f516b206d5e3..234c99847a44 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery_test.py +++ b/sdks/python/apache_beam/io/gcp/bigquery_test.py @@ -81,7 +81,7 @@ from apache_beam.transforms.display_test import DisplayDataItemMatcher # Protect against environments where bigquery library is not available. -# pylint: disable=wrong-import-order, wrong-import-position +# pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports try: from apitools.base.py.exceptions import HttpError @@ -97,7 +97,7 @@ HttpError = None HttpForbiddenError = None exceptions = None -# pylint: enable=wrong-import-order, wrong-import-position +# pylint: enable=wrong-import-order, wrong-import-position, ungrouped-imports _LOGGER = logging.getLogger(__name__) diff --git a/sdks/python/apache_beam/io/gcp/gcsio_retry_test.py b/sdks/python/apache_beam/io/gcp/gcsio_retry_test.py index f53ea5244244..2572a72ae05c 100644 --- a/sdks/python/apache_beam/io/gcp/gcsio_retry_test.py +++ b/sdks/python/apache_beam/io/gcp/gcsio_retry_test.py @@ -26,6 +26,7 @@ from apache_beam.runners.worker import statesampler from apache_beam.utils import counters +# pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports try: from google.api_core import exceptions as api_exceptions @@ -34,6 +35,8 @@ gcsio_retry = None api_exceptions = None +# pylint: enable=wrong-import-order, wrong-import-position, ungrouped-imports + @unittest.skipIf((gcsio_retry is None or api_exceptions is None), 'GCP dependencies are not installed') diff --git a/sdks/python/apache_beam/io/gcp/gcsio_test.py b/sdks/python/apache_beam/io/gcp/gcsio_test.py index 60ba3bda8eb2..d2b873f566cb 100644 --- a/sdks/python/apache_beam/io/gcp/gcsio_test.py +++ b/sdks/python/apache_beam/io/gcp/gcsio_test.py @@ -34,7 +34,7 @@ from apache_beam.runners.worker import statesampler from apache_beam.utils import counters -# pylint: disable=wrong-import-order, wrong-import-position +# pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports try: from google.cloud.exceptions import BadRequest @@ -44,7 +44,7 @@ from apache_beam.io.gcp.gcsio_retry import DEFAULT_RETRY_WITH_THROTTLING_COUNTER except ImportError: NotFound = None -# pylint: enable=wrong-import-order, wrong-import-position +# pylint: enable=wrong-import-order, wrong-import-position, ungrouped-imports DEFAULT_GCP_PROJECT = 'apache-beam-testing' diff --git a/sdks/python/apache_beam/io/gcp/healthcare/dicomio_integration_test.py b/sdks/python/apache_beam/io/gcp/healthcare/dicomio_integration_test.py index ca602c79a464..b585466aef36 100644 --- a/sdks/python/apache_beam/io/gcp/healthcare/dicomio_integration_test.py +++ b/sdks/python/apache_beam/io/gcp/healthcare/dicomio_integration_test.py @@ -38,7 +38,7 @@ from apache_beam.testing.util import assert_that from apache_beam.testing.util import equal_to -# pylint: disable=wrong-import-order, wrong-import-position +# pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports try: from google.auth import default from google.auth.transport import requests @@ -48,7 +48,7 @@ from apache_beam.io.gcp.healthcare.dicomio import UploadToDicomStore except ImportError: DicomSearch = None -# pylint: enable=wrong-import-order, wrong-import-position +# pylint: enable=wrong-import-order, wrong-import-position, ungrouped-imports REGION = 'us-central1' DATA_SET_ID = 'apache-beam-integration-testing' diff --git a/sdks/python/apache_beam/runners/__init__.py b/sdks/python/apache_beam/runners/__init__.py index b4ead2885bf3..a37e1417e443 100644 --- a/sdks/python/apache_beam/runners/__init__.py +++ b/sdks/python/apache_beam/runners/__init__.py @@ -19,11 +19,13 @@ This package defines runners, which are used to execute a pipeline. """ - -from apache_beam.runners.dataflow.dataflow_runner import DataflowRunner -from apache_beam.runners.dataflow.test_dataflow_runner import TestDataflowRunner from apache_beam.runners.direct.direct_runner import DirectRunner from apache_beam.runners.direct.test_direct_runner import TestDirectRunner from apache_beam.runners.runner import PipelineRunner from apache_beam.runners.runner import PipelineState from apache_beam.runners.runner import create_runner + +# isort: off +# initialize these last to avoid a circular dependency +from apache_beam.runners.dataflow.dataflow_runner import DataflowRunner +from apache_beam.runners.dataflow.test_dataflow_runner import TestDataflowRunner \ No newline at end of file diff --git a/sdks/python/apache_beam/typehints/decorators.py b/sdks/python/apache_beam/typehints/decorators.py index 6f27676d6466..2d2f7981dd29 100644 --- a/sdks/python/apache_beam/typehints/decorators.py +++ b/sdks/python/apache_beam/typehints/decorators.py @@ -123,7 +123,7 @@ def foo((a, b)): _ANY_VAR_POSITIONAL = typehints.Tuple[typehints.Any, ...] _ANY_VAR_KEYWORD = typehints.Dict[typehints.Any, typehints.Any] -_disable_from_callable = False # pylint: disable=invalid-name +_disable_from_callable = False # pylint: disable=invalid-name def get_signature(func): From 5524d33d95d0196f25f77e78f9583ae16634f38b Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Thu, 30 Oct 2025 14:17:37 -0400 Subject: [PATCH 23/29] last bit of pylint fixes --- sdks/python/apache_beam/ml/gcp/videointelligenceml_test_it.py | 1 + sdks/python/apache_beam/ml/gcp/visionml_test_it.py | 1 + sdks/python/apache_beam/ml/rag/chunking/langchain_test.py | 1 + .../apache_beam/ml/transforms/embeddings/vertex_ai_test.py | 2 +- sdks/python/apache_beam/runners/__init__.py | 2 +- 5 files changed, 5 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/ml/gcp/videointelligenceml_test_it.py b/sdks/python/apache_beam/ml/gcp/videointelligenceml_test_it.py index eac951697d3d..37ebe8145b4e 100644 --- a/sdks/python/apache_beam/ml/gcp/videointelligenceml_test_it.py +++ b/sdks/python/apache_beam/ml/gcp/videointelligenceml_test_it.py @@ -31,6 +31,7 @@ # Protect against environments where Google Cloud VideoIntelligence client is # not available. +# pylint: disable=ungrouped-imports try: from google.cloud.videointelligence import enums from google.cloud.videointelligence import types diff --git a/sdks/python/apache_beam/ml/gcp/visionml_test_it.py b/sdks/python/apache_beam/ml/gcp/visionml_test_it.py index 68b1aeccfb6e..f7f61c60552b 100644 --- a/sdks/python/apache_beam/ml/gcp/visionml_test_it.py +++ b/sdks/python/apache_beam/ml/gcp/visionml_test_it.py @@ -27,6 +27,7 @@ # Protect against environments where Google Cloud Vision client is not # available. +# pylint: disable=ungrouped-imports try: from google.cloud import vision diff --git a/sdks/python/apache_beam/ml/rag/chunking/langchain_test.py b/sdks/python/apache_beam/ml/rag/chunking/langchain_test.py index 580d2b17e25b..5ee496c036a1 100644 --- a/sdks/python/apache_beam/ml/rag/chunking/langchain_test.py +++ b/sdks/python/apache_beam/ml/rag/chunking/langchain_test.py @@ -26,6 +26,7 @@ from apache_beam.testing.util import equal_to from apache_beam.testing.util import is_not_empty +# pylint: disable=ungrouped-imports try: from langchain.text_splitter import CharacterTextSplitter from langchain.text_splitter import RecursiveCharacterTextSplitter diff --git a/sdks/python/apache_beam/ml/transforms/embeddings/vertex_ai_test.py b/sdks/python/apache_beam/ml/transforms/embeddings/vertex_ai_test.py index 7f820d4da2f3..458a04c7c810 100644 --- a/sdks/python/apache_beam/ml/transforms/embeddings/vertex_ai_test.py +++ b/sdks/python/apache_beam/ml/transforms/embeddings/vertex_ai_test.py @@ -25,6 +25,7 @@ from apache_beam.ml.transforms import base from apache_beam.ml.transforms.base import MLTransform +# pylint: disable=ungrouped-imports try: from vertexai.vision_models import Image from vertexai.vision_models import Video @@ -42,7 +43,6 @@ VertexAITextEmbeddings = None # type: ignore VertexAIImageEmbeddings = None # type: ignore -# pylint: disable=ungrouped-imports try: import tensorflow_transform as tft diff --git a/sdks/python/apache_beam/runners/__init__.py b/sdks/python/apache_beam/runners/__init__.py index a37e1417e443..d22024c255d3 100644 --- a/sdks/python/apache_beam/runners/__init__.py +++ b/sdks/python/apache_beam/runners/__init__.py @@ -28,4 +28,4 @@ # isort: off # initialize these last to avoid a circular dependency from apache_beam.runners.dataflow.dataflow_runner import DataflowRunner -from apache_beam.runners.dataflow.test_dataflow_runner import TestDataflowRunner \ No newline at end of file +from apache_beam.runners.dataflow.test_dataflow_runner import TestDataflowRunner From 60e8552e28928072dae48d42bfe4a59ecee1944a Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Thu, 30 Oct 2025 14:38:54 -0400 Subject: [PATCH 24/29] exclude portability api files --- sdks/python/.isort.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/.isort.cfg b/sdks/python/.isort.cfg index ae37f37a113f..2cde8073e216 100644 --- a/sdks/python/.isort.cfg +++ b/sdks/python/.isort.cfg @@ -55,4 +55,4 @@ skip=apiclient.py, cloudbuild_v1_client.py, cloudbuild_v1_messages.py, boto3_client.py -skip_glob=*.pxd,*.pyx,*pb2*.py,**/examples/**/*.py \ No newline at end of file +skip_glob=*.pxd,*.pyx,*pb2*.py,**/examples/**/*.py,**/portability/api/**/*.py \ No newline at end of file From a4990055acee7da0caf8959bed33f9fb84bd81e6 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Thu, 30 Oct 2025 14:58:14 -0400 Subject: [PATCH 25/29] extra exclusion of portability/api/__init__.py --- sdks/python/.isort.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/python/.isort.cfg b/sdks/python/.isort.cfg index 2cde8073e216..a29f98cc90be 100644 --- a/sdks/python/.isort.cfg +++ b/sdks/python/.isort.cfg @@ -54,5 +54,5 @@ skip=apiclient.py, proto2_coder_test_messages_pb2.py, cloudbuild_v1_client.py, cloudbuild_v1_messages.py, - boto3_client.py -skip_glob=*.pxd,*.pyx,*pb2*.py,**/examples/**/*.py,**/portability/api/**/*.py \ No newline at end of file + boto3_client.py, +skip_glob=*.pxd,*.pyx,*pb2*.py,**/examples/**/*.py,**/portability/api/**/*.py,**/portability/api/__init__.py \ No newline at end of file From 5e031e7b2556365e0600f23c6edf24a9b5b09640 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Thu, 30 Oct 2025 15:00:55 -0400 Subject: [PATCH 26/29] fix python version in flink container --- .github/workflows/beam_PreCommit_Flink_Container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/beam_PreCommit_Flink_Container.yml b/.github/workflows/beam_PreCommit_Flink_Container.yml index 0c42fe0d4d87..51bc3c092f6e 100644 --- a/.github/workflows/beam_PreCommit_Flink_Container.yml +++ b/.github/workflows/beam_PreCommit_Flink_Container.yml @@ -79,7 +79,7 @@ env: ARTIFACTS_DIR: gs://beam-flink-cluster/beam-precommit-flink-container-${{ github.run_id }} DOCKER_REGISTRY: gcr.io DOCKER_REPOSITORY_ROOT: ${{ github.event_name == 'pull_request_target' && 'gcr.io/apache-beam-testing/beam-sdk-pr' || 'gcr.io/apache-beam-testing/beam-sdk' }} - PYTHON_VERSION: 3.10 + PYTHON_VERSION: '3.10' PYTHON_SDK_IMAGE_TAG: latest jobs: From ab79134d861b6600867a89410aa4bdb09cfae01d Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Thu, 30 Oct 2025 15:23:36 -0400 Subject: [PATCH 27/29] make the whitespace precommit happy --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index ca5c00e18fd4..4aadd391b287 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -83,7 +83,7 @@ ## Deprecations * X behavior is deprecated and will be removed in X versions ([#X](https://github.com/apache/beam/issues/X)). -* (Python) Python 3.9 reached EOL in October 2025 and support for the lanugage version has been removed. ([#36665](https://github.com/apache/beam/pull/36665)). +* (Python) Python 3.9 reached EOL in October 2025 and support for the lanugage version has been removed. ([#36665](https://github.com/apache/beam/issues/36665)). ## Bugfixes From 3da12926b57bcdd6c074cdd95f51471bc0e27c97 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Fri, 31 Oct 2025 10:52:36 -0400 Subject: [PATCH 28/29] restore io init --- sdks/python/apache_beam/io/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/io/__init__.py b/sdks/python/apache_beam/io/__init__.py index a3e51150d8a1..00944f188f77 100644 --- a/sdks/python/apache_beam/io/__init__.py +++ b/sdks/python/apache_beam/io/__init__.py @@ -18,6 +18,7 @@ """A package defining several input sources and output sinks.""" # pylint: disable=wildcard-import +# isort: off from apache_beam.io.avroio import * from apache_beam.io.filebasedsink import * from apache_beam.io.iobase import Read @@ -26,16 +27,16 @@ from apache_beam.io.iobase import Writer from apache_beam.io.mongodbio import * from apache_beam.io.parquetio import * -from apache_beam.io.range_trackers import * from apache_beam.io.textio import * from apache_beam.io.tfrecordio import * +from apache_beam.io.range_trackers import * # Protect against environments where clientslibrary is not available. # pylint: disable=wrong-import-order, wrong-import-position try: - from apache_beam.io.gcp import gcsio from apache_beam.io.gcp.bigquery import * from apache_beam.io.gcp.pubsub import * + from apache_beam.io.gcp import gcsio from apache_beam.transforms.xlang.io import * except ImportError: pass From 15636f837c9da252b73464a55122ca01caf88c71 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Fri, 31 Oct 2025 13:00:04 -0400 Subject: [PATCH 29/29] restore vertex test ordering --- .../ml/transforms/embeddings/vertex_ai_test.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/sdks/python/apache_beam/ml/transforms/embeddings/vertex_ai_test.py b/sdks/python/apache_beam/ml/transforms/embeddings/vertex_ai_test.py index 458a04c7c810..50507c54e36d 100644 --- a/sdks/python/apache_beam/ml/transforms/embeddings/vertex_ai_test.py +++ b/sdks/python/apache_beam/ml/transforms/embeddings/vertex_ai_test.py @@ -26,11 +26,8 @@ from apache_beam.ml.transforms.base import MLTransform # pylint: disable=ungrouped-imports +# isort: off try: - from vertexai.vision_models import Image - from vertexai.vision_models import Video - from vertexai.vision_models import VideoSegmentConfig - from apache_beam.ml.rag.types import Chunk from apache_beam.ml.rag.types import Content from apache_beam.ml.transforms.embeddings.vertex_ai import VertexAIImageEmbeddings @@ -38,6 +35,12 @@ from apache_beam.ml.transforms.embeddings.vertex_ai import VertexAITextEmbeddings from apache_beam.ml.transforms.embeddings.vertex_ai import VertexImage from apache_beam.ml.transforms.embeddings.vertex_ai import VertexVideo + + # Load the Vertex dependencies last so type resolution still pulls in RAG + # types. + from vertexai.vision_models import Image + from vertexai.vision_models import Video + from vertexai.vision_models import VideoSegmentConfig except ImportError: VertexAIMultiModalEmbeddings = None # type: ignore VertexAITextEmbeddings = None # type: ignore