Support Python 3.11 (#26121)

.github/actions/setup-default-test-properties/test-properties.json

@@ -1,12 +1,12 @@
 {
   "PythonTestProperties": {
-    "ALL_SUPPORTED_VERSIONS": ["3.7", "3.8", "3.9", "3.10"],
+    "ALL_SUPPORTED_VERSIONS": ["3.7", "3.8", "3.9", "3.10", "3.11"],
     "LOWEST_SUPPORTED": ["3.7"],
-    "HIGHEST_SUPPORTED": ["3.10"],
-    "ESSENTIAL_VERSIONS": ["3.7", "3.10"],
-    "CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS": ["3.7", "3.10"],
-    "CROSS_LANGUAGE_VALIDATES_RUNNER_DATAFLOW_USING_SQL_PYTHON_VERSIONS": ["3.10"],
-    "VALIDATES_CONTAINER_DATAFLOW_PYTHON_VERSIONS": ["3.7", "3.8", "3.9", "3.10"],
+    "HIGHEST_SUPPORTED": ["3.11"],
+    "ESSENTIAL_VERSIONS": ["3.7", "3.11"],
+    "CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS": ["3.7", "3.11"],
+    "CROSS_LANGUAGE_VALIDATES_RUNNER_DATAFLOW_USING_SQL_PYTHON_VERSIONS": ["3.11"],
+    "VALIDATES_CONTAINER_DATAFLOW_PYTHON_VERSIONS": ["3.7", "3.8", "3.9", "3.10", "3.11" ]
     "LOAD_TEST_PYTHON_VERSION": "3.7",
     "CHICAGO_TAXI_EXAMPLE_FLINK_PYTHON_VERSION": "3.7",
     "DEFAULT_INTERPRETER": "python3.8",

.github/workflows/build_wheels.yml

@@ -48,7 +48,7 @@ jobs:
     runs-on: ubuntu-latest
     env:
       EVENT_NAME: ${{ github.event_name }}
-      PY_VERSIONS_FULL: "cp37-* cp38-* cp39-* cp310-*"
+      PY_VERSIONS_FULL: "cp37-* cp38-* cp39-* cp310-* cp311-*"
     outputs:
       gcp-variables-set: ${{ steps.check_gcp_variables.outputs.gcp-variables-set }}
       py-versions-full: ${{ steps.set-py-versions.outputs.py-versions-full }}

.github/workflows/python_tests.yml

@@ -105,6 +105,7 @@ jobs:
           {"py_ver": "3.8", "tox_env": "py38"},
           {"py_ver": "3.9", "tox_env": "py39"},
           {"py_ver": "3.10", "tox_env": "py310" },
+          { "py_ver": "3.11", "tox_env": "py311" },
         ]
     steps:
       - name: Checkout code
@@ -115,7 +116,7 @@ jobs:
           python-version: ${{ matrix.params.py_ver }}
       - name: Get build dependencies
         working-directory: ./sdks/python
-        run: pip install -r build-requirements.txt
+        run: pip install -r build-requirements.txt --use-pep517
       - name: Install tox
         run: pip install tox
       - name: Run tests basic unix
@@ -140,7 +141,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python: ["3.7", "3.8", "3.9", "3.10"]
+        python: ["3.7", "3.8", "3.9", "3.10", "3.11"]
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
@@ -168,7 +169,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python: ["3.7", "3.8", "3.9", "3.10"]
+        python: ["3.7", "3.8", "3.9", "3.10", "3.11"]
     steps:
       - name: Checkout code
         uses: actions/checkout@v3

.test-infra/jenkins/PythonTestProperties.groovy

@@ -23,7 +23,8 @@ class PythonTestProperties {
     '3.7',
     '3.8',
     '3.9',
-    '3.10'
+    '3.10',
+    '3.11'
   ]
   final static List<String> SUPPORTED_CONTAINER_TASKS = ALL_SUPPORTED_VERSIONS.collect {
     "py${it.replace('.', '')}"

.test-infra/jenkins/job_PreCommit_Portable_Python.groovy

@@ -18,6 +18,8 @@
 
 import CommonJobProperties as commonJobProperties
 import PrecommitJobBuilder
+import static PythonTestProperties.LOWEST_SUPPORTED
+import static PythonTestProperties.HIGHEST_SUPPORTED
 
 PrecommitJobBuilder builder = new PrecommitJobBuilder(
     scope: this,
@@ -40,15 +42,17 @@ builder.build {
   // Due to BEAM-7993, run multiple Python version of portable precommit
   // tests in parallel could lead python3 container crash. We manually
   // config gradle steps here to run tests in sequential.
+  def lowestSupported = LOWEST_SUPPORTED.replace('.', '')
+  def highestSupported = HIGHEST_SUPPORTED.replace('.', '')
   steps {
     gradle {
       rootBuildScriptDir(commonJobProperties.checkoutDir)
-      tasks(':sdks:python:test-suites:portable:py37:preCommitPy37')
+      tasks(':sdks:python:test-suites:portable:py${lowestSupported}:preCommitPy${lowestSupported}')
       commonJobProperties.setGradleSwitches(delegate)
     }
     gradle {
       rootBuildScriptDir(commonJobProperties.checkoutDir)
-      tasks(':sdks:python:test-suites:portable:py310:preCommitPy310')
+      tasks(':sdks:python:test-suites:portable:${highestSupported}:preCommitPy${highestSupported}')
       commonJobProperties.setGradleSwitches(delegate)
     }
   }

CHANGES.md

@@ -54,8 +54,7 @@
 
 ## Highlights
 
-* New highly anticipated feature X added to Python SDK ([#X](https://github.com/apache/beam/issues/X)).
-* New highly anticipated feature Y added to Java SDK ([#Y](https://github.com/apache/beam/issues/Y)).
+* Apache Beam adds Python 3.11 support ([#23848](https://github.com/apache/beam/issues/23848)).
 
 ## I/Os
 

build.gradle.kts

@@ -456,6 +456,7 @@ tasks.register("pythonPreCommit") {
   dependsOn(":sdks:python:test-suites:tox:py38:preCommitPy38")
   dependsOn(":sdks:python:test-suites:tox:py39:preCommitPy39")
   dependsOn(":sdks:python:test-suites:tox:py310:preCommitPy310")
+  dependsOn(":sdks:python:test-suites:tox:py311:preCommitPy311")
 }
 
 tasks.register("pythonPreCommitIT") {
@@ -473,6 +474,7 @@ tasks.register("pythonDockerBuildPreCommit") {
   dependsOn(":sdks:python:container:py38:docker")
   dependsOn(":sdks:python:container:py39:docker")
   dependsOn(":sdks:python:container:py310:docker")
+  dependsOn(":sdks:python:container:py311:docker")
 }
 
 tasks.register("pythonLintPreCommit") {
@@ -502,6 +504,9 @@ tasks.register("python38PostCommit") {
   dependsOn(":sdks:python:test-suites:direct:py38:hdfsIntegrationTest")
   dependsOn(":sdks:python:test-suites:portable:py38:postCommitPy38")
   // TODO: https://github.com/apache/beam/issues/22651
+  // The default container uses Python 3.8. The goal here is to
+  // duild Docker images for TensorRT tests during run time for python versions
+  // other than 3.8 and add these tests in other python postcommit suites.
   dependsOn(":sdks:python:test-suites:dataflow:py38:inferencePostCommitIT")
   dependsOn(":sdks:python:test-suites:direct:py38:inferencePostCommitIT")
 }
@@ -523,6 +528,13 @@ tasks.register("python310PostCommit") {
   dependsOn(":sdks:python:test-suites:portable:py310:postCommitPy310")
 }
 
+tasks.register("python311PostCommit") {
+  dependsOn(":sdks:python:test-suites:dataflow:py311:postCommitIT")
+  dependsOn(":sdks:python:test-suites:direct:py311:postCommitIT")
+  dependsOn(":sdks:python:test-suites:direct:py311:hdfsIntegrationTest")
+  dependsOn(":sdks:python:test-suites:portable:py311:postCommitPy311")
+}
+
 task("python37SickbayPostCommit") {
   dependsOn(":sdks:python:test-suites:dataflow:py37:postCommitSickbay")
 }
@@ -537,14 +549,14 @@ task("python39SickbayPostCommit") {
 
 tasks.register("portablePythonPreCommit") {
   dependsOn(":sdks:python:test-suites:portable:py37:preCommitPy37")
-  dependsOn(":sdks:python:test-suites:portable:py310:preCommitPy310")
+  dependsOn(":sdks:python:test-suites:portable:py311:preCommitPy311")
 }
 
 tasks.register("pythonSparkPostCommit") {
   dependsOn(":sdks:python:test-suites:portable:py37:sparkValidatesRunner")
   dependsOn(":sdks:python:test-suites:portable:py38:sparkValidatesRunner")
   dependsOn(":sdks:python:test-suites:portable:py39:sparkValidatesRunner")
-  dependsOn(":sdks:python:test-suites:portable:py310:sparkValidatesRunner")
+  dependsOn(":sdks:python:test-suites:portable:py311:sparkValidatesRunner")
 }
 
 tasks.register("websitePreCommit") {

sdks/python/apache_beam/__init__.py

@@ -70,7 +70,7 @@
 import warnings
 
 if sys.version_info.major == 3:
-  if sys.version_info.minor <= 6 or sys.version_info.minor >= 11:
+  if sys.version_info.minor <= 6 or sys.version_info.minor >= 12:
     warnings.warn(
         'This version of Apache Beam has not been sufficiently tested on '
         'Python %s.%s. You may encounter bugs or missing features.' %

sdks/python/apache_beam/internal/dill_pickler.py

@@ -46,6 +46,125 @@
 
 settings = {'dill_byref': None}
 
+if sys.version_info >= (3, 11) and dill.__version__ == "0.3.1.1":
+  # Let's make dill 0.3.1.1 support Python 3.11.
+
+  # The following function is based on 'save_code' from 'dill'
+  # Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
+  # Copyright (c) 2008-2015 California Institute of Technology.
+  # Copyright (c) 2016-2023 The Uncertainty Quantification Foundation.
+  # License: 3-clause BSD.  The full license text is available at:
+  #  - https://github.com/uqfoundation/dill/blob/master/LICENSE
+
+  # The following function is also based on 'save_codeobject' from 'cloudpickle'
+  # Copyright (c) 2012, Regents of the University of California.
+  # Copyright (c) 2009 `PiCloud, Inc. <http://www.picloud.com>`_.
+  # License: 3-clause BSD.  The full license text is available at:
+  #  - https://github.com/cloudpipe/cloudpickle/blob/master/LICENSE
+
+  from types import CodeType
+
+  @dill.register(CodeType)
+  def save_code(pickler, obj):
+    if hasattr(obj, "co_endlinetable"):  # python 3.11a (20 args)
+      args = (
+          obj.co_argcount,
+          obj.co_posonlyargcount,
+          obj.co_kwonlyargcount,
+          obj.co_nlocals,
+          obj.co_stacksize,
+          obj.co_flags,
+          obj.co_code,
+          obj.co_consts,
+          obj.co_names,
+          obj.co_varnames,
+          obj.co_filename,
+          obj.co_name,
+          obj.co_qualname,
+          obj.co_firstlineno,
+          obj.co_linetable,
+          obj.co_endlinetable,
+          obj.co_columntable,
+          obj.co_exceptiontable,
+          obj.co_freevars,
+          obj.co_cellvars)
+    elif hasattr(obj, "co_exceptiontable"):  # python 3.11 (18 args)
+      args = (
+          obj.co_argcount,
+          obj.co_posonlyargcount,
+          obj.co_kwonlyargcount,
+          obj.co_nlocals,
+          obj.co_stacksize,
+          obj.co_flags,
+          obj.co_code,
+          obj.co_consts,
+          obj.co_names,
+          obj.co_varnames,
+          obj.co_filename,
+          obj.co_name,
+          obj.co_qualname,
+          obj.co_firstlineno,
+          obj.co_linetable,
+          obj.co_exceptiontable,
+          obj.co_freevars,
+          obj.co_cellvars)
+    elif hasattr(obj, "co_linetable"):  # python 3.10 (16 args)
+      args = (
+          obj.co_argcount,
+          obj.co_posonlyargcount,
+          obj.co_kwonlyargcount,
+          obj.co_nlocals,
+          obj.co_stacksize,
+          obj.co_flags,
+          obj.co_code,
+          obj.co_consts,
+          obj.co_names,
+          obj.co_varnames,
+          obj.co_filename,
+          obj.co_name,
+          obj.co_firstlineno,
+          obj.co_linetable,
+          obj.co_freevars,
+          obj.co_cellvars)
+    elif hasattr(obj, "co_posonlyargcount"):  # python 3.8 (16 args)
+      args = (
+          obj.co_argcount,
+          obj.co_posonlyargcount,
+          obj.co_kwonlyargcount,
+          obj.co_nlocals,
+          obj.co_stacksize,
+          obj.co_flags,
+          obj.co_code,
+          obj.co_consts,
+          obj.co_names,
+          obj.co_varnames,
+          obj.co_filename,
+          obj.co_name,
+          obj.co_firstlineno,
+          obj.co_lnotab,
+          obj.co_freevars,
+          obj.co_cellvars)
+    else:  # python 3.7 (15 args)
+      args = (
+          obj.co_argcount,
+          obj.co_kwonlyargcount,
+          obj.co_nlocals,
+          obj.co_stacksize,
+          obj.co_flags,
+          obj.co_code,
+          obj.co_consts,
+          obj.co_names,
+          obj.co_varnames,
+          obj.co_filename,
+          obj.co_name,
+          obj.co_firstlineno,
+          obj.co_lnotab,
+          obj.co_freevars,
+          obj.co_cellvars)
+    pickler.save_reduce(CodeType, args, obj=obj)
+
+  dill._dill.save_code = save_code
+
 
 class _NoOpContextManager(object):
   def __enter__(self):

sdks/python/apache_beam/runners/dataflow/internal/apiclient.py

@@ -83,7 +83,7 @@
 
 _LOGGER = logging.getLogger(__name__)
 
-_PYTHON_VERSIONS_SUPPORTED_BY_DATAFLOW = ['3.7', '3.8', '3.9', '3.10']
+_PYTHON_VERSIONS_SUPPORTED_BY_DATAFLOW = ['3.7', '3.8', '3.9', '3.10', '3.11']
 
 
 class Step(object):

sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py

@@ -972,7 +972,7 @@ def test_interpreter_version_check_passes_py38(self):
 
   @mock.patch(
       'apache_beam.runners.dataflow.internal.apiclient.sys.version_info',
-      (3, 11, 0))
+      (3, 12, 0))
   @mock.patch(
       'apache_beam.runners.dataflow.internal.apiclient.'
       'beam_version.__version__',

sdks/python/container/base_image_requirements_manual.txt

@@ -32,11 +32,15 @@ cython<1
 # some versions of libraries that launch Beam pipelines, like tensorflow-transform.
 # Leaving 'future' in our containers for now prevent breaking tft users.
 future
-guppy3  # Memory profiler
+# TODO: Uncomment python version once guppy supports it.
+# https://github.com/apache/beam/issues/25984
+guppy3;python_version<"3.11"  # Memory profiler
 mmh3  # Optimizes execution of some Beam codepaths. TODO: Make it Beam's dependency.
 nltk  # Commonly used for natural language processing.
 nose==1.3.7  # For Dataflow internal testing. TODO: remove this.
-python-snappy  # Optimizes execution of some Beam codepaths.
+# TODO: Uncomment python version once python-snappy supports it.
+# https://github.com/apache/beam/issues/25985
+python-snappy;python_version<"3.11"  # Optimizes execution of some Beam codepaths.
 scipy
 scikit-learn
 tensorflow>=2.12.0;python_version>="3.8"

sdks/python/container/build.gradle

@@ -40,20 +40,23 @@ tasks.register("buildAll") {
   dependsOn ':sdks:python:container:py38:docker'
   dependsOn ':sdks:python:container:py39:docker'
   dependsOn ':sdks:python:container:py310:docker'
+  dependsOn ':sdks:python:container:py311:docker'
 }
 
 tasks.register("pushAll") {
   dependsOn ':sdks:python:container:py37:dockerPush'
   dependsOn ':sdks:python:container:py38:dockerPush'
   dependsOn ':sdks:python:container:py39:dockerPush'
-  dependsOn ':sdks:python:container:py310:dockerPush'
+  dependsOn ':sdks:python:container:py310:docker'
+  dependsOn ':sdks:python:container:py311:docker'
 }
 
 tasks.register("generatePythonRequirementsAll") {
   dependsOn ':sdks:python:container:py37:generatePythonRequirements'
   dependsOn ':sdks:python:container:py38:generatePythonRequirements'
   dependsOn ':sdks:python:container:py39:generatePythonRequirements'
   dependsOn ':sdks:python:container:py310:generatePythonRequirements'
+  dependsOn ':sdks:python:container:py311:generatePythonRequirements'
 }
 
 artifacts {