From c370fe928f3a33c2c7aff297949a6a5240c1bbfc Mon Sep 17 00:00:00 2001
From: Yi Hu <yathu@google.com>
Date: Tue, 4 Oct 2022 11:10:41 -0400
Subject: [PATCH 1/6] Bump google-cloud-spanner version for py containers

---
 sdks/python/container/py37/base_image_requirements.txt | 2 +-
 sdks/python/container/py38/base_image_requirements.txt | 2 +-
 sdks/python/container/py39/base_image_requirements.txt | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/sdks/python/container/py37/base_image_requirements.txt b/sdks/python/container/py37/base_image_requirements.txt
index a4aaf62964a8..2d3dea0392c5 100644
--- a/sdks/python/container/py37/base_image_requirements.txt
+++ b/sdks/python/container/py37/base_image_requirements.txt
@@ -66,7 +66,7 @@ google-cloud-profiler==3.1.0
 google-cloud-pubsub==2.13.7
 google-cloud-pubsublite==1.5.0
 google-cloud-recommendations-ai==0.7.1
-google-cloud-spanner==1.19.3
+google-cloud-spanner==3.22.0
 google-cloud-storage==2.5.0
 google-cloud-videointelligence==1.16.3
 google-cloud-vision==1.0.2
diff --git a/sdks/python/container/py38/base_image_requirements.txt b/sdks/python/container/py38/base_image_requirements.txt
index 96904402dda6..d0b8620b0809 100644
--- a/sdks/python/container/py38/base_image_requirements.txt
+++ b/sdks/python/container/py38/base_image_requirements.txt
@@ -66,7 +66,7 @@ google-cloud-profiler==3.1.0
 google-cloud-pubsub==2.13.7
 google-cloud-pubsublite==1.5.0
 google-cloud-recommendations-ai==0.7.1
-google-cloud-spanner==1.19.3
+google-cloud-spanner==3.22.0
 google-cloud-storage==2.5.0
 google-cloud-videointelligence==1.16.3
 google-cloud-vision==1.0.2
diff --git a/sdks/python/container/py39/base_image_requirements.txt b/sdks/python/container/py39/base_image_requirements.txt
index 961eebae43eb..93996d460564 100644
--- a/sdks/python/container/py39/base_image_requirements.txt
+++ b/sdks/python/container/py39/base_image_requirements.txt
@@ -66,7 +66,7 @@ google-cloud-profiler==3.1.0
 google-cloud-pubsub==2.13.7
 google-cloud-pubsublite==1.5.0
 google-cloud-recommendations-ai==0.7.1
-google-cloud-spanner==1.19.3
+google-cloud-spanner==3.22.0
 google-cloud-storage==2.5.0
 google-cloud-videointelligence==1.16.3
 google-cloud-vision==1.0.2

From a5da89182a33a2fd0c4ff14d2f3db526f8d63697 Mon Sep 17 00:00:00 2001
From: Yi Hu <yathu@google.com>
Date: Tue, 4 Oct 2022 16:22:24 -0400
Subject: [PATCH 2/6] Pin spanner client version to 3.x

---
 CHANGES.md                                         | 3 +++
 sdks/python/container/run_generate_requirements.sh | 2 +-
 sdks/python/setup.py                               | 6 +-----
 3 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index 87d3d685eb31..92b578d16c85 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -62,6 +62,7 @@
 * Decreased TextSource CPU utilization by 2.3x (Java) ([#23193](https://github.com/apache/beam/issues/23193)).
 * Fixed bug when using SpannerIO with RuntimeValueProvider options (Java) ([#22146](https://github.com/apache/beam/issues/22146)).
 * Fixed issue for unicode rendering on WriteToBigQuery ([#10785](https://github.com/apache/beam/issues/10785))
+* Bumped google-cloud-spanner dependency version to 3.x for Python SDK ([#21198](https://github.com/apache/beam/issues/21198)).
 
 ## New Features / Improvements
 
@@ -78,6 +79,8 @@
 ## Bugfixes
 
 * Fixed X (Java/Python) ([#X](https://github.com/apache/beam/issues/X)).
+* Fixed Python cross-language JDBC IO Connector cannot read or write rows containing Numeric/Decimal type values ([#19817](https://github.com/apache/beam/issues/19817)).
+
 ## Known Issues
 
 * ([#X](https://github.com/apache/beam/issues/X)).
diff --git a/sdks/python/container/run_generate_requirements.sh b/sdks/python/container/run_generate_requirements.sh
index cbcb0ada7865..55173bb8cd93 100755
--- a/sdks/python/container/run_generate_requirements.sh
+++ b/sdks/python/container/run_generate_requirements.sh
@@ -23,7 +23,7 @@
 # 2) A list of dependencies from base_image_requirements_manual.txt, which we
 # maintain manually.
 
-# It is recommended to run this script via gralde commands such as:
+# It is recommended to run this script via gradle commands such as:
 # ./gradlew :sdks:python:container:generatePythonRequirementsAll
 # ./gradlew :sdks:python:container:py38:generatePythonRequirements
 
diff --git a/sdks/python/setup.py b/sdks/python/setup.py
index 238488c5d603..feb08fa80101 100644
--- a/sdks/python/setup.py
+++ b/sdks/python/setup.py
@@ -285,9 +285,6 @@ def get_portability_package_data():
           'gcp': [
             'cachetools>=3.1.0,<5',
             'google-apitools>=0.5.31,<0.5.32',
-            # Transitive dep. Required for google-cloud-spanner v1.
-            # See: https://github.com/apache/beam/issues/22454
-            'google-api-core!=2.8.2,<3',
             # NOTE: Maintainers, please do not require google-auth>=2.x.x
             # Until this issue is closed
             # https://github.com/googleapis/google-cloud-python/issues/10566
@@ -301,8 +298,7 @@ def get_portability_package_data():
             'google-cloud-bigquery-storage>=2.6.3,<2.14',
             'google-cloud-core>=0.28.1,<3',
             'google-cloud-bigtable>=0.31.1,<2',
-            # google-cloud-spanner 2.x causes dependency parsing backoff
-            'google-cloud-spanner>=1.13.0,!=2,<4',
+            'google-cloud-spanner>=3.0.0,<4',
             'grpcio-gcp>=0.2.2,<1',
             # GCP Packages required by ML functionality
             'google-cloud-dlp>=3.0.0,<4',

From e70160873bb6dbd10699646e94629dcbaab03e9c Mon Sep 17 00:00:00 2001
From: Yi Hu <yathu@google.com>
Date: Tue, 4 Oct 2022 17:36:50 -0400
Subject: [PATCH 3/6] regenerate base_image_requirements

---
 .../py37/base_image_requirements.txt          | 21 ++++++++++---------
 .../py38/base_image_requirements.txt          | 21 ++++++++++---------
 .../py39/base_image_requirements.txt          | 21 ++++++++++---------
 3 files changed, 33 insertions(+), 30 deletions(-)

diff --git a/sdks/python/container/py37/base_image_requirements.txt b/sdks/python/container/py37/base_image_requirements.txt
index 2d3dea0392c5..be0bf21cae59 100644
--- a/sdks/python/container/py37/base_image_requirements.txt
+++ b/sdks/python/container/py37/base_image_requirements.txt
@@ -28,7 +28,7 @@ beautifulsoup4==4.11.1
 bs4==0.0.1
 CacheControl==0.12.11
 cachetools==4.2.4
-certifi==2022.9.14
+certifi==2022.9.24
 cffi==1.15.1
 charset-normalizer==2.1.1
 click==8.1.3
@@ -44,17 +44,17 @@ execnet==1.9.0
 fastavro==1.6.1
 fasteners==0.18
 firebase-admin==5.2.0
-flatbuffers==2.0.7
+flatbuffers==22.9.24
 freezegun==1.2.2
 future==0.18.2
 gast==0.4.0
 google-api-core==2.8.2
-google-api-python-client==2.62.0
+google-api-python-client==2.64.0
 google-apitools==0.5.31
-google-auth==2.11.1
+google-auth==2.12.0
 google-auth-httplib2==0.1.0
 google-auth-oauthlib==0.4.6
-google-cloud-bigquery==3.3.2
+google-cloud-bigquery==3.3.3
 google-cloud-bigquery-storage==2.13.2
 google-cloud-bigtable==1.7.2
 google-cloud-core==2.3.2
@@ -73,7 +73,7 @@ google-cloud-vision==1.0.2
 google-crc32c==1.5.0
 google-pasta==0.2.0
 google-python-cloud-debugger==3.1
-google-resumable-media==2.3.3
+google-resumable-media==2.4.0
 googleapis-common-protos==1.56.4
 greenlet==1.1.3
 grpc-google-iam-v1==0.12.4
@@ -85,7 +85,7 @@ h5py==3.7.0
 hdfs==2.7.0
 httplib2==0.20.4
 idna==3.4
-importlib-metadata==4.12.0
+importlib-metadata==5.0.0
 iniconfig==1.1.1
 joblib==1.2.0
 keras==2.10.0
@@ -111,7 +111,7 @@ parameterized==0.8.1
 pbr==5.10.0
 pluggy==1.0.0
 proto-plus==1.22.1
-protobuf==3.19.5
+protobuf==3.19.6
 psycopg2-binary==2.9.3
 py==1.11.0
 pyarrow==9.0.0
@@ -129,7 +129,7 @@ pytest-timeout==2.1.0
 pytest-xdist==2.5.0
 python-dateutil==2.8.2
 python-snappy==0.6.1
-pytz==2022.2.1
+pytz==2022.4
 PyYAML==6.0
 regex==2022.9.13
 requests==2.28.1
@@ -141,8 +141,9 @@ scipy==1.7.3
 six==1.16.0
 soupsieve==2.3.2.post1
 SQLAlchemy==1.4.41
+sqlparse==0.4.3
 tenacity==5.1.5
-tensorboard==2.10.0
+tensorboard==2.10.1
 tensorboard-data-server==0.6.1
 tensorboard-plugin-wit==1.8.1
 tensorflow==2.10.0
diff --git a/sdks/python/container/py38/base_image_requirements.txt b/sdks/python/container/py38/base_image_requirements.txt
index d0b8620b0809..4f54074e6e41 100644
--- a/sdks/python/container/py38/base_image_requirements.txt
+++ b/sdks/python/container/py38/base_image_requirements.txt
@@ -28,7 +28,7 @@ beautifulsoup4==4.11.1
 bs4==0.0.1
 CacheControl==0.12.11
 cachetools==4.2.4
-certifi==2022.9.14
+certifi==2022.9.24
 cffi==1.15.1
 charset-normalizer==2.1.1
 click==8.1.3
@@ -44,17 +44,17 @@ execnet==1.9.0
 fastavro==1.6.1
 fasteners==0.18
 firebase-admin==5.2.0
-flatbuffers==2.0.7
+flatbuffers==22.9.24
 freezegun==1.2.2
 future==0.18.2
 gast==0.4.0
 google-api-core==2.8.2
-google-api-python-client==2.62.0
+google-api-python-client==2.64.0
 google-apitools==0.5.31
-google-auth==2.11.1
+google-auth==2.12.0
 google-auth-httplib2==0.1.0
 google-auth-oauthlib==0.4.6
-google-cloud-bigquery==3.3.2
+google-cloud-bigquery==3.3.3
 google-cloud-bigquery-storage==2.13.2
 google-cloud-bigtable==1.7.2
 google-cloud-core==2.3.2
@@ -73,7 +73,7 @@ google-cloud-vision==1.0.2
 google-crc32c==1.5.0
 google-pasta==0.2.0
 google-python-cloud-debugger==3.1
-google-resumable-media==2.3.3
+google-resumable-media==2.4.0
 googleapis-common-protos==1.56.4
 greenlet==1.1.3
 grpc-google-iam-v1==0.12.4
@@ -85,7 +85,7 @@ h5py==3.7.0
 hdfs==2.7.0
 httplib2==0.20.4
 idna==3.4
-importlib-metadata==4.12.0
+importlib-metadata==5.0.0
 iniconfig==1.1.1
 joblib==1.2.0
 keras==2.10.0
@@ -111,7 +111,7 @@ parameterized==0.8.1
 pbr==5.10.0
 pluggy==1.0.0
 proto-plus==1.22.1
-protobuf==3.19.5
+protobuf==3.19.6
 psycopg2-binary==2.9.3
 py==1.11.0
 pyarrow==9.0.0
@@ -129,7 +129,7 @@ pytest-timeout==2.1.0
 pytest-xdist==2.5.0
 python-dateutil==2.8.2
 python-snappy==0.6.1
-pytz==2022.2.1
+pytz==2022.4
 PyYAML==6.0
 regex==2022.9.13
 requests==2.28.1
@@ -141,8 +141,9 @@ scipy==1.9.1
 six==1.16.0
 soupsieve==2.3.2.post1
 SQLAlchemy==1.4.41
+sqlparse==0.4.3
 tenacity==5.1.5
-tensorboard==2.10.0
+tensorboard==2.10.1
 tensorboard-data-server==0.6.1
 tensorboard-plugin-wit==1.8.1
 tensorflow==2.10.0
diff --git a/sdks/python/container/py39/base_image_requirements.txt b/sdks/python/container/py39/base_image_requirements.txt
index 93996d460564..f7e27d2599ef 100644
--- a/sdks/python/container/py39/base_image_requirements.txt
+++ b/sdks/python/container/py39/base_image_requirements.txt
@@ -28,7 +28,7 @@ beautifulsoup4==4.11.1
 bs4==0.0.1
 CacheControl==0.12.11
 cachetools==4.2.4
-certifi==2022.9.14
+certifi==2022.9.24
 cffi==1.15.1
 charset-normalizer==2.1.1
 click==8.1.3
@@ -44,17 +44,17 @@ execnet==1.9.0
 fastavro==1.6.1
 fasteners==0.18
 firebase-admin==5.2.0
-flatbuffers==2.0.7
+flatbuffers==22.9.24
 freezegun==1.2.2
 future==0.18.2
 gast==0.4.0
 google-api-core==2.8.2
-google-api-python-client==2.62.0
+google-api-python-client==2.64.0
 google-apitools==0.5.31
-google-auth==2.11.1
+google-auth==2.12.0
 google-auth-httplib2==0.1.0
 google-auth-oauthlib==0.4.6
-google-cloud-bigquery==3.3.2
+google-cloud-bigquery==3.3.3
 google-cloud-bigquery-storage==2.13.2
 google-cloud-bigtable==1.7.2
 google-cloud-core==2.3.2
@@ -73,7 +73,7 @@ google-cloud-vision==1.0.2
 google-crc32c==1.5.0
 google-pasta==0.2.0
 google-python-cloud-debugger==3.1
-google-resumable-media==2.3.3
+google-resumable-media==2.4.0
 googleapis-common-protos==1.56.4
 greenlet==1.1.3
 grpc-google-iam-v1==0.12.4
@@ -85,7 +85,7 @@ h5py==3.7.0
 hdfs==2.7.0
 httplib2==0.20.4
 idna==3.4
-importlib-metadata==4.12.0
+importlib-metadata==5.0.0
 iniconfig==1.1.1
 joblib==1.2.0
 keras==2.10.0
@@ -111,7 +111,7 @@ parameterized==0.8.1
 pbr==5.10.0
 pluggy==1.0.0
 proto-plus==1.22.1
-protobuf==3.19.5
+protobuf==3.19.6
 psycopg2-binary==2.9.3
 py==1.11.0
 pyarrow==9.0.0
@@ -129,7 +129,7 @@ pytest-timeout==2.1.0
 pytest-xdist==2.5.0
 python-dateutil==2.8.2
 python-snappy==0.6.1
-pytz==2022.2.1
+pytz==2022.4
 PyYAML==6.0
 regex==2022.9.13
 requests==2.28.1
@@ -141,8 +141,9 @@ scipy==1.9.1
 six==1.16.0
 soupsieve==2.3.2.post1
 SQLAlchemy==1.4.41
+sqlparse==0.4.3
 tenacity==5.1.5
-tensorboard==2.10.0
+tensorboard==2.10.1
 tensorboard-data-server==0.6.1
 tensorboard-plugin-wit==1.8.1
 tensorflow==2.10.0

From 71fb18739affca17ac901c045a586b0f660856c5 Mon Sep 17 00:00:00 2001
From: Yi Hu <yathu@google.com>
Date: Tue, 4 Oct 2022 17:46:03 -0400
Subject: [PATCH 4/6] Remove grpcio-gcp dependency

---
 sdks/python/container/py37/base_image_requirements.txt | 1 -
 sdks/python/container/py38/base_image_requirements.txt | 1 -
 sdks/python/container/py39/base_image_requirements.txt | 1 -
 sdks/python/setup.py                                   | 1 -
 4 files changed, 4 deletions(-)

diff --git a/sdks/python/container/py37/base_image_requirements.txt b/sdks/python/container/py37/base_image_requirements.txt
index be0bf21cae59..af778d61bdd1 100644
--- a/sdks/python/container/py37/base_image_requirements.txt
+++ b/sdks/python/container/py37/base_image_requirements.txt
@@ -78,7 +78,6 @@ googleapis-common-protos==1.56.4
 greenlet==1.1.3
 grpc-google-iam-v1==0.12.4
 grpcio==1.49.1
-grpcio-gcp==0.2.2
 grpcio-status==1.48.2
 guppy3==3.1.2
 h5py==3.7.0
diff --git a/sdks/python/container/py38/base_image_requirements.txt b/sdks/python/container/py38/base_image_requirements.txt
index 4f54074e6e41..bc5b1f5fba77 100644
--- a/sdks/python/container/py38/base_image_requirements.txt
+++ b/sdks/python/container/py38/base_image_requirements.txt
@@ -78,7 +78,6 @@ googleapis-common-protos==1.56.4
 greenlet==1.1.3
 grpc-google-iam-v1==0.12.4
 grpcio==1.49.1
-grpcio-gcp==0.2.2
 grpcio-status==1.48.2
 guppy3==3.1.2
 h5py==3.7.0
diff --git a/sdks/python/container/py39/base_image_requirements.txt b/sdks/python/container/py39/base_image_requirements.txt
index f7e27d2599ef..58e06d472967 100644
--- a/sdks/python/container/py39/base_image_requirements.txt
+++ b/sdks/python/container/py39/base_image_requirements.txt
@@ -78,7 +78,6 @@ googleapis-common-protos==1.56.4
 greenlet==1.1.3
 grpc-google-iam-v1==0.12.4
 grpcio==1.49.1
-grpcio-gcp==0.2.2
 grpcio-status==1.48.2
 guppy3==3.1.2
 h5py==3.7.0
diff --git a/sdks/python/setup.py b/sdks/python/setup.py
index feb08fa80101..dc4dde575287 100644
--- a/sdks/python/setup.py
+++ b/sdks/python/setup.py
@@ -299,7 +299,6 @@ def get_portability_package_data():
             'google-cloud-core>=0.28.1,<3',
             'google-cloud-bigtable>=0.31.1,<2',
             'google-cloud-spanner>=3.0.0,<4',
-            'grpcio-gcp>=0.2.2,<1',
             # GCP Packages required by ML functionality
             'google-cloud-dlp>=3.0.0,<4',
             'google-cloud-language>=1.3.0,<2',

From a3158eacc230328dcecd3286feeafe54aae268ed Mon Sep 17 00:00:00 2001
From: Yi Hu <yathu@google.com>
Date: Tue, 4 Oct 2022 21:28:17 -0400
Subject: [PATCH 5/6] [Test only] only run spanner test for postcommit

---
 build.gradle.kts | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/build.gradle.kts b/build.gradle.kts
index 40122094e023..4ec819693ae7 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -335,16 +335,16 @@ tasks.register("pythonFormatterPreCommit") {
 }
 
 tasks.register("python37PostCommit") {
-  dependsOn(":sdks:python:test-suites:dataflow:py37:postCommitIT")
-  dependsOn(":sdks:python:test-suites:direct:py37:postCommitIT")
-  dependsOn(":sdks:python:test-suites:direct:py37:directRunnerIT")
-  dependsOn(":sdks:python:test-suites:direct:py37:hdfsIntegrationTest")
-  dependsOn(":sdks:python:test-suites:direct:py37:mongodbioIT")
-  dependsOn(":sdks:python:test-suites:portable:py37:postCommitPy37")
+  //dependsOn(":sdks:python:test-suites:dataflow:py37:postCommitIT")
+  //dependsOn(":sdks:python:test-suites:direct:py37:postCommitIT")
+  //dependsOn(":sdks:python:test-suites:direct:py37:directRunnerIT")
+  //dependsOn(":sdks:python:test-suites:direct:py37:hdfsIntegrationTest")
+  //dependsOn(":sdks:python:test-suites:direct:py37:mongodbioIT")
+  //dependsOn(":sdks:python:test-suites:portable:py37:postCommitPy37")
   dependsOn(":sdks:python:test-suites:dataflow:py37:spannerioIT")
   dependsOn(":sdks:python:test-suites:direct:py37:spannerioIT")
-  dependsOn(":sdks:python:test-suites:portable:py37:xlangSpannerIOIT")
-  dependsOn(":sdks:python:test-suites:direct:py37:inferencePostCommitIT")
+  //dependsOn(":sdks:python:test-suites:portable:py37:xlangSpannerIOIT")
+  //dependsOn(":sdks:python:test-suites:direct:py37:inferencePostCommitIT")
 }
 
 tasks.register("python38PostCommit") {

From 2cd1c31f286dd3d5a45edd9c905f822642a21be6 Mon Sep 17 00:00:00 2001
From: Yi Hu <yathu@google.com>
Date: Tue, 4 Oct 2022 22:09:47 -0400
Subject: [PATCH 6/6] Revert "[Test only] only run spanner test for postcommit"

This reverts commit a3158eacc230328dcecd3286feeafe54aae268ed.
---
 build.gradle.kts | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/build.gradle.kts b/build.gradle.kts
index 4ec819693ae7..40122094e023 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -335,16 +335,16 @@ tasks.register("pythonFormatterPreCommit") {
 }
 
 tasks.register("python37PostCommit") {
-  //dependsOn(":sdks:python:test-suites:dataflow:py37:postCommitIT")
-  //dependsOn(":sdks:python:test-suites:direct:py37:postCommitIT")
-  //dependsOn(":sdks:python:test-suites:direct:py37:directRunnerIT")
-  //dependsOn(":sdks:python:test-suites:direct:py37:hdfsIntegrationTest")
-  //dependsOn(":sdks:python:test-suites:direct:py37:mongodbioIT")
-  //dependsOn(":sdks:python:test-suites:portable:py37:postCommitPy37")
+  dependsOn(":sdks:python:test-suites:dataflow:py37:postCommitIT")
+  dependsOn(":sdks:python:test-suites:direct:py37:postCommitIT")
+  dependsOn(":sdks:python:test-suites:direct:py37:directRunnerIT")
+  dependsOn(":sdks:python:test-suites:direct:py37:hdfsIntegrationTest")
+  dependsOn(":sdks:python:test-suites:direct:py37:mongodbioIT")
+  dependsOn(":sdks:python:test-suites:portable:py37:postCommitPy37")
   dependsOn(":sdks:python:test-suites:dataflow:py37:spannerioIT")
   dependsOn(":sdks:python:test-suites:direct:py37:spannerioIT")
-  //dependsOn(":sdks:python:test-suites:portable:py37:xlangSpannerIOIT")
-  //dependsOn(":sdks:python:test-suites:direct:py37:inferencePostCommitIT")
+  dependsOn(":sdks:python:test-suites:portable:py37:xlangSpannerIOIT")
+  dependsOn(":sdks:python:test-suites:direct:py37:inferencePostCommitIT")
 }
 
 tasks.register("python38PostCommit") {