From 8ee721dccf5d4908023a19ab169aa4c6aa492673 Mon Sep 17 00:00:00 2001 From: Zili Ge Date: Mon, 30 Oct 2023 11:51:49 -0400 Subject: [PATCH 1/2] Update gcs_to_bigquery.py There is now the option to preserve ASCII control characters directly through the BQ load API. Add in this option --- airflow/providers/google/cloud/transfers/gcs_to_bigquery.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py b/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py index 798bc8a52b408..d2cf2fdde276d 100644 --- a/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +++ b/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py @@ -531,7 +531,7 @@ def _create_external_table(self): "skipLeadingRows", "quote", "encoding", - "preserveAsciiControlCharacters", + "preserveAsciiControlCharacters" ], "googleSheetsOptions": ["skipLeadingRows"], } @@ -657,6 +657,7 @@ def _use_existing_table(self): "nullMarker", "quote", "encoding", + "preserveAsciiControlCharacters" ], "DATASTORE_BACKUP": ["projectionFields"], "NEWLINE_DELIMITED_JSON": ["autodetect", "ignoreUnknownValues"], From 9d6a2571d773b8552e16c2e12f610ffd012cc66d Mon Sep 17 00:00:00 2001 From: romsharon98 Date: Tue, 2 Jan 2024 18:11:22 +0200 Subject: [PATCH 2/2] add comma --- airflow/providers/google/cloud/transfers/gcs_to_bigquery.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py b/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py index d2cf2fdde276d..b22b7d8f8da54 100644 --- a/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +++ b/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py @@ -531,7 +531,7 @@ def _create_external_table(self): "skipLeadingRows", "quote", "encoding", - "preserveAsciiControlCharacters" + "preserveAsciiControlCharacters", ], "googleSheetsOptions": ["skipLeadingRows"], } @@ -657,7 +657,7 @@ def _use_existing_table(self): "nullMarker", "quote", "encoding", - "preserveAsciiControlCharacters" + "preserveAsciiControlCharacters", ], "DATASTORE_BACKUP": ["projectionFields"], "NEWLINE_DELIMITED_JSON": ["autodetect", "ignoreUnknownValues"],