From 1502faf042dac21631e4c8980b0aec872707f1c9 Mon Sep 17 00:00:00 2001 From: Zili Ge Date: Mon, 30 Oct 2023 11:51:49 -0400 Subject: [PATCH] Update gcs_to_bigquery.py There is now the option to preserve ASCII control characters directly through the BQ load API. Add in this option --- airflow/providers/google/cloud/transfers/gcs_to_bigquery.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py b/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py index 94c233d6c5218..79ff8a4dfe285 100644 --- a/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +++ b/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py @@ -528,7 +528,7 @@ def _create_external_table(self): "skipLeadingRows", "quote", "encoding", - "preserveAsciiControlCharacters", + "preserveAsciiControlCharacters" ], "googleSheetsOptions": ["skipLeadingRows"], } @@ -654,6 +654,7 @@ def _use_existing_table(self): "nullMarker", "quote", "encoding", + "preserveAsciiControlCharacters" ], "DATASTORE_BACKUP": ["projectionFields"], "NEWLINE_DELIMITED_JSON": ["autodetect", "ignoreUnknownValues"],