From d9209d9c876449a6ebe7d4d15a0bd3057fe3ca57 Mon Sep 17 00:00:00 2001 From: Tarun Annapareddy Date: Fri, 26 Sep 2025 13:50:43 -0700 Subject: [PATCH 1/4] Fix BigQuery Integration tests --- .../apache_beam/yaml/extended_tests/data/enrichment.yaml | 6 +++++- sdks/python/apache_beam/yaml/integration_tests.py | 5 +++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/sdks/python/apache_beam/yaml/extended_tests/data/enrichment.yaml b/sdks/python/apache_beam/yaml/extended_tests/data/enrichment.yaml index 6469c094b8b4..b1db7ba5110e 100644 --- a/sdks/python/apache_beam/yaml/extended_tests/data/enrichment.yaml +++ b/sdks/python/apache_beam/yaml/extended_tests/data/enrichment.yaml @@ -40,6 +40,9 @@ pipelines: - type: WriteToBigQuery config: table: "{BQ_TABLE}" + options: + project: "apache-beam-testing" + temp_location: "TEMP_DIR" - pipeline: type: chain @@ -81,4 +84,5 @@ pipelines: elements: - {label: '37a', rank: 1, name: 'S2'} options: - yaml_experimental_features: [ 'Enrichment' ] \ No newline at end of file + yaml_experimental_features: [ 'Enrichment' ] + temp_location: "gs://temp-storage-for-end-to-end-tests/temp-it" diff --git a/sdks/python/apache_beam/yaml/integration_tests.py b/sdks/python/apache_beam/yaml/integration_tests.py index 733dd10d0286..5c73be511a41 100644 --- a/sdks/python/apache_beam/yaml/integration_tests.py +++ b/sdks/python/apache_beam/yaml/integration_tests.py @@ -721,8 +721,9 @@ def test(self, providers=providers): # default arg to capture loop value for pipeline_spec in spec['pipelines']: with beam.Pipeline(options=PipelineOptions( pickle_library='cloudpickle', - **yaml_transform.SafeLineLoader.strip_metadata(pipeline_spec.get( - 'options', {})))) as p: + **replace_recursive( + yaml_transform.SafeLineLoader.strip_metadata(pipeline_spec.get( + 'options', {})), vars))) as p: yaml_transform.expand_pipeline( p, replace_recursive(pipeline_spec, vars)) From b9ba54d20b8aff0c9b6c85729202faa1a82e1fc6 Mon Sep 17 00:00:00 2001 From: Tarun Annapareddy Date: Fri, 26 Sep 2025 13:53:58 -0700 Subject: [PATCH 2/4] Fix location --- .../python/apache_beam/yaml/extended_tests/data/enrichment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/yaml/extended_tests/data/enrichment.yaml b/sdks/python/apache_beam/yaml/extended_tests/data/enrichment.yaml index b1db7ba5110e..4cebb990dddb 100644 --- a/sdks/python/apache_beam/yaml/extended_tests/data/enrichment.yaml +++ b/sdks/python/apache_beam/yaml/extended_tests/data/enrichment.yaml @@ -42,7 +42,7 @@ pipelines: table: "{BQ_TABLE}" options: project: "apache-beam-testing" - temp_location: "TEMP_DIR" + temp_location: "{TEMP_DIR}" - pipeline: type: chain From 8bde4429a3c785b37bfb86ee9e9998810d4b57a0 Mon Sep 17 00:00:00 2001 From: Tarun Annapareddy Date: Fri, 26 Sep 2025 13:58:19 -0700 Subject: [PATCH 3/4] fix location --- .../apache_beam/yaml/extended_tests/data/enrichment.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/yaml/extended_tests/data/enrichment.yaml b/sdks/python/apache_beam/yaml/extended_tests/data/enrichment.yaml index 4cebb990dddb..c191b2c1fc9c 100644 --- a/sdks/python/apache_beam/yaml/extended_tests/data/enrichment.yaml +++ b/sdks/python/apache_beam/yaml/extended_tests/data/enrichment.yaml @@ -84,5 +84,4 @@ pipelines: elements: - {label: '37a', rank: 1, name: 'S2'} options: - yaml_experimental_features: [ 'Enrichment' ] - temp_location: "gs://temp-storage-for-end-to-end-tests/temp-it" + yaml_experimental_features: [ 'Enrichment' ] \ No newline at end of file From 08e76a0ca12b65570655280804368887f0a2d155 Mon Sep 17 00:00:00 2001 From: Tarun Annapareddy Date: Fri, 26 Sep 2025 14:15:37 -0700 Subject: [PATCH 4/4] fix formatting --- sdks/python/apache_beam/yaml/integration_tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdks/python/apache_beam/yaml/integration_tests.py b/sdks/python/apache_beam/yaml/integration_tests.py index 5c73be511a41..0e25a7fd224c 100644 --- a/sdks/python/apache_beam/yaml/integration_tests.py +++ b/sdks/python/apache_beam/yaml/integration_tests.py @@ -721,9 +721,9 @@ def test(self, providers=providers): # default arg to capture loop value for pipeline_spec in spec['pipelines']: with beam.Pipeline(options=PipelineOptions( pickle_library='cloudpickle', - **replace_recursive( - yaml_transform.SafeLineLoader.strip_metadata(pipeline_spec.get( - 'options', {})), vars))) as p: + **replace_recursive(yaml_transform.SafeLineLoader.strip_metadata( + pipeline_spec.get('options', {})), + vars))) as p: yaml_transform.expand_pipeline( p, replace_recursive(pipeline_spec, vars))