From 6c8fdf1ea39feaec361934d907b53281124cca2d Mon Sep 17 00:00:00 2001 From: Bhargavkonidena Date: Fri, 23 May 2025 14:07:09 +0530 Subject: [PATCH 1/3] Fix for issue 34772 to include user provided providers --- .../python/apache_beam/yaml/generate_yaml_docs.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/yaml/generate_yaml_docs.py b/sdks/python/apache_beam/yaml/generate_yaml_docs.py index 592177dc2bc0..aae3b4f82acc 100644 --- a/sdks/python/apache_beam/yaml/generate_yaml_docs.py +++ b/sdks/python/apache_beam/yaml/generate_yaml_docs.py @@ -32,6 +32,7 @@ from apache_beam.version import __version__ as beam_version from apache_beam.yaml import json_utils from apache_beam.yaml import yaml_provider +from apache_beam.yaml import yaml_utils from apache_beam.yaml.yaml_errors import ErrorHandlingConfig @@ -263,7 +264,14 @@ def create_index(include, exclude, options): with subprocess_server.SubprocessServer.cache_subprocesses(): json_config_schemas = [] markdown_out = io.StringIO() - providers = yaml_provider.standard_providers() + if options.provider_source_file: + providers = yaml_provider.merge_providers( + yaml_provider.load_providers( + yaml_utils.locate_data_file(options.provider_source_file) + ) + ) + else: + providers = yaml_provider.standard_providers() for transform_base, transforms in itertools.groupby( sorted(providers.keys(), key=io_grouping_key), key=lambda s: s.split('-')[0]): @@ -557,6 +565,11 @@ def main(): parser.add_argument('--schema_file') parser.add_argument('--include', default='.*') parser.add_argument('--exclude', default='') + parser.add_argument( + "--provider-source-file", + help="Path to a YAML file containing custom providers. " + "If not provided, uses standard Beam providers.", + ) options = parser.parse_args() include = re.compile(options.include).match exclude = ( From 6cf0bd50f170be2987f62695a3715e73a3daf1cb Mon Sep 17 00:00:00 2001 From: Bhargavkonidena Date: Sun, 25 May 2025 13:25:12 +0530 Subject: [PATCH 2/3] Fix CI checks --- sdks/python/apache_beam/yaml/generate_yaml_docs.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/sdks/python/apache_beam/yaml/generate_yaml_docs.py b/sdks/python/apache_beam/yaml/generate_yaml_docs.py index aae3b4f82acc..1a838308f614 100644 --- a/sdks/python/apache_beam/yaml/generate_yaml_docs.py +++ b/sdks/python/apache_beam/yaml/generate_yaml_docs.py @@ -266,10 +266,8 @@ def create_index(include, exclude, options): markdown_out = io.StringIO() if options.provider_source_file: providers = yaml_provider.merge_providers( - yaml_provider.load_providers( - yaml_utils.locate_data_file(options.provider_source_file) - ) - ) + yaml_provider.load_providers( + yaml_utils.locate_data_file(options.provider_source_file))) else: providers = yaml_provider.standard_providers() for transform_base, transforms in itertools.groupby( @@ -566,9 +564,9 @@ def main(): parser.add_argument('--include', default='.*') parser.add_argument('--exclude', default='') parser.add_argument( - "--provider-source-file", - help="Path to a YAML file containing custom providers. " - "If not provided, uses standard Beam providers.", + "--provider-source-file", + help="Path to a YAML file containing custom providers. " + "If not provided, uses standard Beam providers.", ) options = parser.parse_args() include = re.compile(options.include).match From 1267810bc2630dcb2290c95d363c7af1ad7591d2 Mon Sep 17 00:00:00 2001 From: Bhargavkonidena Date: Sat, 31 May 2025 09:12:16 +0530 Subject: [PATCH 3/3] Incorporate review comments --- sdks/python/apache_beam/yaml/generate_yaml_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/yaml/generate_yaml_docs.py b/sdks/python/apache_beam/yaml/generate_yaml_docs.py index 1a838308f614..ae1a04661e62 100644 --- a/sdks/python/apache_beam/yaml/generate_yaml_docs.py +++ b/sdks/python/apache_beam/yaml/generate_yaml_docs.py @@ -564,7 +564,7 @@ def main(): parser.add_argument('--include', default='.*') parser.add_argument('--exclude', default='') parser.add_argument( - "--provider-source-file", + "--provider_source_file", help="Path to a YAML file containing custom providers. " "If not provided, uses standard Beam providers.", )