From f4340c1854718dcdb5777cd21b6a6b7fc4593f1a Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Fri, 15 Jan 2021 12:00:12 +0000 Subject: [PATCH 001/113] docs: Add extension types to extension docs Signed-off-by: Ryan Northey --- api/envoy/config/bootstrap/v3/bootstrap.proto | 1 + api/envoy/config/listener/v3/listener_components.proto | 1 + 2 files changed, 2 insertions(+) diff --git a/api/envoy/config/bootstrap/v3/bootstrap.proto b/api/envoy/config/bootstrap/v3/bootstrap.proto index c3fee5e2dfeb6..6341c9812f408 100644 --- a/api/envoy/config/bootstrap/v3/bootstrap.proto +++ b/api/envoy/config/bootstrap/v3/bootstrap.proto @@ -250,6 +250,7 @@ message Bootstrap { // Specifies optional bootstrap extensions to be instantiated at startup time. // Each item contains extension specific configuration. + // [#protodoc-extension: Bootstrap] repeated core.v3.TypedExtensionConfig bootstrap_extensions = 21; // Specifies optional extensions instantiated at startup time and diff --git a/api/envoy/config/listener/v3/listener_components.proto b/api/envoy/config/listener/v3/listener_components.proto index c73103bfad5db..e12aef17a8655 100644 --- a/api/envoy/config/listener/v3/listener_components.proto +++ b/api/envoy/config/listener/v3/listener_components.proto @@ -39,6 +39,7 @@ message Filter { oneof config_type { // Filter specific configuration which depends on the filter being // instantiated. See the supported filters for further documentation. + // [#protodoc-extension: Filter] google.protobuf.Any typed_config = 4; // Configuration source specifier for an extension configuration discovery From 3acd6e1731f27f61b7c72e14f8b947a8121e5e67 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Fri, 15 Jan 2021 12:27:33 +0000 Subject: [PATCH 002/113] tools Signed-off-by: Ryan Northey --- tools/api_proto_plugin/annotations.py | 4 ++++ tools/protodoc/protodoc.py | 8 ++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/tools/api_proto_plugin/annotations.py b/tools/api_proto_plugin/annotations.py index 48452742125a7..d4c1b9d62bdbb 100644 --- a/tools/api_proto_plugin/annotations.py +++ b/tools/api_proto_plugin/annotations.py @@ -10,6 +10,9 @@ # Page/section titles with special prefixes in the proto comments DOC_TITLE_ANNOTATION = 'protodoc-title' +# Specify the type of extensions at extension points +EXTENSION_TYPE_ANNOTATION = 'protodoc-extension' + # When documenting an extension, this should be used to specify the qualified # name that the extension registers as in the static registry, e.g. # envoy.filters.network.http_connection_manager. @@ -32,6 +35,7 @@ VALID_ANNOTATIONS = set([ DOC_TITLE_ANNOTATION, EXTENSION_ANNOTATION, + EXTENSION_TYPE_ANNOTATION, NOT_IMPLEMENTED_HIDE_ANNOTATION, NEXT_FREE_FIELD_ANNOTATION, NEXT_MAJOR_VERSION_ANNOTATION, diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index 10faf4473fe63..210f9da176022 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -223,11 +223,15 @@ def FormatHeaderFromFile(style, source_code_info, proto_name): if annotations.EXTENSION_ANNOTATION in source_code_info.file_level_annotations: extension = source_code_info.file_level_annotations[annotations.EXTENSION_ANNOTATION] formatted_extension = FormatExtension(extension) + formatted_extension_type = '' + if annotations.EXTENSION_TYPE_ANNOTATION in source_code_info.file_level_annotations: + formatted_extension_type = "INFO ABOUT THE EXTENSION TYPE 8)" if annotations.DOC_TITLE_ANNOTATION in source_code_info.file_level_annotations: return anchor + FormatHeader( style, source_code_info.file_level_annotations[ - annotations.DOC_TITLE_ANNOTATION]) + formatted_extension, stripped_comment - return anchor + FormatHeader(style, proto_name) + formatted_extension, stripped_comment + annotations.DOC_TITLE_ANNOTATION]) + formatted_extension + formatted_extension_type, stripped_comment + + return anchor + FormatHeader(style, proto_name) + formatted_extension + formatted_extension_type, stripped_comment def FormatFieldTypeAsJson(type_context, field): From 4b49c7fd805db86b07605c2262e74b517da99b05 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Fri, 15 Jan 2021 12:27:58 +0000 Subject: [PATCH 003/113] api/ Signed-off-by: Ryan Northey --- api/envoy/config/listener/v3/listener_components.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/envoy/config/listener/v3/listener_components.proto b/api/envoy/config/listener/v3/listener_components.proto index e12aef17a8655..56e9ad00cd1c7 100644 --- a/api/envoy/config/listener/v3/listener_components.proto +++ b/api/envoy/config/listener/v3/listener_components.proto @@ -34,12 +34,12 @@ message Filter { // The name of the filter to instantiate. The name must match a // :ref:`supported filter `. + // [#protodoc-extension: Filter] string name = 1 [(validate.rules).string = {min_len: 1}]; oneof config_type { // Filter specific configuration which depends on the filter being // instantiated. See the supported filters for further documentation. - // [#protodoc-extension: Filter] google.protobuf.Any typed_config = 4; // Configuration source specifier for an extension configuration discovery From 1c04199e8d5e2460c82da783583c50cf7a30c219 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Fri, 15 Jan 2021 13:22:24 +0000 Subject: [PATCH 004/113] tools Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index 210f9da176022..4d0b487fd1ded 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -132,7 +132,10 @@ def FormatCommentWithAnnotations(comment, type_name=''): if annotations.EXTENSION_ANNOTATION in comment.annotations: extension = comment.annotations[annotations.EXTENSION_ANNOTATION] formatted_extension = FormatExtension(extension) - return annotations.WithoutAnnotations(StripLeadingSpace(comment.raw) + '\n') + formatted_extension + formatted_extension_type = '' + if annotations.EXTENSION_TYPE_ANNOTATION in source_code_info.file_level_annotations: + formatted_extension_type = "INFO ABOUT THE EXTENSION TYPE 8)" + return annotations.WithoutAnnotations(StripLeadingSpace(comment.raw) + '\n') + formatted_extension + formatted_extension_type def MapLines(f, s): From 8101ff3d2dc851fb8a1ac981276d2ff05c6ffe2a Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Fri, 15 Jan 2021 13:40:26 +0000 Subject: [PATCH 005/113] tools Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index 4d0b487fd1ded..35c76e1452176 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -133,7 +133,7 @@ def FormatCommentWithAnnotations(comment, type_name=''): extension = comment.annotations[annotations.EXTENSION_ANNOTATION] formatted_extension = FormatExtension(extension) formatted_extension_type = '' - if annotations.EXTENSION_TYPE_ANNOTATION in source_code_info.file_level_annotations: + if annotations.EXTENSION_TYPE_ANNOTATION in content.annotations: formatted_extension_type = "INFO ABOUT THE EXTENSION TYPE 8)" return annotations.WithoutAnnotations(StripLeadingSpace(comment.raw) + '\n') + formatted_extension + formatted_extension_type From 4e9082f84b8ad49bbe6f569c2422f6dcaa9fe893 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Fri, 15 Jan 2021 14:23:03 +0000 Subject: [PATCH 006/113] tools Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index 35c76e1452176..0ca8214d31aab 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -133,7 +133,7 @@ def FormatCommentWithAnnotations(comment, type_name=''): extension = comment.annotations[annotations.EXTENSION_ANNOTATION] formatted_extension = FormatExtension(extension) formatted_extension_type = '' - if annotations.EXTENSION_TYPE_ANNOTATION in content.annotations: + if annotations.EXTENSION_TYPE_ANNOTATION in comment.annotations: formatted_extension_type = "INFO ABOUT THE EXTENSION TYPE 8)" return annotations.WithoutAnnotations(StripLeadingSpace(comment.raw) + '\n') + formatted_extension + formatted_extension_type From 621d0039eb8d391e2fa29ac7e3eaadd3370d37da Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Fri, 15 Jan 2021 14:56:06 +0000 Subject: [PATCH 007/113] tools Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index 0ca8214d31aab..11ee58c239862 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -134,7 +134,7 @@ def FormatCommentWithAnnotations(comment, type_name=''): formatted_extension = FormatExtension(extension) formatted_extension_type = '' if annotations.EXTENSION_TYPE_ANNOTATION in comment.annotations: - formatted_extension_type = "INFO ABOUT THE EXTENSION TYPE 8)" + formatted_extension_type = "INFO ABOUT THE EXTENSION TYPE 8)\n\n" return annotations.WithoutAnnotations(StripLeadingSpace(comment.raw) + '\n') + formatted_extension + formatted_extension_type @@ -228,7 +228,7 @@ def FormatHeaderFromFile(style, source_code_info, proto_name): formatted_extension = FormatExtension(extension) formatted_extension_type = '' if annotations.EXTENSION_TYPE_ANNOTATION in source_code_info.file_level_annotations: - formatted_extension_type = "INFO ABOUT THE EXTENSION TYPE 8)" + formatted_extension_type = "INFO ABOUT THE EXTENSION TYPE 8)\n\n" if annotations.DOC_TITLE_ANNOTATION in source_code_info.file_level_annotations: return anchor + FormatHeader( style, source_code_info.file_level_annotations[ From 69d2cec50be3c68b2af347d94e415dc92043d552 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Mon, 18 Jan 2021 11:11:27 +0000 Subject: [PATCH 008/113] extensiondb Signed-off-by: Ryan Northey --- docs/generate_extension_db.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/generate_extension_db.py b/docs/generate_extension_db.py index 9eab52d3d6f3c..65af316dc7415 100755 --- a/docs/generate_extension_db.py +++ b/docs/generate_extension_db.py @@ -71,4 +71,8 @@ def GetExtensionMetadata(target): extension_db['envoy.upstreams.http.http_protocol_options'] = GetExtensionMetadata( '//source/extensions/upstreams/http:config') + print('EXTENSION DB') + print(extesion_db) + print() + pathlib.Path(output_path).write_text(json.dumps(extension_db)) From e81f19fa53a8f994a96a72e5857562eb3de06320 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Mon, 18 Jan 2021 12:55:25 +0000 Subject: [PATCH 009/113] extensiondb Signed-off-by: Ryan Northey --- docs/generate_extension_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/generate_extension_db.py b/docs/generate_extension_db.py index 65af316dc7415..f0d70d0df3a6c 100755 --- a/docs/generate_extension_db.py +++ b/docs/generate_extension_db.py @@ -72,7 +72,7 @@ def GetExtensionMetadata(target): '//source/extensions/upstreams/http:config') print('EXTENSION DB') - print(extesion_db) + print(extension_db) print() pathlib.Path(output_path).write_text(json.dumps(extension_db)) From 66538be25d3090c7bbfd9b6b3e7aa5221e41bc4b Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Mon, 18 Jan 2021 16:13:37 +0000 Subject: [PATCH 010/113] add category kw/arg to envoy_cc_extension Signed-off-by: Ryan Northey --- bazel/envoy_library.bzl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bazel/envoy_library.bzl b/bazel/envoy_library.bzl index ca64d5a09e000..59e4d3343bf41 100644 --- a/bazel/envoy_library.bzl +++ b/bazel/envoy_library.bzl @@ -80,6 +80,8 @@ EXTENSION_STATUS_VALUES = [ def envoy_cc_extension( name, security_posture, + # Make this mandatory once all extensions have had their cat added. + category = None, # Only set this for internal, undocumented extensions. undocumented = False, status = "stable", From fa4d5ed1b10ae39ebfc1d2a30cc7bc2e07e9c553 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Mon, 18 Jan 2021 16:21:57 +0000 Subject: [PATCH 011/113] mark extension Signed-off-by: Ryan Northey --- source/extensions/access_loggers/file/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/source/extensions/access_loggers/file/BUILD b/source/extensions/access_loggers/file/BUILD index 93e2ad5b5c614..aced900487fd8 100644 --- a/source/extensions/access_loggers/file/BUILD +++ b/source/extensions/access_loggers/file/BUILD @@ -16,6 +16,7 @@ envoy_cc_library( name = "file_access_log_lib", srcs = ["file_access_log_impl.cc"], hdrs = ["file_access_log_impl.h"], + category = "envoy.extensions.access_loggers.file", # The file based access logger is core code. visibility = ["//visibility:public"], deps = [ From dea0769fdafc4cd27ea464b2d9e01c4584254adc Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Mon, 18 Jan 2021 16:39:17 +0000 Subject: [PATCH 012/113] docs/ Signed-off-by: Ryan Northey --- docs/generate_extension_db.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/generate_extension_db.py b/docs/generate_extension_db.py index f0d70d0df3a6c..814c127d08f22 100755 --- a/docs/generate_extension_db.py +++ b/docs/generate_extension_db.py @@ -37,10 +37,10 @@ def GetExtensionMetadata(target): if not BUILDOZER_PATH: raise ExtensionDbError('Buildozer not found!') r = subprocess.run( - [BUILDOZER_PATH, '-stdout', 'print security_posture status undocumented', target], + [BUILDOZER_PATH, '-stdout', 'print security_posture status category undocumented', target], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - security_posture, status, undocumented = r.stdout.decode('utf-8').strip().split(' ') + security_posture, status, category, undocumented = r.stdout.decode('utf-8').strip().split(' ') if IsMissing(security_posture): raise ExtensionDbError( 'Missing security posture for %s. Please make sure the target is an envoy_cc_extension and security_posture is set' @@ -49,6 +49,7 @@ def GetExtensionMetadata(target): 'security_posture': security_posture, 'undocumented': False if IsMissing(undocumented) else bool(undocumented), 'status': 'stable' if IsMissing(status) else status, + 'category': 'nocategory' if IsMissing(category) else category, } From ebc27cfbbb442ccfb43d3dbaade21df7c6a4fdcb Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Mon, 18 Jan 2021 17:25:39 +0000 Subject: [PATCH 013/113] category Signed-off-by: Ryan Northey --- source/extensions/filters/network/client_ssl_auth/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/source/extensions/filters/network/client_ssl_auth/BUILD b/source/extensions/filters/network/client_ssl_auth/BUILD index d77c4abae5949..7a2f71411471d 100644 --- a/source/extensions/filters/network/client_ssl_auth/BUILD +++ b/source/extensions/filters/network/client_ssl_auth/BUILD @@ -16,6 +16,7 @@ envoy_cc_library( name = "client_ssl_auth", srcs = ["client_ssl_auth.cc"], hdrs = ["client_ssl_auth.h"], + category = "envoy.filters.network", deps = [ "//include/envoy/network:connection_interface", "//include/envoy/network:filter_interface", From 494392a93cdaefd673af44850d0363991a28c7c4 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Fri, 22 Jan 2021 13:06:57 +0000 Subject: [PATCH 014/113] docs/ Signed-off-by: Ryan Northey --- docs/generate_extension_db.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/generate_extension_db.py b/docs/generate_extension_db.py index 814c127d08f22..1a30228874271 100755 --- a/docs/generate_extension_db.py +++ b/docs/generate_extension_db.py @@ -37,10 +37,11 @@ def GetExtensionMetadata(target): if not BUILDOZER_PATH: raise ExtensionDbError('Buildozer not found!') r = subprocess.run( - [BUILDOZER_PATH, '-stdout', 'print security_posture status category undocumented', target], + [BUILDOZER_PATH, '-stdout', 'print security_posture status undocumented category', target], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - security_posture, status, category, undocumented = r.stdout.decode('utf-8').strip().split(' ') + security_posture, status, undocumented = r.stdout.decode('utf-8').strip().split(' ')[:3] + categories = r.stdout.decode('utf-8').strip().split(' ')[3:] if IsMissing(security_posture): raise ExtensionDbError( 'Missing security posture for %s. Please make sure the target is an envoy_cc_extension and security_posture is set' @@ -49,7 +50,7 @@ def GetExtensionMetadata(target): 'security_posture': security_posture, 'undocumented': False if IsMissing(undocumented) else bool(undocumented), 'status': 'stable' if IsMissing(status) else status, - 'category': 'nocategory' if IsMissing(category) else category, + 'categories': 'nocategory' if IsMissing(categories) else categories, } From b72b1a5947b955ee6f1cd20e04d514e57f6e9238 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Fri, 22 Jan 2021 13:42:33 +0000 Subject: [PATCH 015/113] docs/ Signed-off-by: Ryan Northey --- docs/build.sh | 6 ++++++ docs/generate_extension_cats_db.py | 25 +++++++++++++++++++++++++ docs/generate_extension_db.py | 4 ---- 3 files changed, 31 insertions(+), 4 deletions(-) create mode 100755 docs/generate_extension_cats_db.py diff --git a/docs/build.sh b/docs/build.sh index 7890fcf6f3ee1..5594c5f39b6a8 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -63,6 +63,8 @@ rm -rf bazel-bin/external/envoy_api_canonical EXTENSION_DB_PATH="$(realpath "${BUILD_DIR}/extension_db.json")" export EXTENSION_DB_PATH +EXTENSION_CAT_DB_PATH="$(realpath "${BUILD_DIR}/extension_cat_db.json")" + # This is for local RBE setup, should be no-op for builds without RBE setting in bazelrc files. IFS=" " read -ra BAZEL_BUILD_OPTIONS <<< "${BAZEL_BUILD_OPTIONS:-}" BAZEL_BUILD_OPTIONS+=( @@ -75,6 +77,10 @@ BAZEL_BUILD_OPTIONS+=( # metadata, based on the envoy_cc_extension() Bazel target attributes. ./docs/generate_extension_db.py "${EXTENSION_DB_PATH}" +# Generate extension category database. +./docs/generate_extension_cats_db.py "${EXTENSION_DB_PATH}" "${EXTENSION_CAT_DB_PATH}" + + # Generate RST for the lists of trusted/untrusted extensions in # intro/arch_overview/security docs. mkdir -p "${GENERATED_RST_DIR}"/intro/arch_overview/security diff --git a/docs/generate_extension_cats_db.py b/docs/generate_extension_cats_db.py new file mode 100755 index 0000000000000..c5e67c51a468b --- /dev/null +++ b/docs/generate_extension_cats_db.py @@ -0,0 +1,25 @@ + +import json + + +def main(): + extension_db = sys.argv[1] + extension_cat_db = sys.argv[2] + + with open(extension_db) as f: + data = json.load(f) + + categories = {} + for k, v in data.items: + for cat in v['categories']: + # not sure if this needs to be ordered + categories[cat] = categories.get(cat, set()) + categories[cat].add(k) + + print(categories) + + with open(extension_cat_db, 'w') as f: + json.dump(f) + +if __name__ == '__main__': + main() diff --git a/docs/generate_extension_db.py b/docs/generate_extension_db.py index 1a30228874271..d269476f86645 100755 --- a/docs/generate_extension_db.py +++ b/docs/generate_extension_db.py @@ -73,8 +73,4 @@ def GetExtensionMetadata(target): extension_db['envoy.upstreams.http.http_protocol_options'] = GetExtensionMetadata( '//source/extensions/upstreams/http:config') - print('EXTENSION DB') - print(extension_db) - print() - pathlib.Path(output_path).write_text(json.dumps(extension_db)) From 9218861477daf71e55d66c4a97e6398a46e08fde Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Fri, 22 Jan 2021 13:48:37 +0000 Subject: [PATCH 016/113] docs/ Signed-off-by: Ryan Northey --- docs/generate_extension_cats_db.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/generate_extension_cats_db.py b/docs/generate_extension_cats_db.py index c5e67c51a468b..94587a5f0e914 100755 --- a/docs/generate_extension_cats_db.py +++ b/docs/generate_extension_cats_db.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 import json From 643a9b01503d8f7aa1e054649b71358e4672d260 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Fri, 22 Jan 2021 13:59:05 +0000 Subject: [PATCH 017/113] docs/ Signed-off-by: Ryan Northey --- docs/generate_extension_cats_db.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/generate_extension_cats_db.py b/docs/generate_extension_cats_db.py index 94587a5f0e914..ddcaa3af9cf85 100755 --- a/docs/generate_extension_cats_db.py +++ b/docs/generate_extension_cats_db.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import json +import sys def main(): From 3dd09911d56808e2f70c3ec67118901fbfe6b43b Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Fri, 22 Jan 2021 14:22:28 +0000 Subject: [PATCH 018/113] docs/ Signed-off-by: Ryan Northey --- docs/generate_extension_cats_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/generate_extension_cats_db.py b/docs/generate_extension_cats_db.py index ddcaa3af9cf85..4d37e35c55ae8 100755 --- a/docs/generate_extension_cats_db.py +++ b/docs/generate_extension_cats_db.py @@ -12,7 +12,7 @@ def main(): data = json.load(f) categories = {} - for k, v in data.items: + for k, v in data.items(): for cat in v['categories']: # not sure if this needs to be ordered categories[cat] = categories.get(cat, set()) From 32be692581a131365308e7250f989e5ec9900a17 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Fri, 22 Jan 2021 14:34:55 +0000 Subject: [PATCH 019/113] docs/ Signed-off-by: Ryan Northey --- docs/generate_extension_cats_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/generate_extension_cats_db.py b/docs/generate_extension_cats_db.py index 4d37e35c55ae8..90e7611e3f197 100755 --- a/docs/generate_extension_cats_db.py +++ b/docs/generate_extension_cats_db.py @@ -21,7 +21,7 @@ def main(): print(categories) with open(extension_cat_db, 'w') as f: - json.dump(f) + json.dump(f, categories) if __name__ == '__main__': main() From b20776b1d96da29fb77816481ceef752d0fcd652 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Fri, 22 Jan 2021 14:41:51 +0000 Subject: [PATCH 020/113] docs/ Signed-off-by: Ryan Northey --- docs/generate_extension_cats_db.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/generate_extension_cats_db.py b/docs/generate_extension_cats_db.py index 90e7611e3f197..a94441db90d6a 100755 --- a/docs/generate_extension_cats_db.py +++ b/docs/generate_extension_cats_db.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 - +65;6201;1c import json import sys @@ -21,7 +21,7 @@ def main(): print(categories) with open(extension_cat_db, 'w') as f: - json.dump(f, categories) + json.dump(categories, f) if __name__ == '__main__': main() From 807eff2a8e333499ec4a13648c2c4b112949f908 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Fri, 22 Jan 2021 15:16:17 +0000 Subject: [PATCH 021/113] grrmacs Signed-off-by: Ryan Northey --- docs/generate_extension_cats_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/generate_extension_cats_db.py b/docs/generate_extension_cats_db.py index a94441db90d6a..002b332176911 100755 --- a/docs/generate_extension_cats_db.py +++ b/docs/generate_extension_cats_db.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -65;6201;1c + import json import sys From 190706ab5f708519968e2409bbffc3f66fa972b6 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Fri, 22 Jan 2021 15:22:47 +0000 Subject: [PATCH 022/113] docs Signed-off-by: Ryan Northey --- docs/generate_extension_cats_db.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/generate_extension_cats_db.py b/docs/generate_extension_cats_db.py index 002b332176911..c0728a18a512c 100755 --- a/docs/generate_extension_cats_db.py +++ b/docs/generate_extension_cats_db.py @@ -14,9 +14,8 @@ def main(): categories = {} for k, v in data.items(): for cat in v['categories']: - # not sure if this needs to be ordered - categories[cat] = categories.get(cat, set()) - categories[cat].add(k) + categories[cat] = categories.get(cat, []) + categories[cat].append(k) print(categories) From 34a2cafa9e19d3fef74e2d634ce59e57e455da90 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Fri, 22 Jan 2021 15:38:57 +0000 Subject: [PATCH 023/113] source/ Signed-off-by: Ryan Northey --- source/extensions/access_loggers/file/BUILD | 2 +- source/extensions/access_loggers/grpc/BUILD | 1 + source/extensions/quic_listeners/quiche/BUILD | 5 +++++ source/extensions/transport_sockets/alts/BUILD | 4 ++++ 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/source/extensions/access_loggers/file/BUILD b/source/extensions/access_loggers/file/BUILD index aced900487fd8..36a550e33bb91 100644 --- a/source/extensions/access_loggers/file/BUILD +++ b/source/extensions/access_loggers/file/BUILD @@ -16,7 +16,6 @@ envoy_cc_library( name = "file_access_log_lib", srcs = ["file_access_log_impl.cc"], hdrs = ["file_access_log_impl.h"], - category = "envoy.extensions.access_loggers.file", # The file based access logger is core code. visibility = ["//visibility:public"], deps = [ @@ -28,6 +27,7 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], + category = "envoy.access_loggers", # TODO(#9953) determine if this is core or should be cleaned up. extra_visibility = [ "//test:__subpackages__", diff --git a/source/extensions/access_loggers/grpc/BUILD b/source/extensions/access_loggers/grpc/BUILD index b958ccba79950..1894222eee3f8 100644 --- a/source/extensions/access_loggers/grpc/BUILD +++ b/source/extensions/access_loggers/grpc/BUILD @@ -97,6 +97,7 @@ envoy_cc_extension( name = "http_config", srcs = ["http_config.cc"], hdrs = ["http_config.h"], + category = "envoy.access_loggers", # TODO(#9953) clean up. extra_visibility = [ "//test/common/access_log:__subpackages__", diff --git a/source/extensions/quic_listeners/quiche/BUILD b/source/extensions/quic_listeners/quiche/BUILD index a90cfde6ddc66..ef15c3e71833e 100644 --- a/source/extensions/quic_listeners/quiche/BUILD +++ b/source/extensions/quic_listeners/quiche/BUILD @@ -372,6 +372,11 @@ envoy_cc_extension( security_posture = "unknown", tags = ["nofips"], + category = ( + "envoy.transport_sockets.downstream", + "envoy.transport_sockets.upstream", + ), + # QUICHE can't build against FIPS BoringSSL until the FIPS build # is on a new enough version to have QUIC support. Remove it from # the build until then. Re-enable as part of #7433. diff --git a/source/extensions/transport_sockets/alts/BUILD b/source/extensions/transport_sockets/alts/BUILD index 631c74a1c8d3f..3e9fc2c2b3c13 100644 --- a/source/extensions/transport_sockets/alts/BUILD +++ b/source/extensions/transport_sockets/alts/BUILD @@ -37,6 +37,10 @@ envoy_cc_extension( external_deps = [ "abseil_node_hash_set", ], + category = ( + "envoy.transport_sockets.downstream", + "envoy.transport_sockets.upstream", + ), security_posture = "robust_to_untrusted_downstream_and_upstream", deps = [ ":tsi_handshaker", From e47597c837b8365381fb698b4185aa63398bd4bf Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Fri, 22 Jan 2021 16:16:49 +0000 Subject: [PATCH 024/113] api/ Signed-off-by: Ryan Northey --- api/envoy/config/listener/v3/listener_components.proto | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/envoy/config/listener/v3/listener_components.proto b/api/envoy/config/listener/v3/listener_components.proto index 56e9ad00cd1c7..addb015629872 100644 --- a/api/envoy/config/listener/v3/listener_components.proto +++ b/api/envoy/config/listener/v3/listener_components.proto @@ -34,7 +34,7 @@ message Filter { // The name of the filter to instantiate. The name must match a // :ref:`supported filter `. - // [#protodoc-extension: Filter] + // [#protodoc-extension: envoy.filters.network] string name = 1 [(validate.rules).string = {min_len: 1}]; oneof config_type { @@ -333,6 +333,7 @@ message ListenerFilter { // The name of the filter to instantiate. The name must match a // :ref:`supported filter `. + // [#protodoc-extension: envoy.transport_sockets.downstream] string name = 1 [(validate.rules).string = {min_len: 1}]; // Filter specific configuration which depends on the filter being instantiated. From 1feb978e66e8a0d26972e9402a4c16d879b1fe00 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Tue, 26 Jan 2021 16:45:56 +0000 Subject: [PATCH 025/113] docs/ Signed-off-by: Ryan Northey --- docs/build.sh | 3 +-- tools/protodoc/protodoc.py | 7 ++++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/build.sh b/docs/build.sh index 5594c5f39b6a8..7dc06d0f16bad 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -61,9 +61,8 @@ pip3 install --require-hashes -r "${SCRIPT_DIR}"/requirements.txt rm -rf bazel-bin/external/envoy_api_canonical EXTENSION_DB_PATH="$(realpath "${BUILD_DIR}/extension_db.json")" -export EXTENSION_DB_PATH - EXTENSION_CAT_DB_PATH="$(realpath "${BUILD_DIR}/extension_cat_db.json")" +export EXTENSION_DB_PATH EXTENSION_CAT_DB_PATH # This is for local RBE setup, should be no-op for builds without RBE setting in bazelrc files. IFS=" " read -ra BAZEL_BUILD_OPTIONS <<< "${BAZEL_BUILD_OPTIONS:-}" diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index 11ee58c239862..a93f4e73d61db 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -92,6 +92,11 @@ 'This extension is work-in-progress. Functionality is incomplete and it is not intended for production use.', } +with open(EXTENSION_CATEGORY_DB) as f: + EXTENSION_CATEGORIES = json.load( + pathlib.Path( + os.getenv('EXTENSION_CAT_DB_PATH')).read_text()) + class ProtodocError(Exception): """Base error class for the protodoc module.""" @@ -134,7 +139,7 @@ def FormatCommentWithAnnotations(comment, type_name=''): formatted_extension = FormatExtension(extension) formatted_extension_type = '' if annotations.EXTENSION_TYPE_ANNOTATION in comment.annotations: - formatted_extension_type = "INFO ABOUT THE EXTENSION TYPE 8)\n\n" + formatted_extension_type = "INFO ABOUT THE EXTENSION TYPE %s\n\n" % comment.annotations[annotations.EXTENSION_TYPE_ANNOTATION] return annotations.WithoutAnnotations(StripLeadingSpace(comment.raw) + '\n') + formatted_extension + formatted_extension_type From b273c2b770b85ed6e313fd14db4aede2f65b559a Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Tue, 26 Jan 2021 17:10:46 +0000 Subject: [PATCH 026/113] tools/ Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index a93f4e73d61db..46b87c310ba71 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -92,10 +92,9 @@ 'This extension is work-in-progress. Functionality is incomplete and it is not intended for production use.', } -with open(EXTENSION_CATEGORY_DB) as f: - EXTENSION_CATEGORIES = json.load( - pathlib.Path( - os.getenv('EXTENSION_CAT_DB_PATH')).read_text()) +EXTENSION_CATEGORIES = json.loads( + pathlib.Path( + os.getenv('EXTENSION_CAT_DB_PATH')).read_text()) class ProtodocError(Exception): From ab31ffcbab5e0a9101bb61d44a29b01eca81a790 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Tue, 26 Jan 2021 17:59:13 +0000 Subject: [PATCH 027/113] docs/ Signed-off-by: Ryan Northey --- docs/build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/build.sh b/docs/build.sh index 7dc06d0f16bad..57b5182b82846 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -70,7 +70,8 @@ BAZEL_BUILD_OPTIONS+=( "--remote_download_outputs=all" "--strategy=protodoc=sandboxed,local" "--action_env=ENVOY_BLOB_SHA" - "--action_env=EXTENSION_DB_PATH") + "--action_env=EXTENSION_DB_PATH" + "--action_env=EXTENSION_CAT_DB_PATH") # Generate extension database. This maps from extension name to extension # metadata, based on the envoy_cc_extension() Bazel target attributes. From 6f6e1c58fa7696879c1b496ceb37631554decbc1 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Wed, 27 Jan 2021 14:51:10 +0000 Subject: [PATCH 028/113] tools/ Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index 46b87c310ba71..dcef2ae858c29 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -138,7 +138,7 @@ def FormatCommentWithAnnotations(comment, type_name=''): formatted_extension = FormatExtension(extension) formatted_extension_type = '' if annotations.EXTENSION_TYPE_ANNOTATION in comment.annotations: - formatted_extension_type = "INFO ABOUT THE EXTENSION TYPE %s\n\n" % comment.annotations[annotations.EXTENSION_TYPE_ANNOTATION] + formatted_extension_type = FormatExtensionType(comment.annotations[annotations.EXTENSION_TYPE_ANNOTATION]) return annotations.WithoutAnnotations(StripLeadingSpace(comment.raw) + '\n') + formatted_extension + formatted_extension_type @@ -211,6 +211,23 @@ def FormatExtension(extension): exit(1) # Raising the error buries the above message in tracebacks. +def FormatExtensionType(extension_type): + """Format extension metadata as RST. + + Args: + extension: the name of the extension, e.g. com.acme.foo. + + Returns: + RST formatted extension description. + """ + try: + "INFO ABOUT THE EXTENSION TYPE %s\n\n" % EXTENSION_CATEGORIES[extension_type] + except KeyError as e: + sys.stderr.write( + '\n\nUnable to find extension type: %s\n\n' % extension_type) + exit(1) # Raising the error buries the above message in tracebacks. + + def FormatHeaderFromFile(style, source_code_info, proto_name): """Format RST header based on special file level title From fb6f2e0f23507169d3063e9d8d88e78eab9e2895 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Wed, 27 Jan 2021 15:36:34 +0000 Subject: [PATCH 029/113] docs/ Signed-off-by: Ryan Northey --- docs/generate_extension_cats_db.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/generate_extension_cats_db.py b/docs/generate_extension_cats_db.py index c0728a18a512c..e46c351cc20dd 100755 --- a/docs/generate_extension_cats_db.py +++ b/docs/generate_extension_cats_db.py @@ -13,12 +13,13 @@ def main(): categories = {} for k, v in data.items(): + print(">> %s" % k) + print(v) + print() for cat in v['categories']: categories[cat] = categories.get(cat, []) categories[cat].append(k) - print(categories) - with open(extension_cat_db, 'w') as f: json.dump(categories, f) From 3797a2c25ded7a58b02c441a03878a5103cef177 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Wed, 27 Jan 2021 16:06:52 +0000 Subject: [PATCH 030/113] docs/ Signed-off-by: Ryan Northey --- docs/generate_extension_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/generate_extension_db.py b/docs/generate_extension_db.py index d269476f86645..b4e82ab2e4b18 100755 --- a/docs/generate_extension_db.py +++ b/docs/generate_extension_db.py @@ -50,7 +50,7 @@ def GetExtensionMetadata(target): 'security_posture': security_posture, 'undocumented': False if IsMissing(undocumented) else bool(undocumented), 'status': 'stable' if IsMissing(status) else status, - 'categories': 'nocategory' if IsMissing(categories) else categories, + 'categories': json.loads(categories) if categories else 'missing', } From c910d3a3223d611cd5581bc1cc4b8846fb27439c Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Wed, 27 Jan 2021 16:14:09 +0000 Subject: [PATCH 031/113] docs/ Signed-off-by: Ryan Northey --- docs/generate_extension_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/generate_extension_db.py b/docs/generate_extension_db.py index b4e82ab2e4b18..b019e00fe13e5 100755 --- a/docs/generate_extension_db.py +++ b/docs/generate_extension_db.py @@ -41,7 +41,7 @@ def GetExtensionMetadata(target): stdout=subprocess.PIPE, stderr=subprocess.PIPE) security_posture, status, undocumented = r.stdout.decode('utf-8').strip().split(' ')[:3] - categories = r.stdout.decode('utf-8').strip().split(' ')[3:] + categories = ' '.join(r.stdout.decode('utf-8').strip().split(' ')[3:]) if IsMissing(security_posture): raise ExtensionDbError( 'Missing security posture for %s. Please make sure the target is an envoy_cc_extension and security_posture is set' From 6b29f0a0dbc2a62f764c01d30546c54c1b1269cc Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Wed, 27 Jan 2021 16:18:57 +0000 Subject: [PATCH 032/113] docs/ Signed-off-by: Ryan Northey --- docs/generate_extension_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/generate_extension_db.py b/docs/generate_extension_db.py index b019e00fe13e5..44a26011c9f7e 100755 --- a/docs/generate_extension_db.py +++ b/docs/generate_extension_db.py @@ -50,7 +50,7 @@ def GetExtensionMetadata(target): 'security_posture': security_posture, 'undocumented': False if IsMissing(undocumented) else bool(undocumented), 'status': 'stable' if IsMissing(status) else status, - 'categories': json.loads(categories) if categories else 'missing', + 'categories': json.loads(categories or '["missing"]'), } From 4e79e80f05b77d478d7a4104d2963dd584447122 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Wed, 27 Jan 2021 16:23:26 +0000 Subject: [PATCH 033/113] docs/ Signed-off-by: Ryan Northey --- docs/generate_extension_db.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/generate_extension_db.py b/docs/generate_extension_db.py index 44a26011c9f7e..f4071c4c8fa38 100755 --- a/docs/generate_extension_db.py +++ b/docs/generate_extension_db.py @@ -46,6 +46,8 @@ def GetExtensionMetadata(target): raise ExtensionDbError( 'Missing security posture for %s. Please make sure the target is an envoy_cc_extension and security_posture is set' % target) + print(target) + print(categories) return { 'security_posture': security_posture, 'undocumented': False if IsMissing(undocumented) else bool(undocumented), From db1fb4c97402fb0d05e596eb33df1d12b59159a0 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Wed, 27 Jan 2021 16:36:51 +0000 Subject: [PATCH 034/113] docs/ Signed-off-by: Ryan Northey --- docs/generate_extension_db.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/generate_extension_db.py b/docs/generate_extension_db.py index f4071c4c8fa38..3e9b61eef79d8 100755 --- a/docs/generate_extension_db.py +++ b/docs/generate_extension_db.py @@ -47,12 +47,17 @@ def GetExtensionMetadata(target): 'Missing security posture for %s. Please make sure the target is an envoy_cc_extension and security_posture is set' % target) print(target) + categories = categories or '[]' + categories = ( + json.loads(categories) + if '[' in categories + else [categories]) print(categories) return { 'security_posture': security_posture, 'undocumented': False if IsMissing(undocumented) else bool(undocumented), 'status': 'stable' if IsMissing(status) else status, - 'categories': json.loads(categories or '["missing"]'), + 'categories': categories, } From 7888129d2379450889690106c0d08889aab77e7f Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Wed, 27 Jan 2021 16:48:24 +0000 Subject: [PATCH 035/113] docs/ Signed-off-by: Ryan Northey --- docs/generate_extension_db.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/generate_extension_db.py b/docs/generate_extension_db.py index 3e9b61eef79d8..c31eb6209d960 100755 --- a/docs/generate_extension_db.py +++ b/docs/generate_extension_db.py @@ -3,6 +3,7 @@ # Generate an extension database, a JSON file mapping from qualified well known # extension name to metadata derived from the envoy_cc_extension target. +import ast import json import os import pathlib @@ -48,9 +49,12 @@ def GetExtensionMetadata(target): % target) print(target) categories = categories or '[]' + # evaluate tuples/lists + # wrap strings in a list categories = ( - json.loads(categories) - if '[' in categories + ast.literal_eval(categories) + if ('[' in categories + or '(' in categories) else [categories]) print(categories) return { From 86c1135879926d90345057ec82f8b4d7434a9c4b Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Wed, 27 Jan 2021 16:53:25 +0000 Subject: [PATCH 036/113] docs/ Signed-off-by: Ryan Northey --- docs/generate_extension_db.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/generate_extension_db.py b/docs/generate_extension_db.py index c31eb6209d960..f71f07bf4f7b3 100755 --- a/docs/generate_extension_db.py +++ b/docs/generate_extension_db.py @@ -49,6 +49,7 @@ def GetExtensionMetadata(target): % target) print(target) categories = categories or '[]' + print(categories) # evaluate tuples/lists # wrap strings in a list categories = ( @@ -56,6 +57,7 @@ def GetExtensionMetadata(target): if ('[' in categories or '(' in categories) else [categories]) + print('mangled cats...') print(categories) return { 'security_posture': security_posture, From 3e19c574f133c71f59e779d63d27151f500ef692 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Wed, 27 Jan 2021 16:58:13 +0000 Subject: [PATCH 037/113] docs/ Signed-off-by: Ryan Northey --- docs/generate_extension_db.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/generate_extension_db.py b/docs/generate_extension_db.py index f71f07bf4f7b3..f3fa80b565a60 100755 --- a/docs/generate_extension_db.py +++ b/docs/generate_extension_db.py @@ -48,7 +48,10 @@ def GetExtensionMetadata(target): 'Missing security posture for %s. Please make sure the target is an envoy_cc_extension and security_posture is set' % target) print(target) - categories = categories or '[]' + categories = ( + '[]' + if IsMissing(categories) + else categories) print(categories) # evaluate tuples/lists # wrap strings in a list From 97f4ac7b763e2e66bb259914e504fec590627750 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Wed, 27 Jan 2021 17:09:18 +0000 Subject: [PATCH 038/113] source/ Signed-off-by: Ryan Northey --- source/extensions/filters/network/client_ssl_auth/BUILD | 2 +- source/extensions/transport_sockets/alts/BUILD | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/extensions/filters/network/client_ssl_auth/BUILD b/source/extensions/filters/network/client_ssl_auth/BUILD index 7a2f71411471d..184ef95404aa4 100644 --- a/source/extensions/filters/network/client_ssl_auth/BUILD +++ b/source/extensions/filters/network/client_ssl_auth/BUILD @@ -16,7 +16,6 @@ envoy_cc_library( name = "client_ssl_auth", srcs = ["client_ssl_auth.cc"], hdrs = ["client_ssl_auth.h"], - category = "envoy.filters.network", deps = [ "//include/envoy/network:connection_interface", "//include/envoy/network:filter_interface", @@ -41,6 +40,7 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], + category = "envoy.filters.network", security_posture = "robust_to_untrusted_downstream", deps = [ ":client_ssl_auth", diff --git a/source/extensions/transport_sockets/alts/BUILD b/source/extensions/transport_sockets/alts/BUILD index 3e9fc2c2b3c13..e066df38d2402 100644 --- a/source/extensions/transport_sockets/alts/BUILD +++ b/source/extensions/transport_sockets/alts/BUILD @@ -37,10 +37,10 @@ envoy_cc_extension( external_deps = [ "abseil_node_hash_set", ], - category = ( + category = [ "envoy.transport_sockets.downstream", "envoy.transport_sockets.upstream", - ), + ], security_posture = "robust_to_untrusted_downstream_and_upstream", deps = [ ":tsi_handshaker", From d8f55379a6b837f2f1615c2f7990902c85298501 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Wed, 27 Jan 2021 17:15:11 +0000 Subject: [PATCH 039/113] tools/ Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index dcef2ae858c29..1ff4619a53a36 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -248,8 +248,8 @@ def FormatHeaderFromFile(style, source_code_info, proto_name): extension = source_code_info.file_level_annotations[annotations.EXTENSION_ANNOTATION] formatted_extension = FormatExtension(extension) formatted_extension_type = '' - if annotations.EXTENSION_TYPE_ANNOTATION in source_code_info.file_level_annotations: - formatted_extension_type = "INFO ABOUT THE EXTENSION TYPE 8)\n\n" + if annotations.EXTENSION_TYPE_ANNOTATION in comment.annotations: + formatted_extension_type = FormatExtensionType(comment.annotations[annotations.EXTENSION_TYPE_ANNOTATION]) if annotations.DOC_TITLE_ANNOTATION in source_code_info.file_level_annotations: return anchor + FormatHeader( style, source_code_info.file_level_annotations[ From 18d6a9b5e83b6c6e18e92117533e261b3be158d5 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Wed, 27 Jan 2021 17:31:33 +0000 Subject: [PATCH 040/113] docs/ Signed-off-by: Ryan Northey --- docs/generate_extension_db.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/generate_extension_db.py b/docs/generate_extension_db.py index f3fa80b565a60..919c1fda04cfb 100755 --- a/docs/generate_extension_db.py +++ b/docs/generate_extension_db.py @@ -41,8 +41,9 @@ def GetExtensionMetadata(target): [BUILDOZER_PATH, '-stdout', 'print security_posture status undocumented category', target], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - security_posture, status, undocumented = r.stdout.decode('utf-8').strip().split(' ')[:3] - categories = ' '.join(r.stdout.decode('utf-8').strip().split(' ')[3:]) + rout = r.stdout.decode('utf-8').strip().split(' ') + security_posture, status, undocumented = rout[:3] + categories = ' '.join(rout[3:]) if IsMissing(security_posture): raise ExtensionDbError( 'Missing security posture for %s. Please make sure the target is an envoy_cc_extension and security_posture is set' From 56f1449aca89f040b01e538d7a3bbcb76abfaee1 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Wed, 27 Jan 2021 17:31:36 +0000 Subject: [PATCH 041/113] source/ Signed-off-by: Ryan Northey --- source/extensions/transport_sockets/alts/BUILD | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/extensions/transport_sockets/alts/BUILD b/source/extensions/transport_sockets/alts/BUILD index e066df38d2402..3e9fc2c2b3c13 100644 --- a/source/extensions/transport_sockets/alts/BUILD +++ b/source/extensions/transport_sockets/alts/BUILD @@ -37,10 +37,10 @@ envoy_cc_extension( external_deps = [ "abseil_node_hash_set", ], - category = [ + category = ( "envoy.transport_sockets.downstream", "envoy.transport_sockets.upstream", - ], + ), security_posture = "robust_to_untrusted_downstream_and_upstream", deps = [ ":tsi_handshaker", From ccc0d159a7ed9b470404bb57fb503c602877de62 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Wed, 27 Jan 2021 18:30:04 +0000 Subject: [PATCH 042/113] tools/ Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index 1ff4619a53a36..aaa3d396cc215 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -248,8 +248,9 @@ def FormatHeaderFromFile(style, source_code_info, proto_name): extension = source_code_info.file_level_annotations[annotations.EXTENSION_ANNOTATION] formatted_extension = FormatExtension(extension) formatted_extension_type = '' - if annotations.EXTENSION_TYPE_ANNOTATION in comment.annotations: - formatted_extension_type = FormatExtensionType(comment.annotations[annotations.EXTENSION_TYPE_ANNOTATION]) + if annotations.EXTENSION_TYPE_ANNOTATION in source_code_info.file_level_annotations: + formatted_extension_type = FormatExtensionType( + source_code_info.file_level_annotations[annotations.EXTENSION_TYPE_ANNOTATION]) if annotations.DOC_TITLE_ANNOTATION in source_code_info.file_level_annotations: return anchor + FormatHeader( style, source_code_info.file_level_annotations[ From ff39112c81a2fffceeb9e955358c1d229206d846 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Wed, 27 Jan 2021 19:06:39 +0000 Subject: [PATCH 043/113] tools/ Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index aaa3d396cc215..7a16da6c3ac9b 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -221,7 +221,7 @@ def FormatExtensionType(extension_type): RST formatted extension description. """ try: - "INFO ABOUT THE EXTENSION TYPE %s\n\n" % EXTENSION_CATEGORIES[extension_type] + return "INFO ABOUT THE EXTENSION TYPE %s\n\n" % EXTENSION_CATEGORIES[extension_type] except KeyError as e: sys.stderr.write( '\n\nUnable to find extension type: %s\n\n' % extension_type) From f7178d1a7d6b4581f957af90dfc2cb8e3bf568ee Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Wed, 27 Jan 2021 19:51:28 +0000 Subject: [PATCH 044/113] api/ Signed-off-by: Ryan Northey --- api/envoy/config/bootstrap/v3/bootstrap.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/envoy/config/bootstrap/v3/bootstrap.proto b/api/envoy/config/bootstrap/v3/bootstrap.proto index 6341c9812f408..61207097e4943 100644 --- a/api/envoy/config/bootstrap/v3/bootstrap.proto +++ b/api/envoy/config/bootstrap/v3/bootstrap.proto @@ -250,7 +250,7 @@ message Bootstrap { // Specifies optional bootstrap extensions to be instantiated at startup time. // Each item contains extension specific configuration. - // [#protodoc-extension: Bootstrap] + // [#protodoc-extension: envoy.bootstrap] repeated core.v3.TypedExtensionConfig bootstrap_extensions = 21; // Specifies optional extensions instantiated at startup time and From 5814c5201091c128f0fc350127ecc7f873854c04 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Wed, 27 Jan 2021 19:56:15 +0000 Subject: [PATCH 045/113] docs/ Signed-off-by: Ryan Northey --- docs/generate_extension_db.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/docs/generate_extension_db.py b/docs/generate_extension_db.py index 919c1fda04cfb..cc6bd3e71bbd5 100755 --- a/docs/generate_extension_db.py +++ b/docs/generate_extension_db.py @@ -48,21 +48,18 @@ def GetExtensionMetadata(target): raise ExtensionDbError( 'Missing security posture for %s. Please make sure the target is an envoy_cc_extension and security_posture is set' % target) - print(target) + # evaluate tuples/lists + # wrap strings in a list + # use empty list for missing (this should not be necessary) categories = ( '[]' if IsMissing(categories) else categories) - print(categories) - # evaluate tuples/lists - # wrap strings in a list categories = ( ast.literal_eval(categories) if ('[' in categories or '(' in categories) else [categories]) - print('mangled cats...') - print(categories) return { 'security_posture': security_posture, 'undocumented': False if IsMissing(undocumented) else bool(undocumented), From b6a8d3ced2d92eac72aad2c0ee78889a721f5919 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Wed, 27 Jan 2021 20:04:34 +0000 Subject: [PATCH 046/113] tools/ Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index 7a16da6c3ac9b..a1f0e623eaaaf 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -225,6 +225,8 @@ def FormatExtensionType(extension_type): except KeyError as e: sys.stderr.write( '\n\nUnable to find extension type: %s\n\n' % extension_type) + for k in EXTENSION_CATEGORIES: + sys.stderr.write("category: %s" % k) exit(1) # Raising the error buries the above message in tracebacks. From 800398a0a2cff4c9a94debeaf3b80234e8a06032 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Wed, 27 Jan 2021 20:24:49 +0000 Subject: [PATCH 047/113] source/s Signed-off-by: Ryan Northey --- source/extensions/bootstrap/wasm/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/source/extensions/bootstrap/wasm/BUILD b/source/extensions/bootstrap/wasm/BUILD index e23ac8fc84a2f..fe58c86f94c02 100644 --- a/source/extensions/bootstrap/wasm/BUILD +++ b/source/extensions/bootstrap/wasm/BUILD @@ -16,6 +16,7 @@ envoy_cc_extension( hdrs = [ "config.h", ], + category = "envoy.bootstrap", security_posture = "unknown", status = "alpha", deps = [ From 08056fad7a033429b149b8180d85c569f22ed6d3 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Wed, 27 Jan 2021 20:25:23 +0000 Subject: [PATCH 048/113] tools/ Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index a1f0e623eaaaf..7a16da6c3ac9b 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -225,8 +225,6 @@ def FormatExtensionType(extension_type): except KeyError as e: sys.stderr.write( '\n\nUnable to find extension type: %s\n\n' % extension_type) - for k in EXTENSION_CATEGORIES: - sys.stderr.write("category: %s" % k) exit(1) # Raising the error buries the above message in tracebacks. From 8ecce344f38eb58a2c536836973b2fa8f98c0b0b Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Wed, 27 Jan 2021 21:32:33 +0000 Subject: [PATCH 049/113] tools/ Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index 7a16da6c3ac9b..ba8b07052ad45 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -61,6 +61,8 @@ $security_posture + $category + """) # A map from the extension security postures (as defined in the @@ -204,7 +206,8 @@ def FormatExtension(extension): return EXTENSION_TEMPLATE.substitute(anchor=anchor, extension=extension, status=status, - security_posture=security_posture) + security_posture=security_posture, + category=extension_metadata['category']) except KeyError as e: sys.stderr.write( '\n\nDid you forget to add an entry to source/extensions/extensions_build_config.bzl?\n\n') From f1b86ae676f29b81ce4ac3de75e401470ff87f93 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Wed, 27 Jan 2021 21:52:57 +0000 Subject: [PATCH 050/113] tools/ Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index ba8b07052ad45..748c6ae7c5ac5 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -207,7 +207,8 @@ def FormatExtension(extension): extension=extension, status=status, security_posture=security_posture, - category=extension_metadata['category']) + # this shouldnt need get once all have cats... + category=extension_metadata.get('category', [])) except KeyError as e: sys.stderr.write( '\n\nDid you forget to add an entry to source/extensions/extensions_build_config.bzl?\n\n') From dd29ceb64ed89cb4d54cd9e1aecf01c3c3bd246b Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Thu, 28 Jan 2021 16:46:57 +0000 Subject: [PATCH 051/113] tools/ Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index 748c6ae7c5ac5..6019185d8cfb5 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -65,6 +65,17 @@ """) + +EXTENSION_TYPE_TEMPLATE = string.Template("""$anchor + +.. tip:: + This extension type has the following known extensions + + $extensions + +""") + + # A map from the extension security postures (as defined in the # envoy_cc_extension build macro) to human readable text for extension docs. EXTENSION_SECURITY_POSTURES = { @@ -219,13 +230,15 @@ def FormatExtensionType(extension_type): """Format extension metadata as RST. Args: - extension: the name of the extension, e.g. com.acme.foo. + extension_type: the name of the extension_type, e.g. com.acme. Returns: - RST formatted extension description. + RST formatted extension type description. """ try: - return "INFO ABOUT THE EXTENSION TYPE %s\n\n" % EXTENSION_CATEGORIES[extension_type] + anchor = FormatAnchor('extension_type_' + extension_type) + return EXTENSION_TYPE_TEMPLATE.substitute(anchor=anchor, + extensions=EXTENSION_CATEGORIES[extension_type]) except KeyError as e: sys.stderr.write( '\n\nUnable to find extension type: %s\n\n' % extension_type) From 98e0fa647ddedddd4b968c09356a8f5bf3e750b9 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Thu, 28 Jan 2021 17:24:25 +0000 Subject: [PATCH 052/113] tools/ Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index 6019185d8cfb5..98aa91108dbe9 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -61,6 +61,8 @@ $security_posture + This extension extends the following extension type: + $category """) @@ -214,6 +216,8 @@ def FormatExtension(extension): anchor = FormatAnchor('extension_' + extension) status = EXTENSION_STATUS_VALUES.get(extension_metadata['status'], '') security_posture = EXTENSION_SECURITY_POSTURES[extension_metadata['security_posture']] + print(extension) + print(extension_metadata) return EXTENSION_TEMPLATE.substitute(anchor=anchor, extension=extension, status=status, From 66faaa66317961e067ddbbb59cd3f6267fc14b5c Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Thu, 28 Jan 2021 17:52:18 +0000 Subject: [PATCH 053/113] tools/ Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index 98aa91108dbe9..8703b6b10b92b 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -63,7 +63,7 @@ This extension extends the following extension type: - $category + $categories """) @@ -216,14 +216,12 @@ def FormatExtension(extension): anchor = FormatAnchor('extension_' + extension) status = EXTENSION_STATUS_VALUES.get(extension_metadata['status'], '') security_posture = EXTENSION_SECURITY_POSTURES[extension_metadata['security_posture']] - print(extension) - print(extension_metadata) return EXTENSION_TEMPLATE.substitute(anchor=anchor, extension=extension, status=status, security_posture=security_posture, # this shouldnt need get once all have cats... - category=extension_metadata.get('category', [])) + categories=extension_metadata.get('categories', [])) except KeyError as e: sys.stderr.write( '\n\nDid you forget to add an entry to source/extensions/extensions_build_config.bzl?\n\n') From a9cfe79be47087226b7d61446bd3b5ad38abb57f Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Thu, 28 Jan 2021 18:16:09 +0000 Subject: [PATCH 054/113] tools/ Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index 8703b6b10b92b..ba0fb93af32eb 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -239,8 +239,12 @@ def FormatExtensionType(extension_type): """ try: anchor = FormatAnchor('extension_type_' + extension_type) + extensions = "\n- ".join( + ":ref:`%s `" % (ext, ext) + for ext + in EXTENSION_CATEGORIES[extension_type]) return EXTENSION_TYPE_TEMPLATE.substitute(anchor=anchor, - extensions=EXTENSION_CATEGORIES[extension_type]) + extensions=extensions) except KeyError as e: sys.stderr.write( '\n\nUnable to find extension type: %s\n\n' % extension_type) From 6c4f686515a58d32c76a7e88063cc16981b2a710 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Thu, 28 Jan 2021 18:31:45 +0000 Subject: [PATCH 055/113] tools/ Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index ba0fb93af32eb..0a6bbcad9b531 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -244,7 +244,7 @@ def FormatExtensionType(extension_type): for ext in EXTENSION_CATEGORIES[extension_type]) return EXTENSION_TYPE_TEMPLATE.substitute(anchor=anchor, - extensions=extensions) + extensions="%s\n" % extensions) except KeyError as e: sys.stderr.write( '\n\nUnable to find extension type: %s\n\n' % extension_type) From 6f2c2ae1045c616c9fbd2d3617856c32589f0cca Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Thu, 28 Jan 2021 19:29:14 +0000 Subject: [PATCH 056/113] tools/ Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index 0a6bbcad9b531..f71ff9b656199 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -239,8 +239,8 @@ def FormatExtensionType(extension_type): """ try: anchor = FormatAnchor('extension_type_' + extension_type) - extensions = "\n- ".join( - ":ref:`%s `" % (ext, ext) + extensions = "\n".join( + " - :ref:`%s `" % (ext, ext) for ext in EXTENSION_CATEGORIES[extension_type]) return EXTENSION_TYPE_TEMPLATE.substitute(anchor=anchor, From 83f48e526910ca6e18d4566b90b7a027318b26c6 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Thu, 28 Jan 2021 19:53:35 +0000 Subject: [PATCH 057/113] tools/ Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index f71ff9b656199..108325ed5afcb 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -73,8 +73,6 @@ .. tip:: This extension type has the following known extensions - $extensions - """) @@ -244,7 +242,7 @@ def FormatExtensionType(extension_type): for ext in EXTENSION_CATEGORIES[extension_type]) return EXTENSION_TYPE_TEMPLATE.substitute(anchor=anchor, - extensions="%s\n" % extensions) + extensions="%s\n\n" % extensions) except KeyError as e: sys.stderr.write( '\n\nUnable to find extension type: %s\n\n' % extension_type) From 683195e8113cf3183cf67abe75e2a0fbd5f95aa2 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Mon, 8 Feb 2021 13:51:30 +0000 Subject: [PATCH 058/113] api Signed-off-by: Ryan Northey --- api/envoy/config/cluster/v3/cluster.proto | 2 ++ .../extensions/filters/http/compressor/v3/compressor.proto | 1 + .../extensions/filters/http/decompressor/v3/decompressor.proto | 1 + 3 files changed, 4 insertions(+) diff --git a/api/envoy/config/cluster/v3/cluster.proto b/api/envoy/config/cluster/v3/cluster.proto index 145a8b6951971..14f5ccf3bd5f7 100644 --- a/api/envoy/config/cluster/v3/cluster.proto +++ b/api/envoy/config/cluster/v3/cluster.proto @@ -171,6 +171,7 @@ message Cluster { // Cluster specific configuration which depends on the cluster being instantiated. // See the supported cluster for further documentation. + // [#protodoc-extension: envoy.clusters] google.protobuf.Any typed_config = 2; } @@ -1004,6 +1005,7 @@ message Cluster { // If users desire custom connection pool or upstream behavior, for example terminating // CONNECT only if a custom filter indicates it is appropriate, the custom factories // can be registered and configured here. + // [#protodoc-extension: envoy.upstreams] core.v3.TypedExtensionConfig upstream_config = 48; // Configuration to track optional cluster stats. diff --git a/api/envoy/extensions/filters/http/compressor/v3/compressor.proto b/api/envoy/extensions/filters/http/compressor/v3/compressor.proto index 766560d9b5864..a58f246d4ffb3 100644 --- a/api/envoy/extensions/filters/http/compressor/v3/compressor.proto +++ b/api/envoy/extensions/filters/http/compressor/v3/compressor.proto @@ -96,6 +96,7 @@ message Compressor { // :ref:`envoy.compression.gzip.compressor` // is included in Envoy. // This field is ignored if used in the context of the gzip http-filter, but is mandatory otherwise. + // [#protodoc-extension: envoy.compression.compressor] config.core.v3.TypedExtensionConfig compressor_library = 6; // Configuration for request compression. Compression is disabled by default if left empty. diff --git a/api/envoy/extensions/filters/http/decompressor/v3/decompressor.proto b/api/envoy/extensions/filters/http/decompressor/v3/decompressor.proto index 058172fbaf64f..fb0d6b7172044 100644 --- a/api/envoy/extensions/filters/http/decompressor/v3/decompressor.proto +++ b/api/envoy/extensions/filters/http/decompressor/v3/decompressor.proto @@ -43,6 +43,7 @@ message Decompressor { // A decompressor library to use for both request and response decompression. Currently only // :ref:`envoy.compression.gzip.compressor` // is included in Envoy. + // [#protodoc-extension: envoy.compression.decompressor] config.core.v3.TypedExtensionConfig decompressor_library = 1 [(validate.rules).message = {required: true}]; From 3bf48f5dc80b6f773c820dfced414e1056cff29a Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Mon, 8 Feb 2021 14:32:37 +0000 Subject: [PATCH 059/113] tools/ Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index 108325ed5afcb..fcf45f6a63d70 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -73,6 +73,8 @@ .. tip:: This extension type has the following known extensions +$extensions + """) @@ -238,11 +240,11 @@ def FormatExtensionType(extension_type): try: anchor = FormatAnchor('extension_type_' + extension_type) extensions = "\n".join( - " - :ref:`%s `" % (ext, ext) + " - :ref:`%s `" % (ext, ext) for ext in EXTENSION_CATEGORIES[extension_type]) return EXTENSION_TYPE_TEMPLATE.substitute(anchor=anchor, - extensions="%s\n\n" % extensions) + extensions="%s\n \n" % extensions) except KeyError as e: sys.stderr.write( '\n\nUnable to find extension type: %s\n\n' % extension_type) From f78a0b386c46a2b56e90eb73021a34db91292831 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Mon, 8 Feb 2021 14:57:24 +0000 Subject: [PATCH 060/113] tools/o Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index fcf45f6a63d70..c2b883f6041b1 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -216,12 +216,15 @@ def FormatExtension(extension): anchor = FormatAnchor('extension_' + extension) status = EXTENSION_STATUS_VALUES.get(extension_metadata['status'], '') security_posture = EXTENSION_SECURITY_POSTURES[extension_metadata['security_posture']] + extension_categories = "\n".join( + " - :ref:`%s `" % (ext, ext) + for ext + in extension_metadata.get('categories', [])) return EXTENSION_TEMPLATE.substitute(anchor=anchor, extension=extension, status=status, security_posture=security_posture, - # this shouldnt need get once all have cats... - categories=extension_metadata.get('categories', [])) + categories=extension_categories) except KeyError as e: sys.stderr.write( '\n\nDid you forget to add an entry to source/extensions/extensions_build_config.bzl?\n\n') From a47409e7e24aad6b94b9f17f76bb56654274fe24 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Mon, 8 Feb 2021 15:11:54 +0000 Subject: [PATCH 061/113] tools/ Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index c2b883f6041b1..a9c929e456f0e 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -224,7 +224,7 @@ def FormatExtension(extension): extension=extension, status=status, security_posture=security_posture, - categories=extension_categories) + categories="%s\n \n" % extension_categories) except KeyError as e: sys.stderr.write( '\n\nDid you forget to add an entry to source/extensions/extensions_build_config.bzl?\n\n') From 21c68c07abb17297122571a5a8b298917082dbb8 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Mon, 8 Feb 2021 15:32:47 +0000 Subject: [PATCH 062/113] tools/ Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index a9c929e456f0e..44f5ab42d2e31 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -63,7 +63,7 @@ This extension extends the following extension type: - $categories +$categories """) From 2453fb23be4cf7093363878c18455f4b7c250351 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Mon, 8 Feb 2021 17:26:26 +0000 Subject: [PATCH 063/113] api/ Signed-off-by: Ryan Northey --- api/envoy/config/accesslog/v3/accesslog.proto | 1 + api/envoy/config/cluster/v3/cluster.proto | 1 + .../http_connection_manager/v3/http_connection_manager.proto | 1 + 3 files changed, 3 insertions(+) diff --git a/api/envoy/config/accesslog/v3/accesslog.proto b/api/envoy/config/accesslog/v3/accesslog.proto index b709e8919944b..4083ff8ed7d92 100644 --- a/api/envoy/config/accesslog/v3/accesslog.proto +++ b/api/envoy/config/accesslog/v3/accesslog.proto @@ -50,6 +50,7 @@ message AccessLog { // #. "envoy.access_loggers.tcp_grpc": :ref:`TcpGrpcAccessLogConfig // ` oneof config_type { + // [#protodoc-extension: envoy.access_loggers] google.protobuf.Any typed_config = 4; } } diff --git a/api/envoy/config/cluster/v3/cluster.proto b/api/envoy/config/cluster/v3/cluster.proto index 14f5ccf3bd5f7..e2d9740c9e5ac 100644 --- a/api/envoy/config/cluster/v3/cluster.proto +++ b/api/envoy/config/cluster/v3/cluster.proto @@ -158,6 +158,7 @@ message Cluster { google.protobuf.Struct match = 2; // The configuration of the transport socket. + // [#protodoc-extension: envoy.transport_socket.upstream] core.v3.TransportSocket transport_socket = 3; } diff --git a/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto b/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto index c00085e1149d6..3827447a0159f 100644 --- a/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto +++ b/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto @@ -268,6 +268,7 @@ message HttpConnectionManager { // A list of individual HTTP filters that make up the filter chain for // requests made to the connection manager. :ref:`Order matters ` // as the filters are processed sequentially as request events happen. + // [#protodoc-extension: envoy.filters.http] repeated HttpFilter http_filters = 5; // Whether the connection manager manipulates the :ref:`config_http_conn_man_headers_user-agent` From 7f022bb4ca8f2cadd1ee58279bfc647970d6b64e Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Mon, 8 Feb 2021 17:48:08 +0000 Subject: [PATCH 064/113] api/ Signed-off-by: Ryan Northey --- api/envoy/config/cluster/v3/cluster.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/envoy/config/cluster/v3/cluster.proto b/api/envoy/config/cluster/v3/cluster.proto index e2d9740c9e5ac..1c5af078554d0 100644 --- a/api/envoy/config/cluster/v3/cluster.proto +++ b/api/envoy/config/cluster/v3/cluster.proto @@ -158,7 +158,7 @@ message Cluster { google.protobuf.Struct match = 2; // The configuration of the transport socket. - // [#protodoc-extension: envoy.transport_socket.upstream] + // [#protodoc-extension: envoy.transport_sockets.upstream] core.v3.TransportSocket transport_socket = 3; } From 5749b625d7c2cebb0fd05678cd73e66b1a8baa5c Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Tue, 9 Feb 2021 15:06:09 +0000 Subject: [PATCH 065/113] api/ Signed-off-by: Ryan Northey --- api/envoy/config/accesslog/v3/accesslog.proto | 2 +- api/envoy/config/core/v3/grpc_service.proto | 1 + api/envoy/config/listener/v3/listener_components.proto | 3 ++- api/envoy/config/listener/v3/udp_listener_config.proto | 1 + .../filters/network/thrift_proxy/v3/thrift_proxy.proto | 1 + 5 files changed, 6 insertions(+), 2 deletions(-) diff --git a/api/envoy/config/accesslog/v3/accesslog.proto b/api/envoy/config/accesslog/v3/accesslog.proto index 4083ff8ed7d92..cdde08b918632 100644 --- a/api/envoy/config/accesslog/v3/accesslog.proto +++ b/api/envoy/config/accesslog/v3/accesslog.proto @@ -49,8 +49,8 @@ message AccessLog { // ` // #. "envoy.access_loggers.tcp_grpc": :ref:`TcpGrpcAccessLogConfig // ` + // [#protodoc-extension: envoy.access_loggers] oneof config_type { - // [#protodoc-extension: envoy.access_loggers] google.protobuf.Any typed_config = 4; } } diff --git a/api/envoy/config/core/v3/grpc_service.proto b/api/envoy/config/core/v3/grpc_service.proto index e3730d017410b..70b1a4438dff0 100644 --- a/api/envoy/config/core/v3/grpc_service.proto +++ b/api/envoy/config/core/v3/grpc_service.proto @@ -124,6 +124,7 @@ message GrpcService { string name = 1; + // [#protodoc-extension: envoy.filters.network] oneof config_type { google.protobuf.Any typed_config = 3; } diff --git a/api/envoy/config/listener/v3/listener_components.proto b/api/envoy/config/listener/v3/listener_components.proto index dfc871d477818..ed0cd30e1d89e 100644 --- a/api/envoy/config/listener/v3/listener_components.proto +++ b/api/envoy/config/listener/v3/listener_components.proto @@ -33,9 +33,9 @@ message Filter { // The name of the filter to instantiate. The name must match a // :ref:`supported filter `. - // [#protodoc-extension: envoy.filters.network] string name = 1 [(validate.rules).string = {min_len: 1}]; + // [#protodoc-extension: envoy.filters.network] oneof config_type { // Filter specific configuration which depends on the filter being // instantiated. See the supported filters for further documentation. @@ -337,6 +337,7 @@ message ListenerFilter { // Filter specific configuration which depends on the filter being instantiated. // See the supported filters for further documentation. + // [#protodoc-extension: envoy.filters.listener] oneof config_type { google.protobuf.Any typed_config = 3; } diff --git a/api/envoy/config/listener/v3/udp_listener_config.proto b/api/envoy/config/listener/v3/udp_listener_config.proto index fe39be9b35258..c45616822480e 100644 --- a/api/envoy/config/listener/v3/udp_listener_config.proto +++ b/api/envoy/config/listener/v3/udp_listener_config.proto @@ -30,6 +30,7 @@ message UdpListenerConfig { // Used to create a specific listener factory. To some factory, e.g. // "raw_udp_listener", config is not needed. + // [#protodoc-extension: envoy.filters.udp_listener] oneof config_type { google.protobuf.Any typed_config = 3; } diff --git a/api/envoy/extensions/filters/network/thrift_proxy/v3/thrift_proxy.proto b/api/envoy/extensions/filters/network/thrift_proxy/v3/thrift_proxy.proto index 11d87e986fd59..be5f21f1837c0 100644 --- a/api/envoy/extensions/filters/network/thrift_proxy/v3/thrift_proxy.proto +++ b/api/envoy/extensions/filters/network/thrift_proxy/v3/thrift_proxy.proto @@ -80,6 +80,7 @@ message ThriftProxy { // Thrift proxy. Order matters as the filters are processed sequentially. For backwards // compatibility, if no thrift_filters are specified, a default Thrift router filter // (`envoy.filters.thrift.router`) is used. + // [#protodoc-extension: envoy.thrift_proxy.filters] repeated ThriftFilter thrift_filters = 5; // If set to true, Envoy will try to skip decode data after metadata in the Thrift message. From 8d71825924fcbb6aeef426a175bf5381cd1ffa95 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Tue, 9 Feb 2021 15:38:23 +0000 Subject: [PATCH 066/113] api/ Signed-off-by: Ryan Northey --- api/envoy/config/core/v3/grpc_service.proto | 2 +- api/envoy/config/core/v3/health_check.proto | 1 + api/envoy/config/metrics/v3/stats.proto | 1 + api/envoy/config/overload/v3/overload.proto | 1 + api/envoy/config/route/v3/route_components.proto | 2 ++ api/envoy/config/trace/v3/http_tracer.proto | 1 + 6 files changed, 7 insertions(+), 1 deletion(-) diff --git a/api/envoy/config/core/v3/grpc_service.proto b/api/envoy/config/core/v3/grpc_service.proto index 70b1a4438dff0..2d29ada789582 100644 --- a/api/envoy/config/core/v3/grpc_service.proto +++ b/api/envoy/config/core/v3/grpc_service.proto @@ -124,7 +124,7 @@ message GrpcService { string name = 1; - // [#protodoc-extension: envoy.filters.network] + // [#protodoc-extension: envoy.grpc_credentials] oneof config_type { google.protobuf.Any typed_config = 3; } diff --git a/api/envoy/config/core/v3/health_check.proto b/api/envoy/config/core/v3/health_check.proto index e4ad7429e0d6a..a27a5c7533d82 100644 --- a/api/envoy/config/core/v3/health_check.proto +++ b/api/envoy/config/core/v3/health_check.proto @@ -189,6 +189,7 @@ message HealthCheck { // A custom health checker specific configuration which depends on the custom health checker // being instantiated. See :api:`envoy/config/health_checker` for reference. + // [#protodoc-extension: envoy.health_checkers] oneof config_type { google.protobuf.Any typed_config = 3; } diff --git a/api/envoy/config/metrics/v3/stats.proto b/api/envoy/config/metrics/v3/stats.proto index 16335b2a1c13d..28519d2ef8b83 100644 --- a/api/envoy/config/metrics/v3/stats.proto +++ b/api/envoy/config/metrics/v3/stats.proto @@ -41,6 +41,7 @@ message StatsSink { // Stats sink specific configuration which depends on the sink being instantiated. See // :ref:`StatsdSink ` for an example. + // [#protodoc-extension: envoy.stats_sinks] oneof config_type { google.protobuf.Any typed_config = 3; } diff --git a/api/envoy/config/overload/v3/overload.proto b/api/envoy/config/overload/v3/overload.proto index 7c51d493b4918..1a27fe7b6ed3f 100644 --- a/api/envoy/config/overload/v3/overload.proto +++ b/api/envoy/config/overload/v3/overload.proto @@ -41,6 +41,7 @@ message ResourceMonitor { string name = 1 [(validate.rules).string = {min_len: 1}]; // Configuration for the resource monitor being instantiated. + // [#protodoc-extension: envoy.resource_monitors] oneof config_type { google.protobuf.Any typed_config = 3; } diff --git a/api/envoy/config/route/v3/route_components.proto b/api/envoy/config/route/v3/route_components.proto index 1ac866e01a2c7..3aaf7edf793ff 100644 --- a/api/envoy/config/route/v3/route_components.proto +++ b/api/envoy/config/route/v3/route_components.proto @@ -1119,6 +1119,7 @@ message RetryPolicy { string name = 1 [(validate.rules).string = {min_len: 1}]; + // [#protodoc-extension: envoy.retry_priority] oneof config_type { google.protobuf.Any typed_config = 3; } @@ -1134,6 +1135,7 @@ message RetryPolicy { string name = 1 [(validate.rules).string = {min_len: 1}]; + // [#protodoc-extension: envoy.retry_host_predicates] oneof config_type { google.protobuf.Any typed_config = 3; } diff --git a/api/envoy/config/trace/v3/http_tracer.proto b/api/envoy/config/trace/v3/http_tracer.proto index 7146687b831db..a6dc8a87f3be2 100644 --- a/api/envoy/config/trace/v3/http_tracer.proto +++ b/api/envoy/config/trace/v3/http_tracer.proto @@ -62,6 +62,7 @@ message Tracing { // - :ref:`DatadogConfig ` // - :ref:`OpenCensusConfig ` // - :ref:`AWS X-Ray ` + // [#protodoc-extension: envoy.tracers] oneof config_type { google.protobuf.Any typed_config = 3; } From 2e1d44d9011e67a7389454afcddaa92a768a5637 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Tue, 9 Feb 2021 15:51:03 +0000 Subject: [PATCH 067/113] api Signed-off-by: Ryan Northey --- api/envoy/config/route/v3/route_components.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/envoy/config/route/v3/route_components.proto b/api/envoy/config/route/v3/route_components.proto index 3aaf7edf793ff..246b0a7b9438f 100644 --- a/api/envoy/config/route/v3/route_components.proto +++ b/api/envoy/config/route/v3/route_components.proto @@ -1119,7 +1119,7 @@ message RetryPolicy { string name = 1 [(validate.rules).string = {min_len: 1}]; - // [#protodoc-extension: envoy.retry_priority] + // [#protodoc-extension: envoy.retry_priorities] oneof config_type { google.protobuf.Any typed_config = 3; } From 16b95eeb2b9bb244e6cbc6636a764871f55d7b9e Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Tue, 9 Feb 2021 16:13:31 +0000 Subject: [PATCH 068/113] api/ Signed-off-by: Ryan Northey --- api/envoy/config/bootstrap/v3/bootstrap.proto | 1 + api/envoy/config/route/v3/route_components.proto | 2 ++ 2 files changed, 3 insertions(+) diff --git a/api/envoy/config/bootstrap/v3/bootstrap.proto b/api/envoy/config/bootstrap/v3/bootstrap.proto index 5a231af19e7f6..a1008c3c661b5 100644 --- a/api/envoy/config/bootstrap/v3/bootstrap.proto +++ b/api/envoy/config/bootstrap/v3/bootstrap.proto @@ -190,6 +190,7 @@ message Bootstrap { // Optional watchdogs configuration. // This is used for specifying different watchdogs for the different subsystems. + // [#protodoc-extension: envoy.guarddog_actions] Watchdogs watchdogs = 27; // Configuration for an external tracing provider. diff --git a/api/envoy/config/route/v3/route_components.proto b/api/envoy/config/route/v3/route_components.proto index 246b0a7b9438f..b6af1b6d81014 100644 --- a/api/envoy/config/route/v3/route_components.proto +++ b/api/envoy/config/route/v3/route_components.proto @@ -1749,6 +1749,7 @@ message RateLimit { MetaData metadata = 8; // Rate limit descriptor extension. See the rate limit descriptor extensions documentation. + // [#protodoc-extension: envoy.rate_limit_descriptors] core.v3.TypedExtensionConfig extension = 9; } } @@ -1940,6 +1941,7 @@ message InternalRedirectPolicy { // Specifies a list of predicates that are queried when an upstream response is deemed // to trigger an internal redirect by all other criteria. Any predicate in the list can reject // the redirect, causing the response to be proxied to downstream. + // [#protodoc-extension: envoy.internal_redirect_predicates] repeated core.v3.TypedExtensionConfig predicates = 3; // Allow internal redirect to follow a target URI with a different scheme than the value of From 97dea40189984cd90b74019b09679dbf84d69856 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Tue, 9 Feb 2021 16:22:39 +0000 Subject: [PATCH 069/113] empty Signed-off-by: Ryan Northey From 72422a40ab7bfcd5087e84eb40005431cd6a1ce2 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Tue, 9 Feb 2021 16:41:51 +0000 Subject: [PATCH 070/113] tools/ Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index 44f5ab42d2e31..ee1435a4a5cc2 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -240,12 +240,15 @@ def FormatExtensionType(extension_type): Returns: RST formatted extension type description. """ + IGNORED_EXTENSIONS = [ + "envoy.tracers.xray", "envoy.filters.http.cache.simple_http_cache"] try: anchor = FormatAnchor('extension_type_' + extension_type) extensions = "\n".join( " - :ref:`%s `" % (ext, ext) for ext - in EXTENSION_CATEGORIES[extension_type]) + in EXTENSION_CATEGORIES[extension_type] + if ext not in IGNORED_EXTENSIONS) return EXTENSION_TYPE_TEMPLATE.substitute(anchor=anchor, extensions="%s\n \n" % extensions) except KeyError as e: From 327f5486dbc45b3a9bdcc976e8a7213e760a897b Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Tue, 9 Feb 2021 17:13:17 +0000 Subject: [PATCH 071/113] tools/ Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index ee1435a4a5cc2..8e57c755facbf 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -57,6 +57,8 @@ This extension may be referenced by the qualified name *$extension* .. note:: + EXTENSION_MARKER + $status $security_posture @@ -71,6 +73,8 @@ EXTENSION_TYPE_TEMPLATE = string.Template("""$anchor .. tip:: + EXTENSION_TYPE_MARKER + This extension type has the following known extensions $extensions From ddb104e49cf2216ea21bb12d9708a450ad869cf7 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Wed, 10 Feb 2021 10:37:11 +0000 Subject: [PATCH 072/113] updates Signed-off-by: Ryan Northey --- api/envoy/extensions/wasm/v3/wasm.proto | 1 + bazel/envoy_library.bzl | 2 -- source/extensions/transport_sockets/alts/BUILD | 4 ---- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/api/envoy/extensions/wasm/v3/wasm.proto b/api/envoy/extensions/wasm/v3/wasm.proto index 2232b11075c8d..239d7291b5a30 100644 --- a/api/envoy/extensions/wasm/v3/wasm.proto +++ b/api/envoy/extensions/wasm/v3/wasm.proto @@ -59,6 +59,7 @@ message VmConfig { // (proxy_on_start). `google.protobuf.Struct` is serialized as JSON before // passing it to the plugin. `google.protobuf.BytesValue` and // `google.protobuf.StringValue` are passed directly without the wrapper. + // [#protodoc-extension: envoy.wasm.runtime] google.protobuf.Any configuration = 4; // Allow the wasm file to include pre-compiled code on VMs which support it. diff --git a/bazel/envoy_library.bzl b/bazel/envoy_library.bzl index d052c481ace5c..0131f3c132c75 100644 --- a/bazel/envoy_library.bzl +++ b/bazel/envoy_library.bzl @@ -77,14 +77,12 @@ EXTENSION_CATEGORIES = [ "envoy.filters.listener", "envoy.filters.network", "envoy.filters.udp_listener", - "envoy.formatter", "envoy.grpc_credentials", "envoy.guarddog_actions", "envoy.health_checkers", "envoy.internal_redirect_predicates", "envoy.io_socket", "envoy.rate_limit_descriptors", - "envoy.resolvers", "envoy.resource_monitors", "envoy.retry_host_predicates", "envoy.retry_priorities", diff --git a/source/extensions/transport_sockets/alts/BUILD b/source/extensions/transport_sockets/alts/BUILD index edc53da484e4b..2d3df5920e8fb 100644 --- a/source/extensions/transport_sockets/alts/BUILD +++ b/source/extensions/transport_sockets/alts/BUILD @@ -41,10 +41,6 @@ envoy_cc_extension( external_deps = [ "abseil_node_hash_set", ], - category = ( - "envoy.transport_sockets.downstream", - "envoy.transport_sockets.upstream", - ), security_posture = "robust_to_untrusted_downstream_and_upstream", deps = [ ":tsi_handshaker", From 307ac3fcf4eeb7f2aa899d64d5dfdc6948ab0666 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Wed, 10 Feb 2021 16:38:05 +0000 Subject: [PATCH 073/113] docs/ Signed-off-by: Ryan Northey --- docs/generate_extension_db.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docs/generate_extension_db.py b/docs/generate_extension_db.py index cc6bd3e71bbd5..5717927247c36 100755 --- a/docs/generate_extension_db.py +++ b/docs/generate_extension_db.py @@ -50,11 +50,6 @@ def GetExtensionMetadata(target): % target) # evaluate tuples/lists # wrap strings in a list - # use empty list for missing (this should not be necessary) - categories = ( - '[]' - if IsMissing(categories) - else categories) categories = ( ast.literal_eval(categories) if ('[' in categories From 3be54d39e1127785af9365b6eed95472731d0797 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Wed, 10 Feb 2021 17:05:39 +0000 Subject: [PATCH 074/113] docs/ Signed-off-by: Ryan Northey --- docs/generate_extension_cats_db.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/generate_extension_cats_db.py b/docs/generate_extension_cats_db.py index e46c351cc20dd..be4c09411951c 100755 --- a/docs/generate_extension_cats_db.py +++ b/docs/generate_extension_cats_db.py @@ -13,9 +13,6 @@ def main(): categories = {} for k, v in data.items(): - print(">> %s" % k) - print(v) - print() for cat in v['categories']: categories[cat] = categories.get(cat, []) categories[cat].append(k) @@ -23,5 +20,6 @@ def main(): with open(extension_cat_db, 'w') as f: json.dump(categories, f) + if __name__ == '__main__': main() From 6621d90c33c4b0c0e89c597dcb63bd7f7aaaf2f0 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Wed, 10 Feb 2021 17:27:56 +0000 Subject: [PATCH 075/113] tools/ Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index 8e57c755facbf..d59fc50bb2dcd 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -221,7 +221,7 @@ def FormatExtension(extension): status = EXTENSION_STATUS_VALUES.get(extension_metadata['status'], '') security_posture = EXTENSION_SECURITY_POSTURES[extension_metadata['security_posture']] extension_categories = "\n".join( - " - :ref:`%s `" % (ext, ext) + f" - :ref:`{ext} `" for ext in extension_metadata.get('categories', [])) return EXTENSION_TEMPLATE.substitute(anchor=anchor, @@ -249,7 +249,7 @@ def FormatExtensionType(extension_type): try: anchor = FormatAnchor('extension_type_' + extension_type) extensions = "\n".join( - " - :ref:`%s `" % (ext, ext) + f" - :ref:`{ext} `" for ext in EXTENSION_CATEGORIES[extension_type] if ext not in IGNORED_EXTENSIONS) @@ -282,6 +282,7 @@ def FormatHeaderFromFile(style, source_code_info, proto_name): formatted_extension = FormatExtension(extension) formatted_extension_type = '' if annotations.EXTENSION_TYPE_ANNOTATION in source_code_info.file_level_annotations: + raise formatted_extension_type = FormatExtensionType( source_code_info.file_level_annotations[annotations.EXTENSION_TYPE_ANNOTATION]) if annotations.DOC_TITLE_ANNOTATION in source_code_info.file_level_annotations: From 1dd6c62eb5b08242eaf52cf119bc5ebc079ddd9d Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Wed, 10 Feb 2021 17:45:32 +0000 Subject: [PATCH 076/113] tools/ Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index d59fc50bb2dcd..f5974004cb1c1 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -280,17 +280,12 @@ def FormatHeaderFromFile(style, source_code_info, proto_name): if annotations.EXTENSION_ANNOTATION in source_code_info.file_level_annotations: extension = source_code_info.file_level_annotations[annotations.EXTENSION_ANNOTATION] formatted_extension = FormatExtension(extension) - formatted_extension_type = '' - if annotations.EXTENSION_TYPE_ANNOTATION in source_code_info.file_level_annotations: - raise - formatted_extension_type = FormatExtensionType( - source_code_info.file_level_annotations[annotations.EXTENSION_TYPE_ANNOTATION]) if annotations.DOC_TITLE_ANNOTATION in source_code_info.file_level_annotations: return anchor + FormatHeader( style, source_code_info.file_level_annotations[ - annotations.DOC_TITLE_ANNOTATION]) + formatted_extension + formatted_extension_type, stripped_comment + annotations.DOC_TITLE_ANNOTATION]) + formatted_extension, stripped_comment - return anchor + FormatHeader(style, proto_name) + formatted_extension + formatted_extension_type, stripped_comment + return anchor + FormatHeader(style, proto_name) + formatted_extension, stripped_comment def FormatFieldTypeAsJson(type_context, field): From 29366d31d0543281b801a5a18c38e44c2e67c290 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Thu, 11 Feb 2021 09:27:25 +0000 Subject: [PATCH 077/113] protodoc-extension -> extenion-category Signed-off-by: Ryan Northey --- api/envoy/config/accesslog/v3/accesslog.proto | 2 +- api/envoy/config/bootstrap/v3/bootstrap.proto | 4 ++-- api/envoy/config/cluster/v3/cluster.proto | 6 +++--- api/envoy/config/core/v3/grpc_service.proto | 2 +- api/envoy/config/core/v3/health_check.proto | 2 +- api/envoy/config/listener/v3/listener_components.proto | 6 +++--- api/envoy/config/listener/v3/udp_listener_config.proto | 2 +- api/envoy/config/metrics/v3/stats.proto | 2 +- api/envoy/config/overload/v3/overload.proto | 2 +- api/envoy/config/route/v3/route_components.proto | 8 ++++---- api/envoy/config/trace/v3/http_tracer.proto | 2 +- .../filters/http/compressor/v3/compressor.proto | 2 +- .../filters/http/decompressor/v3/decompressor.proto | 2 +- .../v3/http_connection_manager.proto | 2 +- .../filters/network/thrift_proxy/v3/thrift_proxy.proto | 2 +- api/envoy/extensions/wasm/v3/wasm.proto | 2 +- tools/api_proto_plugin/annotations.py | 2 +- 17 files changed, 25 insertions(+), 25 deletions(-) diff --git a/api/envoy/config/accesslog/v3/accesslog.proto b/api/envoy/config/accesslog/v3/accesslog.proto index cdde08b918632..6d494be4aea6d 100644 --- a/api/envoy/config/accesslog/v3/accesslog.proto +++ b/api/envoy/config/accesslog/v3/accesslog.proto @@ -49,7 +49,7 @@ message AccessLog { // ` // #. "envoy.access_loggers.tcp_grpc": :ref:`TcpGrpcAccessLogConfig // ` - // [#protodoc-extension: envoy.access_loggers] + // [#extension-category: envoy.access_loggers] oneof config_type { google.protobuf.Any typed_config = 4; } diff --git a/api/envoy/config/bootstrap/v3/bootstrap.proto b/api/envoy/config/bootstrap/v3/bootstrap.proto index a1008c3c661b5..7b2ab471c47bf 100644 --- a/api/envoy/config/bootstrap/v3/bootstrap.proto +++ b/api/envoy/config/bootstrap/v3/bootstrap.proto @@ -190,7 +190,7 @@ message Bootstrap { // Optional watchdogs configuration. // This is used for specifying different watchdogs for the different subsystems. - // [#protodoc-extension: envoy.guarddog_actions] + // [#extension-category: envoy.guarddog_actions] Watchdogs watchdogs = 27; // Configuration for an external tracing provider. @@ -250,7 +250,7 @@ message Bootstrap { // Specifies optional bootstrap extensions to be instantiated at startup time. // Each item contains extension specific configuration. - // [#protodoc-extension: envoy.bootstrap] + // [#extension-category: envoy.bootstrap] repeated core.v3.TypedExtensionConfig bootstrap_extensions = 21; // Specifies optional extensions instantiated at startup time and diff --git a/api/envoy/config/cluster/v3/cluster.proto b/api/envoy/config/cluster/v3/cluster.proto index 1c5af078554d0..6b89257cb23ae 100644 --- a/api/envoy/config/cluster/v3/cluster.proto +++ b/api/envoy/config/cluster/v3/cluster.proto @@ -158,7 +158,7 @@ message Cluster { google.protobuf.Struct match = 2; // The configuration of the transport socket. - // [#protodoc-extension: envoy.transport_sockets.upstream] + // [#extension-category: envoy.transport_sockets.upstream] core.v3.TransportSocket transport_socket = 3; } @@ -172,7 +172,7 @@ message Cluster { // Cluster specific configuration which depends on the cluster being instantiated. // See the supported cluster for further documentation. - // [#protodoc-extension: envoy.clusters] + // [#extension-category: envoy.clusters] google.protobuf.Any typed_config = 2; } @@ -1006,7 +1006,7 @@ message Cluster { // If users desire custom connection pool or upstream behavior, for example terminating // CONNECT only if a custom filter indicates it is appropriate, the custom factories // can be registered and configured here. - // [#protodoc-extension: envoy.upstreams] + // [#extension-category: envoy.upstreams] core.v3.TypedExtensionConfig upstream_config = 48; // Configuration to track optional cluster stats. diff --git a/api/envoy/config/core/v3/grpc_service.proto b/api/envoy/config/core/v3/grpc_service.proto index 2d29ada789582..103c8b90f6349 100644 --- a/api/envoy/config/core/v3/grpc_service.proto +++ b/api/envoy/config/core/v3/grpc_service.proto @@ -124,7 +124,7 @@ message GrpcService { string name = 1; - // [#protodoc-extension: envoy.grpc_credentials] + // [#extension-category: envoy.grpc_credentials] oneof config_type { google.protobuf.Any typed_config = 3; } diff --git a/api/envoy/config/core/v3/health_check.proto b/api/envoy/config/core/v3/health_check.proto index a27a5c7533d82..27710830536c8 100644 --- a/api/envoy/config/core/v3/health_check.proto +++ b/api/envoy/config/core/v3/health_check.proto @@ -189,7 +189,7 @@ message HealthCheck { // A custom health checker specific configuration which depends on the custom health checker // being instantiated. See :api:`envoy/config/health_checker` for reference. - // [#protodoc-extension: envoy.health_checkers] + // [#extension-category: envoy.health_checkers] oneof config_type { google.protobuf.Any typed_config = 3; } diff --git a/api/envoy/config/listener/v3/listener_components.proto b/api/envoy/config/listener/v3/listener_components.proto index ed0cd30e1d89e..952056ddce0e8 100644 --- a/api/envoy/config/listener/v3/listener_components.proto +++ b/api/envoy/config/listener/v3/listener_components.proto @@ -35,7 +35,7 @@ message Filter { // :ref:`supported filter `. string name = 1 [(validate.rules).string = {min_len: 1}]; - // [#protodoc-extension: envoy.filters.network] + // [#extension-category: envoy.filters.network] oneof config_type { // Filter specific configuration which depends on the filter being // instantiated. See the supported filters for further documentation. @@ -332,12 +332,12 @@ message ListenerFilter { // The name of the filter to instantiate. The name must match a // :ref:`supported filter `. - // [#protodoc-extension: envoy.transport_sockets.downstream] + // [#extension-category: envoy.transport_sockets.downstream] string name = 1 [(validate.rules).string = {min_len: 1}]; // Filter specific configuration which depends on the filter being instantiated. // See the supported filters for further documentation. - // [#protodoc-extension: envoy.filters.listener] + // [#extension-category: envoy.filters.listener] oneof config_type { google.protobuf.Any typed_config = 3; } diff --git a/api/envoy/config/listener/v3/udp_listener_config.proto b/api/envoy/config/listener/v3/udp_listener_config.proto index c45616822480e..058bca55acfdb 100644 --- a/api/envoy/config/listener/v3/udp_listener_config.proto +++ b/api/envoy/config/listener/v3/udp_listener_config.proto @@ -30,7 +30,7 @@ message UdpListenerConfig { // Used to create a specific listener factory. To some factory, e.g. // "raw_udp_listener", config is not needed. - // [#protodoc-extension: envoy.filters.udp_listener] + // [#extension-category: envoy.filters.udp_listener] oneof config_type { google.protobuf.Any typed_config = 3; } diff --git a/api/envoy/config/metrics/v3/stats.proto b/api/envoy/config/metrics/v3/stats.proto index 28519d2ef8b83..307805e65ad8e 100644 --- a/api/envoy/config/metrics/v3/stats.proto +++ b/api/envoy/config/metrics/v3/stats.proto @@ -41,7 +41,7 @@ message StatsSink { // Stats sink specific configuration which depends on the sink being instantiated. See // :ref:`StatsdSink ` for an example. - // [#protodoc-extension: envoy.stats_sinks] + // [#extension-category: envoy.stats_sinks] oneof config_type { google.protobuf.Any typed_config = 3; } diff --git a/api/envoy/config/overload/v3/overload.proto b/api/envoy/config/overload/v3/overload.proto index 1a27fe7b6ed3f..8817fe0c291b4 100644 --- a/api/envoy/config/overload/v3/overload.proto +++ b/api/envoy/config/overload/v3/overload.proto @@ -41,7 +41,7 @@ message ResourceMonitor { string name = 1 [(validate.rules).string = {min_len: 1}]; // Configuration for the resource monitor being instantiated. - // [#protodoc-extension: envoy.resource_monitors] + // [#extension-category: envoy.resource_monitors] oneof config_type { google.protobuf.Any typed_config = 3; } diff --git a/api/envoy/config/route/v3/route_components.proto b/api/envoy/config/route/v3/route_components.proto index b6af1b6d81014..05ede76394fec 100644 --- a/api/envoy/config/route/v3/route_components.proto +++ b/api/envoy/config/route/v3/route_components.proto @@ -1119,7 +1119,7 @@ message RetryPolicy { string name = 1 [(validate.rules).string = {min_len: 1}]; - // [#protodoc-extension: envoy.retry_priorities] + // [#extension-category: envoy.retry_priorities] oneof config_type { google.protobuf.Any typed_config = 3; } @@ -1135,7 +1135,7 @@ message RetryPolicy { string name = 1 [(validate.rules).string = {min_len: 1}]; - // [#protodoc-extension: envoy.retry_host_predicates] + // [#extension-category: envoy.retry_host_predicates] oneof config_type { google.protobuf.Any typed_config = 3; } @@ -1749,7 +1749,7 @@ message RateLimit { MetaData metadata = 8; // Rate limit descriptor extension. See the rate limit descriptor extensions documentation. - // [#protodoc-extension: envoy.rate_limit_descriptors] + // [#extension-category: envoy.rate_limit_descriptors] core.v3.TypedExtensionConfig extension = 9; } } @@ -1941,7 +1941,7 @@ message InternalRedirectPolicy { // Specifies a list of predicates that are queried when an upstream response is deemed // to trigger an internal redirect by all other criteria. Any predicate in the list can reject // the redirect, causing the response to be proxied to downstream. - // [#protodoc-extension: envoy.internal_redirect_predicates] + // [#extension-category: envoy.internal_redirect_predicates] repeated core.v3.TypedExtensionConfig predicates = 3; // Allow internal redirect to follow a target URI with a different scheme than the value of diff --git a/api/envoy/config/trace/v3/http_tracer.proto b/api/envoy/config/trace/v3/http_tracer.proto index a6dc8a87f3be2..d0ba3ba6e4fdb 100644 --- a/api/envoy/config/trace/v3/http_tracer.proto +++ b/api/envoy/config/trace/v3/http_tracer.proto @@ -62,7 +62,7 @@ message Tracing { // - :ref:`DatadogConfig ` // - :ref:`OpenCensusConfig ` // - :ref:`AWS X-Ray ` - // [#protodoc-extension: envoy.tracers] + // [#extension-category: envoy.tracers] oneof config_type { google.protobuf.Any typed_config = 3; } diff --git a/api/envoy/extensions/filters/http/compressor/v3/compressor.proto b/api/envoy/extensions/filters/http/compressor/v3/compressor.proto index a58f246d4ffb3..ed347cc439eb7 100644 --- a/api/envoy/extensions/filters/http/compressor/v3/compressor.proto +++ b/api/envoy/extensions/filters/http/compressor/v3/compressor.proto @@ -96,7 +96,7 @@ message Compressor { // :ref:`envoy.compression.gzip.compressor` // is included in Envoy. // This field is ignored if used in the context of the gzip http-filter, but is mandatory otherwise. - // [#protodoc-extension: envoy.compression.compressor] + // [#extension-category: envoy.compression.compressor] config.core.v3.TypedExtensionConfig compressor_library = 6; // Configuration for request compression. Compression is disabled by default if left empty. diff --git a/api/envoy/extensions/filters/http/decompressor/v3/decompressor.proto b/api/envoy/extensions/filters/http/decompressor/v3/decompressor.proto index fb0d6b7172044..54d5f23bc449e 100644 --- a/api/envoy/extensions/filters/http/decompressor/v3/decompressor.proto +++ b/api/envoy/extensions/filters/http/decompressor/v3/decompressor.proto @@ -43,7 +43,7 @@ message Decompressor { // A decompressor library to use for both request and response decompression. Currently only // :ref:`envoy.compression.gzip.compressor` // is included in Envoy. - // [#protodoc-extension: envoy.compression.decompressor] + // [#extension-category: envoy.compression.decompressor] config.core.v3.TypedExtensionConfig decompressor_library = 1 [(validate.rules).message = {required: true}]; diff --git a/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto b/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto index 3827447a0159f..cd7bdba52decf 100644 --- a/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto +++ b/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto @@ -268,7 +268,7 @@ message HttpConnectionManager { // A list of individual HTTP filters that make up the filter chain for // requests made to the connection manager. :ref:`Order matters ` // as the filters are processed sequentially as request events happen. - // [#protodoc-extension: envoy.filters.http] + // [#extension-category: envoy.filters.http] repeated HttpFilter http_filters = 5; // Whether the connection manager manipulates the :ref:`config_http_conn_man_headers_user-agent` diff --git a/api/envoy/extensions/filters/network/thrift_proxy/v3/thrift_proxy.proto b/api/envoy/extensions/filters/network/thrift_proxy/v3/thrift_proxy.proto index be5f21f1837c0..659ee14604401 100644 --- a/api/envoy/extensions/filters/network/thrift_proxy/v3/thrift_proxy.proto +++ b/api/envoy/extensions/filters/network/thrift_proxy/v3/thrift_proxy.proto @@ -80,7 +80,7 @@ message ThriftProxy { // Thrift proxy. Order matters as the filters are processed sequentially. For backwards // compatibility, if no thrift_filters are specified, a default Thrift router filter // (`envoy.filters.thrift.router`) is used. - // [#protodoc-extension: envoy.thrift_proxy.filters] + // [#extension-category: envoy.thrift_proxy.filters] repeated ThriftFilter thrift_filters = 5; // If set to true, Envoy will try to skip decode data after metadata in the Thrift message. diff --git a/api/envoy/extensions/wasm/v3/wasm.proto b/api/envoy/extensions/wasm/v3/wasm.proto index 239d7291b5a30..af2b7a38e310a 100644 --- a/api/envoy/extensions/wasm/v3/wasm.proto +++ b/api/envoy/extensions/wasm/v3/wasm.proto @@ -59,7 +59,7 @@ message VmConfig { // (proxy_on_start). `google.protobuf.Struct` is serialized as JSON before // passing it to the plugin. `google.protobuf.BytesValue` and // `google.protobuf.StringValue` are passed directly without the wrapper. - // [#protodoc-extension: envoy.wasm.runtime] + // [#extension-category: envoy.wasm.runtime] google.protobuf.Any configuration = 4; // Allow the wasm file to include pre-compiled code on VMs which support it. diff --git a/tools/api_proto_plugin/annotations.py b/tools/api_proto_plugin/annotations.py index d4c1b9d62bdbb..db0bfce534351 100644 --- a/tools/api_proto_plugin/annotations.py +++ b/tools/api_proto_plugin/annotations.py @@ -11,7 +11,7 @@ DOC_TITLE_ANNOTATION = 'protodoc-title' # Specify the type of extensions at extension points -EXTENSION_TYPE_ANNOTATION = 'protodoc-extension' +EXTENSION_TYPE_ANNOTATION = 'extension-category' # When documenting an extension, this should be used to specify the qualified # name that the extension registers as in the static registry, e.g. From aa2f47ea36ceaa8b9b0f2d2160b57d7b6d8b7b6d Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Thu, 11 Feb 2021 09:37:01 +0000 Subject: [PATCH 078/113] type -> category Signed-off-by: Ryan Northey --- source/extensions/quic_listeners/quiche/BUILD | 5 --- tools/protodoc/protodoc.py | 33 +++++++++---------- 2 files changed, 16 insertions(+), 22 deletions(-) diff --git a/source/extensions/quic_listeners/quiche/BUILD b/source/extensions/quic_listeners/quiche/BUILD index 804c9a0062e89..6dd9192114b03 100644 --- a/source/extensions/quic_listeners/quiche/BUILD +++ b/source/extensions/quic_listeners/quiche/BUILD @@ -378,11 +378,6 @@ envoy_cc_extension( security_posture = "unknown", tags = ["nofips"], - category = ( - "envoy.transport_sockets.downstream", - "envoy.transport_sockets.upstream", - ), - # QUICHE can't build against FIPS BoringSSL until the FIPS build # is on a new enough version to have QUIC support. Remove it from # the build until then. Re-enable as part of #7433. diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index f5974004cb1c1..bf31e04f17015 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -63,19 +63,19 @@ $security_posture - This extension extends the following extension type: + This extension extends the following extension category: $categories """) -EXTENSION_TYPE_TEMPLATE = string.Template("""$anchor +EXTENSION_CATEGORY_TEMPLATE = string.Template("""$anchor .. tip:: - EXTENSION_TYPE_MARKER + EXTENSION_CATEGORY_MARKER - This extension type has the following known extensions + This extension category has the following known extensions $extensions @@ -155,10 +155,10 @@ def FormatCommentWithAnnotations(comment, type_name=''): if annotations.EXTENSION_ANNOTATION in comment.annotations: extension = comment.annotations[annotations.EXTENSION_ANNOTATION] formatted_extension = FormatExtension(extension) - formatted_extension_type = '' - if annotations.EXTENSION_TYPE_ANNOTATION in comment.annotations: - formatted_extension_type = FormatExtensionType(comment.annotations[annotations.EXTENSION_TYPE_ANNOTATION]) - return annotations.WithoutAnnotations(StripLeadingSpace(comment.raw) + '\n') + formatted_extension + formatted_extension_type + formatted_extension_category = '' + if annotations.EXTENSION_CATEGORY_ANNOTATION in comment.annotations: + formatted_extension_category = FormatExtensionType(comment.annotations[annotations.EXTENSION_CATEGORY_ANNOTATION]) + return annotations.WithoutAnnotations(StripLeadingSpace(comment.raw) + '\n') + formatted_extension + formatted_extension_category def MapLines(f, s): @@ -221,7 +221,7 @@ def FormatExtension(extension): status = EXTENSION_STATUS_VALUES.get(extension_metadata['status'], '') security_posture = EXTENSION_SECURITY_POSTURES[extension_metadata['security_posture']] extension_categories = "\n".join( - f" - :ref:`{ext} `" + f" - :ref:`{ext} `" for ext in extension_metadata.get('categories', [])) return EXTENSION_TEMPLATE.substitute(anchor=anchor, @@ -235,29 +235,29 @@ def FormatExtension(extension): exit(1) # Raising the error buries the above message in tracebacks. -def FormatExtensionType(extension_type): +def FormatExtensionType(extension_category): """Format extension metadata as RST. Args: - extension_type: the name of the extension_type, e.g. com.acme. + extension_category: the name of the extension_category, e.g. com.acme. Returns: - RST formatted extension type description. + RST formatted extension category description. """ IGNORED_EXTENSIONS = [ "envoy.tracers.xray", "envoy.filters.http.cache.simple_http_cache"] try: - anchor = FormatAnchor('extension_type_' + extension_type) + anchor = FormatAnchor('extension_category_' + extension_category) extensions = "\n".join( f" - :ref:`{ext} `" for ext - in EXTENSION_CATEGORIES[extension_type] + in EXTENSION_CATEGORIES[extension_category] if ext not in IGNORED_EXTENSIONS) - return EXTENSION_TYPE_TEMPLATE.substitute(anchor=anchor, + return EXTENSION_CATEGORY_TEMPLATE.substitute(anchor=anchor, extensions="%s\n \n" % extensions) except KeyError as e: sys.stderr.write( - '\n\nUnable to find extension type: %s\n\n' % extension_type) + '\n\nUnable to find extension category: %s\n\n' % extension_category) exit(1) # Raising the error buries the above message in tracebacks. @@ -284,7 +284,6 @@ def FormatHeaderFromFile(style, source_code_info, proto_name): return anchor + FormatHeader( style, source_code_info.file_level_annotations[ annotations.DOC_TITLE_ANNOTATION]) + formatted_extension, stripped_comment - return anchor + FormatHeader(style, proto_name) + formatted_extension, stripped_comment From 1a607a5a32378e1a007ffdbb9f5eafb9a1bff89f Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Thu, 11 Feb 2021 09:39:57 +0000 Subject: [PATCH 079/113] type -> category Signed-off-by: Ryan Northey --- tools/api_proto_plugin/annotations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/api_proto_plugin/annotations.py b/tools/api_proto_plugin/annotations.py index db0bfce534351..cb8abdf3b91be 100644 --- a/tools/api_proto_plugin/annotations.py +++ b/tools/api_proto_plugin/annotations.py @@ -11,7 +11,7 @@ DOC_TITLE_ANNOTATION = 'protodoc-title' # Specify the type of extensions at extension points -EXTENSION_TYPE_ANNOTATION = 'extension-category' +EXTENSION_CATEGORY_ANNOTATION = 'extension-category' # When documenting an extension, this should be used to specify the qualified # name that the extension registers as in the static registry, e.g. @@ -35,7 +35,7 @@ VALID_ANNOTATIONS = set([ DOC_TITLE_ANNOTATION, EXTENSION_ANNOTATION, - EXTENSION_TYPE_ANNOTATION, + EXTENSION_CATEGORY_ANNOTATION, NOT_IMPLEMENTED_HIDE_ANNOTATION, NEXT_FREE_FIELD_ANNOTATION, NEXT_MAJOR_VERSION_ANNOTATION, From 62fc64e383ece6b8d5358f726c145588aa63f435 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Thu, 11 Feb 2021 10:09:39 +0000 Subject: [PATCH 080/113] docs/ Signed-off-by: Ryan Northey --- docs/build.sh | 2 +- ...nerate_extension_cats_db.py => generate_extension_cat_db.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename docs/{generate_extension_cats_db.py => generate_extension_cat_db.py} (100%) diff --git a/docs/build.sh b/docs/build.sh index 57b5182b82846..abf623ee228cd 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -78,7 +78,7 @@ BAZEL_BUILD_OPTIONS+=( ./docs/generate_extension_db.py "${EXTENSION_DB_PATH}" # Generate extension category database. -./docs/generate_extension_cats_db.py "${EXTENSION_DB_PATH}" "${EXTENSION_CAT_DB_PATH}" +./docs/generate_extension_cat_db.py "${EXTENSION_DB_PATH}" "${EXTENSION_CAT_DB_PATH}" # Generate RST for the lists of trusted/untrusted extensions in diff --git a/docs/generate_extension_cats_db.py b/docs/generate_extension_cat_db.py similarity index 100% rename from docs/generate_extension_cats_db.py rename to docs/generate_extension_cat_db.py From c7b70b84660d4d6a30a4f68ecd529a611153b2eb Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Thu, 11 Feb 2021 10:10:32 +0000 Subject: [PATCH 081/113] markers Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index bf31e04f17015..e82da9617bea6 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -57,8 +57,6 @@ This extension may be referenced by the qualified name *$extension* .. note:: - EXTENSION_MARKER - $status $security_posture @@ -73,8 +71,6 @@ EXTENSION_CATEGORY_TEMPLATE = string.Template("""$anchor .. tip:: - EXTENSION_CATEGORY_MARKER - This extension category has the following known extensions $extensions From bc5c0590ce03623b9bbe0011ad30f546449b9205 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Thu, 11 Feb 2021 10:29:14 +0000 Subject: [PATCH 082/113] cleanup Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index e82da9617bea6..19e2f56bd10f1 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -153,7 +153,7 @@ def FormatCommentWithAnnotations(comment, type_name=''): formatted_extension = FormatExtension(extension) formatted_extension_category = '' if annotations.EXTENSION_CATEGORY_ANNOTATION in comment.annotations: - formatted_extension_category = FormatExtensionType(comment.annotations[annotations.EXTENSION_CATEGORY_ANNOTATION]) + formatted_extension_category = FormatExtensionCategory(comment.annotations[annotations.EXTENSION_CATEGORY_ANNOTATION]) return annotations.WithoutAnnotations(StripLeadingSpace(comment.raw) + '\n') + formatted_extension + formatted_extension_category @@ -216,22 +216,28 @@ def FormatExtension(extension): anchor = FormatAnchor('extension_' + extension) status = EXTENSION_STATUS_VALUES.get(extension_metadata['status'], '') security_posture = EXTENSION_SECURITY_POSTURES[extension_metadata['security_posture']] - extension_categories = "\n".join( - f" - :ref:`{ext} `" - for ext - in extension_metadata.get('categories', [])) + categories = FormatExtensionList(extension_metadata["categories"], "extension_category") return EXTENSION_TEMPLATE.substitute(anchor=anchor, extension=extension, status=status, security_posture=security_posture, - categories="%s\n \n" % extension_categories) + categories=extension_categories) except KeyError as e: sys.stderr.write( '\n\nDid you forget to add an entry to source/extensions/extensions_build_config.bzl?\n\n') exit(1) # Raising the error buries the above message in tracebacks. -def FormatExtensionType(extension_category): +def FormatExtensionList(items, prefix="extension", indent=2): + indent = " " * indent + formatted_list = "\n".join( + f"{indent}- :ref:`{ext} <{prefix}_{ext}>`" + for ext + in items) + return f"%s\n{indent}\n" % formatted_list) + + +def FormatExtensionCategory(extension_category): """Format extension metadata as RST. Args: @@ -243,18 +249,14 @@ def FormatExtensionType(extension_category): IGNORED_EXTENSIONS = [ "envoy.tracers.xray", "envoy.filters.http.cache.simple_http_cache"] try: - anchor = FormatAnchor('extension_category_' + extension_category) - extensions = "\n".join( - f" - :ref:`{ext} `" - for ext - in EXTENSION_CATEGORIES[extension_category] - if ext not in IGNORED_EXTENSIONS) - return EXTENSION_CATEGORY_TEMPLATE.substitute(anchor=anchor, - extensions="%s\n \n" % extensions) + extensions = EXTENSION_CATEGORIES[extension_category] except KeyError as e: sys.stderr.write( '\n\nUnable to find extension category: %s\n\n' % extension_category) exit(1) # Raising the error buries the above message in tracebacks. + anchor = FormatAnchor('extension_category_' + extension_category) + extensions = FormatExtensionList([x for x in extensions if x not in IGNORED_EXTENSIONS]) + return EXTENSION_CATEGORY_TEMPLATE.substitute(anchor=anchor, extensions=extensions) def FormatHeaderFromFile(style, source_code_info, proto_name): From adb1347efdd5388dba8d609e36d0b1ed68ab66f8 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Thu, 11 Feb 2021 10:32:06 +0000 Subject: [PATCH 083/113] cleanup Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index 19e2f56bd10f1..108e2fc2a1768 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -234,7 +234,7 @@ def FormatExtensionList(items, prefix="extension", indent=2): f"{indent}- :ref:`{ext} <{prefix}_{ext}>`" for ext in items) - return f"%s\n{indent}\n" % formatted_list) + return f"{formatted_list}\n{indent}\n" def FormatExtensionCategory(extension_category): From c0fc9edf7fdd130841d0c2485d4479eca429915a Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Thu, 11 Feb 2021 11:18:39 +0000 Subject: [PATCH 084/113] tools/ Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index 108e2fc2a1768..e9ebec9814022 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -221,7 +221,7 @@ def FormatExtension(extension): extension=extension, status=status, security_posture=security_posture, - categories=extension_categories) + categories=categories) except KeyError as e: sys.stderr.write( '\n\nDid you forget to add an entry to source/extensions/extensions_build_config.bzl?\n\n') From d5d9ae81832cd27c126147a52e09ad5f1d9f9efb Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Thu, 11 Feb 2021 11:58:35 +0000 Subject: [PATCH 085/113] format Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index e9ebec9814022..302686fde02f5 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -67,7 +67,6 @@ """) - EXTENSION_CATEGORY_TEMPLATE = string.Template("""$anchor .. tip:: @@ -77,7 +76,6 @@ """) - # A map from the extension security postures (as defined in the # envoy_cc_extension build macro) to human readable text for extension docs. EXTENSION_SECURITY_POSTURES = { @@ -107,9 +105,7 @@ 'This extension is work-in-progress. Functionality is incomplete and it is not intended for production use.', } -EXTENSION_CATEGORIES = json.loads( - pathlib.Path( - os.getenv('EXTENSION_CAT_DB_PATH')).read_text()) +EXTENSION_CATEGORIES = json.loads(pathlib.Path(os.getenv('EXTENSION_CAT_DB_PATH')).read_text()) class ProtodocError(Exception): @@ -154,7 +150,8 @@ def FormatCommentWithAnnotations(comment, type_name=''): formatted_extension_category = '' if annotations.EXTENSION_CATEGORY_ANNOTATION in comment.annotations: formatted_extension_category = FormatExtensionCategory(comment.annotations[annotations.EXTENSION_CATEGORY_ANNOTATION]) - return annotations.WithoutAnnotations(StripLeadingSpace(comment.raw) + '\n') + formatted_extension + formatted_extension_category + comment = annotations.WithoutAnnotations(StripLeadingSpace(comment.raw) + '\n') + return comment + formatted_extension + formatted_extension_category def MapLines(f, s): @@ -230,10 +227,7 @@ def FormatExtension(extension): def FormatExtensionList(items, prefix="extension", indent=2): indent = " " * indent - formatted_list = "\n".join( - f"{indent}- :ref:`{ext} <{prefix}_{ext}>`" - for ext - in items) + formatted_list = "\n".join(f"{indent}- :ref:`{ext} <{prefix}_{ext}>`" for ext in items) return f"{formatted_list}\n{indent}\n" @@ -246,13 +240,11 @@ def FormatExtensionCategory(extension_category): Returns: RST formatted extension category description. """ - IGNORED_EXTENSIONS = [ - "envoy.tracers.xray", "envoy.filters.http.cache.simple_http_cache"] + IGNORED_EXTENSIONS = ["envoy.tracers.xray", "envoy.filters.http.cache.simple_http_cache"] try: extensions = EXTENSION_CATEGORIES[extension_category] except KeyError as e: - sys.stderr.write( - '\n\nUnable to find extension category: %s\n\n' % extension_category) + sys.stderr.write(f"\n\nUnable to find extension category: {extension_category}\n\n") exit(1) # Raising the error buries the above message in tracebacks. anchor = FormatAnchor('extension_category_' + extension_category) extensions = FormatExtensionList([x for x in extensions if x not in IGNORED_EXTENSIONS]) From 8e89914f71289c7f902ca67173332163672e69c7 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Thu, 11 Feb 2021 13:18:57 +0000 Subject: [PATCH 086/113] tools/ Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index 302686fde02f5..2f2fd500b56b7 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -149,7 +149,8 @@ def FormatCommentWithAnnotations(comment, type_name=''): formatted_extension = FormatExtension(extension) formatted_extension_category = '' if annotations.EXTENSION_CATEGORY_ANNOTATION in comment.annotations: - formatted_extension_category = FormatExtensionCategory(comment.annotations[annotations.EXTENSION_CATEGORY_ANNOTATION]) + formatted_extension_category = FormatExtensionCategory( + comment.annotations[annotations.EXTENSION_CATEGORY_ANNOTATION]) comment = annotations.WithoutAnnotations(StripLeadingSpace(comment.raw) + '\n') return comment + formatted_extension + formatted_extension_category From 816d7dc5fe0d940981c8750cc11fd9cf58d1960e Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Thu, 11 Feb 2021 14:33:44 +0000 Subject: [PATCH 087/113] docs/ Signed-off-by: Ryan Northey --- docs/generate_extension_cat_db.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/generate_extension_cat_db.py b/docs/generate_extension_cat_db.py index be4c09411951c..ff600bbb323c8 100755 --- a/docs/generate_extension_cat_db.py +++ b/docs/generate_extension_cat_db.py @@ -5,21 +5,21 @@ def main(): - extension_db = sys.argv[1] - extension_cat_db = sys.argv[2] + extension_db = sys.argv[1] + extension_cat_db = sys.argv[2] - with open(extension_db) as f: - data = json.load(f) + with open(extension_db) as f: + data = json.load(f) - categories = {} - for k, v in data.items(): - for cat in v['categories']: - categories[cat] = categories.get(cat, []) - categories[cat].append(k) + categories = {} + for k, v in data.items(): + for cat in v['categories']: + categories[cat] = categories.get(cat, []) + categories[cat].append(k) - with open(extension_cat_db, 'w') as f: - json.dump(categories, f) + with open(extension_cat_db, 'w') as f: + json.dump(categories, f) if __name__ == '__main__': - main() + main() From 7ba92c9f15233795492a5b205872341985df4936 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Thu, 11 Feb 2021 15:28:09 +0000 Subject: [PATCH 088/113] generated_api_shadow/ Signed-off-by: Ryan Northey --- generated_api_shadow/envoy/config/accesslog/v3/accesslog.proto | 3 ++- generated_api_shadow/envoy/config/bootstrap/v3/bootstrap.proto | 1 + generated_api_shadow/envoy/config/cluster/v3/cluster.proto | 1 + generated_api_shadow/envoy/config/core/v3/grpc_service.proto | 1 + generated_api_shadow/envoy/config/core/v3/health_check.proto | 1 + .../envoy/config/listener/v3/listener_components.proto | 1 + .../envoy/config/listener/v3/udp_listener_config.proto | 1 + generated_api_shadow/envoy/config/metrics/v3/stats.proto | 1 + generated_api_shadow/envoy/config/overload/v3/overload.proto | 1 + .../envoy/config/route/v3/route_components.proto | 1 + generated_api_shadow/envoy/config/trace/v3/http_tracer.proto | 1 + .../extensions/filters/http/compressor/v3/compressor.proto | 1 + .../extensions/filters/http/decompressor/v3/decompressor.proto | 1 + .../http_connection_manager/v3/http_connection_manager.proto | 1 + .../filters/network/thrift_proxy/v3/thrift_proxy.proto | 1 + generated_api_shadow/envoy/extensions/wasm/v3/wasm.proto | 1 + 16 files changed, 17 insertions(+), 1 deletion(-) diff --git a/generated_api_shadow/envoy/config/accesslog/v3/accesslog.proto b/generated_api_shadow/envoy/config/accesslog/v3/accesslog.proto index 3d8f3bb8e16e2..1f94cb7f0fe69 100644 --- a/generated_api_shadow/envoy/config/accesslog/v3/accesslog.proto +++ b/generated_api_shadow/envoy/config/accesslog/v3/accesslog.proto @@ -1,4 +1,4 @@ -syntax = "proto3"; + syntax = "proto3"; package envoy.config.accesslog.v3; @@ -46,6 +46,7 @@ message AccessLog { // ` // #. "envoy.access_loggers.tcp_grpc": :ref:`TcpGrpcAccessLogConfig // ` + // [#extension-category: envoy.access_loggers] oneof config_type { google.protobuf.Any typed_config = 4; diff --git a/generated_api_shadow/envoy/config/bootstrap/v3/bootstrap.proto b/generated_api_shadow/envoy/config/bootstrap/v3/bootstrap.proto index 9ac8927e19840..a6e6af054e6fa 100644 --- a/generated_api_shadow/envoy/config/bootstrap/v3/bootstrap.proto +++ b/generated_api_shadow/envoy/config/bootstrap/v3/bootstrap.proto @@ -189,6 +189,7 @@ message Bootstrap { // Optional watchdogs configuration. // This is used for specifying different watchdogs for the different subsystems. + // [#extension-category: envoy.guarddog_actions] Watchdogs watchdogs = 27; // Configuration for an external tracing provider. diff --git a/generated_api_shadow/envoy/config/cluster/v3/cluster.proto b/generated_api_shadow/envoy/config/cluster/v3/cluster.proto index 894ab6c188be9..556df43f51225 100644 --- a/generated_api_shadow/envoy/config/cluster/v3/cluster.proto +++ b/generated_api_shadow/envoy/config/cluster/v3/cluster.proto @@ -159,6 +159,7 @@ message Cluster { google.protobuf.Struct match = 2; // The configuration of the transport socket. + // [#extension-category: envoy.transport_sockets.upstream] core.v3.TransportSocket transport_socket = 3; } diff --git a/generated_api_shadow/envoy/config/core/v3/grpc_service.proto b/generated_api_shadow/envoy/config/core/v3/grpc_service.proto index fb05f3b73a5fd..648a972031cad 100644 --- a/generated_api_shadow/envoy/config/core/v3/grpc_service.proto +++ b/generated_api_shadow/envoy/config/core/v3/grpc_service.proto @@ -120,6 +120,7 @@ message GrpcService { string name = 1; + // [#extension-category: envoy.grpc_credentials] oneof config_type { google.protobuf.Any typed_config = 3; diff --git a/generated_api_shadow/envoy/config/core/v3/health_check.proto b/generated_api_shadow/envoy/config/core/v3/health_check.proto index 5f8fd325aa959..2e72212d31f11 100644 --- a/generated_api_shadow/envoy/config/core/v3/health_check.proto +++ b/generated_api_shadow/envoy/config/core/v3/health_check.proto @@ -187,6 +187,7 @@ message HealthCheck { // A custom health checker specific configuration which depends on the custom health checker // being instantiated. See :api:`envoy/config/health_checker` for reference. + // [#extension-category: envoy.health_checkers] oneof config_type { google.protobuf.Any typed_config = 3; diff --git a/generated_api_shadow/envoy/config/listener/v3/listener_components.proto b/generated_api_shadow/envoy/config/listener/v3/listener_components.proto index f96647e5d0b11..90d005adfa062 100644 --- a/generated_api_shadow/envoy/config/listener/v3/listener_components.proto +++ b/generated_api_shadow/envoy/config/listener/v3/listener_components.proto @@ -35,6 +35,7 @@ message Filter { // :ref:`supported filter `. string name = 1 [(validate.rules).string = {min_len: 1}]; + // [#extension-category: envoy.filters.network] oneof config_type { // Filter specific configuration which depends on the filter being // instantiated. See the supported filters for further documentation. diff --git a/generated_api_shadow/envoy/config/listener/v3/udp_listener_config.proto b/generated_api_shadow/envoy/config/listener/v3/udp_listener_config.proto index 7d9bef4b67690..8573e240d3c57 100644 --- a/generated_api_shadow/envoy/config/listener/v3/udp_listener_config.proto +++ b/generated_api_shadow/envoy/config/listener/v3/udp_listener_config.proto @@ -27,6 +27,7 @@ message UdpListenerConfig { // Used to create a specific listener factory. To some factory, e.g. // "raw_udp_listener", config is not needed. + // [#extension-category: envoy.filters.udp_listener] oneof config_type { google.protobuf.Any typed_config = 3; diff --git a/generated_api_shadow/envoy/config/metrics/v3/stats.proto b/generated_api_shadow/envoy/config/metrics/v3/stats.proto index 2d53df66f9c11..77da528d50269 100644 --- a/generated_api_shadow/envoy/config/metrics/v3/stats.proto +++ b/generated_api_shadow/envoy/config/metrics/v3/stats.proto @@ -38,6 +38,7 @@ message StatsSink { // Stats sink specific configuration which depends on the sink being instantiated. See // :ref:`StatsdSink ` for an example. + // [#extension-category: envoy.stats_sinks] oneof config_type { google.protobuf.Any typed_config = 3; diff --git a/generated_api_shadow/envoy/config/overload/v3/overload.proto b/generated_api_shadow/envoy/config/overload/v3/overload.proto index 567f9405c693f..ae23dd3c69bb1 100644 --- a/generated_api_shadow/envoy/config/overload/v3/overload.proto +++ b/generated_api_shadow/envoy/config/overload/v3/overload.proto @@ -38,6 +38,7 @@ message ResourceMonitor { string name = 1 [(validate.rules).string = {min_len: 1}]; // Configuration for the resource monitor being instantiated. + // [#extension-category: envoy.resource_monitors] oneof config_type { google.protobuf.Any typed_config = 3; diff --git a/generated_api_shadow/envoy/config/route/v3/route_components.proto b/generated_api_shadow/envoy/config/route/v3/route_components.proto index e4704dbebd175..a4c12d98d82d1 100644 --- a/generated_api_shadow/envoy/config/route/v3/route_components.proto +++ b/generated_api_shadow/envoy/config/route/v3/route_components.proto @@ -1127,6 +1127,7 @@ message RetryPolicy { string name = 1 [(validate.rules).string = {min_len: 1}]; + // [#extension-category: envoy.retry_priorities] oneof config_type { google.protobuf.Any typed_config = 3; diff --git a/generated_api_shadow/envoy/config/trace/v3/http_tracer.proto b/generated_api_shadow/envoy/config/trace/v3/http_tracer.proto index 23d0efc47a38e..b679310769a57 100644 --- a/generated_api_shadow/envoy/config/trace/v3/http_tracer.proto +++ b/generated_api_shadow/envoy/config/trace/v3/http_tracer.proto @@ -59,6 +59,7 @@ message Tracing { // - :ref:`DatadogConfig ` // - :ref:`OpenCensusConfig ` // - :ref:`AWS X-Ray ` + // [#extension-category: envoy.tracers] oneof config_type { google.protobuf.Any typed_config = 3; diff --git a/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto b/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto index 766560d9b5864..ed347cc439eb7 100644 --- a/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto +++ b/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto @@ -96,6 +96,7 @@ message Compressor { // :ref:`envoy.compression.gzip.compressor` // is included in Envoy. // This field is ignored if used in the context of the gzip http-filter, but is mandatory otherwise. + // [#extension-category: envoy.compression.compressor] config.core.v3.TypedExtensionConfig compressor_library = 6; // Configuration for request compression. Compression is disabled by default if left empty. diff --git a/generated_api_shadow/envoy/extensions/filters/http/decompressor/v3/decompressor.proto b/generated_api_shadow/envoy/extensions/filters/http/decompressor/v3/decompressor.proto index 058172fbaf64f..54d5f23bc449e 100644 --- a/generated_api_shadow/envoy/extensions/filters/http/decompressor/v3/decompressor.proto +++ b/generated_api_shadow/envoy/extensions/filters/http/decompressor/v3/decompressor.proto @@ -43,6 +43,7 @@ message Decompressor { // A decompressor library to use for both request and response decompression. Currently only // :ref:`envoy.compression.gzip.compressor` // is included in Envoy. + // [#extension-category: envoy.compression.decompressor] config.core.v3.TypedExtensionConfig decompressor_library = 1 [(validate.rules).message = {required: true}]; diff --git a/generated_api_shadow/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto b/generated_api_shadow/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto index 844313e1ef972..09f721ac896a4 100644 --- a/generated_api_shadow/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto +++ b/generated_api_shadow/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto @@ -272,6 +272,7 @@ message HttpConnectionManager { // A list of individual HTTP filters that make up the filter chain for // requests made to the connection manager. :ref:`Order matters ` // as the filters are processed sequentially as request events happen. + // [#extension-category: envoy.filters.http] repeated HttpFilter http_filters = 5; // Whether the connection manager manipulates the :ref:`config_http_conn_man_headers_user-agent` diff --git a/generated_api_shadow/envoy/extensions/filters/network/thrift_proxy/v3/thrift_proxy.proto b/generated_api_shadow/envoy/extensions/filters/network/thrift_proxy/v3/thrift_proxy.proto index 4fc04f2d88033..8485693caaf9e 100644 --- a/generated_api_shadow/envoy/extensions/filters/network/thrift_proxy/v3/thrift_proxy.proto +++ b/generated_api_shadow/envoy/extensions/filters/network/thrift_proxy/v3/thrift_proxy.proto @@ -81,6 +81,7 @@ message ThriftProxy { // Thrift proxy. Order matters as the filters are processed sequentially. For backwards // compatibility, if no thrift_filters are specified, a default Thrift router filter // (`envoy.filters.thrift.router`) is used. + // [#extension-category: envoy.thrift_proxy.filters] repeated ThriftFilter thrift_filters = 5; // If set to true, Envoy will try to skip decode data after metadata in the Thrift message. diff --git a/generated_api_shadow/envoy/extensions/wasm/v3/wasm.proto b/generated_api_shadow/envoy/extensions/wasm/v3/wasm.proto index 2232b11075c8d..af2b7a38e310a 100644 --- a/generated_api_shadow/envoy/extensions/wasm/v3/wasm.proto +++ b/generated_api_shadow/envoy/extensions/wasm/v3/wasm.proto @@ -59,6 +59,7 @@ message VmConfig { // (proxy_on_start). `google.protobuf.Struct` is serialized as JSON before // passing it to the plugin. `google.protobuf.BytesValue` and // `google.protobuf.StringValue` are passed directly without the wrapper. + // [#extension-category: envoy.wasm.runtime] google.protobuf.Any configuration = 4; // Allow the wasm file to include pre-compiled code on VMs which support it. From 689d9d880b1a934da3af1726edf28e9d9e1340a6 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Fri, 12 Feb 2021 13:27:56 +0000 Subject: [PATCH 089/113] tools/ Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 2 +- tools/spelling/spelling_dictionary.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index 2f2fd500b56b7..339ecbb79ade6 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -150,7 +150,7 @@ def FormatCommentWithAnnotations(comment, type_name=''): formatted_extension_category = '' if annotations.EXTENSION_CATEGORY_ANNOTATION in comment.annotations: formatted_extension_category = FormatExtensionCategory( - comment.annotations[annotations.EXTENSION_CATEGORY_ANNOTATION]) + comment.annotations[annotations.EXTENSION_CATEGORY_ANNOTATION]) comment = annotations.WithoutAnnotations(StripLeadingSpace(comment.raw) + '\n') return comment + formatted_extension + formatted_extension_category diff --git a/tools/spelling/spelling_dictionary.txt b/tools/spelling/spelling_dictionary.txt index 48f4c1d953de0..a9e4cdc025de4 100644 --- a/tools/spelling/spelling_dictionary.txt +++ b/tools/spelling/spelling_dictionary.txt @@ -115,6 +115,7 @@ FQDN FREEBIND FUZZER FUZZERS +guarddog GC GCC GCE From c907f4681577108e67e7bdbc5cd6dfdb5b5d3a3b Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Fri, 12 Feb 2021 13:28:42 +0000 Subject: [PATCH 090/113] generated_api_shadow/ Signed-off-by: Ryan Northey --- generated_api_shadow/envoy/config/accesslog/v3/accesslog.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generated_api_shadow/envoy/config/accesslog/v3/accesslog.proto b/generated_api_shadow/envoy/config/accesslog/v3/accesslog.proto index 1f94cb7f0fe69..b266ea21b9886 100644 --- a/generated_api_shadow/envoy/config/accesslog/v3/accesslog.proto +++ b/generated_api_shadow/envoy/config/accesslog/v3/accesslog.proto @@ -1,4 +1,4 @@ - syntax = "proto3"; +syntax = "proto3"; package envoy.config.accesslog.v3; From ab0cf37ba9089d3707e1b058299b58dd67d964c3 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Fri, 12 Feb 2021 16:10:09 +0000 Subject: [PATCH 091/113] api/ Signed-off-by: Ryan Northey --- api/envoy/config/accesslog/v4alpha/accesslog.proto | 1 + api/envoy/config/bootstrap/v4alpha/bootstrap.proto | 2 ++ api/envoy/config/cluster/v4alpha/cluster.proto | 3 +++ api/envoy/config/core/v4alpha/grpc_service.proto | 1 + api/envoy/config/core/v4alpha/health_check.proto | 1 + api/envoy/config/listener/v4alpha/listener_components.proto | 3 +++ api/envoy/config/listener/v4alpha/udp_listener_config.proto | 1 + api/envoy/config/metrics/v4alpha/stats.proto | 1 + api/envoy/config/route/v4alpha/route_components.proto | 4 ++++ api/envoy/config/trace/v4alpha/http_tracer.proto | 1 + .../filters/http/compressor/v4alpha/compressor.proto | 1 + .../v4alpha/http_connection_manager.proto | 1 + .../filters/network/thrift_proxy/v4alpha/thrift_proxy.proto | 1 + 13 files changed, 21 insertions(+) diff --git a/api/envoy/config/accesslog/v4alpha/accesslog.proto b/api/envoy/config/accesslog/v4alpha/accesslog.proto index b812d77e446aa..62bc8af6913bb 100644 --- a/api/envoy/config/accesslog/v4alpha/accesslog.proto +++ b/api/envoy/config/accesslog/v4alpha/accesslog.proto @@ -49,6 +49,7 @@ message AccessLog { // ` // #. "envoy.access_loggers.tcp_grpc": :ref:`TcpGrpcAccessLogConfig // ` + // [#extension-category: envoy.access_loggers] oneof config_type { google.protobuf.Any typed_config = 4; } diff --git a/api/envoy/config/bootstrap/v4alpha/bootstrap.proto b/api/envoy/config/bootstrap/v4alpha/bootstrap.proto index 9d9552e425c20..0b055dac26810 100644 --- a/api/envoy/config/bootstrap/v4alpha/bootstrap.proto +++ b/api/envoy/config/bootstrap/v4alpha/bootstrap.proto @@ -180,6 +180,7 @@ message Bootstrap { // Optional watchdogs configuration. // This is used for specifying different watchdogs for the different subsystems. + // [#extension-category: envoy.guarddog_actions] Watchdogs watchdogs = 27; // Configuration for the runtime configuration provider. If not @@ -232,6 +233,7 @@ message Bootstrap { // Specifies optional bootstrap extensions to be instantiated at startup time. // Each item contains extension specific configuration. + // [#extension-category: envoy.bootstrap] repeated core.v4alpha.TypedExtensionConfig bootstrap_extensions = 21; // Specifies optional extensions instantiated at startup time and diff --git a/api/envoy/config/cluster/v4alpha/cluster.proto b/api/envoy/config/cluster/v4alpha/cluster.proto index 4e6015b815958..062123fc42060 100644 --- a/api/envoy/config/cluster/v4alpha/cluster.proto +++ b/api/envoy/config/cluster/v4alpha/cluster.proto @@ -160,6 +160,7 @@ message Cluster { google.protobuf.Struct match = 2; // The configuration of the transport socket. + // [#extension-category: envoy.transport_sockets.upstream] core.v4alpha.TransportSocket transport_socket = 3; } @@ -173,6 +174,7 @@ message Cluster { // Cluster specific configuration which depends on the cluster being instantiated. // See the supported cluster for further documentation. + // [#extension-category: envoy.clusters] google.protobuf.Any typed_config = 2; } @@ -945,6 +947,7 @@ message Cluster { // If users desire custom connection pool or upstream behavior, for example terminating // CONNECT only if a custom filter indicates it is appropriate, the custom factories // can be registered and configured here. + // [#extension-category: envoy.upstreams] core.v4alpha.TypedExtensionConfig upstream_config = 48; // Configuration to track optional cluster stats. diff --git a/api/envoy/config/core/v4alpha/grpc_service.proto b/api/envoy/config/core/v4alpha/grpc_service.proto index 9ea35b4564709..c7c284f1bdcd8 100644 --- a/api/envoy/config/core/v4alpha/grpc_service.proto +++ b/api/envoy/config/core/v4alpha/grpc_service.proto @@ -124,6 +124,7 @@ message GrpcService { string name = 1; + // [#extension-category: envoy.grpc_credentials] oneof config_type { google.protobuf.Any typed_config = 3; } diff --git a/api/envoy/config/core/v4alpha/health_check.proto b/api/envoy/config/core/v4alpha/health_check.proto index 8d870680a466d..ddc7d65e00075 100644 --- a/api/envoy/config/core/v4alpha/health_check.proto +++ b/api/envoy/config/core/v4alpha/health_check.proto @@ -189,6 +189,7 @@ message HealthCheck { // A custom health checker specific configuration which depends on the custom health checker // being instantiated. See :api:`envoy/config/health_checker` for reference. + // [#extension-category: envoy.health_checkers] oneof config_type { google.protobuf.Any typed_config = 3; } diff --git a/api/envoy/config/listener/v4alpha/listener_components.proto b/api/envoy/config/listener/v4alpha/listener_components.proto index 04d8ef644300a..682334b3fdb5b 100644 --- a/api/envoy/config/listener/v4alpha/listener_components.proto +++ b/api/envoy/config/listener/v4alpha/listener_components.proto @@ -35,6 +35,7 @@ message Filter { // :ref:`supported filter `. string name = 1 [(validate.rules).string = {min_len: 1}]; + // [#extension-category: envoy.filters.network] oneof config_type { // Filter specific configuration which depends on the filter being // instantiated. See the supported filters for further documentation. @@ -323,10 +324,12 @@ message ListenerFilter { // The name of the filter to instantiate. The name must match a // :ref:`supported filter `. + // [#extension-category: envoy.transport_sockets.downstream] string name = 1 [(validate.rules).string = {min_len: 1}]; // Filter specific configuration which depends on the filter being instantiated. // See the supported filters for further documentation. + // [#extension-category: envoy.filters.listener] oneof config_type { google.protobuf.Any typed_config = 3; } diff --git a/api/envoy/config/listener/v4alpha/udp_listener_config.proto b/api/envoy/config/listener/v4alpha/udp_listener_config.proto index 96eebee2d237e..9d6a3917a1526 100644 --- a/api/envoy/config/listener/v4alpha/udp_listener_config.proto +++ b/api/envoy/config/listener/v4alpha/udp_listener_config.proto @@ -30,6 +30,7 @@ message UdpListenerConfig { // Used to create a specific listener factory. To some factory, e.g. // "raw_udp_listener", config is not needed. + // [#extension-category: envoy.filters.udp_listener] oneof config_type { google.protobuf.Any typed_config = 3; } diff --git a/api/envoy/config/metrics/v4alpha/stats.proto b/api/envoy/config/metrics/v4alpha/stats.proto index 02d7c18db5ce6..3a351cdd094c6 100644 --- a/api/envoy/config/metrics/v4alpha/stats.proto +++ b/api/envoy/config/metrics/v4alpha/stats.proto @@ -41,6 +41,7 @@ message StatsSink { // Stats sink specific configuration which depends on the sink being instantiated. See // :ref:`StatsdSink ` for an example. + // [#extension-category: envoy.stats_sinks] oneof config_type { google.protobuf.Any typed_config = 3; } diff --git a/api/envoy/config/route/v4alpha/route_components.proto b/api/envoy/config/route/v4alpha/route_components.proto index 044fe5ff20025..7aa36354ec269 100644 --- a/api/envoy/config/route/v4alpha/route_components.proto +++ b/api/envoy/config/route/v4alpha/route_components.proto @@ -1061,6 +1061,7 @@ message RetryPolicy { string name = 1 [(validate.rules).string = {min_len: 1}]; + // [#extension-category: envoy.retry_priorities] oneof config_type { google.protobuf.Any typed_config = 3; } @@ -1076,6 +1077,7 @@ message RetryPolicy { string name = 1 [(validate.rules).string = {min_len: 1}]; + // [#extension-category: envoy.retry_host_predicates] oneof config_type { google.protobuf.Any typed_config = 3; } @@ -1699,6 +1701,7 @@ message RateLimit { MetaData metadata = 8; // Rate limit descriptor extension. See the rate limit descriptor extensions documentation. + // [#extension-category: envoy.rate_limit_descriptors] core.v4alpha.TypedExtensionConfig extension = 9; } } @@ -1901,6 +1904,7 @@ message InternalRedirectPolicy { // Specifies a list of predicates that are queried when an upstream response is deemed // to trigger an internal redirect by all other criteria. Any predicate in the list can reject // the redirect, causing the response to be proxied to downstream. + // [#extension-category: envoy.internal_redirect_predicates] repeated core.v4alpha.TypedExtensionConfig predicates = 3; // Allow internal redirect to follow a target URI with a different scheme than the value of diff --git a/api/envoy/config/trace/v4alpha/http_tracer.proto b/api/envoy/config/trace/v4alpha/http_tracer.proto index 2a7d3134312ba..4b8c681688425 100644 --- a/api/envoy/config/trace/v4alpha/http_tracer.proto +++ b/api/envoy/config/trace/v4alpha/http_tracer.proto @@ -62,6 +62,7 @@ message Tracing { // - :ref:`DatadogConfig ` // - :ref:`OpenCensusConfig ` // - :ref:`AWS X-Ray ` + // [#extension-category: envoy.tracers] oneof config_type { google.protobuf.Any typed_config = 3; } diff --git a/api/envoy/extensions/filters/http/compressor/v4alpha/compressor.proto b/api/envoy/extensions/filters/http/compressor/v4alpha/compressor.proto index 419c27da2efb1..d379bf48ce699 100644 --- a/api/envoy/extensions/filters/http/compressor/v4alpha/compressor.proto +++ b/api/envoy/extensions/filters/http/compressor/v4alpha/compressor.proto @@ -83,6 +83,7 @@ message Compressor { // :ref:`envoy.compression.gzip.compressor` // is included in Envoy. // This field is ignored if used in the context of the gzip http-filter, but is mandatory otherwise. + // [#extension-category: envoy.compression.compressor] config.core.v4alpha.TypedExtensionConfig compressor_library = 6; // Configuration for request compression. Compression is disabled by default if left empty. diff --git a/api/envoy/extensions/filters/network/http_connection_manager/v4alpha/http_connection_manager.proto b/api/envoy/extensions/filters/network/http_connection_manager/v4alpha/http_connection_manager.proto index 619054bba4a06..7d55b26e71005 100644 --- a/api/envoy/extensions/filters/network/http_connection_manager/v4alpha/http_connection_manager.proto +++ b/api/envoy/extensions/filters/network/http_connection_manager/v4alpha/http_connection_manager.proto @@ -267,6 +267,7 @@ message HttpConnectionManager { // A list of individual HTTP filters that make up the filter chain for // requests made to the connection manager. :ref:`Order matters ` // as the filters are processed sequentially as request events happen. + // [#extension-category: envoy.filters.http] repeated HttpFilter http_filters = 5; // Whether the connection manager manipulates the :ref:`config_http_conn_man_headers_user-agent` diff --git a/api/envoy/extensions/filters/network/thrift_proxy/v4alpha/thrift_proxy.proto b/api/envoy/extensions/filters/network/thrift_proxy/v4alpha/thrift_proxy.proto index 05f8c54f6c63b..1834202272362 100644 --- a/api/envoy/extensions/filters/network/thrift_proxy/v4alpha/thrift_proxy.proto +++ b/api/envoy/extensions/filters/network/thrift_proxy/v4alpha/thrift_proxy.proto @@ -80,6 +80,7 @@ message ThriftProxy { // Thrift proxy. Order matters as the filters are processed sequentially. For backwards // compatibility, if no thrift_filters are specified, a default Thrift router filter // (`envoy.filters.thrift.router`) is used. + // [#extension-category: envoy.thrift_proxy.filters] repeated ThriftFilter thrift_filters = 5; // If set to true, Envoy will try to skip decode data after metadata in the Thrift message. From 2e56eb6be52de13af2a1a17b940de959913c9681 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Mon, 15 Feb 2021 13:42:12 +0000 Subject: [PATCH 092/113] generated_api_shadow/ Signed-off-by: Ryan Northey --- .../envoy/config/accesslog/v4alpha/accesslog.proto | 1 + .../envoy/config/bootstrap/v3/bootstrap.proto | 1 + .../envoy/config/bootstrap/v4alpha/bootstrap.proto | 2 ++ generated_api_shadow/envoy/config/cluster/v3/cluster.proto | 2 ++ .../envoy/config/cluster/v4alpha/cluster.proto | 3 +++ .../envoy/config/core/v4alpha/grpc_service.proto | 1 + .../envoy/config/core/v4alpha/health_check.proto | 1 + .../envoy/config/listener/v3/listener_components.proto | 1 + .../envoy/config/listener/v4alpha/listener_components.proto | 3 +++ .../envoy/config/listener/v4alpha/udp_listener_config.proto | 1 + generated_api_shadow/envoy/config/metrics/v4alpha/stats.proto | 1 + .../envoy/config/route/v3/route_components.proto | 3 +++ .../envoy/config/route/v4alpha/route_components.proto | 4 ++++ .../envoy/config/trace/v4alpha/http_tracer.proto | 1 + .../filters/http/compressor/v4alpha/compressor.proto | 1 + .../v4alpha/http_connection_manager.proto | 1 + .../filters/network/thrift_proxy/v4alpha/thrift_proxy.proto | 1 + 17 files changed, 28 insertions(+) diff --git a/generated_api_shadow/envoy/config/accesslog/v4alpha/accesslog.proto b/generated_api_shadow/envoy/config/accesslog/v4alpha/accesslog.proto index b812d77e446aa..62bc8af6913bb 100644 --- a/generated_api_shadow/envoy/config/accesslog/v4alpha/accesslog.proto +++ b/generated_api_shadow/envoy/config/accesslog/v4alpha/accesslog.proto @@ -49,6 +49,7 @@ message AccessLog { // ` // #. "envoy.access_loggers.tcp_grpc": :ref:`TcpGrpcAccessLogConfig // ` + // [#extension-category: envoy.access_loggers] oneof config_type { google.protobuf.Any typed_config = 4; } diff --git a/generated_api_shadow/envoy/config/bootstrap/v3/bootstrap.proto b/generated_api_shadow/envoy/config/bootstrap/v3/bootstrap.proto index a6e6af054e6fa..30b03dae8a3be 100644 --- a/generated_api_shadow/envoy/config/bootstrap/v3/bootstrap.proto +++ b/generated_api_shadow/envoy/config/bootstrap/v3/bootstrap.proto @@ -249,6 +249,7 @@ message Bootstrap { // Specifies optional bootstrap extensions to be instantiated at startup time. // Each item contains extension specific configuration. + // [#extension-category: envoy.bootstrap] repeated core.v3.TypedExtensionConfig bootstrap_extensions = 21; // Specifies optional extensions instantiated at startup time and diff --git a/generated_api_shadow/envoy/config/bootstrap/v4alpha/bootstrap.proto b/generated_api_shadow/envoy/config/bootstrap/v4alpha/bootstrap.proto index 69c499186bb5d..5928b89f00494 100644 --- a/generated_api_shadow/envoy/config/bootstrap/v4alpha/bootstrap.proto +++ b/generated_api_shadow/envoy/config/bootstrap/v4alpha/bootstrap.proto @@ -186,6 +186,7 @@ message Bootstrap { // Optional watchdogs configuration. // This is used for specifying different watchdogs for the different subsystems. + // [#extension-category: envoy.guarddog_actions] Watchdogs watchdogs = 27; // Configuration for an external tracing provider. @@ -245,6 +246,7 @@ message Bootstrap { // Specifies optional bootstrap extensions to be instantiated at startup time. // Each item contains extension specific configuration. + // [#extension-category: envoy.bootstrap] repeated core.v4alpha.TypedExtensionConfig bootstrap_extensions = 21; // Specifies optional extensions instantiated at startup time and diff --git a/generated_api_shadow/envoy/config/cluster/v3/cluster.proto b/generated_api_shadow/envoy/config/cluster/v3/cluster.proto index 556df43f51225..e47d3bf00e114 100644 --- a/generated_api_shadow/envoy/config/cluster/v3/cluster.proto +++ b/generated_api_shadow/envoy/config/cluster/v3/cluster.proto @@ -173,6 +173,7 @@ message Cluster { // Cluster specific configuration which depends on the cluster being instantiated. // See the supported cluster for further documentation. + // [#extension-category: envoy.clusters] google.protobuf.Any typed_config = 2; } @@ -1004,6 +1005,7 @@ message Cluster { // If users desire custom connection pool or upstream behavior, for example terminating // CONNECT only if a custom filter indicates it is appropriate, the custom factories // can be registered and configured here. + // [#extension-category: envoy.upstreams] core.v3.TypedExtensionConfig upstream_config = 48; // Configuration to track optional cluster stats. diff --git a/generated_api_shadow/envoy/config/cluster/v4alpha/cluster.proto b/generated_api_shadow/envoy/config/cluster/v4alpha/cluster.proto index 3ffa88ddde6a6..ddac036831180 100644 --- a/generated_api_shadow/envoy/config/cluster/v4alpha/cluster.proto +++ b/generated_api_shadow/envoy/config/cluster/v4alpha/cluster.proto @@ -161,6 +161,7 @@ message Cluster { google.protobuf.Struct match = 2; // The configuration of the transport socket. + // [#extension-category: envoy.transport_sockets.upstream] core.v4alpha.TransportSocket transport_socket = 3; } @@ -174,6 +175,7 @@ message Cluster { // Cluster specific configuration which depends on the cluster being instantiated. // See the supported cluster for further documentation. + // [#extension-category: envoy.clusters] google.protobuf.Any typed_config = 2; } @@ -1017,6 +1019,7 @@ message Cluster { // If users desire custom connection pool or upstream behavior, for example terminating // CONNECT only if a custom filter indicates it is appropriate, the custom factories // can be registered and configured here. + // [#extension-category: envoy.upstreams] core.v4alpha.TypedExtensionConfig upstream_config = 48; // Configuration to track optional cluster stats. diff --git a/generated_api_shadow/envoy/config/core/v4alpha/grpc_service.proto b/generated_api_shadow/envoy/config/core/v4alpha/grpc_service.proto index 9ea35b4564709..c7c284f1bdcd8 100644 --- a/generated_api_shadow/envoy/config/core/v4alpha/grpc_service.proto +++ b/generated_api_shadow/envoy/config/core/v4alpha/grpc_service.proto @@ -124,6 +124,7 @@ message GrpcService { string name = 1; + // [#extension-category: envoy.grpc_credentials] oneof config_type { google.protobuf.Any typed_config = 3; } diff --git a/generated_api_shadow/envoy/config/core/v4alpha/health_check.proto b/generated_api_shadow/envoy/config/core/v4alpha/health_check.proto index 8d870680a466d..ddc7d65e00075 100644 --- a/generated_api_shadow/envoy/config/core/v4alpha/health_check.proto +++ b/generated_api_shadow/envoy/config/core/v4alpha/health_check.proto @@ -189,6 +189,7 @@ message HealthCheck { // A custom health checker specific configuration which depends on the custom health checker // being instantiated. See :api:`envoy/config/health_checker` for reference. + // [#extension-category: envoy.health_checkers] oneof config_type { google.protobuf.Any typed_config = 3; } diff --git a/generated_api_shadow/envoy/config/listener/v3/listener_components.proto b/generated_api_shadow/envoy/config/listener/v3/listener_components.proto index 90d005adfa062..6c42a9645aaf0 100644 --- a/generated_api_shadow/envoy/config/listener/v3/listener_components.proto +++ b/generated_api_shadow/envoy/config/listener/v3/listener_components.proto @@ -33,6 +33,7 @@ message Filter { // The name of the filter to instantiate. The name must match a // :ref:`supported filter `. + // [#extension-category: envoy.transport_sockets.downstream] string name = 1 [(validate.rules).string = {min_len: 1}]; // [#extension-category: envoy.filters.network] diff --git a/generated_api_shadow/envoy/config/listener/v4alpha/listener_components.proto b/generated_api_shadow/envoy/config/listener/v4alpha/listener_components.proto index 2b3c218c7bdfe..bb03a9226834f 100644 --- a/generated_api_shadow/envoy/config/listener/v4alpha/listener_components.proto +++ b/generated_api_shadow/envoy/config/listener/v4alpha/listener_components.proto @@ -35,6 +35,7 @@ message Filter { // :ref:`supported filter `. string name = 1 [(validate.rules).string = {min_len: 1}]; + // [#extension-category: envoy.filters.network] oneof config_type { // Filter specific configuration which depends on the filter being // instantiated. See the supported filters for further documentation. @@ -335,10 +336,12 @@ message ListenerFilter { // The name of the filter to instantiate. The name must match a // :ref:`supported filter `. + // [#extension-category: envoy.transport_sockets.downstream] string name = 1 [(validate.rules).string = {min_len: 1}]; // Filter specific configuration which depends on the filter being instantiated. // See the supported filters for further documentation. + // [#extension-category: envoy.filters.listener] oneof config_type { google.protobuf.Any typed_config = 3; } diff --git a/generated_api_shadow/envoy/config/listener/v4alpha/udp_listener_config.proto b/generated_api_shadow/envoy/config/listener/v4alpha/udp_listener_config.proto index 96eebee2d237e..9d6a3917a1526 100644 --- a/generated_api_shadow/envoy/config/listener/v4alpha/udp_listener_config.proto +++ b/generated_api_shadow/envoy/config/listener/v4alpha/udp_listener_config.proto @@ -30,6 +30,7 @@ message UdpListenerConfig { // Used to create a specific listener factory. To some factory, e.g. // "raw_udp_listener", config is not needed. + // [#extension-category: envoy.filters.udp_listener] oneof config_type { google.protobuf.Any typed_config = 3; } diff --git a/generated_api_shadow/envoy/config/metrics/v4alpha/stats.proto b/generated_api_shadow/envoy/config/metrics/v4alpha/stats.proto index 02d7c18db5ce6..3a351cdd094c6 100644 --- a/generated_api_shadow/envoy/config/metrics/v4alpha/stats.proto +++ b/generated_api_shadow/envoy/config/metrics/v4alpha/stats.proto @@ -41,6 +41,7 @@ message StatsSink { // Stats sink specific configuration which depends on the sink being instantiated. See // :ref:`StatsdSink ` for an example. + // [#extension-category: envoy.stats_sinks] oneof config_type { google.protobuf.Any typed_config = 3; } diff --git a/generated_api_shadow/envoy/config/route/v3/route_components.proto b/generated_api_shadow/envoy/config/route/v3/route_components.proto index a4c12d98d82d1..a8e1f9478572c 100644 --- a/generated_api_shadow/envoy/config/route/v3/route_components.proto +++ b/generated_api_shadow/envoy/config/route/v3/route_components.proto @@ -1141,6 +1141,7 @@ message RetryPolicy { string name = 1 [(validate.rules).string = {min_len: 1}]; + // [#extension-category: envoy.retry_host_predicates] oneof config_type { google.protobuf.Any typed_config = 3; @@ -1761,6 +1762,7 @@ message RateLimit { MetaData metadata = 8; // Rate limit descriptor extension. See the rate limit descriptor extensions documentation. + // [#extension-category: envoy.rate_limit_descriptors] core.v3.TypedExtensionConfig extension = 9; } } @@ -1958,6 +1960,7 @@ message InternalRedirectPolicy { // Specifies a list of predicates that are queried when an upstream response is deemed // to trigger an internal redirect by all other criteria. Any predicate in the list can reject // the redirect, causing the response to be proxied to downstream. + // [#extension-category: envoy.internal_redirect_predicates] repeated core.v3.TypedExtensionConfig predicates = 3; // Allow internal redirect to follow a target URI with a different scheme than the value of diff --git a/generated_api_shadow/envoy/config/route/v4alpha/route_components.proto b/generated_api_shadow/envoy/config/route/v4alpha/route_components.proto index 83c496d059044..c912b38d23f3f 100644 --- a/generated_api_shadow/envoy/config/route/v4alpha/route_components.proto +++ b/generated_api_shadow/envoy/config/route/v4alpha/route_components.proto @@ -1128,6 +1128,7 @@ message RetryPolicy { string name = 1 [(validate.rules).string = {min_len: 1}]; + // [#extension-category: envoy.retry_priorities] oneof config_type { google.protobuf.Any typed_config = 3; } @@ -1143,6 +1144,7 @@ message RetryPolicy { string name = 1 [(validate.rules).string = {min_len: 1}]; + // [#extension-category: envoy.retry_host_predicates] oneof config_type { google.protobuf.Any typed_config = 3; } @@ -1769,6 +1771,7 @@ message RateLimit { MetaData metadata = 8; // Rate limit descriptor extension. See the rate limit descriptor extensions documentation. + // [#extension-category: envoy.rate_limit_descriptors] core.v4alpha.TypedExtensionConfig extension = 9; } } @@ -1971,6 +1974,7 @@ message InternalRedirectPolicy { // Specifies a list of predicates that are queried when an upstream response is deemed // to trigger an internal redirect by all other criteria. Any predicate in the list can reject // the redirect, causing the response to be proxied to downstream. + // [#extension-category: envoy.internal_redirect_predicates] repeated core.v4alpha.TypedExtensionConfig predicates = 3; // Allow internal redirect to follow a target URI with a different scheme than the value of diff --git a/generated_api_shadow/envoy/config/trace/v4alpha/http_tracer.proto b/generated_api_shadow/envoy/config/trace/v4alpha/http_tracer.proto index 2a7d3134312ba..4b8c681688425 100644 --- a/generated_api_shadow/envoy/config/trace/v4alpha/http_tracer.proto +++ b/generated_api_shadow/envoy/config/trace/v4alpha/http_tracer.proto @@ -62,6 +62,7 @@ message Tracing { // - :ref:`DatadogConfig ` // - :ref:`OpenCensusConfig ` // - :ref:`AWS X-Ray ` + // [#extension-category: envoy.tracers] oneof config_type { google.protobuf.Any typed_config = 3; } diff --git a/generated_api_shadow/envoy/extensions/filters/http/compressor/v4alpha/compressor.proto b/generated_api_shadow/envoy/extensions/filters/http/compressor/v4alpha/compressor.proto index 40dfaa636408e..419e690051e38 100644 --- a/generated_api_shadow/envoy/extensions/filters/http/compressor/v4alpha/compressor.proto +++ b/generated_api_shadow/envoy/extensions/filters/http/compressor/v4alpha/compressor.proto @@ -106,6 +106,7 @@ message Compressor { // :ref:`envoy.compression.gzip.compressor` // is included in Envoy. // This field is ignored if used in the context of the gzip http-filter, but is mandatory otherwise. + // [#extension-category: envoy.compression.compressor] config.core.v4alpha.TypedExtensionConfig compressor_library = 6; // Configuration for request compression. Compression is disabled by default if left empty. diff --git a/generated_api_shadow/envoy/extensions/filters/network/http_connection_manager/v4alpha/http_connection_manager.proto b/generated_api_shadow/envoy/extensions/filters/network/http_connection_manager/v4alpha/http_connection_manager.proto index 619054bba4a06..7d55b26e71005 100644 --- a/generated_api_shadow/envoy/extensions/filters/network/http_connection_manager/v4alpha/http_connection_manager.proto +++ b/generated_api_shadow/envoy/extensions/filters/network/http_connection_manager/v4alpha/http_connection_manager.proto @@ -267,6 +267,7 @@ message HttpConnectionManager { // A list of individual HTTP filters that make up the filter chain for // requests made to the connection manager. :ref:`Order matters ` // as the filters are processed sequentially as request events happen. + // [#extension-category: envoy.filters.http] repeated HttpFilter http_filters = 5; // Whether the connection manager manipulates the :ref:`config_http_conn_man_headers_user-agent` diff --git a/generated_api_shadow/envoy/extensions/filters/network/thrift_proxy/v4alpha/thrift_proxy.proto b/generated_api_shadow/envoy/extensions/filters/network/thrift_proxy/v4alpha/thrift_proxy.proto index 05f8c54f6c63b..1834202272362 100644 --- a/generated_api_shadow/envoy/extensions/filters/network/thrift_proxy/v4alpha/thrift_proxy.proto +++ b/generated_api_shadow/envoy/extensions/filters/network/thrift_proxy/v4alpha/thrift_proxy.proto @@ -80,6 +80,7 @@ message ThriftProxy { // Thrift proxy. Order matters as the filters are processed sequentially. For backwards // compatibility, if no thrift_filters are specified, a default Thrift router filter // (`envoy.filters.thrift.router`) is used. + // [#extension-category: envoy.thrift_proxy.filters] repeated ThriftFilter thrift_filters = 5; // If set to true, Envoy will try to skip decode data after metadata in the Thrift message. From cccc21deaa54543198468a0a307e6e738d5d26c2 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Mon, 15 Feb 2021 14:21:58 +0000 Subject: [PATCH 093/113] generated_api_shadow/ Signed-off-by: Ryan Northey --- .../envoy/config/listener/v3/listener_components.proto | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/generated_api_shadow/envoy/config/listener/v3/listener_components.proto b/generated_api_shadow/envoy/config/listener/v3/listener_components.proto index 6c42a9645aaf0..1124e820467db 100644 --- a/generated_api_shadow/envoy/config/listener/v3/listener_components.proto +++ b/generated_api_shadow/envoy/config/listener/v3/listener_components.proto @@ -33,7 +33,6 @@ message Filter { // The name of the filter to instantiate. The name must match a // :ref:`supported filter `. - // [#extension-category: envoy.transport_sockets.downstream] string name = 1 [(validate.rules).string = {min_len: 1}]; // [#extension-category: envoy.filters.network] @@ -330,10 +329,12 @@ message ListenerFilter { // The name of the filter to instantiate. The name must match a // :ref:`supported filter `. + // [#extension-category: envoy.transport_sockets.downstream] string name = 1 [(validate.rules).string = {min_len: 1}]; // Filter specific configuration which depends on the filter being instantiated. // See the supported filters for further documentation. + // [#extension-category: envoy.filters.listener] oneof config_type { google.protobuf.Any typed_config = 3; From 0207b35ccfd1ddefe50c7387e89e42305f22c1f6 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Mon, 15 Feb 2021 14:53:48 +0000 Subject: [PATCH 094/113] genBUILD Signed-off-by: Ryan Northey --- generated_api_shadow/BUILD | 1 - 1 file changed, 1 deletion(-) diff --git a/generated_api_shadow/BUILD b/generated_api_shadow/BUILD index e2e5b3bfeb772..046fac3d012de 100644 --- a/generated_api_shadow/BUILD +++ b/generated_api_shadow/BUILD @@ -145,7 +145,6 @@ proto_library( "//envoy/config/resource_monitor/injected_resource/v2alpha:pkg", "//envoy/config/retry/omit_canary_hosts/v2:pkg", "//envoy/config/retry/previous_hosts/v2:pkg", - "//envoy/config/retry/previous_hosts/v3:pkg", "//envoy/config/route/v3:pkg", "//envoy/config/tap/v3:pkg", "//envoy/config/trace/v3:pkg", From 0b6aa7681160d35a57c494f92c3aa12936ee1127 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Mon, 15 Feb 2021 14:54:36 +0000 Subject: [PATCH 095/113] docs/ Signed-off-by: Ryan Northey --- docs/generate_extension_db.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/docs/generate_extension_db.py b/docs/generate_extension_db.py index 4418bbc8de19f..19b87f3fa7224 100755 --- a/docs/generate_extension_db.py +++ b/docs/generate_extension_db.py @@ -66,11 +66,8 @@ def GetExtensionMetadata(target): % target) # evaluate tuples/lists # wrap strings in a list - categories = ( - ast.literal_eval(categories) - if ('[' in categories - or '(' in categories) - else [categories]) + categories = (ast.literal_eval(categories) if + ('[' in categories or '(' in categories) else [categories]) return { 'security_posture': security_posture, 'undocumented': False if IsMissing(undocumented) else bool(undocumented), From 5525d7535b848d2d90c84ce080c9326fb43168b9 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Mon, 15 Feb 2021 17:14:44 +0000 Subject: [PATCH 096/113] api/ Signed-off-by: Ryan Northey --- api/envoy/config/trace/v3/xray.proto | 1 + 1 file changed, 1 insertion(+) diff --git a/api/envoy/config/trace/v3/xray.proto b/api/envoy/config/trace/v3/xray.proto index a904a10892360..208170b60c3f7 100644 --- a/api/envoy/config/trace/v3/xray.proto +++ b/api/envoy/config/trace/v3/xray.proto @@ -21,6 +21,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: AWS X-Ray Tracer Configuration] // Configuration for AWS X-Ray tracer +// [#extension: envoy.tracers.xray] message XRayConfig { option (udpa.annotations.versioning).previous_message_type = "envoy.config.trace.v2alpha.XRayConfig"; From 4f2fcf72963a1dbe7aef08416c4647c3ae1e9a7b Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Mon, 15 Feb 2021 17:15:12 +0000 Subject: [PATCH 097/113] tools/ Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index 339ecbb79ade6..9eff5a8f0fcb0 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -241,7 +241,7 @@ def FormatExtensionCategory(extension_category): Returns: RST formatted extension category description. """ - IGNORED_EXTENSIONS = ["envoy.tracers.xray", "envoy.filters.http.cache.simple_http_cache"] + IGNORED_EXTENSIONS = ["envoy.filters.http.cache.simple_http_cache"] try: extensions = EXTENSION_CATEGORIES[extension_category] except KeyError as e: From a1cc9a5d712d638dfb85c53b5c97dd49f7c0d18a Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Mon, 15 Feb 2021 17:24:24 +0000 Subject: [PATCH 098/113] tools/ Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index 9eff5a8f0fcb0..46671f0698099 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -241,14 +241,13 @@ def FormatExtensionCategory(extension_category): Returns: RST formatted extension category description. """ - IGNORED_EXTENSIONS = ["envoy.filters.http.cache.simple_http_cache"] try: extensions = EXTENSION_CATEGORIES[extension_category] except KeyError as e: sys.stderr.write(f"\n\nUnable to find extension category: {extension_category}\n\n") exit(1) # Raising the error buries the above message in tracebacks. anchor = FormatAnchor('extension_category_' + extension_category) - extensions = FormatExtensionList([x for x in extensions if x not in IGNORED_EXTENSIONS]) + extensions = FormatExtensionList(extensions) return EXTENSION_CATEGORY_TEMPLATE.substitute(anchor=anchor, extensions=extensions) From e10d68452a526aff7d58064e75562c013f7d05fb Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Mon, 15 Feb 2021 17:24:28 +0000 Subject: [PATCH 099/113] api/ Signed-off-by: Ryan Northey --- api/envoy/extensions/filters/http/cache/v3alpha/cache.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/envoy/extensions/filters/http/cache/v3alpha/cache.proto b/api/envoy/extensions/filters/http/cache/v3alpha/cache.proto index 9260abe94a96c..f96642e9b69a7 100644 --- a/api/envoy/extensions/filters/http/cache/v3alpha/cache.proto +++ b/api/envoy/extensions/filters/http/cache/v3alpha/cache.proto @@ -18,8 +18,8 @@ option (udpa.annotations.file_status).work_in_progress = true; option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: HTTP Cache Filter] -// [#extension: envoy.filters.http.cache] +// [#extension: envoy.filters.http.cache] message CacheConfig { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.http.cache.v2alpha.CacheConfig"; From 6564038b1c43d29b93842b48099719302fae31f4 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Mon, 15 Feb 2021 17:48:53 +0000 Subject: [PATCH 100/113] api/ Signed-off-by: Ryan Northey --- api/envoy/extensions/filters/http/cache/v3alpha/cache.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/envoy/extensions/filters/http/cache/v3alpha/cache.proto b/api/envoy/extensions/filters/http/cache/v3alpha/cache.proto index f96642e9b69a7..9e140a38a902e 100644 --- a/api/envoy/extensions/filters/http/cache/v3alpha/cache.proto +++ b/api/envoy/extensions/filters/http/cache/v3alpha/cache.proto @@ -19,7 +19,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: HTTP Cache Filter] -// [#extension: envoy.filters.http.cache] +// [#extension: envoy.filters.http.cache.simple_http_cache] message CacheConfig { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.http.cache.v2alpha.CacheConfig"; From 833682ccbf2ae933116f7f45e741b3bf44698d39 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Mon, 15 Feb 2021 18:55:39 +0000 Subject: [PATCH 101/113] shadows Signed-off-by: Ryan Northey --- api/envoy/extensions/tracers/xray/v4alpha/xray.proto | 1 + generated_api_shadow/envoy/config/trace/v3/xray.proto | 1 + .../envoy/extensions/filters/http/cache/v3alpha/cache.proto | 2 +- .../envoy/extensions/filters/http/cache/v4alpha/cache.proto | 2 +- .../envoy/extensions/tracers/xray/v4alpha/xray.proto | 1 + 5 files changed, 5 insertions(+), 2 deletions(-) diff --git a/api/envoy/extensions/tracers/xray/v4alpha/xray.proto b/api/envoy/extensions/tracers/xray/v4alpha/xray.proto index c21ce9a7cf828..649f294b4273b 100644 --- a/api/envoy/extensions/tracers/xray/v4alpha/xray.proto +++ b/api/envoy/extensions/tracers/xray/v4alpha/xray.proto @@ -19,6 +19,7 @@ option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSIO // [#protodoc-title: AWS X-Ray Tracer Configuration] // Configuration for AWS X-Ray tracer +// [#extension: envoy.tracers.xray] message XRayConfig { option (udpa.annotations.versioning).previous_message_type = "envoy.config.trace.v3.XRayConfig"; diff --git a/generated_api_shadow/envoy/config/trace/v3/xray.proto b/generated_api_shadow/envoy/config/trace/v3/xray.proto index a904a10892360..208170b60c3f7 100644 --- a/generated_api_shadow/envoy/config/trace/v3/xray.proto +++ b/generated_api_shadow/envoy/config/trace/v3/xray.proto @@ -21,6 +21,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: AWS X-Ray Tracer Configuration] // Configuration for AWS X-Ray tracer +// [#extension: envoy.tracers.xray] message XRayConfig { option (udpa.annotations.versioning).previous_message_type = "envoy.config.trace.v2alpha.XRayConfig"; diff --git a/generated_api_shadow/envoy/extensions/filters/http/cache/v3alpha/cache.proto b/generated_api_shadow/envoy/extensions/filters/http/cache/v3alpha/cache.proto index 9260abe94a96c..9e140a38a902e 100644 --- a/generated_api_shadow/envoy/extensions/filters/http/cache/v3alpha/cache.proto +++ b/generated_api_shadow/envoy/extensions/filters/http/cache/v3alpha/cache.proto @@ -18,8 +18,8 @@ option (udpa.annotations.file_status).work_in_progress = true; option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: HTTP Cache Filter] -// [#extension: envoy.filters.http.cache] +// [#extension: envoy.filters.http.cache.simple_http_cache] message CacheConfig { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.http.cache.v2alpha.CacheConfig"; diff --git a/generated_api_shadow/envoy/extensions/filters/http/cache/v4alpha/cache.proto b/generated_api_shadow/envoy/extensions/filters/http/cache/v4alpha/cache.proto index ad9bb4c639a4d..283e8f53aebb2 100644 --- a/generated_api_shadow/envoy/extensions/filters/http/cache/v4alpha/cache.proto +++ b/generated_api_shadow/envoy/extensions/filters/http/cache/v4alpha/cache.proto @@ -18,8 +18,8 @@ option (udpa.annotations.file_status).work_in_progress = true; option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; // [#protodoc-title: HTTP Cache Filter] -// [#extension: envoy.filters.http.cache] +// [#extension: envoy.filters.http.cache.simple_http_cache] message CacheConfig { option (udpa.annotations.versioning).previous_message_type = "envoy.extensions.filters.http.cache.v3alpha.CacheConfig"; diff --git a/generated_api_shadow/envoy/extensions/tracers/xray/v4alpha/xray.proto b/generated_api_shadow/envoy/extensions/tracers/xray/v4alpha/xray.proto index c21ce9a7cf828..649f294b4273b 100644 --- a/generated_api_shadow/envoy/extensions/tracers/xray/v4alpha/xray.proto +++ b/generated_api_shadow/envoy/extensions/tracers/xray/v4alpha/xray.proto @@ -19,6 +19,7 @@ option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSIO // [#protodoc-title: AWS X-Ray Tracer Configuration] // Configuration for AWS X-Ray tracer +// [#extension: envoy.tracers.xray] message XRayConfig { option (udpa.annotations.versioning).previous_message_type = "envoy.config.trace.v3.XRayConfig"; From 4d52072342887812b03607350db9a06884aaedca Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Mon, 15 Feb 2021 20:21:40 +0000 Subject: [PATCH 102/113] api/ Signed-off-by: Ryan Northey --- api/envoy/extensions/filters/http/cache/v4alpha/cache.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/envoy/extensions/filters/http/cache/v4alpha/cache.proto b/api/envoy/extensions/filters/http/cache/v4alpha/cache.proto index ad9bb4c639a4d..283e8f53aebb2 100644 --- a/api/envoy/extensions/filters/http/cache/v4alpha/cache.proto +++ b/api/envoy/extensions/filters/http/cache/v4alpha/cache.proto @@ -18,8 +18,8 @@ option (udpa.annotations.file_status).work_in_progress = true; option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; // [#protodoc-title: HTTP Cache Filter] -// [#extension: envoy.filters.http.cache] +// [#extension: envoy.filters.http.cache.simple_http_cache] message CacheConfig { option (udpa.annotations.versioning).previous_message_type = "envoy.extensions.filters.http.cache.v3alpha.CacheConfig"; From 985f5c25d00e54fc076bcdffb41ac2bd1a029414 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Tue, 16 Feb 2021 11:21:48 +0000 Subject: [PATCH 103/113] extension-cat-db Signed-off-by: Ryan Northey --- docs/build.sh | 10 ++-------- docs/generate_extension_cat_db.py | 25 ------------------------- tools/protodoc/protodoc.py | 13 ++++++++++--- 3 files changed, 12 insertions(+), 36 deletions(-) delete mode 100755 docs/generate_extension_cat_db.py diff --git a/docs/build.sh b/docs/build.sh index abf623ee228cd..7890fcf6f3ee1 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -61,8 +61,7 @@ pip3 install --require-hashes -r "${SCRIPT_DIR}"/requirements.txt rm -rf bazel-bin/external/envoy_api_canonical EXTENSION_DB_PATH="$(realpath "${BUILD_DIR}/extension_db.json")" -EXTENSION_CAT_DB_PATH="$(realpath "${BUILD_DIR}/extension_cat_db.json")" -export EXTENSION_DB_PATH EXTENSION_CAT_DB_PATH +export EXTENSION_DB_PATH # This is for local RBE setup, should be no-op for builds without RBE setting in bazelrc files. IFS=" " read -ra BAZEL_BUILD_OPTIONS <<< "${BAZEL_BUILD_OPTIONS:-}" @@ -70,17 +69,12 @@ BAZEL_BUILD_OPTIONS+=( "--remote_download_outputs=all" "--strategy=protodoc=sandboxed,local" "--action_env=ENVOY_BLOB_SHA" - "--action_env=EXTENSION_DB_PATH" - "--action_env=EXTENSION_CAT_DB_PATH") + "--action_env=EXTENSION_DB_PATH") # Generate extension database. This maps from extension name to extension # metadata, based on the envoy_cc_extension() Bazel target attributes. ./docs/generate_extension_db.py "${EXTENSION_DB_PATH}" -# Generate extension category database. -./docs/generate_extension_cat_db.py "${EXTENSION_DB_PATH}" "${EXTENSION_CAT_DB_PATH}" - - # Generate RST for the lists of trusted/untrusted extensions in # intro/arch_overview/security docs. mkdir -p "${GENERATED_RST_DIR}"/intro/arch_overview/security diff --git a/docs/generate_extension_cat_db.py b/docs/generate_extension_cat_db.py deleted file mode 100755 index ff600bbb323c8..0000000000000 --- a/docs/generate_extension_cat_db.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python3 - -import json -import sys - - -def main(): - extension_db = sys.argv[1] - extension_cat_db = sys.argv[2] - - with open(extension_db) as f: - data = json.load(f) - - categories = {} - for k, v in data.items(): - for cat in v['categories']: - categories[cat] = categories.get(cat, []) - categories[cat].append(k) - - with open(extension_cat_db, 'w') as f: - json.dump(categories, f) - - -if __name__ == '__main__': - main() diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index 46671f0698099..ea69a164fc2de 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -105,7 +105,15 @@ 'This extension is work-in-progress. Functionality is incomplete and it is not intended for production use.', } -EXTENSION_CATEGORIES = json.loads(pathlib.Path(os.getenv('EXTENSION_CAT_DB_PATH')).read_text()) +EXTENSION_DB = json.loads(pathlib.Path(os.getenv('EXTENSION_DB_PATH')).read_text()) + +categories = {} +for k, v in data.items(): + for cat in v['categories']: + categories[cat] = categories.get(cat, []) + categories[cat].append(k) + +EXTENSION_CATEGORIES = categories class ProtodocError(Exception): @@ -209,8 +217,7 @@ def FormatExtension(extension): RST formatted extension description. """ try: - extension_metadata = json.loads(pathlib.Path( - os.getenv('EXTENSION_DB_PATH')).read_text())[extension] + extension_metadata = EXTENSION_DB[extension] anchor = FormatAnchor('extension_' + extension) status = EXTENSION_STATUS_VALUES.get(extension_metadata['status'], '') security_posture = EXTENSION_SECURITY_POSTURES[extension_metadata['security_posture']] From dd4c63ea94e4f8940adc69866d7458c40c6f6e40 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Tue, 16 Feb 2021 11:42:03 +0000 Subject: [PATCH 104/113] extension-cat-db Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index ea69a164fc2de..c8408da8be6b0 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -107,13 +107,12 @@ EXTENSION_DB = json.loads(pathlib.Path(os.getenv('EXTENSION_DB_PATH')).read_text()) -categories = {} -for k, v in data.items(): - for cat in v['categories']: - categories[cat] = categories.get(cat, []) - categories[cat].append(k) - -EXTENSION_CATEGORIES = categories +# create an index of extension categories from extension db +EXTENSION_CATEGORIES = {} +for _k, _v in EXTENSION_DB.items(): + for _cat in _v['categories']: + EXTENSION_CATEGORIES.setdefault(_cat, []) + EXTENSION_CATEGORIES[_cat].append(_k) class ProtodocError(Exception): From d3b6927ab285494df25659861abbdf4730b6260b Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Tue, 16 Feb 2021 11:57:07 +0000 Subject: [PATCH 105/113] tools/ Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index c8408da8be6b0..f731cfa091bd6 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -111,8 +111,7 @@ EXTENSION_CATEGORIES = {} for _k, _v in EXTENSION_DB.items(): for _cat in _v['categories']: - EXTENSION_CATEGORIES.setdefault(_cat, []) - EXTENSION_CATEGORIES[_cat].append(_k) + EXTENSION_CATEGORIES.setdefault(_cat, []).append(_k) class ProtodocError(Exception): From ae2389c4f14b61ba4b0e61d0037bc784bada1e93 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Tue, 16 Feb 2021 12:34:18 +0000 Subject: [PATCH 106/113] introduce error... Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index f731cfa091bd6..044c50d9bfbea 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -247,6 +247,8 @@ def FormatExtensionCategory(extension_category): RST formatted extension category description. """ try: + if 'xray' in extension_category: + extension_category = f"{extension_category}.foo" extensions = EXTENSION_CATEGORIES[extension_category] except KeyError as e: sys.stderr.write(f"\n\nUnable to find extension category: {extension_category}\n\n") From f4848a06f40301a053149012a464922e979986e7 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Tue, 16 Feb 2021 13:19:33 +0000 Subject: [PATCH 107/113] introduce error... Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index 044c50d9bfbea..e6253920bc78a 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -247,7 +247,7 @@ def FormatExtensionCategory(extension_category): RST formatted extension category description. """ try: - if 'xray' in extension_category: + if 'tracers' in extension_category: extension_category = f"{extension_category}.foo" extensions = EXTENSION_CATEGORIES[extension_category] except KeyError as e: From 41c39bfb85ff152d261f5e3cb1463175f0592f7c Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Tue, 16 Feb 2021 13:44:54 +0000 Subject: [PATCH 108/113] tools/ Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index e6253920bc78a..bb5ec6f03ce53 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -251,10 +251,9 @@ def FormatExtensionCategory(extension_category): extension_category = f"{extension_category}.foo" extensions = EXTENSION_CATEGORIES[extension_category] except KeyError as e: - sys.stderr.write(f"\n\nUnable to find extension category: {extension_category}\n\n") - exit(1) # Raising the error buries the above message in tracebacks. + raise ProtodocError(f"\n\nUnable to find extension category: {extension_category}\n\n") anchor = FormatAnchor('extension_category_' + extension_category) - extensions = FormatExtensionList(extensions) + extensions = FormatExtensionList(sorted(extensions)) return EXTENSION_CATEGORY_TEMPLATE.substitute(anchor=anchor, extensions=extensions) From 3fa04d4fffc00014f01bbf7a97981cdda93fea1e Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Tue, 16 Feb 2021 14:08:27 +0000 Subject: [PATCH 109/113] tools/ Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index bb5ec6f03ce53..d30bafd396121 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -247,8 +247,6 @@ def FormatExtensionCategory(extension_category): RST formatted extension category description. """ try: - if 'tracers' in extension_category: - extension_category = f"{extension_category}.foo" extensions = EXTENSION_CATEGORIES[extension_category] except KeyError as e: raise ProtodocError(f"\n\nUnable to find extension category: {extension_category}\n\n") From e5bef2862e42dc7cdf4c3efe8d1a7fa372dde16a Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Tue, 16 Feb 2021 14:16:22 +0000 Subject: [PATCH 110/113] messaging Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index d30bafd396121..f973e2b10f6c3 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -61,12 +61,18 @@ $security_posture - This extension extends the following extension category: +""") + +# Template for formating extension categories for extension. +EXTENSION_CATEGORIES_TEMPLATE = string.Template(""" +.. tip:: + $message: $categories """) + EXTENSION_CATEGORY_TEMPLATE = string.Template("""$anchor .. tip:: @@ -219,12 +225,17 @@ def FormatExtension(extension): anchor = FormatAnchor('extension_' + extension) status = EXTENSION_STATUS_VALUES.get(extension_metadata['status'], '') security_posture = EXTENSION_SECURITY_POSTURES[extension_metadata['security_posture']] + extension = EXTENSION_TEMPLATE.substitute(anchor=anchor, + extension=extension, + status=status, + security_posture=security_posture) + categories = FormatExtensionList(extension_metadata["categories"], "extension_category") - return EXTENSION_TEMPLATE.substitute(anchor=anchor, - extension=extension, - status=status, - security_posture=security_posture, - categories=categories) + cat_or_cats = "categories" if len(categories) > 1 else "category" + category_message = ("This extension extends and can be used with the following extension {cat_or_cats}" + extension_category = EXTENSION_CATEGORIES_TEMPLATE.substitute(message=category_message, + categories=categories) + return f"{extension}\n\n{extension_category}" except KeyError as e: sys.stderr.write( '\n\nDid you forget to add an entry to source/extensions/extensions_build_config.bzl?\n\n') From fdab6b8d240bb7a6b0e544ad452f8ef22e9aaf76 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Tue, 16 Feb 2021 14:23:30 +0000 Subject: [PATCH 111/113] messaging Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index f973e2b10f6c3..d392ca541e587 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -63,7 +63,7 @@ """) -# Template for formating extension categories for extension. +# Template for formating extension's category/ies. EXTENSION_CATEGORIES_TEMPLATE = string.Template(""" .. tip:: $message: @@ -72,7 +72,7 @@ """) - +# Template for formating an extension category. EXTENSION_CATEGORY_TEMPLATE = string.Template("""$anchor .. tip:: @@ -232,7 +232,7 @@ def FormatExtension(extension): categories = FormatExtensionList(extension_metadata["categories"], "extension_category") cat_or_cats = "categories" if len(categories) > 1 else "category" - category_message = ("This extension extends and can be used with the following extension {cat_or_cats}" + category_message = "This extension extends and can be used with the following extension {cat_or_cats}" extension_category = EXTENSION_CATEGORIES_TEMPLATE.substitute(message=category_message, categories=categories) return f"{extension}\n\n{extension_category}" From cbb90a75633e1e7117ea64c3d36a8c0cf34a64c5 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Tue, 16 Feb 2021 14:41:35 +0000 Subject: [PATCH 112/113] tools/ Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index d392ca541e587..3a7dc98b1fee6 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -232,7 +232,7 @@ def FormatExtension(extension): categories = FormatExtensionList(extension_metadata["categories"], "extension_category") cat_or_cats = "categories" if len(categories) > 1 else "category" - category_message = "This extension extends and can be used with the following extension {cat_or_cats}" + category_message = f"This extension extends and can be used with the following extension {cat_or_cats}" extension_category = EXTENSION_CATEGORIES_TEMPLATE.substitute(message=category_message, categories=categories) return f"{extension}\n\n{extension_category}" From 3dd38d036273a11c8a15555585276dec5bc45d40 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Tue, 16 Feb 2021 14:41:56 +0000 Subject: [PATCH 113/113] tools/ Signed-off-by: Ryan Northey --- tools/protodoc/protodoc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index 3a7dc98b1fee6..7911d33e89426 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -76,7 +76,7 @@ EXTENSION_CATEGORY_TEMPLATE = string.Template("""$anchor .. tip:: - This extension category has the following known extensions + This extension category has the following known extensions: $extensions