diff --git a/generator/CMakeLists.txt b/generator/CMakeLists.txt index e4abeb7990ce7..070e66965cbed 100644 --- a/generator/CMakeLists.txt +++ b/generator/CMakeLists.txt @@ -96,6 +96,8 @@ add_library( internal/metadata_decorator_generator.h internal/metadata_decorator_rest_generator.cc internal/metadata_decorator_rest_generator.h + internal/mixin_utils.cc + internal/mixin_utils.h internal/mock_connection_generator.cc internal/mock_connection_generator.h internal/option_defaults_generator.cc @@ -251,6 +253,7 @@ function (google_cloud_cpp_generator_define_tests) internal/http_option_utils_test.cc internal/longrunning_test.cc internal/make_generators_test.cc + internal/mixin_utils_test.cc internal/pagination_test.cc internal/predicate_utils_test.cc internal/printer_test.cc diff --git a/generator/google_cloud_cpp_generator.bzl b/generator/google_cloud_cpp_generator.bzl index 435745a90bde5..29f7a3c9dbe36 100644 --- a/generator/google_cloud_cpp_generator.bzl +++ b/generator/google_cloud_cpp_generator.bzl @@ -50,6 +50,7 @@ google_cloud_cpp_generator_hdrs = [ "internal/make_generators.h", "internal/metadata_decorator_generator.h", "internal/metadata_decorator_rest_generator.h", + "internal/mixin_utils.h", "internal/mock_connection_generator.h", "internal/option_defaults_generator.h", "internal/options_generator.h", @@ -108,6 +109,7 @@ google_cloud_cpp_generator_srcs = [ "internal/make_generators.cc", "internal/metadata_decorator_generator.cc", "internal/metadata_decorator_rest_generator.cc", + "internal/mixin_utils.cc", "internal/mock_connection_generator.cc", "internal/option_defaults_generator.cc", "internal/options_generator.cc", diff --git a/generator/google_cloud_cpp_generator_unit_tests.bzl b/generator/google_cloud_cpp_generator_unit_tests.bzl index 1ce613e2fce11..dedd2494c91b5 100644 --- a/generator/google_cloud_cpp_generator_unit_tests.bzl +++ b/generator/google_cloud_cpp_generator_unit_tests.bzl @@ -32,6 +32,7 @@ google_cloud_cpp_generator_unit_tests = [ "internal/http_option_utils_test.cc", "internal/longrunning_test.cc", "internal/make_generators_test.cc", + "internal/mixin_utils_test.cc", "internal/pagination_test.cc", "internal/predicate_utils_test.cc", "internal/printer_test.cc", diff --git a/generator/internal/mixin_utils.cc b/generator/internal/mixin_utils.cc new file mode 100644 index 0000000000000..9f9624ffc7be9 --- /dev/null +++ b/generator/internal/mixin_utils.cc @@ -0,0 +1,193 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "generator/internal/mixin_utils.h" +#include "generator/internal/codegen_utils.h" +#include "google/cloud/internal/absl_str_join_quiet.h" +#include "google/cloud/log.h" +#include "absl/base/no_destructor.h" +#include "absl/strings/ascii.h" +#include "absl/types/optional.h" +#include +#include +#include +#include +#include +#include +#include + +using ::google::protobuf::DescriptorPool; +using ::google::protobuf::FileDescriptor; +using ::google::protobuf::MethodDescriptor; +using ::google::protobuf::ServiceDescriptor; + +namespace google { +namespace cloud { +namespace generator_internal { +namespace { + +std::unordered_map const& GetMixinProtoPathMap() { + static absl::NoDestructor> const + kMixinProtoPathMap{{ + {"google.cloud.location.Locations", + "google/cloud/location/locations.proto"}, + {"google.iam.v1.IAMPolicy", "google/iam/v1/iam_policy.proto"}, + {"google.longrunning.Operations", + "google/longrunning/operations.proto"}, + }}; + return *kMixinProtoPathMap; +} + +std::unordered_map const& GetHttpVerbs() { + static absl::NoDestructor> const + kHttpVerbs{{ + {"get", "Get"}, + {"post", "Post"}, + {"put", "Put"}, + {"patch", "Patch"}, + {"delete", "Delete"}, + }}; + return *kHttpVerbs; +} + +/** + * Extract Mixin methods from the YAML file, together with the overwritten http + * info. + */ +std::unordered_map GetMixinMethodOverrides( + YAML::Node const& service_config) { + std::unordered_map mixin_method_overrides; + if (service_config.Type() != YAML::NodeType::Map) + return mixin_method_overrides; + if (!service_config["http"]) return mixin_method_overrides; + auto const& http = service_config["http"]; + if (http.Type() != YAML::NodeType::Map) return mixin_method_overrides; + auto const& rules = http["rules"]; + if (rules.Type() != YAML::NodeType::Sequence) return mixin_method_overrides; + for (auto const& rule : rules) { + if (rule.Type() != YAML::NodeType::Map) continue; + + auto const& selector = rule["selector"]; + if (selector.Type() != YAML::NodeType::Scalar) continue; + std::string const method_full_name = selector.as(); + + for (auto const& kv : rule) { + if (kv.first.Type() != YAML::NodeType::Scalar || + kv.second.Type() != YAML::NodeType::Scalar) + continue; + + std::string const http_verb_lower = + absl::AsciiStrToLower(kv.first.as()); + auto const& http_verbs = GetHttpVerbs(); + auto const it = http_verbs.find(http_verb_lower); + if (it == http_verbs.end()) continue; + std::string const& http_verb = it->second; + std::string const http_path = kv.second.as(); + absl::optional http_body; + if (rule["body"]) { + http_body = rule["body"].as(); + } + + mixin_method_overrides[method_full_name] = + MixinMethodOverride{http_verb, http_path, http_body}; + } + } + return mixin_method_overrides; +} + +/** + * Get all methods' names from a service. + */ +std::unordered_set GetMethodNames( + ServiceDescriptor const& service) { + std::unordered_set method_names; + for (int i = 0; i < service.method_count(); ++i) { + auto const* method = service.method(i); + method_names.insert(method->name()); + } + return method_names; +} + +} // namespace + +std::vector GetMixinProtoPaths(YAML::Node const& service_config) { + std::vector proto_paths; + if (service_config.Type() != YAML::NodeType::Map) return proto_paths; + auto const& apis = service_config["apis"]; + if (apis.Type() != YAML::NodeType::Sequence) return proto_paths; + for (auto const& api : apis) { + if (api.Type() != YAML::NodeType::Map) continue; + auto const& name = api["name"]; + if (name.Type() != YAML::NodeType::Scalar) continue; + std::string const package_path = name.as(); + auto const& mixin_proto_path_map = GetMixinProtoPathMap(); + auto const it = mixin_proto_path_map.find(package_path); + if (it == mixin_proto_path_map.end()) continue; + proto_paths.push_back(it->second); + } + return proto_paths; +} + +std::vector GetMixinProtoPaths( + std::string const& service_yaml_path) { + return GetMixinProtoPaths(YAML::LoadFile(service_yaml_path)); +} + +std::vector GetMixinMethods(YAML::Node const& service_config, + ServiceDescriptor const& service) { + std::vector mixin_methods; + DescriptorPool const* pool = service.file()->pool(); + if (pool == nullptr) { + GCP_LOG(FATAL) << __FILE__ << ":" << __LINE__ + << " DescriptorPool doesn't exist for service: " + << service.full_name(); + } + std::unordered_set const method_names = GetMethodNames(service); + auto const mixin_proto_paths = GetMixinProtoPaths(service_config); + auto const mixin_method_overrides = GetMixinMethodOverrides(service_config); + + for (auto const& mixin_proto_path : mixin_proto_paths) { + FileDescriptor const* mixin_file = pool->FindFileByName(mixin_proto_path); + if (mixin_file == nullptr) { + GCP_LOG(FATAL) << __FILE__ << ":" << __LINE__ + << " Mixin FileDescriptor is not found for path: " + << mixin_proto_path + << " in service: " << service.full_name(); + } + for (int i = 0; i < mixin_file->service_count(); ++i) { + ServiceDescriptor const* mixin_service = mixin_file->service(i); + for (int j = 0; j < mixin_service->method_count(); ++j) { + MethodDescriptor const* mixin_method = mixin_service->method(j); + auto mixin_method_full_name = mixin_method->full_name(); + auto const it = mixin_method_overrides.find(mixin_method_full_name); + if (it == mixin_method_overrides.end()) continue; + + // if the mixin method name required from YAML appears in the original + // service proto, ignore the mixin. + if (method_names.find(mixin_method->name()) != method_names.end()) + continue; + + mixin_methods.push_back( + {absl::AsciiStrToLower(mixin_service->name()) + "_stub", + ProtoNameToCppName(mixin_service->full_name()), *mixin_method, + it->second}); + } + } + } + return mixin_methods; +} + +} // namespace generator_internal +} // namespace cloud +} // namespace google diff --git a/generator/internal/mixin_utils.h b/generator/internal/mixin_utils.h new file mode 100644 index 0000000000000..965830e7e002e --- /dev/null +++ b/generator/internal/mixin_utils.h @@ -0,0 +1,73 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GOOGLE_CLOUD_CPP_GENERATOR_INTERNAL_MIXIN_UTILS_H +#define GOOGLE_CLOUD_CPP_GENERATOR_INTERNAL_MIXIN_UTILS_H + +#include "absl/types/optional.h" +#include +#include +#include +#include + +namespace google { +namespace cloud { +namespace generator_internal { + +/** + * Override of http info of a mixin method. + */ +struct MixinMethodOverride { + std::string http_verb; + std::string http_path; + absl::optional http_body; +}; + +/** + * All required info for a mixin method + * including grpc stub name, grpc stub full qualified name, + * method descriptor and method http overrides. + */ +struct MixinMethod { + std::string grpc_stub_name; + std::string grpc_stub_fqn; + std::reference_wrapper method; + MixinMethodOverride method_override; +}; + +/** + * Extract Mixin proto file paths from the YAML Node. + */ +std::vector GetMixinProtoPaths(YAML::Node const& service_config); + +/** + * Extract Mixin proto file paths from the YAML Node loaded from a YAML file + * path. + */ +std::vector GetMixinProtoPaths( + std::string const& service_yaml_path); + +/** + * Get Mixin methods' descriptors and services' info from proto files, + * and get the http info overrides from YAML file. + */ +std::vector GetMixinMethods( + YAML::Node const& service_config, + google::protobuf::ServiceDescriptor const& service); + +} // namespace generator_internal +} // namespace cloud +} // namespace google + +#endif // GOOGLE_CLOUD_CPP_GENERATOR_INTERNAL_MIXIN_UTILS_H diff --git a/generator/internal/mixin_utils_test.cc b/generator/internal/mixin_utils_test.cc new file mode 100644 index 0000000000000..1d5889934978d --- /dev/null +++ b/generator/internal/mixin_utils_test.cc @@ -0,0 +1,283 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "generator/internal/mixin_utils.h" +#include "generator/testing/descriptor_pool_fixture.h" +#include + +namespace google { +namespace cloud { +namespace generator_internal { +namespace { + +using ::google::protobuf::FileDescriptor; +using ::google::protobuf::ServiceDescriptor; +using ::testing::AllOf; +using ::testing::Contains; +using ::testing::Eq; +using ::testing::NotNull; +using ::testing::Optional; + +auto constexpr kServiceConfigYaml = R"""( +apis: + - name: test.v1.Service + - name: google.cloud.location.Locations + - name: google.iam.v1.IAMPolicy + - name: google.longrunning.Operations +http: + rules: + - selector: google.cloud.location.Locations.GetLocation + get: 'OverwriteGetLocationPath' + - selector: google.cloud.location.Locations.ListLocations + get: 'OverwriteListLocationPath' + - selector: google.iam.v1.IAMPolicy.SetIamPolicy + post: 'OverwriteSetIamPolicyPath' + body: '*' +)"""; + +auto constexpr kServiceConfigRedundantRulesYaml = R"""( +apis: + - name: test.v1.Service + - name: google.cloud.location.Locations + - name: google.iam.v1.IAMPolicy +http: + rules: + - selector: google.cloud.location.Locations.GetLocation + get: 'OverwriteGetLocationPath' + - selector: google.cloud.location.Locations.ListLocations + get: 'OverwriteListLocationPath' + - selector: google.iam.v1.IAMPolicy.SetIamPolicy + post: 'OverwriteSetIamPolicyPath' + body: '*' + - selector: google.cloud.Redundant.RedundantGet + get: 'OverwriteListLocationPath' +)"""; + +auto constexpr kAnnotationsProto = R"""( + syntax = "proto3"; + package google.api; + import "google/api/http.proto"; + import "google/protobuf/descriptor.proto"; + extend google.protobuf.MethodOptions { + // See `HttpRule`. + HttpRule http = 72295728; + }; +)"""; + +auto constexpr kHttpProto = R"""( + syntax = "proto3"; + package google.api; + option cc_enable_arenas = true; + message Http { + repeated HttpRule rules = 1; + bool fully_decode_reserved_expansion = 2; + } + message HttpRule { + string selector = 1; + oneof pattern { + string get = 2; + string put = 3; + string post = 4; + string delete = 5; + string patch = 6; + CustomHttpPattern custom = 8; + } + string body = 7; + string response_body = 12; + repeated HttpRule additional_bindings = 11; + } + message CustomHttpPattern { + string kind = 1; + string path = 2; + }; +)"""; + +auto constexpr kMixinLocationProto = R"""( +syntax = "proto3"; +package google.cloud.location; +import "google/api/annotations.proto"; +import "google/api/http.proto"; +import "test/v1/common.proto"; + +service Locations { + rpc GetLocation(test.v1.Request) returns (test.v1.Response) { + option (google.api.http) = { + get: "ToBeOverwrittenPath" + }; + } + rpc ListLocations(test.v1.Request) returns (test.v1.Response) { + option (google.api.http) = { + get: "ToBeOverwrittenPath" + }; + } +} +)"""; + +auto constexpr kMixinIAMPolicyProto = R"""( +syntax = "proto3"; +package google.iam.v1; +import "google/api/annotations.proto"; +import "google/api/http.proto"; +import "test/v1/common.proto"; + +service IAMPolicy { + rpc SetIamPolicy(test.v1.Request) returns (test.v1.Response) { + option (google.api.http) = { + get: "ToBeOverwrittenPath" + }; + } +} +)"""; + +auto constexpr kClientProto1 = R"""( +syntax = "proto3"; +package test.v1; +import "test/v1/common.proto"; + +service Service0 { + rpc method0(Request) returns (Response) {} +} +)"""; + +auto constexpr kClientProto2 = R"""( +syntax = "proto3"; +package test.v1; +import "test/v1/common.proto"; + +service Service1 { + rpc method0(Request) returns (Response) {} + rpc GetLocation(Request) returns (Response) {} + rpc ListLocations(Request) returns (Response) {} +} +)"""; + +class MixinUtilsTest : public generator_testing::DescriptorPoolFixture { + public: + void SetUp() override { + ASSERT_TRUE(AddProtoFile("google/api/http.proto", kHttpProto)); + ASSERT_TRUE( + AddProtoFile("google/api/annotations.proto", kAnnotationsProto)); + ASSERT_TRUE(AddProtoFile("test/v1/service1.proto", kClientProto1)); + ASSERT_TRUE(AddProtoFile("test/v1/service2.proto", kClientProto2)); + ASSERT_TRUE(AddProtoFile("google/cloud/location/locations.proto", + kMixinLocationProto)); + ASSERT_TRUE( + AddProtoFile("google/iam/v1/iam_policy.proto", kMixinIAMPolicyProto)); + } + + protected: + YAML::Node service_config_ = YAML::Load(kServiceConfigYaml); + YAML::Node service_config_redundant_ = + YAML::Load(kServiceConfigRedundantRulesYaml); +}; + +TEST_F(MixinUtilsTest, FilesParseSuccessfully) { + ASSERT_THAT(FindFile("google/protobuf/descriptor.proto"), NotNull()); + ASSERT_THAT(FindFile("google/api/http.proto"), NotNull()); + ASSERT_THAT(FindFile("google/api/annotations.proto"), NotNull()); + ASSERT_THAT(FindFile("test/v1/common.proto"), NotNull()); + ASSERT_THAT(FindFile("test/v1/service1.proto"), NotNull()); + ASSERT_THAT(FindFile("test/v1/service2.proto"), NotNull()); + ASSERT_THAT(FindFile("google/cloud/location/locations.proto"), NotNull()); + ASSERT_THAT(FindFile("google/iam/v1/iam_policy.proto"), NotNull()); +} + +TEST_F(MixinUtilsTest, ExtractMixinProtoPathsFromYaml) { + auto const mixin_proto_paths = GetMixinProtoPaths(service_config_); + EXPECT_THAT(mixin_proto_paths, + AllOf(Contains("google/cloud/location/locations.proto"), + Contains("google/iam/v1/iam_policy.proto"), + Contains("google/longrunning/operations.proto"))); +} + +TEST_F(MixinUtilsTest, GetMixinMethods) { + FileDescriptor const* file = FindFile("test/v1/service1.proto"); + EXPECT_THAT(file, NotNull()); + + ServiceDescriptor const* service = file->service(0); + EXPECT_THAT(service, NotNull()); + + auto const& mixin_methods = GetMixinMethods(service_config_, *service); + EXPECT_EQ(mixin_methods.size(), 3); + + MixinMethod const& get_location = mixin_methods[0]; + MixinMethod const& list_locations = mixin_methods[1]; + MixinMethod const& set_iam_policy = mixin_methods[2]; + + EXPECT_EQ(get_location.method.get().full_name(), + "google.cloud.location.Locations.GetLocation"); + EXPECT_EQ(get_location.grpc_stub_name, "locations_stub"); + EXPECT_EQ(get_location.grpc_stub_fqn, "google::cloud::location::Locations"); + EXPECT_THAT(get_location.method_override.http_body, Eq(absl::nullopt)); + EXPECT_EQ(get_location.method_override.http_path, "OverwriteGetLocationPath"); + EXPECT_EQ(get_location.method_override.http_verb, "Get"); + + EXPECT_EQ(list_locations.method.get().full_name(), + "google.cloud.location.Locations.ListLocations"); + EXPECT_EQ(list_locations.grpc_stub_name, "locations_stub"); + EXPECT_EQ(list_locations.grpc_stub_fqn, "google::cloud::location::Locations"); + EXPECT_THAT(list_locations.method_override.http_body, Eq(absl::nullopt)); + EXPECT_EQ(list_locations.method_override.http_path, + "OverwriteListLocationPath"); + EXPECT_EQ(list_locations.method_override.http_verb, "Get"); + + EXPECT_EQ(set_iam_policy.method.get().full_name(), + "google.iam.v1.IAMPolicy.SetIamPolicy"); + EXPECT_EQ(set_iam_policy.grpc_stub_name, "iampolicy_stub"); + EXPECT_EQ(set_iam_policy.grpc_stub_fqn, "google::iam::v1::IAMPolicy"); + EXPECT_THAT(set_iam_policy.method_override.http_body, + Optional(std::string("*"))); + EXPECT_EQ(set_iam_policy.method_override.http_path, + "OverwriteSetIamPolicyPath"); + EXPECT_EQ(set_iam_policy.method_override.http_verb, "Post"); +} + +TEST_F(MixinUtilsTest, GetMixinMethodsWithDuplicatedMixinNames) { + FileDescriptor const* file = FindFile("test/v1/service2.proto"); + EXPECT_THAT(file, NotNull()); + + ServiceDescriptor const* service = file->service(0); + EXPECT_THAT(service, NotNull()); + + auto const& mixin_methods = GetMixinMethods(service_config_, *service); + EXPECT_EQ(mixin_methods.size(), 1); + EXPECT_EQ(mixin_methods[0].method.get().full_name(), + "google.iam.v1.IAMPolicy.SetIamPolicy"); +} + +TEST_F(MixinUtilsTest, GetMixinMethodsWithRedundantRules) { + FileDescriptor const* file = FindFile("test/v1/service1.proto"); + EXPECT_THAT(file, NotNull()); + + ServiceDescriptor const* service = file->service(0); + EXPECT_THAT(service, NotNull()); + + auto const& mixin_methods = + GetMixinMethods(service_config_redundant_, *service); + EXPECT_EQ(mixin_methods.size(), 3); + for (auto const& mixin_method : mixin_methods) { + EXPECT_NE(mixin_method.method.get().full_name(), + "google.cloud.Redundant.RedundantGet"); + } +} + +} // namespace +} // namespace generator_internal +} // namespace cloud +} // namespace google + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/generator/testing/fake_source_tree.cc b/generator/testing/fake_source_tree.cc index c85e892da5a12..e24c39fdfaa1a 100644 --- a/generator/testing/fake_source_tree.cc +++ b/generator/testing/fake_source_tree.cc @@ -22,7 +22,7 @@ FakeSourceTree::FakeSourceTree(std::map files) : files_(std::move(files)) {} void FakeSourceTree::Insert(std::string const& filename, std::string contents) { - files_.emplace(filename, std::move(contents)); + files_[filename] = std::move(contents); } google::protobuf::io::ZeroCopyInputStream* FakeSourceTree::Open(