diff --git a/src/main/java/com/google/api/generator/gapic/composer/BUILD.bazel b/src/main/java/com/google/api/generator/gapic/composer/BUILD.bazel index a4500c029d..9e5bbfd444 100644 --- a/src/main/java/com/google/api/generator/gapic/composer/BUILD.bazel +++ b/src/main/java/com/google/api/generator/gapic/composer/BUILD.bazel @@ -17,8 +17,8 @@ java_library( "//src/main/java/com/google/api/generator/engine/ast", "//src/main/java/com/google/api/generator/engine/writer", "//src/main/java/com/google/api/generator/gapic:status_java_proto", - "//src/main/java/com/google/api/generator/gapic/model", "//src/main/java/com/google/api/generator/gapic/composer/samplecode", + "//src/main/java/com/google/api/generator/gapic/model", "//src/main/java/com/google/api/generator/gapic/utils", "@com_google_api_api_common//jar", "@com_google_api_gax_java//gax", diff --git a/src/main/java/com/google/api/generator/gapic/composer/BatchingDescriptorComposer.java b/src/main/java/com/google/api/generator/gapic/composer/BatchingDescriptorComposer.java new file mode 100644 index 0000000000..4202809ce1 --- /dev/null +++ b/src/main/java/com/google/api/generator/gapic/composer/BatchingDescriptorComposer.java @@ -0,0 +1,557 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.api.generator.gapic.composer; + +import com.google.api.gax.batching.PartitionKey; +import com.google.api.gax.batching.RequestBuilder; +import com.google.api.gax.rpc.BatchedRequestIssuer; +import com.google.api.gax.rpc.BatchingDescriptor; +import com.google.api.generator.engine.ast.AnonymousClassExpr; +import com.google.api.generator.engine.ast.AssignmentExpr; +import com.google.api.generator.engine.ast.ConcreteReference; +import com.google.api.generator.engine.ast.Expr; +import com.google.api.generator.engine.ast.ExprStatement; +import com.google.api.generator.engine.ast.ForStatement; +import com.google.api.generator.engine.ast.GeneralForStatement; +import com.google.api.generator.engine.ast.IfStatement; +import com.google.api.generator.engine.ast.MethodDefinition; +import com.google.api.generator.engine.ast.MethodInvocationExpr; +import com.google.api.generator.engine.ast.NewObjectExpr; +import com.google.api.generator.engine.ast.PrimitiveValue; +import com.google.api.generator.engine.ast.Reference; +import com.google.api.generator.engine.ast.ScopeNode; +import com.google.api.generator.engine.ast.Statement; +import com.google.api.generator.engine.ast.TypeNode; +import com.google.api.generator.engine.ast.UnaryOperationExpr; +import com.google.api.generator.engine.ast.ValueExpr; +import com.google.api.generator.engine.ast.VaporReference; +import com.google.api.generator.engine.ast.Variable; +import com.google.api.generator.engine.ast.VariableExpr; +import com.google.api.generator.gapic.model.Field; +import com.google.api.generator.gapic.model.GapicBatchingSettings; +import com.google.api.generator.gapic.model.Message; +import com.google.api.generator.gapic.model.Method; +import com.google.api.generator.gapic.utils.JavaStyle; +import com.google.common.base.Preconditions; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +public class BatchingDescriptorComposer { + private static final String BATCHING_DESC_PATTERN = "%s_BATCHING_DESC"; + + private static final Reference BATCHING_DESCRIPTOR_REF = + ConcreteReference.withClazz(BatchingDescriptor.class); + private static final Reference REQUEST_BUILDER_REF = + ConcreteReference.withClazz(RequestBuilder.class); + private static final Reference BATCHED_REQUEST_ISSUER_REF = + ConcreteReference.withClazz(BatchedRequestIssuer.class); + + private static final TypeNode PARTITION_KEY_TYPE = toType(PartitionKey.class); + + private static final String ADD_ALL_METHOD_PATTERN = "addAll%s"; + private static final String BATCH_FOO_INDEX_PATTERN = "batch%sIndex"; + private static final String GET_LIST_METHOD_PATTERN = "get%sList"; + private static final String GET_COUNT_METHOD_PATTERN = "get%sCount"; + + public static Expr createBatchingDescriptorFieldDeclExpr( + Method method, GapicBatchingSettings batchingSettings, Map messageTypes) { + List javaMethods = new ArrayList<>(); + javaMethods.add(createGetBatchPartitionKeyMethod(method, batchingSettings, messageTypes)); + javaMethods.add(createGetRequestBuilderMethod(method, batchingSettings)); + javaMethods.add(createSplitResponseMethod(method, batchingSettings, messageTypes)); + javaMethods.add(createSplitExceptionMethod(method)); + javaMethods.add(createCountElementsMethod(method, batchingSettings)); + javaMethods.add(createCountByteSMethod(method)); + + TypeNode batchingDescriptorType = + toType(BATCHING_DESCRIPTOR_REF, method.inputType(), method.outputType()); + AnonymousClassExpr batchingDescriptorClassExpr = + AnonymousClassExpr.builder() + .setType(batchingDescriptorType) + .setMethods(javaMethods) + .build(); + + String varName = + String.format(BATCHING_DESC_PATTERN, JavaStyle.toUpperSnakeCase(method.name())); + return AssignmentExpr.builder() + .setVariableExpr( + VariableExpr.builder() + .setVariable( + Variable.builder().setType(batchingDescriptorType).setName(varName).build()) + .setIsDecl(true) + .setScope(ScopeNode.PRIVATE) + .setIsStatic(true) + .setIsFinal(true) + .build()) + .setValueExpr(batchingDescriptorClassExpr) + .build(); + } + + private static MethodDefinition createGetBatchPartitionKeyMethod( + Method method, GapicBatchingSettings batchingSettings, Map messageTypes) { + String methodInputTypeName = method.inputType().reference().name(); + Message inputMessage = messageTypes.get(methodInputTypeName); + Preconditions.checkNotNull( + inputMessage, + String.format( + "Message %s not found for RPC method %s", methodInputTypeName, method.name())); + + VariableExpr requestVarExpr = + VariableExpr.withVariable( + Variable.builder().setType(method.inputType()).setName("request").build()); + + List partitionKeyArgExprs = new ArrayList<>(); + for (String discriminatorFieldName : batchingSettings.discriminatorFieldNames()) { + Preconditions.checkNotNull( + inputMessage.fieldMap().get(discriminatorFieldName), + String.format( + "Batching discriminator field %s not found in message %s", + discriminatorFieldName, inputMessage.name())); + String getterMethodName = + String.format("get%s", JavaStyle.toUpperCamelCase(discriminatorFieldName)); + partitionKeyArgExprs.add( + MethodInvocationExpr.builder() + .setExprReferenceExpr(requestVarExpr) + .setMethodName(getterMethodName) + .build()); + } + Expr returnExpr = + NewObjectExpr.builder() + .setType(PARTITION_KEY_TYPE) + .setArguments(partitionKeyArgExprs) + .build(); + + return MethodDefinition.builder() + .setIsOverride(true) + .setScope(ScopeNode.PUBLIC) + .setReturnType(PARTITION_KEY_TYPE) + .setName("getBatchPartitionKey") + .setArguments(requestVarExpr.toBuilder().setIsDecl(true).build()) + .setReturnExpr(returnExpr) + .build(); + } + + private static MethodDefinition createGetRequestBuilderMethod( + Method method, GapicBatchingSettings batchingSettings) { + TypeNode builderType = + TypeNode.withReference( + VaporReference.builder() + .setEnclosingClassNames(method.inputType().reference().name()) + .setName("Builder") + .setPakkage(method.inputType().reference().pakkage()) + .build()); + + VariableExpr builderVarExpr = + VariableExpr.withVariable( + Variable.builder().setType(builderType).setName("builder").build()); + VariableExpr requestVarExpr = + VariableExpr.withVariable( + Variable.builder().setType(method.inputType()).setName("request").build()); + + Expr toBuilderExpr = + AssignmentExpr.builder() + .setVariableExpr(builderVarExpr) + .setValueExpr( + MethodInvocationExpr.builder() + .setExprReferenceExpr(requestVarExpr) + .setMethodName("toBuilder") + .setReturnType(builderType) + .build()) + .build(); + + String upperBatchedFieldName = JavaStyle.toUpperCamelCase(batchingSettings.batchedFieldName()); + String getFooListMethodName = String.format(GET_LIST_METHOD_PATTERN, upperBatchedFieldName); + Expr getFooListExpr = + MethodInvocationExpr.builder() + .setExprReferenceExpr(requestVarExpr) + .setMethodName(getFooListMethodName) + .build(); + + String addAllMethodName = String.format(ADD_ALL_METHOD_PATTERN, upperBatchedFieldName); + Expr addAllExpr = + MethodInvocationExpr.builder() + .setExprReferenceExpr(builderVarExpr) + .setMethodName(addAllMethodName) + .setArguments(getFooListExpr) + .build(); + + MethodDefinition appendRequestMethod = + MethodDefinition.builder() + .setIsOverride(true) + .setScope(ScopeNode.PUBLIC) + .setReturnType(TypeNode.VOID) + .setName("appendRequest") + .setArguments(requestVarExpr.toBuilder().setIsDecl(true).build()) + .setBody( + Arrays.asList( + IfStatement.builder() + .setConditionExpr( + MethodInvocationExpr.builder() + .setStaticReferenceType(toType(Objects.class)) + .setMethodName("isNull") + .setArguments(builderVarExpr) + .setReturnType(TypeNode.BOOLEAN) + .build()) + .setBody(Arrays.asList(ExprStatement.withExpr(toBuilderExpr))) + .setElseBody(Arrays.asList(ExprStatement.withExpr(addAllExpr))) + .build())) + .build(); + + MethodDefinition buildMethod = + MethodDefinition.builder() + .setIsOverride(true) + .setScope(ScopeNode.PUBLIC) + .setReturnType(method.inputType()) + .setName("build") + .setReturnExpr( + MethodInvocationExpr.builder() + .setExprReferenceExpr(builderVarExpr) + .setMethodName("build") + .setReturnType(method.inputType()) + .build()) + .build(); + + TypeNode anonClassType = toType(REQUEST_BUILDER_REF, method.inputType()); + AnonymousClassExpr requestBuilderAnonClassExpr = + AnonymousClassExpr.builder() + .setType(anonClassType) + .setStatements( + Arrays.asList( + ExprStatement.withExpr( + builderVarExpr + .toBuilder() + .setIsDecl(true) + .setScope(ScopeNode.PRIVATE) + .build()))) + .setMethods(Arrays.asList(appendRequestMethod, buildMethod)) + .build(); + + return MethodDefinition.builder() + .setIsOverride(true) + .setScope(ScopeNode.PUBLIC) + .setReturnType(anonClassType) + .setName("getRequestBuilder") + .setReturnExpr(requestBuilderAnonClassExpr) + .build(); + } + + private static MethodDefinition createSplitResponseMethod( + Method method, GapicBatchingSettings batchingSettings, Map messageTypes) { + VariableExpr batchResponseVarExpr = + VariableExpr.withVariable( + Variable.builder().setType(method.outputType()).setName("batchResponse").build()); + + TypeNode batchedRequestIssuerType = toType(BATCHED_REQUEST_ISSUER_REF, method.outputType()); + TypeNode batchVarType = + TypeNode.withReference( + ConcreteReference.builder() + .setClazz(Collection.class) + .setGenerics( + Arrays.asList( + ConcreteReference.wildcardWithUpperBound( + batchedRequestIssuerType.reference()))) + .build()); + VariableExpr batchVarExpr = + VariableExpr.withVariable( + Variable.builder().setType(batchVarType).setName("batch").build()); + + VariableExpr responderVarExpr = + VariableExpr.withVariable( + Variable.builder().setType(batchedRequestIssuerType).setName("responder").build()); + + String upperCamelBatchedFieldName = + JavaStyle.toUpperCamelCase(batchingSettings.batchedFieldName()); + VariableExpr batchMessageIndexVarExpr = + VariableExpr.withVariable( + Variable.builder().setType(TypeNode.INT).setName("batchMessageIndex").build()); + + VariableExpr subresponseElementsVarExpr = null; + boolean hasSubresponseField = batchingSettings.subresponseFieldName() != null; + + List outerForBody = new ArrayList<>(); + if (hasSubresponseField) { + Message outputMessage = messageTypes.get(method.outputType().reference().name()); + Preconditions.checkNotNull( + outputMessage, String.format("Output message not found for RPC %s", method.name())); + + Field subresponseElementField = + outputMessage.fieldMap().get(batchingSettings.subresponseFieldName()); + Preconditions.checkNotNull( + subresponseElementField, + String.format( + "Subresponse field %s not found in message %s", + batchingSettings.subresponseFieldName(), outputMessage.name())); + TypeNode subresponseElementType = subresponseElementField.type(); + subresponseElementsVarExpr = + VariableExpr.withVariable( + Variable.builder() + .setType(subresponseElementType) + .setName("subresponseElements") + .build()); + + VariableExpr subresponseCountVarExpr = + VariableExpr.withVariable( + Variable.builder().setType(TypeNode.LONG).setName("subresponseCount").build()); + + outerForBody.add( + ExprStatement.withExpr( + AssignmentExpr.builder() + .setVariableExpr(subresponseElementsVarExpr.toBuilder().setIsDecl(true).build()) + .setValueExpr( + NewObjectExpr.builder() + .setType( + TypeNode.withReference(ConcreteReference.withClazz(ArrayList.class))) + .setIsGeneric(true) + .build()) + .build())); + + String getFooCountMethodName = "getMessageCount"; + outerForBody.add( + ExprStatement.withExpr( + AssignmentExpr.builder() + .setVariableExpr(subresponseCountVarExpr.toBuilder().setIsDecl(true).build()) + .setValueExpr( + MethodInvocationExpr.builder() + .setExprReferenceExpr(responderVarExpr) + .setMethodName(getFooCountMethodName) + .setReturnType(subresponseCountVarExpr.type()) + .build()) + .build())); + + List innerSubresponseForExprs = new ArrayList<>(); + String getSubresponseFieldMethodName = + String.format( + "get%s", JavaStyle.toUpperCamelCase(batchingSettings.subresponseFieldName())); + Expr addMethodArgExpr = + MethodInvocationExpr.builder() + .setExprReferenceExpr(batchResponseVarExpr) + .setMethodName(getSubresponseFieldMethodName) + .setArguments(UnaryOperationExpr.postfixIncrementWithExpr(batchMessageIndexVarExpr)) + .build(); + innerSubresponseForExprs.add( + MethodInvocationExpr.builder() + .setExprReferenceExpr(subresponseElementsVarExpr) + .setMethodName("add") + .setArguments(addMethodArgExpr) + .build()); + // TODO(miraleung): Increment batchMessageIndexVarExpr. + + VariableExpr forIndexVarExpr = + VariableExpr.builder() + .setIsDecl(true) + .setVariable(Variable.builder().setType(TypeNode.INT).setName("i").build()) + .build(); + ValueExpr initValueExpr = + ValueExpr.withValue(PrimitiveValue.builder().setValue("0").setType(TypeNode.INT).build()); + GeneralForStatement innerSubresponseForStatement = + GeneralForStatement.incrementWith( + forIndexVarExpr, + initValueExpr, + subresponseCountVarExpr, + innerSubresponseForExprs.stream() + .map(e -> ExprStatement.withExpr(e)) + .collect(Collectors.toList())); + + outerForBody.add(innerSubresponseForStatement); + } + + TypeNode responseType = method.outputType(); + Expr responseBuilderExpr = + MethodInvocationExpr.builder() + .setStaticReferenceType(responseType) + .setMethodName("newBuilder") + .build(); + if (hasSubresponseField) { + Preconditions.checkNotNull( + subresponseElementsVarExpr, + String.format( + "subresponseElements variable should not be null for method %s", method.name())); + + responseBuilderExpr = + MethodInvocationExpr.builder() + .setExprReferenceExpr(responseBuilderExpr) + .setMethodName( + String.format( + "addAll%s", + JavaStyle.toUpperCamelCase(batchingSettings.subresponseFieldName()))) + .setArguments(subresponseElementsVarExpr) + .build(); + } + responseBuilderExpr = + MethodInvocationExpr.builder() + .setExprReferenceExpr(responseBuilderExpr) + .setMethodName("build") + .setReturnType(responseType) + .build(); + + VariableExpr responseVarExpr = + VariableExpr.withVariable( + Variable.builder().setType(responseType).setName("response").build()); + outerForBody.add( + ExprStatement.withExpr( + AssignmentExpr.builder() + .setVariableExpr(responseVarExpr.toBuilder().setIsDecl(true).build()) + .setValueExpr(responseBuilderExpr) + .build())); + + outerForBody.add( + ExprStatement.withExpr( + MethodInvocationExpr.builder() + .setExprReferenceExpr(responderVarExpr) + .setMethodName("setResponse") + .setArguments(responseVarExpr) + .build())); + + ForStatement outerForStatement = + ForStatement.builder() + .setLocalVariableExpr(responderVarExpr.toBuilder().setIsDecl(true).build()) + .setCollectionExpr(batchVarExpr) + .setBody(outerForBody) + .build(); + + List bodyStatements = new ArrayList<>(); + if (hasSubresponseField) { + bodyStatements.add( + ExprStatement.withExpr( + AssignmentExpr.builder() + .setVariableExpr(batchMessageIndexVarExpr.toBuilder().setIsDecl(true).build()) + .setValueExpr( + ValueExpr.withValue( + PrimitiveValue.builder().setType(TypeNode.INT).setValue("0").build())) + .build())); + } + bodyStatements.add(outerForStatement); + + return MethodDefinition.builder() + .setIsOverride(true) + .setScope(ScopeNode.PUBLIC) + .setReturnType(TypeNode.VOID) + .setName("splitResponse") + .setArguments( + Arrays.asList(batchResponseVarExpr, batchVarExpr).stream() + .map(v -> v.toBuilder().setIsDecl(true).build()) + .collect(Collectors.toList())) + .setBody(bodyStatements) + .build(); + } + + private static MethodDefinition createSplitExceptionMethod(Method method) { + VariableExpr throwableVarExpr = + VariableExpr.withVariable( + Variable.builder().setType(toType(Throwable.class)).setName("throwable").build()); + + TypeNode batchedRequestIssuerType = toType(BATCHED_REQUEST_ISSUER_REF, method.outputType()); + TypeNode batchVarType = + TypeNode.withReference( + ConcreteReference.builder() + .setClazz(Collection.class) + .setGenerics( + Arrays.asList( + ConcreteReference.wildcardWithUpperBound( + batchedRequestIssuerType.reference()))) + .build()); + VariableExpr batchVarExpr = + VariableExpr.withVariable( + Variable.builder().setType(batchVarType).setName("batch").build()); + VariableExpr responderVarExpr = + VariableExpr.withVariable( + Variable.builder().setType(batchedRequestIssuerType).setName("responder").build()); + + ForStatement forStatement = + ForStatement.builder() + .setLocalVariableExpr(responderVarExpr.toBuilder().setIsDecl(true).build()) + .setCollectionExpr(batchVarExpr) + .setBody( + Arrays.asList( + ExprStatement.withExpr( + MethodInvocationExpr.builder() + .setExprReferenceExpr(responderVarExpr) + .setMethodName("setException") + .setArguments(throwableVarExpr) + .build()))) + .build(); + + return MethodDefinition.builder() + .setIsOverride(true) + .setScope(ScopeNode.PUBLIC) + .setReturnType(TypeNode.VOID) + .setName("splitException") + .setArguments( + Arrays.asList(throwableVarExpr, batchVarExpr).stream() + .map(v -> v.toBuilder().setIsDecl(true).build()) + .collect(Collectors.toList())) + .setBody(Arrays.asList(forStatement)) + .build(); + } + + private static MethodDefinition createCountElementsMethod( + Method method, GapicBatchingSettings batchingSettings) { + String getFooCountMethodName = + String.format( + GET_COUNT_METHOD_PATTERN, + JavaStyle.toUpperCamelCase(batchingSettings.batchedFieldName())); + VariableExpr requestVarExpr = + VariableExpr.withVariable( + Variable.builder().setType(method.inputType()).setName("request").build()); + + return MethodDefinition.builder() + .setIsOverride(true) + .setScope(ScopeNode.PUBLIC) + .setReturnType(TypeNode.LONG) + .setName("countElements") + .setArguments(requestVarExpr.toBuilder().setIsDecl(true).build()) + .setReturnExpr( + MethodInvocationExpr.builder() + .setExprReferenceExpr(requestVarExpr) + .setMethodName(getFooCountMethodName) + .setReturnType(TypeNode.LONG) + .build()) + .build(); + } + + private static MethodDefinition createCountByteSMethod(Method method) { + VariableExpr requestVarExpr = + VariableExpr.withVariable( + Variable.builder().setType(method.inputType()).setName("request").build()); + return MethodDefinition.builder() + .setIsOverride(true) + .setScope(ScopeNode.PUBLIC) + .setReturnType(TypeNode.LONG) + .setName("countBytes") + .setArguments(requestVarExpr.toBuilder().setIsDecl(true).build()) + .setReturnExpr( + MethodInvocationExpr.builder() + .setExprReferenceExpr(requestVarExpr) + .setMethodName("getSerializedSize") + .setReturnType(TypeNode.LONG) + .build()) + .build(); + } + + private static TypeNode toType(Class clazz) { + return TypeNode.withReference(ConcreteReference.withClazz(clazz)); + } + + private static TypeNode toType(Reference reference, TypeNode... types) { + return TypeNode.withReference( + reference.copyAndSetGenerics( + Arrays.asList(types).stream().map(t -> t.reference()).collect(Collectors.toList()))); + } +} diff --git a/src/main/java/com/google/api/generator/gapic/composer/RetrySettingsComposer.java b/src/main/java/com/google/api/generator/gapic/composer/RetrySettingsComposer.java index a1f9acd2d2..1f5aa03569 100644 --- a/src/main/java/com/google/api/generator/gapic/composer/RetrySettingsComposer.java +++ b/src/main/java/com/google/api/generator/gapic/composer/RetrySettingsComposer.java @@ -14,6 +14,9 @@ package com.google.api.generator.gapic.composer; +import com.google.api.gax.batching.BatchingSettings; +import com.google.api.gax.batching.FlowControlSettings; +import com.google.api.gax.batching.FlowController; import com.google.api.gax.grpc.ProtoOperationTransformers; import com.google.api.gax.longrunning.OperationSnapshot; import com.google.api.gax.longrunning.OperationTimedPollAlgorithm; @@ -34,6 +37,7 @@ import com.google.api.generator.engine.ast.ValueExpr; import com.google.api.generator.engine.ast.Variable; import com.google.api.generator.engine.ast.VariableExpr; +import com.google.api.generator.gapic.model.GapicBatchingSettings; import com.google.api.generator.gapic.model.GapicRetrySettings; import com.google.api.generator.gapic.model.GapicServiceConfig; import com.google.api.generator.gapic.model.Method; @@ -363,6 +367,108 @@ public static Expr createLroSettingsBuilderExpr( return builderSettingsExpr; } + public static Expr createBatchingBuilderSettingsExpr( + String settingsGetterMethodName, + GapicBatchingSettings batchingSettings, + VariableExpr builderVarExpr) { + + Expr batchingSettingsBuilderExpr = + MethodInvocationExpr.builder() + .setStaticReferenceType(STATIC_TYPES.get("BatchingSettings")) + .setMethodName("newBuilder") + .build(); + + batchingSettingsBuilderExpr = + MethodInvocationExpr.builder() + .setExprReferenceExpr(batchingSettingsBuilderExpr) + .setMethodName("setElementCountThreshold") + .setArguments(toValExpr(batchingSettings.elementCountThreshold())) + .build(); + + batchingSettingsBuilderExpr = + MethodInvocationExpr.builder() + .setExprReferenceExpr(batchingSettingsBuilderExpr) + .setMethodName("setRequestByteThreshold") + .setArguments(toValExpr(batchingSettings.requestByteThreshold())) + .build(); + + batchingSettingsBuilderExpr = + MethodInvocationExpr.builder() + .setExprReferenceExpr(batchingSettingsBuilderExpr) + .setMethodName("setDelayThreshold") + .setArguments( + createDurationOfMillisExpr(toValExpr(batchingSettings.delayThresholdMillis()))) + .build(); + + // FlowControlSettings. + Expr flowControlSettingsExpr = + MethodInvocationExpr.builder() + .setStaticReferenceType(STATIC_TYPES.get("FlowControlSettings")) + .setMethodName("newBuilder") + .build(); + if (batchingSettings.flowControlElementLimit() != null) { + flowControlSettingsExpr = + MethodInvocationExpr.builder() + .setExprReferenceExpr(flowControlSettingsExpr) + .setMethodName("setMaxOutstandingElementCount") + .setArguments(toValExpr(batchingSettings.flowControlElementLimit())) + .build(); + } + if (batchingSettings.flowControlByteLimit() != null) { + flowControlSettingsExpr = + MethodInvocationExpr.builder() + .setExprReferenceExpr(flowControlSettingsExpr) + .setMethodName("setMaxOutstandingRequestBytes") + .setArguments(toValExpr(batchingSettings.flowControlByteLimit())) + .build(); + } + flowControlSettingsExpr = + MethodInvocationExpr.builder() + .setExprReferenceExpr(flowControlSettingsExpr) + .setMethodName("setLimitExceededBehavior") + .setArguments( + EnumRefExpr.builder() + .setType(STATIC_TYPES.get("LimitExceededBehavior")) + .setName( + JavaStyle.toUpperCamelCase( + batchingSettings + .flowControlLimitExceededBehavior() + .name() + .toLowerCase())) + .build()) + .build(); + flowControlSettingsExpr = + MethodInvocationExpr.builder() + .setExprReferenceExpr(flowControlSettingsExpr) + .setMethodName("build") + .build(); + + batchingSettingsBuilderExpr = + MethodInvocationExpr.builder() + .setExprReferenceExpr(batchingSettingsBuilderExpr) + .setMethodName("setFlowControlSettings") + .setArguments(flowControlSettingsExpr) + .build(); + + batchingSettingsBuilderExpr = + MethodInvocationExpr.builder() + .setExprReferenceExpr(batchingSettingsBuilderExpr) + .setMethodName("build") + .build(); + + // Put everything together. + Expr builderExpr = + MethodInvocationExpr.builder() + .setExprReferenceExpr(builderVarExpr) + .setMethodName(settingsGetterMethodName) + .build(); + return MethodInvocationExpr.builder() + .setExprReferenceExpr(builderExpr) + .setMethodName("setBatchingSettings") + .setArguments(batchingSettingsBuilderExpr) + .build(); + } + private static Expr createRetryCodeDefinitionExpr( String codeName, List retryCodes, VariableExpr definitionsVarExpr) { // Construct something like `definitions.put("code_name", @@ -597,7 +703,10 @@ private static MethodInvocationExpr createDurationOfMillisExpr(ValueExpr valExpr private static Map createStaticTypes() { List concreteClazzes = Arrays.asList( + BatchingSettings.class, org.threeten.bp.Duration.class, + FlowControlSettings.class, + FlowController.LimitExceededBehavior.class, ImmutableMap.class, ImmutableSet.class, Lists.class, diff --git a/src/main/java/com/google/api/generator/gapic/composer/ServiceStubSettingsClassComposer.java b/src/main/java/com/google/api/generator/gapic/composer/ServiceStubSettingsClassComposer.java index 9fe65b37b6..46e458100b 100644 --- a/src/main/java/com/google/api/generator/gapic/composer/ServiceStubSettingsClassComposer.java +++ b/src/main/java/com/google/api/generator/gapic/composer/ServiceStubSettingsClassComposer.java @@ -18,6 +18,11 @@ import com.google.api.core.ApiFunction; import com.google.api.core.ApiFuture; import com.google.api.core.BetaApi; +import com.google.api.gax.batching.BatchingSettings; +import com.google.api.gax.batching.FlowControlSettings; +import com.google.api.gax.batching.FlowController.LimitExceededBehavior; +import com.google.api.gax.batching.PartitionKey; +import com.google.api.gax.batching.RequestBuilder; import com.google.api.gax.core.GaxProperties; import com.google.api.gax.core.GoogleCredentialsProvider; import com.google.api.gax.core.InstantiatingExecutorProvider; @@ -30,6 +35,9 @@ import com.google.api.gax.retrying.RetrySettings; import com.google.api.gax.rpc.ApiCallContext; import com.google.api.gax.rpc.ApiClientHeaderProvider; +import com.google.api.gax.rpc.BatchedRequestIssuer; +import com.google.api.gax.rpc.BatchingCallSettings; +import com.google.api.gax.rpc.BatchingDescriptor; import com.google.api.gax.rpc.ClientContext; import com.google.api.gax.rpc.OperationCallSettings; import com.google.api.gax.rpc.PageContext; @@ -74,6 +82,7 @@ import com.google.api.generator.engine.ast.Variable; import com.google.api.generator.engine.ast.VariableExpr; import com.google.api.generator.gapic.model.Field; +import com.google.api.generator.gapic.model.GapicBatchingSettings; import com.google.api.generator.gapic.model.GapicClass; import com.google.api.generator.gapic.model.GapicServiceConfig; import com.google.api.generator.gapic.model.Message; @@ -222,8 +231,12 @@ private static Map createMethodSettingsClassMemberVarExprs Map varExprs = new LinkedHashMap<>(); // Creates class variables Settings, e.g. echoSettings. + // TODO(miraleung): Handle batching here. for (Method method : service.methods()) { - TypeNode settingsType = getCallSettingsType(method, types, isNestedClass); + boolean hasBatchingSettings = + !Objects.isNull(serviceConfig) && serviceConfig.hasBatchingSetting(service, method); + TypeNode settingsType = + getCallSettingsType(method, types, hasBatchingSettings, isNestedClass); String varName = JavaStyle.toLowerCamelCase(String.format("%sSettings", method.name())); varExprs.put( varName, @@ -310,6 +323,20 @@ private static List createClassStatements( statements.add(EMPTY_LINE_STATEMENT); } + for (Method method : service.methods()) { + Optional batchingSettingOpt = + Objects.isNull(serviceConfig) + ? Optional.empty() + : serviceConfig.getBatchingSetting(service, method); + if (batchingSettingOpt.isPresent()) { + statements.add( + exprToStatementFn.apply( + BatchingDescriptorComposer.createBatchingDescriptorFieldDeclExpr( + method, batchingSettingOpt.get(), messageTypes))); + } + statements.add(EMPTY_LINE_STATEMENT); + } + return statements; } @@ -1244,6 +1271,22 @@ private static MethodDefinition createNestedClassInitDefaultsMethod( if (streamKind.equals(Method.Stream.CLIENT) || streamKind.equals(Method.Stream.BIDI)) { continue; } + if (!Objects.isNull(serviceConfig) && serviceConfig.hasBatchingSetting(service, method)) { + Optional batchingSettingOpt = + serviceConfig.getBatchingSetting(service, method); + Preconditions.checkState( + batchingSettingOpt.isPresent(), + String.format( + "No batching setting found for service %s, method %s", + service.name(), method.name())); + String settingsGetterMethodName = + String.format("%sSettings", JavaStyle.toLowerCamelCase(method.name())); + bodyStatements.add( + ExprStatement.withExpr( + RetrySettingsComposer.createBatchingBuilderSettingsExpr( + settingsGetterMethodName, batchingSettingOpt.get(), builderVarExpr))); + bodyStatements.add(EMPTY_LINE_STATEMENT); + } bodyStatements.add( ExprStatement.withExpr( @@ -1319,6 +1362,8 @@ private static List createNestedClassConstructorMethods( .build()); Reference pagedSettingsBuilderRef = ConcreteReference.withClazz(PagedCallSettings.Builder.class); + Reference batchingSettingsBuilderRef = + ConcreteReference.withClazz(BatchingCallSettings.Builder.class); Reference unaryCallSettingsBuilderRef = ConcreteReference.withClazz(UnaryCallSettings.Builder.class); Function isUnaryCallSettingsBuilderFn = @@ -1328,6 +1373,9 @@ private static List createNestedClassConstructorMethods( .equals(unaryCallSettingsBuilderRef); Function isPagedCallSettingsBuilderFn = t -> t.reference().copyAndSetGenerics(ImmutableList.of()).equals(pagedSettingsBuilderRef); + Function isBatchingCallSettingsBuilderFn = + t -> + t.reference().copyAndSetGenerics(ImmutableList.of()).equals(batchingSettingsBuilderRef); Function builderToCallSettingsFn = t -> TypeNode.withReference( @@ -1358,20 +1406,61 @@ private static List createNestedClassConstructorMethods( String methodName = getMethodNameFromSettingsVarName(e.getKey()); if (!isPagedCallSettingsBuilderFn.apply(varType)) { - boolean isUnaryCallSettings = isUnaryCallSettingsBuilderFn.apply(varType); + if (!isBatchingCallSettingsBuilderFn.apply(varType)) { + boolean isUnaryCallSettings = isUnaryCallSettingsBuilderFn.apply(varType); + Expr builderExpr = + AssignmentExpr.builder() + .setVariableExpr(varExpr) + .setValueExpr( + MethodInvocationExpr.builder() + .setStaticReferenceType( + builderToCallSettingsFn.apply(varExpr.type())) + .setMethodName( + isUnaryCallSettings + ? "newUnaryCallSettingsBuilder" + : "newBuilder") + .setReturnType(varExpr.type()) + .build()) + .build(); + return ExprStatement.withExpr(builderExpr); + } + Expr newBatchingSettingsExpr = + MethodInvocationExpr.builder() + .setStaticReferenceType(STATIC_TYPES.get("BatchingSettings")) + .setMethodName("newBuilder") + .build(); + newBatchingSettingsExpr = + MethodInvocationExpr.builder() + .setExprReferenceExpr(newBatchingSettingsExpr) + .setMethodName("build") + .build(); + + String batchingDescVarName = + String.format( + BATCHING_DESC_PATTERN, JavaStyle.toUpperSnakeCase(methodName)); + Expr batchingSettingsBuilderExpr = + MethodInvocationExpr.builder() + .setStaticReferenceType(builderToCallSettingsFn.apply(varType)) + .setMethodName("newBuilder") + .setArguments( + VariableExpr.withVariable( + Variable.builder() + .setType(STATIC_TYPES.get("BatchingDescriptor")) + .setName(batchingDescVarName) + .build())) + .build(); + batchingSettingsBuilderExpr = + MethodInvocationExpr.builder() + .setExprReferenceExpr(batchingSettingsBuilderExpr) + .setMethodName("setBatchingSettings") + .setArguments(newBatchingSettingsExpr) + .setReturnType(varType) + .build(); + Expr builderExpr = AssignmentExpr.builder() .setVariableExpr(varExpr) - .setValueExpr( - MethodInvocationExpr.builder() - .setStaticReferenceType( - builderToCallSettingsFn.apply(varExpr.type())) - .setMethodName( - isUnaryCallSettings - ? "newUnaryCallSettingsBuilder" - : "newBuilder") - .setReturnType(varExpr.type()) - .build()) + .setValueExpr(batchingSettingsBuilderExpr) .build(); return ExprStatement.withExpr(builderExpr); } @@ -1417,7 +1506,8 @@ private static List createNestedClassConstructorMethods( .filter( v -> isUnaryCallSettingsBuilderFn.apply(v.type()) - || isPagedCallSettingsBuilderFn.apply(v.type())) + || isPagedCallSettingsBuilderFn.apply(v.type()) + || isBatchingCallSettingsBuilderFn.apply(v.type())) .collect(Collectors.toList())) .setReturnType(NESTED_UNARY_METHOD_SETTINGS_BUILDERS_VAR_EXPR.type()) .build()) @@ -1699,10 +1789,15 @@ private static Map createStaticTypes() { ApiClientHeaderProvider.class, ApiFunction.class, ApiFuture.class, + BatchedRequestIssuer.class, + BatchingCallSettings.class, + BatchingDescriptor.class, + BatchingSettings.class, BetaApi.class, ClientContext.class, Duration.class, Empty.class, + FlowControlSettings.class, GaxGrpcProperties.class, GaxProperties.class, Generated.class, @@ -1714,6 +1809,7 @@ private static Map createStaticTypes() { ImmutableSet.class, InstantiatingExecutorProvider.class, InstantiatingGrpcChannelProvider.class, + LimitExceededBehavior.class, List.class, Lists.class, MonitoredResourceDescriptor.class, @@ -1725,7 +1821,9 @@ private static Map createStaticTypes() { PagedCallSettings.class, PagedListDescriptor.class, PagedListResponseFactory.class, + PartitionKey.class, ProtoOperationTransformers.class, + RequestBuilder.class, RetrySettings.class, ServerStreamingCallSettings.class, StatusCode.class, @@ -1881,7 +1979,10 @@ private static String getGrpcServiceStubTypeName(String serviceName) { } private static TypeNode getCallSettingsType( - Method method, Map types, final boolean isSettingsBuilder) { + Method method, + Map types, + boolean isBatchingSettings, + final boolean isSettingsBuilder) { Function typeMakerFn = clz -> TypeNode.withReference(ConcreteReference.withClazz(clz)); // Default: No streaming. @@ -1891,6 +1992,11 @@ private static TypeNode getCallSettingsType( isSettingsBuilder ? PagedCallSettings.Builder.class : PagedCallSettings.class) : typeMakerFn.apply( isSettingsBuilder ? UnaryCallSettings.Builder.class : UnaryCallSettings.class); + if (isBatchingSettings) { + callSettingsType = + typeMakerFn.apply( + isSettingsBuilder ? BatchingCallSettings.Builder.class : BatchingCallSettings.class); + } // Streaming takes precedence over paging, as per the monolith's existing behavior. switch (method.stream()) { diff --git a/src/main/java/com/google/api/generator/gapic/model/GapicBatchingSettings.java b/src/main/java/com/google/api/generator/gapic/model/GapicBatchingSettings.java new file mode 100644 index 0000000000..2f7d4de00d --- /dev/null +++ b/src/main/java/com/google/api/generator/gapic/model/GapicBatchingSettings.java @@ -0,0 +1,99 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.api.generator.gapic.model; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; +import java.util.List; +import javax.annotation.Nullable; + +@AutoValue +public abstract class GapicBatchingSettings { + public enum FlowControlLimitExceededBehavior { + THROW_EXCEPTION, + BLOCK, + IGNORE + }; + + // Threshold fields. + public abstract String protoPakkage(); + + public abstract String serviceName(); + + public abstract String methodName(); + + public abstract int elementCountThreshold(); + + public abstract long requestByteThreshold(); + + public abstract long delayThresholdMillis(); + + @Nullable + public abstract Integer flowControlElementLimit(); + + @Nullable + public abstract Integer flowControlByteLimit(); + + public abstract FlowControlLimitExceededBehavior flowControlLimitExceededBehavior(); + + // Batch descriptor fields. + public abstract String batchedFieldName(); + + public abstract ImmutableList discriminatorFieldNames(); + + @Nullable + public abstract String subresponseFieldName(); + + public boolean matches(Service service, Method method) { + return protoPakkage().equals(service.protoPakkage()) + && serviceName().equals(service.name()) + && methodName().equals(method.name()); + } + + public static Builder builder() { + return new AutoValue_GapicBatchingSettings.Builder() + .setFlowControlLimitExceededBehavior(FlowControlLimitExceededBehavior.IGNORE); + } + + @AutoValue.Builder + public abstract static class Builder { + public abstract Builder setProtoPakkage(String protoPakkage); + + public abstract Builder setServiceName(String serviceName); + + public abstract Builder setMethodName(String methodName); + + public abstract Builder setElementCountThreshold(int elementCountThreshold); + + public abstract Builder setRequestByteThreshold(long requestByteThreshold); + + public abstract Builder setDelayThresholdMillis(long delalyThresholdMillis); + + public abstract Builder setFlowControlElementLimit(Integer flowControlElementLimit); + + public abstract Builder setFlowControlByteLimit(Integer flowControlByteLimit); + + public abstract Builder setFlowControlLimitExceededBehavior( + FlowControlLimitExceededBehavior behavior); + + public abstract Builder setBatchedFieldName(String batchedFieldName); + + public abstract Builder setDiscriminatorFieldNames(List discriminatorFieldNames); + + public abstract Builder setSubresponseFieldName(String subresponseFieldName); + + public abstract GapicBatchingSettings build(); + } +} diff --git a/src/main/java/com/google/api/generator/gapic/model/GapicServiceConfig.java b/src/main/java/com/google/api/generator/gapic/model/GapicServiceConfig.java index 73d0b38b4d..d161f44861 100644 --- a/src/main/java/com/google/api/generator/gapic/model/GapicServiceConfig.java +++ b/src/main/java/com/google/api/generator/gapic/model/GapicServiceConfig.java @@ -42,14 +42,19 @@ public class GapicServiceConfig { private final List methodConfigs; private final Map methodConfigTable; + private final Map batchingSettingsTable; private GapicServiceConfig( - List methodConfigs, Map methodConfigTable) { + List methodConfigs, + Map methodConfigTable, + Map batchingSettingsTable) { this.methodConfigs = methodConfigs; this.methodConfigTable = methodConfigTable; + this.batchingSettingsTable = batchingSettingsTable; } - public static GapicServiceConfig create(ServiceConfig serviceConfig) { + public static GapicServiceConfig create( + ServiceConfig serviceConfig, Optional> batchingSettingsOpt) { // Keep this processing logic out of the constructor. Map methodConfigTable = new HashMap<>(); List methodConfigs = serviceConfig.getMethodConfigList(); @@ -60,7 +65,21 @@ public static GapicServiceConfig create(ServiceConfig serviceConfig) { } } - return new GapicServiceConfig(methodConfigs, methodConfigTable); + Map batchingSettingsTable = new HashMap<>(); + if (batchingSettingsOpt.isPresent()) { + for (GapicBatchingSettings batchingSetting : batchingSettingsOpt.get()) { + batchingSettingsTable.put( + MethodConfig.Name.newBuilder() + .setService( + String.format( + "%s.%s", batchingSetting.protoPakkage(), batchingSetting.serviceName())) + .setMethod(batchingSetting.methodName()) + .build(), + batchingSetting); + } + } + + return new GapicServiceConfig(methodConfigs, methodConfigTable, batchingSettingsTable); } public Map getAllGapicRetrySettings(Service service) { @@ -99,6 +118,16 @@ public String getRetryParamsName(Service service, Method method) { return NO_RETRY_PARAMS_NAME; } + public boolean hasBatchingSetting(Service service, Method method) { + return batchingSettingsTable.containsKey(toName(service, method)); + } + + public Optional getBatchingSetting(Service service, Method method) { + return hasBatchingSetting(service, method) + ? Optional.of(batchingSettingsTable.get(toName(service, method))) + : Optional.empty(); + } + private GapicRetrySettings toGapicRetrySettings(Service service, Method method) { GapicRetrySettings.Kind kind = GapicRetrySettings.Kind.FULL; Optional retryPolicyIndexOpt = retryPolicyIndexLookup(service, method); diff --git a/src/main/java/com/google/api/generator/gapic/protoparser/BatchingSettingsConfigParser.java b/src/main/java/com/google/api/generator/gapic/protoparser/BatchingSettingsConfigParser.java new file mode 100644 index 0000000000..79a1c9817d --- /dev/null +++ b/src/main/java/com/google/api/generator/gapic/protoparser/BatchingSettingsConfigParser.java @@ -0,0 +1,190 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.api.generator.gapic.protoparser; + +import com.google.api.generator.gapic.model.GapicBatchingSettings; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.constructor.SafeConstructor; + +public class BatchingSettingsConfigParser { + private static String LIMIT_EXCEEDED_BEHAVIOR_THROW_EXCEPTION_YAML_VALUE = "THROW_EXCEPTION"; + private static String LIMIT_EXCEEDED_BEHAVIOR_BLOCK_YAML_VALUE = "BLOCK"; + + private static String YAML_KEY_INTERFACES = "interfaces"; + private static String YAML_KEY_NAME = "name"; + private static String YAML_KEY_METHODS = "methods"; + private static String YAML_KEY_BATCHING = "batching"; + private static String YAML_KEY_THRESHOLDS = "thresholds"; + private static String YAML_KEY_DESCRIPTOR = "batch_descriptor"; + + private static String YAML_KEY_BATCHING_ELEMENT_COUNT_THRESHOLD = "element_count_threshold"; + private static String YAML_KEY_BATCHING_DELAY_THRESHOLD_MILLIS = "delay_threshold_millis"; + private static String YAML_KEY_BATCHING_REQUEST_BYTE_THRESHOLD = "request_byte_threshold"; + private static String YAML_KEY_BATCHING_FLOW_CONTROL_ELEMENT_LIMIT = "flow_control_element_limit"; + private static String YAML_KEY_BATCHING_FLOW_CONTROL_BYTE_LIMIT = "flow_control_byte_limit"; + private static String YAML_KEY_BATCHING_FLOW_CONTROL_LIMIT_EXCEEDED_BEHAVIOR = + "flow_control_limit_exceeded_behavior"; + + private static String YAML_KEY_DESCRIPTOR_BATCHED_FIELD = "batched_field"; + private static String YAML_KEY_DESCRIPTOR_DISCRIMINATOR_FIELD = "discriminator_fields"; + private static String YAML_KEY_DESCRIPTOR_SUBRESPONSE_FIELD = "subresponse_field"; + + public static Optional> parse( + Optional gapicYamlConfigFilePathOpt) { + return gapicYamlConfigFilePathOpt.isPresent() + ? parse(gapicYamlConfigFilePathOpt.get()) + : Optional.empty(); + } + + @VisibleForTesting + static Optional> parse(String gapicYamlConfigFilePath) { + if (Strings.isNullOrEmpty(gapicYamlConfigFilePath) + || !(new File(gapicYamlConfigFilePath)).exists()) { + return Optional.empty(); + } + + String fileContents = null; + + try { + fileContents = new String(Files.readAllBytes(Paths.get(gapicYamlConfigFilePath))); + } catch (IOException e) { + return Optional.empty(); + } + + Yaml yaml = new Yaml(new SafeConstructor()); + Map yamlMap = yaml.load(fileContents); + return parseFromMap(yamlMap); + } + + private static Optional> parseFromMap(Map yamlMap) { + if (!yamlMap.containsKey(YAML_KEY_INTERFACES)) { + return Optional.empty(); + } + + List settings = new ArrayList<>(); + for (Map serviceYamlConfig : + (List>) yamlMap.get(YAML_KEY_INTERFACES)) { + if (!serviceYamlConfig.containsKey(YAML_KEY_METHODS)) { + continue; + } + for (Map methodYamlConfig : + (List>) serviceYamlConfig.get(YAML_KEY_METHODS)) { + if (!methodYamlConfig.containsKey(YAML_KEY_BATCHING)) { + continue; + } + Map batchingOuterYamlConfig = + (Map) methodYamlConfig.get(YAML_KEY_BATCHING); + if (!batchingOuterYamlConfig.containsKey(YAML_KEY_THRESHOLDS)) { + continue; + } + Preconditions.checkState( + batchingOuterYamlConfig.containsKey(YAML_KEY_DESCRIPTOR), + String.format( + "%s key expected but not found for method %s", + YAML_KEY_DESCRIPTOR, (String) methodYamlConfig.get(YAML_KEY_NAME))); + + // Parse the threshold values first. + Map batchingYamlConfig = + (Map) batchingOuterYamlConfig.get(YAML_KEY_THRESHOLDS); + Preconditions.checkState( + batchingYamlConfig.containsKey(YAML_KEY_BATCHING_ELEMENT_COUNT_THRESHOLD) + && batchingYamlConfig.containsKey(YAML_KEY_BATCHING_REQUEST_BYTE_THRESHOLD) + && batchingYamlConfig.containsKey(YAML_KEY_BATCHING_DELAY_THRESHOLD_MILLIS), + String.format( + "Batching YAML config is missing one of %s, %s, or %s fields", + YAML_KEY_BATCHING_ELEMENT_COUNT_THRESHOLD, + YAML_KEY_BATCHING_REQUEST_BYTE_THRESHOLD, + YAML_KEY_BATCHING_DELAY_THRESHOLD_MILLIS)); + + String interfaceName = (String) serviceYamlConfig.get(YAML_KEY_NAME); + int lastDotIndex = interfaceName.lastIndexOf("."); + String protoPakkage = interfaceName.substring(0, lastDotIndex); + String serviceName = interfaceName.substring(lastDotIndex + 1); + String methodName = (String) methodYamlConfig.get(YAML_KEY_NAME); + GapicBatchingSettings.Builder settingsBuilder = + GapicBatchingSettings.builder() + .setProtoPakkage(protoPakkage) + .setServiceName(serviceName) + .setMethodName(methodName) + .setElementCountThreshold( + (Integer) batchingYamlConfig.get(YAML_KEY_BATCHING_ELEMENT_COUNT_THRESHOLD)) + .setRequestByteThreshold( + (Integer) batchingYamlConfig.get(YAML_KEY_BATCHING_REQUEST_BYTE_THRESHOLD)) + .setDelayThresholdMillis( + (Integer) batchingYamlConfig.get(YAML_KEY_BATCHING_DELAY_THRESHOLD_MILLIS)); + + if (batchingYamlConfig.containsKey(YAML_KEY_BATCHING_FLOW_CONTROL_ELEMENT_LIMIT)) { + settingsBuilder.setFlowControlElementLimit( + (Integer) batchingYamlConfig.get(YAML_KEY_BATCHING_FLOW_CONTROL_ELEMENT_LIMIT)); + } + if (batchingYamlConfig.containsKey(YAML_KEY_BATCHING_FLOW_CONTROL_BYTE_LIMIT)) { + settingsBuilder.setFlowControlByteLimit( + (Integer) batchingYamlConfig.get(YAML_KEY_BATCHING_FLOW_CONTROL_BYTE_LIMIT)); + } + if (batchingYamlConfig.containsKey( + YAML_KEY_BATCHING_FLOW_CONTROL_LIMIT_EXCEEDED_BEHAVIOR)) { + String behaviorYamlValue = + (String) + batchingYamlConfig.get(YAML_KEY_BATCHING_FLOW_CONTROL_LIMIT_EXCEEDED_BEHAVIOR); + GapicBatchingSettings.FlowControlLimitExceededBehavior behaviorSetting = + GapicBatchingSettings.FlowControlLimitExceededBehavior.IGNORE; + // IGNORE or UNSET_BEHAVIOR YAML values map to FlowControlLimitExceededBehavior.IGNOER. + if (behaviorYamlValue.equals(LIMIT_EXCEEDED_BEHAVIOR_THROW_EXCEPTION_YAML_VALUE)) { + behaviorSetting = + GapicBatchingSettings.FlowControlLimitExceededBehavior.THROW_EXCEPTION; + } else if (behaviorYamlValue.equals(LIMIT_EXCEEDED_BEHAVIOR_BLOCK_YAML_VALUE)) { + behaviorSetting = GapicBatchingSettings.FlowControlLimitExceededBehavior.BLOCK; + } + settingsBuilder.setFlowControlLimitExceededBehavior(behaviorSetting); + } + + // Parse the descriptor values. + Map descriptorYamlConfig = + (Map) batchingOuterYamlConfig.get(YAML_KEY_DESCRIPTOR); + Preconditions.checkState( + descriptorYamlConfig.containsKey(YAML_KEY_DESCRIPTOR_BATCHED_FIELD) + && descriptorYamlConfig.containsKey(YAML_KEY_DESCRIPTOR_DISCRIMINATOR_FIELD), + String.format( + "Batching descriptor YAML config is missing one of %s or %s fields", + YAML_KEY_DESCRIPTOR_BATCHED_FIELD, YAML_KEY_DESCRIPTOR_DISCRIMINATOR_FIELD)); + + settingsBuilder.setBatchedFieldName( + (String) descriptorYamlConfig.get(YAML_KEY_DESCRIPTOR_BATCHED_FIELD)); + settingsBuilder.setDiscriminatorFieldNames( + (List) descriptorYamlConfig.get(YAML_KEY_DESCRIPTOR_DISCRIMINATOR_FIELD)); + + if (descriptorYamlConfig.containsKey(YAML_KEY_DESCRIPTOR_SUBRESPONSE_FIELD)) { + settingsBuilder.setSubresponseFieldName( + (String) descriptorYamlConfig.get(YAML_KEY_DESCRIPTOR_SUBRESPONSE_FIELD)); + } + + settings.add(settingsBuilder.build()); + } + } + + return settings.isEmpty() ? Optional.empty() : Optional.of(settings); + } +} diff --git a/src/main/java/com/google/api/generator/gapic/protoparser/Parser.java b/src/main/java/com/google/api/generator/gapic/protoparser/Parser.java index eea185b777..aebf249d60 100644 --- a/src/main/java/com/google/api/generator/gapic/protoparser/Parser.java +++ b/src/main/java/com/google/api/generator/gapic/protoparser/Parser.java @@ -19,6 +19,7 @@ import com.google.api.ResourceProto; import com.google.api.generator.engine.ast.TypeNode; import com.google.api.generator.gapic.model.Field; +import com.google.api.generator.gapic.model.GapicBatchingSettings; import com.google.api.generator.gapic.model.GapicContext; import com.google.api.generator.gapic.model.GapicServiceConfig; import com.google.api.generator.gapic.model.LongrunningOperation; @@ -79,9 +80,15 @@ public GapicParserException(String errorMessage) { } public static GapicContext parse(CodeGeneratorRequest request) { + Optional gapicYamlConfigPathOpt = + PluginArgumentParser.parseGapicYamlConfigPath(request); + Optional> batchingSettingsOpt = + BatchingSettingsConfigParser.parse(gapicYamlConfigPathOpt); + Optional serviceConfigPathOpt = PluginArgumentParser.parseJsonConfigPath(request); String serviceConfigPath = serviceConfigPathOpt.isPresent() ? serviceConfigPathOpt.get() : null; - Optional serviceConfigOpt = ServiceConfigParser.parse(serviceConfigPath); + Optional serviceConfigOpt = + ServiceConfigParser.parse(serviceConfigPath, batchingSettingsOpt); Optional serviceYamlConfigPathOpt = PluginArgumentParser.parseServiceYamlConfigPath(request); diff --git a/src/main/java/com/google/api/generator/gapic/protoparser/PluginArgumentParser.java b/src/main/java/com/google/api/generator/gapic/protoparser/PluginArgumentParser.java index 0cd4a17bc8..7cf29a1d29 100644 --- a/src/main/java/com/google/api/generator/gapic/protoparser/PluginArgumentParser.java +++ b/src/main/java/com/google/api/generator/gapic/protoparser/PluginArgumentParser.java @@ -26,6 +26,7 @@ public class PluginArgumentParser { // Synced to rules_java_gapic/java_gapic.bzl. @VisibleForTesting static final String KEY_GRPC_SERVICE_CONFIG = "grpc-service-config"; + @VisibleForTesting static final String KEY_GAPIC_CONFIG = "gapic-config"; @VisibleForTesting static final String KEY_SERVICE_YAML_CONFIG = "gapic-service-config"; private static final String JSON_FILE_ENDING = "grpc_service_config.json"; @@ -36,6 +37,10 @@ static Optional parseJsonConfigPath(CodeGeneratorRequest request) { return parseJsonConfigPath(request.getParameter()); } + static Optional parseGapicYamlConfigPath(CodeGeneratorRequest request) { + return parseGapicYamlConfigPath(request.getParameter()); + } + static Optional parseServiceYamlConfigPath(CodeGeneratorRequest request) { return parseServiceYamlConfigPath(request.getParameter()); } @@ -46,6 +51,11 @@ static Optional parseJsonConfigPath(String pluginProtocArgument) { return parseArgument(pluginProtocArgument, KEY_GRPC_SERVICE_CONFIG, JSON_FILE_ENDING); } + @VisibleForTesting + static Optional parseGapicYamlConfigPath(String pluginProtocArgument) { + return parseArgument(pluginProtocArgument, KEY_GAPIC_CONFIG, GAPIC_YAML_FILE_ENDING); + } + @VisibleForTesting static Optional parseServiceYamlConfigPath(String pluginProtocArgument) { return parseArgument(pluginProtocArgument, KEY_SERVICE_YAML_CONFIG, SERVICE_YAML_FILE_ENDING); diff --git a/src/main/java/com/google/api/generator/gapic/protoparser/ServiceConfigParser.java b/src/main/java/com/google/api/generator/gapic/protoparser/ServiceConfigParser.java index 24bbe84716..dcab2c4ec3 100644 --- a/src/main/java/com/google/api/generator/gapic/protoparser/ServiceConfigParser.java +++ b/src/main/java/com/google/api/generator/gapic/protoparser/ServiceConfigParser.java @@ -14,6 +14,7 @@ package com.google.api.generator.gapic.protoparser; +import com.google.api.generator.gapic.model.GapicBatchingSettings; import com.google.api.generator.gapic.model.GapicServiceConfig; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; @@ -22,15 +23,17 @@ import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; +import java.util.List; import java.util.Optional; public class ServiceConfigParser { - public static Optional parse(String serviceConfigFilePath) { + public static Optional parse( + String serviceConfigFilePath, Optional> batchingSettingsOpt) { Optional rawConfigOpt = parseFile(serviceConfigFilePath); if (!rawConfigOpt.isPresent()) { return Optional.empty(); } - return Optional.of(GapicServiceConfig.create(rawConfigOpt.get())); + return Optional.of(GapicServiceConfig.create(rawConfigOpt.get(), batchingSettingsOpt)); } @VisibleForTesting diff --git a/src/test/java/com/google/api/generator/gapic/composer/BUILD.bazel b/src/test/java/com/google/api/generator/gapic/composer/BUILD.bazel index d95a2dc2be..0e0e941a90 100644 --- a/src/test/java/com/google/api/generator/gapic/composer/BUILD.bazel +++ b/src/test/java/com/google/api/generator/gapic/composer/BUILD.bazel @@ -4,6 +4,7 @@ load("//:rules_bazel/java/java_diff_test.bzl", "golden_update") package(default_visibility = ["//visibility:public"]) UPDATE_GOLDENS_TESTS = [ + "BatchingDescriptorComposerTest", "ComposerTest", "GrpcServiceCallableFactoryClassComposerTest", "GrpcServiceStubClassComposerTest", @@ -74,6 +75,7 @@ java_proto_library( ], data = [ "//src/test/java/com/google/api/generator/gapic/composer/goldens:goldens_files", + "//src/test/java/com/google/api/generator/gapic/testdata:gapic_config_files", "//src/test/java/com/google/api/generator/gapic/testdata:service_config_files", ], test_class = "com.google.api.generator.gapic.composer.{0}".format(test_name), @@ -93,6 +95,7 @@ TEST_CLASS_DIR = "com.google.api.generator.gapic.composer." ], data = [ "//src/test/java/com/google/api/generator/gapic/composer/goldens:goldens_files", + "//src/test/java/com/google/api/generator/gapic/testdata:gapic_config_files", "//src/test/java/com/google/api/generator/gapic/testdata:service_config_files", ], test_class = "com.google.api.generator.gapic.composer.{0}".format(test_name), diff --git a/src/test/java/com/google/api/generator/gapic/composer/BatchingDescriptorComposerTest.java b/src/test/java/com/google/api/generator/gapic/composer/BatchingDescriptorComposerTest.java new file mode 100644 index 0000000000..e2d366370e --- /dev/null +++ b/src/test/java/com/google/api/generator/gapic/composer/BatchingDescriptorComposerTest.java @@ -0,0 +1,193 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.api.generator.gapic.composer; + +import static junit.framework.Assert.assertEquals; +import static junit.framework.Assert.assertTrue; + +import com.google.api.generator.engine.ast.Expr; +import com.google.api.generator.engine.writer.JavaWriterVisitor; +import com.google.api.generator.gapic.model.GapicBatchingSettings; +import com.google.api.generator.gapic.model.GapicServiceConfig; +import com.google.api.generator.gapic.model.Message; +import com.google.api.generator.gapic.model.Method; +import com.google.api.generator.gapic.model.ResourceName; +import com.google.api.generator.gapic.model.Service; +import com.google.api.generator.gapic.protoparser.BatchingSettingsConfigParser; +import com.google.api.generator.gapic.protoparser.Parser; +import com.google.api.generator.gapic.protoparser.ServiceConfigParser; +import com.google.api.generator.test.framework.Assert; +import com.google.api.generator.test.framework.Utils; +import com.google.logging.v2.LogEntryProto; +import com.google.logging.v2.LoggingConfigProto; +import com.google.logging.v2.LoggingMetricsProto; +import com.google.logging.v2.LoggingProto; +import com.google.protobuf.Descriptors.FileDescriptor; +import com.google.protobuf.Descriptors.ServiceDescriptor; +import com.google.pubsub.v1.PubsubProto; +import google.cloud.CommonResources; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import org.junit.Before; +import org.junit.Test; + +public class BatchingDescriptorComposerTest { + private JavaWriterVisitor writerVisitor; + + @Before + public void setUp() { + writerVisitor = new JavaWriterVisitor(); + } + + @Test + public void batchingDescriptor_hasSubresponseField() { + FileDescriptor serviceFileDescriptor = PubsubProto.getDescriptor(); + FileDescriptor commonResourcesFileDescriptor = CommonResources.getDescriptor(); + ServiceDescriptor serviceDescriptor = serviceFileDescriptor.getServices().get(0); + assertEquals("Publisher", serviceDescriptor.getName()); + + Map resourceNames = new HashMap<>(); + resourceNames.putAll(Parser.parseResourceNames(serviceFileDescriptor)); + resourceNames.putAll(Parser.parseResourceNames(commonResourcesFileDescriptor)); + + Map messageTypes = Parser.parseMessages(serviceFileDescriptor); + + Set outputResourceNames = new HashSet<>(); + List services = + Parser.parseService( + serviceFileDescriptor, + messageTypes, + resourceNames, + Optional.empty(), + outputResourceNames); + + String filename = "pubsub_gapic.yaml"; + Path path = Paths.get(ComposerConstants.TESTFILES_DIRECTORY, filename); + Optional> batchingSettingsOpt = + BatchingSettingsConfigParser.parse(Optional.of(path.toString())); + assertTrue(batchingSettingsOpt.isPresent()); + + String jsonFilename = "pubsub_grpc_service_config.json"; + Path jsonPath = Paths.get(ComposerConstants.TESTFILES_DIRECTORY, jsonFilename); + Optional configOpt = + ServiceConfigParser.parse(jsonPath.toString(), batchingSettingsOpt); + assertTrue(configOpt.isPresent()); + GapicServiceConfig config = configOpt.get(); + + Service service = services.get(0); + assertEquals("Publisher", service.name()); + Method method = findMethod(service, "Publish"); + + GapicBatchingSettings batchingSetting = batchingSettingsOpt.get().get(0); + assertEquals("Publish", batchingSetting.methodName()); + Expr batchingDescriptorExpr = + BatchingDescriptorComposer.createBatchingDescriptorFieldDeclExpr( + method, batchingSetting, messageTypes); + + batchingDescriptorExpr.accept(writerVisitor); + Utils.saveCodegenToFile( + this.getClass(), "BatchingDescriptorComposerTestSubresponse.golden", writerVisitor.write()); + Path goldenFilePath = + Paths.get( + ComposerConstants.GOLDENFILES_DIRECTORY, + "BatchingDescriptorComposerTestSubresponse.golden"); + Assert.assertCodeEquals(goldenFilePath, writerVisitor.write()); + } + + @Test + public void batchingDescriptor_noSubresponseField() { + FileDescriptor serviceFileDescriptor = LoggingProto.getDescriptor(); + ServiceDescriptor serviceDescriptor = serviceFileDescriptor.getServices().get(0); + assertEquals(serviceDescriptor.getName(), "LoggingServiceV2"); + + List protoFiles = + Arrays.asList( + serviceFileDescriptor, + LogEntryProto.getDescriptor(), + LoggingConfigProto.getDescriptor(), + LoggingMetricsProto.getDescriptor()); + + Map resourceNames = new HashMap<>(); + Map messageTypes = new HashMap<>(); + for (FileDescriptor fileDescriptor : protoFiles) { + resourceNames.putAll(Parser.parseResourceNames(fileDescriptor)); + messageTypes.putAll(Parser.parseMessages(fileDescriptor)); + } + + Set outputResourceNames = new HashSet<>(); + List services = + Parser.parseService( + serviceFileDescriptor, + messageTypes, + resourceNames, + Optional.empty(), + outputResourceNames); + + String filename = "logging_gapic.yaml"; + Path path = Paths.get(ComposerConstants.TESTFILES_DIRECTORY, filename); + Optional> batchingSettingsOpt = + BatchingSettingsConfigParser.parse(Optional.of(path.toString())); + assertTrue(batchingSettingsOpt.isPresent()); + + String jsonFilename = "logging_grpc_service_config.json"; + Path jsonPath = Paths.get(ComposerConstants.TESTFILES_DIRECTORY, jsonFilename); + Optional configOpt = + ServiceConfigParser.parse(jsonPath.toString(), batchingSettingsOpt); + assertTrue(configOpt.isPresent()); + GapicServiceConfig config = configOpt.get(); + + Service service = services.get(0); + assertEquals("LoggingServiceV2", service.name()); + Method method = findMethod(service, "WriteLogEntries"); + + GapicBatchingSettings batchingSetting = batchingSettingsOpt.get().get(0); + assertEquals("WriteLogEntries", batchingSetting.methodName()); + Expr batchingDescriptorExpr = + BatchingDescriptorComposer.createBatchingDescriptorFieldDeclExpr( + method, batchingSetting, messageTypes); + + batchingDescriptorExpr.accept(writerVisitor); + Utils.saveCodegenToFile( + this.getClass(), + "BatchingDescriptorComposerTestNoSubresponse.golden", + writerVisitor.write()); + Path goldenFilePath = + Paths.get( + ComposerConstants.GOLDENFILES_DIRECTORY, + "BatchingDescriptorComposerTestNoSubresponse.golden"); + Assert.assertCodeEquals(goldenFilePath, writerVisitor.write()); + } + + private static Method findMethod(Service service, String methodName) { + for (Method m : service.methods()) { + if (m.name().equals(methodName)) { + return m; + } + } + return null; + } + + private static String createLines(String... lines) { + // Cast to get rid of warnings. + return String.format(new String(new char[lines.length]).replace("\0", "%s"), (Object[]) lines); + } +} diff --git a/src/test/java/com/google/api/generator/gapic/composer/RetrySettingsComposerTest.java b/src/test/java/com/google/api/generator/gapic/composer/RetrySettingsComposerTest.java index 80a8364fd5..7a1f2ab030 100644 --- a/src/test/java/com/google/api/generator/gapic/composer/RetrySettingsComposerTest.java +++ b/src/test/java/com/google/api/generator/gapic/composer/RetrySettingsComposerTest.java @@ -27,22 +27,31 @@ import com.google.api.generator.engine.ast.Variable; import com.google.api.generator.engine.ast.VariableExpr; import com.google.api.generator.engine.writer.JavaWriterVisitor; +import com.google.api.generator.gapic.model.GapicBatchingSettings; import com.google.api.generator.gapic.model.GapicServiceConfig; import com.google.api.generator.gapic.model.Message; import com.google.api.generator.gapic.model.Method; import com.google.api.generator.gapic.model.ResourceName; import com.google.api.generator.gapic.model.Service; +import com.google.api.generator.gapic.protoparser.BatchingSettingsConfigParser; import com.google.api.generator.gapic.protoparser.Parser; import com.google.api.generator.gapic.protoparser.ServiceConfigParser; import com.google.api.generator.testutils.LineFormatter; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.logging.v2.LogEntryProto; +import com.google.logging.v2.LoggingConfigProto; +import com.google.logging.v2.LoggingMetricsProto; +import com.google.logging.v2.LoggingProto; import com.google.protobuf.Descriptors.FileDescriptor; import com.google.protobuf.Descriptors.ServiceDescriptor; +import com.google.pubsub.v1.PubsubProto; import com.google.showcase.v1beta1.EchoOuterClass; +import google.cloud.CommonResources; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -81,7 +90,8 @@ public void paramDefinitionsBlock_noConfigsFound() { String jsonFilename = "retrying_grpc_service_config.json"; Path jsonPath = Paths.get(ComposerConstants.TESTFILES_DIRECTORY, jsonFilename); - Optional serviceConfigOpt = ServiceConfigParser.parse(jsonPath.toString()); + Optional serviceConfigOpt = + ServiceConfigParser.parse(jsonPath.toString(), Optional.empty()); assertTrue(serviceConfigOpt.isPresent()); GapicServiceConfig serviceConfig = serviceConfigOpt.get(); @@ -118,7 +128,8 @@ public void paramDefinitionsBlock_basic() { String jsonFilename = "showcase_grpc_service_config.json"; Path jsonPath = Paths.get(ComposerConstants.TESTFILES_DIRECTORY, jsonFilename); - Optional serviceConfigOpt = ServiceConfigParser.parse(jsonPath.toString()); + Optional serviceConfigOpt = + ServiceConfigParser.parse(jsonPath.toString(), Optional.empty()); assertTrue(serviceConfigOpt.isPresent()); GapicServiceConfig serviceConfig = serviceConfigOpt.get(); @@ -169,7 +180,8 @@ public void codesDefinitionsBlock_noConfigsFound() { String jsonFilename = "retrying_grpc_service_config.json"; Path jsonPath = Paths.get(ComposerConstants.TESTFILES_DIRECTORY, jsonFilename); - Optional serviceConfigOpt = ServiceConfigParser.parse(jsonPath.toString()); + Optional serviceConfigOpt = + ServiceConfigParser.parse(jsonPath.toString(), Optional.empty()); assertTrue(serviceConfigOpt.isPresent()); GapicServiceConfig serviceConfig = serviceConfigOpt.get(); @@ -206,7 +218,8 @@ public void codesDefinitionsBlock_basic() { String jsonFilename = "showcase_grpc_service_config.json"; Path jsonPath = Paths.get(ComposerConstants.TESTFILES_DIRECTORY, jsonFilename); - Optional serviceConfigOpt = ServiceConfigParser.parse(jsonPath.toString()); + Optional serviceConfigOpt = + ServiceConfigParser.parse(jsonPath.toString(), Optional.empty()); assertTrue(serviceConfigOpt.isPresent()); GapicServiceConfig serviceConfig = serviceConfigOpt.get(); @@ -246,7 +259,8 @@ public void simpleBuilderExpr_basic() { String jsonFilename = "showcase_grpc_service_config.json"; Path jsonPath = Paths.get(ComposerConstants.TESTFILES_DIRECTORY, jsonFilename); - Optional serviceConfigOpt = ServiceConfigParser.parse(jsonPath.toString()); + Optional serviceConfigOpt = + ServiceConfigParser.parse(jsonPath.toString(), Optional.empty()); assertTrue(serviceConfigOpt.isPresent()); GapicServiceConfig serviceConfig = serviceConfigOpt.get(); @@ -328,7 +342,8 @@ public void lroBuilderExpr() { String jsonFilename = "showcase_grpc_service_config.json"; Path jsonPath = Paths.get(ComposerConstants.TESTFILES_DIRECTORY, jsonFilename); - Optional serviceConfigOpt = ServiceConfigParser.parse(jsonPath.toString()); + Optional serviceConfigOpt = + ServiceConfigParser.parse(jsonPath.toString(), Optional.empty()); assertTrue(serviceConfigOpt.isPresent()); GapicServiceConfig serviceConfig = serviceConfigOpt.get(); @@ -365,6 +380,164 @@ public void lroBuilderExpr() { assertEquals(expected, writerVisitor.write()); } + @Test + public void batchingSettings_minimalFlowControlSettings() { + FileDescriptor serviceFileDescriptor = PubsubProto.getDescriptor(); + FileDescriptor commonResourcesFileDescriptor = CommonResources.getDescriptor(); + ServiceDescriptor serviceDescriptor = serviceFileDescriptor.getServices().get(0); + assertEquals("Publisher", serviceDescriptor.getName()); + + Map resourceNames = new HashMap<>(); + resourceNames.putAll(Parser.parseResourceNames(serviceFileDescriptor)); + resourceNames.putAll(Parser.parseResourceNames(commonResourcesFileDescriptor)); + + Map messageTypes = Parser.parseMessages(serviceFileDescriptor); + + Set outputResourceNames = new HashSet<>(); + List services = + Parser.parseService( + serviceFileDescriptor, + messageTypes, + resourceNames, + Optional.empty(), + outputResourceNames); + + String filename = "pubsub_gapic.yaml"; + Path path = Paths.get(ComposerConstants.TESTFILES_DIRECTORY, filename); + Optional> batchingSettingsOpt = + BatchingSettingsConfigParser.parse(Optional.of(path.toString())); + assertTrue(batchingSettingsOpt.isPresent()); + + String jsonFilename = "pubsub_grpc_service_config.json"; + Path jsonPath = Paths.get(ComposerConstants.TESTFILES_DIRECTORY, jsonFilename); + Optional configOpt = + ServiceConfigParser.parse(jsonPath.toString(), batchingSettingsOpt); + assertTrue(configOpt.isPresent()); + GapicServiceConfig config = configOpt.get(); + + Service service = services.get(0); + assertEquals("Publisher", service.name()); + + VariableExpr builderVarExpr = createBuilderVarExpr(service); + String methodSettingsName = "publishSettings"; + GapicBatchingSettings batchingSettings = + GapicBatchingSettings.builder() + .setProtoPakkage("com.google.pubsub.v1") + .setServiceName("Publishing") + .setMethodName("Publish") + .setElementCountThreshold(100) + .setRequestByteThreshold(1048576) + .setDelayThresholdMillis(10) + .setBatchedFieldName("messages") + .setDiscriminatorFieldNames(Arrays.asList("topic")) + .setSubresponseFieldName("message_ids") + .build(); + + Expr builderExpr = + RetrySettingsComposer.createBatchingBuilderSettingsExpr( + methodSettingsName, batchingSettings, builderVarExpr); + builderExpr.accept(writerVisitor); + String expected = + "builder" + + ".publishSettings()" + + ".setBatchingSettings(" + + "BatchingSettings.newBuilder()" + + ".setElementCountThreshold(100L)" + + ".setRequestByteThreshold(1048576L)" + + ".setDelayThreshold(Duration.ofMillis(10L))" + + ".setFlowControlSettings(" + + "FlowControlSettings.newBuilder()" + + ".setLimitExceededBehavior(FlowController.LimitExceededBehavior.Ignore)" + + ".build())" + + ".build())"; + assertEquals(expected, writerVisitor.write()); + } + + @Test + public void batchingSettings_fullFlowControlSettings() { + FileDescriptor serviceFileDescriptor = LoggingProto.getDescriptor(); + ServiceDescriptor serviceDescriptor = serviceFileDescriptor.getServices().get(0); + assertEquals(serviceDescriptor.getName(), "LoggingServiceV2"); + + List protoFiles = + Arrays.asList( + serviceFileDescriptor, + LogEntryProto.getDescriptor(), + LoggingConfigProto.getDescriptor(), + LoggingMetricsProto.getDescriptor()); + + Map resourceNames = new HashMap<>(); + Map messageTypes = new HashMap<>(); + for (FileDescriptor fileDescriptor : protoFiles) { + resourceNames.putAll(Parser.parseResourceNames(fileDescriptor)); + messageTypes.putAll(Parser.parseMessages(fileDescriptor)); + } + + Set outputResourceNames = new HashSet<>(); + List services = + Parser.parseService( + serviceFileDescriptor, + messageTypes, + resourceNames, + Optional.empty(), + outputResourceNames); + + String filename = "logging_gapic.yaml"; + Path path = Paths.get(ComposerConstants.TESTFILES_DIRECTORY, filename); + Optional> batchingSettingsOpt = + BatchingSettingsConfigParser.parse(Optional.of(path.toString())); + assertTrue(batchingSettingsOpt.isPresent()); + + String jsonFilename = "logging_grpc_service_config.json"; + Path jsonPath = Paths.get(ComposerConstants.TESTFILES_DIRECTORY, jsonFilename); + Optional configOpt = + ServiceConfigParser.parse(jsonPath.toString(), batchingSettingsOpt); + assertTrue(configOpt.isPresent()); + GapicServiceConfig config = configOpt.get(); + + Service service = services.get(0); + assertEquals("LoggingServiceV2", service.name()); + + VariableExpr builderVarExpr = createBuilderVarExpr(service); + String methodSettingsName = "writeLogEntriesSettings"; + GapicBatchingSettings batchingSettings = + GapicBatchingSettings.builder() + .setProtoPakkage("com.google.logging.v2") + .setServiceName("LoggingServiceV2") + .setMethodName("WriteLogEntries") + .setElementCountThreshold(1000) + .setRequestByteThreshold(1048576) + .setDelayThresholdMillis(50) + .setFlowControlElementLimit(100000) + .setFlowControlByteLimit(10485760) + .setFlowControlLimitExceededBehavior( + GapicBatchingSettings.FlowControlLimitExceededBehavior.THROW_EXCEPTION) + .setBatchedFieldName("entries") + .setDiscriminatorFieldNames(Arrays.asList("log_name", "resource", "labels")) + .build(); + + Expr builderExpr = + RetrySettingsComposer.createBatchingBuilderSettingsExpr( + methodSettingsName, batchingSettings, builderVarExpr); + builderExpr.accept(writerVisitor); + String expected = + "builder" + + ".writeLogEntriesSettings()" + + ".setBatchingSettings(" + + "BatchingSettings.newBuilder()" + + ".setElementCountThreshold(1000L)" + + ".setRequestByteThreshold(1048576L)" + + ".setDelayThreshold(Duration.ofMillis(50L))" + + ".setFlowControlSettings(" + + "FlowControlSettings.newBuilder()" + + ".setMaxOutstandingElementCount(100000L)" + + ".setMaxOutstandingRequestBytes(10485760L)" + + ".setLimitExceededBehavior(FlowController.LimitExceededBehavior.ThrowException)" + + ".build())" + + ".build())"; + assertEquals(expected, writerVisitor.write()); + } + private static Method findMethod(Service service, String methodName) { for (Method m : service.methods()) { if (m.name().equals(methodName)) { diff --git a/src/test/java/com/google/api/generator/gapic/composer/ServiceStubSettingsClassComposerTest.java b/src/test/java/com/google/api/generator/gapic/composer/ServiceStubSettingsClassComposerTest.java index dfc35f44d5..527e513101 100644 --- a/src/test/java/com/google/api/generator/gapic/composer/ServiceStubSettingsClassComposerTest.java +++ b/src/test/java/com/google/api/generator/gapic/composer/ServiceStubSettingsClassComposerTest.java @@ -18,11 +18,13 @@ import static junit.framework.Assert.assertTrue; import com.google.api.generator.engine.writer.JavaWriterVisitor; +import com.google.api.generator.gapic.model.GapicBatchingSettings; import com.google.api.generator.gapic.model.GapicClass; import com.google.api.generator.gapic.model.GapicServiceConfig; import com.google.api.generator.gapic.model.Message; import com.google.api.generator.gapic.model.ResourceName; import com.google.api.generator.gapic.model.Service; +import com.google.api.generator.gapic.protoparser.BatchingSettingsConfigParser; import com.google.api.generator.gapic.protoparser.Parser; import com.google.api.generator.gapic.protoparser.ServiceConfigParser; import com.google.api.generator.test.framework.Assert; @@ -51,7 +53,7 @@ public class ServiceStubSettingsClassComposerTest { @Test - public void generateServiceStubSettingsClasses_batchingNotHandled() throws IOException { + public void generateServiceStubSettingsClasses_batchingWithEmptyResponses() throws IOException { FileDescriptor serviceFileDescriptor = LoggingProto.getDescriptor(); ServiceDescriptor serviceDescriptor = serviceFileDescriptor.getServices().get(0); assertEquals(serviceDescriptor.getName(), "LoggingServiceV2"); @@ -73,9 +75,16 @@ public void generateServiceStubSettingsClasses_batchingNotHandled() throws IOExc List services = parseServices(serviceFileDescriptor, serviceDescriptor, messageTypes, resourceNames); + String filename = "logging_gapic.yaml"; + Path path = Paths.get(ComposerConstants.TESTFILES_DIRECTORY, filename); + Optional> batchingSettingsOpt = + BatchingSettingsConfigParser.parse(Optional.of(path.toString())); + assertTrue(batchingSettingsOpt.isPresent()); + String jsonFilename = "logging_grpc_service_config.json"; Path jsonPath = Paths.get(ComposerConstants.TESTFILES_DIRECTORY, jsonFilename); - Optional configOpt = ServiceConfigParser.parse(jsonPath.toString()); + Optional configOpt = + ServiceConfigParser.parse(jsonPath.toString(), batchingSettingsOpt); assertTrue(configOpt.isPresent()); GapicServiceConfig config = configOpt.get(); @@ -93,7 +102,8 @@ public void generateServiceStubSettingsClasses_batchingNotHandled() throws IOExc } @Test - public void generateServiceStubSettingsClasses_batchingNotHandledInPubSub() throws IOException { + public void generateServiceStubSettingsClasses_batchingWithNonemptyResponses() + throws IOException { FileDescriptor serviceFileDescriptor = PubsubProto.getDescriptor(); FileDescriptor commonResourcesFileDescriptor = CommonResources.getDescriptor(); ServiceDescriptor serviceDescriptor = serviceFileDescriptor.getServices().get(0); @@ -108,9 +118,16 @@ public void generateServiceStubSettingsClasses_batchingNotHandledInPubSub() thro List services = parseServices(serviceFileDescriptor, serviceDescriptor, messageTypes, resourceNames); + String filename = "pubsub_gapic.yaml"; + Path path = Paths.get(ComposerConstants.TESTFILES_DIRECTORY, filename); + Optional> batchingSettingsOpt = + BatchingSettingsConfigParser.parse(Optional.of(path.toString())); + assertTrue(batchingSettingsOpt.isPresent()); + String jsonFilename = "pubsub_grpc_service_config.json"; Path jsonPath = Paths.get(ComposerConstants.TESTFILES_DIRECTORY, jsonFilename); - Optional configOpt = ServiceConfigParser.parse(jsonPath.toString()); + Optional configOpt = + ServiceConfigParser.parse(jsonPath.toString(), batchingSettingsOpt); assertTrue(configOpt.isPresent()); GapicServiceConfig config = configOpt.get(); @@ -140,7 +157,8 @@ public void generateServiceStubSettingsClasses_basic() throws IOException { String jsonFilename = "showcase_grpc_service_config.json"; Path jsonPath = Paths.get(ComposerConstants.TESTFILES_DIRECTORY, jsonFilename); - Optional configOpt = ServiceConfigParser.parse(jsonPath.toString()); + Optional configOpt = + ServiceConfigParser.parse(jsonPath.toString(), Optional.empty()); assertTrue(configOpt.isPresent()); GapicServiceConfig config = configOpt.get(); diff --git a/src/test/java/com/google/api/generator/gapic/composer/goldens/BatchingDescriptorComposerTestNoSubresponse.golden b/src/test/java/com/google/api/generator/gapic/composer/goldens/BatchingDescriptorComposerTestNoSubresponse.golden new file mode 100644 index 0000000000..b9c9eba122 --- /dev/null +++ b/src/test/java/com/google/api/generator/gapic/composer/goldens/BatchingDescriptorComposerTestNoSubresponse.golden @@ -0,0 +1,53 @@ +private static final BatchingDescriptor WRITE_LOG_ENTRIES_BATCHING_DESC = new BatchingDescriptor() { +@Override +public PartitionKey getBatchPartitionKey(WriteLogEntriesRequest request) { +return new PartitionKey(request.getLogName(), request.getResource(), request.getLabels()); +} + +@Override +public RequestBuilder getRequestBuilder() { +return new RequestBuilder() { +private WriteLogEntriesRequest.Builder builder; +@Override +public void appendRequest(WriteLogEntriesRequest request) { +if (Objects.isNull(builder)) { +builder = request.toBuilder(); +} else { +builder.addAllEntries(request.getEntriesList()); +} +} + +@Override +public WriteLogEntriesRequest build() { +return builder.build(); +} + +}; +} + +@Override +public void splitResponse(WriteLogEntriesResponse batchResponse, Collection> batch) { +for (BatchedRequestIssuer responder : batch) { +WriteLogEntriesResponse response = WriteLogEntriesResponse.newBuilder().build(); +responder.setResponse(response); +} +} + +@Override +public void splitException(Throwable throwable, Collection> batch) { +for (BatchedRequestIssuer responder : batch) { +responder.setException(throwable); +} +} + +@Override +public long countElements(WriteLogEntriesRequest request) { +return request.getEntriesCount(); +} + +@Override +public long countBytes(WriteLogEntriesRequest request) { +return request.getSerializedSize(); +} + +} \ No newline at end of file diff --git a/src/test/java/com/google/api/generator/gapic/composer/goldens/BatchingDescriptorComposerTestSubresponse.golden b/src/test/java/com/google/api/generator/gapic/composer/goldens/BatchingDescriptorComposerTestSubresponse.golden new file mode 100644 index 0000000000..a322c69aec --- /dev/null +++ b/src/test/java/com/google/api/generator/gapic/composer/goldens/BatchingDescriptorComposerTestSubresponse.golden @@ -0,0 +1,59 @@ +private static final BatchingDescriptor PUBLISH_BATCHING_DESC = new BatchingDescriptor() { +@Override +public PartitionKey getBatchPartitionKey(PublishRequest request) { +return new PartitionKey(request.getTopic()); +} + +@Override +public RequestBuilder getRequestBuilder() { +return new RequestBuilder() { +private PublishRequest.Builder builder; +@Override +public void appendRequest(PublishRequest request) { +if (Objects.isNull(builder)) { +builder = request.toBuilder(); +} else { +builder.addAllMessages(request.getMessagesList()); +} +} + +@Override +public PublishRequest build() { +return builder.build(); +} + +}; +} + +@Override +public void splitResponse(PublishResponse batchResponse, Collection> batch) { +int batchMessageIndex = 0; +for (BatchedRequestIssuer responder : batch) { +List subresponseElements = new ArrayList<>(); +long subresponseCount = responder.getMessageCount(); +for (int i = 0; i < subresponseCount; i++) { +subresponseElements.add(batchResponse.getMessageIds(batchMessageIndex++)); +} +PublishResponse response = PublishResponse.newBuilder().addAllMessageIds(subresponseElements).build(); +responder.setResponse(response); +} +} + +@Override +public void splitException(Throwable throwable, Collection> batch) { +for (BatchedRequestIssuer responder : batch) { +responder.setException(throwable); +} +} + +@Override +public long countElements(PublishRequest request) { +return request.getMessagesCount(); +} + +@Override +public long countBytes(PublishRequest request) { +return request.getSerializedSize(); +} + +} \ No newline at end of file diff --git a/src/test/java/com/google/api/generator/gapic/composer/goldens/LoggingServiceV2StubSettings.golden b/src/test/java/com/google/api/generator/gapic/composer/goldens/LoggingServiceV2StubSettings.golden index 329fc273fc..ef336d4033 100644 --- a/src/test/java/com/google/api/generator/gapic/composer/goldens/LoggingServiceV2StubSettings.golden +++ b/src/test/java/com/google/api/generator/gapic/composer/goldens/LoggingServiceV2StubSettings.golden @@ -8,6 +8,11 @@ import com.google.api.MonitoredResourceDescriptor; import com.google.api.core.ApiFunction; import com.google.api.core.ApiFuture; import com.google.api.core.BetaApi; +import com.google.api.gax.batching.BatchingSettings; +import com.google.api.gax.batching.FlowControlSettings; +import com.google.api.gax.batching.FlowController; +import com.google.api.gax.batching.PartitionKey; +import com.google.api.gax.batching.RequestBuilder; import com.google.api.gax.core.GaxProperties; import com.google.api.gax.core.GoogleCredentialsProvider; import com.google.api.gax.core.InstantiatingExecutorProvider; @@ -17,6 +22,9 @@ import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; import com.google.api.gax.retrying.RetrySettings; import com.google.api.gax.rpc.ApiCallContext; import com.google.api.gax.rpc.ApiClientHeaderProvider; +import com.google.api.gax.rpc.BatchedRequestIssuer; +import com.google.api.gax.rpc.BatchingCallSettings; +import com.google.api.gax.rpc.BatchingDescriptor; import com.google.api.gax.rpc.ClientContext; import com.google.api.gax.rpc.PageContext; import com.google.api.gax.rpc.PagedCallSettings; @@ -43,6 +51,7 @@ import com.google.logging.v2.WriteLogEntriesRequest; import com.google.logging.v2.WriteLogEntriesResponse; import com.google.protobuf.Empty; import java.io.IOException; +import java.util.Collection; import java.util.List; import java.util.Objects; import javax.annotation.Generated; @@ -94,7 +103,7 @@ public class LoggingServiceV2StubSettings extends StubSettings deleteLogSettings; - private final UnaryCallSettings + private final BatchingCallSettings writeLogEntriesSettings; private final PagedCallSettings< ListLogEntriesRequest, ListLogEntriesResponse, ListLogEntriesPagedResponse> @@ -295,13 +304,73 @@ public class LoggingServiceV2StubSettings extends StubSettings + WRITE_LOG_ENTRIES_BATCHING_DESC = + new BatchingDescriptor() { + @Override + public PartitionKey getBatchPartitionKey(WriteLogEntriesRequest request) { + return new PartitionKey( + request.getLogName(), request.getResource(), request.getLabels()); + } + + @Override + public RequestBuilder getRequestBuilder() { + return new RequestBuilder() { + private WriteLogEntriesRequest.Builder builder; + + @Override + public void appendRequest(WriteLogEntriesRequest request) { + if (Objects.isNull(builder)) { + builder = request.toBuilder(); + } else { + builder.addAllEntries(request.getEntriesList()); + } + } + + @Override + public WriteLogEntriesRequest build() { + return builder.build(); + } + }; + } + + @Override + public void splitResponse( + WriteLogEntriesResponse batchResponse, + Collection> batch) { + for (BatchedRequestIssuer responder : batch) { + WriteLogEntriesResponse response = WriteLogEntriesResponse.newBuilder().build(); + responder.setResponse(response); + } + } + + @Override + public void splitException( + Throwable throwable, + Collection> batch) { + for (BatchedRequestIssuer responder : batch) { + responder.setException(throwable); + } + } + + @Override + public long countElements(WriteLogEntriesRequest request) { + return request.getEntriesCount(); + } + + @Override + public long countBytes(WriteLogEntriesRequest request) { + return request.getSerializedSize(); + } + }; + /** Returns the object with the settings used for calls to deleteLog. */ public UnaryCallSettings deleteLogSettings() { return deleteLogSettings; } /** Returns the object with the settings used for calls to writeLogEntries. */ - public UnaryCallSettings + public BatchingCallSettings writeLogEntriesSettings() { return writeLogEntriesSettings; } @@ -409,7 +478,7 @@ public class LoggingServiceV2StubSettings extends StubSettings { private final ImmutableList> unaryMethodSettingsBuilders; private final UnaryCallSettings.Builder deleteLogSettings; - private final UnaryCallSettings.Builder + private final BatchingCallSettings.Builder writeLogEntriesSettings; private final PagedCallSettings.Builder< ListLogEntriesRequest, ListLogEntriesResponse, ListLogEntriesPagedResponse> @@ -465,7 +534,9 @@ public class LoggingServiceV2StubSettings extends StubSettings + public BatchingCallSettings.Builder writeLogEntriesSettings() { return writeLogEntriesSettings; } diff --git a/src/test/java/com/google/api/generator/gapic/composer/goldens/PublisherStubSettings.golden b/src/test/java/com/google/api/generator/gapic/composer/goldens/PublisherStubSettings.golden index efd6ed6fff..4f27847cd4 100644 --- a/src/test/java/com/google/api/generator/gapic/composer/goldens/PublisherStubSettings.golden +++ b/src/test/java/com/google/api/generator/gapic/composer/goldens/PublisherStubSettings.golden @@ -7,6 +7,11 @@ import static com.google.pubsub.v1.PublisherClient.ListTopicsPagedResponse; import com.google.api.core.ApiFunction; import com.google.api.core.ApiFuture; import com.google.api.core.BetaApi; +import com.google.api.gax.batching.BatchingSettings; +import com.google.api.gax.batching.FlowControlSettings; +import com.google.api.gax.batching.FlowController; +import com.google.api.gax.batching.PartitionKey; +import com.google.api.gax.batching.RequestBuilder; import com.google.api.gax.core.GaxProperties; import com.google.api.gax.core.GoogleCredentialsProvider; import com.google.api.gax.core.InstantiatingExecutorProvider; @@ -16,6 +21,9 @@ import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; import com.google.api.gax.retrying.RetrySettings; import com.google.api.gax.rpc.ApiCallContext; import com.google.api.gax.rpc.ApiClientHeaderProvider; +import com.google.api.gax.rpc.BatchedRequestIssuer; +import com.google.api.gax.rpc.BatchingCallSettings; +import com.google.api.gax.rpc.BatchingDescriptor; import com.google.api.gax.rpc.ClientContext; import com.google.api.gax.rpc.PageContext; import com.google.api.gax.rpc.PagedCallSettings; @@ -46,6 +54,8 @@ import com.google.pubsub.v1.PublishResponse; import com.google.pubsub.v1.Topic; import com.google.pubsub.v1.UpdateTopicRequest; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Objects; import javax.annotation.Generated; @@ -94,7 +104,7 @@ public class PublisherStubSettings extends StubSettings { private final UnaryCallSettings createTopicSettings; private final UnaryCallSettings updateTopicSettings; - private final UnaryCallSettings publishSettings; + private final BatchingCallSettings publishSettings; private final UnaryCallSettings getTopicSettings; private final PagedCallSettings listTopicsSettings; @@ -289,6 +299,71 @@ public class PublisherStubSettings extends StubSettings { } }; + private static final BatchingDescriptor PUBLISH_BATCHING_DESC = + new BatchingDescriptor() { + @Override + public PartitionKey getBatchPartitionKey(PublishRequest request) { + return new PartitionKey(request.getTopic()); + } + + @Override + public RequestBuilder getRequestBuilder() { + return new RequestBuilder() { + private PublishRequest.Builder builder; + + @Override + public void appendRequest(PublishRequest request) { + if (Objects.isNull(builder)) { + builder = request.toBuilder(); + } else { + builder.addAllMessages(request.getMessagesList()); + } + } + + @Override + public PublishRequest build() { + return builder.build(); + } + }; + } + + @Override + public void splitResponse( + PublishResponse batchResponse, + Collection> batch) { + int batchMessageIndex = 0; + for (BatchedRequestIssuer responder : batch) { + List subresponseElements = new ArrayList<>(); + long subresponseCount = responder.getMessageCount(); + for (int i = 0; i < subresponseCount; i++) { + subresponseElements.add(batchResponse.getMessageIds(batchMessageIndex++)); + } + PublishResponse response = + PublishResponse.newBuilder().addAllMessageIds(subresponseElements).build(); + responder.setResponse(response); + } + } + + @Override + public void splitException( + Throwable throwable, + Collection> batch) { + for (BatchedRequestIssuer responder : batch) { + responder.setException(throwable); + } + } + + @Override + public long countElements(PublishRequest request) { + return request.getMessagesCount(); + } + + @Override + public long countBytes(PublishRequest request) { + return request.getSerializedSize(); + } + }; + /** Returns the object with the settings used for calls to createTopic. */ public UnaryCallSettings createTopicSettings() { return createTopicSettings; @@ -300,7 +375,7 @@ public class PublisherStubSettings extends StubSettings { } /** Returns the object with the settings used for calls to publish. */ - public UnaryCallSettings publishSettings() { + public BatchingCallSettings publishSettings() { return publishSettings; } @@ -426,7 +501,7 @@ public class PublisherStubSettings extends StubSettings { private final ImmutableList> unaryMethodSettingsBuilders; private final UnaryCallSettings.Builder createTopicSettings; private final UnaryCallSettings.Builder updateTopicSettings; - private final UnaryCallSettings.Builder publishSettings; + private final BatchingCallSettings.Builder publishSettings; private final UnaryCallSettings.Builder getTopicSettings; private final PagedCallSettings.Builder< ListTopicsRequest, ListTopicsResponse, ListTopicsPagedResponse> @@ -520,7 +595,9 @@ public class PublisherStubSettings extends StubSettings { createTopicSettings = UnaryCallSettings.newUnaryCallSettingsBuilder(); updateTopicSettings = UnaryCallSettings.newUnaryCallSettingsBuilder(); - publishSettings = UnaryCallSettings.newUnaryCallSettingsBuilder(); + publishSettings = + BatchingCallSettings.newBuilder(PUBLISH_BATCHING_DESC) + .setBatchingSettings(BatchingSettings.newBuilder().build()); getTopicSettings = UnaryCallSettings.newUnaryCallSettingsBuilder(); listTopicsSettings = PagedCallSettings.newBuilder(LIST_TOPICS_PAGE_STR_FACT); listTopicSubscriptionsSettings = @@ -591,6 +668,19 @@ public class PublisherStubSettings extends StubSettings { .setRetryableCodes(RETRYABLE_CODE_DEFINITIONS.get("retry_policy_0_codes")) .setRetrySettings(RETRY_PARAM_DEFINITIONS.get("retry_policy_0_params")); + builder + .publishSettings() + .setBatchingSettings( + BatchingSettings.newBuilder() + .setElementCountThreshold(100L) + .setRequestByteThreshold(1048576L) + .setDelayThreshold(Duration.ofMillis(10L)) + .setFlowControlSettings( + FlowControlSettings.newBuilder() + .setLimitExceededBehavior(FlowController.LimitExceededBehavior.Ignore) + .build()) + .build()); + builder .publishSettings() .setRetryableCodes(RETRYABLE_CODE_DEFINITIONS.get("retry_policy_1_codes")) @@ -656,7 +746,7 @@ public class PublisherStubSettings extends StubSettings { } /** Returns the builder for the settings used for calls to publish. */ - public UnaryCallSettings.Builder publishSettings() { + public BatchingCallSettings.Builder publishSettings() { return publishSettings; } diff --git a/src/test/java/com/google/api/generator/gapic/model/GapicServiceConfigTest.java b/src/test/java/com/google/api/generator/gapic/model/GapicServiceConfigTest.java index 435b4e7402..43564350d5 100644 --- a/src/test/java/com/google/api/generator/gapic/model/GapicServiceConfigTest.java +++ b/src/test/java/com/google/api/generator/gapic/model/GapicServiceConfigTest.java @@ -16,6 +16,7 @@ import static com.google.common.truth.Truth.assertThat; import static junit.framework.Assert.assertEquals; +import static junit.framework.Assert.assertFalse; import static junit.framework.Assert.assertTrue; import com.google.api.generator.gapic.protoparser.Parser; @@ -28,6 +29,7 @@ import io.grpc.serviceconfig.MethodConfig; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -48,7 +50,9 @@ public void serviceConfig_noConfigsFound() { String jsonFilename = "retrying_grpc_service_config.json"; Path jsonPath = Paths.get(JSON_DIRECTORY, jsonFilename); - Optional serviceConfigOpt = ServiceConfigParser.parse(jsonPath.toString()); + Optional> batchingSettingsOpt = Optional.empty(); + Optional serviceConfigOpt = + ServiceConfigParser.parse(jsonPath.toString(), batchingSettingsOpt); assertTrue(serviceConfigOpt.isPresent()); GapicServiceConfig serviceConfig = serviceConfigOpt.get(); @@ -77,7 +81,9 @@ public void serviceConfig_basic() { String jsonFilename = "showcase_grpc_service_config.json"; Path jsonPath = Paths.get(JSON_DIRECTORY, jsonFilename); - Optional serviceConfigOpt = ServiceConfigParser.parse(jsonPath.toString()); + Optional> batchingSettingsOpt = Optional.empty(); + Optional serviceConfigOpt = + ServiceConfigParser.parse(jsonPath.toString(), batchingSettingsOpt); assertTrue(serviceConfigOpt.isPresent()); GapicServiceConfig serviceConfig = serviceConfigOpt.get(); @@ -124,6 +130,80 @@ public void serviceConfig_basic() { assertEquals(GapicServiceConfig.EMPTY_RETRY_CODES, retryCodes.get(retryCodeName)); } + @Test + public void serviceConfig_withBatchingSettings() { + FileDescriptor echoFileDescriptor = EchoOuterClass.getDescriptor(); + ServiceDescriptor echoServiceDescriptor = echoFileDescriptor.getServices().get(0); + Service service = parseService(echoFileDescriptor); + + String jsonFilename = "showcase_grpc_service_config.json"; + Path jsonPath = Paths.get(JSON_DIRECTORY, jsonFilename); + + GapicBatchingSettings origBatchingSetting = + GapicBatchingSettings.builder() + .setProtoPakkage("google.showcase.v1beta1") + .setServiceName("Echo") + .setMethodName("Echo") + .setElementCountThreshold(1000) + .setRequestByteThreshold(2000) + .setDelayThresholdMillis(3000) + .setBatchedFieldName("name") + .setDiscriminatorFieldNames(Arrays.asList("severity")) + .build(); + Optional> batchingSettingsOpt = + Optional.of(Arrays.asList(origBatchingSetting)); + + Optional serviceConfigOpt = + ServiceConfigParser.parse(jsonPath.toString(), batchingSettingsOpt); + assertTrue(serviceConfigOpt.isPresent()); + GapicServiceConfig serviceConfig = serviceConfigOpt.get(); + + Map retrySettings = serviceConfig.getAllGapicRetrySettings(service); + assertEquals(2, retrySettings.size()); + Map> retryCodes = serviceConfig.getAllRetryCodes(service); + assertEquals(2, retryCodes.size()); + + // Echo method has an explicitly-defined setting. + Method method = findMethod(service, "Echo"); + assertThat(method).isNotNull(); + + // No change to the retry settings. + String retryParamsName = serviceConfig.getRetryParamsName(service, method); + assertEquals("retry_policy_1_params", retryParamsName); + GapicRetrySettings settings = retrySettings.get(retryParamsName); + assertThat(settings).isNotNull(); + assertEquals(10, settings.timeout().getSeconds()); + assertEquals(GapicRetrySettings.Kind.FULL, settings.kind()); + + // No changge to the retry codes. + String retryCodeName = serviceConfig.getRetryCodeName(service, method); + assertEquals("retry_policy_1_codes", retryCodeName); + List retryCode = retryCodes.get(retryCodeName); + assertThat(retryCode).containsExactly(Code.UNAVAILABLE, Code.UNKNOWN); + + // Check batching settings. + assertTrue(serviceConfig.hasBatchingSetting(service, method)); + Optional batchingSettingOpt = + serviceConfig.getBatchingSetting(service, method); + assertTrue(batchingSettingOpt.isPresent()); + GapicBatchingSettings batchingSetting = batchingSettingOpt.get(); + assertEquals( + origBatchingSetting.elementCountThreshold(), batchingSetting.elementCountThreshold()); + assertEquals( + origBatchingSetting.requestByteThreshold(), batchingSetting.requestByteThreshold()); + assertEquals( + origBatchingSetting.delayThresholdMillis(), batchingSetting.delayThresholdMillis()); + + // Chat method defaults to the service-defined setting. + method = findMethod(service, "Chat"); + assertThat(method).isNotNull(); + retryParamsName = serviceConfig.getRetryParamsName(service, method); + assertEquals("no_retry_0_params", retryParamsName); + retryCodeName = serviceConfig.getRetryCodeName(service, method); + assertEquals("no_retry_0_codes", retryCodeName); + assertFalse(serviceConfig.hasBatchingSetting(service, method)); + } + private static Service parseService(FileDescriptor fileDescriptor) { Map messageTypes = Parser.parseMessages(fileDescriptor); Map resourceNames = Parser.parseResourceNames(fileDescriptor); diff --git a/src/test/java/com/google/api/generator/gapic/protoparser/BUILD.bazel b/src/test/java/com/google/api/generator/gapic/protoparser/BUILD.bazel index eefb4b9789..7df82d359f 100644 --- a/src/test/java/com/google/api/generator/gapic/protoparser/BUILD.bazel +++ b/src/test/java/com/google/api/generator/gapic/protoparser/BUILD.bazel @@ -3,6 +3,7 @@ load("@rules_java//java:defs.bzl", "java_test") package(default_visibility = ["//visibility:public"]) TESTS = [ + "BatchingSettingsConfigParserTest", "HttpRuleParserTest", "MethodSignatureParserTest", "ParserTest", @@ -25,6 +26,7 @@ filegroup( srcs = ["{0}.java".format(test_name)], data = [ "//src/test/java/com/google/api/generator/gapic/testdata:basic_proto_descriptor", + "//src/test/java/com/google/api/generator/gapic/testdata:gapic_config_files", "//src/test/java/com/google/api/generator/gapic/testdata:service_config_files", "//src/test/java/com/google/api/generator/gapic/testdata:service_yaml_files", ], diff --git a/src/test/java/com/google/api/generator/gapic/protoparser/BatchingSettingsConfigParserTest.java b/src/test/java/com/google/api/generator/gapic/protoparser/BatchingSettingsConfigParserTest.java new file mode 100644 index 0000000000..92ba2e5094 --- /dev/null +++ b/src/test/java/com/google/api/generator/gapic/protoparser/BatchingSettingsConfigParserTest.java @@ -0,0 +1,114 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.api.generator.gapic.protoparser; + +import static com.google.common.truth.Truth.assertThat; +import static junit.framework.Assert.assertEquals; +import static junit.framework.Assert.assertFalse; +import static junit.framework.Assert.assertTrue; + +import com.google.api.generator.gapic.model.GapicBatchingSettings; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.Optional; +import org.junit.Test; + +public class BatchingSettingsConfigParserTest { + private static final String YAML_DIRECTORY = + "src/test/java/com/google/api/generator/gapic/testdata/"; + + @Test + public void parseGapicSettings_plain() { + String filename = "datastore_gapic.yaml"; + Path path = Paths.get(YAML_DIRECTORY, filename); + Optional> settingsOpt = + BatchingSettingsConfigParser.parse(path.toString()); + assertFalse(settingsOpt.isPresent()); + } + + @Test + public void parseGapicSettings_noMethodSettings() { + String filename = "showcase_gapic.yaml"; + Path path = Paths.get(YAML_DIRECTORY, filename); + Optional> settingsOpt = + BatchingSettingsConfigParser.parse(path.toString()); + assertFalse(settingsOpt.isPresent()); + } + + @Test + public void parseBatchingSettings_logging() { + String filename = "logging_gapic.yaml"; + Path path = Paths.get(YAML_DIRECTORY, filename); + Optional> settingsOpt = + BatchingSettingsConfigParser.parse(path.toString()); + assertTrue(settingsOpt.isPresent()); + + List batchingSettings = settingsOpt.get(); + assertEquals(1, batchingSettings.size()); + GapicBatchingSettings setting = batchingSettings.get(0); + + assertEquals("google.logging.v2", setting.protoPakkage()); + assertEquals("LoggingServiceV2", setting.serviceName()); + assertEquals("WriteLogEntries", setting.methodName()); + + assertEquals(1000, setting.elementCountThreshold()); + assertEquals(1048576, setting.requestByteThreshold()); + assertEquals(50, setting.delayThresholdMillis()); + + assertThat(setting.flowControlElementLimit()).isNotNull(); + assertEquals(100000, (long) setting.flowControlElementLimit()); + assertThat(setting.flowControlByteLimit()).isNotNull(); + assertEquals(10485760, (long) setting.flowControlByteLimit()); + assertEquals( + GapicBatchingSettings.FlowControlLimitExceededBehavior.THROW_EXCEPTION, + setting.flowControlLimitExceededBehavior()); + + assertEquals("entries", setting.batchedFieldName()); + assertThat(setting.discriminatorFieldNames()).containsExactly("log_name", "resource", "labels"); + assertThat(setting.subresponseFieldName()).isNull(); + } + + @Test + public void parseBatchingSettings_pubsub() { + String filename = "pubsub_gapic.yaml"; + Path path = Paths.get(YAML_DIRECTORY, filename); + Optional> settingsOpt = + BatchingSettingsConfigParser.parse(path.toString()); + assertTrue(settingsOpt.isPresent()); + + List batchingSettings = settingsOpt.get(); + assertEquals(1, batchingSettings.size()); + GapicBatchingSettings setting = batchingSettings.get(0); + + assertEquals("google.pubsub.v1", setting.protoPakkage()); + assertEquals("Publisher", setting.serviceName()); + assertEquals("Publish", setting.methodName()); + + assertEquals(100, setting.elementCountThreshold()); + assertEquals(1048576, setting.requestByteThreshold()); + assertEquals(10, setting.delayThresholdMillis()); + + assertThat(setting.flowControlElementLimit()).isNull(); + assertThat(setting.flowControlByteLimit()).isNull(); + assertEquals( + GapicBatchingSettings.FlowControlLimitExceededBehavior.IGNORE, + setting.flowControlLimitExceededBehavior()); + + assertEquals("messages", setting.batchedFieldName()); + assertThat(setting.discriminatorFieldNames()).containsExactly("topic"); + assertEquals("message_ids", setting.subresponseFieldName()); + } +} diff --git a/src/test/java/com/google/api/generator/gapic/protoparser/PluginArgumentParserTest.java b/src/test/java/com/google/api/generator/gapic/protoparser/PluginArgumentParserTest.java index b843a734b7..c8a84d4f82 100644 --- a/src/test/java/com/google/api/generator/gapic/protoparser/PluginArgumentParserTest.java +++ b/src/test/java/com/google/api/generator/gapic/protoparser/PluginArgumentParserTest.java @@ -53,43 +53,101 @@ public void parseJsonPath_similarFileAppearsFirst() { String.join( ",", Arrays.asList( - createServiceConfig(gapicPath), + createGapicConfig(gapicPath), createGrpcServiceConfig("/tmp/something.json"), createGrpcServiceConfig("/tmp/some_grpc_service_configjson"), createGrpcServiceConfig(jsonPath), - createServiceConfig(gapicPath))); + createGapicConfig(gapicPath))); assertEquals(jsonPath, PluginArgumentParser.parseJsonConfigPath(rawArgument).get()); } @Test public void parseJsonPath_argumentHasSpaces() { String jsonPath = "/tmp/foo_grpc_service_config.json"; + String gapicPath = "/tmp/something_gapic.yaml"; String rawArgument = String.join( " , ", Arrays.asList( + createGapicConfig(gapicPath), createGrpcServiceConfig("/tmp/something.json"), createGrpcServiceConfig("/tmp/some_grpc_service_configjson"), - createGrpcServiceConfig(jsonPath))); + createGrpcServiceConfig(jsonPath), + createGapicConfig(gapicPath))); assertEquals(jsonPath, PluginArgumentParser.parseJsonConfigPath(rawArgument).get()); } @Test public void parseJsonPath_restAreEmpty() { String jsonPath = "/tmp/foobar_grpc_service_config.json"; - String emptyPath = ""; + String gapicPath = ""; String rawArgument = - String.join(",", Arrays.asList(emptyPath, createGrpcServiceConfig(jsonPath), emptyPath)); + String.join(",", Arrays.asList(gapicPath, createGrpcServiceConfig(jsonPath), gapicPath)); assertEquals(jsonPath, PluginArgumentParser.parseJsonConfigPath(rawArgument).get()); } @Test public void parseJsonPath_noneFound() { - String someOtherPath = "/tmp/something_gapic.yaml"; - String rawArgument = String.join(",", Arrays.asList(someOtherPath)); + String gapicPath = "/tmp/something_gapic.yaml"; + String rawArgument = String.join(",", Arrays.asList(gapicPath)); assertFalse(PluginArgumentParser.parseJsonConfigPath(rawArgument).isPresent()); } + @Test + public void parseGapicYamlPath_onlyOnePresent() { + String gapicPath = "/tmp/something_gapic.yaml"; + assertEquals( + gapicPath, + PluginArgumentParser.parseGapicYamlConfigPath(createGapicConfig(gapicPath)).get()); + } + + @Test + public void parseGapicYamlPath_returnsFirstOneFound() { + String gapicPathOne = "/tmp/something_gapic.yaml"; + String gapicPathTwo = "/tmp/other_gapic.yaml"; + assertEquals( + gapicPathOne, + PluginArgumentParser.parseGapicYamlConfigPath( + String.join( + ",", + Arrays.asList( + createGapicConfig(gapicPathOne), createGapicConfig(gapicPathTwo)))) + .get()); + } + + @Test + public void parseGapicYamlPath_similarFileAppearsFirst() { + String jsonPath = "/tmp/foo_grpc_service_config.json"; + String gapicPath = "/tmp/something_gapic.yaml"; + String rawArgument = + String.join( + ",", + Arrays.asList( + createGrpcServiceConfig(jsonPath), + createGapicConfig("/tmp/something.yaml"), + createGapicConfig("/tmp/some_gapicyaml"), + createGapicConfig(gapicPath))); + assertEquals(gapicPath, PluginArgumentParser.parseGapicYamlConfigPath(rawArgument).get()); + } + + @Test + public void parseGapicYamlPath_restAreEmpty() { + String jsonPath = ""; + String gapicPath = "/tmp/something_gapic.yaml"; + String rawArgument = + String.join(",", Arrays.asList(jsonPath, createGapicConfig(gapicPath), jsonPath)); + assertEquals(gapicPath, PluginArgumentParser.parseGapicYamlConfigPath(rawArgument).get()); + } + + @Test + public void parseGapicYamlPath_noneFound() { + String jsonPath = "/tmp/foo_grpc_service_config.json"; + String gapicPath = ""; + String rawArgument = + String.join(",", Arrays.asList(createGrpcServiceConfig(jsonPath), gapicPath)); + assertFalse(PluginArgumentParser.parseGapicYamlConfigPath(rawArgument).isPresent()); + } + @Test public void parseServiceYamlPath_onlyOnePresent() { String servicePath = "/tmp/something.yaml"; @@ -125,13 +183,19 @@ public void parseServiceYamlPath_gapicFilePresent() { // Passed under the right flags. rawArgument = String.join( - ",", Arrays.asList(createServiceConfig(gapicPath), createServiceConfig(servicePath))); + ",", Arrays.asList(createGapicConfig(gapicPath), createServiceConfig(servicePath))); assertEquals(servicePath, PluginArgumentParser.parseServiceYamlConfigPath(rawArgument).get()); // Swapped flags. rawArgument = String.join( - ",", Arrays.asList(createServiceConfig(gapicPath), createServiceConfig(gapicPath))); + ",", Arrays.asList(createGapicConfig(servicePath), createServiceConfig(gapicPath))); + assertFalse(PluginArgumentParser.parseServiceYamlConfigPath(rawArgument).isPresent()); + + // Both passed under the Gapic yaml flag. + rawArgument = + String.join( + ",", Arrays.asList(createGapicConfig(gapicPath), createGapicConfig(servicePath))); assertFalse(PluginArgumentParser.parseServiceYamlConfigPath(rawArgument).isPresent()); } @@ -145,8 +209,8 @@ public void parseServiceYamlPath_similarFileAppearsFirst() { ",", Arrays.asList( createGrpcServiceConfig(jsonPath), - createServiceConfig("/tmp/something.yaml"), - createServiceConfig("/tmp/some_gapicyaml"), + createGapicConfig("/tmp/something.yaml"), + createGapicConfig("/tmp/some_gapicyaml"), createServiceConfig(gapicPath), createServiceConfig(servicePath))); assertEquals(servicePath, PluginArgumentParser.parseServiceYamlConfigPath(rawArgument).get()); @@ -165,6 +229,10 @@ private static String createGrpcServiceConfig(String path) { return String.format("%s=%s", PluginArgumentParser.KEY_GRPC_SERVICE_CONFIG, path); } + private static String createGapicConfig(String path) { + return String.format("%s=%s", PluginArgumentParser.KEY_GAPIC_CONFIG, path); + } + private static String createServiceConfig(String path) { return String.format("%s=%s", PluginArgumentParser.KEY_SERVICE_YAML_CONFIG, path); } diff --git a/src/test/java/com/google/api/generator/gapic/testdata/BUILD.bazel b/src/test/java/com/google/api/generator/gapic/testdata/BUILD.bazel index ed949b95fa..4909db60e6 100644 --- a/src/test/java/com/google/api/generator/gapic/testdata/BUILD.bazel +++ b/src/test/java/com/google/api/generator/gapic/testdata/BUILD.bazel @@ -8,6 +8,11 @@ filegroup( srcs = glob(["*.json"]), ) +filegroup( + name = "gapic_config_files", + srcs = glob(["*_gapic.yaml"]), +) + filegroup( name = "service_yaml_files", srcs = ["logging.yaml"], diff --git a/src/test/java/com/google/api/generator/gapic/testdata/datastore_gapic.yaml b/src/test/java/com/google/api/generator/gapic/testdata/datastore_gapic.yaml new file mode 100644 index 0000000000..e9860d5efe --- /dev/null +++ b/src/test/java/com/google/api/generator/gapic/testdata/datastore_gapic.yaml @@ -0,0 +1,38 @@ +type: com.google.api.codegen.ConfigProto +config_schema_version: 2.0.0 +language_settings: + java: + package_name: com.google.cloud.datastore.v1 + python: + package_name: google.cloud.datastore_v1.gapic + go: + package_name: cloud.google.com/go/datastore/apiv1 + csharp: + package_name: Google.Cloud.Datastore.V1 + release_level: GA + ruby: + package_name: Google::Cloud::Datastore::V1 + release_level: GA + php: + package_name: Google\Cloud\Datastore\V1 + nodejs: + package_name: datastore.v1 + domain_layer_location: google-cloud +interfaces: +- name: google.datastore.v1.Datastore + retry_params_def: + - name: default + initial_retry_delay_millis: 100 + retry_delay_multiplier: 1.3 + max_retry_delay_millis: 60000 + initial_rpc_timeout_millis: 60000 + rpc_timeout_multiplier: 1 + max_rpc_timeout_millis: 60000 + total_timeout_millis: 600000 + methods: + - name: Lookup + retry_codes_name: idempotent + - name: RunQuery + retry_codes_name: idempotent + - name: ReserveIds + retry_codes_name: idempotent diff --git a/src/test/java/com/google/api/generator/gapic/testdata/logging_gapic.yaml b/src/test/java/com/google/api/generator/gapic/testdata/logging_gapic.yaml new file mode 100644 index 0000000000..75c6deda26 --- /dev/null +++ b/src/test/java/com/google/api/generator/gapic/testdata/logging_gapic.yaml @@ -0,0 +1,79 @@ +type: com.google.api.codegen.ConfigProto +config_schema_version: 2.0.0 +language_settings: + java: + package_name: com.google.cloud.logging.v2 + interface_names: + google.logging.v2.ConfigServiceV2: Config + google.logging.v2.LoggingServiceV2: Logging + google.logging.v2.MetricsServiceV2: Metrics + python: + package_name: google.cloud.logging_v2.gapic + go: + package_name: cloud.google.com/go/logging/apiv2 + domain_layer_location: cloud.google.com/go/logging + csharp: + package_name: Google.Cloud.Logging.V2 + release_level: GA + ruby: + package_name: Google::Cloud::Logging::V2 + php: + package_name: Google\Cloud\Logging\V2 + nodejs: + package_name: logging.v2 + domain_layer_location: google-cloud +interfaces: +- name: google.logging.v2.ConfigServiceV2 + retry_codes_def: + - name: idempotent + retry_codes: + - UNAVAILABLE + - DEADLINE_EXCEEDED + - INTERNAL + methods: + - name: DeleteSink + retry_codes_name: idempotent + - name: UpdateSink + retry_codes_name: idempotent + - name: DeleteExclusion + retry_codes_name: idempotent +- name: google.logging.v2.MetricsServiceV2 + retry_codes_def: + - name: idempotent + retry_codes: + - UNAVAILABLE + - DEADLINE_EXCEEDED + - INTERNAL + methods: + - name: UpdateLogMetric + retry_codes_name: idempotent + - name: DeleteLogMetric + retry_codes_name: idempotent +- name: google.logging.v2.LoggingServiceV2 + retry_codes_def: + - name: idempotent + retry_codes: + - UNAVAILABLE + - DEADLINE_EXCEEDED + - INTERNAL + methods: + - name: DeleteLog + retry_codes_name: idempotent + - name: ListLogEntries + retry_codes_name: idempotent + - name: WriteLogEntries + retry_codes_name: idempotent + batching: + thresholds: + element_count_threshold: 1000 + request_byte_threshold: 1048576 + delay_threshold_millis: 50 + flow_control_element_limit: 100000 + flow_control_byte_limit: 10485760 + flow_control_limit_exceeded_behavior: THROW_EXCEPTION + batch_descriptor: + batched_field: entries + discriminator_fields: + - log_name + - resource + - labels diff --git a/src/test/java/com/google/api/generator/gapic/testdata/pubsub_gapic.yaml b/src/test/java/com/google/api/generator/gapic/testdata/pubsub_gapic.yaml new file mode 100644 index 0000000000..e26d535faf --- /dev/null +++ b/src/test/java/com/google/api/generator/gapic/testdata/pubsub_gapic.yaml @@ -0,0 +1,296 @@ +type: com.google.api.codegen.ConfigProto +config_schema_version: 2.0.0 +language_settings: + java: + package_name: com.google.cloud.pubsub.v1 + interface_names: + google.pubsub.v1.Publisher: TopicAdmin + google.pubsub.v1.Subscriber: SubscriptionAdmin + release_level: GA + python: + package_name: google.cloud.pubsub_v1.gapic + release_level: GA + go: + package_name: cloud.google.com/go/pubsub/apiv1 + domain_layer_location: cloud.google.com/go/pubsub + release_level: GA + csharp: + package_name: Google.Cloud.PubSub.V1 + interface_names: + google.pubsub.v1.Publisher: PublisherServiceApi + google.pubsub.v1.Subscriber: SubscriberServiceApi + release_level: GA + ruby: + package_name: Google::Cloud::PubSub::V1 + release_level: BETA + php: + package_name: Google\Cloud\PubSub\V1 + release_level: GA + nodejs: + package_name: pubsub.v1 + domain_layer_location: google-cloud + release_level: GA +collections: +# Language overrides are for backward-compatibility in Java. +- entity_name: snapshot + language_overrides: + - language: java + entity_name: project_snapshot +# Language overrides are for backward-compatibility in Java. +- entity_name: subscription + language_overrides: + - language: java + entity_name: project_subscription +interfaces: +- name: google.pubsub.v1.Subscriber + # Deprecated collections are for backward-compatibility in Ruby, PHP and Python + deprecated_collections: + - name_pattern: "projects/{project}/topics/{topic}" + entity_name: topic + lang_doc: + java: To retrieve messages from a subscription, see the Subscriber class. + retry_codes_def: + - name: idempotent + retry_codes: + - UNKNOWN + - ABORTED + - UNAVAILABLE + - name: non_idempotent + retry_codes: + - UNAVAILABLE + - name: streaming_pull + retry_codes: + - DEADLINE_EXCEEDED + - RESOURCE_EXHAUSTED + - ABORTED + - INTERNAL + - UNAVAILABLE + retry_params_def: + - name: default + initial_retry_delay_millis: 100 + retry_delay_multiplier: 1.3 + max_retry_delay_millis: 60000 # 60 seconds + initial_rpc_timeout_millis: 60000 # 60 seconds + rpc_timeout_multiplier: 1 + max_rpc_timeout_millis: 60000 # 60 seconds + total_timeout_millis: 600000 # 10 minutes + - name: messaging + initial_retry_delay_millis: 100 + retry_delay_multiplier: 1.3 + max_retry_delay_millis: 60000 # 60 seconds + initial_rpc_timeout_millis: 25000 # 25 seconds + rpc_timeout_multiplier: 1 + max_rpc_timeout_millis: 25000 # 25 seconds + total_timeout_millis: 600000 # 10 minutes + - name: streaming_messaging + initial_retry_delay_millis: 100 + retry_delay_multiplier: 1.3 + max_retry_delay_millis: 60000 # 60 seconds + initial_rpc_timeout_millis: 600000 # 10 minutes + rpc_timeout_multiplier: 1 + max_rpc_timeout_millis: 600000 # 10 minutes + total_timeout_millis: 600000 # 10 minutes + methods: + # TODO: remove the per method retry_codes_name field once + # https://github.com/googleapis/gapic-generator/issues/3137 + # is fixed + - name: CreateSubscription + retry_codes_name: idempotent + - name: GetSubscription + retry_codes_name: idempotent + - name: UpdateSubscription + retry_codes_name: non_idempotent + sample_code_init_fields: + - update_mask.paths[0]="ack_deadline_seconds" + - subscription.ack_deadline_seconds=42 + - name: ListSubscriptions + retry_codes_name: idempotent + - name: DeleteSubscription + retry_codes_name: non_idempotent + - name: GetSnapshot + surface_treatments: + - include_languages: + - go + - java + - csharp + - ruby + - nodejs + - python + - php + visibility: PACKAGE + - name: ModifyAckDeadline + retry_codes_name: non_idempotent + surface_treatments: + - include_languages: + - java + visibility: PACKAGE + - name: Acknowledge + retry_codes_name: non_idempotent + retry_params_name: messaging + surface_treatments: + - include_languages: + - java + visibility: PACKAGE + - name: Pull + retry_codes_name: idempotent + retry_params_name: messaging + surface_treatments: + - include_languages: + - java + visibility: PACKAGE + - name: StreamingPull + retry_codes_name: streaming_pull + retry_params_name: streaming_messaging + timeout_millis: 900000 + surface_treatments: + - include_languages: + - java + visibility: PACKAGE + - name: ModifyPushConfig + retry_codes_name: non_idempotent + - name: ListSnapshots + retry_codes_name: idempotent + - name: CreateSnapshot + retry_codes_name: non_idempotent + - name: UpdateSnapshot + retry_codes_name: non_idempotent + sample_code_init_fields: + - update_mask.paths[0]="expire_time" + - snapshot.expire_time.seconds=123456 + - name: DeleteSnapshot + retry_codes_name: non_idempotent + - name: Seek + retry_codes_name: idempotent + - name: SetIamPolicy + retry_codes_name: non_idempotent + reroute_to_grpc_interface: google.iam.v1.IAMPolicy + surface_treatments: + - include_languages: + - go + visibility: DISABLED + - name: GetIamPolicy + retry_codes_name: idempotent + reroute_to_grpc_interface: google.iam.v1.IAMPolicy + surface_treatments: + - include_languages: + - go + visibility: DISABLED + - name: TestIamPermissions + retry_codes_name: non_idempotent + reroute_to_grpc_interface: google.iam.v1.IAMPolicy + surface_treatments: + - include_languages: + - go + visibility: DISABLED +- name: google.pubsub.v1.Publisher + lang_doc: + java: To publish messages to a topic, see the Publisher class. + smoke_test: + method: ListTopics + init_fields: + - project%project=$PROJECT_ID + retry_codes_def: + - name: idempotent + retry_codes: + - UNKNOWN + - ABORTED + - UNAVAILABLE + - name: non_idempotent + retry_codes: + - UNAVAILABLE + - name: none + retry_codes: [] + - name: publish + retry_codes: + - ABORTED + - CANCELLED + - INTERNAL + - RESOURCE_EXHAUSTED + - UNKNOWN + - UNAVAILABLE + - DEADLINE_EXCEEDED + retry_params_def: + - name: default + initial_retry_delay_millis: 100 + retry_delay_multiplier: 1.3 + max_retry_delay_millis: 60000 # 60 seconds + initial_rpc_timeout_millis: 60000 # 60 seconds + rpc_timeout_multiplier: 1 + max_rpc_timeout_millis: 60000 # 60 seconds + total_timeout_millis: 600000 # 10 minutes + - name: messaging + initial_retry_delay_millis: 100 + retry_delay_multiplier: 1.3 + max_retry_delay_millis: 60000 # 60 seconds + initial_rpc_timeout_millis: 5000 # 5 seconds + rpc_timeout_multiplier: 1.3 + max_rpc_timeout_millis: 60000 # 60 seconds + total_timeout_millis: 60000 # 60 seconds + methods: + - name: CreateTopic + retry_codes_name: non_idempotent + - name: UpdateTopic + retry_codes_name: non_idempotent + - name: Publish + retry_codes_name: publish + retry_params_name: messaging + batching: + thresholds: + element_count_threshold: 100 + element_count_limit: 1000 # TO BE REMOVED LATER + request_byte_threshold: 1048576 # 1 MiB + request_byte_limit: 10485760 # TO BE REMOVED LATER + delay_threshold_millis: 10 + batch_descriptor: + batched_field: messages + discriminator_fields: + - topic + subresponse_field: message_ids + sample_code_init_fields: + - messages[0].data + surface_treatments: + - include_languages: + - java + visibility: PACKAGE + - name: GetTopic + retry_codes_name: idempotent + - name: ListTopics + retry_codes_name: idempotent + - name: ListTopicSubscriptions + retry_codes_name: idempotent + - name: ListTopicSnapshots + surface_treatments: + - include_languages: + - go + - java + - csharp + - ruby + - nodejs + - python + - php + visibility: PACKAGE + - name: DeleteTopic + retry_codes_name: non_idempotent + - name: SetIamPolicy + retry_codes_name: non_idempotent + reroute_to_grpc_interface: google.iam.v1.IAMPolicy + # TODO: surface_treatments are here only to make bazel presubmit pass + # they can be removed once presubmit starts using gapic-generator-go + surface_treatments: + - include_languages: + - go + visibility: DISABLED + - name: GetIamPolicy + retry_codes_name: idempotent + reroute_to_grpc_interface: google.iam.v1.IAMPolicy + surface_treatments: + - include_languages: + - go + visibility: DISABLED + - name: TestIamPermissions + retry_codes_name: non_idempotent + reroute_to_grpc_interface: google.iam.v1.IAMPolicy + surface_treatments: + - include_languages: + - go + visibility: DISABLED diff --git a/src/test/java/com/google/api/generator/gapic/testdata/showcase_gapic.yaml b/src/test/java/com/google/api/generator/gapic/testdata/showcase_gapic.yaml new file mode 100644 index 0000000000..8c50128718 --- /dev/null +++ b/src/test/java/com/google/api/generator/gapic/testdata/showcase_gapic.yaml @@ -0,0 +1,22 @@ +# FIXME: Address all the FIXMEs in this generated config before using it for +# client generation. Remove this paragraph after you closed all the FIXMEs. The +# retry_codes_name, required_fields, flattening, and timeout properties cannot +# be precisely decided by the tooling and may require some configuration. +type: com.google.api.codegen.ConfigProto +config_schema_version: 2.0.0 +# The settings of generated code in a specific language. +language_settings: + java: + package_name: com.google.showcase.v1beta1 + python: + package_name: google.showcase_v1beta1.gapic + go: + package_name: cloud.google.com/go/showcase/apiv1beta1 + csharp: + package_name: Google.Showcase.V1beta1 + ruby: + package_name: Google::Showcase::V1beta1 + php: + package_name: Google\Showcase\V1beta1 + nodejs: + package_name: showcase.v1beta1