[ggj] feat(drift): Generate gapic_metadata.json, add integ test framework support

rules_bazel/java/integration_test.bzl

@@ -19,7 +19,7 @@ def _diff_integration_goldens_impl(ctx):
     unzip -j {input_test} -d codegen_tmp
     cd codegen_tmp
     # Remove unneeded non-Java files, like MANIFEST
-    rm -rf $(find . -type f ! -name "*.java")
+    rm -rf $(find ./ -type f ! -name '*.java' -a ! -name '*gapic_metadata.json')
     cd ..
     diff codegen_tmp test/integration/goldens/{api_name}/ > {diff_output}
     # Bash `diff` command will return exit code 1 when there are differences between the two
@@ -119,7 +119,7 @@ def _overwrite_golden_impl(ctx):
     unzip -j {input_test} -d codegen_tmp
     cd codegen_tmp
     # Remove unneeded non-Java files, like MANIFEST
-    rm -rf $(find . -type f ! -name "*.java")
+    rm -rf $(find ./ -type f ! -name '*.java' -a ! -name '*gapic_metadata.json')
     zip -r ../{goldens_output_zip} .
     """.format(
         goldens_output_zip = goldens_output_zip.path,
@@ -142,6 +142,7 @@ def _overwrite_golden_impl(ctx):
     golden_update_script_content = """
     cd ${{BUILD_WORKSPACE_DIRECTORY}}
     rm -r test/integration/goldens/{api_name}/*.java
+    rm -r test/integration/goldens/{api_name}/gapic_metadata.json
     unzip -ao {goldens_output_zip} -d test/integration/goldens/{api_name}
     """.format(
         goldens_output_zip = goldens_output_zip.path,

src/main/java/com/google/api/generator/gapic/Generator.java

@@ -30,7 +30,8 @@ public static CodeGeneratorResponse generateGapic(CodeGeneratorRequest request)
     List<GapicClass> clazzes = Composer.composeServiceClasses(context);
     GapicPackageInfo packageInfo = Composer.composePackageInfo(context);
     String outputFilename = "temp-codegen.srcjar";
-    CodeGeneratorResponse response = Writer.writeCode(clazzes, packageInfo, outputFilename);
+    CodeGeneratorResponse response =
+        Writer.write(clazzes, packageInfo, context.gapicMetadata(), outputFilename);
     return response;
   }
 }

src/main/java/com/google/api/generator/gapic/composer/BUILD.bazel

@@ -30,6 +30,7 @@ java_library(
         "@com_google_api_gax_java//gax",
         "@com_google_api_gax_java//gax-grpc:gax_grpc",
         "@com_google_code_findbugs_jsr305//jar",
+        "@com_google_googleapis//gapic/metadata:metadata_java_proto",
         "@com_google_googleapis//google/api:api_java_proto",
         "@com_google_googleapis//google/longrunning:longrunning_java_proto",
         "@com_google_googleapis//google/rpc:rpc_java_proto",

src/main/java/com/google/api/generator/gapic/composer/Composer.java

@@ -63,7 +63,7 @@ public static List<GapicClass> generateServiceClasses(
     List<GapicClass> clazzes = new ArrayList<>();
     clazzes.addAll(
         generateStubClasses(service, context.serviceConfig(), context.messages(), resourceNames));
-    clazzes.addAll(generateClientSettingsClasses(service, context.messages(), resourceNames));
+    clazzes.addAll(generateClientSettingsClasses(service, context, resourceNames));
     clazzes.addAll(generateMockClasses(service, resourceNames, context.messages()));
     clazzes.addAll(generateTestClasses(service, context, resourceNames));
     // TODO(miraleung): Generate test classes.
@@ -93,11 +93,10 @@ public static List<GapicClass> generateStubClasses(
   }
 
   public static List<GapicClass> generateClientSettingsClasses(
-      Service service, Map<String, Message> messageTypes, Map<String, ResourceName> resourceNames) {
+      Service service, GapicContext context, Map<String, ResourceName> resourceNames) {
     List<GapicClass> clazzes = new ArrayList<>();
-    clazzes.add(
-        ServiceClientClassComposer.instance().generate(service, messageTypes, resourceNames));
-    clazzes.add(ServiceSettingsClassComposer.instance().generate(service, messageTypes));
+    clazzes.add(ServiceClientClassComposer.instance().generate(service, context, resourceNames));
+    clazzes.add(ServiceSettingsClassComposer.instance().generate(service, context.messages()));
     return clazzes;
   }
 

src/main/java/com/google/api/generator/gapic/composer/ServiceClientClassComposer.java

@@ -62,6 +62,7 @@
 import com.google.api.generator.gapic.model.Field;
 import com.google.api.generator.gapic.model.GapicClass;
 import com.google.api.generator.gapic.model.GapicClass.Kind;
+import com.google.api.generator.gapic.model.GapicContext;
 import com.google.api.generator.gapic.model.LongrunningOperation;
 import com.google.api.generator.gapic.model.Message;
 import com.google.api.generator.gapic.model.Method;
@@ -76,6 +77,7 @@
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
 import com.google.common.util.concurrent.MoreExecutors;
+import com.google.gapic.metadata.GapicMetadata;
 import com.google.longrunning.Operation;
 import com.google.rpc.Status;
 import java.io.IOException;
@@ -120,13 +122,16 @@ public static ServiceClientClassComposer instance() {
   }
 
   public GapicClass generate(
-      Service service, Map<String, Message> messageTypes, Map<String, ResourceName> resourceNames) {
+      Service service, GapicContext context, Map<String, ResourceName> resourceNames) {
+    Map<String, Message> messageTypes = context.messages();
     TypeStore typeStore = createTypes(service, messageTypes);
     String className = ClassNames.getServiceClientClassName(service);
     GapicClass.Kind kind = Kind.MAIN;
     String pakkage = service.pakkage();
     boolean hasLroClient = hasLroMethods(service);
 
+    Map<String, List<String>> grpcRpcsToJavaMethodNames = new HashMap<>();
+
     ClassDefinition classDef =
         ClassDefinition.builder()
             .setHeaderCommentStatements(
@@ -138,9 +143,17 @@ public GapicClass generate(
             .setImplementsTypes(createClassImplements(typeStore))
             .setStatements(createFieldDeclarations(service, typeStore, hasLroClient))
             .setMethods(
-                createClassMethods(service, messageTypes, typeStore, resourceNames, hasLroClient))
+                createClassMethods(
+                    service,
+                    messageTypes,
+                    typeStore,
+                    resourceNames,
+                    hasLroClient,
+                    grpcRpcsToJavaMethodNames))
             .setNestedClasses(createNestedPagingClasses(service, messageTypes, typeStore))
             .build();
+
+    updateGapicMetadata(context, service, className, grpcRpcsToJavaMethodNames);
     return GapicClass.create(kind, classDef);
   }
 
@@ -186,12 +199,15 @@ private static List<MethodDefinition> createClassMethods(
       Map<String, Message> messageTypes,
       TypeStore typeStore,
       Map<String, ResourceName> resourceNames,
-      boolean hasLroClient) {
+      boolean hasLroClient,
+      Map<String, List<String>> grpcRpcToJavaMethodMetadata) {
     List<MethodDefinition> methods = new ArrayList<>();
     methods.addAll(createStaticCreatorMethods(service, typeStore));
     methods.addAll(createConstructorMethods(service, typeStore, hasLroClient));
     methods.addAll(createGetterMethods(service, typeStore, hasLroClient));
-    methods.addAll(createServiceMethods(service, messageTypes, typeStore, resourceNames));
+    methods.addAll(
+        createServiceMethods(
+            service, messageTypes, typeStore, resourceNames, grpcRpcToJavaMethodMetadata));
     methods.addAll(createBackgroundResourceMethods(service, typeStore));
     return methods;
   }
@@ -511,35 +527,56 @@ private static List<MethodDefinition> createServiceMethods(
       Service service,
       Map<String, Message> messageTypes,
       TypeStore typeStore,
-      Map<String, ResourceName> resourceNames) {
+      Map<String, ResourceName> resourceNames,
+      Map<String, List<String>> grpcRpcToJavaMethodMetadata) {
     List<MethodDefinition> javaMethods = new ArrayList<>();
+    Function<MethodDefinition, String> javaMethodNameFn = m -> m.methodIdentifier().name();
     for (Method method : service.methods()) {
+      if (!grpcRpcToJavaMethodMetadata.containsKey(method.name())) {
+        grpcRpcToJavaMethodMetadata.put(method.name(), new ArrayList<>());
+      }
       if (method.stream().equals(Stream.NONE)) {
-        javaMethods.addAll(
+        List<MethodDefinition> generatedMethods =
             createMethodVariants(
                 method,
                 ClassNames.getServiceClientClassName(service),
                 messageTypes,
                 typeStore,
-                resourceNames));
-        javaMethods.add(
+                resourceNames);
+        grpcRpcToJavaMethodMetadata
+            .get(method.name())
+            .addAll(
+                generatedMethods.stream()
+                    .map(m -> javaMethodNameFn.apply(m))
+                    .collect(Collectors.toList()));
+        javaMethods.addAll(generatedMethods);
+
+        MethodDefinition generatedMethod =
             createMethodDefaultMethod(
                 method,
                 ClassNames.getServiceClientClassName(service),
                 messageTypes,
                 typeStore,
-                resourceNames));
+                resourceNames);
+        grpcRpcToJavaMethodMetadata.get(method.name()).add(javaMethodNameFn.apply(generatedMethod));
+        javaMethods.add(generatedMethod);
       }
       if (method.hasLro()) {
-        javaMethods.add(
-            createLroCallableMethod(service, method, typeStore, messageTypes, resourceNames));
+        MethodDefinition generatedMethod =
+            createLroCallableMethod(service, method, typeStore, messageTypes, resourceNames);
+        grpcRpcToJavaMethodMetadata.get(method.name()).add(javaMethodNameFn.apply(generatedMethod));
+        javaMethods.add(generatedMethod);
       }
       if (method.isPaged()) {
-        javaMethods.add(
-            createPagedCallableMethod(service, method, typeStore, messageTypes, resourceNames));
+        MethodDefinition generatedMethod =
+            createPagedCallableMethod(service, method, typeStore, messageTypes, resourceNames);
+        grpcRpcToJavaMethodMetadata.get(method.name()).add(javaMethodNameFn.apply(generatedMethod));
+        javaMethods.add(generatedMethod);
       }
-      javaMethods.add(
-          createCallableMethod(service, method, typeStore, messageTypes, resourceNames));
+      MethodDefinition generatedMethod =
+          createCallableMethod(service, method, typeStore, messageTypes, resourceNames);
+      grpcRpcToJavaMethodMetadata.get(method.name()).add(javaMethodNameFn.apply(generatedMethod));
+      javaMethods.add(generatedMethod);
     }
     return javaMethods;
   }
@@ -1667,4 +1704,49 @@ private static boolean isProtoEmptyType(TypeNode type) {
     return type.reference().pakkage().equals("com.google.protobuf")
         && type.reference().name().equals("Empty");
   }
+
+  private static void updateGapicMetadata(
+      GapicContext context, String protoPackage, String javaPackage) {
+    context.updateGapicMetadata(
+        context
+            .gapicMetadata()
+            .toBuilder()
+            .setProtoPackage(protoPackage)
+            .setLibraryPackage(javaPackage)
+            .build());
+  }
+
+  private static void updateGapicMetadata(
+      GapicContext context,
+      Service service,
+      String clientClassName,
+      Map<String, List<String>> grpcRpcToJavaMethodNames) {
+    GapicMetadata.Builder metadataBuilder = context.gapicMetadata().toBuilder();
+    metadataBuilder =
+        metadataBuilder
+            .setProtoPackage(service.protoPakkage())
+            .setLibraryPackage(service.pakkage());
+
+    GapicMetadata.ServiceAsClient.Builder serviceClientProtoBuilder =
+        GapicMetadata.ServiceAsClient.newBuilder().setLibraryClient(clientClassName);
+
+    // Sort for deterministic tests.
+    List<String> sortedRpcNames = new ArrayList<>(grpcRpcToJavaMethodNames.keySet());
+    Collections.sort(sortedRpcNames);
+    for (String rpcName : sortedRpcNames) {
+      GapicMetadata.MethodList methodList =
+          GapicMetadata.MethodList.newBuilder()
+              .addAllMethods(grpcRpcToJavaMethodNames.get(rpcName))
+              .build();
+      serviceClientProtoBuilder.putRpcs(rpcName, methodList);
+    }
+
+    metadataBuilder =
+        metadataBuilder.putServices(
+            service.name(),
+            GapicMetadata.ServiceForTransport.newBuilder()
+                .putClients("grpc", serviceClientProtoBuilder.build())
+                .build());
+    context.updateGapicMetadata(metadataBuilder.build());
+  }
 }

src/main/java/com/google/api/generator/gapic/model/BUILD.bazel

@@ -21,6 +21,7 @@ java_library(
         "@com_google_auto_value_auto_value//jar",
         "@com_google_auto_value_auto_value_annotations//jar",
         "@com_google_code_findbugs_jsr305//jar",
+        "@com_google_googleapis//gapic/metadata:metadata_java_proto",
         "@com_google_googleapis//google/api:api_java_proto",
         "@com_google_googleapis//google/rpc:rpc_java_proto",
         "@com_google_guava_guava//jar",

src/main/java/com/google/api/generator/gapic/model/GapicContext.java

@@ -18,6 +18,7 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
+import com.google.gapic.metadata.GapicMetadata;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -26,6 +27,10 @@
 
 @AutoValue
 public abstract class GapicContext {
+  // Keep a non-AutoValue reference to GapicMetadata, since we need to update
+  // it iteratively as we generate client methods.
+  private GapicMetadata gapicMetadata = defaultGapicMetadata();
+
   // Maps the message name (as it appears in the protobuf) to Messages.
   public abstract ImmutableMap<String, Message> messages();
 
@@ -39,6 +44,10 @@ public abstract class GapicContext {
 
   public abstract ImmutableSet<ResourceName> helperResourceNames();
 
+  public GapicMetadata gapicMetadata() {
+    return gapicMetadata;
+  }
+
   @Nullable
   public abstract GapicServiceConfig serviceConfig();
 
@@ -49,6 +58,21 @@ public boolean hasServiceYamlProto() {
     return serviceYamlProto() != null;
   }
 
+  public void updateGapicMetadata(GapicMetadata newMetadata) {
+    gapicMetadata = newMetadata;
+  }
+
+  static GapicMetadata defaultGapicMetadata() {
+    return GapicMetadata.newBuilder()
+        .setSchema("1.0")
+        .setComment(
+            "This file maps proto services/RPCs to the corresponding library clients/methods")
+        .setLanguage("java")
+        .build();
+  }
+
+  public abstract Builder toBuilder();
+
   public static Builder builder() {
     return new AutoValue_GapicContext.Builder().setMixinServices(Collections.emptyList());
   }

src/main/java/com/google/api/generator/gapic/protowriter/BUILD.bazel

@@ -16,6 +16,8 @@ java_library(
         "//src/main/java/com/google/api/generator/engine/ast",
         "//src/main/java/com/google/api/generator/engine/writer",
         "//src/main/java/com/google/api/generator/gapic/model",
+        "@com_google_googleapis//gapic/metadata:metadata_java_proto",
         "@com_google_protobuf//:protobuf_java",
+        "@com_google_protobuf//:protobuf_java_util",
     ],
 )

src/main/java/com/google/api/generator/gapic/protowriter/Writer.java

@@ -19,8 +19,10 @@
 import com.google.api.generator.engine.writer.JavaWriterVisitor;
 import com.google.api.generator.gapic.model.GapicClass;
 import com.google.api.generator.gapic.model.GapicPackageInfo;
+import com.google.gapic.metadata.GapicMetadata;
 import com.google.protobuf.ByteString;
 import com.google.protobuf.compiler.PluginProtos.CodeGeneratorResponse;
+import com.google.protobuf.util.JsonFormat;
 import java.io.IOException;
 import java.util.List;
 import java.util.jar.JarEntry;
@@ -33,8 +35,11 @@ public GapicWriterException(String errorMessage) {
     }
   }
 
-  public static CodeGeneratorResponse writeCode(
-      List<GapicClass> clazzes, GapicPackageInfo gapicPackageInfo, String outputFilePath) {
+  public static CodeGeneratorResponse write(
+      List<GapicClass> clazzes,
+      GapicPackageInfo gapicPackageInfo,
+      GapicMetadata gapicMetadata,
+      String outputFilePath) {
     ByteString.Output output = ByteString.newOutput();
     JavaWriterVisitor codeWriter = new JavaWriterVisitor();
     JarOutputStream jos = null;
@@ -80,6 +85,15 @@ public static CodeGeneratorResponse writeCode(
       throw new GapicWriterException("Could not write code for package-info.java");
     }
 
+    // Write the mdatadata file.
+    jarEntry = new JarEntry(String.format("%s/gapic_metadata.json", path));
+    try {
+      jos.putNextEntry(jarEntry);
+      jos.write(JsonFormat.printer().print(gapicMetadata).getBytes());
+    } catch (IOException e) {
+      throw new GapicWriterException("Could not write gapic_metadata.json");
+    }
+
     try {
       jos.finish();
       jos.flush();