feat: ingest structured logs using stdout in LoggingHandler

google-cloud-logging/clirr-ignored-differences.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- see http://www.mojohaus.org/clirr-maven-plugin/examples/ignored-differences.html -->
+<differences>
+  <difference>
+    <differenceType>7012</differenceType>
+    <className>com/google/cloud/logging/Logging</className>
+    <method>java.lang.Iterable populateMetadata(java.lang.Iterable, com.google.cloud.MonitoredResource, java.lang.String[])</method>
+  </difference>
+</differences>

google-cloud-logging/pom.xml

@@ -21,6 +21,11 @@
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.google.code.gson</groupId>
+      <artifactId>gson</artifactId>
+      <version>2.8.9</version>
+    </dependency>
     <dependency>
       <groupId>io.grpc</groupId>
       <artifactId>grpc-api</artifactId>

google-cloud-logging/src/main/java/com/google/cloud/logging/LogEntry.java

@@ -19,9 +19,17 @@
 import static com.google.common.base.Preconditions.checkNotNull;
 
 import com.google.cloud.MonitoredResource;
+import com.google.cloud.logging.Payload.Type;
 import com.google.common.base.Function;
 import com.google.common.base.MoreObjects;
 import com.google.common.collect.ImmutableMap;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonPrimitive;
+import com.google.gson.JsonSerializationContext;
+import com.google.gson.JsonSerializer;
 import com.google.logging.v2.LogEntryOperation;
 import com.google.logging.v2.LogEntrySourceLocation;
 import com.google.logging.v2.LogName;
@@ -61,8 +69,8 @@ public LogEntry apply(com.google.logging.v2.LogEntry pb) {
   private final HttpRequest httpRequest;
   private final Map<String, String> labels;
   private final Operation operation;
-  private final Object trace;
-  private final Object spanId;
+  private final String trace;
+  private final String spanId;
   private final boolean traceSampled;
   private final SourceLocation sourceLocation;
   private final Payload<?> payload;
@@ -80,8 +88,8 @@ public static class Builder {
     private HttpRequest httpRequest;
     private Map<String, String> labels = new HashMap<>();
     private Operation operation;
-    private Object trace;
-    private Object spanId;
+    private String trace;
+    private String spanId;
     private boolean traceSampled;
     private SourceLocation sourceLocation;
     private Payload<?> payload;
@@ -245,7 +253,7 @@ public Builder setTrace(String trace) {
      * relative resource name, the name is assumed to be relative to `//tracing.googleapis.com`.
      */
     public Builder setTrace(Object trace) {
-      this.trace = trace;
+      this.trace = trace != null ? trace.toString() : null;
       return this;
     }
 
@@ -257,7 +265,7 @@ public Builder setSpanId(String spanId) {
 
     /** Sets the ID of the trace span associated with the log entry, if any. */
     public Builder setSpanId(Object spanId) {
-      this.spanId = spanId;
+      this.spanId = spanId != null ? spanId.toString() : null;
       return this;
     }
 
@@ -575,6 +583,142 @@ com.google.logging.v2.LogEntry toPb(String projectId) {
     return builder.build();
   }
 
+  /**
+   * Customized serializers to match the expected format for timestamp, source location and request
+   * method
+   */
+  static final class InstantSerializer implements JsonSerializer<Instant> {
+    @Override
+    public JsonElement serialize(
+        Instant src, java.lang.reflect.Type typeOfSrc, JsonSerializationContext context) {
+      return new JsonPrimitive(src.toString());
+    }
+  }
+
+  static final class SourceLocationSerializer implements JsonSerializer<SourceLocation> {
+    @Override
+    public JsonElement serialize(
+        SourceLocation src, java.lang.reflect.Type typeOfSrc, JsonSerializationContext context) {
+      JsonObject obj = new JsonObject();
+      if (src.getFile() != null) {
+        obj.addProperty("file", src.getFile());
+      }
+      if (src.getLine() != null) {
+        obj.addProperty("line", src.getLine().toString());
+      }
+      if (src.getFunction() != null) {
+        obj.addProperty("function", src.getFunction());
+      }
+      return obj;
+    }
+  }
+
+  static final class RequestMethodSerializer implements JsonSerializer<HttpRequest.RequestMethod> {
+    @Override
+    public JsonElement serialize(
+        HttpRequest.RequestMethod src,
+        java.lang.reflect.Type typeOfSrc,
+        JsonSerializationContext context) {
+      return new JsonPrimitive(src.name());
+    }
+  }
+
+  /** Helper class to format one line Json representation of the LogEntry for structured log. */
+  static final class StructuredLogFormatter {
+    private final Gson gson;
+    private final StringBuilder builder;
+
+    public StructuredLogFormatter(StringBuilder builder) {
+      checkNotNull(builder);
+      this.gson =
+          new GsonBuilder()
+              .registerTypeAdapter(Instant.class, new InstantSerializer())
+              .registerTypeAdapter(SourceLocation.class, new SourceLocationSerializer())
+              .registerTypeAdapter(HttpRequest.RequestMethod.class, new RequestMethodSerializer())
+              .create();
+      this.builder = builder;
+    }
+
+    /**
+     * Adds a Json field and value pair to the current string representation. Method does not
+     * validate parameters to be multi-line strings. Nothing is added if {@code value} parameter is
+     * {@code null}.
+     *
+     * @param name a valid Json field name string.
+     * @param value an object to be serialized to Json using {@link Gson}.
+     * @param appendComma a flag to add a trailing comma.
+     * @return a reference to this object.
+     */
+    public StructuredLogFormatter appendField(String name, Object value, boolean appendComma) {
+      checkNotNull(name);
+      if (value != null) {
+        builder.append(gson.toJson(name)).append(":").append(gson.toJson(value));
+        if (!appendComma) {
+          builder.append(",");
+        }
+      }
+      return this;
+    }
+
+    public StructuredLogFormatter appendField(String name, Object value) {
+      return appendField(name, value, false);
+    }
+
+    /**
+     * Serializes a dictionary of key, values as Json fields.
+     *
+     * @param value a {@link Map} of key, value arguments to be serialized using {@link Gson}.
+     * @param appendComma a flag to add a trailing comma.
+     * @return a reference to this object.
+     */
+    public StructuredLogFormatter appendDict(Map<String, Object> value, boolean appendComma) {
+      if (value != null) {
+        String json = gson.toJson(value);
+        // append json object without brackets
+        if (json.length() > 1) {
+          builder.append(json.substring(0, json.length() - 1).substring(1));
+          if (!appendComma) {
+            builder.append(",");
+          }
+        }
+      }
+      return this;
+    }
+  }
+
+  /**
+   * Serializes the object to a one line JSON string in the simplified format that can be parsed by
+   * the logging agents that run on Google Cloud resources.
+   */
+  public String toStructuredJsonString() {
+    if (payload.getType() == Type.PROTO) {
+      throw new UnsupportedOperationException("LogEntry with protobuf payload cannot be converted");
+    }
+
+    final StringBuilder builder = new StringBuilder("{");
+    final StructuredLogFormatter formatter = new StructuredLogFormatter(builder);
+
+    formatter
+        .appendField("severity", severity)
+        .appendField("timestamp", timestamp)
+        .appendField("httpRequest", httpRequest)
+        .appendField("logging.googleapis.com/insertId", insertId)
+        .appendField("logging.googleapis.com/labels", labels)
+        .appendField("logging.googleapis.com/operation", operation)
+        .appendField("logging.googleapis.com/sourceLocation", sourceLocation)
+        .appendField("logging.googleapis.com/spanId", spanId)
+        .appendField("logging.googleapis.com/trace", trace)
+        .appendField("logging.googleapis.com/trace_sampled", traceSampled);
+    if (payload.getType() == Type.STRING) {
+      formatter.appendField("message", payload.getData(), true);
+    } else if (payload.getType() == Type.JSON) {
+      Payload.JsonPayload jsonPayload = (Payload.JsonPayload) payload;
+      formatter.appendDict(jsonPayload.getDataAsMap(), true);
+    }
+    builder.append("}");
+    return builder.toString();
+  }
+
   /** Returns a builder for {@code LogEntry} objects given the entry payload. */
   public static Builder newBuilder(Payload<?> payload) {
     return new Builder(payload);

google-cloud-logging/src/main/java/com/google/cloud/logging/Logging.java

@@ -1286,8 +1286,30 @@ ApiFuture<AsyncPage<MonitoredResourceDescriptor>> listMonitoredResourceDescripto
    * </pre>
    */
   @BetaApi("The surface for the tail streaming is not stable yet and may change in the future.")
-  default LogEntryServerStream tailLogEntries(TailOption... options) {
+  LogEntryServerStream tailLogEntries(TailOption... options);
+
+  /**
+   * Populates metadata fields of the immutable collection of {@link LogEntry} items. Only empty
+   * fields are populated. The {@link SourceLocation} is populated only for items with the severity
+   * set to {@link Severity.DEBUG}. The information about {@link HttpRequest}, trace and span Id is
+   * retrieved using {@link ContextHandler}.
+   *
+   * @param logEntries an immutable collection of {@link LogEntry} items.
+   * @param customResource a customized instance of the {@link MonitoredResource}. If this parameter
+   *     is {@code null} then the new instance will be generated using {@link
+   *     MonitoredResourceUtil#getResource(String, String)}.
+   * @param exclusionClassPaths a list of exclussion class path prefixes. If left empty then {@link
+   *     SourceLocation} instance is built based on the caller's stack trace information. Otherwise,
+   *     the information from the first {@link StackTraceElement} along the call stack which class
+   *     name does not start with any not {@code null} exclusion class paths is used.
+   * @return A collection of {@link LogEntry} items composed from the {@code logEntries} parameter
+   *     with populated metadata fields.
+   */
+  default Iterable<LogEntry> populateMetadata(
+      Iterable<LogEntry> logEntries,
+      MonitoredResource customResource,
+      String... exclusionClassPaths) {
     throw new UnsupportedOperationException(
-        "method tailLogEntriesCallable() does not have default implementation");
+        "method populateMetadata() does not have default implementation");
   }
 }

google-cloud-logging/src/main/java/com/google/cloud/logging/LoggingConfig.java

@@ -42,6 +42,7 @@ class LoggingConfig {
   private static final String ENHANCERS_TAG = "enhancers";
   private static final String USE_INHERITED_CONTEXT = "useInheritedContext";
   private static final String AUTO_POPULATE_METADATA = "autoPopulateMetadata";
+  private static final String REDIRECT_TO_STDOUT = "redirectToStdout";
 
   public LoggingConfig(String className) {
     this.className = className;
@@ -78,11 +79,11 @@ Formatter getFormatter() {
   }
 
   Boolean getAutoPopulateMetadata() {
-    String flag = getProperty(AUTO_POPULATE_METADATA);
-    if (flag != null) {
-      return Boolean.parseBoolean(flag);
-    }
-    return null;
+    return getBooleanProperty(AUTO_POPULATE_METADATA, null);
+  }
+
+  Boolean getRedirectToStdout() {
+    return getBooleanProperty(REDIRECT_TO_STDOUT, null);
   }
 
   MonitoredResource getMonitoredResource(String projectId) {
@@ -127,6 +128,14 @@ private String getProperty(String name, String defaultValue) {
     return firstNonNull(getProperty(name), defaultValue);
   }
 
+  private Boolean getBooleanProperty(String name, Boolean defaultValue) {
+    String flag = getProperty(name);
+    if (flag != null) {
+      return Boolean.parseBoolean(flag);
+    }
+    return defaultValue;
+  }
+
   private Level getLevelProperty(String name, Level defaultValue) {
     String stringLevel = getProperty(name);
     if (stringLevel == null) {