[BEAM-725] Migrate to use the generic Google credentials library

pom.xml

@@ -112,6 +112,7 @@
     <datastore.proto.version>1.2.0</datastore.proto.version>
     <google-auto-service.version>1.0-rc2</google-auto-service.version>
     <google-auto-value.version>1.1</google-auto-value.version>
+    <google-auth.version>0.6.0</google-auth.version>
     <google-clients.version>1.22.0</google-clients.version>
     <google-cloud-bigdataoss.version>1.4.5</google-cloud-bigdataoss.version>
     <google-cloud-dataflow-java-proto-library-all.version>0.5.160304</google-cloud-dataflow-java-proto-library-all.version>
@@ -524,6 +525,26 @@
         </exclusions>
       </dependency>
 
+      <dependency>
+        <groupId>com.google.auth</groupId>
+        <artifactId>google-auth-library-credentials</artifactId>
+        <version>${google-auth.version}</version>
+      </dependency>
+
+      <dependency>
+        <groupId>com.google.auth</groupId>
+        <artifactId>google-auth-library-oauth2-http</artifactId>
+        <version>${google-auth.version}</version>
+        <exclusions>
+          <!-- Exclude an old version of guava that is being pulled in by a transitive 
+            dependency of google-api-client -->
+          <exclusion>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava-jdk5</artifactId>
+          </exclusion>
+        </exclusions>
+      </dependency>
+
       <dependency>
         <groupId>com.google.apis</groupId>
         <artifactId>google-api-services-bigquery</artifactId>

runners/google-cloud-dataflow-java/pom.xml

@@ -163,11 +163,6 @@
       <artifactId>google-api-client</artifactId>
     </dependency>
 
-    <dependency>
-      <groupId>com.google.oauth-client</groupId>
-      <artifactId>google-oauth-client</artifactId>
-    </dependency>
-
     <dependency>
       <groupId>com.google.http-client</groupId>
       <artifactId>google-http-client</artifactId>
@@ -194,6 +189,16 @@
       <artifactId>google-api-services-clouddebugger</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>com.google.auth</groupId>
+      <artifactId>google-auth-library-credentials</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>com.google.auth</groupId>
+      <artifactId>google-auth-library-oauth2-http</artifactId>
+    </dependency>
+
     <dependency>
       <groupId>com.google.cloud.bigdataoss</groupId>
       <artifactId>util</artifactId>

runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DataflowTransport.java

@@ -20,10 +20,11 @@
 import static org.apache.beam.sdk.util.Transport.getJsonFactory;
 import static org.apache.beam.sdk.util.Transport.getTransport;
 
-import com.google.api.client.auth.oauth2.Credential;
 import com.google.api.client.http.HttpRequestInitializer;
 import com.google.api.services.clouddebugger.v2.Clouddebugger;
 import com.google.api.services.dataflow.Dataflow;
+import com.google.auth.Credentials;
+import com.google.auth.http.HttpCredentialsAdapter;
 import com.google.cloud.hadoop.util.ChainingHttpRequestInitializer;
 import com.google.common.collect.ImmutableList;
 import java.net.MalformedURLException;
@@ -91,11 +92,13 @@ public static Clouddebugger.Builder newClouddebuggerClient(DataflowPipelineOptio
   }
 
   private static HttpRequestInitializer chainHttpRequestInitializer(
-      Credential credential, HttpRequestInitializer httpRequestInitializer) {
+      Credentials credential, HttpRequestInitializer httpRequestInitializer) {
     if (credential == null) {
       return httpRequestInitializer;
     } else {
-      return new ChainingHttpRequestInitializer(credential, httpRequestInitializer);
+      return new ChainingHttpRequestInitializer(
+          new HttpCredentialsAdapter(credential),
+          httpRequestInitializer);
     }
   }
 }

sdks/java/core/pom.xml

@@ -248,18 +248,14 @@
       <scope>runtime</scope>
     </dependency>
 
+    <dependency>
+      <groupId>com.google.auth</groupId>
+      <artifactId>google-auth-library-credentials</artifactId>
+    </dependency>
+
     <dependency>
       <groupId>com.google.auth</groupId>
       <artifactId>google-auth-library-oauth2-http</artifactId>
-      <version>0.4.0</version>
-      <exclusions>
-        <!-- Exclude an old version of guava that is being pulled in by a transitive 
-          dependency of google-api-client -->
-        <exclusion>
-          <groupId>com.google.guava</groupId>
-          <artifactId>guava-jdk5</artifactId>
-        </exclusion>
-      </exclusions>
     </dependency>
 
     <dependency>
@@ -321,11 +317,6 @@
       <scope>runtime</scope>
     </dependency>
 
-    <dependency>
-      <groupId>com.google.oauth-client</groupId>
-      <artifactId>google-oauth-client-java6</artifactId>
-    </dependency>
-
     <dependency>
       <groupId>com.google.oauth-client</groupId>
       <artifactId>google-oauth-client</artifactId>

sdks/java/core/src/main/java/org/apache/beam/sdk/options/GcpOptions.java

@@ -18,8 +18,7 @@
 package org.apache.beam.sdk.options;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.google.api.client.auth.oauth2.Credential;
-import com.google.api.client.googleapis.auth.oauth2.GoogleOAuthConstants;
+import com.google.auth.Credentials;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Strings;
 import com.google.common.io.Files;
@@ -40,31 +39,14 @@
 import org.slf4j.LoggerFactory;
 
 /**
- * Options used to configure Google Cloud Platform project and credentials.
+ * Options used to configure Google Cloud Platform specific options such as the project
+ * and credentials.
  *
- * <p>These options configure which of the following three different mechanisms for obtaining a
- * credential are used:
- * <ol>
- *   <li>
- *     It can fetch the
- *     <a href="https://developers.google.com/accounts/docs/application-default-credentials">
- *     application default credentials</a>.
- *   </li>
- *   <li>
- *     The user can specify a client secrets file and go through the OAuth2
- *     webflow. The credential will then be cached in the user's home
- *     directory for reuse.
- *   </li>
- *   <li>
- *     The user can specify a file containing a service account private key along
- *     with the service account name.
- *   </li>
- * </ol>
- *
- * <p>The default mechanism is to use the
+ * <p>These options defer to the
  * <a href="https://developers.google.com/accounts/docs/application-default-credentials">
- * application default credentials</a>. The other options can be
- * used by setting the corresponding properties.
+ * application default credentials</a> for authentication. See the
+ * <a href="https://github.com/google/google-auth-library-java">Google Auth Library</a> for
+ * alternative mechanisms for creating credentials.
  */
 @Description("Options used to configure Google Cloud Platform project and credentials.")
 public interface GcpOptions extends GoogleApiDebugOptions, PipelineOptions {
@@ -77,79 +59,6 @@ public interface GcpOptions extends GoogleApiDebugOptions, PipelineOptions {
   String getProject();
   void setProject(String value);
 
-  /**
-   * This option controls which file to use when attempting to create the credentials using the
-   * service account method.
-   *
-   * <p>This option if specified, needs be combined with the
-   * {@link GcpOptions#getServiceAccountName() serviceAccountName}.
-   */
-  @JsonIgnore
-  @Description("Controls which file to use when attempting to create the credentials "
-      + "using the service account method. This option if specified, needs to be combined with "
-      + "the serviceAccountName option.")
-  String getServiceAccountKeyfile();
-  void setServiceAccountKeyfile(String value);
-
-  /**
-   * This option controls which service account to use when attempting to create the credentials
-   * using the service account method.
-   *
-   * <p>This option if specified, needs be combined with the
-   * {@link GcpOptions#getServiceAccountKeyfile() serviceAccountKeyfile}.
-   */
-  @JsonIgnore
-  @Description("Controls which service account to use when attempting to create the credentials "
-      + "using the service account method. This option if specified, needs to be combined with "
-      + "the serviceAccountKeyfile option.")
-  String getServiceAccountName();
-  void setServiceAccountName(String value);
-
-  /**
-   * This option controls which file to use when attempting to create the credentials
-   * using the OAuth 2 webflow. After the OAuth2 webflow, the credentials will be stored
-   * within credentialDir.
-   */
-  @JsonIgnore
-  @Description("This option controls which file to use when attempting to create the credentials "
-      + "using the OAuth 2 webflow. After the OAuth2 webflow, the credentials will be stored "
-      + "within credentialDir.")
-  String getSecretsFile();
-  void setSecretsFile(String value);
-
-  /**
-   * This option controls which credential store to use when creating the credentials
-   * using the OAuth 2 webflow.
-   */
-  @Description("This option controls which credential store to use when creating the credentials "
-      + "using the OAuth 2 webflow.")
-  @Default.String("cloud_dataflow")
-  String getCredentialId();
-  void setCredentialId(String value);
-
-  /**
-   * Directory for storing dataflow credentials after execution of the OAuth 2 webflow. Defaults
-   * to using the $HOME/.store/data-flow directory.
-   */
-  @Description("Directory for storing dataflow credentials after execution of the OAuth 2 webflow. "
-      + "Defaults to using the $HOME/.store/data-flow directory.")
-  @Default.InstanceFactory(CredentialDirFactory.class)
-  String getCredentialDir();
-  void setCredentialDir(String value);
-
-  /**
-   * Returns the default credential directory of ${user.home}/.store/data-flow.
-   */
-  class CredentialDirFactory implements DefaultValueFactory<String> {
-    @Override
-    public String create(PipelineOptions options) {
-      File home = new File(System.getProperty("user.home"));
-      File store = new File(home, ".store");
-      File dataflow = new File(store, "data-flow");
-      return dataflow.getPath();
-    }
-  }
-
   /**
    * The class of the credential factory that should be created and used to create
    * credentials. If gcpCredential has not been set explicitly, an instance of this class will
@@ -173,9 +82,8 @@ void setCredentialFactoryClass(
       + "If no credential has been set explicitly, the default is to use the instance factory "
       + "that constructs a credential based upon the currently set credentialFactoryClass.")
   @Default.InstanceFactory(GcpUserCredentialsFactory.class)
-  @Hidden
-  Credential getGcpCredential();
-  void setGcpCredential(Credential value);
+  Credentials getGcpCredential();
+  void setGcpCredential(Credentials value);
 
   /**
    * Attempts to infer the default project based upon the environment this application
@@ -251,9 +159,9 @@ Map<String, String> getEnvironment() {
    * Attempts to load the GCP credentials. See
    * {@link CredentialFactory#getCredential()} for more details.
    */
-  class GcpUserCredentialsFactory implements DefaultValueFactory<Credential> {
+  class GcpUserCredentialsFactory implements DefaultValueFactory<Credentials> {
     @Override
-    public Credential create(PipelineOptions options) {
+    public Credentials create(PipelineOptions options) {
       GcpOptions gcpOptions = options.as(GcpOptions.class);
       try {
         CredentialFactory factory = InstanceBuilder.ofType(CredentialFactory.class)
@@ -268,28 +176,6 @@ public Credential create(PipelineOptions options) {
     }
   }
 
-  /**
-   * The token server URL to use for OAuth 2 authentication. Normally, the default is sufficient,
-   * but some specialized use cases may want to override this value.
-   */
-  @Description("The token server URL to use for OAuth 2 authentication. Normally, the default "
-      + "is sufficient, but some specialized use cases may want to override this value.")
-  @Default.String(GoogleOAuthConstants.TOKEN_SERVER_URL)
-  @Hidden
-  String getTokenServerUrl();
-  void setTokenServerUrl(String value);
-
-  /**
-   * The authorization server URL to use for OAuth 2 authentication. Normally, the default is
-   * sufficient, but some specialized use cases may want to override this value.
-   */
-  @Description("The authorization server URL to use for OAuth 2 authentication. Normally, the "
-      + "default is sufficient, but some specialized use cases may want to override this value.")
-  @Default.String(GoogleOAuthConstants.AUTHORIZATION_SERVER_URL)
-  @Hidden
-  String getAuthorizationServerEncodedUrl();
-  void setAuthorizationServerEncodedUrl(String value);
-
   /**
    * A GCS path for storing temporary files in GCP.
    *

sdks/java/core/src/main/java/org/apache/beam/sdk/testing/BigqueryMatcher.java

@@ -19,8 +19,6 @@
 
 import static com.google.common.base.Preconditions.checkArgument;
 
-import com.google.api.client.auth.oauth2.Credential;
-import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
 import com.google.api.client.http.HttpTransport;
 import com.google.api.client.json.JsonFactory;
 import com.google.api.client.util.BackOff;
@@ -32,6 +30,9 @@
 import com.google.api.services.bigquery.model.QueryResponse;
 import com.google.api.services.bigquery.model.TableCell;
 import com.google.api.services.bigquery.model.TableRow;
+import com.google.auth.Credentials;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Strings;
 import com.google.common.collect.Lists;
@@ -136,9 +137,9 @@ protected boolean matchesSafely(PipelineResult pipelineResult) {
   Bigquery newBigqueryClient(String applicationName) {
     HttpTransport transport = Transport.getTransport();
     JsonFactory jsonFactory = Transport.getJsonFactory();
-    Credential credential = getDefaultCredential(transport, jsonFactory);
+    Credentials credential = getDefaultCredential();
 
-    return new Bigquery.Builder(transport, jsonFactory, credential)
+    return new Bigquery.Builder(transport, jsonFactory, new HttpCredentialsAdapter(credential))
         .setApplicationName(applicationName)
         .build();
   }
@@ -168,10 +169,10 @@ private void validateArgument(String name, String value) {
         !Strings.isNullOrEmpty(value), "Expected valid %s, but was %s", name, value);
   }
 
-  private Credential getDefaultCredential(HttpTransport transport, JsonFactory jsonFactory) {
-    GoogleCredential credential;
+  private Credentials getDefaultCredential() {
+    GoogleCredentials credential;
     try {
-      credential = GoogleCredential.getApplicationDefault(transport, jsonFactory);
+      credential = GoogleCredentials.getApplicationDefault();
     } catch (IOException e) {
       throw new RuntimeException("Failed to get application default credential.", e);
     }

sdks/java/core/src/main/java/org/apache/beam/sdk/util/CredentialFactory.java

@@ -17,13 +17,13 @@
  */
 package org.apache.beam.sdk.util;
 
-import com.google.api.client.auth.oauth2.Credential;
+import com.google.auth.Credentials;
 import java.io.IOException;
 import java.security.GeneralSecurityException;
 
 /**
  * Construct an oauth credential to be used by the SDK and the SDK workers.
  */
 public interface CredentialFactory {
-  Credential getCredential() throws IOException, GeneralSecurityException;
+  Credentials getCredential() throws IOException, GeneralSecurityException;
 }