diff --git a/dataproc/pom.xml b/dataproc/pom.xml new file mode 100644 index 00000000000..5d4cfcc9603 --- /dev/null +++ b/dataproc/pom.xml @@ -0,0 +1,53 @@ + + + 4.0.0 + + com.example + dataproc + 1.0-SNAPSHOT + + + + com.google.cloud.samples + shared-configuration + 1.0.11 + + + + 1.8 + 1.8 + UTF-8 + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + + + + + + + + junit + junit + 4.12 + + + com.google.cloud + google-cloud-dataproc + 0.117.0 + + + + \ No newline at end of file diff --git a/dataproc/src/main/java/CreateCluster.java b/dataproc/src/main/java/CreateCluster.java new file mode 100644 index 00000000000..a5e5f4264a1 --- /dev/null +++ b/dataproc/src/main/java/CreateCluster.java @@ -0,0 +1,84 @@ +/* + * Copyright 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// [START create_cluster] +import com.google.api.gax.longrunning.OperationFuture; +import com.google.cloud.dataproc.v1.Cluster; +import com.google.cloud.dataproc.v1.ClusterConfig; +import com.google.cloud.dataproc.v1.ClusterControllerClient; +import com.google.cloud.dataproc.v1.ClusterControllerSettings; +import com.google.cloud.dataproc.v1.ClusterOperationMetadata; +import com.google.cloud.dataproc.v1.InstanceGroupConfig; +import java.io.IOException; +import java.io.InterruptedIOException; +import java.util.concurrent.ExecutionException; + +public class CreateCluster { + + public static void createCluster(String projectId, String region, String clusterName) + throws IOException, InterruptedException { + String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); + + // Configure the settings for the cluster controller client + ClusterControllerSettings clusterControllerSettings = + ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); + + // Create a cluster controller client with the configured settings. We only need to create + // the client once, and can be reused for multiple requests. Using a try-with-resources + // will close the client for us, but this can also be done manually with the .close() method. + try (ClusterControllerClient clusterControllerClient = ClusterControllerClient + .create(clusterControllerSettings)) { + // Configure the settings for our cluster + InstanceGroupConfig masterConfig = InstanceGroupConfig.newBuilder() + .setMachineTypeUri("n1-standard-1") + .setNumInstances(1) + .build(); + InstanceGroupConfig workerConfig = InstanceGroupConfig.newBuilder() + .setMachineTypeUri("n1-standard-1") + .setNumInstances(2) + .build(); + ClusterConfig clusterConfig = ClusterConfig.newBuilder() + .setMasterConfig(masterConfig) + .setWorkerConfig(workerConfig) + .build(); + // Create the cluster object with the desired cluster config + Cluster cluster = Cluster.newBuilder() + .setClusterName(clusterName) + .setConfig(clusterConfig) + .build(); + + // Send a request to create a Dataproc cluster. + OperationFuture createClusterAsyncRequest = + clusterControllerClient.createClusterAsync(projectId, region, cluster); + Cluster response = createClusterAsyncRequest.get(); + + // Print out the response + System.out.println( + String.format("Cluster created successfully: %s", response.getClusterName()) + ); + + } catch (IOException e) { + // Likely this would occur due to issues authenticating with GCP. Make sure the environment + // variable GOOGLE_APPLICATION_CREDENTIALS is configured. + System.out.println("Error creating the cluster controller client: \n" + e.toString()); + } catch (ExecutionException e) { + // Common issues for this include needing to increase compute engine quotas or a cluster of + // the same name already exists. + System.out.println("Error during cluster creation request: \n" + e.toString()); + } + } +} +// [END create_cluster] \ No newline at end of file diff --git a/dataproc/src/test/java/CreateClusterTest.java b/dataproc/src/test/java/CreateClusterTest.java new file mode 100644 index 00000000000..39ab75f8233 --- /dev/null +++ b/dataproc/src/test/java/CreateClusterTest.java @@ -0,0 +1,88 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import static junit.framework.TestCase.assertNotNull; +import static org.hamcrest.MatcherAssert.assertThat; + +import com.google.api.gax.longrunning.OperationFuture; +import com.google.cloud.dataproc.v1.ClusterControllerClient; +import com.google.cloud.dataproc.v1.ClusterOperationMetadata; +import com.google.protobuf.Empty; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import org.hamcrest.CoreMatchers; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class CreateClusterTest { + + private static final String BASE_CLUSTER_NAME = "test-cluster"; + private static final String REGION = "us-central1"; + + private static String projectId = System.getenv("GOOGLE_CLOUD_PROJECT"); + private String clusterName; + private ByteArrayOutputStream bout; + + private static void requireEnv(String varName) { + assertNotNull( + System.getenv(varName), + String.format("Environment variable '%s' is required to perform these tests.", varName) + ); + } + + @BeforeClass + public static void checkRequirements() { + requireEnv("GOOGLE_APPLICATION_CREDENTIALS"); + requireEnv("GOOGLE_CLOUD_PROJECT"); + } + + @Before + public void setUp() { + clusterName = String.format("%s-%s", BASE_CLUSTER_NAME, UUID.randomUUID().toString()); + + bout = new ByteArrayOutputStream(); + System.setOut(new PrintStream(bout)); + } + + @Test + public void createClusterTest() throws IOException, InterruptedException { + CreateCluster.createCluster(projectId, REGION, clusterName); + String output = bout.toString(); + + assertThat(output, CoreMatchers.containsString(clusterName)); + } + + @After + public void tearDown() throws IOException, InterruptedException { + try (ClusterControllerClient clusterControllerClient = ClusterControllerClient + .create()) { + OperationFuture deleteClusterAsyncRequest = + clusterControllerClient.deleteClusterAsync(projectId, REGION, clusterName); + deleteClusterAsyncRequest.get(); + + } catch (ExecutionException e) { + System.out.println("Error during cluster deletion: \n" + e.toString()); + } + } +} \ No newline at end of file