From 8b74842a3bfc9471d9d0bac4fb5b0e8d3a694ea8 Mon Sep 17 00:00:00 2001 From: bradmiro Date: Mon, 2 Dec 2019 12:42:12 -0500 Subject: [PATCH 01/21] Added dataproc quickstart samples --- dataproc/pom.xml | 5 + dataproc/src/main/java/CreateCluster.java | 42 +++--- dataproc/src/main/java/DeleteCluster.java | 61 ++++++++ dataproc/src/main/java/ListClusters.java | 49 +++++++ dataproc/src/main/java/SubmitJob.java | 69 +++++++++ dataproc/src/main/java/UpdateCluster.java | 92 ++++++++++++ dataproc/src/test/java/CreateClusterTest.java | 9 +- dataproc/src/test/java/DeleteClusterTest.java | 95 ++++++++++++ dataproc/src/test/java/ListClustersTest.java | 115 +++++++++++++++ dataproc/src/test/java/SubmitJobTest.java | 135 ++++++++++++++++++ dataproc/src/test/java/UpdateClusterTest.java | 130 +++++++++++++++++ 11 files changed, 776 insertions(+), 26 deletions(-) create mode 100644 dataproc/src/main/java/DeleteCluster.java create mode 100644 dataproc/src/main/java/ListClusters.java create mode 100644 dataproc/src/main/java/SubmitJob.java create mode 100644 dataproc/src/main/java/UpdateCluster.java create mode 100644 dataproc/src/test/java/DeleteClusterTest.java create mode 100644 dataproc/src/test/java/ListClustersTest.java create mode 100644 dataproc/src/test/java/SubmitJobTest.java create mode 100644 dataproc/src/test/java/UpdateClusterTest.java diff --git a/dataproc/pom.xml b/dataproc/pom.xml index c28c912b081..ef58507b3c9 100644 --- a/dataproc/pom.xml +++ b/dataproc/pom.xml @@ -49,6 +49,11 @@ google-cloud-dataproc 0.117.0 + + com.google.cloud + google-cloud-storage + 1.101.0 + \ No newline at end of file diff --git a/dataproc/src/main/java/CreateCluster.java b/dataproc/src/main/java/CreateCluster.java index f68852e8e2e..ed2f507dc6b 100644 --- a/dataproc/src/main/java/CreateCluster.java +++ b/dataproc/src/main/java/CreateCluster.java @@ -38,26 +38,27 @@ public static void createCluster(String projectId, String region, String cluster // Create a cluster controller client with the configured settings. We only need to create // the client once, and can be reused for multiple requests. Using a try-with-resources // will close the client for us, but this can also be done manually with the .close() method. - try (ClusterControllerClient clusterControllerClient = ClusterControllerClient - .create(clusterControllerSettings)) { + try (ClusterControllerClient clusterControllerClient = + ClusterControllerClient.create(clusterControllerSettings)) { // Configure the settings for our cluster - InstanceGroupConfig masterConfig = InstanceGroupConfig.newBuilder() - .setMachineTypeUri("n1-standard-1") - .setNumInstances(1) - .build(); - InstanceGroupConfig workerConfig = InstanceGroupConfig.newBuilder() - .setMachineTypeUri("n1-standard-1") - .setNumInstances(2) - .build(); - ClusterConfig clusterConfig = ClusterConfig.newBuilder() - .setMasterConfig(masterConfig) - .setWorkerConfig(workerConfig) - .build(); + InstanceGroupConfig masterConfig = + InstanceGroupConfig.newBuilder() + .setMachineTypeUri("n1-standard-1") + .setNumInstances(1) + .build(); + InstanceGroupConfig workerConfig = + InstanceGroupConfig.newBuilder() + .setMachineTypeUri("n1-standard-1") + .setNumInstances(2) + .build(); + ClusterConfig clusterConfig = + ClusterConfig.newBuilder() + .setMasterConfig(masterConfig) + .setWorkerConfig(workerConfig) + .build(); // Create the cluster object with the desired cluster config - Cluster cluster = Cluster.newBuilder() - .setClusterName(clusterName) - .setConfig(clusterConfig) - .build(); + Cluster cluster = + Cluster.newBuilder().setClusterName(clusterName).setConfig(clusterConfig).build(); // Create the Cloud Dataproc cluster OperationFuture createClusterAsyncRequest = @@ -66,8 +67,7 @@ public static void createCluster(String projectId, String region, String cluster // Print out a success message System.out.println( - String.format("Cluster created successfully: %s", response.getClusterName()) - ); + String.format("Cluster created successfully: %s", response.getClusterName())); } catch (IOException e) { // Likely this would occur due to issues authenticating with GCP. Make sure the environment @@ -80,4 +80,4 @@ public static void createCluster(String projectId, String region, String cluster } } } -// [END dataproc_create_cluster] \ No newline at end of file +// [END dataproc_create_cluster] diff --git a/dataproc/src/main/java/DeleteCluster.java b/dataproc/src/main/java/DeleteCluster.java new file mode 100644 index 00000000000..f94761179c2 --- /dev/null +++ b/dataproc/src/main/java/DeleteCluster.java @@ -0,0 +1,61 @@ +/* + * Copyright 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// [START dataproc_delete_cluster] +import com.google.api.gax.longrunning.OperationFuture; +import com.google.cloud.dataproc.v1.ClusterControllerClient; +import com.google.cloud.dataproc.v1.ClusterControllerSettings; +import com.google.cloud.dataproc.v1.ClusterOperationMetadata; +import com.google.protobuf.Empty; +import java.io.IOException; +import java.util.concurrent.ExecutionException; + +public class DeleteCluster { + + public static void deleteCluster(String projectId, String region, String clusterName) + throws IOException, InterruptedException { + String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); + + // Configure the settings for the cluster controller client + ClusterControllerSettings clusterControllerSettings = + ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); + + // Create a cluster controller client with the configured settings. We only need to create + // the client once, and can be reused for multiple requests. Using a try-with-resources + // will close the client for us, but this can also be done manually with the .close() method. + try (ClusterControllerClient clusterControllerClient = + ClusterControllerClient.create(clusterControllerSettings)) { + // Configure the settings for our cluster + + // Delete the Cloud Dataproc cluster + OperationFuture deleteClusterAsyncRequest = + clusterControllerClient.deleteClusterAsync(projectId, region, clusterName); + deleteClusterAsyncRequest.get(); + + // Print out a success message + System.out.println(String.format("Cluster deleted successfully: %s", clusterName)); + + } catch (IOException e) { + // Likely this would occur due to issues authenticating with GCP. Make sure the environment + // variable GOOGLE_APPLICATION_CREDENTIALS is configured. + System.out.println("Error deleting the cluster controller client: \n" + e.toString()); + } catch (ExecutionException e) { + // This will likely be due to a cluster of the given name not existing in the given region. + System.out.println("Error during cluster deletion request: \n" + e.toString()); + } + } +} +// [END dataproc_delete_cluster] diff --git a/dataproc/src/main/java/ListClusters.java b/dataproc/src/main/java/ListClusters.java new file mode 100644 index 00000000000..de8483bbc06 --- /dev/null +++ b/dataproc/src/main/java/ListClusters.java @@ -0,0 +1,49 @@ +/* + * Copyright 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// [START dataproc_list_clusters] +import com.google.cloud.dataproc.v1.Cluster; +import com.google.cloud.dataproc.v1.ClusterControllerClient; +import com.google.cloud.dataproc.v1.ClusterControllerSettings; +import java.io.IOException; + +public class ListClusters { + + public static void ListClusters(String projectId, String region) throws IOException { + String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); + + // Configure the settings for the cluster controller client + ClusterControllerSettings clusterControllerSettings = + ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); + + // Create a cluster controller client with the configured settings. We only need to create + // the client once, and can be reused for multiple requests. Using a try-with-resources + // will close the client for us, but this can also be done manually with the .close() method. + try (ClusterControllerClient clusterControllerClient = + ClusterControllerClient.create(clusterControllerSettings)) { + + for (Cluster element : clusterControllerClient.listClusters(projectId, region).iterateAll()) { + System.out.println( + String.format("%s-%s", element.getClusterName(), element.getStatus().getState())); + } + } catch (IOException e) { + // Likely this would occur due to issues authenticating with GCP. Make sure the environment + // variable GOOGLE_APPLICATION_CREDENTIALS is configured. + System.out.println("Error deleting the cluster controller client: \n" + e.toString()); + } + } +} +// [END dataproc_list_clusters] diff --git a/dataproc/src/main/java/SubmitJob.java b/dataproc/src/main/java/SubmitJob.java new file mode 100644 index 00000000000..506ac243960 --- /dev/null +++ b/dataproc/src/main/java/SubmitJob.java @@ -0,0 +1,69 @@ +/* + * Copyright 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// [START dataproc_submit_job] +import com.google.cloud.dataproc.v1.Job; +import com.google.cloud.dataproc.v1.JobControllerClient; +import com.google.cloud.dataproc.v1.JobControllerSettings; +import com.google.cloud.dataproc.v1.JobPlacement; +import com.google.cloud.dataproc.v1.PySparkJob; +import java.io.IOException; + +public class SubmitJob { + + public static void submitJob( + String projectId, String region, String clusterName, String job_file_path) + throws IOException { + String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); + + // Configure the settings for the cluster controller client + JobControllerSettings jobControllerSettings = + JobControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); + + // Create a cluster controller client with the configured settings. We only need to create + // the client once, and can be reused for multiple requests. Using a try-with-resources + // will close the client for us, but this can also be done manually with the .close() method. + try (JobControllerClient jobControllerClient = + JobControllerClient.create(jobControllerSettings)) { + + // Configure the settings for our job + JobPlacement jobPlacement = JobPlacement.newBuilder().setClusterName(clusterName).build(); + PySparkJob pySparkJob = PySparkJob.newBuilder().setMainPythonFileUri(job_file_path).build(); + Job job = Job.newBuilder().setPlacement(jobPlacement).setPysparkJob(pySparkJob).build(); + + Job jobOperation = jobControllerClient.submitJob(projectId, region, job); + + String jobId = jobOperation.getReference().getJobId(); + + while (true) { + Job jobInfo = jobControllerClient.getJob(projectId, region, jobId); + if (jobInfo.getStatus().getState().toString() == "ERROR") { + System.out.println( + String.format("Job %s failed: %s", jobId, jobInfo.getStatus().getDetails())); + break; + } else if (jobInfo.getStatus().getState().toString() == "DONE") { + System.out.println(String.format("Job %s finished.", jobId)); + break; + } + } + } catch (IOException e) { + // Likely this would occur due to issues authenticating with GCP. Make sure the environment + // variable GOOGLE_APPLICATION_CREDENTIALS is configured. + System.out.println("Error creating the cluster controller client: \n" + e.toString()); + } + } +} +// [END dataproc_submit_job] diff --git a/dataproc/src/main/java/UpdateCluster.java b/dataproc/src/main/java/UpdateCluster.java new file mode 100644 index 00000000000..3727c8f47de --- /dev/null +++ b/dataproc/src/main/java/UpdateCluster.java @@ -0,0 +1,92 @@ +/* + * Copyright 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// [START dataproc_update_cluster] +import com.google.api.gax.longrunning.OperationFuture; +import com.google.cloud.dataproc.v1.Cluster; +import com.google.cloud.dataproc.v1.ClusterConfig; +import com.google.cloud.dataproc.v1.ClusterControllerClient; +import com.google.cloud.dataproc.v1.ClusterControllerSettings; +import com.google.cloud.dataproc.v1.ClusterOperationMetadata; +import com.google.cloud.dataproc.v1.InstanceGroupConfig; +import com.google.protobuf.FieldMask; +import java.io.IOException; +import java.util.concurrent.ExecutionException; + +public class UpdateCluster { + + public static void updateCluster( + String projectId, String region, String clusterName, int numWorkers) + throws IOException, InterruptedException { + String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); + + // Configure the settings for the cluster controller client + ClusterControllerSettings clusterControllerSettings = + ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); + + // Create a cluster controller client with the configured settings. We only need to create + // the client once, and can be reused for multiple requests. Using a try-with-resources + // will close the client for us, but this can also be done manually with the .close() method. + try (ClusterControllerClient clusterControllerClient = + ClusterControllerClient.create(clusterControllerSettings)) { + // Configure the settings for our cluster + InstanceGroupConfig masterConfig = + InstanceGroupConfig.newBuilder() + .setMachineTypeUri("n1-standard-1") + .setNumInstances(1) + .build(); + InstanceGroupConfig workerConfig = + InstanceGroupConfig.newBuilder() + .setMachineTypeUri("n1-standard-1") + .setNumInstances(numWorkers) + .build(); + ClusterConfig clusterConfig = + ClusterConfig.newBuilder() + .setMasterConfig(masterConfig) + .setWorkerConfig(workerConfig) + .build(); + // Create the cluster object with the desired cluster config + Cluster cluster = + Cluster.newBuilder().setClusterName(clusterName).setConfig(clusterConfig).build(); + + // Create a field mask to indicate what field of the cluster to update. + FieldMask updateMask = + FieldMask.newBuilder().addPaths("config.worker_config.num_instances").build(); + + // Submit the request to update the cluster + OperationFuture updateClusterAsyncRequest = + clusterControllerClient.updateClusterAsync( + projectId, region, clusterName, cluster, updateMask); + Cluster response = updateClusterAsyncRequest.get(); + + // Print out a success message + System.out.println( + String.format( + "Cluster %s now has %d workers.", + response.getClusterName(), response.getConfig().getWorkerConfig().getNumInstances())); + + } catch (IOException e) { + // Likely this would occur due to issues authenticating with GCP. Make sure the environment + // variable GOOGLE_APPLICATION_CREDENTIALS is configured. + System.out.println("Error creating the cluster controller client: \n" + e.toString()); + } catch (ExecutionException e) { + // Common issues for this include needing to increase compute engine quotas or a cluster of + // the same name already exists. + System.out.println("Error during cluster creation request: \n" + e.toString()); + } + } +} +// [END dataproc_update_cluster] diff --git a/dataproc/src/test/java/CreateClusterTest.java b/dataproc/src/test/java/CreateClusterTest.java index 800856b02dc..f98c24a9b4d 100644 --- a/dataproc/src/test/java/CreateClusterTest.java +++ b/dataproc/src/test/java/CreateClusterTest.java @@ -48,8 +48,7 @@ public class CreateClusterTest { private static void requireEnv(String varName) { assertNotNull( System.getenv(varName), - String.format("Environment variable '%s' is required to perform these tests.", varName) - ); + String.format("Environment variable '%s' is required to perform these tests.", varName)); } @BeforeClass @@ -81,8 +80,8 @@ public void tearDown() throws IOException, InterruptedException { ClusterControllerSettings clusterControllerSettings = ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); - try (ClusterControllerClient clusterControllerClient = ClusterControllerClient - .create(clusterControllerSettings)) { + try (ClusterControllerClient clusterControllerClient = + ClusterControllerClient.create(clusterControllerSettings)) { OperationFuture deleteClusterAsyncRequest = clusterControllerClient.deleteClusterAsync(projectId, REGION, clusterName); deleteClusterAsyncRequest.get(); @@ -91,4 +90,4 @@ public void tearDown() throws IOException, InterruptedException { System.out.println("Error during cluster deletion: \n" + e.toString()); } } -} \ No newline at end of file +} diff --git a/dataproc/src/test/java/DeleteClusterTest.java b/dataproc/src/test/java/DeleteClusterTest.java new file mode 100644 index 00000000000..82d4c453e6e --- /dev/null +++ b/dataproc/src/test/java/DeleteClusterTest.java @@ -0,0 +1,95 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import static junit.framework.TestCase.assertNotNull; +import static org.hamcrest.MatcherAssert.assertThat; + +import com.google.api.gax.longrunning.OperationFuture; +import com.google.cloud.dataproc.v1.Cluster; +import com.google.cloud.dataproc.v1.ClusterConfig; +import com.google.cloud.dataproc.v1.ClusterControllerClient; +import com.google.cloud.dataproc.v1.ClusterControllerSettings; +import com.google.cloud.dataproc.v1.ClusterOperationMetadata; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import org.hamcrest.CoreMatchers; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class DeleteClusterTest { + + private static final String BASE_CLUSTER_NAME = "test-cluster"; + private static final String REGION = "us-central1"; + + private static String projectId = System.getenv("GOOGLE_CLOUD_PROJECT"); + private String clusterName; + private ByteArrayOutputStream bout; + + private static void requireEnv(String varName) { + assertNotNull( + System.getenv(varName), + String.format("Environment variable '%s' is required to perform these tests.", varName)); + } + + @BeforeClass + public static void checkRequirements() { + requireEnv("GOOGLE_APPLICATION_CREDENTIALS"); + requireEnv("GOOGLE_CLOUD_PROJECT"); + } + + @Before + public void setUp() throws IOException, InterruptedException { + clusterName = String.format("%s-%s", BASE_CLUSTER_NAME, UUID.randomUUID().toString()); + + bout = new ByteArrayOutputStream(); + System.setOut(new PrintStream(bout)); + + String myEndpoint = String.format("%s-dataproc.googleapis.com:443", REGION); + + ClusterControllerSettings clusterControllerSettings = + ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); + + Cluster cluster = + Cluster.newBuilder() + .setClusterName(clusterName) + .setConfig(ClusterConfig.newBuilder().build()) + .build(); + + try (ClusterControllerClient clusterControllerClient = + ClusterControllerClient.create(clusterControllerSettings)) { + OperationFuture createClusterAsyncRequest = + clusterControllerClient.createClusterAsync(projectId, REGION, cluster); + createClusterAsyncRequest.get(); + } catch (ExecutionException e) { + System.out.println("[deleteCluster] Error during test cluster creation: \n" + e.toString()); + } + } + + @Test + public void DeleteClusterTest() throws IOException, InterruptedException { + DeleteCluster.deleteCluster(projectId, REGION, clusterName); + String output = bout.toString(); + + assertThat(output, CoreMatchers.containsString("deleted successfully")); + } +} diff --git a/dataproc/src/test/java/ListClustersTest.java b/dataproc/src/test/java/ListClustersTest.java new file mode 100644 index 00000000000..067dfed0a6e --- /dev/null +++ b/dataproc/src/test/java/ListClustersTest.java @@ -0,0 +1,115 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import static junit.framework.TestCase.assertNotNull; +import static org.hamcrest.MatcherAssert.assertThat; + +import com.google.api.gax.longrunning.OperationFuture; +import com.google.cloud.dataproc.v1.Cluster; +import com.google.cloud.dataproc.v1.ClusterConfig; +import com.google.cloud.dataproc.v1.ClusterControllerClient; +import com.google.cloud.dataproc.v1.ClusterControllerSettings; +import com.google.cloud.dataproc.v1.ClusterOperationMetadata; +import com.google.protobuf.Empty; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import org.hamcrest.CoreMatchers; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class ListClustersTest { + + private static final String BASE_CLUSTER_NAME = "test-cluster"; + private static final String REGION = "us-central1"; + private static final String projectId = System.getenv("GOOGLE_CLOUD_PROJECT"); + + private String clusterName; + private String myEndpoint; + private ByteArrayOutputStream bout; + private PrintStream standardOutOrig; + + private static void requireEnv(String varName) { + assertNotNull( + System.getenv(varName), + String.format("Environment variable '%s' is required to perform these tests.", varName)); + } + + @BeforeClass + public static void checkRequirements() { + requireEnv("GOOGLE_APPLICATION_CREDENTIALS"); + requireEnv("GOOGLE_CLOUD_PROJECT"); + } + + @Before + public void setUp() throws IOException, InterruptedException { + clusterName = String.format("%s-%s", BASE_CLUSTER_NAME, UUID.randomUUID().toString()); + bout = new ByteArrayOutputStream(); + standardOutOrig = System.out; + System.setOut(new PrintStream(bout)); + myEndpoint = String.format("%s-dataproc.googleapis.com:443", REGION); + + ClusterControllerSettings clusterControllerSettings = + ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); + + Cluster cluster = + Cluster.newBuilder() + .setClusterName(clusterName) + .setConfig(ClusterConfig.newBuilder().build()) + .build(); + + try (ClusterControllerClient clusterControllerClient = + ClusterControllerClient.create(clusterControllerSettings)) { + OperationFuture createClusterAsyncRequest = + clusterControllerClient.createClusterAsync(projectId, REGION, cluster); + createClusterAsyncRequest.get(); + } catch (ExecutionException e) { + System.out.println("[deleteCluster] Error during test cluster creation: \n" + e.toString()); + } + } + + @Test + public void ListClustersTest() throws IOException, InterruptedException { + ListClusters.ListClusters(projectId, REGION); + String output = bout.toString(); + + assertThat(output, CoreMatchers.containsString(clusterName)); + assertThat(output, CoreMatchers.containsString("RUNNING")); + } + + @After + public void teardown() throws IOException, InterruptedException { + System.setOut(standardOutOrig); + ClusterControllerSettings clusterControllerSettings = + ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); + + try (ClusterControllerClient clusterControllerClient = + ClusterControllerClient.create(clusterControllerSettings)) { + OperationFuture deleteClusterAsyncRequest = + clusterControllerClient.deleteClusterAsync(projectId, REGION, clusterName); + deleteClusterAsyncRequest.get(); + } catch (ExecutionException e) { + System.out.println("[deleteCluster] Error during test cluster creation: \n" + e.toString()); + } + } +} diff --git a/dataproc/src/test/java/SubmitJobTest.java b/dataproc/src/test/java/SubmitJobTest.java new file mode 100644 index 00000000000..cc6413e02c8 --- /dev/null +++ b/dataproc/src/test/java/SubmitJobTest.java @@ -0,0 +1,135 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import static java.nio.charset.StandardCharsets.UTF_8; +import static junit.framework.TestCase.assertNotNull; +import static org.hamcrest.MatcherAssert.assertThat; + +import com.google.api.gax.longrunning.OperationFuture; +import com.google.cloud.dataproc.v1.Cluster; +import com.google.cloud.dataproc.v1.ClusterConfig; +import com.google.cloud.dataproc.v1.ClusterControllerClient; +import com.google.cloud.dataproc.v1.ClusterControllerSettings; +import com.google.cloud.dataproc.v1.ClusterOperationMetadata; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.Bucket; +import com.google.cloud.storage.BucketInfo; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageOptions; +import com.google.protobuf.Empty; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import org.hamcrest.CoreMatchers; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class SubmitJobTest { + + private static final String MY_UUID = UUID.randomUUID().toString(); + private static final String REGION = "us-central1"; + private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); + private static final String ENDPOINT = String.format("%s-dataproc.googleapis.com:443", REGION); + private static final String CLUSTER_NAME = String.format("%s-%s", "test-cluster", MY_UUID); + private static final String BUCKET_NAME = String.format("%s-%s", "test-bucket", MY_UUID); + private static final String JOB_FILE_NAME = "sum.py"; + private static final String JOB_FILE_PATH = + String.format("gs://%s/%s", BUCKET_NAME, JOB_FILE_NAME); + private static final String SORT_CODE = + "import pyspark\n" + + "sc = pyspark.SparkContext()\n" + + "rdd = sc.parallelize((1,2,3,4,5))\n" + + "sum = rdd.reduce(lambda x, y: x + y)\n"; + + private ByteArrayOutputStream bout; + private PrintStream standardOutOrig; + private Storage storage; + private Blob blob; + + private static void requireEnv(String varName) { + assertNotNull( + System.getenv(varName), + String.format("Environment variable '%s' is required to perform these tests.", varName)); + } + + @BeforeClass + public static void checkRequirements() { + requireEnv("GOOGLE_APPLICATION_CREDENTIALS"); + requireEnv("GOOGLE_CLOUD_PROJECT"); + } + + @Before + public void setUp() throws IOException, InterruptedException { + bout = new ByteArrayOutputStream(); + standardOutOrig = System.out; + System.setOut(new PrintStream(bout)); + + storage = StorageOptions.getDefaultInstance().getService(); + Bucket bucket = storage.create(BucketInfo.of(BUCKET_NAME)); + blob = bucket.create(JOB_FILE_NAME, SORT_CODE.getBytes(UTF_8), "text/plain"); + + ClusterControllerSettings clusterControllerSettings = + ClusterControllerSettings.newBuilder().setEndpoint(ENDPOINT).build(); + + Cluster cluster = + Cluster.newBuilder() + .setClusterName(CLUSTER_NAME) + .setConfig(ClusterConfig.newBuilder().build()) + .build(); + + try (ClusterControllerClient clusterControllerClient = + ClusterControllerClient.create(clusterControllerSettings)) { + OperationFuture createClusterAsyncRequest = + clusterControllerClient.createClusterAsync(PROJECT_ID, REGION, cluster); + createClusterAsyncRequest.get(); + } catch (ExecutionException e) { + System.out.println("[SubmitJob] Error during test cluster creation: \n" + e.toString()); + } + } + + @Test + public void submitJobTest() throws IOException { + SubmitJob.submitJob(PROJECT_ID, REGION, CLUSTER_NAME, JOB_FILE_PATH); + String output = bout.toString(); + + assertThat(output, CoreMatchers.containsString("finished")); + } + + @After + public void teardown() throws IOException, InterruptedException { + System.setOut(standardOutOrig); + blob.delete(); + + ClusterControllerSettings clusterControllerSettings = + ClusterControllerSettings.newBuilder().setEndpoint(ENDPOINT).build(); + + try (ClusterControllerClient clusterControllerClient = + ClusterControllerClient.create(clusterControllerSettings)) { + OperationFuture deleteClusterAsyncRequest = + clusterControllerClient.deleteClusterAsync(PROJECT_ID, REGION, CLUSTER_NAME); + deleteClusterAsyncRequest.get(); + } catch (ExecutionException e) { + System.out.println("[SubmitJob] Error during test cluster deletion: \n" + e.toString()); + } + } +} diff --git a/dataproc/src/test/java/UpdateClusterTest.java b/dataproc/src/test/java/UpdateClusterTest.java new file mode 100644 index 00000000000..7b19fdc1ebf --- /dev/null +++ b/dataproc/src/test/java/UpdateClusterTest.java @@ -0,0 +1,130 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import static junit.framework.TestCase.assertNotNull; +import static org.hamcrest.MatcherAssert.assertThat; + +import com.google.api.gax.longrunning.OperationFuture; +import com.google.cloud.dataproc.v1.Cluster; +import com.google.cloud.dataproc.v1.ClusterConfig; +import com.google.cloud.dataproc.v1.ClusterControllerClient; +import com.google.cloud.dataproc.v1.ClusterControllerSettings; +import com.google.cloud.dataproc.v1.ClusterOperationMetadata; +import com.google.cloud.dataproc.v1.InstanceGroupConfig; +import com.google.protobuf.Empty; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import org.hamcrest.CoreMatchers; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class UpdateClusterTest { + + private static final String MY_UUID = UUID.randomUUID().toString(); + private static final String REGION = "us-central1"; + private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); + private static final String ENDPOINT = String.format("%s-dataproc.googleapis.com:443", REGION); + private static final String CLUSTER_NAME = String.format("%s-%s", "test-cluster", MY_UUID); + private static final int NUM_WORKERS = 2; + private static final int NEW_WORKERS = NUM_WORKERS * 2; + + private ByteArrayOutputStream bout; + private PrintStream standardOutOrig; + + private static void requireEnv(String varName) { + assertNotNull( + System.getenv(varName), + String.format("Environment variable '%s' is required to perform these tests.", varName)); + } + + @BeforeClass + public static void checkRequirements() { + requireEnv("GOOGLE_APPLICATION_CREDENTIALS"); + requireEnv("GOOGLE_CLOUD_PROJECT"); + } + + @Before + public void setUp() throws IOException, InterruptedException { + bout = new ByteArrayOutputStream(); + standardOutOrig = System.out; + System.setOut(new PrintStream(bout)); + + ClusterControllerSettings clusterControllerSettings = + ClusterControllerSettings.newBuilder().setEndpoint(ENDPOINT).build(); + + // Configure the settings for our cluster + InstanceGroupConfig masterConfig = + InstanceGroupConfig.newBuilder() + .setMachineTypeUri("n1-standard-1") + .setNumInstances(1) + .build(); + InstanceGroupConfig workerConfig = + InstanceGroupConfig.newBuilder() + .setMachineTypeUri("n1-standard-1") + .setNumInstances(NUM_WORKERS) + .build(); + ClusterConfig clusterConfig = + ClusterConfig.newBuilder() + .setMasterConfig(masterConfig) + .setWorkerConfig(workerConfig) + .build(); + // Create the cluster object with the desired cluster config + Cluster cluster = + Cluster.newBuilder().setClusterName(CLUSTER_NAME).setConfig(clusterConfig).build(); + + try (ClusterControllerClient clusterControllerClient = + ClusterControllerClient.create(clusterControllerSettings)) { + OperationFuture createClusterAsyncRequest = + clusterControllerClient.createClusterAsync(PROJECT_ID, REGION, cluster); + createClusterAsyncRequest.get(); + } catch (ExecutionException e) { + System.out.println("[UpdateCluster] Error during test cluster creation: \n" + e.toString()); + } + } + + @Test + public void updateClusterTest() throws IOException, InterruptedException { + UpdateCluster.updateCluster(PROJECT_ID, REGION, CLUSTER_NAME, NEW_WORKERS); + String output = bout.toString(); + + assertThat(output, CoreMatchers.containsString(String.format("%d workers", NEW_WORKERS))); + } + + @After + public void teardown() throws IOException, InterruptedException { + System.setOut(standardOutOrig); + + ClusterControllerSettings clusterControllerSettings = + ClusterControllerSettings.newBuilder().setEndpoint(ENDPOINT).build(); + + try (ClusterControllerClient clusterControllerClient = + ClusterControllerClient.create(clusterControllerSettings)) { + OperationFuture deleteClusterAsyncRequest = + clusterControllerClient.deleteClusterAsync(PROJECT_ID, REGION, CLUSTER_NAME); + deleteClusterAsyncRequest.get(); + } catch (ExecutionException e) { + System.out.println("[UpdateCluster] Error during test cluster update: \n" + e.toString()); + } + } +} From 0c132cdcbeabea169b6d32811da461cf80f925d9 Mon Sep 17 00:00:00 2001 From: bradmiro Date: Mon, 2 Dec 2019 14:37:20 -0500 Subject: [PATCH 02/21] Fixed linting, string formatting, copyrights --- dataproc/src/main/java/CreateCluster.java | 17 ++++++------ dataproc/src/main/java/DeleteCluster.java | 16 +++++------ dataproc/src/main/java/ListClusters.java | 17 ++++++------ dataproc/src/main/java/SubmitJob.java | 27 +++++++++---------- dataproc/src/main/java/UpdateCluster.java | 21 +++++++-------- dataproc/src/test/java/CreateClusterTest.java | 20 +++++++------- dataproc/src/test/java/DeleteClusterTest.java | 27 +++++++++++-------- dataproc/src/test/java/ListClustersTest.java | 25 +++++++++-------- dataproc/src/test/java/SubmitJobTest.java | 13 +++++---- dataproc/src/test/java/UpdateClusterTest.java | 9 +++---- 10 files changed, 95 insertions(+), 97 deletions(-) diff --git a/dataproc/src/main/java/CreateCluster.java b/dataproc/src/main/java/CreateCluster.java index ed2f507dc6b..ebae59dd53c 100644 --- a/dataproc/src/main/java/CreateCluster.java +++ b/dataproc/src/main/java/CreateCluster.java @@ -1,5 +1,5 @@ /* - * Copyright 2019 Google Inc. + * Copyright 2019 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,15 +29,15 @@ public class CreateCluster { public static void createCluster(String projectId, String region, String clusterName) throws IOException, InterruptedException { - String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); + String myEndpoint = region + "-dataproc.googleapis.com:443"; // Configure the settings for the cluster controller client ClusterControllerSettings clusterControllerSettings = ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); - // Create a cluster controller client with the configured settings. We only need to create - // the client once, and can be reused for multiple requests. Using a try-with-resources - // will close the client for us, but this can also be done manually with the .close() method. + // Create a cluster controller client with the configured settings. The client only needs to be + // created once and can be reused for multiple requests. Using a try-with-resources + // closes the client, but this can also be done manually with the .close() method. try (ClusterControllerClient clusterControllerClient = ClusterControllerClient.create(clusterControllerSettings)) { // Configure the settings for our cluster @@ -66,17 +66,16 @@ public static void createCluster(String projectId, String region, String cluster Cluster response = createClusterAsyncRequest.get(); // Print out a success message - System.out.println( - String.format("Cluster created successfully: %s", response.getClusterName())); + System.out.println("Cluster created successfully: " + response.getClusterName()); } catch (IOException e) { // Likely this would occur due to issues authenticating with GCP. Make sure the environment // variable GOOGLE_APPLICATION_CREDENTIALS is configured. - System.out.println("Error creating the cluster controller client: \n" + e.toString()); + System.err.println("Error creating the cluster controller client: \n" + e.getMessage()); } catch (ExecutionException e) { // Common issues for this include needing to increase compute engine quotas or a cluster of // the same name already exists. - System.out.println("Error during cluster creation request: \n" + e.toString()); + System.err.println("Error during cluster creation request: \n" + e.getMessage()); } } } diff --git a/dataproc/src/main/java/DeleteCluster.java b/dataproc/src/main/java/DeleteCluster.java index f94761179c2..84736b773cf 100644 --- a/dataproc/src/main/java/DeleteCluster.java +++ b/dataproc/src/main/java/DeleteCluster.java @@ -1,5 +1,5 @@ /* - * Copyright 2019 Google Inc. + * Copyright 2019 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,15 +27,15 @@ public class DeleteCluster { public static void deleteCluster(String projectId, String region, String clusterName) throws IOException, InterruptedException { - String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); + String myEndpoint = region + "-dataproc.googleapis.com:443"; // Configure the settings for the cluster controller client ClusterControllerSettings clusterControllerSettings = ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); - // Create a cluster controller client with the configured settings. We only need to create - // the client once, and can be reused for multiple requests. Using a try-with-resources - // will close the client for us, but this can also be done manually with the .close() method. + // Create a cluster controller client with the configured settings. The client only needs to be + // created once and can be reused for multiple requests. Using a try-with-resources + // closes the client, but this can also be done manually with the .close() method. try (ClusterControllerClient clusterControllerClient = ClusterControllerClient.create(clusterControllerSettings)) { // Configure the settings for our cluster @@ -46,15 +46,15 @@ public static void deleteCluster(String projectId, String region, String cluster deleteClusterAsyncRequest.get(); // Print out a success message - System.out.println(String.format("Cluster deleted successfully: %s", clusterName)); + System.out.println("Cluster deleted successfully: " + clusterName); } catch (IOException e) { // Likely this would occur due to issues authenticating with GCP. Make sure the environment // variable GOOGLE_APPLICATION_CREDENTIALS is configured. - System.out.println("Error deleting the cluster controller client: \n" + e.toString()); + System.err.println("Error deleting the cluster controller client: \n" + e.getMessage()); } catch (ExecutionException e) { // This will likely be due to a cluster of the given name not existing in the given region. - System.out.println("Error during cluster deletion request: \n" + e.toString()); + System.err.println("Error during cluster deletion request: \n" + e.getMessage()); } } } diff --git a/dataproc/src/main/java/ListClusters.java b/dataproc/src/main/java/ListClusters.java index de8483bbc06..b0b0a68b905 100644 --- a/dataproc/src/main/java/ListClusters.java +++ b/dataproc/src/main/java/ListClusters.java @@ -1,5 +1,5 @@ /* - * Copyright 2019 Google Inc. + * Copyright 2019 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,27 +22,26 @@ public class ListClusters { - public static void ListClusters(String projectId, String region) throws IOException { - String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); + public static void listClusters(String projectId, String region) throws IOException { + String myEndpoint = region + "-dataproc.googleapis.com:443"; // Configure the settings for the cluster controller client ClusterControllerSettings clusterControllerSettings = ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); - // Create a cluster controller client with the configured settings. We only need to create - // the client once, and can be reused for multiple requests. Using a try-with-resources - // will close the client for us, but this can also be done manually with the .close() method. + // Create a cluster controller client with the configured settings. The client only needs to be + // created once and can be reused for multiple requests. Using a try-with-resources + // closes the client, but this can also be done manually with the .close() method. try (ClusterControllerClient clusterControllerClient = ClusterControllerClient.create(clusterControllerSettings)) { for (Cluster element : clusterControllerClient.listClusters(projectId, region).iterateAll()) { - System.out.println( - String.format("%s-%s", element.getClusterName(), element.getStatus().getState())); + System.out.println(element.getClusterName() + ": " + element.getStatus().getState()); } } catch (IOException e) { // Likely this would occur due to issues authenticating with GCP. Make sure the environment // variable GOOGLE_APPLICATION_CREDENTIALS is configured. - System.out.println("Error deleting the cluster controller client: \n" + e.toString()); + System.out.println("Error deleting the cluster controller client: \n" + e.getMessage()); } } } diff --git a/dataproc/src/main/java/SubmitJob.java b/dataproc/src/main/java/SubmitJob.java index 506ac243960..01bce05569b 100644 --- a/dataproc/src/main/java/SubmitJob.java +++ b/dataproc/src/main/java/SubmitJob.java @@ -1,5 +1,5 @@ /* - * Copyright 2019 Google Inc. + * Copyright 2019 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,29 +19,29 @@ import com.google.cloud.dataproc.v1.JobControllerClient; import com.google.cloud.dataproc.v1.JobControllerSettings; import com.google.cloud.dataproc.v1.JobPlacement; +import com.google.cloud.dataproc.v1.JobStatus; import com.google.cloud.dataproc.v1.PySparkJob; import java.io.IOException; public class SubmitJob { public static void submitJob( - String projectId, String region, String clusterName, String job_file_path) - throws IOException { - String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); + String projectId, String region, String clusterName, String jobFilePath) throws IOException { + String myEndpoint = region + "-dataproc.googleapis.com:443"; // Configure the settings for the cluster controller client JobControllerSettings jobControllerSettings = JobControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); - // Create a cluster controller client with the configured settings. We only need to create - // the client once, and can be reused for multiple requests. Using a try-with-resources - // will close the client for us, but this can also be done manually with the .close() method. + // Create a job controller client with the configured settings. The client only needs to be + // created once and can be reused for multiple requests. Using a try-with-resources + // closes the client, but this can also be done manually with the .close() method. try (JobControllerClient jobControllerClient = JobControllerClient.create(jobControllerSettings)) { // Configure the settings for our job JobPlacement jobPlacement = JobPlacement.newBuilder().setClusterName(clusterName).build(); - PySparkJob pySparkJob = PySparkJob.newBuilder().setMainPythonFileUri(job_file_path).build(); + PySparkJob pySparkJob = PySparkJob.newBuilder().setMainPythonFileUri(jobFilePath).build(); Job job = Job.newBuilder().setPlacement(jobPlacement).setPysparkJob(pySparkJob).build(); Job jobOperation = jobControllerClient.submitJob(projectId, region, job); @@ -50,19 +50,18 @@ public static void submitJob( while (true) { Job jobInfo = jobControllerClient.getJob(projectId, region, jobId); - if (jobInfo.getStatus().getState().toString() == "ERROR") { - System.out.println( - String.format("Job %s failed: %s", jobId, jobInfo.getStatus().getDetails())); + if (jobInfo.getStatus().getState().equals(JobStatus.State.ERROR)) { + System.err.printf("Job %s failed: %s", jobId, jobInfo.getStatus().getDetails()); break; - } else if (jobInfo.getStatus().getState().toString() == "DONE") { - System.out.println(String.format("Job %s finished.", jobId)); + } else if (jobInfo.getStatus().getState().equals(JobStatus.State.DONE)) { + System.out.printf("Job %s finished.", jobId); break; } } } catch (IOException e) { // Likely this would occur due to issues authenticating with GCP. Make sure the environment // variable GOOGLE_APPLICATION_CREDENTIALS is configured. - System.out.println("Error creating the cluster controller client: \n" + e.toString()); + System.err.println("Error creating the cluster controller client: \n" + e.getMessage()); } } } diff --git a/dataproc/src/main/java/UpdateCluster.java b/dataproc/src/main/java/UpdateCluster.java index 3727c8f47de..6cfba7bcace 100644 --- a/dataproc/src/main/java/UpdateCluster.java +++ b/dataproc/src/main/java/UpdateCluster.java @@ -1,5 +1,5 @@ /* - * Copyright 2019 Google Inc. + * Copyright 2019 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,15 +31,15 @@ public class UpdateCluster { public static void updateCluster( String projectId, String region, String clusterName, int numWorkers) throws IOException, InterruptedException { - String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); + String myEndpoint = region + "-dataproc.googleapis.com:443"; // Configure the settings for the cluster controller client ClusterControllerSettings clusterControllerSettings = ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); - // Create a cluster controller client with the configured settings. We only need to create - // the client once, and can be reused for multiple requests. Using a try-with-resources - // will close the client for us, but this can also be done manually with the .close() method. + // Create a cluster controller client with the configured settings. The client only needs to be + // created once and can be reused for multiple requests. Using a try-with-resources + // closes the client, but this can also be done manually with the .close() method. try (ClusterControllerClient clusterControllerClient = ClusterControllerClient.create(clusterControllerSettings)) { // Configure the settings for our cluster @@ -73,19 +73,18 @@ public static void updateCluster( Cluster response = updateClusterAsyncRequest.get(); // Print out a success message - System.out.println( - String.format( - "Cluster %s now has %d workers.", - response.getClusterName(), response.getConfig().getWorkerConfig().getNumInstances())); + System.out.printf( + "Cluster %s now has %d workers.", + response.getClusterName(), response.getConfig().getWorkerConfig().getNumInstances()); } catch (IOException e) { // Likely this would occur due to issues authenticating with GCP. Make sure the environment // variable GOOGLE_APPLICATION_CREDENTIALS is configured. - System.out.println("Error creating the cluster controller client: \n" + e.toString()); + System.err.println("Error creating the cluster controller client: \n" + e.getMessage()); } catch (ExecutionException e) { // Common issues for this include needing to increase compute engine quotas or a cluster of // the same name already exists. - System.out.println("Error during cluster creation request: \n" + e.toString()); + System.out.println("Error during cluster creation request: \n" + e.getMessage()); } } } diff --git a/dataproc/src/test/java/CreateClusterTest.java b/dataproc/src/test/java/CreateClusterTest.java index f98c24a9b4d..fc661e371d3 100644 --- a/dataproc/src/test/java/CreateClusterTest.java +++ b/dataproc/src/test/java/CreateClusterTest.java @@ -38,12 +38,12 @@ @RunWith(JUnit4.class) public class CreateClusterTest { - private static final String BASE_CLUSTER_NAME = "test-cluster"; + private static final String CLUSTER_NAME = "test-cluster-" + UUID.randomUUID().toString(); private static final String REGION = "us-central1"; + private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); - private static String projectId = System.getenv("GOOGLE_CLOUD_PROJECT"); - private String clusterName; private ByteArrayOutputStream bout; + private PrintStream standardOutOrig; private static void requireEnv(String varName) { assertNotNull( @@ -59,23 +59,23 @@ public static void checkRequirements() { @Before public void setUp() { - clusterName = String.format("%s-%s", BASE_CLUSTER_NAME, UUID.randomUUID().toString()); - bout = new ByteArrayOutputStream(); System.setOut(new PrintStream(bout)); + standardOutOrig = System.out; } @Test public void createClusterTest() throws IOException, InterruptedException { - CreateCluster.createCluster(projectId, REGION, clusterName); + CreateCluster.createCluster(PROJECT_ID, REGION, CLUSTER_NAME); String output = bout.toString(); - assertThat(output, CoreMatchers.containsString(clusterName)); + assertThat(output, CoreMatchers.containsString(CLUSTER_NAME)); } @After public void tearDown() throws IOException, InterruptedException { - String myEndpoint = String.format("%s-dataproc.googleapis.com:443", REGION); + String myEndpoint = REGION + "-dataproc.googleapis.com:443"; + System.setOut(standardOutOrig); ClusterControllerSettings clusterControllerSettings = ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); @@ -83,11 +83,11 @@ public void tearDown() throws IOException, InterruptedException { try (ClusterControllerClient clusterControllerClient = ClusterControllerClient.create(clusterControllerSettings)) { OperationFuture deleteClusterAsyncRequest = - clusterControllerClient.deleteClusterAsync(projectId, REGION, clusterName); + clusterControllerClient.deleteClusterAsync(PROJECT_ID, REGION, CLUSTER_NAME); deleteClusterAsyncRequest.get(); } catch (ExecutionException e) { - System.out.println("Error during cluster deletion: \n" + e.toString()); + System.err.println("Error during cluster deletion: \n" + e.getMessage()); } } } diff --git a/dataproc/src/test/java/DeleteClusterTest.java b/dataproc/src/test/java/DeleteClusterTest.java index 82d4c453e6e..0ec6797463c 100644 --- a/dataproc/src/test/java/DeleteClusterTest.java +++ b/dataproc/src/test/java/DeleteClusterTest.java @@ -29,6 +29,7 @@ import java.util.UUID; import java.util.concurrent.ExecutionException; import org.hamcrest.CoreMatchers; +import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -38,12 +39,12 @@ @RunWith(JUnit4.class) public class DeleteClusterTest { - private static final String BASE_CLUSTER_NAME = "test-cluster"; + private static final String CLUSTER_NAME = "test-cluster-" + UUID.randomUUID().toString(); private static final String REGION = "us-central1"; + private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); - private static String projectId = System.getenv("GOOGLE_CLOUD_PROJECT"); - private String clusterName; private ByteArrayOutputStream bout; + private PrintStream standardOutOrig; private static void requireEnv(String varName) { assertNotNull( @@ -59,37 +60,41 @@ public static void checkRequirements() { @Before public void setUp() throws IOException, InterruptedException { - clusterName = String.format("%s-%s", BASE_CLUSTER_NAME, UUID.randomUUID().toString()); - bout = new ByteArrayOutputStream(); System.setOut(new PrintStream(bout)); + standardOutOrig = System.out; - String myEndpoint = String.format("%s-dataproc.googleapis.com:443", REGION); + String myEndpoint = REGION + "-dataproc.googleapis.com:443"; ClusterControllerSettings clusterControllerSettings = ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); Cluster cluster = Cluster.newBuilder() - .setClusterName(clusterName) + .setClusterName(CLUSTER_NAME) .setConfig(ClusterConfig.newBuilder().build()) .build(); try (ClusterControllerClient clusterControllerClient = ClusterControllerClient.create(clusterControllerSettings)) { OperationFuture createClusterAsyncRequest = - clusterControllerClient.createClusterAsync(projectId, REGION, cluster); + clusterControllerClient.createClusterAsync(PROJECT_ID, REGION, cluster); createClusterAsyncRequest.get(); } catch (ExecutionException e) { - System.out.println("[deleteCluster] Error during test cluster creation: \n" + e.toString()); + System.err.println("[deleteCluster] Error during test cluster creation: \n" + e.getMessage()); } } @Test - public void DeleteClusterTest() throws IOException, InterruptedException { - DeleteCluster.deleteCluster(projectId, REGION, clusterName); + public void deleteClusterTest() throws IOException, InterruptedException { + DeleteCluster.deleteCluster(PROJECT_ID, REGION, CLUSTER_NAME); String output = bout.toString(); assertThat(output, CoreMatchers.containsString("deleted successfully")); } + + @After + public void teardown() { + System.setOut(standardOutOrig); + } } diff --git a/dataproc/src/test/java/ListClustersTest.java b/dataproc/src/test/java/ListClustersTest.java index 067dfed0a6e..4d0272ea797 100644 --- a/dataproc/src/test/java/ListClustersTest.java +++ b/dataproc/src/test/java/ListClustersTest.java @@ -40,11 +40,10 @@ @RunWith(JUnit4.class) public class ListClustersTest { - private static final String BASE_CLUSTER_NAME = "test-cluster"; + private static final String CLUSTER_NAME = "test-cluster-" + UUID.randomUUID().toString(); private static final String REGION = "us-central1"; - private static final String projectId = System.getenv("GOOGLE_CLOUD_PROJECT"); + private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); - private String clusterName; private String myEndpoint; private ByteArrayOutputStream bout; private PrintStream standardOutOrig; @@ -63,53 +62,53 @@ public static void checkRequirements() { @Before public void setUp() throws IOException, InterruptedException { - clusterName = String.format("%s-%s", BASE_CLUSTER_NAME, UUID.randomUUID().toString()); bout = new ByteArrayOutputStream(); standardOutOrig = System.out; System.setOut(new PrintStream(bout)); - myEndpoint = String.format("%s-dataproc.googleapis.com:443", REGION); + myEndpoint = REGION + "-dataproc.googleapis.com:443"; ClusterControllerSettings clusterControllerSettings = ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); Cluster cluster = Cluster.newBuilder() - .setClusterName(clusterName) + .setClusterName(CLUSTER_NAME) .setConfig(ClusterConfig.newBuilder().build()) .build(); try (ClusterControllerClient clusterControllerClient = ClusterControllerClient.create(clusterControllerSettings)) { OperationFuture createClusterAsyncRequest = - clusterControllerClient.createClusterAsync(projectId, REGION, cluster); + clusterControllerClient.createClusterAsync(PROJECT_ID, REGION, cluster); createClusterAsyncRequest.get(); } catch (ExecutionException e) { - System.out.println("[deleteCluster] Error during test cluster creation: \n" + e.toString()); + System.err.println("[deleteCluster] Error during test cluster creation: \n" + e.getMessage()); } } @Test - public void ListClustersTest() throws IOException, InterruptedException { - ListClusters.ListClusters(projectId, REGION); + public void listClustersTest() throws IOException { + ListClusters.listClusters(PROJECT_ID, REGION); String output = bout.toString(); - assertThat(output, CoreMatchers.containsString(clusterName)); + assertThat(output, CoreMatchers.containsString(CLUSTER_NAME)); assertThat(output, CoreMatchers.containsString("RUNNING")); } @After public void teardown() throws IOException, InterruptedException { System.setOut(standardOutOrig); + ClusterControllerSettings clusterControllerSettings = ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); try (ClusterControllerClient clusterControllerClient = ClusterControllerClient.create(clusterControllerSettings)) { OperationFuture deleteClusterAsyncRequest = - clusterControllerClient.deleteClusterAsync(projectId, REGION, clusterName); + clusterControllerClient.deleteClusterAsync(PROJECT_ID, REGION, CLUSTER_NAME); deleteClusterAsyncRequest.get(); } catch (ExecutionException e) { - System.out.println("[deleteCluster] Error during test cluster creation: \n" + e.toString()); + System.err.println("[deleteCluster] Error during test cluster creation: \n" + e.getMessage()); } } } diff --git a/dataproc/src/test/java/SubmitJobTest.java b/dataproc/src/test/java/SubmitJobTest.java index cc6413e02c8..ab4cd1139cc 100644 --- a/dataproc/src/test/java/SubmitJobTest.java +++ b/dataproc/src/test/java/SubmitJobTest.java @@ -49,9 +49,9 @@ public class SubmitJobTest { private static final String MY_UUID = UUID.randomUUID().toString(); private static final String REGION = "us-central1"; private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); - private static final String ENDPOINT = String.format("%s-dataproc.googleapis.com:443", REGION); - private static final String CLUSTER_NAME = String.format("%s-%s", "test-cluster", MY_UUID); - private static final String BUCKET_NAME = String.format("%s-%s", "test-bucket", MY_UUID); + private static final String ENDPOINT = REGION + "-dataproc.googleapis.com:443"; + private static final String CLUSTER_NAME = "test-cluster-" + MY_UUID; + private static final String BUCKET_NAME = "test-bucket-" + MY_UUID; private static final String JOB_FILE_NAME = "sum.py"; private static final String JOB_FILE_PATH = String.format("gs://%s/%s", BUCKET_NAME, JOB_FILE_NAME); @@ -63,7 +63,6 @@ public class SubmitJobTest { private ByteArrayOutputStream bout; private PrintStream standardOutOrig; - private Storage storage; private Blob blob; private static void requireEnv(String varName) { @@ -84,7 +83,7 @@ public void setUp() throws IOException, InterruptedException { standardOutOrig = System.out; System.setOut(new PrintStream(bout)); - storage = StorageOptions.getDefaultInstance().getService(); + Storage storage = StorageOptions.getDefaultInstance().getService(); Bucket bucket = storage.create(BucketInfo.of(BUCKET_NAME)); blob = bucket.create(JOB_FILE_NAME, SORT_CODE.getBytes(UTF_8), "text/plain"); @@ -103,7 +102,7 @@ public void setUp() throws IOException, InterruptedException { clusterControllerClient.createClusterAsync(PROJECT_ID, REGION, cluster); createClusterAsyncRequest.get(); } catch (ExecutionException e) { - System.out.println("[SubmitJob] Error during test cluster creation: \n" + e.toString()); + System.err.println("[SubmitJob] Error during test cluster creation: \n" + e.getMessage()); } } @@ -129,7 +128,7 @@ public void teardown() throws IOException, InterruptedException { clusterControllerClient.deleteClusterAsync(PROJECT_ID, REGION, CLUSTER_NAME); deleteClusterAsyncRequest.get(); } catch (ExecutionException e) { - System.out.println("[SubmitJob] Error during test cluster deletion: \n" + e.toString()); + System.err.println("[SubmitJob] Error during test cluster deletion: \n" + e.getMessage()); } } } diff --git a/dataproc/src/test/java/UpdateClusterTest.java b/dataproc/src/test/java/UpdateClusterTest.java index 7b19fdc1ebf..8eb12ec26e1 100644 --- a/dataproc/src/test/java/UpdateClusterTest.java +++ b/dataproc/src/test/java/UpdateClusterTest.java @@ -41,11 +41,10 @@ @RunWith(JUnit4.class) public class UpdateClusterTest { - private static final String MY_UUID = UUID.randomUUID().toString(); private static final String REGION = "us-central1"; private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); - private static final String ENDPOINT = String.format("%s-dataproc.googleapis.com:443", REGION); - private static final String CLUSTER_NAME = String.format("%s-%s", "test-cluster", MY_UUID); + private static final String ENDPOINT = REGION + "-dataproc.googleapis.com:443"; + private static final String CLUSTER_NAME = "test-cluster-" + UUID.randomUUID().toString(); private static final int NUM_WORKERS = 2; private static final int NEW_WORKERS = NUM_WORKERS * 2; @@ -99,7 +98,7 @@ public void setUp() throws IOException, InterruptedException { clusterControllerClient.createClusterAsync(PROJECT_ID, REGION, cluster); createClusterAsyncRequest.get(); } catch (ExecutionException e) { - System.out.println("[UpdateCluster] Error during test cluster creation: \n" + e.toString()); + System.err.println("[UpdateCluster] Error during test cluster creation: \n" + e.getMessage()); } } @@ -124,7 +123,7 @@ public void teardown() throws IOException, InterruptedException { clusterControllerClient.deleteClusterAsync(PROJECT_ID, REGION, CLUSTER_NAME); deleteClusterAsyncRequest.get(); } catch (ExecutionException e) { - System.out.println("[UpdateCluster] Error during test cluster update: \n" + e.toString()); + System.err.println("[UpdateCluster] Error during test cluster update: \n" + e.getMessage()); } } } From c242a0cd0caef77b348c3a124f99d64d14f1efa6 Mon Sep 17 00:00:00 2001 From: bradmiro Date: Mon, 2 Dec 2019 15:16:44 -0500 Subject: [PATCH 03/21] added overloaded functions to all samples --- dataproc/src/main/java/CreateCluster.java | 23 ++++++++++--------- dataproc/src/main/java/DeleteCluster.java | 23 ++++++++++--------- dataproc/src/main/java/ListClusters.java | 17 +++++++++----- dataproc/src/main/java/SubmitJob.java | 19 ++++++++++----- dataproc/src/main/java/UpdateCluster.java | 23 ++++++++++--------- dataproc/src/test/java/CreateClusterTest.java | 6 ++--- dataproc/src/test/java/DeleteClusterTest.java | 2 +- dataproc/src/test/java/ListClustersTest.java | 4 +--- dataproc/src/test/java/SubmitJobTest.java | 4 +--- dataproc/src/test/java/UpdateClusterTest.java | 6 ++--- 10 files changed, 67 insertions(+), 60 deletions(-) diff --git a/dataproc/src/main/java/CreateCluster.java b/dataproc/src/main/java/CreateCluster.java index ebae59dd53c..02365e8661f 100644 --- a/dataproc/src/main/java/CreateCluster.java +++ b/dataproc/src/main/java/CreateCluster.java @@ -27,9 +27,18 @@ public class CreateCluster { + public static void createCluster() + throws IOException, InterruptedException, ExecutionException { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String region = "your-project-region"; + String clusterName = "your-cluster-name"; + createCluster(projectId, region, clusterName); + } + public static void createCluster(String projectId, String region, String clusterName) - throws IOException, InterruptedException { - String myEndpoint = region + "-dataproc.googleapis.com:443"; + throws IOException, InterruptedException, ExecutionException { + String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); // Configure the settings for the cluster controller client ClusterControllerSettings clusterControllerSettings = @@ -66,16 +75,8 @@ public static void createCluster(String projectId, String region, String cluster Cluster response = createClusterAsyncRequest.get(); // Print out a success message - System.out.println("Cluster created successfully: " + response.getClusterName()); + System.out.printf("Cluster created successfully: %s", response.getClusterName()); - } catch (IOException e) { - // Likely this would occur due to issues authenticating with GCP. Make sure the environment - // variable GOOGLE_APPLICATION_CREDENTIALS is configured. - System.err.println("Error creating the cluster controller client: \n" + e.getMessage()); - } catch (ExecutionException e) { - // Common issues for this include needing to increase compute engine quotas or a cluster of - // the same name already exists. - System.err.println("Error during cluster creation request: \n" + e.getMessage()); } } } diff --git a/dataproc/src/main/java/DeleteCluster.java b/dataproc/src/main/java/DeleteCluster.java index 84736b773cf..4bdb5aa70a5 100644 --- a/dataproc/src/main/java/DeleteCluster.java +++ b/dataproc/src/main/java/DeleteCluster.java @@ -25,9 +25,18 @@ public class DeleteCluster { + public static void deleteCluster() + throws IOException, InterruptedException, ExecutionException { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String region = "your-project-region"; + String clusterName = "your-cluster-name"; + deleteCluster(projectId, region, clusterName); + } + public static void deleteCluster(String projectId, String region, String clusterName) - throws IOException, InterruptedException { - String myEndpoint = region + "-dataproc.googleapis.com:443"; + throws IOException, InterruptedException, ExecutionException { + String myEndpoint = String.format("-dataproc.googleapis.com:443", region); // Configure the settings for the cluster controller client ClusterControllerSettings clusterControllerSettings = @@ -46,15 +55,7 @@ public static void deleteCluster(String projectId, String region, String cluster deleteClusterAsyncRequest.get(); // Print out a success message - System.out.println("Cluster deleted successfully: " + clusterName); - - } catch (IOException e) { - // Likely this would occur due to issues authenticating with GCP. Make sure the environment - // variable GOOGLE_APPLICATION_CREDENTIALS is configured. - System.err.println("Error deleting the cluster controller client: \n" + e.getMessage()); - } catch (ExecutionException e) { - // This will likely be due to a cluster of the given name not existing in the given region. - System.err.println("Error during cluster deletion request: \n" + e.getMessage()); + System.out.printf("Cluster deleted successfully: %s", clusterName); } } } diff --git a/dataproc/src/main/java/ListClusters.java b/dataproc/src/main/java/ListClusters.java index b0b0a68b905..6b35a18e4f8 100644 --- a/dataproc/src/main/java/ListClusters.java +++ b/dataproc/src/main/java/ListClusters.java @@ -19,11 +19,20 @@ import com.google.cloud.dataproc.v1.ClusterControllerClient; import com.google.cloud.dataproc.v1.ClusterControllerSettings; import java.io.IOException; +import java.util.concurrent.ExecutionException; public class ListClusters { + public static void listClusters() + throws IOException, InterruptedException, ExecutionException { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String region = "your-project-region"; + listClusters(projectId, region); + } + public static void listClusters(String projectId, String region) throws IOException { - String myEndpoint = region + "-dataproc.googleapis.com:443"; + String myEndpoint = String.format("-dataproc.googleapis.com:443"); // Configure the settings for the cluster controller client ClusterControllerSettings clusterControllerSettings = @@ -36,12 +45,8 @@ public static void listClusters(String projectId, String region) throws IOExcept ClusterControllerClient.create(clusterControllerSettings)) { for (Cluster element : clusterControllerClient.listClusters(projectId, region).iterateAll()) { - System.out.println(element.getClusterName() + ": " + element.getStatus().getState()); + System.out.printf("%s: %s", element.getClusterName(), element.getStatus().getState()); } - } catch (IOException e) { - // Likely this would occur due to issues authenticating with GCP. Make sure the environment - // variable GOOGLE_APPLICATION_CREDENTIALS is configured. - System.out.println("Error deleting the cluster controller client: \n" + e.getMessage()); } } } diff --git a/dataproc/src/main/java/SubmitJob.java b/dataproc/src/main/java/SubmitJob.java index 01bce05569b..7e16224d661 100644 --- a/dataproc/src/main/java/SubmitJob.java +++ b/dataproc/src/main/java/SubmitJob.java @@ -22,12 +22,23 @@ import com.google.cloud.dataproc.v1.JobStatus; import com.google.cloud.dataproc.v1.PySparkJob; import java.io.IOException; +import java.util.concurrent.ExecutionException; public class SubmitJob { + public static void submitJob() + throws IOException, InterruptedException, ExecutionException { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String region = "your-project-region"; + String clusterName = "your-cluster-name"; + String jobFilePath = "your-job-file-path"; + submitJob(projectId, region, clusterName, jobFilePath); + } + public static void submitJob( String projectId, String region, String clusterName, String jobFilePath) throws IOException { - String myEndpoint = region + "-dataproc.googleapis.com:443"; + String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); // Configure the settings for the cluster controller client JobControllerSettings jobControllerSettings = @@ -51,17 +62,13 @@ public static void submitJob( while (true) { Job jobInfo = jobControllerClient.getJob(projectId, region, jobId); if (jobInfo.getStatus().getState().equals(JobStatus.State.ERROR)) { - System.err.printf("Job %s failed: %s", jobId, jobInfo.getStatus().getDetails()); + System.out.printf("Job %s failed: %s", jobId, jobInfo.getStatus().getDetails()); break; } else if (jobInfo.getStatus().getState().equals(JobStatus.State.DONE)) { System.out.printf("Job %s finished.", jobId); break; } } - } catch (IOException e) { - // Likely this would occur due to issues authenticating with GCP. Make sure the environment - // variable GOOGLE_APPLICATION_CREDENTIALS is configured. - System.err.println("Error creating the cluster controller client: \n" + e.getMessage()); } } } diff --git a/dataproc/src/main/java/UpdateCluster.java b/dataproc/src/main/java/UpdateCluster.java index 6cfba7bcace..aae45cd7724 100644 --- a/dataproc/src/main/java/UpdateCluster.java +++ b/dataproc/src/main/java/UpdateCluster.java @@ -28,10 +28,20 @@ public class UpdateCluster { + public static void updateCluster() + throws IOException, InterruptedException, ExecutionException { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String region = "your-project-region"; + String clusterName = "your-cluster-name"; + int numWorkers = 0; //your number of workers + updateCluster(projectId, region, clusterName, numWorkers); + } + public static void updateCluster( String projectId, String region, String clusterName, int numWorkers) - throws IOException, InterruptedException { - String myEndpoint = region + "-dataproc.googleapis.com:443"; + throws IOException, InterruptedException, ExecutionException { + String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); // Configure the settings for the cluster controller client ClusterControllerSettings clusterControllerSettings = @@ -76,15 +86,6 @@ public static void updateCluster( System.out.printf( "Cluster %s now has %d workers.", response.getClusterName(), response.getConfig().getWorkerConfig().getNumInstances()); - - } catch (IOException e) { - // Likely this would occur due to issues authenticating with GCP. Make sure the environment - // variable GOOGLE_APPLICATION_CREDENTIALS is configured. - System.err.println("Error creating the cluster controller client: \n" + e.getMessage()); - } catch (ExecutionException e) { - // Common issues for this include needing to increase compute engine quotas or a cluster of - // the same name already exists. - System.out.println("Error during cluster creation request: \n" + e.getMessage()); } } } diff --git a/dataproc/src/test/java/CreateClusterTest.java b/dataproc/src/test/java/CreateClusterTest.java index fc661e371d3..feedfa37eaa 100644 --- a/dataproc/src/test/java/CreateClusterTest.java +++ b/dataproc/src/test/java/CreateClusterTest.java @@ -65,7 +65,7 @@ public void setUp() { } @Test - public void createClusterTest() throws IOException, InterruptedException { + public void createClusterTest() throws IOException, InterruptedException, ExecutionException { CreateCluster.createCluster(PROJECT_ID, REGION, CLUSTER_NAME); String output = bout.toString(); @@ -73,7 +73,7 @@ public void createClusterTest() throws IOException, InterruptedException { } @After - public void tearDown() throws IOException, InterruptedException { + public void tearDown() throws IOException, InterruptedException, ExecutionException { String myEndpoint = REGION + "-dataproc.googleapis.com:443"; System.setOut(standardOutOrig); @@ -86,8 +86,6 @@ public void tearDown() throws IOException, InterruptedException { clusterControllerClient.deleteClusterAsync(PROJECT_ID, REGION, CLUSTER_NAME); deleteClusterAsyncRequest.get(); - } catch (ExecutionException e) { - System.err.println("Error during cluster deletion: \n" + e.getMessage()); } } } diff --git a/dataproc/src/test/java/DeleteClusterTest.java b/dataproc/src/test/java/DeleteClusterTest.java index 0ec6797463c..ceb2e8841a0 100644 --- a/dataproc/src/test/java/DeleteClusterTest.java +++ b/dataproc/src/test/java/DeleteClusterTest.java @@ -86,7 +86,7 @@ public void setUp() throws IOException, InterruptedException { } @Test - public void deleteClusterTest() throws IOException, InterruptedException { + public void deleteClusterTest() throws IOException, InterruptedException, ExecutionException { DeleteCluster.deleteCluster(PROJECT_ID, REGION, CLUSTER_NAME); String output = bout.toString(); diff --git a/dataproc/src/test/java/ListClustersTest.java b/dataproc/src/test/java/ListClustersTest.java index 4d0272ea797..8a4e798ddc3 100644 --- a/dataproc/src/test/java/ListClustersTest.java +++ b/dataproc/src/test/java/ListClustersTest.java @@ -96,7 +96,7 @@ public void listClustersTest() throws IOException { } @After - public void teardown() throws IOException, InterruptedException { + public void teardown() throws IOException, InterruptedException, ExecutionException { System.setOut(standardOutOrig); ClusterControllerSettings clusterControllerSettings = @@ -107,8 +107,6 @@ public void teardown() throws IOException, InterruptedException { OperationFuture deleteClusterAsyncRequest = clusterControllerClient.deleteClusterAsync(PROJECT_ID, REGION, CLUSTER_NAME); deleteClusterAsyncRequest.get(); - } catch (ExecutionException e) { - System.err.println("[deleteCluster] Error during test cluster creation: \n" + e.getMessage()); } } } diff --git a/dataproc/src/test/java/SubmitJobTest.java b/dataproc/src/test/java/SubmitJobTest.java index ab4cd1139cc..6d1ed867606 100644 --- a/dataproc/src/test/java/SubmitJobTest.java +++ b/dataproc/src/test/java/SubmitJobTest.java @@ -115,7 +115,7 @@ public void submitJobTest() throws IOException { } @After - public void teardown() throws IOException, InterruptedException { + public void teardown() throws IOException, InterruptedException, ExecutionException { System.setOut(standardOutOrig); blob.delete(); @@ -127,8 +127,6 @@ public void teardown() throws IOException, InterruptedException { OperationFuture deleteClusterAsyncRequest = clusterControllerClient.deleteClusterAsync(PROJECT_ID, REGION, CLUSTER_NAME); deleteClusterAsyncRequest.get(); - } catch (ExecutionException e) { - System.err.println("[SubmitJob] Error during test cluster deletion: \n" + e.getMessage()); } } } diff --git a/dataproc/src/test/java/UpdateClusterTest.java b/dataproc/src/test/java/UpdateClusterTest.java index 8eb12ec26e1..08cb5db584a 100644 --- a/dataproc/src/test/java/UpdateClusterTest.java +++ b/dataproc/src/test/java/UpdateClusterTest.java @@ -103,7 +103,7 @@ public void setUp() throws IOException, InterruptedException { } @Test - public void updateClusterTest() throws IOException, InterruptedException { + public void updateClusterTest() throws IOException, InterruptedException, ExecutionException { UpdateCluster.updateCluster(PROJECT_ID, REGION, CLUSTER_NAME, NEW_WORKERS); String output = bout.toString(); @@ -111,7 +111,7 @@ public void updateClusterTest() throws IOException, InterruptedException { } @After - public void teardown() throws IOException, InterruptedException { + public void teardown() throws IOException, InterruptedException, ExecutionException { System.setOut(standardOutOrig); ClusterControllerSettings clusterControllerSettings = @@ -122,8 +122,6 @@ public void teardown() throws IOException, InterruptedException { OperationFuture deleteClusterAsyncRequest = clusterControllerClient.deleteClusterAsync(PROJECT_ID, REGION, CLUSTER_NAME); deleteClusterAsyncRequest.get(); - } catch (ExecutionException e) { - System.err.println("[UpdateCluster] Error during test cluster update: \n" + e.getMessage()); } } } From aa03238598300e0b37399a6ca66da5307920701d Mon Sep 17 00:00:00 2001 From: bradmiro Date: Mon, 2 Dec 2019 16:29:55 -0500 Subject: [PATCH 04/21] Formatting changes --- dataproc/src/main/java/CreateCluster.java | 4 +--- dataproc/src/main/java/DeleteCluster.java | 5 ++--- dataproc/src/main/java/ListClusters.java | 3 +-- dataproc/src/main/java/SubmitJob.java | 3 +-- dataproc/src/main/java/UpdateCluster.java | 5 ++--- dataproc/src/test/java/CreateClusterTest.java | 6 +++--- dataproc/src/test/java/DeleteClusterTest.java | 2 +- dataproc/src/test/java/ListClustersTest.java | 5 +++-- dataproc/src/test/java/SubmitJobTest.java | 6 +++--- dataproc/src/test/java/UpdateClusterTest.java | 4 +--- 10 files changed, 18 insertions(+), 25 deletions(-) diff --git a/dataproc/src/main/java/CreateCluster.java b/dataproc/src/main/java/CreateCluster.java index 02365e8661f..0f0bd1c3ead 100644 --- a/dataproc/src/main/java/CreateCluster.java +++ b/dataproc/src/main/java/CreateCluster.java @@ -27,8 +27,7 @@ public class CreateCluster { - public static void createCluster() - throws IOException, InterruptedException, ExecutionException { + public static void createCluster() throws IOException, InterruptedException, ExecutionException { // TODO(developer): Replace these variables before running the sample. String projectId = "your-project-id"; String region = "your-project-region"; @@ -76,7 +75,6 @@ public static void createCluster(String projectId, String region, String cluster // Print out a success message System.out.printf("Cluster created successfully: %s", response.getClusterName()); - } } } diff --git a/dataproc/src/main/java/DeleteCluster.java b/dataproc/src/main/java/DeleteCluster.java index 4bdb5aa70a5..301fd4b296f 100644 --- a/dataproc/src/main/java/DeleteCluster.java +++ b/dataproc/src/main/java/DeleteCluster.java @@ -25,8 +25,7 @@ public class DeleteCluster { - public static void deleteCluster() - throws IOException, InterruptedException, ExecutionException { + public static void deleteCluster() throws IOException, InterruptedException, ExecutionException { // TODO(developer): Replace these variables before running the sample. String projectId = "your-project-id"; String region = "your-project-region"; @@ -36,7 +35,7 @@ public static void deleteCluster() public static void deleteCluster(String projectId, String region, String clusterName) throws IOException, InterruptedException, ExecutionException { - String myEndpoint = String.format("-dataproc.googleapis.com:443", region); + String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); // Configure the settings for the cluster controller client ClusterControllerSettings clusterControllerSettings = diff --git a/dataproc/src/main/java/ListClusters.java b/dataproc/src/main/java/ListClusters.java index 6b35a18e4f8..840783d81fd 100644 --- a/dataproc/src/main/java/ListClusters.java +++ b/dataproc/src/main/java/ListClusters.java @@ -23,8 +23,7 @@ public class ListClusters { - public static void listClusters() - throws IOException, InterruptedException, ExecutionException { + public static void listClusters() throws IOException, InterruptedException, ExecutionException { // TODO(developer): Replace these variables before running the sample. String projectId = "your-project-id"; String region = "your-project-region"; diff --git a/dataproc/src/main/java/SubmitJob.java b/dataproc/src/main/java/SubmitJob.java index 7e16224d661..a2f558e1ddf 100644 --- a/dataproc/src/main/java/SubmitJob.java +++ b/dataproc/src/main/java/SubmitJob.java @@ -26,8 +26,7 @@ public class SubmitJob { - public static void submitJob() - throws IOException, InterruptedException, ExecutionException { + public static void submitJob() throws IOException, InterruptedException, ExecutionException { // TODO(developer): Replace these variables before running the sample. String projectId = "your-project-id"; String region = "your-project-region"; diff --git a/dataproc/src/main/java/UpdateCluster.java b/dataproc/src/main/java/UpdateCluster.java index aae45cd7724..7eb7afc9f62 100644 --- a/dataproc/src/main/java/UpdateCluster.java +++ b/dataproc/src/main/java/UpdateCluster.java @@ -28,13 +28,12 @@ public class UpdateCluster { - public static void updateCluster() - throws IOException, InterruptedException, ExecutionException { + public static void updateCluster() throws IOException, InterruptedException, ExecutionException { // TODO(developer): Replace these variables before running the sample. String projectId = "your-project-id"; String region = "your-project-region"; String clusterName = "your-cluster-name"; - int numWorkers = 0; //your number of workers + int numWorkers = 0; // your number of workers updateCluster(projectId, region, clusterName, numWorkers); } diff --git a/dataproc/src/test/java/CreateClusterTest.java b/dataproc/src/test/java/CreateClusterTest.java index feedfa37eaa..da4c321b80e 100644 --- a/dataproc/src/test/java/CreateClusterTest.java +++ b/dataproc/src/test/java/CreateClusterTest.java @@ -38,7 +38,8 @@ @RunWith(JUnit4.class) public class CreateClusterTest { - private static final String CLUSTER_NAME = "test-cluster-" + UUID.randomUUID().toString(); + private static final String CLUSTER_NAME = + String.format("test-cluster-%s", UUID.randomUUID().toString()); private static final String REGION = "us-central1"; private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); @@ -74,7 +75,7 @@ public void createClusterTest() throws IOException, InterruptedException, Execut @After public void tearDown() throws IOException, InterruptedException, ExecutionException { - String myEndpoint = REGION + "-dataproc.googleapis.com:443"; + String myEndpoint = String.format("%s-dataproc.googleapis.com:443", REGION); System.setOut(standardOutOrig); ClusterControllerSettings clusterControllerSettings = @@ -85,7 +86,6 @@ public void tearDown() throws IOException, InterruptedException, ExecutionExcept OperationFuture deleteClusterAsyncRequest = clusterControllerClient.deleteClusterAsync(PROJECT_ID, REGION, CLUSTER_NAME); deleteClusterAsyncRequest.get(); - } } } diff --git a/dataproc/src/test/java/DeleteClusterTest.java b/dataproc/src/test/java/DeleteClusterTest.java index ceb2e8841a0..1adac0f5699 100644 --- a/dataproc/src/test/java/DeleteClusterTest.java +++ b/dataproc/src/test/java/DeleteClusterTest.java @@ -64,7 +64,7 @@ public void setUp() throws IOException, InterruptedException { System.setOut(new PrintStream(bout)); standardOutOrig = System.out; - String myEndpoint = REGION + "-dataproc.googleapis.com:443"; + String myEndpoint = String.format("%s-dataproc.googleapis.com:443", REGION); ClusterControllerSettings clusterControllerSettings = ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); diff --git a/dataproc/src/test/java/ListClustersTest.java b/dataproc/src/test/java/ListClustersTest.java index 8a4e798ddc3..93c0b970b41 100644 --- a/dataproc/src/test/java/ListClustersTest.java +++ b/dataproc/src/test/java/ListClustersTest.java @@ -40,7 +40,8 @@ @RunWith(JUnit4.class) public class ListClustersTest { - private static final String CLUSTER_NAME = "test-cluster-" + UUID.randomUUID().toString(); + private static final String CLUSTER_NAME = + String.format("test-cluster-%s", UUID.randomUUID().toString()); private static final String REGION = "us-central1"; private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); @@ -65,7 +66,7 @@ public void setUp() throws IOException, InterruptedException { bout = new ByteArrayOutputStream(); standardOutOrig = System.out; System.setOut(new PrintStream(bout)); - myEndpoint = REGION + "-dataproc.googleapis.com:443"; + myEndpoint = String.format("%s-dataproc.googleapis.com:443", REGION); ClusterControllerSettings clusterControllerSettings = ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); diff --git a/dataproc/src/test/java/SubmitJobTest.java b/dataproc/src/test/java/SubmitJobTest.java index 6d1ed867606..8484f4b7dc8 100644 --- a/dataproc/src/test/java/SubmitJobTest.java +++ b/dataproc/src/test/java/SubmitJobTest.java @@ -49,9 +49,9 @@ public class SubmitJobTest { private static final String MY_UUID = UUID.randomUUID().toString(); private static final String REGION = "us-central1"; private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); - private static final String ENDPOINT = REGION + "-dataproc.googleapis.com:443"; - private static final String CLUSTER_NAME = "test-cluster-" + MY_UUID; - private static final String BUCKET_NAME = "test-bucket-" + MY_UUID; + private static final String ENDPOINT = String.format("%s-dataproc.googleapis.com:443", REGION); + private static final String CLUSTER_NAME = String.format("test-cluster-%s", MY_UUID); + private static final String BUCKET_NAME = String.format("test-bucket-%s", MY_UUID); private static final String JOB_FILE_NAME = "sum.py"; private static final String JOB_FILE_PATH = String.format("gs://%s/%s", BUCKET_NAME, JOB_FILE_NAME); diff --git a/dataproc/src/test/java/UpdateClusterTest.java b/dataproc/src/test/java/UpdateClusterTest.java index 08cb5db584a..0353c3cc9fc 100644 --- a/dataproc/src/test/java/UpdateClusterTest.java +++ b/dataproc/src/test/java/UpdateClusterTest.java @@ -64,7 +64,7 @@ public static void checkRequirements() { } @Before - public void setUp() throws IOException, InterruptedException { + public void setUp() throws IOException, InterruptedException, ExecutionException { bout = new ByteArrayOutputStream(); standardOutOrig = System.out; System.setOut(new PrintStream(bout)); @@ -97,8 +97,6 @@ public void setUp() throws IOException, InterruptedException { OperationFuture createClusterAsyncRequest = clusterControllerClient.createClusterAsync(PROJECT_ID, REGION, cluster); createClusterAsyncRequest.get(); - } catch (ExecutionException e) { - System.err.println("[UpdateCluster] Error during test cluster creation: \n" + e.getMessage()); } } From e2b74e551af7c5bc2549425a063188ee84ed3ade Mon Sep 17 00:00:00 2001 From: bradmiro Date: Mon, 2 Dec 2019 17:28:59 -0500 Subject: [PATCH 05/21] small bug fixes --- dataproc/src/main/java/ListClusters.java | 2 +- dataproc/src/test/java/DeleteClusterTest.java | 3 ++- dataproc/src/test/java/UpdateClusterTest.java | 5 +++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/dataproc/src/main/java/ListClusters.java b/dataproc/src/main/java/ListClusters.java index 840783d81fd..c6c4b697afe 100644 --- a/dataproc/src/main/java/ListClusters.java +++ b/dataproc/src/main/java/ListClusters.java @@ -31,7 +31,7 @@ public static void listClusters() throws IOException, InterruptedException, Exec } public static void listClusters(String projectId, String region) throws IOException { - String myEndpoint = String.format("-dataproc.googleapis.com:443"); + String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); // Configure the settings for the cluster controller client ClusterControllerSettings clusterControllerSettings = diff --git a/dataproc/src/test/java/DeleteClusterTest.java b/dataproc/src/test/java/DeleteClusterTest.java index 1adac0f5699..62c96557612 100644 --- a/dataproc/src/test/java/DeleteClusterTest.java +++ b/dataproc/src/test/java/DeleteClusterTest.java @@ -39,7 +39,8 @@ @RunWith(JUnit4.class) public class DeleteClusterTest { - private static final String CLUSTER_NAME = "test-cluster-" + UUID.randomUUID().toString(); + private static final String CLUSTER_NAME = + String.format("test-cluster-%s", UUID.randomUUID().toString()); private static final String REGION = "us-central1"; private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); diff --git a/dataproc/src/test/java/UpdateClusterTest.java b/dataproc/src/test/java/UpdateClusterTest.java index 0353c3cc9fc..ac3aa869431 100644 --- a/dataproc/src/test/java/UpdateClusterTest.java +++ b/dataproc/src/test/java/UpdateClusterTest.java @@ -43,8 +43,9 @@ public class UpdateClusterTest { private static final String REGION = "us-central1"; private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); - private static final String ENDPOINT = REGION + "-dataproc.googleapis.com:443"; - private static final String CLUSTER_NAME = "test-cluster-" + UUID.randomUUID().toString(); + private static final String ENDPOINT = String.format("%s-dataproc.googleapis.com:443", REGION); + private static final String CLUSTER_NAME = + String.format("test-cluster-%s", UUID.randomUUID().toString()); private static final int NUM_WORKERS = 2; private static final int NEW_WORKERS = NUM_WORKERS * 2; From ee02b72d99d8c0737d183fbdfaac96abe53c31f4 Mon Sep 17 00:00:00 2001 From: bradmiro Date: Fri, 20 Dec 2019 12:13:08 -0500 Subject: [PATCH 06/21] Fixed CreateCluster sample and added Quickstart --- dataproc/src/main/java/DeleteCluster.java | 61 --------- dataproc/src/main/java/ListClusters.java | 52 -------- dataproc/src/main/java/SubmitJob.java | 74 ---------- dataproc/src/main/java/UpdateCluster.java | 91 ------------- .../{DeleteClusterTest.java => CreateCluster} | 46 +++---- dataproc/src/test/java/ListClustersTest.java | 113 ---------------- ...SubmitJobTest.java => QuickstartTest.java} | 42 +++--- dataproc/src/test/java/UpdateClusterTest.java | 126 ------------------ 8 files changed, 34 insertions(+), 571 deletions(-) delete mode 100644 dataproc/src/main/java/DeleteCluster.java delete mode 100644 dataproc/src/main/java/ListClusters.java delete mode 100644 dataproc/src/main/java/SubmitJob.java delete mode 100644 dataproc/src/main/java/UpdateCluster.java rename dataproc/src/test/java/{DeleteClusterTest.java => CreateCluster} (73%) delete mode 100644 dataproc/src/test/java/ListClustersTest.java rename dataproc/src/test/java/{SubmitJobTest.java => QuickstartTest.java} (75%) delete mode 100644 dataproc/src/test/java/UpdateClusterTest.java diff --git a/dataproc/src/main/java/DeleteCluster.java b/dataproc/src/main/java/DeleteCluster.java deleted file mode 100644 index 301fd4b296f..00000000000 --- a/dataproc/src/main/java/DeleteCluster.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2019 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// [START dataproc_delete_cluster] -import com.google.api.gax.longrunning.OperationFuture; -import com.google.cloud.dataproc.v1.ClusterControllerClient; -import com.google.cloud.dataproc.v1.ClusterControllerSettings; -import com.google.cloud.dataproc.v1.ClusterOperationMetadata; -import com.google.protobuf.Empty; -import java.io.IOException; -import java.util.concurrent.ExecutionException; - -public class DeleteCluster { - - public static void deleteCluster() throws IOException, InterruptedException, ExecutionException { - // TODO(developer): Replace these variables before running the sample. - String projectId = "your-project-id"; - String region = "your-project-region"; - String clusterName = "your-cluster-name"; - deleteCluster(projectId, region, clusterName); - } - - public static void deleteCluster(String projectId, String region, String clusterName) - throws IOException, InterruptedException, ExecutionException { - String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); - - // Configure the settings for the cluster controller client - ClusterControllerSettings clusterControllerSettings = - ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); - - // Create a cluster controller client with the configured settings. The client only needs to be - // created once and can be reused for multiple requests. Using a try-with-resources - // closes the client, but this can also be done manually with the .close() method. - try (ClusterControllerClient clusterControllerClient = - ClusterControllerClient.create(clusterControllerSettings)) { - // Configure the settings for our cluster - - // Delete the Cloud Dataproc cluster - OperationFuture deleteClusterAsyncRequest = - clusterControllerClient.deleteClusterAsync(projectId, region, clusterName); - deleteClusterAsyncRequest.get(); - - // Print out a success message - System.out.printf("Cluster deleted successfully: %s", clusterName); - } - } -} -// [END dataproc_delete_cluster] diff --git a/dataproc/src/main/java/ListClusters.java b/dataproc/src/main/java/ListClusters.java deleted file mode 100644 index c6c4b697afe..00000000000 --- a/dataproc/src/main/java/ListClusters.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2019 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// [START dataproc_list_clusters] -import com.google.cloud.dataproc.v1.Cluster; -import com.google.cloud.dataproc.v1.ClusterControllerClient; -import com.google.cloud.dataproc.v1.ClusterControllerSettings; -import java.io.IOException; -import java.util.concurrent.ExecutionException; - -public class ListClusters { - - public static void listClusters() throws IOException, InterruptedException, ExecutionException { - // TODO(developer): Replace these variables before running the sample. - String projectId = "your-project-id"; - String region = "your-project-region"; - listClusters(projectId, region); - } - - public static void listClusters(String projectId, String region) throws IOException { - String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); - - // Configure the settings for the cluster controller client - ClusterControllerSettings clusterControllerSettings = - ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); - - // Create a cluster controller client with the configured settings. The client only needs to be - // created once and can be reused for multiple requests. Using a try-with-resources - // closes the client, but this can also be done manually with the .close() method. - try (ClusterControllerClient clusterControllerClient = - ClusterControllerClient.create(clusterControllerSettings)) { - - for (Cluster element : clusterControllerClient.listClusters(projectId, region).iterateAll()) { - System.out.printf("%s: %s", element.getClusterName(), element.getStatus().getState()); - } - } - } -} -// [END dataproc_list_clusters] diff --git a/dataproc/src/main/java/SubmitJob.java b/dataproc/src/main/java/SubmitJob.java deleted file mode 100644 index a2f558e1ddf..00000000000 --- a/dataproc/src/main/java/SubmitJob.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright 2019 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// [START dataproc_submit_job] -import com.google.cloud.dataproc.v1.Job; -import com.google.cloud.dataproc.v1.JobControllerClient; -import com.google.cloud.dataproc.v1.JobControllerSettings; -import com.google.cloud.dataproc.v1.JobPlacement; -import com.google.cloud.dataproc.v1.JobStatus; -import com.google.cloud.dataproc.v1.PySparkJob; -import java.io.IOException; -import java.util.concurrent.ExecutionException; - -public class SubmitJob { - - public static void submitJob() throws IOException, InterruptedException, ExecutionException { - // TODO(developer): Replace these variables before running the sample. - String projectId = "your-project-id"; - String region = "your-project-region"; - String clusterName = "your-cluster-name"; - String jobFilePath = "your-job-file-path"; - submitJob(projectId, region, clusterName, jobFilePath); - } - - public static void submitJob( - String projectId, String region, String clusterName, String jobFilePath) throws IOException { - String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); - - // Configure the settings for the cluster controller client - JobControllerSettings jobControllerSettings = - JobControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); - - // Create a job controller client with the configured settings. The client only needs to be - // created once and can be reused for multiple requests. Using a try-with-resources - // closes the client, but this can also be done manually with the .close() method. - try (JobControllerClient jobControllerClient = - JobControllerClient.create(jobControllerSettings)) { - - // Configure the settings for our job - JobPlacement jobPlacement = JobPlacement.newBuilder().setClusterName(clusterName).build(); - PySparkJob pySparkJob = PySparkJob.newBuilder().setMainPythonFileUri(jobFilePath).build(); - Job job = Job.newBuilder().setPlacement(jobPlacement).setPysparkJob(pySparkJob).build(); - - Job jobOperation = jobControllerClient.submitJob(projectId, region, job); - - String jobId = jobOperation.getReference().getJobId(); - - while (true) { - Job jobInfo = jobControllerClient.getJob(projectId, region, jobId); - if (jobInfo.getStatus().getState().equals(JobStatus.State.ERROR)) { - System.out.printf("Job %s failed: %s", jobId, jobInfo.getStatus().getDetails()); - break; - } else if (jobInfo.getStatus().getState().equals(JobStatus.State.DONE)) { - System.out.printf("Job %s finished.", jobId); - break; - } - } - } - } -} -// [END dataproc_submit_job] diff --git a/dataproc/src/main/java/UpdateCluster.java b/dataproc/src/main/java/UpdateCluster.java deleted file mode 100644 index 7eb7afc9f62..00000000000 --- a/dataproc/src/main/java/UpdateCluster.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright 2019 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// [START dataproc_update_cluster] -import com.google.api.gax.longrunning.OperationFuture; -import com.google.cloud.dataproc.v1.Cluster; -import com.google.cloud.dataproc.v1.ClusterConfig; -import com.google.cloud.dataproc.v1.ClusterControllerClient; -import com.google.cloud.dataproc.v1.ClusterControllerSettings; -import com.google.cloud.dataproc.v1.ClusterOperationMetadata; -import com.google.cloud.dataproc.v1.InstanceGroupConfig; -import com.google.protobuf.FieldMask; -import java.io.IOException; -import java.util.concurrent.ExecutionException; - -public class UpdateCluster { - - public static void updateCluster() throws IOException, InterruptedException, ExecutionException { - // TODO(developer): Replace these variables before running the sample. - String projectId = "your-project-id"; - String region = "your-project-region"; - String clusterName = "your-cluster-name"; - int numWorkers = 0; // your number of workers - updateCluster(projectId, region, clusterName, numWorkers); - } - - public static void updateCluster( - String projectId, String region, String clusterName, int numWorkers) - throws IOException, InterruptedException, ExecutionException { - String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); - - // Configure the settings for the cluster controller client - ClusterControllerSettings clusterControllerSettings = - ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); - - // Create a cluster controller client with the configured settings. The client only needs to be - // created once and can be reused for multiple requests. Using a try-with-resources - // closes the client, but this can also be done manually with the .close() method. - try (ClusterControllerClient clusterControllerClient = - ClusterControllerClient.create(clusterControllerSettings)) { - // Configure the settings for our cluster - InstanceGroupConfig masterConfig = - InstanceGroupConfig.newBuilder() - .setMachineTypeUri("n1-standard-1") - .setNumInstances(1) - .build(); - InstanceGroupConfig workerConfig = - InstanceGroupConfig.newBuilder() - .setMachineTypeUri("n1-standard-1") - .setNumInstances(numWorkers) - .build(); - ClusterConfig clusterConfig = - ClusterConfig.newBuilder() - .setMasterConfig(masterConfig) - .setWorkerConfig(workerConfig) - .build(); - // Create the cluster object with the desired cluster config - Cluster cluster = - Cluster.newBuilder().setClusterName(clusterName).setConfig(clusterConfig).build(); - - // Create a field mask to indicate what field of the cluster to update. - FieldMask updateMask = - FieldMask.newBuilder().addPaths("config.worker_config.num_instances").build(); - - // Submit the request to update the cluster - OperationFuture updateClusterAsyncRequest = - clusterControllerClient.updateClusterAsync( - projectId, region, clusterName, cluster, updateMask); - Cluster response = updateClusterAsyncRequest.get(); - - // Print out a success message - System.out.printf( - "Cluster %s now has %d workers.", - response.getClusterName(), response.getConfig().getWorkerConfig().getNumInstances()); - } - } -} -// [END dataproc_update_cluster] diff --git a/dataproc/src/test/java/DeleteClusterTest.java b/dataproc/src/test/java/CreateCluster similarity index 73% rename from dataproc/src/test/java/DeleteClusterTest.java rename to dataproc/src/test/java/CreateCluster index 62c96557612..07d78a28adf 100644 --- a/dataproc/src/test/java/DeleteClusterTest.java +++ b/dataproc/src/test/java/CreateCluster @@ -18,11 +18,10 @@ import static org.hamcrest.MatcherAssert.assertThat; import com.google.api.gax.longrunning.OperationFuture; -import com.google.cloud.dataproc.v1.Cluster; -import com.google.cloud.dataproc.v1.ClusterConfig; import com.google.cloud.dataproc.v1.ClusterControllerClient; import com.google.cloud.dataproc.v1.ClusterControllerSettings; import com.google.cloud.dataproc.v1.ClusterOperationMetadata; +import com.google.protobuf.Empty; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintStream; @@ -37,7 +36,7 @@ import org.junit.runners.JUnit4; @RunWith(JUnit4.class) -public class DeleteClusterTest { +public class CreateCluster { private static final String CLUSTER_NAME = String.format("test-cluster-%s", UUID.randomUUID().toString()); @@ -60,42 +59,33 @@ public static void checkRequirements() { } @Before - public void setUp() throws IOException, InterruptedException { + public void setUp() { bout = new ByteArrayOutputStream(); System.setOut(new PrintStream(bout)); standardOutOrig = System.out; - - String myEndpoint = String.format("%s-dataproc.googleapis.com:443", REGION); - - ClusterControllerSettings clusterControllerSettings = - ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); - - Cluster cluster = - Cluster.newBuilder() - .setClusterName(CLUSTER_NAME) - .setConfig(ClusterConfig.newBuilder().build()) - .build(); - - try (ClusterControllerClient clusterControllerClient = - ClusterControllerClient.create(clusterControllerSettings)) { - OperationFuture createClusterAsyncRequest = - clusterControllerClient.createClusterAsync(PROJECT_ID, REGION, cluster); - createClusterAsyncRequest.get(); - } catch (ExecutionException e) { - System.err.println("[deleteCluster] Error during test cluster creation: \n" + e.getMessage()); - } } @Test - public void deleteClusterTest() throws IOException, InterruptedException, ExecutionException { - DeleteCluster.deleteCluster(PROJECT_ID, REGION, CLUSTER_NAME); + public void createClusterTest() throws IOException, InterruptedException, ExecutionException { + Quickstart.createCluster(PROJECT_ID, REGION, CLUSTER_NAME); String output = bout.toString(); - assertThat(output, CoreMatchers.containsString("deleted successfully")); + assertThat(output, CoreMatchers.containsString(CLUSTER_NAME)); } @After - public void teardown() { + public void tearDown() throws IOException, InterruptedException, ExecutionException { + String myEndpoint = String.format("%s-dataproc.googleapis.com:443", REGION); System.setOut(standardOutOrig); + + ClusterControllerSettings clusterControllerSettings = + ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); + + try (ClusterControllerClient clusterControllerClient = + ClusterControllerClient.create(clusterControllerSettings)) { + OperationFuture deleteClusterAsyncRequest = + clusterControllerClient.deleteClusterAsync(PROJECT_ID, REGION, CLUSTER_NAME); + deleteClusterAsyncRequest.get(); + } } } diff --git a/dataproc/src/test/java/ListClustersTest.java b/dataproc/src/test/java/ListClustersTest.java deleted file mode 100644 index 93c0b970b41..00000000000 --- a/dataproc/src/test/java/ListClustersTest.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright 2019 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import static junit.framework.TestCase.assertNotNull; -import static org.hamcrest.MatcherAssert.assertThat; - -import com.google.api.gax.longrunning.OperationFuture; -import com.google.cloud.dataproc.v1.Cluster; -import com.google.cloud.dataproc.v1.ClusterConfig; -import com.google.cloud.dataproc.v1.ClusterControllerClient; -import com.google.cloud.dataproc.v1.ClusterControllerSettings; -import com.google.cloud.dataproc.v1.ClusterOperationMetadata; -import com.google.protobuf.Empty; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.PrintStream; -import java.util.UUID; -import java.util.concurrent.ExecutionException; -import org.hamcrest.CoreMatchers; -import org.junit.After; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -@RunWith(JUnit4.class) -public class ListClustersTest { - - private static final String CLUSTER_NAME = - String.format("test-cluster-%s", UUID.randomUUID().toString()); - private static final String REGION = "us-central1"; - private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); - - private String myEndpoint; - private ByteArrayOutputStream bout; - private PrintStream standardOutOrig; - - private static void requireEnv(String varName) { - assertNotNull( - System.getenv(varName), - String.format("Environment variable '%s' is required to perform these tests.", varName)); - } - - @BeforeClass - public static void checkRequirements() { - requireEnv("GOOGLE_APPLICATION_CREDENTIALS"); - requireEnv("GOOGLE_CLOUD_PROJECT"); - } - - @Before - public void setUp() throws IOException, InterruptedException { - bout = new ByteArrayOutputStream(); - standardOutOrig = System.out; - System.setOut(new PrintStream(bout)); - myEndpoint = String.format("%s-dataproc.googleapis.com:443", REGION); - - ClusterControllerSettings clusterControllerSettings = - ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); - - Cluster cluster = - Cluster.newBuilder() - .setClusterName(CLUSTER_NAME) - .setConfig(ClusterConfig.newBuilder().build()) - .build(); - - try (ClusterControllerClient clusterControllerClient = - ClusterControllerClient.create(clusterControllerSettings)) { - OperationFuture createClusterAsyncRequest = - clusterControllerClient.createClusterAsync(PROJECT_ID, REGION, cluster); - createClusterAsyncRequest.get(); - } catch (ExecutionException e) { - System.err.println("[deleteCluster] Error during test cluster creation: \n" + e.getMessage()); - } - } - - @Test - public void listClustersTest() throws IOException { - ListClusters.listClusters(PROJECT_ID, REGION); - String output = bout.toString(); - - assertThat(output, CoreMatchers.containsString(CLUSTER_NAME)); - assertThat(output, CoreMatchers.containsString("RUNNING")); - } - - @After - public void teardown() throws IOException, InterruptedException, ExecutionException { - System.setOut(standardOutOrig); - - ClusterControllerSettings clusterControllerSettings = - ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); - - try (ClusterControllerClient clusterControllerClient = - ClusterControllerClient.create(clusterControllerSettings)) { - OperationFuture deleteClusterAsyncRequest = - clusterControllerClient.deleteClusterAsync(PROJECT_ID, REGION, CLUSTER_NAME); - deleteClusterAsyncRequest.get(); - } - } -} diff --git a/dataproc/src/test/java/SubmitJobTest.java b/dataproc/src/test/java/QuickstartTest.java similarity index 75% rename from dataproc/src/test/java/SubmitJobTest.java rename to dataproc/src/test/java/QuickstartTest.java index 8484f4b7dc8..7351fc67ac3 100644 --- a/dataproc/src/test/java/SubmitJobTest.java +++ b/dataproc/src/test/java/QuickstartTest.java @@ -20,7 +20,6 @@ import com.google.api.gax.longrunning.OperationFuture; import com.google.cloud.dataproc.v1.Cluster; -import com.google.cloud.dataproc.v1.ClusterConfig; import com.google.cloud.dataproc.v1.ClusterControllerClient; import com.google.cloud.dataproc.v1.ClusterControllerSettings; import com.google.cloud.dataproc.v1.ClusterOperationMetadata; @@ -44,7 +43,7 @@ import org.junit.runners.JUnit4; @RunWith(JUnit4.class) -public class SubmitJobTest { +public class QuickstartTest { private static final String MY_UUID = UUID.randomUUID().toString(); private static final String REGION = "us-central1"; @@ -86,32 +85,17 @@ public void setUp() throws IOException, InterruptedException { Storage storage = StorageOptions.getDefaultInstance().getService(); Bucket bucket = storage.create(BucketInfo.of(BUCKET_NAME)); blob = bucket.create(JOB_FILE_NAME, SORT_CODE.getBytes(UTF_8), "text/plain"); - - ClusterControllerSettings clusterControllerSettings = - ClusterControllerSettings.newBuilder().setEndpoint(ENDPOINT).build(); - - Cluster cluster = - Cluster.newBuilder() - .setClusterName(CLUSTER_NAME) - .setConfig(ClusterConfig.newBuilder().build()) - .build(); - - try (ClusterControllerClient clusterControllerClient = - ClusterControllerClient.create(clusterControllerSettings)) { - OperationFuture createClusterAsyncRequest = - clusterControllerClient.createClusterAsync(PROJECT_ID, REGION, cluster); - createClusterAsyncRequest.get(); - } catch (ExecutionException e) { - System.err.println("[SubmitJob] Error during test cluster creation: \n" + e.getMessage()); - } } @Test - public void submitJobTest() throws IOException { - SubmitJob.submitJob(PROJECT_ID, REGION, CLUSTER_NAME, JOB_FILE_PATH); + public void quickstartTest() throws IOException, InterruptedException, ExecutionException { + Quickstart.quickstart(PROJECT_ID, REGION, CLUSTER_NAME, JOB_FILE_PATH); String output = bout.toString(); - assertThat(output, CoreMatchers.containsString("finished")); + assertThat(output, CoreMatchers.containsString("Cluster created successfully")); + assertThat(output, CoreMatchers.containsString("Submitted job")); + assertThat(output, CoreMatchers.containsString("finished with state DONE:")); + assertThat(output, CoreMatchers.containsString("successfully deleted")); } @After @@ -124,9 +108,15 @@ public void teardown() throws IOException, InterruptedException, ExecutionExcept try (ClusterControllerClient clusterControllerClient = ClusterControllerClient.create(clusterControllerSettings)) { - OperationFuture deleteClusterAsyncRequest = - clusterControllerClient.deleteClusterAsync(PROJECT_ID, REGION, CLUSTER_NAME); - deleteClusterAsyncRequest.get(); + for (Cluster element : + clusterControllerClient.listClusters(PROJECT_ID, REGION).iterateAll()) { + if (element.getClusterName() == CLUSTER_NAME) { + OperationFuture deleteClusterAsyncRequest = + clusterControllerClient.deleteClusterAsync(PROJECT_ID, REGION, CLUSTER_NAME); + deleteClusterAsyncRequest.get(); + break; + } + } } } } diff --git a/dataproc/src/test/java/UpdateClusterTest.java b/dataproc/src/test/java/UpdateClusterTest.java deleted file mode 100644 index ac3aa869431..00000000000 --- a/dataproc/src/test/java/UpdateClusterTest.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright 2019 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import static junit.framework.TestCase.assertNotNull; -import static org.hamcrest.MatcherAssert.assertThat; - -import com.google.api.gax.longrunning.OperationFuture; -import com.google.cloud.dataproc.v1.Cluster; -import com.google.cloud.dataproc.v1.ClusterConfig; -import com.google.cloud.dataproc.v1.ClusterControllerClient; -import com.google.cloud.dataproc.v1.ClusterControllerSettings; -import com.google.cloud.dataproc.v1.ClusterOperationMetadata; -import com.google.cloud.dataproc.v1.InstanceGroupConfig; -import com.google.protobuf.Empty; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.PrintStream; -import java.util.UUID; -import java.util.concurrent.ExecutionException; -import org.hamcrest.CoreMatchers; -import org.junit.After; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -@RunWith(JUnit4.class) -public class UpdateClusterTest { - - private static final String REGION = "us-central1"; - private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); - private static final String ENDPOINT = String.format("%s-dataproc.googleapis.com:443", REGION); - private static final String CLUSTER_NAME = - String.format("test-cluster-%s", UUID.randomUUID().toString()); - private static final int NUM_WORKERS = 2; - private static final int NEW_WORKERS = NUM_WORKERS * 2; - - private ByteArrayOutputStream bout; - private PrintStream standardOutOrig; - - private static void requireEnv(String varName) { - assertNotNull( - System.getenv(varName), - String.format("Environment variable '%s' is required to perform these tests.", varName)); - } - - @BeforeClass - public static void checkRequirements() { - requireEnv("GOOGLE_APPLICATION_CREDENTIALS"); - requireEnv("GOOGLE_CLOUD_PROJECT"); - } - - @Before - public void setUp() throws IOException, InterruptedException, ExecutionException { - bout = new ByteArrayOutputStream(); - standardOutOrig = System.out; - System.setOut(new PrintStream(bout)); - - ClusterControllerSettings clusterControllerSettings = - ClusterControllerSettings.newBuilder().setEndpoint(ENDPOINT).build(); - - // Configure the settings for our cluster - InstanceGroupConfig masterConfig = - InstanceGroupConfig.newBuilder() - .setMachineTypeUri("n1-standard-1") - .setNumInstances(1) - .build(); - InstanceGroupConfig workerConfig = - InstanceGroupConfig.newBuilder() - .setMachineTypeUri("n1-standard-1") - .setNumInstances(NUM_WORKERS) - .build(); - ClusterConfig clusterConfig = - ClusterConfig.newBuilder() - .setMasterConfig(masterConfig) - .setWorkerConfig(workerConfig) - .build(); - // Create the cluster object with the desired cluster config - Cluster cluster = - Cluster.newBuilder().setClusterName(CLUSTER_NAME).setConfig(clusterConfig).build(); - - try (ClusterControllerClient clusterControllerClient = - ClusterControllerClient.create(clusterControllerSettings)) { - OperationFuture createClusterAsyncRequest = - clusterControllerClient.createClusterAsync(PROJECT_ID, REGION, cluster); - createClusterAsyncRequest.get(); - } - } - - @Test - public void updateClusterTest() throws IOException, InterruptedException, ExecutionException { - UpdateCluster.updateCluster(PROJECT_ID, REGION, CLUSTER_NAME, NEW_WORKERS); - String output = bout.toString(); - - assertThat(output, CoreMatchers.containsString(String.format("%d workers", NEW_WORKERS))); - } - - @After - public void teardown() throws IOException, InterruptedException, ExecutionException { - System.setOut(standardOutOrig); - - ClusterControllerSettings clusterControllerSettings = - ClusterControllerSettings.newBuilder().setEndpoint(ENDPOINT).build(); - - try (ClusterControllerClient clusterControllerClient = - ClusterControllerClient.create(clusterControllerSettings)) { - OperationFuture deleteClusterAsyncRequest = - clusterControllerClient.deleteClusterAsync(PROJECT_ID, REGION, CLUSTER_NAME); - deleteClusterAsyncRequest.get(); - } - } -} From 400617c65cae60022640b3a38da578a34c1678cd Mon Sep 17 00:00:00 2001 From: bradmiro Date: Fri, 20 Dec 2019 12:40:00 -0500 Subject: [PATCH 07/21] Added quickstart sample --- dataproc/pom.xml | 14 +- dataproc/src/main/java/Quickstart.java | 172 +++++++++++++++++++++++++ dataproc/src/test/java/CreateCluster | 91 ------------- 3 files changed, 184 insertions(+), 93 deletions(-) create mode 100644 dataproc/src/main/java/Quickstart.java delete mode 100644 dataproc/src/test/java/CreateCluster diff --git a/dataproc/pom.xml b/dataproc/pom.xml index ef58507b3c9..fdf40d6cbf8 100644 --- a/dataproc/pom.xml +++ b/dataproc/pom.xml @@ -36,6 +36,18 @@ + + + + com.google.cloud + libraries-bom + 3.0.0 + pom + import + + + + @@ -47,12 +59,10 @@ com.google.cloud google-cloud-dataproc - 0.117.0 com.google.cloud google-cloud-storage - 1.101.0 diff --git a/dataproc/src/main/java/Quickstart.java b/dataproc/src/main/java/Quickstart.java new file mode 100644 index 00000000000..3153abc18ee --- /dev/null +++ b/dataproc/src/main/java/Quickstart.java @@ -0,0 +1,172 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// [START dataproc_quickstart] +import com.google.api.gax.longrunning.OperationFuture; +import com.google.cloud.dataproc.v1.Cluster; +import com.google.cloud.dataproc.v1.ClusterConfig; +import com.google.cloud.dataproc.v1.ClusterControllerClient; +import com.google.cloud.dataproc.v1.ClusterControllerSettings; +import com.google.cloud.dataproc.v1.ClusterOperationMetadata; +import com.google.cloud.dataproc.v1.InstanceGroupConfig; +import com.google.cloud.dataproc.v1.Job; +import com.google.cloud.dataproc.v1.JobControllerClient; +import com.google.cloud.dataproc.v1.JobControllerSettings; +import com.google.cloud.dataproc.v1.JobPlacement; +import com.google.cloud.dataproc.v1.PySparkJob; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageOptions; +import com.google.protobuf.Empty; +import java.io.IOException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +public class Quickstart { + + public static void quickstart() throws IOException, InterruptedException { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String region = "your-project-region"; + String clusterName = "your-cluster-name"; + String jobFilePath = "your-job-file-path"; + quickstart(projectId, region, clusterName, jobFilePath); + } + + public static void quickstart( + String projectId, String region, String clusterName, String jobFilePath) + throws IOException, InterruptedException { + String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); + + // Configure the settings for the cluster controller client + ClusterControllerSettings clusterControllerSettings = + ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); + + // Configure the settings for the job controller client + JobControllerSettings jobControllerSettings = + JobControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); + + // Create both a cluster controller client and job controller client with the configured + // settings. The client only needs to be created once and can be reused for multiple requests. + // Using a try-with-resources closes the client, but this can also be done manually with + // the .close() method. + try (ClusterControllerClient clusterControllerClient = + ClusterControllerClient.create(clusterControllerSettings); + JobControllerClient jobControllerClient = + JobControllerClient.create(jobControllerSettings)) { + // Configure the settings for our cluster + InstanceGroupConfig masterConfig = + InstanceGroupConfig.newBuilder() + .setMachineTypeUri("n1-standard-1") + .setNumInstances(1) + .build(); + InstanceGroupConfig workerConfig = + InstanceGroupConfig.newBuilder() + .setMachineTypeUri("n1-standard-1") + .setNumInstances(2) + .build(); + ClusterConfig clusterConfig = + ClusterConfig.newBuilder() + .setMasterConfig(masterConfig) + .setWorkerConfig(workerConfig) + .build(); + // Create the cluster object with the desired cluster config + Cluster cluster = + Cluster.newBuilder().setClusterName(clusterName).setConfig(clusterConfig).build(); + + // Create the Cloud Dataproc cluster + OperationFuture createClusterAsyncRequest = + clusterControllerClient.createClusterAsync(projectId, region, cluster); + Cluster response = createClusterAsyncRequest.get(); + System.out.printf("Cluster created successfully: %s", response.getClusterName()); + + // Configure the settings for our job + JobPlacement jobPlacement = JobPlacement.newBuilder().setClusterName(clusterName).build(); + PySparkJob pySparkJob = PySparkJob.newBuilder().setMainPythonFileUri(jobFilePath).build(); + Job job = Job.newBuilder().setPlacement(jobPlacement).setPysparkJob(pySparkJob).build(); + + // Submit an asynchronous request to execute the job + Job request = jobControllerClient.submitJob(projectId, region, job); + String jobId = request.getReference().getJobId(); + System.out.println(String.format("Submitted job \"%s\"", jobId)); + + // Wait for the job to finish + CompletableFuture finishedJobFuture = + CompletableFuture.supplyAsync( + () -> { + // States that indicate a job is "finished" + while (true) { + // Poll the service periodically until the Job is in a finished state. + Job jobInfo = jobControllerClient.getJob(projectId, region, jobId); + switch (jobInfo.getStatus().getState()) { + case DONE: + return jobInfo; + case CANCELLED: + case ERROR: + case ATTEMPT_FAILURE: + throw new RuntimeException( + String.format( + "Job %s failed due to %s: %s", + jobId, + jobInfo.getStatus().getState(), + jobInfo.getStatus().getDetails())); + default: + try { + // Wait a second in between polling attempts. + TimeUnit.SECONDS.sleep(1); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } + }); + int timeout = 10; + try { + Job jobInfo = finishedJobFuture.get(timeout, TimeUnit.MINUTES); + System.out.printf("Job %s finished successfully.", jobId); + + // Cloud Dataproc job output gets saved to a GCS bucket allocated to it. + Cluster clusterInfo = clusterControllerClient.getCluster(projectId, region, clusterName); + Storage storage = StorageOptions.getDefaultInstance().getService(); + Blob blob = + storage.get( + clusterInfo.getConfig().getConfigBucket(), + String.format( + "google-cloud-dataproc-metainfo/%s/jobs/%s/driveroutput.000000000", + clusterInfo.getClusterUuid(), jobId)); + System.out.println( + String.format( + "Job \"%s\" finished with state %s:\n%s", + jobId, jobInfo.getStatus().getState(), new String(blob.getContent()))); + } catch (TimeoutException e) { + System.err.println( + String.format("Job timed out after %d minutes: %s", timeout, e.getMessage())); + } + + // Delete the cluster + OperationFuture deleteClusterAsyncRequest = + clusterControllerClient.deleteClusterAsync(projectId, region, clusterName); + deleteClusterAsyncRequest.get(); + System.out.println(String.format("Cluster \"%s\" successfully deleted.", clusterName)); + + } catch (ExecutionException e) { + System.err.println(String.format("Error executing quickstart: " + e.getMessage())); + } + } +} +// [END dataproc_quickstart] \ No newline at end of file diff --git a/dataproc/src/test/java/CreateCluster b/dataproc/src/test/java/CreateCluster deleted file mode 100644 index 07d78a28adf..00000000000 --- a/dataproc/src/test/java/CreateCluster +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright 2019 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import static junit.framework.TestCase.assertNotNull; -import static org.hamcrest.MatcherAssert.assertThat; - -import com.google.api.gax.longrunning.OperationFuture; -import com.google.cloud.dataproc.v1.ClusterControllerClient; -import com.google.cloud.dataproc.v1.ClusterControllerSettings; -import com.google.cloud.dataproc.v1.ClusterOperationMetadata; -import com.google.protobuf.Empty; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.PrintStream; -import java.util.UUID; -import java.util.concurrent.ExecutionException; -import org.hamcrest.CoreMatchers; -import org.junit.After; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -@RunWith(JUnit4.class) -public class CreateCluster { - - private static final String CLUSTER_NAME = - String.format("test-cluster-%s", UUID.randomUUID().toString()); - private static final String REGION = "us-central1"; - private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); - - private ByteArrayOutputStream bout; - private PrintStream standardOutOrig; - - private static void requireEnv(String varName) { - assertNotNull( - System.getenv(varName), - String.format("Environment variable '%s' is required to perform these tests.", varName)); - } - - @BeforeClass - public static void checkRequirements() { - requireEnv("GOOGLE_APPLICATION_CREDENTIALS"); - requireEnv("GOOGLE_CLOUD_PROJECT"); - } - - @Before - public void setUp() { - bout = new ByteArrayOutputStream(); - System.setOut(new PrintStream(bout)); - standardOutOrig = System.out; - } - - @Test - public void createClusterTest() throws IOException, InterruptedException, ExecutionException { - Quickstart.createCluster(PROJECT_ID, REGION, CLUSTER_NAME); - String output = bout.toString(); - - assertThat(output, CoreMatchers.containsString(CLUSTER_NAME)); - } - - @After - public void tearDown() throws IOException, InterruptedException, ExecutionException { - String myEndpoint = String.format("%s-dataproc.googleapis.com:443", REGION); - System.setOut(standardOutOrig); - - ClusterControllerSettings clusterControllerSettings = - ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); - - try (ClusterControllerClient clusterControllerClient = - ClusterControllerClient.create(clusterControllerSettings)) { - OperationFuture deleteClusterAsyncRequest = - clusterControllerClient.deleteClusterAsync(PROJECT_ID, REGION, CLUSTER_NAME); - deleteClusterAsyncRequest.get(); - } - } -} From d0efbbc7003afe83b8572c0a69cae8fce15727dc Mon Sep 17 00:00:00 2001 From: bradmiro Date: Mon, 2 Dec 2019 12:42:12 -0500 Subject: [PATCH 08/21] Added dataproc quickstart samples --- dataproc/pom.xml | 5 + dataproc/src/main/java/CreateCluster.java | 42 +++--- dataproc/src/main/java/DeleteCluster.java | 61 ++++++++ dataproc/src/main/java/ListClusters.java | 49 +++++++ dataproc/src/main/java/SubmitJob.java | 69 +++++++++ dataproc/src/main/java/UpdateCluster.java | 92 ++++++++++++ dataproc/src/test/java/CreateClusterTest.java | 9 +- dataproc/src/test/java/DeleteClusterTest.java | 95 ++++++++++++ dataproc/src/test/java/ListClustersTest.java | 115 +++++++++++++++ dataproc/src/test/java/SubmitJobTest.java | 135 ++++++++++++++++++ dataproc/src/test/java/UpdateClusterTest.java | 130 +++++++++++++++++ 11 files changed, 776 insertions(+), 26 deletions(-) create mode 100644 dataproc/src/main/java/DeleteCluster.java create mode 100644 dataproc/src/main/java/ListClusters.java create mode 100644 dataproc/src/main/java/SubmitJob.java create mode 100644 dataproc/src/main/java/UpdateCluster.java create mode 100644 dataproc/src/test/java/DeleteClusterTest.java create mode 100644 dataproc/src/test/java/ListClustersTest.java create mode 100644 dataproc/src/test/java/SubmitJobTest.java create mode 100644 dataproc/src/test/java/UpdateClusterTest.java diff --git a/dataproc/pom.xml b/dataproc/pom.xml index 87993bcc842..578889c4a60 100644 --- a/dataproc/pom.xml +++ b/dataproc/pom.xml @@ -49,6 +49,11 @@ google-cloud-dataproc 0.118.0 + + com.google.cloud + google-cloud-storage + 1.101.0 + \ No newline at end of file diff --git a/dataproc/src/main/java/CreateCluster.java b/dataproc/src/main/java/CreateCluster.java index f68852e8e2e..ed2f507dc6b 100644 --- a/dataproc/src/main/java/CreateCluster.java +++ b/dataproc/src/main/java/CreateCluster.java @@ -38,26 +38,27 @@ public static void createCluster(String projectId, String region, String cluster // Create a cluster controller client with the configured settings. We only need to create // the client once, and can be reused for multiple requests. Using a try-with-resources // will close the client for us, but this can also be done manually with the .close() method. - try (ClusterControllerClient clusterControllerClient = ClusterControllerClient - .create(clusterControllerSettings)) { + try (ClusterControllerClient clusterControllerClient = + ClusterControllerClient.create(clusterControllerSettings)) { // Configure the settings for our cluster - InstanceGroupConfig masterConfig = InstanceGroupConfig.newBuilder() - .setMachineTypeUri("n1-standard-1") - .setNumInstances(1) - .build(); - InstanceGroupConfig workerConfig = InstanceGroupConfig.newBuilder() - .setMachineTypeUri("n1-standard-1") - .setNumInstances(2) - .build(); - ClusterConfig clusterConfig = ClusterConfig.newBuilder() - .setMasterConfig(masterConfig) - .setWorkerConfig(workerConfig) - .build(); + InstanceGroupConfig masterConfig = + InstanceGroupConfig.newBuilder() + .setMachineTypeUri("n1-standard-1") + .setNumInstances(1) + .build(); + InstanceGroupConfig workerConfig = + InstanceGroupConfig.newBuilder() + .setMachineTypeUri("n1-standard-1") + .setNumInstances(2) + .build(); + ClusterConfig clusterConfig = + ClusterConfig.newBuilder() + .setMasterConfig(masterConfig) + .setWorkerConfig(workerConfig) + .build(); // Create the cluster object with the desired cluster config - Cluster cluster = Cluster.newBuilder() - .setClusterName(clusterName) - .setConfig(clusterConfig) - .build(); + Cluster cluster = + Cluster.newBuilder().setClusterName(clusterName).setConfig(clusterConfig).build(); // Create the Cloud Dataproc cluster OperationFuture createClusterAsyncRequest = @@ -66,8 +67,7 @@ public static void createCluster(String projectId, String region, String cluster // Print out a success message System.out.println( - String.format("Cluster created successfully: %s", response.getClusterName()) - ); + String.format("Cluster created successfully: %s", response.getClusterName())); } catch (IOException e) { // Likely this would occur due to issues authenticating with GCP. Make sure the environment @@ -80,4 +80,4 @@ public static void createCluster(String projectId, String region, String cluster } } } -// [END dataproc_create_cluster] \ No newline at end of file +// [END dataproc_create_cluster] diff --git a/dataproc/src/main/java/DeleteCluster.java b/dataproc/src/main/java/DeleteCluster.java new file mode 100644 index 00000000000..f94761179c2 --- /dev/null +++ b/dataproc/src/main/java/DeleteCluster.java @@ -0,0 +1,61 @@ +/* + * Copyright 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// [START dataproc_delete_cluster] +import com.google.api.gax.longrunning.OperationFuture; +import com.google.cloud.dataproc.v1.ClusterControllerClient; +import com.google.cloud.dataproc.v1.ClusterControllerSettings; +import com.google.cloud.dataproc.v1.ClusterOperationMetadata; +import com.google.protobuf.Empty; +import java.io.IOException; +import java.util.concurrent.ExecutionException; + +public class DeleteCluster { + + public static void deleteCluster(String projectId, String region, String clusterName) + throws IOException, InterruptedException { + String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); + + // Configure the settings for the cluster controller client + ClusterControllerSettings clusterControllerSettings = + ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); + + // Create a cluster controller client with the configured settings. We only need to create + // the client once, and can be reused for multiple requests. Using a try-with-resources + // will close the client for us, but this can also be done manually with the .close() method. + try (ClusterControllerClient clusterControllerClient = + ClusterControllerClient.create(clusterControllerSettings)) { + // Configure the settings for our cluster + + // Delete the Cloud Dataproc cluster + OperationFuture deleteClusterAsyncRequest = + clusterControllerClient.deleteClusterAsync(projectId, region, clusterName); + deleteClusterAsyncRequest.get(); + + // Print out a success message + System.out.println(String.format("Cluster deleted successfully: %s", clusterName)); + + } catch (IOException e) { + // Likely this would occur due to issues authenticating with GCP. Make sure the environment + // variable GOOGLE_APPLICATION_CREDENTIALS is configured. + System.out.println("Error deleting the cluster controller client: \n" + e.toString()); + } catch (ExecutionException e) { + // This will likely be due to a cluster of the given name not existing in the given region. + System.out.println("Error during cluster deletion request: \n" + e.toString()); + } + } +} +// [END dataproc_delete_cluster] diff --git a/dataproc/src/main/java/ListClusters.java b/dataproc/src/main/java/ListClusters.java new file mode 100644 index 00000000000..de8483bbc06 --- /dev/null +++ b/dataproc/src/main/java/ListClusters.java @@ -0,0 +1,49 @@ +/* + * Copyright 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// [START dataproc_list_clusters] +import com.google.cloud.dataproc.v1.Cluster; +import com.google.cloud.dataproc.v1.ClusterControllerClient; +import com.google.cloud.dataproc.v1.ClusterControllerSettings; +import java.io.IOException; + +public class ListClusters { + + public static void ListClusters(String projectId, String region) throws IOException { + String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); + + // Configure the settings for the cluster controller client + ClusterControllerSettings clusterControllerSettings = + ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); + + // Create a cluster controller client with the configured settings. We only need to create + // the client once, and can be reused for multiple requests. Using a try-with-resources + // will close the client for us, but this can also be done manually with the .close() method. + try (ClusterControllerClient clusterControllerClient = + ClusterControllerClient.create(clusterControllerSettings)) { + + for (Cluster element : clusterControllerClient.listClusters(projectId, region).iterateAll()) { + System.out.println( + String.format("%s-%s", element.getClusterName(), element.getStatus().getState())); + } + } catch (IOException e) { + // Likely this would occur due to issues authenticating with GCP. Make sure the environment + // variable GOOGLE_APPLICATION_CREDENTIALS is configured. + System.out.println("Error deleting the cluster controller client: \n" + e.toString()); + } + } +} +// [END dataproc_list_clusters] diff --git a/dataproc/src/main/java/SubmitJob.java b/dataproc/src/main/java/SubmitJob.java new file mode 100644 index 00000000000..506ac243960 --- /dev/null +++ b/dataproc/src/main/java/SubmitJob.java @@ -0,0 +1,69 @@ +/* + * Copyright 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// [START dataproc_submit_job] +import com.google.cloud.dataproc.v1.Job; +import com.google.cloud.dataproc.v1.JobControllerClient; +import com.google.cloud.dataproc.v1.JobControllerSettings; +import com.google.cloud.dataproc.v1.JobPlacement; +import com.google.cloud.dataproc.v1.PySparkJob; +import java.io.IOException; + +public class SubmitJob { + + public static void submitJob( + String projectId, String region, String clusterName, String job_file_path) + throws IOException { + String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); + + // Configure the settings for the cluster controller client + JobControllerSettings jobControllerSettings = + JobControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); + + // Create a cluster controller client with the configured settings. We only need to create + // the client once, and can be reused for multiple requests. Using a try-with-resources + // will close the client for us, but this can also be done manually with the .close() method. + try (JobControllerClient jobControllerClient = + JobControllerClient.create(jobControllerSettings)) { + + // Configure the settings for our job + JobPlacement jobPlacement = JobPlacement.newBuilder().setClusterName(clusterName).build(); + PySparkJob pySparkJob = PySparkJob.newBuilder().setMainPythonFileUri(job_file_path).build(); + Job job = Job.newBuilder().setPlacement(jobPlacement).setPysparkJob(pySparkJob).build(); + + Job jobOperation = jobControllerClient.submitJob(projectId, region, job); + + String jobId = jobOperation.getReference().getJobId(); + + while (true) { + Job jobInfo = jobControllerClient.getJob(projectId, region, jobId); + if (jobInfo.getStatus().getState().toString() == "ERROR") { + System.out.println( + String.format("Job %s failed: %s", jobId, jobInfo.getStatus().getDetails())); + break; + } else if (jobInfo.getStatus().getState().toString() == "DONE") { + System.out.println(String.format("Job %s finished.", jobId)); + break; + } + } + } catch (IOException e) { + // Likely this would occur due to issues authenticating with GCP. Make sure the environment + // variable GOOGLE_APPLICATION_CREDENTIALS is configured. + System.out.println("Error creating the cluster controller client: \n" + e.toString()); + } + } +} +// [END dataproc_submit_job] diff --git a/dataproc/src/main/java/UpdateCluster.java b/dataproc/src/main/java/UpdateCluster.java new file mode 100644 index 00000000000..3727c8f47de --- /dev/null +++ b/dataproc/src/main/java/UpdateCluster.java @@ -0,0 +1,92 @@ +/* + * Copyright 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// [START dataproc_update_cluster] +import com.google.api.gax.longrunning.OperationFuture; +import com.google.cloud.dataproc.v1.Cluster; +import com.google.cloud.dataproc.v1.ClusterConfig; +import com.google.cloud.dataproc.v1.ClusterControllerClient; +import com.google.cloud.dataproc.v1.ClusterControllerSettings; +import com.google.cloud.dataproc.v1.ClusterOperationMetadata; +import com.google.cloud.dataproc.v1.InstanceGroupConfig; +import com.google.protobuf.FieldMask; +import java.io.IOException; +import java.util.concurrent.ExecutionException; + +public class UpdateCluster { + + public static void updateCluster( + String projectId, String region, String clusterName, int numWorkers) + throws IOException, InterruptedException { + String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); + + // Configure the settings for the cluster controller client + ClusterControllerSettings clusterControllerSettings = + ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); + + // Create a cluster controller client with the configured settings. We only need to create + // the client once, and can be reused for multiple requests. Using a try-with-resources + // will close the client for us, but this can also be done manually with the .close() method. + try (ClusterControllerClient clusterControllerClient = + ClusterControllerClient.create(clusterControllerSettings)) { + // Configure the settings for our cluster + InstanceGroupConfig masterConfig = + InstanceGroupConfig.newBuilder() + .setMachineTypeUri("n1-standard-1") + .setNumInstances(1) + .build(); + InstanceGroupConfig workerConfig = + InstanceGroupConfig.newBuilder() + .setMachineTypeUri("n1-standard-1") + .setNumInstances(numWorkers) + .build(); + ClusterConfig clusterConfig = + ClusterConfig.newBuilder() + .setMasterConfig(masterConfig) + .setWorkerConfig(workerConfig) + .build(); + // Create the cluster object with the desired cluster config + Cluster cluster = + Cluster.newBuilder().setClusterName(clusterName).setConfig(clusterConfig).build(); + + // Create a field mask to indicate what field of the cluster to update. + FieldMask updateMask = + FieldMask.newBuilder().addPaths("config.worker_config.num_instances").build(); + + // Submit the request to update the cluster + OperationFuture updateClusterAsyncRequest = + clusterControllerClient.updateClusterAsync( + projectId, region, clusterName, cluster, updateMask); + Cluster response = updateClusterAsyncRequest.get(); + + // Print out a success message + System.out.println( + String.format( + "Cluster %s now has %d workers.", + response.getClusterName(), response.getConfig().getWorkerConfig().getNumInstances())); + + } catch (IOException e) { + // Likely this would occur due to issues authenticating with GCP. Make sure the environment + // variable GOOGLE_APPLICATION_CREDENTIALS is configured. + System.out.println("Error creating the cluster controller client: \n" + e.toString()); + } catch (ExecutionException e) { + // Common issues for this include needing to increase compute engine quotas or a cluster of + // the same name already exists. + System.out.println("Error during cluster creation request: \n" + e.toString()); + } + } +} +// [END dataproc_update_cluster] diff --git a/dataproc/src/test/java/CreateClusterTest.java b/dataproc/src/test/java/CreateClusterTest.java index 800856b02dc..f98c24a9b4d 100644 --- a/dataproc/src/test/java/CreateClusterTest.java +++ b/dataproc/src/test/java/CreateClusterTest.java @@ -48,8 +48,7 @@ public class CreateClusterTest { private static void requireEnv(String varName) { assertNotNull( System.getenv(varName), - String.format("Environment variable '%s' is required to perform these tests.", varName) - ); + String.format("Environment variable '%s' is required to perform these tests.", varName)); } @BeforeClass @@ -81,8 +80,8 @@ public void tearDown() throws IOException, InterruptedException { ClusterControllerSettings clusterControllerSettings = ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); - try (ClusterControllerClient clusterControllerClient = ClusterControllerClient - .create(clusterControllerSettings)) { + try (ClusterControllerClient clusterControllerClient = + ClusterControllerClient.create(clusterControllerSettings)) { OperationFuture deleteClusterAsyncRequest = clusterControllerClient.deleteClusterAsync(projectId, REGION, clusterName); deleteClusterAsyncRequest.get(); @@ -91,4 +90,4 @@ public void tearDown() throws IOException, InterruptedException { System.out.println("Error during cluster deletion: \n" + e.toString()); } } -} \ No newline at end of file +} diff --git a/dataproc/src/test/java/DeleteClusterTest.java b/dataproc/src/test/java/DeleteClusterTest.java new file mode 100644 index 00000000000..82d4c453e6e --- /dev/null +++ b/dataproc/src/test/java/DeleteClusterTest.java @@ -0,0 +1,95 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import static junit.framework.TestCase.assertNotNull; +import static org.hamcrest.MatcherAssert.assertThat; + +import com.google.api.gax.longrunning.OperationFuture; +import com.google.cloud.dataproc.v1.Cluster; +import com.google.cloud.dataproc.v1.ClusterConfig; +import com.google.cloud.dataproc.v1.ClusterControllerClient; +import com.google.cloud.dataproc.v1.ClusterControllerSettings; +import com.google.cloud.dataproc.v1.ClusterOperationMetadata; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import org.hamcrest.CoreMatchers; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class DeleteClusterTest { + + private static final String BASE_CLUSTER_NAME = "test-cluster"; + private static final String REGION = "us-central1"; + + private static String projectId = System.getenv("GOOGLE_CLOUD_PROJECT"); + private String clusterName; + private ByteArrayOutputStream bout; + + private static void requireEnv(String varName) { + assertNotNull( + System.getenv(varName), + String.format("Environment variable '%s' is required to perform these tests.", varName)); + } + + @BeforeClass + public static void checkRequirements() { + requireEnv("GOOGLE_APPLICATION_CREDENTIALS"); + requireEnv("GOOGLE_CLOUD_PROJECT"); + } + + @Before + public void setUp() throws IOException, InterruptedException { + clusterName = String.format("%s-%s", BASE_CLUSTER_NAME, UUID.randomUUID().toString()); + + bout = new ByteArrayOutputStream(); + System.setOut(new PrintStream(bout)); + + String myEndpoint = String.format("%s-dataproc.googleapis.com:443", REGION); + + ClusterControllerSettings clusterControllerSettings = + ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); + + Cluster cluster = + Cluster.newBuilder() + .setClusterName(clusterName) + .setConfig(ClusterConfig.newBuilder().build()) + .build(); + + try (ClusterControllerClient clusterControllerClient = + ClusterControllerClient.create(clusterControllerSettings)) { + OperationFuture createClusterAsyncRequest = + clusterControllerClient.createClusterAsync(projectId, REGION, cluster); + createClusterAsyncRequest.get(); + } catch (ExecutionException e) { + System.out.println("[deleteCluster] Error during test cluster creation: \n" + e.toString()); + } + } + + @Test + public void DeleteClusterTest() throws IOException, InterruptedException { + DeleteCluster.deleteCluster(projectId, REGION, clusterName); + String output = bout.toString(); + + assertThat(output, CoreMatchers.containsString("deleted successfully")); + } +} diff --git a/dataproc/src/test/java/ListClustersTest.java b/dataproc/src/test/java/ListClustersTest.java new file mode 100644 index 00000000000..067dfed0a6e --- /dev/null +++ b/dataproc/src/test/java/ListClustersTest.java @@ -0,0 +1,115 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import static junit.framework.TestCase.assertNotNull; +import static org.hamcrest.MatcherAssert.assertThat; + +import com.google.api.gax.longrunning.OperationFuture; +import com.google.cloud.dataproc.v1.Cluster; +import com.google.cloud.dataproc.v1.ClusterConfig; +import com.google.cloud.dataproc.v1.ClusterControllerClient; +import com.google.cloud.dataproc.v1.ClusterControllerSettings; +import com.google.cloud.dataproc.v1.ClusterOperationMetadata; +import com.google.protobuf.Empty; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import org.hamcrest.CoreMatchers; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class ListClustersTest { + + private static final String BASE_CLUSTER_NAME = "test-cluster"; + private static final String REGION = "us-central1"; + private static final String projectId = System.getenv("GOOGLE_CLOUD_PROJECT"); + + private String clusterName; + private String myEndpoint; + private ByteArrayOutputStream bout; + private PrintStream standardOutOrig; + + private static void requireEnv(String varName) { + assertNotNull( + System.getenv(varName), + String.format("Environment variable '%s' is required to perform these tests.", varName)); + } + + @BeforeClass + public static void checkRequirements() { + requireEnv("GOOGLE_APPLICATION_CREDENTIALS"); + requireEnv("GOOGLE_CLOUD_PROJECT"); + } + + @Before + public void setUp() throws IOException, InterruptedException { + clusterName = String.format("%s-%s", BASE_CLUSTER_NAME, UUID.randomUUID().toString()); + bout = new ByteArrayOutputStream(); + standardOutOrig = System.out; + System.setOut(new PrintStream(bout)); + myEndpoint = String.format("%s-dataproc.googleapis.com:443", REGION); + + ClusterControllerSettings clusterControllerSettings = + ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); + + Cluster cluster = + Cluster.newBuilder() + .setClusterName(clusterName) + .setConfig(ClusterConfig.newBuilder().build()) + .build(); + + try (ClusterControllerClient clusterControllerClient = + ClusterControllerClient.create(clusterControllerSettings)) { + OperationFuture createClusterAsyncRequest = + clusterControllerClient.createClusterAsync(projectId, REGION, cluster); + createClusterAsyncRequest.get(); + } catch (ExecutionException e) { + System.out.println("[deleteCluster] Error during test cluster creation: \n" + e.toString()); + } + } + + @Test + public void ListClustersTest() throws IOException, InterruptedException { + ListClusters.ListClusters(projectId, REGION); + String output = bout.toString(); + + assertThat(output, CoreMatchers.containsString(clusterName)); + assertThat(output, CoreMatchers.containsString("RUNNING")); + } + + @After + public void teardown() throws IOException, InterruptedException { + System.setOut(standardOutOrig); + ClusterControllerSettings clusterControllerSettings = + ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); + + try (ClusterControllerClient clusterControllerClient = + ClusterControllerClient.create(clusterControllerSettings)) { + OperationFuture deleteClusterAsyncRequest = + clusterControllerClient.deleteClusterAsync(projectId, REGION, clusterName); + deleteClusterAsyncRequest.get(); + } catch (ExecutionException e) { + System.out.println("[deleteCluster] Error during test cluster creation: \n" + e.toString()); + } + } +} diff --git a/dataproc/src/test/java/SubmitJobTest.java b/dataproc/src/test/java/SubmitJobTest.java new file mode 100644 index 00000000000..cc6413e02c8 --- /dev/null +++ b/dataproc/src/test/java/SubmitJobTest.java @@ -0,0 +1,135 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import static java.nio.charset.StandardCharsets.UTF_8; +import static junit.framework.TestCase.assertNotNull; +import static org.hamcrest.MatcherAssert.assertThat; + +import com.google.api.gax.longrunning.OperationFuture; +import com.google.cloud.dataproc.v1.Cluster; +import com.google.cloud.dataproc.v1.ClusterConfig; +import com.google.cloud.dataproc.v1.ClusterControllerClient; +import com.google.cloud.dataproc.v1.ClusterControllerSettings; +import com.google.cloud.dataproc.v1.ClusterOperationMetadata; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.Bucket; +import com.google.cloud.storage.BucketInfo; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageOptions; +import com.google.protobuf.Empty; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import org.hamcrest.CoreMatchers; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class SubmitJobTest { + + private static final String MY_UUID = UUID.randomUUID().toString(); + private static final String REGION = "us-central1"; + private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); + private static final String ENDPOINT = String.format("%s-dataproc.googleapis.com:443", REGION); + private static final String CLUSTER_NAME = String.format("%s-%s", "test-cluster", MY_UUID); + private static final String BUCKET_NAME = String.format("%s-%s", "test-bucket", MY_UUID); + private static final String JOB_FILE_NAME = "sum.py"; + private static final String JOB_FILE_PATH = + String.format("gs://%s/%s", BUCKET_NAME, JOB_FILE_NAME); + private static final String SORT_CODE = + "import pyspark\n" + + "sc = pyspark.SparkContext()\n" + + "rdd = sc.parallelize((1,2,3,4,5))\n" + + "sum = rdd.reduce(lambda x, y: x + y)\n"; + + private ByteArrayOutputStream bout; + private PrintStream standardOutOrig; + private Storage storage; + private Blob blob; + + private static void requireEnv(String varName) { + assertNotNull( + System.getenv(varName), + String.format("Environment variable '%s' is required to perform these tests.", varName)); + } + + @BeforeClass + public static void checkRequirements() { + requireEnv("GOOGLE_APPLICATION_CREDENTIALS"); + requireEnv("GOOGLE_CLOUD_PROJECT"); + } + + @Before + public void setUp() throws IOException, InterruptedException { + bout = new ByteArrayOutputStream(); + standardOutOrig = System.out; + System.setOut(new PrintStream(bout)); + + storage = StorageOptions.getDefaultInstance().getService(); + Bucket bucket = storage.create(BucketInfo.of(BUCKET_NAME)); + blob = bucket.create(JOB_FILE_NAME, SORT_CODE.getBytes(UTF_8), "text/plain"); + + ClusterControllerSettings clusterControllerSettings = + ClusterControllerSettings.newBuilder().setEndpoint(ENDPOINT).build(); + + Cluster cluster = + Cluster.newBuilder() + .setClusterName(CLUSTER_NAME) + .setConfig(ClusterConfig.newBuilder().build()) + .build(); + + try (ClusterControllerClient clusterControllerClient = + ClusterControllerClient.create(clusterControllerSettings)) { + OperationFuture createClusterAsyncRequest = + clusterControllerClient.createClusterAsync(PROJECT_ID, REGION, cluster); + createClusterAsyncRequest.get(); + } catch (ExecutionException e) { + System.out.println("[SubmitJob] Error during test cluster creation: \n" + e.toString()); + } + } + + @Test + public void submitJobTest() throws IOException { + SubmitJob.submitJob(PROJECT_ID, REGION, CLUSTER_NAME, JOB_FILE_PATH); + String output = bout.toString(); + + assertThat(output, CoreMatchers.containsString("finished")); + } + + @After + public void teardown() throws IOException, InterruptedException { + System.setOut(standardOutOrig); + blob.delete(); + + ClusterControllerSettings clusterControllerSettings = + ClusterControllerSettings.newBuilder().setEndpoint(ENDPOINT).build(); + + try (ClusterControllerClient clusterControllerClient = + ClusterControllerClient.create(clusterControllerSettings)) { + OperationFuture deleteClusterAsyncRequest = + clusterControllerClient.deleteClusterAsync(PROJECT_ID, REGION, CLUSTER_NAME); + deleteClusterAsyncRequest.get(); + } catch (ExecutionException e) { + System.out.println("[SubmitJob] Error during test cluster deletion: \n" + e.toString()); + } + } +} diff --git a/dataproc/src/test/java/UpdateClusterTest.java b/dataproc/src/test/java/UpdateClusterTest.java new file mode 100644 index 00000000000..7b19fdc1ebf --- /dev/null +++ b/dataproc/src/test/java/UpdateClusterTest.java @@ -0,0 +1,130 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import static junit.framework.TestCase.assertNotNull; +import static org.hamcrest.MatcherAssert.assertThat; + +import com.google.api.gax.longrunning.OperationFuture; +import com.google.cloud.dataproc.v1.Cluster; +import com.google.cloud.dataproc.v1.ClusterConfig; +import com.google.cloud.dataproc.v1.ClusterControllerClient; +import com.google.cloud.dataproc.v1.ClusterControllerSettings; +import com.google.cloud.dataproc.v1.ClusterOperationMetadata; +import com.google.cloud.dataproc.v1.InstanceGroupConfig; +import com.google.protobuf.Empty; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import org.hamcrest.CoreMatchers; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class UpdateClusterTest { + + private static final String MY_UUID = UUID.randomUUID().toString(); + private static final String REGION = "us-central1"; + private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); + private static final String ENDPOINT = String.format("%s-dataproc.googleapis.com:443", REGION); + private static final String CLUSTER_NAME = String.format("%s-%s", "test-cluster", MY_UUID); + private static final int NUM_WORKERS = 2; + private static final int NEW_WORKERS = NUM_WORKERS * 2; + + private ByteArrayOutputStream bout; + private PrintStream standardOutOrig; + + private static void requireEnv(String varName) { + assertNotNull( + System.getenv(varName), + String.format("Environment variable '%s' is required to perform these tests.", varName)); + } + + @BeforeClass + public static void checkRequirements() { + requireEnv("GOOGLE_APPLICATION_CREDENTIALS"); + requireEnv("GOOGLE_CLOUD_PROJECT"); + } + + @Before + public void setUp() throws IOException, InterruptedException { + bout = new ByteArrayOutputStream(); + standardOutOrig = System.out; + System.setOut(new PrintStream(bout)); + + ClusterControllerSettings clusterControllerSettings = + ClusterControllerSettings.newBuilder().setEndpoint(ENDPOINT).build(); + + // Configure the settings for our cluster + InstanceGroupConfig masterConfig = + InstanceGroupConfig.newBuilder() + .setMachineTypeUri("n1-standard-1") + .setNumInstances(1) + .build(); + InstanceGroupConfig workerConfig = + InstanceGroupConfig.newBuilder() + .setMachineTypeUri("n1-standard-1") + .setNumInstances(NUM_WORKERS) + .build(); + ClusterConfig clusterConfig = + ClusterConfig.newBuilder() + .setMasterConfig(masterConfig) + .setWorkerConfig(workerConfig) + .build(); + // Create the cluster object with the desired cluster config + Cluster cluster = + Cluster.newBuilder().setClusterName(CLUSTER_NAME).setConfig(clusterConfig).build(); + + try (ClusterControllerClient clusterControllerClient = + ClusterControllerClient.create(clusterControllerSettings)) { + OperationFuture createClusterAsyncRequest = + clusterControllerClient.createClusterAsync(PROJECT_ID, REGION, cluster); + createClusterAsyncRequest.get(); + } catch (ExecutionException e) { + System.out.println("[UpdateCluster] Error during test cluster creation: \n" + e.toString()); + } + } + + @Test + public void updateClusterTest() throws IOException, InterruptedException { + UpdateCluster.updateCluster(PROJECT_ID, REGION, CLUSTER_NAME, NEW_WORKERS); + String output = bout.toString(); + + assertThat(output, CoreMatchers.containsString(String.format("%d workers", NEW_WORKERS))); + } + + @After + public void teardown() throws IOException, InterruptedException { + System.setOut(standardOutOrig); + + ClusterControllerSettings clusterControllerSettings = + ClusterControllerSettings.newBuilder().setEndpoint(ENDPOINT).build(); + + try (ClusterControllerClient clusterControllerClient = + ClusterControllerClient.create(clusterControllerSettings)) { + OperationFuture deleteClusterAsyncRequest = + clusterControllerClient.deleteClusterAsync(PROJECT_ID, REGION, CLUSTER_NAME); + deleteClusterAsyncRequest.get(); + } catch (ExecutionException e) { + System.out.println("[UpdateCluster] Error during test cluster update: \n" + e.toString()); + } + } +} From 2970060f2193040fd435ac999be73bae509d37e1 Mon Sep 17 00:00:00 2001 From: bradmiro Date: Mon, 2 Dec 2019 14:37:20 -0500 Subject: [PATCH 09/21] Fixed linting, string formatting, copyrights --- dataproc/src/main/java/CreateCluster.java | 17 ++++++------ dataproc/src/main/java/DeleteCluster.java | 16 +++++------ dataproc/src/main/java/ListClusters.java | 17 ++++++------ dataproc/src/main/java/SubmitJob.java | 27 +++++++++---------- dataproc/src/main/java/UpdateCluster.java | 21 +++++++-------- dataproc/src/test/java/CreateClusterTest.java | 20 +++++++------- dataproc/src/test/java/DeleteClusterTest.java | 27 +++++++++++-------- dataproc/src/test/java/ListClustersTest.java | 25 +++++++++-------- dataproc/src/test/java/SubmitJobTest.java | 13 +++++---- dataproc/src/test/java/UpdateClusterTest.java | 9 +++---- 10 files changed, 95 insertions(+), 97 deletions(-) diff --git a/dataproc/src/main/java/CreateCluster.java b/dataproc/src/main/java/CreateCluster.java index ed2f507dc6b..ebae59dd53c 100644 --- a/dataproc/src/main/java/CreateCluster.java +++ b/dataproc/src/main/java/CreateCluster.java @@ -1,5 +1,5 @@ /* - * Copyright 2019 Google Inc. + * Copyright 2019 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,15 +29,15 @@ public class CreateCluster { public static void createCluster(String projectId, String region, String clusterName) throws IOException, InterruptedException { - String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); + String myEndpoint = region + "-dataproc.googleapis.com:443"; // Configure the settings for the cluster controller client ClusterControllerSettings clusterControllerSettings = ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); - // Create a cluster controller client with the configured settings. We only need to create - // the client once, and can be reused for multiple requests. Using a try-with-resources - // will close the client for us, but this can also be done manually with the .close() method. + // Create a cluster controller client with the configured settings. The client only needs to be + // created once and can be reused for multiple requests. Using a try-with-resources + // closes the client, but this can also be done manually with the .close() method. try (ClusterControllerClient clusterControllerClient = ClusterControllerClient.create(clusterControllerSettings)) { // Configure the settings for our cluster @@ -66,17 +66,16 @@ public static void createCluster(String projectId, String region, String cluster Cluster response = createClusterAsyncRequest.get(); // Print out a success message - System.out.println( - String.format("Cluster created successfully: %s", response.getClusterName())); + System.out.println("Cluster created successfully: " + response.getClusterName()); } catch (IOException e) { // Likely this would occur due to issues authenticating with GCP. Make sure the environment // variable GOOGLE_APPLICATION_CREDENTIALS is configured. - System.out.println("Error creating the cluster controller client: \n" + e.toString()); + System.err.println("Error creating the cluster controller client: \n" + e.getMessage()); } catch (ExecutionException e) { // Common issues for this include needing to increase compute engine quotas or a cluster of // the same name already exists. - System.out.println("Error during cluster creation request: \n" + e.toString()); + System.err.println("Error during cluster creation request: \n" + e.getMessage()); } } } diff --git a/dataproc/src/main/java/DeleteCluster.java b/dataproc/src/main/java/DeleteCluster.java index f94761179c2..84736b773cf 100644 --- a/dataproc/src/main/java/DeleteCluster.java +++ b/dataproc/src/main/java/DeleteCluster.java @@ -1,5 +1,5 @@ /* - * Copyright 2019 Google Inc. + * Copyright 2019 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,15 +27,15 @@ public class DeleteCluster { public static void deleteCluster(String projectId, String region, String clusterName) throws IOException, InterruptedException { - String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); + String myEndpoint = region + "-dataproc.googleapis.com:443"; // Configure the settings for the cluster controller client ClusterControllerSettings clusterControllerSettings = ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); - // Create a cluster controller client with the configured settings. We only need to create - // the client once, and can be reused for multiple requests. Using a try-with-resources - // will close the client for us, but this can also be done manually with the .close() method. + // Create a cluster controller client with the configured settings. The client only needs to be + // created once and can be reused for multiple requests. Using a try-with-resources + // closes the client, but this can also be done manually with the .close() method. try (ClusterControllerClient clusterControllerClient = ClusterControllerClient.create(clusterControllerSettings)) { // Configure the settings for our cluster @@ -46,15 +46,15 @@ public static void deleteCluster(String projectId, String region, String cluster deleteClusterAsyncRequest.get(); // Print out a success message - System.out.println(String.format("Cluster deleted successfully: %s", clusterName)); + System.out.println("Cluster deleted successfully: " + clusterName); } catch (IOException e) { // Likely this would occur due to issues authenticating with GCP. Make sure the environment // variable GOOGLE_APPLICATION_CREDENTIALS is configured. - System.out.println("Error deleting the cluster controller client: \n" + e.toString()); + System.err.println("Error deleting the cluster controller client: \n" + e.getMessage()); } catch (ExecutionException e) { // This will likely be due to a cluster of the given name not existing in the given region. - System.out.println("Error during cluster deletion request: \n" + e.toString()); + System.err.println("Error during cluster deletion request: \n" + e.getMessage()); } } } diff --git a/dataproc/src/main/java/ListClusters.java b/dataproc/src/main/java/ListClusters.java index de8483bbc06..b0b0a68b905 100644 --- a/dataproc/src/main/java/ListClusters.java +++ b/dataproc/src/main/java/ListClusters.java @@ -1,5 +1,5 @@ /* - * Copyright 2019 Google Inc. + * Copyright 2019 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,27 +22,26 @@ public class ListClusters { - public static void ListClusters(String projectId, String region) throws IOException { - String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); + public static void listClusters(String projectId, String region) throws IOException { + String myEndpoint = region + "-dataproc.googleapis.com:443"; // Configure the settings for the cluster controller client ClusterControllerSettings clusterControllerSettings = ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); - // Create a cluster controller client with the configured settings. We only need to create - // the client once, and can be reused for multiple requests. Using a try-with-resources - // will close the client for us, but this can also be done manually with the .close() method. + // Create a cluster controller client with the configured settings. The client only needs to be + // created once and can be reused for multiple requests. Using a try-with-resources + // closes the client, but this can also be done manually with the .close() method. try (ClusterControllerClient clusterControllerClient = ClusterControllerClient.create(clusterControllerSettings)) { for (Cluster element : clusterControllerClient.listClusters(projectId, region).iterateAll()) { - System.out.println( - String.format("%s-%s", element.getClusterName(), element.getStatus().getState())); + System.out.println(element.getClusterName() + ": " + element.getStatus().getState()); } } catch (IOException e) { // Likely this would occur due to issues authenticating with GCP. Make sure the environment // variable GOOGLE_APPLICATION_CREDENTIALS is configured. - System.out.println("Error deleting the cluster controller client: \n" + e.toString()); + System.out.println("Error deleting the cluster controller client: \n" + e.getMessage()); } } } diff --git a/dataproc/src/main/java/SubmitJob.java b/dataproc/src/main/java/SubmitJob.java index 506ac243960..01bce05569b 100644 --- a/dataproc/src/main/java/SubmitJob.java +++ b/dataproc/src/main/java/SubmitJob.java @@ -1,5 +1,5 @@ /* - * Copyright 2019 Google Inc. + * Copyright 2019 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,29 +19,29 @@ import com.google.cloud.dataproc.v1.JobControllerClient; import com.google.cloud.dataproc.v1.JobControllerSettings; import com.google.cloud.dataproc.v1.JobPlacement; +import com.google.cloud.dataproc.v1.JobStatus; import com.google.cloud.dataproc.v1.PySparkJob; import java.io.IOException; public class SubmitJob { public static void submitJob( - String projectId, String region, String clusterName, String job_file_path) - throws IOException { - String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); + String projectId, String region, String clusterName, String jobFilePath) throws IOException { + String myEndpoint = region + "-dataproc.googleapis.com:443"; // Configure the settings for the cluster controller client JobControllerSettings jobControllerSettings = JobControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); - // Create a cluster controller client with the configured settings. We only need to create - // the client once, and can be reused for multiple requests. Using a try-with-resources - // will close the client for us, but this can also be done manually with the .close() method. + // Create a job controller client with the configured settings. The client only needs to be + // created once and can be reused for multiple requests. Using a try-with-resources + // closes the client, but this can also be done manually with the .close() method. try (JobControllerClient jobControllerClient = JobControllerClient.create(jobControllerSettings)) { // Configure the settings for our job JobPlacement jobPlacement = JobPlacement.newBuilder().setClusterName(clusterName).build(); - PySparkJob pySparkJob = PySparkJob.newBuilder().setMainPythonFileUri(job_file_path).build(); + PySparkJob pySparkJob = PySparkJob.newBuilder().setMainPythonFileUri(jobFilePath).build(); Job job = Job.newBuilder().setPlacement(jobPlacement).setPysparkJob(pySparkJob).build(); Job jobOperation = jobControllerClient.submitJob(projectId, region, job); @@ -50,19 +50,18 @@ public static void submitJob( while (true) { Job jobInfo = jobControllerClient.getJob(projectId, region, jobId); - if (jobInfo.getStatus().getState().toString() == "ERROR") { - System.out.println( - String.format("Job %s failed: %s", jobId, jobInfo.getStatus().getDetails())); + if (jobInfo.getStatus().getState().equals(JobStatus.State.ERROR)) { + System.err.printf("Job %s failed: %s", jobId, jobInfo.getStatus().getDetails()); break; - } else if (jobInfo.getStatus().getState().toString() == "DONE") { - System.out.println(String.format("Job %s finished.", jobId)); + } else if (jobInfo.getStatus().getState().equals(JobStatus.State.DONE)) { + System.out.printf("Job %s finished.", jobId); break; } } } catch (IOException e) { // Likely this would occur due to issues authenticating with GCP. Make sure the environment // variable GOOGLE_APPLICATION_CREDENTIALS is configured. - System.out.println("Error creating the cluster controller client: \n" + e.toString()); + System.err.println("Error creating the cluster controller client: \n" + e.getMessage()); } } } diff --git a/dataproc/src/main/java/UpdateCluster.java b/dataproc/src/main/java/UpdateCluster.java index 3727c8f47de..6cfba7bcace 100644 --- a/dataproc/src/main/java/UpdateCluster.java +++ b/dataproc/src/main/java/UpdateCluster.java @@ -1,5 +1,5 @@ /* - * Copyright 2019 Google Inc. + * Copyright 2019 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,15 +31,15 @@ public class UpdateCluster { public static void updateCluster( String projectId, String region, String clusterName, int numWorkers) throws IOException, InterruptedException { - String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); + String myEndpoint = region + "-dataproc.googleapis.com:443"; // Configure the settings for the cluster controller client ClusterControllerSettings clusterControllerSettings = ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); - // Create a cluster controller client with the configured settings. We only need to create - // the client once, and can be reused for multiple requests. Using a try-with-resources - // will close the client for us, but this can also be done manually with the .close() method. + // Create a cluster controller client with the configured settings. The client only needs to be + // created once and can be reused for multiple requests. Using a try-with-resources + // closes the client, but this can also be done manually with the .close() method. try (ClusterControllerClient clusterControllerClient = ClusterControllerClient.create(clusterControllerSettings)) { // Configure the settings for our cluster @@ -73,19 +73,18 @@ public static void updateCluster( Cluster response = updateClusterAsyncRequest.get(); // Print out a success message - System.out.println( - String.format( - "Cluster %s now has %d workers.", - response.getClusterName(), response.getConfig().getWorkerConfig().getNumInstances())); + System.out.printf( + "Cluster %s now has %d workers.", + response.getClusterName(), response.getConfig().getWorkerConfig().getNumInstances()); } catch (IOException e) { // Likely this would occur due to issues authenticating with GCP. Make sure the environment // variable GOOGLE_APPLICATION_CREDENTIALS is configured. - System.out.println("Error creating the cluster controller client: \n" + e.toString()); + System.err.println("Error creating the cluster controller client: \n" + e.getMessage()); } catch (ExecutionException e) { // Common issues for this include needing to increase compute engine quotas or a cluster of // the same name already exists. - System.out.println("Error during cluster creation request: \n" + e.toString()); + System.out.println("Error during cluster creation request: \n" + e.getMessage()); } } } diff --git a/dataproc/src/test/java/CreateClusterTest.java b/dataproc/src/test/java/CreateClusterTest.java index f98c24a9b4d..fc661e371d3 100644 --- a/dataproc/src/test/java/CreateClusterTest.java +++ b/dataproc/src/test/java/CreateClusterTest.java @@ -38,12 +38,12 @@ @RunWith(JUnit4.class) public class CreateClusterTest { - private static final String BASE_CLUSTER_NAME = "test-cluster"; + private static final String CLUSTER_NAME = "test-cluster-" + UUID.randomUUID().toString(); private static final String REGION = "us-central1"; + private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); - private static String projectId = System.getenv("GOOGLE_CLOUD_PROJECT"); - private String clusterName; private ByteArrayOutputStream bout; + private PrintStream standardOutOrig; private static void requireEnv(String varName) { assertNotNull( @@ -59,23 +59,23 @@ public static void checkRequirements() { @Before public void setUp() { - clusterName = String.format("%s-%s", BASE_CLUSTER_NAME, UUID.randomUUID().toString()); - bout = new ByteArrayOutputStream(); System.setOut(new PrintStream(bout)); + standardOutOrig = System.out; } @Test public void createClusterTest() throws IOException, InterruptedException { - CreateCluster.createCluster(projectId, REGION, clusterName); + CreateCluster.createCluster(PROJECT_ID, REGION, CLUSTER_NAME); String output = bout.toString(); - assertThat(output, CoreMatchers.containsString(clusterName)); + assertThat(output, CoreMatchers.containsString(CLUSTER_NAME)); } @After public void tearDown() throws IOException, InterruptedException { - String myEndpoint = String.format("%s-dataproc.googleapis.com:443", REGION); + String myEndpoint = REGION + "-dataproc.googleapis.com:443"; + System.setOut(standardOutOrig); ClusterControllerSettings clusterControllerSettings = ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); @@ -83,11 +83,11 @@ public void tearDown() throws IOException, InterruptedException { try (ClusterControllerClient clusterControllerClient = ClusterControllerClient.create(clusterControllerSettings)) { OperationFuture deleteClusterAsyncRequest = - clusterControllerClient.deleteClusterAsync(projectId, REGION, clusterName); + clusterControllerClient.deleteClusterAsync(PROJECT_ID, REGION, CLUSTER_NAME); deleteClusterAsyncRequest.get(); } catch (ExecutionException e) { - System.out.println("Error during cluster deletion: \n" + e.toString()); + System.err.println("Error during cluster deletion: \n" + e.getMessage()); } } } diff --git a/dataproc/src/test/java/DeleteClusterTest.java b/dataproc/src/test/java/DeleteClusterTest.java index 82d4c453e6e..0ec6797463c 100644 --- a/dataproc/src/test/java/DeleteClusterTest.java +++ b/dataproc/src/test/java/DeleteClusterTest.java @@ -29,6 +29,7 @@ import java.util.UUID; import java.util.concurrent.ExecutionException; import org.hamcrest.CoreMatchers; +import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -38,12 +39,12 @@ @RunWith(JUnit4.class) public class DeleteClusterTest { - private static final String BASE_CLUSTER_NAME = "test-cluster"; + private static final String CLUSTER_NAME = "test-cluster-" + UUID.randomUUID().toString(); private static final String REGION = "us-central1"; + private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); - private static String projectId = System.getenv("GOOGLE_CLOUD_PROJECT"); - private String clusterName; private ByteArrayOutputStream bout; + private PrintStream standardOutOrig; private static void requireEnv(String varName) { assertNotNull( @@ -59,37 +60,41 @@ public static void checkRequirements() { @Before public void setUp() throws IOException, InterruptedException { - clusterName = String.format("%s-%s", BASE_CLUSTER_NAME, UUID.randomUUID().toString()); - bout = new ByteArrayOutputStream(); System.setOut(new PrintStream(bout)); + standardOutOrig = System.out; - String myEndpoint = String.format("%s-dataproc.googleapis.com:443", REGION); + String myEndpoint = REGION + "-dataproc.googleapis.com:443"; ClusterControllerSettings clusterControllerSettings = ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); Cluster cluster = Cluster.newBuilder() - .setClusterName(clusterName) + .setClusterName(CLUSTER_NAME) .setConfig(ClusterConfig.newBuilder().build()) .build(); try (ClusterControllerClient clusterControllerClient = ClusterControllerClient.create(clusterControllerSettings)) { OperationFuture createClusterAsyncRequest = - clusterControllerClient.createClusterAsync(projectId, REGION, cluster); + clusterControllerClient.createClusterAsync(PROJECT_ID, REGION, cluster); createClusterAsyncRequest.get(); } catch (ExecutionException e) { - System.out.println("[deleteCluster] Error during test cluster creation: \n" + e.toString()); + System.err.println("[deleteCluster] Error during test cluster creation: \n" + e.getMessage()); } } @Test - public void DeleteClusterTest() throws IOException, InterruptedException { - DeleteCluster.deleteCluster(projectId, REGION, clusterName); + public void deleteClusterTest() throws IOException, InterruptedException { + DeleteCluster.deleteCluster(PROJECT_ID, REGION, CLUSTER_NAME); String output = bout.toString(); assertThat(output, CoreMatchers.containsString("deleted successfully")); } + + @After + public void teardown() { + System.setOut(standardOutOrig); + } } diff --git a/dataproc/src/test/java/ListClustersTest.java b/dataproc/src/test/java/ListClustersTest.java index 067dfed0a6e..4d0272ea797 100644 --- a/dataproc/src/test/java/ListClustersTest.java +++ b/dataproc/src/test/java/ListClustersTest.java @@ -40,11 +40,10 @@ @RunWith(JUnit4.class) public class ListClustersTest { - private static final String BASE_CLUSTER_NAME = "test-cluster"; + private static final String CLUSTER_NAME = "test-cluster-" + UUID.randomUUID().toString(); private static final String REGION = "us-central1"; - private static final String projectId = System.getenv("GOOGLE_CLOUD_PROJECT"); + private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); - private String clusterName; private String myEndpoint; private ByteArrayOutputStream bout; private PrintStream standardOutOrig; @@ -63,53 +62,53 @@ public static void checkRequirements() { @Before public void setUp() throws IOException, InterruptedException { - clusterName = String.format("%s-%s", BASE_CLUSTER_NAME, UUID.randomUUID().toString()); bout = new ByteArrayOutputStream(); standardOutOrig = System.out; System.setOut(new PrintStream(bout)); - myEndpoint = String.format("%s-dataproc.googleapis.com:443", REGION); + myEndpoint = REGION + "-dataproc.googleapis.com:443"; ClusterControllerSettings clusterControllerSettings = ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); Cluster cluster = Cluster.newBuilder() - .setClusterName(clusterName) + .setClusterName(CLUSTER_NAME) .setConfig(ClusterConfig.newBuilder().build()) .build(); try (ClusterControllerClient clusterControllerClient = ClusterControllerClient.create(clusterControllerSettings)) { OperationFuture createClusterAsyncRequest = - clusterControllerClient.createClusterAsync(projectId, REGION, cluster); + clusterControllerClient.createClusterAsync(PROJECT_ID, REGION, cluster); createClusterAsyncRequest.get(); } catch (ExecutionException e) { - System.out.println("[deleteCluster] Error during test cluster creation: \n" + e.toString()); + System.err.println("[deleteCluster] Error during test cluster creation: \n" + e.getMessage()); } } @Test - public void ListClustersTest() throws IOException, InterruptedException { - ListClusters.ListClusters(projectId, REGION); + public void listClustersTest() throws IOException { + ListClusters.listClusters(PROJECT_ID, REGION); String output = bout.toString(); - assertThat(output, CoreMatchers.containsString(clusterName)); + assertThat(output, CoreMatchers.containsString(CLUSTER_NAME)); assertThat(output, CoreMatchers.containsString("RUNNING")); } @After public void teardown() throws IOException, InterruptedException { System.setOut(standardOutOrig); + ClusterControllerSettings clusterControllerSettings = ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); try (ClusterControllerClient clusterControllerClient = ClusterControllerClient.create(clusterControllerSettings)) { OperationFuture deleteClusterAsyncRequest = - clusterControllerClient.deleteClusterAsync(projectId, REGION, clusterName); + clusterControllerClient.deleteClusterAsync(PROJECT_ID, REGION, CLUSTER_NAME); deleteClusterAsyncRequest.get(); } catch (ExecutionException e) { - System.out.println("[deleteCluster] Error during test cluster creation: \n" + e.toString()); + System.err.println("[deleteCluster] Error during test cluster creation: \n" + e.getMessage()); } } } diff --git a/dataproc/src/test/java/SubmitJobTest.java b/dataproc/src/test/java/SubmitJobTest.java index cc6413e02c8..ab4cd1139cc 100644 --- a/dataproc/src/test/java/SubmitJobTest.java +++ b/dataproc/src/test/java/SubmitJobTest.java @@ -49,9 +49,9 @@ public class SubmitJobTest { private static final String MY_UUID = UUID.randomUUID().toString(); private static final String REGION = "us-central1"; private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); - private static final String ENDPOINT = String.format("%s-dataproc.googleapis.com:443", REGION); - private static final String CLUSTER_NAME = String.format("%s-%s", "test-cluster", MY_UUID); - private static final String BUCKET_NAME = String.format("%s-%s", "test-bucket", MY_UUID); + private static final String ENDPOINT = REGION + "-dataproc.googleapis.com:443"; + private static final String CLUSTER_NAME = "test-cluster-" + MY_UUID; + private static final String BUCKET_NAME = "test-bucket-" + MY_UUID; private static final String JOB_FILE_NAME = "sum.py"; private static final String JOB_FILE_PATH = String.format("gs://%s/%s", BUCKET_NAME, JOB_FILE_NAME); @@ -63,7 +63,6 @@ public class SubmitJobTest { private ByteArrayOutputStream bout; private PrintStream standardOutOrig; - private Storage storage; private Blob blob; private static void requireEnv(String varName) { @@ -84,7 +83,7 @@ public void setUp() throws IOException, InterruptedException { standardOutOrig = System.out; System.setOut(new PrintStream(bout)); - storage = StorageOptions.getDefaultInstance().getService(); + Storage storage = StorageOptions.getDefaultInstance().getService(); Bucket bucket = storage.create(BucketInfo.of(BUCKET_NAME)); blob = bucket.create(JOB_FILE_NAME, SORT_CODE.getBytes(UTF_8), "text/plain"); @@ -103,7 +102,7 @@ public void setUp() throws IOException, InterruptedException { clusterControllerClient.createClusterAsync(PROJECT_ID, REGION, cluster); createClusterAsyncRequest.get(); } catch (ExecutionException e) { - System.out.println("[SubmitJob] Error during test cluster creation: \n" + e.toString()); + System.err.println("[SubmitJob] Error during test cluster creation: \n" + e.getMessage()); } } @@ -129,7 +128,7 @@ public void teardown() throws IOException, InterruptedException { clusterControllerClient.deleteClusterAsync(PROJECT_ID, REGION, CLUSTER_NAME); deleteClusterAsyncRequest.get(); } catch (ExecutionException e) { - System.out.println("[SubmitJob] Error during test cluster deletion: \n" + e.toString()); + System.err.println("[SubmitJob] Error during test cluster deletion: \n" + e.getMessage()); } } } diff --git a/dataproc/src/test/java/UpdateClusterTest.java b/dataproc/src/test/java/UpdateClusterTest.java index 7b19fdc1ebf..8eb12ec26e1 100644 --- a/dataproc/src/test/java/UpdateClusterTest.java +++ b/dataproc/src/test/java/UpdateClusterTest.java @@ -41,11 +41,10 @@ @RunWith(JUnit4.class) public class UpdateClusterTest { - private static final String MY_UUID = UUID.randomUUID().toString(); private static final String REGION = "us-central1"; private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); - private static final String ENDPOINT = String.format("%s-dataproc.googleapis.com:443", REGION); - private static final String CLUSTER_NAME = String.format("%s-%s", "test-cluster", MY_UUID); + private static final String ENDPOINT = REGION + "-dataproc.googleapis.com:443"; + private static final String CLUSTER_NAME = "test-cluster-" + UUID.randomUUID().toString(); private static final int NUM_WORKERS = 2; private static final int NEW_WORKERS = NUM_WORKERS * 2; @@ -99,7 +98,7 @@ public void setUp() throws IOException, InterruptedException { clusterControllerClient.createClusterAsync(PROJECT_ID, REGION, cluster); createClusterAsyncRequest.get(); } catch (ExecutionException e) { - System.out.println("[UpdateCluster] Error during test cluster creation: \n" + e.toString()); + System.err.println("[UpdateCluster] Error during test cluster creation: \n" + e.getMessage()); } } @@ -124,7 +123,7 @@ public void teardown() throws IOException, InterruptedException { clusterControllerClient.deleteClusterAsync(PROJECT_ID, REGION, CLUSTER_NAME); deleteClusterAsyncRequest.get(); } catch (ExecutionException e) { - System.out.println("[UpdateCluster] Error during test cluster update: \n" + e.toString()); + System.err.println("[UpdateCluster] Error during test cluster update: \n" + e.getMessage()); } } } From ebe059d6b838eaa9e526c49c958ba1224b47ea28 Mon Sep 17 00:00:00 2001 From: bradmiro Date: Mon, 2 Dec 2019 15:16:44 -0500 Subject: [PATCH 10/21] added overloaded functions to all samples --- dataproc/src/main/java/CreateCluster.java | 23 ++++++++++--------- dataproc/src/main/java/DeleteCluster.java | 23 ++++++++++--------- dataproc/src/main/java/ListClusters.java | 17 +++++++++----- dataproc/src/main/java/SubmitJob.java | 19 ++++++++++----- dataproc/src/main/java/UpdateCluster.java | 23 ++++++++++--------- dataproc/src/test/java/CreateClusterTest.java | 6 ++--- dataproc/src/test/java/DeleteClusterTest.java | 2 +- dataproc/src/test/java/ListClustersTest.java | 4 +--- dataproc/src/test/java/SubmitJobTest.java | 4 +--- dataproc/src/test/java/UpdateClusterTest.java | 6 ++--- 10 files changed, 67 insertions(+), 60 deletions(-) diff --git a/dataproc/src/main/java/CreateCluster.java b/dataproc/src/main/java/CreateCluster.java index ebae59dd53c..02365e8661f 100644 --- a/dataproc/src/main/java/CreateCluster.java +++ b/dataproc/src/main/java/CreateCluster.java @@ -27,9 +27,18 @@ public class CreateCluster { + public static void createCluster() + throws IOException, InterruptedException, ExecutionException { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String region = "your-project-region"; + String clusterName = "your-cluster-name"; + createCluster(projectId, region, clusterName); + } + public static void createCluster(String projectId, String region, String clusterName) - throws IOException, InterruptedException { - String myEndpoint = region + "-dataproc.googleapis.com:443"; + throws IOException, InterruptedException, ExecutionException { + String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); // Configure the settings for the cluster controller client ClusterControllerSettings clusterControllerSettings = @@ -66,16 +75,8 @@ public static void createCluster(String projectId, String region, String cluster Cluster response = createClusterAsyncRequest.get(); // Print out a success message - System.out.println("Cluster created successfully: " + response.getClusterName()); + System.out.printf("Cluster created successfully: %s", response.getClusterName()); - } catch (IOException e) { - // Likely this would occur due to issues authenticating with GCP. Make sure the environment - // variable GOOGLE_APPLICATION_CREDENTIALS is configured. - System.err.println("Error creating the cluster controller client: \n" + e.getMessage()); - } catch (ExecutionException e) { - // Common issues for this include needing to increase compute engine quotas or a cluster of - // the same name already exists. - System.err.println("Error during cluster creation request: \n" + e.getMessage()); } } } diff --git a/dataproc/src/main/java/DeleteCluster.java b/dataproc/src/main/java/DeleteCluster.java index 84736b773cf..4bdb5aa70a5 100644 --- a/dataproc/src/main/java/DeleteCluster.java +++ b/dataproc/src/main/java/DeleteCluster.java @@ -25,9 +25,18 @@ public class DeleteCluster { + public static void deleteCluster() + throws IOException, InterruptedException, ExecutionException { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String region = "your-project-region"; + String clusterName = "your-cluster-name"; + deleteCluster(projectId, region, clusterName); + } + public static void deleteCluster(String projectId, String region, String clusterName) - throws IOException, InterruptedException { - String myEndpoint = region + "-dataproc.googleapis.com:443"; + throws IOException, InterruptedException, ExecutionException { + String myEndpoint = String.format("-dataproc.googleapis.com:443", region); // Configure the settings for the cluster controller client ClusterControllerSettings clusterControllerSettings = @@ -46,15 +55,7 @@ public static void deleteCluster(String projectId, String region, String cluster deleteClusterAsyncRequest.get(); // Print out a success message - System.out.println("Cluster deleted successfully: " + clusterName); - - } catch (IOException e) { - // Likely this would occur due to issues authenticating with GCP. Make sure the environment - // variable GOOGLE_APPLICATION_CREDENTIALS is configured. - System.err.println("Error deleting the cluster controller client: \n" + e.getMessage()); - } catch (ExecutionException e) { - // This will likely be due to a cluster of the given name not existing in the given region. - System.err.println("Error during cluster deletion request: \n" + e.getMessage()); + System.out.printf("Cluster deleted successfully: %s", clusterName); } } } diff --git a/dataproc/src/main/java/ListClusters.java b/dataproc/src/main/java/ListClusters.java index b0b0a68b905..6b35a18e4f8 100644 --- a/dataproc/src/main/java/ListClusters.java +++ b/dataproc/src/main/java/ListClusters.java @@ -19,11 +19,20 @@ import com.google.cloud.dataproc.v1.ClusterControllerClient; import com.google.cloud.dataproc.v1.ClusterControllerSettings; import java.io.IOException; +import java.util.concurrent.ExecutionException; public class ListClusters { + public static void listClusters() + throws IOException, InterruptedException, ExecutionException { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String region = "your-project-region"; + listClusters(projectId, region); + } + public static void listClusters(String projectId, String region) throws IOException { - String myEndpoint = region + "-dataproc.googleapis.com:443"; + String myEndpoint = String.format("-dataproc.googleapis.com:443"); // Configure the settings for the cluster controller client ClusterControllerSettings clusterControllerSettings = @@ -36,12 +45,8 @@ public static void listClusters(String projectId, String region) throws IOExcept ClusterControllerClient.create(clusterControllerSettings)) { for (Cluster element : clusterControllerClient.listClusters(projectId, region).iterateAll()) { - System.out.println(element.getClusterName() + ": " + element.getStatus().getState()); + System.out.printf("%s: %s", element.getClusterName(), element.getStatus().getState()); } - } catch (IOException e) { - // Likely this would occur due to issues authenticating with GCP. Make sure the environment - // variable GOOGLE_APPLICATION_CREDENTIALS is configured. - System.out.println("Error deleting the cluster controller client: \n" + e.getMessage()); } } } diff --git a/dataproc/src/main/java/SubmitJob.java b/dataproc/src/main/java/SubmitJob.java index 01bce05569b..7e16224d661 100644 --- a/dataproc/src/main/java/SubmitJob.java +++ b/dataproc/src/main/java/SubmitJob.java @@ -22,12 +22,23 @@ import com.google.cloud.dataproc.v1.JobStatus; import com.google.cloud.dataproc.v1.PySparkJob; import java.io.IOException; +import java.util.concurrent.ExecutionException; public class SubmitJob { + public static void submitJob() + throws IOException, InterruptedException, ExecutionException { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String region = "your-project-region"; + String clusterName = "your-cluster-name"; + String jobFilePath = "your-job-file-path"; + submitJob(projectId, region, clusterName, jobFilePath); + } + public static void submitJob( String projectId, String region, String clusterName, String jobFilePath) throws IOException { - String myEndpoint = region + "-dataproc.googleapis.com:443"; + String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); // Configure the settings for the cluster controller client JobControllerSettings jobControllerSettings = @@ -51,17 +62,13 @@ public static void submitJob( while (true) { Job jobInfo = jobControllerClient.getJob(projectId, region, jobId); if (jobInfo.getStatus().getState().equals(JobStatus.State.ERROR)) { - System.err.printf("Job %s failed: %s", jobId, jobInfo.getStatus().getDetails()); + System.out.printf("Job %s failed: %s", jobId, jobInfo.getStatus().getDetails()); break; } else if (jobInfo.getStatus().getState().equals(JobStatus.State.DONE)) { System.out.printf("Job %s finished.", jobId); break; } } - } catch (IOException e) { - // Likely this would occur due to issues authenticating with GCP. Make sure the environment - // variable GOOGLE_APPLICATION_CREDENTIALS is configured. - System.err.println("Error creating the cluster controller client: \n" + e.getMessage()); } } } diff --git a/dataproc/src/main/java/UpdateCluster.java b/dataproc/src/main/java/UpdateCluster.java index 6cfba7bcace..aae45cd7724 100644 --- a/dataproc/src/main/java/UpdateCluster.java +++ b/dataproc/src/main/java/UpdateCluster.java @@ -28,10 +28,20 @@ public class UpdateCluster { + public static void updateCluster() + throws IOException, InterruptedException, ExecutionException { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String region = "your-project-region"; + String clusterName = "your-cluster-name"; + int numWorkers = 0; //your number of workers + updateCluster(projectId, region, clusterName, numWorkers); + } + public static void updateCluster( String projectId, String region, String clusterName, int numWorkers) - throws IOException, InterruptedException { - String myEndpoint = region + "-dataproc.googleapis.com:443"; + throws IOException, InterruptedException, ExecutionException { + String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); // Configure the settings for the cluster controller client ClusterControllerSettings clusterControllerSettings = @@ -76,15 +86,6 @@ public static void updateCluster( System.out.printf( "Cluster %s now has %d workers.", response.getClusterName(), response.getConfig().getWorkerConfig().getNumInstances()); - - } catch (IOException e) { - // Likely this would occur due to issues authenticating with GCP. Make sure the environment - // variable GOOGLE_APPLICATION_CREDENTIALS is configured. - System.err.println("Error creating the cluster controller client: \n" + e.getMessage()); - } catch (ExecutionException e) { - // Common issues for this include needing to increase compute engine quotas or a cluster of - // the same name already exists. - System.out.println("Error during cluster creation request: \n" + e.getMessage()); } } } diff --git a/dataproc/src/test/java/CreateClusterTest.java b/dataproc/src/test/java/CreateClusterTest.java index fc661e371d3..feedfa37eaa 100644 --- a/dataproc/src/test/java/CreateClusterTest.java +++ b/dataproc/src/test/java/CreateClusterTest.java @@ -65,7 +65,7 @@ public void setUp() { } @Test - public void createClusterTest() throws IOException, InterruptedException { + public void createClusterTest() throws IOException, InterruptedException, ExecutionException { CreateCluster.createCluster(PROJECT_ID, REGION, CLUSTER_NAME); String output = bout.toString(); @@ -73,7 +73,7 @@ public void createClusterTest() throws IOException, InterruptedException { } @After - public void tearDown() throws IOException, InterruptedException { + public void tearDown() throws IOException, InterruptedException, ExecutionException { String myEndpoint = REGION + "-dataproc.googleapis.com:443"; System.setOut(standardOutOrig); @@ -86,8 +86,6 @@ public void tearDown() throws IOException, InterruptedException { clusterControllerClient.deleteClusterAsync(PROJECT_ID, REGION, CLUSTER_NAME); deleteClusterAsyncRequest.get(); - } catch (ExecutionException e) { - System.err.println("Error during cluster deletion: \n" + e.getMessage()); } } } diff --git a/dataproc/src/test/java/DeleteClusterTest.java b/dataproc/src/test/java/DeleteClusterTest.java index 0ec6797463c..ceb2e8841a0 100644 --- a/dataproc/src/test/java/DeleteClusterTest.java +++ b/dataproc/src/test/java/DeleteClusterTest.java @@ -86,7 +86,7 @@ public void setUp() throws IOException, InterruptedException { } @Test - public void deleteClusterTest() throws IOException, InterruptedException { + public void deleteClusterTest() throws IOException, InterruptedException, ExecutionException { DeleteCluster.deleteCluster(PROJECT_ID, REGION, CLUSTER_NAME); String output = bout.toString(); diff --git a/dataproc/src/test/java/ListClustersTest.java b/dataproc/src/test/java/ListClustersTest.java index 4d0272ea797..8a4e798ddc3 100644 --- a/dataproc/src/test/java/ListClustersTest.java +++ b/dataproc/src/test/java/ListClustersTest.java @@ -96,7 +96,7 @@ public void listClustersTest() throws IOException { } @After - public void teardown() throws IOException, InterruptedException { + public void teardown() throws IOException, InterruptedException, ExecutionException { System.setOut(standardOutOrig); ClusterControllerSettings clusterControllerSettings = @@ -107,8 +107,6 @@ public void teardown() throws IOException, InterruptedException { OperationFuture deleteClusterAsyncRequest = clusterControllerClient.deleteClusterAsync(PROJECT_ID, REGION, CLUSTER_NAME); deleteClusterAsyncRequest.get(); - } catch (ExecutionException e) { - System.err.println("[deleteCluster] Error during test cluster creation: \n" + e.getMessage()); } } } diff --git a/dataproc/src/test/java/SubmitJobTest.java b/dataproc/src/test/java/SubmitJobTest.java index ab4cd1139cc..6d1ed867606 100644 --- a/dataproc/src/test/java/SubmitJobTest.java +++ b/dataproc/src/test/java/SubmitJobTest.java @@ -115,7 +115,7 @@ public void submitJobTest() throws IOException { } @After - public void teardown() throws IOException, InterruptedException { + public void teardown() throws IOException, InterruptedException, ExecutionException { System.setOut(standardOutOrig); blob.delete(); @@ -127,8 +127,6 @@ public void teardown() throws IOException, InterruptedException { OperationFuture deleteClusterAsyncRequest = clusterControllerClient.deleteClusterAsync(PROJECT_ID, REGION, CLUSTER_NAME); deleteClusterAsyncRequest.get(); - } catch (ExecutionException e) { - System.err.println("[SubmitJob] Error during test cluster deletion: \n" + e.getMessage()); } } } diff --git a/dataproc/src/test/java/UpdateClusterTest.java b/dataproc/src/test/java/UpdateClusterTest.java index 8eb12ec26e1..08cb5db584a 100644 --- a/dataproc/src/test/java/UpdateClusterTest.java +++ b/dataproc/src/test/java/UpdateClusterTest.java @@ -103,7 +103,7 @@ public void setUp() throws IOException, InterruptedException { } @Test - public void updateClusterTest() throws IOException, InterruptedException { + public void updateClusterTest() throws IOException, InterruptedException, ExecutionException { UpdateCluster.updateCluster(PROJECT_ID, REGION, CLUSTER_NAME, NEW_WORKERS); String output = bout.toString(); @@ -111,7 +111,7 @@ public void updateClusterTest() throws IOException, InterruptedException { } @After - public void teardown() throws IOException, InterruptedException { + public void teardown() throws IOException, InterruptedException, ExecutionException { System.setOut(standardOutOrig); ClusterControllerSettings clusterControllerSettings = @@ -122,8 +122,6 @@ public void teardown() throws IOException, InterruptedException { OperationFuture deleteClusterAsyncRequest = clusterControllerClient.deleteClusterAsync(PROJECT_ID, REGION, CLUSTER_NAME); deleteClusterAsyncRequest.get(); - } catch (ExecutionException e) { - System.err.println("[UpdateCluster] Error during test cluster update: \n" + e.getMessage()); } } } From 6b20c68672ecca358cbcbc2c6875785f72842845 Mon Sep 17 00:00:00 2001 From: bradmiro Date: Mon, 2 Dec 2019 16:29:55 -0500 Subject: [PATCH 11/21] Formatting changes --- dataproc/src/main/java/CreateCluster.java | 4 +--- dataproc/src/main/java/DeleteCluster.java | 5 ++--- dataproc/src/main/java/ListClusters.java | 3 +-- dataproc/src/main/java/SubmitJob.java | 3 +-- dataproc/src/main/java/UpdateCluster.java | 5 ++--- dataproc/src/test/java/CreateClusterTest.java | 6 +++--- dataproc/src/test/java/DeleteClusterTest.java | 2 +- dataproc/src/test/java/ListClustersTest.java | 5 +++-- dataproc/src/test/java/SubmitJobTest.java | 6 +++--- dataproc/src/test/java/UpdateClusterTest.java | 4 +--- 10 files changed, 18 insertions(+), 25 deletions(-) diff --git a/dataproc/src/main/java/CreateCluster.java b/dataproc/src/main/java/CreateCluster.java index 02365e8661f..0f0bd1c3ead 100644 --- a/dataproc/src/main/java/CreateCluster.java +++ b/dataproc/src/main/java/CreateCluster.java @@ -27,8 +27,7 @@ public class CreateCluster { - public static void createCluster() - throws IOException, InterruptedException, ExecutionException { + public static void createCluster() throws IOException, InterruptedException, ExecutionException { // TODO(developer): Replace these variables before running the sample. String projectId = "your-project-id"; String region = "your-project-region"; @@ -76,7 +75,6 @@ public static void createCluster(String projectId, String region, String cluster // Print out a success message System.out.printf("Cluster created successfully: %s", response.getClusterName()); - } } } diff --git a/dataproc/src/main/java/DeleteCluster.java b/dataproc/src/main/java/DeleteCluster.java index 4bdb5aa70a5..301fd4b296f 100644 --- a/dataproc/src/main/java/DeleteCluster.java +++ b/dataproc/src/main/java/DeleteCluster.java @@ -25,8 +25,7 @@ public class DeleteCluster { - public static void deleteCluster() - throws IOException, InterruptedException, ExecutionException { + public static void deleteCluster() throws IOException, InterruptedException, ExecutionException { // TODO(developer): Replace these variables before running the sample. String projectId = "your-project-id"; String region = "your-project-region"; @@ -36,7 +35,7 @@ public static void deleteCluster() public static void deleteCluster(String projectId, String region, String clusterName) throws IOException, InterruptedException, ExecutionException { - String myEndpoint = String.format("-dataproc.googleapis.com:443", region); + String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); // Configure the settings for the cluster controller client ClusterControllerSettings clusterControllerSettings = diff --git a/dataproc/src/main/java/ListClusters.java b/dataproc/src/main/java/ListClusters.java index 6b35a18e4f8..840783d81fd 100644 --- a/dataproc/src/main/java/ListClusters.java +++ b/dataproc/src/main/java/ListClusters.java @@ -23,8 +23,7 @@ public class ListClusters { - public static void listClusters() - throws IOException, InterruptedException, ExecutionException { + public static void listClusters() throws IOException, InterruptedException, ExecutionException { // TODO(developer): Replace these variables before running the sample. String projectId = "your-project-id"; String region = "your-project-region"; diff --git a/dataproc/src/main/java/SubmitJob.java b/dataproc/src/main/java/SubmitJob.java index 7e16224d661..a2f558e1ddf 100644 --- a/dataproc/src/main/java/SubmitJob.java +++ b/dataproc/src/main/java/SubmitJob.java @@ -26,8 +26,7 @@ public class SubmitJob { - public static void submitJob() - throws IOException, InterruptedException, ExecutionException { + public static void submitJob() throws IOException, InterruptedException, ExecutionException { // TODO(developer): Replace these variables before running the sample. String projectId = "your-project-id"; String region = "your-project-region"; diff --git a/dataproc/src/main/java/UpdateCluster.java b/dataproc/src/main/java/UpdateCluster.java index aae45cd7724..7eb7afc9f62 100644 --- a/dataproc/src/main/java/UpdateCluster.java +++ b/dataproc/src/main/java/UpdateCluster.java @@ -28,13 +28,12 @@ public class UpdateCluster { - public static void updateCluster() - throws IOException, InterruptedException, ExecutionException { + public static void updateCluster() throws IOException, InterruptedException, ExecutionException { // TODO(developer): Replace these variables before running the sample. String projectId = "your-project-id"; String region = "your-project-region"; String clusterName = "your-cluster-name"; - int numWorkers = 0; //your number of workers + int numWorkers = 0; // your number of workers updateCluster(projectId, region, clusterName, numWorkers); } diff --git a/dataproc/src/test/java/CreateClusterTest.java b/dataproc/src/test/java/CreateClusterTest.java index feedfa37eaa..da4c321b80e 100644 --- a/dataproc/src/test/java/CreateClusterTest.java +++ b/dataproc/src/test/java/CreateClusterTest.java @@ -38,7 +38,8 @@ @RunWith(JUnit4.class) public class CreateClusterTest { - private static final String CLUSTER_NAME = "test-cluster-" + UUID.randomUUID().toString(); + private static final String CLUSTER_NAME = + String.format("test-cluster-%s", UUID.randomUUID().toString()); private static final String REGION = "us-central1"; private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); @@ -74,7 +75,7 @@ public void createClusterTest() throws IOException, InterruptedException, Execut @After public void tearDown() throws IOException, InterruptedException, ExecutionException { - String myEndpoint = REGION + "-dataproc.googleapis.com:443"; + String myEndpoint = String.format("%s-dataproc.googleapis.com:443", REGION); System.setOut(standardOutOrig); ClusterControllerSettings clusterControllerSettings = @@ -85,7 +86,6 @@ public void tearDown() throws IOException, InterruptedException, ExecutionExcept OperationFuture deleteClusterAsyncRequest = clusterControllerClient.deleteClusterAsync(PROJECT_ID, REGION, CLUSTER_NAME); deleteClusterAsyncRequest.get(); - } } } diff --git a/dataproc/src/test/java/DeleteClusterTest.java b/dataproc/src/test/java/DeleteClusterTest.java index ceb2e8841a0..1adac0f5699 100644 --- a/dataproc/src/test/java/DeleteClusterTest.java +++ b/dataproc/src/test/java/DeleteClusterTest.java @@ -64,7 +64,7 @@ public void setUp() throws IOException, InterruptedException { System.setOut(new PrintStream(bout)); standardOutOrig = System.out; - String myEndpoint = REGION + "-dataproc.googleapis.com:443"; + String myEndpoint = String.format("%s-dataproc.googleapis.com:443", REGION); ClusterControllerSettings clusterControllerSettings = ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); diff --git a/dataproc/src/test/java/ListClustersTest.java b/dataproc/src/test/java/ListClustersTest.java index 8a4e798ddc3..93c0b970b41 100644 --- a/dataproc/src/test/java/ListClustersTest.java +++ b/dataproc/src/test/java/ListClustersTest.java @@ -40,7 +40,8 @@ @RunWith(JUnit4.class) public class ListClustersTest { - private static final String CLUSTER_NAME = "test-cluster-" + UUID.randomUUID().toString(); + private static final String CLUSTER_NAME = + String.format("test-cluster-%s", UUID.randomUUID().toString()); private static final String REGION = "us-central1"; private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); @@ -65,7 +66,7 @@ public void setUp() throws IOException, InterruptedException { bout = new ByteArrayOutputStream(); standardOutOrig = System.out; System.setOut(new PrintStream(bout)); - myEndpoint = REGION + "-dataproc.googleapis.com:443"; + myEndpoint = String.format("%s-dataproc.googleapis.com:443", REGION); ClusterControllerSettings clusterControllerSettings = ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); diff --git a/dataproc/src/test/java/SubmitJobTest.java b/dataproc/src/test/java/SubmitJobTest.java index 6d1ed867606..8484f4b7dc8 100644 --- a/dataproc/src/test/java/SubmitJobTest.java +++ b/dataproc/src/test/java/SubmitJobTest.java @@ -49,9 +49,9 @@ public class SubmitJobTest { private static final String MY_UUID = UUID.randomUUID().toString(); private static final String REGION = "us-central1"; private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); - private static final String ENDPOINT = REGION + "-dataproc.googleapis.com:443"; - private static final String CLUSTER_NAME = "test-cluster-" + MY_UUID; - private static final String BUCKET_NAME = "test-bucket-" + MY_UUID; + private static final String ENDPOINT = String.format("%s-dataproc.googleapis.com:443", REGION); + private static final String CLUSTER_NAME = String.format("test-cluster-%s", MY_UUID); + private static final String BUCKET_NAME = String.format("test-bucket-%s", MY_UUID); private static final String JOB_FILE_NAME = "sum.py"; private static final String JOB_FILE_PATH = String.format("gs://%s/%s", BUCKET_NAME, JOB_FILE_NAME); diff --git a/dataproc/src/test/java/UpdateClusterTest.java b/dataproc/src/test/java/UpdateClusterTest.java index 08cb5db584a..0353c3cc9fc 100644 --- a/dataproc/src/test/java/UpdateClusterTest.java +++ b/dataproc/src/test/java/UpdateClusterTest.java @@ -64,7 +64,7 @@ public static void checkRequirements() { } @Before - public void setUp() throws IOException, InterruptedException { + public void setUp() throws IOException, InterruptedException, ExecutionException { bout = new ByteArrayOutputStream(); standardOutOrig = System.out; System.setOut(new PrintStream(bout)); @@ -97,8 +97,6 @@ public void setUp() throws IOException, InterruptedException { OperationFuture createClusterAsyncRequest = clusterControllerClient.createClusterAsync(PROJECT_ID, REGION, cluster); createClusterAsyncRequest.get(); - } catch (ExecutionException e) { - System.err.println("[UpdateCluster] Error during test cluster creation: \n" + e.getMessage()); } } From 9990637939e6c453c6bf19e01bb37568a9c4227f Mon Sep 17 00:00:00 2001 From: bradmiro Date: Mon, 2 Dec 2019 17:28:59 -0500 Subject: [PATCH 12/21] small bug fixes --- dataproc/src/main/java/ListClusters.java | 2 +- dataproc/src/test/java/DeleteClusterTest.java | 3 ++- dataproc/src/test/java/UpdateClusterTest.java | 5 +++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/dataproc/src/main/java/ListClusters.java b/dataproc/src/main/java/ListClusters.java index 840783d81fd..c6c4b697afe 100644 --- a/dataproc/src/main/java/ListClusters.java +++ b/dataproc/src/main/java/ListClusters.java @@ -31,7 +31,7 @@ public static void listClusters() throws IOException, InterruptedException, Exec } public static void listClusters(String projectId, String region) throws IOException { - String myEndpoint = String.format("-dataproc.googleapis.com:443"); + String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); // Configure the settings for the cluster controller client ClusterControllerSettings clusterControllerSettings = diff --git a/dataproc/src/test/java/DeleteClusterTest.java b/dataproc/src/test/java/DeleteClusterTest.java index 1adac0f5699..62c96557612 100644 --- a/dataproc/src/test/java/DeleteClusterTest.java +++ b/dataproc/src/test/java/DeleteClusterTest.java @@ -39,7 +39,8 @@ @RunWith(JUnit4.class) public class DeleteClusterTest { - private static final String CLUSTER_NAME = "test-cluster-" + UUID.randomUUID().toString(); + private static final String CLUSTER_NAME = + String.format("test-cluster-%s", UUID.randomUUID().toString()); private static final String REGION = "us-central1"; private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); diff --git a/dataproc/src/test/java/UpdateClusterTest.java b/dataproc/src/test/java/UpdateClusterTest.java index 0353c3cc9fc..ac3aa869431 100644 --- a/dataproc/src/test/java/UpdateClusterTest.java +++ b/dataproc/src/test/java/UpdateClusterTest.java @@ -43,8 +43,9 @@ public class UpdateClusterTest { private static final String REGION = "us-central1"; private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); - private static final String ENDPOINT = REGION + "-dataproc.googleapis.com:443"; - private static final String CLUSTER_NAME = "test-cluster-" + UUID.randomUUID().toString(); + private static final String ENDPOINT = String.format("%s-dataproc.googleapis.com:443", REGION); + private static final String CLUSTER_NAME = + String.format("test-cluster-%s", UUID.randomUUID().toString()); private static final int NUM_WORKERS = 2; private static final int NEW_WORKERS = NUM_WORKERS * 2; From 5a9a3a365baa40b0ee92a64b76f07505a3e38197 Mon Sep 17 00:00:00 2001 From: bradmiro Date: Fri, 20 Dec 2019 12:13:08 -0500 Subject: [PATCH 13/21] Fixed CreateCluster sample and added Quickstart --- dataproc/src/main/java/DeleteCluster.java | 61 --------- dataproc/src/main/java/ListClusters.java | 52 -------- dataproc/src/main/java/SubmitJob.java | 74 ---------- dataproc/src/main/java/UpdateCluster.java | 91 ------------- .../{DeleteClusterTest.java => CreateCluster} | 46 +++---- dataproc/src/test/java/ListClustersTest.java | 113 ---------------- ...SubmitJobTest.java => QuickstartTest.java} | 42 +++--- dataproc/src/test/java/UpdateClusterTest.java | 126 ------------------ 8 files changed, 34 insertions(+), 571 deletions(-) delete mode 100644 dataproc/src/main/java/DeleteCluster.java delete mode 100644 dataproc/src/main/java/ListClusters.java delete mode 100644 dataproc/src/main/java/SubmitJob.java delete mode 100644 dataproc/src/main/java/UpdateCluster.java rename dataproc/src/test/java/{DeleteClusterTest.java => CreateCluster} (73%) delete mode 100644 dataproc/src/test/java/ListClustersTest.java rename dataproc/src/test/java/{SubmitJobTest.java => QuickstartTest.java} (75%) delete mode 100644 dataproc/src/test/java/UpdateClusterTest.java diff --git a/dataproc/src/main/java/DeleteCluster.java b/dataproc/src/main/java/DeleteCluster.java deleted file mode 100644 index 301fd4b296f..00000000000 --- a/dataproc/src/main/java/DeleteCluster.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2019 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// [START dataproc_delete_cluster] -import com.google.api.gax.longrunning.OperationFuture; -import com.google.cloud.dataproc.v1.ClusterControllerClient; -import com.google.cloud.dataproc.v1.ClusterControllerSettings; -import com.google.cloud.dataproc.v1.ClusterOperationMetadata; -import com.google.protobuf.Empty; -import java.io.IOException; -import java.util.concurrent.ExecutionException; - -public class DeleteCluster { - - public static void deleteCluster() throws IOException, InterruptedException, ExecutionException { - // TODO(developer): Replace these variables before running the sample. - String projectId = "your-project-id"; - String region = "your-project-region"; - String clusterName = "your-cluster-name"; - deleteCluster(projectId, region, clusterName); - } - - public static void deleteCluster(String projectId, String region, String clusterName) - throws IOException, InterruptedException, ExecutionException { - String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); - - // Configure the settings for the cluster controller client - ClusterControllerSettings clusterControllerSettings = - ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); - - // Create a cluster controller client with the configured settings. The client only needs to be - // created once and can be reused for multiple requests. Using a try-with-resources - // closes the client, but this can also be done manually with the .close() method. - try (ClusterControllerClient clusterControllerClient = - ClusterControllerClient.create(clusterControllerSettings)) { - // Configure the settings for our cluster - - // Delete the Cloud Dataproc cluster - OperationFuture deleteClusterAsyncRequest = - clusterControllerClient.deleteClusterAsync(projectId, region, clusterName); - deleteClusterAsyncRequest.get(); - - // Print out a success message - System.out.printf("Cluster deleted successfully: %s", clusterName); - } - } -} -// [END dataproc_delete_cluster] diff --git a/dataproc/src/main/java/ListClusters.java b/dataproc/src/main/java/ListClusters.java deleted file mode 100644 index c6c4b697afe..00000000000 --- a/dataproc/src/main/java/ListClusters.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2019 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// [START dataproc_list_clusters] -import com.google.cloud.dataproc.v1.Cluster; -import com.google.cloud.dataproc.v1.ClusterControllerClient; -import com.google.cloud.dataproc.v1.ClusterControllerSettings; -import java.io.IOException; -import java.util.concurrent.ExecutionException; - -public class ListClusters { - - public static void listClusters() throws IOException, InterruptedException, ExecutionException { - // TODO(developer): Replace these variables before running the sample. - String projectId = "your-project-id"; - String region = "your-project-region"; - listClusters(projectId, region); - } - - public static void listClusters(String projectId, String region) throws IOException { - String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); - - // Configure the settings for the cluster controller client - ClusterControllerSettings clusterControllerSettings = - ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); - - // Create a cluster controller client with the configured settings. The client only needs to be - // created once and can be reused for multiple requests. Using a try-with-resources - // closes the client, but this can also be done manually with the .close() method. - try (ClusterControllerClient clusterControllerClient = - ClusterControllerClient.create(clusterControllerSettings)) { - - for (Cluster element : clusterControllerClient.listClusters(projectId, region).iterateAll()) { - System.out.printf("%s: %s", element.getClusterName(), element.getStatus().getState()); - } - } - } -} -// [END dataproc_list_clusters] diff --git a/dataproc/src/main/java/SubmitJob.java b/dataproc/src/main/java/SubmitJob.java deleted file mode 100644 index a2f558e1ddf..00000000000 --- a/dataproc/src/main/java/SubmitJob.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright 2019 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// [START dataproc_submit_job] -import com.google.cloud.dataproc.v1.Job; -import com.google.cloud.dataproc.v1.JobControllerClient; -import com.google.cloud.dataproc.v1.JobControllerSettings; -import com.google.cloud.dataproc.v1.JobPlacement; -import com.google.cloud.dataproc.v1.JobStatus; -import com.google.cloud.dataproc.v1.PySparkJob; -import java.io.IOException; -import java.util.concurrent.ExecutionException; - -public class SubmitJob { - - public static void submitJob() throws IOException, InterruptedException, ExecutionException { - // TODO(developer): Replace these variables before running the sample. - String projectId = "your-project-id"; - String region = "your-project-region"; - String clusterName = "your-cluster-name"; - String jobFilePath = "your-job-file-path"; - submitJob(projectId, region, clusterName, jobFilePath); - } - - public static void submitJob( - String projectId, String region, String clusterName, String jobFilePath) throws IOException { - String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); - - // Configure the settings for the cluster controller client - JobControllerSettings jobControllerSettings = - JobControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); - - // Create a job controller client with the configured settings. The client only needs to be - // created once and can be reused for multiple requests. Using a try-with-resources - // closes the client, but this can also be done manually with the .close() method. - try (JobControllerClient jobControllerClient = - JobControllerClient.create(jobControllerSettings)) { - - // Configure the settings for our job - JobPlacement jobPlacement = JobPlacement.newBuilder().setClusterName(clusterName).build(); - PySparkJob pySparkJob = PySparkJob.newBuilder().setMainPythonFileUri(jobFilePath).build(); - Job job = Job.newBuilder().setPlacement(jobPlacement).setPysparkJob(pySparkJob).build(); - - Job jobOperation = jobControllerClient.submitJob(projectId, region, job); - - String jobId = jobOperation.getReference().getJobId(); - - while (true) { - Job jobInfo = jobControllerClient.getJob(projectId, region, jobId); - if (jobInfo.getStatus().getState().equals(JobStatus.State.ERROR)) { - System.out.printf("Job %s failed: %s", jobId, jobInfo.getStatus().getDetails()); - break; - } else if (jobInfo.getStatus().getState().equals(JobStatus.State.DONE)) { - System.out.printf("Job %s finished.", jobId); - break; - } - } - } - } -} -// [END dataproc_submit_job] diff --git a/dataproc/src/main/java/UpdateCluster.java b/dataproc/src/main/java/UpdateCluster.java deleted file mode 100644 index 7eb7afc9f62..00000000000 --- a/dataproc/src/main/java/UpdateCluster.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright 2019 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// [START dataproc_update_cluster] -import com.google.api.gax.longrunning.OperationFuture; -import com.google.cloud.dataproc.v1.Cluster; -import com.google.cloud.dataproc.v1.ClusterConfig; -import com.google.cloud.dataproc.v1.ClusterControllerClient; -import com.google.cloud.dataproc.v1.ClusterControllerSettings; -import com.google.cloud.dataproc.v1.ClusterOperationMetadata; -import com.google.cloud.dataproc.v1.InstanceGroupConfig; -import com.google.protobuf.FieldMask; -import java.io.IOException; -import java.util.concurrent.ExecutionException; - -public class UpdateCluster { - - public static void updateCluster() throws IOException, InterruptedException, ExecutionException { - // TODO(developer): Replace these variables before running the sample. - String projectId = "your-project-id"; - String region = "your-project-region"; - String clusterName = "your-cluster-name"; - int numWorkers = 0; // your number of workers - updateCluster(projectId, region, clusterName, numWorkers); - } - - public static void updateCluster( - String projectId, String region, String clusterName, int numWorkers) - throws IOException, InterruptedException, ExecutionException { - String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); - - // Configure the settings for the cluster controller client - ClusterControllerSettings clusterControllerSettings = - ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); - - // Create a cluster controller client with the configured settings. The client only needs to be - // created once and can be reused for multiple requests. Using a try-with-resources - // closes the client, but this can also be done manually with the .close() method. - try (ClusterControllerClient clusterControllerClient = - ClusterControllerClient.create(clusterControllerSettings)) { - // Configure the settings for our cluster - InstanceGroupConfig masterConfig = - InstanceGroupConfig.newBuilder() - .setMachineTypeUri("n1-standard-1") - .setNumInstances(1) - .build(); - InstanceGroupConfig workerConfig = - InstanceGroupConfig.newBuilder() - .setMachineTypeUri("n1-standard-1") - .setNumInstances(numWorkers) - .build(); - ClusterConfig clusterConfig = - ClusterConfig.newBuilder() - .setMasterConfig(masterConfig) - .setWorkerConfig(workerConfig) - .build(); - // Create the cluster object with the desired cluster config - Cluster cluster = - Cluster.newBuilder().setClusterName(clusterName).setConfig(clusterConfig).build(); - - // Create a field mask to indicate what field of the cluster to update. - FieldMask updateMask = - FieldMask.newBuilder().addPaths("config.worker_config.num_instances").build(); - - // Submit the request to update the cluster - OperationFuture updateClusterAsyncRequest = - clusterControllerClient.updateClusterAsync( - projectId, region, clusterName, cluster, updateMask); - Cluster response = updateClusterAsyncRequest.get(); - - // Print out a success message - System.out.printf( - "Cluster %s now has %d workers.", - response.getClusterName(), response.getConfig().getWorkerConfig().getNumInstances()); - } - } -} -// [END dataproc_update_cluster] diff --git a/dataproc/src/test/java/DeleteClusterTest.java b/dataproc/src/test/java/CreateCluster similarity index 73% rename from dataproc/src/test/java/DeleteClusterTest.java rename to dataproc/src/test/java/CreateCluster index 62c96557612..07d78a28adf 100644 --- a/dataproc/src/test/java/DeleteClusterTest.java +++ b/dataproc/src/test/java/CreateCluster @@ -18,11 +18,10 @@ import static org.hamcrest.MatcherAssert.assertThat; import com.google.api.gax.longrunning.OperationFuture; -import com.google.cloud.dataproc.v1.Cluster; -import com.google.cloud.dataproc.v1.ClusterConfig; import com.google.cloud.dataproc.v1.ClusterControllerClient; import com.google.cloud.dataproc.v1.ClusterControllerSettings; import com.google.cloud.dataproc.v1.ClusterOperationMetadata; +import com.google.protobuf.Empty; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintStream; @@ -37,7 +36,7 @@ import org.junit.runners.JUnit4; @RunWith(JUnit4.class) -public class DeleteClusterTest { +public class CreateCluster { private static final String CLUSTER_NAME = String.format("test-cluster-%s", UUID.randomUUID().toString()); @@ -60,42 +59,33 @@ public static void checkRequirements() { } @Before - public void setUp() throws IOException, InterruptedException { + public void setUp() { bout = new ByteArrayOutputStream(); System.setOut(new PrintStream(bout)); standardOutOrig = System.out; - - String myEndpoint = String.format("%s-dataproc.googleapis.com:443", REGION); - - ClusterControllerSettings clusterControllerSettings = - ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); - - Cluster cluster = - Cluster.newBuilder() - .setClusterName(CLUSTER_NAME) - .setConfig(ClusterConfig.newBuilder().build()) - .build(); - - try (ClusterControllerClient clusterControllerClient = - ClusterControllerClient.create(clusterControllerSettings)) { - OperationFuture createClusterAsyncRequest = - clusterControllerClient.createClusterAsync(PROJECT_ID, REGION, cluster); - createClusterAsyncRequest.get(); - } catch (ExecutionException e) { - System.err.println("[deleteCluster] Error during test cluster creation: \n" + e.getMessage()); - } } @Test - public void deleteClusterTest() throws IOException, InterruptedException, ExecutionException { - DeleteCluster.deleteCluster(PROJECT_ID, REGION, CLUSTER_NAME); + public void createClusterTest() throws IOException, InterruptedException, ExecutionException { + Quickstart.createCluster(PROJECT_ID, REGION, CLUSTER_NAME); String output = bout.toString(); - assertThat(output, CoreMatchers.containsString("deleted successfully")); + assertThat(output, CoreMatchers.containsString(CLUSTER_NAME)); } @After - public void teardown() { + public void tearDown() throws IOException, InterruptedException, ExecutionException { + String myEndpoint = String.format("%s-dataproc.googleapis.com:443", REGION); System.setOut(standardOutOrig); + + ClusterControllerSettings clusterControllerSettings = + ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); + + try (ClusterControllerClient clusterControllerClient = + ClusterControllerClient.create(clusterControllerSettings)) { + OperationFuture deleteClusterAsyncRequest = + clusterControllerClient.deleteClusterAsync(PROJECT_ID, REGION, CLUSTER_NAME); + deleteClusterAsyncRequest.get(); + } } } diff --git a/dataproc/src/test/java/ListClustersTest.java b/dataproc/src/test/java/ListClustersTest.java deleted file mode 100644 index 93c0b970b41..00000000000 --- a/dataproc/src/test/java/ListClustersTest.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright 2019 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import static junit.framework.TestCase.assertNotNull; -import static org.hamcrest.MatcherAssert.assertThat; - -import com.google.api.gax.longrunning.OperationFuture; -import com.google.cloud.dataproc.v1.Cluster; -import com.google.cloud.dataproc.v1.ClusterConfig; -import com.google.cloud.dataproc.v1.ClusterControllerClient; -import com.google.cloud.dataproc.v1.ClusterControllerSettings; -import com.google.cloud.dataproc.v1.ClusterOperationMetadata; -import com.google.protobuf.Empty; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.PrintStream; -import java.util.UUID; -import java.util.concurrent.ExecutionException; -import org.hamcrest.CoreMatchers; -import org.junit.After; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -@RunWith(JUnit4.class) -public class ListClustersTest { - - private static final String CLUSTER_NAME = - String.format("test-cluster-%s", UUID.randomUUID().toString()); - private static final String REGION = "us-central1"; - private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); - - private String myEndpoint; - private ByteArrayOutputStream bout; - private PrintStream standardOutOrig; - - private static void requireEnv(String varName) { - assertNotNull( - System.getenv(varName), - String.format("Environment variable '%s' is required to perform these tests.", varName)); - } - - @BeforeClass - public static void checkRequirements() { - requireEnv("GOOGLE_APPLICATION_CREDENTIALS"); - requireEnv("GOOGLE_CLOUD_PROJECT"); - } - - @Before - public void setUp() throws IOException, InterruptedException { - bout = new ByteArrayOutputStream(); - standardOutOrig = System.out; - System.setOut(new PrintStream(bout)); - myEndpoint = String.format("%s-dataproc.googleapis.com:443", REGION); - - ClusterControllerSettings clusterControllerSettings = - ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); - - Cluster cluster = - Cluster.newBuilder() - .setClusterName(CLUSTER_NAME) - .setConfig(ClusterConfig.newBuilder().build()) - .build(); - - try (ClusterControllerClient clusterControllerClient = - ClusterControllerClient.create(clusterControllerSettings)) { - OperationFuture createClusterAsyncRequest = - clusterControllerClient.createClusterAsync(PROJECT_ID, REGION, cluster); - createClusterAsyncRequest.get(); - } catch (ExecutionException e) { - System.err.println("[deleteCluster] Error during test cluster creation: \n" + e.getMessage()); - } - } - - @Test - public void listClustersTest() throws IOException { - ListClusters.listClusters(PROJECT_ID, REGION); - String output = bout.toString(); - - assertThat(output, CoreMatchers.containsString(CLUSTER_NAME)); - assertThat(output, CoreMatchers.containsString("RUNNING")); - } - - @After - public void teardown() throws IOException, InterruptedException, ExecutionException { - System.setOut(standardOutOrig); - - ClusterControllerSettings clusterControllerSettings = - ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); - - try (ClusterControllerClient clusterControllerClient = - ClusterControllerClient.create(clusterControllerSettings)) { - OperationFuture deleteClusterAsyncRequest = - clusterControllerClient.deleteClusterAsync(PROJECT_ID, REGION, CLUSTER_NAME); - deleteClusterAsyncRequest.get(); - } - } -} diff --git a/dataproc/src/test/java/SubmitJobTest.java b/dataproc/src/test/java/QuickstartTest.java similarity index 75% rename from dataproc/src/test/java/SubmitJobTest.java rename to dataproc/src/test/java/QuickstartTest.java index 8484f4b7dc8..7351fc67ac3 100644 --- a/dataproc/src/test/java/SubmitJobTest.java +++ b/dataproc/src/test/java/QuickstartTest.java @@ -20,7 +20,6 @@ import com.google.api.gax.longrunning.OperationFuture; import com.google.cloud.dataproc.v1.Cluster; -import com.google.cloud.dataproc.v1.ClusterConfig; import com.google.cloud.dataproc.v1.ClusterControllerClient; import com.google.cloud.dataproc.v1.ClusterControllerSettings; import com.google.cloud.dataproc.v1.ClusterOperationMetadata; @@ -44,7 +43,7 @@ import org.junit.runners.JUnit4; @RunWith(JUnit4.class) -public class SubmitJobTest { +public class QuickstartTest { private static final String MY_UUID = UUID.randomUUID().toString(); private static final String REGION = "us-central1"; @@ -86,32 +85,17 @@ public void setUp() throws IOException, InterruptedException { Storage storage = StorageOptions.getDefaultInstance().getService(); Bucket bucket = storage.create(BucketInfo.of(BUCKET_NAME)); blob = bucket.create(JOB_FILE_NAME, SORT_CODE.getBytes(UTF_8), "text/plain"); - - ClusterControllerSettings clusterControllerSettings = - ClusterControllerSettings.newBuilder().setEndpoint(ENDPOINT).build(); - - Cluster cluster = - Cluster.newBuilder() - .setClusterName(CLUSTER_NAME) - .setConfig(ClusterConfig.newBuilder().build()) - .build(); - - try (ClusterControllerClient clusterControllerClient = - ClusterControllerClient.create(clusterControllerSettings)) { - OperationFuture createClusterAsyncRequest = - clusterControllerClient.createClusterAsync(PROJECT_ID, REGION, cluster); - createClusterAsyncRequest.get(); - } catch (ExecutionException e) { - System.err.println("[SubmitJob] Error during test cluster creation: \n" + e.getMessage()); - } } @Test - public void submitJobTest() throws IOException { - SubmitJob.submitJob(PROJECT_ID, REGION, CLUSTER_NAME, JOB_FILE_PATH); + public void quickstartTest() throws IOException, InterruptedException, ExecutionException { + Quickstart.quickstart(PROJECT_ID, REGION, CLUSTER_NAME, JOB_FILE_PATH); String output = bout.toString(); - assertThat(output, CoreMatchers.containsString("finished")); + assertThat(output, CoreMatchers.containsString("Cluster created successfully")); + assertThat(output, CoreMatchers.containsString("Submitted job")); + assertThat(output, CoreMatchers.containsString("finished with state DONE:")); + assertThat(output, CoreMatchers.containsString("successfully deleted")); } @After @@ -124,9 +108,15 @@ public void teardown() throws IOException, InterruptedException, ExecutionExcept try (ClusterControllerClient clusterControllerClient = ClusterControllerClient.create(clusterControllerSettings)) { - OperationFuture deleteClusterAsyncRequest = - clusterControllerClient.deleteClusterAsync(PROJECT_ID, REGION, CLUSTER_NAME); - deleteClusterAsyncRequest.get(); + for (Cluster element : + clusterControllerClient.listClusters(PROJECT_ID, REGION).iterateAll()) { + if (element.getClusterName() == CLUSTER_NAME) { + OperationFuture deleteClusterAsyncRequest = + clusterControllerClient.deleteClusterAsync(PROJECT_ID, REGION, CLUSTER_NAME); + deleteClusterAsyncRequest.get(); + break; + } + } } } } diff --git a/dataproc/src/test/java/UpdateClusterTest.java b/dataproc/src/test/java/UpdateClusterTest.java deleted file mode 100644 index ac3aa869431..00000000000 --- a/dataproc/src/test/java/UpdateClusterTest.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright 2019 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import static junit.framework.TestCase.assertNotNull; -import static org.hamcrest.MatcherAssert.assertThat; - -import com.google.api.gax.longrunning.OperationFuture; -import com.google.cloud.dataproc.v1.Cluster; -import com.google.cloud.dataproc.v1.ClusterConfig; -import com.google.cloud.dataproc.v1.ClusterControllerClient; -import com.google.cloud.dataproc.v1.ClusterControllerSettings; -import com.google.cloud.dataproc.v1.ClusterOperationMetadata; -import com.google.cloud.dataproc.v1.InstanceGroupConfig; -import com.google.protobuf.Empty; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.PrintStream; -import java.util.UUID; -import java.util.concurrent.ExecutionException; -import org.hamcrest.CoreMatchers; -import org.junit.After; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -@RunWith(JUnit4.class) -public class UpdateClusterTest { - - private static final String REGION = "us-central1"; - private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); - private static final String ENDPOINT = String.format("%s-dataproc.googleapis.com:443", REGION); - private static final String CLUSTER_NAME = - String.format("test-cluster-%s", UUID.randomUUID().toString()); - private static final int NUM_WORKERS = 2; - private static final int NEW_WORKERS = NUM_WORKERS * 2; - - private ByteArrayOutputStream bout; - private PrintStream standardOutOrig; - - private static void requireEnv(String varName) { - assertNotNull( - System.getenv(varName), - String.format("Environment variable '%s' is required to perform these tests.", varName)); - } - - @BeforeClass - public static void checkRequirements() { - requireEnv("GOOGLE_APPLICATION_CREDENTIALS"); - requireEnv("GOOGLE_CLOUD_PROJECT"); - } - - @Before - public void setUp() throws IOException, InterruptedException, ExecutionException { - bout = new ByteArrayOutputStream(); - standardOutOrig = System.out; - System.setOut(new PrintStream(bout)); - - ClusterControllerSettings clusterControllerSettings = - ClusterControllerSettings.newBuilder().setEndpoint(ENDPOINT).build(); - - // Configure the settings for our cluster - InstanceGroupConfig masterConfig = - InstanceGroupConfig.newBuilder() - .setMachineTypeUri("n1-standard-1") - .setNumInstances(1) - .build(); - InstanceGroupConfig workerConfig = - InstanceGroupConfig.newBuilder() - .setMachineTypeUri("n1-standard-1") - .setNumInstances(NUM_WORKERS) - .build(); - ClusterConfig clusterConfig = - ClusterConfig.newBuilder() - .setMasterConfig(masterConfig) - .setWorkerConfig(workerConfig) - .build(); - // Create the cluster object with the desired cluster config - Cluster cluster = - Cluster.newBuilder().setClusterName(CLUSTER_NAME).setConfig(clusterConfig).build(); - - try (ClusterControllerClient clusterControllerClient = - ClusterControllerClient.create(clusterControllerSettings)) { - OperationFuture createClusterAsyncRequest = - clusterControllerClient.createClusterAsync(PROJECT_ID, REGION, cluster); - createClusterAsyncRequest.get(); - } - } - - @Test - public void updateClusterTest() throws IOException, InterruptedException, ExecutionException { - UpdateCluster.updateCluster(PROJECT_ID, REGION, CLUSTER_NAME, NEW_WORKERS); - String output = bout.toString(); - - assertThat(output, CoreMatchers.containsString(String.format("%d workers", NEW_WORKERS))); - } - - @After - public void teardown() throws IOException, InterruptedException, ExecutionException { - System.setOut(standardOutOrig); - - ClusterControllerSettings clusterControllerSettings = - ClusterControllerSettings.newBuilder().setEndpoint(ENDPOINT).build(); - - try (ClusterControllerClient clusterControllerClient = - ClusterControllerClient.create(clusterControllerSettings)) { - OperationFuture deleteClusterAsyncRequest = - clusterControllerClient.deleteClusterAsync(PROJECT_ID, REGION, CLUSTER_NAME); - deleteClusterAsyncRequest.get(); - } - } -} From b9eaf2574fa6f99f2d5c1bef000457d5dbad8c11 Mon Sep 17 00:00:00 2001 From: bradmiro Date: Fri, 20 Dec 2019 12:40:00 -0500 Subject: [PATCH 14/21] Added quickstart sample --- dataproc/pom.xml | 13 +++- dataproc/src/test/java/CreateCluster | 91 ---------------------------- 2 files changed, 12 insertions(+), 92 deletions(-) delete mode 100644 dataproc/src/test/java/CreateCluster diff --git a/dataproc/pom.xml b/dataproc/pom.xml index 578889c4a60..c23e5411d9c 100644 --- a/dataproc/pom.xml +++ b/dataproc/pom.xml @@ -36,6 +36,18 @@ + + + + com.google.cloud + libraries-bom + 3.0.0 + pom + import + + + + @@ -52,7 +64,6 @@ com.google.cloud google-cloud-storage - 1.101.0 diff --git a/dataproc/src/test/java/CreateCluster b/dataproc/src/test/java/CreateCluster deleted file mode 100644 index 07d78a28adf..00000000000 --- a/dataproc/src/test/java/CreateCluster +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright 2019 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import static junit.framework.TestCase.assertNotNull; -import static org.hamcrest.MatcherAssert.assertThat; - -import com.google.api.gax.longrunning.OperationFuture; -import com.google.cloud.dataproc.v1.ClusterControllerClient; -import com.google.cloud.dataproc.v1.ClusterControllerSettings; -import com.google.cloud.dataproc.v1.ClusterOperationMetadata; -import com.google.protobuf.Empty; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.PrintStream; -import java.util.UUID; -import java.util.concurrent.ExecutionException; -import org.hamcrest.CoreMatchers; -import org.junit.After; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -@RunWith(JUnit4.class) -public class CreateCluster { - - private static final String CLUSTER_NAME = - String.format("test-cluster-%s", UUID.randomUUID().toString()); - private static final String REGION = "us-central1"; - private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); - - private ByteArrayOutputStream bout; - private PrintStream standardOutOrig; - - private static void requireEnv(String varName) { - assertNotNull( - System.getenv(varName), - String.format("Environment variable '%s' is required to perform these tests.", varName)); - } - - @BeforeClass - public static void checkRequirements() { - requireEnv("GOOGLE_APPLICATION_CREDENTIALS"); - requireEnv("GOOGLE_CLOUD_PROJECT"); - } - - @Before - public void setUp() { - bout = new ByteArrayOutputStream(); - System.setOut(new PrintStream(bout)); - standardOutOrig = System.out; - } - - @Test - public void createClusterTest() throws IOException, InterruptedException, ExecutionException { - Quickstart.createCluster(PROJECT_ID, REGION, CLUSTER_NAME); - String output = bout.toString(); - - assertThat(output, CoreMatchers.containsString(CLUSTER_NAME)); - } - - @After - public void tearDown() throws IOException, InterruptedException, ExecutionException { - String myEndpoint = String.format("%s-dataproc.googleapis.com:443", REGION); - System.setOut(standardOutOrig); - - ClusterControllerSettings clusterControllerSettings = - ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); - - try (ClusterControllerClient clusterControllerClient = - ClusterControllerClient.create(clusterControllerSettings)) { - OperationFuture deleteClusterAsyncRequest = - clusterControllerClient.deleteClusterAsync(PROJECT_ID, REGION, CLUSTER_NAME); - deleteClusterAsyncRequest.get(); - } - } -} From ea1965f25b62bbb4711f9d9745eb271f2fc01409 Mon Sep 17 00:00:00 2001 From: bradmiro Date: Thu, 26 Dec 2019 13:18:53 -0500 Subject: [PATCH 15/21] Updates to createCluster and quickstart --- dataproc/src/main/java/Quickstart.java | 172 ++++++++++++++++++ dataproc/src/test/java/CreateClusterTest.java | 2 +- dataproc/src/test/java/QuickstartTest.java | 10 +- 3 files changed, 180 insertions(+), 4 deletions(-) create mode 100644 dataproc/src/main/java/Quickstart.java diff --git a/dataproc/src/main/java/Quickstart.java b/dataproc/src/main/java/Quickstart.java new file mode 100644 index 00000000000..3153abc18ee --- /dev/null +++ b/dataproc/src/main/java/Quickstart.java @@ -0,0 +1,172 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// [START dataproc_quickstart] +import com.google.api.gax.longrunning.OperationFuture; +import com.google.cloud.dataproc.v1.Cluster; +import com.google.cloud.dataproc.v1.ClusterConfig; +import com.google.cloud.dataproc.v1.ClusterControllerClient; +import com.google.cloud.dataproc.v1.ClusterControllerSettings; +import com.google.cloud.dataproc.v1.ClusterOperationMetadata; +import com.google.cloud.dataproc.v1.InstanceGroupConfig; +import com.google.cloud.dataproc.v1.Job; +import com.google.cloud.dataproc.v1.JobControllerClient; +import com.google.cloud.dataproc.v1.JobControllerSettings; +import com.google.cloud.dataproc.v1.JobPlacement; +import com.google.cloud.dataproc.v1.PySparkJob; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageOptions; +import com.google.protobuf.Empty; +import java.io.IOException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +public class Quickstart { + + public static void quickstart() throws IOException, InterruptedException { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String region = "your-project-region"; + String clusterName = "your-cluster-name"; + String jobFilePath = "your-job-file-path"; + quickstart(projectId, region, clusterName, jobFilePath); + } + + public static void quickstart( + String projectId, String region, String clusterName, String jobFilePath) + throws IOException, InterruptedException { + String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); + + // Configure the settings for the cluster controller client + ClusterControllerSettings clusterControllerSettings = + ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); + + // Configure the settings for the job controller client + JobControllerSettings jobControllerSettings = + JobControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); + + // Create both a cluster controller client and job controller client with the configured + // settings. The client only needs to be created once and can be reused for multiple requests. + // Using a try-with-resources closes the client, but this can also be done manually with + // the .close() method. + try (ClusterControllerClient clusterControllerClient = + ClusterControllerClient.create(clusterControllerSettings); + JobControllerClient jobControllerClient = + JobControllerClient.create(jobControllerSettings)) { + // Configure the settings for our cluster + InstanceGroupConfig masterConfig = + InstanceGroupConfig.newBuilder() + .setMachineTypeUri("n1-standard-1") + .setNumInstances(1) + .build(); + InstanceGroupConfig workerConfig = + InstanceGroupConfig.newBuilder() + .setMachineTypeUri("n1-standard-1") + .setNumInstances(2) + .build(); + ClusterConfig clusterConfig = + ClusterConfig.newBuilder() + .setMasterConfig(masterConfig) + .setWorkerConfig(workerConfig) + .build(); + // Create the cluster object with the desired cluster config + Cluster cluster = + Cluster.newBuilder().setClusterName(clusterName).setConfig(clusterConfig).build(); + + // Create the Cloud Dataproc cluster + OperationFuture createClusterAsyncRequest = + clusterControllerClient.createClusterAsync(projectId, region, cluster); + Cluster response = createClusterAsyncRequest.get(); + System.out.printf("Cluster created successfully: %s", response.getClusterName()); + + // Configure the settings for our job + JobPlacement jobPlacement = JobPlacement.newBuilder().setClusterName(clusterName).build(); + PySparkJob pySparkJob = PySparkJob.newBuilder().setMainPythonFileUri(jobFilePath).build(); + Job job = Job.newBuilder().setPlacement(jobPlacement).setPysparkJob(pySparkJob).build(); + + // Submit an asynchronous request to execute the job + Job request = jobControllerClient.submitJob(projectId, region, job); + String jobId = request.getReference().getJobId(); + System.out.println(String.format("Submitted job \"%s\"", jobId)); + + // Wait for the job to finish + CompletableFuture finishedJobFuture = + CompletableFuture.supplyAsync( + () -> { + // States that indicate a job is "finished" + while (true) { + // Poll the service periodically until the Job is in a finished state. + Job jobInfo = jobControllerClient.getJob(projectId, region, jobId); + switch (jobInfo.getStatus().getState()) { + case DONE: + return jobInfo; + case CANCELLED: + case ERROR: + case ATTEMPT_FAILURE: + throw new RuntimeException( + String.format( + "Job %s failed due to %s: %s", + jobId, + jobInfo.getStatus().getState(), + jobInfo.getStatus().getDetails())); + default: + try { + // Wait a second in between polling attempts. + TimeUnit.SECONDS.sleep(1); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } + }); + int timeout = 10; + try { + Job jobInfo = finishedJobFuture.get(timeout, TimeUnit.MINUTES); + System.out.printf("Job %s finished successfully.", jobId); + + // Cloud Dataproc job output gets saved to a GCS bucket allocated to it. + Cluster clusterInfo = clusterControllerClient.getCluster(projectId, region, clusterName); + Storage storage = StorageOptions.getDefaultInstance().getService(); + Blob blob = + storage.get( + clusterInfo.getConfig().getConfigBucket(), + String.format( + "google-cloud-dataproc-metainfo/%s/jobs/%s/driveroutput.000000000", + clusterInfo.getClusterUuid(), jobId)); + System.out.println( + String.format( + "Job \"%s\" finished with state %s:\n%s", + jobId, jobInfo.getStatus().getState(), new String(blob.getContent()))); + } catch (TimeoutException e) { + System.err.println( + String.format("Job timed out after %d minutes: %s", timeout, e.getMessage())); + } + + // Delete the cluster + OperationFuture deleteClusterAsyncRequest = + clusterControllerClient.deleteClusterAsync(projectId, region, clusterName); + deleteClusterAsyncRequest.get(); + System.out.println(String.format("Cluster \"%s\" successfully deleted.", clusterName)); + + } catch (ExecutionException e) { + System.err.println(String.format("Error executing quickstart: " + e.getMessage())); + } + } +} +// [END dataproc_quickstart] \ No newline at end of file diff --git a/dataproc/src/test/java/CreateClusterTest.java b/dataproc/src/test/java/CreateClusterTest.java index da4c321b80e..1afe81fc769 100644 --- a/dataproc/src/test/java/CreateClusterTest.java +++ b/dataproc/src/test/java/CreateClusterTest.java @@ -39,7 +39,7 @@ public class CreateClusterTest { private static final String CLUSTER_NAME = - String.format("test-cluster-%s", UUID.randomUUID().toString()); + String.format("java-dataproc-create-cluster-test-cluster-%s", UUID.randomUUID().toString()); private static final String REGION = "us-central1"; private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); diff --git a/dataproc/src/test/java/QuickstartTest.java b/dataproc/src/test/java/QuickstartTest.java index 7351fc67ac3..82f1ba2519c 100644 --- a/dataproc/src/test/java/QuickstartTest.java +++ b/dataproc/src/test/java/QuickstartTest.java @@ -49,8 +49,10 @@ public class QuickstartTest { private static final String REGION = "us-central1"; private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); private static final String ENDPOINT = String.format("%s-dataproc.googleapis.com:443", REGION); - private static final String CLUSTER_NAME = String.format("test-cluster-%s", MY_UUID); - private static final String BUCKET_NAME = String.format("test-bucket-%s", MY_UUID); + private static final String CLUSTER_NAME = + String.format("java-dataproc-quickstart-test-bucket%s", MY_UUID); + private static final String BUCKET_NAME = + String.format("java-dataproc-quickstart-test-bucket-%s", MY_UUID); private static final String JOB_FILE_NAME = "sum.py"; private static final String JOB_FILE_PATH = String.format("gs://%s/%s", BUCKET_NAME, JOB_FILE_NAME); @@ -62,6 +64,7 @@ public class QuickstartTest { private ByteArrayOutputStream bout; private PrintStream standardOutOrig; + private Bucket bucket; private Blob blob; private static void requireEnv(String varName) { @@ -83,7 +86,7 @@ public void setUp() throws IOException, InterruptedException { System.setOut(new PrintStream(bout)); Storage storage = StorageOptions.getDefaultInstance().getService(); - Bucket bucket = storage.create(BucketInfo.of(BUCKET_NAME)); + bucket = storage.create(BucketInfo.of(BUCKET_NAME)); blob = bucket.create(JOB_FILE_NAME, SORT_CODE.getBytes(UTF_8), "text/plain"); } @@ -102,6 +105,7 @@ public void quickstartTest() throws IOException, InterruptedException, Execution public void teardown() throws IOException, InterruptedException, ExecutionException { System.setOut(standardOutOrig); blob.delete(); + bucket.delete(); ClusterControllerSettings clusterControllerSettings = ClusterControllerSettings.newBuilder().setEndpoint(ENDPOINT).build(); From 539746f343f8d945e7d2eb81131e76bc14df366d Mon Sep 17 00:00:00 2001 From: bradmiro Date: Thu, 26 Dec 2019 13:36:27 -0500 Subject: [PATCH 16/21] Fixed quickstart and tests --- dataproc/src/main/java/Quickstart.java | 50 ++++++++++++-------------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/dataproc/src/main/java/Quickstart.java b/dataproc/src/main/java/Quickstart.java index 3153abc18ee..dfba42ec86e 100644 --- a/dataproc/src/main/java/Quickstart.java +++ b/dataproc/src/main/java/Quickstart.java @@ -39,6 +39,26 @@ public class Quickstart { + public static Job waitForJobCompletion(JobControllerClient jobControllerClient, String projectId, String region, String jobId) { + while (true) { + // Poll the service periodically until the Job is in a finished state. + Job jobInfo = jobControllerClient.getJob(projectId, region, jobId); + switch (jobInfo.getStatus().getState()) { + case DONE: + case CANCELLED: + case ERROR: + return jobInfo; + default: + try { + // Wait a second in between polling attempts. + TimeUnit.SECONDS.sleep(1); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } + } + public static void quickstart() throws IOException, InterruptedException { // TODO(developer): Replace these variables before running the sample. String projectId = "your-project-id"; @@ -107,34 +127,8 @@ public static void quickstart( // Wait for the job to finish CompletableFuture finishedJobFuture = - CompletableFuture.supplyAsync( - () -> { - // States that indicate a job is "finished" - while (true) { - // Poll the service periodically until the Job is in a finished state. - Job jobInfo = jobControllerClient.getJob(projectId, region, jobId); - switch (jobInfo.getStatus().getState()) { - case DONE: - return jobInfo; - case CANCELLED: - case ERROR: - case ATTEMPT_FAILURE: - throw new RuntimeException( - String.format( - "Job %s failed due to %s: %s", - jobId, - jobInfo.getStatus().getState(), - jobInfo.getStatus().getDetails())); - default: - try { - // Wait a second in between polling attempts. - TimeUnit.SECONDS.sleep(1); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - } - }); + CompletableFuture.supplyAsync(() -> + waitForJobCompletion(jobControllerClient, projectId, region, jobId)); int timeout = 10; try { Job jobInfo = finishedJobFuture.get(timeout, TimeUnit.MINUTES); From cdc616e544e69ba9fd86f74b334503d2edb1608a Mon Sep 17 00:00:00 2001 From: bradmiro Date: Fri, 27 Dec 2019 18:16:16 -0500 Subject: [PATCH 17/21] Changes to tests --- dataproc/src/test/java/CreateClusterTest.java | 7 ++----- dataproc/src/test/java/QuickstartTest.java | 11 ++++------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/dataproc/src/test/java/CreateClusterTest.java b/dataproc/src/test/java/CreateClusterTest.java index 574130a2730..9aae14b4e7b 100644 --- a/dataproc/src/test/java/CreateClusterTest.java +++ b/dataproc/src/test/java/CreateClusterTest.java @@ -44,12 +44,11 @@ public class CreateClusterTest { private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); private ByteArrayOutputStream bout; - private PrintStream standardOutOrig; private static void requireEnv(String varName) { assertNotNull( - System.getenv(varName), - String.format("Environment variable '%s' is required to perform these tests.", varName)); + String.format("Environment variable '%s' is required to perform these tests.", varName), + System.getenv(varName)); } @BeforeClass @@ -62,7 +61,6 @@ public static void checkRequirements() { public void setUp() { bout = new ByteArrayOutputStream(); System.setOut(new PrintStream(bout)); - standardOutOrig = System.out; } @Test @@ -76,7 +74,6 @@ public void createClusterTest() throws IOException, InterruptedException, Execut @After public void tearDown() throws IOException, InterruptedException, ExecutionException { String myEndpoint = String.format("%s-dataproc.googleapis.com:443", REGION); - System.setOut(standardOutOrig); ClusterControllerSettings clusterControllerSettings = ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); diff --git a/dataproc/src/test/java/QuickstartTest.java b/dataproc/src/test/java/QuickstartTest.java index 3b876a4fdd7..41ebc20cbf0 100644 --- a/dataproc/src/test/java/QuickstartTest.java +++ b/dataproc/src/test/java/QuickstartTest.java @@ -50,9 +50,9 @@ public class QuickstartTest { private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); private static final String ENDPOINT = String.format("%s-dataproc.googleapis.com:443", REGION); private static final String CLUSTER_NAME = - String.format("java-dataproc-quickstart-test-%s", MY_UUID); + String.format("java-qs-test-%s", MY_UUID); private static final String BUCKET_NAME = - String.format("java-dataproc-quickstart-test-%s", MY_UUID); + String.format("java-dataproc-qs-test-%s", MY_UUID); private static final String JOB_FILE_NAME = "sum.py"; private static final String JOB_FILE_PATH = String.format("gs://%s/%s", BUCKET_NAME, JOB_FILE_NAME); @@ -63,14 +63,13 @@ public class QuickstartTest { + "sum = rdd.reduce(lambda x, y: x + y)\n"; private ByteArrayOutputStream bout; - private PrintStream standardOutOrig; private Bucket bucket; private Blob blob; private static void requireEnv(String varName) { assertNotNull( - System.getenv(varName), - String.format("Environment variable '%s' is required to perform these tests.", varName)); + String.format("Environment variable '%s' is required to perform these tests.", varName), + System.getenv(varName)); } @BeforeClass @@ -82,7 +81,6 @@ public static void checkRequirements() { @Before public void setUp() throws IOException, InterruptedException { bout = new ByteArrayOutputStream(); - standardOutOrig = System.out; System.setOut(new PrintStream(bout)); Storage storage = StorageOptions.getDefaultInstance().getService(); @@ -103,7 +101,6 @@ public void quickstartTest() throws IOException, InterruptedException, Execution @After public void teardown() throws IOException, InterruptedException, ExecutionException { - System.setOut(standardOutOrig); blob.delete(); bucket.delete(); From 561ca346948df0aeba0f646b86b9f60d0c3d54d5 Mon Sep 17 00:00:00 2001 From: bradmiro Date: Fri, 27 Dec 2019 18:31:28 -0500 Subject: [PATCH 18/21] Added periods to comments --- dataproc/src/main/java/CreateCluster.java | 10 +++++----- dataproc/src/main/java/Quickstart.java | 18 +++++++++--------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/dataproc/src/main/java/CreateCluster.java b/dataproc/src/main/java/CreateCluster.java index 0f0bd1c3ead..ff77511f5f7 100644 --- a/dataproc/src/main/java/CreateCluster.java +++ b/dataproc/src/main/java/CreateCluster.java @@ -39,7 +39,7 @@ public static void createCluster(String projectId, String region, String cluster throws IOException, InterruptedException, ExecutionException { String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); - // Configure the settings for the cluster controller client + // Configure the settings for the cluster controller client. ClusterControllerSettings clusterControllerSettings = ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); @@ -48,7 +48,7 @@ public static void createCluster(String projectId, String region, String cluster // closes the client, but this can also be done manually with the .close() method. try (ClusterControllerClient clusterControllerClient = ClusterControllerClient.create(clusterControllerSettings)) { - // Configure the settings for our cluster + // Configure the settings for our cluster. InstanceGroupConfig masterConfig = InstanceGroupConfig.newBuilder() .setMachineTypeUri("n1-standard-1") @@ -64,16 +64,16 @@ public static void createCluster(String projectId, String region, String cluster .setMasterConfig(masterConfig) .setWorkerConfig(workerConfig) .build(); - // Create the cluster object with the desired cluster config + // Create the cluster object with the desired cluster config. Cluster cluster = Cluster.newBuilder().setClusterName(clusterName).setConfig(clusterConfig).build(); - // Create the Cloud Dataproc cluster + // Create the Cloud Dataproc cluster. OperationFuture createClusterAsyncRequest = clusterControllerClient.createClusterAsync(projectId, region, cluster); Cluster response = createClusterAsyncRequest.get(); - // Print out a success message + // Print out a success message. System.out.printf("Cluster created successfully: %s", response.getClusterName()); } } diff --git a/dataproc/src/main/java/Quickstart.java b/dataproc/src/main/java/Quickstart.java index 7683920cde2..cd6592e2889 100644 --- a/dataproc/src/main/java/Quickstart.java +++ b/dataproc/src/main/java/Quickstart.java @@ -73,11 +73,11 @@ public static void quickstart( throws IOException, InterruptedException { String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); - // Configure the settings for the cluster controller client + // Configure the settings for the cluster controller client. ClusterControllerSettings clusterControllerSettings = ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); - // Configure the settings for the job controller client + // Configure the settings for the job controller client. JobControllerSettings jobControllerSettings = JobControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); @@ -89,7 +89,7 @@ public static void quickstart( ClusterControllerClient.create(clusterControllerSettings); JobControllerClient jobControllerClient = JobControllerClient.create(jobControllerSettings)) { - // Configure the settings for our cluster + // Configure the settings for our cluster. InstanceGroupConfig masterConfig = InstanceGroupConfig.newBuilder() .setMachineTypeUri("n1-standard-1") @@ -105,27 +105,27 @@ public static void quickstart( .setMasterConfig(masterConfig) .setWorkerConfig(workerConfig) .build(); - // Create the cluster object with the desired cluster config + // Create the cluster object with the desired cluster config. Cluster cluster = Cluster.newBuilder().setClusterName(clusterName).setConfig(clusterConfig).build(); - // Create the Cloud Dataproc cluster + // Create the Cloud Dataproc cluster. OperationFuture createClusterAsyncRequest = clusterControllerClient.createClusterAsync(projectId, region, cluster); Cluster response = createClusterAsyncRequest.get(); System.out.printf("Cluster created successfully: %s", response.getClusterName()); - // Configure the settings for our job + // Configure the settings for our job. JobPlacement jobPlacement = JobPlacement.newBuilder().setClusterName(clusterName).build(); PySparkJob pySparkJob = PySparkJob.newBuilder().setMainPythonFileUri(jobFilePath).build(); Job job = Job.newBuilder().setPlacement(jobPlacement).setPysparkJob(pySparkJob).build(); - // Submit an asynchronous request to execute the job + // Submit an asynchronous request to execute the job. Job request = jobControllerClient.submitJob(projectId, region, job); String jobId = request.getReference().getJobId(); System.out.println(String.format("Submitted job \"%s\"", jobId)); - // Wait for the job to finish + // Wait for the job to finish. CompletableFuture finishedJobFuture = CompletableFuture.supplyAsync(() -> waitForJobCompletion(jobControllerClient, projectId, region, jobId)); @@ -152,7 +152,7 @@ public static void quickstart( String.format("Job timed out after %d minutes: %s", timeout, e.getMessage())); } - // Delete the cluster + // Delete the cluster. OperationFuture deleteClusterAsyncRequest = clusterControllerClient.deleteClusterAsync(projectId, region, clusterName); deleteClusterAsyncRequest.get(); From 1ad40b6860b5746c26c2730489606a64459a7f0c Mon Sep 17 00:00:00 2001 From: bradmiro Date: Mon, 30 Dec 2019 15:51:36 -0500 Subject: [PATCH 19/21] Fixed pom and added handling for ExecutionException --- dataproc/pom.xml | 17 ++++++++--------- dataproc/src/main/java/CreateCluster.java | 7 +++++-- dataproc/src/test/java/CreateClusterTest.java | 2 +- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/dataproc/pom.xml b/dataproc/pom.xml index c23e5411d9c..240b527bc1b 100644 --- a/dataproc/pom.xml +++ b/dataproc/pom.xml @@ -41,7 +41,7 @@ com.google.cloud libraries-bom - 3.0.0 + 3.3.0 pom import @@ -49,22 +49,21 @@ - - - junit - junit - 4.12 - test - com.google.cloud google-cloud-dataproc - 0.118.0 com.google.cloud google-cloud-storage + + + junit + junit + 4.12 + test + \ No newline at end of file diff --git a/dataproc/src/main/java/CreateCluster.java b/dataproc/src/main/java/CreateCluster.java index ff77511f5f7..0815a35ffe0 100644 --- a/dataproc/src/main/java/CreateCluster.java +++ b/dataproc/src/main/java/CreateCluster.java @@ -27,7 +27,7 @@ public class CreateCluster { - public static void createCluster() throws IOException, InterruptedException, ExecutionException { + public static void createCluster() throws IOException, InterruptedException { // TODO(developer): Replace these variables before running the sample. String projectId = "your-project-id"; String region = "your-project-region"; @@ -36,7 +36,7 @@ public static void createCluster() throws IOException, InterruptedException, Exe } public static void createCluster(String projectId, String region, String clusterName) - throws IOException, InterruptedException, ExecutionException { + throws IOException, InterruptedException { String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); // Configure the settings for the cluster controller client. @@ -75,6 +75,9 @@ public static void createCluster(String projectId, String region, String cluster // Print out a success message. System.out.printf("Cluster created successfully: %s", response.getClusterName()); + + } catch (ExecutionException e) { + System.err.println(String.format("Error executing createCluster: %s ", e.getMessage())); } } } diff --git a/dataproc/src/test/java/CreateClusterTest.java b/dataproc/src/test/java/CreateClusterTest.java index 9aae14b4e7b..14ca6a9893d 100644 --- a/dataproc/src/test/java/CreateClusterTest.java +++ b/dataproc/src/test/java/CreateClusterTest.java @@ -39,7 +39,7 @@ public class CreateClusterTest { private static final String CLUSTER_NAME = - String.format("java-dataproc-create-cluster-test-%s", UUID.randomUUID().toString()); + String.format("java-cc-test-%s", UUID.randomUUID().toString()); private static final String REGION = "us-central1"; private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); From 64dbb5ef3e79066b8780d3f99f66651d9dc83b82 Mon Sep 17 00:00:00 2001 From: bradmiro Date: Mon, 30 Dec 2019 16:18:34 -0500 Subject: [PATCH 20/21] Fixed lint errors --- dataproc/src/main/java/Quickstart.java | 9 +++++---- dataproc/src/test/java/CreateClusterTest.java | 4 ++-- dataproc/src/test/java/QuickstartTest.java | 7 +++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/dataproc/src/main/java/Quickstart.java b/dataproc/src/main/java/Quickstart.java index cd6592e2889..0a0d177b1d0 100644 --- a/dataproc/src/main/java/Quickstart.java +++ b/dataproc/src/main/java/Quickstart.java @@ -39,7 +39,8 @@ public class Quickstart { - public static Job waitForJobCompletion(JobControllerClient jobControllerClient, String projectId, String region, String jobId) { + public static Job waitForJobCompletion( + JobControllerClient jobControllerClient, String projectId, String region, String jobId) { while (true) { // Poll the service periodically until the Job is in a finished state. Job jobInfo = jobControllerClient.getJob(projectId, region, jobId); @@ -127,8 +128,8 @@ public static void quickstart( // Wait for the job to finish. CompletableFuture finishedJobFuture = - CompletableFuture.supplyAsync(() -> - waitForJobCompletion(jobControllerClient, projectId, region, jobId)); + CompletableFuture.supplyAsync( + () -> waitForJobCompletion(jobControllerClient, projectId, region, jobId)); int timeout = 10; try { Job jobInfo = finishedJobFuture.get(timeout, TimeUnit.MINUTES); @@ -163,4 +164,4 @@ public static void quickstart( } } } -// [END dataproc_quickstart] \ No newline at end of file +// [END dataproc_quickstart] diff --git a/dataproc/src/test/java/CreateClusterTest.java b/dataproc/src/test/java/CreateClusterTest.java index 14ca6a9893d..539e0e2b624 100644 --- a/dataproc/src/test/java/CreateClusterTest.java +++ b/dataproc/src/test/java/CreateClusterTest.java @@ -39,7 +39,7 @@ public class CreateClusterTest { private static final String CLUSTER_NAME = - String.format("java-cc-test-%s", UUID.randomUUID().toString()); + String.format("java-cc-test-%s", UUID.randomUUID().toString()); private static final String REGION = "us-central1"; private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); @@ -64,7 +64,7 @@ public void setUp() { } @Test - public void createClusterTest() throws IOException, InterruptedException, ExecutionException { + public void createClusterTest() throws IOException, InterruptedException { CreateCluster.createCluster(PROJECT_ID, REGION, CLUSTER_NAME); String output = bout.toString(); diff --git a/dataproc/src/test/java/QuickstartTest.java b/dataproc/src/test/java/QuickstartTest.java index 41ebc20cbf0..9200082571a 100644 --- a/dataproc/src/test/java/QuickstartTest.java +++ b/dataproc/src/test/java/QuickstartTest.java @@ -49,8 +49,7 @@ public class QuickstartTest { private static final String REGION = "us-central1"; private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); private static final String ENDPOINT = String.format("%s-dataproc.googleapis.com:443", REGION); - private static final String CLUSTER_NAME = - String.format("java-qs-test-%s", MY_UUID); + private static final String CLUSTER_NAME = String.format("java-qs-test-%s", MY_UUID); private static final String BUCKET_NAME = String.format("java-dataproc-qs-test-%s", MY_UUID); private static final String JOB_FILE_NAME = "sum.py"; @@ -79,7 +78,7 @@ public static void checkRequirements() { } @Before - public void setUp() throws IOException, InterruptedException { + public void setUp() { bout = new ByteArrayOutputStream(); System.setOut(new PrintStream(bout)); @@ -89,7 +88,7 @@ public void setUp() throws IOException, InterruptedException { } @Test - public void quickstartTest() throws IOException, InterruptedException, ExecutionException { + public void quickstartTest() throws IOException, InterruptedException { Quickstart.quickstart(PROJECT_ID, REGION, CLUSTER_NAME, JOB_FILE_PATH); String output = bout.toString(); From db2c76f517d3409bced5fb2ff4ca43af5149dba4 Mon Sep 17 00:00:00 2001 From: bradmiro Date: Mon, 30 Dec 2019 16:35:21 -0500 Subject: [PATCH 21/21] Fixed linting errors --- dataproc/src/test/java/QuickstartTest.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dataproc/src/test/java/QuickstartTest.java b/dataproc/src/test/java/QuickstartTest.java index 9200082571a..3296e224828 100644 --- a/dataproc/src/test/java/QuickstartTest.java +++ b/dataproc/src/test/java/QuickstartTest.java @@ -50,8 +50,7 @@ public class QuickstartTest { private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); private static final String ENDPOINT = String.format("%s-dataproc.googleapis.com:443", REGION); private static final String CLUSTER_NAME = String.format("java-qs-test-%s", MY_UUID); - private static final String BUCKET_NAME = - String.format("java-dataproc-qs-test-%s", MY_UUID); + private static final String BUCKET_NAME = String.format("java-dataproc-qs-test-%s", MY_UUID); private static final String JOB_FILE_NAME = "sum.py"; private static final String JOB_FILE_PATH = String.format("gs://%s/%s", BUCKET_NAME, JOB_FILE_NAME); @@ -68,7 +67,7 @@ public class QuickstartTest { private static void requireEnv(String varName) { assertNotNull( String.format("Environment variable '%s' is required to perform these tests.", varName), - System.getenv(varName)); + System.getenv(varName)); } @BeforeClass