diff --git a/google-cloud-examples/pom.xml b/google-cloud-examples/pom.xml index 4aded1d268bb..cf784ce7da15 100644 --- a/google-cloud-examples/pom.xml +++ b/google-cloud-examples/pom.xml @@ -89,6 +89,20 @@ 4.12 test + + com.google.cloud + google-cloud-bigtable + + + com.google.cloud + google-cloud-bigtable-admin + + + org.mockito + mockito-all + RELEASE + test + diff --git a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigtable/HelloWorld.java b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigtable/HelloWorld.java new file mode 100644 index 000000000000..cac8d1febacc --- /dev/null +++ b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigtable/HelloWorld.java @@ -0,0 +1,167 @@ +/* + * Copyright 2018 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.examples.bigtable; + +import com.google.api.gax.rpc.NotFoundException; +import com.google.api.gax.rpc.ServerStream; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; +import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.BigtableDataSettings; +import com.google.cloud.bigtable.data.v2.models.InstanceName; +import com.google.cloud.bigtable.data.v2.models.Query; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.cloud.bigtable.data.v2.models.RowCell; +import com.google.cloud.bigtable.data.v2.models.RowMutation; +import java.io.IOException; + +public class HelloWorld { + + private static final String COLUMN_FAMILY = "cf1"; + private static final String COLUMN_QUALIFIER = "greeting"; + private static final String ROW_KEY_PREFIX = "rowKey"; + private final String tableId; + private final BigtableDataClient dataClient; + private final BigtableTableAdminClient adminClient; + + public static void main(String[] args) throws Exception { + + if (args.length != 2) { + System.out.println("Missing required project id or instance id"); + } + String projectId = args[0]; + String instanceId = args[1]; + + HelloWorld helloWorld = new HelloWorld(projectId, instanceId, "test-table"); + helloWorld.run(); + } + + public HelloWorld(String projectId, String instanceId, String tableId) throws IOException { + this.tableId = tableId; + + // [START connecting_to_bigtable] + // Create the settings to configure a bigtable data client + BigtableDataSettings settings = + BigtableDataSettings.newBuilder() + .setInstanceName(InstanceName.of(projectId, instanceId)) + .build(); + + // Create bigtable data client + dataClient = BigtableDataClient.create(settings); + + // Create the settings to configure a bigtable admin client + BigtableTableAdminSettings adminSettings = + BigtableTableAdminSettings.newBuilder() + .setInstanceName(com.google.bigtable.admin.v2.InstanceName.of(projectId, instanceId)) + .build(); + + // Create bigtable admin client + adminClient = BigtableTableAdminClient.create(adminSettings); + // [END connecting_to_bigtable] + } + + public void run() throws Exception { + createTable(); + writeToTable(); + readSingleRow(); + readTable(); + deleteTable(); + + dataClient.close(); + adminClient.close(); + } + + public void createTable() { + // [START creating_a_table] + // Check if table exists, create table if does not exist + if (!adminClient.exists(tableId)) { + System.out.println("Creating table: " + tableId); + CreateTableRequest createTableRequest = + CreateTableRequest.of(tableId).addFamily(COLUMN_FAMILY); + adminClient.createTable(createTableRequest); + System.out.printf("Table %s created successfully%n", tableId); + } + // [END creating_a_table] + } + + public void writeToTable() { + // [START writing_rows] + try { + System.out.println("\nWriting some greetings to the table"); + String[] greetings = {"Hello World!", "Hello Bigtable!", "Hello Java!"}; + for (int i = 0; i < greetings.length; i++) { + RowMutation rowMutation = + RowMutation.create(tableId, ROW_KEY_PREFIX + i) + .setCell(COLUMN_FAMILY, COLUMN_QUALIFIER, greetings[i]); + dataClient.mutateRow(rowMutation); + System.out.println(greetings[i]); + } + } catch (NotFoundException e) { + System.err.println("Exception while writing to table: " + e.getMessage()); + } + // [END writing_rows] + } + + public void readSingleRow() { + // [START reading_a_row] + try { + System.out.println("\nReading a single row by row key"); + Row row = dataClient.readRow(tableId, ROW_KEY_PREFIX + 0); + System.out.println("Row: " + row.getKey().toStringUtf8()); + for (RowCell cell : row.getCells()) { + System.out.printf( + "Family: %s Qualifier: %s Value: %s%n", + cell.getFamily(), cell.getQualifier().toStringUtf8(), cell.getValue().toStringUtf8()); + } + } catch (NotFoundException e) { + System.err.println("Exception while reading a single row: " + e.getMessage()); + } + // [END reading_a_row] + } + + public void readTable() { + // [START scanning_all_rows] + try { + System.out.println("\nReading the entire table"); + Query query = Query.create(tableId); + ServerStream rowStream = dataClient.readRows(query); + for (Row r : rowStream) { + System.out.println("Row Key: " + r.getKey().toStringUtf8()); + for (RowCell cell : r.getCells()) { + System.out.printf( + "Family: %s Qualifier: %s Value: %s%n", + cell.getFamily(), cell.getQualifier().toStringUtf8(), cell.getValue().toStringUtf8()); + } + } + } catch (NotFoundException e) { + System.err.println("Exception while reading table: " + e.getMessage()); + } + // [END scanning_all_rows] + } + + public void deleteTable() { + // [START deleting_a_table] + System.out.println("\nDeleting table: " + tableId); + try { + adminClient.deleteTable(tableId); + System.out.printf("Table %s deleted successfully%n", tableId); + } catch (NotFoundException e) { + System.err.println("Exception while deleting table: " + e.getMessage()); + } + // [END deleting_a_table] + } +} diff --git a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigtable/InstanceAdmin.java b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigtable/InstanceAdmin.java new file mode 100644 index 000000000000..7c81f85a6077 --- /dev/null +++ b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigtable/InstanceAdmin.java @@ -0,0 +1,223 @@ +/* + * Copyright 2018 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.examples.bigtable; + +import com.google.api.gax.rpc.ApiException; +import com.google.api.gax.rpc.NotFoundException; +import com.google.bigtable.admin.v2.ProjectName; +import com.google.cloud.bigtable.admin.v2.BigtableInstanceAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableInstanceAdminSettings; +import com.google.cloud.bigtable.admin.v2.models.Cluster; +import com.google.cloud.bigtable.admin.v2.models.CreateClusterRequest; +import com.google.cloud.bigtable.admin.v2.models.CreateInstanceRequest; +import com.google.cloud.bigtable.admin.v2.models.Instance; +import com.google.cloud.bigtable.admin.v2.models.Instance.Type; +import com.google.cloud.bigtable.admin.v2.models.StorageType; +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public class InstanceAdmin { + + public static void main(String[] args) { + + final String GCLOUD_PROJECT = args[0]; + final String PRODUCTION_INSTANCE = "ssd-instance"; + final String PRODUCTION_CLUSTER = "ssd-cluster"; + + if (args.length != 1) { + System.out.println("Missing required project id"); + return; + } + + try { + // Create the settings to configure a bigtable admin client + BigtableInstanceAdminSettings instanceAdminSettings = + BigtableInstanceAdminSettings.newBuilder() + .setProjectName(ProjectName.of(GCLOUD_PROJECT)) + .build(); + + // Create bigtable admin client + BigtableInstanceAdminClient adminClient = + BigtableInstanceAdminClient.create(instanceAdminSettings); + + // Create PRODUCTION instance + createProdInstance(adminClient, PRODUCTION_INSTANCE, PRODUCTION_CLUSTER); + + // List instances + listInstances(adminClient); + + // Get PRODUCTION instance + getInstance(adminClient, PRODUCTION_INSTANCE); + + // Get PRODUCTION clusters + listClusters(adminClient, PRODUCTION_INSTANCE); + + // Add cluster to PRODUCTION instance + addCluster(adminClient, PRODUCTION_INSTANCE, PRODUCTION_CLUSTER); + + // Delete cluster from PRODUCTION instance + deleteCluster(adminClient, PRODUCTION_INSTANCE, PRODUCTION_CLUSTER); + + // End operations with deleting PRODUCTION instance + deleteInstance(adminClient, PRODUCTION_INSTANCE); + + } catch (IOException ex) { + System.err.println("Exception while running InstanceAdmin: " + ex.getMessage()); + } + } + + public static Instance createProdInstance( + BigtableInstanceAdminClient adminClient, String instanceID, String clusterID) { + System.out.println("Check if instance exists:"); + // [START bigtable_check_instance_exists] + boolean found = false; + try { + found = adminClient.exists(instanceID); + } catch (ApiException e) { + System.err.println("Error checking if instance exists: " + e.getMessage()); + } + // [END bigtable_check_instance_exists] + + Instance instance = null; + // Create an instance if does not exists + if (!found) { + System.out.println("Instance does not exist, creating a PRODUCTION instance:"); + // [START bigtable_create_prod_instance] + // Create a Production Instance with the ID "ssd-instance" + // cluster id "ssd-cluster", 3 nodes and location us-central1-f + CreateInstanceRequest createInstanceRequest = + CreateInstanceRequest.of(instanceID) + .addCluster(clusterID, "us-central1-f", 3, StorageType.SSD) + .setType(Type.PRODUCTION) + .addLabel("example", "instance_admin"); + // Create production instance with given request + try { + instance = adminClient.createInstance(createInstanceRequest); + System.out.printf("PRODUCTION type instance: %s, created successfully", instance.getId()); + } catch (ApiException e) { + System.err.println("Error creating PRODUCTION instance: " + e.getMessage()); + System.exit(0); + } + // [END bigtable_create_prod_instance] + } else { + System.out.printf("Instance: %s exists", instanceID); + instance = adminClient.getInstance(instanceID); + } + return instance; + } + + public static List listInstances(BigtableInstanceAdminClient adminClient) { + System.out.println("\nListing Instances:"); + // [START bigtable_list_instances] + List instances = null; + try { + instances = adminClient.listInstances(); + for (Instance instance : instances) { + System.out.println(instance.getId()); + } + } catch (ApiException e) { + System.err.println("Error listing instances: " + e.getMessage()); + } + // [END bigtable_list_instances] + return instances; + } + + public static Instance getInstance(BigtableInstanceAdminClient adminClient, String instanceID) { + System.out.println("\nGet Instance:"); + // [START bigtable_get_instance] + Instance instance = null; + try { + instance = adminClient.getInstance(instanceID); + System.out.println("Instance ID: " + instance.getId()); + System.out.println("Instance Meta:"); + System.out.println("Display Name: " + instance.getDisplayName()); + System.out.println("Labels:"); + Map labels = instance.getLabels(); + for (String key : labels.keySet()) { + System.out.printf("%s: %s", key, labels.get(key)); + } + System.out.println("State: " + instance.getState()); + System.out.println("Type: " + instance.getType()); + } catch (ApiException e) { + System.err.println("Error getting instance: " + e.getMessage()); + } + // [END bigtable_get_instance] + return instance; + } + + public static List listClusters( + BigtableInstanceAdminClient adminClient, String instanceID) { + System.out.println("\nListing Clusters:"); + // [START bigtable_get_clusters] + List clusters = null; + try { + clusters = adminClient.listClusters(instanceID); + for (Cluster cluster : clusters) { + System.out.println(cluster.getId()); + } + } catch (ApiException e) { + System.err.println("Error listing clusters: " + e.getMessage()); + } + // [END bigtable_get_clusters] + return clusters; + } + + public static void deleteInstance(BigtableInstanceAdminClient adminClient, String instanceID) { + System.out.println("\nDeleting Instance:"); + // [START bigtable_delete_instance] + try { + adminClient.deleteInstance(instanceID); + System.out.println("Instance deleted: " + instanceID); + } catch (NotFoundException e) { + System.err.println("Error deleting instance: " + instanceID + " " + e.getMessage()); + } + // [END bigtable_delete_instance] + } + + public static Cluster addCluster( + BigtableInstanceAdminClient adminClient, String instanceID, String clusterID) { + Cluster cluster = null; + System.out.println("\nAdding cluster to instance: " + instanceID); + // [START bigtable_create_cluster] + try { + cluster = + adminClient.createCluster( + CreateClusterRequest.of(instanceID, clusterID) + .setZone("us-central1-c") + .setServeNodes(3) + .setStorageType(StorageType.SSD)); + System.out.printf("Cluster: %s created successfully", cluster.getId()); + } catch (ApiException e) { + System.err.println("Error creating cluster: " + e.getMessage()); + } + // [END bigtable_create_cluster] + return cluster; + } + + public static void deleteCluster( + BigtableInstanceAdminClient adminClient, String instanceID, String clusterID) { + System.out.println("\nDeleting Cluster"); + // [START bigtable_delete_cluster] + try { + adminClient.deleteCluster(instanceID, clusterID); + System.out.printf("Cluster: %s deleted successfully", clusterID); + } catch (ApiException e) { + System.err.println("Error deleting cluster: " + e.getMessage()); + } + // [END bigtable_delete_cluster] + } +} diff --git a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigtable/TableAdmin.java b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigtable/TableAdmin.java new file mode 100644 index 000000000000..04e02a659658 --- /dev/null +++ b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigtable/TableAdmin.java @@ -0,0 +1,390 @@ +/* + * Copyright 2018 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.examples.bigtable; + +import com.google.api.gax.rpc.ApiException; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; +import com.google.cloud.bigtable.admin.v2.models.ColumnFamily; +import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest; +import com.google.cloud.bigtable.admin.v2.models.GCRules; +import com.google.cloud.bigtable.admin.v2.models.ModifyColumnFamiliesRequest; +import com.google.cloud.bigtable.admin.v2.models.Table; +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Random; +import java.util.concurrent.TimeUnit; + +public class TableAdmin { + + private static final String tablePrefix = "table"; + + public static void main(String[] args) { + + final String GCLOUD_PROJECT = args[0]; + final String INSTANCE_ID = args[1]; + final String TABLE_ID = generateTableId(); + + if (args.length != 2) { + System.out.println("Missing required project id or instance id"); + return; + } + + try { + // Create the settings to configure a bigtable admin client + BigtableTableAdminSettings adminSettings = + BigtableTableAdminSettings.newBuilder() + .setInstanceName( + com.google.bigtable.admin.v2.InstanceName.of(GCLOUD_PROJECT, INSTANCE_ID)) + .build(); + + // Create bigtable admin client + BigtableTableAdminClient adminClient = BigtableTableAdminClient.create(adminSettings); + + // Check if table exists, create table if does not exist + createTable(adminClient, TABLE_ID); + + // List all tables in current instance + listAllTables(adminClient); + + // Print table metadata + getTableMeta(adminClient, TABLE_ID); + + // Create column family with max age GC rule + maxAgeRule(adminClient, TABLE_ID, "cf1"); + + // Create column family with max versions GC rule + maxVersionsRule(adminClient, TABLE_ID, "cf2"); + + // Create column family with union GC rule + unionRule(adminClient, TABLE_ID, "cf3"); + + // Create column family with intersection GC rule + intersectionRule(adminClient, TABLE_ID, "cf4"); + + // Create column family with nested GC rule + nestedRule(adminClient, TABLE_ID, "cf5"); + + // List column families for given table + listColumnFamilies(adminClient, TABLE_ID); + + // Modify column family cf1's GC rule + modifyColumnFamilyRule(adminClient, TABLE_ID, "cf1"); + + // Print updated column family cf1's rule + printModifiedColumnFamily(adminClient, TABLE_ID, "cf1"); + + // Delete column family cf2 + deleteColumnFamily(adminClient, TABLE_ID, "cf2"); + + // Delete table + deleteTable(adminClient, TABLE_ID); + + } catch (IOException ex) { + System.err.println("Exception while running BigtableTableAdmin: " + ex.getMessage()); + } + } + + public static Table createTable(BigtableTableAdminClient adminClient, String TABLE_ID) { + // [START bigtable_create_table] + // Check if table exists + Table table = null; + if (!adminClient.exists(TABLE_ID)) { + // Create table if does not exist + System.out.println("Table does not exist. Creating table: " + TABLE_ID); + // Creating table + try { + CreateTableRequest createTableRequest = CreateTableRequest.of(TABLE_ID).addFamily("cf"); + table = adminClient.createTable(createTableRequest); + System.out.printf("Table: %s created successfully%n", table.getId()); + } catch (ApiException ex) { + System.err.println("Error creating table: " + ex.getMessage()); + } + } else { + System.out.println("Table exists"); + table = adminClient.getTable(TABLE_ID); + } + // [END bigtable_create_table] + return table; + } + + public static List listAllTables(BigtableTableAdminClient adminClient) { + System.out.println("\nListing tables in current instance:"); + // [START bigtable_list_tables] + // List tables in current instance + List listTables = null; + try { + listTables = adminClient.listTables(); + for (String tableName : listTables) { + System.out.println(tableName); + } + } catch (ApiException ex) { + System.err.println("Error listing tables in current instance: " + ex.getMessage()); + } + // [END bigtable_list_tables] + return listTables; + } + + public static Table getTableMeta(BigtableTableAdminClient adminClient, String TABLE_ID) { + System.out.println("\nPrinting table metadata:"); + // [START bigtable_get_table_metadata] + // Get table metadata, and apply a view to the table fields + Table table = null; + try { + table = adminClient.getTable(TABLE_ID); + System.out.println("Table: " + table.getId()); + Collection columnFamilies = table.getColumnFamilies(); + for (ColumnFamily columnFamily : columnFamilies) { + printColumnFamily(columnFamily); + } + } catch (ApiException ex) { + System.err.println("Error retrieving table metadata: " + ex.getMessage()); + } + // [END bigtable_get_table_metadata] + return table; + } + + public static Table maxAgeRule(BigtableTableAdminClient adminClient, String TABLE_ID, String cf) { + System.out.printf("%nCreating column family %s with max age GC rule:%n", cf); + // [START bigtable_create_family_gc_max_age] + // Create a column family with GC policy : maximum age + // where age = current time minus cell timestamp + + // Define the GC rule to retain data with max age of 5 days + GCRules.DurationRule maxAgeRule1 = GCRules.GCRULES.maxAge(5, TimeUnit.DAYS); + + // Create column family with given GC rule + Table table = null; + try { + ModifyColumnFamiliesRequest columnFamiliesRequest1 = + ModifyColumnFamiliesRequest.of(TABLE_ID).addFamily(cf, maxAgeRule1); + table = adminClient.modifyFamilies(columnFamiliesRequest1); + System.out.println("Created column family: " + cf); + } catch (ApiException ex) { + System.err.println("Error creating column family: " + ex.getMessage()); + } + // [END bigtable_create_family_gc_max_age] + return table; + } + + public static Table maxVersionsRule( + BigtableTableAdminClient adminClient, String TABLE_ID, String cf) { + System.out.printf("%nCreating column family %s with max versions GC rule:%n", cf); + // [START bigtable_create_family_gc_max_versions] + // Create a column family with GC policy : most recent N versions + // where 1 = most recent version + + // Define the GC policy to retain only the most recent 2 versions + GCRules.VersionRule versionRule1 = GCRules.GCRULES.maxVersions(2); + + // Create column family with given GC rule + Table table = null; + try { + ModifyColumnFamiliesRequest columnFamiliesRequest2 = + ModifyColumnFamiliesRequest.of(TABLE_ID).addFamily(cf, versionRule1); + table = adminClient.modifyFamilies(columnFamiliesRequest2); + System.out.println("Created column family: " + cf); + } catch (ApiException ex) { + System.err.println("Error creating column family: " + ex.getMessage()); + } + // [END bigtable_create_family_gc_max_versions] + return table; + } + + public static Table unionRule(BigtableTableAdminClient adminClient, String TABLE_ID, String cf) { + System.out.printf("%nCreating column family %s with union GC rule:%n", cf); + // [START bigtable_create_family_gc_union] + // Create a column family with GC policy to drop data that matches at least one condition. + + // Define a GC rule to drop cells older than 5 days OR not the most recent version + GCRules.DurationRule maxAgeRule2 = GCRules.GCRULES.maxAge(5, TimeUnit.DAYS); + GCRules.VersionRule versionRule2 = GCRules.GCRULES.maxVersions(1); + // Add rules to union rule list + GCRules.UnionRule unionRule1 = GCRules.GCRULES.union().rule(maxAgeRule2).rule(versionRule2); + + // Create column family with given GC rule + Table table = null; + try { + ModifyColumnFamiliesRequest columnFamiliesRequest3 = + ModifyColumnFamiliesRequest.of(TABLE_ID).addFamily(cf, unionRule1); + table = adminClient.modifyFamilies(columnFamiliesRequest3); + System.out.println("Created column family: " + cf); + } catch (ApiException ex) { + System.err.println("Error creating column family: " + ex.getMessage()); + } + // [END bigtable_create_family_gc_union] + return table; + } + + public static Table intersectionRule( + BigtableTableAdminClient adminClient, String TABLE_ID, String cf) { + System.out.printf("%nCreating column family %s with intersection GC rule:%n", cf); + // [START bigtable_create_family_gc_intersection] + // Create a column family with GC policy to drop data that matches all conditions + + // GC rule: Drop cells older than 5 days AND older than the most recent 2 versions + GCRules.DurationRule maxAgeRule3 = GCRules.GCRULES.maxAge(5, TimeUnit.DAYS); + GCRules.VersionRule versionRule3 = GCRules.GCRULES.maxVersions(2); + GCRules.IntersectionRule intersectionRule1 = + GCRules.GCRULES.intersection().rule(maxAgeRule3).rule(versionRule3); + + // Create column family with given GC rule + Table table = null; + try { + ModifyColumnFamiliesRequest columnFamiliesRequest4 = + ModifyColumnFamiliesRequest.of(TABLE_ID).addFamily(cf, intersectionRule1); + table = adminClient.modifyFamilies(columnFamiliesRequest4); + System.out.println("Created column family: " + cf); + } catch (ApiException ex) { + System.err.println("Error creating column family: " + ex.getMessage()); + } + // [END bigtable_create_family_gc_intersection] + return table; + } + + public static Table nestedRule(BigtableTableAdminClient adminClient, String TABLE_ID, String cf) { + System.out.printf("%nCreating column family %s with a nested GC rule:%n", cf); + // [START bigtable_create_family_gc_nested] + // Create a nested GC rule: + // Drop cells that are either older than the 10 recent versions + // OR + // Drop cells that are older than a month AND older than the 2 recent versions + GCRules.VersionRule versionRule4 = GCRules.GCRULES.maxVersions(10); + GCRules.DurationRule maxAgeRule4 = GCRules.GCRULES.maxAge(30, TimeUnit.DAYS); + GCRules.VersionRule versionRule5 = GCRules.GCRULES.maxVersions(2); + GCRules.IntersectionRule intersectionRule2 = + GCRules.GCRULES.intersection().rule(maxAgeRule4).rule(versionRule5); + GCRules.UnionRule unionRule2 = + GCRules.GCRULES.union().rule(intersectionRule2).rule(versionRule4); + + // Create column family with given GC rule + Table table = null; + try { + ModifyColumnFamiliesRequest columnFamiliesRequest5 = + ModifyColumnFamiliesRequest.of(TABLE_ID).addFamily(cf, unionRule2); + table = adminClient.modifyFamilies(columnFamiliesRequest5); + System.out.println("Created column family: " + cf); + } catch (ApiException ex) { + System.err.println("Error creating column family: " + ex.getMessage()); + } + // [END bigtable_create_family_gc_nested] + return table; + } + + public static Collection listColumnFamilies( + BigtableTableAdminClient adminClient, String TABLE_ID) { + System.out.println("\nPrinting ID and GC Rule for all column families:"); + // [START bigtable_list_column_families] + // List all families in the table with GC rules + Collection columnFamilies = null; + try { + Table table = adminClient.getTable(TABLE_ID); + columnFamilies = table.getColumnFamilies(); + for (ColumnFamily columnFamily : columnFamilies) { + printColumnFamily(columnFamily); + } + } catch (ApiException ex) { + System.err.println("Error retrieving families: " + ex.getMessage()); + } + // [END bigtable_list_column_families] + return columnFamilies; + } + + public static Table modifyColumnFamilyRule( + BigtableTableAdminClient adminClient, String TABLE_ID, String cf) { + System.out.printf("%nUpdating column family %s GC rule:%n", cf); + // [START bigtable_update_gc_rule] + // Update the column family metadata to update the GC rule + // Update a column family GC rule + GCRules.VersionRule versionRule6 = GCRules.GCRULES.maxVersions(1); + Table table = null; + try { + // Update column family with given GC rule + ModifyColumnFamiliesRequest updateRequest = + ModifyColumnFamiliesRequest.of(TABLE_ID).updateFamily(cf, versionRule6); + table = adminClient.modifyFamilies(updateRequest); + System.out.printf("Column family %s GC rule updated%n", cf); + } catch (ApiException ex) { + System.err.printf("Error updating GC rule for %s: %s%n", cf, ex.getMessage()); + } + // [END bigtable_update_gc_rule] + return table; + } + + public static ColumnFamily printModifiedColumnFamily( + BigtableTableAdminClient adminClient, String TABLE_ID, String cf) { + System.out.printf("%nPrint updated GC rule for column family: %s%n", cf); + // [START bigtable_family_get_gc_rule] + ColumnFamily colFamily = null; + try { + Table table = adminClient.getTable(TABLE_ID); + Collection columnFamilies = table.getColumnFamilies(); + for (ColumnFamily columnFamily : columnFamilies) { + if (columnFamily.getId().equals(cf)) { + printColumnFamily(columnFamily); + colFamily = columnFamily; + } + } + } catch (ApiException ex) { + System.err.printf( + "Error retrieving metadata for column family: %s%n%s%n", cf, ex.getMessage()); + } + // [END bigtable_family_get_gc_rule] + return colFamily; + } + + public static Table deleteColumnFamily( + BigtableTableAdminClient adminClient, String TABLE_ID, String cf) { + System.out.println("\nDelete column family: " + cf); + // [START bigtable_delete_family] + // Delete a column family + Table table = null; + try { + ModifyColumnFamiliesRequest deleted = ModifyColumnFamiliesRequest.of(TABLE_ID).dropFamily(cf); + table = adminClient.modifyFamilies(deleted); + System.out.printf("Column family %s deleted successfully%n", cf); + } catch (ApiException ex) { + System.err.printf("Error deleting family: %s%n%s%n", cf, ex.getMessage()); + } + // [END bigtable_delete_family] + return table; + } + + public static void deleteTable(BigtableTableAdminClient adminClient, String TABLE_ID) { + // [START bigtable_delete_table] + // Delete the entire table + System.out.println("Delete table: " + TABLE_ID); + try { + adminClient.deleteTable(TABLE_ID); + System.out.println("Table deleted: " + TABLE_ID); + } catch (ApiException ex) { + System.err.printf("Error deleting table %s: %s%n", TABLE_ID, ex.getMessage()); + } + // [END bigtable_delete_table] + } + + private static void printColumnFamily(ColumnFamily columnFamily) { + System.out.printf( + "Column family: %s%nMetadata: %s%n", + columnFamily.getId(), columnFamily.getGCRule().toString()); + } + + private static String generateTableId() { + return String.format( + "%s-%016x-%d", tablePrefix, System.currentTimeMillis(), new Random().nextLong()); + } +} diff --git a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITHelloWorld.java b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITHelloWorld.java new file mode 100644 index 000000000000..306ee34d1a7a --- /dev/null +++ b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITHelloWorld.java @@ -0,0 +1,162 @@ +/* + * Copyright 2018 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.examples.bigtable; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; +import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.BigtableDataSettings; +import com.google.cloud.bigtable.data.v2.models.InstanceName; +import com.google.cloud.bigtable.data.v2.models.Row; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.util.Random; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.junit.AfterClass; +import org.junit.AssumptionViolatedException; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class ITHelloWorld { + + private static final String INSTANCE_PROPERTY_NAME = "bigtable.instance"; + private static final String TABLE_PREFIX = "table"; + private static final String COLUMN_FAMILY = "cf1"; + private static String tableId; + private static BigtableDataClient dataClient; + private static BigtableTableAdminClient adminClient; + private static HelloWorld helloWorld; + private static InstanceName instanceName; + + @BeforeClass + public static void beforeClass() throws IOException { + String targetInstance = System.getProperty(INSTANCE_PROPERTY_NAME); + if (targetInstance == null) { + dataClient = null; + adminClient = null; + return; + } + instanceName = InstanceName.parse(targetInstance); + BigtableDataSettings settings = + BigtableDataSettings.newBuilder().setInstanceName(instanceName).build(); + dataClient = BigtableDataClient.create(settings); + BigtableTableAdminSettings adminSettings = + BigtableTableAdminSettings.newBuilder() + .setInstanceName(com.google.bigtable.admin.v2.InstanceName.parse(targetInstance)) + .build(); + adminClient = BigtableTableAdminClient.create(adminSettings); + } + + @AfterClass + public static void afterClass() throws Exception { + adminClient.deleteTable(tableId); + garbageCollect(); + dataClient.close(); + adminClient.close(); + } + + @Before + public void setup() throws IOException { + if (adminClient == null || dataClient == null) { + throw new AssumptionViolatedException( + INSTANCE_PROPERTY_NAME + " property is not set, skipping integration tests."); + } + tableId = generateTableId(); + helloWorld = new HelloWorld(instanceName.getProject(), instanceName.getInstance(), tableId); + if (!adminClient.exists(tableId)) { + adminClient.createTable(CreateTableRequest.of(tableId).addFamily(COLUMN_FAMILY)); + } + } + + @Test + public void testCreateAndDeleteTable() throws IOException { + // Create table + String fakeTable = generateTableId(); + HelloWorld testHelloWorld = + new HelloWorld(instanceName.getProject(), instanceName.getInstance(), fakeTable); + testHelloWorld.createTable(); + assertTrue(adminClient.exists(fakeTable)); + + // Delete table + testHelloWorld.deleteTable(); + assertTrue(!adminClient.exists(fakeTable)); + } + + @Test + public void testWriteToAndReadFromTable() { + // Write to table + helloWorld.writeToTable(); + Row row = dataClient.readRow(tableId, "rowKey0"); + assertNotNull(row); + + // Read a single row + OutputStream outputStream = new ByteArrayOutputStream(); + System.setOut(new PrintStream(outputStream)); + helloWorld.readSingleRow(); + assertTrue(outputStream.toString().contains("Reading a single row by row key")); + assertTrue(outputStream.toString().contains("Row: rowKey0")); + assertTrue( + outputStream + .toString() + .contains("Family: cf1 Qualifier: greeting Value: Hello World!")); + + // Restore normal output + System.setOut(new PrintStream(System.out)); + } + + @Test + public void testReadTable() { + // Read whole table + OutputStream outputStream = new ByteArrayOutputStream(); + System.setOut(new PrintStream(outputStream)); + helloWorld.readTable(); + assertTrue(outputStream.toString().contains("Reading the entire table")); + + // Restore normal output + System.setOut(new PrintStream(System.out)); + } + + private static String generateTableId() { + return String.format( + "%s-%016x-%d", TABLE_PREFIX, System.currentTimeMillis(), new Random().nextLong()); + } + + public static void garbageCollect() { + Pattern timestampPattern = Pattern.compile(TABLE_PREFIX + "-([0-9]+)"); + for (String tableId : adminClient.listTables()) { + Matcher matcher = timestampPattern.matcher(tableId); + if (!matcher.matches()) { + continue; + } + String timestampStr = matcher.group(1); + long timestamp = Long.parseLong(timestampStr); + if (System.currentTimeMillis() - timestamp < TimeUnit.MINUTES.toMillis(15)) { + continue; + } + System.out.println("Garbage collecting orphaned table: " + tableId); + adminClient.deleteTable(tableId); + } + } +} diff --git a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITInstanceAdmin.java b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITInstanceAdmin.java new file mode 100644 index 000000000000..7694bc5f23ec --- /dev/null +++ b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITInstanceAdmin.java @@ -0,0 +1,124 @@ +/* + * Copyright 2018 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.examples.bigtable; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import com.google.api.gax.rpc.NotFoundException; +import com.google.bigtable.admin.v2.InstanceName; +import com.google.bigtable.admin.v2.ProjectName; +import com.google.cloud.bigtable.admin.v2.BigtableInstanceAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableInstanceAdminSettings; +import com.google.cloud.bigtable.admin.v2.models.Cluster; +import com.google.cloud.bigtable.admin.v2.models.CreateInstanceRequest; +import com.google.cloud.bigtable.admin.v2.models.Instance; +import com.google.cloud.bigtable.admin.v2.models.Instance.Type; +import com.google.cloud.bigtable.admin.v2.models.StorageType; +import java.io.IOException; +import java.util.List; +import org.junit.AfterClass; +import org.junit.AssumptionViolatedException; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class ITInstanceAdmin { + + private static final String INSTANCE_PROPERTY_NAME = "bigtable.instance"; + private static final String PRODUCTION_INSTANCE = "test-instance"; + private static final String PRODUCTION_CLUSTER = "test-cluster" + System.currentTimeMillis(); + private static BigtableInstanceAdminClient adminClient; + + @BeforeClass + public static void beforeClass() throws IOException { + String targetProject = System.getProperty(INSTANCE_PROPERTY_NAME); + if (targetProject == null) { + adminClient = null; + return; + } + ProjectName projectName = ProjectName.of(InstanceName.parse(targetProject).getProject()); + BigtableInstanceAdminSettings instanceAdminSettings = + BigtableInstanceAdminSettings.newBuilder().setProjectName(projectName).build(); + adminClient = BigtableInstanceAdminClient.create(instanceAdminSettings); + if (!adminClient.exists(PRODUCTION_INSTANCE)) { + adminClient.createInstance( + CreateInstanceRequest.of(PRODUCTION_INSTANCE) + .addCluster(PRODUCTION_INSTANCE, "us-central1-f", 3, StorageType.SSD) + .setType(Type.PRODUCTION) + .addLabel("example", "instance_admin")); + } + } + + @AfterClass + public static void afterClass() { + adminClient.deleteInstance(PRODUCTION_INSTANCE); + adminClient.close(); + } + + @Before + public void setup() { + if (adminClient == null) { + throw new AssumptionViolatedException( + INSTANCE_PROPERTY_NAME + " property is not set, skipping integration tests."); + } + } + + @Test + public void testCreateAndDeleteInstance() { + // Create instance + Instance instance = + InstanceAdmin.createProdInstance(adminClient, "fake-instance", "fake-cluster"); + assertNotNull(instance); + + // Delete instance + InstanceAdmin.deleteInstance(adminClient, "fake-instance"); + assertTrue(!adminClient.exists("fake-instance")); + } + + @Test + public void testListInstances() { + // List instances + List instance = InstanceAdmin.listInstances(adminClient); + assertTrue(instance.size() > 0); + } + + @Test + public void testGetInstance() { + // Get instance + Instance instance = InstanceAdmin.getInstance(adminClient, PRODUCTION_INSTANCE); + assertNotNull(instance); + } + + @Test + public void testListClusters() { + // List clusters + List clusters = InstanceAdmin.listClusters(adminClient, PRODUCTION_INSTANCE); + assertTrue(clusters.size() > 0); + } + + @Test(expected = NotFoundException.class) + public void testAddAndDeleteCluster() { + // Add cluster + Cluster cluster = + InstanceAdmin.addCluster(adminClient, PRODUCTION_INSTANCE, PRODUCTION_CLUSTER); + assertNotNull(cluster); + + // Delete cluster + InstanceAdmin.deleteCluster(adminClient, PRODUCTION_INSTANCE, PRODUCTION_CLUSTER); + adminClient.getCluster(PRODUCTION_INSTANCE, PRODUCTION_CLUSTER); + } +} diff --git a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITTableAdmin.java b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITTableAdmin.java new file mode 100644 index 000000000000..cb28bbbfe453 --- /dev/null +++ b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITTableAdmin.java @@ -0,0 +1,151 @@ +/* + * Copyright 2018 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.examples.bigtable; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import com.google.bigtable.admin.v2.InstanceName; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; +import com.google.cloud.bigtable.admin.v2.models.ColumnFamily; +import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest; +import com.google.cloud.bigtable.admin.v2.models.Table; +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import org.junit.AfterClass; +import org.junit.AssumptionViolatedException; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class ITTableAdmin { + + private static final String INSTANCE_PROPERTY_NAME = "bigtable.instance"; + private static final String TABLE_ID = "test-table" + System.currentTimeMillis(); + private static BigtableTableAdminClient adminClient; + + @BeforeClass + public static void beforeClass() throws IOException { + String targetInstance = System.getProperty(INSTANCE_PROPERTY_NAME); + if (targetInstance == null) { + adminClient = null; + return; + } + BigtableTableAdminSettings adminSettings = + BigtableTableAdminSettings.newBuilder() + .setInstanceName(InstanceName.parse(targetInstance)) + .build(); + adminClient = BigtableTableAdminClient.create(adminSettings); + if (!adminClient.exists(TABLE_ID)) { + adminClient.createTable(CreateTableRequest.of(TABLE_ID).addFamily("cf")); + } + } + + @AfterClass + public static void afterClass() { + adminClient.deleteTable(TABLE_ID); + adminClient.close(); + } + + @Before + public void setup() { + if (adminClient == null) { + throw new AssumptionViolatedException( + INSTANCE_PROPERTY_NAME + " property is not set, skipping integration tests."); + } + } + + @Test + public void testCreateAndDeleteTable() { + // Create table + Table table = TableAdmin.createTable(adminClient, "fake-table"); + assertNotNull(table); + + // Delete table + TableAdmin.deleteTable(adminClient, "fake-table"); + assertTrue(!adminClient.exists("fake-table")); + } + + @Test + public void testListAllTables() { + // List all tables in instance + List tables = TableAdmin.listAllTables(adminClient); + assertTrue(tables.size() > 0); + } + + @Test + public void testGetTableMeta() { + // Get table meta + Table table = TableAdmin.getTableMeta(adminClient, TABLE_ID); + assertNotNull(table); + } + + @Test + public void testCreateMaxAgeRuleAndModifyAndPrintColumnFamily() { + // Max age rule + Table table = TableAdmin.maxAgeRule(adminClient, TABLE_ID, "cf1"); + assertNotNull(table); + + // Modify cf1 + table = TableAdmin.modifyColumnFamilyRule(adminClient, TABLE_ID, "cf1"); + assertNotNull(table); + + // Print modified cf1 + ColumnFamily columnFamily = TableAdmin.printModifiedColumnFamily(adminClient, TABLE_ID, "cf1"); + assertNotNull(columnFamily); + } + + @Test + public void testCreateMaxVersionsRuleAndDeleteColumnFamily() { + // Max versions rule + Table table = TableAdmin.maxVersionsRule(adminClient, TABLE_ID, "cf2"); + assertNotNull(table); + + // Delete cf2 + table = TableAdmin.deleteColumnFamily(adminClient, TABLE_ID, "cf2"); + assertNotNull(table); + } + + @Test + public void testCreateUnionRule() { + // Union rule + Table table = TableAdmin.unionRule(adminClient, TABLE_ID, "cf3"); + assertNotNull(table); + } + + @Test + public void testCreateIntersectionRule() { + // Intersection rule + Table table = TableAdmin.intersectionRule(adminClient, TABLE_ID, "cf4"); + assertNotNull(table); + } + + @Test + public void testCreateNestedRule() { + // Nested rule + Table table = TableAdmin.nestedRule(adminClient, TABLE_ID, "cf5"); + assertNotNull(table); + } + + @Test + public void testListColumnFamilies() { + // List column families + Collection columnFamilies = TableAdmin.listColumnFamilies(adminClient, TABLE_ID); + assertTrue(columnFamilies.size() > 0); + } +}