Bigtable: 19 - Implement integration tests

.circleci/config.yml

@@ -79,6 +79,19 @@ jobs:
       - run:
           name: Run integration tests for google-cloud-bigquery
           command: ./utilities/verify_single_it.sh google-cloud-bigquery
+
+  bigtable_it:
+    working_directory: ~/googleapis
+    <<: *anchor_docker
+    <<: *anchor_auth_vars
+    steps:
+      - checkout
+      - run:
+          <<: *anchor_run_decrypt
+      - run:
+          name: Run integration tests for google-cloud-bigtable
+          command: ./utilities/verify_single_it.sh google-cloud-bigtable -Dbigtable.env=prod -Dbigtable.table=projects/gcloud-devel/instances/google-cloud-bigtable/tables/integration-tests
+
   compute_it:
     working_directory: ~/googleapis
     <<: *anchor_docker
@@ -220,6 +233,10 @@ workflows:
           filters:
             branches:
               only: master
+      - bigtable_it:
+          filters:
+            branches:
+              only: master
       - compute_it:
           filters:
             branches:

TESTING.md

@@ -3,6 +3,7 @@
 This library provides tools to help write tests for code that uses the following google-cloud services:
 
 -  [BigQuery](#testing-code-that-uses-bigquery)
+-  [Bigtable](#testing-code-that-uses-bigtable)
 -  [Compute](#testing-code-that-uses-compute)
 -  [Datastore](#testing-code-that-uses-datastore)
 -  [DNS](#testing-code-that-uses-dns)
@@ -41,6 +42,29 @@ Here is an example that clears the dataset created in Step 3.
   RemoteBigQueryHelper.forceDelete(bigquery, dataset);
   ```
 
+### Testing code that uses Bigtable
+
+Bigtable integration tests can either be run against an emulator or a real Bigtable table. The
+target environment can be selected via the `bigtable.env` system property. By default it is set to
+`emulator` and the other option is `prod`.
+
+To use the `emulator` environment, please install the gcloud sdk and use it to install the
+`cbtemulator` via `gcloud components install bigtable`.
+
+To use the `prod` environment:
+1. Set up the target table using `google-cloud-bigtable/scripts/setup-test-table.sh`
+2. Download the [JSON service account credentials file][create-service-account] from the Google 
+   Developer's Console.
+3. Set the environment variable `GOOGLE_APPLICATION_CREDENTIALS` to the path of the credentials file
+4. Set the system property `bigtable.env=prod` and `bigtable.table` to the full table name you 
+    created earlier. Example: 
+    ```shell
+    mvn verify -am -pl google-cloud-bigtable \
+      -Dbigtable.env=prod \
+      -Dbigtable.table=projects/my-project/instances/my-instance/tables/my-table
+    ```
+
+
 ### Testing code that uses Compute
 
 Currently, there isn't an emulator for Google Compute, so an alternative is to create a test

google-cloud-bigtable/pom.xml

@@ -127,6 +127,24 @@
           </execution>
         </executions>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-failsafe-plugin</artifactId>
+        <version>2.19.1</version>
+        <executions>
+          <execution>
+            <goals>
+              <goal>integration-test</goal>
+              <goal>verify</goal>
+            </goals>
+          </execution>
+        </executions>
+        <configuration>
+          <parallel>classes</parallel>
+          <perCoreThreadCount>true</perCoreThreadCount>
+          <threadCount>2</threadCount>
+        </configuration>
+      </plugin>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-javadoc-plugin</artifactId>

google-cloud-bigtable/scripts/setup-test-table.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+# Set up a table to use for integration tests.
+
+set -x
+
+# Format: projects/<project-id>/instances/<instance-id>/tables/<table-id>
+TABLE_NAME=$1
+
+if [[ ${TABLE_NAME} =~ projects\/([^/]+)\/instances\/([^/]+)\/tables\/([^/]+) ]]; then
+  PROJECT_ID=${BASH_REMATCH[1]}
+  INSTANCE_ID=${BASH_REMATCH[2]}
+  TABLE_ID=${BASH_REMATCH[3]}
+else
+  echo "Invalid table name: $TABLE_NAME" 1>&2
+  exit 1
+fi
+
+cbt -project $PROJECT_ID -instance $INSTANCE_ID createtable $TABLE_ID
+cbt -project $PROJECT_ID -instance $INSTANCE_ID createfamily $TABLE_ID cf
+cbt -project $PROJECT_ID -instance $INSTANCE_ID setgcpolicy $TABLE_ID cf maxversions=1

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/InstanceName.java

@@ -58,26 +58,20 @@ private InstanceName(Builder builder) {
   }
 
   public static InstanceName of(String project, String instance) {
-    return newBuilder()
-      .setProject(project)
-      .setInstance(instance)
-      .build();
+    return newBuilder().setProject(project).setInstance(instance).build();
   }
 
   public static String format(String project, String instance) {
-    return newBuilder()
-      .setProject(project)
-      .setInstance(instance)
-      .build()
-      .toString();
+    return newBuilder().setProject(project).setInstance(instance).build().toString();
   }
 
   public static InstanceName parse(String formattedString) {
     if (formattedString.isEmpty()) {
       return null;
     }
     Map<String, String> matchMap =
-        PATH_TEMPLATE.validatedMatch(formattedString, "InstanceName.parse: formattedString not in valid format");
+        PATH_TEMPLATE.validatedMatch(
+            formattedString, "InstanceName.parse: formattedString not in valid format");
     return of(matchMap.get("project"), matchMap.get("instance"));
   }
 
@@ -123,9 +117,7 @@ public String getFieldValue(String fieldName) {
     return getFieldValuesMap().get(fieldName);
   }
 
-  /**
-   * @deprecated This method is only present to satisfy the ResourceName interface.
-   */
+  /** @deprecated This method is only present to satisfy the ResourceName interface. */
   @Deprecated
   public ResourceNameType getType() {
     throw new UnsupportedOperationException("InstanceName.getType() not supported");
@@ -160,8 +152,7 @@ public Builder setInstance(String instance) {
       return this;
     }
 
-    private Builder() {
-    }
+    private Builder() {}
 
     private Builder(InstanceName instanceName) {
       project = instanceName.project;
@@ -180,8 +171,7 @@ public boolean equals(Object o) {
     }
     if (o instanceof InstanceName) {
       InstanceName that = (InstanceName) o;
-      return (this.project.equals(that.project))
-          && (this.instance.equals(that.instance));
+      return (this.project.equals(that.project)) && (this.instance.equals(that.instance));
     }
     return false;
   }
@@ -196,4 +186,3 @@ public int hashCode() {
     return h;
   }
 }
-

google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BulkMutationBatcherIT.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.cloud.bigtable.data.v2.it;
+
+import com.google.api.gax.rpc.ServerStream;
+import com.google.cloud.bigtable.data.v2.BigtableDataClient;
+import com.google.cloud.bigtable.data.v2.it.env.TestEnvRule;
+import com.google.cloud.bigtable.data.v2.models.BulkMutationBatcher;
+import com.google.cloud.bigtable.data.v2.models.Query;
+import com.google.cloud.bigtable.data.v2.models.Row;
+import com.google.cloud.bigtable.data.v2.models.RowCell;
+import com.google.cloud.bigtable.data.v2.models.RowMutation;
+import com.google.common.collect.Lists;
+import com.google.common.truth.Truth;
+import com.google.protobuf.ByteString;
+import java.util.List;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class BulkMutationBatcherIT {
+  @ClassRule public static TestEnvRule testEnvRule = new TestEnvRule();
+
+  @Test
+  public void test() throws Exception {
+    BigtableDataClient client = testEnvRule.env().getDataClient();
+    String tableId = testEnvRule.env().getTableName().getTable();
+    String family = testEnvRule.env().getFamilyId();
+    String rowPrefix = testEnvRule.env().getRowPrefix();
+
+    try (BulkMutationBatcher batcher = client.newBulkMutationBatcher()) {
+      for (int i = 0; i < 10; i++) {
+        batcher.add(
+            RowMutation.create(tableId, rowPrefix + "-" + i).setCell(family, "q", 10_000, "value"));
+      }
+    }
+
+    List<Row> expectedRows = Lists.newArrayList();
+    for (int i = 0; i < 10; i++) {
+      expectedRows.add(
+          Row.create(
+              ByteString.copyFromUtf8(rowPrefix + "-" + i),
+              Lists.newArrayList(
+                  RowCell.create(
+                      family,
+                      ByteString.copyFromUtf8("q"),
+                      10_000,
+                      Lists.<String>newArrayList(),
+                      ByteString.copyFromUtf8("value")))));
+    }
+    ServerStream<Row> actualRows = client.readRows(Query.create(tableId).prefix(rowPrefix));
+
+    Truth.assertThat(actualRows).containsExactlyElementsIn(expectedRows);
+  }
+}

google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/CheckAndMutateIT.java

@@ -0,0 +1,73 @@
+/*
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.cloud.bigtable.data.v2.it;
+
+import static com.google.cloud.bigtable.data.v2.models.Filters.FILTERS;
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.cloud.bigtable.data.v2.it.env.TestEnvRule;
+import com.google.cloud.bigtable.data.v2.models.ConditionalRowMutation;
+import com.google.cloud.bigtable.data.v2.models.Mutation;
+import com.google.cloud.bigtable.data.v2.models.Query;
+import com.google.cloud.bigtable.data.v2.models.Row;
+import com.google.cloud.bigtable.data.v2.models.RowMutation;
+import com.google.protobuf.ByteString;
+import java.util.concurrent.TimeUnit;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class CheckAndMutateIT {
+  @ClassRule public static TestEnvRule testEnvRule = new TestEnvRule();
+
+  @Test
+  public void test() throws Exception {
+    String tableId = testEnvRule.env().getTableName().getTable();
+    String rowKey = testEnvRule.env().getRowPrefix();
+    String familyId = testEnvRule.env().getFamilyId();
+
+    testEnvRule
+        .env()
+        .getDataClient()
+        .mutateRowCallable()
+        .call(
+            RowMutation.create(tableId, rowKey)
+                .setCell(familyId, "q1", "val1")
+                .setCell(familyId, "q2", "val2"));
+
+    testEnvRule
+        .env()
+        .getDataClient()
+        .checkAndMutateRowAsync(
+            ConditionalRowMutation.create(tableId, rowKey)
+                .condition(FILTERS.qualifier().exactMatch("q1"))
+                .then(Mutation.create().setCell(familyId, "q3", "q1")))
+        .get(1, TimeUnit.MINUTES);
+
+    Row row =
+        testEnvRule
+            .env()
+            .getDataClient()
+            .readRowsCallable()
+            .first()
+            .call(Query.create(tableId).rowKey(rowKey));
+
+    assertThat(row.getCells()).hasSize(3);
+    assertThat(row.getCells().get(2).getValue()).isEqualTo(ByteString.copyFromUtf8("q1"));
+  }
+}

google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/MutateRowIT.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.cloud.bigtable.data.v2.it;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.cloud.bigtable.data.v2.it.env.TestEnvRule;
+import com.google.cloud.bigtable.data.v2.models.Query;
+import com.google.cloud.bigtable.data.v2.models.Row;
+import com.google.cloud.bigtable.data.v2.models.RowMutation;
+import com.google.protobuf.ByteString;
+import java.util.concurrent.TimeUnit;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class MutateRowIT {
+  @ClassRule public static TestEnvRule testEnvRule = new TestEnvRule();
+
+  @Test
+  public void test() throws Exception {
+    String rowKey = testEnvRule.env().getRowPrefix() + "testA";
+    String familyId = testEnvRule.env().getFamilyId();
+
+    testEnvRule
+        .env()
+        .getDataClient()
+        .mutateRowAsync(
+            RowMutation.create(testEnvRule.env().getTableName().getTable(), rowKey)
+                .setCell(familyId, "q", "myVal")
+                .setCell(familyId, "q2", "myVal2")
+                .setCell(familyId, "q3", "myVal3"))
+        .get(1, TimeUnit.MINUTES);
+
+    testEnvRule
+        .env()
+        .getDataClient()
+        .mutateRowAsync(
+            RowMutation.create(testEnvRule.env().getTableName().getTable(), rowKey)
+                .deleteCells(familyId, "q2"))
+        .get(1, TimeUnit.MINUTES);
+
+    Row row =
+        testEnvRule
+            .env()
+            .getDataClient()
+            .readRowsCallable()
+            .first()
+            .call(Query.create(testEnvRule.env().getTableName().getTable()).rowKey(rowKey));
+
+    assertThat(row.getCells()).hasSize(2);
+    assertThat(row.getCells().get(0).getValue()).isEqualTo(ByteString.copyFromUtf8("myVal"));
+    assertThat(row.getCells().get(1).getValue()).isEqualTo(ByteString.copyFromUtf8("myVal3"));
+  }
+}