diff --git a/.circleci/config.yml b/.circleci/config.yml index 4eb1d2231cc6..bb1c01f41222 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -79,6 +79,19 @@ jobs: - run: name: Run integration tests for google-cloud-bigquery command: ./utilities/verify_single_it.sh google-cloud-bigquery + + bigtable_it: + working_directory: ~/googleapis + <<: *anchor_docker + <<: *anchor_auth_vars + steps: + - checkout + - run: + <<: *anchor_run_decrypt + - run: + name: Run integration tests for google-cloud-bigtable + command: ./utilities/verify_single_it.sh google-cloud-bigtable -Dbigtable.env=prod -Dbigtable.table=projects/gcloud-devel/instances/google-cloud-bigtable/tables/integration-tests + compute_it: working_directory: ~/googleapis <<: *anchor_docker @@ -220,6 +233,10 @@ workflows: filters: branches: only: master + - bigtable_it: + filters: + branches: + only: master - compute_it: filters: branches: diff --git a/TESTING.md b/TESTING.md index 5b77b99ac43e..f5a8242b6cae 100644 --- a/TESTING.md +++ b/TESTING.md @@ -3,6 +3,7 @@ This library provides tools to help write tests for code that uses the following google-cloud services: - [BigQuery](#testing-code-that-uses-bigquery) +- [Bigtable](#testing-code-that-uses-bigtable) - [Compute](#testing-code-that-uses-compute) - [Datastore](#testing-code-that-uses-datastore) - [DNS](#testing-code-that-uses-dns) @@ -41,6 +42,29 @@ Here is an example that clears the dataset created in Step 3. RemoteBigQueryHelper.forceDelete(bigquery, dataset); ``` +### Testing code that uses Bigtable + +Bigtable integration tests can either be run against an emulator or a real Bigtable table. The +target environment can be selected via the `bigtable.env` system property. By default it is set to +`emulator` and the other option is `prod`. + +To use the `emulator` environment, please install the gcloud sdk and use it to install the +`cbtemulator` via `gcloud components install bigtable`. + +To use the `prod` environment: +1. Set up the target table using `google-cloud-bigtable/scripts/setup-test-table.sh` +2. Download the [JSON service account credentials file][create-service-account] from the Google + Developer's Console. +3. Set the environment variable `GOOGLE_APPLICATION_CREDENTIALS` to the path of the credentials file +4. Set the system property `bigtable.env=prod` and `bigtable.table` to the full table name you + created earlier. Example: + ```shell + mvn verify -am -pl google-cloud-bigtable \ + -Dbigtable.env=prod \ + -Dbigtable.table=projects/my-project/instances/my-instance/tables/my-table + ``` + + ### Testing code that uses Compute Currently, there isn't an emulator for Google Compute, so an alternative is to create a test diff --git a/google-cloud-bigtable/pom.xml b/google-cloud-bigtable/pom.xml index 2af77c7773a4..383b8dc08da1 100644 --- a/google-cloud-bigtable/pom.xml +++ b/google-cloud-bigtable/pom.xml @@ -127,6 +127,24 @@ + + org.apache.maven.plugins + maven-failsafe-plugin + 2.19.1 + + + + integration-test + verify + + + + + classes + true + 2 + + org.apache.maven.plugins maven-javadoc-plugin diff --git a/google-cloud-bigtable/scripts/setup-test-table.sh b/google-cloud-bigtable/scripts/setup-test-table.sh new file mode 100755 index 000000000000..1ebedb0f6007 --- /dev/null +++ b/google-cloud-bigtable/scripts/setup-test-table.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +# Set up a table to use for integration tests. + +set -x + +# Format: projects//instances//tables/ +TABLE_NAME=$1 + +if [[ ${TABLE_NAME} =~ projects\/([^/]+)\/instances\/([^/]+)\/tables\/([^/]+) ]]; then + PROJECT_ID=${BASH_REMATCH[1]} + INSTANCE_ID=${BASH_REMATCH[2]} + TABLE_ID=${BASH_REMATCH[3]} +else + echo "Invalid table name: $TABLE_NAME" 1>&2 + exit 1 +fi + +cbt -project $PROJECT_ID -instance $INSTANCE_ID createtable $TABLE_ID +cbt -project $PROJECT_ID -instance $INSTANCE_ID createfamily $TABLE_ID cf +cbt -project $PROJECT_ID -instance $INSTANCE_ID setgcpolicy $TABLE_ID cf maxversions=1 diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/InstanceName.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/InstanceName.java index dc4247ab4e96..8465d5eb8e92 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/InstanceName.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/InstanceName.java @@ -58,18 +58,11 @@ private InstanceName(Builder builder) { } public static InstanceName of(String project, String instance) { - return newBuilder() - .setProject(project) - .setInstance(instance) - .build(); + return newBuilder().setProject(project).setInstance(instance).build(); } public static String format(String project, String instance) { - return newBuilder() - .setProject(project) - .setInstance(instance) - .build() - .toString(); + return newBuilder().setProject(project).setInstance(instance).build().toString(); } public static InstanceName parse(String formattedString) { @@ -77,7 +70,8 @@ public static InstanceName parse(String formattedString) { return null; } Map matchMap = - PATH_TEMPLATE.validatedMatch(formattedString, "InstanceName.parse: formattedString not in valid format"); + PATH_TEMPLATE.validatedMatch( + formattedString, "InstanceName.parse: formattedString not in valid format"); return of(matchMap.get("project"), matchMap.get("instance")); } @@ -123,9 +117,7 @@ public String getFieldValue(String fieldName) { return getFieldValuesMap().get(fieldName); } - /** - * @deprecated This method is only present to satisfy the ResourceName interface. - */ + /** @deprecated This method is only present to satisfy the ResourceName interface. */ @Deprecated public ResourceNameType getType() { throw new UnsupportedOperationException("InstanceName.getType() not supported"); @@ -160,8 +152,7 @@ public Builder setInstance(String instance) { return this; } - private Builder() { - } + private Builder() {} private Builder(InstanceName instanceName) { project = instanceName.project; @@ -180,8 +171,7 @@ public boolean equals(Object o) { } if (o instanceof InstanceName) { InstanceName that = (InstanceName) o; - return (this.project.equals(that.project)) - && (this.instance.equals(that.instance)); + return (this.project.equals(that.project)) && (this.instance.equals(that.instance)); } return false; } @@ -196,4 +186,3 @@ public int hashCode() { return h; } } - diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BulkMutationBatcherIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BulkMutationBatcherIT.java new file mode 100644 index 000000000000..b0d82d8d9388 --- /dev/null +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BulkMutationBatcherIT.java @@ -0,0 +1,70 @@ +/* + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.it; + +import com.google.api.gax.rpc.ServerStream; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.it.env.TestEnvRule; +import com.google.cloud.bigtable.data.v2.models.BulkMutationBatcher; +import com.google.cloud.bigtable.data.v2.models.Query; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.cloud.bigtable.data.v2.models.RowCell; +import com.google.cloud.bigtable.data.v2.models.RowMutation; +import com.google.common.collect.Lists; +import com.google.common.truth.Truth; +import com.google.protobuf.ByteString; +import java.util.List; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class BulkMutationBatcherIT { + @ClassRule public static TestEnvRule testEnvRule = new TestEnvRule(); + + @Test + public void test() throws Exception { + BigtableDataClient client = testEnvRule.env().getDataClient(); + String tableId = testEnvRule.env().getTableName().getTable(); + String family = testEnvRule.env().getFamilyId(); + String rowPrefix = testEnvRule.env().getRowPrefix(); + + try (BulkMutationBatcher batcher = client.newBulkMutationBatcher()) { + for (int i = 0; i < 10; i++) { + batcher.add( + RowMutation.create(tableId, rowPrefix + "-" + i).setCell(family, "q", 10_000, "value")); + } + } + + List expectedRows = Lists.newArrayList(); + for (int i = 0; i < 10; i++) { + expectedRows.add( + Row.create( + ByteString.copyFromUtf8(rowPrefix + "-" + i), + Lists.newArrayList( + RowCell.create( + family, + ByteString.copyFromUtf8("q"), + 10_000, + Lists.newArrayList(), + ByteString.copyFromUtf8("value"))))); + } + ServerStream actualRows = client.readRows(Query.create(tableId).prefix(rowPrefix)); + + Truth.assertThat(actualRows).containsExactlyElementsIn(expectedRows); + } +} diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/CheckAndMutateIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/CheckAndMutateIT.java new file mode 100644 index 000000000000..2b65517f232d --- /dev/null +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/CheckAndMutateIT.java @@ -0,0 +1,73 @@ +/* + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.it; + +import static com.google.cloud.bigtable.data.v2.models.Filters.FILTERS; +import static com.google.common.truth.Truth.assertThat; + +import com.google.cloud.bigtable.data.v2.it.env.TestEnvRule; +import com.google.cloud.bigtable.data.v2.models.ConditionalRowMutation; +import com.google.cloud.bigtable.data.v2.models.Mutation; +import com.google.cloud.bigtable.data.v2.models.Query; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.cloud.bigtable.data.v2.models.RowMutation; +import com.google.protobuf.ByteString; +import java.util.concurrent.TimeUnit; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class CheckAndMutateIT { + @ClassRule public static TestEnvRule testEnvRule = new TestEnvRule(); + + @Test + public void test() throws Exception { + String tableId = testEnvRule.env().getTableName().getTable(); + String rowKey = testEnvRule.env().getRowPrefix(); + String familyId = testEnvRule.env().getFamilyId(); + + testEnvRule + .env() + .getDataClient() + .mutateRowCallable() + .call( + RowMutation.create(tableId, rowKey) + .setCell(familyId, "q1", "val1") + .setCell(familyId, "q2", "val2")); + + testEnvRule + .env() + .getDataClient() + .checkAndMutateRowAsync( + ConditionalRowMutation.create(tableId, rowKey) + .condition(FILTERS.qualifier().exactMatch("q1")) + .then(Mutation.create().setCell(familyId, "q3", "q1"))) + .get(1, TimeUnit.MINUTES); + + Row row = + testEnvRule + .env() + .getDataClient() + .readRowsCallable() + .first() + .call(Query.create(tableId).rowKey(rowKey)); + + assertThat(row.getCells()).hasSize(3); + assertThat(row.getCells().get(2).getValue()).isEqualTo(ByteString.copyFromUtf8("q1")); + } +} diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/MutateRowIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/MutateRowIT.java new file mode 100644 index 000000000000..4fdfef18e665 --- /dev/null +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/MutateRowIT.java @@ -0,0 +1,70 @@ +/* + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.it; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.cloud.bigtable.data.v2.it.env.TestEnvRule; +import com.google.cloud.bigtable.data.v2.models.Query; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.cloud.bigtable.data.v2.models.RowMutation; +import com.google.protobuf.ByteString; +import java.util.concurrent.TimeUnit; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class MutateRowIT { + @ClassRule public static TestEnvRule testEnvRule = new TestEnvRule(); + + @Test + public void test() throws Exception { + String rowKey = testEnvRule.env().getRowPrefix() + "testA"; + String familyId = testEnvRule.env().getFamilyId(); + + testEnvRule + .env() + .getDataClient() + .mutateRowAsync( + RowMutation.create(testEnvRule.env().getTableName().getTable(), rowKey) + .setCell(familyId, "q", "myVal") + .setCell(familyId, "q2", "myVal2") + .setCell(familyId, "q3", "myVal3")) + .get(1, TimeUnit.MINUTES); + + testEnvRule + .env() + .getDataClient() + .mutateRowAsync( + RowMutation.create(testEnvRule.env().getTableName().getTable(), rowKey) + .deleteCells(familyId, "q2")) + .get(1, TimeUnit.MINUTES); + + Row row = + testEnvRule + .env() + .getDataClient() + .readRowsCallable() + .first() + .call(Query.create(testEnvRule.env().getTableName().getTable()).rowKey(rowKey)); + + assertThat(row.getCells()).hasSize(2); + assertThat(row.getCells().get(0).getValue()).isEqualTo(ByteString.copyFromUtf8("myVal")); + assertThat(row.getCells().get(1).getValue()).isEqualTo(ByteString.copyFromUtf8("myVal3")); + } +} diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java new file mode 100644 index 000000000000..a4fab6e2ac08 --- /dev/null +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java @@ -0,0 +1,144 @@ +/* + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.it; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.api.core.SettableApiFuture; +import com.google.api.gax.rpc.ResponseObserver; +import com.google.api.gax.rpc.StreamController; +import com.google.cloud.bigtable.data.v2.it.env.TestEnvRule; +import com.google.cloud.bigtable.data.v2.models.Query; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.cloud.bigtable.data.v2.models.RowCell; +import com.google.cloud.bigtable.data.v2.models.RowMutation; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.protobuf.ByteString; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class ReadIT { + private String prefix; + + @ClassRule public static TestEnvRule testEnvRule = new TestEnvRule(); + + @Before + public void setUp() { + prefix = testEnvRule.env().getRowPrefix(); + } + + @Test + public void readEmpty() throws Throwable { + String uniqueKey = prefix + "-readEmpty"; + + Query query = Query.create(testEnvRule.env().getTableName().getTable()).rowKey(uniqueKey); + + // Sync + ArrayList rows = Lists.newArrayList(testEnvRule.env().getDataClient().readRows(query)); + assertThat(rows).isEmpty(); + + // Async + AccumulatingObserver observer = new AccumulatingObserver(); + testEnvRule.env().getDataClient().readRowsAsync(query, observer); + observer.awaitCompletion(); + assertThat(observer.responses).isEmpty(); + } + + @Test + public void read() throws Throwable { + int numRows = 5; + List expectedRows = Lists.newArrayList(); + String uniqueKey = prefix + "-read"; + + long timestampMicros = System.nanoTime() * 1_000; + + for (int i = 0; i < numRows; i++) { + testEnvRule + .env() + .getDataClient() + .mutateRowCallable() + .call( + RowMutation.create(testEnvRule.env().getTableName().getTable(), uniqueKey + "-" + i) + .setCell(testEnvRule.env().getFamilyId(), "q", timestampMicros, "my-value")); + + expectedRows.add( + Row.create( + ByteString.copyFromUtf8(uniqueKey + "-" + i), + ImmutableList.of( + RowCell.create( + testEnvRule.env().getFamilyId(), + ByteString.copyFromUtf8("q"), + timestampMicros, + ImmutableList.of(), + ByteString.copyFromUtf8("my-value"))))); + } + + // Sync + Query query = + Query.create(testEnvRule.env().getTableName().getTable()) + .range(uniqueKey + "-0", uniqueKey + "-" + numRows); + ArrayList actualResults = + Lists.newArrayList(testEnvRule.env().getDataClient().readRows(query)); + + assertThat(actualResults).containsExactlyElementsIn(expectedRows); + + // Async + AccumulatingObserver observer = new AccumulatingObserver(); + testEnvRule.env().getDataClient().readRowsAsync(query, observer); + observer.awaitCompletion(); + assertThat(observer.responses).containsExactlyElementsIn(expectedRows); + } + + static class AccumulatingObserver implements ResponseObserver { + final List responses = Lists.newArrayList(); + final SettableApiFuture completionFuture = SettableApiFuture.create(); + + void awaitCompletion() throws Throwable { + try { + completionFuture.get(10, TimeUnit.MINUTES); + } catch (ExecutionException e) { + throw e.getCause(); + } + } + + @Override + public void onStart(StreamController controller) {} + + @Override + public void onResponse(Row row) { + responses.add(row); + } + + @Override + public void onError(Throwable t) { + completionFuture.setException(t); + } + + @Override + public void onComplete() { + completionFuture.set(null); + } + } +} diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadModifyWriteIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadModifyWriteIT.java new file mode 100644 index 000000000000..296f43e2ddbe --- /dev/null +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadModifyWriteIT.java @@ -0,0 +1,57 @@ +/* + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.it; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.cloud.bigtable.data.v2.it.env.TestEnvRule; +import com.google.cloud.bigtable.data.v2.models.ReadModifyWriteRow; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.protobuf.ByteString; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class ReadModifyWriteIT { + @ClassRule public static TestEnvRule testEnvRule = new TestEnvRule(); + + @Test + public void test() throws InterruptedException, ExecutionException, TimeoutException { + String tableId = testEnvRule.env().getTableName().getTable(); + String family = testEnvRule.env().getFamilyId(); + String rowKey = testEnvRule.env().getRowPrefix(); + + Row row = + testEnvRule + .env() + .getDataClient() + .readModifyWriteRowAsync( + ReadModifyWriteRow.create(tableId, rowKey) + .append(family, "q1", "a") + .increment(family, "q2", 3)) + .get(1, TimeUnit.MINUTES); + + assertThat(row.getCells()).hasSize(2); + assertThat(row.getCells().get(0).getValue()).isEqualTo(ByteString.copyFromUtf8("a")); + assertThat(row.getCells().get(1).getValue()) + .isEqualTo(ByteString.copyFrom(new byte[] {0, 0, 0, 0, 0, 0, 0, 3})); + } +} diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/SampleRowsIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/SampleRowsIT.java new file mode 100644 index 000000000000..e77fcdd98a16 --- /dev/null +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/SampleRowsIT.java @@ -0,0 +1,65 @@ +/* + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.it; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.api.core.ApiFuture; +import com.google.api.core.ApiFutures; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.it.env.TestEnvRule; +import com.google.cloud.bigtable.data.v2.models.KeyOffset; +import com.google.cloud.bigtable.data.v2.models.RowMutation; +import com.google.common.collect.Lists; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class SampleRowsIT { + @ClassRule public static TestEnvRule testEnvRule = new TestEnvRule(); + + @Test + public void test() throws InterruptedException, ExecutionException, TimeoutException { + BigtableDataClient client = testEnvRule.env().getDataClient(); + + // Create some data so that sample row keys has something to show + List> futures = Lists.newArrayList(); + for (int i = 0; i < 10; i++) { + ApiFuture future = + client.mutateRowAsync( + RowMutation.create( + testEnvRule.env().getTableName().getTable(), + testEnvRule.env().getRowPrefix() + "-" + i) + .setCell(testEnvRule.env().getFamilyId(), "", "value")); + futures.add(future); + } + ApiFutures.allAsList(futures).get(1, TimeUnit.MINUTES); + + ApiFuture> future = + client.sampleRowKeysAsync(testEnvRule.env().getTableName().getTable()); + + List results = future.get(1, TimeUnit.MINUTES); + + assertThat(results).isNotEmpty(); + assertThat(results.get(results.size() - 1).getOffsetBytes()).isGreaterThan(0L); + } +} diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java new file mode 100644 index 000000000000..476fe8974b51 --- /dev/null +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java @@ -0,0 +1,231 @@ +/* + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.it.env; + +import com.google.api.gax.core.NoCredentialsProvider; +import com.google.api.gax.grpc.GrpcTransportChannel; +import com.google.api.gax.rpc.ClientSettings; +import com.google.api.gax.rpc.FixedTransportChannelProvider; +import com.google.cloud.bigtable.data.v2.models.InstanceName; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.BigtableDataSettings; +import com.google.common.io.CharStreams; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.ServerSocket; +import java.net.Socket; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.FutureTask; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** Java wrapper around the gcloud bigtable emulator. */ +// TODO(igorbernstein): Clean this up and externalize this in a separate artifact +// TODO(igorbernstein): Stop depending on gcloud for the binary, instead wrap it in a jar. +class Emulator { + private static final Logger LOGGER = Logger.getLogger(Emulator.class.getName()); + + private final Path executable; + private Process process; + private boolean isStopped = true; + private ManagedChannel channel; + private BigtableTableAdminClient tableAdminClient; + private BigtableDataClient dataClient; + + private static final InstanceName INSTANCE_NAME = + InstanceName.of("fake-project", "fake-instance"); + + // Use the gcloud installed emulator + static Emulator createGCloud() { + final Path gcloudSdkPath; + + try { + gcloudSdkPath = getGcloudSdkPath(); + } catch (Exception e) { + throw new RuntimeException("Failed to get the gcloud SDK path. Is it installed?", e); + } + + Path emulatorPath = + gcloudSdkPath.resolve(Paths.get("platform", "bigtable-emulator", "cbtemulator")); + + if (!Files.exists(emulatorPath)) { + throw new RuntimeException( + "cbtemulator is not installed, please install with `gcloud components install bigtable`"); + } + + return new Emulator(emulatorPath); + } + + private Emulator(Path executable) { + this.executable = executable; + } + + void start() throws Exception { + int availablePort = getAvailablePort(); + + process = Runtime.getRuntime().exec(executable + " -port " + "" + availablePort); + pipeStreamToLog(process.getInputStream(), Level.INFO); + pipeStreamToLog(process.getErrorStream(), Level.WARNING); + isStopped = false; + + waitForPort(availablePort); + + channel = createChannel(availablePort); + + tableAdminClient = + BigtableTableAdminClient.create( + configureClient(BigtableTableAdminSettings.newBuilder()).build()); + + dataClient = + BigtableDataClient.create( + configureClient(BigtableDataSettings.newBuilder().setInstanceName(INSTANCE_NAME)) + .build()); + + Runtime.getRuntime() + .addShutdownHook( + new Thread() { + @Override + public void run() { + if (!isStopped) { + isStopped = true; + process.destroy(); + } + } + }); + } + + void stop() throws Exception { + try { + dataClient.close(); + tableAdminClient.close(); + channel.shutdownNow(); + channel.awaitTermination(1, TimeUnit.MINUTES); + } finally { + isStopped = true; + process.destroy(); + } + } + + BigtableDataClient getDataClient() { + return dataClient; + } + + BigtableTableAdminClient getTableAdminClient() { + return tableAdminClient; + } + + // + private static Path getGcloudSdkPath() throws Exception { + Process p = Runtime.getRuntime().exec("gcloud info --format=value(installation.sdk_root)"); + pipeStreamToLog(p.getErrorStream(), Level.WARNING); + + String sdkRoot = bufferOutput(p.getInputStream()).get(1, TimeUnit.MINUTES).trim(); + + if (p.waitFor() != 0) { + throw new RuntimeException("Failed to get sdk root, is gcloud sdk installed?"); + } + return Paths.get(sdkRoot); + } + + private static int getAvailablePort() { + try (ServerSocket serverSocket = new ServerSocket(0)) { + return serverSocket.getLocalPort(); + } catch (IOException e) { + throw new RuntimeException("Failed to find open port"); + } + } + + private void waitForPort(int port) throws InterruptedException, TimeoutException { + for (int i = 0; i < 100; i++) { + try (Socket ignored = new Socket("localhost", port)) { + return; + } catch (IOException e) { + Thread.sleep(200); + } + } + + throw new TimeoutException("Timed out waiting for server to start"); + } + + private ManagedChannel createChannel(int port) { + // NOTE: usePlaintext is currently @ExperimentalAPI. In grpc 1.11 it be became parameterless. + // In 1.12 it should be stable. See https://github.com/grpc/grpc-java/issues/1772 for discussion + return ManagedChannelBuilder.forAddress("localhost", port) + .usePlaintext(true) + .maxInboundMessageSize(256 * 1024 * 1024) + .build(); + } + + private > T configureClient(T settings) { + settings + .setCredentialsProvider(new NoCredentialsProvider()) + .setTransportChannelProvider( + FixedTransportChannelProvider.create(GrpcTransportChannel.create(channel))); + + return settings; + } + + private static void pipeStreamToLog(final InputStream stream, final Level level) { + final BufferedReader reader = new BufferedReader(new InputStreamReader(stream)); + + Thread thread = + new Thread( + new Runnable() { + @Override + public void run() { + try { + String line; + while ((line = reader.readLine()) != null) { + LOGGER.log(level, line); + } + } catch (IOException e) { + LOGGER.log(Level.WARNING, "Failed to read process stream", e); + } + } + }); + thread.setDaemon(true); + thread.start(); + } + + private static Future bufferOutput(final InputStream stream) { + FutureTask task = + new FutureTask<>( + new Callable() { + @Override + public String call() throws Exception { + return CharStreams.toString(new InputStreamReader(stream)); + } + }); + task.run(); + + return task; + } + // +} diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/EmulatorEnv.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/EmulatorEnv.java new file mode 100644 index 000000000000..4181adc16ce1 --- /dev/null +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/EmulatorEnv.java @@ -0,0 +1,76 @@ +/* + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.it.env; + +import com.google.bigtable.admin.v2.ColumnFamily; +import com.google.bigtable.admin.v2.CreateTableRequest; +import com.google.cloud.bigtable.data.v2.models.InstanceName; +import com.google.bigtable.admin.v2.Table; +import com.google.bigtable.v2.TableName; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import java.io.IOException; +import java.util.concurrent.TimeoutException; + +public class EmulatorEnv implements TestEnv { + private static final InstanceName INSTANCE_NAME = + InstanceName.of("fake-project", "fake-instance"); + private static final String TABLE_ID = "default-table"; + private static final String FAMILY_ID = "cf"; + + private Emulator emulator; + + @Override + public void start() throws Exception { + emulator = Emulator.createGCloud(); + emulator.start(); + + emulator + .getTableAdminClient() + .createTable( + CreateTableRequest.newBuilder() + .setParent(INSTANCE_NAME.toString()) + .setTableId(TABLE_ID) + .setTable( + Table.newBuilder() + .putColumnFamilies(FAMILY_ID, ColumnFamily.getDefaultInstance())) + .build()); + } + + @Override + public void stop() throws Exception { + emulator.stop(); + } + + @Override + public TableName getTableName() { + return TableName.of(INSTANCE_NAME.getProject(), INSTANCE_NAME.getInstance(), TABLE_ID); + } + + @Override + public String getRowPrefix() { + return "fake-"; + } + + @Override + public BigtableDataClient getDataClient() { + return emulator.getDataClient(); + } + + @Override + public String getFamilyId() { + return FAMILY_ID; + } +} diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/ProdEnv.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/ProdEnv.java new file mode 100644 index 000000000000..cdd35c9a9b1b --- /dev/null +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/ProdEnv.java @@ -0,0 +1,120 @@ +/* + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.it.env; + +import com.google.api.core.ApiFuture; +import com.google.api.core.ApiFutures; +import com.google.api.gax.rpc.ServerStream; +import com.google.cloud.bigtable.data.v2.models.InstanceName; +import com.google.bigtable.v2.TableName; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.models.Query; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.cloud.bigtable.data.v2.models.RowMutation; +import com.google.common.collect.Lists; +import java.io.IOException; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** + * Test environment that uses an existing bigtable table. The table must have a pre-existing family + * {@code cf}. The target table is configured via the system properties: + * + *
    + *
  • {@code bigtable.project} + *
  • {@code bigtable.instance} + *
  • {@code bigtable.table} + *
+ */ +public class ProdEnv implements TestEnv { + private static final String TABLE_PROPERTY_NAME = "bigtable.table"; + + private TableName tableName; + private static final String FAMILY_ID = "cf"; + private String rowPrefix; + + private BigtableDataClient dataClient; + + static ProdEnv fromSystemProperties() { + String tableNameStr = getRequiredProperty(TABLE_PROPERTY_NAME); + TableName tableName = TableName.parse(tableNameStr); + + return new ProdEnv(tableName); + } + + ProdEnv(TableName tableName) { + this.tableName = tableName; + } + + @Override + public void start() throws IOException { + rowPrefix = UUID.randomUUID() + "-"; + dataClient = + BigtableDataClient.create(InstanceName.of(tableName.getProject(), tableName.getInstance())); + } + + @Override + public void stop() throws Exception { + deleteRows(); + dataClient.close(); + } + + @Override + public BigtableDataClient getDataClient() { + return dataClient; + } + + @Override + public TableName getTableName() { + return tableName; + } + + @Override + public String getFamilyId() { + return FAMILY_ID; + } + + @Override + public String getRowPrefix() { + return rowPrefix; + } + + private void deleteRows() throws InterruptedException, ExecutionException, TimeoutException { + Query query = Query.create(tableName.getTable()).prefix(rowPrefix); + + List> futures = Lists.newArrayList(); + ServerStream rows = dataClient.readRows(query); + for (Row row : rows) { + ApiFuture future = + dataClient.mutateRowAsync( + RowMutation.create(tableName.getTable(), row.getKey()).deleteRow()); + futures.add(future); + } + + ApiFutures.allAsList(futures).get(10, TimeUnit.MINUTES); + } + + private static String getRequiredProperty(String prop) { + String value = System.getProperty(prop); + if (value == null || value.isEmpty()) { + throw new RuntimeException("Missing system property: " + prop); + } + return value; + } +} diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/TestEnv.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/TestEnv.java new file mode 100644 index 000000000000..ccc4eb7336fe --- /dev/null +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/TestEnv.java @@ -0,0 +1,40 @@ +/* + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.it.env; + +import com.google.bigtable.v2.TableName; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import java.io.IOException; +import java.util.concurrent.TimeoutException; + +/** + * Defines the interface of a target environment. + * + *

This allows for integration tests to run against either production or an emulator. + */ +public interface TestEnv { + void start() throws Exception; + + void stop() throws Exception; + + BigtableDataClient getDataClient(); + + TableName getTableName(); + + String getFamilyId(); + + String getRowPrefix(); +} diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/TestEnvRule.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/TestEnvRule.java new file mode 100644 index 000000000000..06f4c6d3ef54 --- /dev/null +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/TestEnvRule.java @@ -0,0 +1,76 @@ +/* + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.it.env; + +import java.util.logging.Level; +import java.util.logging.Logger; +import org.junit.rules.ExternalResource; + +/** + * Simple JUnit rule to start and stop the target test environment. + * + *

The environment can be specified via the system property {@code bigtable.env}. The choices + * are: + * + *

    + *
  • {@code emulator}: uses the cbtemulator component that can be installed by gcloud + *
  • {@code prod}: uses a pre-existing production table. The target table is defined using + * system properties listed in {@link ProdEnv} and application default credentials + *
+ * + *

By default, {@code emulator} will be used + */ +public class TestEnvRule extends ExternalResource { + private static final Logger LOGGER = Logger.getLogger(TestEnvRule.class.getName()); + + private static final String ENV_PROPERTY = "bigtable.env"; + + private TestEnv testEnv; + + @Override + protected void before() throws Throwable { + String env = System.getProperty(ENV_PROPERTY, "emulator"); + + switch (env) { + case "emulator": + testEnv = new EmulatorEnv(); + break; + case "prod": + testEnv = ProdEnv.fromSystemProperties(); + break; + default: + throw new IllegalArgumentException( + String.format( + "Unknown env: %s. Please set the system property %s to either 'emulator' or 'prod'.", + env, ENV_PROPERTY)); + } + testEnv.start(); + } + + @Override + protected void after() { + try { + testEnv.stop(); + } catch (Exception e) { + LOGGER.log(Level.WARNING, "Failed to stop the environment", e); + } + testEnv = null; + } + + public TestEnv env() { + return testEnv; + } +} diff --git a/utilities/verify_single_it.sh b/utilities/verify_single_it.sh index 8437edf60b46..83a9e3f746c9 100755 --- a/utilities/verify_single_it.sh +++ b/utilities/verify_single_it.sh @@ -5,6 +5,7 @@ set -e MODULE=$1 +ARGS=${@:2:99} if [ -z $MODULE ]; then echo "First arg (module) not provided, so we're exiting." @@ -26,4 +27,4 @@ echo "----- building and installing shared modules -----" mvn -B -pl google-cloud-core,google-cloud-core-http,google-cloud-core-grpc,google-cloud-storage,google-cloud-pubsub install -DskipTests echo "----- running integration tests -----" -mvn -B -pl $MODULE -DtrimStackTrace=false -fae verify +mvn -B -pl $MODULE -DtrimStackTrace=false -fae verify ${ARGS}