From dc087fbba806338ad9d945188afc4d4a9fca55ae Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Sun, 4 Mar 2018 00:35:53 -0500 Subject: [PATCH 01/15] First round of integration tests --- .../bigtable/data/v2/it/CheckAndMutateIT.java | 64 ++++++ .../bigtable/data/v2/it/MutateRowIT.java | 63 ++++++ .../cloud/bigtable/data/v2/it/ReadIT.java | 144 ++++++++++++ .../data/v2/it/ReadModifyWriteIT.java | 53 +++++ .../bigtable/data/v2/it/SampleRowsIT.java | 66 ++++++ .../bigtable/data/v2/it/env/Emulator.java | 208 ++++++++++++++++++ .../bigtable/data/v2/it/env/EmulatorEnv.java | 76 +++++++ .../bigtable/data/v2/it/env/ProdEnv.java | 131 +++++++++++ .../bigtable/data/v2/it/env/TestEnv.java | 35 +++ .../bigtable/data/v2/it/env/TestEnvRule.java | 64 ++++++ 10 files changed, 904 insertions(+) create mode 100644 google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/CheckAndMutateIT.java create mode 100644 google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/MutateRowIT.java create mode 100644 google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java create mode 100644 google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadModifyWriteIT.java create mode 100644 google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/SampleRowsIT.java create mode 100644 google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java create mode 100644 google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/EmulatorEnv.java create mode 100644 google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/ProdEnv.java create mode 100644 google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/TestEnv.java create mode 100644 google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/TestEnvRule.java diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/CheckAndMutateIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/CheckAndMutateIT.java new file mode 100644 index 000000000000..60e14aa7c861 --- /dev/null +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/CheckAndMutateIT.java @@ -0,0 +1,64 @@ +/* + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.it; + +import static com.google.cloud.bigtable.data.v2.models.Filters.FILTERS; +import static com.google.common.truth.Truth.assertThat; + +import com.google.cloud.bigtable.data.v2.it.env.TestEnvRule; +import com.google.cloud.bigtable.data.v2.models.ConditionalRowMutation; +import com.google.cloud.bigtable.data.v2.models.Mutation; +import com.google.cloud.bigtable.data.v2.models.Query; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.cloud.bigtable.data.v2.models.RowMutation; +import com.google.protobuf.ByteString; +import java.util.concurrent.TimeUnit; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class CheckAndMutateIT { + @ClassRule + public static TestEnvRule testEnvRule = new TestEnvRule(); + + @Test + public void test() throws Exception { + String tableId = testEnvRule.env().getTableName().getTable(); + String rowKey = testEnvRule.env().getRowPrefix(); + String familyId = testEnvRule.env().getFamilyId(); + + testEnvRule.env().getDataClient().mutateRowCallable().call( + RowMutation.create(tableId, rowKey) + .setCell(familyId, "q1", "val1") + .setCell(familyId, "q2", "val2") + ); + + testEnvRule.env().getDataClient().checkAndMutateRowAsync( + ConditionalRowMutation.create(tableId, rowKey) + .condition(FILTERS.qualifier().exactMatch("q1")) + .then(Mutation.create().setCell(familyId, "q3", "q1")) + ).get(1, TimeUnit.MINUTES); + + Row row = testEnvRule.env().getDataClient().readRowsCallable().first().call( + Query.create(tableId).rowKey(rowKey) + ); + + assertThat(row.getCells()).hasSize(3); + assertThat(row.getCells().get(2).getValue()).isEqualTo(ByteString.copyFromUtf8("q1")); + } +} diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/MutateRowIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/MutateRowIT.java new file mode 100644 index 000000000000..5cad5a576187 --- /dev/null +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/MutateRowIT.java @@ -0,0 +1,63 @@ +/* + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.it; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.cloud.bigtable.data.v2.it.env.TestEnvRule; +import com.google.cloud.bigtable.data.v2.models.Query; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.cloud.bigtable.data.v2.models.RowMutation; +import com.google.protobuf.ByteString; +import java.util.concurrent.TimeUnit; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class MutateRowIT { + @ClassRule + public static TestEnvRule testEnvRule = new TestEnvRule(); + + @Test + public void test() throws Exception { + String rowKey = testEnvRule.env().getRowPrefix() + "testA"; + String familyId = testEnvRule.env().getFamilyId(); + + + testEnvRule.env().getDataClient().mutateRowAsync( + RowMutation.create(testEnvRule.env().getTableName().getTable(), rowKey) + .setCell(familyId, "q", "myVal") + .setCell(familyId, "q2", "myVal2") + .setCell(familyId, "q3", "myVal3") + ).get(1, TimeUnit.MINUTES); + + testEnvRule.env().getDataClient().mutateRowAsync( + RowMutation.create(testEnvRule.env().getTableName().getTable(), rowKey) + .deleteCells(familyId, "q2") + ).get(1, TimeUnit.MINUTES); + + Row row = testEnvRule.env().getDataClient().readRowsCallable().first().call( + Query.create(testEnvRule.env().getTableName().getTable()) + .rowKey(rowKey) + ); + + assertThat(row.getCells()).hasSize(2); + assertThat(row.getCells().get(0).getValue()).isEqualTo(ByteString.copyFromUtf8("myVal")); + assertThat(row.getCells().get(1).getValue()).isEqualTo(ByteString.copyFromUtf8("myVal3")); + } +} diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java new file mode 100644 index 000000000000..a4fab6e2ac08 --- /dev/null +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java @@ -0,0 +1,144 @@ +/* + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.it; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.api.core.SettableApiFuture; +import com.google.api.gax.rpc.ResponseObserver; +import com.google.api.gax.rpc.StreamController; +import com.google.cloud.bigtable.data.v2.it.env.TestEnvRule; +import com.google.cloud.bigtable.data.v2.models.Query; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.cloud.bigtable.data.v2.models.RowCell; +import com.google.cloud.bigtable.data.v2.models.RowMutation; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.protobuf.ByteString; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class ReadIT { + private String prefix; + + @ClassRule public static TestEnvRule testEnvRule = new TestEnvRule(); + + @Before + public void setUp() { + prefix = testEnvRule.env().getRowPrefix(); + } + + @Test + public void readEmpty() throws Throwable { + String uniqueKey = prefix + "-readEmpty"; + + Query query = Query.create(testEnvRule.env().getTableName().getTable()).rowKey(uniqueKey); + + // Sync + ArrayList rows = Lists.newArrayList(testEnvRule.env().getDataClient().readRows(query)); + assertThat(rows).isEmpty(); + + // Async + AccumulatingObserver observer = new AccumulatingObserver(); + testEnvRule.env().getDataClient().readRowsAsync(query, observer); + observer.awaitCompletion(); + assertThat(observer.responses).isEmpty(); + } + + @Test + public void read() throws Throwable { + int numRows = 5; + List expectedRows = Lists.newArrayList(); + String uniqueKey = prefix + "-read"; + + long timestampMicros = System.nanoTime() * 1_000; + + for (int i = 0; i < numRows; i++) { + testEnvRule + .env() + .getDataClient() + .mutateRowCallable() + .call( + RowMutation.create(testEnvRule.env().getTableName().getTable(), uniqueKey + "-" + i) + .setCell(testEnvRule.env().getFamilyId(), "q", timestampMicros, "my-value")); + + expectedRows.add( + Row.create( + ByteString.copyFromUtf8(uniqueKey + "-" + i), + ImmutableList.of( + RowCell.create( + testEnvRule.env().getFamilyId(), + ByteString.copyFromUtf8("q"), + timestampMicros, + ImmutableList.of(), + ByteString.copyFromUtf8("my-value"))))); + } + + // Sync + Query query = + Query.create(testEnvRule.env().getTableName().getTable()) + .range(uniqueKey + "-0", uniqueKey + "-" + numRows); + ArrayList actualResults = + Lists.newArrayList(testEnvRule.env().getDataClient().readRows(query)); + + assertThat(actualResults).containsExactlyElementsIn(expectedRows); + + // Async + AccumulatingObserver observer = new AccumulatingObserver(); + testEnvRule.env().getDataClient().readRowsAsync(query, observer); + observer.awaitCompletion(); + assertThat(observer.responses).containsExactlyElementsIn(expectedRows); + } + + static class AccumulatingObserver implements ResponseObserver { + final List responses = Lists.newArrayList(); + final SettableApiFuture completionFuture = SettableApiFuture.create(); + + void awaitCompletion() throws Throwable { + try { + completionFuture.get(10, TimeUnit.MINUTES); + } catch (ExecutionException e) { + throw e.getCause(); + } + } + + @Override + public void onStart(StreamController controller) {} + + @Override + public void onResponse(Row row) { + responses.add(row); + } + + @Override + public void onError(Throwable t) { + completionFuture.setException(t); + } + + @Override + public void onComplete() { + completionFuture.set(null); + } + } +} diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadModifyWriteIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadModifyWriteIT.java new file mode 100644 index 000000000000..b682ce439413 --- /dev/null +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadModifyWriteIT.java @@ -0,0 +1,53 @@ +/* + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.it; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.cloud.bigtable.data.v2.it.env.TestEnvRule; +import com.google.cloud.bigtable.data.v2.models.ReadModifyWriteRow; +import com.google.cloud.bigtable.data.v2.models.Row; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class ReadModifyWriteIT { + @ClassRule + public static TestEnvRule testEnvRule = new TestEnvRule(); + + @Test + public void test() throws InterruptedException, ExecutionException, TimeoutException { + String tableId = testEnvRule.env().getTableName().getTable(); + String family = testEnvRule.env().getFamilyId(); + String rowKey = testEnvRule.env().getRowPrefix(); + + Row row = testEnvRule.env().getDataClient().readModifyWriteRowAsync( + ReadModifyWriteRow.create(tableId, rowKey) + .append(family, "q1", "a") + .increment(family, "q2", 3) + ).get(1, TimeUnit.MINUTES); + + assertThat(row.getCells()).hasSize(2); + assertThat(row.getCells().get(0).getValue()).isEqualTo(ByteString.copyFromUtf8("a")); + assertThat(row.getCells().get(1).getValue()) + .isEqualTo(ByteString.copyFrom(new byte[] {0, 0, 0, 0, 0, 0, 0, 3})); + } +} diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/SampleRowsIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/SampleRowsIT.java new file mode 100644 index 000000000000..0f2c58e9c296 --- /dev/null +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/SampleRowsIT.java @@ -0,0 +1,66 @@ +/* + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.it; + +import static com.google.common.truth.Truth.*; + +import com.google.api.core.ApiFuture; +import com.google.api.core.ApiFutures; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.it.env.TestEnvRule; +import com.google.cloud.bigtable.data.v2.models.KeyOffset; +import com.google.cloud.bigtable.data.v2.models.RowMutation; +import com.google.common.collect.Lists; +import com.google.common.truth.Truth; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class SampleRowsIT { + @ClassRule + public static TestEnvRule testEnvRule = new TestEnvRule(); + + @Test + public void test() throws InterruptedException, ExecutionException, TimeoutException { + BigtableDataClient client = testEnvRule.env().getDataClient(); + + // Create some data so that sample row keys has something to show + List> futures = Lists.newArrayList(); + for (int i = 0; i < 10; i++) { + ApiFuture future = client.mutateRowAsync( + RowMutation.create(testEnvRule.env().getTableName().getTable(), + testEnvRule.env().getRowPrefix() + "-" + i) + .setCell(testEnvRule.env().getFamilyId(), "", "value") + ); + futures.add(future); + } + ApiFutures.allAsList(futures).get(1, TimeUnit.MINUTES); + + ApiFuture> future = client + .sampleRowKeysAsync(testEnvRule.env().getTableName().getTable()); + + List results = future.get(1, TimeUnit.MINUTES); + + assertThat(results).isNotEmpty(); + assertThat(results.get(results.size() - 1).getOffsetBytes()).isGreaterThan(0L); + } +} diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java new file mode 100644 index 000000000000..3cfc8f07f39f --- /dev/null +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java @@ -0,0 +1,208 @@ +/* + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.it.env; + +import com.google.api.gax.core.NoCredentialsProvider; +import com.google.api.gax.grpc.GrpcTransportChannel; +import com.google.api.gax.rpc.ClientSettings; +import com.google.api.gax.rpc.FixedTransportChannelProvider; +import com.google.bigtable.admin.v2.InstanceName; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.BigtableDataSettings; +import com.google.common.io.CharStreams; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.ServerSocket; +import java.net.Socket; +import java.nio.file.Paths; +import java.util.concurrent.Callable; +import java.util.concurrent.Future; +import java.util.concurrent.FutureTask; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Java wrapper around the gcloud bigtable emulator. + */ +// TODO(igorbernstein): Clean this up and externalize this in a separate artifact +// TODO(igorbernstein): Stop depending on gcloud for the binary, instead wrap it in a jar. +class Emulator { + private static final Logger LOGGER = Logger.getLogger(Emulator.class.getName()); + + private final String executable; + private Process process; + private ManagedChannel channel; + private BigtableTableAdminClient tableAdminClient; + private BigtableDataClient dataClient; + + private static final InstanceName INSTANCE_NAME = + InstanceName.of("fake-project", "fake-instance"); + + static Emulator createGCloud() { + final Process p; + + try { + p = Runtime.getRuntime().exec("gcloud info --format=value(installation.sdk_root)"); + pipeStreamToLog(p.getErrorStream(), Level.WARNING); + } catch (IOException e) { + throw new RuntimeException("Failed to run gcloud info", e); + } + + String sdkRoot; + try { + sdkRoot = bufferOutput(p.getInputStream()).get(1, TimeUnit.MINUTES).trim(); + } catch (Exception e) { + throw new RuntimeException("Failed to get gcloud sdk install path", e); + } + + try { + if (p.waitFor() != 0) { + throw new RuntimeException("Failed to get sdk root, is gcloud sdk installed?"); + } + } catch (InterruptedException e) { + throw new RuntimeException("Interrupted invoking gcloud", e); + } + + String emulatorPath = + Paths.get(sdkRoot, "platform", "bigtable-emulator", "cbtemulator").toString(); + + if (!new File(emulatorPath).exists()) { + throw new RuntimeException( + "cbtemulator is not installed, please install with `gcloud components install bigtable`"); + } + + return new Emulator(emulatorPath); + } + + private Emulator(String executable) { + this.executable = executable; + } + + void start() throws InterruptedException, IOException, TimeoutException { + int availablePort = getAvailablePort(); + + process = Runtime.getRuntime().exec(executable + " -port " + "" + availablePort); + pipeStreamToLog(process.getInputStream(), Level.INFO); + pipeStreamToLog(process.getErrorStream(), Level.WARNING); + + waitForPort(availablePort); + + channel = createChannel(availablePort); + + tableAdminClient = + BigtableTableAdminClient.create( + configureClient(BigtableTableAdminSettings.newBuilder()).build()); + + dataClient = + BigtableDataClient.create( + configureClient(BigtableDataSettings.newBuilder().setInstanceName(INSTANCE_NAME)) + .build()); + } + + void stop() throws Exception { + dataClient.close(); + tableAdminClient.close(); + channel.shutdownNow(); + channel.awaitTermination(1, TimeUnit.MINUTES); + process.destroy(); + } + + BigtableDataClient getDataClient() { + return dataClient; + } + + BigtableTableAdminClient getTableAdminClient() { + return tableAdminClient; + } + + // + private static int getAvailablePort() { + try (ServerSocket serverSocket = new ServerSocket(0)) { + return serverSocket.getLocalPort(); + } catch (IOException e) { + throw new RuntimeException("Failed to find open port"); + } + } + + private void waitForPort(int port) throws InterruptedException, TimeoutException { + for (int i = 0; i < 100; i++) { + try (Socket ignored = new Socket("localhost", port)) { + return; + } catch (IOException e) { + Thread.sleep(200); + } + } + + throw new TimeoutException("Timed out waiting for server to start"); + } + + private ManagedChannel createChannel(int port) { + return ManagedChannelBuilder.forAddress("localhost", port) + .usePlaintext(true) + .maxInboundMessageSize(256 * 1024 * 1024) + .build(); + } + + private > T configureClient(T settings) { + settings + .setCredentialsProvider(new NoCredentialsProvider()) + .setTransportChannelProvider( + FixedTransportChannelProvider.create(GrpcTransportChannel.create(channel))); + + return settings; + } + + private static void pipeStreamToLog(final InputStream stream, final Level level) { + final BufferedReader reader = new BufferedReader(new InputStreamReader(stream)); + + Thread thread = new Thread(new Runnable() { + @Override + public void run() { + try { + String line; + while ((line = reader.readLine()) != null) + LOGGER.log(level, line); + } catch (IOException e) { + LOGGER.log(Level.WARNING, "Failed to read process stream", e); + } + } + }); + thread.setDaemon(true); + thread.start(); + } + + private static Future bufferOutput(final InputStream stream) { + FutureTask task = new FutureTask<>(new Callable() { + @Override + public String call() throws Exception { + return CharStreams.toString(new InputStreamReader(stream)); + } + }); + task.run(); + + return task; + } + // +} diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/EmulatorEnv.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/EmulatorEnv.java new file mode 100644 index 000000000000..b4a4fdcf7b64 --- /dev/null +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/EmulatorEnv.java @@ -0,0 +1,76 @@ +/* + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.it.env; + +import com.google.bigtable.admin.v2.ColumnFamily; +import com.google.bigtable.admin.v2.CreateTableRequest; +import com.google.bigtable.admin.v2.InstanceName; +import com.google.bigtable.admin.v2.Table; +import com.google.bigtable.v2.TableName; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import java.io.IOException; +import java.util.concurrent.TimeoutException; + +public class EmulatorEnv implements TestEnv { + private static final InstanceName INSTANCE_NAME = + InstanceName.of("fake-project", "fake-instance"); + private static final String TABLE_ID = "default-table"; + private static final String FAMILY_ID = "cf"; + + private Emulator emulator; + + @Override + public void start() throws IOException, InterruptedException, TimeoutException { + emulator = Emulator.createGCloud(); + emulator.start(); + + emulator + .getTableAdminClient() + .createTable( + CreateTableRequest.newBuilder() + .setParent(INSTANCE_NAME.toString()) + .setTableId(TABLE_ID) + .setTable( + Table.newBuilder() + .putColumnFamilies(FAMILY_ID, ColumnFamily.getDefaultInstance())) + .build()); + } + + @Override + public void stop() throws Exception { + emulator.stop(); + } + + @Override + public TableName getTableName() { + return TableName.of(INSTANCE_NAME.getProject(), INSTANCE_NAME.getInstance(), TABLE_ID); + } + + @Override + public String getRowPrefix() { + return "fake-"; + } + + @Override + public BigtableDataClient getDataClient() { + return emulator.getDataClient(); + } + + @Override + public String getFamilyId() { + return FAMILY_ID; + } +} diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/ProdEnv.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/ProdEnv.java new file mode 100644 index 000000000000..9fa7ffc59024 --- /dev/null +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/ProdEnv.java @@ -0,0 +1,131 @@ +/* + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.it.env; + +import com.google.api.core.ApiFuture; +import com.google.api.core.ApiFutures; +import com.google.api.gax.rpc.ServerStream; +import com.google.bigtable.admin.v2.InstanceName; +import com.google.bigtable.v2.TableName; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.models.Query; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.cloud.bigtable.data.v2.models.RowMutation; +import com.google.common.collect.Lists; +import java.io.IOException; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** + * Test environment that uses an existing bigtable table. The table must have a pre-existing family + * {@code cf}. The target table is configured via the system properties: + * + *
    + *
  • {@code bigtable.project} + *
  • {@code bigtable.instance} + *
  • {@code bigtable.table} + *
+ * + * Expects the instance to exist with a table created: cbt -project igorbernstein-dev -instance + * integration-tests createtable my-table cbt -project igorbernstein-dev -instance integration-tests + * createfamily my-table cf + */ +public class ProdEnv implements TestEnv { + private static final String PROJECT_PROPERTY_NAME = "bigtable.project"; + private static final String INSTANCE_PROPERTY_NAME = "bigtable.instance"; + private static final String TABLE_PROPERTY_NAME = "bigtable.table"; + + private TableName tableName; + private static final String FAMILY_ID = "cf"; + private String rowPrefix; + + private BigtableDataClient dataClient; + + static ProdEnv fromSystemProperties() { + String projectId = getRequiredProperty(PROJECT_PROPERTY_NAME); + String instanceId = getRequiredProperty(INSTANCE_PROPERTY_NAME); + String tableId = getRequiredProperty(TABLE_PROPERTY_NAME); + + return new ProdEnv(TableName.of(projectId, instanceId, tableId)); + } + + ProdEnv(TableName tableName) { + this.tableName = tableName; + } + + @Override + public void start() throws IOException { + String projectId = getRequiredProperty(PROJECT_PROPERTY_NAME); + String instanceId = getRequiredProperty(INSTANCE_PROPERTY_NAME); + String tableId = getRequiredProperty(TABLE_PROPERTY_NAME); + + tableName = TableName.of(projectId, instanceId, tableId); + rowPrefix = UUID.randomUUID() + "-"; + dataClient = BigtableDataClient.create(InstanceName.of(projectId, instanceId)); + } + + @Override + public void stop() throws Exception { + deleteRows(); + dataClient.close(); + } + + @Override + public BigtableDataClient getDataClient() { + return dataClient; + } + + @Override + public TableName getTableName() { + return tableName; + } + + @Override + public String getFamilyId() { + return FAMILY_ID; + } + + @Override + public String getRowPrefix() { + return rowPrefix; + } + + private void deleteRows() throws InterruptedException, ExecutionException, TimeoutException { + Query query = Query.create(tableName.getTable()).prefix(rowPrefix); + + List> futures = Lists.newArrayList(); + ServerStream rows = dataClient.readRows(query); + for (Row row : rows) { + ApiFuture future = + dataClient.mutateRowAsync( + RowMutation.create(tableName.getTable(), row.getKey()).deleteRow()); + futures.add(future); + } + + ApiFutures.allAsList(futures).get(10, TimeUnit.MINUTES); + } + + private static String getRequiredProperty(String prop) { + String value = System.getProperty(prop); + if (value == null || value.isEmpty()) { + throw new RuntimeException("Missing system property: " + prop); + } + return value; + } +} diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/TestEnv.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/TestEnv.java new file mode 100644 index 000000000000..790849ecdeae --- /dev/null +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/TestEnv.java @@ -0,0 +1,35 @@ +/* + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.it.env; + +import com.google.bigtable.v2.TableName; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import java.io.IOException; +import java.util.concurrent.TimeoutException; + +public interface TestEnv { + void start() throws IOException, InterruptedException, TimeoutException; + + void stop() throws Exception; + + BigtableDataClient getDataClient(); + + TableName getTableName(); + + String getFamilyId(); + + String getRowPrefix(); +} diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/TestEnvRule.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/TestEnvRule.java new file mode 100644 index 000000000000..40b4fad8a6fd --- /dev/null +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/TestEnvRule.java @@ -0,0 +1,64 @@ +/* + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.it.env; + +import java.util.logging.Level; +import java.util.logging.Logger; +import org.junit.rules.ExternalResource; + +public class TestEnvRule extends ExternalResource { + private static final Logger LOGGER = Logger.getLogger(TestEnvRule.class.getName()); + + private static final String ENV_PROPERTY = "bigtable.env"; + + private TestEnv testEnv; + + @Override + protected void before() throws Throwable { + String env = System.getProperty(ENV_PROPERTY, "emulator"); + + switch (env) { + case "emulator": + testEnv = new EmulatorEnv(); + break; + case "prod": + testEnv = ProdEnv.fromSystemProperties(); + break; + default: + throw new RuntimeException( + "Unknown env: " + + env + + ". Please set the system propert " + + ENV_PROPERTY + + " to either 'emulator' or 'prod'."); + } + testEnv.start(); + } + + @Override + protected void after() { + try { + testEnv.stop(); + } catch (Exception e) { + LOGGER.log(Level.WARNING, "Failed to stop the environment", e); + } + testEnv = null; + } + + public TestEnv env() { + return testEnv; + } +} From 2a3687d53bd100208354c2ae99f3b55e3c1a7009 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Wed, 7 Mar 2018 22:09:40 -0500 Subject: [PATCH 02/15] wip --- .../data/v2/it/BulkMutationBatcherIT.java | 70 +++++++++++++++++++ .../bigtable/data/v2/it/CheckAndMutateIT.java | 39 +++++++---- .../bigtable/data/v2/it/MutateRowIT.java | 41 ++++++----- .../data/v2/it/ReadModifyWriteIT.java | 18 +++-- .../bigtable/data/v2/it/SampleRowsIT.java | 18 ++--- .../bigtable/data/v2/it/env/Emulator.java | 47 +++++++------ 6 files changed, 162 insertions(+), 71 deletions(-) create mode 100644 google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BulkMutationBatcherIT.java diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BulkMutationBatcherIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BulkMutationBatcherIT.java new file mode 100644 index 000000000000..b0d82d8d9388 --- /dev/null +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BulkMutationBatcherIT.java @@ -0,0 +1,70 @@ +/* + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.it; + +import com.google.api.gax.rpc.ServerStream; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.it.env.TestEnvRule; +import com.google.cloud.bigtable.data.v2.models.BulkMutationBatcher; +import com.google.cloud.bigtable.data.v2.models.Query; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.cloud.bigtable.data.v2.models.RowCell; +import com.google.cloud.bigtable.data.v2.models.RowMutation; +import com.google.common.collect.Lists; +import com.google.common.truth.Truth; +import com.google.protobuf.ByteString; +import java.util.List; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class BulkMutationBatcherIT { + @ClassRule public static TestEnvRule testEnvRule = new TestEnvRule(); + + @Test + public void test() throws Exception { + BigtableDataClient client = testEnvRule.env().getDataClient(); + String tableId = testEnvRule.env().getTableName().getTable(); + String family = testEnvRule.env().getFamilyId(); + String rowPrefix = testEnvRule.env().getRowPrefix(); + + try (BulkMutationBatcher batcher = client.newBulkMutationBatcher()) { + for (int i = 0; i < 10; i++) { + batcher.add( + RowMutation.create(tableId, rowPrefix + "-" + i).setCell(family, "q", 10_000, "value")); + } + } + + List expectedRows = Lists.newArrayList(); + for (int i = 0; i < 10; i++) { + expectedRows.add( + Row.create( + ByteString.copyFromUtf8(rowPrefix + "-" + i), + Lists.newArrayList( + RowCell.create( + family, + ByteString.copyFromUtf8("q"), + 10_000, + Lists.newArrayList(), + ByteString.copyFromUtf8("value"))))); + } + ServerStream actualRows = client.readRows(Query.create(tableId).prefix(rowPrefix)); + + Truth.assertThat(actualRows).containsExactlyElementsIn(expectedRows); + } +} diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/CheckAndMutateIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/CheckAndMutateIT.java index 60e14aa7c861..2b65517f232d 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/CheckAndMutateIT.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/CheckAndMutateIT.java @@ -33,8 +33,7 @@ @RunWith(JUnit4.class) public class CheckAndMutateIT { - @ClassRule - public static TestEnvRule testEnvRule = new TestEnvRule(); + @ClassRule public static TestEnvRule testEnvRule = new TestEnvRule(); @Test public void test() throws Exception { @@ -42,21 +41,31 @@ public void test() throws Exception { String rowKey = testEnvRule.env().getRowPrefix(); String familyId = testEnvRule.env().getFamilyId(); - testEnvRule.env().getDataClient().mutateRowCallable().call( - RowMutation.create(tableId, rowKey) - .setCell(familyId, "q1", "val1") - .setCell(familyId, "q2", "val2") - ); + testEnvRule + .env() + .getDataClient() + .mutateRowCallable() + .call( + RowMutation.create(tableId, rowKey) + .setCell(familyId, "q1", "val1") + .setCell(familyId, "q2", "val2")); - testEnvRule.env().getDataClient().checkAndMutateRowAsync( - ConditionalRowMutation.create(tableId, rowKey) - .condition(FILTERS.qualifier().exactMatch("q1")) - .then(Mutation.create().setCell(familyId, "q3", "q1")) - ).get(1, TimeUnit.MINUTES); + testEnvRule + .env() + .getDataClient() + .checkAndMutateRowAsync( + ConditionalRowMutation.create(tableId, rowKey) + .condition(FILTERS.qualifier().exactMatch("q1")) + .then(Mutation.create().setCell(familyId, "q3", "q1"))) + .get(1, TimeUnit.MINUTES); - Row row = testEnvRule.env().getDataClient().readRowsCallable().first().call( - Query.create(tableId).rowKey(rowKey) - ); + Row row = + testEnvRule + .env() + .getDataClient() + .readRowsCallable() + .first() + .call(Query.create(tableId).rowKey(rowKey)); assertThat(row.getCells()).hasSize(3); assertThat(row.getCells().get(2).getValue()).isEqualTo(ByteString.copyFromUtf8("q1")); diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/MutateRowIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/MutateRowIT.java index 5cad5a576187..4fdfef18e665 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/MutateRowIT.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/MutateRowIT.java @@ -30,31 +30,38 @@ @RunWith(JUnit4.class) public class MutateRowIT { - @ClassRule - public static TestEnvRule testEnvRule = new TestEnvRule(); + @ClassRule public static TestEnvRule testEnvRule = new TestEnvRule(); @Test public void test() throws Exception { String rowKey = testEnvRule.env().getRowPrefix() + "testA"; String familyId = testEnvRule.env().getFamilyId(); + testEnvRule + .env() + .getDataClient() + .mutateRowAsync( + RowMutation.create(testEnvRule.env().getTableName().getTable(), rowKey) + .setCell(familyId, "q", "myVal") + .setCell(familyId, "q2", "myVal2") + .setCell(familyId, "q3", "myVal3")) + .get(1, TimeUnit.MINUTES); - testEnvRule.env().getDataClient().mutateRowAsync( - RowMutation.create(testEnvRule.env().getTableName().getTable(), rowKey) - .setCell(familyId, "q", "myVal") - .setCell(familyId, "q2", "myVal2") - .setCell(familyId, "q3", "myVal3") - ).get(1, TimeUnit.MINUTES); + testEnvRule + .env() + .getDataClient() + .mutateRowAsync( + RowMutation.create(testEnvRule.env().getTableName().getTable(), rowKey) + .deleteCells(familyId, "q2")) + .get(1, TimeUnit.MINUTES); - testEnvRule.env().getDataClient().mutateRowAsync( - RowMutation.create(testEnvRule.env().getTableName().getTable(), rowKey) - .deleteCells(familyId, "q2") - ).get(1, TimeUnit.MINUTES); - - Row row = testEnvRule.env().getDataClient().readRowsCallable().first().call( - Query.create(testEnvRule.env().getTableName().getTable()) - .rowKey(rowKey) - ); + Row row = + testEnvRule + .env() + .getDataClient() + .readRowsCallable() + .first() + .call(Query.create(testEnvRule.env().getTableName().getTable()).rowKey(rowKey)); assertThat(row.getCells()).hasSize(2); assertThat(row.getCells().get(0).getValue()).isEqualTo(ByteString.copyFromUtf8("myVal")); diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadModifyWriteIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadModifyWriteIT.java index b682ce439413..296f43e2ddbe 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadModifyWriteIT.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadModifyWriteIT.java @@ -20,6 +20,7 @@ import com.google.cloud.bigtable.data.v2.it.env.TestEnvRule; import com.google.cloud.bigtable.data.v2.models.ReadModifyWriteRow; import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.protobuf.ByteString; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -30,8 +31,7 @@ @RunWith(JUnit4.class) public class ReadModifyWriteIT { - @ClassRule - public static TestEnvRule testEnvRule = new TestEnvRule(); + @ClassRule public static TestEnvRule testEnvRule = new TestEnvRule(); @Test public void test() throws InterruptedException, ExecutionException, TimeoutException { @@ -39,11 +39,15 @@ public void test() throws InterruptedException, ExecutionException, TimeoutExcep String family = testEnvRule.env().getFamilyId(); String rowKey = testEnvRule.env().getRowPrefix(); - Row row = testEnvRule.env().getDataClient().readModifyWriteRowAsync( - ReadModifyWriteRow.create(tableId, rowKey) - .append(family, "q1", "a") - .increment(family, "q2", 3) - ).get(1, TimeUnit.MINUTES); + Row row = + testEnvRule + .env() + .getDataClient() + .readModifyWriteRowAsync( + ReadModifyWriteRow.create(tableId, rowKey) + .append(family, "q1", "a") + .increment(family, "q2", 3)) + .get(1, TimeUnit.MINUTES); assertThat(row.getCells()).hasSize(2); assertThat(row.getCells().get(0).getValue()).isEqualTo(ByteString.copyFromUtf8("a")); diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/SampleRowsIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/SampleRowsIT.java index 0f2c58e9c296..352c2385b5bc 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/SampleRowsIT.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/SampleRowsIT.java @@ -36,8 +36,7 @@ @RunWith(JUnit4.class) public class SampleRowsIT { - @ClassRule - public static TestEnvRule testEnvRule = new TestEnvRule(); + @ClassRule public static TestEnvRule testEnvRule = new TestEnvRule(); @Test public void test() throws InterruptedException, ExecutionException, TimeoutException { @@ -46,17 +45,18 @@ public void test() throws InterruptedException, ExecutionException, TimeoutExcep // Create some data so that sample row keys has something to show List> futures = Lists.newArrayList(); for (int i = 0; i < 10; i++) { - ApiFuture future = client.mutateRowAsync( - RowMutation.create(testEnvRule.env().getTableName().getTable(), - testEnvRule.env().getRowPrefix() + "-" + i) - .setCell(testEnvRule.env().getFamilyId(), "", "value") - ); + ApiFuture future = + client.mutateRowAsync( + RowMutation.create( + testEnvRule.env().getTableName().getTable(), + testEnvRule.env().getRowPrefix() + "-" + i) + .setCell(testEnvRule.env().getFamilyId(), "", "value")); futures.add(future); } ApiFutures.allAsList(futures).get(1, TimeUnit.MINUTES); - ApiFuture> future = client - .sampleRowKeysAsync(testEnvRule.env().getTableName().getTable()); + ApiFuture> future = + client.sampleRowKeysAsync(testEnvRule.env().getTableName().getTable()); List results = future.get(1, TimeUnit.MINUTES); diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java index 3cfc8f07f39f..0456246e891b 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java @@ -43,9 +43,7 @@ import java.util.logging.Level; import java.util.logging.Logger; -/** - * Java wrapper around the gcloud bigtable emulator. - */ +/** Java wrapper around the gcloud bigtable emulator. */ // TODO(igorbernstein): Clean this up and externalize this in a separate artifact // TODO(igorbernstein): Stop depending on gcloud for the binary, instead wrap it in a jar. class Emulator { @@ -137,7 +135,7 @@ BigtableTableAdminClient getTableAdminClient() { return tableAdminClient; } - // + // private static int getAvailablePort() { try (ServerSocket serverSocket = new ServerSocket(0)) { return serverSocket.getLocalPort(); @@ -177,32 +175,35 @@ private ManagedChannel createChannel(int port) { private static void pipeStreamToLog(final InputStream stream, final Level level) { final BufferedReader reader = new BufferedReader(new InputStreamReader(stream)); - Thread thread = new Thread(new Runnable() { - @Override - public void run() { - try { - String line; - while ((line = reader.readLine()) != null) - LOGGER.log(level, line); - } catch (IOException e) { - LOGGER.log(Level.WARNING, "Failed to read process stream", e); - } - } - }); + Thread thread = + new Thread( + new Runnable() { + @Override + public void run() { + try { + String line; + while ((line = reader.readLine()) != null) LOGGER.log(level, line); + } catch (IOException e) { + LOGGER.log(Level.WARNING, "Failed to read process stream", e); + } + } + }); thread.setDaemon(true); thread.start(); } private static Future bufferOutput(final InputStream stream) { - FutureTask task = new FutureTask<>(new Callable() { - @Override - public String call() throws Exception { - return CharStreams.toString(new InputStreamReader(stream)); - } - }); + FutureTask task = + new FutureTask<>( + new Callable() { + @Override + public String call() throws Exception { + return CharStreams.toString(new InputStreamReader(stream)); + } + }); task.run(); return task; } - // + // } From 1f212c9a20220981a83756b441dcc3c20d2926fb Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Wed, 7 Mar 2018 22:56:58 -0500 Subject: [PATCH 03/15] wip --- TESTING.md | 19 ++++++ .../bigtable/data/v2/it/env/Emulator.java | 63 ++++++++++--------- .../bigtable/data/v2/it/env/EmulatorEnv.java | 2 +- .../bigtable/data/v2/it/env/ProdEnv.java | 4 -- .../bigtable/data/v2/it/env/TestEnv.java | 7 ++- .../bigtable/data/v2/it/env/TestEnvRule.java | 14 +++++ 6 files changed, 73 insertions(+), 36 deletions(-) diff --git a/TESTING.md b/TESTING.md index 5b77b99ac43e..afa04b4d8a54 100644 --- a/TESTING.md +++ b/TESTING.md @@ -3,6 +3,7 @@ This library provides tools to help write tests for code that uses the following google-cloud services: - [BigQuery](#testing-code-that-uses-bigquery) +- [Bigtable](#testing-code-that-uses-bigtable) - [Compute](#testing-code-that-uses-compute) - [Datastore](#testing-code-that-uses-datastore) - [DNS](#testing-code-that-uses-dns) @@ -41,6 +42,24 @@ Here is an example that clears the dataset created in Step 3. RemoteBigQueryHelper.forceDelete(bigquery, dataset); ``` +### Testing code that uses Bigtable + +Bigtable integration tests can either be run against an emulator or a real Bigtable table. The +target environment can be selected via the `bigtable.env` system property. By default it is set to +`emulator` and the other option is `prod`. + +To use the `emulator` environment, please install the gcloud sdk and use it to install the +`cbtemulator` via `gcloud components install bigtable`. + +To use the `prod` environment: +1. Create a table with a column family named `cf`. +2. Download the [JSON service account credentials file][create-service-account] from the Google + Developer's Console. +3. Set the environment variable `GOOGLE_APPLICATION_CREDENTIALS` to the path of the credentials file +4. Set the system properties `bigtable.project`, `bigtable.instance` and `bigtable.table` to the + table you created earlier. + + ### Testing code that uses Compute Currently, there isn't an emulator for Google Compute, so an alternative is to create a test diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java index 0456246e891b..cc3e076f25ec 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java @@ -34,8 +34,11 @@ import java.io.InputStreamReader; import java.net.ServerSocket; import java.net.Socket; +import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.FutureTask; import java.util.concurrent.TimeUnit; @@ -49,7 +52,7 @@ class Emulator { private static final Logger LOGGER = Logger.getLogger(Emulator.class.getName()); - private final String executable; + private final Path executable; private Process process; private ManagedChannel channel; private BigtableTableAdminClient tableAdminClient; @@ -58,35 +61,20 @@ class Emulator { private static final InstanceName INSTANCE_NAME = InstanceName.of("fake-project", "fake-instance"); + // Use the gcloud installed emulator static Emulator createGCloud() { - final Process p; + final Path gcloudSdkPath; try { - p = Runtime.getRuntime().exec("gcloud info --format=value(installation.sdk_root)"); - pipeStreamToLog(p.getErrorStream(), Level.WARNING); - } catch (IOException e) { - throw new RuntimeException("Failed to run gcloud info", e); - } - - String sdkRoot; - try { - sdkRoot = bufferOutput(p.getInputStream()).get(1, TimeUnit.MINUTES).trim(); + gcloudSdkPath = getGcloudSdkPath(); } catch (Exception e) { - throw new RuntimeException("Failed to get gcloud sdk install path", e); - } - - try { - if (p.waitFor() != 0) { - throw new RuntimeException("Failed to get sdk root, is gcloud sdk installed?"); - } - } catch (InterruptedException e) { - throw new RuntimeException("Interrupted invoking gcloud", e); + throw new RuntimeException("Failed to get the gcloud SDK path. Is it installed?", e); } - String emulatorPath = - Paths.get(sdkRoot, "platform", "bigtable-emulator", "cbtemulator").toString(); + Path emulatorPath = + gcloudSdkPath.resolve(Paths.get("platform", "bigtable-emulator", "cbtemulator")); - if (!new File(emulatorPath).exists()) { + if (!Files.exists(emulatorPath)) { throw new RuntimeException( "cbtemulator is not installed, please install with `gcloud components install bigtable`"); } @@ -94,11 +82,11 @@ static Emulator createGCloud() { return new Emulator(emulatorPath); } - private Emulator(String executable) { + private Emulator(Path executable) { this.executable = executable; } - void start() throws InterruptedException, IOException, TimeoutException { + void start() throws Exception { int availablePort = getAvailablePort(); process = Runtime.getRuntime().exec(executable + " -port " + "" + availablePort); @@ -120,11 +108,14 @@ void start() throws InterruptedException, IOException, TimeoutException { } void stop() throws Exception { - dataClient.close(); - tableAdminClient.close(); - channel.shutdownNow(); - channel.awaitTermination(1, TimeUnit.MINUTES); - process.destroy(); + try { + dataClient.close(); + tableAdminClient.close(); + channel.shutdownNow(); + channel.awaitTermination(1, TimeUnit.MINUTES); + } finally { + process.destroy(); + } } BigtableDataClient getDataClient() { @@ -136,6 +127,18 @@ BigtableTableAdminClient getTableAdminClient() { } // + private static Path getGcloudSdkPath() throws Exception { + Process p = Runtime.getRuntime().exec("gcloud info --format=value(installation.sdk_root)"); + pipeStreamToLog(p.getErrorStream(), Level.WARNING); + + String sdkRoot = bufferOutput(p.getInputStream()).get(1, TimeUnit.MINUTES).trim(); + + if (p.waitFor() != 0) { + throw new RuntimeException("Failed to get sdk root, is gcloud sdk installed?"); + } + return Paths.get(sdkRoot); + } + private static int getAvailablePort() { try (ServerSocket serverSocket = new ServerSocket(0)) { return serverSocket.getLocalPort(); diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/EmulatorEnv.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/EmulatorEnv.java index b4a4fdcf7b64..cf16d83909db 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/EmulatorEnv.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/EmulatorEnv.java @@ -33,7 +33,7 @@ public class EmulatorEnv implements TestEnv { private Emulator emulator; @Override - public void start() throws IOException, InterruptedException, TimeoutException { + public void start() throws Exception { emulator = Emulator.createGCloud(); emulator.start(); diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/ProdEnv.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/ProdEnv.java index 9fa7ffc59024..98952f1fb36d 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/ProdEnv.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/ProdEnv.java @@ -41,10 +41,6 @@ *
  • {@code bigtable.instance} *
  • {@code bigtable.table} * - * - * Expects the instance to exist with a table created: cbt -project igorbernstein-dev -instance - * integration-tests createtable my-table cbt -project igorbernstein-dev -instance integration-tests - * createfamily my-table cf */ public class ProdEnv implements TestEnv { private static final String PROJECT_PROPERTY_NAME = "bigtable.project"; diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/TestEnv.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/TestEnv.java index 790849ecdeae..ccc4eb7336fe 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/TestEnv.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/TestEnv.java @@ -20,8 +20,13 @@ import java.io.IOException; import java.util.concurrent.TimeoutException; +/** + * Defines the interface of a target environment. + * + *

    This allows for integration tests to run against either production or an emulator. + */ public interface TestEnv { - void start() throws IOException, InterruptedException, TimeoutException; + void start() throws Exception; void stop() throws Exception; diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/TestEnvRule.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/TestEnvRule.java index 40b4fad8a6fd..25c505efe26e 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/TestEnvRule.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/TestEnvRule.java @@ -19,6 +19,20 @@ import java.util.logging.Logger; import org.junit.rules.ExternalResource; +/** + * Simple JUnit rule to start and stop the target test environment. + * + *

    The environment can be specified via the system property {@code bigtable.env}. The choices + * are: + * + *

      + *
    • {@code emulator}: uses the cbtemulator component that can be installed by gcloud + *
    • {@code prod}: uses a pre-existing production table. The target table is defined using + * system properties listed in {@link ProdEnv} and application default credentials + *
    + * + *

    By default, {@code emulator} will be used + */ public class TestEnvRule extends ExternalResource { private static final Logger LOGGER = Logger.getLogger(TestEnvRule.class.getName()); From 1dac3add9d80a75c23b11b02013d411a285b82ac Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Thu, 15 Mar 2018 18:16:35 -0400 Subject: [PATCH 04/15] tweaks --- .../com/google/cloud/bigtable/data/v2/it/SampleRowsIT.java | 3 +-- .../com/google/cloud/bigtable/data/v2/it/env/Emulator.java | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/SampleRowsIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/SampleRowsIT.java index 352c2385b5bc..e77fcdd98a16 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/SampleRowsIT.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/SampleRowsIT.java @@ -15,7 +15,7 @@ */ package com.google.cloud.bigtable.data.v2.it; -import static com.google.common.truth.Truth.*; +import static com.google.common.truth.Truth.assertThat; import com.google.api.core.ApiFuture; import com.google.api.core.ApiFutures; @@ -24,7 +24,6 @@ import com.google.cloud.bigtable.data.v2.models.KeyOffset; import com.google.cloud.bigtable.data.v2.models.RowMutation; import com.google.common.collect.Lists; -import com.google.common.truth.Truth; import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java index cc3e076f25ec..5e59bf63429c 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java @@ -160,6 +160,8 @@ private void waitForPort(int port) throws InterruptedException, TimeoutException } private ManagedChannel createChannel(int port) { + // NOTE: usePlaintext is currently @ExperimentalAPI. In grpc 1.11 it be become parameterless. + // In 1.12 it should be stable. See https://github.com/grpc/grpc-java/issues/1772 for discussion return ManagedChannelBuilder.forAddress("localhost", port) .usePlaintext(true) .maxInboundMessageSize(256 * 1024 * 1024) From c6d0e785e167410b7848b3a2aff842470ecf6eb4 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Thu, 15 Mar 2018 18:28:12 -0400 Subject: [PATCH 05/15] english --- .../java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java index 5e59bf63429c..d9b06312afd9 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java @@ -160,7 +160,7 @@ private void waitForPort(int port) throws InterruptedException, TimeoutException } private ManagedChannel createChannel(int port) { - // NOTE: usePlaintext is currently @ExperimentalAPI. In grpc 1.11 it be become parameterless. + // NOTE: usePlaintext is currently @ExperimentalAPI. In grpc 1.11 it be became parameterless. // In 1.12 it should be stable. See https://github.com/grpc/grpc-java/issues/1772 for discussion return ManagedChannelBuilder.forAddress("localhost", port) .usePlaintext(true) From 9efb094d496ece18e196731075e583ba9c677268 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Mon, 19 Mar 2018 21:06:45 -0400 Subject: [PATCH 06/15] add shutdown hook for the emulator --- .../cloud/bigtable/data/v2/it/env/Emulator.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java index d9b06312afd9..2c700961e660 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java @@ -54,6 +54,7 @@ class Emulator { private final Path executable; private Process process; + private boolean isStopped = true; private ManagedChannel channel; private BigtableTableAdminClient tableAdminClient; private BigtableDataClient dataClient; @@ -92,6 +93,7 @@ void start() throws Exception { process = Runtime.getRuntime().exec(executable + " -port " + "" + availablePort); pipeStreamToLog(process.getInputStream(), Level.INFO); pipeStreamToLog(process.getErrorStream(), Level.WARNING); + isStopped = false; waitForPort(availablePort); @@ -105,6 +107,16 @@ void start() throws Exception { BigtableDataClient.create( configureClient(BigtableDataSettings.newBuilder().setInstanceName(INSTANCE_NAME)) .build()); + + Runtime.getRuntime().addShutdownHook(new Thread() { + @Override + public void run() { + if (!isStopped) { + isStopped = true; + process.destroy(); + } + } + }); } void stop() throws Exception { @@ -114,6 +126,7 @@ void stop() throws Exception { channel.shutdownNow(); channel.awaitTermination(1, TimeUnit.MINUTES); } finally { + isStopped = true; process.destroy(); } } From 3859fee3d86cecaa9452147ae9203a206ccc2862 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Mon, 19 Mar 2018 21:06:58 -0400 Subject: [PATCH 07/15] run integration tests in parallel --- google-cloud-bigtable/pom.xml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/google-cloud-bigtable/pom.xml b/google-cloud-bigtable/pom.xml index 2af77c7773a4..383b8dc08da1 100644 --- a/google-cloud-bigtable/pom.xml +++ b/google-cloud-bigtable/pom.xml @@ -127,6 +127,24 @@ + + org.apache.maven.plugins + maven-failsafe-plugin + 2.19.1 + + + + integration-test + verify + + + + + classes + true + 2 + + org.apache.maven.plugins maven-javadoc-plugin From ab1af8171c675be3dc23fa4cd70a9307f752a374 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Mon, 19 Mar 2018 21:07:21 -0400 Subject: [PATCH 08/15] use single property for target prod table --- .../cloud/bigtable/data/v2/it/env/ProdEnv.java | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/ProdEnv.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/ProdEnv.java index 98952f1fb36d..54caa4b67c19 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/ProdEnv.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/ProdEnv.java @@ -43,8 +43,6 @@ * */ public class ProdEnv implements TestEnv { - private static final String PROJECT_PROPERTY_NAME = "bigtable.project"; - private static final String INSTANCE_PROPERTY_NAME = "bigtable.instance"; private static final String TABLE_PROPERTY_NAME = "bigtable.table"; private TableName tableName; @@ -54,11 +52,10 @@ public class ProdEnv implements TestEnv { private BigtableDataClient dataClient; static ProdEnv fromSystemProperties() { - String projectId = getRequiredProperty(PROJECT_PROPERTY_NAME); - String instanceId = getRequiredProperty(INSTANCE_PROPERTY_NAME); - String tableId = getRequiredProperty(TABLE_PROPERTY_NAME); + String tableNameStr = getRequiredProperty(TABLE_PROPERTY_NAME); + TableName tableName = TableName.parse(tableNameStr); - return new ProdEnv(TableName.of(projectId, instanceId, tableId)); + return new ProdEnv(tableName); } ProdEnv(TableName tableName) { @@ -67,13 +64,8 @@ static ProdEnv fromSystemProperties() { @Override public void start() throws IOException { - String projectId = getRequiredProperty(PROJECT_PROPERTY_NAME); - String instanceId = getRequiredProperty(INSTANCE_PROPERTY_NAME); - String tableId = getRequiredProperty(TABLE_PROPERTY_NAME); - - tableName = TableName.of(projectId, instanceId, tableId); rowPrefix = UUID.randomUUID() + "-"; - dataClient = BigtableDataClient.create(InstanceName.of(projectId, instanceId)); + dataClient = BigtableDataClient.create(InstanceName.of(tableName.getProject(), tableName.getInstance())); } @Override From fadc70df383d4d63fac9d507a20118ec21397681 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Mon, 19 Mar 2018 21:07:30 -0400 Subject: [PATCH 09/15] add script to setup the table --- .../scripts/setup-test-table.sh | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100755 google-cloud-bigtable/scripts/setup-test-table.sh diff --git a/google-cloud-bigtable/scripts/setup-test-table.sh b/google-cloud-bigtable/scripts/setup-test-table.sh new file mode 100755 index 000000000000..e405f084576a --- /dev/null +++ b/google-cloud-bigtable/scripts/setup-test-table.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +# Setup a table to use for integration tests. + +set -x + +# Format: projects//instances//tables/ +TABLE_NAME=$1 + +if [[ ${TABLE_NAME} =~ projects\/([^/]+)\/instances\/([^/]+)\/tables\/([^/]+) ]]; then + PROJECT_ID=${BASH_REMATCH[1]} + INSTANCE_ID=${BASH_REMATCH[2]} + TABLE_ID=${BASH_REMATCH[3]} +else + echo "Invalid table name: $TABLE_NAME" 1>&2 + exit 1 +fi + +cbt -project $PROJECT_ID -instance $INSTANCE_ID createtable $TABLE_ID +cbt -project $PROJECT_ID -instance $INSTANCE_ID createfamily $TABLE_ID cf +cbt -project $PROJECT_ID -instance $INSTANCE_ID setgcpolicy $TABLE_ID cf maxversions=1 From 603fa39a48ce93856f1534a5d202aac7bb918df4 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Mon, 19 Mar 2018 21:18:59 -0400 Subject: [PATCH 10/15] add bigtable integration tests to circleci --- .circleci/config.yml | 13 +++++++++++++ utilities/verify_single_it.sh | 3 ++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4eb1d2231cc6..53288dd4c878 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -79,6 +79,19 @@ jobs: - run: name: Run integration tests for google-cloud-bigquery command: ./utilities/verify_single_it.sh google-cloud-bigquery + + bigtable_it: + working_directory: ~/googleapis + <<: *anchor_docker + <<: *anchor_auth_vars + steps: + - checkout + - run: + <<: *anchor_run_decrypt + - run: + name: Run integration tests for google-cloud-bigquery + command: ./utilities/verify_single_it.sh google-cloud-bigtable -Dbigtable.env=prod -Dbigtable.table=projects/gcloud-devel/instances/google-cloud-bigtable/tables/integration-tests + compute_it: working_directory: ~/googleapis <<: *anchor_docker diff --git a/utilities/verify_single_it.sh b/utilities/verify_single_it.sh index 8437edf60b46..83a9e3f746c9 100755 --- a/utilities/verify_single_it.sh +++ b/utilities/verify_single_it.sh @@ -5,6 +5,7 @@ set -e MODULE=$1 +ARGS=${@:2:99} if [ -z $MODULE ]; then echo "First arg (module) not provided, so we're exiting." @@ -26,4 +27,4 @@ echo "----- building and installing shared modules -----" mvn -B -pl google-cloud-core,google-cloud-core-http,google-cloud-core-grpc,google-cloud-storage,google-cloud-pubsub install -DskipTests echo "----- running integration tests -----" -mvn -B -pl $MODULE -DtrimStackTrace=false -fae verify +mvn -B -pl $MODULE -DtrimStackTrace=false -fae verify ${ARGS} From c81396c319420afa99cc8c2d5d66fe68bd4dfafd Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Mon, 19 Mar 2018 21:23:18 -0400 Subject: [PATCH 11/15] update testing doc --- TESTING.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/TESTING.md b/TESTING.md index afa04b4d8a54..8b26b4ced9ad 100644 --- a/TESTING.md +++ b/TESTING.md @@ -52,12 +52,17 @@ To use the `emulator` environment, please install the gcloud sdk and use it to i `cbtemulator` via `gcloud components install bigtable`. To use the `prod` environment: -1. Create a table with a column family named `cf`. +1. Setup the target table using `google-cloud-bigtable/scripts/setup-test-table.sh` 2. Download the [JSON service account credentials file][create-service-account] from the Google Developer's Console. 3. Set the environment variable `GOOGLE_APPLICATION_CREDENTIALS` to the path of the credentials file -4. Set the system properties `bigtable.project`, `bigtable.instance` and `bigtable.table` to the - table you created earlier. +4. Set the system property `bigtable.env=prod` and `bigtable.table` to the full table name you + created earlier. Example: + ```shell + mvn verify -am -pl google-cloud-bigtable \ + -Dbigtable.env=prod \ + -Dbigtable.table=projects/my-project/instances/my-instance/tables/my-table + ``` ### Testing code that uses Compute From b8eb76976ec0fbe5fd96c9c602b307f90089d2f2 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Mon, 19 Mar 2018 21:23:38 -0400 Subject: [PATCH 12/15] reformat --- .../bigtable/data/v2/it/env/Emulator.java | 20 ++++++++++--------- .../bigtable/data/v2/it/env/ProdEnv.java | 3 ++- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java index 2c700961e660..c784ecde668d 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java @@ -108,15 +108,17 @@ void start() throws Exception { configureClient(BigtableDataSettings.newBuilder().setInstanceName(INSTANCE_NAME)) .build()); - Runtime.getRuntime().addShutdownHook(new Thread() { - @Override - public void run() { - if (!isStopped) { - isStopped = true; - process.destroy(); - } - } - }); + Runtime.getRuntime() + .addShutdownHook( + new Thread() { + @Override + public void run() { + if (!isStopped) { + isStopped = true; + process.destroy(); + } + } + }); } void stop() throws Exception { diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/ProdEnv.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/ProdEnv.java index 54caa4b67c19..19facfb9f74e 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/ProdEnv.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/ProdEnv.java @@ -65,7 +65,8 @@ static ProdEnv fromSystemProperties() { @Override public void start() throws IOException { rowPrefix = UUID.randomUUID() + "-"; - dataClient = BigtableDataClient.create(InstanceName.of(tableName.getProject(), tableName.getInstance())); + dataClient = + BigtableDataClient.create(InstanceName.of(tableName.getProject(), tableName.getInstance())); } @Override From acf2c69b57d537363c9f093d0f1de9cb8b202a87 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Mon, 19 Mar 2018 21:35:49 -0400 Subject: [PATCH 13/15] circleci fixes --- .circleci/config.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 53288dd4c878..bb1c01f41222 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -89,7 +89,7 @@ jobs: - run: <<: *anchor_run_decrypt - run: - name: Run integration tests for google-cloud-bigquery + name: Run integration tests for google-cloud-bigtable command: ./utilities/verify_single_it.sh google-cloud-bigtable -Dbigtable.env=prod -Dbigtable.table=projects/gcloud-devel/instances/google-cloud-bigtable/tables/integration-tests compute_it: @@ -233,6 +233,10 @@ workflows: filters: branches: only: master + - bigtable_it: + filters: + branches: + only: master - compute_it: filters: branches: From d75e0fa6398e88ec11defedf5a38dfff6948ee2c Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Tue, 20 Mar 2018 14:47:08 -0400 Subject: [PATCH 14/15] update InstanceNames to the local copy --- .../java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java | 2 +- .../com/google/cloud/bigtable/data/v2/it/env/EmulatorEnv.java | 2 +- .../java/com/google/cloud/bigtable/data/v2/it/env/ProdEnv.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java index c784ecde668d..fed53116cffd 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java @@ -19,7 +19,7 @@ import com.google.api.gax.grpc.GrpcTransportChannel; import com.google.api.gax.rpc.ClientSettings; import com.google.api.gax.rpc.FixedTransportChannelProvider; -import com.google.bigtable.admin.v2.InstanceName; +import com.google.cloud.bigtable.data.v2.models.InstanceName; import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; import com.google.cloud.bigtable.data.v2.BigtableDataClient; diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/EmulatorEnv.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/EmulatorEnv.java index cf16d83909db..4181adc16ce1 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/EmulatorEnv.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/EmulatorEnv.java @@ -17,7 +17,7 @@ import com.google.bigtable.admin.v2.ColumnFamily; import com.google.bigtable.admin.v2.CreateTableRequest; -import com.google.bigtable.admin.v2.InstanceName; +import com.google.cloud.bigtable.data.v2.models.InstanceName; import com.google.bigtable.admin.v2.Table; import com.google.bigtable.v2.TableName; import com.google.cloud.bigtable.data.v2.BigtableDataClient; diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/ProdEnv.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/ProdEnv.java index 19facfb9f74e..cdd35c9a9b1b 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/ProdEnv.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/ProdEnv.java @@ -18,7 +18,7 @@ import com.google.api.core.ApiFuture; import com.google.api.core.ApiFutures; import com.google.api.gax.rpc.ServerStream; -import com.google.bigtable.admin.v2.InstanceName; +import com.google.cloud.bigtable.data.v2.models.InstanceName; import com.google.bigtable.v2.TableName; import com.google.cloud.bigtable.data.v2.BigtableDataClient; import com.google.cloud.bigtable.data.v2.models.Query; From fc30c49c368f714a01b52f37d5a6a43d04891002 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Tue, 20 Mar 2018 14:53:12 -0400 Subject: [PATCH 15/15] address feedback & reformat --- TESTING.md | 2 +- .../scripts/setup-test-table.sh | 2 +- .../bigtable/data/v2/models/InstanceName.java | 25 ++++++------------- .../bigtable/data/v2/it/env/Emulator.java | 4 ++- .../bigtable/data/v2/it/env/TestEnvRule.java | 10 +++----- 5 files changed, 16 insertions(+), 27 deletions(-) diff --git a/TESTING.md b/TESTING.md index 8b26b4ced9ad..f5a8242b6cae 100644 --- a/TESTING.md +++ b/TESTING.md @@ -52,7 +52,7 @@ To use the `emulator` environment, please install the gcloud sdk and use it to i `cbtemulator` via `gcloud components install bigtable`. To use the `prod` environment: -1. Setup the target table using `google-cloud-bigtable/scripts/setup-test-table.sh` +1. Set up the target table using `google-cloud-bigtable/scripts/setup-test-table.sh` 2. Download the [JSON service account credentials file][create-service-account] from the Google Developer's Console. 3. Set the environment variable `GOOGLE_APPLICATION_CREDENTIALS` to the path of the credentials file diff --git a/google-cloud-bigtable/scripts/setup-test-table.sh b/google-cloud-bigtable/scripts/setup-test-table.sh index e405f084576a..1ebedb0f6007 100755 --- a/google-cloud-bigtable/scripts/setup-test-table.sh +++ b/google-cloud-bigtable/scripts/setup-test-table.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Setup a table to use for integration tests. +# Set up a table to use for integration tests. set -x diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/InstanceName.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/InstanceName.java index dc4247ab4e96..8465d5eb8e92 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/InstanceName.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/InstanceName.java @@ -58,18 +58,11 @@ private InstanceName(Builder builder) { } public static InstanceName of(String project, String instance) { - return newBuilder() - .setProject(project) - .setInstance(instance) - .build(); + return newBuilder().setProject(project).setInstance(instance).build(); } public static String format(String project, String instance) { - return newBuilder() - .setProject(project) - .setInstance(instance) - .build() - .toString(); + return newBuilder().setProject(project).setInstance(instance).build().toString(); } public static InstanceName parse(String formattedString) { @@ -77,7 +70,8 @@ public static InstanceName parse(String formattedString) { return null; } Map matchMap = - PATH_TEMPLATE.validatedMatch(formattedString, "InstanceName.parse: formattedString not in valid format"); + PATH_TEMPLATE.validatedMatch( + formattedString, "InstanceName.parse: formattedString not in valid format"); return of(matchMap.get("project"), matchMap.get("instance")); } @@ -123,9 +117,7 @@ public String getFieldValue(String fieldName) { return getFieldValuesMap().get(fieldName); } - /** - * @deprecated This method is only present to satisfy the ResourceName interface. - */ + /** @deprecated This method is only present to satisfy the ResourceName interface. */ @Deprecated public ResourceNameType getType() { throw new UnsupportedOperationException("InstanceName.getType() not supported"); @@ -160,8 +152,7 @@ public Builder setInstance(String instance) { return this; } - private Builder() { - } + private Builder() {} private Builder(InstanceName instanceName) { project = instanceName.project; @@ -180,8 +171,7 @@ public boolean equals(Object o) { } if (o instanceof InstanceName) { InstanceName that = (InstanceName) o; - return (this.project.equals(that.project)) - && (this.instance.equals(that.instance)); + return (this.project.equals(that.project)) && (this.instance.equals(that.instance)); } return false; } @@ -196,4 +186,3 @@ public int hashCode() { return h; } } - diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java index fed53116cffd..476fe8974b51 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java @@ -202,7 +202,9 @@ private static void pipeStreamToLog(final InputStream stream, final Level level) public void run() { try { String line; - while ((line = reader.readLine()) != null) LOGGER.log(level, line); + while ((line = reader.readLine()) != null) { + LOGGER.log(level, line); + } } catch (IOException e) { LOGGER.log(Level.WARNING, "Failed to read process stream", e); } diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/TestEnvRule.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/TestEnvRule.java index 25c505efe26e..06f4c6d3ef54 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/TestEnvRule.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/TestEnvRule.java @@ -52,12 +52,10 @@ protected void before() throws Throwable { testEnv = ProdEnv.fromSystemProperties(); break; default: - throw new RuntimeException( - "Unknown env: " - + env - + ". Please set the system propert " - + ENV_PROPERTY - + " to either 'emulator' or 'prod'."); + throw new IllegalArgumentException( + String.format( + "Unknown env: %s. Please set the system property %s to either 'emulator' or 'prod'.", + env, ENV_PROPERTY)); } testEnv.start(); }