From c9371c93cf4760f004e7804926ffa1caf6c21e57 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Wed, 17 Dec 2025 14:44:36 -0500 Subject: [PATCH 1/6] test: Add picosecond test cases --- .../it/ITBigQueryStorageReadClientTest.java | 86 +++++--- .../it/ITBigQueryStorageWriteClientTest.java | 185 ++++++++++++------ .../bigquery/storage/v1/it/util/Helper.java | 49 +++-- .../v1/it/util/SimpleRowReaderArrow.java | 22 ++- 4 files changed, 241 insertions(+), 101 deletions(-) diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryStorageReadClientTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryStorageReadClientTest.java index ad827e930..0fa7c1add 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryStorageReadClientTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryStorageReadClientTest.java @@ -16,6 +16,8 @@ package com.google.cloud.bigquery.storage.v1.it; +import static com.google.cloud.bigquery.storage.v1.it.util.Helper.TIMESTAMP_COLUMN_NAME; +import static com.google.cloud.bigquery.storage.v1.it.util.Helper.TIMESTAMP_HIGHER_PRECISION_COLUMN_NAME; import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth.assertWithMessage; import static org.junit.Assert.assertArrayEquals; @@ -53,6 +55,7 @@ import com.google.cloud.bigquery.TableInfo; import com.google.cloud.bigquery.TimePartitioning; import com.google.cloud.bigquery.storage.v1.AppendRowsResponse; +import com.google.cloud.bigquery.storage.v1.ArrowSerializationOptions; import com.google.cloud.bigquery.storage.v1.BigQueryReadClient; import com.google.cloud.bigquery.storage.v1.BigQueryReadSettings; import com.google.cloud.bigquery.storage.v1.CreateReadSessionRequest; @@ -77,6 +80,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.MoreExecutors; import com.google.protobuf.Descriptors.DescriptorValidationException; +import com.google.protobuf.Int64Value; import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.sdk.OpenTelemetrySdk; @@ -123,10 +127,8 @@ public class ITBigQueryStorageReadClientTest { private static final String DATASET = RemoteBigQueryHelper.generateDatasetName(); private static final String DESCRIPTION = "BigQuery Storage Java client test dataset"; private static final String BQSTORAGE_TIMESTAMP_READ_TABLE = "bqstorage_timestamp_read"; - private static final int SHAKESPEARE_SAMPLE_ROW_COUNT = 164_656; private static final int SHAKESPEARE_SAMPELS_ROWS_MORE_THAN_100_WORDS = 1_333; - private static final int MAX_STREAM_COUNT = 1; private static BigQueryReadClient readClient; @@ -526,43 +528,48 @@ public static void beforeClass() private static void setupTimestampTable() throws DescriptorValidationException, IOException, InterruptedException { + // Schema to create a BQ table com.google.cloud.bigquery.Schema timestampSchema = com.google.cloud.bigquery.Schema.of( - Field.newBuilder("timestamp", StandardSQLTypeName.TIMESTAMP) + Field.newBuilder(TIMESTAMP_COLUMN_NAME, StandardSQLTypeName.TIMESTAMP) + .setMode(Mode.NULLABLE) + .build(), + Field.newBuilder(TIMESTAMP_HIGHER_PRECISION_COLUMN_NAME, StandardSQLTypeName.TIMESTAMP) + .setTimestampPrecision(12L) .setMode(Mode.NULLABLE) .build()); + // Create BQ table with timestamps + TableId tableId = TableId.of(DATASET, BQSTORAGE_TIMESTAMP_READ_TABLE); + bigquery.create(TableInfo.of(tableId, StandardTableDefinition.of(timestampSchema))); + + TableName parentTable = TableName.of(projectName, DATASET, BQSTORAGE_TIMESTAMP_READ_TABLE); + + // Define the BQStorage schema to write to TableSchema timestampTableSchema = TableSchema.newBuilder() .addFields( TableFieldSchema.newBuilder() - .setName("timestamp") + .setName(TIMESTAMP_COLUMN_NAME) + .setType(TableFieldSchema.Type.TIMESTAMP) + .setMode(TableFieldSchema.Mode.NULLABLE) + .build()) + .addFields( + TableFieldSchema.newBuilder() + .setName(TIMESTAMP_HIGHER_PRECISION_COLUMN_NAME) + .setTimestampPrecision(Int64Value.newBuilder().setValue(12).build()) .setType(TableFieldSchema.Type.TIMESTAMP) .setMode(TableFieldSchema.Mode.NULLABLE) .build()) - .build(); - - // Create table with Range fields. - TableId tableId = TableId.of(DATASET, BQSTORAGE_TIMESTAMP_READ_TABLE); - bigquery.create(TableInfo.of(tableId, StandardTableDefinition.of(timestampSchema))); - - TableName parentTable = TableName.of(projectName, DATASET, BQSTORAGE_TIMESTAMP_READ_TABLE); - RetrySettings retrySettings = - RetrySettings.newBuilder() - .setInitialRetryDelayDuration(Duration.ofMillis(500)) - .setRetryDelayMultiplier(1.1) - .setMaxAttempts(5) - .setMaxRetryDelayDuration(Duration.ofSeconds(10)) .build(); try (JsonStreamWriter writer = - JsonStreamWriter.newBuilder(parentTable.toString(), timestampTableSchema) - .setRetrySettings(retrySettings) - .build()) { + JsonStreamWriter.newBuilder(parentTable.toString(), timestampTableSchema).build()) { JSONArray data = new JSONArray(); - for (long timestampMicro : Helper.INPUT_TIMESTAMPS_MICROS) { + for (Object[] timestampData : Helper.INPUT_TIMESTAMPS) { JSONObject row = new JSONObject(); - row.put("timestamp", timestampMicro); + row.put(TIMESTAMP_COLUMN_NAME, timestampData[0]); + row.put(TIMESTAMP_HIGHER_PRECISION_COLUMN_NAME, timestampData[1]); data.put(row); } @@ -858,6 +865,7 @@ public void testRangeTypeWrite() } } + // Tests that inputs for micros and picos can be read properly via Arrow @Test public void timestamp_readArrow() throws IOException { String table = @@ -865,7 +873,21 @@ public void timestamp_readArrow() throws IOException { ReadSession session = readClient.createReadSession( parentProjectId, - ReadSession.newBuilder().setTable(table).setDataFormat(DataFormat.ARROW).build(), + ReadSession.newBuilder() + .setTable(table) + .setDataFormat(DataFormat.ARROW) + .setReadOptions( + TableReadOptions.newBuilder() + .setArrowSerializationOptions( + ArrowSerializationOptions.newBuilder() + // This serialization option only impacts columns that are type + // `TIMESTAMP_PICOS` and has no impact on other columns types + .setPicosTimestampPrecision( + ArrowSerializationOptions.PicosTimestampPrecision + .TIMESTAMP_PRECISION_PICOS) + .build()) + .build()) + .build(), MAX_STREAM_COUNT); assertEquals( String.format( @@ -896,22 +918,32 @@ public void timestamp_readArrow() throws IOException { reader.processRows( response.getArrowRecordBatch(), new SimpleRowReaderArrow.ArrowTimestampBatchConsumer( - Arrays.asList(Helper.INPUT_TIMESTAMPS_MICROS))); + Helper.EXPECTED_TIMESTAMPS_HIGHER_PRECISION_ISO_OUTPUT)); rowCount += response.getRowCount(); } - assertEquals(Helper.EXPECTED_TIMESTAMPS_MICROS.length, rowCount); + assertEquals(Helper.EXPECTED_TIMESTAMPS_HIGHER_PRECISION_ISO_OUTPUT.length, rowCount); } } + // Tests that inputs for micros and picos can be read properly via Avro @Test public void timestamp_readAvro() throws IOException { String table = BigQueryResource.formatTableResource(projectName, DATASET, BQSTORAGE_TIMESTAMP_READ_TABLE); List rows = Helper.readAllRows(readClient, parentProjectId, table, null); List timestamps = - rows.stream().map(x -> (Long) x.get("timestamp")).collect(Collectors.toList()); + rows.stream().map(x -> (Long) x.get(TIMESTAMP_COLUMN_NAME)).collect(Collectors.toList()); + List timestampHigherPrecision = + rows.stream() + .map(x -> x.get(TIMESTAMP_HIGHER_PRECISION_COLUMN_NAME).toString()) + .collect(Collectors.toList()); for (int i = 0; i < timestamps.size(); i++) { - assertEquals(Helper.EXPECTED_TIMESTAMPS_MICROS[i], timestamps.get(i)); + assertEquals(Helper.EXPECTED_TIMESTAMPS_HIGHER_PRECISION_ISO_OUTPUT[i][0], timestamps.get(i)); + } + for (int i = 0; i < timestampHigherPrecision.size(); i++) { + assertEquals( + Helper.EXPECTED_TIMESTAMPS_HIGHER_PRECISION_ISO_OUTPUT[i][1], + timestampHigherPrecision.get(i)); } } diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryStorageWriteClientTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryStorageWriteClientTest.java index ccb979fe2..ae182f710 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryStorageWriteClientTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryStorageWriteClientTest.java @@ -16,6 +16,9 @@ package com.google.cloud.bigquery.storage.v1.it; +import static com.google.cloud.bigquery.storage.v1.it.util.Helper.EXPECTED_TIMESTAMPS_HIGHER_PRECISION_ISO_OUTPUT; +import static com.google.cloud.bigquery.storage.v1.it.util.Helper.TIMESTAMP_COLUMN_NAME; +import static com.google.cloud.bigquery.storage.v1.it.util.Helper.TIMESTAMP_HIGHER_PRECISION_COLUMN_NAME; import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -25,6 +28,7 @@ import com.google.api.client.util.Sleeper; import com.google.api.core.ApiFuture; +import com.google.api.core.ApiFutures; import com.google.api.gax.retrying.RetrySettings; import com.google.api.gax.rpc.FixedHeaderProvider; import com.google.api.gax.rpc.HeaderProvider; @@ -45,11 +49,13 @@ import com.google.cloud.bigquery.storage.v1.it.util.Helper; import com.google.cloud.bigquery.testing.RemoteBigQueryHelper; import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.MoreExecutors; import com.google.protobuf.ByteString; import com.google.protobuf.DescriptorProtos.DescriptorProto; import com.google.protobuf.DescriptorProtos.FieldDescriptorProto; import com.google.protobuf.Descriptors; import com.google.protobuf.Descriptors.DescriptorValidationException; +import com.google.protobuf.Int64Value; import io.grpc.Status; import io.grpc.Status.Code; import java.io.ByteArrayOutputStream; @@ -116,6 +122,26 @@ public class ITBigQueryStorageWriteClientTest { private static final BufferAllocator allocator = new RootAllocator(); + // Arrow is a bit special in that timestamps are limited to nanoseconds precision. + // The data will be padded to fit into the higher precision columns. + public static final Object[][] INPUT_ARROW_WRITE_TIMESTAMPS = + new Object[][] { + {1735734896123456L /* 2025-01-01T12:34:56.123456Z */, 1735734896123456789L}, + {1580646896123456L /* 2020-02-02T12:34:56.123456Z */, 1580646896123456789L}, + {636467696123456L /* 1990-03-03T12:34:56.123456Z */, 636467696123456789L}, + {165846896123456L /* 1975-04-04T12:34:56.123456Z */, 165846896123456789L} + }; + + // Arrow's higher precision column is padded with extra 0's if configured to return + // ISO as output for any picosecond enabled column. + public static final Object[][] EXPECTED_ARROW_WRITE_TIMESTAMPS_ISO_OUTPUT = + new Object[][] { + {1735734896123456L /* 2025-01-01T12:34:56.123456Z */, "2025-01-01T12:34:56.123456789000Z"}, + {1580646896123456L /* 2020-02-02T12:34:56.123456Z */, "2020-02-02T12:34:56.123456789000Z"}, + {636467696123456L /* 1990-03-03T12:34:56.123456Z */, "1990-03-03T12:34:56.123456789000Z"}, + {165846896123456L /* 1975-04-04T12:34:56.123456Z */, "1975-04-04T12:34:56.123456789000Z"} + }; + public static class StringWithSecondsNanos { public String foo; public long seconds; @@ -2272,44 +2298,54 @@ public void testLargeRequest() throws IOException, InterruptedException, Executi } } + // Tests that inputs for micro and picos are able to use Arrow to write + // to BQ @Test public void timestamp_arrowWrite() throws IOException { - String timestampFieldName = "timestamp"; - com.google.cloud.bigquery.Schema tableSchema = - com.google.cloud.bigquery.Schema.of( - Field.newBuilder(timestampFieldName, StandardSQLTypeName.TIMESTAMP) - .setMode(Mode.REQUIRED) - .build()); - String tableName = "bqstorage_timestamp_write_arrow"; - TableId testTableId = TableId.of(DATASET, tableName); - bigquery.create( - TableInfo.of( - testTableId, StandardTableDefinition.newBuilder().setSchema(tableSchema).build())); + // Opt to create a new table to write to instead of re-using table to prevent + // the test from failing due to any issues with deleting data after test. + // Increases the test time duration, but would be more resilient to transient + // failures + createTimestampTable(tableName); + // Define the fields as Arrow types that are compatible with BQ Schema types List fields = ImmutableList.of( new org.apache.arrow.vector.types.pojo.Field( - timestampFieldName, + TIMESTAMP_COLUMN_NAME, FieldType.nullable( new ArrowType.Timestamp( org.apache.arrow.vector.types.TimeUnit.MICROSECOND, "UTC")), + null), + new org.apache.arrow.vector.types.pojo.Field( + TIMESTAMP_HIGHER_PRECISION_COLUMN_NAME, + FieldType.nullable( + new ArrowType.Timestamp( + org.apache.arrow.vector.types.TimeUnit.NANOSECOND, "UTC")), null)); - org.apache.arrow.vector.types.pojo.Schema schema = + org.apache.arrow.vector.types.pojo.Schema arrowSchema = new org.apache.arrow.vector.types.pojo.Schema(fields, null); + int numRows = INPUT_ARROW_WRITE_TIMESTAMPS.length; TableName parent = TableName.of(ServiceOptions.getDefaultProjectId(), DATASET, tableName); try (StreamWriter streamWriter = - StreamWriter.newBuilder(parent.toString() + "/_default").setWriterSchema(schema).build()) { - try (VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { + StreamWriter.newBuilder(parent.toString() + "/_default") + .setWriterSchema(arrowSchema) + .build()) { + try (VectorSchemaRoot root = VectorSchemaRoot.create(arrowSchema, allocator)) { TimeStampMicroTZVector timestampVector = - (TimeStampMicroTZVector) root.getVector(timestampFieldName); - timestampVector.allocateNew(Helper.INPUT_TIMESTAMPS_MICROS.length); - - for (int i = 0; i < Helper.INPUT_TIMESTAMPS_MICROS.length; i++) { - timestampVector.set(i, Helper.INPUT_TIMESTAMPS_MICROS[i]); + (TimeStampMicroTZVector) root.getVector(TIMESTAMP_COLUMN_NAME); + TimeStampNanoTZVector timestampHigherPrecisionVector = + (TimeStampNanoTZVector) root.getVector(TIMESTAMP_HIGHER_PRECISION_COLUMN_NAME); + timestampVector.allocateNew(numRows); + timestampHigherPrecisionVector.allocateNew(numRows); + + for (int i = 0; i < numRows; i++) { + timestampVector.set(i, (Long) INPUT_ARROW_WRITE_TIMESTAMPS[i][0]); + timestampHigherPrecisionVector.set(i, (Long) INPUT_ARROW_WRITE_TIMESTAMPS[i][1]); } - root.setRowCount(Helper.INPUT_TIMESTAMPS_MICROS.length); + root.setRowCount(numRows); CompressionCodec codec = NoCompressionCodec.Factory.INSTANCE.createCodec( @@ -2319,66 +2355,103 @@ public void timestamp_arrowWrite() throws IOException { org.apache.arrow.vector.ipc.message.ArrowRecordBatch batch = vectorUnloader.getRecordBatch(); // Asynchronous append. - streamWriter.append(batch, -1); + ApiFuture future = streamWriter.append(batch); + ApiFutures.addCallback( + future, new Helper.AppendCompleteCallback(), MoreExecutors.directExecutor()); } } - String table = - BigQueryResource.formatTableResource( - ServiceOptions.getDefaultProjectId(), DATASET, tableName); - List rows = Helper.readAllRows(readClient, parentProjectId, table, null); - List timestamps = - rows.stream().map(x -> (Long) x.get(timestampFieldName)).collect(Collectors.toList()); - assertEquals(timestamps.size(), Helper.EXPECTED_TIMESTAMPS_MICROS.length); - for (int i = 0; i < timestamps.size(); i++) { - assertEquals(timestamps.get(i), Helper.EXPECTED_TIMESTAMPS_MICROS[i]); - } + assertTimestamps(tableName, EXPECTED_ARROW_WRITE_TIMESTAMPS_ISO_OUTPUT); } + // Tests that inputs for micro and picos are able to converted to protobuf + // and written to BQ @Test public void timestamp_protobufWrite() throws IOException, DescriptorValidationException, InterruptedException { - String timestampFieldName = "timestamp"; - com.google.cloud.bigquery.Schema bqTableSchema = - com.google.cloud.bigquery.Schema.of( - Field.newBuilder(timestampFieldName, StandardSQLTypeName.TIMESTAMP) - .setMode(Mode.REQUIRED) - .build()); - - String tableName = "bqstorage_timestamp_write_avro"; - TableId testTableId = TableId.of(DATASET, tableName); - bigquery.create( - TableInfo.of( - testTableId, StandardTableDefinition.newBuilder().setSchema(bqTableSchema).build())); - - TableFieldSchema TEST_TIMESTAMP = + String tableName = "bqstorage_timestamp_write_protobuf"; + // Opt to create a new table to write to instead of re-using table to prevent + // the test from failing due to any issues with deleting data after test. + // Increases the test time duration, but would be more resilient to transient + // failures + createTimestampTable(tableName); + + // Define the table schema so that the automatic converter is able to + // determine how to convert from Json -> Protobuf + TableFieldSchema testTimestamp = TableFieldSchema.newBuilder() - .setName(timestampFieldName) + .setName(TIMESTAMP_COLUMN_NAME) .setType(TableFieldSchema.Type.TIMESTAMP) .setMode(TableFieldSchema.Mode.NULLABLE) .build(); - TableSchema tableSchema = TableSchema.newBuilder().addFields(TEST_TIMESTAMP).build(); + TableFieldSchema testTimestampHighPrecision = + TableFieldSchema.newBuilder() + .setName(TIMESTAMP_HIGHER_PRECISION_COLUMN_NAME) + .setTimestampPrecision(Int64Value.newBuilder().setValue(12).build()) + .setType(TableFieldSchema.Type.TIMESTAMP) + .setMode(TableFieldSchema.Mode.NULLABLE) + .build(); + TableSchema tableSchema = + TableSchema.newBuilder() + .addFields(testTimestamp) + .addFields(testTimestampHighPrecision) + .build(); TableName parent = TableName.of(ServiceOptions.getDefaultProjectId(), DATASET, tableName); try (JsonStreamWriter jsonStreamWriter = JsonStreamWriter.newBuilder(parent.toString(), tableSchema).build()) { - for (long timestampMicro : Helper.INPUT_TIMESTAMPS_MICROS) { - JSONArray jsonArray = new JSONArray(); + JSONArray jsonArray = new JSONArray(); + for (Object[] timestampData : Helper.INPUT_TIMESTAMPS) { JSONObject row = new JSONObject(); - row.put(timestampFieldName, timestampMicro); + row.put(TIMESTAMP_COLUMN_NAME, timestampData[0]); + row.put(TIMESTAMP_HIGHER_PRECISION_COLUMN_NAME, timestampData[1]); jsonArray.put(row); - jsonStreamWriter.append(jsonArray); } + ApiFuture future = jsonStreamWriter.append(jsonArray); + ApiFutures.addCallback( + future, new Helper.AppendCompleteCallback(), MoreExecutors.directExecutor()); } + assertTimestamps(tableName, EXPECTED_TIMESTAMPS_HIGHER_PRECISION_ISO_OUTPUT); + } + + private void createTimestampTable(String tableName) { + Schema bqTableSchema = + Schema.of( + Field.newBuilder(TIMESTAMP_COLUMN_NAME, StandardSQLTypeName.TIMESTAMP) + .setMode(Mode.NULLABLE) + .build(), + Field.newBuilder(TIMESTAMP_HIGHER_PRECISION_COLUMN_NAME, StandardSQLTypeName.TIMESTAMP) + .setMode(Mode.NULLABLE) + .setTimestampPrecision(12L) + .build()); + + TableId testTableId = TableId.of(DATASET, tableName); + bigquery.create( + TableInfo.of( + testTableId, StandardTableDefinition.newBuilder().setSchema(bqTableSchema).build())); + } + private void assertTimestamps(String tableName, Object[][] expected) throws IOException { String table = BigQueryResource.formatTableResource( ServiceOptions.getDefaultProjectId(), DATASET, tableName); + + // Read all the data as Avro GenericRecords List rows = Helper.readAllRows(readClient, parentProjectId, table, null); + + // Each timestamp response is expected to contain two fields: + // 1. Micros from timestamp as a Long and 2. ISO8601 instant with picos precision List timestamps = - rows.stream().map(x -> (Long) x.get(timestampFieldName)).collect(Collectors.toList()); - assertEquals(timestamps.size(), Helper.EXPECTED_TIMESTAMPS_MICROS.length); - for (int i = 0; i < timestamps.size(); i++) { - assertEquals(timestamps.get(i), Helper.EXPECTED_TIMESTAMPS_MICROS[i]); + rows.stream().map(x -> (Long) x.get(TIMESTAMP_COLUMN_NAME)).collect(Collectors.toList()); + List timestampHigherPrecision = + rows.stream() + .map(x -> x.get(TIMESTAMP_HIGHER_PRECISION_COLUMN_NAME).toString()) + .collect(Collectors.toList()); + + assertEquals(timestamps.size(), expected.length); + assertEquals(timestampHigherPrecision.size(), expected.length); + for (int i = 0; i < timestampHigherPrecision.size(); i++) { + assertEquals(timestamps.get(i), expected[i][0]); + assertEquals(timestampHigherPrecision.get(i), expected[i][1]); } } } diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/util/Helper.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/util/Helper.java index 26883f59f..05aa4e8d8 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/util/Helper.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/util/Helper.java @@ -23,6 +23,7 @@ import com.google.api.gax.rpc.ServerStream; import com.google.auth.oauth2.ServiceAccountCredentials; import com.google.cloud.bigquery.storage.v1.AppendRowsResponse; +import com.google.cloud.bigquery.storage.v1.AvroSerializationOptions; import com.google.cloud.bigquery.storage.v1.BigQueryReadClient; import com.google.cloud.bigquery.storage.v1.CreateReadSessionRequest; import com.google.cloud.bigquery.storage.v1.DataFormat; @@ -42,20 +43,27 @@ public class Helper { - public static final Long[] INPUT_TIMESTAMPS_MICROS = - new Long[] { - 1735734896123456L, // 2025-01-01T12:34:56.123456Z - 1580646896123456L, // 2020-02-02T12:34:56.123456Z - 636467696123456L, // 1990-03-03T12:34:56.123456Z - 165846896123456L // 1975-04-04T12:34:56.123456Z + public static final String TIMESTAMP_COLUMN_NAME = "timestamp"; + public static final String TIMESTAMP_HIGHER_PRECISION_COLUMN_NAME = "timestampHigherPrecision"; + + // Sample test cases for timestamps. First element is micros from epcoh and the second element + // is the ISO format in with picosecond precision + public static final Object[][] INPUT_TIMESTAMPS = + new Object[][] { + {1735734896123456L /* 2025-01-01T12:34:56.123456Z */, "2025-01-01T12:34:56.123456789123Z"}, + {1580646896123456L /* 2020-02-02T12:34:56.123456Z */, "2020-02-02T12:34:56.123456789123Z"}, + {636467696123456L /* 1990-03-03T12:34:56.123456Z */, "1990-03-03T12:34:56.123456789123Z"}, + {165846896123456L /* 1975-04-04T12:34:56.123456Z */, "1975-04-04T12:34:56.123456789123Z"} }; - public static final Long[] EXPECTED_TIMESTAMPS_MICROS = - new Long[] { - 1735734896123456L, // 2025-01-01T12:34:56.123456Z - 1580646896123456L, // 2020-02-02T12:34:56.123456Z - 636467696123456L, // 1990-03-03T12:34:56.123456Z - 165846896123456L // 1975-04-04T12:34:56.123456Z + // Expected response for timestamps from the input. If enabled with ISO as output, it will + // ISO8601 format for any picosecond enabled column. + public static final Object[][] EXPECTED_TIMESTAMPS_HIGHER_PRECISION_ISO_OUTPUT = + new Object[][] { + {1735734896123456L /* 2025-01-01T12:34:56.123456Z */, "2025-01-01T12:34:56.123456789123Z"}, + {1580646896123456L /* 2020-02-02T12:34:56.123456Z */, "2020-02-02T12:34:56.123456789123Z"}, + {636467696123456L /* 1990-03-03T12:34:56.123456Z */, "1990-03-03T12:34:56.123456789123Z"}, + {165846896123456L /* 1975-04-04T12:34:56.123456Z */, "1975-04-04T12:34:56.123456789123Z"} }; public static ServiceAccountCredentials loadCredentials(String credentialFile) { @@ -114,7 +122,22 @@ public static void processRowsAtSnapshot( .setParent(parentProjectId) .setMaxStreamCount(1) .setReadSession( - ReadSession.newBuilder().setTable(table).setDataFormat(DataFormat.AVRO).build()); + ReadSession.newBuilder() + .setTable(table) + .setDataFormat(DataFormat.AVRO) + .setReadOptions( + ReadSession.TableReadOptions.newBuilder() + .setAvroSerializationOptions( + AvroSerializationOptions.newBuilder() + .setPicosTimestampPrecision( + // This serialization option only impacts columns that are + // type. `TIMESTAMP_PICOS` and has no impact on other + // columns types. + AvroSerializationOptions.PicosTimestampPrecision + .TIMESTAMP_PRECISION_PICOS) + .build()) + .build()) + .build()); if (snapshotInMillis != null) { createSessionRequestBuilder diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/util/SimpleRowReaderArrow.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/util/SimpleRowReaderArrow.java index e4afb9b1a..ff5b423c2 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/util/SimpleRowReaderArrow.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/util/SimpleRowReaderArrow.java @@ -16,6 +16,8 @@ package com.google.cloud.bigquery.storage.v1.it.util; +import static com.google.cloud.bigquery.storage.v1.it.util.Helper.TIMESTAMP_COLUMN_NAME; +import static com.google.cloud.bigquery.storage.v1.it.util.Helper.TIMESTAMP_HIGHER_PRECISION_COLUMN_NAME; import static com.google.common.truth.Truth.assertThat; import com.google.cloud.bigquery.FieldElementType; @@ -50,19 +52,29 @@ public interface ArrowBatchConsumer { } public static class ArrowTimestampBatchConsumer implements ArrowBatchConsumer { - private final List expectedTimestampValues; + private final Object[][] expectedTimestampValues; - public ArrowTimestampBatchConsumer(List expectedTimestampValues) { + public ArrowTimestampBatchConsumer(Object[][] expectedTimestampValues) { this.expectedTimestampValues = expectedTimestampValues; } @Override public void accept(VectorSchemaRoot root) { - FieldVector timestampFieldVector = root.getVector("timestamp"); + FieldVector timestampFieldVector = root.getVector(TIMESTAMP_COLUMN_NAME); + FieldVector timestampHigherPrecisionFieldVector = + root.getVector(TIMESTAMP_HIGHER_PRECISION_COLUMN_NAME); + assertThat(timestampFieldVector.getValueCount()) + .isEqualTo(timestampHigherPrecisionFieldVector.getValueCount()); int count = timestampFieldVector.getValueCount(); for (int i = 0; i < count; i++) { - long value = (Long) timestampFieldVector.getObject(i); - assertThat(value).isEqualTo(expectedTimestampValues.get(i)); + long timestampMicros = (Long) timestampFieldVector.getObject(i); + assertThat(timestampMicros).isEqualTo(expectedTimestampValues[i][0]); + + // The Object comes back as `Text` which cannot be cast to String + // (use `toString()` instead) + String timestampHigherPrecisionISO = + timestampHigherPrecisionFieldVector.getObject(i).toString(); + assertThat(timestampHigherPrecisionISO).isEqualTo(expectedTimestampValues[i][1]); } } } From 4d19c1d334b7cb00d3c3bd82748aefa5636f2cc6 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Wed, 17 Dec 2025 14:54:09 -0500 Subject: [PATCH 2/6] chore: Add constant and fix assertions --- .../v1/it/ITBigQueryStorageReadClientTest.java | 5 +++-- .../v1/it/ITBigQueryStorageWriteClientTest.java | 16 ++++++++++------ .../bigquery/storage/v1/it/util/Helper.java | 1 + 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryStorageReadClientTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryStorageReadClientTest.java index 0fa7c1add..dce201a2e 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryStorageReadClientTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryStorageReadClientTest.java @@ -535,7 +535,7 @@ private static void setupTimestampTable() .setMode(Mode.NULLABLE) .build(), Field.newBuilder(TIMESTAMP_HIGHER_PRECISION_COLUMN_NAME, StandardSQLTypeName.TIMESTAMP) - .setTimestampPrecision(12L) + .setTimestampPrecision(Helper.PICOSECOND_PRECISION) .setMode(Mode.NULLABLE) .build()); @@ -557,7 +557,8 @@ private static void setupTimestampTable() .addFields( TableFieldSchema.newBuilder() .setName(TIMESTAMP_HIGHER_PRECISION_COLUMN_NAME) - .setTimestampPrecision(Int64Value.newBuilder().setValue(12).build()) + .setTimestampPrecision( + Int64Value.newBuilder().setValue(Helper.PICOSECOND_PRECISION).build()) .setType(TableFieldSchema.Type.TIMESTAMP) .setMode(TableFieldSchema.Mode.NULLABLE) .build()) diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryStorageWriteClientTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryStorageWriteClientTest.java index ae182f710..58babb50b 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryStorageWriteClientTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryStorageWriteClientTest.java @@ -2386,7 +2386,8 @@ public void timestamp_protobufWrite() TableFieldSchema testTimestampHighPrecision = TableFieldSchema.newBuilder() .setName(TIMESTAMP_HIGHER_PRECISION_COLUMN_NAME) - .setTimestampPrecision(Int64Value.newBuilder().setValue(12).build()) + .setTimestampPrecision( + Int64Value.newBuilder().setValue(Helper.PICOSECOND_PRECISION).build()) .setType(TableFieldSchema.Type.TIMESTAMP) .setMode(TableFieldSchema.Mode.NULLABLE) .build(); @@ -2399,6 +2400,9 @@ public void timestamp_protobufWrite() TableName parent = TableName.of(ServiceOptions.getDefaultProjectId(), DATASET, tableName); try (JsonStreamWriter jsonStreamWriter = JsonStreamWriter.newBuilder(parent.toString(), tableSchema).build()) { + + // Creates a single payload to append (JsonArray with multiple JsonObjects) + // Each JsonObject contains a row (one micros, one picos) JSONArray jsonArray = new JSONArray(); for (Object[] timestampData : Helper.INPUT_TIMESTAMPS) { JSONObject row = new JSONObject(); @@ -2421,7 +2425,7 @@ private void createTimestampTable(String tableName) { .build(), Field.newBuilder(TIMESTAMP_HIGHER_PRECISION_COLUMN_NAME, StandardSQLTypeName.TIMESTAMP) .setMode(Mode.NULLABLE) - .setTimestampPrecision(12L) + .setTimestampPrecision(Helper.PICOSECOND_PRECISION) .build()); TableId testTableId = TableId.of(DATASET, tableName); @@ -2447,11 +2451,11 @@ private void assertTimestamps(String tableName, Object[][] expected) throws IOEx .map(x -> x.get(TIMESTAMP_HIGHER_PRECISION_COLUMN_NAME).toString()) .collect(Collectors.toList()); - assertEquals(timestamps.size(), expected.length); - assertEquals(timestampHigherPrecision.size(), expected.length); + assertEquals(expected.length, timestamps.size()); + assertEquals(expected.length, timestampHigherPrecision.size()); for (int i = 0; i < timestampHigherPrecision.size(); i++) { - assertEquals(timestamps.get(i), expected[i][0]); - assertEquals(timestampHigherPrecision.get(i), expected[i][1]); + assertEquals(expected[i][0], timestamps.get(i)); + assertEquals(expected[i][1], timestampHigherPrecision.get(i)); } } } diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/util/Helper.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/util/Helper.java index 05aa4e8d8..e1ee637f6 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/util/Helper.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/util/Helper.java @@ -43,6 +43,7 @@ public class Helper { + public static final long PICOSECOND_PRECISION = 12; public static final String TIMESTAMP_COLUMN_NAME = "timestamp"; public static final String TIMESTAMP_HIGHER_PRECISION_COLUMN_NAME = "timestampHigherPrecision"; From 1cd7809900c72905f4caaea18ac9f5bfaa4a6d4b Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Wed, 17 Dec 2025 17:49:26 -0500 Subject: [PATCH 3/6] test: Add ITs for writing pico timestamps using Protobuf Message --- .../it/ITBigQueryStorageWriteClientTest.java | 148 +++++++++++++++++- 1 file changed, 145 insertions(+), 3 deletions(-) diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryStorageWriteClientTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryStorageWriteClientTest.java index 58babb50b..9d2729f20 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryStorageWriteClientTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryStorageWriteClientTest.java @@ -51,11 +51,14 @@ import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.MoreExecutors; import com.google.protobuf.ByteString; +import com.google.protobuf.DescriptorProtos; import com.google.protobuf.DescriptorProtos.DescriptorProto; import com.google.protobuf.DescriptorProtos.FieldDescriptorProto; import com.google.protobuf.Descriptors; import com.google.protobuf.Descriptors.DescriptorValidationException; +import com.google.protobuf.DynamicMessage; import com.google.protobuf.Int64Value; +import com.google.protobuf.Message; import io.grpc.Status; import io.grpc.Status.Code; import java.io.ByteArrayOutputStream; @@ -124,7 +127,7 @@ public class ITBigQueryStorageWriteClientTest { // Arrow is a bit special in that timestamps are limited to nanoseconds precision. // The data will be padded to fit into the higher precision columns. - public static final Object[][] INPUT_ARROW_WRITE_TIMESTAMPS = + private static final Object[][] INPUT_ARROW_WRITE_TIMESTAMPS = new Object[][] { {1735734896123456L /* 2025-01-01T12:34:56.123456Z */, 1735734896123456789L}, {1580646896123456L /* 2020-02-02T12:34:56.123456Z */, 1580646896123456789L}, @@ -134,7 +137,7 @@ public class ITBigQueryStorageWriteClientTest { // Arrow's higher precision column is padded with extra 0's if configured to return // ISO as output for any picosecond enabled column. - public static final Object[][] EXPECTED_ARROW_WRITE_TIMESTAMPS_ISO_OUTPUT = + private static final Object[][] EXPECTED_ARROW_WRITE_TIMESTAMPS_ISO_OUTPUT = new Object[][] { {1735734896123456L /* 2025-01-01T12:34:56.123456Z */, "2025-01-01T12:34:56.123456789000Z"}, {1580646896123456L /* 2020-02-02T12:34:56.123456Z */, "2020-02-02T12:34:56.123456789000Z"}, @@ -142,6 +145,26 @@ public class ITBigQueryStorageWriteClientTest { {165846896123456L /* 1975-04-04T12:34:56.123456Z */, "1975-04-04T12:34:56.123456789000Z"} }; + // Arrow is a bit special in that timestamps are limited to nanoseconds precision. + // The data will be padded to fit into the higher precision columns. + private static final Long[][] INPUT_PROTO_DESCRIPTOR_WRITE_TIMESTAMPS = + new Long[][] { + {1735734896L, 123456789123L}, /* 2025-01-01T12:34:56.123456789123Z */ + {1580646896L, 123456789123L}, /* 2020-02-02T12:34:56.123456789123Z */ + {636467696L, 123456789123L}, /* 1990-03-03T12:34:56.123456789123Z */ + {165846896L, 123456789123L} /* 1975-04-04T12:34:56.123456789123Z */ + }; + + // Arrow's higher precision column is padded with extra 0's if configured to return + // ISO as output for any picosecond enabled column. + private static final String[] EXPECTED_PROTO_DESCRIPTOR_WRITE_TIMESTAMPS_ISO_OUTPUT = + new String[] { + "2025-01-01T12:34:56.123456789123Z", + "2020-02-02T12:34:56.123456789123Z", + "1990-03-03T12:34:56.123456789123Z", + "1975-04-04T12:34:56.123456789123Z" + }; + public static class StringWithSecondsNanos { public String foo; public long seconds; @@ -2368,7 +2391,7 @@ public void timestamp_arrowWrite() throws IOException { @Test public void timestamp_protobufWrite() throws IOException, DescriptorValidationException, InterruptedException { - String tableName = "bqstorage_timestamp_write_protobuf"; + String tableName = "bqstorage_timestamp_write_protobuf_schema_aware"; // Opt to create a new table to write to instead of re-using table to prevent // the test from failing due to any issues with deleting data after test. // Increases the test time duration, but would be more resilient to transient @@ -2417,6 +2440,125 @@ public void timestamp_protobufWrite() assertTimestamps(tableName, EXPECTED_TIMESTAMPS_HIGHER_PRECISION_ISO_OUTPUT); } + // Tests that users can use a Protobuf message that contains second a fractional + // part (pico) to be written to BQ + @Test + public void timestamp_protobufWrite_customMessage_higherPrecision() + throws IOException, DescriptorValidationException { + String tableName = "bqstorage_timestamp_write_protobuf_custom_descriptor"; + // Opt to create a new table to write to instead of re-using table to prevent + // the test from failing due to any issues with deleting data after test. + // Increases the test time duration, but would be more resilient to transient + // failures + createTimestampTable(tableName); + + /* + A sample protobuf format : + message Wrapper { + message TimestampPicos { + int64 seconds = 1; + int64 picoseconds = 2; + } + Wrapper timestampHigherPrecision = 1; + // ... + } + */ + String wrapperProtoName = "Wrapper"; + String timestampPicosProtoName = "TimestampPicos"; + String secondsProtoName = "seconds"; + String picosProtoName = "picoseconds"; + DescriptorProto timestampPicosDescriptor = + DescriptorProto.newBuilder() + .setName(timestampPicosProtoName) + .addField( + DescriptorProtos.FieldDescriptorProto.newBuilder() + .setName(secondsProtoName) + .setNumber(1) + .setType(DescriptorProtos.FieldDescriptorProto.Type.TYPE_INT64) + .build()) + .addField( + DescriptorProtos.FieldDescriptorProto.newBuilder() + .setName(picosProtoName) + .setNumber(2) + .setType(DescriptorProtos.FieldDescriptorProto.Type.TYPE_INT64) + .build()) + .build(); + DescriptorProto wrapperDescriptor = + DescriptorProto.newBuilder() + .setName(wrapperProtoName) // random name + .addField( + DescriptorProtos.FieldDescriptorProto.newBuilder() + .setName(TIMESTAMP_HIGHER_PRECISION_COLUMN_NAME) + .setNumber(3) + .setType(DescriptorProtos.FieldDescriptorProto.Type.TYPE_MESSAGE) + .setTypeName(timestampPicosDescriptor.getName()) + .build()) + .addNestedType(timestampPicosDescriptor) + .build(); + ProtoSchema protoSchema = + ProtoSchema.newBuilder().setProtoDescriptor(wrapperDescriptor).build(); + + TableName parent = TableName.of(ServiceOptions.getDefaultProjectId(), DATASET, tableName); + try (StreamWriter streamWriter = + StreamWriter.newBuilder(parent.toString() + "/_default", writeClient) + .setWriterSchema(protoSchema) + .build()) { + DescriptorProtos.FileDescriptorProto fileProto = + DescriptorProtos.FileDescriptorProto.newBuilder() + .setName("test.proto") // dummy proto file + .addMessageType(wrapperDescriptor) + .build(); + + // Build the runtime descriptor (resolves types and names) + Descriptors.FileDescriptor file = + Descriptors.FileDescriptor.buildFrom(fileProto, new Descriptors.FileDescriptor[] {}); + + // Get the handle to the "wrapper" message type + Descriptors.Descriptor descriptor = file.findMessageTypeByName(wrapperProtoName); + + ProtoRows.Builder rowsBuilder = ProtoRows.newBuilder(); + for (Long[] timestampParts : INPUT_PROTO_DESCRIPTOR_WRITE_TIMESTAMPS) { + Message message = + DynamicMessage.newBuilder(descriptor) + .setField( + descriptor.findFieldByName(TIMESTAMP_HIGHER_PRECISION_COLUMN_NAME), + DynamicMessage.newBuilder(descriptor.findNestedTypeByName(timestampPicosProtoName)) + .setField( + descriptor + .findNestedTypeByName(timestampPicosProtoName) + .findFieldByName(secondsProtoName), + timestampParts[0]) + .setField( + descriptor + .findNestedTypeByName(timestampPicosProtoName) + .findFieldByName(picosProtoName), + timestampParts[1]) + .build()) + .build(); + rowsBuilder.addSerializedRows(message.toByteString()); + } + ApiFuture future = streamWriter.append(rowsBuilder.build()); + ApiFutures.addCallback( + future, new Helper.AppendCompleteCallback(), MoreExecutors.directExecutor()); + } + String table = + BigQueryResource.formatTableResource( + ServiceOptions.getDefaultProjectId(), DATASET, tableName); + List rows = Helper.readAllRows(readClient, parentProjectId, table, null); + List timestampHigherPrecision = + rows.stream() + .map(x -> x.get(TIMESTAMP_HIGHER_PRECISION_COLUMN_NAME).toString()) + .collect(Collectors.toList()); + assertEquals( + EXPECTED_PROTO_DESCRIPTOR_WRITE_TIMESTAMPS_ISO_OUTPUT.length, + timestampHigherPrecision.size()); + for (int i = 0; i < EXPECTED_PROTO_DESCRIPTOR_WRITE_TIMESTAMPS_ISO_OUTPUT.length; i++) { + assertEquals( + EXPECTED_PROTO_DESCRIPTOR_WRITE_TIMESTAMPS_ISO_OUTPUT[i], + timestampHigherPrecision.get(i)); + } + } + private void createTimestampTable(String tableName) { Schema bqTableSchema = Schema.of( From 19a261db6c1b0fec4efa2c403f28a3828077cf37 Mon Sep 17 00:00:00 2001 From: cloud-java-bot Date: Wed, 17 Dec 2025 22:53:51 +0000 Subject: [PATCH 4/6] chore: generate libraries at Wed Dec 17 22:51:39 UTC 2025 --- .../storage/v1/it/ITBigQueryStorageWriteClientTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryStorageWriteClientTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryStorageWriteClientTest.java index 9d2729f20..0e759a3e4 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryStorageWriteClientTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryStorageWriteClientTest.java @@ -2522,7 +2522,8 @@ public void timestamp_protobufWrite_customMessage_higherPrecision() DynamicMessage.newBuilder(descriptor) .setField( descriptor.findFieldByName(TIMESTAMP_HIGHER_PRECISION_COLUMN_NAME), - DynamicMessage.newBuilder(descriptor.findNestedTypeByName(timestampPicosProtoName)) + DynamicMessage.newBuilder( + descriptor.findNestedTypeByName(timestampPicosProtoName)) .setField( descriptor .findNestedTypeByName(timestampPicosProtoName) From 40d21f3ea6fd229f77eaf6487f6d0d96e955cce0 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Wed, 17 Dec 2025 18:02:05 -0500 Subject: [PATCH 5/6] chore: Add additional test comments --- .../it/ITBigQueryStorageWriteClientTest.java | 33 ++++++++++--------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryStorageWriteClientTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryStorageWriteClientTest.java index 0e759a3e4..d221e3c00 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryStorageWriteClientTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryStorageWriteClientTest.java @@ -145,8 +145,9 @@ public class ITBigQueryStorageWriteClientTest { {165846896123456L /* 1975-04-04T12:34:56.123456Z */, "1975-04-04T12:34:56.123456789000Z"} }; - // Arrow is a bit special in that timestamps are limited to nanoseconds precision. - // The data will be padded to fit into the higher precision columns. + // Special case where users can use the Write API with Protobuf messages + // The format is two fields: 1. Seconds from epoch and 2. Subsecond fractional (millis, micros, + // nano, or pico). This test case is using picos sub-second fractional private static final Long[][] INPUT_PROTO_DESCRIPTOR_WRITE_TIMESTAMPS = new Long[][] { {1735734896L, 123456789123L}, /* 2025-01-01T12:34:56.123456789123Z */ @@ -155,15 +156,15 @@ public class ITBigQueryStorageWriteClientTest { {165846896L, 123456789123L} /* 1975-04-04T12:34:56.123456789123Z */ }; - // Arrow's higher precision column is padded with extra 0's if configured to return - // ISO as output for any picosecond enabled column. - private static final String[] EXPECTED_PROTO_DESCRIPTOR_WRITE_TIMESTAMPS_ISO_OUTPUT = - new String[] { - "2025-01-01T12:34:56.123456789123Z", - "2020-02-02T12:34:56.123456789123Z", - "1990-03-03T12:34:56.123456789123Z", - "1975-04-04T12:34:56.123456789123Z" - }; + // Expected ISO8601 output when using proto descriptors to write to BQ with pico precision + private static final String[] + EXPECTED_PROTO_DESCRIPTOR_WRITE_TIMESTAMPS_HIGH_PRECISION_ISO_OUTPUT = + new String[] { + "2025-01-01T12:34:56.123456789123Z", + "2020-02-02T12:34:56.123456789123Z", + "1990-03-03T12:34:56.123456789123Z", + "1975-04-04T12:34:56.123456789123Z" + }; public static class StringWithSecondsNanos { public String foo; @@ -2453,7 +2454,7 @@ public void timestamp_protobufWrite_customMessage_higherPrecision() createTimestampTable(tableName); /* - A sample protobuf format : + A sample protobuf format: message Wrapper { message TimestampPicos { int64 seconds = 1; @@ -2551,11 +2552,13 @@ public void timestamp_protobufWrite_customMessage_higherPrecision() .map(x -> x.get(TIMESTAMP_HIGHER_PRECISION_COLUMN_NAME).toString()) .collect(Collectors.toList()); assertEquals( - EXPECTED_PROTO_DESCRIPTOR_WRITE_TIMESTAMPS_ISO_OUTPUT.length, + EXPECTED_PROTO_DESCRIPTOR_WRITE_TIMESTAMPS_HIGH_PRECISION_ISO_OUTPUT.length, timestampHigherPrecision.size()); - for (int i = 0; i < EXPECTED_PROTO_DESCRIPTOR_WRITE_TIMESTAMPS_ISO_OUTPUT.length; i++) { + for (int i = 0; + i < EXPECTED_PROTO_DESCRIPTOR_WRITE_TIMESTAMPS_HIGH_PRECISION_ISO_OUTPUT.length; + i++) { assertEquals( - EXPECTED_PROTO_DESCRIPTOR_WRITE_TIMESTAMPS_ISO_OUTPUT[i], + EXPECTED_PROTO_DESCRIPTOR_WRITE_TIMESTAMPS_HIGH_PRECISION_ISO_OUTPUT[i], timestampHigherPrecision.get(i)); } } From 34d4f01de8bb03da34030403edde335e81de5e2a Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Thu, 18 Dec 2025 11:12:09 -0500 Subject: [PATCH 6/6] chore: Fix compile issues --- .../storage/v1/it/ITBigQueryStorageWriteClientTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryStorageWriteClientTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryStorageWriteClientTest.java index e4133a8d2..e28d8fcc7 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryStorageWriteClientTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryStorageWriteClientTest.java @@ -56,7 +56,9 @@ import com.google.protobuf.DescriptorProtos.FieldDescriptorProto; import com.google.protobuf.Descriptors; import com.google.protobuf.Descriptors.DescriptorValidationException; +import com.google.protobuf.DynamicMessage; import com.google.protobuf.Int64Value; +import com.google.protobuf.Message; import io.grpc.Status; import io.grpc.Status.Code; import java.io.ByteArrayOutputStream;