From 6f3f4c648b08d8c845e17565dc7e942ad1c799e1 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Tue, 9 Dec 2025 17:38:25 -0500 Subject: [PATCH 1/8] feat: Add picosecond timestamp support for Json to Proto converter --- .../v1/BQTableSchemaToProtoDescriptor.java | 52 ++++-- .../storage/v1/JsonToProtoMessage.java | 103 +++++++++++ .../BQTableSchemaToProtoDescriptorTest.java | 163 ++++++++++++++---- .../storage/v1/JsonToProtoMessageTest.java | 57 ++++++ .../src/test/proto/jsonTest.proto | 6 + 5 files changed, 333 insertions(+), 48 deletions(-) diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptor.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptor.java index 60bb739b23..53d835f524 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptor.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptor.java @@ -30,6 +30,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Map; /** * Converts a BQ table schema to protobuf descriptor. All field names will be converted to lowercase @@ -37,15 +38,14 @@ * shown in the ImmutableMaps below. */ public class BQTableSchemaToProtoDescriptor { - private static ImmutableMap - BQTableSchemaModeMap = - ImmutableMap.of( - TableFieldSchema.Mode.NULLABLE, FieldDescriptorProto.Label.LABEL_OPTIONAL, - TableFieldSchema.Mode.REPEATED, FieldDescriptorProto.Label.LABEL_REPEATED, - TableFieldSchema.Mode.REQUIRED, FieldDescriptorProto.Label.LABEL_REQUIRED); + private static Map DEFAULT_BQ_TABLE_SCHEMA_MODE_MAP = + ImmutableMap.of( + TableFieldSchema.Mode.NULLABLE, FieldDescriptorProto.Label.LABEL_OPTIONAL, + TableFieldSchema.Mode.REPEATED, FieldDescriptorProto.Label.LABEL_REPEATED, + TableFieldSchema.Mode.REQUIRED, FieldDescriptorProto.Label.LABEL_REQUIRED); - private static ImmutableMap - BQTableSchemaTypeMap = + private static Map + DEFAULT_BQ_TABLE_SCHEMA_TYPE_MAP = new ImmutableMap.Builder() .put(TableFieldSchema.Type.BOOL, FieldDescriptorProto.Type.TYPE_BOOL) .put(TableFieldSchema.Type.BYTES, FieldDescriptorProto.Type.TYPE_BYTES) @@ -142,11 +142,13 @@ private static Descriptor convertBQTableSchemaToProtoDescriptorImpl( .setType(BQTableField.getRangeElementType().getType()) .setName("start") .setMode(Mode.NULLABLE) + .setTimestampPrecision(BQTableField.getTimestampPrecision()) .build(), TableFieldSchema.newBuilder() .setType(BQTableField.getRangeElementType().getType()) .setName("end") .setMode(Mode.NULLABLE) + .setTimestampPrecision(BQTableField.getTimestampPrecision()) .build()); if (dependencyMap.containsKey(rangeFields)) { @@ -189,7 +191,7 @@ private static Descriptor convertBQTableSchemaToProtoDescriptorImpl( * @param index Index for protobuf fields. * @param scope used to name descriptors */ - private static FieldDescriptorProto convertBQTableFieldToProtoField( + static FieldDescriptorProto convertBQTableFieldToProtoField( TableFieldSchema BQTableField, int index, String scope) { TableFieldSchema.Mode mode = BQTableField.getMode(); String fieldName = BQTableField.getName().toLowerCase(); @@ -198,7 +200,7 @@ private static FieldDescriptorProto convertBQTableFieldToProtoField( FieldDescriptorProto.newBuilder() .setName(fieldName) .setNumber(index) - .setLabel((FieldDescriptorProto.Label) BQTableSchemaModeMap.get(mode)); + .setLabel((FieldDescriptorProto.Label) DEFAULT_BQ_TABLE_SCHEMA_MODE_MAP.get(mode)); switch (BQTableField.getType()) { case STRUCT: @@ -206,12 +208,38 @@ private static FieldDescriptorProto convertBQTableFieldToProtoField( break; case RANGE: fieldDescriptor.setType( - (FieldDescriptorProto.Type) BQTableSchemaTypeMap.get(BQTableField.getType())); + (FieldDescriptorProto.Type) + DEFAULT_BQ_TABLE_SCHEMA_TYPE_MAP.get(BQTableField.getType())); fieldDescriptor.setTypeName(scope); break; + case TIMESTAMP: + // Can map to either int64 or string based on the BQ Field's timestamp precision + // Default: microsecond (6) maps to int64 and picosecond (12) maps to string. + switch ((int) BQTableField.getTimestampPrecision().getValue()) { + case 12: + fieldDescriptor.setType( + (FieldDescriptorProto.Type) FieldDescriptorProto.Type.TYPE_STRING); + break; + case 6: + case 0: + // If the timestampPrecision value coems back as a null result from the server, a + // default value + // of 0L is set. Map this value as default precision as 6 (microsecond). + fieldDescriptor.setType( + (FieldDescriptorProto.Type) FieldDescriptorProto.Type.TYPE_INT64); + break; + default: + // This should never happen as it's an invalid value from server + throw new IllegalStateException( + "BigQuery Timestamp field " + + BQTableField.getName() + + " has timestamp precision that is not 6 or 12"); + } + break; default: fieldDescriptor.setType( - (FieldDescriptorProto.Type) BQTableSchemaTypeMap.get(BQTableField.getType())); + (FieldDescriptorProto.Type) + DEFAULT_BQ_TABLE_SCHEMA_TYPE_MAP.get(BQTableField.getType())); break; } diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java index 9a4fecf780..86bfe43dcf 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java @@ -17,6 +17,7 @@ import com.google.api.pathtemplate.ValidationException; import com.google.cloud.bigquery.storage.v1.Exceptions.RowIndexToErrorException; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.primitives.Doubles; @@ -26,15 +27,18 @@ import com.google.protobuf.Descriptors.Descriptor; import com.google.protobuf.Descriptors.FieldDescriptor; import com.google.protobuf.DynamicMessage; +import com.google.protobuf.Timestamp; import com.google.protobuf.UninitializedMessageException; import java.math.BigDecimal; import java.math.RoundingMode; +import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatterBuilder; +import java.time.format.DateTimeParseException; import java.time.format.TextStyle; import java.time.temporal.ChronoField; import java.time.temporal.TemporalAccessor; @@ -42,6 +46,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; @@ -120,6 +126,14 @@ public class JsonToProtoMessage implements ToProtoConverter { .parseDefaulting(ChronoField.SECOND_OF_MINUTE, 0) .toFormatter(); + // Regex to identify >9 digits in the fraction part (e.g. `.123456789123`) + // Matches the dot, followed by 10+ digits (fractional part), followed by non-digits (like `+00`) + // or end of string + private static final Pattern ISO8601_TIMESTAMP_HIGH_PRECISION_PATTERN = + Pattern.compile("\\.(\\d{10,})(?:\\D|$)"); + private static final long MICROS_PER_SECOND = 1_000_000; + private static final int NANOS_PER_MICRO = 1_000; + /** You can use {@link #INSTANCE} instead */ public JsonToProtoMessage() {} @@ -685,6 +699,14 @@ private void fillField( } break; case STRING: + // Timestamp fields will be transmitted as a String if BQ's timestamp field is + // enabled to support picosecond. Check that the schema's field is timestamp before + // proceeding with the rest of the logic. Converts the supported types into a String. + // Supported types: https://docs.cloud.google.com/bigquery/docs/supported-data-types + if (fieldSchema != null && fieldSchema.getType() == TableFieldSchema.Type.TIMESTAMP) { + protoMsg.setField(fieldDescriptor, getTimestampAsString(val)); + return; + } if (val instanceof String) { protoMsg.setField(fieldDescriptor, val); return; @@ -958,6 +980,14 @@ private void fillRepeatedField( } break; case STRING: + // Timestamp fields will be transmitted as a String if BQ's timestamp field is + // enabled to support picosecond. Check that the schema's field is timestamp before + // proceeding with the rest of the logic. Converts the supported types into a String. + // Supported types: https://docs.cloud.google.com/bigquery/docs/supported-data-types + if (fieldSchema != null && fieldSchema.getType() == TableFieldSchema.Type.TIMESTAMP) { + protoMsg.addRepeatedField(fieldDescriptor, getTimestampAsString(val)); + return; + } if (val instanceof String) { protoMsg.addRepeatedField(fieldDescriptor, val); } else if (val instanceof Short @@ -1002,6 +1032,40 @@ private void fillRepeatedField( } } + /** + * Converts microseconds from epoch to a Java Instant. + * + * @param micros the number of microseconds from 1970-01-01T00:00:00Z + * @return the Instant corresponding to the microseconds + */ + @VisibleForTesting + static Instant fromEpochMicros(long micros) { + long seconds = Math.floorDiv(micros, MICROS_PER_SECOND); + int nanos = (int) Math.floorMod(micros, MICROS_PER_SECOND) * NANOS_PER_MICRO; + + return Instant.ofEpochSecond(seconds, nanos); + } + + /** Best effort to try and convert a timestamp to an ISO8601 string */ + @VisibleForTesting + static String getTimestampAsString(Object val) { + if (val instanceof String) { + // Validate the ISO8601 values before sending it to the server + String value = (String) val; + validateTimestamp(value); + return value; + } else if (val instanceof Short || val instanceof Integer || val instanceof Long) { + // Micros from epoch will most likely will be represented a Long, but any non-float + // numeric value can be used + return fromEpochMicros((Long) val).toString(); + } else if (val instanceof Timestamp) { + // Convert the Protobuf timestamp class to ISO8601 string + Timestamp timestamp = (Timestamp) val; + return Instant.ofEpochSecond(timestamp.getSeconds(), timestamp.getNanos()).toString(); + } + throw new IllegalArgumentException("The timestamp value passed in is not from a valid type"); + } + private static void throwWrongFieldType( FieldDescriptor fieldDescriptor, String currentScope, int index) { throw new IllegalArgumentException( @@ -1009,4 +1073,43 @@ private static void throwWrongFieldType( "JSONObject does not have a %s field at %s[%d].", FIELD_TYPE_TO_DEBUG_MESSAGE.get(fieldDescriptor.getType()), currentScope, index)); } + + /** + * Internal helper method to check that the timestamp follows the expected String input of ISO8601 + * string. Allows the fractional portion of the timestamp to support up to 12 digits of precision + * (up to picosecond). + * + * @throws IllegalArgumentException if timestamp is invalid or exceeds picosecond precision + */ + @VisibleForTesting + static void validateTimestamp(String timestamp) { + // Check if the string has greater than nanosecond precision (>9 digits in fractional second) + Matcher matcher = ISO8601_TIMESTAMP_HIGH_PRECISION_PATTERN.matcher(timestamp); + if (matcher.find()) { + // Group 1 is the fractional second part of the ISO8601 string + String fraction = matcher.group(1); + // Pos 10-12 of the fractional second are guaranteed to be digits. The regex only + // matches the fraction section as long as they are digits. + if (fraction.length() > 12) { + throw new IllegalArgumentException( + "Fractional second portion of ISO8601 only supports up to picosecond (12 digits) in" + + " BigQuery"); + } + + // Replace the entire fractional second portion with just the nanosecond portion. + // The new timestamp will be validated against the JDK's DateTimeFormatter + String truncatedFraction = fraction.substring(0, 9); + timestamp = + new StringBuilder(timestamp) + .replace(matcher.start(1), matcher.end(1), truncatedFraction) + .toString(); + } + + // It is valid as long as DateTimeFormatter doesn't throw an exception + try { + TIMESTAMP_FORMATTER.parse((String) timestamp); + } catch (DateTimeParseException e) { + throw new IllegalArgumentException(e.getMessage(), e); + } + } } diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptorTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptorTest.java index ba845c1c12..56263bca61 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptorTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptorTest.java @@ -15,13 +15,19 @@ */ package com.google.cloud.bigquery.storage.v1; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; -import com.google.cloud.bigquery.storage.test.JsonTest.*; -import com.google.cloud.bigquery.storage.test.SchemaTest.*; +import com.google.cloud.bigquery.storage.test.JsonTest; +import com.google.cloud.bigquery.storage.test.SchemaTest; import com.google.common.collect.ImmutableMap; +import com.google.protobuf.DescriptorProtos; +import com.google.protobuf.Descriptors; import com.google.protobuf.Descriptors.Descriptor; import com.google.protobuf.Descriptors.FieldDescriptor; +import com.google.protobuf.Int64Value; import java.util.HashMap; import java.util.Map; import org.junit.Test; @@ -32,21 +38,20 @@ public class BQTableSchemaToProtoDescriptorTest { // This is a map between the TableFieldSchema.Type and the descriptor it is supposed to // produce. The produced descriptor will be used to check against the entry values here. - private static ImmutableMap - BQTableTypeToCorrectProtoDescriptorTest = - new ImmutableMap.Builder() - .put(TableFieldSchema.Type.BOOL, BoolType.getDescriptor()) - .put(TableFieldSchema.Type.BYTES, BytesType.getDescriptor()) - .put(TableFieldSchema.Type.DATE, Int32Type.getDescriptor()) - .put(TableFieldSchema.Type.DATETIME, Int64Type.getDescriptor()) - .put(TableFieldSchema.Type.DOUBLE, DoubleType.getDescriptor()) - .put(TableFieldSchema.Type.GEOGRAPHY, StringType.getDescriptor()) - .put(TableFieldSchema.Type.INT64, Int64Type.getDescriptor()) - .put(TableFieldSchema.Type.NUMERIC, BytesType.getDescriptor()) - .put(TableFieldSchema.Type.STRING, StringType.getDescriptor()) - .put(TableFieldSchema.Type.TIME, Int64Type.getDescriptor()) - .put(TableFieldSchema.Type.TIMESTAMP, Int64Type.getDescriptor()) - .build(); + private static Map BQTableTypeToCorrectProtoDescriptorTest = + new ImmutableMap.Builder() + .put(TableFieldSchema.Type.BOOL, SchemaTest.BoolType.getDescriptor()) + .put(TableFieldSchema.Type.BYTES, SchemaTest.BytesType.getDescriptor()) + .put(TableFieldSchema.Type.DATE, SchemaTest.Int32Type.getDescriptor()) + .put(TableFieldSchema.Type.DATETIME, SchemaTest.Int64Type.getDescriptor()) + .put(TableFieldSchema.Type.DOUBLE, SchemaTest.DoubleType.getDescriptor()) + .put(TableFieldSchema.Type.GEOGRAPHY, SchemaTest.StringType.getDescriptor()) + .put(TableFieldSchema.Type.INT64, SchemaTest.Int64Type.getDescriptor()) + .put(TableFieldSchema.Type.NUMERIC, SchemaTest.BytesType.getDescriptor()) + .put(TableFieldSchema.Type.STRING, SchemaTest.StringType.getDescriptor()) + .put(TableFieldSchema.Type.TIME, SchemaTest.Int64Type.getDescriptor()) + .put(TableFieldSchema.Type.TIMESTAMP, SchemaTest.Int64Type.getDescriptor()) + .build(); // Creates mapping from descriptor to how many times it was reused. private void mapDescriptorToCount(Descriptor descriptor, HashMap map) { @@ -64,25 +69,29 @@ private void mapDescriptorToCount(Descriptor descriptor, HashMap + BQTableSchemaToProtoDescriptor.convertBQTableFieldToProtoField( + timestampField, 0, null)); + + TableFieldSchema timestampField1 = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.TIMESTAMP) + .setTimestampPrecision(Int64Value.newBuilder().setValue(7).build()) + .setMode(TableFieldSchema.Mode.NULLABLE) + .build(); + assertThrows( + IllegalStateException.class, + () -> + BQTableSchemaToProtoDescriptor.convertBQTableFieldToProtoField( + timestampField1, 0, null)); } } diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java index 2e622e8966..12277607cf 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java @@ -16,6 +16,7 @@ package com.google.cloud.bigquery.storage.v1; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import com.google.cloud.bigquery.storage.test.JsonTest.*; @@ -27,7 +28,9 @@ import com.google.protobuf.Descriptors.Descriptor; import com.google.protobuf.DynamicMessage; import com.google.protobuf.Message; +import com.google.protobuf.Timestamp; import java.math.BigDecimal; +import java.time.Instant; import java.time.LocalTime; import java.util.ArrayList; import java.util.Collection; @@ -1834,4 +1837,58 @@ public void testDoubleAndFloatToRepeatedBigNumericConversion() { JsonToProtoMessage.INSTANCE.convertToProtoMessage(TestBignumeric.getDescriptor(), ts, json); assertEquals(expectedProto, protoMsg); } + + @Test + public void testGetTimestampAsString() { + assertEquals( + "2025-10-01 12:34:56.123456+00:00", + JsonToProtoMessage.getTimestampAsString("2025-10-01 12:34:56.123456+00:00")); + assertEquals( + "2025-10-01 12:34:56.123456789123+00:00", + JsonToProtoMessage.getTimestampAsString("2025-10-01 12:34:56.123456789123+00:00")); + + assertEquals("1970-01-01T00:00:00.000001Z", JsonToProtoMessage.getTimestampAsString(1L)); + assertEquals("1969-12-31T23:59:59.999999Z", JsonToProtoMessage.getTimestampAsString(-1L)); + + assertEquals( + "1970-01-02T10:17:36.000123456Z", + JsonToProtoMessage.getTimestampAsString( + Timestamp.newBuilder().setSeconds(123456).setNanos(123456).build())); + assertEquals( + "1969-12-30T13:42:23.999876544Z", + JsonToProtoMessage.getTimestampAsString( + Timestamp.newBuilder().setSeconds(-123456).setNanos(-123456).build())); + + assertThrows( + IllegalArgumentException.class, + () -> JsonToProtoMessage.getTimestampAsString("2025-10-01")); + assertThrows( + IllegalArgumentException.class, () -> JsonToProtoMessage.getTimestampAsString("1234")); + assertThrows( + IllegalArgumentException.class, () -> JsonToProtoMessage.getTimestampAsString("abc")); + assertThrows( + IllegalArgumentException.class, () -> JsonToProtoMessage.getTimestampAsString(10.4)); + assertThrows( + IllegalArgumentException.class, () -> JsonToProtoMessage.getTimestampAsString(-1000.4)); + assertThrows( + IllegalArgumentException.class, + () -> JsonToProtoMessage.getTimestampAsString(Timestamp.newBuilder())); + assertThrows( + IllegalArgumentException.class, + () -> JsonToProtoMessage.getTimestampAsString(new Object())); + assertThrows( + IllegalArgumentException.class, () -> JsonToProtoMessage.getTimestampAsString(null)); + } + + @Test + public void testFromEpochMicros() { + // The `+` is added if there are more than 4 digits for years + assertEquals( + "+294247-01-10T04:00:54.775807Z", + JsonToProtoMessage.fromEpochMicros(Long.MAX_VALUE).toString()); + assertEquals( + "-290308-12-21T19:59:05.224192Z", + JsonToProtoMessage.fromEpochMicros(Long.MIN_VALUE).toString()); + assertEquals(Instant.EPOCH.toString(), JsonToProtoMessage.fromEpochMicros(0L).toString()); + } } diff --git a/google-cloud-bigquerystorage/src/test/proto/jsonTest.proto b/google-cloud-bigquerystorage/src/test/proto/jsonTest.proto index 618bcc0a03..84248ad6f3 100644 --- a/google-cloud-bigquerystorage/src/test/proto/jsonTest.proto +++ b/google-cloud-bigquerystorage/src/test/proto/jsonTest.proto @@ -221,6 +221,7 @@ message TestRange { optional TestRangeDate range_date_mixed_case = 4; optional TestRangeDatetime range_datetime_mixed_case = 5; optional TestRangeTimestamp range_timestamp_mixed_case = 6; + optional TestRangeTimestampHigherPrecision range_timestamp_higher_precision_mixed_case = 7; } message TestRangeDate { @@ -236,4 +237,9 @@ message TestRangeDatetime { message TestRangeTimestamp { optional int64 start = 1; optional int64 end = 2; +} + +message TestRangeTimestampHigherPrecision { + optional string start = 1; + optional string end = 2; } \ No newline at end of file From 7862f389fd3b3a23ea8329c6f5abf47ca1c1906a Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Tue, 9 Dec 2025 21:50:24 -0500 Subject: [PATCH 2/8] chore: Add edge cases for user input --- .../storage/v1/JsonToProtoMessage.java | 116 ++++++----- .../BQTableSchemaToProtoDescriptorTest.java | 34 ++-- .../storage/v1/JsonToProtoMessageTest.java | 187 ++++++++++++++++-- .../src/test/proto/jsonTest.proto | 32 ++- 4 files changed, 291 insertions(+), 78 deletions(-) diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java index 86bfe43dcf..2dd685d66d 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java @@ -15,6 +15,11 @@ */ package com.google.cloud.bigquery.storage.v1; +import static java.time.temporal.ChronoField.HOUR_OF_DAY; +import static java.time.temporal.ChronoField.MINUTE_OF_HOUR; +import static java.time.temporal.ChronoField.NANO_OF_SECOND; +import static java.time.temporal.ChronoField.SECOND_OF_MINUTE; + import com.google.api.pathtemplate.ValidationException; import com.google.cloud.bigquery.storage.v1.Exceptions.RowIndexToErrorException; import com.google.common.annotations.VisibleForTesting; @@ -69,7 +74,31 @@ public class JsonToProtoMessage implements ToProtoConverter { .put(FieldDescriptor.Type.STRING, "string") .put(FieldDescriptor.Type.MESSAGE, "object") .build(); - private static final DateTimeFormatter TIMESTAMP_FORMATTER = + + private static final DateTimeFormatter TO_TIMESTAMP_FORMATTER = + new DateTimeFormatterBuilder() + .parseLenient() + .append(DateTimeFormatter.ISO_LOCAL_DATE) + .optionalStart() + .appendLiteral('T') + .optionalEnd() + .appendValue(HOUR_OF_DAY, 2) + .appendLiteral(':') + .appendValue(MINUTE_OF_HOUR, 2) + .optionalStart() + .appendLiteral(':') + .appendValue(SECOND_OF_MINUTE, 2) + .optionalEnd() + .optionalStart() + .appendFraction(NANO_OF_SECOND, 6, 9, true) + .optionalEnd() + .optionalStart() + .appendOffset("+HHMM", "+00:00") + .optionalEnd() + .toFormatter() + .withZone(ZoneOffset.UTC); + + private static final DateTimeFormatter FROM_TIMESTAMP_FORMATTER = new DateTimeFormatterBuilder() .parseLenient() .append(DateTimeFormatter.ofPattern("yyyy[/][-]MM[/][-]dd")) @@ -634,25 +663,8 @@ private void fillField( return; } } else if (fieldSchema.getType() == TableFieldSchema.Type.TIMESTAMP) { - if (val instanceof String) { - Double parsed = Doubles.tryParse((String) val); - if (parsed != null) { - protoMsg.setField(fieldDescriptor, parsed.longValue()); - return; - } - TemporalAccessor parsedTime = TIMESTAMP_FORMATTER.parse((String) val); - protoMsg.setField( - fieldDescriptor, - parsedTime.getLong(ChronoField.INSTANT_SECONDS) * 1000000 - + parsedTime.getLong(ChronoField.MICRO_OF_SECOND)); - return; - } else if (val instanceof Long) { - protoMsg.setField(fieldDescriptor, val); - return; - } else if (val instanceof Integer) { - protoMsg.setField(fieldDescriptor, Long.valueOf((Integer) val)); - return; - } + protoMsg.setField(fieldDescriptor, getTimestampAsLong(val)); + return; } } if (val instanceof Integer) { @@ -919,24 +931,7 @@ private void fillRepeatedField( } } else if (fieldSchema != null && fieldSchema.getType() == TableFieldSchema.Type.TIMESTAMP) { - if (val instanceof String) { - Double parsed = Doubles.tryParse((String) val); - if (parsed != null) { - protoMsg.addRepeatedField(fieldDescriptor, parsed.longValue()); - } else { - TemporalAccessor parsedTime = TIMESTAMP_FORMATTER.parse((String) val); - protoMsg.addRepeatedField( - fieldDescriptor, - parsedTime.getLong(ChronoField.INSTANT_SECONDS) * 1000000 - + parsedTime.getLong(ChronoField.MICRO_OF_SECOND)); - } - } else if (val instanceof Long) { - protoMsg.addRepeatedField(fieldDescriptor, val); - } else if (val instanceof Integer) { - protoMsg.addRepeatedField(fieldDescriptor, Long.valueOf((Integer) val)); - } else { - throwWrongFieldType(fieldDescriptor, currentScope, index); - } + protoMsg.addRepeatedField(fieldDescriptor, getTimestampAsLong(val)); } else if (val instanceof Integer) { protoMsg.addRepeatedField(fieldDescriptor, Long.valueOf((Integer) val)); } else if (val instanceof Long) { @@ -1050,18 +1045,51 @@ static Instant fromEpochMicros(long micros) { @VisibleForTesting static String getTimestampAsString(Object val) { if (val instanceof String) { - // Validate the ISO8601 values before sending it to the server String value = (String) val; + Double parsed = Doubles.tryParse(value); + // If true, it was a numeric value inside a String + if (parsed != null) { + return getTimestampAsString(parsed.longValue()); + } + // Validate the ISO8601 values before sending it to the server. No need to format + // if it's valid. validateTimestamp(value); - return value; - } else if (val instanceof Short || val instanceof Integer || val instanceof Long) { + + // If it's high precision (more than 9 digits), then return the ISO8601 string as-is + // as JDK does not have a DateTimeFormatter that supports more than nanosecond precision. + Matcher matcher = ISO8601_TIMESTAMP_HIGH_PRECISION_PATTERN.matcher(value); + if (matcher.find()) { + return value; + } + // Otherwise, output the timestamp to a standard format before sending it to BQ + Instant instant = FROM_TIMESTAMP_FORMATTER.parse(value, Instant::from); + return TO_TIMESTAMP_FORMATTER.format(instant); + } else if (val instanceof Number) { // Micros from epoch will most likely will be represented a Long, but any non-float // numeric value can be used - return fromEpochMicros((Long) val).toString(); + Instant instant = fromEpochMicros(((Number) val).longValue()); + return TO_TIMESTAMP_FORMATTER.format(instant); } else if (val instanceof Timestamp) { // Convert the Protobuf timestamp class to ISO8601 string Timestamp timestamp = (Timestamp) val; - return Instant.ofEpochSecond(timestamp.getSeconds(), timestamp.getNanos()).toString(); + return TO_TIMESTAMP_FORMATTER.format( + Instant.ofEpochSecond(timestamp.getSeconds(), timestamp.getNanos())); + } + throw new IllegalArgumentException("The timestamp value passed in is not from a valid type"); + } + + /* Best effort to try and convert the Object to a long (microseconds since epoch) */ + private long getTimestampAsLong(Object val) { + if (val instanceof String) { + Double parsed = Doubles.tryParse((String) val); + if (parsed != null) { + return parsed.longValue(); + } + TemporalAccessor parsedTime = FROM_TIMESTAMP_FORMATTER.parse((String) val); + return parsedTime.getLong(ChronoField.INSTANT_SECONDS) * 1000000 + + parsedTime.getLong(ChronoField.MICRO_OF_SECOND); + } else if (val instanceof Number) { + return ((Number) val).longValue(); } throw new IllegalArgumentException("The timestamp value passed in is not from a valid type"); } @@ -1107,7 +1135,7 @@ static void validateTimestamp(String timestamp) { // It is valid as long as DateTimeFormatter doesn't throw an exception try { - TIMESTAMP_FORMATTER.parse((String) timestamp); + FROM_TIMESTAMP_FORMATTER.parse((String) timestamp); } catch (DateTimeParseException e) { throw new IllegalArgumentException(e.getMessage(), e); } diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptorTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptorTest.java index 56263bca61..23484050a8 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptorTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptorTest.java @@ -85,10 +85,9 @@ private void assertDesciptorsAreEqual(Descriptor expected, Descriptor actual) { FieldDescriptor.Type originalType = expectedField.getType(); assertEquals(convertedField.getName(), convertedType, originalType); // Check mode - assertTrue( - (expectedField.isRepeated() == convertedField.isRepeated()) - && (expectedField.isRequired() == convertedField.isRequired()) - && (expectedField.isOptional() == convertedField.isOptional())); + assertEquals(expectedField.isRepeated(), convertedField.isRepeated()); + assertEquals(expectedField.isRequired(), convertedField.isRequired()); + assertEquals(expectedField.isOptional(), convertedField.isOptional()); // Recursively check nested messages if (convertedType == FieldDescriptor.Type.MESSAGE) { assertDesciptorsAreEqual(expectedField.getMessageType(), convertedField.getMessageType()); @@ -195,17 +194,6 @@ public void testRange() throws Exception { .setType(TableFieldSchema.Type.TIMESTAMP) .build()) .build()) - .addFields( - TableFieldSchema.newBuilder() - .setName("range_timestamp_higher_precision_miXEd_caSE") - .setType(TableFieldSchema.Type.RANGE) - .setMode(TableFieldSchema.Mode.NULLABLE) - .setRangeElementType( - TableFieldSchema.FieldElementType.newBuilder() - .setType(TableFieldSchema.Type.TIMESTAMP) - .build()) - .setTimestampPrecision(Int64Value.newBuilder().setValue(12).build()) - .build()) .build(); final Descriptor descriptor = BQTableSchemaToProtoDescriptor.convertBQTableSchemaToProtoDescriptor(tableSchema); @@ -424,6 +412,20 @@ public void testStructComplex() throws Exception { .setMode(TableFieldSchema.Mode.REPEATED) .setName("test_json") .build(); + final TableFieldSchema TEST_TIMESTAMP_HIGHER_PRECISION = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.TIMESTAMP) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setName("test_timestamp_higher_precision") + .setTimestampPrecision(Int64Value.newBuilder().setValue(12).build()) + .build(); + final TableFieldSchema TEST_TIMESTAMP_HIGHER_PRECISION_REPEATED = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.TIMESTAMP) + .setMode(TableFieldSchema.Mode.REPEATED) + .setName("test_timestamp_higher_precision_repeated") + .setTimestampPrecision(Int64Value.newBuilder().setValue(12).build()) + .build(); final TableSchema tableSchema = TableSchema.newBuilder() .addFields(0, test_int) @@ -457,6 +459,8 @@ public void testStructComplex() throws Exception { .addFields(28, TEST_BIGNUMERIC_DOUBLE) .addFields(29, TEST_INTERVAL) .addFields(30, TEST_JSON) + .addFields(31, TEST_TIMESTAMP_HIGHER_PRECISION) + .addFields(32, TEST_TIMESTAMP_HIGHER_PRECISION_REPEATED) .build(); final Descriptor descriptor = BQTableSchemaToProtoDescriptor.convertBQTableSchemaToProtoDescriptor(tableSchema); diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java index 12277607cf..d3514e1746 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java @@ -27,6 +27,7 @@ import com.google.protobuf.ByteString; import com.google.protobuf.Descriptors.Descriptor; import com.google.protobuf.DynamicMessage; +import com.google.protobuf.Int64Value; import com.google.protobuf.Message; import com.google.protobuf.Timestamp; import java.math.BigDecimal; @@ -516,6 +517,20 @@ public class JsonToProtoMessageTest { .setMode(TableFieldSchema.Mode.REPEATED) .setName("test_json") .build(); + final TableFieldSchema TEST_TIMESTAMP_HIGHER_PRECISION = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.TIMESTAMP) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setName("test_timestamp_higher_precision") + .setTimestampPrecision(Int64Value.newBuilder().setValue(12).build()) + .build(); + private final TableFieldSchema TEST_TIMESTAMP_HIGHER_PRECISION_REPEATED = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.TIMESTAMP) + .setMode(TableFieldSchema.Mode.REPEATED) + .setName("test_timestamp_higher_precision_repeated") + .setTimestampPrecision(Int64Value.newBuilder().setValue(12).build()) + .build(); private final TableSchema COMPLEX_TABLE_SCHEMA = TableSchema.newBuilder() .addFields(0, TEST_INT) @@ -549,6 +564,8 @@ public class JsonToProtoMessageTest { .addFields(28, TEST_BIGNUMERIC_DOUBLE) .addFields(29, TEST_INTERVAL) .addFields(30, TEST_JSON) + .addFields(31, TEST_TIMESTAMP_HIGHER_PRECISION) + .addFields(32, TEST_TIMESTAMP_HIGHER_PRECISION_REPEATED) .build(); @Test @@ -876,6 +893,76 @@ public void testTimestamp() throws Exception { assertEquals(expectedProto, protoMsg); } + @Test + public void testTimestamp_higherPrecision() throws Exception { + TableSchema tableSchema = + TableSchema.newBuilder() + .addFields( + TableFieldSchema.newBuilder(TEST_TIMESTAMP_HIGHER_PRECISION) + .setName("test_string") + .build()) + .addFields( + TableFieldSchema.newBuilder(TEST_TIMESTAMP_HIGHER_PRECISION) + .setName("test_string_T_Z") + .build()) + .addFields( + TableFieldSchema.newBuilder(TEST_TIMESTAMP_HIGHER_PRECISION) + .setName("test_long") + .build()) + .addFields( + TableFieldSchema.newBuilder(TEST_TIMESTAMP_HIGHER_PRECISION) + .setName("test_int") + .build()) + .addFields( + TableFieldSchema.newBuilder(TEST_TIMESTAMP_HIGHER_PRECISION) + .setName("test_float") + .build()) + .addFields( + TableFieldSchema.newBuilder(TEST_TIMESTAMP_HIGHER_PRECISION) + .setName("test_offset") + .build()) + .addFields( + TableFieldSchema.newBuilder(TEST_TIMESTAMP_HIGHER_PRECISION) + .setName("test_zero_offset") + .build()) + .addFields( + TableFieldSchema.newBuilder(TEST_TIMESTAMP_HIGHER_PRECISION) + .setName("test_timezone") + .build()) + .addFields( + TableFieldSchema.newBuilder(TEST_TIMESTAMP_HIGHER_PRECISION) + .setName("test_saformat") + .build()) + .build(); + + TestTimestampHigherPrecision expectedProto = + TestTimestampHigherPrecision.newBuilder() + .setTestString("1970-01-01T00:00:00.000010+00:00") + .setTestStringTZ("2022-03-28T18:47:59.010000+00:00") + .setTestLong("2023-06-28T20:28:05.000000+00:00") + .setTestInt("1970-01-01T00:02:33.480695+00:00") + .setTestFloat("1970-01-02T18:37:48.069500+00:00") + .setTestOffset("2022-04-05T05:06:11.000000+00:00") + .setTestZeroOffset("2022-03-28T18:47:59.010000+00:00") + .setTestTimezone("2022-04-05T16:06:11.000000+00:00") + .setTestSaformat("2018-08-19T12:11:00.000000+00:00") + .build(); + JSONObject json = new JSONObject(); + json.put("test_string", "1970-01-01 00:00:00.000010"); + json.put("test_string_T_Z", "2022-03-28T18:47:59.01Z"); + json.put("test_long", 1687984085000000L); + json.put("test_int", 153480695); + json.put("test_float", "1.534680695e11"); + json.put("test_offset", "2022-04-05T09:06:11+04:00"); + json.put("test_zero_offset", "2022-03-28T18:47:59.01+00:00"); + json.put("test_timezone", "2022-04-05 09:06:11 PST"); + json.put("test_saformat", "2018/08/19 12:11"); + DynamicMessage protoMsg = + JsonToProtoMessage.INSTANCE.convertToProtoMessage( + TestTimestampHigherPrecision.getDescriptor(), tableSchema, json); + assertEquals(expectedProto, protoMsg); + } + @Test public void testTimestampRepeated() throws Exception { TableSchema tableSchema = @@ -946,6 +1033,77 @@ public void testTimestampRepeated() throws Exception { assertEquals(expectedProto, protoMsg); } + @Test + public void testTimestampRepeated_higherPrecision() throws Exception { + TableSchema tableSchema = + TableSchema.newBuilder() + .addFields( + TableFieldSchema.newBuilder(TEST_TIMESTAMP_HIGHER_PRECISION_REPEATED) + .setName("test_string_repeated") + .build()) + .addFields( + TableFieldSchema.newBuilder(TEST_TIMESTAMP_HIGHER_PRECISION_REPEATED) + .setName("test_string_T_Z_repeated") + .build()) + .addFields( + TableFieldSchema.newBuilder(TEST_TIMESTAMP_HIGHER_PRECISION_REPEATED) + .setName("test_long_repeated") + .build()) + .addFields( + TableFieldSchema.newBuilder(TEST_TIMESTAMP_HIGHER_PRECISION_REPEATED) + .setName("test_int_repeated") + .build()) + .addFields( + TableFieldSchema.newBuilder(TEST_TIMESTAMP_HIGHER_PRECISION_REPEATED) + .setName("test_float_repeated") + .build()) + .addFields( + TableFieldSchema.newBuilder(TEST_TIMESTAMP_HIGHER_PRECISION_REPEATED) + .setName("test_offset_repeated") + .build()) + .addFields( + TableFieldSchema.newBuilder(TEST_TIMESTAMP_HIGHER_PRECISION_REPEATED) + .setName("test_zero_offset_repeated") + .build()) + .addFields( + TableFieldSchema.newBuilder(TEST_TIMESTAMP_HIGHER_PRECISION_REPEATED) + .setName("test_timezone_repeated") + .build()) + .addFields( + TableFieldSchema.newBuilder(TEST_TIMESTAMP_HIGHER_PRECISION_REPEATED) + .setName("test_saformat_repeated") + .build()) + .build(); + + TestRepeatedTimestampHigherPrecision expectedProto = + TestRepeatedTimestampHigherPrecision.newBuilder() + .addTestStringRepeated("1970-01-01T00:00:00.000010+00:00") + .addTestStringTZRepeated("2022-03-28T18:47:59.010000+00:00") + .addTestLongRepeated("2023-06-28T20:28:05.000000+00:00") + .addTestIntRepeated("1970-01-01T00:02:33.480695+00:00") + .addTestFloatRepeated("1970-01-02T18:37:48.069500+00:00") + .addTestOffsetRepeated("2022-04-05T05:06:11.000000+00:00") + .addTestZeroOffsetRepeated("2022-03-28T18:47:59.010000+00:00") + .addTestTimezoneRepeated("2022-04-05T16:06:11.000000+00:00") + .addTestSaformatRepeated("2018-08-19T12:11:00.000000+00:00") + .build(); + JSONObject json = new JSONObject(); + json.put("test_string_repeated", new JSONArray(new String[] {"1970-01-01 00:00:00.000010"})); + json.put("test_string_T_Z_repeated", new JSONArray(new String[] {"2022-03-28T18:47:59.01Z"})); + json.put("test_long_repeated", new JSONArray(new Long[] {1687984085000000L})); + json.put("test_int_repeated", new JSONArray(new Integer[] {153480695})); + json.put("test_float_repeated", new JSONArray(new String[] {"1.534680695e11"})); + json.put("test_offset_repeated", new JSONArray(new String[] {"2022-04-05T09:06:11+04:00"})); + json.put( + "test_zero_offset_repeated", new JSONArray(new String[] {"2022-03-28T18:47:59.01+00:00"})); + json.put("test_timezone_repeated", new JSONArray(new String[] {"2022-04-05 09:06:11 PST"})); + json.put("test_saformat_repeated", new JSONArray(new String[] {"2018/08/19 12:11"})); + DynamicMessage protoMsg = + JsonToProtoMessage.INSTANCE.convertToProtoMessage( + TestRepeatedTimestampHigherPrecision.getDescriptor(), tableSchema, json); + assertEquals(expectedProto, protoMsg); + } + @Test public void testDate() throws Exception { TableSchema tableSchema = @@ -1308,6 +1466,7 @@ public void testStructComplex() throws Exception { BigDecimalByteStringEncoder.encodeToBigNumericByteString(new BigDecimal(5D))) .setTestInterval("0-0 0 0:0:0.000005") .addTestJson("{'a':'b'}") + .setTestTimestampHigherPrecision("2025-12-01 12:34:56.123456789123+00:00") .build(); JSONObject complex_lvl2 = new JSONObject(); complex_lvl2.put("test_int", 3); @@ -1373,6 +1532,7 @@ public void testStructComplex() throws Exception { json.put("test_bignumeric_double", 5D); json.put("test_interval", "0-0 0 0:0:0.000005"); json.put("test_json", new JSONArray(new String[] {"{'a':'b'}"})); + json.put("test_timestamp_higher_precision", "2025-12-01 12:34:56.123456789123+00:00"); DynamicMessage protoMsg = JsonToProtoMessage.INSTANCE.convertToProtoMessage( ComplexRoot.getDescriptor(), COMPLEX_TABLE_SCHEMA, json); @@ -1840,36 +2000,37 @@ public void testDoubleAndFloatToRepeatedBigNumericConversion() { @Test public void testGetTimestampAsString() { + // String case must be in ISO8601 format assertEquals( - "2025-10-01 12:34:56.123456+00:00", + "2025-10-01T12:34:56.123456+00:00", JsonToProtoMessage.getTimestampAsString("2025-10-01 12:34:56.123456+00:00")); assertEquals( - "2025-10-01 12:34:56.123456789123+00:00", - JsonToProtoMessage.getTimestampAsString("2025-10-01 12:34:56.123456789123+00:00")); + "2025-10-01T12:34:56.123456789123+00:00", + JsonToProtoMessage.getTimestampAsString("2025-10-01T12:34:56.123456789123+00:00")); - assertEquals("1970-01-01T00:00:00.000001Z", JsonToProtoMessage.getTimestampAsString(1L)); - assertEquals("1969-12-31T23:59:59.999999Z", JsonToProtoMessage.getTimestampAsString(-1L)); + // Numeric case must be micros from epoch + assertEquals("1970-01-01T00:00:00.000001+00:00", JsonToProtoMessage.getTimestampAsString(1L)); + assertEquals("1969-12-31T23:59:59.999999+00:00", JsonToProtoMessage.getTimestampAsString(-1L)); + assertEquals( + "1970-01-01T00:00:00.001234+00:00", JsonToProtoMessage.getTimestampAsString("1234")); + assertEquals("1970-01-01T00:00:00.000010+00:00", JsonToProtoMessage.getTimestampAsString(10.4)); + assertEquals("1969-12-31T23:59:59.999000+00:00", JsonToProtoMessage.getTimestampAsString("-1000.4")); + // Protobuf timestamp format is converted to ISO8601 string assertEquals( - "1970-01-02T10:17:36.000123456Z", + "1970-01-02T10:17:36.000123456+00:00", JsonToProtoMessage.getTimestampAsString( Timestamp.newBuilder().setSeconds(123456).setNanos(123456).build())); assertEquals( - "1969-12-30T13:42:23.999876544Z", + "1969-12-30T13:42:23.999876544+00:00", JsonToProtoMessage.getTimestampAsString( Timestamp.newBuilder().setSeconds(-123456).setNanos(-123456).build())); assertThrows( IllegalArgumentException.class, () -> JsonToProtoMessage.getTimestampAsString("2025-10-01")); - assertThrows( - IllegalArgumentException.class, () -> JsonToProtoMessage.getTimestampAsString("1234")); assertThrows( IllegalArgumentException.class, () -> JsonToProtoMessage.getTimestampAsString("abc")); - assertThrows( - IllegalArgumentException.class, () -> JsonToProtoMessage.getTimestampAsString(10.4)); - assertThrows( - IllegalArgumentException.class, () -> JsonToProtoMessage.getTimestampAsString(-1000.4)); assertThrows( IllegalArgumentException.class, () -> JsonToProtoMessage.getTimestampAsString(Timestamp.newBuilder())); diff --git a/google-cloud-bigquerystorage/src/test/proto/jsonTest.proto b/google-cloud-bigquerystorage/src/test/proto/jsonTest.proto index 84248ad6f3..d878f7bdc9 100644 --- a/google-cloud-bigquerystorage/src/test/proto/jsonTest.proto +++ b/google-cloud-bigquerystorage/src/test/proto/jsonTest.proto @@ -35,6 +35,8 @@ message ComplexRoot { optional bytes test_bignumeric_double = 29; optional string test_interval = 30; repeated string test_json = 31; + optional string test_timestamp_higher_precision = 32; + repeated string test_timestamp_higher_precision_repeated = 33; } message CasingComplex { @@ -157,6 +159,18 @@ message TestTimestamp { optional int64 test_saformat = 9; } +message TestTimestampHigherPrecision { + optional string test_string = 1; + optional string test_string_t_z = 2; + optional string test_long = 3; + optional string test_int = 4; + optional string test_float = 5; + optional string test_offset = 6; + optional string test_zero_offset = 7; + optional string test_timezone = 8; + optional string test_saformat = 9; +} + message TestRepeatedTimestamp { repeated int64 test_string_repeated = 1; repeated int64 test_string_t_z_repeated = 2; @@ -169,6 +183,18 @@ message TestRepeatedTimestamp { repeated int64 test_saformat_repeated = 9; } +message TestRepeatedTimestampHigherPrecision { + repeated string test_string_repeated = 1; + repeated string test_string_t_z_repeated = 2; + repeated string test_long_repeated = 3; + repeated string test_int_repeated = 4; + repeated string test_float_repeated = 5; + repeated string test_offset_repeated = 6; + repeated string test_zero_offset_repeated = 7; + repeated string test_timezone_repeated = 8; + repeated string test_saformat_repeated = 9; +} + message TestDate { optional int32 test_string = 1; optional int32 test_long = 2; @@ -221,7 +247,6 @@ message TestRange { optional TestRangeDate range_date_mixed_case = 4; optional TestRangeDatetime range_datetime_mixed_case = 5; optional TestRangeTimestamp range_timestamp_mixed_case = 6; - optional TestRangeTimestampHigherPrecision range_timestamp_higher_precision_mixed_case = 7; } message TestRangeDate { @@ -238,8 +263,3 @@ message TestRangeTimestamp { optional int64 start = 1; optional int64 end = 2; } - -message TestRangeTimestampHigherPrecision { - optional string start = 1; - optional string end = 2; -} \ No newline at end of file From 78076291493899cba7b315d5542db2464d63ba3d Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Tue, 9 Dec 2025 21:51:34 -0500 Subject: [PATCH 3/8] chore: Fix lint issues --- .../cloud/bigquery/storage/v1/JsonToProtoMessageTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java index d3514e1746..c3ca556993 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java @@ -2014,7 +2014,8 @@ public void testGetTimestampAsString() { assertEquals( "1970-01-01T00:00:00.001234+00:00", JsonToProtoMessage.getTimestampAsString("1234")); assertEquals("1970-01-01T00:00:00.000010+00:00", JsonToProtoMessage.getTimestampAsString(10.4)); - assertEquals("1969-12-31T23:59:59.999000+00:00", JsonToProtoMessage.getTimestampAsString("-1000.4")); + assertEquals( + "1969-12-31T23:59:59.999000+00:00", JsonToProtoMessage.getTimestampAsString("-1000.4")); // Protobuf timestamp format is converted to ISO8601 string assertEquals( From 8acb8900a0980b16f44aaae06d216ce0bea1ddea Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Wed, 10 Dec 2025 10:58:39 -0500 Subject: [PATCH 4/8] chore: Disable check for v1beta2 --- .../v1beta2/BQTableSchemaToProtoDescriptorTest.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/BQTableSchemaToProtoDescriptorTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/BQTableSchemaToProtoDescriptorTest.java index 8e08418237..b8887f2443 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/BQTableSchemaToProtoDescriptorTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/BQTableSchemaToProtoDescriptorTest.java @@ -24,6 +24,8 @@ import com.google.protobuf.Descriptors.FieldDescriptor; import java.util.HashMap; import java.util.Map; + +import com.google.protobuf.Int64Value; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -65,9 +67,9 @@ private void mapDescriptorToCount(Descriptor descriptor, HashMap Date: Wed, 10 Dec 2025 16:01:14 +0000 Subject: [PATCH 5/8] chore: generate libraries at Wed Dec 10 15:59:07 UTC 2025 --- .../storage/v1beta2/BQTableSchemaToProtoDescriptorTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/BQTableSchemaToProtoDescriptorTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/BQTableSchemaToProtoDescriptorTest.java index b8887f2443..06faf91959 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/BQTableSchemaToProtoDescriptorTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/BQTableSchemaToProtoDescriptorTest.java @@ -24,8 +24,6 @@ import com.google.protobuf.Descriptors.FieldDescriptor; import java.util.HashMap; import java.util.Map; - -import com.google.protobuf.Int64Value; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; From f3b380068b227e883351f4fe1eb3b93310f63860 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Wed, 10 Dec 2025 14:00:44 -0500 Subject: [PATCH 6/8] chore: Address PR feedback --- .../v1/BQTableSchemaToProtoDescriptor.java | 41 +++++++------- .../storage/v1/JsonToProtoMessage.java | 13 +++-- .../BQTableSchemaToProtoDescriptorTest.java | 56 +++++++++++-------- 3 files changed, 62 insertions(+), 48 deletions(-) diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptor.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptor.java index 53d835f524..e69e628ef7 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptor.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptor.java @@ -31,6 +31,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.logging.Logger; /** * Converts a BQ table schema to protobuf descriptor. All field names will be converted to lowercase @@ -38,6 +39,10 @@ * shown in the ImmutableMaps below. */ public class BQTableSchemaToProtoDescriptor { + + private static final Logger LOG = + Logger.getLogger(BQTableSchemaToProtoDescriptor.class.getName()); + private static Map DEFAULT_BQ_TABLE_SCHEMA_MODE_MAP = ImmutableMap.of( TableFieldSchema.Mode.NULLABLE, FieldDescriptorProto.Label.LABEL_OPTIONAL, @@ -215,26 +220,24 @@ static FieldDescriptorProto convertBQTableFieldToProtoField( case TIMESTAMP: // Can map to either int64 or string based on the BQ Field's timestamp precision // Default: microsecond (6) maps to int64 and picosecond (12) maps to string. - switch ((int) BQTableField.getTimestampPrecision().getValue()) { - case 12: - fieldDescriptor.setType( - (FieldDescriptorProto.Type) FieldDescriptorProto.Type.TYPE_STRING); - break; - case 6: - case 0: - // If the timestampPrecision value coems back as a null result from the server, a - // default value - // of 0L is set. Map this value as default precision as 6 (microsecond). - fieldDescriptor.setType( - (FieldDescriptorProto.Type) FieldDescriptorProto.Type.TYPE_INT64); - break; - default: - // This should never happen as it's an invalid value from server - throw new IllegalStateException( - "BigQuery Timestamp field " - + BQTableField.getName() - + " has timestamp precision that is not 6 or 12"); + long timestampPrecision = BQTableField.getTimestampPrecision().getValue(); + if (timestampPrecision == 12L) { + fieldDescriptor.setType( + (FieldDescriptorProto.Type) FieldDescriptorProto.Type.TYPE_STRING); + break; + } + // This should never happen as this is a server response issue. If this is the case, + // warn the user and use INT64 as the default is microsecond precision. + if (timestampPrecision != 6L || timestampPrecision != 0L) { + LOG.warning( + "BigQuery Timestamp field " + + BQTableField.getName() + + " has timestamp precision that is not 6 or 12. Defaulting to microsecond precision and mapping to INT64 protobuf type."); } + // If the timestampPrecision value comes back as a null result from the server, + // timestampPrecision has a value of 0L. Use the INT64 to map to the type used + // for the default precision (microsecond). + fieldDescriptor.setType((FieldDescriptorProto.Type) FieldDescriptorProto.Type.TYPE_INT64); break; default: fieldDescriptor.setType( diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java index 2dd685d66d..6e5643f002 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java @@ -1041,7 +1041,11 @@ static Instant fromEpochMicros(long micros) { return Instant.ofEpochSecond(seconds, nanos); } - /** Best effort to try and convert a timestamp to an ISO8601 string */ + /** + * Best effort to try and convert a timestamp to an ISO8601 string. Standardize the timestamp + * output to be ISO_DATE_TIME (e.g. 2011-12-03T10:15:30+01:00) for timestamps up to nanosecond + * precision. For higher precision, the ISO8601 input is used as long as it is valid. + */ @VisibleForTesting static String getTimestampAsString(Object val) { if (val instanceof String) { @@ -1051,8 +1055,7 @@ static String getTimestampAsString(Object val) { if (parsed != null) { return getTimestampAsString(parsed.longValue()); } - // Validate the ISO8601 values before sending it to the server. No need to format - // if it's valid. + // Validate the ISO8601 values before sending it to the server. validateTimestamp(value); // If it's high precision (more than 9 digits), then return the ISO8601 string as-is @@ -1065,8 +1068,8 @@ static String getTimestampAsString(Object val) { Instant instant = FROM_TIMESTAMP_FORMATTER.parse(value, Instant::from); return TO_TIMESTAMP_FORMATTER.format(instant); } else if (val instanceof Number) { - // Micros from epoch will most likely will be represented a Long, but any non-float - // numeric value can be used + // Micros from epoch will most likely will be represented a Long, but any numeric + // value can be used Instant instant = fromEpochMicros(((Number) val).longValue()); return TO_TIMESTAMP_FORMATTER.format(instant); } else if (val instanceof Timestamp) { diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptorTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptorTest.java index 23484050a8..51b78df183 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptorTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptorTest.java @@ -17,7 +17,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import com.google.cloud.bigquery.storage.test.JsonTest; @@ -71,9 +70,9 @@ private void mapDescriptorToCount(Descriptor descriptor, HashMap - BQTableSchemaToProtoDescriptor.convertBQTableFieldToProtoField( - timestampField, 0, null)); + DescriptorProtos.FieldDescriptorProto fieldDescriptorProto = + BQTableSchemaToProtoDescriptor.convertBQTableFieldToProtoField(timestampField, 0, null); + assertEquals( + DescriptorProtos.FieldDescriptorProto.Type.TYPE_INT64, fieldDescriptorProto.getType()); TableFieldSchema timestampField1 = TableFieldSchema.newBuilder() @@ -710,10 +708,20 @@ public void timestampField_picosecondPrecision_invalid() throws Exception { .setTimestampPrecision(Int64Value.newBuilder().setValue(7).build()) .setMode(TableFieldSchema.Mode.NULLABLE) .build(); - assertThrows( - IllegalStateException.class, - () -> - BQTableSchemaToProtoDescriptor.convertBQTableFieldToProtoField( - timestampField1, 0, null)); + DescriptorProtos.FieldDescriptorProto fieldDescriptorProto1 = + BQTableSchemaToProtoDescriptor.convertBQTableFieldToProtoField(timestampField1, 0, null); + assertEquals( + DescriptorProtos.FieldDescriptorProto.Type.TYPE_INT64, fieldDescriptorProto1.getType()); + + TableFieldSchema timestampField2 = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.TIMESTAMP) + .setTimestampPrecision(Int64Value.newBuilder().setValue(-1).build()) + .setMode(TableFieldSchema.Mode.NULLABLE) + .build(); + DescriptorProtos.FieldDescriptorProto fieldDescriptorProto2 = + BQTableSchemaToProtoDescriptor.convertBQTableFieldToProtoField(timestampField2, 0, null); + assertEquals( + DescriptorProtos.FieldDescriptorProto.Type.TYPE_INT64, fieldDescriptorProto2.getType()); } } From 37c2fabdccefb8378e4c449023a0fe50b2db0663 Mon Sep 17 00:00:00 2001 From: cloud-java-bot Date: Wed, 10 Dec 2025 19:03:27 +0000 Subject: [PATCH 7/8] chore: generate libraries at Wed Dec 10 19:01:14 UTC 2025 --- .../bigquery/storage/v1/BQTableSchemaToProtoDescriptor.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptor.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptor.java index e69e628ef7..af405ae4b5 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptor.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptor.java @@ -232,7 +232,8 @@ static FieldDescriptorProto convertBQTableFieldToProtoField( LOG.warning( "BigQuery Timestamp field " + BQTableField.getName() - + " has timestamp precision that is not 6 or 12. Defaulting to microsecond precision and mapping to INT64 protobuf type."); + + " has timestamp precision that is not 6 or 12. Defaulting to microsecond" + + " precision and mapping to INT64 protobuf type."); } // If the timestampPrecision value comes back as a null result from the server, // timestampPrecision has a value of 0L. Use the INT64 to map to the type used From 19021a832c06bc624c756a50e155666606a48ae9 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Wed, 10 Dec 2025 15:19:32 -0500 Subject: [PATCH 8/8] chore: Fix timestamp precision check condition --- .../bigquery/storage/v1/BQTableSchemaToProtoDescriptor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptor.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptor.java index af405ae4b5..5842f6d068 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptor.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptor.java @@ -228,7 +228,7 @@ static FieldDescriptorProto convertBQTableFieldToProtoField( } // This should never happen as this is a server response issue. If this is the case, // warn the user and use INT64 as the default is microsecond precision. - if (timestampPrecision != 6L || timestampPrecision != 0L) { + if (timestampPrecision != 6L && timestampPrecision != 0L) { LOG.warning( "BigQuery Timestamp field " + BQTableField.getName()