From e222007fb501d30215a365cf43f382da29f37690 Mon Sep 17 00:00:00 2001 From: Matthew Pope Date: Wed, 18 Oct 2023 14:07:58 -0700 Subject: [PATCH 1/4] Adding support for timestamps --- .../amazon/ion/impl/bin/IonEncoder_1_1.java | 218 +++++++++++++++- .../ion/impl/bin/Ion_1_1_Constants.java | 36 +++ src/com/amazon/ion/impl/bin/OpCodes.java | 15 ++ src/com/amazon/ion/impl/bin/WriteBuffer.java | 35 ++- .../ion/impl/bin/IonEncoder_1_1Test.java | 245 +++++++++++++++++- .../amazon/ion/impl/bin/WriteBufferTest.java | 32 ++- 6 files changed, 567 insertions(+), 14 deletions(-) create mode 100644 src/com/amazon/ion/impl/bin/Ion_1_1_Constants.java diff --git a/src/com/amazon/ion/impl/bin/IonEncoder_1_1.java b/src/com/amazon/ion/impl/bin/IonEncoder_1_1.java index cca35421f7..3ec31c03f1 100644 --- a/src/com/amazon/ion/impl/bin/IonEncoder_1_1.java +++ b/src/com/amazon/ion/impl/bin/IonEncoder_1_1.java @@ -1,13 +1,12 @@ package com.amazon.ion.impl.bin; -import com.amazon.ion.Decimal; import com.amazon.ion.IonType; import com.amazon.ion.Timestamp; -import com.amazon.ion.impl.bin.utf8.Utf8StringEncoder; import java.math.BigDecimal; import java.math.BigInteger; +import static com.amazon.ion.impl.bin.Ion_1_1_Constants.*; import static java.lang.Double.doubleToRawLongBits; import static java.lang.Float.floatToIntBits; @@ -158,4 +157,219 @@ public static int writeFloat(WriteBuffer buffer, final double value) { return 9; } } + + /** + * Writes a Timestamp to the given WriteBuffer using the Ion 1.1 encoding for Ion Timestamps. + * @return the number of bytes written + */ + public static int writeTimestampValue(WriteBuffer buffer, Timestamp value) { + if (value == null) { + return writeNullValue(buffer, IonType.TIMESTAMP); + } + // Timestamps may be encoded using the short form if they meet certain conditions. + // Condition 1: The year is between 1970 and 2097. + if (value.getYear() < 1970 || value.getYear() > 2097) { + return writeLongFormTimestampValue(buffer, value); + } + + // If the precision is year, month, or day, we can skip the remaining checks. + if (!value.getPrecision().includes(Timestamp.Precision.MINUTE)) { + return writeShortFormTimestampValue(buffer, value); + } + + // Condition 2: The fractional seconds are a common precision. + int secondsScale = value.getDecimalSecond().scale(); + if (secondsScale != 0 && secondsScale != 3 && secondsScale != 6 && secondsScale != 9) { + return writeLongFormTimestampValue(buffer, value); + } + // Condition 3: The local offset is either UTC, unknown, or falls between -14:00 to +14:00 and is divisible by 15 minutes. + Integer offset = value.getLocalOffset(); + if (offset != null && (offset < -14 * 60 || offset > 14 * 60 || offset % 15 != 0)) { + return writeLongFormTimestampValue(buffer, value); + } + return writeShortFormTimestampValue(buffer, value); + } + + /** + * Writes a short-form timestamp. + * Value cannot be null. + * If calling from outside this class, use writeTimestampValue instead. + */ + private static int writeShortFormTimestampValue(WriteBuffer buffer, Timestamp value) { + long bits = (value.getYear() - 1970L); + if (value.getPrecision() == Timestamp.Precision.YEAR) { + buffer.writeByte(OpCodes.TIMESTAMP_YEAR_PRECISION); + buffer.writeFixedIntOrUInt(bits, 1); + return 2; + } + + bits |= ((long) value.getMonth()) << S_TIMESTAMP_MONTH_BIT_OFFSET; + if (value.getPrecision() == Timestamp.Precision.MONTH) { + buffer.writeByte(OpCodes.TIMESTAMP_MONTH_PRECISION); + buffer.writeFixedIntOrUInt(bits, 2); + return 3; + } + + bits |= ((long) value.getDay()) << S_TIMESTAMP_DAY_BIT_OFFSET; + if (value.getPrecision() == Timestamp.Precision.DAY) { + buffer.writeByte(OpCodes.TIMESTAMP_DAY_PRECISION); + buffer.writeFixedIntOrUInt(bits, 2); + return 3; + } + + bits |= ((long) value.getHour()) << S_TIMESTAMP_HOUR_BIT_OFFSET; + bits |= ((long) value.getMinute()) << S_TIMESTAMP_MINUTE_BIT_OFFSET; + if (value.getLocalOffset() == null || value.getLocalOffset() == 0) { + if (value.getLocalOffset() != null) { + bits |= S_U_TIMESTAMP_UTC_FLAG; + } + + if (value.getPrecision() == Timestamp.Precision.MINUTE) { + buffer.writeByte(OpCodes.TIMESTAMP_MINUTE_PRECISION); + buffer.writeFixedIntOrUInt(bits, 4); + return 5; + } + + bits |= ((long) value.getSecond()) << S_U_TIMESTAMP_SECOND_BIT_OFFSET; + + int secondsScale = value.getDecimalSecond().scale(); + if (secondsScale != 0) { + long fractionalSeconds = value.getDecimalSecond().remainder(BigDecimal.ONE).movePointRight(secondsScale).longValue(); + bits |= fractionalSeconds << S_U_TIMESTAMP_FRACTION_BIT_OFFSET; + } + switch (secondsScale) { + case 0: + buffer.writeByte(OpCodes.TIMESTAMP_SECOND_PRECISION); + buffer.writeFixedIntOrUInt(bits, 5); + return 6; + case 3: + buffer.writeByte(OpCodes.TIMESTAMP_MILLIS_PRECISION); + buffer.writeFixedIntOrUInt(bits, 6); + return 7; + case 6: + buffer.writeByte(OpCodes.TIMESTAMP_MICROS_PRECISION); + buffer.writeFixedIntOrUInt(bits, 7); + return 8; + case 9: + buffer.writeByte(OpCodes.TIMESTAMP_NANOS_PRECISION); + buffer.writeFixedIntOrUInt(bits, 8); + return 9; + default: + throw new IllegalStateException("This is unreachable!"); + } + } else { + long localOffset = value.getLocalOffset().longValue() / 15; + bits |= (localOffset & LEAST_SIGNIFICANT_7_BITS) << S_O_TIMESTAMP_OFFSET_BIT_OFFSET; + + if (value.getPrecision() == Timestamp.Precision.MINUTE) { + buffer.writeByte(OpCodes.TIMESTAMP_MINUTE_PRECISION_WITH_OFFSET); + buffer.writeFixedIntOrUInt(bits, 5); + return 6; + } + + bits |= ((long) value.getSecond()) << S_O_TIMESTAMP_SECOND_BIT_OFFSET; + + // The fractional seconds bits will be put into a separate long because we need nine bytes total + // if there are nanoseconds (which is too much for one long) and the boundary between the seconds + // and fractional seconds subfields conveniently aligns with a byte boundary. + long fractionBits = 0; + int secondsScale = value.getDecimalSecond().scale(); + if (secondsScale != 0) { + fractionBits = value.getDecimalSecond().remainder(BigDecimal.ONE).movePointRight(secondsScale).longValue(); + } + switch (secondsScale) { + case 0: + buffer.writeByte(OpCodes.TIMESTAMP_SECOND_PRECISION_WITH_OFFSET); + buffer.writeFixedIntOrUInt(bits, 5); + return 6; + case 3: + buffer.writeByte(OpCodes.TIMESTAMP_MILLIS_PRECISION_WITH_OFFSET); + buffer.writeFixedIntOrUInt(bits, 5); + buffer.writeFixedIntOrUInt(fractionBits, 2); + return 8; + case 6: + buffer.writeByte(OpCodes.TIMESTAMP_MICROS_PRECISION_WITH_OFFSET); + buffer.writeFixedIntOrUInt(bits, 5); + buffer.writeFixedIntOrUInt(fractionBits, 3); + return 9; + case 9: + buffer.writeByte(OpCodes.TIMESTAMP_NANOS_PRECISION_WITH_OFFSET); + buffer.writeFixedIntOrUInt(bits, 5); + buffer.writeFixedIntOrUInt(fractionBits, 4); + return 10; + default: + throw new IllegalStateException("This is unreachable!"); + } + } + } + + /** + * Writes a long-form timestamp. + * Value may not be null. + * Only visible for testing. If calling from outside this class, use writeTimestampValue instead. + */ + static int writeLongFormTimestampValue(WriteBuffer buffer, Timestamp value) { + buffer.writeByte(OpCodes.VARIABLE_LENGTH_TIMESTAMP); + + long bits = value.getYear(); + if (value.getPrecision() == Timestamp.Precision.YEAR) { + buffer.writeFlexUInt(2); + buffer.writeFixedIntOrUInt(bits, 2); + return 4; // OpCode + FlexUInt + 2 bytes data + } + + bits |= ((long) value.getMonth()) << L_TIMESTAMP_MONTH_BIT_OFFSET; + if (value.getPrecision() == Timestamp.Precision.MONTH) { + buffer.writeFlexUInt(3); + buffer.writeFixedIntOrUInt(bits, 3); + return 5; // OpCode + FlexUInt + 3 bytes data + } + + bits |= ((long) value.getDay()) << L_TIMESTAMP_DAY_BIT_OFFSET; + if (value.getPrecision() == Timestamp.Precision.DAY) { + buffer.writeFlexUInt(3); + buffer.writeFixedIntOrUInt(bits, 3); + return 5; // OpCode + FlexUInt + 3 bytes data + } + + bits |= ((long) value.getHour()) << L_TIMESTAMP_HOUR_BIT_OFFSET; + bits |= ((long) value.getMinute()) << L_TIMESTAMP_MINUTE_BIT_OFFSET; + long localOffsetValue = L_TIMESTAMP_UNKNOWN_OFFSET_VALUE; + if (value.getLocalOffset() != null) { + localOffsetValue = value.getLocalOffset() + (24 * 60); + } + bits |= localOffsetValue << L_TIMESTAMP_OFFSET_BIT_OFFSET; + + if (value.getPrecision() == Timestamp.Precision.MINUTE) { + buffer.writeFlexUInt(6); + buffer.writeFixedIntOrUInt(bits, 6); + return 8; // OpCode + FlexUInt + 6 bytes data + } + + + bits |= ((long) value.getSecond()) << L_TIMESTAMP_SECOND_BIT_OFFSET; + int secondsScale = value.getDecimalSecond().scale(); + if (secondsScale == 0) { + buffer.writeFlexUInt(7); + buffer.writeFixedIntOrUInt(bits, 7); + return 9; // OpCode + FlexUInt + 7 bytes data + } + + BigDecimal fractionalSeconds = value.getDecimalSecond().remainder(BigDecimal.ONE); + BigInteger coefficient = fractionalSeconds.unscaledValue(); + long exponent = fractionalSeconds.scale(); + int numCoefficientBytes = WriteBuffer.flexUIntLength(coefficient); + int numExponentBytes = WriteBuffer.fixedUIntLength(exponent); + // Years-seconds data (7 bytes) + fraction coefficient + fraction exponent + int dataLength = 7 + numCoefficientBytes + numExponentBytes; + + buffer.writeFlexUInt(dataLength); + buffer.writeFixedIntOrUInt(bits, 7); + buffer.writeFlexUInt(coefficient); + buffer.writeFixedUInt(exponent); + + // OpCode + FlexUInt length + dataLength + return 1 + WriteBuffer.flexUIntLength(dataLength) + dataLength; + } + } diff --git a/src/com/amazon/ion/impl/bin/Ion_1_1_Constants.java b/src/com/amazon/ion/impl/bin/Ion_1_1_Constants.java new file mode 100644 index 0000000000..e0c031ee93 --- /dev/null +++ b/src/com/amazon/ion/impl/bin/Ion_1_1_Constants.java @@ -0,0 +1,36 @@ +package com.amazon.ion.impl.bin; + +/** + * Contains constants (other than OpCodes) which are generally applicable to both reading and writing binary Ion 1.1 + */ +public class Ion_1_1_Constants { + private Ion_1_1_Constants() {} + + //////// Timestamp Field Constants //////// + + // S_TIMESTAMP_* is applicable to all short-form timestamps + static final int S_TIMESTAMP_MONTH_BIT_OFFSET = 7; + static final int S_TIMESTAMP_DAY_BIT_OFFSET = 11; + static final int S_TIMESTAMP_HOUR_BIT_OFFSET = 16; + static final int S_TIMESTAMP_MINUTE_BIT_OFFSET = 21; + // S_U_TIMESTAMP_* is applicable to all short-form timestamps with a `U` bit + static final int S_U_TIMESTAMP_UTC_FLAG = 1 << 27; + static final int S_U_TIMESTAMP_SECOND_BIT_OFFSET = 28; + static final int S_U_TIMESTAMP_FRACTION_BIT_OFFSET = 34; + // S_O_TIMESTAMP_* is applicable to all short-form timestamps with `o` (offset) bits + static final int S_O_TIMESTAMP_OFFSET_BIT_OFFSET = 27; + static final int S_O_TIMESTAMP_SECOND_BIT_OFFSET = 34; + + // L_TIMESTAMP_* is applicable to all long-form timestamps + static final int L_TIMESTAMP_MONTH_BIT_OFFSET = 14; + static final int L_TIMESTAMP_DAY_BIT_OFFSET = 18; + static final int L_TIMESTAMP_HOUR_BIT_OFFSET = 23; + static final int L_TIMESTAMP_MINUTE_BIT_OFFSET = 28; + static final int L_TIMESTAMP_OFFSET_BIT_OFFSET = 34; + static final int L_TIMESTAMP_SECOND_BIT_OFFSET = 44; + static final int L_TIMESTAMP_UNKNOWN_OFFSET_VALUE = 0b111111111111; + + //////// Bit masks //////// + + static final long LEAST_SIGNIFICANT_7_BITS = 0b01111111L; +} diff --git a/src/com/amazon/ion/impl/bin/OpCodes.java b/src/com/amazon/ion/impl/bin/OpCodes.java index 0ea5bf843d..4e3f3c6468 100644 --- a/src/com/amazon/ion/impl/bin/OpCodes.java +++ b/src/com/amazon/ion/impl/bin/OpCodes.java @@ -20,9 +20,24 @@ private OpCodes() {} // 0x61-0x6E are additional lengths of decimals. public static final byte NEGATIVE_ZERO_DECIMAL = 0x6F; + public static final byte TIMESTAMP_YEAR_PRECISION = 0x70; + public static final byte TIMESTAMP_MONTH_PRECISION = 0x71; + public static final byte TIMESTAMP_DAY_PRECISION = 0x72; + public static final byte TIMESTAMP_MINUTE_PRECISION = 0x73; + public static final byte TIMESTAMP_SECOND_PRECISION = 0x74; + public static final byte TIMESTAMP_MILLIS_PRECISION = 0x75; + public static final byte TIMESTAMP_MICROS_PRECISION = 0x76; + public static final byte TIMESTAMP_NANOS_PRECISION = 0x77; + public static final byte TIMESTAMP_MINUTE_PRECISION_WITH_OFFSET = 0x78; + public static final byte TIMESTAMP_SECOND_PRECISION_WITH_OFFSET = 0x79; + public static final byte TIMESTAMP_MILLIS_PRECISION_WITH_OFFSET = 0x7A; + public static final byte TIMESTAMP_MICROS_PRECISION_WITH_OFFSET = 0x7B; + public static final byte TIMESTAMP_NANOS_PRECISION_WITH_OFFSET = 0x7C; + // 0x7D-0x7F Reserved public static final byte NULL_UNTYPED = (byte) 0xEA; public static final byte NULL_TYPED = (byte) 0xEB; public static final byte VARIABLE_LENGTH_INTEGER = (byte) 0xF5; + public static final byte VARIABLE_LENGTH_TIMESTAMP = (byte) 0xF7; } diff --git a/src/com/amazon/ion/impl/bin/WriteBuffer.java b/src/com/amazon/ion/impl/bin/WriteBuffer.java index 5a493e8ca0..2aa1496a09 100644 --- a/src/com/amazon/ion/impl/bin/WriteBuffer.java +++ b/src/com/amazon/ion/impl/bin/WriteBuffer.java @@ -1437,7 +1437,7 @@ public static int fixedIntLength(final long value) { */ public int writeFixedInt(final long value) { int numBytes = fixedIntLength(value); - return writeFixedIntOrUInt(value, numBytes); + return _writeFixedIntOrUInt(value, numBytes); } /** Get the length of FixedUInt for the provided value. */ @@ -1453,17 +1453,40 @@ public static int fixedUIntLength(final long value) { */ public int writeFixedUInt(final long value) { if (value < 0) { - throw new IllegalArgumentException("Attempted to write a FlexUInt for " + value); + throw new IllegalArgumentException("Attempted to write a FixedUInt for " + value); } int numBytes = fixedUIntLength(value); - return writeFixedIntOrUInt(value, numBytes); + return _writeFixedIntOrUInt(value, numBytes); } /** - * Because the fixed int and fixed uint encodings are so similar, we can use this method to write either one as long - * as we provide the correct number of bytes needed to encode the value. + * Writes the bytes of a {@code long} as a {@code FixedInt} or {@code FixedUInt} using {@code numBytes} bytes. + *

+ * {@code numBytes} should be an integer from 1 to 8 inclusive. If {@code numBytes} is out of bounds, that is a + * programmer error and will result in an IllegalArgumentException. + *

+ * Because the {@code FixedInt} and {@code FixedUInt} encodings are so similar, we can use this method to write + * either one as long as we provide the correct number of bytes needed to encode the value. + *

+ * Most of the time, you should not use this method. Instead, use {@link WriteBuffer#writeFixedInt} or + * {@link WriteBuffer#writeFixedUInt}, which calculate the minimum number of required bytes to represent the value. + *

+ * You should use this method when the spec requires a {@code FixedInt} or {@code FixedUInt} of a specific + * size when it's possible that the value could fit in a smaller FixedInt or FixedUInt than the size required in + * the spec. + */ + public int writeFixedIntOrUInt(final long value, final int numBytes) { + if (0 > numBytes || numBytes > 8) { + throw new IllegalArgumentException("numBytes is out of bounds; was " + numBytes); + } + return _writeFixedIntOrUInt(value, numBytes); + } + + /** + * Because the {@code FixedInt} and {@code FixedUInt} encodings are so similar, we can use this method to write + * either one as long as we provide the correct number of bytes needed to encode the value. */ - private int writeFixedIntOrUInt(final long value, final int numBytes) { + private int _writeFixedIntOrUInt(final long value, final int numBytes) { writeByte((byte) value); if (numBytes > 1) { writeByte((byte) (value >> 8)); diff --git a/test/com/amazon/ion/impl/bin/IonEncoder_1_1Test.java b/test/com/amazon/ion/impl/bin/IonEncoder_1_1Test.java index 4d26345d77..0916e70b9c 100644 --- a/test/com/amazon/ion/impl/bin/IonEncoder_1_1Test.java +++ b/test/com/amazon/ion/impl/bin/IonEncoder_1_1Test.java @@ -1,10 +1,14 @@ package com.amazon.ion.impl.bin; import com.amazon.ion.IonType; +import com.amazon.ion.Timestamp; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.converter.ArgumentConversionException; +import org.junit.jupiter.params.converter.ConvertWith; +import org.junit.jupiter.params.converter.TypedArgumentConverter; import org.junit.jupiter.params.provider.CsvSource; import java.io.ByteArrayOutputStream; @@ -33,8 +37,8 @@ private byte[] bytes() { } /** - * Checks that the function writes the expected bytes and returns the expected - * count of written bytes for the given input value. + * Checks that the function writes the expected bytes and returns the expected count of written bytes for the + * given input value. The expected bytes should be a string of space-separated hexadecimal pairs. */ private void assertWritingValue(String expectedBytes, T value, BiFunction writeOperation) { int numBytes = writeOperation.apply(buf, value); @@ -42,6 +46,16 @@ private void assertWritingValue(String expectedBytes, T value, BiFunction void assertWritingValueWithBinary(String expectedBytes, T value, BiFunction writeOperation) { + int numBytes = writeOperation.apply(buf, value); + Assertions.assertEquals(expectedBytes, byteArrayToBitString(bytes())); + Assertions.assertEquals(byteLengthFromBitString(expectedBytes), numBytes); + } + @ParameterizedTest @CsvSource({ " NULL, EA", @@ -72,7 +86,7 @@ public void testWriteNullValueForDatagram() { "true, 5E", "false, 5F", }) - public void testWriteBooleanValue(Boolean value, String expectedBytes) { + public void testWriteBooleanValue(boolean value, String expectedBytes) { assertWritingValue(expectedBytes, value, IonEncoder_1_1::writeBoolValue); } @@ -144,8 +158,8 @@ public void testWriteIntegerValue(long value, String expectedBytes) { " -9223372036854775809, F5 13 FF FF FF FF FF FF FF 7F FF", "-99999999999999999999999999999, F5 1B 01 00 00 60 35 E8 8D 92 51 F0 E1 BC FE", }) - public void testWriteIntegerValueForBigInteger(String value, String expectedBytes) { - assertWritingValue(expectedBytes, new BigInteger(value), IonEncoder_1_1::writeIntValue); + public void testWriteIntegerValueForBigInteger(BigInteger value, String expectedBytes) { + assertWritingValue(expectedBytes, value, IonEncoder_1_1::writeIntValue); } @Test @@ -205,6 +219,186 @@ public void testWriteFloatValueForDouble(double value, String expectedBytes) { assertWritingValue(expectedBytes, value, IonEncoder_1_1::writeFloat); } + // Because timestamp subfields are smeared across bytes, it's easier to reason about them in 1s and 0s + // instead of hex digits + @ParameterizedTest + @CsvSource({ + // OpCode MYYYYYYY DDDDDMMM mmmHHHHH ssssUmmm ffffffss ffffffff ffffffff ffffffff + "2023-10-15T01:00Z, 01110011 00110101 01111101 00000001 00001000", + "2023-10-15T01:59Z, 01110011 00110101 01111101 01100001 00001111", + "2023-10-15T11:22Z, 01110011 00110101 01111101 11001011 00001010", + "2023-10-15T23:00Z, 01110011 00110101 01111101 00010111 00001000", + "2023-10-15T23:59Z, 01110011 00110101 01111101 01110111 00001111", + "2023-10-15T11:22:00Z, 01110100 00110101 01111101 11001011 00001010 00000000", + "2023-10-15T11:22:33Z, 01110100 00110101 01111101 11001011 00011010 00000010", + "2023-10-15T11:22:59Z, 01110100 00110101 01111101 11001011 10111010 00000011", + "2023-10-15T11:22:33.000Z, 01110101 00110101 01111101 11001011 00011010 00000010 00000000", + "2023-10-15T11:22:33.444Z, 01110101 00110101 01111101 11001011 00011010 11110010 00000110", + "2023-10-15T11:22:33.999Z, 01110101 00110101 01111101 11001011 00011010 10011110 00001111", + "2023-10-15T11:22:33.000000Z, 01110110 00110101 01111101 11001011 00011010 00000010 00000000 00000000", + "2023-10-15T11:22:33.444555Z, 01110110 00110101 01111101 11001011 00011010 00101110 00100010 00011011", + "2023-10-15T11:22:33.999999Z, 01110110 00110101 01111101 11001011 00011010 11111110 00001000 00111101", + "2023-10-15T11:22:33.000000000Z, 01110111 00110101 01111101 11001011 00011010 00000010 00000000 00000000 00000000", + "2023-10-15T11:22:33.444555666Z, 01110111 00110101 01111101 11001011 00011010 01001010 10000110 11111101 01101001", + "2023-10-15T11:22:33.999999999Z, 01110111 00110101 01111101 11001011 00011010 11111110 00100111 01101011 11101110", + }) + public void testWriteTimestampValueWithUtcShortForm(@ConvertWith(StringToTimestamp.class) Timestamp value, String expectedBytes) { + assertWritingValueWithBinary(expectedBytes, value, IonEncoder_1_1::writeTimestampValue); + } + + + @ParameterizedTest + @CsvSource({ + // OpCode MYYYYYYY DDDDDMMM mmmHHHHH ssssUmmm ffffffss ffffffff ffffffff ffffffff + "1970T, 01110000 00000000", + "2023T, 01110000 00110101", + "2097T, 01110000 01111111", + "2023-01T, 01110001 10110101 00000000", + "2023-10T, 01110001 00110101 00000101", + "2023-12T, 01110001 00110101 00000110", + "2023-10-01T, 01110010 00110101 00001101", + "2023-10-15T, 01110010 00110101 01111101", + "2023-10-31T, 01110010 00110101 11111101", + "2023-10-15T01:00-00:00, 01110011 00110101 01111101 00000001 00000000", + "2023-10-15T01:59-00:00, 01110011 00110101 01111101 01100001 00000111", + "2023-10-15T11:22-00:00, 01110011 00110101 01111101 11001011 00000010", + "2023-10-15T23:00-00:00, 01110011 00110101 01111101 00010111 00000000", + "2023-10-15T23:59-00:00, 01110011 00110101 01111101 01110111 00000111", + "2023-10-15T11:22:00-00:00, 01110100 00110101 01111101 11001011 00000010 00000000", + "2023-10-15T11:22:33-00:00, 01110100 00110101 01111101 11001011 00010010 00000010", + "2023-10-15T11:22:59-00:00, 01110100 00110101 01111101 11001011 10110010 00000011", + "2023-10-15T11:22:33.000-00:00, 01110101 00110101 01111101 11001011 00010010 00000010 00000000", + "2023-10-15T11:22:33.444-00:00, 01110101 00110101 01111101 11001011 00010010 11110010 00000110", + "2023-10-15T11:22:33.999-00:00, 01110101 00110101 01111101 11001011 00010010 10011110 00001111", + "2023-10-15T11:22:33.000000-00:00, 01110110 00110101 01111101 11001011 00010010 00000010 00000000 00000000", + "2023-10-15T11:22:33.444555-00:00, 01110110 00110101 01111101 11001011 00010010 00101110 00100010 00011011", + "2023-10-15T11:22:33.999999-00:00, 01110110 00110101 01111101 11001011 00010010 11111110 00001000 00111101", + "2023-10-15T11:22:33.000000000-00:00, 01110111 00110101 01111101 11001011 00010010 00000010 00000000 00000000 00000000", + "2023-10-15T11:22:33.444555666-00:00, 01110111 00110101 01111101 11001011 00010010 01001010 10000110 11111101 01101001", + "2023-10-15T11:22:33.999999999-00:00, 01110111 00110101 01111101 11001011 00010010 11111110 00100111 01101011 11101110", + }) + public void testWriteTimestampValueWithUnknownOffsetShortForm(@ConvertWith(StringToTimestamp.class) Timestamp value, String expectedBytes) { + assertWritingValueWithBinary(expectedBytes, value, IonEncoder_1_1::writeTimestampValue); + } + + @ParameterizedTest + @CsvSource({ + // OpCode MYYYYYYY DDDDDMMM mmmHHHHH ooooommm ssssssoo ffffffff ffffffff ffffffff ..ffffff + "2023-10-15T01:00-14:00, 01111000 00110101 01111101 00000001 01000000 00000010", + "2023-10-15T01:00+14:00, 01111000 00110101 01111101 00000001 11000000 00000001", + "2023-10-15T01:00-01:15, 01111000 00110101 01111101 00000001 11011000 00000011", + "2023-10-15T01:00+01:15, 01111000 00110101 01111101 00000001 00101000 00000000", + "2023-10-15T01:59+01:15, 01111000 00110101 01111101 01100001 00101111 00000000", + "2023-10-15T11:22+01:15, 01111000 00110101 01111101 11001011 00101010 00000000", + "2023-10-15T23:00+01:15, 01111000 00110101 01111101 00010111 00101000 00000000", + "2023-10-15T23:59+01:15, 01111000 00110101 01111101 01110111 00101111 00000000", + "2023-10-15T11:22:00+01:15, 01111001 00110101 01111101 11001011 00101010 00000000", + "2023-10-15T11:22:33+01:15, 01111001 00110101 01111101 11001011 00101010 10000100", + "2023-10-15T11:22:59+01:15, 01111001 00110101 01111101 11001011 00101010 11101100", + "2023-10-15T11:22:33.000+01:15, 01111010 00110101 01111101 11001011 00101010 10000100 00000000 00000000", + "2023-10-15T11:22:33.444+01:15, 01111010 00110101 01111101 11001011 00101010 10000100 10111100 00000001", + "2023-10-15T11:22:33.999+01:15, 01111010 00110101 01111101 11001011 00101010 10000100 11100111 00000011", + "2023-10-15T11:22:33.000000+01:15, 01111011 00110101 01111101 11001011 00101010 10000100 00000000 00000000 00000000", + "2023-10-15T11:22:33.444555+01:15, 01111011 00110101 01111101 11001011 00101010 10000100 10001011 11001000 00000110", + "2023-10-15T11:22:33.999999+01:15, 01111011 00110101 01111101 11001011 00101010 10000100 00111111 01000010 00001111", + "2023-10-15T11:22:33.000000000+01:15, 01111100 00110101 01111101 11001011 00101010 10000100 00000000 00000000 00000000 00000000", + "2023-10-15T11:22:33.444555666+01:15, 01111100 00110101 01111101 11001011 00101010 10000100 10010010 01100001 01111111 00011010", + "2023-10-15T11:22:33.999999999+01:15, 01111100 00110101 01111101 11001011 00101010 10000100 11111111 11001001 10011010 00111011", + + }) + public void testWriteTimestampValueWithKnownOffsetShortForm(@ConvertWith(StringToTimestamp.class) Timestamp value, String expectedBytes) { + assertWritingValueWithBinary(expectedBytes, value, IonEncoder_1_1::writeTimestampValue); + } + + @ParameterizedTest + @CsvSource({ + // OpCode Length YYYYYYYY MMYYYYYY HDDDDDMM mmmmHHHH oooooomm ssoooooo ....ssss Coefficient+ Scale + "0001T, 11110111 00000101 00000001 00000000", + "1947T, 11110111 00000101 10011011 00000111", + "9999T, 11110111 00000101 00001111 00100111", + "1947-01T, 11110111 00000111 10011011 01000111 00000000", + "1947-12T, 11110111 00000111 10011011 00000111 00000011", + "1947-01-01T, 11110111 00000111 10011011 01000111 00000100", + "1947-12-23T, 11110111 00000111 10011011 00000111 01011111", + "1947-12-31T, 11110111 00000111 10011011 00000111 01111111", + "1947-12-23T00:00Z, 11110111 00001101 10011011 00000111 01011111 00000000 10000000 00010110", + "1947-12-23T23:59Z, 11110111 00001101 10011011 00000111 11011111 10111011 10000011 00010110", + "1947-12-23T23:59:00Z, 11110111 00001111 10011011 00000111 11011111 10111011 10000011 00010110 00000000", + "1947-12-23T23:59:59Z, 11110111 00001111 10011011 00000111 11011111 10111011 10000011 10110110 00000011", + "1947-12-23T23:59:00.0Z, 11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000001 00000001", + "1947-12-23T23:59:00.00Z, 11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000001 00000010", + "1947-12-23T23:59:00.000Z, 11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000001 00000011", + "1947-12-23T23:59:00.0000Z, 11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000001 00000100", + "1947-12-23T23:59:00.00000Z, 11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000001 00000101", + "1947-12-23T23:59:00.000000Z, 11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000001 00000110", + "1947-12-23T23:59:00.0000000Z, 11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000001 00000111", + "1947-12-23T23:59:00.00000000Z, 11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000001 00001000", + "1947-12-23T23:59:00.9Z, 11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00010011 00000001", + "1947-12-23T23:59:00.99Z, 11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 11000111 00000010", + "1947-12-23T23:59:00.999Z, 11110111 00010101 10011011 00000111 11011111 10111011 10000011 00010110 00000000 10011110 00001111 00000011", + "1947-12-23T23:59:00.9999Z, 11110111 00010101 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00111110 10011100 00000100", + "1947-12-23T23:59:00.99999Z, 11110111 00010111 10011011 00000111 11011111 10111011 10000011 00010110 00000000 11111100 00110100 00001100 00000101", + "1947-12-23T23:59:00.999999Z, 11110111 00010111 10011011 00000111 11011111 10111011 10000011 00010110 00000000 11111100 00010001 01111010 00000110", + "1947-12-23T23:59:00.9999999Z, 11110111 00011001 10011011 00000111 11011111 10111011 10000011 00010110 00000000 11111000 01100111 10001001 00001001 00000111", + "1947-12-23T23:59:00.99999999Z, 11110111 00011001 10011011 00000111 11011111 10111011 10000011 00010110 00000000 11111000 00001111 01011110 01011111 00001000", + + "1947-12-23T23:59:00.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000Z, " + + "11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000001 10001101", + + "1947-12-23T23:59:00.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000Z, " + + "11110111 00010101 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000001 01101000 00000001", + + "1947-12-23T23:59:00.999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999Z, " + + "11110111 10010111 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 " + + "11111100 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 " + + "11111111 10010100 10001001 01111001 01101100 11001110 01111000 11110010 01000000 01111101 10100110 11000111 10101000 01000110 01011001 01110001 01001101 " + + "00100000 11110101 01101110 01111010 00001100 00001001 11101111 01111111 11110011 00011110 00010100 11010111 01101000 01110111 10101100 01101100 10001110 " + + "00110010 10110111 10000010 11110010 00110110 01101000 11110010 10100111 10001101", + + + // Offsets + "2048-01-01T01:01-23:59, 11110111 00001101 00000000 01001000 10000100 00010000 00000100 00000000", + "2048-01-01T01:01-00:02, 11110111 00001101 00000000 01001000 10000100 00010000 01111000 00010110", + "2048-01-01T01:01-00:01, 11110111 00001101 00000000 01001000 10000100 00010000 01111100 00010110", + "2048-01-01T01:01-00:00, 11110111 00001101 00000000 01001000 10000100 00010000 11111100 00111111", + "2048-01-01T01:01+00:00, 11110111 00001101 00000000 01001000 10000100 00010000 10000000 00010110", + "2048-01-01T01:01+00:01, 11110111 00001101 00000000 01001000 10000100 00010000 10000100 00010110", + "2048-01-01T01:01+00:02, 11110111 00001101 00000000 01001000 10000100 00010000 10001000 00010110", + "2048-01-01T01:01+23:59, 11110111 00001101 00000000 01001000 10000100 00010000 11111100 00101100", + }) + public void testWriteTimestampValueLongForm(@ConvertWith(StringToTimestamp.class) Timestamp value, String expectedBytes) { + assertWritingValueWithBinary(expectedBytes, value, IonEncoder_1_1::writeLongFormTimestampValue); + } + + @ParameterizedTest + @CsvSource({ + // Long form because it's out of the year range + "0001T, 11110111 00000101 00000001 00000000", + "9999T, 11110111 00000101 00001111 00100111", + + // Long form because the offset is too high/low + "2048-01-01T01:01+14:15, 11110111 00001101 00000000 01001000 10000100 00010000 11011100 00100011", + "2048-01-01T01:01-14:15, 11110111 00001101 00000000 01001000 10000100 00010000 00100100 00001001", + + // Long form because the offset is not a multiple of 15 + "2048-01-01T01:01+00:01, 11110111 00001101 00000000 01001000 10000100 00010000 10000100 00010110", + + // Long form because the fractional seconds are millis, micros, or nanos + "2023-12-31T23:59:00.0Z, 11110111 00010011 11100111 00000111 11111111 10111011 10000011 00010110 00000000 00000001 00000001", + }) + public void testWriteTimestampDelegatesCorrectlyToLongForm(@ConvertWith(StringToTimestamp.class) Timestamp value, String expectedBytes) { + assertWritingValueWithBinary(expectedBytes, value, IonEncoder_1_1::writeTimestampValue); + } + + @Test + public void testWriteTimestampValueForNullTimestamp() { + int numBytes = IonEncoder_1_1.writeTimestampValue(buf, null); + Assertions.assertEquals("EB 04", byteArrayToHex(bytes())); + Assertions.assertEquals(2, numBytes); + } + /** * Utility method to make it easier to write test cases that assert specific sequences of bytes. */ @@ -222,4 +416,45 @@ private static String byteArrayToHex(byte[] bytes) { private static int byteLengthFromHexString(String hexString) { return (hexString.replaceAll("[^\\dA-F]", "").length() - 1) / 2 + 1; } + + /** + * Converts a byte array to a string of bits, such as "00110110 10001001". + * The purpose of this method is to make it easier to read and write test assertions. + */ + private static String byteArrayToBitString(byte[] bytes) { + StringBuilder s = new StringBuilder(); + for (byte aByte : bytes) { + for (int bit = 7; bit >= 0; bit--) { + if (((0x01 << bit) & aByte) != 0) { + s.append("1"); + } else { + s.append("0"); + } + } + s.append(" "); + } + return s.toString().trim(); + } + + /** + * Determines the number of bytes needed to represent a series of hexadecimal digits. + */ + private static int byteLengthFromBitString(String bitString) { + return (bitString.replaceAll("[^01]", "").length() - 1) / 8 + 1; + } + + /** + * Converts a String to a Timestamp for a @Parameterized test + */ + static class StringToTimestamp extends TypedArgumentConverter { + protected StringToTimestamp() { + super(String.class, Timestamp.class); + } + + @Override + protected Timestamp convert(String source) throws ArgumentConversionException { + if (source == null) return null; + return Timestamp.valueOf(source); + } + } } diff --git a/test/com/amazon/ion/impl/bin/WriteBufferTest.java b/test/com/amazon/ion/impl/bin/WriteBufferTest.java index a46e4c2c75..1a8e83b80f 100644 --- a/test/com/amazon/ion/impl/bin/WriteBufferTest.java +++ b/test/com/amazon/ion/impl/bin/WriteBufferTest.java @@ -24,7 +24,6 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.math.BigDecimal; import java.math.BigInteger; import java.util.Arrays; import org.junit.jupiter.api.Test; @@ -1620,6 +1619,37 @@ public void testWriteFixedUIntForNegativeNumber() { Assertions.assertThrows(IllegalArgumentException.class, () -> buf.writeFixedUInt(-1)); } + @ParameterizedTest + @CsvSource({ + " 0, 1, 00000000", + " 0, 2, 00000000 00000000", + " 0, 8, 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000", + " 1, 1, 00000001", + " 1, 2, 00000001 00000000", + " 1, 8, 00000001 00000000 00000000 00000000 00000000 00000000 00000000 00000000", + " 255, 1, 11111111", + " 255, 2, 11111111 00000000", + " 255, 3, 11111111 00000000 00000000", + " -1, 1, 11111111", + " -1, 2, 11111111 11111111", + " -1, 8, 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111", + // Long.MIN_VALUE and Long.MAX_VALUE + " 9223372036854775807, 8, 11111111 11111111 11111111 11111111 11111111 11111111 11111111 01111111", + "-9223372036854775808, 8, 00000000 00000000 00000000 00000000 00000000 00000000 00000000 10000000", + }) + public void testWriteFixedIntOrUInt(long value, int numBytes, String expectedBits) { + int actualNumBytes = buf.writeFixedIntOrUInt(value, numBytes); + String actualBits = byteArrayToBitString(bytes()); + Assertions.assertEquals(expectedBits, actualBits); + Assertions.assertEquals(numBytes, actualNumBytes); + } + + @Test + public void testWriteFixedIntOrUIntThrowsExceptionWhenNumBytesIsOutOfBounds() { + Assertions.assertThrows(IllegalArgumentException.class, () -> buf.writeFixedIntOrUInt(0, -1)); + Assertions.assertThrows(IllegalArgumentException.class, () -> buf.writeFixedIntOrUInt(0, 9)); + } + /** * Converts a byte array to a string of bits, such as "00110110 10001001". * The purpose of this method is to make it easier to read and write test assertions. From 54753cc0f1074bb6dbc7870b98ce64ee89245afc Mon Sep 17 00:00:00 2001 From: Matthew Pope Date: Wed, 1 Nov 2023 11:50:09 -0700 Subject: [PATCH 2/4] Adds suggested changes --- src/com/amazon/ion/impl/bin/IonEncoder_1_1.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/com/amazon/ion/impl/bin/IonEncoder_1_1.java b/src/com/amazon/ion/impl/bin/IonEncoder_1_1.java index 3ec31c03f1..c7bfa1ee85 100644 --- a/src/com/amazon/ion/impl/bin/IonEncoder_1_1.java +++ b/src/com/amazon/ion/impl/bin/IonEncoder_1_1.java @@ -178,9 +178,11 @@ public static int writeTimestampValue(WriteBuffer buffer, Timestamp value) { } // Condition 2: The fractional seconds are a common precision. - int secondsScale = value.getDecimalSecond().scale(); - if (secondsScale != 0 && secondsScale != 3 && secondsScale != 6 && secondsScale != 9) { - return writeLongFormTimestampValue(buffer, value); + if (value.getFractionalSecond() != null) { + int secondsScale = value.getFractionalSecond().scale(); + if (secondsScale != 0 && secondsScale != 3 && secondsScale != 6 && secondsScale != 9) { + return writeLongFormTimestampValue(buffer, value); + } } // Condition 3: The local offset is either UTC, unknown, or falls between -14:00 to +14:00 and is divisible by 15 minutes. Integer offset = value.getLocalOffset(); From ea7cad8c4221da3b20a509eb2c4502927d800e74 Mon Sep 17 00:00:00 2001 From: Matthew Pope Date: Thu, 2 Nov 2023 12:05:07 -0700 Subject: [PATCH 3/4] Adds suggested changes --- .../amazon/ion/impl/bin/IonEncoder_1_1.java | 25 +++++++++++++------ .../ion/impl/bin/Ion_1_1_Constants.java | 2 +- .../ion/impl/bin/IonEncoder_1_1Test.java | 2 +- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/com/amazon/ion/impl/bin/IonEncoder_1_1.java b/src/com/amazon/ion/impl/bin/IonEncoder_1_1.java index c7bfa1ee85..a349be844e 100644 --- a/src/com/amazon/ion/impl/bin/IonEncoder_1_1.java +++ b/src/com/amazon/ion/impl/bin/IonEncoder_1_1.java @@ -178,8 +178,8 @@ public static int writeTimestampValue(WriteBuffer buffer, Timestamp value) { } // Condition 2: The fractional seconds are a common precision. - if (value.getFractionalSecond() != null) { - int secondsScale = value.getFractionalSecond().scale(); + if (value.getZFractionalSecond() != null) { + int secondsScale = value.getZFractionalSecond().scale(); if (secondsScale != 0 && secondsScale != 3 && secondsScale != 6 && secondsScale != 9) { return writeLongFormTimestampValue(buffer, value); } @@ -234,9 +234,12 @@ private static int writeShortFormTimestampValue(WriteBuffer buffer, Timestamp va bits |= ((long) value.getSecond()) << S_U_TIMESTAMP_SECOND_BIT_OFFSET; - int secondsScale = value.getDecimalSecond().scale(); + int secondsScale = 0; + if (value.getZFractionalSecond() != null) { + secondsScale = value.getZFractionalSecond().scale(); + } if (secondsScale != 0) { - long fractionalSeconds = value.getDecimalSecond().remainder(BigDecimal.ONE).movePointRight(secondsScale).longValue(); + long fractionalSeconds = value.getZFractionalSecond().unscaledValue().longValue(); bits |= fractionalSeconds << S_U_TIMESTAMP_FRACTION_BIT_OFFSET; } switch (secondsScale) { @@ -275,9 +278,12 @@ private static int writeShortFormTimestampValue(WriteBuffer buffer, Timestamp va // if there are nanoseconds (which is too much for one long) and the boundary between the seconds // and fractional seconds subfields conveniently aligns with a byte boundary. long fractionBits = 0; - int secondsScale = value.getDecimalSecond().scale(); + int secondsScale = 0; + if (value.getZFractionalSecond() != null) { + secondsScale = value.getZFractionalSecond().scale(); + } if (secondsScale != 0) { - fractionBits = value.getDecimalSecond().remainder(BigDecimal.ONE).movePointRight(secondsScale).longValue(); + fractionBits = value.getZFractionalSecond().unscaledValue().longValue(); } switch (secondsScale) { case 0: @@ -350,14 +356,17 @@ static int writeLongFormTimestampValue(WriteBuffer buffer, Timestamp value) { bits |= ((long) value.getSecond()) << L_TIMESTAMP_SECOND_BIT_OFFSET; - int secondsScale = value.getDecimalSecond().scale(); + int secondsScale = 0; + if (value.getZFractionalSecond() != null) { + secondsScale = value.getZFractionalSecond().scale(); + } if (secondsScale == 0) { buffer.writeFlexUInt(7); buffer.writeFixedIntOrUInt(bits, 7); return 9; // OpCode + FlexUInt + 7 bytes data } - BigDecimal fractionalSeconds = value.getDecimalSecond().remainder(BigDecimal.ONE); + BigDecimal fractionalSeconds = value.getZFractionalSecond(); BigInteger coefficient = fractionalSeconds.unscaledValue(); long exponent = fractionalSeconds.scale(); int numCoefficientBytes = WriteBuffer.flexUIntLength(coefficient); diff --git a/src/com/amazon/ion/impl/bin/Ion_1_1_Constants.java b/src/com/amazon/ion/impl/bin/Ion_1_1_Constants.java index e0c031ee93..91a7647042 100644 --- a/src/com/amazon/ion/impl/bin/Ion_1_1_Constants.java +++ b/src/com/amazon/ion/impl/bin/Ion_1_1_Constants.java @@ -27,7 +27,7 @@ private Ion_1_1_Constants() {} static final int L_TIMESTAMP_HOUR_BIT_OFFSET = 23; static final int L_TIMESTAMP_MINUTE_BIT_OFFSET = 28; static final int L_TIMESTAMP_OFFSET_BIT_OFFSET = 34; - static final int L_TIMESTAMP_SECOND_BIT_OFFSET = 44; + static final int L_TIMESTAMP_SECOND_BIT_OFFSET = 46; static final int L_TIMESTAMP_UNKNOWN_OFFSET_VALUE = 0b111111111111; //////// Bit masks //////// diff --git a/test/com/amazon/ion/impl/bin/IonEncoder_1_1Test.java b/test/com/amazon/ion/impl/bin/IonEncoder_1_1Test.java index 0916e70b9c..60de3f611f 100644 --- a/test/com/amazon/ion/impl/bin/IonEncoder_1_1Test.java +++ b/test/com/amazon/ion/impl/bin/IonEncoder_1_1Test.java @@ -324,7 +324,7 @@ public void testWriteTimestampValueWithKnownOffsetShortForm(@ConvertWith(StringT "1947-12-23T00:00Z, 11110111 00001101 10011011 00000111 01011111 00000000 10000000 00010110", "1947-12-23T23:59Z, 11110111 00001101 10011011 00000111 11011111 10111011 10000011 00010110", "1947-12-23T23:59:00Z, 11110111 00001111 10011011 00000111 11011111 10111011 10000011 00010110 00000000", - "1947-12-23T23:59:59Z, 11110111 00001111 10011011 00000111 11011111 10111011 10000011 10110110 00000011", + "1947-12-23T23:59:59Z, 11110111 00001111 10011011 00000111 11011111 10111011 10000011 11010110 00001110", "1947-12-23T23:59:00.0Z, 11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000001 00000001", "1947-12-23T23:59:00.00Z, 11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000001 00000010", "1947-12-23T23:59:00.000Z, 11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000001 00000011", From 7fce291f1215a20e427a2fdaa8c8748edce704b6 Mon Sep 17 00:00:00 2001 From: Matthew Pope Date: Wed, 8 Nov 2023 10:47:15 -0500 Subject: [PATCH 4/4] Fix short form timestamp offsets --- .../amazon/ion/impl/bin/IonEncoder_1_1.java | 2 +- .../ion/impl/bin/IonEncoder_1_1Test.java | 40 +++++++++---------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/com/amazon/ion/impl/bin/IonEncoder_1_1.java b/src/com/amazon/ion/impl/bin/IonEncoder_1_1.java index a349be844e..fe379df851 100644 --- a/src/com/amazon/ion/impl/bin/IonEncoder_1_1.java +++ b/src/com/amazon/ion/impl/bin/IonEncoder_1_1.java @@ -263,7 +263,7 @@ private static int writeShortFormTimestampValue(WriteBuffer buffer, Timestamp va throw new IllegalStateException("This is unreachable!"); } } else { - long localOffset = value.getLocalOffset().longValue() / 15; + long localOffset = (value.getLocalOffset().longValue() / 15) + (14 * 4); bits |= (localOffset & LEAST_SIGNIFICANT_7_BITS) << S_O_TIMESTAMP_OFFSET_BIT_OFFSET; if (value.getPrecision() == Timestamp.Precision.MINUTE) { diff --git a/test/com/amazon/ion/impl/bin/IonEncoder_1_1Test.java b/test/com/amazon/ion/impl/bin/IonEncoder_1_1Test.java index 60de3f611f..fd756db309 100644 --- a/test/com/amazon/ion/impl/bin/IonEncoder_1_1Test.java +++ b/test/com/amazon/ion/impl/bin/IonEncoder_1_1Test.java @@ -284,26 +284,26 @@ public void testWriteTimestampValueWithUnknownOffsetShortForm(@ConvertWith(Strin @ParameterizedTest @CsvSource({ // OpCode MYYYYYYY DDDDDMMM mmmHHHHH ooooommm ssssssoo ffffffff ffffffff ffffffff ..ffffff - "2023-10-15T01:00-14:00, 01111000 00110101 01111101 00000001 01000000 00000010", - "2023-10-15T01:00+14:00, 01111000 00110101 01111101 00000001 11000000 00000001", - "2023-10-15T01:00-01:15, 01111000 00110101 01111101 00000001 11011000 00000011", - "2023-10-15T01:00+01:15, 01111000 00110101 01111101 00000001 00101000 00000000", - "2023-10-15T01:59+01:15, 01111000 00110101 01111101 01100001 00101111 00000000", - "2023-10-15T11:22+01:15, 01111000 00110101 01111101 11001011 00101010 00000000", - "2023-10-15T23:00+01:15, 01111000 00110101 01111101 00010111 00101000 00000000", - "2023-10-15T23:59+01:15, 01111000 00110101 01111101 01110111 00101111 00000000", - "2023-10-15T11:22:00+01:15, 01111001 00110101 01111101 11001011 00101010 00000000", - "2023-10-15T11:22:33+01:15, 01111001 00110101 01111101 11001011 00101010 10000100", - "2023-10-15T11:22:59+01:15, 01111001 00110101 01111101 11001011 00101010 11101100", - "2023-10-15T11:22:33.000+01:15, 01111010 00110101 01111101 11001011 00101010 10000100 00000000 00000000", - "2023-10-15T11:22:33.444+01:15, 01111010 00110101 01111101 11001011 00101010 10000100 10111100 00000001", - "2023-10-15T11:22:33.999+01:15, 01111010 00110101 01111101 11001011 00101010 10000100 11100111 00000011", - "2023-10-15T11:22:33.000000+01:15, 01111011 00110101 01111101 11001011 00101010 10000100 00000000 00000000 00000000", - "2023-10-15T11:22:33.444555+01:15, 01111011 00110101 01111101 11001011 00101010 10000100 10001011 11001000 00000110", - "2023-10-15T11:22:33.999999+01:15, 01111011 00110101 01111101 11001011 00101010 10000100 00111111 01000010 00001111", - "2023-10-15T11:22:33.000000000+01:15, 01111100 00110101 01111101 11001011 00101010 10000100 00000000 00000000 00000000 00000000", - "2023-10-15T11:22:33.444555666+01:15, 01111100 00110101 01111101 11001011 00101010 10000100 10010010 01100001 01111111 00011010", - "2023-10-15T11:22:33.999999999+01:15, 01111100 00110101 01111101 11001011 00101010 10000100 11111111 11001001 10011010 00111011", + "2023-10-15T01:00-14:00, 01111000 00110101 01111101 00000001 00000000 00000000", + "2023-10-15T01:00+14:00, 01111000 00110101 01111101 00000001 10000000 00000011", + "2023-10-15T01:00-01:15, 01111000 00110101 01111101 00000001 10011000 00000001", + "2023-10-15T01:00+01:15, 01111000 00110101 01111101 00000001 11101000 00000001", + "2023-10-15T01:59+01:15, 01111000 00110101 01111101 01100001 11101111 00000001", + "2023-10-15T11:22+01:15, 01111000 00110101 01111101 11001011 11101010 00000001", + "2023-10-15T23:00+01:15, 01111000 00110101 01111101 00010111 11101000 00000001", + "2023-10-15T23:59+01:15, 01111000 00110101 01111101 01110111 11101111 00000001", + "2023-10-15T11:22:00+01:15, 01111001 00110101 01111101 11001011 11101010 00000001", + "2023-10-15T11:22:33+01:15, 01111001 00110101 01111101 11001011 11101010 10000101", + "2023-10-15T11:22:59+01:15, 01111001 00110101 01111101 11001011 11101010 11101101", + "2023-10-15T11:22:33.000+01:15, 01111010 00110101 01111101 11001011 11101010 10000101 00000000 00000000", + "2023-10-15T11:22:33.444+01:15, 01111010 00110101 01111101 11001011 11101010 10000101 10111100 00000001", + "2023-10-15T11:22:33.999+01:15, 01111010 00110101 01111101 11001011 11101010 10000101 11100111 00000011", + "2023-10-15T11:22:33.000000+01:15, 01111011 00110101 01111101 11001011 11101010 10000101 00000000 00000000 00000000", + "2023-10-15T11:22:33.444555+01:15, 01111011 00110101 01111101 11001011 11101010 10000101 10001011 11001000 00000110", + "2023-10-15T11:22:33.999999+01:15, 01111011 00110101 01111101 11001011 11101010 10000101 00111111 01000010 00001111", + "2023-10-15T11:22:33.000000000+01:15, 01111100 00110101 01111101 11001011 11101010 10000101 00000000 00000000 00000000 00000000", + "2023-10-15T11:22:33.444555666+01:15, 01111100 00110101 01111101 11001011 11101010 10000101 10010010 01100001 01111111 00011010", + "2023-10-15T11:22:33.999999999+01:15, 01111100 00110101 01111101 11001011 11101010 10000101 11111111 11001001 10011010 00111011", }) public void testWriteTimestampValueWithKnownOffsetShortForm(@ConvertWith(StringToTimestamp.class) Timestamp value, String expectedBytes) {