Skip to content

Adds support for FixedInt and FixedUInt in WriteBuffer #610

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
popematt merged 2 commits into from
Oct 16, 2023
Merged

Adds support for FixedInt and FixedUInt in WriteBuffer #610

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
11ac792
Adds support for FixedInt and FixedUInt in WriteBuffer
popematt Oct 16, 2023
4d65f6d
Merge branch 'master' into fixedint
popematt Oct 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 71 additions & 0 deletions src/com/amazon/ion/impl/bin/WriteBuffer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1359,6 +1359,77 @@ private int writeFlexIntOrUInt(final long value, final int numBytes) {
return numBytes;
}

/** Get the length of FixedInt for the provided value. */
public static int fixedIntLength(final long value) {
int numMagnitudeBitsRequired;
if (value < 0) {
int numLeadingOnes = Long.numberOfLeadingZeros(~value);
numMagnitudeBitsRequired = 64 - numLeadingOnes;
} else {
int numLeadingZeros = Long.numberOfLeadingZeros(value);
numMagnitudeBitsRequired = 64 - numLeadingZeros;
}
return numMagnitudeBitsRequired / 8 + 1;
}

/**
* Writes a FixedInt to this WriteBuffer, using the minimum number of bytes needed to represent the number.
* Returns the number of bytes that were needed to encode the value.
*/
public int writeFixedInt(final long value) {
int numBytes = fixedIntLength(value);
return writeFixedIntOrUInt(value, numBytes);
}

/** Get the length of FixedUInt for the provided value. */
public static int fixedUIntLength(final long value) {
int numLeadingZeros = Long.numberOfLeadingZeros(value);
int numMagnitudeBitsRequired = 64 - numLeadingZeros;
return (numMagnitudeBitsRequired - 1) / 8 + 1;
}

/**
* Writes a FixedUInt to this WriteBuffer, using the minimum number of bytes needed to represent the number.
* Returns the number of bytes that were needed to encode the value.
*/
public int writeFixedUInt(final long value) {
if (value < 0) {
throw new IllegalArgumentException("Attempted to write a FlexUInt for " + value);
}
int numBytes = fixedUIntLength(value);
return writeFixedIntOrUInt(value, numBytes);
}

/**
* Because the fixed int and fixed uint encodings are so similar, we can use this method to write either one as long
* as we provide the correct number of bytes needed to encode the value.
*/
private int writeFixedIntOrUInt(final long value, final int numBytes) {
writeByte((byte) value);
if (numBytes > 1) {
writeByte((byte) (value >> 8));
if (numBytes > 2) {
writeByte((byte) (value >> 8 * 2));
if (numBytes > 3) {
writeByte((byte) (value >> 8 * 3));
if (numBytes > 4) {
writeByte((byte) (value >> 8 * 4));
if (numBytes > 5) {
writeByte((byte) (value >> 8 * 5));
if (numBytes > 6) {
writeByte((byte) (value >> 8 * 6));
if (numBytes > 7) {
writeByte((byte) (value >> 8 * 7));
}
}
}
}
}
}
}
Comment on lines +1408 to +1429
Copy link
Contributor

@tgregg tgregg Oct 17, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As an optimization, we could have fast/slow variants of this too, where we do the current technique when remaining() < numBytes, and otherwise we just write directly into the underlying array.

return numBytes;
}

/** Write the entire buffer to output stream. */
public void writeTo(final OutputStream out) throws IOException
{
Expand Down
111 changes: 106 additions & 5 deletions test/com/amazon/ion/impl/bin/WriteBufferTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,10 @@
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.Arrays;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;

Expand All @@ -42,15 +41,14 @@ public class WriteBufferTest

private ByteArrayOutputStream out;

@Before
@BeforeEach
public void setup()
{
buf = new WriteBuffer(ALLOCATOR);
out = new ByteArrayOutputStream();
}

@After
@AfterEach
public void teardown()
{
buf = null;
Expand Down Expand Up @@ -1330,6 +1328,8 @@ public void testVarUIntLength9() {
" 281474976710656, 10000000 00000000 00000000 00000000 00000000 00000000 00000000 00000001",
" 36028797018963967, 10000000 11111111 11111111 11111111 11111111 11111111 11111111 01111111",
" 36028797018963968, 00000000 00000001 00000000 00000000 00000000 00000000 00000000 00000000 00000001",
// Different one-bits in every byte, making it easy to see if any bytes are out of order
" 72624976668147840, 00000000 00000001 10000001 01000000 00100000 00010000 00001000 00000100 00000010",
" 4611686018427387903, 00000000 11111111 11111111 11111111 11111111 11111111 11111111 11111111 01111111",
" 4611686018427387904, 00000000 00000010 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000001",
// Long.MAX_VALUE
Expand All @@ -1353,6 +1353,8 @@ public void testVarUIntLength9() {
" -281474976710657, 10000000 11111111 11111111 11111111 11111111 11111111 11111111 11111110",
" -36028797018963968, 10000000 00000000 00000000 00000000 00000000 00000000 00000000 10000000",
" -36028797018963969, 00000000 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111110",
// Different zero-bits in every byte, making it easy to see if any bytes are out of order
" -72624976668147841, 00000000 11111111 01111110 10111111 11011111 11101111 11110111 11111011 11111101",
"-4611686018427387904, 00000000 00000001 00000000 00000000 00000000 00000000 00000000 00000000 10000000",
"-4611686018427387905, 00000000 11111110 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111110",
// Long.MIN_VALUE
Expand Down Expand Up @@ -1392,6 +1394,8 @@ public void testWriteFlexInt(long value, String expectedBits) {
" 562949953421312, 10000000 00000000 00000000 00000000 00000000 00000000 00000000 00000010",
" 72057594037927935, 10000000 11111111 11111111 11111111 11111111 11111111 11111111 11111111",
" 72057594037927936, 00000000 00000001 00000000 00000000 00000000 00000000 00000000 00000000 00000010",
// Different one-bits in every byte, making it easy to see if any bytes are out of order
" 72624976668147840, 00000000 00000001 10000001 01000000 00100000 00010000 00001000 00000100 00000010",
// Long.MAX_VALUE
"9223372036854775807, 00000000 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111",
})
Expand All @@ -1407,6 +1411,103 @@ public void testWriteFlexUIntForNegativeNumber() {
Assertions.assertThrows(IllegalArgumentException.class, () -> buf.writeFlexUInt(-1));
}

@ParameterizedTest
@CsvSource({
" 0, 00000000",
" 1, 00000001",
" 2, 00000010",
" 14, 00001110",
" 127, 01111111",
" 128, 10000000 00000000",
" 32767, 11111111 01111111",
" 32768, 00000000 10000000 00000000",
" 3954261, 01010101 01010110 00111100",
" 8388607, 11111111 11111111 01111111",
" 8388608, 00000000 00000000 10000000 00000000",
" 2147483647, 11111111 11111111 11111111 01111111",
" 2147483648, 00000000 00000000 00000000 10000000 00000000",
" 549755813887, 11111111 11111111 11111111 11111111 01111111",
" 549755813888, 00000000 00000000 00000000 00000000 10000000 00000000",
" 140737488355327, 11111111 11111111 11111111 11111111 11111111 01111111",
" 140737488355328, 00000000 00000000 00000000 00000000 00000000 10000000 00000000",
" 36028797018963967, 11111111 11111111 11111111 11111111 11111111 11111111 01111111",
" 36028797018963968, 00000000 00000000 00000000 00000000 00000000 00000000 10000000 00000000",
// Different one-bit in every byte, making it easy to see if any bytes are out of order
" 72624976668147840, 10000000 01000000 00100000 00010000 00001000 00000100 00000010 00000001",
// Long.MAX_VALUE
" 9223372036854775807, 11111111 11111111 11111111 11111111 11111111 11111111 11111111 01111111",
" -1, 11111111",
" -2, 11111110",
" -14, 11110010",
" -128, 10000000",
" -129, 01111111 11111111",
" -32768, 00000000 10000000",
" -32769, 11111111 01111111 11111111",
" -3954261, 10101011 10101001 11000011",
" -8388608, 00000000 00000000 10000000",
" -8388609, 11111111 11111111 01111111 11111111",
" -2147483648, 00000000 00000000 00000000 10000000",
" -2147483649, 11111111 11111111 11111111 01111111 11111111",
" -549755813888, 00000000 00000000 00000000 00000000 10000000",
" -549755813889, 11111111 11111111 11111111 11111111 01111111 11111111",
" -140737488355328, 00000000 00000000 00000000 00000000 00000000 10000000",
" -140737488355329, 11111111 11111111 11111111 11111111 11111111 01111111 11111111",
" -36028797018963968, 00000000 00000000 00000000 00000000 00000000 00000000 10000000",
" -36028797018963969, 11111111 11111111 11111111 11111111 11111111 11111111 01111111 11111111",
// Different zero-bit in every byte, making it easy to see if any bytes are out of order
" -72624976668147841, 01111111 10111111 11011111 11101111 11110111 11111011 11111101 11111110",
// Long.MIN_VALUE
"-9223372036854775808, 00000000 00000000 00000000 00000000 00000000 00000000 00000000 10000000",

})
public void testWriteFixedInt(long value, String expectedBits) {
int numBytes = buf.writeFixedInt(value);
String actualBits = byteArrayToBitString(bytes());
Assertions.assertEquals(expectedBits, actualBits);
Assertions.assertEquals((expectedBits.length() + 1)/9, numBytes);
}

@ParameterizedTest
@CsvSource({
" 0, 00000000",
" 1, 00000001",
" 2, 00000010",
" 14, 00001110",
" 127, 01111111",
" 128, 10000000",
" 255, 11111111",
" 256, 00000000 00000001",
" 65535, 11111111 11111111",
" 65536, 00000000 00000000 00000001",
" 3954261, 01010101 01010110 00111100",
" 16777215, 11111111 11111111 11111111",
" 16777216, 00000000 00000000 00000000 00000001",
" 4294967295, 11111111 11111111 11111111 11111111",
" 4294967296, 00000000 00000000 00000000 00000000 00000001",
" 1099511627775, 11111111 11111111 11111111 11111111 11111111",
" 1099511627776, 00000000 00000000 00000000 00000000 00000000 00000001",
" 281474976710655, 11111111 11111111 11111111 11111111 11111111 11111111",
" 281474976710656, 00000000 00000000 00000000 00000000 00000000 00000000 00000001",
" 5023487023698435, 00000011 11010010 10010100 10110111 11010101 11011000 00010001",
" 72057594037927935, 11111111 11111111 11111111 11111111 11111111 11111111 11111111",
" 72057594037927936, 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000001",
// Different one-bit in every byte, making it easy to see if any bytes are out of order
" 72624976668147840, 10000000 01000000 00100000 00010000 00001000 00000100 00000010 00000001",
// Long.MAX_VALUE 72057594037927936
" 9223372036854775807, 11111111 11111111 11111111 11111111 11111111 11111111 11111111 01111111",
})
public void testWriteFixedUInt(long value, String expectedBits) {
int numBytes = buf.writeFixedUInt(value);
String actualBits = byteArrayToBitString(bytes());
Assertions.assertEquals(expectedBits, actualBits);
Assertions.assertEquals((expectedBits.length() + 1)/9, numBytes);
}

@Test
public void testWriteFixedUIntForNegativeNumber() {
Assertions.assertThrows(IllegalArgumentException.class, () -> buf.writeFixedUInt(-1));
}

/**
* Converts a byte array to a string of bits, such as "00110110 10001001".
* The purpose of this method is to make it easier to read and write test assertions.
Expand Down