Skip to content

Adds support for FlexInt and FlexUInt in WriteBuffer #609

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
popematt merged 2 commits into from
Oct 16, 2023
Merged

Adds support for FlexInt and FlexUInt in WriteBuffer #609

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
dd07b8c
Adds support for FlexInt and FlexUInt in WriteBuffer
popematt Oct 13, 2023
1704ce2
Update WriteBufferTest.java
popematt Oct 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 87 additions & 0 deletions src/com/amazon/ion/impl/bin/WriteBuffer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1272,6 +1272,93 @@ public void writeUInt8At(final long position, final long value)
block.data[offset] = (byte) value;
}

/** Get the length of FlexInt for the provided value. */
public static int flexIntLength(final long value) {
int numMagnitudeBitsRequired;
if (value < 0) {
int numLeadingOnes = Long.numberOfLeadingZeros(~value);
numMagnitudeBitsRequired = 64 - numLeadingOnes;
} else {
int numLeadingZeros = Long.numberOfLeadingZeros(value);
numMagnitudeBitsRequired = 64 - numLeadingZeros;
}
return numMagnitudeBitsRequired / 7 + 1;
}

/** Writes a FlexInt to this WriteBuffer, returning the number of bytes that were needed to encode the value */
public int writeFlexInt(final long value) {
int numBytes = flexIntLength(value);
return writeFlexIntOrUInt(value, numBytes);
}

/** Get the length of FlexUInt for the provided value. */
public static int flexUIntLength(final long value) {
int numLeadingZeros = Long.numberOfLeadingZeros(value);
int numMagnitudeBitsRequired = 64 - numLeadingZeros;
return (numMagnitudeBitsRequired - 1) / 7 + 1;
}

/** Writes a FlexUInt to this WriteBuffer, returning the number of bytes that were needed to encode the value */
public int writeFlexUInt(final long value) {
if (value < 0) {
throw new IllegalArgumentException("Attempted to write a FlexUInt for " + value);
}
int numBytes = flexUIntLength(value);
return writeFlexIntOrUInt(value, numBytes);
}

/**
* Because the flex int and flex uint encodings are so similar, we can use this method to write either one as long
* as we provide the correct number of bytes needed to encode the value.
*/
private int writeFlexIntOrUInt(final long value, final int numBytes) {
if (numBytes == 1) {
writeByte((byte) (0x01 | (byte)(value << 1)));
} else if (numBytes == 2) {
writeByte((byte) (0x02 | (byte)(value << 2)));
writeByte((byte) (value >> 6));
} else if (numBytes == 3) {
writeByte((byte) (0x04 | (byte)(value << 3)));
writeByte((byte) (value >> 5));
writeByte((byte) (value >> 13));
} else if (numBytes == 4) {
writeByte((byte) (0x08 | (byte)(value << 4)));
writeByte((byte) (value >> 4));
writeByte((byte) (value >> 12));
writeByte((byte) (value >> 20));
Comment on lines +1315 to +1328
Copy link
Contributor

@tgregg tgregg Oct 17, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As an optimization, we could go into the fallback loop branch if remaining() < numBytes, then in the short branches we can write directly into the buffer rather than repetitively calling writeByte, which requires some redundant logic. See e.g. writeUInt64 vs writeUInt64Slow.

} else {
// Finally, fall back to a loop based approach.

int i = 0; // `i` gets incremented for every byte written.

// Start with leading zero bytes.
// If there's 1-8 total bytes, we need no leading zero-bytes.
// If there's 9-16 total bytes, we need one zero-byte
// If there's 17-24 total bytes, we need two zero-bytes, etc.
for (; i < (numBytes - 1)/8; i++) {
writeByte((byte) 0);
}

// Write the last length bits, possibly also containing some value bits.
int remainingLengthBits = (numBytes - 1) % 8;
byte lengthPart = (byte) (0x01 << remainingLengthBits);

int valueBitOffset = remainingLengthBits + 1;
byte valuePart = (byte) (value << valueBitOffset);

writeByte((byte) (valuePart | lengthPart));
i++;

int valueByteOffset = 1;
for (; i < numBytes; i++) {
writeByte((byte) (value >> (8 * valueByteOffset - valueBitOffset)));
valueByteOffset++;
}

}
return numBytes;
}

/** Write the entire buffer to output stream. */
public void writeTo(final OutputStream out) throws IOException
{
Expand Down
125 changes: 125 additions & 0 deletions test/com/amazon/ion/impl/bin/WriteBufferTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,10 @@
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;

public class WriteBufferTest
{
Expand All @@ -39,6 +43,7 @@ public class WriteBufferTest
private ByteArrayOutputStream out;

@Before
@BeforeEach
public void setup()
{
buf = new WriteBuffer(ALLOCATOR);
Expand Down Expand Up @@ -1300,4 +1305,124 @@ public void testVarUIntLength9() {
int length = varUIntLength(0x7FFFFFFFFFFFFFFCL);
assertEquals(9, length);
}

@ParameterizedTest
@CsvSource({
" 0, 00000001",
" 1, 00000011",
" 2, 00000101",
" 3, 00000111",
" 4, 00001001",
" 5, 00001011",
" 14, 00011101",
" 63, 01111111",
" 64, 00000010 00000001",
" 729, 01100110 00001011",
" 8191, 11111110 01111111",
" 8192, 00000100 00000000 00000001",
" 1048575, 11111100 11111111 01111111",
" 1048576, 00001000 00000000 00000000 00000001",
" 134217727, 11111000 11111111 11111111 01111111",
" 134217728, 00010000 00000000 00000000 00000000 00000001",
" 17179869184, 00100000 00000000 00000000 00000000 00000000 00000001",
" 2199023255552, 01000000 00000000 00000000 00000000 00000000 00000000 00000001",
" 281474976710655, 11000000 11111111 11111111 11111111 11111111 11111111 01111111",
" 281474976710656, 10000000 00000000 00000000 00000000 00000000 00000000 00000000 00000001",
" 36028797018963967, 10000000 11111111 11111111 11111111 11111111 11111111 11111111 01111111",
" 36028797018963968, 00000000 00000001 00000000 00000000 00000000 00000000 00000000 00000000 00000001",
" 4611686018427387903, 00000000 11111111 11111111 11111111 11111111 11111111 11111111 11111111 01111111",
" 4611686018427387904, 00000000 00000010 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000001",
// Long.MAX_VALUE
" 9223372036854775807, 00000000 11111110 11111111 11111111 11111111 11111111 11111111 11111111 11111111 00000001",
Copy link
Contributor

@zslayton zslayton Oct 16, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this (and its negative counterpart below) are the limits for long. If that's the case, would you mind adding a comment stating that? It's a good test, and I assume many don't have the limits memorized.

" -1, 11111111",
" -2, 11111101",
" -3, 11111011",
" -14, 11100101",
" -64, 10000001",
" -65, 11111110 11111110",
" -729, 10011110 11110100",
" -8192, 00000010 10000000",
" -8193, 11111100 11111111 11111110",
" -1048576, 00000100 00000000 10000000",
" -1048577, 11111000 11111111 11111111 11111110",
" -134217728, 00001000 00000000 00000000 10000000",
" -134217729, 11110000 11111111 11111111 11111111 11111110",
" -17179869184, 00010000 00000000 00000000 00000000 10000000",
" -17179869185, 11100000 11111111 11111111 11111111 11111111 11111110",
" -281474976710656, 01000000 00000000 00000000 00000000 00000000 00000000 10000000",
" -281474976710657, 10000000 11111111 11111111 11111111 11111111 11111111 11111111 11111110",
" -36028797018963968, 10000000 00000000 00000000 00000000 00000000 00000000 00000000 10000000",
" -36028797018963969, 00000000 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111110",
"-4611686018427387904, 00000000 00000001 00000000 00000000 00000000 00000000 00000000 00000000 10000000",
"-4611686018427387905, 00000000 11111110 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111110",
// Long.MIN_VALUE
"-9223372036854775808, 00000000 00000010 00000000 00000000 00000000 00000000 00000000 00000000 00000000 11111110",

})
public void testWriteFlexInt(long value, String expectedBits) {
int numBytes = buf.writeFlexInt(value);
String actualBits = byteArrayToBitString(bytes());
Assertions.assertEquals(expectedBits, actualBits);
Assertions.assertEquals((expectedBits.length() + 1)/9, numBytes);
}

@ParameterizedTest
@CsvSource({
" 0, 00000001",
" 1, 00000011",
" 2, 00000101",
" 3, 00000111",
" 4, 00001001",
" 5, 00001011",
" 14, 00011101",
" 63, 01111111",
" 64, 10000001",
" 127, 11111111",
" 128, 00000010 00000010",
" 729, 01100110 00001011",
" 16383, 11111110 11111111",
" 16384, 00000100 00000000 00000010",
" 2097151, 11111100 11111111 11111111",
" 2097152, 00001000 00000000 00000000 00000010",
" 268435455, 11111000 11111111 11111111 11111111",
" 268435456, 00010000 00000000 00000000 00000000 00000010",
" 34359738368, 00100000 00000000 00000000 00000000 00000000 00000010",
" 4398046511104, 01000000 00000000 00000000 00000000 00000000 00000000 00000010",
" 562949953421311, 11000000 11111111 11111111 11111111 11111111 11111111 11111111",
" 562949953421312, 10000000 00000000 00000000 00000000 00000000 00000000 00000000 00000010",
" 72057594037927935, 10000000 11111111 11111111 11111111 11111111 11111111 11111111 11111111",
" 72057594037927936, 00000000 00000001 00000000 00000000 00000000 00000000 00000000 00000000 00000010",
// Long.MAX_VALUE
"9223372036854775807, 00000000 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111",
})
public void testWriteFlexUInt(long value, String expectedBits) {
int numBytes = buf.writeFlexUInt(value);
String actualBits = byteArrayToBitString(bytes());
Assertions.assertEquals(expectedBits, actualBits);
Assertions.assertEquals((expectedBits.length() + 1)/9, numBytes);
}

@Test
public void testWriteFlexUIntForNegativeNumber() {
Assertions.assertThrows(IllegalArgumentException.class, () -> buf.writeFlexUInt(-1));
}

/**
* Converts a byte array to a string of bits, such as "00110110 10001001".
* The purpose of this method is to make it easier to read and write test assertions.
*/
private static String byteArrayToBitString(byte[] bytes) {
StringBuilder s = new StringBuilder();
for (byte aByte : bytes) {
for (int bit = 7; bit >= 0; bit--) {
if (((0x01 << bit) & aByte) != 0) {
s.append("1");
} else {
s.append("0");
}
}
s.append(" ");
}
return s.toString().trim();
}
}