Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -275,7 +275,7 @@ private void setValueFromParser(ValueVector valueVector, int i) throws IOExcepti
break;
case TIMESTAMPNANO:
((TimeStampNanoVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class));
break;
break;
default:
throw new UnsupportedOperationException("minor type: " + valueVector.getMinorType());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,12 @@
import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
import org.apache.arrow.vector.complex.writer.BigIntWriter;
import org.apache.arrow.vector.complex.writer.DateMilliWriter;
import org.apache.arrow.vector.complex.writer.IntWriter;
import org.apache.arrow.vector.complex.writer.TimeMilliWriter;
import org.apache.arrow.vector.complex.writer.TimeStampMilliWriter;
import org.apache.arrow.vector.holders.NullableTimeStampMilliHolder;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.junit.After;
import org.junit.Assert;
Expand Down Expand Up @@ -138,6 +142,48 @@ protected void validateComplexContent(int count, VectorSchemaRoot root) {
}
}

private DateTime makeDateTimeFromCount(int i) {
return new DateTime(2000 + i, 1 + i, 1 + i, i, i, i, i, DateTimeZone.UTC);
}

protected void writeDateTimeData(int count, NullableMapVector parent) {
Assert.assertTrue(count < 100);
ComplexWriter writer = new ComplexWriterImpl("root", parent);
MapWriter rootWriter = writer.rootAsMap();
DateMilliWriter dateWriter = rootWriter.dateMilli("date");
TimeMilliWriter timeWriter = rootWriter.timeMilli("time");
TimeStampMilliWriter timeStampMilliWriter = rootWriter.timeStampMilli("timestamp-milli");
for (int i = 0; i < count; i++) {
DateTime dt = makeDateTimeFromCount(i);
// Number of days in milliseconds since epoch, stored as 64-bit integer, only date part is used
dateWriter.setPosition(i);
long dateLong = dt.minusMillis(dt.getMillisOfDay()).getMillis();
dateWriter.writeDateMilli(dateLong);
// Time is a value in milliseconds since midnight, stored as 32-bit integer
timeWriter.setPosition(i);
timeWriter.writeTimeMilli(dt.getMillisOfDay());
// Timestamp is milliseconds since the epoch, stored as 64-bit integer
timeStampMilliWriter.setPosition(i);
timeStampMilliWriter.writeTimeStampMilli(dt.getMillis());
}
writer.setValueCount(count);
}

protected void validateDateTimeContent(int count, VectorSchemaRoot root) {
Assert.assertEquals(count, root.getRowCount());
printVectors(root.getFieldVectors());
for (int i = 0; i < count; i++) {
Object dateVal = root.getVector("date").getAccessor().getObject(i);
DateTime dt = makeDateTimeFromCount(i);
DateTime dateExpected = dt.minusMillis(dt.getMillisOfDay());
Assert.assertEquals(dateExpected.getMillis(), dateVal);
Object timeVal = root.getVector("time").getAccessor().getObject(i);
Assert.assertEquals(dt.getMillisOfDay(), timeVal);
Object timestampMilliVal = root.getVector("timestamp-milli").getAccessor().getObject(i);
Assert.assertTrue(dt.withZoneRetainFields(DateTimeZone.getDefault()).equals(timestampMilliVal));
}
}

protected void writeData(int count, MapVector parent) {
ComplexWriter writer = new ComplexWriterImpl("root", parent);
MapWriter rootWriter = writer.rootAsMap();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ public void testWriteReadUnionJSON() throws IOException {

writeJSON(file, root);
}
// read
// read
try (
BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
BufferAllocator vectorAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
Expand All @@ -119,6 +119,42 @@ public void testWriteReadUnionJSON() throws IOException {
}
}

@Test
public void testWriteReadDateTimeJSON() throws IOException {
File file = new File("target/mytest_datetime.json");
int count = COUNT;

// write
try (
BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
NullableMapVector parent = new NullableMapVector("parent", vectorAllocator, null, null)) {

writeDateTimeData(count, parent);

printVectors(parent.getChildrenFromFields());

VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root"));
validateDateTimeContent(count, root);

writeJSON(file, new VectorSchemaRoot(parent.getChild("root")));
}

// read
try (
BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
) {
JsonFileReader reader = new JsonFileReader(file, readerAllocator);
Schema schema = reader.start();
LOGGER.debug("reading schema: " + schema);

// initialize vectors
try (VectorSchemaRoot root = reader.read();) {
validateDateTimeContent(count, root);
}
reader.close();
}
}

@Test
public void testSetStructLength() throws IOException {
File file = new File("../../integration/data/struct_example.json");
Expand Down