diff --git a/csharp/README.md b/csharp/README.md index fc393c20c90..121a4b25847 100644 --- a/csharp/README.md +++ b/csharp/README.md @@ -148,3 +148,11 @@ Build from the Apache Arrow project root. dotnet test test/Apache.Arrow.Tests All build artifacts are placed in the **artifacts** folder in the project root. + +# Updating FlatBuffers code + +See https://google.github.io/flatbuffers/flatbuffers_guide_use_java_c-sharp.html for how to get the `flatc` executable. + +Run `flatc --csharp` on each `.fbs` file in the [format](../format) folder. And replace the checked in `.cs` files under [FlatBuf](src/Apache.Arrow/Flatbuf) with the generated files. + +Update the non-generated [FlatBuffers](src/Apache.Arrow/Flatbuf/FlatBuffers) `.cs` files with the files from the [google/flatbuffers repo](https://github.com/google/flatbuffers/tree/master/net/FlatBuffers). diff --git a/csharp/src/Apache.Arrow/Flatbuf/Block.cs b/csharp/src/Apache.Arrow/Flatbuf/Block.cs index ec97fd19bec..89c065b202c 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/Block.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/Block.cs @@ -8,7 +8,7 @@ namespace Apache.Arrow.Flatbuf using global::System; using global::FlatBuffers; -public struct Block : IFlatbufferObject +internal struct Block : IFlatbufferObject { private Struct __p; public ByteBuffer ByteBuffer { get { return __p.bb; } } diff --git a/csharp/src/Apache.Arrow/Flatbuf/Buffer.cs b/csharp/src/Apache.Arrow/Flatbuf/Buffer.cs index 25506f8f578..7b2315cab4b 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/Buffer.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/Buffer.cs @@ -10,7 +10,7 @@ namespace Apache.Arrow.Flatbuf /// ---------------------------------------------------------------------- /// A Buffer represents a single contiguous memory segment -public struct Buffer : IFlatbufferObject +internal struct Buffer : IFlatbufferObject { private Struct __p; public ByteBuffer ByteBuffer { get { return __p.bb; } } diff --git a/csharp/src/Apache.Arrow/Flatbuf/DictionaryBatch.cs b/csharp/src/Apache.Arrow/Flatbuf/DictionaryBatch.cs index e16bf159927..e3afafdd5cb 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/DictionaryBatch.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/DictionaryBatch.cs @@ -14,7 +14,7 @@ namespace Apache.Arrow.Flatbuf /// There is one vector / column per dictionary, but that vector / column /// may be spread across multiple dictionary batches by using the isDelta /// flag -public struct DictionaryBatch : IFlatbufferObject +internal struct DictionaryBatch : IFlatbufferObject { private Table __p; public ByteBuffer ByteBuffer { get { return __p.bb; } } diff --git a/csharp/src/Apache.Arrow/Flatbuf/DictionaryEncoding.cs b/csharp/src/Apache.Arrow/Flatbuf/DictionaryEncoding.cs index 282caf304eb..02a35fdd41b 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/DictionaryEncoding.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/DictionaryEncoding.cs @@ -10,7 +10,7 @@ namespace Apache.Arrow.Flatbuf /// ---------------------------------------------------------------------- /// Dictionary encoding metadata -public struct DictionaryEncoding : IFlatbufferObject +internal struct DictionaryEncoding : IFlatbufferObject { private Table __p; public ByteBuffer ByteBuffer { get { return __p.bb; } } diff --git a/csharp/src/Apache.Arrow/Flatbuf/Enums/DateUnit.cs b/csharp/src/Apache.Arrow/Flatbuf/Enums/DateUnit.cs index 66c0740204f..46fd0cc4c83 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/Enums/DateUnit.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/Enums/DateUnit.cs @@ -5,7 +5,7 @@ namespace Apache.Arrow.Flatbuf { -public enum DateUnit : short +internal enum DateUnit : short { DAY = 0, MILLISECOND = 1, diff --git a/csharp/src/Apache.Arrow/Flatbuf/Enums/Endianness.cs b/csharp/src/Apache.Arrow/Flatbuf/Enums/Endianness.cs index 0609f674c6b..a0e64f4ff0e 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/Enums/Endianness.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/Enums/Endianness.cs @@ -7,7 +7,7 @@ namespace Apache.Arrow.Flatbuf /// ---------------------------------------------------------------------- /// Endianness of the platform producing the data -public enum Endianness : short +internal enum Endianness : short { Little = 0, Big = 1, diff --git a/csharp/src/Apache.Arrow/Flatbuf/Enums/IntervalUnit.cs b/csharp/src/Apache.Arrow/Flatbuf/Enums/IntervalUnit.cs index 9134b704072..d1363968df0 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/Enums/IntervalUnit.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/Enums/IntervalUnit.cs @@ -5,7 +5,7 @@ namespace Apache.Arrow.Flatbuf { -public enum IntervalUnit : short +internal enum IntervalUnit : short { YEAR_MONTH = 0, DAY_TIME = 1, diff --git a/csharp/src/Apache.Arrow/Flatbuf/Enums/MessageHeader.cs b/csharp/src/Apache.Arrow/Flatbuf/Enums/MessageHeader.cs index 7334de64896..94d239bfa93 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/Enums/MessageHeader.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/Enums/MessageHeader.cs @@ -13,7 +13,7 @@ namespace Apache.Arrow.Flatbuf /// Arrow implementations do not need to implement all of the message types, /// which may include experimental metadata types. For maximum compatibility, /// it is best to send data using RecordBatch -public enum MessageHeader : byte +internal enum MessageHeader : byte { NONE = 0, Schema = 1, diff --git a/csharp/src/Apache.Arrow/Flatbuf/Enums/MetadataVersion.cs b/csharp/src/Apache.Arrow/Flatbuf/Enums/MetadataVersion.cs index 3b97c2f623a..9d5c93512d4 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/Enums/MetadataVersion.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/Enums/MetadataVersion.cs @@ -5,7 +5,7 @@ namespace Apache.Arrow.Flatbuf { -public enum MetadataVersion : short +internal enum MetadataVersion : short { /// 0.1.0 V1 = 0, diff --git a/csharp/src/Apache.Arrow/Flatbuf/Enums/Precision.cs b/csharp/src/Apache.Arrow/Flatbuf/Enums/Precision.cs index 8c8552c405d..3f47a2c0b92 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/Enums/Precision.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/Enums/Precision.cs @@ -5,7 +5,7 @@ namespace Apache.Arrow.Flatbuf { -public enum Precision : short +internal enum Precision : short { HALF = 0, SINGLE = 1, diff --git a/csharp/src/Apache.Arrow/Flatbuf/Enums/TimeUnit.cs b/csharp/src/Apache.Arrow/Flatbuf/Enums/TimeUnit.cs index 8a96de02566..300b835d948 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/Enums/TimeUnit.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/Enums/TimeUnit.cs @@ -5,7 +5,7 @@ namespace Apache.Arrow.Flatbuf { -public enum TimeUnit : short +internal enum TimeUnit : short { SECOND = 0, MILLISECOND = 1, diff --git a/csharp/src/Apache.Arrow/Flatbuf/Enums/Type.cs b/csharp/src/Apache.Arrow/Flatbuf/Enums/Type.cs index dce9ac8dd0f..5dcb1260b49 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/Enums/Type.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/Enums/Type.cs @@ -8,7 +8,7 @@ namespace Apache.Arrow.Flatbuf /// ---------------------------------------------------------------------- /// Top-level Type value, enabling extensible type-specific metadata. We can /// add new logical types to Type without breaking backwards compatibility -public enum Type : byte +internal enum Type : byte { NONE = 0, Null = 1, diff --git a/csharp/src/Apache.Arrow/Flatbuf/Enums/UnionMode.cs b/csharp/src/Apache.Arrow/Flatbuf/Enums/UnionMode.cs index bb6ddd3627f..724ff4ac0e3 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/Enums/UnionMode.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/Enums/UnionMode.cs @@ -5,7 +5,7 @@ namespace Apache.Arrow.Flatbuf { -public enum UnionMode : short +internal enum UnionMode : short { Sparse = 0, Dense = 1, diff --git a/csharp/src/Apache.Arrow/Flatbuf/Field.cs b/csharp/src/Apache.Arrow/Flatbuf/Field.cs index a75f2b563fa..a4f9e305734 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/Field.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/Field.cs @@ -15,7 +15,7 @@ namespace Apache.Arrow.Flatbuf /// - children is only for nested Arrow arrays /// - For primitive types, children will have length 0 /// - nullable should default to true in general -public struct Field : IFlatbufferObject +internal struct Field : IFlatbufferObject { private Table __p; public ByteBuffer ByteBuffer { get { return __p.bb; } } @@ -25,7 +25,12 @@ public struct Field : IFlatbufferObject public Field __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; } public string Name { get { int o = __p.__offset(4); return o != 0 ? __p.__string(o + __p.bb_pos) : null; } } +#if ENABLE_SPAN_T + public Span GetNameBytes() { return __p.__vector_as_span(4); } +#else public ArraySegment? GetNameBytes() { return __p.__vector_as_arraysegment(4); } +#endif + public byte[] GetNameArray() { return __p.__vector_as_array(4); } public bool Nullable { get { int o = __p.__offset(6); return o != 0 ? 0!=__p.bb.Get(o + __p.bb_pos) : (bool)false; } } public Type TypeType { get { int o = __p.__offset(8); return o != 0 ? (Type)__p.bb.Get(o + __p.bb_pos) : Flatbuf.Type.NONE; } } public TTable? Type() where TTable : struct, IFlatbufferObject { int o = __p.__offset(10); return o != 0 ? (TTable?)__p.__union(o) : null; } @@ -62,9 +67,11 @@ public static Offset CreateField(FlatBufferBuilder builder, public static void AddDictionary(FlatBufferBuilder builder, Offset dictionaryOffset) { builder.AddOffset(4, dictionaryOffset.Value, 0); } public static void AddChildren(FlatBufferBuilder builder, VectorOffset childrenOffset) { builder.AddOffset(5, childrenOffset.Value, 0); } public static VectorOffset CreateChildrenVector(FlatBufferBuilder builder, Offset[] data) { builder.StartVector(4, data.Length, 4); for (int i = data.Length - 1; i >= 0; i--) builder.AddOffset(data[i].Value); return builder.EndVector(); } + public static VectorOffset CreateChildrenVectorBlock(FlatBufferBuilder builder, Offset[] data) { builder.StartVector(4, data.Length, 4); builder.Add(data); return builder.EndVector(); } public static void StartChildrenVector(FlatBufferBuilder builder, int numElems) { builder.StartVector(4, numElems, 4); } public static void AddCustomMetadata(FlatBufferBuilder builder, VectorOffset customMetadataOffset) { builder.AddOffset(6, customMetadataOffset.Value, 0); } public static VectorOffset CreateCustomMetadataVector(FlatBufferBuilder builder, Offset[] data) { builder.StartVector(4, data.Length, 4); for (int i = data.Length - 1; i >= 0; i--) builder.AddOffset(data[i].Value); return builder.EndVector(); } + public static VectorOffset CreateCustomMetadataVectorBlock(FlatBufferBuilder builder, Offset[] data) { builder.StartVector(4, data.Length, 4); builder.Add(data); return builder.EndVector(); } public static void StartCustomMetadataVector(FlatBufferBuilder builder, int numElems) { builder.StartVector(4, numElems, 4); } public static Offset EndField(FlatBufferBuilder builder) { int o = builder.EndObject(); diff --git a/csharp/src/Apache.Arrow/Flatbuf/FieldNode.cs b/csharp/src/Apache.Arrow/Flatbuf/FieldNode.cs index 53b01b8f9a6..811e10ea46b 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/FieldNode.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/FieldNode.cs @@ -17,7 +17,7 @@ namespace Apache.Arrow.Flatbuf /// For example, a List with values [[1, 2, 3], null, [4], [5, 6], null] /// would have {length: 5, null_count: 2} for its List node, and {length: 6, /// null_count: 0} for its Int16 node, as separate FieldNode structs -public struct FieldNode : IFlatbufferObject +internal struct FieldNode : IFlatbufferObject { private Struct __p; public ByteBuffer ByteBuffer { get { return __p.bb; } } diff --git a/csharp/src/Apache.Arrow/Flatbuf/FixedSizeBinary.cs b/csharp/src/Apache.Arrow/Flatbuf/FixedSizeBinary.cs index a022af7e0c1..b6414a23b5c 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/FixedSizeBinary.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/FixedSizeBinary.cs @@ -8,7 +8,7 @@ namespace Apache.Arrow.Flatbuf using global::System; using global::FlatBuffers; -public struct FixedSizeBinary : IFlatbufferObject +internal struct FixedSizeBinary : IFlatbufferObject { private Table __p; public ByteBuffer ByteBuffer { get { return __p.bb; } } diff --git a/csharp/src/Apache.Arrow/Flatbuf/FixedSizeList.cs b/csharp/src/Apache.Arrow/Flatbuf/FixedSizeList.cs index 12a23d980d8..0ca69b7a196 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/FixedSizeList.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/FixedSizeList.cs @@ -8,7 +8,7 @@ namespace Apache.Arrow.Flatbuf using global::System; using global::FlatBuffers; -public struct FixedSizeList : IFlatbufferObject +internal struct FixedSizeList : IFlatbufferObject { private Table __p; public ByteBuffer ByteBuffer { get { return __p.bb; } } diff --git a/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/ByteBuffer.cs b/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/ByteBuffer.cs index 307a98fadca..63a35bef1a7 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/ByteBuffer.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/ByteBuffer.cs @@ -14,7 +14,7 @@ * limitations under the License. */ -// There are 2 #defines that have an impact on performance of this ByteBuffer implementation +// There are 3 #defines that have an impact on performance / features of this ByteBuffer implementation // // UNSAFE_BYTEBUFFER // This will use unsafe code to manipulate the underlying byte array. This @@ -24,6 +24,12 @@ // This will disable the bounds check asserts to the byte array. This can // yield a small performance gain in normal code.. // +// ENABLE_SPAN_T +// This will enable reading and writing blocks of memory with a Span instead if just +// T[]. You can also enable writing directly to shared memory or other types of memory +// by providing a custom implementation of ByteBufferAllocator. +// ENABLE_SPAN_T also requires UNSAFE_BYTEBUFFER to be defined +// // Using UNSAFE_BYTEBUFFER and BYTEBUFFER_NO_BOUNDS_CHECK together can yield a // performance gain of ~15% for some operations, however doing so is potentially // dangerous. Do so at your own risk! @@ -32,19 +38,135 @@ using System; using System.Collections.Generic; using System.IO; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using System.Text; +#if ENABLE_SPAN_T && !UNSAFE_BYTEBUFFER +#error ENABLE_SPAN_T requires UNSAFE_BYTEBUFFER to also be defined +#endif + namespace FlatBuffers { + internal abstract class ByteBufferAllocator : IDisposable + { +#if UNSAFE_BYTEBUFFER + public unsafe byte* Buffer + { + get; + protected set; + } +#else + public byte[] Buffer + { + get; + protected set; + } +#endif + + public int Length + { + get; + protected set; + } + + public abstract void Dispose(); + + public abstract void GrowFront(int newSize); + +#if !ENABLE_SPAN_T + public abstract byte[] ByteArray { get; } +#endif + } + + internal class ByteArrayAllocator : ByteBufferAllocator + { + private byte[] _buffer; + + public ByteArrayAllocator(byte[] buffer) + { + _buffer = buffer; + InitPointer(); + } + + public override void GrowFront(int newSize) + { + if ((Length & 0xC0000000) != 0) + throw new Exception( + "ByteBuffer: cannot grow buffer beyond 2 gigabytes."); + + if (newSize < Length) + throw new Exception("ByteBuffer: cannot truncate buffer."); + + byte[] newBuffer = new byte[newSize]; + System.Buffer.BlockCopy(_buffer, 0, newBuffer, newSize - Length, Length); + _buffer = newBuffer; + InitPointer(); + } + + public override void Dispose() + { + GC.SuppressFinalize(this); +#if UNSAFE_BYTEBUFFER + if (_handle.IsAllocated) + { + _handle.Free(); + } +#endif + } + +#if !ENABLE_SPAN_T + public override byte[] ByteArray + { + get { return _buffer; } + } +#endif + +#if UNSAFE_BYTEBUFFER + private GCHandle _handle; + + ~ByteArrayAllocator() + { + if (_handle.IsAllocated) + { + _handle.Free(); + } + } +#endif + + private void InitPointer() + { + Length = _buffer.Length; +#if UNSAFE_BYTEBUFFER + if (_handle.IsAllocated) + { + _handle.Free(); + } + _handle = GCHandle.Alloc(_buffer, GCHandleType.Pinned); + unsafe + { + Buffer = (byte*)_handle.AddrOfPinnedObject().ToPointer(); + } +#else + Buffer = _buffer; +#endif + } + } + + /// /// Class to mimic Java's ByteBuffer which is used heavily in Flatbuffers. /// - public class ByteBuffer + internal class ByteBuffer : IDisposable { - protected byte[] _buffer; + private ByteBufferAllocator _buffer; private int _pos; // Must track start of the buffer. - public int Length { get { return _buffer.Length; } } + public ByteBuffer(ByteBufferAllocator allocator, int position) + { + _buffer = allocator; + _pos = position; + } public ByteBuffer(int size) : this(new byte[size]) { } @@ -52,15 +174,25 @@ public ByteBuffer(byte[] buffer) : this(buffer, 0) { } public ByteBuffer(byte[] buffer, int pos) { - _buffer = buffer; + _buffer = new ByteArrayAllocator(buffer); _pos = pos; } + public void Dispose() + { + if (_buffer != null) + { + _buffer.Dispose(); + } + } + public int Position { get { return _pos; } set { _pos = value; } } + public int Length { get { return _buffer.Length; } } + public void Reset() { _pos = 0; @@ -77,17 +209,7 @@ public ByteBuffer Duplicate() // the end of the new buffer. public void GrowFront(int newSize) { - if ((Length & 0xC0000000) != 0) - throw new Exception( - "ByteBuffer: cannot grow buffer beyond 2 gigabytes."); - - if (newSize < Length) - throw new Exception("ByteBuffer: cannot truncate buffer."); - - byte[] newBuffer = new byte[newSize]; - Buffer.BlockCopy(_buffer, 0, newBuffer, newSize - Length, - Length); - _buffer = newBuffer; + _buffer.GrowFront(newSize); } public byte[] ToArray(int pos, int len) @@ -145,16 +267,38 @@ public static int ArraySize(T[] x) return SizeOf() * x.Length; } +#if ENABLE_SPAN_T + public static int ArraySize(Span x) + { + return SizeOf() * x.Length; + } +#endif + // Get a portion of the buffer casted into an array of type T, given // the buffer position and length. +#if ENABLE_SPAN_T public T[] ToArray(int pos, int len) - where T: struct + where T : struct + { + unsafe + { + AssertOffsetAndLength(pos, len); + T[] arr = new T[len]; + var typed = MemoryMarshal.Cast(new Span(_buffer.Buffer + pos, _buffer.Length)); + typed.Slice(0, arr.Length).CopyTo(arr); + return arr; + } + } +#else + public T[] ToArray(int pos, int len) + where T : struct { AssertOffsetAndLength(pos, len); T[] arr = new T[len]; - Buffer.BlockCopy(_buffer, pos, arr, 0, ArraySize(arr)); + Buffer.BlockCopy(_buffer.ByteArray, pos, arr, 0, ArraySize(arr)); return arr; } +#endif public byte[] ToSizedArray() { @@ -166,15 +310,25 @@ public byte[] ToFullArray() return ToArray(0, Length); } + +#if ENABLE_SPAN_T + public unsafe Span ToSpan(int pos, int len) + { + return new Span(_buffer.Buffer, _buffer.Length).Slice(pos, len); + } +#else public ArraySegment ToArraySegment(int pos, int len) { - return new ArraySegment(_buffer, pos, len); + return new ArraySegment(_buffer.ByteArray, pos, len); } +#endif +#if !ENABLE_SPAN_T public MemoryStream ToMemoryStream(int pos, int len) { - return new MemoryStream(_buffer, pos, len); + return new MemoryStream(_buffer.ByteArray, pos, len); } +#endif #if !UNSAFE_BYTEBUFFER // Pre-allocated helper arrays for convertion. @@ -217,14 +371,14 @@ protected void WriteLittleEndian(int offset, int count, ulong data) { for (int i = 0; i < count; i++) { - _buffer[offset + i] = (byte)(data >> i * 8); + _buffer.Buffer[offset + i] = (byte)(data >> i * 8); } } else { for (int i = 0; i < count; i++) { - _buffer[offset + count - 1 - i] = (byte)(data >> i * 8); + _buffer.Buffer[offset + count - 1 - i] = (byte)(data >> i * 8); } } } @@ -237,14 +391,14 @@ protected ulong ReadLittleEndian(int offset, int count) { for (int i = 0; i < count; i++) { - r |= (ulong)_buffer[offset + i] << i * 8; + r |= (ulong)_buffer.Buffer[offset + i] << i * 8; } } else { for (int i = 0; i < count; i++) { - r |= (ulong)_buffer[offset + count - 1 - i] << i * 8; + r |= (ulong)_buffer.Buffer[offset + count - 1 - i] << i * 8; } } return r; @@ -253,30 +407,57 @@ protected ulong ReadLittleEndian(int offset, int count) private void AssertOffsetAndLength(int offset, int length) { - #if !BYTEBUFFER_NO_BOUNDS_CHECK +#if !BYTEBUFFER_NO_BOUNDS_CHECK if (offset < 0 || offset > _buffer.Length - length) throw new ArgumentOutOfRangeException(); - #endif +#endif } +#if UNSAFE_BYTEBUFFER + + public unsafe void PutSbyte(int offset, sbyte value) + { + AssertOffsetAndLength(offset, sizeof(sbyte)); + _buffer.Buffer[offset] = (byte)value; + } + + public unsafe void PutByte(int offset, byte value) + { + AssertOffsetAndLength(offset, sizeof(byte)); + _buffer.Buffer[offset] = value; + } + + public unsafe void PutByte(int offset, byte value, int count) + { + AssertOffsetAndLength(offset, sizeof(byte) * count); + for (var i = 0; i < count; ++i) + _buffer.Buffer[offset + i] = value; + } + + // this method exists in order to conform with Java ByteBuffer standards + public void Put(int offset, byte value) + { + PutByte(offset, value); + } +#else public void PutSbyte(int offset, sbyte value) { AssertOffsetAndLength(offset, sizeof(sbyte)); - _buffer[offset] = (byte)value; + _buffer.Buffer[offset] = (byte)value; } public void PutByte(int offset, byte value) { AssertOffsetAndLength(offset, sizeof(byte)); - _buffer[offset] = value; + _buffer.Buffer[offset] = value; } public void PutByte(int offset, byte value, int count) { AssertOffsetAndLength(offset, sizeof(byte) * count); for (var i = 0; i < count; ++i) - _buffer[offset + i] = value; + _buffer.Buffer[offset + i] = value; } // this method exists in order to conform with Java ByteBuffer standards @@ -284,13 +465,25 @@ public void Put(int offset, byte value) { PutByte(offset, value); } +#endif +#if ENABLE_SPAN_T + public unsafe void PutStringUTF8(int offset, string value) + { + AssertOffsetAndLength(offset, value.Length); + fixed (char* s = value) + { + Encoding.UTF8.GetBytes(s, value.Length, _buffer.Buffer + offset, Length - offset); + } + } +#else public void PutStringUTF8(int offset, string value) { AssertOffsetAndLength(offset, value.Length); Encoding.UTF8.GetBytes(value, 0, value.Length, - _buffer, offset); + _buffer.ByteArray, offset); } +#endif #if UNSAFE_BYTEBUFFER // Unsafe but more efficient versions of Put*. @@ -302,12 +495,10 @@ public void PutShort(int offset, short value) public unsafe void PutUshort(int offset, ushort value) { AssertOffsetAndLength(offset, sizeof(ushort)); - fixed (byte* ptr = _buffer) - { - *(ushort*)(ptr + offset) = BitConverter.IsLittleEndian - ? value - : ReverseBytes(value); - } + byte* ptr = _buffer.Buffer; + *(ushort*)(ptr + offset) = BitConverter.IsLittleEndian + ? value + : ReverseBytes(value); } public void PutInt(int offset, int value) @@ -318,12 +509,10 @@ public void PutInt(int offset, int value) public unsafe void PutUint(int offset, uint value) { AssertOffsetAndLength(offset, sizeof(uint)); - fixed (byte* ptr = _buffer) - { - *(uint*)(ptr + offset) = BitConverter.IsLittleEndian - ? value - : ReverseBytes(value); - } + byte* ptr = _buffer.Buffer; + *(uint*)(ptr + offset) = BitConverter.IsLittleEndian + ? value + : ReverseBytes(value); } public unsafe void PutLong(int offset, long value) @@ -334,44 +523,38 @@ public unsafe void PutLong(int offset, long value) public unsafe void PutUlong(int offset, ulong value) { AssertOffsetAndLength(offset, sizeof(ulong)); - fixed (byte* ptr = _buffer) - { - *(ulong*)(ptr + offset) = BitConverter.IsLittleEndian - ? value - : ReverseBytes(value); - } + byte* ptr = _buffer.Buffer; + *(ulong*)(ptr + offset) = BitConverter.IsLittleEndian + ? value + : ReverseBytes(value); } public unsafe void PutFloat(int offset, float value) { AssertOffsetAndLength(offset, sizeof(float)); - fixed (byte* ptr = _buffer) + byte* ptr = _buffer.Buffer; + if (BitConverter.IsLittleEndian) { - if (BitConverter.IsLittleEndian) - { - *(float*)(ptr + offset) = value; - } - else - { - *(uint*)(ptr + offset) = ReverseBytes(*(uint*)(&value)); - } + *(float*)(ptr + offset) = value; + } + else + { + *(uint*)(ptr + offset) = ReverseBytes(*(uint*)(&value)); } } public unsafe void PutDouble(int offset, double value) { AssertOffsetAndLength(offset, sizeof(double)); - fixed (byte* ptr = _buffer) + byte* ptr = _buffer.Buffer; + if (BitConverter.IsLittleEndian) { - if (BitConverter.IsLittleEndian) - { - *(double*)(ptr + offset) = value; + *(double*)(ptr + offset) = value; - } - else - { - *(ulong*)(ptr + offset) = ReverseBytes(*(ulong*)(ptr + offset)); - } + } + else + { + *(ulong*)(ptr + offset) = ReverseBytes(*(ulong*)(&value)); } } #else // !UNSAFE_BYTEBUFFER @@ -428,74 +611,45 @@ public void PutDouble(int offset, double value) WriteLittleEndian(offset, sizeof(double), ulonghelper[0]); } - /// - /// Copies an array of type T into this buffer, ending at the given - /// offset into this buffer. The starting offset is calculated based on the length - /// of the array and is the value returned. - /// - /// The type of the input data (must be a struct) - /// The offset into this buffer where the copy will end - /// The array to copy data from - /// The 'start' location of this buffer now, after the copy completed - public int Put(int offset, T[] x) - where T : struct - { - if(x == null) - { - throw new ArgumentNullException("Cannot put a null array"); - } - - if(x.Length == 0) - { - throw new ArgumentException("Cannot put an empty array"); - } - - if(!IsSupportedType()) - { - throw new ArgumentException("Cannot put an array of type " - + typeof(T) + " into this buffer"); - } +#endif // UNSAFE_BYTEBUFFER - if (BitConverter.IsLittleEndian) - { - int numBytes = ByteBuffer.ArraySize(x); - offset -= numBytes; - AssertOffsetAndLength(offset, numBytes); - // if we are LE, just do a block copy - Buffer.BlockCopy(x, 0, _buffer, offset, numBytes); - } - else - { - throw new NotImplementedException("Big Endian Support not implemented yet " + - "for putting typed arrays"); - // if we are BE, we have to swap each element by itself - //for(int i = x.Length - 1; i >= 0; i--) - //{ - // todo: low priority, but need to genericize the Put() functions - //} - } - return offset; +#if UNSAFE_BYTEBUFFER + public unsafe sbyte GetSbyte(int index) + { + AssertOffsetAndLength(index, sizeof(sbyte)); + return (sbyte)_buffer.Buffer[index]; } - -#endif // UNSAFE_BYTEBUFFER - + public unsafe byte Get(int index) + { + AssertOffsetAndLength(index, sizeof(byte)); + return _buffer.Buffer[index]; + } +#else public sbyte GetSbyte(int index) { AssertOffsetAndLength(index, sizeof(sbyte)); - return (sbyte)_buffer[index]; + return (sbyte)_buffer.Buffer[index]; } public byte Get(int index) { AssertOffsetAndLength(index, sizeof(byte)); - return _buffer[index]; + return _buffer.Buffer[index]; } +#endif +#if ENABLE_SPAN_T + public unsafe string GetStringUTF8(int startPos, int len) + { + return Encoding.UTF8.GetString(_buffer.Buffer + startPos, len); + } +#else public string GetStringUTF8(int startPos, int len) { - return Encoding.UTF8.GetString(_buffer, startPos, len); + return Encoding.UTF8.GetString(_buffer.ByteArray, startPos, len); } +#endif #if UNSAFE_BYTEBUFFER // Unsafe but more efficient versions of Get*. @@ -507,7 +661,7 @@ public short GetShort(int offset) public unsafe ushort GetUshort(int offset) { AssertOffsetAndLength(offset, sizeof(ushort)); - fixed (byte* ptr = _buffer) + byte* ptr = _buffer.Buffer; { return BitConverter.IsLittleEndian ? *(ushort*)(ptr + offset) @@ -523,7 +677,7 @@ public int GetInt(int offset) public unsafe uint GetUint(int offset) { AssertOffsetAndLength(offset, sizeof(uint)); - fixed (byte* ptr = _buffer) + byte* ptr = _buffer.Buffer; { return BitConverter.IsLittleEndian ? *(uint*)(ptr + offset) @@ -539,7 +693,7 @@ public long GetLong(int offset) public unsafe ulong GetUlong(int offset) { AssertOffsetAndLength(offset, sizeof(ulong)); - fixed (byte* ptr = _buffer) + byte* ptr = _buffer.Buffer; { return BitConverter.IsLittleEndian ? *(ulong*)(ptr + offset) @@ -550,7 +704,7 @@ public unsafe ulong GetUlong(int offset) public unsafe float GetFloat(int offset) { AssertOffsetAndLength(offset, sizeof(float)); - fixed (byte* ptr = _buffer) + byte* ptr = _buffer.Buffer; { if (BitConverter.IsLittleEndian) { @@ -567,7 +721,7 @@ public unsafe float GetFloat(int offset) public unsafe double GetDouble(int offset) { AssertOffsetAndLength(offset, sizeof(double)); - fixed (byte* ptr = _buffer) + byte* ptr = _buffer.Buffer; { if (BitConverter.IsLittleEndian) { @@ -629,5 +783,98 @@ public double GetDouble(int index) return doublehelper[0]; } #endif // UNSAFE_BYTEBUFFER + + /// + /// Copies an array of type T into this buffer, ending at the given + /// offset into this buffer. The starting offset is calculated based on the length + /// of the array and is the value returned. + /// + /// The type of the input data (must be a struct) + /// The offset into this buffer where the copy will end + /// The array to copy data from + /// The 'start' location of this buffer now, after the copy completed + public int Put(int offset, T[] x) + where T : struct + { + if (x == null) + { + throw new ArgumentNullException("Cannot put a null array"); + } + + if (x.Length == 0) + { + throw new ArgumentException("Cannot put an empty array"); + } + + if (!IsSupportedType()) + { + throw new ArgumentException("Cannot put an array of type " + + typeof(T) + " into this buffer"); + } + + if (BitConverter.IsLittleEndian) + { + int numBytes = ByteBuffer.ArraySize(x); + offset -= numBytes; + AssertOffsetAndLength(offset, numBytes); + // if we are LE, just do a block copy +#if ENABLE_SPAN_T + unsafe + { + MemoryMarshal.Cast(x).CopyTo(new Span(_buffer.Buffer, _buffer.Length).Slice(offset, numBytes)); + } +#else + Buffer.BlockCopy(x, 0, _buffer.ByteArray, offset, numBytes); +#endif + } + else + { + throw new NotImplementedException("Big Endian Support not implemented yet " + + "for putting typed arrays"); + // if we are BE, we have to swap each element by itself + //for(int i = x.Length - 1; i >= 0; i--) + //{ + // todo: low priority, but need to genericize the Put() functions + //} + } + return offset; + } + +#if ENABLE_SPAN_T + public unsafe int Put(int offset, Span x) + where T : struct + { + if (x.Length == 0) + { + throw new ArgumentException("Cannot put an empty array"); + } + + if (!IsSupportedType()) + { + throw new ArgumentException("Cannot put an array of type " + + typeof(T) + " into this buffer"); + } + + if (BitConverter.IsLittleEndian) + { + int numBytes = ByteBuffer.ArraySize(x); + offset -= numBytes; + AssertOffsetAndLength(offset, numBytes); + // if we are LE, just do a block copy + MemoryMarshal.Cast(x).CopyTo(new Span(_buffer.Buffer, _buffer.Length).Slice(offset, numBytes)); + } + else + { + throw new NotImplementedException("Big Endian Support not implemented yet " + + "for putting typed arrays"); + // if we are BE, we have to swap each element by itself + //for(int i = x.Length - 1; i >= 0; i--) + //{ + // todo: low priority, but need to genericize the Put() functions + //} + } + return offset; + } +#endif } } diff --git a/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/ByteBufferUtil.cs b/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/ByteBufferUtil.cs index 66e8266843b..cfba4305e17 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/ByteBufferUtil.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/ByteBufferUtil.cs @@ -21,7 +21,7 @@ namespace FlatBuffers /// /// Class that collects utility functions around `ByteBuffer`. /// - public class ByteBufferUtil + internal class ByteBufferUtil { // Extract the size prefix from a `ByteBuffer`. public static int GetSizePrefix(ByteBuffer bb) { diff --git a/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/FlatBufferBuilder.cs b/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/FlatBufferBuilder.cs index 33bba969dda..0485068cc89 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/FlatBufferBuilder.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/FlatBufferBuilder.cs @@ -28,7 +28,7 @@ namespace FlatBuffers /// Responsible for building up and accessing a FlatBuffer formatted byte /// array (via ByteBuffer). /// - public class FlatBufferBuilder + internal class FlatBufferBuilder { private int _space; private ByteBuffer _bb; @@ -62,6 +62,17 @@ public FlatBufferBuilder(int initialSize) _bb = new ByteBuffer(initialSize); } + /// + /// Create a FlatBufferBuilder backed by the pased in ByteBuffer + /// + /// The ByteBuffer to write to + public FlatBufferBuilder(ByteBuffer buffer) + { + _bb = buffer; + _space = buffer.Length; + buffer.Reset(); + } + /// /// Reset the FlatBufferBuilder by purging all data that it holds. /// @@ -191,6 +202,20 @@ public void Put(T[] x) _space = _bb.Put(_space, x); } +#if ENABLE_SPAN_T + /// + /// Puts a span of type T into this builder at the + /// current offset + /// + /// The type of the input data + /// The span to copy data from + public void Put(Span x) + where T : struct + { + _space = _bb.Put(_space, x); + } +#endif + public void PutDouble(double x) { _bb.PutDouble(_space -= sizeof(double), x); @@ -288,6 +313,28 @@ public void Add(T[] x) Put(x); } +#if ENABLE_SPAN_T + /// + /// Add a span of type T to the buffer (aligns the data and grows if necessary). + /// + /// The type of the input data + /// The span to copy data from + public void Add(Span x) + where T : struct + { + if (!ByteBuffer.IsSupportedType()) + { + throw new ArgumentException("Cannot add this Type array to the builder"); + } + + int size = ByteBuffer.SizeOf(); + // Need to prep on size (for data alignment) and then we pass the + // rest of the length (minus 1) as additional bytes + Prep(size, size * (x.Length - 1)); + Put(x); + } +#endif + /// /// Add a `double` to the buffer (aligns the data and grows if necessary). /// @@ -511,6 +558,27 @@ public StringOffset CreateString(string s) return new StringOffset(EndVector().Value); } + +#if ENABLE_SPAN_T + /// + /// Creates a string in the buffer from a Span containing + /// a UTF8 string. + /// + /// the UTF8 string to add to the buffer + /// + /// The offset in the buffer where the encoded string starts. + /// + public StringOffset CreateUTF8String(Span chars) + { + NotNested(); + AddByte(0); + var utf8StringLen = chars.Length; + StartVector(1, utf8StringLen, 1); + _space = _bb.Put(_space, chars); + return new StringOffset(EndVector().Value); + } +#endif + /// @cond FLATBUFFERS_INTERNAL // Structs are stored inline, so nothing additional is being added. // `d` is always 0. @@ -568,7 +636,7 @@ public int EndObject() break; } - endLoop: { } + endLoop: { } } if (existingVtable != 0) { diff --git a/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/FlatBufferConstants.cs b/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/FlatBufferConstants.cs index e30f3f3944f..68a4e9cad09 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/FlatBufferConstants.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/FlatBufferConstants.cs @@ -21,7 +21,7 @@ namespace FlatBuffers { - public static class FlatBufferConstants + internal static class FlatBufferConstants { public const int FileIdentifierLength = 4; public const int SizePrefixLength = 4; diff --git a/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/IFlatbufferObject.cs b/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/IFlatbufferObject.cs index 6a15aba6e5b..1be932c9867 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/IFlatbufferObject.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/IFlatbufferObject.cs @@ -19,7 +19,7 @@ namespace FlatBuffers /// /// This is the base for both structs and tables. /// - public interface IFlatbufferObject + internal interface IFlatbufferObject { void __init(int _i, ByteBuffer _bb); diff --git a/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/Offset.cs b/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/Offset.cs index 2b17cec85d4..70b75b6de7e 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/Offset.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/Offset.cs @@ -19,7 +19,7 @@ namespace FlatBuffers /// /// Offset class for typesafe assignments. /// - public struct Offset where T : struct + internal struct Offset where T : struct { public int Value; public Offset(int value) @@ -28,7 +28,7 @@ public Offset(int value) } } - public struct StringOffset + internal struct StringOffset { public int Value; public StringOffset(int value) @@ -37,7 +37,7 @@ public StringOffset(int value) } } - public struct VectorOffset + internal struct VectorOffset { public int Value; public VectorOffset(int value) diff --git a/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/Struct.cs b/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/Struct.cs index 61da32f77bc..ca44ff92044 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/Struct.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/Struct.cs @@ -19,7 +19,7 @@ namespace FlatBuffers /// /// All structs in the generated code derive from this class, and add their own accessors. /// - public struct Struct + internal struct Struct { public int bb_pos; public ByteBuffer bb; diff --git a/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/Table.cs b/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/Table.cs index 07db5f42315..f809072ba83 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/Table.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/Table.cs @@ -22,7 +22,7 @@ namespace FlatBuffers /// /// All tables in the generated code derive from this struct, and add their own accessors. /// - public struct Table + internal struct Table { public int bb_pos; public ByteBuffer bb; @@ -78,6 +78,23 @@ public int __vector(int offset) return offset + bb.GetInt(offset) + sizeof(int); // data starts after the length } +#if ENABLE_SPAN_T + // Get the data of a vector whoses offset is stored at "offset" in this object as an + // Spant<byte>. If the vector is not present in the ByteBuffer, + // then an empty span will be returned. + public Span __vector_as_span(int offset) + { + var o = this.__offset(offset); + if (0 == o) + { + return new Span(); + } + + var pos = this.__vector(o); + var len = this.__vector_len(o); + return bb.ToSpan(pos, len); + } +#else // Get the data of a vector whoses offset is stored at "offset" in this object as an // ArraySegment<byte>. If the vector is not present in the ByteBuffer, // then a null value will be returned. @@ -93,6 +110,7 @@ public int __vector(int offset) var len = this.__vector_len(o); return bb.ToArraySegment(pos, len); } +#endif // Get the data of a vector whoses offset is stored at "offset" in this object as an // T[]. If the vector is not present in the ByteBuffer, then a null value will be diff --git a/csharp/src/Apache.Arrow/Flatbuf/Footer.cs b/csharp/src/Apache.Arrow/Flatbuf/Footer.cs index f45790a1ca5..d9f50a23c84 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/Footer.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/Footer.cs @@ -11,7 +11,7 @@ namespace Apache.Arrow.Flatbuf /// ---------------------------------------------------------------------- /// Arrow File metadata /// -public struct Footer : IFlatbufferObject +internal struct Footer : IFlatbufferObject { private Table __p; public ByteBuffer ByteBuffer { get { return __p.bb; } } diff --git a/csharp/src/Apache.Arrow/Flatbuf/KeyValue.cs b/csharp/src/Apache.Arrow/Flatbuf/KeyValue.cs index f5a062a6ec8..5b8c2efcaac 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/KeyValue.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/KeyValue.cs @@ -11,7 +11,7 @@ namespace Apache.Arrow.Flatbuf /// ---------------------------------------------------------------------- /// user defined key value pairs to add custom metadata to arrow /// key namespacing is the responsibility of the user -public struct KeyValue : IFlatbufferObject +internal struct KeyValue : IFlatbufferObject { private Table __p; public ByteBuffer ByteBuffer { get { return __p.bb; } } @@ -21,9 +21,19 @@ public struct KeyValue : IFlatbufferObject public KeyValue __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; } public string Key { get { int o = __p.__offset(4); return o != 0 ? __p.__string(o + __p.bb_pos) : null; } } +#if ENABLE_SPAN_T + public Span GetKeyBytes() { return __p.__vector_as_span(4); } +#else public ArraySegment? GetKeyBytes() { return __p.__vector_as_arraysegment(4); } +#endif + public byte[] GetKeyArray() { return __p.__vector_as_array(4); } public string Value { get { int o = __p.__offset(6); return o != 0 ? __p.__string(o + __p.bb_pos) : null; } } +#if ENABLE_SPAN_T + public Span GetValueBytes() { return __p.__vector_as_span(6); } +#else public ArraySegment? GetValueBytes() { return __p.__vector_as_arraysegment(6); } +#endif + public byte[] GetValueArray() { return __p.__vector_as_array(6); } public static Offset CreateKeyValue(FlatBufferBuilder builder, StringOffset keyOffset = default(StringOffset), diff --git a/csharp/src/Apache.Arrow/Flatbuf/Map.cs b/csharp/src/Apache.Arrow/Flatbuf/Map.cs index a47ff4bc6c1..9141f4c932d 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/Map.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/Map.cs @@ -32,7 +32,7 @@ namespace Apache.Arrow.Flatbuf /// The metadata is structured so that Arrow systems without special handling /// for Map can make Map an alias for List. The "layout" attribute for the Map /// field must have the same contents as a List. -public struct Map : IFlatbufferObject +internal struct Map : IFlatbufferObject { private Table __p; public ByteBuffer ByteBuffer { get { return __p.bb; } } diff --git a/csharp/src/Apache.Arrow/Flatbuf/Message.cs b/csharp/src/Apache.Arrow/Flatbuf/Message.cs index 71422fe6be0..0bf9ffa5314 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/Message.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/Message.cs @@ -8,7 +8,7 @@ namespace Apache.Arrow.Flatbuf using global::System; using global::FlatBuffers; -public struct Message : IFlatbufferObject +internal struct Message : IFlatbufferObject { private Table __p; public ByteBuffer ByteBuffer { get { return __p.bb; } } diff --git a/csharp/src/Apache.Arrow/Flatbuf/RecordBatch.cs b/csharp/src/Apache.Arrow/Flatbuf/RecordBatch.cs index 322175c88f5..9bd1acfc374 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/RecordBatch.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/RecordBatch.cs @@ -11,7 +11,7 @@ namespace Apache.Arrow.Flatbuf /// A data header describing the shared memory layout of a "record" or "row" /// batch. Some systems call this a "row batch" internally and others a "record /// batch". -public struct RecordBatch : IFlatbufferObject +internal struct RecordBatch : IFlatbufferObject { private Table __p; public ByteBuffer ByteBuffer { get { return __p.bb; } } diff --git a/csharp/src/Apache.Arrow/Flatbuf/Schema.cs b/csharp/src/Apache.Arrow/Flatbuf/Schema.cs index 94bbf18b196..206a8a1983d 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/Schema.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/Schema.cs @@ -10,7 +10,7 @@ namespace Apache.Arrow.Flatbuf /// ---------------------------------------------------------------------- /// A Schema describes the columns in a row batch -public struct Schema : IFlatbufferObject +internal struct Schema : IFlatbufferObject { private Table __p; public ByteBuffer ByteBuffer { get { return __p.bb; } } @@ -43,9 +43,11 @@ public static Offset CreateSchema(FlatBufferBuilder builder, public static void AddEndianness(FlatBufferBuilder builder, Endianness endianness) { builder.AddShort(0, (short)endianness, 0); } public static void AddFields(FlatBufferBuilder builder, VectorOffset fieldsOffset) { builder.AddOffset(1, fieldsOffset.Value, 0); } public static VectorOffset CreateFieldsVector(FlatBufferBuilder builder, Offset[] data) { builder.StartVector(4, data.Length, 4); for (int i = data.Length - 1; i >= 0; i--) builder.AddOffset(data[i].Value); return builder.EndVector(); } + public static VectorOffset CreateFieldsVectorBlock(FlatBufferBuilder builder, Offset[] data) { builder.StartVector(4, data.Length, 4); builder.Add(data); return builder.EndVector(); } public static void StartFieldsVector(FlatBufferBuilder builder, int numElems) { builder.StartVector(4, numElems, 4); } public static void AddCustomMetadata(FlatBufferBuilder builder, VectorOffset customMetadataOffset) { builder.AddOffset(2, customMetadataOffset.Value, 0); } public static VectorOffset CreateCustomMetadataVector(FlatBufferBuilder builder, Offset[] data) { builder.StartVector(4, data.Length, 4); for (int i = data.Length - 1; i >= 0; i--) builder.AddOffset(data[i].Value); return builder.EndVector(); } + public static VectorOffset CreateCustomMetadataVectorBlock(FlatBufferBuilder builder, Offset[] data) { builder.StartVector(4, data.Length, 4); builder.Add(data); return builder.EndVector(); } public static void StartCustomMetadataVector(FlatBufferBuilder builder, int numElems) { builder.StartVector(4, numElems, 4); } public static Offset EndSchema(FlatBufferBuilder builder) { int o = builder.EndObject(); diff --git a/csharp/src/Apache.Arrow/Flatbuf/Tensor.cs b/csharp/src/Apache.Arrow/Flatbuf/Tensor.cs index f1d7285892a..918d7d6ea73 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/Tensor.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/Tensor.cs @@ -8,7 +8,7 @@ namespace Apache.Arrow.Flatbuf using global::System; using global::FlatBuffers; -public struct Tensor : IFlatbufferObject +internal struct Tensor : IFlatbufferObject { private Table __p; public ByteBuffer ByteBuffer { get { return __p.bb; } } @@ -27,7 +27,12 @@ public struct Tensor : IFlatbufferObject /// Non-negative byte offsets to advance one value cell along each dimension public long Strides(int j) { int o = __p.__offset(10); return o != 0 ? __p.bb.GetLong(__p.__vector(o) + j * 8) : (long)0; } public int StridesLength { get { int o = __p.__offset(10); return o != 0 ? __p.__vector_len(o) : 0; } } +#if ENABLE_SPAN_T + public Span GetStridesBytes() { return __p.__vector_as_span(10); } +#else public ArraySegment? GetStridesBytes() { return __p.__vector_as_arraysegment(10); } +#endif + public long[] GetStridesArray() { return __p.__vector_as_array(10); } /// The location and size of the tensor's data public Buffer? Data { get { int o = __p.__offset(12); return o != 0 ? (Buffer?)(new Buffer()).__assign(o + __p.bb_pos, __p.bb) : null; } } @@ -36,9 +41,11 @@ public struct Tensor : IFlatbufferObject public static void AddType(FlatBufferBuilder builder, int typeOffset) { builder.AddOffset(1, typeOffset, 0); } public static void AddShape(FlatBufferBuilder builder, VectorOffset shapeOffset) { builder.AddOffset(2, shapeOffset.Value, 0); } public static VectorOffset CreateShapeVector(FlatBufferBuilder builder, Offset[] data) { builder.StartVector(4, data.Length, 4); for (int i = data.Length - 1; i >= 0; i--) builder.AddOffset(data[i].Value); return builder.EndVector(); } + public static VectorOffset CreateShapeVectorBlock(FlatBufferBuilder builder, Offset[] data) { builder.StartVector(4, data.Length, 4); builder.Add(data); return builder.EndVector(); } public static void StartShapeVector(FlatBufferBuilder builder, int numElems) { builder.StartVector(4, numElems, 4); } public static void AddStrides(FlatBufferBuilder builder, VectorOffset stridesOffset) { builder.AddOffset(3, stridesOffset.Value, 0); } public static VectorOffset CreateStridesVector(FlatBufferBuilder builder, long[] data) { builder.StartVector(8, data.Length, 8); for (int i = data.Length - 1; i >= 0; i--) builder.AddLong(data[i]); return builder.EndVector(); } + public static VectorOffset CreateStridesVectorBlock(FlatBufferBuilder builder, long[] data) { builder.StartVector(8, data.Length, 8); builder.Add(data); return builder.EndVector(); } public static void StartStridesVector(FlatBufferBuilder builder, int numElems) { builder.StartVector(8, numElems, 8); } public static void AddData(FlatBufferBuilder builder, Offset dataOffset) { builder.AddStruct(4, dataOffset.Value, 0); } public static Offset EndTensor(FlatBufferBuilder builder) { diff --git a/csharp/src/Apache.Arrow/Flatbuf/TensorDim.cs b/csharp/src/Apache.Arrow/Flatbuf/TensorDim.cs index ee8e94e083c..a4c51824b8e 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/TensorDim.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/TensorDim.cs @@ -8,8 +8,10 @@ namespace Apache.Arrow.Flatbuf using global::System; using global::FlatBuffers; +/// ---------------------------------------------------------------------- +/// Data structures for dense tensors /// Shape data for a single axis in a tensor -public struct TensorDim : IFlatbufferObject +internal struct TensorDim : IFlatbufferObject { private Table __p; public ByteBuffer ByteBuffer { get { return __p.bb; } } @@ -22,7 +24,12 @@ public struct TensorDim : IFlatbufferObject public long Size { get { int o = __p.__offset(4); return o != 0 ? __p.bb.GetLong(o + __p.bb_pos) : (long)0; } } /// Name of the dimension, optional public string Name { get { int o = __p.__offset(6); return o != 0 ? __p.__string(o + __p.bb_pos) : null; } } +#if ENABLE_SPAN_T + public Span GetNameBytes() { return __p.__vector_as_span(6); } +#else public ArraySegment? GetNameBytes() { return __p.__vector_as_arraysegment(6); } +#endif + public byte[] GetNameArray() { return __p.__vector_as_array(6); } public static Offset CreateTensorDim(FlatBufferBuilder builder, long size = 0, diff --git a/csharp/src/Apache.Arrow/Flatbuf/Types/Binary.cs b/csharp/src/Apache.Arrow/Flatbuf/Types/Binary.cs index 119c00f2038..0738d11e903 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/Types/Binary.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/Types/Binary.cs @@ -8,7 +8,7 @@ namespace Apache.Arrow.Flatbuf using global::System; using global::FlatBuffers; -public struct Binary : IFlatbufferObject +internal struct Binary : IFlatbufferObject { private Table __p; public ByteBuffer ByteBuffer { get { return __p.bb; } } diff --git a/csharp/src/Apache.Arrow/Flatbuf/Types/Bool.cs b/csharp/src/Apache.Arrow/Flatbuf/Types/Bool.cs index 4f90845bd40..da488abfe01 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/Types/Bool.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/Types/Bool.cs @@ -8,7 +8,7 @@ namespace Apache.Arrow.Flatbuf using global::System; using global::FlatBuffers; -public struct Bool : IFlatbufferObject +internal struct Bool : IFlatbufferObject { private Table __p; public ByteBuffer ByteBuffer { get { return __p.bb; } } diff --git a/csharp/src/Apache.Arrow/Flatbuf/Types/Date.cs b/csharp/src/Apache.Arrow/Flatbuf/Types/Date.cs index 0074a4d3980..e9b7fb3ccae 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/Types/Date.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/Types/Date.cs @@ -14,7 +14,7 @@ namespace Apache.Arrow.Flatbuf /// * Milliseconds (64 bits) indicating UNIX time elapsed since the epoch (no /// leap seconds), where the values are evenly divisible by 86400000 /// * Days (32 bits) since the UNIX epoch -public struct Date : IFlatbufferObject +internal struct Date : IFlatbufferObject { private Table __p; public ByteBuffer ByteBuffer { get { return __p.bb; } } diff --git a/csharp/src/Apache.Arrow/Flatbuf/Types/Decimal.cs b/csharp/src/Apache.Arrow/Flatbuf/Types/Decimal.cs index c10a63d672c..b661709f722 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/Types/Decimal.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/Types/Decimal.cs @@ -8,7 +8,7 @@ namespace Apache.Arrow.Flatbuf using global::System; using global::FlatBuffers; -public struct Decimal : IFlatbufferObject +internal struct Decimal : IFlatbufferObject { private Table __p; public ByteBuffer ByteBuffer { get { return __p.bb; } } diff --git a/csharp/src/Apache.Arrow/Flatbuf/Types/FloatingPoint.cs b/csharp/src/Apache.Arrow/Flatbuf/Types/FloatingPoint.cs index 69590427594..0cb58ecf4ef 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/Types/FloatingPoint.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/Types/FloatingPoint.cs @@ -8,7 +8,7 @@ namespace Apache.Arrow.Flatbuf using global::System; using global::FlatBuffers; -public struct FloatingPoint : IFlatbufferObject +internal struct FloatingPoint : IFlatbufferObject { private Table __p; public ByteBuffer ByteBuffer { get { return __p.bb; } } diff --git a/csharp/src/Apache.Arrow/Flatbuf/Types/Int.cs b/csharp/src/Apache.Arrow/Flatbuf/Types/Int.cs index 13608a5c518..754080279f5 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/Types/Int.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/Types/Int.cs @@ -8,7 +8,7 @@ namespace Apache.Arrow.Flatbuf using global::System; using global::FlatBuffers; -public struct Int : IFlatbufferObject +internal struct Int : IFlatbufferObject { private Table __p; public ByteBuffer ByteBuffer { get { return __p.bb; } } diff --git a/csharp/src/Apache.Arrow/Flatbuf/Types/Interval.cs b/csharp/src/Apache.Arrow/Flatbuf/Types/Interval.cs index 4e6191dabe0..bb92448ac43 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/Types/Interval.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/Types/Interval.cs @@ -8,7 +8,7 @@ namespace Apache.Arrow.Flatbuf using global::System; using global::FlatBuffers; -public struct Interval : IFlatbufferObject +internal struct Interval : IFlatbufferObject { private Table __p; public ByteBuffer ByteBuffer { get { return __p.bb; } } diff --git a/csharp/src/Apache.Arrow/Flatbuf/Types/List.cs b/csharp/src/Apache.Arrow/Flatbuf/Types/List.cs index 3051cd8a85c..8f4985bb538 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/Types/List.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/Types/List.cs @@ -8,7 +8,7 @@ namespace Apache.Arrow.Flatbuf using global::System; using global::FlatBuffers; -public struct List : IFlatbufferObject +internal struct List : IFlatbufferObject { private Table __p; public ByteBuffer ByteBuffer { get { return __p.bb; } } diff --git a/csharp/src/Apache.Arrow/Flatbuf/Types/Null.cs b/csharp/src/Apache.Arrow/Flatbuf/Types/Null.cs index 3a0fdfc16b1..85fa5bb9919 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/Types/Null.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/Types/Null.cs @@ -9,7 +9,7 @@ namespace Apache.Arrow.Flatbuf using global::FlatBuffers; /// These are stored in the flatbuffer in the Type union below -public struct Null : IFlatbufferObject +internal struct Null : IFlatbufferObject { private Table __p; public ByteBuffer ByteBuffer { get { return __p.bb; } } diff --git a/csharp/src/Apache.Arrow/Flatbuf/Types/Struct_.cs b/csharp/src/Apache.Arrow/Flatbuf/Types/Struct_.cs index f1d7aa27dcf..8f3d708cd9a 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/Types/Struct_.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/Types/Struct_.cs @@ -11,7 +11,7 @@ namespace Apache.Arrow.Flatbuf /// A Struct_ in the flatbuffer metadata is the same as an Arrow Struct /// (according to the physical memory layout). We used Struct_ here as /// Struct is a reserved word in Flatbuffers -public struct Struct_ : IFlatbufferObject +internal struct Struct_ : IFlatbufferObject { private Table __p; public ByteBuffer ByteBuffer { get { return __p.bb; } } diff --git a/csharp/src/Apache.Arrow/Flatbuf/Types/Time.cs b/csharp/src/Apache.Arrow/Flatbuf/Types/Time.cs index 5d1d3947dab..1d7c0881cca 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/Types/Time.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/Types/Time.cs @@ -11,7 +11,7 @@ namespace Apache.Arrow.Flatbuf /// Time type. The physical storage type depends on the unit /// - SECOND and MILLISECOND: 32 bits /// - MICROSECOND and NANOSECOND: 64 bits -public struct Time : IFlatbufferObject +internal struct Time : IFlatbufferObject { private Table __p; public ByteBuffer ByteBuffer { get { return __p.bb; } } diff --git a/csharp/src/Apache.Arrow/Flatbuf/Types/Timestamp.cs b/csharp/src/Apache.Arrow/Flatbuf/Types/Timestamp.cs index 576b076dd2c..c0168b4cbf8 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/Types/Timestamp.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/Types/Timestamp.cs @@ -14,7 +14,7 @@ namespace Apache.Arrow.Flatbuf /// /// The Timestamp metadata supports both "time zone naive" and "time zone /// aware" timestamps. Read about the timezone attribute for more detail -public struct Timestamp : IFlatbufferObject +internal struct Timestamp : IFlatbufferObject { private Table __p; public ByteBuffer ByteBuffer { get { return __p.bb; } } @@ -45,7 +45,12 @@ public struct Timestamp : IFlatbufferObject /// between time zones is a metadata-only operation and does not change the /// underlying values public string Timezone { get { int o = __p.__offset(6); return o != 0 ? __p.__string(o + __p.bb_pos) : null; } } +#if ENABLE_SPAN_T + public Span GetTimezoneBytes() { return __p.__vector_as_span(6); } +#else public ArraySegment? GetTimezoneBytes() { return __p.__vector_as_arraysegment(6); } +#endif + public byte[] GetTimezoneArray() { return __p.__vector_as_array(6); } public static Offset CreateTimestamp(FlatBufferBuilder builder, TimeUnit unit = TimeUnit.SECOND, diff --git a/csharp/src/Apache.Arrow/Flatbuf/Types/Union.cs b/csharp/src/Apache.Arrow/Flatbuf/Types/Union.cs index b3ca36e984c..de2a85a17de 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/Types/Union.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/Types/Union.cs @@ -12,7 +12,7 @@ namespace Apache.Arrow.Flatbuf /// By default ids in the type vector refer to the offsets in the children /// optionally typeIds provides an indirection between the child offset and the type id /// for each child typeIds[offset] is the id used in the type vector -public struct Union : IFlatbufferObject +internal struct Union : IFlatbufferObject { private Table __p; public ByteBuffer ByteBuffer { get { return __p.bb; } } @@ -24,7 +24,12 @@ public struct Union : IFlatbufferObject public UnionMode Mode { get { int o = __p.__offset(4); return o != 0 ? (UnionMode)__p.bb.GetShort(o + __p.bb_pos) : UnionMode.Sparse; } } public int TypeIds(int j) { int o = __p.__offset(6); return o != 0 ? __p.bb.GetInt(__p.__vector(o) + j * 4) : (int)0; } public int TypeIdsLength { get { int o = __p.__offset(6); return o != 0 ? __p.__vector_len(o) : 0; } } +#if ENABLE_SPAN_T + public Span GetTypeIdsBytes() { return __p.__vector_as_span(6); } +#else public ArraySegment? GetTypeIdsBytes() { return __p.__vector_as_arraysegment(6); } +#endif + public int[] GetTypeIdsArray() { return __p.__vector_as_array(6); } public static Offset CreateUnion(FlatBufferBuilder builder, UnionMode mode = UnionMode.Sparse, @@ -39,6 +44,7 @@ public static Offset CreateUnion(FlatBufferBuilder builder, public static void AddMode(FlatBufferBuilder builder, UnionMode mode) { builder.AddShort(0, (short)mode, 0); } public static void AddTypeIds(FlatBufferBuilder builder, VectorOffset typeIdsOffset) { builder.AddOffset(1, typeIdsOffset.Value, 0); } public static VectorOffset CreateTypeIdsVector(FlatBufferBuilder builder, int[] data) { builder.StartVector(4, data.Length, 4); for (int i = data.Length - 1; i >= 0; i--) builder.AddInt(data[i]); return builder.EndVector(); } + public static VectorOffset CreateTypeIdsVectorBlock(FlatBufferBuilder builder, int[] data) { builder.StartVector(4, data.Length, 4); builder.Add(data); return builder.EndVector(); } public static void StartTypeIdsVector(FlatBufferBuilder builder, int numElems) { builder.StartVector(4, numElems, 4); } public static Offset EndUnion(FlatBufferBuilder builder) { int o = builder.EndObject(); diff --git a/csharp/src/Apache.Arrow/Flatbuf/Types/Utf8.cs b/csharp/src/Apache.Arrow/Flatbuf/Types/Utf8.cs index c3ebb70b798..e2b80f4070c 100644 --- a/csharp/src/Apache.Arrow/Flatbuf/Types/Utf8.cs +++ b/csharp/src/Apache.Arrow/Flatbuf/Types/Utf8.cs @@ -9,7 +9,7 @@ namespace Apache.Arrow.Flatbuf using global::FlatBuffers; /// Unicode with UTF-8 encoding -public struct Utf8 : IFlatbufferObject +internal struct Utf8 : IFlatbufferObject { private Table __p; public ByteBuffer ByteBuffer { get { return __p.bb; } } diff --git a/csharp/src/Apache.Arrow/Ipc/ArrowStreamReader.cs b/csharp/src/Apache.Arrow/Ipc/ArrowStreamReader.cs index f6e1ca5d069..c97b7cf9a8f 100644 --- a/csharp/src/Apache.Arrow/Ipc/ArrowStreamReader.cs +++ b/csharp/src/Apache.Arrow/Ipc/ArrowStreamReader.cs @@ -159,7 +159,7 @@ public void Dispose() #region Static Helper Functions - protected static IEnumerable BuildArrays(Schema schema, + private static IEnumerable BuildArrays(Schema schema, FlatBuffers.ByteBuffer messageBuffer, Flatbuf.RecordBatch recordBatchMessage) { @@ -180,7 +180,7 @@ protected static IEnumerable BuildArrays(Schema schema, return arrays.Select(ArrowArrayFactory.BuildArray); } - protected static T ReadMessage(FlatBuffers.ByteBuffer bb) where T : struct, FlatBuffers.IFlatbufferObject + private static T ReadMessage(FlatBuffers.ByteBuffer bb) where T : struct, FlatBuffers.IFlatbufferObject { var returnType = typeof(T); var msg = Flatbuf.Message.GetRootAsMessage(bb); diff --git a/csharp/src/Apache.Arrow/Ipc/ArrowStreamWriter.cs b/csharp/src/Apache.Arrow/Ipc/ArrowStreamWriter.cs index 639c64a0363..6105e926aec 100644 --- a/csharp/src/Apache.Arrow/Ipc/ArrowStreamWriter.cs +++ b/csharp/src/Apache.Arrow/Ipc/ArrowStreamWriter.cs @@ -143,13 +143,13 @@ public Block(int offset, int length, int metadataLength) protected ArrayPool Buffers { get; } - protected FlatBufferBuilder Builder { get; } + private protected FlatBufferBuilder Builder { get; } protected bool HasWrittenSchema { get; set; } protected Schema Schema { get; } - protected const Flatbuf.MetadataVersion CurrentMetadataVersion = Flatbuf.MetadataVersion.V4; + private protected const Flatbuf.MetadataVersion CurrentMetadataVersion = Flatbuf.MetadataVersion.V4; private static readonly byte[] Padding = new byte[64]; @@ -280,7 +280,7 @@ public Task WriteBufferAsync(ArrowBuffer arrowBuffer, CancellationToken cancella } } - protected Offset SerializeSchema(Schema schema) + private protected Offset SerializeSchema(Schema schema) { // TODO: Serialize schema metadata @@ -313,7 +313,7 @@ public Task WriteBufferAsync(ArrowBuffer arrowBuffer, CancellationToken cancella } - protected async Task> WriteSchemaAsync(Schema schema, CancellationToken cancellationToken) + private async Task> WriteSchemaAsync(Schema schema, CancellationToken cancellationToken) { Builder.Clear(); @@ -328,7 +328,7 @@ public Task WriteBufferAsync(ArrowBuffer arrowBuffer, CancellationToken cancella return schemaOffset; } - protected async Task WriteMessageAsync( + private async Task WriteMessageAsync( Flatbuf.MessageHeader headerType, Offset headerOffset, int bodyLength, CancellationToken cancellationToken) where T: struct @@ -353,7 +353,7 @@ await Buffers.RentReturnAsync(4, (buffer) => await WritePaddingAsync(messagePaddingLength); } - protected async Task WriteFlatBufferAsync(CancellationToken cancellationToken = default) + private protected async Task WriteFlatBufferAsync(CancellationToken cancellationToken = default) { var segment = Builder.DataBuffer.ToArraySegment(Builder.DataBuffer.Position, Builder.Offset);