Skip to content

C# support for directly reading and writing to memory other than byte[] #4886

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
aardappel merged 1 commit into from
Aug 23, 2018
Merged

C# support for directly reading and writing to memory other than byte[] #4886

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
486 changes: 364 additions & 122 deletions net/FlatBuffers/ByteBuffer.cs
View file
Open in desktop

Large diffs are not rendered by default.

70 changes: 69 additions & 1 deletion net/FlatBuffers/FlatBufferBuilder.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,17 @@ public FlatBufferBuilder(int initialSize)
_bb = new ByteBuffer(initialSize);
}

/// <summary>
/// Create a FlatBufferBuilder backed by the pased in ByteBuffer
/// </summary>
/// <param name="buffer">The ByteBuffer to write to</param>
public FlatBufferBuilder(ByteBuffer buffer)
{
_bb = buffer;
_space = buffer.Length;
buffer.Reset();
}

/// <summary>
/// Reset the FlatBufferBuilder by purging all data that it holds.
/// </summary>
Expand Down Expand Up @@ -191,6 +202,20 @@ public void Put<T>(T[] x)
_space = _bb.Put(_space, x);
}

#if ENABLE_SPAN_T
/// <summary>
/// Puts a span of type T into this builder at the
/// current offset
/// </summary>
/// <typeparam name="T">The type of the input data </typeparam>
/// <param name="x">The span to copy data from</param>
public void Put<T>(Span<T> x)
where T : struct
{
_space = _bb.Put(_space, x);
}
#endif

public void PutDouble(double x)
{
_bb.PutDouble(_space -= sizeof(double), x);
Expand Down Expand Up @@ -288,6 +313,28 @@ public void Add<T>(T[] x)
Put(x);
}

#if ENABLE_SPAN_T
/// <summary>
/// Add a span of type T to the buffer (aligns the data and grows if necessary).
/// </summary>
/// <typeparam name="T">The type of the input data</typeparam>
/// <param name="x">The span to copy data from</param>
public void Add<T>(Span<T> x)
where T : struct
{
if (!ByteBuffer.IsSupportedType<T>())
{
throw new ArgumentException("Cannot add this Type array to the builder");
}

int size = ByteBuffer.SizeOf<T>();
// Need to prep on size (for data alignment) and then we pass the
// rest of the length (minus 1) as additional bytes
Prep(size, size * (x.Length - 1));
Put(x);
}
#endif

/// <summary>
/// Add a `double` to the buffer (aligns the data and grows if necessary).
/// </summary>
Expand Down Expand Up @@ -511,6 +558,27 @@ public StringOffset CreateString(string s)
return new StringOffset(EndVector().Value);
}


#if ENABLE_SPAN_T
/// <summary>
/// Creates a string in the buffer from a Span containing
/// a UTF8 string.
/// </summary>
/// <param name="chars">the UTF8 string to add to the buffer</param>
/// <returns>
/// The offset in the buffer where the encoded string starts.
/// </returns>
public StringOffset CreateUTF8String(Span<byte> chars)
{
NotNested();
AddByte(0);
var utf8StringLen = chars.Length;
StartVector(1, utf8StringLen, 1);
_space = _bb.Put(_space, chars);
return new StringOffset(EndVector().Value);
}
#endif

/// @cond FLATBUFFERS_INTERNAL
// Structs are stored inline, so nothing additional is being added.
// `d` is always 0.
Expand Down Expand Up @@ -568,7 +636,7 @@ public int EndObject()
break;
}

endLoop: { }
endLoop: { }
}

if (existingVtable != 0) {
Expand Down
3 changes: 3 additions & 0 deletions net/FlatBuffers/FlatBuffers.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,7 @@
<PackageLicenseUrl>https://github.com/google/flatbuffers/blob/master/LICENSE.txt</PackageLicenseUrl>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="System.Memory" Version="4.5.1" />
</ItemGroup>
</Project>
18 changes: 18 additions & 0 deletions net/FlatBuffers/Table.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,23 @@ public int __vector(int offset)
return offset + bb.GetInt(offset) + sizeof(int); // data starts after the length
}

#if ENABLE_SPAN_T
// Get the data of a vector whoses offset is stored at "offset" in this object as an
// Spant&lt;byte&gt;. If the vector is not present in the ByteBuffer,
// then an empty span will be returned.
public Span<byte> __vector_as_span(int offset)
Copy link
Collaborator

@aardappel aardappel Aug 20, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is pretty awesome.

{
var o = this.__offset(offset);
if (0 == o)
{
return new Span<byte>();
}

var pos = this.__vector(o);
var len = this.__vector_len(o);
return bb.ToSpan(pos, len);
}
#else
// Get the data of a vector whoses offset is stored at "offset" in this object as an
// ArraySegment&lt;byte&gt;. If the vector is not present in the ByteBuffer,
// then a null value will be returned.
Copy link
Collaborator

@aardappel aardappel Aug 20, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should this function maybe always be available, for backwards compatibility?

Copy link
Collaborator

@aardappel aardappel Aug 20, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

actually never mind, the user has to explicitly enable ENABLE_SPAN_T so it is ok

Copy link

@et1975 et1975 Aug 20, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the user has to explicitly enable ENABLE_SPAN_T

Given that this is a preprocessor directive, how would this surface to the user?

Copy link
Contributor Author

@ccifra ccifra Aug 21, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You will have to build the library with ENABLE_SPAN_T defined in much the same way to need to build the library with UNSAFE_BYTEBUFFER defined to get the performance benefits.

Expand All @@ -93,6 +110,7 @@ public int __vector(int offset)
var len = this.__vector_len(o);
return bb.ToArraySegment(pos, len);
}
#endif

// Get the data of a vector whoses offset is stored at "offset" in this object as an
// T[]. If the vector is not present in the ByteBuffer, then a null value will be
Expand Down
9 changes: 9 additions & 0 deletions src/idl_gen_general.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1113,12 +1113,21 @@ class GeneralGenerator : public BaseGenerator {
code += "); }\n";
break;
case IDLOptions::kCSharp:
code += "#if ENABLE_SPAN_T\n";
code += " public Span<byte> Get";
code += MakeCamel(field.name, lang_.first_camel_upper);
code += "Bytes() { return ";
code += lang_.accessor_prefix + "__vector_as_span(";
code += NumToString(field.value.offset);
code += "); }\n";
code += "#else\n";
code += " public ArraySegment<byte>? Get";
code += MakeCamel(field.name, lang_.first_camel_upper);
code += "Bytes() { return ";
code += lang_.accessor_prefix + "__vector_as_arraysegment(";
code += NumToString(field.value.offset);
code += "); }\n";
code += "#endif\n";

// For direct blockcopying the data into a typed array
code += " public ";
Expand Down
21 changes: 20 additions & 1 deletion tests/FlatBuffers.Test/FlatBuffersExampleTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,14 +110,19 @@ private void CanCreateNewFlatBufferFromScratch(bool sizePrefix)
Monster.FinishMonsterBuffer(fbb, mon);
}


// Dump to output directory so we can inspect later, if needed
#if ENABLE_SPAN_T
var data = fbb.DataBuffer.ToSizedArray();
string filename = @"Resources/monsterdata_cstest" + (sizePrefix ? "_sp" : "") + ".mon";
File.WriteAllBytes(filename, data);
#else
using (var ms = fbb.DataBuffer.ToMemoryStream(fbb.DataBuffer.Position, fbb.Offset))
{
var data = ms.ToArray();
string filename = @"Resources/monsterdata_cstest" + (sizePrefix ? "_sp" : "") + ".mon";
File.WriteAllBytes(filename, data);
}
#endif

// Remove the size prefix if necessary for further testing
ByteBuffer dataBuffer = fbb.DataBuffer;
Expand Down Expand Up @@ -243,6 +248,19 @@ private void TestBuffer(ByteBuffer bb)

Assert.AreEqual(false, monster.Testbool);

#if ENABLE_SPAN_T
var nameBytes = monster.GetNameBytes();
Assert.AreEqual("MyMonster", Encoding.UTF8.GetString(nameBytes.ToArray(), 0, nameBytes.Length));

if (0 == monster.TestarrayofboolsLength)
{
Assert.IsFalse(monster.GetTestarrayofboolsBytes().Length != 0);
}
else
{
Assert.IsTrue(monster.GetTestarrayofboolsBytes().Length == 0);
}
#else
var nameBytes = monster.GetNameBytes().Value;
Assert.AreEqual("MyMonster", Encoding.UTF8.GetString(nameBytes.Array, nameBytes.Offset, nameBytes.Count));

Expand All @@ -254,6 +272,7 @@ private void TestBuffer(ByteBuffer bb)
{
Assert.IsTrue(monster.GetTestarrayofboolsBytes().HasValue);
}
#endif
}

[FlatBuffersTestMethod]
Expand Down
40 changes: 40 additions & 0 deletions tests/MyGame/Example/Monster.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,19 @@ public struct Monster : IFlatbufferObject
public short Hp { get { int o = __p.__offset(8); return o != 0 ? __p.bb.GetShort(o + __p.bb_pos) : (short)100; } }
public bool MutateHp(short hp) { int o = __p.__offset(8); if (o != 0) { __p.bb.PutShort(o + __p.bb_pos, hp); return true; } else { return false; } }
public string Name { get { int o = __p.__offset(10); return o != 0 ? __p.__string(o + __p.bb_pos) : null; } }
#if ENABLE_SPAN_T
public Span<byte> GetNameBytes() { return __p.__vector_as_span(10); }
#else
public ArraySegment<byte>? GetNameBytes() { return __p.__vector_as_arraysegment(10); }
#endif
public byte[] GetNameArray() { return __p.__vector_as_array<byte>(10); }
public byte Inventory(int j) { int o = __p.__offset(14); return o != 0 ? __p.bb.Get(__p.__vector(o) + j * 1) : (byte)0; }
public int InventoryLength { get { int o = __p.__offset(14); return o != 0 ? __p.__vector_len(o) : 0; } }
#if ENABLE_SPAN_T
public Span<byte> GetInventoryBytes() { return __p.__vector_as_span(14); }
#else
public ArraySegment<byte>? GetInventoryBytes() { return __p.__vector_as_arraysegment(14); }
#endif
public byte[] GetInventoryArray() { return __p.__vector_as_array<byte>(14); }
public bool MutateInventory(int j, byte inventory) { int o = __p.__offset(14); if (o != 0) { __p.bb.Put(__p.__vector(o) + j * 1, inventory); return true; } else { return false; } }
public Color Color { get { int o = __p.__offset(16); return o != 0 ? (Color)__p.bb.GetSbyte(o + __p.bb_pos) : Color.Blue; } }
Expand All @@ -49,7 +57,11 @@ public struct Monster : IFlatbufferObject
public Monster? Enemy { get { int o = __p.__offset(28); return o != 0 ? (Monster?)(new Monster()).__assign(__p.__indirect(o + __p.bb_pos), __p.bb) : null; } }
public byte Testnestedflatbuffer(int j) { int o = __p.__offset(30); return o != 0 ? __p.bb.Get(__p.__vector(o) + j * 1) : (byte)0; }
public int TestnestedflatbufferLength { get { int o = __p.__offset(30); return o != 0 ? __p.__vector_len(o) : 0; } }
#if ENABLE_SPAN_T
public Span<byte> GetTestnestedflatbufferBytes() { return __p.__vector_as_span(30); }
#else
public ArraySegment<byte>? GetTestnestedflatbufferBytes() { return __p.__vector_as_arraysegment(30); }
#endif
public byte[] GetTestnestedflatbufferArray() { return __p.__vector_as_array<byte>(30); }
public Monster? GetTestnestedflatbufferAsMonster() { int o = __p.__offset(30); return o != 0 ? (Monster?)(new Monster()).__assign(__p.__indirect(__p.__vector(o)), __p.bb) : null; }
public bool MutateTestnestedflatbuffer(int j, byte testnestedflatbuffer) { int o = __p.__offset(30); if (o != 0) { __p.bb.Put(__p.__vector(o) + j * 1, testnestedflatbuffer); return true; } else { return false; } }
Expand All @@ -74,7 +86,11 @@ public struct Monster : IFlatbufferObject
public bool MutateTesthashu64Fnv1a(ulong testhashu64_fnv1a) { int o = __p.__offset(50); if (o != 0) { __p.bb.PutUlong(o + __p.bb_pos, testhashu64_fnv1a); return true; } else { return false; } }
public bool Testarrayofbools(int j) { int o = __p.__offset(52); return o != 0 ? 0!=__p.bb.Get(__p.__vector(o) + j * 1) : false; }
public int TestarrayofboolsLength { get { int o = __p.__offset(52); return o != 0 ? __p.__vector_len(o) : 0; } }
#if ENABLE_SPAN_T
public Span<byte> GetTestarrayofboolsBytes() { return __p.__vector_as_span(52); }
#else
public ArraySegment<byte>? GetTestarrayofboolsBytes() { return __p.__vector_as_arraysegment(52); }
#endif
public bool[] GetTestarrayofboolsArray() { return __p.__vector_as_array<bool>(52); }
public bool MutateTestarrayofbools(int j, bool testarrayofbools) { int o = __p.__offset(52); if (o != 0) { __p.bb.Put(__p.__vector(o) + j * 1, (byte)(testarrayofbools ? 1 : 0)); return true; } else { return false; } }
public float Testf { get { int o = __p.__offset(54); return o != 0 ? __p.bb.GetFloat(o + __p.bb_pos) : (float)3.14159f; } }
Expand All @@ -89,19 +105,31 @@ public struct Monster : IFlatbufferObject
public int TestarrayofsortedstructLength { get { int o = __p.__offset(62); return o != 0 ? __p.__vector_len(o) : 0; } }
public byte Flex(int j) { int o = __p.__offset(64); return o != 0 ? __p.bb.Get(__p.__vector(o) + j * 1) : (byte)0; }
public int FlexLength { get { int o = __p.__offset(64); return o != 0 ? __p.__vector_len(o) : 0; } }
#if ENABLE_SPAN_T
public Span<byte> GetFlexBytes() { return __p.__vector_as_span(64); }
#else
public ArraySegment<byte>? GetFlexBytes() { return __p.__vector_as_arraysegment(64); }
#endif
public byte[] GetFlexArray() { return __p.__vector_as_array<byte>(64); }
public bool MutateFlex(int j, byte flex) { int o = __p.__offset(64); if (o != 0) { __p.bb.Put(__p.__vector(o) + j * 1, flex); return true; } else { return false; } }
public Test? Test5(int j) { int o = __p.__offset(66); return o != 0 ? (Test?)(new Test()).__assign(__p.__vector(o) + j * 4, __p.bb) : null; }
public int Test5Length { get { int o = __p.__offset(66); return o != 0 ? __p.__vector_len(o) : 0; } }
public long VectorOfLongs(int j) { int o = __p.__offset(68); return o != 0 ? __p.bb.GetLong(__p.__vector(o) + j * 8) : (long)0; }
public int VectorOfLongsLength { get { int o = __p.__offset(68); return o != 0 ? __p.__vector_len(o) : 0; } }
#if ENABLE_SPAN_T
public Span<byte> GetVectorOfLongsBytes() { return __p.__vector_as_span(68); }
#else
public ArraySegment<byte>? GetVectorOfLongsBytes() { return __p.__vector_as_arraysegment(68); }
#endif
public long[] GetVectorOfLongsArray() { return __p.__vector_as_array<long>(68); }
public bool MutateVectorOfLongs(int j, long vector_of_longs) { int o = __p.__offset(68); if (o != 0) { __p.bb.PutLong(__p.__vector(o) + j * 8, vector_of_longs); return true; } else { return false; } }
public double VectorOfDoubles(int j) { int o = __p.__offset(70); return o != 0 ? __p.bb.GetDouble(__p.__vector(o) + j * 8) : (double)0; }
public int VectorOfDoublesLength { get { int o = __p.__offset(70); return o != 0 ? __p.__vector_len(o) : 0; } }
#if ENABLE_SPAN_T
public Span<byte> GetVectorOfDoublesBytes() { return __p.__vector_as_span(70); }
#else
public ArraySegment<byte>? GetVectorOfDoublesBytes() { return __p.__vector_as_arraysegment(70); }
#endif
public double[] GetVectorOfDoublesArray() { return __p.__vector_as_array<double>(70); }
public bool MutateVectorOfDoubles(int j, double vector_of_doubles) { int o = __p.__offset(70); if (o != 0) { __p.bb.PutDouble(__p.__vector(o) + j * 8, vector_of_doubles); return true; } else { return false; } }
public MyGame.InParentNamespace? ParentNamespaceTest { get { int o = __p.__offset(72); return o != 0 ? (MyGame.InParentNamespace?)(new MyGame.InParentNamespace()).__assign(__p.__indirect(o + __p.bb_pos), __p.bb) : null; } }
Expand All @@ -112,7 +140,11 @@ public struct Monster : IFlatbufferObject
public bool MutateSingleWeakReference(ulong single_weak_reference) { int o = __p.__offset(76); if (o != 0) { __p.bb.PutUlong(o + __p.bb_pos, single_weak_reference); return true; } else { return false; } }
public ulong VectorOfWeakReferences(int j) { int o = __p.__offset(78); return o != 0 ? __p.bb.GetUlong(__p.__vector(o) + j * 8) : (ulong)0; }
public int VectorOfWeakReferencesLength { get { int o = __p.__offset(78); return o != 0 ? __p.__vector_len(o) : 0; } }
#if ENABLE_SPAN_T
public Span<byte> GetVectorOfWeakReferencesBytes() { return __p.__vector_as_span(78); }
#else
public ArraySegment<byte>? GetVectorOfWeakReferencesBytes() { return __p.__vector_as_arraysegment(78); }
#endif
public ulong[] GetVectorOfWeakReferencesArray() { return __p.__vector_as_array<ulong>(78); }
public bool MutateVectorOfWeakReferences(int j, ulong vector_of_weak_references) { int o = __p.__offset(78); if (o != 0) { __p.bb.PutUlong(__p.__vector(o) + j * 8, vector_of_weak_references); return true; } else { return false; } }
public Referrable? VectorOfStrongReferrables(int j) { int o = __p.__offset(80); return o != 0 ? (Referrable?)(new Referrable()).__assign(__p.__indirect(__p.__vector(o) + j * 4), __p.bb) : null; }
Expand All @@ -122,14 +154,22 @@ public struct Monster : IFlatbufferObject
public bool MutateCoOwningReference(ulong co_owning_reference) { int o = __p.__offset(82); if (o != 0) { __p.bb.PutUlong(o + __p.bb_pos, co_owning_reference); return true; } else { return false; } }
public ulong VectorOfCoOwningReferences(int j) { int o = __p.__offset(84); return o != 0 ? __p.bb.GetUlong(__p.__vector(o) + j * 8) : (ulong)0; }
public int VectorOfCoOwningReferencesLength { get { int o = __p.__offset(84); return o != 0 ? __p.__vector_len(o) : 0; } }
#if ENABLE_SPAN_T
public Span<byte> GetVectorOfCoOwningReferencesBytes() { return __p.__vector_as_span(84); }
#else
public ArraySegment<byte>? GetVectorOfCoOwningReferencesBytes() { return __p.__vector_as_arraysegment(84); }
#endif
public ulong[] GetVectorOfCoOwningReferencesArray() { return __p.__vector_as_array<ulong>(84); }
public bool MutateVectorOfCoOwningReferences(int j, ulong vector_of_co_owning_references) { int o = __p.__offset(84); if (o != 0) { __p.bb.PutUlong(__p.__vector(o) + j * 8, vector_of_co_owning_references); return true; } else { return false; } }
public ulong NonOwningReference { get { int o = __p.__offset(86); return o != 0 ? __p.bb.GetUlong(o + __p.bb_pos) : (ulong)0; } }
public bool MutateNonOwningReference(ulong non_owning_reference) { int o = __p.__offset(86); if (o != 0) { __p.bb.PutUlong(o + __p.bb_pos, non_owning_reference); return true; } else { return false; } }
public ulong VectorOfNonOwningReferences(int j) { int o = __p.__offset(88); return o != 0 ? __p.bb.GetUlong(__p.__vector(o) + j * 8) : (ulong)0; }
public int VectorOfNonOwningReferencesLength { get { int o = __p.__offset(88); return o != 0 ? __p.__vector_len(o) : 0; } }
#if ENABLE_SPAN_T
public Span<byte> GetVectorOfNonOwningReferencesBytes() { return __p.__vector_as_span(88); }
#else
public ArraySegment<byte>? GetVectorOfNonOwningReferencesBytes() { return __p.__vector_as_arraysegment(88); }
#endif
public ulong[] GetVectorOfNonOwningReferencesArray() { return __p.__vector_as_array<ulong>(88); }
public bool MutateVectorOfNonOwningReferences(int j, ulong vector_of_non_owning_references) { int o = __p.__offset(88); if (o != 0) { __p.bb.PutUlong(__p.__vector(o) + j * 8, vector_of_non_owning_references); return true; } else { return false; } }

Expand Down
4 changes: 4 additions & 0 deletions tests/MyGame/Example/Stat.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,11 @@ public struct Stat : IFlatbufferObject
public Stat __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; }

public string Id { get { int o = __p.__offset(4); return o != 0 ? __p.__string(o + __p.bb_pos) : null; } }
#if ENABLE_SPAN_T
public Span<byte> GetIdBytes() { return __p.__vector_as_span(4); }
#else
public ArraySegment<byte>? GetIdBytes() { return __p.__vector_as_arraysegment(4); }
#endif
public byte[] GetIdArray() { return __p.__vector_as_array<byte>(4); }
public long Val { get { int o = __p.__offset(6); return o != 0 ? __p.bb.GetLong(o + __p.bb_pos) : (long)0; } }
public bool MutateVal(long val) { int o = __p.__offset(6); if (o != 0) { __p.bb.PutLong(o + __p.bb_pos, val); return true; } else { return false; } }
Expand Down
Loading