diff --git a/pkg/Microsoft.Private.PackageBaseline/packageIndex.json b/pkg/Microsoft.Private.PackageBaseline/packageIndex.json index 2bfa5afbaa55..c76c5cd87c19 100644 --- a/pkg/Microsoft.Private.PackageBaseline/packageIndex.json +++ b/pkg/Microsoft.Private.PackageBaseline/packageIndex.json @@ -2543,9 +2543,10 @@ ], "InboxOn": { "netcoreapp2.1": "4.1.0.0", + "netcoreapp3.0": "4.2.0.0", "monoandroid10": "Any", "monotouch10": "Any", - "uap10.0.16300": "4.1.0.0", + "uap10.0.16300": "4.2.0.0", "xamarinios10": "Any", "xamarinmac20": "Any", "xamarintvos10": "Any", diff --git a/pkg/descriptions.json b/pkg/descriptions.json index a103b5e8e72e..f1ec48f83ce2 100644 --- a/pkg/descriptions.json +++ b/pkg/descriptions.json @@ -1050,6 +1050,7 @@ "System.ReadOnlyMemory", "System.Buffers.MemoryPool", "System.Buffers.ReadOnlySequence", + "System.Buffers.SequenceReader", "System.Buffers.Text.Utf8Parser", "System.Buffers.Text.Utf8Formatter" ] diff --git a/src/System.Memory/Directory.Build.props b/src/System.Memory/Directory.Build.props index d056cea58f58..1627cb108724 100644 --- a/src/System.Memory/Directory.Build.props +++ b/src/System.Memory/Directory.Build.props @@ -4,7 +4,7 @@ - 4.1.0.0 + 4.2.0.0 Open true true diff --git a/src/System.Memory/ref/System.Memory.cs b/src/System.Memory/ref/System.Memory.cs index 1b5df25b35f3..93ed84648ef8 100644 --- a/src/System.Memory/ref/System.Memory.cs +++ b/src/System.Memory/ref/System.Memory.cs @@ -97,7 +97,6 @@ public static void Reverse(this System.Span span) { } private readonly object _dummy; private readonly int _dummyPrimitive; public SequencePosition(object @object, int integer) { throw null; } - [System.ComponentModel.EditorBrowsableAttribute((System.ComponentModel.EditorBrowsableState)(1))] public override bool Equals(object obj) { throw null; } public bool Equals(System.SequencePosition other) { throw null; } [System.ComponentModel.EditorBrowsableAttribute((System.ComponentModel.EditorBrowsableState)(1))] @@ -184,6 +183,49 @@ public partial struct Enumerator public bool MoveNext() { throw null; } } } + public static partial class SequenceReaderExtensions + { + public static bool TryReadBigEndian(this ref System.Buffers.SequenceReader reader, out short value) { throw null; } + public static bool TryReadBigEndian(this ref System.Buffers.SequenceReader reader, out int value) { throw null; } + public static bool TryReadBigEndian(this ref System.Buffers.SequenceReader reader, out long value) { throw null; } + public static bool TryReadLittleEndian(this ref System.Buffers.SequenceReader reader, out short value) { throw null; } + public static bool TryReadLittleEndian(this ref System.Buffers.SequenceReader reader, out int value) { throw null; } + public static bool TryReadLittleEndian(this ref System.Buffers.SequenceReader reader, out long value) { throw null; } + } + public ref partial struct SequenceReader where T : unmanaged, System.IEquatable + { + public SequenceReader(System.Buffers.ReadOnlySequence buffer) { throw null; } + public long Consumed { get { throw null; } } + public System.ReadOnlySpan CurrentSpan { get { throw null; } } + public int CurrentSpanIndex { get { throw null; } } + public bool End { get { throw null; } } + public long Length { get { throw null; } } + public System.SequencePosition Position { get { throw null; } } + public long Remaining { get { throw null; } } + public System.Buffers.ReadOnlySequence Sequence { get { throw null; } } + public System.ReadOnlySpan UnreadSpan { get { throw null; } } + public void Advance(long count) { } + public long AdvancePast(T value) { throw null; } + public long AdvancePastAny(System.ReadOnlySpan values) { throw null; } + public long AdvancePastAny(T value0, T value1) { throw null; } + public long AdvancePastAny(T value0, T value1, T value2) { throw null; } + public long AdvancePastAny(T value0, T value1, T value2, T value3) { throw null; } + public bool IsNext(System.ReadOnlySpan next, bool advancePast = false) { throw null; } + public bool IsNext(T next, bool advancePast = false) { throw null; } + public void Rewind(long count) { } + public bool TryAdvanceTo(T delimiter, bool advancePastDelimiter = true) { throw null; } + public bool TryAdvanceToAny(System.ReadOnlySpan delimiters, bool advancePastDelimiter = true) { throw null; } + public bool TryCopyTo(System.Span destination) { throw null; } + public bool TryPeek(out T value) { throw null; } + public bool TryRead(out T value) { throw null; } + public bool TryReadTo(out System.Buffers.ReadOnlySequence sequence, System.ReadOnlySpan delimiter, bool advancePastDelimiter = true) { throw null; } + public bool TryReadTo(out System.Buffers.ReadOnlySequence sequence, T delimiter, bool advancePastDelimiter = true) { throw null; } + public bool TryReadTo(out System.Buffers.ReadOnlySequence sequence, T delimiter, T delimiterEscape, bool advancePastDelimiter = true) { throw null; } + public bool TryReadTo(out System.ReadOnlySpan span, T delimiter, bool advancePastDelimiter = true) { throw null; } + public bool TryReadTo(out System.ReadOnlySpan span, T delimiter, T delimiterEscape, bool advancePastDelimiter = true) { throw null; } + public bool TryReadToAny(out System.Buffers.ReadOnlySequence sequence, System.ReadOnlySpan delimiters, bool advancePastDelimiter = true) { throw null; } + public bool TryReadToAny(out System.ReadOnlySpan span, System.ReadOnlySpan delimiters, bool advancePastDelimiter = true) { throw null; } + } public readonly partial struct StandardFormat : System.IEquatable { private readonly int _dummyPrimitive; @@ -382,6 +424,7 @@ public static void Write(System.Span destination, ref T value) where T public static partial class SequenceMarshal { public static bool TryGetArray(System.Buffers.ReadOnlySequence sequence, out System.ArraySegment segment) { throw null; } + public static bool TryRead(ref System.Buffers.SequenceReader reader, out T value) where T : unmanaged { throw null; } public static bool TryGetReadOnlyMemory(System.Buffers.ReadOnlySequence sequence, out System.ReadOnlyMemory memory) { throw null; } public static bool TryGetReadOnlySequenceSegment(System.Buffers.ReadOnlySequence sequence, out System.Buffers.ReadOnlySequenceSegment startSegment, out int startIndex, out System.Buffers.ReadOnlySequenceSegment endSegment, out int endIndex) { throw null; } } diff --git a/src/System.Memory/src/System.Memory.csproj b/src/System.Memory/src/System.Memory.csproj index 1aab519e55c7..866a8a4d985c 100644 --- a/src/System.Memory/src/System.Memory.csproj +++ b/src/System.Memory/src/System.Memory.csproj @@ -1,4 +1,4 @@ - + {4BBC8F69-D03E-4432-AA8A-D458FA5B235A} true @@ -19,7 +19,10 @@ - + + + + diff --git a/src/System.Memory/src/System/Buffers/ReadOnlySequence_helpers.cs b/src/System.Memory/src/System/Buffers/ReadOnlySequence.Helpers.cs similarity index 84% rename from src/System.Memory/src/System/Buffers/ReadOnlySequence_helpers.cs rename to src/System.Memory/src/System/Buffers/ReadOnlySequence.Helpers.cs index 2264fc29ab70..2a95e09e8fa8 100644 --- a/src/System.Memory/src/System/Buffers/ReadOnlySequence_helpers.cs +++ b/src/System.Memory/src/System/Buffers/ReadOnlySequence.Helpers.cs @@ -4,6 +4,7 @@ using System.Diagnostics; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using Internal.Runtime.CompilerServices; namespace System.Buffers @@ -467,5 +468,81 @@ private static bool InRange(ulong value, ulong start, ulong end) // Equivalent to: return (start <= value && value <= start) return (value - start) <= (end - start); } + + /// + /// Helper to efficiently prepare the + /// + /// The first span in the sequence. + /// The next position. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void GetFirstSpan(out ReadOnlySpan first, out SequencePosition next) + { + first = default; + next = default; + SequencePosition start = Start; + int startIndex = start.GetInteger(); + object startObject = start.GetObject(); + + if (startObject != null) + { + SequencePosition end = End; + int endIndex = end.GetInteger(); + bool hasMultipleSegments = startObject != end.GetObject(); + + if (startIndex >= 0) + { + if (endIndex >= 0) + { + // Positive start and end index == ReadOnlySequenceSegment + ReadOnlySequenceSegment segment = (ReadOnlySequenceSegment)startObject; + next = new SequencePosition(segment.Next, 0); + first = segment.Memory.Span; + if (hasMultipleSegments) + { + first = first.Slice(startIndex); + } + else + { + first = first.Slice(startIndex, endIndex - startIndex); + } + } + else + { + // Positive start and negative end index == T[] + if (hasMultipleSegments) + ThrowHelper.ThrowInvalidOperationException_EndPositionNotReached(); + + first = new ReadOnlySpan((T[])startObject, startIndex, (endIndex & ReadOnlySequence.IndexBitMask) - startIndex); + } + } + else + { + first = GetFirstSpanSlow(startObject, startIndex, endIndex, hasMultipleSegments); + } + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ReadOnlySpan GetFirstSpanSlow(object startObject, int startIndex, int endIndex, bool hasMultipleSegments) + { + Debug.Assert(startIndex < 0); + if (hasMultipleSegments) + ThrowHelper.ThrowInvalidOperationException_EndPositionNotReached(); + + // The type == char check here is redundant. However, we still have it to allow + // the JIT to see when that the code is unreachable and eliminate it. + if (typeof(T) == typeof(char) && endIndex < 0) + { + // Negative start and negative end index == string + ReadOnlySpan spanOfChar = ((string)startObject).AsSpan(startIndex & ReadOnlySequence.IndexBitMask, endIndex - startIndex); + return MemoryMarshal.CreateSpan(ref Unsafe.As(ref MemoryMarshal.GetReference(spanOfChar)), spanOfChar.Length); + } + else + { + // Negative start and positive end index == MemoryManager + startIndex &= ReadOnlySequence.IndexBitMask; + return ((MemoryManager)startObject).Memory.Span.Slice(startIndex, endIndex - startIndex); + } + } } } diff --git a/src/System.Memory/src/System/Buffers/SequenceReader.Search.cs b/src/System.Memory/src/System/Buffers/SequenceReader.Search.cs new file mode 100644 index 000000000000..771b39490b7e --- /dev/null +++ b/src/System.Memory/src/System/Buffers/SequenceReader.Search.cs @@ -0,0 +1,770 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Diagnostics; +using System.Runtime.CompilerServices; + +namespace System.Buffers +{ + public ref partial struct SequenceReader where T : unmanaged, IEquatable + { + /// + /// Try to read everything up to the given . + /// + /// The read data, if any. + /// The delimiter to look for. + /// True to move past the if found. + /// True if the was found. + public bool TryReadTo(out ReadOnlySpan span, T delimiter, bool advancePastDelimiter = true) + { + ReadOnlySpan remaining = UnreadSpan; + int index = remaining.IndexOf(delimiter); + + if (index != -1) + { + span = index == 0 ? default : remaining.Slice(0, index); + AdvanceCurrentSpan(index + (advancePastDelimiter ? 1 : 0)); + return true; + } + + return TryReadToSlow(out span, delimiter, advancePastDelimiter); + } + + private bool TryReadToSlow(out ReadOnlySpan span, T delimiter, bool advancePastDelimiter) + { + if (!TryReadToInternal(out ReadOnlySequence sequence, delimiter, advancePastDelimiter, CurrentSpan.Length - CurrentSpanIndex)) + { + span = default; + return false; + } + + span = sequence.IsSingleSegment ? sequence.First.Span : sequence.ToArray(); + return true; + } + + /// + /// Try to read everything up to the given , ignoring delimiters that are + /// preceded by . + /// + /// The read data, if any. + /// The delimiter to look for. + /// If found prior to it will skip that occurrence. + /// True to move past the if found. + /// True if the was found. + public bool TryReadTo(out ReadOnlySpan span, T delimiter, T delimiterEscape, bool advancePastDelimiter = true) + { + ReadOnlySpan remaining = UnreadSpan; + int index = remaining.IndexOf(delimiter); + + if ((index > 0 && !remaining[index - 1].Equals(delimiterEscape)) || index == 0) + { + span = remaining.Slice(0, index); + AdvanceCurrentSpan(index + (advancePastDelimiter ? 1 : 0)); + return true; + } + + // This delimiter might be skipped, go down the slow path + return TryReadToSlow(out span, delimiter, delimiterEscape, index, advancePastDelimiter); + } + + private bool TryReadToSlow(out ReadOnlySpan span, T delimiter, T delimiterEscape, int index, bool advancePastDelimiter) + { + if (!TryReadToSlow(out ReadOnlySequence sequence, delimiter, delimiterEscape, index, advancePastDelimiter)) + { + span = default; + return false; + } + + Debug.Assert(sequence.Length > 0); + span = sequence.IsSingleSegment ? sequence.First.Span : sequence.ToArray(); + return true; + } + + private bool TryReadToSlow(out ReadOnlySequence sequence, T delimiter, T delimiterEscape, int index, bool advancePastDelimiter) + { + SequenceReader copy = this; + + ReadOnlySpan remaining = UnreadSpan; + bool priorEscape = false; + + do + { + if (index >= 0) + { + if (index == 0 && priorEscape) + { + // We were in the escaped state, so skip this delimiter + priorEscape = false; + Advance(index + 1); + remaining = UnreadSpan; + goto Continue; + } + else if (index > 0 && remaining[index - 1].Equals(delimiterEscape)) + { + // This delimiter might be skipped + + // Count our escapes + int escapeCount = 1; + int i = index - 2; + for (; i >= 0; i--) + { + if (!remaining[i].Equals(delimiterEscape)) + break; + } + if (i < 0 && priorEscape) + { + // Started and ended with escape, increment once more + escapeCount++; + } + escapeCount += index - 2 - i; + + if ((escapeCount & 1) != 0) + { + // An odd escape count means we're currently escaped, + // skip the delimiter and reset escaped state. + Advance(index + 1); + priorEscape = false; + remaining = UnreadSpan; + goto Continue; + } + } + + // Found the delimiter. Move to it, slice, then move past it. + AdvanceCurrentSpan(index); + + sequence = Sequence.Slice(copy.Position, Position); + if (advancePastDelimiter) + { + Advance(1); + } + return true; + } + else + { + // No delimiter, need to check the end of the span for odd number of escapes then advance + if (remaining.Length > 0 && remaining[remaining.Length - 1].Equals(delimiterEscape)) + { + int escapeCount = 1; + int i = remaining.Length - 2; + for (; i >= 0; i--) + { + if (!remaining[i].Equals(delimiterEscape)) + break; + } + + escapeCount += remaining.Length - 2 - i; + if (i < 0 && priorEscape) + priorEscape = (escapeCount & 1) == 0; // equivalent to incrementing escapeCount before setting priorEscape + else + priorEscape = (escapeCount & 1) != 0; + } + else + { + priorEscape = false; + } + } + + // Nothing in the current span, move to the end, checking for the skip delimiter + AdvanceCurrentSpan(remaining.Length); + remaining = CurrentSpan; + + Continue: + index = remaining.IndexOf(delimiter); + } while (!End); + + // Didn't find anything, reset our original state. + this = copy; + sequence = default; + return false; + } + + /// + /// Try to read everything up to the given . + /// + /// The read data, if any. + /// The delimiter to look for. + /// True to move past the if found. + /// True if the was found. + public bool TryReadTo(out ReadOnlySequence sequence, T delimiter, bool advancePastDelimiter = true) + { + return TryReadToInternal(out sequence, delimiter, advancePastDelimiter); + } + + private bool TryReadToInternal(out ReadOnlySequence sequence, T delimiter, bool advancePastDelimiter, int skip = 0) + { + Debug.Assert(skip >= 0); + SequenceReader copy = this; + if (skip > 0) + Advance(skip); + ReadOnlySpan remaining = UnreadSpan; + + while (_moreData) + { + int index = remaining.IndexOf(delimiter); + if (index != -1) + { + // Found the delimiter. Move to it, slice, then move past it. + if (index > 0) + { + AdvanceCurrentSpan(index); + } + + sequence = Sequence.Slice(copy.Position, Position); + if (advancePastDelimiter) + { + Advance(1); + } + return true; + } + + AdvanceCurrentSpan(remaining.Length); + remaining = CurrentSpan; + } + + // Didn't find anything, reset our original state. + this = copy; + sequence = default; + return false; + } + + /// + /// Try to read everything up to the given , ignoring delimiters that are + /// preceded by . + /// + /// The read data, if any. + /// The delimiter to look for. + /// If found prior to it will skip that occurrence. + /// True to move past the if found. + /// True if the was found. + public bool TryReadTo(out ReadOnlySequence sequence, T delimiter, T delimiterEscape, bool advancePastDelimiter = true) + { + SequenceReader copy = this; + + ReadOnlySpan remaining = UnreadSpan; + bool priorEscape = false; + + while (_moreData) + { + int index = remaining.IndexOf(delimiter); + if (index != -1) + { + if (index == 0 && priorEscape) + { + // We were in the escaped state, so skip this delimiter + priorEscape = false; + Advance(index + 1); + remaining = UnreadSpan; + continue; + } + else if (index > 0 && remaining[index - 1].Equals(delimiterEscape)) + { + // This delimiter might be skipped + + // Count our escapes + int escapeCount = 0; + for (int i = index; i > 0 && remaining[i - 1].Equals(delimiterEscape); i--, escapeCount++) + ; + if (escapeCount == index && priorEscape) + { + // Started and ended with escape, increment once more + escapeCount++; + } + + priorEscape = false; + if ((escapeCount & 1) != 0) + { + // Odd escape count means we're in the escaped state, so skip this delimiter + Advance(index + 1); + remaining = UnreadSpan; + continue; + } + } + + // Found the delimiter. Move to it, slice, then move past it. + if (index > 0) + { + Advance(index); + } + + sequence = Sequence.Slice(copy.Position, Position); + if (advancePastDelimiter) + { + Advance(1); + } + return true; + } + + // No delimiter, need to check the end of the span for odd number of escapes then advance + { + int escapeCount = 0; + for (int i = remaining.Length; i > 0 && remaining[i - 1].Equals(delimiterEscape); i--, escapeCount++) + ; + if (priorEscape && escapeCount == remaining.Length) + { + escapeCount++; + } + priorEscape = escapeCount % 2 != 0; + } + + // Nothing in the current span, move to the end, checking for the skip delimiter + Advance(remaining.Length); + remaining = CurrentSpan; + } + + // Didn't find anything, reset our original state. + this = copy; + sequence = default; + return false; + } + + /// + /// Try to read everything up to the given . + /// + /// The read data, if any. + /// The delimiters to look for. + /// True to move past the first found instance of any of the given . + /// True if any of the the were found. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool TryReadToAny(out ReadOnlySpan span, ReadOnlySpan delimiters, bool advancePastDelimiter = true) + { + ReadOnlySpan remaining = UnreadSpan; + int index = delimiters.Length == 2 + ? remaining.IndexOfAny(delimiters[0], delimiters[1]) + : remaining.IndexOfAny(delimiters); + + if (index != -1) + { + span = remaining.Slice(0, index); + Advance(index + (advancePastDelimiter ? 1 : 0)); + return true; + } + + return TryReadToAnySlow(out span, delimiters, advancePastDelimiter); + } + + private bool TryReadToAnySlow(out ReadOnlySpan span, ReadOnlySpan delimiters, bool advancePastDelimiter) + { + if (!TryReadToAnyInternal(out ReadOnlySequence sequence, delimiters, advancePastDelimiter, CurrentSpan.Length - CurrentSpanIndex)) + { + span = default; + return false; + } + + span = sequence.IsSingleSegment ? sequence.First.Span : sequence.ToArray(); + return true; + } + + /// + /// Try to read everything up to the given . + /// + /// The read data, if any. + /// The delimiters to look for. + /// True to move past the first found instance of any of the given . + /// True if any of the the were found. + public bool TryReadToAny(out ReadOnlySequence sequence, ReadOnlySpan delimiters, bool advancePastDelimiter = true) + { + return TryReadToAnyInternal(out sequence, delimiters, advancePastDelimiter); + } + + private bool TryReadToAnyInternal(out ReadOnlySequence sequence, ReadOnlySpan delimiters, bool advancePastDelimiter, int skip = 0) + { + SequenceReader copy = this; + if (skip > 0) + Advance(skip); + ReadOnlySpan remaining = UnreadSpan; + + while (!End) + { + int index = delimiters.Length == 2 + ? remaining.IndexOfAny(delimiters[0], delimiters[1]) + : remaining.IndexOfAny(delimiters); + + if (index != -1) + { + // Found one of the delimiters. Move to it, slice, then move past it. + if (index > 0) + { + AdvanceCurrentSpan(index); + } + + sequence = Sequence.Slice(copy.Position, Position); + if (advancePastDelimiter) + { + Advance(1); + } + return true; + } + + Advance(remaining.Length); + remaining = CurrentSpan; + } + + // Didn't find anything, reset our original state. + this = copy; + sequence = default; + return false; + } + + /// + /// Try to read data until the entire given matches. + /// + /// The read data, if any. + /// The multi (T) delimiter. + /// True to move past the if found. + /// True if the was found. + public bool TryReadTo(out ReadOnlySequence sequence, ReadOnlySpan delimiter, bool advancePastDelimiter = true) + { + if (delimiter.Length == 0) + { + sequence = default; + return true; + } + + SequenceReader copy = this; + + bool advanced = false; + while (!End) + { + if (!TryReadTo(out sequence, delimiter[0], advancePastDelimiter: false)) + { + this = copy; + return false; + } + + if (delimiter.Length == 1) + { + return true; + } + + if (IsNext(delimiter)) + { + // Probably a faster way to do this, potentially by avoiding the Advance in the previous TryReadTo call + if (advanced) + { + sequence = copy.Sequence.Slice(copy.Consumed, Consumed - copy.Consumed); + } + + if (advancePastDelimiter) + { + Advance(delimiter.Length); + } + return true; + } + else + { + Advance(1); + advanced = true; + } + } + + this = copy; + sequence = default; + return false; + } + + /// + /// Advance until the given , if found. + /// + /// The delimiter to search for. + /// True to move past the if found. + /// True if the given was found. + public bool TryAdvanceTo(T delimiter, bool advancePastDelimiter = true) + { + ReadOnlySpan remaining = UnreadSpan; + int index = remaining.IndexOf(delimiter); + if (index != -1) + { + Advance(advancePastDelimiter ? index + 1 : index); + return true; + } + + return TryReadToInternal(out _, delimiter, advancePastDelimiter); + } + + /// + /// Advance until any of the given , if found. + /// + /// The delimiters to search for. + /// True to move past the first found instance of any of the given . + /// True if any of the given were found. + public bool TryAdvanceToAny(ReadOnlySpan delimiters, bool advancePastDelimiter = true) + { + ReadOnlySpan remaining = UnreadSpan; + int index = remaining.IndexOfAny(delimiters); + if (index != -1) + { + AdvanceCurrentSpan(index + (advancePastDelimiter ? 1 : 0)); + return true; + } + + return TryReadToAnyInternal(out _, delimiters, advancePastDelimiter); + } + + /// + /// Advance past consecutive instances of the given . + /// + /// How many positions the reader has been advanced. + public long AdvancePast(T value) + { + long start = Consumed; + + do + { + // Advance past all matches in the current span + int i; + for (i = CurrentSpanIndex; i < CurrentSpan.Length && CurrentSpan[i].Equals(value); i++) + { + } + + int advanced = i - CurrentSpanIndex; + if (advanced == 0) + { + // Didn't advance at all in this span, exit. + break; + } + + AdvanceCurrentSpan(advanced); + + // If we're at postion 0 after advancing and not at the End, + // we're in a new span and should continue the loop. + } while (CurrentSpanIndex == 0 && !End); + + return Consumed - start; + } + + /// + /// Skip consecutive instances of any of the given . + /// + /// How many positions the reader has been advanced. + public long AdvancePastAny(ReadOnlySpan values) + { + long start = Consumed; + + do + { + // Advance past all matches in the current span + int i; + for (i = CurrentSpanIndex; i < CurrentSpan.Length && values.IndexOf(CurrentSpan[i]) != -1; i++) + { + } + + int advanced = i - CurrentSpanIndex; + if (advanced == 0) + { + // Didn't advance at all in this span, exit. + break; + } + + AdvanceCurrentSpan(advanced); + + // If we're at postion 0 after advancing and not at the End, + // we're in a new span and should continue the loop. + } while (CurrentSpanIndex == 0 && !End); + + return Consumed - start; + } + + /// + /// Advance past consecutive instances of any of the given values. + /// + /// How many positions the reader has been advanced. + public long AdvancePastAny(T value0, T value1, T value2, T value3) + { + long start = Consumed; + + do + { + // Advance past all matches in the current span + int i; + for (i = CurrentSpanIndex; i < CurrentSpan.Length; i++) + { + T value = CurrentSpan[i]; + if (!value.Equals(value0) && !value.Equals(value1) && !value.Equals(value2) && !value.Equals(value3)) + { + break; + } + } + + int advanced = i - CurrentSpanIndex; + if (advanced == 0) + { + // Didn't advance at all in this span, exit. + break; + } + + AdvanceCurrentSpan(advanced); + + // If we're at postion 0 after advancing and not at the End, + // we're in a new span and should continue the loop. + } while (CurrentSpanIndex == 0 && !End); + + return Consumed - start; + } + + /// + /// Advance past consecutive instances of any of the given values. + /// + /// How many positions the reader has been advanced. + public long AdvancePastAny(T value0, T value1, T value2) + { + long start = Consumed; + + do + { + // Advance past all matches in the current span + int i; + for (i = CurrentSpanIndex; i < CurrentSpan.Length; i++) + { + T value = CurrentSpan[i]; + if (!value.Equals(value0) && !value.Equals(value1) && !value.Equals(value2)) + { + break; + } + } + + int advanced = i - CurrentSpanIndex; + if (advanced == 0) + { + // Didn't advance at all in this span, exit. + break; + } + + AdvanceCurrentSpan(advanced); + + // If we're at postion 0 after advancing and not at the End, + // we're in a new span and should continue the loop. + } while (CurrentSpanIndex == 0 && !End); + + return Consumed - start; + } + + /// + /// Advance past consecutive instances of any of the given values. + /// + /// How many positions the reader has been advanced. + public long AdvancePastAny(T value0, T value1) + { + long start = Consumed; + + do + { + // Advance past all matches in the current span + int i; + for (i = CurrentSpanIndex; i < CurrentSpan.Length; i++) + { + T value = CurrentSpan[i]; + if (!value.Equals(value0) && !value.Equals(value1)) + { + break; + } + } + + int advanced = i - CurrentSpanIndex; + if (advanced == 0) + { + // Didn't advance at all in this span, exit. + break; + } + + AdvanceCurrentSpan(advanced); + + // If we're at postion 0 after advancing and not at the End, + // we're in a new span and should continue the loop. + } while (CurrentSpanIndex == 0 && !End); + + return Consumed - start; + } + + /// + /// Check to see if the given value is next. + /// + /// The value to compare the next items to. + /// Move past the value if found. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool IsNext(T next, bool advancePast = false) + { + if (End) + return false; + + if (CurrentSpan[CurrentSpanIndex].Equals(next)) + { + if (advancePast) + { + AdvanceCurrentSpan(1); + } + return true; + } + return false; + } + + /// + /// Check to see if the given values are next. + /// + /// The span to compare the next items to. + /// Move past the values if found. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool IsNext(ReadOnlySpan next, bool advancePast = false) + { + ReadOnlySpan unread = UnreadSpan; + if (unread.StartsWith(next)) + { + if (advancePast) + { + AdvanceCurrentSpan(next.Length); + } + return true; + } + + // Only check the slow path if there wasn't enough to satisfy next + return unread.Length < next.Length && IsNextSlow(next, advancePast); + } + + private unsafe bool IsNextSlow(ReadOnlySpan next, bool advancePast) + { + ReadOnlySpan currentSpan = UnreadSpan; + + // We should only come in here if we need more data than we have in our current span + Debug.Assert(currentSpan.Length < next.Length); + + int fullLength = next.Length; + SequencePosition nextPosition = _nextPosition; + + while (next.StartsWith(currentSpan)) + { + if (next.Length == currentSpan.Length) + { + // Fully matched + if (advancePast) + { + Advance(fullLength); + } + return true; + } + + // Need to check the next segment + while (true) + { + if (!Sequence.TryGet(ref nextPosition, out ReadOnlyMemory nextSegment, advance: true)) + { + // Nothing left + return false; + } + + if (nextSegment.Length > 0) + { + next = next.Slice(currentSpan.Length); + currentSpan = nextSegment.Span; + if (currentSpan.Length > next.Length) + { + currentSpan = currentSpan.Slice(0, next.Length); + } + break; + } + } + } + + return false; + } + } +} diff --git a/src/System.Memory/src/System/Buffers/SequenceReader.cs b/src/System.Memory/src/System/Buffers/SequenceReader.cs new file mode 100644 index 000000000000..52a24e3ae0b2 --- /dev/null +++ b/src/System.Memory/src/System/Buffers/SequenceReader.cs @@ -0,0 +1,373 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Diagnostics; +using System.Runtime.CompilerServices; + +namespace System.Buffers +{ + public ref partial struct SequenceReader where T : unmanaged, IEquatable + { + private SequencePosition _currentPosition; + private SequencePosition _nextPosition; + private bool _moreData; + private long _length; + + /// + /// Create a over the given . + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public SequenceReader(ReadOnlySequence sequence) + { + CurrentSpanIndex = 0; + Consumed = 0; + Sequence = sequence; + _currentPosition = sequence.Start; + _length = -1; + + sequence.GetFirstSpan(out ReadOnlySpan first, out _nextPosition); + CurrentSpan = first; + _moreData = first.Length > 0; + + if (!_moreData && !sequence.IsSingleSegment) + { + _moreData = true; + GetNextSpan(); + } + } + + /// + /// True when there is no more data in the . + /// + public bool End => !_moreData; + + /// + /// The underlying for the reader. + /// + public ReadOnlySequence Sequence { get; } + + /// + /// The current position in the . + /// + public SequencePosition Position + => Sequence.GetPosition(CurrentSpanIndex, _currentPosition); + + /// + /// The current segment in the as a span. + /// + public ReadOnlySpan CurrentSpan { get; private set; } + + /// + /// The index in the . + /// + public int CurrentSpanIndex { get; private set; } + + /// + /// The unread portion of the . + /// + public ReadOnlySpan UnreadSpan + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => CurrentSpan.Slice(CurrentSpanIndex); + } + + /// + /// The total number of 's processed by the reader. + /// + public long Consumed { get; private set; } + + /// + /// Remaining 's in the reader's . + /// + public long Remaining => Length - Consumed; + + /// + /// Count of in the reader's . + /// + public long Length + { + get + { + if (_length < 0) + { + // Cache the length + _length = Sequence.Length; + } + return _length; + } + } + + /// + /// Peeks at the next value without advancing the reader. + /// + /// The next value or default if at the end. + /// False if at the end of the reader. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool TryPeek(out T value) + { + if (_moreData) + { + value = CurrentSpan[CurrentSpanIndex]; + return true; + } + else + { + value = default; + return false; + } + } + + /// + /// Read the next value and advance the reader. + /// + /// The next value or default if at the end. + /// False if at the end of the reader. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool TryRead(out T value) + { + if (End) + { + value = default; + return false; + } + + value = CurrentSpan[CurrentSpanIndex]; + CurrentSpanIndex++; + Consumed++; + + if (CurrentSpanIndex >= CurrentSpan.Length) + { + GetNextSpan(); + } + + return true; + } + + /// + /// Move the reader back the specified number of items. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Rewind(long count) + { + if (count < 0) + { + ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.count); + } + + Consumed -= count; + + if (CurrentSpanIndex >= count) + { + CurrentSpanIndex -= (int)count; + _moreData = true; + } + else + { + // Current segment doesn't have enough data, scan backward through segments + RetreatToPreviousSpan(Consumed); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private void RetreatToPreviousSpan(long consumed) + { + ResetReader(); + Advance(consumed); + } + + private void ResetReader() + { + CurrentSpanIndex = 0; + Consumed = 0; + _currentPosition = Sequence.Start; + _nextPosition = _currentPosition; + + if (Sequence.TryGet(ref _nextPosition, out ReadOnlyMemory memory, advance: true)) + { + _moreData = true; + + if (memory.Length == 0) + { + CurrentSpan = default; + // No data in the first span, move to one with data + GetNextSpan(); + } + else + { + CurrentSpan = memory.Span; + } + } + else + { + // No data in any spans and at end of sequence + _moreData = false; + CurrentSpan = default; + } + } + + /// + /// Get the next segment with available data, if any. + /// + private void GetNextSpan() + { + if (!Sequence.IsSingleSegment) + { + SequencePosition previousNextPosition = _nextPosition; + while (Sequence.TryGet(ref _nextPosition, out ReadOnlyMemory memory, advance: true)) + { + _currentPosition = previousNextPosition; + if (memory.Length > 0) + { + CurrentSpan = memory.Span; + CurrentSpanIndex = 0; + return; + } + else + { + CurrentSpan = default; + CurrentSpanIndex = 0; + previousNextPosition = _nextPosition; + } + } + } + _moreData = false; + } + + /// + /// Move the reader ahead the specified number of items. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Advance(long count) + { + const long TooBigOrNegative = unchecked((long)0xFFFFFFFF80000000); + if ((count & TooBigOrNegative) == 0 && CurrentSpan.Length - CurrentSpanIndex > (int)count) + { + CurrentSpanIndex += (int)count; + Consumed += count; + } + else + { + // Can't satisfy from the current span + AdvanceToNextSpan(count); + } + } + + /// + /// Unchecked helper to avoid unnecessary checks where you know count is valid. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void AdvanceCurrentSpan(long count) + { + Debug.Assert(count >= 0); + + Consumed += count; + CurrentSpanIndex += (int)count; + if (CurrentSpanIndex >= CurrentSpan.Length) + GetNextSpan(); + } + + /// + /// Only call this helper if you know that you are advancing in the current span + /// with valid count and there is no need to fetch the next one. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void AdvanceWithinSpan(long count) + { + Debug.Assert(count >= 0); + + Consumed += count; + CurrentSpanIndex += (int)count; + + Debug.Assert(CurrentSpanIndex < CurrentSpan.Length); + } + + private void AdvanceToNextSpan(long count) + { + if (count < 0) + { + ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.count); + } + + Consumed += count; + while (_moreData) + { + int remaining = CurrentSpan.Length - CurrentSpanIndex; + + if (remaining > count) + { + CurrentSpanIndex += (int)count; + count = 0; + break; + } + + // As there may not be any further segments we need to + // push the current index to the end of the span. + CurrentSpanIndex += remaining; + count -= remaining; + Debug.Assert(count >= 0); + + GetNextSpan(); + + if (count == 0) + { + break; + } + } + + if (count != 0) + { + // Not enough data left- adjust for where we actually ended and throw + Consumed -= count; + ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.count); + } + } + + /// + /// Copies data from the current to the given span. + /// + /// Destination to copy to. + /// True if there is enough data to copy to the . + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool TryCopyTo(Span destination) + { + ReadOnlySpan firstSpan = UnreadSpan; + if (firstSpan.Length >= destination.Length) + { + firstSpan.Slice(0, destination.Length).CopyTo(destination); + return true; + } + + return TryCopyMultisegment(destination); + } + + internal bool TryCopyMultisegment(Span destination) + { + if (Remaining < destination.Length) + return false; + + ReadOnlySpan firstSpan = UnreadSpan; + Debug.Assert(firstSpan.Length < destination.Length); + firstSpan.CopyTo(destination); + int copied = firstSpan.Length; + + SequencePosition next = _nextPosition; + while (Sequence.TryGet(ref next, out ReadOnlyMemory nextSegment, true)) + { + if (nextSegment.Length > 0) + { + ReadOnlySpan nextSpan = nextSegment.Span; + int toCopy = Math.Min(nextSpan.Length, destination.Length - copied); + nextSpan.Slice(0, toCopy).CopyTo(destination.Slice(copied)); + copied += toCopy; + if (copied >= destination.Length) + { + break; + } + } + } + + return true; + } + } +} diff --git a/src/System.Memory/src/System/Buffers/SequenceReaderExtensions.Binary.cs b/src/System.Memory/src/System/Buffers/SequenceReaderExtensions.Binary.cs new file mode 100644 index 000000000000..7dc5b84f4d29 --- /dev/null +++ b/src/System.Memory/src/System/Buffers/SequenceReaderExtensions.Binary.cs @@ -0,0 +1,175 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Buffers.Binary; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using Internal.Runtime.CompilerServices; + +namespace System.Buffers +{ + public static partial class SequenceReaderExtensions + { + /// + /// Try to read the given type out of the buffer if possible. Warning: this is dangerous to use with arbitrary + /// structs- see remarks for full details. + /// + /// + /// IMPORTANT: The read is a straight copy of bits. If a struct depends on specific state of it's members to + /// behave correctly this can lead to exceptions, etc. If reading endian specific integers, use the explicit + /// overloads such as + /// + /// + /// True if successful. will be default if failed (due to lack of space). + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static unsafe bool TryRead(ref this SequenceReader reader, out T value) where T : unmanaged + { + ReadOnlySpan span = reader.UnreadSpan; + if (span.Length < sizeof(T)) + return TryReadMultisegment(ref reader, out value); + + value = Unsafe.ReadUnaligned(ref MemoryMarshal.GetReference(span)); + reader.Advance(sizeof(T)); + return true; + } + + private static unsafe bool TryReadMultisegment(ref SequenceReader reader, out T value) where T : unmanaged + { + Debug.Assert(reader.UnreadSpan.Length < sizeof(T)); + + // Not enough data in the current segment, try to peek for the data we need. + T buffer = default; + Span tempSpan = new Span(&buffer, sizeof(T)); + + if (!reader.TryCopyTo(tempSpan)) + { + value = default; + return false; + } + + value = Unsafe.ReadUnaligned(ref MemoryMarshal.GetReference(tempSpan)); + reader.Advance(sizeof(T)); + return true; + } + + /// + /// Reads an as little endian. + /// + /// False if there wasn't enough data for an . + public static bool TryReadLittleEndian(ref this SequenceReader reader, out short value) + { + if (BitConverter.IsLittleEndian) + { + return reader.TryRead(out value); + } + + return TryReadReverseEndianness(ref reader, out value); + } + + /// + /// Reads an as big endian. + /// + /// False if there wasn't enough data for an . + public static bool TryReadBigEndian(ref this SequenceReader reader, out short value) + { + if (!BitConverter.IsLittleEndian) + { + return reader.TryRead(out value); + } + + return TryReadReverseEndianness(ref reader, out value); + } + + private static bool TryReadReverseEndianness(ref SequenceReader reader, out short value) + { + if (reader.TryRead(out value)) + { + value = BinaryPrimitives.ReverseEndianness(value); + return true; + } + + return false; + } + + /// + /// Reads an as little endian. + /// + /// False if there wasn't enough data for an . + public static bool TryReadLittleEndian(ref this SequenceReader reader, out int value) + { + if (BitConverter.IsLittleEndian) + { + return reader.TryRead(out value); + } + + return TryReadReverseEndianness(ref reader, out value); + } + + /// + /// Reads an as big endian. + /// + /// False if there wasn't enough data for an . + public static bool TryReadBigEndian(ref this SequenceReader reader, out int value) + { + if (!BitConverter.IsLittleEndian) + { + return reader.TryRead(out value); + } + + return TryReadReverseEndianness(ref reader, out value); + } + + private static bool TryReadReverseEndianness(ref SequenceReader reader, out int value) + { + if (reader.TryRead(out value)) + { + value = BinaryPrimitives.ReverseEndianness(value); + return true; + } + + return false; + } + + /// + /// Reads an as little endian. + /// + /// False if there wasn't enough data for an . + public static bool TryReadLittleEndian(ref this SequenceReader reader, out long value) + { + if (BitConverter.IsLittleEndian) + { + return reader.TryRead(out value); + } + + return TryReadReverseEndianness(ref reader, out value); + } + + /// + /// Reads an as big endian. + /// + /// False if there wasn't enough data for an . + public static bool TryReadBigEndian(ref this SequenceReader reader, out long value) + { + if (!BitConverter.IsLittleEndian) + { + return reader.TryRead(out value); + } + + return TryReadReverseEndianness(ref reader, out value); + } + + private static bool TryReadReverseEndianness(ref SequenceReader reader, out long value) + { + if (reader.TryRead(out value)) + { + value = BinaryPrimitives.ReverseEndianness(value); + return true; + } + + return false; + } + } +} diff --git a/src/System.Memory/src/System/Runtime/InteropServices/SequenceMarshal.cs b/src/System.Memory/src/System/Runtime/InteropServices/SequenceMarshal.cs index 8335a4144bef..ded0b44d9d98 100644 --- a/src/System.Memory/src/System/Runtime/InteropServices/SequenceMarshal.cs +++ b/src/System.Memory/src/System/Runtime/InteropServices/SequenceMarshal.cs @@ -57,5 +57,22 @@ internal static bool TryGetString(ReadOnlySequence sequence, out string te { return sequence.TryGetString(out text, out start, out length); } + + /// + /// Try to read the given type out of the buffer if possible. Warning: this is dangerous to use with arbitrary + /// structs- see remarks for full details. + /// + /// + /// IMPORTANT: The read is a straight copy of bits. If a struct depends on specific state of it's members to + /// behave correctly this can lead to exceptions, etc. If reading endian specific integers, use the explicit + /// overloads such as + /// + /// + /// True if successful. will be default if failed (due to lack of space). + /// + public static bool TryRead(ref SequenceReader reader, out T value) where T : unmanaged + { + return reader.TryRead(out value); + } } } diff --git a/src/System.Memory/src/System/ThrowHelper.cs b/src/System.Memory/src/System/ThrowHelper.cs index 70b1a1ca8597..13be346f202d 100644 --- a/src/System.Memory/src/System/ThrowHelper.cs +++ b/src/System.Memory/src/System/ThrowHelper.cs @@ -152,6 +152,7 @@ internal enum ExceptionArgument endIndex, array, culture, - manager + manager, + count } } diff --git a/src/System.Memory/tests/SequenceReader/Advance.cs b/src/System.Memory/tests/SequenceReader/Advance.cs new file mode 100644 index 000000000000..bec4afb32948 --- /dev/null +++ b/src/System.Memory/tests/SequenceReader/Advance.cs @@ -0,0 +1,61 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Buffers; +using Xunit; + +namespace System.Memory.Tests.SequenceReader +{ + public class Advance + { + [Theory, + InlineData(true), + InlineData(false)] + public void Basic(bool singleSegment) + { + byte[] buffer = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + ReadOnlySequence bytes = singleSegment + ? new ReadOnlySequence(buffer) + : SequenceFactory.CreateSplit(buffer, 2, 4); + + SequenceReader skipReader = new SequenceReader(bytes); + Assert.False(skipReader.TryAdvanceTo(10)); + Assert.True(skipReader.TryAdvanceTo(4, advancePastDelimiter: false)); + Assert.True(skipReader.TryRead(out byte value)); + Assert.Equal(4, value); + + Assert.True(skipReader.TryAdvanceToAny(new byte[] { 3, 12, 7 }, advancePastDelimiter: false)); + Assert.True(skipReader.TryRead(out value)); + Assert.Equal(7, value); + Assert.Equal(1, skipReader.AdvancePast(8)); + Assert.True(skipReader.TryRead(out value)); + Assert.Equal(9, value); + + skipReader = new SequenceReader(bytes); + Assert.Equal(0, skipReader.AdvancePast(2)); + Assert.Equal(3, skipReader.AdvancePastAny(new byte[] { 2, 3, 1 })); + Assert.True(skipReader.TryRead(out value)); + Assert.Equal(4, value); + } + + [Fact] + public void PastEmptySegments() + { + ReadOnlySequence bytes = SequenceFactory.Create(new byte[][] { + new byte[] { 0 }, + new byte[] { }, + new byte[] { }, + new byte[] { } + }); + + SequenceReader reader = new SequenceReader(bytes); + reader.Advance(1); + Assert.Equal(0, reader.CurrentSpanIndex); + Assert.Equal(0, reader.CurrentSpan.Length); + Assert.False(reader.TryPeek(out byte value)); + ReadOnlySequence sequence = reader.Sequence.Slice(reader.Position); + Assert.Equal(0, sequence.Length); + } + } +} diff --git a/src/System.Memory/tests/SequenceReader/BasicTests.cs b/src/System.Memory/tests/SequenceReader/BasicTests.cs new file mode 100644 index 000000000000..3986dc64ea9b --- /dev/null +++ b/src/System.Memory/tests/SequenceReader/BasicTests.cs @@ -0,0 +1,542 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Buffers; +using Xunit; + +namespace System.Memory.Tests.SequenceReader +{ + public class ArrayByte : SingleSegment + { + public ArrayByte() : base(ReadOnlySequenceFactory.ArrayFactory, s_byteInputData) { } + } + + public class ArrayChar : SingleSegment + { + public ArrayChar() : base(ReadOnlySequenceFactory.ArrayFactory, s_charInputData) { } + } + + public class MemoryByte : SingleSegment + { + public MemoryByte() : base(ReadOnlySequenceFactory.MemoryFactory, s_byteInputData) { } + } + + public class MemoryChar : SingleSegment + { + public MemoryChar() : base(ReadOnlySequenceFactory.MemoryFactory, s_charInputData) { } + } + + public class SingleSegmentByte : SingleSegment + { + public SingleSegmentByte() : base(s_byteInputData) { } + } + + public class SingleSegmentChar : SingleSegment + { + public SingleSegmentChar() : base(s_charInputData) { } + } + + public abstract class SingleSegment : ReaderBasicTests where T : unmanaged, IEquatable + { + public SingleSegment(T[] inputData) : base(ReadOnlySequenceFactory.SingleSegmentFactory, inputData) { } + internal SingleSegment(ReadOnlySequenceFactory factory, T[] inputData) : base(factory, inputData) { } + + [Fact] + public void AdvanceSingleBufferSkipsValues() + { + SequenceReader reader = new SequenceReader(SequenceFactory.Create(GetInputData(5))); + Assert.Equal(5, reader.Length); + Assert.Equal(5, reader.Remaining); + Assert.Equal(0, reader.Consumed); + Assert.Equal(0, reader.CurrentSpanIndex); + + // Advance 2 positions + reader.Advance(2); + Assert.Equal(5, reader.Length); + Assert.Equal(3, reader.Remaining); + Assert.Equal(2, reader.Consumed); + Assert.Equal(2, reader.CurrentSpanIndex); + Assert.Equal(InputData[2], reader.CurrentSpan[reader.CurrentSpanIndex]); + Assert.True(reader.TryPeek(out T value)); + Assert.Equal(InputData[2], value); + + // Advance 2 positions + reader.Advance(2); + Assert.Equal(1, reader.Remaining); + Assert.Equal(4, reader.Consumed); + Assert.Equal(4, reader.CurrentSpanIndex); + Assert.Equal(InputData[4], reader.CurrentSpan[reader.CurrentSpanIndex]); + Assert.True(reader.TryPeek(out value)); + Assert.Equal(InputData[4], value); + } + + [Fact] + public void TryReadReturnsValueAndAdvances() + { + SequenceReader reader = new SequenceReader(Factory.CreateWithContent(GetInputData(2))); + Assert.Equal(2, reader.Length); + Assert.Equal(2, reader.Remaining); + Assert.Equal(0, reader.Consumed); + Assert.Equal(0, reader.CurrentSpanIndex); + Assert.Equal(InputData[0], reader.CurrentSpan[reader.CurrentSpanIndex]); + + // Read 1st value + Assert.True(reader.TryRead(out T value)); + Assert.Equal(InputData[0], value); + Assert.Equal(1, reader.Remaining); + Assert.Equal(1, reader.Consumed); + Assert.Equal(1, reader.CurrentSpanIndex); + Assert.Equal(InputData[1], reader.CurrentSpan[reader.CurrentSpanIndex]); + + // Read 2nd value + Assert.True(reader.TryRead(out value)); + Assert.Equal(InputData[1], value); + Assert.Equal(0, reader.Remaining); + Assert.Equal(2, reader.Consumed); + Assert.Equal(2, reader.CurrentSpanIndex); + + // Read at end + Assert.False(reader.TryRead(out value)); + Assert.Equal(default, value); + Assert.Equal(0, reader.Remaining); + Assert.Equal(2, reader.Consumed); + Assert.Equal(2, reader.CurrentSpanIndex); + Assert.True(reader.End); + } + + [Fact] + public void DefaultState() + { + T[] array = new T[] { default }; + SequenceReader reader = default; + Assert.Equal(0, reader.CurrentSpan.Length); + Assert.Equal(0, reader.UnreadSpan.Length); + Assert.Equal(0, reader.Consumed); + Assert.Equal(0, reader.CurrentSpanIndex); + Assert.Equal(0, reader.Length); + Assert.Equal(0, reader.Remaining); + Assert.True(reader.End); + Assert.False(reader.TryPeek(out T value)); + Assert.Equal(default, value); + Assert.False(reader.TryRead(out value)); + Assert.Equal(default, value); + Assert.Equal(0, reader.AdvancePast(default)); + Assert.Equal(0, reader.AdvancePastAny(array)); + Assert.Equal(0, reader.AdvancePastAny(default)); + Assert.False(reader.TryReadTo(out ReadOnlySequence sequence, default(T))); + Assert.True(sequence.IsEmpty); + Assert.False(reader.TryReadTo(out sequence, array)); + Assert.True(sequence.IsEmpty); + Assert.False(reader.TryReadTo(out ReadOnlySpan span, default)); + Assert.True(span.IsEmpty); + Assert.False(reader.TryReadToAny(out sequence, array)); + Assert.True(sequence.IsEmpty); + Assert.False(reader.TryReadToAny(out span, array)); + Assert.True(span.IsEmpty); + Assert.False(reader.TryAdvanceTo(default)); + Assert.False(reader.TryAdvanceToAny(array)); + Assert.Equal(0, reader.CurrentSpan.Length); + Assert.Equal(0, reader.UnreadSpan.Length); + Assert.Equal(0, reader.Consumed); + Assert.Equal(0, reader.CurrentSpanIndex); + Assert.Equal(0, reader.Length); + Assert.Equal(0, reader.Remaining); + } + } + + public class SegmentPerByte : ReaderBasicTests + { + public SegmentPerByte() : base(ReadOnlySequenceFactory.SegmentPerItemFactory, s_byteInputData) { } + } + + public class SegmentPerChar : ReaderBasicTests + { + public SegmentPerChar() : base(ReadOnlySequenceFactory.SegmentPerItemFactory, s_charInputData) { } + } + + public abstract class ReaderBasicTests where T : unmanaged, IEquatable + { + internal static byte[] s_byteInputData = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; + internal static char[] s_charInputData = new char[] { '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A' }; + private T[] _inputData; + + internal ReadOnlySequenceFactory Factory { get; } + protected ReadOnlySpan InputData { get => _inputData; } + + public T[] GetInputData(int count) => InputData.Slice(0, count).ToArray(); + + internal ReaderBasicTests(ReadOnlySequenceFactory factory, T[] inputData) + { + Factory = factory; + _inputData = inputData; + } + + [Fact] + public void TryPeekReturnsWithoutMoving() + { + SequenceReader reader = new SequenceReader(Factory.CreateWithContent(GetInputData(2))); + Assert.Equal(0, reader.Consumed); + Assert.Equal(2, reader.Remaining); + Assert.True(reader.TryPeek(out T value)); + Assert.Equal(InputData[0], value); + Assert.Equal(0, reader.Consumed); + Assert.Equal(2, reader.Remaining); + Assert.True(reader.TryPeek(out value)); + Assert.Equal(InputData[0], value); + Assert.Equal(0, reader.Consumed); + Assert.Equal(2, reader.Remaining); + } + + [Fact] + public void CursorIsCorrectAtEnd() + { + SequenceReader reader = new SequenceReader(Factory.CreateWithContent(GetInputData(2))); + reader.TryRead(out T _); + reader.TryRead(out T _); + Assert.True(reader.End); + } + + [Fact] + public void CursorIsCorrectWithEmptyLastBlock() + { + SequenceSegment last = new SequenceSegment(); + last.SetMemory(new OwnedArray(new T[4]), 0, 4); + + SequenceSegment first = new SequenceSegment(); + first.SetMemory(new OwnedArray(GetInputData(2)), 0, 2); + first.SetNext(last); + + SequenceReader reader = new SequenceReader(new ReadOnlySequence(first, first.Start, last, last.Start)); + reader.TryRead(out T _); + reader.TryRead(out T _); + reader.TryRead(out T _); + Assert.Same(last, reader.Position.GetObject()); + Assert.Equal(0, reader.Position.GetInteger()); + Assert.True(reader.End); + } + + [Fact] + public void TryPeekReturnsDefaultInTheEnd() + { + SequenceReader reader = new SequenceReader(Factory.CreateWithContent(GetInputData(2))); + Assert.True(reader.TryRead(out T value)); + Assert.Equal(InputData[0], value); + Assert.True(reader.TryRead(out value)); + Assert.Equal(InputData[1], value); + Assert.False(reader.TryRead(out value)); + Assert.Equal(default, value); + } + + [Fact] + public void AdvanceToEndThenPeekReturnsDefault() + { + SequenceReader reader = new SequenceReader(Factory.CreateWithContent(GetInputData(5))); + reader.Advance(5); + Assert.True(reader.End); + Assert.False(reader.TryPeek(out T value)); + Assert.Equal(default, value); + } + + [Fact] + public void AdvancingPastLengthThrows() + { + SequenceReader reader = new SequenceReader(Factory.CreateWithContent(GetInputData(5))); + try + { + reader.Advance(6); + Assert.True(false); + } + catch (Exception ex) + { + Assert.True(ex is ArgumentOutOfRangeException); + Assert.Equal(5, reader.Consumed); + Assert.Equal(0, reader.Remaining); + Assert.True(reader.End); + } + } + + [Fact] + public void CtorFindsFirstNonEmptySegment() + { + ReadOnlySequence buffer = Factory.CreateWithContent(GetInputData(1)); + SequenceReader reader = new SequenceReader(buffer); + + Assert.True(reader.TryPeek(out T value)); + Assert.Equal(InputData[0], value); + Assert.Equal(0, reader.Consumed); + Assert.Equal(1, reader.Remaining); + } + + [Fact] + public void EmptySegmentsAreSkippedOnMoveNext() + { + ReadOnlySequence buffer = Factory.CreateWithContent(GetInputData(2)); + SequenceReader reader = new SequenceReader(buffer); + + Assert.True(reader.TryPeek(out T value)); + Assert.Equal(InputData[0], value); + reader.Advance(1); + Assert.True(reader.TryPeek(out value)); + Assert.Equal(InputData[1], value); + } + + [Fact] + public void TryPeekGoesToEndIfAllEmptySegments() + { + ReadOnlySequence buffer = SequenceFactory.Create(new[] { new T[] { }, new T[] { }, new T[] { }, new T[] { } }); + SequenceReader reader = new SequenceReader(buffer); + + Assert.False(reader.TryPeek(out T value)); + Assert.Equal(default, value); + Assert.True(reader.End); + } + + [Fact] + public void AdvanceTraversesSegments() + { + ReadOnlySequence buffer = Factory.CreateWithContent(GetInputData(3)); + SequenceReader reader = new SequenceReader(buffer); + + reader.Advance(2); + Assert.Equal(InputData[2], reader.CurrentSpan[reader.CurrentSpanIndex]); + Assert.True(reader.TryRead(out T value)); + Assert.Equal(InputData[2], value); + } + + [Fact] + public void AdvanceThrowsPastLengthMultipleSegments() + { + ReadOnlySequence buffer = Factory.CreateWithContent(GetInputData(3)); + SequenceReader reader = new SequenceReader(buffer); + + try + { + reader.Advance(4); + Assert.True(false); + } + catch (Exception ex) + { + Assert.True(ex is ArgumentOutOfRangeException); + Assert.Equal(3, reader.Consumed); + Assert.Equal(0, reader.Remaining); + Assert.True(reader.End); + } + } + + [Fact] + public void TryReadTraversesSegments() + { + ReadOnlySequence buffer = Factory.CreateWithContent(GetInputData(3)); + SequenceReader reader = new SequenceReader(buffer); + + Assert.True(reader.TryRead(out T value)); + Assert.Equal(InputData[0], value); + Assert.True(reader.TryRead(out value)); + Assert.Equal(InputData[1], value); + Assert.True(reader.TryRead(out value)); + Assert.Equal(InputData[2], value); + Assert.False(reader.TryRead(out value)); + Assert.Equal(default, value); + Assert.True(reader.End); + } + + [Fact] + public void TryPeekTraversesSegments() + { + ReadOnlySequence buffer = Factory.CreateWithContent(GetInputData(2)); + SequenceReader reader = new SequenceReader(buffer); + + Assert.Equal(InputData[0], reader.CurrentSpan[reader.CurrentSpanIndex]); + Assert.True(reader.TryRead(out T value)); + Assert.Equal(InputData[0], value); + + Assert.Equal(InputData[1], reader.CurrentSpan[reader.CurrentSpanIndex]); + Assert.True(reader.TryPeek(out value)); + Assert.Equal(InputData[1], value); + Assert.True(reader.TryRead(out value)); + Assert.Equal(InputData[1], value); + Assert.False(reader.TryPeek(out value)); + Assert.False(reader.TryRead(out value)); + Assert.Equal(default, value); + Assert.Equal(default, value); + } + + [Fact] + public void PeekWorkesWithEmptySegments() + { + ReadOnlySequence buffer = Factory.CreateWithContent(GetInputData(1)); + SequenceReader reader = new SequenceReader(buffer); + + Assert.Equal(0, reader.CurrentSpanIndex); + Assert.Equal(1, reader.CurrentSpan.Length); + Assert.True(reader.TryPeek(out T value)); + Assert.Equal(InputData[0], value); + Assert.True(reader.TryRead(out value)); + Assert.Equal(InputData[0], value); + Assert.False(reader.TryPeek(out value)); + Assert.False(reader.TryRead(out value)); + Assert.Equal(default, value); + Assert.Equal(default, value); + } + + [Fact] + public void WorksWithEmptyBuffer() + { + SequenceReader reader = new SequenceReader(Factory.CreateWithContent(new T[] { })); + + Assert.Equal(0, reader.CurrentSpanIndex); + Assert.Equal(0, reader.CurrentSpan.Length); + Assert.Equal(0, reader.Length); + Assert.Equal(0, reader.Remaining); + Assert.True(reader.End); + Assert.False(reader.TryPeek(out T value)); + Assert.Equal(default, value); + Assert.False(reader.TryRead(out value)); + Assert.Equal(default, value); + Assert.True(reader.End); + } + + [Theory, + InlineData(0, false), + InlineData(5, false), + InlineData(10, false), + InlineData(11, true), + InlineData(12, true), + InlineData(15, true)] + public void ReturnsCorrectCursor(int takes, bool end) + { + ReadOnlySequence readableBuffer = Factory.CreateWithContent(GetInputData(10)); + SequenceReader reader = new SequenceReader(readableBuffer); + for (int i = 0; i < takes; i++) + { + reader.TryRead(out _); + } + + T[] expected = end ? new T[] { } : readableBuffer.Slice(takes).ToArray(); + Assert.Equal(expected, readableBuffer.Slice(reader.Position).ToArray()); + } + + [Fact] + public void SlicingBufferReturnsCorrectCursor() + { + ReadOnlySequence buffer = Factory.CreateWithContent(GetInputData(10)); + ReadOnlySequence sliced = buffer.Slice(2L); + + SequenceReader reader = new SequenceReader(sliced); + Assert.Equal(sliced.ToArray(), buffer.Slice(reader.Position).ToArray()); + Assert.True(reader.TryPeek(out T value)); + Assert.Equal(InputData[2], value); + Assert.Equal(0, reader.CurrentSpanIndex); + } + + [Fact] + public void ReaderIndexIsCorrect() + { + ReadOnlySequence buffer = Factory.CreateWithContent(GetInputData(10)); + SequenceReader reader = new SequenceReader(buffer); + + int counter = 0; + while (!reader.End) + { + ReadOnlySpan span = reader.CurrentSpan; + for (int i = reader.CurrentSpanIndex; i < span.Length; i++) + { + Assert.Equal(InputData[counter++], reader.CurrentSpan[i]); + } + reader.Advance(span.Length); + } + Assert.Equal(buffer.Length, reader.Consumed); + } + + [Theory, + InlineData(1), + InlineData(2), + InlineData(3)] + public void Advance_PositionIsCorrect(int advanceBy) + { + // Check that advancing through the reader gives the same position + // as returned directly from the buffer. + + ReadOnlySequence buffer = Factory.CreateWithContent(GetInputData(10)); + SequenceReader reader = new SequenceReader(buffer); + + SequencePosition readerPosition = reader.Position; + SequencePosition bufferPosition = buffer.GetPosition(0); + Assert.Equal(readerPosition.GetInteger(), bufferPosition.GetInteger()); + Assert.Same(readerPosition.GetObject(), readerPosition.GetObject()); + + for (int i = advanceBy; i <= buffer.Length; i += advanceBy) + { + reader.Advance(advanceBy); + readerPosition = reader.Position; + bufferPosition = buffer.GetPosition(i); + Assert.Equal(readerPosition.GetInteger(), bufferPosition.GetInteger()); + Assert.Same(readerPosition.GetObject(), readerPosition.GetObject()); + } + } + + [Fact] + public void AdvanceTo() + { + // Ensure we can advance to each of the items in the buffer + + T[] inputData = GetInputData(10); + ReadOnlySequence buffer = Factory.CreateWithContent(inputData); + + for (int i = 0; i < buffer.Length; i++) + { + SequenceReader reader = new SequenceReader(buffer); + Assert.True(reader.TryAdvanceTo(inputData[i], advancePastDelimiter: false)); + Assert.True(reader.TryPeek(out T value)); + Assert.Equal(inputData[i], value); + } + } + + [Fact] + public void AdvanceTo_AdvancePast() + { + // Ensure we can advance to each of the items in the buffer (skipping what we advanced to) + + T[] inputData = GetInputData(10); + ReadOnlySequence buffer = Factory.CreateWithContent(inputData); + + for (int start = 0; start < 2; start++) + { + for (int i = start; i < buffer.Length - 1; i += 2) + { + SequenceReader reader = new SequenceReader(buffer); + Assert.True(reader.TryAdvanceTo(inputData[i], advancePastDelimiter: true)); + Assert.True(reader.TryPeek(out T value)); + Assert.Equal(inputData[i + 1], value); + } + } + } + + [Fact] + public void CopyToSmallerBufferWorks() + { + T[] content = (T[])_inputData.Clone(); + + Span buffer = new T[content.Length]; + SequenceReader reader = new SequenceReader(Factory.CreateWithContent(content)); + + // this loop skips more and more items in the reader + for (int i = 0; i < content.Length; i++) + { + // this loop makes the destination buffer smaller and smaller + for (int j = 0; j < buffer.Length - i; j++) + { + Span bufferSlice = buffer.Slice(0, j); + bufferSlice.Clear(); + Assert.True(reader.TryCopyTo(bufferSlice)); + Assert.Equal(Math.Min(bufferSlice.Length, content.Length - i), bufferSlice.Length); + + Assert.True(bufferSlice.SequenceEqual(content.AsSpan(i, j))); + } + + reader.Advance(1); + } + } + } +} diff --git a/src/System.Memory/tests/SequenceReader/BinaryExtensions.cs b/src/System.Memory/tests/SequenceReader/BinaryExtensions.cs new file mode 100644 index 000000000000..f1a89ae48d23 --- /dev/null +++ b/src/System.Memory/tests/SequenceReader/BinaryExtensions.cs @@ -0,0 +1,70 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Buffers; +using System.Runtime.InteropServices; +using Xunit; + +namespace System.Memory.Tests.SequenceReader +{ + public class BinaryExtensions + { + [Fact] + public void MultiSegmentBytesReaderNumbers() + { + ReadOnlySequence bytes = SequenceFactory.Create(new byte[][] { + new byte[] { 0 }, + new byte[] { 1, 2 }, + new byte[] { 3, 4 }, + new byte[] { 5, 6, 7, 8 }, + new byte[] { 8, 0 }, + new byte[] { 1, }, + new byte[] { 0, 2, }, + new byte[] { 1, 2, 3, 4 }, + new byte[] { 5, 6 }, + new byte[] { 7, 8, 9, }, + new byte[] { 0, 1, 2, 3 }, + new byte[] { 4, 5 }, + new byte[] { 6, 7, 8, 9 }, + new byte[] { 0, 1, 2, 3 }, + new byte[] { 4 }, + }); + + SequenceReader reader = new SequenceReader(bytes); + + Assert.True(reader.TryReadTo(out ReadOnlySequence bytesValue, 2)); + Span span = bytesValue.ToArray(); + Assert.Equal(0, span[0]); + Assert.Equal(1, span[1]); + + Assert.True(reader.TryReadTo(out bytesValue, 5)); + span = bytesValue.ToArray(); + Assert.Equal(3, span[0]); + Assert.Equal(4, span[1]); + + Assert.True(reader.TryReadTo(out bytesValue, new byte[] { 8, 8 })); + span = bytesValue.ToArray(); + Assert.Equal(6, span[0]); + Assert.Equal(7, span[1]); + + Assert.True(SequenceMarshal.TryRead(ref reader, out int intValue)); + Assert.Equal(BitConverter.ToInt32(new byte[] { 0, 1, 0, 2 }), intValue); + + Assert.True(reader.TryReadBigEndian(out intValue)); + Assert.Equal(BitConverter.ToInt32(new byte[] { 4, 3, 2, 1 }), intValue); + + Assert.True(reader.TryReadLittleEndian(out long longValue)); + Assert.Equal(BitConverter.ToInt64(new byte[] { 5, 6, 7, 8, 9, 0, 1, 2 }), longValue); + + Assert.True(reader.TryReadBigEndian(out longValue)); + Assert.Equal(BitConverter.ToInt64(new byte[] { 0, 9, 8, 7, 6, 5, 4, 3 }), longValue); + + Assert.True(reader.TryReadLittleEndian(out short shortValue)); + Assert.Equal(BitConverter.ToInt16(new byte[] { 1, 2 }), shortValue); + + Assert.True(reader.TryReadBigEndian(out shortValue)); + Assert.Equal(BitConverter.ToInt16(new byte[] { 4, 3 }), shortValue); + } + } +} diff --git a/src/System.Memory/tests/SequenceReader/IsNext.cs b/src/System.Memory/tests/SequenceReader/IsNext.cs new file mode 100644 index 000000000000..8141de4b199c --- /dev/null +++ b/src/System.Memory/tests/SequenceReader/IsNext.cs @@ -0,0 +1,46 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Buffers; +using Xunit; + +namespace System.Memory.Tests.SequenceReader +{ + public class IsNext + { + [Fact] + public void IsNext_Span() + { + ReadOnlySequence bytes = SequenceFactory.Create(new byte[][] { + new byte[] { 0 }, + new byte[] { 1, 2 }, + new byte[] { 3, 4 }, + new byte[] { 5, 6, 7, 8 } + }); + + SequenceReader reader = new SequenceReader(bytes); + Assert.True(reader.IsNext(ReadOnlySpan.Empty, advancePast: false)); + Assert.True(reader.IsNext(ReadOnlySpan.Empty, advancePast: true)); + Assert.True(reader.IsNext(new byte[] { 0 }, advancePast: false)); + Assert.False(reader.IsNext(new byte[] { 0, 2 }, advancePast: false)); + Assert.False(reader.IsNext(new byte[] { 0, 2 }, advancePast: true)); + Assert.True(reader.IsNext(new byte[] { 0, 1 }, advancePast: false)); + Assert.False(reader.IsNext(new byte[] { 0, 1, 3 }, advancePast: false)); + Assert.True(reader.IsNext(new byte[] { 0, 1, 2 }, advancePast: false)); + Assert.False(reader.IsNext(new byte[] { 0, 1, 2, 4 }, advancePast: false)); + Assert.True(reader.IsNext(new byte[] { 0, 1, 2, 3 }, advancePast: false)); + Assert.True(reader.IsNext(new byte[] { 0, 1, 2, 3, 4 }, advancePast: false)); + Assert.True(reader.IsNext(new byte[] { 0, 1, 2, 3, 4, 5 }, advancePast: false)); + Assert.True(reader.IsNext(new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8 }, advancePast: false)); + Assert.False(reader.IsNext(new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, advancePast: false)); + Assert.False(reader.IsNext(new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, advancePast: true)); + Assert.Equal(0, reader.Consumed); + + Assert.True(reader.IsNext(new byte[] { 0, 1, 2, 3 }, advancePast: true)); + Assert.True(reader.IsNext(new byte[] { 4, 5, 6 }, advancePast: true)); + Assert.True(reader.TryPeek(out byte value)); + Assert.Equal(7, value); + } + } +} diff --git a/src/System.Memory/tests/SequenceReader/OwnedArray.cs b/src/System.Memory/tests/SequenceReader/OwnedArray.cs new file mode 100644 index 000000000000..37c20a236058 --- /dev/null +++ b/src/System.Memory/tests/SequenceReader/OwnedArray.cs @@ -0,0 +1,103 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Buffers; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Threading; + +namespace System.Memory.Tests.SequenceReader +{ + public class OwnedArray : MemoryManager + { + T[] _array; + int _referenceCount; + + public OwnedArray(int length) + { + _array = new T[length]; + } + + public OwnedArray(T[] array) + { + if (array == null) + ThrowArgumentNullException(nameof(array)); + _array = array; + } + + public static implicit operator OwnedArray(T[] array) => new OwnedArray(array); + + public override Span GetSpan() + { + if (IsDisposed) + ThrowObjectDisposedException(nameof(OwnedArray)); + return new Span(_array); + } + + public override MemoryHandle Pin(int elementIndex = 0) + { + unsafe + { + Retain(); + if ((uint)elementIndex > (uint)_array.Length) + throw new ArgumentOutOfRangeException(nameof(elementIndex)); + var handle = GCHandle.Alloc(_array, GCHandleType.Pinned); + void* pointer = Unsafe.Add((void*)handle.AddrOfPinnedObject(), elementIndex); + return new MemoryHandle(pointer, handle, this); + } + } + + protected override bool TryGetArray(out ArraySegment arraySegment) + { + if (IsDisposed) + ThrowObjectDisposedException(nameof(OwnedArray)); + arraySegment = new ArraySegment(_array); + return true; + } + + protected override void Dispose(bool disposing) + { + _array = null; + } + + public void Retain() + { + if (IsDisposed) + ThrowObjectDisposedException(nameof(OwnedArray)); + Interlocked.Increment(ref _referenceCount); + } + + public override void Unpin() + { + int newRefCount = Interlocked.Decrement(ref _referenceCount); + if (newRefCount < 0) + ThrowInvalidOperationException(); + if (newRefCount == 0) + { + OnNoReferences(); + } + } + + protected virtual void OnNoReferences() + { + } + + protected bool IsRetained => _referenceCount > 0; + + public bool IsDisposed => _array == null; + + [MethodImpl(MethodImplOptions.NoInlining)] + public static void ThrowObjectDisposedException(string objectName) + => throw new ObjectDisposedException(objectName); + + [MethodImpl(MethodImplOptions.NoInlining)] + public static void ThrowInvalidOperationException() + => throw new InvalidOperationException(); + + [MethodImpl(MethodImplOptions.NoInlining)] + public static void ThrowArgumentNullException(string argumentName) + => throw new ArgumentNullException(argumentName); + } +} diff --git a/src/System.Memory/tests/SequenceReader/ReadTo.cs b/src/System.Memory/tests/SequenceReader/ReadTo.cs new file mode 100644 index 000000000000..2742f283d5fe --- /dev/null +++ b/src/System.Memory/tests/SequenceReader/ReadTo.cs @@ -0,0 +1,177 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Buffers; +using Xunit; + +namespace System.Memory.Tests.SequenceReader +{ + public class ReadTo + { + [Theory, + InlineData(false, false), + InlineData(false, true), + InlineData(true, false), + InlineData(true, true)] + public void TryReadTo_Span(bool advancePastDelimiter, bool useEscapeOverload) + { + ReadOnlySequence bytes = SequenceFactory.Create(new byte[][] { + new byte[] { 0 }, + new byte[] { 1, 2 }, + new byte[] { }, + new byte[] { 3, 4, 5, 6 } + }); + + SequenceReader reader = new SequenceReader(bytes); + + // Read to 0-5 + for (byte i = 0; i < bytes.Length - 1; i++) + { + SequenceReader copy = reader; + + // Can read to the first integer (0-5) + Assert.True( + useEscapeOverload + ? copy.TryReadTo(out ReadOnlySpan span, i, 255, advancePastDelimiter) + : copy.TryReadTo(out span, i, advancePastDelimiter)); + + // Should never have a null Position object + Assert.NotNull(copy.Position.GetObject()); + + // Should be able to then read to 6 + Assert.True( + useEscapeOverload + ? copy.TryReadTo(out span, 6, 255, advancePastDelimiter) + : copy.TryReadTo(out span, 6, advancePastDelimiter)); + + Assert.NotNull(copy.Position.GetObject()); + + // If we didn't advance, we should still be able to read to 6 + Assert.Equal(!advancePastDelimiter, + useEscapeOverload + ? copy.TryReadTo(out span, 6, 255, advancePastDelimiter) + : copy.TryReadTo(out span, 6, advancePastDelimiter)); + } + } + + [Theory, + InlineData(false, false), + InlineData(false, true), + InlineData(true, false), + InlineData(true, true)] + public void TryReadTo_Sequence(bool advancePastDelimiter, bool useEscapeOverload) + { + ReadOnlySequence bytes = SequenceFactory.Create(new byte[][] { + new byte[] { 0 }, + new byte[] { 1, 2 }, + new byte[] { }, + new byte[] { 3, 4, 5, 6 } + }); + + SequenceReader reader = new SequenceReader(bytes); + + // Read to 0-5 + for (byte i = 0; i < bytes.Length - 1; i++) + { + SequenceReader copy = reader; + + // Can read to the first integer (0-5) + Assert.True( + useEscapeOverload + ? copy.TryReadTo(out ReadOnlySequence sequence, i, 255, advancePastDelimiter) + : copy.TryReadTo(out sequence, i, advancePastDelimiter)); + + // Should never have a null Position object + Assert.NotNull(copy.Position.GetObject()); + ReadOnlySequence.Enumerator enumerator = sequence.GetEnumerator(); + while (enumerator.MoveNext()) + ; + + // Should be able to read to final 6 + Assert.True( + useEscapeOverload + ? copy.TryReadTo(out sequence, 6, 255, advancePastDelimiter) + : copy.TryReadTo(out sequence, 6, advancePastDelimiter)); + + Assert.NotNull(copy.Position.GetObject()); + enumerator = sequence.GetEnumerator(); + while (enumerator.MoveNext()) + ; + + // If we didn't advance, we should still be able to read to 6 + Assert.Equal(!advancePastDelimiter, + useEscapeOverload + ? copy.TryReadTo(out sequence, 6, 255, advancePastDelimiter) + : copy.TryReadTo(out sequence, 6, advancePastDelimiter)); + } + } + + [Theory, + InlineData(false), + InlineData(true),] + public void TryReadToSpan_Sequence(bool advancePastDelimiter) + { + ReadOnlySequence bytes = SequenceFactory.Create(new byte[][] { + new byte[] { 0, 0 }, + new byte[] { 1, 1, 2, 2 }, + new byte[] { }, + new byte[] { 3, 3, 4, 4, 5, 5, 6, 6 } + }); + + SequenceReader reader = new SequenceReader(bytes); + for (byte i = 0; i < bytes.Length / 2 - 1; i++) + { + byte[] expected = new byte[i * 2 + 1]; + for (int j = 0; j < expected.Length - 1; j++) + { + expected[j] = (byte)(j / 2); + } + expected[i * 2] = i; + ReadOnlySpan searchFor = new byte[] { i, (byte)(i + 1) }; + SequenceReader copy = reader; + Assert.True(copy.TryReadTo(out ReadOnlySequence seq, searchFor, advancePastDelimiter)); + Assert.True(seq.ToArray().AsSpan().SequenceEqual(expected)); + } + + bytes = SequenceFactory.Create(new byte[][] { + new byte[] { 47, 42, 66, 32, 42, 32, 66, 42, 47 } // /*b * b*/ + }); + + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out ReadOnlySequence sequence, new byte[] { 42, 47 }, advancePastDelimiter)); // */ + Assert.True(sequence.ToArray().AsSpan().SequenceEqual(new byte[] { 47, 42, 66, 32, 42, 32, 66 })); + } + + [Theory, + InlineData(false), + InlineData(true)] + public void TryReadTo_NotFound_Span(bool advancePastDelimiter) + { + ReadOnlySequence bytes = SequenceFactory.Create(new byte[][] { + new byte[] { 1 }, + new byte[] { 2, 3, 255 } + }); + + SequenceReader reader = new SequenceReader(bytes); + reader.Advance(4); + Assert.False(reader.TryReadTo(out ReadOnlySpan span, 255, 0, advancePastDelimiter)); + } + + [Theory, + InlineData(false), + InlineData(true)] + public void TryReadTo_NotFound_Sequence(bool advancePastDelimiter) + { + ReadOnlySequence bytes = SequenceFactory.Create(new byte[][] { + new byte[] { 1 }, + new byte[] { 2, 3, 255 } + }); + + SequenceReader reader = new SequenceReader(bytes); + reader.Advance(4); + Assert.False(reader.TryReadTo(out ReadOnlySequence span, 255, 0, advancePastDelimiter)); + } + } +} diff --git a/src/System.Memory/tests/SequenceReader/Rewind.cs b/src/System.Memory/tests/SequenceReader/Rewind.cs new file mode 100644 index 000000000000..2d91cf3f5de3 --- /dev/null +++ b/src/System.Memory/tests/SequenceReader/Rewind.cs @@ -0,0 +1,71 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Buffers; +using Xunit; + +namespace System.Memory.Tests.SequenceReader +{ + public class Rewind + { + [Fact] + public void Rewind_Full() + { + ReadOnlySequence bytes = SequenceFactory.Create(new byte[][] { + new byte[] { 0 }, + new byte[] { 1, 2 }, + new byte[] { 3, 4 }, + new byte[] { 5, 6, 7, 8 } + }); + + SequenceReader reader = new SequenceReader(bytes); + reader.Advance(1); + SequenceReader copy = reader; + for (int i = 1; i < bytes.Length; i++) + { + reader.Advance(i); + reader.Rewind(i); + + Assert.Equal(copy.Position, reader.Position); + Assert.Equal(copy.Consumed, reader.Consumed); + Assert.Equal(copy.CurrentSpanIndex, reader.CurrentSpanIndex); + Assert.Equal(copy.End, reader.End); + Assert.True(copy.CurrentSpan.SequenceEqual(reader.CurrentSpan)); + } + } + + [Fact] + public void Rewind_ByOne() + { + ReadOnlySequence bytes = SequenceFactory.Create(new byte[][] { + new byte[] { 0 }, + new byte[] { 1, 2 }, + new byte[] { 3, 4 }, + new byte[] { 5, 6, 7, 8 } + }); + + SequenceReader reader = new SequenceReader(bytes); + reader.Advance(1); + SequenceReader copy = reader; + for (int i = 1; i < bytes.Length; i++) + { + reader.Advance(i); + Assert.Equal(i + 1, reader.Consumed); + + for (int j = 0; j < i; j++) + { + reader.Rewind(1); + Assert.Equal(i - j, reader.Consumed); + Assert.False(reader.End); + } + + Assert.Equal(copy.Position, reader.Position); + Assert.Equal(copy.Consumed, reader.Consumed); + Assert.Equal(copy.CurrentSpanIndex, reader.CurrentSpanIndex); + Assert.Equal(copy.End, reader.End); + Assert.True(copy.CurrentSpan.SequenceEqual(reader.CurrentSpan)); + } + } + } +} diff --git a/src/System.Memory/tests/SequenceReader/SequenceFactory.cs b/src/System.Memory/tests/SequenceReader/SequenceFactory.cs new file mode 100644 index 000000000000..eea28a00d657 --- /dev/null +++ b/src/System.Memory/tests/SequenceReader/SequenceFactory.cs @@ -0,0 +1,119 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Buffers; +using System.Text; + +namespace System.Memory.Tests.SequenceReader +{ + public static class SequenceFactory + { + public static ReadOnlySequence CreateSplit(T[] buffer, int minSize, int maxSize) where T : struct + { + if (buffer == null || buffer.Length == 0 || minSize <= 0 || maxSize <= 0 || minSize > maxSize) + { + throw new InvalidOperationException(); + } + + Random r = new Random(0xFEED); + + SequenceSegment last = null; + SequenceSegment first = null; + OwnedArray ownedBuffer = new OwnedArray(buffer); + + int remaining = buffer.Length; + int position = 0; + while (remaining > 0) + { + int take = Math.Min(r.Next(minSize, maxSize), remaining); + SequenceSegment current = new SequenceSegment(); + current.SetMemory(ownedBuffer, position, position + take); + if (first == null) + { + first = current; + last = current; + } + else + { + last.SetNext(current); + last = current; + } + remaining -= take; + position += take; + } + + return new ReadOnlySequence(first, 0, last, last.Length); + } + + public static ReadOnlySequence Create(params T[][] inputs) where T : struct + { + if (inputs == null || inputs.Length == 0) + { + throw new InvalidOperationException(); + } + + SequenceSegment last = null; + SequenceSegment first = null; + + for (int i = 0; i < inputs.Length; i++) + { + T[] source = inputs[i]; + int length = source.Length; + int dataOffset = length; + + // Shift the incoming data for testing + T[] chars = new T[length * 8]; + for (int j = 0; j < length; j++) + { + chars[dataOffset + j] = source[j]; + } + + // Create a segment that has offset relative to the OwnedMemory and OwnedMemory itself has offset relative to array + OwnedArray ownedBuffer = new OwnedArray(chars); + SequenceSegment current = new SequenceSegment(); + current.SetMemory(ownedBuffer, length, length * 2); + if (first == null) + { + first = current; + last = current; + } + else + { + last.SetNext(current); + last = current; + } + } + + return new ReadOnlySequence(first, 0, last, last.Length); + } + + public static ReadOnlySequence CreateUtf8(params string[] inputs) + { + byte[][] buffers = new byte[inputs.Length][]; + for (int i = 0; i < inputs.Length; i++) + { + buffers[i] = Encoding.UTF8.GetBytes(inputs[i]); + } + return Create(buffers); + } + + public static ReadOnlySequence Create(params int[] inputs) where T : struct + { + T[][] buffers; + if (inputs.Length == 0) + { + buffers = new[] { new T[] { } }; + } + else + { + buffers = new T[inputs.Length][]; + for (int i = 0; i < inputs.Length; i++) + { + buffers[i] = new T[inputs[i]]; + } + } + return Create(buffers); + } + } +} diff --git a/src/System.Memory/tests/SequenceReader/SequenceSegment.cs b/src/System.Memory/tests/SequenceReader/SequenceSegment.cs new file mode 100644 index 000000000000..60132e41501c --- /dev/null +++ b/src/System.Memory/tests/SequenceReader/SequenceSegment.cs @@ -0,0 +1,131 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Buffers; +using System.Diagnostics; +using System.Runtime.InteropServices; +using System.Text; + +namespace System.Memory.Tests.SequenceReader +{ + public class SequenceSegment : ReadOnlySequenceSegment where T : struct + { + private SequenceSegment _next; + + public int Start { get; private set; } + + public int End + { + get => _end; + set + { + Debug.Assert(Start - value <= AvailableMemory.Length); + + _end = value; + Memory = AvailableMemory.Slice(Start, _end - Start); + } + } + + /// + /// Reference to the next block of data when the overall "active" bytes spans multiple blocks. At the point when the block is + /// leased Next is guaranteed to be null. Start, End, and Next are used together in order to create a linked-list of discontiguous + /// working memory. The "active" memory is grown when bytes are copied in, End is increased, and Next is assigned. The "active" + /// memory is shrunk when bytes are consumed, Start is increased, and blocks are returned to the pool. + /// + public SequenceSegment NextSegment + { + get => _next; + set + { + _next = value; + Next = value; + } + } + + /// + /// The buffer being tracked if segment owns the memory + /// + private IMemoryOwner _ownedMemory; + + private int _end; + + public void SetMemory(IMemoryOwner buffer) + { + SetMemory(buffer, 0, 0); + } + + public void SetMemory(IMemoryOwner ownedMemory, int start, int end, bool readOnly = false) + { + _ownedMemory = ownedMemory; + + AvailableMemory = _ownedMemory.Memory; + + ReadOnly = readOnly; + RunningIndex = 0; + Start = start; + End = end; + NextSegment = null; + } + + public void ResetMemory() + { + _ownedMemory.Dispose(); + _ownedMemory = null; + AvailableMemory = default; + } + + public Memory AvailableMemory { get; private set; } + + public int Length => End - Start; + + /// + /// If true, data should not be written into the backing block after the End offset. Data between start and end should never be modified + /// since this would break cloning. + /// + public bool ReadOnly { get; private set; } + + /// + /// The amount of writable bytes in this segment. It is the amount of bytes between Length and End + /// + public int WritableBytes => AvailableMemory.Length - End; + + /// + /// ToString overridden for debugger convenience. This displays the "active" byte information in this block as ASCII characters. + /// + /// + public override string ToString() + { + if (Memory.IsEmpty) + { + return ""; + } + + if (typeof(T) == typeof(byte)) + { + var builder = new StringBuilder(); + SpanLiteralExtensions.AppendAsLiteral(MemoryMarshal.AsBytes(Memory.Span), builder); + return builder.ToString(); + } + + return Memory.Span.ToString(); + } + + public void SetNext(SequenceSegment segment) + { + Debug.Assert(segment != null); + Debug.Assert(Next == null); + + NextSegment = segment; + + segment = this; + + while (segment.Next != null) + { + segment.NextSegment.RunningIndex = segment.RunningIndex + segment.Length; + segment = segment.NextSegment; + } + } + } +} diff --git a/src/System.Memory/tests/SequenceReader/SkipDelimiter.cs b/src/System.Memory/tests/SequenceReader/SkipDelimiter.cs new file mode 100644 index 000000000000..ec102056f05f --- /dev/null +++ b/src/System.Memory/tests/SequenceReader/SkipDelimiter.cs @@ -0,0 +1,262 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Buffers; +using System.Text; +using Xunit; + +namespace System.Memory.Tests.SequenceReader +{ + class SkipDelimiter + { + [Fact] + public void TryReadTo_SkipDelimiter() + { + byte[] expected = Encoding.UTF8.GetBytes("This is our ^|understanding^|"); + ReadOnlySequence bytes = SequenceFactory.CreateUtf8("This is our ^|understanding^|| you see."); + SequenceReader reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out ReadOnlySpan span, (byte)'|', (byte)'^', advancePastDelimiter: true)); + Assert.Equal(expected, span.ToArray()); + Assert.True(reader.IsNext((byte)' ')); + Assert.Equal(30, reader.Consumed); + + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false)); + Assert.Equal(expected, span.ToArray()); + Assert.True(reader.IsNext((byte)'|')); + Assert.Equal(29, reader.Consumed); + + // Put the skip delimiter in another segment + bytes = SequenceFactory.CreateUtf8("This is our ^|understanding", "^|| you see."); + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: true)); + Assert.Equal(expected, span.ToArray()); + Assert.True(reader.IsNext((byte)' ')); + Assert.Equal(30, reader.Consumed); + + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false)); + Assert.Equal(expected, span.ToArray()); + Assert.True(reader.IsNext((byte)'|')); + Assert.Equal(29, reader.Consumed); + + // Put the skip delimiter at the end of the segment + bytes = SequenceFactory.CreateUtf8("This is our ^|understanding^", "|| you see."); + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: true)); + Assert.Equal(expected, span.ToArray()); + Assert.True(reader.IsNext((byte)' ')); + Assert.Equal(30, reader.Consumed); + + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false)); + Assert.Equal(expected, span.ToArray()); + Assert.True(reader.IsNext((byte)'|')); + Assert.Equal(29, reader.Consumed); + + // No trailing data + bytes = SequenceFactory.CreateUtf8("This is our ^|understanding^||"); + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false)); + Assert.Equal(expected, span.ToArray()); + Assert.True(reader.IsNext((byte)'|')); + Assert.Equal(29, reader.Consumed); + + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: true)); + Assert.Equal(expected, span.ToArray()); + Assert.True(reader.End); + Assert.Equal(30, reader.Consumed); + + // All delimiters skipped + bytes = SequenceFactory.CreateUtf8("This is our ^|understanding^|"); + reader = new SequenceReader(bytes); + Assert.False(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false)); + Assert.Equal(0, reader.Consumed); + + reader = new SequenceReader(bytes); + Assert.False(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: true)); + Assert.Equal(0, reader.Consumed); + + bytes = SequenceFactory.CreateUtf8("abc^|de|"); + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: true)); + Assert.Equal(Encoding.UTF8.GetBytes("abc^|de"), span.ToArray()); + Assert.True(reader.End); + Assert.Equal(8, reader.Consumed); + + // Escape leads + bytes = SequenceFactory.CreateUtf8("^|a|b"); + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: true)); + Assert.Equal(Encoding.UTF8.GetBytes("^|a"), span.ToArray()); + Assert.True(reader.IsNext((byte)'b')); + Assert.Equal(4, reader.Consumed); + + // Delimiter starts second segment. + bytes = SequenceFactory.CreateUtf8("^", "|a|b"); + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: true)); + Assert.Equal(Encoding.UTF8.GetBytes("^|a"), span.ToArray()); + Assert.True(reader.IsNext((byte)'b')); + Assert.Equal(4, reader.Consumed); + } + + [Fact] + public void TryReadTo_SkipDelimiter_Runs() + { + ReadOnlySequence bytes = SequenceFactory.CreateUtf8("abc^^|def"); + SequenceReader reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out ReadOnlySpan span, (byte)'|', (byte)'^', advancePastDelimiter: false)); + Assert.Equal(Encoding.UTF8.GetBytes("abc^^"), span.ToArray()); + Assert.True(reader.IsNext((byte)'|')); + Assert.Equal(5, reader.Consumed); + + // Split after escape char + bytes = SequenceFactory.CreateUtf8("abc^^", "|def"); + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false)); + Assert.Equal(Encoding.UTF8.GetBytes("abc^^"), span.ToArray()); + Assert.True(reader.IsNext((byte)'|')); + Assert.Equal(5, reader.Consumed); + + // Split before and after escape char + bytes = SequenceFactory.CreateUtf8("abc^", "^", "|def"); + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false)); + Assert.Equal(Encoding.UTF8.GetBytes("abc^^"), span.ToArray()); + Assert.True(reader.IsNext((byte)'|')); + Assert.Equal(5, reader.Consumed); + + // Check advance past delimiter + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: true)); + Assert.Equal(Encoding.UTF8.GetBytes("abc^^"), span.ToArray()); + Assert.True(reader.IsNext((byte)'d')); + Assert.Equal(6, reader.Consumed); + + // Leading run of 2 + bytes = SequenceFactory.CreateUtf8("^^|abc"); + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false)); + Assert.Equal(Encoding.UTF8.GetBytes("^^"), span.ToArray()); + Assert.True(reader.IsNext((byte)'|')); + Assert.Equal(2, reader.Consumed); + + // Leading run of 3 + bytes = SequenceFactory.CreateUtf8("^^^|abc"); + reader = new SequenceReader(bytes); + Assert.False(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false)); + Assert.True(reader.IsNext((byte)'^')); + Assert.Equal(0, reader.Consumed); + + // Trailing run of 3 + bytes = SequenceFactory.CreateUtf8("abc^^^|"); + reader = new SequenceReader(bytes); + Assert.False(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false)); + Assert.True(reader.IsNext((byte)'a')); + Assert.Equal(0, reader.Consumed); + + // Trailing run of 3, split + bytes = SequenceFactory.CreateUtf8("abc^^^", "|"); + reader = new SequenceReader(bytes); + Assert.False(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false)); + Assert.True(reader.IsNext((byte)'a')); + Assert.Equal(0, reader.Consumed); + } + + [Fact] + public void TryReadTo_SkipDelimiter_ReadOnlySequence() + { + byte[] expected = Encoding.UTF8.GetBytes("This is our ^|understanding^|"); + ReadOnlySequence bytes = SequenceFactory.CreateUtf8("This is our ^|understanding^|| you see."); + SequenceReader reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out ReadOnlySequence sequence, (byte)'|', (byte)'^', advancePastDelimiter: true)); + Assert.Equal(expected, sequence.ToArray()); + Assert.True(reader.IsNext((byte)' ')); + Assert.Equal(30, reader.Consumed); + + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out sequence, (byte)'|', (byte)'^', advancePastDelimiter: false)); + Assert.Equal(expected, sequence.ToArray()); + Assert.True(reader.IsNext((byte)'|')); + Assert.Equal(29, reader.Consumed); + + // Put the skip delimiter in another segment + bytes = SequenceFactory.CreateUtf8("This is our ^|understanding", "^|| you see."); + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out sequence, (byte)'|', (byte)'^', advancePastDelimiter: true)); + Assert.Equal(expected, sequence.ToArray()); + Assert.True(reader.IsNext((byte)' ')); + Assert.Equal(30, reader.Consumed); + + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out sequence, (byte)'|', (byte)'^', advancePastDelimiter: false)); + Assert.Equal(expected, sequence.ToArray()); + Assert.True(reader.IsNext((byte)'|')); + Assert.Equal(29, reader.Consumed); + + // Put the skip delimiter at the end of the segment + bytes = SequenceFactory.CreateUtf8("This is our ^|understanding^", "|| you see."); + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out sequence, (byte)'|', (byte)'^', advancePastDelimiter: true)); + Assert.Equal(expected, sequence.ToArray()); + Assert.True(reader.IsNext((byte)' ')); + Assert.Equal(30, reader.Consumed); + + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out sequence, (byte)'|', (byte)'^', advancePastDelimiter: false)); + Assert.Equal(expected, sequence.ToArray()); + Assert.True(reader.IsNext((byte)'|')); + Assert.Equal(29, reader.Consumed); + + // No trailing data + bytes = SequenceFactory.CreateUtf8("This is our ^|understanding^||"); + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out sequence, (byte)'|', (byte)'^', advancePastDelimiter: false)); + Assert.Equal(expected, sequence.ToArray()); + Assert.True(reader.IsNext((byte)'|')); + Assert.Equal(29, reader.Consumed); + + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out sequence, (byte)'|', (byte)'^', advancePastDelimiter: true)); + Assert.Equal(expected, sequence.ToArray()); + Assert.True(reader.End); + Assert.Equal(30, reader.Consumed); + + // All delimiters skipped + bytes = SequenceFactory.CreateUtf8("This is our ^|understanding^|"); + reader = new SequenceReader(bytes); + Assert.False(reader.TryReadTo(out sequence, (byte)'|', (byte)'^', advancePastDelimiter: false)); + Assert.Equal(0, reader.Consumed); + + reader = new SequenceReader(bytes); + Assert.False(reader.TryReadTo(out sequence, (byte)'|', (byte)'^', advancePastDelimiter: true)); + Assert.Equal(0, reader.Consumed); + + bytes = SequenceFactory.CreateUtf8("abc^|de|"); + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out sequence, (byte)'|', (byte)'^', advancePastDelimiter: true)); + Assert.Equal(Encoding.UTF8.GetBytes("abc^|de"), sequence.ToArray()); + Assert.True(reader.End); + Assert.Equal(8, reader.Consumed); + + // Escape leads + bytes = SequenceFactory.CreateUtf8("^|a|b"); + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out sequence, (byte)'|', (byte)'^', advancePastDelimiter: true)); + Assert.Equal(Encoding.UTF8.GetBytes("^|a"), sequence.ToArray()); + Assert.True(reader.IsNext((byte)'b')); + Assert.Equal(4, reader.Consumed); + + // Delimiter starts second segment. + bytes = SequenceFactory.CreateUtf8("^", "|a|b"); + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out sequence, (byte)'|', (byte)'^', advancePastDelimiter: true)); + Assert.Equal(Encoding.UTF8.GetBytes("^|a"), sequence.ToArray()); + Assert.True(reader.IsNext((byte)'b')); + Assert.Equal(4, reader.Consumed); + } + } +} diff --git a/src/System.Memory/tests/SequenceReader/SpanLiteralExtensions.cs b/src/System.Memory/tests/SequenceReader/SpanLiteralExtensions.cs new file mode 100644 index 000000000000..5f750f986ffe --- /dev/null +++ b/src/System.Memory/tests/SequenceReader/SpanLiteralExtensions.cs @@ -0,0 +1,72 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Text; + +namespace System.Memory.Tests.SequenceReader +{ + public class SpanLiteralExtensions + { + internal static void AppendAsLiteral(ReadOnlySpan span, StringBuilder sb) + { + for (int i = 0; i < span.Length; i++) + { + AppendCharLiteral((char)span[i], sb); + } + } + + internal static void AppendCharLiteral(char c, StringBuilder sb) + { + switch (c) + { + case '\'': + sb.Append(@"\'"); + break; + case '\"': + sb.Append("\\\""); + break; + case '\\': + sb.Append(@"\\"); + break; + case '\0': + sb.Append(@"\0"); + break; + case '\a': + sb.Append(@"\a"); + break; + case '\b': + sb.Append(@"\b"); + break; + case '\f': + sb.Append(@"\f"); + break; + case '\n': + sb.Append(@"\n"); + break; + case '\r': + sb.Append(@"\r"); + break; + case '\t': + sb.Append(@"\t"); + break; + case '\v': + sb.Append(@"\v"); + break; + default: + // ASCII printable character + if (!char.IsControl(c)) + { + sb.Append(c); + // As UTF16 escaped character + } + else + { + sb.Append(@"\u"); + sb.Append(((int)c).ToString("x4")); + } + break; + } + } + } +} diff --git a/src/System.Memory/tests/System.Memory.Tests.csproj b/src/System.Memory/tests/System.Memory.Tests.csproj index a8753e5a5cbd..8c03849c4e1d 100644 --- a/src/System.Memory/tests/System.Memory.Tests.csproj +++ b/src/System.Memory/tests/System.Memory.Tests.csproj @@ -7,7 +7,7 @@ true - + @@ -15,6 +15,17 @@ + + + + + + + + + + +