From 82930ec443d7bd3ce0053f7daa309e3742580646 Mon Sep 17 00:00:00 2001 From: Jeremy Kuhne Date: Tue, 6 Nov 2018 17:43:31 -0800 Subject: [PATCH 01/10] Add SequenceReader Initial add of ReadOnlySequence reader. Includes search and binary read functionality. Perf tests will be ported from CoreFXlab later. Parse functionality still being discussed. --- .../packageIndex.json | 4 +- src/System.Memory/Directory.Build.props | 2 +- src/System.Memory/ref/System.Memory.cs | 46 + src/System.Memory/src/System.Memory.csproj | 3 + .../Buffers/ReadOnlySequence_helpers.cs | 76 ++ .../src/System/Buffers/SequenceReader.cs | 373 ++++++++ .../SequenceReaderExtensions_binary.cs | 170 ++++ .../System/Buffers/SequenceReader_search.cs | 798 ++++++++++++++++++ src/System.Memory/src/System/ThrowHelper.cs | 3 +- .../tests/SequenceReader/Advance.cs | 61 ++ .../tests/SequenceReader/BasicTests.cs | 568 +++++++++++++ .../tests/SequenceReader/BinaryExtensions.cs | 69 ++ .../tests/SequenceReader/IsNext.cs | 46 + .../tests/SequenceReader/OwnedArray.cs | 103 +++ .../tests/SequenceReader/ReadTo.cs | 177 ++++ .../tests/SequenceReader/Rewind.cs | 71 ++ .../tests/SequenceReader/SequenceFactory.cs | 119 +++ .../tests/SequenceReader/SequenceSegment.cs | 131 +++ .../tests/SequenceReader/SkipDelimiter.cs | 262 ++++++ .../SequenceReader/SpanLiteralExtensions.cs | 72 ++ .../tests/System.Memory.Tests.csproj | 11 + 21 files changed, 3162 insertions(+), 3 deletions(-) create mode 100644 src/System.Memory/src/System/Buffers/SequenceReader.cs create mode 100644 src/System.Memory/src/System/Buffers/SequenceReaderExtensions_binary.cs create mode 100644 src/System.Memory/src/System/Buffers/SequenceReader_search.cs create mode 100644 src/System.Memory/tests/SequenceReader/Advance.cs create mode 100644 src/System.Memory/tests/SequenceReader/BasicTests.cs create mode 100644 src/System.Memory/tests/SequenceReader/BinaryExtensions.cs create mode 100644 src/System.Memory/tests/SequenceReader/IsNext.cs create mode 100644 src/System.Memory/tests/SequenceReader/OwnedArray.cs create mode 100644 src/System.Memory/tests/SequenceReader/ReadTo.cs create mode 100644 src/System.Memory/tests/SequenceReader/Rewind.cs create mode 100644 src/System.Memory/tests/SequenceReader/SequenceFactory.cs create mode 100644 src/System.Memory/tests/SequenceReader/SequenceSegment.cs create mode 100644 src/System.Memory/tests/SequenceReader/SkipDelimiter.cs create mode 100644 src/System.Memory/tests/SequenceReader/SpanLiteralExtensions.cs diff --git a/pkg/Microsoft.Private.PackageBaseline/packageIndex.json b/pkg/Microsoft.Private.PackageBaseline/packageIndex.json index 2bfa5afbaa55..c023d5e2cf42 100644 --- a/pkg/Microsoft.Private.PackageBaseline/packageIndex.json +++ b/pkg/Microsoft.Private.PackageBaseline/packageIndex.json @@ -2543,6 +2543,7 @@ ], "InboxOn": { "netcoreapp2.1": "4.1.0.0", + "netcoreapp3.0": "4.2.0.0", "monoandroid10": "Any", "monotouch10": "Any", "uap10.0.16300": "4.1.0.0", @@ -2553,7 +2554,8 @@ }, "AssemblyVersionInPackageVersion": { "4.0.1.0": "4.5.0", - "4.1.0.0": "4.6.0" + "4.1.0.0": "4.6.0", + "4.2.0.0": "4.7.0" } }, "System.Messaging": { diff --git a/src/System.Memory/Directory.Build.props b/src/System.Memory/Directory.Build.props index d056cea58f58..1627cb108724 100644 --- a/src/System.Memory/Directory.Build.props +++ b/src/System.Memory/Directory.Build.props @@ -4,7 +4,7 @@ - 4.1.0.0 + 4.2.0.0 Open true true diff --git a/src/System.Memory/ref/System.Memory.cs b/src/System.Memory/ref/System.Memory.cs index 1b5df25b35f3..794a01bf8c2e 100644 --- a/src/System.Memory/ref/System.Memory.cs +++ b/src/System.Memory/ref/System.Memory.cs @@ -184,6 +184,52 @@ public partial struct Enumerator public bool MoveNext() { throw null; } } } + public ref partial struct SequenceReader where T : unmanaged, System.IEquatable + { + public SequenceReader(System.Buffers.ReadOnlySequence buffer) { } + public bool IsLastSegment { get { throw null; } } + public bool End { get { throw null; } } + public ReadOnlySequence Sequence { get { throw null; } } + public SequencePosition Position { get { throw null; } } + public ReadOnlySpan CurrentSpan { get { throw null; } } + public int CurrentSpanIndex { get { throw null; } } + public ReadOnlySpan UnreadSpan { get { throw null; } } + public long Consumed { get { throw null; } } + public long Remaining { get { throw null; } } + public long Length { get { throw null; } } + public ReadOnlySpan Peek(Span copyBuffer) { throw null; } + public bool TryPeek(out T value) { throw null; } + public void Advance(long count) { throw null; } + public void Rewind(long count) { throw null; } + public bool TryRead(out T value) { throw null; } + public bool TryReadTo(out ReadOnlySpan span, T delimiter) { throw null; } + public bool TryReadTo(out ReadOnlySpan span, T delimiter, bool advancePastDelimiter) { throw null; } + public bool TryReadTo(out ReadOnlySpan span, T delimiter, T delimiterEscape, bool advancePastDelimiter = true) { throw null; } + public bool TryReadTo(out ReadOnlySequence sequence, T delimiter, bool advancePastDelimiter = true) { throw null; } + public bool TryReadTo(out ReadOnlySequence sequence, T delimiter, T delimiterEscape, bool advancePastDelimiter = true) { throw null; } + public bool TryReadToAny(out ReadOnlySpan span, ReadOnlySpan delimiters, bool advancePastDelimiter = true) { throw null; } + public bool TryReadToAny(out ReadOnlySequence sequence, ReadOnlySpan delimiters, bool advancePastDelimiter = true) { throw null; } + public bool TryReadTo(out ReadOnlySequence sequence, ReadOnlySpan delimiter, bool advancePastDelimiter = true) { throw null; } + public bool TryAdvanceTo(T delimiter, bool advancePastDelimiter = true) { throw null; } + public bool TryAdvanceToAny(ReadOnlySpan delimiters, bool advancePastDelimiter = true) { throw null; } + public long AdvancePast(T value) { throw null; } + public long AdvancePastAny(T value0, T value1) { throw null; } + public long AdvancePastAny(T value0, T value1, T value2) { throw null; } + public long AdvancePastAny(T value0, T value1, T value2, T value3) { throw null; } + public long AdvancePastAny(ReadOnlySpan values) { throw null; } + public bool IsNext(T next, bool advancePast = false) { throw null; } + public bool IsNext(ReadOnlySpan next, bool advancePast = false) { throw null; } + } + public static partial class SequenceReaderExtensions + { + public static bool TryRead(ref this SequenceReader reader, out T value) where T : unmanaged { throw null; } + public static bool TryReadInt16LittleEndian(ref this SequenceReader reader, out short value) { throw null; } + public static bool TryReadInt16BigEndian(ref this SequenceReader reader, out short value) { throw null; } + public static bool TryReadInt32LittleEndian(ref this SequenceReader reader, out int value) { throw null; } + public static bool TryReadInt32BigEndian(ref this SequenceReader reader, out int value) { throw null; } + public static bool TryReadInt64LittleEndian(ref this SequenceReader reader, out long value) { throw null; } + public static bool TryReadInt64BigEndian(ref this SequenceReader reader, out long value) { throw null; } + } public readonly partial struct StandardFormat : System.IEquatable { private readonly int _dummyPrimitive; diff --git a/src/System.Memory/src/System.Memory.csproj b/src/System.Memory/src/System.Memory.csproj index 1aab519e55c7..baf32b646edd 100644 --- a/src/System.Memory/src/System.Memory.csproj +++ b/src/System.Memory/src/System.Memory.csproj @@ -8,6 +8,9 @@ netcoreapp-Unix-Debug;netcoreapp-Unix-Release;netcoreapp-Windows_NT-Debug;netcoreapp-Windows_NT-Release;netcoreappaot-Windows_NT-Debug;netcoreappaot-Windows_NT-Release;uap-Windows_NT-Debug;uap-Windows_NT-Release + + + diff --git a/src/System.Memory/src/System/Buffers/ReadOnlySequence_helpers.cs b/src/System.Memory/src/System/Buffers/ReadOnlySequence_helpers.cs index 2264fc29ab70..8855788466ed 100644 --- a/src/System.Memory/src/System/Buffers/ReadOnlySequence_helpers.cs +++ b/src/System.Memory/src/System/Buffers/ReadOnlySequence_helpers.cs @@ -467,5 +467,81 @@ private static bool InRange(ulong value, ulong start, ulong end) // Equivalent to: return (start <= value && value <= start) return (value - start) <= (end - start); } + + /// + /// Helper to efficiently prepare the + /// + /// The first span in the sequence. + /// The next position. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void GetFirstSpan(out ReadOnlySpan first, out SequencePosition next) + { + first = default; + next = default; + SequencePosition start = Start; + int startIndex = start.GetInteger(); + object startObject = start.GetObject(); + + if (startObject != null) + { + SequencePosition end = End; + int endIndex = end.GetInteger(); + bool isMultiSegment = startObject != end.GetObject(); + + // A == 0 && B == 0 means SequenceType.MultiSegment + if (startIndex >= 0) + { + if (endIndex >= 0) // SequenceType.MultiSegment + { + ReadOnlySequenceSegment segment = (ReadOnlySequenceSegment)startObject; + next = new SequencePosition(segment.Next, 0); + first = segment.Memory.Span; + if (isMultiSegment) + { + first = first.Slice(startIndex); + } + else + { + first = first.Slice(startIndex, endIndex - startIndex); + } + } + else + { + if (isMultiSegment) + ThrowHelper.ThrowInvalidOperationException_EndPositionNotReached(); + + first = new ReadOnlySpan((T[])startObject, startIndex, (endIndex & ReadOnlySequence.IndexBitMask) - startIndex); + } + } + else + { + first = GetFirstSpanSlow(startObject, startIndex, endIndex, isMultiSegment); + } + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ReadOnlySpan GetFirstSpanSlow(object startObject, int startIndex, int endIndex, bool isMultiSegment) + { + Debug.Assert(startIndex < 0 || endIndex < 0); + if (isMultiSegment) + ThrowHelper.ThrowInvalidOperationException_EndPositionNotReached(); + + // The type == char check here is redundant. However, we still have it to allow + // the JIT to see when that the code is unreachable and eliminate it. + // A == 1 && B == 1 means SequenceType.String + if (typeof(T) == typeof(char) && endIndex < 0) + { + var memory = (ReadOnlyMemory)(object)((string)startObject).AsMemory(); + + // No need to remove the FlagBitMask since (endIndex - startIndex) == (endIndex & ReadOnlySequence.IndexBitMask) - (startIndex & ReadOnlySequence.IndexBitMask) + return memory.Span.Slice(startIndex & ReadOnlySequence.IndexBitMask, endIndex - startIndex); + } + else // endIndex >= 0, A == 1 && B == 0 means SequenceType.MemoryManager + { + startIndex &= ReadOnlySequence.IndexBitMask; + return ((MemoryManager)startObject).Memory.Span.Slice(startIndex, endIndex - startIndex); + } + } } } diff --git a/src/System.Memory/src/System/Buffers/SequenceReader.cs b/src/System.Memory/src/System/Buffers/SequenceReader.cs new file mode 100644 index 000000000000..565ed3e22b1c --- /dev/null +++ b/src/System.Memory/src/System/Buffers/SequenceReader.cs @@ -0,0 +1,373 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Diagnostics; +using System.Runtime.CompilerServices; + +namespace System.Buffers +{ + public ref partial struct SequenceReader where T : unmanaged, IEquatable + { + private SequencePosition _currentPosition; + private SequencePosition _nextPosition; + private bool _moreData; + private long _length; + + /// + /// Create a over the given . + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public SequenceReader(ReadOnlySequence buffer) + { + CurrentSpanIndex = 0; + Consumed = 0; + Sequence = buffer; + _currentPosition = buffer.Start; + _length = -1; + + buffer.GetFirstSpan(out ReadOnlySpan first, out _nextPosition); + CurrentSpan = first; + _moreData = first.Length > 0; + + if (!_moreData && !buffer.IsSingleSegment) + { + _moreData = true; + GetNextSpan(); + } + } + + /// + /// Return true if we're in the last segment + /// + public bool IsLastSegment => _nextPosition.GetObject() == null; + + /// + /// True when there is no more data in the . + /// + public bool End => !_moreData; + + /// + /// The underlying for the reader. + /// + public ReadOnlySequence Sequence { get; } + + /// + /// The current position in the . + /// + public SequencePosition Position + => Sequence.GetPosition(CurrentSpanIndex, _currentPosition); + + /// + /// The current segment in the . + /// + public ReadOnlySpan CurrentSpan { get; private set; } + + /// + /// The index in the . + /// + public int CurrentSpanIndex { get; private set; } + + /// + /// The unread portion of the . + /// + public ReadOnlySpan UnreadSpan + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => CurrentSpan.Slice(CurrentSpanIndex); + } + + /// + /// The total number of 's processed by the reader. + /// + public long Consumed { get; private set; } + + /// + /// Remaining 's in the reader's . + /// + public long Remaining => Length - Consumed; + + /// + /// Count of in the reader's . + /// + public long Length + { + get + { + if (_length < 0) + { + _length = Sequence.Length; + } + return _length; + } + } + + /// + /// Peeks at the next value without advancing the reader. + /// + /// The next value or default if at the end. + /// False if at the end of the reader. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool TryPeek(out T value) + { + if (_moreData) + { + value = CurrentSpan[CurrentSpanIndex]; + return true; + } + else + { + value = default; + return false; + } + } + + /// + /// Read the next value and advance the reader. + /// + /// The next value or default if at the end. + /// False if at the end of the reader. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool TryRead(out T value) + { + if (End) + { + value = default; + return false; + } + + value = CurrentSpan[CurrentSpanIndex]; + CurrentSpanIndex++; + Consumed++; + + if (CurrentSpanIndex >= CurrentSpan.Length) + { + GetNextSpan(); + } + + return true; + } + + /// + /// Move the reader back the specified number of positions. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Rewind(long count) + { + if (count < 0) + { + ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.count); + } + + Consumed -= count; + + if (CurrentSpanIndex >= count) + { + CurrentSpanIndex -= (int)count; + _moreData = true; + } + else + { + // Current segment doesn't have enough space, scan backward through segments + RetreatToPreviousSpan(Consumed); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private void RetreatToPreviousSpan(long consumed) + { + ResetReader(); + Advance(consumed); + } + + private void ResetReader() + { + CurrentSpanIndex = 0; + Consumed = 0; + _currentPosition = Sequence.Start; + _nextPosition = _currentPosition; + + if (Sequence.TryGet(ref _nextPosition, out ReadOnlyMemory memory, advance: true)) + { + _moreData = true; + + if (memory.Length == 0) + { + CurrentSpan = default; + // No space in the first span, move to one with space + GetNextSpan(); + } + else + { + CurrentSpan = memory.Span; + } + } + else + { + // No space in any spans and at end of sequence + _moreData = false; + CurrentSpan = default; + } + } + + /// + /// Get the next segment with available space, if any. + /// + private void GetNextSpan() + { + if (!Sequence.IsSingleSegment) + { + SequencePosition previousNextPosition = _nextPosition; + while (Sequence.TryGet(ref _nextPosition, out ReadOnlyMemory memory, advance: true)) + { + _currentPosition = previousNextPosition; + if (memory.Length > 0) + { + CurrentSpan = memory.Span; + CurrentSpanIndex = 0; + return; + } + else + { + CurrentSpan = default; + CurrentSpanIndex = 0; + previousNextPosition = _nextPosition; + } + } + } + _moreData = false; + } + + /// + /// Move the reader ahead the specified number of positions. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Advance(long count) + { + const long TooBigOrNegative = unchecked((long)0xFFFFFFFF80000000); + if ((count & TooBigOrNegative) == 0 && CurrentSpan.Length - CurrentSpanIndex > (int)count) + { + CurrentSpanIndex += (int)count; + Consumed += count; + } + else + { + // Can't satisfy from the current span + AdvanceToNextSpan(count); + } + } + + /// + /// Unchecked helper to avoid unnecessary checks where you know count is valid. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void AdvanceCurrentSpan(long count) + { + Debug.Assert(count >= 0); + + Consumed += count; + CurrentSpanIndex += (int)count; + if (CurrentSpanIndex >= CurrentSpan.Length) + GetNextSpan(); + } + + /// + /// Only call this helper if you know that you are advancing in the current span + /// with valid count and there is no need to fetch the next one. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void AdvanceWithinSpan(long count) + { + Debug.Assert(count >= 0); + + Consumed += count; + CurrentSpanIndex += (int)count; + + Debug.Assert(CurrentSpanIndex < CurrentSpan.Length); + } + + private void AdvanceToNextSpan(long count) + { + if (count < 0) + { + ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.count); + } + + Consumed += count; + while (_moreData) + { + int remaining = CurrentSpan.Length - CurrentSpanIndex; + + if (remaining > count) + { + CurrentSpanIndex += (int)count; + count = 0; + break; + } + + CurrentSpanIndex += remaining; + count -= remaining; + Debug.Assert(count >= 0); + + GetNextSpan(); + + if (count == 0) + break; + } + + if (count != 0) + { + // Not enough space left- adjust for where we actually ended and throw + Consumed -= count; + ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.count); + } + } + + /// + /// Peek forward the number of positions in (at most), copying into + /// if needed. + /// + /// + /// Temporary buffer to copy into if there isn't a contiguous span within the existing data to return. + /// Also describes the maximum count of positions to peek. + /// + /// + /// Span over the peeked data. The length may be shorter than if there + /// is not enough data left to fill the requested length. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public ReadOnlySpan Peek(Span copyBuffer) + { + ReadOnlySpan firstSpan = UnreadSpan; + if (firstSpan.Length >= copyBuffer.Length) + { + return firstSpan.Slice(0, copyBuffer.Length); + } + + return PeekSlow(copyBuffer); + } + + internal ReadOnlySpan PeekSlow(Span destination) + { + ReadOnlySpan firstSpan = UnreadSpan; + firstSpan.CopyTo(destination); + int copied = firstSpan.Length; + + SequencePosition next = _nextPosition; + while (Sequence.TryGet(ref next, out ReadOnlyMemory nextSegment, true)) + { + ReadOnlySpan nextSpan = nextSegment.Span; + if (nextSpan.Length > 0) + { + int toCopy = Math.Min(nextSpan.Length, destination.Length - copied); + nextSpan.Slice(0, toCopy).CopyTo(destination.Slice(copied)); + copied += toCopy; + if (copied >= destination.Length) + break; + } + } + + return destination.Slice(0, copied); + } + } +} diff --git a/src/System.Memory/src/System/Buffers/SequenceReaderExtensions_binary.cs b/src/System.Memory/src/System/Buffers/SequenceReaderExtensions_binary.cs new file mode 100644 index 000000000000..f53132d4f78c --- /dev/null +++ b/src/System.Memory/src/System/Buffers/SequenceReaderExtensions_binary.cs @@ -0,0 +1,170 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Buffers.Binary; +using System.Diagnostics; +using System.Runtime.InteropServices; +using Internal.Runtime.CompilerServices; + +namespace System.Buffers +{ + public static partial class SequenceReaderExtensions + { + /// + /// Try to read the given type out of the buffer if possible. + /// + /// + /// The read is unaligned. + /// + /// + /// True if successful. will be default if failed. + /// + public static unsafe bool TryRead(ref this SequenceReader reader, out T value) where T : unmanaged + { + ReadOnlySpan span = reader.UnreadSpan; + if (span.Length < sizeof(T)) + return TryReadSlow(ref reader, out value); + + value = Unsafe.ReadUnaligned(ref MemoryMarshal.GetReference(span)); + reader.Advance(sizeof(T)); + return true; + } + + private static unsafe bool TryReadSlow(ref SequenceReader reader, out T value) where T : unmanaged + { + Debug.Assert(reader.UnreadSpan.Length < sizeof(T)); + + // Not enough data in the current segment, try to peek for the data we need. + byte* buffer = stackalloc byte[sizeof(T)]; + Span tempSpan = new Span(buffer, sizeof(T)); + + if (reader.Peek(tempSpan).Length < sizeof(T)) + { + value = default; + return false; + } + + value = Unsafe.ReadUnaligned(ref MemoryMarshal.GetReference(tempSpan)); + reader.Advance(sizeof(T)); + return true; + } + + /// + /// Reads an as little endian. + /// + /// False if there wasn't enough data for an . + public static bool TryReadInt16LittleEndian(ref this SequenceReader reader, out short value) + { + if (BitConverter.IsLittleEndian) + { + return reader.TryRead(out value); + } + + return TryReadReverseEndianness(ref reader, out value); + } + + /// + /// Reads an as big endian. + /// + /// False if there wasn't enough data for an . + public static bool TryReadInt16BigEndian(ref this SequenceReader reader, out short value) + { + if (!BitConverter.IsLittleEndian) + { + return reader.TryRead(out value); + } + + return TryReadReverseEndianness(ref reader, out value); + } + + private static bool TryReadReverseEndianness(ref SequenceReader reader, out short value) + { + if (reader.TryRead(out value)) + { + value = BinaryPrimitives.ReverseEndianness(value); + return true; + } + + return false; + } + + /// + /// Reads an as little endian. + /// + /// False if there wasn't enough data for an . + public static bool TryReadInt32LittleEndian(ref this SequenceReader reader, out int value) + { + if (BitConverter.IsLittleEndian) + { + return reader.TryRead(out value); + } + + return TryReadReverseEndianness(ref reader, out value); + } + + /// + /// Reads an as big endian. + /// + /// False if there wasn't enough data for an . + public static bool TryReadInt32BigEndian(ref this SequenceReader reader, out int value) + { + if (!BitConverter.IsLittleEndian) + { + return reader.TryRead(out value); + } + + return TryReadReverseEndianness(ref reader, out value); + } + + private static bool TryReadReverseEndianness(ref SequenceReader reader, out int value) + { + if (reader.TryRead(out value)) + { + value = BinaryPrimitives.ReverseEndianness(value); + return true; + } + + return false; + } + + /// + /// Reads an as little endian. + /// + /// False if there wasn't enough data for an . + public static bool TryReadInt64LittleEndian(ref this SequenceReader reader, out long value) + { + if (BitConverter.IsLittleEndian) + { + return reader.TryRead(out value); + } + + return TryReadReverseEndianness(ref reader, out value); + } + + /// + /// Reads an as big endian. + /// + /// False if there wasn't enough data for an . + public static bool TryReadInt64BigEndian(ref this SequenceReader reader, out long value) + { + if (!BitConverter.IsLittleEndian) + { + return reader.TryRead(out value); + } + + return TryReadReverseEndianness(ref reader, out value); + } + + private static bool TryReadReverseEndianness(ref SequenceReader reader, out long value) + { + if (reader.TryRead(out value)) + { + value = BinaryPrimitives.ReverseEndianness(value); + return true; + } + + return false; + } + } +} diff --git a/src/System.Memory/src/System/Buffers/SequenceReader_search.cs b/src/System.Memory/src/System/Buffers/SequenceReader_search.cs new file mode 100644 index 000000000000..7a265473e444 --- /dev/null +++ b/src/System.Memory/src/System/Buffers/SequenceReader_search.cs @@ -0,0 +1,798 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Diagnostics; +using System.Runtime.CompilerServices; + +namespace System.Buffers +{ + public ref partial struct SequenceReader where T : unmanaged, IEquatable + { + /// + /// Try to read everything up to the given . The reader will be positioned + /// after the first occurence of when successful. + /// + /// The read data, if any. + /// The delimiter to look for. + /// True if the was found. + public bool TryReadTo(out ReadOnlySpan span, T delimiter) + { + ReadOnlySpan remaining = UnreadSpan; + int index = remaining.IndexOf(delimiter); + + if (index != -1) + { + span = remaining.Slice(0, index); + AdvanceCurrentSpan(index + 1); + return true; + } + + return TryReadToMultisegment(out span, delimiter); + } + + private bool TryReadToMultisegment(out ReadOnlySpan span, T delimiter) + { + return TryReadToSlow(out span, delimiter, advancePastDelimiter: true); + } + + public bool TryReadTo(out ReadOnlySpan span, T delimiter, bool advancePastDelimiter) + { + ReadOnlySpan remaining = UnreadSpan; + int index = remaining.IndexOf(delimiter); + + if (index != -1) + { + span = index == 0 ? default : remaining.Slice(0, index); + AdvanceCurrentSpan(index + (advancePastDelimiter ? 1 : 0)); + return true; + } + + return TryReadToSlow(out span, delimiter, advancePastDelimiter); + } + + private bool TryReadToSlow(out ReadOnlySpan span, T delimiter, bool advancePastDelimiter) + { + if (!TryReadToInternal(out ReadOnlySequence sequence, delimiter, advancePastDelimiter, CurrentSpan.Length - CurrentSpanIndex)) + { + span = default; + return false; + } + + span = sequence.IsSingleSegment ? sequence.First.Span : sequence.ToArray(); + return true; + } + + /// + /// Try to read everything up to the given , ignoring delimiters that are + /// preceeded by . + /// + /// The read data, if any. + /// The delimiter to look for. + /// If found prior to it will skip that occurence. + /// True to move past the if found. + /// True if the was found. + public bool TryReadTo(out ReadOnlySpan span, T delimiter, T delimiterEscape, bool advancePastDelimiter = true) + { + ReadOnlySpan remaining = UnreadSpan; + int index = remaining.IndexOf(delimiter); + + if ((index > 0 && !remaining[index - 1].Equals(delimiterEscape)) || index == 0) + { + span = remaining.Slice(0, index); + Advance(index + (advancePastDelimiter ? 1 : 0)); + return true; + } + + // This delimiter might be skipped, go down the slow path + return TryReadToSlow(out span, delimiter, delimiterEscape, index, advancePastDelimiter); + } + + private bool TryReadToSlow(out ReadOnlySpan span, T delimiter, T delimiterEscape, int index, bool advancePastDelimiter) + { + if (!TryReadToSlow(out ReadOnlySequence sequence, delimiter, delimiterEscape, index, advancePastDelimiter)) + { + span = default; + return false; + } + + Debug.Assert(sequence.Length > 0); + span = sequence.IsSingleSegment ? sequence.First.Span : sequence.ToArray(); + return true; + } + + private bool TryReadToSlow(out ReadOnlySequence sequence, T delimiter, T delimiterEscape, int index, bool advancePastDelimiter) + { + SequenceReader copy = this; + + ReadOnlySpan remaining = UnreadSpan; + bool priorEscape = false; + + do + { + if (index >= 0) + { + if (index == 0 && priorEscape) + { + // We were in the escaped state, so skip this delimiter + priorEscape = false; + Advance(index + 1); + remaining = UnreadSpan; + goto Continue; + } + else if (index > 0 && remaining[index - 1].Equals(delimiterEscape)) + { + // This delimiter might be skipped + + // Count our escapes + int escapeCount = 1; + int i = index - 2; + for (; i >= 0; i--) + { + if (!remaining[i].Equals(delimiterEscape)) + break; + } + if (i < 0 && priorEscape) + { + // Started and ended with escape, increment once more + escapeCount++; + } + escapeCount += index - 2 - i; + + if ((escapeCount & 1) != 0) + { + priorEscape = false; + // We're in the escaped state, so skip this delimiter + Advance(index + 1); + remaining = UnreadSpan; + goto Continue; + } + } + + // Found the delimiter. Move to it, slice, then move past it. + Advance(index); + + sequence = Sequence.Slice(copy.Position, Position); + if (advancePastDelimiter) + { + Advance(1); + } + return true; + } + else + { + // No delimiter, need to check the end of the span for odd number of escapes then advance + if (remaining.Length > 0 && remaining[remaining.Length - 1].Equals(delimiterEscape)) + { + int escapeCount = 1; + int i = remaining.Length - 2; + for (; i >= 0; i--) + { + if (!remaining[i].Equals(delimiterEscape)) + break; + } + + escapeCount += remaining.Length - 2 - i; + if (i < 0 && priorEscape) + priorEscape = (escapeCount & 1) == 0; // equivalent to incrementing escapeCount before setting priorEscape + else + priorEscape = (escapeCount & 1) != 0; + } + else + { + priorEscape = false; + } + } + + // Nothing in the current span, move to the end, checking for the skip delimiter + Advance(remaining.Length); + remaining = CurrentSpan; + + Continue: + index = remaining.IndexOf(delimiter); + } while (!End); + + // Didn't find anything, reset our original state. + this = copy; + sequence = default; + return false; + } + + /// + /// Try to read everything up to the given . + /// + /// The read data, if any. + /// The delimiter to look for. + /// True to move past the if found. + /// True if the was found. + public bool TryReadTo(out ReadOnlySequence sequence, T delimiter, bool advancePastDelimiter = true) + { + return TryReadToInternal(out sequence, delimiter, advancePastDelimiter); + } + + private bool TryReadToInternal(out ReadOnlySequence sequence, T delimiter, bool advancePastDelimiter, int skip = 0) + { + SequenceReader copy = this; + if (skip > 0) + Advance(skip); + ReadOnlySpan remaining = UnreadSpan; + + while (!End) + { + int index = remaining.IndexOf(delimiter); + if (index != -1) + { + // Found the delimiter. Move to it, slice, then move past it. + if (index > 0) + { + Advance(index); + } + + sequence = Sequence.Slice(copy.Position, Position); + if (advancePastDelimiter) + { + Advance(1); + } + return true; + } + + Advance(remaining.Length); + remaining = CurrentSpan; + } + + // Didn't find anything, reset our original state. + this = copy; + sequence = default; + return false; + } + + /// + /// Try to read everything up to the given , ignoring delimiters that are + /// preceeded by . + /// + /// The read data, if any. + /// The delimiter to look for. + /// If found prior to it will skip that occurence. + /// True to move past the if found. + /// True if the was found. + public bool TryReadTo(out ReadOnlySequence sequence, T delimiter, T delimiterEscape, bool advancePastDelimiter = true) + { + SequenceReader copy = this; + + ReadOnlySpan remaining = UnreadSpan; + bool priorEscape = false; + + while (!End) + { + int index = remaining.IndexOf(delimiter); + if (index != -1) + { + if (index == 0 && priorEscape) + { + // We were in the escaped state, so skip this delimiter + priorEscape = false; + Advance(index + 1); + remaining = UnreadSpan; + continue; + } + else if (index > 0 && remaining[index - 1].Equals(delimiterEscape)) + { + // This delimiter might be skipped + + // Count our escapes + int escapeCount = 0; + for (int i = index; i > 0 && remaining[i - 1].Equals(delimiterEscape); i--, escapeCount++) + ; + if (escapeCount == index && priorEscape) + { + // Started and ended with escape, increment once more + escapeCount++; + } + + priorEscape = false; + if (escapeCount % 2 != 0) + { + // We're in the escaped state, so skip this delimiter + Advance(index + 1); + remaining = UnreadSpan; + continue; + } + } + + // Found the delimiter. Move to it, slice, then move past it. + if (index > 0) + { + Advance(index); + } + + sequence = Sequence.Slice(copy.Position, Position); + if (advancePastDelimiter) + { + Advance(1); + } + return true; + } + + // No delimiter, need to check the end of the span for odd number of escapes then advance + { + int escapeCount = 0; + for (int i = remaining.Length; i > 0 && remaining[i - 1].Equals(delimiterEscape); i--, escapeCount++) + ; + if (priorEscape && escapeCount == remaining.Length) + { + escapeCount++; + } + priorEscape = escapeCount % 2 != 0; + } + + // Nothing in the current span, move to the end, checking for the skip delimiter + Advance(remaining.Length); + remaining = CurrentSpan; + } + + // Didn't find anything, reset our original state. + this = copy; + sequence = default; + return false; + } + + /// + /// Try to read everything up to the given . + /// + /// The read data, if any. + /// The delimiters to look for. + /// True to move past the first found instance of any of the given . + /// True if any of the the were found. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool TryReadToAny(out ReadOnlySpan span, ReadOnlySpan delimiters, bool advancePastDelimiter = true) + { + ReadOnlySpan remaining = UnreadSpan; + int index = delimiters.Length == 2 + ? remaining.IndexOfAny(delimiters[0], delimiters[1]) + : remaining.IndexOfAny(delimiters); + + if (index != -1) + { + span = remaining.Slice(0, index); + Advance(index + (advancePastDelimiter ? 1 : 0)); + return true; + } + + return TryReadToAnySlow(out span, delimiters, advancePastDelimiter); + } + + private bool TryReadToAnySlow(out ReadOnlySpan span, ReadOnlySpan delimiters, bool advancePastDelimiter) + { + if (!TryReadToAnyInternal(out ReadOnlySequence sequence, delimiters, advancePastDelimiter, CurrentSpan.Length - CurrentSpanIndex)) + { + span = default; + return false; + } + + span = sequence.IsSingleSegment ? sequence.First.Span : sequence.ToArray(); + return true; + } + + /// + /// Try to read everything up to the given . + /// + /// The read data, if any. + /// The delimiters to look for. + /// True to move past the first found instance of any of the given . + /// True if any of the the were found. + public bool TryReadToAny(out ReadOnlySequence sequence, ReadOnlySpan delimiters, bool advancePastDelimiter = true) + { + return TryReadToAnyInternal(out sequence, delimiters, advancePastDelimiter); + } + + private bool TryReadToAnyInternal(out ReadOnlySequence sequence, ReadOnlySpan delimiters, bool advancePastDelimiter, int skip = 0) + { + SequenceReader copy = this; + if (skip > 0) + Advance(skip); + ReadOnlySpan remaining = CurrentSpanIndex == 0 ? CurrentSpan : UnreadSpan; + + while (!End) + { + int index = delimiters.Length == 2 + ? remaining.IndexOfAny(delimiters[0], delimiters[1]) + : remaining.IndexOfAny(delimiters); + + if (index != -1) + { + // Found one of the delimiters. Move to it, slice, then move past it. + if (index > 0) + { + Advance(index); + } + + sequence = Sequence.Slice(copy.Position, Position); + if (advancePastDelimiter) + { + Advance(1); + } + return true; + } + + Advance(remaining.Length); + remaining = CurrentSpan; + } + + // Didn't find anything, reset our original state. + this = copy; + sequence = default; + return false; + } + + /// + /// Try to read data until the given sequence. + /// + /// The read data, if any. + /// The multi (T) delimiter. + /// True to move past the sequence if found. + /// True if the was found. + public unsafe bool TryReadTo(out ReadOnlySequence sequence, ReadOnlySpan delimiter, bool advancePastDelimiter = true) + { + if (delimiter.Length == 0) + { + sequence = default; + return true; + } + + SequenceReader copy = this; + + Span peekBuffer; + if (delimiter.Length * sizeof(T) < 512) + { + T* t = stackalloc T[delimiter.Length]; + peekBuffer = new Span(t, delimiter.Length); + } + else + { + peekBuffer = new Span(new T[delimiter.Length]); + } + + bool advanced = false; + while (!End) + { + if (!TryReadTo(out sequence, delimiter[0], advancePastDelimiter: false)) + { + this = copy; + return false; + } + + if (delimiter.Length == 1) + { + return true; + } + + ReadOnlySpan next = Peek(peekBuffer); + if (next.SequenceEqual(delimiter)) + { + //TODO: Figure out a faster way to do this, potentially by avoiding the Advance in the previous TryReadTo call + if (advanced) + sequence = copy.Sequence.Slice(copy.Consumed, Consumed - copy.Consumed); + if (advancePastDelimiter) + { + Advance(delimiter.Length); + } + return true; + } + else + { + Advance(1); + advanced = true; + } + } + + this = copy; + sequence = default; + return false; + } + + /// + /// Advance until the given , if found. + /// + /// The delimiter to search for. + /// True to move past the if found. + /// True if the given was found. + public bool TryAdvanceTo(T delimiter, bool advancePastDelimiter = true) + { + ReadOnlySpan remaining = UnreadSpan; + int index = remaining.IndexOf(delimiter); + if (index != -1) + { + Advance(advancePastDelimiter ? index + 1 : index); + return true; + } + + return TryReadToInternal(out _, delimiter, advancePastDelimiter); + } + + /// + /// Advance until any of the given , if found. + /// + /// The delimiters to search for. + /// True to move past the first found instance of any of the given . + /// True if any of the given was found. + public bool TryAdvanceToAny(ReadOnlySpan delimiters, bool advancePastDelimiter = true) + { + ReadOnlySpan remaining = UnreadSpan; + int index = remaining.IndexOfAny(delimiters); + if (index != -1) + { + AdvanceCurrentSpan(advancePastDelimiter ? index + 1 : index); + return true; + } + + return TryReadToAnyInternal(out _, delimiters, advancePastDelimiter); + } + + /// + /// Advance past consecutive instances of the given . + /// + /// How many positions the reader has been advanced. + public long AdvancePast(T value) + { + long start = Consumed; + + do + { + // Advance past all matches in the current span + int i; + for (i = CurrentSpanIndex; i < CurrentSpan.Length && CurrentSpan[i].Equals(value); i++) + { + } + + int advanced = i - CurrentSpanIndex; + if (advanced == 0) + { + // Didn't advance at all in this span, exit. + break; + } + + AdvanceCurrentSpan(advanced); + + // If we're at postion 0 after advancing and not at the End, + // we're in a new span and should continue the loop. + } while (CurrentSpanIndex == 0 && !End); + + return Consumed - start; + } + + /// + /// Skip consecutive instances of any of the given . + /// + /// How many positions the reader has been advanced. + public long AdvancePastAny(ReadOnlySpan values) + { + long start = Consumed; + + do + { + // Advance past all matches in the current span + int i; + for (i = CurrentSpanIndex; i < CurrentSpan.Length && values.IndexOf(CurrentSpan[i]) != -1; i++) + { + } + + int advanced = i - CurrentSpanIndex; + if (advanced == 0) + { + // Didn't advance at all in this span, exit. + break; + } + + AdvanceCurrentSpan(advanced); + + // If we're at postion 0 after advancing and not at the End, + // we're in a new span and should continue the loop. + } while (CurrentSpanIndex == 0 && !End); + + return Consumed - start; + } + + /// + /// Advance past consecutive instances of any of the given values. + /// + /// How many positions the reader has been advanced. + public long AdvancePastAny(T value0, T value1, T value2, T value3) + { + long start = Consumed; + + do + { + // Advance past all matches in the current span + int i; + for (i = CurrentSpanIndex; i < CurrentSpan.Length; i++) + { + T value = CurrentSpan[i]; + if (!value.Equals(value0) && !value.Equals(value1) && !value.Equals(value2) && !value.Equals(value3)) + { + break; + } + } + + int advanced = i - CurrentSpanIndex; + if (advanced == 0) + { + // Didn't advance at all in this span, exit. + break; + } + + AdvanceCurrentSpan(advanced); + + // If we're at postion 0 after advancing and not at the End, + // we're in a new span and should continue the loop. + } while (CurrentSpanIndex == 0 && !End); + + return Consumed - start; + } + + /// + /// Advance past consecutive instances of any of the given values. + /// + /// How many positions the reader has been advanced. + public long AdvancePastAny(T value0, T value1, T value2) + { + long start = Consumed; + + do + { + // Advance past all matches in the current span + int i; + for (i = CurrentSpanIndex; i < CurrentSpan.Length; i++) + { + T value = CurrentSpan[i]; + if (!value.Equals(value0) && !value.Equals(value1) && !value.Equals(value2)) + { + break; + } + } + + int advanced = i - CurrentSpanIndex; + if (advanced == 0) + { + // Didn't advance at all in this span, exit. + break; + } + + AdvanceCurrentSpan(advanced); + + // If we're at postion 0 after advancing and not at the End, + // we're in a new span and should continue the loop. + } while (CurrentSpanIndex == 0 && !End); + + return Consumed - start; + } + + /// + /// Advance past consecutive instances of any of the given values. + /// + /// How many positions the reader has been advanced. + public long AdvancePastAny(T value0, T value1) + { + long start = Consumed; + + do + { + // Advance past all matches in the current span + int i; + for (i = CurrentSpanIndex; i < CurrentSpan.Length; i++) + { + T value = CurrentSpan[i]; + if (!value.Equals(value0) && !value.Equals(value1)) + { + break; + } + } + + int advanced = i - CurrentSpanIndex; + if (advanced == 0) + { + // Didn't advance at all in this span, exit. + break; + } + + AdvanceCurrentSpan(advanced); + + // If we're at postion 0 after advancing and not at the End, + // we're in a new span and should continue the loop. + } while (CurrentSpanIndex == 0 && !End); + + return Consumed - start; + } + + /// + /// Check to see if the given value is next. + /// + /// The value to compare the next position to. + /// Move past the value if found. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool IsNext(T next, bool advancePast = false) + { + if (End) + return false; + + ReadOnlySpan unread = UnreadSpan; + if (unread[0].Equals(next)) + { + if (advancePast) + { + AdvanceCurrentSpan(1); + } + return true; + } + return false; + } + + /// + /// Check to see if the given values are next. + /// + /// The sequence to compare the next position to. + /// Move past the values if found. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool IsNext(ReadOnlySpan next, bool advancePast = false) + { + ReadOnlySpan unread = UnreadSpan; + if (unread.StartsWith(next)) + { + if (advancePast) + { + AdvanceCurrentSpan(next.Length); + } + return true; + } + + // Only check the slow path if there wasn't enough to satisfy next + return unread.Length < next.Length && IsNextSlow(next, advancePast); + } + + private unsafe bool IsNextSlow(ReadOnlySpan next, bool advancePast) + { + ReadOnlySpan currentSpan = UnreadSpan; + + // We should only come in here if we need more data than we have in our current span + Debug.Assert(currentSpan.Length < next.Length); + + int fullLength = next.Length; + SequencePosition nextPosition = _nextPosition; + + while (next.StartsWith(currentSpan)) + { + if (next.Length == currentSpan.Length) + { + // Fully matched + if (advancePast) + { + Advance(fullLength); + } + return true; + } + + // Need to check the next segment + while (true) + { + if (!Sequence.TryGet(ref nextPosition, out ReadOnlyMemory nextSegment, advance: true)) + { + // Nothing left + return false; + } + + if (nextSegment.Length > 0) + { + next = next.Slice(currentSpan.Length); + currentSpan = nextSegment.Span; + if (currentSpan.Length > next.Length) + { + currentSpan = currentSpan.Slice(0, next.Length); + } + break; + } + } + } + + return false; + } + } +} diff --git a/src/System.Memory/src/System/ThrowHelper.cs b/src/System.Memory/src/System/ThrowHelper.cs index 70b1a1ca8597..13be346f202d 100644 --- a/src/System.Memory/src/System/ThrowHelper.cs +++ b/src/System.Memory/src/System/ThrowHelper.cs @@ -152,6 +152,7 @@ internal enum ExceptionArgument endIndex, array, culture, - manager + manager, + count } } diff --git a/src/System.Memory/tests/SequenceReader/Advance.cs b/src/System.Memory/tests/SequenceReader/Advance.cs new file mode 100644 index 000000000000..bec4afb32948 --- /dev/null +++ b/src/System.Memory/tests/SequenceReader/Advance.cs @@ -0,0 +1,61 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Buffers; +using Xunit; + +namespace System.Memory.Tests.SequenceReader +{ + public class Advance + { + [Theory, + InlineData(true), + InlineData(false)] + public void Basic(bool singleSegment) + { + byte[] buffer = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + ReadOnlySequence bytes = singleSegment + ? new ReadOnlySequence(buffer) + : SequenceFactory.CreateSplit(buffer, 2, 4); + + SequenceReader skipReader = new SequenceReader(bytes); + Assert.False(skipReader.TryAdvanceTo(10)); + Assert.True(skipReader.TryAdvanceTo(4, advancePastDelimiter: false)); + Assert.True(skipReader.TryRead(out byte value)); + Assert.Equal(4, value); + + Assert.True(skipReader.TryAdvanceToAny(new byte[] { 3, 12, 7 }, advancePastDelimiter: false)); + Assert.True(skipReader.TryRead(out value)); + Assert.Equal(7, value); + Assert.Equal(1, skipReader.AdvancePast(8)); + Assert.True(skipReader.TryRead(out value)); + Assert.Equal(9, value); + + skipReader = new SequenceReader(bytes); + Assert.Equal(0, skipReader.AdvancePast(2)); + Assert.Equal(3, skipReader.AdvancePastAny(new byte[] { 2, 3, 1 })); + Assert.True(skipReader.TryRead(out value)); + Assert.Equal(4, value); + } + + [Fact] + public void PastEmptySegments() + { + ReadOnlySequence bytes = SequenceFactory.Create(new byte[][] { + new byte[] { 0 }, + new byte[] { }, + new byte[] { }, + new byte[] { } + }); + + SequenceReader reader = new SequenceReader(bytes); + reader.Advance(1); + Assert.Equal(0, reader.CurrentSpanIndex); + Assert.Equal(0, reader.CurrentSpan.Length); + Assert.False(reader.TryPeek(out byte value)); + ReadOnlySequence sequence = reader.Sequence.Slice(reader.Position); + Assert.Equal(0, sequence.Length); + } + } +} diff --git a/src/System.Memory/tests/SequenceReader/BasicTests.cs b/src/System.Memory/tests/SequenceReader/BasicTests.cs new file mode 100644 index 000000000000..d0c48c048450 --- /dev/null +++ b/src/System.Memory/tests/SequenceReader/BasicTests.cs @@ -0,0 +1,568 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Buffers; +using Xunit; + +namespace System.Memory.Tests.SequenceReader +{ + public class ArrayByte : SingleSegment + { + public ArrayByte() : base(ReadOnlySequenceFactory.ArrayFactory, s_byteInputData) { } + } + + public class ArrayChar : SingleSegment + { + public ArrayChar() : base(ReadOnlySequenceFactory.ArrayFactory, s_charInputData) { } + } + + public class MemoryByte : SingleSegment + { + public MemoryByte() : base(ReadOnlySequenceFactory.MemoryFactory, s_byteInputData) { } + } + + public class MemoryChar : SingleSegment + { + public MemoryChar() : base(ReadOnlySequenceFactory.MemoryFactory, s_charInputData) { } + } + + public class SingleSegmentByte : SingleSegment + { + public SingleSegmentByte() : base(s_byteInputData) { } + } + + public class SingleSegmentChar : SingleSegment + { + public SingleSegmentChar() : base(s_charInputData) { } + } + + public abstract class SingleSegment : ReaderBasicTests where T : unmanaged, IEquatable + { + public SingleSegment(T[] inputData) : base(ReadOnlySequenceFactory.SingleSegmentFactory, inputData) { } + internal SingleSegment(ReadOnlySequenceFactory factory, T[] inputData) : base(factory, inputData) { } + + [Fact] + public void AdvanceSingleBufferSkipsValues() + { + SequenceReader reader = new SequenceReader(SequenceFactory.Create(GetInputData(5))); + Assert.Equal(5, reader.Length); + Assert.Equal(5, reader.Remaining); + Assert.Equal(0, reader.Consumed); + Assert.Equal(0, reader.CurrentSpanIndex); + + // Advance 2 positions + reader.Advance(2); + Assert.Equal(5, reader.Length); + Assert.Equal(3, reader.Remaining); + Assert.Equal(2, reader.Consumed); + Assert.Equal(2, reader.CurrentSpanIndex); + Assert.Equal(InputData[2], reader.CurrentSpan[reader.CurrentSpanIndex]); + Assert.True(reader.TryPeek(out T value)); + Assert.Equal(InputData[2], value); + + // Advance 2 positions + reader.Advance(2); + Assert.Equal(1, reader.Remaining); + Assert.Equal(4, reader.Consumed); + Assert.Equal(4, reader.CurrentSpanIndex); + Assert.Equal(InputData[4], reader.CurrentSpan[reader.CurrentSpanIndex]); + Assert.True(reader.TryPeek(out value)); + Assert.Equal(InputData[4], value); + } + + [Fact] + public void TryReadReturnsValueAndAdvances() + { + SequenceReader reader = new SequenceReader(Factory.CreateWithContent(GetInputData(2))); + Assert.Equal(2, reader.Length); + Assert.Equal(2, reader.Remaining); + Assert.Equal(0, reader.Consumed); + Assert.Equal(0, reader.CurrentSpanIndex); + Assert.Equal(InputData[0], reader.CurrentSpan[reader.CurrentSpanIndex]); + + // Read 1st value + Assert.True(reader.TryRead(out T value)); + Assert.Equal(InputData[0], value); + Assert.Equal(1, reader.Remaining); + Assert.Equal(1, reader.Consumed); + Assert.Equal(1, reader.CurrentSpanIndex); + Assert.Equal(InputData[1], reader.CurrentSpan[reader.CurrentSpanIndex]); + + // Read 2nd value + Assert.True(reader.TryRead(out value)); + Assert.Equal(InputData[1], value); + Assert.Equal(0, reader.Remaining); + Assert.Equal(2, reader.Consumed); + Assert.Equal(2, reader.CurrentSpanIndex); + + // Read at end + Assert.False(reader.TryRead(out value)); + Assert.Equal(default, value); + Assert.Equal(0, reader.Remaining); + Assert.Equal(2, reader.Consumed); + Assert.Equal(2, reader.CurrentSpanIndex); + Assert.True(reader.End); + } + + [Fact] + public void DefaultState() + { + T[] array = new T[] { default }; + SequenceReader reader = default; + Assert.Equal(0, reader.CurrentSpan.Length); + Assert.Equal(0, reader.UnreadSpan.Length); + Assert.Equal(0, reader.Consumed); + Assert.Equal(0, reader.CurrentSpanIndex); + Assert.Equal(0, reader.Length); + Assert.Equal(0, reader.Remaining); + Assert.True(reader.End); + Assert.False(reader.TryPeek(out T value)); + Assert.Equal(default, value); + Assert.False(reader.TryRead(out value)); + Assert.Equal(default, value); + Assert.Equal(0, reader.AdvancePast(default)); + Assert.Equal(0, reader.AdvancePastAny(array)); + Assert.Equal(0, reader.AdvancePastAny(default)); + Assert.False(reader.TryReadTo(out ReadOnlySequence sequence, default(T))); + Assert.True(sequence.IsEmpty); + Assert.False(reader.TryReadTo(out sequence, array)); + Assert.True(sequence.IsEmpty); + Assert.False(reader.TryReadTo(out ReadOnlySpan span, default)); + Assert.True(span.IsEmpty); + Assert.False(reader.TryReadToAny(out sequence, array)); + Assert.True(sequence.IsEmpty); + Assert.False(reader.TryReadToAny(out span, array)); + Assert.True(span.IsEmpty); + Assert.False(reader.TryAdvanceTo(default)); + Assert.False(reader.TryAdvanceToAny(array)); + Assert.Equal(0, reader.CurrentSpan.Length); + Assert.Equal(0, reader.UnreadSpan.Length); + Assert.Equal(0, reader.Consumed); + Assert.Equal(0, reader.CurrentSpanIndex); + Assert.Equal(0, reader.Length); + Assert.Equal(0, reader.Remaining); + } + } + + public class SegmentPerByte : ReaderBasicTests + { + public SegmentPerByte() : base(ReadOnlySequenceFactory.SegmentPerItemFactory, s_byteInputData) { } + } + + public class SegmentPerChar : ReaderBasicTests + { + public SegmentPerChar() : base(ReadOnlySequenceFactory.SegmentPerItemFactory, s_charInputData) { } + } + + public abstract class ReaderBasicTests where T : unmanaged, IEquatable + { + internal static byte[] s_byteInputData = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; + internal static char[] s_charInputData = new char[] { '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A' }; + private T[] _inputData; + + internal ReadOnlySequenceFactory Factory { get; } + protected ReadOnlySpan InputData { get => _inputData; } + + public T[] GetInputData(int count) => InputData.Slice(0, count).ToArray(); + + internal ReaderBasicTests(ReadOnlySequenceFactory factory, T[] inputData) + { + Factory = factory; + _inputData = inputData; + } + + [Fact] + public void TryPeekReturnsWithoutMoving() + { + SequenceReader reader = new SequenceReader(Factory.CreateWithContent(GetInputData(2))); + Assert.Equal(0, reader.Consumed); + Assert.Equal(2, reader.Remaining); + Assert.True(reader.TryPeek(out T value)); + Assert.Equal(InputData[0], value); + Assert.Equal(0, reader.Consumed); + Assert.Equal(2, reader.Remaining); + Assert.True(reader.TryPeek(out value)); + Assert.Equal(InputData[0], value); + Assert.Equal(0, reader.Consumed); + Assert.Equal(2, reader.Remaining); + } + + [Fact] + public void CursorIsCorrectAtEnd() + { + SequenceReader reader = new SequenceReader(Factory.CreateWithContent(GetInputData(2))); + reader.TryRead(out T _); + reader.TryRead(out T _); + Assert.True(reader.End); + } + + [Fact] + public void CursorIsCorrectWithEmptyLastBlock() + { + SequenceSegment last = new SequenceSegment(); + last.SetMemory(new OwnedArray(new T[4]), 0, 4); + + SequenceSegment first = new SequenceSegment(); + first.SetMemory(new OwnedArray(GetInputData(2)), 0, 2); + first.SetNext(last); + + SequenceReader reader = new SequenceReader(new ReadOnlySequence(first, first.Start, last, last.Start)); + reader.TryRead(out T _); + reader.TryRead(out T _); + reader.TryRead(out T _); + Assert.Same(last, reader.Position.GetObject()); + Assert.Equal(0, reader.Position.GetInteger()); + Assert.True(reader.End); + } + + [Fact] + public void TryPeekReturnsDefaultInTheEnd() + { + SequenceReader reader = new SequenceReader(Factory.CreateWithContent(GetInputData(2))); + Assert.True(reader.TryRead(out T value)); + Assert.Equal(InputData[0], value); + Assert.True(reader.TryRead(out value)); + Assert.Equal(InputData[1], value); + Assert.False(reader.TryRead(out value)); + Assert.Equal(default, value); + } + + [Fact] + public void AdvanceToEndThenPeekReturnsDefault() + { + SequenceReader reader = new SequenceReader(Factory.CreateWithContent(GetInputData(5))); + reader.Advance(5); + Assert.True(reader.End); + Assert.False(reader.TryPeek(out T value)); + Assert.Equal(default, value); + } + + [Fact] + public void AdvancingPastLengthThrows() + { + SequenceReader reader = new SequenceReader(Factory.CreateWithContent(GetInputData(5))); + try + { + reader.Advance(6); + Assert.True(false); + } + catch (Exception ex) + { + Assert.True(ex is ArgumentOutOfRangeException); + Assert.Equal(5, reader.Consumed); + Assert.Equal(0, reader.Remaining); + Assert.True(reader.End); + } + } + + [Fact] + public void CtorFindsFirstNonEmptySegment() + { + ReadOnlySequence buffer = Factory.CreateWithContent(GetInputData(1)); + SequenceReader reader = new SequenceReader(buffer); + + Assert.True(reader.TryPeek(out T value)); + Assert.Equal(InputData[0], value); + Assert.Equal(0, reader.Consumed); + Assert.Equal(1, reader.Remaining); + } + + [Fact] + public void EmptySegmentsAreSkippedOnMoveNext() + { + ReadOnlySequence buffer = Factory.CreateWithContent(GetInputData(2)); + SequenceReader reader = new SequenceReader(buffer); + + Assert.True(reader.TryPeek(out T value)); + Assert.Equal(InputData[0], value); + reader.Advance(1); + Assert.True(reader.TryPeek(out value)); + Assert.Equal(InputData[1], value); + } + + [Fact] + public void TryPeekGoesToEndIfAllEmptySegments() + { + ReadOnlySequence buffer = SequenceFactory.Create(new[] { new T[] { }, new T[] { }, new T[] { }, new T[] { } }); + SequenceReader reader = new SequenceReader(buffer); + + Assert.False(reader.TryPeek(out T value)); + Assert.Equal(default, value); + Assert.True(reader.End); + } + + [Fact] + public void AdvanceTraversesSegments() + { + ReadOnlySequence buffer = Factory.CreateWithContent(GetInputData(3)); + SequenceReader reader = new SequenceReader(buffer); + + reader.Advance(2); + Assert.Equal(InputData[2], reader.CurrentSpan[reader.CurrentSpanIndex]); + Assert.True(reader.TryRead(out T value)); + Assert.Equal(InputData[2], value); + } + + [Fact] + public void AdvanceThrowsPastLengthMultipleSegments() + { + ReadOnlySequence buffer = Factory.CreateWithContent(GetInputData(3)); + SequenceReader reader = new SequenceReader(buffer); + + try + { + reader.Advance(4); + Assert.True(false); + } + catch (Exception ex) + { + Assert.True(ex is ArgumentOutOfRangeException); + Assert.Equal(3, reader.Consumed); + Assert.Equal(0, reader.Remaining); + Assert.True(reader.End); + } + } + + [Fact] + public void TryReadTraversesSegments() + { + ReadOnlySequence buffer = Factory.CreateWithContent(GetInputData(3)); + SequenceReader reader = new SequenceReader(buffer); + + Assert.True(reader.TryRead(out T value)); + Assert.Equal(InputData[0], value); + Assert.True(reader.TryRead(out value)); + Assert.Equal(InputData[1], value); + Assert.True(reader.TryRead(out value)); + Assert.Equal(InputData[2], value); + Assert.False(reader.TryRead(out value)); + Assert.Equal(default, value); + Assert.True(reader.End); + } + + [Fact] + public void TryPeekTraversesSegments() + { + ReadOnlySequence buffer = Factory.CreateWithContent(GetInputData(2)); + SequenceReader reader = new SequenceReader(buffer); + + Assert.Equal(InputData[0], reader.CurrentSpan[reader.CurrentSpanIndex]); + Assert.True(reader.TryRead(out T value)); + Assert.Equal(InputData[0], value); + + Assert.Equal(InputData[1], reader.CurrentSpan[reader.CurrentSpanIndex]); + Assert.True(reader.TryPeek(out value)); + Assert.Equal(InputData[1], value); + Assert.True(reader.TryRead(out value)); + Assert.Equal(InputData[1], value); + Assert.False(reader.TryPeek(out value)); + Assert.False(reader.TryRead(out value)); + Assert.Equal(default, value); + Assert.Equal(default, value); + } + + [Fact] + public void PeekWorkesWithEmptySegments() + { + ReadOnlySequence buffer = Factory.CreateWithContent(GetInputData(1)); + SequenceReader reader = new SequenceReader(buffer); + + Assert.Equal(0, reader.CurrentSpanIndex); + Assert.Equal(1, reader.CurrentSpan.Length); + Assert.True(reader.TryPeek(out T value)); + Assert.Equal(InputData[0], value); + Assert.True(reader.TryRead(out value)); + Assert.Equal(InputData[0], value); + Assert.False(reader.TryPeek(out value)); + Assert.False(reader.TryRead(out value)); + Assert.Equal(default, value); + Assert.Equal(default, value); + } + + [Fact] + public void WorksWithEmptyBuffer() + { + SequenceReader reader = new SequenceReader(Factory.CreateWithContent(new T[] { })); + + Assert.Equal(0, reader.CurrentSpanIndex); + Assert.Equal(0, reader.CurrentSpan.Length); + Assert.Equal(0, reader.Length); + Assert.Equal(0, reader.Remaining); + Assert.True(reader.End); + Assert.False(reader.TryPeek(out T value)); + Assert.Equal(default, value); + Assert.False(reader.TryRead(out value)); + Assert.Equal(default, value); + Assert.True(reader.End); + } + + [Theory, + InlineData(0, false), + InlineData(5, false), + InlineData(10, false), + InlineData(11, true), + InlineData(12, true), + InlineData(15, true)] + public void ReturnsCorrectCursor(int takes, bool end) + { + ReadOnlySequence readableBuffer = Factory.CreateWithContent(GetInputData(10)); + SequenceReader reader = new SequenceReader(readableBuffer); + for (int i = 0; i < takes; i++) + { + reader.TryRead(out _); + } + + T[] expected = end ? new T[] { } : readableBuffer.Slice(takes).ToArray(); + Assert.Equal(expected, readableBuffer.Slice(reader.Position).ToArray()); + } + + [Fact] + public void SlicingBufferReturnsCorrectCursor() + { + ReadOnlySequence buffer = Factory.CreateWithContent(GetInputData(10)); + ReadOnlySequence sliced = buffer.Slice(2L); + + SequenceReader reader = new SequenceReader(sliced); + Assert.Equal(sliced.ToArray(), buffer.Slice(reader.Position).ToArray()); + Assert.True(reader.TryPeek(out T value)); + Assert.Equal(InputData[2], value); + Assert.Equal(0, reader.CurrentSpanIndex); + } + + [Fact] + public void ReaderIndexIsCorrect() + { + ReadOnlySequence buffer = Factory.CreateWithContent(GetInputData(10)); + SequenceReader reader = new SequenceReader(buffer); + + int counter = 0; + while (!reader.End) + { + ReadOnlySpan span = reader.CurrentSpan; + for (int i = reader.CurrentSpanIndex; i < span.Length; i++) + { + Assert.Equal(InputData[counter++], reader.CurrentSpan[i]); + } + reader.Advance(span.Length); + } + Assert.Equal(buffer.Length, reader.Consumed); + } + + [Theory, + InlineData(1), + InlineData(2), + InlineData(3)] + public void Advance_PositionIsCorrect(int advanceBy) + { + // Check that advancing through the reader gives the same position + // as returned directly from the buffer. + + ReadOnlySequence buffer = Factory.CreateWithContent(GetInputData(10)); + SequenceReader reader = new SequenceReader(buffer); + + SequencePosition readerPosition = reader.Position; + SequencePosition bufferPosition = buffer.GetPosition(0); + Assert.Equal(readerPosition.GetInteger(), bufferPosition.GetInteger()); + Assert.Same(readerPosition.GetObject(), readerPosition.GetObject()); + + for (int i = advanceBy; i <= buffer.Length; i += advanceBy) + { + reader.Advance(advanceBy); + readerPosition = reader.Position; + bufferPosition = buffer.GetPosition(i); + Assert.Equal(readerPosition.GetInteger(), bufferPosition.GetInteger()); + Assert.Same(readerPosition.GetObject(), readerPosition.GetObject()); + } + } + + [Fact] + public void AdvanceTo() + { + // Ensure we can advance to each of the items in the buffer + + T[] inputData = GetInputData(10); + ReadOnlySequence buffer = Factory.CreateWithContent(inputData); + + for (int i = 0; i < buffer.Length; i++) + { + SequenceReader reader = new SequenceReader(buffer); + Assert.True(reader.TryAdvanceTo(inputData[i], advancePastDelimiter: false)); + Assert.True(reader.TryPeek(out T value)); + Assert.Equal(inputData[i], value); + } + } + + [Fact] + public void AdvanceTo_AdvancePast() + { + // Ensure we can advance to each of the items in the buffer (skipping what we advanced to) + + T[] inputData = GetInputData(10); + ReadOnlySequence buffer = Factory.CreateWithContent(inputData); + + for (int start = 0; start < 2; start++) + { + for (int i = start; i < buffer.Length - 1; i += 2) + { + SequenceReader reader = new SequenceReader(buffer); + Assert.True(reader.TryAdvanceTo(inputData[i], advancePastDelimiter: true)); + Assert.True(reader.TryPeek(out T value)); + Assert.Equal(inputData[i + 1], value); + } + } + } + + [Fact] + public void CopyToLargerBufferWorks() + { + T[] content = (T[])_inputData.Clone(); + + Span buffer = new T[content.Length + 1]; + SequenceReader reader = new SequenceReader(Factory.CreateWithContent(content)); + + // this loop skips more and more items in the reader + for (int i = 0; i < content.Length; i++) + { + int copied = reader.Peek(buffer).Length; + Assert.Equal(content.Length - i, copied); + Assert.True(buffer.Slice(0, copied).SequenceEqual(content.AsSpan(i))); + + // make sure that nothing more got written, i.e. tail is empty + for (int r = copied; r < buffer.Length; r++) + { + Assert.Equal(default, buffer[r]); + } + + reader.Advance(1); + buffer.Clear(); + } + } + + [Fact] + public void CopyToSmallerBufferWorks() + { + T[] content = (T[])_inputData.Clone(); + + Span buffer = new T[content.Length]; + SequenceReader reader = new SequenceReader(Factory.CreateWithContent(content)); + + // this loop skips more and more items in the reader + for (int i = 0; i < content.Length; i++) + { + // this loop makes the destination buffer smaller and smaller + for (int j = 0; j < buffer.Length - i; j++) + { + Span bufferSlice = buffer.Slice(0, j); + bufferSlice.Clear(); + ReadOnlySpan peeked = reader.Peek(bufferSlice); + Assert.Equal(Math.Min(bufferSlice.Length, content.Length - i), peeked.Length); + + Assert.True(peeked.SequenceEqual(content.AsSpan(i, j))); + } + + reader.Advance(1); + } + } + } +} diff --git a/src/System.Memory/tests/SequenceReader/BinaryExtensions.cs b/src/System.Memory/tests/SequenceReader/BinaryExtensions.cs new file mode 100644 index 000000000000..a6d9d0ed0f57 --- /dev/null +++ b/src/System.Memory/tests/SequenceReader/BinaryExtensions.cs @@ -0,0 +1,69 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Buffers; +using Xunit; + +namespace System.Memory.Tests.SequenceReader +{ + public class BinaryExtensions + { + [Fact] + public void MultiSegmentBytesReaderNumbers() + { + ReadOnlySequence bytes = SequenceFactory.Create(new byte[][] { + new byte[] { 0 }, + new byte[] { 1, 2 }, + new byte[] { 3, 4 }, + new byte[] { 5, 6, 7, 8 }, + new byte[] { 8, 0 }, + new byte[] { 1, }, + new byte[] { 0, 2, }, + new byte[] { 1, 2, 3, 4 }, + new byte[] { 5, 6 }, + new byte[] { 7, 8, 9, }, + new byte[] { 0, 1, 2, 3 }, + new byte[] { 4, 5 }, + new byte[] { 6, 7, 8, 9 }, + new byte[] { 0, 1, 2, 3 }, + new byte[] { 4 }, + }); + + SequenceReader reader = new SequenceReader(bytes); + + Assert.True(reader.TryReadTo(out ReadOnlySequence bytesValue, 2)); + Span span = bytesValue.ToArray(); + Assert.Equal(0, span[0]); + Assert.Equal(1, span[1]); + + Assert.True(reader.TryReadTo(out bytesValue, 5)); + span = bytesValue.ToArray(); + Assert.Equal(3, span[0]); + Assert.Equal(4, span[1]); + + Assert.True(reader.TryReadTo(out bytesValue, new byte[] { 8, 8 })); + span = bytesValue.ToArray(); + Assert.Equal(6, span[0]); + Assert.Equal(7, span[1]); + + Assert.True(reader.TryRead(out int intValue)); + Assert.Equal(BitConverter.ToInt32(new byte[] { 0, 1, 0, 2 }), intValue); + + Assert.True(reader.TryReadInt32BigEndian(out intValue)); + Assert.Equal(BitConverter.ToInt32(new byte[] { 4, 3, 2, 1 }), intValue); + + Assert.True(reader.TryReadInt64LittleEndian(out long longValue)); + Assert.Equal(BitConverter.ToInt64(new byte[] { 5, 6, 7, 8, 9, 0, 1, 2 }), longValue); + + Assert.True(reader.TryReadInt64BigEndian(out longValue)); + Assert.Equal(BitConverter.ToInt64(new byte[] { 0, 9, 8, 7, 6, 5, 4, 3 }), longValue); + + Assert.True(reader.TryReadInt16LittleEndian(out short shortValue)); + Assert.Equal(BitConverter.ToInt16(new byte[] { 1, 2 }), shortValue); + + Assert.True(reader.TryReadInt16BigEndian(out shortValue)); + Assert.Equal(BitConverter.ToInt16(new byte[] { 4, 3 }), shortValue); + } + } +} diff --git a/src/System.Memory/tests/SequenceReader/IsNext.cs b/src/System.Memory/tests/SequenceReader/IsNext.cs new file mode 100644 index 000000000000..8141de4b199c --- /dev/null +++ b/src/System.Memory/tests/SequenceReader/IsNext.cs @@ -0,0 +1,46 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Buffers; +using Xunit; + +namespace System.Memory.Tests.SequenceReader +{ + public class IsNext + { + [Fact] + public void IsNext_Span() + { + ReadOnlySequence bytes = SequenceFactory.Create(new byte[][] { + new byte[] { 0 }, + new byte[] { 1, 2 }, + new byte[] { 3, 4 }, + new byte[] { 5, 6, 7, 8 } + }); + + SequenceReader reader = new SequenceReader(bytes); + Assert.True(reader.IsNext(ReadOnlySpan.Empty, advancePast: false)); + Assert.True(reader.IsNext(ReadOnlySpan.Empty, advancePast: true)); + Assert.True(reader.IsNext(new byte[] { 0 }, advancePast: false)); + Assert.False(reader.IsNext(new byte[] { 0, 2 }, advancePast: false)); + Assert.False(reader.IsNext(new byte[] { 0, 2 }, advancePast: true)); + Assert.True(reader.IsNext(new byte[] { 0, 1 }, advancePast: false)); + Assert.False(reader.IsNext(new byte[] { 0, 1, 3 }, advancePast: false)); + Assert.True(reader.IsNext(new byte[] { 0, 1, 2 }, advancePast: false)); + Assert.False(reader.IsNext(new byte[] { 0, 1, 2, 4 }, advancePast: false)); + Assert.True(reader.IsNext(new byte[] { 0, 1, 2, 3 }, advancePast: false)); + Assert.True(reader.IsNext(new byte[] { 0, 1, 2, 3, 4 }, advancePast: false)); + Assert.True(reader.IsNext(new byte[] { 0, 1, 2, 3, 4, 5 }, advancePast: false)); + Assert.True(reader.IsNext(new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8 }, advancePast: false)); + Assert.False(reader.IsNext(new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, advancePast: false)); + Assert.False(reader.IsNext(new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, advancePast: true)); + Assert.Equal(0, reader.Consumed); + + Assert.True(reader.IsNext(new byte[] { 0, 1, 2, 3 }, advancePast: true)); + Assert.True(reader.IsNext(new byte[] { 4, 5, 6 }, advancePast: true)); + Assert.True(reader.TryPeek(out byte value)); + Assert.Equal(7, value); + } + } +} diff --git a/src/System.Memory/tests/SequenceReader/OwnedArray.cs b/src/System.Memory/tests/SequenceReader/OwnedArray.cs new file mode 100644 index 000000000000..37c20a236058 --- /dev/null +++ b/src/System.Memory/tests/SequenceReader/OwnedArray.cs @@ -0,0 +1,103 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Buffers; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Threading; + +namespace System.Memory.Tests.SequenceReader +{ + public class OwnedArray : MemoryManager + { + T[] _array; + int _referenceCount; + + public OwnedArray(int length) + { + _array = new T[length]; + } + + public OwnedArray(T[] array) + { + if (array == null) + ThrowArgumentNullException(nameof(array)); + _array = array; + } + + public static implicit operator OwnedArray(T[] array) => new OwnedArray(array); + + public override Span GetSpan() + { + if (IsDisposed) + ThrowObjectDisposedException(nameof(OwnedArray)); + return new Span(_array); + } + + public override MemoryHandle Pin(int elementIndex = 0) + { + unsafe + { + Retain(); + if ((uint)elementIndex > (uint)_array.Length) + throw new ArgumentOutOfRangeException(nameof(elementIndex)); + var handle = GCHandle.Alloc(_array, GCHandleType.Pinned); + void* pointer = Unsafe.Add((void*)handle.AddrOfPinnedObject(), elementIndex); + return new MemoryHandle(pointer, handle, this); + } + } + + protected override bool TryGetArray(out ArraySegment arraySegment) + { + if (IsDisposed) + ThrowObjectDisposedException(nameof(OwnedArray)); + arraySegment = new ArraySegment(_array); + return true; + } + + protected override void Dispose(bool disposing) + { + _array = null; + } + + public void Retain() + { + if (IsDisposed) + ThrowObjectDisposedException(nameof(OwnedArray)); + Interlocked.Increment(ref _referenceCount); + } + + public override void Unpin() + { + int newRefCount = Interlocked.Decrement(ref _referenceCount); + if (newRefCount < 0) + ThrowInvalidOperationException(); + if (newRefCount == 0) + { + OnNoReferences(); + } + } + + protected virtual void OnNoReferences() + { + } + + protected bool IsRetained => _referenceCount > 0; + + public bool IsDisposed => _array == null; + + [MethodImpl(MethodImplOptions.NoInlining)] + public static void ThrowObjectDisposedException(string objectName) + => throw new ObjectDisposedException(objectName); + + [MethodImpl(MethodImplOptions.NoInlining)] + public static void ThrowInvalidOperationException() + => throw new InvalidOperationException(); + + [MethodImpl(MethodImplOptions.NoInlining)] + public static void ThrowArgumentNullException(string argumentName) + => throw new ArgumentNullException(argumentName); + } +} diff --git a/src/System.Memory/tests/SequenceReader/ReadTo.cs b/src/System.Memory/tests/SequenceReader/ReadTo.cs new file mode 100644 index 000000000000..2742f283d5fe --- /dev/null +++ b/src/System.Memory/tests/SequenceReader/ReadTo.cs @@ -0,0 +1,177 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Buffers; +using Xunit; + +namespace System.Memory.Tests.SequenceReader +{ + public class ReadTo + { + [Theory, + InlineData(false, false), + InlineData(false, true), + InlineData(true, false), + InlineData(true, true)] + public void TryReadTo_Span(bool advancePastDelimiter, bool useEscapeOverload) + { + ReadOnlySequence bytes = SequenceFactory.Create(new byte[][] { + new byte[] { 0 }, + new byte[] { 1, 2 }, + new byte[] { }, + new byte[] { 3, 4, 5, 6 } + }); + + SequenceReader reader = new SequenceReader(bytes); + + // Read to 0-5 + for (byte i = 0; i < bytes.Length - 1; i++) + { + SequenceReader copy = reader; + + // Can read to the first integer (0-5) + Assert.True( + useEscapeOverload + ? copy.TryReadTo(out ReadOnlySpan span, i, 255, advancePastDelimiter) + : copy.TryReadTo(out span, i, advancePastDelimiter)); + + // Should never have a null Position object + Assert.NotNull(copy.Position.GetObject()); + + // Should be able to then read to 6 + Assert.True( + useEscapeOverload + ? copy.TryReadTo(out span, 6, 255, advancePastDelimiter) + : copy.TryReadTo(out span, 6, advancePastDelimiter)); + + Assert.NotNull(copy.Position.GetObject()); + + // If we didn't advance, we should still be able to read to 6 + Assert.Equal(!advancePastDelimiter, + useEscapeOverload + ? copy.TryReadTo(out span, 6, 255, advancePastDelimiter) + : copy.TryReadTo(out span, 6, advancePastDelimiter)); + } + } + + [Theory, + InlineData(false, false), + InlineData(false, true), + InlineData(true, false), + InlineData(true, true)] + public void TryReadTo_Sequence(bool advancePastDelimiter, bool useEscapeOverload) + { + ReadOnlySequence bytes = SequenceFactory.Create(new byte[][] { + new byte[] { 0 }, + new byte[] { 1, 2 }, + new byte[] { }, + new byte[] { 3, 4, 5, 6 } + }); + + SequenceReader reader = new SequenceReader(bytes); + + // Read to 0-5 + for (byte i = 0; i < bytes.Length - 1; i++) + { + SequenceReader copy = reader; + + // Can read to the first integer (0-5) + Assert.True( + useEscapeOverload + ? copy.TryReadTo(out ReadOnlySequence sequence, i, 255, advancePastDelimiter) + : copy.TryReadTo(out sequence, i, advancePastDelimiter)); + + // Should never have a null Position object + Assert.NotNull(copy.Position.GetObject()); + ReadOnlySequence.Enumerator enumerator = sequence.GetEnumerator(); + while (enumerator.MoveNext()) + ; + + // Should be able to read to final 6 + Assert.True( + useEscapeOverload + ? copy.TryReadTo(out sequence, 6, 255, advancePastDelimiter) + : copy.TryReadTo(out sequence, 6, advancePastDelimiter)); + + Assert.NotNull(copy.Position.GetObject()); + enumerator = sequence.GetEnumerator(); + while (enumerator.MoveNext()) + ; + + // If we didn't advance, we should still be able to read to 6 + Assert.Equal(!advancePastDelimiter, + useEscapeOverload + ? copy.TryReadTo(out sequence, 6, 255, advancePastDelimiter) + : copy.TryReadTo(out sequence, 6, advancePastDelimiter)); + } + } + + [Theory, + InlineData(false), + InlineData(true),] + public void TryReadToSpan_Sequence(bool advancePastDelimiter) + { + ReadOnlySequence bytes = SequenceFactory.Create(new byte[][] { + new byte[] { 0, 0 }, + new byte[] { 1, 1, 2, 2 }, + new byte[] { }, + new byte[] { 3, 3, 4, 4, 5, 5, 6, 6 } + }); + + SequenceReader reader = new SequenceReader(bytes); + for (byte i = 0; i < bytes.Length / 2 - 1; i++) + { + byte[] expected = new byte[i * 2 + 1]; + for (int j = 0; j < expected.Length - 1; j++) + { + expected[j] = (byte)(j / 2); + } + expected[i * 2] = i; + ReadOnlySpan searchFor = new byte[] { i, (byte)(i + 1) }; + SequenceReader copy = reader; + Assert.True(copy.TryReadTo(out ReadOnlySequence seq, searchFor, advancePastDelimiter)); + Assert.True(seq.ToArray().AsSpan().SequenceEqual(expected)); + } + + bytes = SequenceFactory.Create(new byte[][] { + new byte[] { 47, 42, 66, 32, 42, 32, 66, 42, 47 } // /*b * b*/ + }); + + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out ReadOnlySequence sequence, new byte[] { 42, 47 }, advancePastDelimiter)); // */ + Assert.True(sequence.ToArray().AsSpan().SequenceEqual(new byte[] { 47, 42, 66, 32, 42, 32, 66 })); + } + + [Theory, + InlineData(false), + InlineData(true)] + public void TryReadTo_NotFound_Span(bool advancePastDelimiter) + { + ReadOnlySequence bytes = SequenceFactory.Create(new byte[][] { + new byte[] { 1 }, + new byte[] { 2, 3, 255 } + }); + + SequenceReader reader = new SequenceReader(bytes); + reader.Advance(4); + Assert.False(reader.TryReadTo(out ReadOnlySpan span, 255, 0, advancePastDelimiter)); + } + + [Theory, + InlineData(false), + InlineData(true)] + public void TryReadTo_NotFound_Sequence(bool advancePastDelimiter) + { + ReadOnlySequence bytes = SequenceFactory.Create(new byte[][] { + new byte[] { 1 }, + new byte[] { 2, 3, 255 } + }); + + SequenceReader reader = new SequenceReader(bytes); + reader.Advance(4); + Assert.False(reader.TryReadTo(out ReadOnlySequence span, 255, 0, advancePastDelimiter)); + } + } +} diff --git a/src/System.Memory/tests/SequenceReader/Rewind.cs b/src/System.Memory/tests/SequenceReader/Rewind.cs new file mode 100644 index 000000000000..2d91cf3f5de3 --- /dev/null +++ b/src/System.Memory/tests/SequenceReader/Rewind.cs @@ -0,0 +1,71 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Buffers; +using Xunit; + +namespace System.Memory.Tests.SequenceReader +{ + public class Rewind + { + [Fact] + public void Rewind_Full() + { + ReadOnlySequence bytes = SequenceFactory.Create(new byte[][] { + new byte[] { 0 }, + new byte[] { 1, 2 }, + new byte[] { 3, 4 }, + new byte[] { 5, 6, 7, 8 } + }); + + SequenceReader reader = new SequenceReader(bytes); + reader.Advance(1); + SequenceReader copy = reader; + for (int i = 1; i < bytes.Length; i++) + { + reader.Advance(i); + reader.Rewind(i); + + Assert.Equal(copy.Position, reader.Position); + Assert.Equal(copy.Consumed, reader.Consumed); + Assert.Equal(copy.CurrentSpanIndex, reader.CurrentSpanIndex); + Assert.Equal(copy.End, reader.End); + Assert.True(copy.CurrentSpan.SequenceEqual(reader.CurrentSpan)); + } + } + + [Fact] + public void Rewind_ByOne() + { + ReadOnlySequence bytes = SequenceFactory.Create(new byte[][] { + new byte[] { 0 }, + new byte[] { 1, 2 }, + new byte[] { 3, 4 }, + new byte[] { 5, 6, 7, 8 } + }); + + SequenceReader reader = new SequenceReader(bytes); + reader.Advance(1); + SequenceReader copy = reader; + for (int i = 1; i < bytes.Length; i++) + { + reader.Advance(i); + Assert.Equal(i + 1, reader.Consumed); + + for (int j = 0; j < i; j++) + { + reader.Rewind(1); + Assert.Equal(i - j, reader.Consumed); + Assert.False(reader.End); + } + + Assert.Equal(copy.Position, reader.Position); + Assert.Equal(copy.Consumed, reader.Consumed); + Assert.Equal(copy.CurrentSpanIndex, reader.CurrentSpanIndex); + Assert.Equal(copy.End, reader.End); + Assert.True(copy.CurrentSpan.SequenceEqual(reader.CurrentSpan)); + } + } + } +} diff --git a/src/System.Memory/tests/SequenceReader/SequenceFactory.cs b/src/System.Memory/tests/SequenceReader/SequenceFactory.cs new file mode 100644 index 000000000000..eea28a00d657 --- /dev/null +++ b/src/System.Memory/tests/SequenceReader/SequenceFactory.cs @@ -0,0 +1,119 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Buffers; +using System.Text; + +namespace System.Memory.Tests.SequenceReader +{ + public static class SequenceFactory + { + public static ReadOnlySequence CreateSplit(T[] buffer, int minSize, int maxSize) where T : struct + { + if (buffer == null || buffer.Length == 0 || minSize <= 0 || maxSize <= 0 || minSize > maxSize) + { + throw new InvalidOperationException(); + } + + Random r = new Random(0xFEED); + + SequenceSegment last = null; + SequenceSegment first = null; + OwnedArray ownedBuffer = new OwnedArray(buffer); + + int remaining = buffer.Length; + int position = 0; + while (remaining > 0) + { + int take = Math.Min(r.Next(minSize, maxSize), remaining); + SequenceSegment current = new SequenceSegment(); + current.SetMemory(ownedBuffer, position, position + take); + if (first == null) + { + first = current; + last = current; + } + else + { + last.SetNext(current); + last = current; + } + remaining -= take; + position += take; + } + + return new ReadOnlySequence(first, 0, last, last.Length); + } + + public static ReadOnlySequence Create(params T[][] inputs) where T : struct + { + if (inputs == null || inputs.Length == 0) + { + throw new InvalidOperationException(); + } + + SequenceSegment last = null; + SequenceSegment first = null; + + for (int i = 0; i < inputs.Length; i++) + { + T[] source = inputs[i]; + int length = source.Length; + int dataOffset = length; + + // Shift the incoming data for testing + T[] chars = new T[length * 8]; + for (int j = 0; j < length; j++) + { + chars[dataOffset + j] = source[j]; + } + + // Create a segment that has offset relative to the OwnedMemory and OwnedMemory itself has offset relative to array + OwnedArray ownedBuffer = new OwnedArray(chars); + SequenceSegment current = new SequenceSegment(); + current.SetMemory(ownedBuffer, length, length * 2); + if (first == null) + { + first = current; + last = current; + } + else + { + last.SetNext(current); + last = current; + } + } + + return new ReadOnlySequence(first, 0, last, last.Length); + } + + public static ReadOnlySequence CreateUtf8(params string[] inputs) + { + byte[][] buffers = new byte[inputs.Length][]; + for (int i = 0; i < inputs.Length; i++) + { + buffers[i] = Encoding.UTF8.GetBytes(inputs[i]); + } + return Create(buffers); + } + + public static ReadOnlySequence Create(params int[] inputs) where T : struct + { + T[][] buffers; + if (inputs.Length == 0) + { + buffers = new[] { new T[] { } }; + } + else + { + buffers = new T[inputs.Length][]; + for (int i = 0; i < inputs.Length; i++) + { + buffers[i] = new T[inputs[i]]; + } + } + return Create(buffers); + } + } +} diff --git a/src/System.Memory/tests/SequenceReader/SequenceSegment.cs b/src/System.Memory/tests/SequenceReader/SequenceSegment.cs new file mode 100644 index 000000000000..60132e41501c --- /dev/null +++ b/src/System.Memory/tests/SequenceReader/SequenceSegment.cs @@ -0,0 +1,131 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Buffers; +using System.Diagnostics; +using System.Runtime.InteropServices; +using System.Text; + +namespace System.Memory.Tests.SequenceReader +{ + public class SequenceSegment : ReadOnlySequenceSegment where T : struct + { + private SequenceSegment _next; + + public int Start { get; private set; } + + public int End + { + get => _end; + set + { + Debug.Assert(Start - value <= AvailableMemory.Length); + + _end = value; + Memory = AvailableMemory.Slice(Start, _end - Start); + } + } + + /// + /// Reference to the next block of data when the overall "active" bytes spans multiple blocks. At the point when the block is + /// leased Next is guaranteed to be null. Start, End, and Next are used together in order to create a linked-list of discontiguous + /// working memory. The "active" memory is grown when bytes are copied in, End is increased, and Next is assigned. The "active" + /// memory is shrunk when bytes are consumed, Start is increased, and blocks are returned to the pool. + /// + public SequenceSegment NextSegment + { + get => _next; + set + { + _next = value; + Next = value; + } + } + + /// + /// The buffer being tracked if segment owns the memory + /// + private IMemoryOwner _ownedMemory; + + private int _end; + + public void SetMemory(IMemoryOwner buffer) + { + SetMemory(buffer, 0, 0); + } + + public void SetMemory(IMemoryOwner ownedMemory, int start, int end, bool readOnly = false) + { + _ownedMemory = ownedMemory; + + AvailableMemory = _ownedMemory.Memory; + + ReadOnly = readOnly; + RunningIndex = 0; + Start = start; + End = end; + NextSegment = null; + } + + public void ResetMemory() + { + _ownedMemory.Dispose(); + _ownedMemory = null; + AvailableMemory = default; + } + + public Memory AvailableMemory { get; private set; } + + public int Length => End - Start; + + /// + /// If true, data should not be written into the backing block after the End offset. Data between start and end should never be modified + /// since this would break cloning. + /// + public bool ReadOnly { get; private set; } + + /// + /// The amount of writable bytes in this segment. It is the amount of bytes between Length and End + /// + public int WritableBytes => AvailableMemory.Length - End; + + /// + /// ToString overridden for debugger convenience. This displays the "active" byte information in this block as ASCII characters. + /// + /// + public override string ToString() + { + if (Memory.IsEmpty) + { + return ""; + } + + if (typeof(T) == typeof(byte)) + { + var builder = new StringBuilder(); + SpanLiteralExtensions.AppendAsLiteral(MemoryMarshal.AsBytes(Memory.Span), builder); + return builder.ToString(); + } + + return Memory.Span.ToString(); + } + + public void SetNext(SequenceSegment segment) + { + Debug.Assert(segment != null); + Debug.Assert(Next == null); + + NextSegment = segment; + + segment = this; + + while (segment.Next != null) + { + segment.NextSegment.RunningIndex = segment.RunningIndex + segment.Length; + segment = segment.NextSegment; + } + } + } +} diff --git a/src/System.Memory/tests/SequenceReader/SkipDelimiter.cs b/src/System.Memory/tests/SequenceReader/SkipDelimiter.cs new file mode 100644 index 000000000000..ec102056f05f --- /dev/null +++ b/src/System.Memory/tests/SequenceReader/SkipDelimiter.cs @@ -0,0 +1,262 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Buffers; +using System.Text; +using Xunit; + +namespace System.Memory.Tests.SequenceReader +{ + class SkipDelimiter + { + [Fact] + public void TryReadTo_SkipDelimiter() + { + byte[] expected = Encoding.UTF8.GetBytes("This is our ^|understanding^|"); + ReadOnlySequence bytes = SequenceFactory.CreateUtf8("This is our ^|understanding^|| you see."); + SequenceReader reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out ReadOnlySpan span, (byte)'|', (byte)'^', advancePastDelimiter: true)); + Assert.Equal(expected, span.ToArray()); + Assert.True(reader.IsNext((byte)' ')); + Assert.Equal(30, reader.Consumed); + + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false)); + Assert.Equal(expected, span.ToArray()); + Assert.True(reader.IsNext((byte)'|')); + Assert.Equal(29, reader.Consumed); + + // Put the skip delimiter in another segment + bytes = SequenceFactory.CreateUtf8("This is our ^|understanding", "^|| you see."); + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: true)); + Assert.Equal(expected, span.ToArray()); + Assert.True(reader.IsNext((byte)' ')); + Assert.Equal(30, reader.Consumed); + + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false)); + Assert.Equal(expected, span.ToArray()); + Assert.True(reader.IsNext((byte)'|')); + Assert.Equal(29, reader.Consumed); + + // Put the skip delimiter at the end of the segment + bytes = SequenceFactory.CreateUtf8("This is our ^|understanding^", "|| you see."); + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: true)); + Assert.Equal(expected, span.ToArray()); + Assert.True(reader.IsNext((byte)' ')); + Assert.Equal(30, reader.Consumed); + + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false)); + Assert.Equal(expected, span.ToArray()); + Assert.True(reader.IsNext((byte)'|')); + Assert.Equal(29, reader.Consumed); + + // No trailing data + bytes = SequenceFactory.CreateUtf8("This is our ^|understanding^||"); + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false)); + Assert.Equal(expected, span.ToArray()); + Assert.True(reader.IsNext((byte)'|')); + Assert.Equal(29, reader.Consumed); + + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: true)); + Assert.Equal(expected, span.ToArray()); + Assert.True(reader.End); + Assert.Equal(30, reader.Consumed); + + // All delimiters skipped + bytes = SequenceFactory.CreateUtf8("This is our ^|understanding^|"); + reader = new SequenceReader(bytes); + Assert.False(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false)); + Assert.Equal(0, reader.Consumed); + + reader = new SequenceReader(bytes); + Assert.False(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: true)); + Assert.Equal(0, reader.Consumed); + + bytes = SequenceFactory.CreateUtf8("abc^|de|"); + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: true)); + Assert.Equal(Encoding.UTF8.GetBytes("abc^|de"), span.ToArray()); + Assert.True(reader.End); + Assert.Equal(8, reader.Consumed); + + // Escape leads + bytes = SequenceFactory.CreateUtf8("^|a|b"); + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: true)); + Assert.Equal(Encoding.UTF8.GetBytes("^|a"), span.ToArray()); + Assert.True(reader.IsNext((byte)'b')); + Assert.Equal(4, reader.Consumed); + + // Delimiter starts second segment. + bytes = SequenceFactory.CreateUtf8("^", "|a|b"); + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: true)); + Assert.Equal(Encoding.UTF8.GetBytes("^|a"), span.ToArray()); + Assert.True(reader.IsNext((byte)'b')); + Assert.Equal(4, reader.Consumed); + } + + [Fact] + public void TryReadTo_SkipDelimiter_Runs() + { + ReadOnlySequence bytes = SequenceFactory.CreateUtf8("abc^^|def"); + SequenceReader reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out ReadOnlySpan span, (byte)'|', (byte)'^', advancePastDelimiter: false)); + Assert.Equal(Encoding.UTF8.GetBytes("abc^^"), span.ToArray()); + Assert.True(reader.IsNext((byte)'|')); + Assert.Equal(5, reader.Consumed); + + // Split after escape char + bytes = SequenceFactory.CreateUtf8("abc^^", "|def"); + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false)); + Assert.Equal(Encoding.UTF8.GetBytes("abc^^"), span.ToArray()); + Assert.True(reader.IsNext((byte)'|')); + Assert.Equal(5, reader.Consumed); + + // Split before and after escape char + bytes = SequenceFactory.CreateUtf8("abc^", "^", "|def"); + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false)); + Assert.Equal(Encoding.UTF8.GetBytes("abc^^"), span.ToArray()); + Assert.True(reader.IsNext((byte)'|')); + Assert.Equal(5, reader.Consumed); + + // Check advance past delimiter + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: true)); + Assert.Equal(Encoding.UTF8.GetBytes("abc^^"), span.ToArray()); + Assert.True(reader.IsNext((byte)'d')); + Assert.Equal(6, reader.Consumed); + + // Leading run of 2 + bytes = SequenceFactory.CreateUtf8("^^|abc"); + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false)); + Assert.Equal(Encoding.UTF8.GetBytes("^^"), span.ToArray()); + Assert.True(reader.IsNext((byte)'|')); + Assert.Equal(2, reader.Consumed); + + // Leading run of 3 + bytes = SequenceFactory.CreateUtf8("^^^|abc"); + reader = new SequenceReader(bytes); + Assert.False(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false)); + Assert.True(reader.IsNext((byte)'^')); + Assert.Equal(0, reader.Consumed); + + // Trailing run of 3 + bytes = SequenceFactory.CreateUtf8("abc^^^|"); + reader = new SequenceReader(bytes); + Assert.False(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false)); + Assert.True(reader.IsNext((byte)'a')); + Assert.Equal(0, reader.Consumed); + + // Trailing run of 3, split + bytes = SequenceFactory.CreateUtf8("abc^^^", "|"); + reader = new SequenceReader(bytes); + Assert.False(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false)); + Assert.True(reader.IsNext((byte)'a')); + Assert.Equal(0, reader.Consumed); + } + + [Fact] + public void TryReadTo_SkipDelimiter_ReadOnlySequence() + { + byte[] expected = Encoding.UTF8.GetBytes("This is our ^|understanding^|"); + ReadOnlySequence bytes = SequenceFactory.CreateUtf8("This is our ^|understanding^|| you see."); + SequenceReader reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out ReadOnlySequence sequence, (byte)'|', (byte)'^', advancePastDelimiter: true)); + Assert.Equal(expected, sequence.ToArray()); + Assert.True(reader.IsNext((byte)' ')); + Assert.Equal(30, reader.Consumed); + + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out sequence, (byte)'|', (byte)'^', advancePastDelimiter: false)); + Assert.Equal(expected, sequence.ToArray()); + Assert.True(reader.IsNext((byte)'|')); + Assert.Equal(29, reader.Consumed); + + // Put the skip delimiter in another segment + bytes = SequenceFactory.CreateUtf8("This is our ^|understanding", "^|| you see."); + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out sequence, (byte)'|', (byte)'^', advancePastDelimiter: true)); + Assert.Equal(expected, sequence.ToArray()); + Assert.True(reader.IsNext((byte)' ')); + Assert.Equal(30, reader.Consumed); + + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out sequence, (byte)'|', (byte)'^', advancePastDelimiter: false)); + Assert.Equal(expected, sequence.ToArray()); + Assert.True(reader.IsNext((byte)'|')); + Assert.Equal(29, reader.Consumed); + + // Put the skip delimiter at the end of the segment + bytes = SequenceFactory.CreateUtf8("This is our ^|understanding^", "|| you see."); + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out sequence, (byte)'|', (byte)'^', advancePastDelimiter: true)); + Assert.Equal(expected, sequence.ToArray()); + Assert.True(reader.IsNext((byte)' ')); + Assert.Equal(30, reader.Consumed); + + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out sequence, (byte)'|', (byte)'^', advancePastDelimiter: false)); + Assert.Equal(expected, sequence.ToArray()); + Assert.True(reader.IsNext((byte)'|')); + Assert.Equal(29, reader.Consumed); + + // No trailing data + bytes = SequenceFactory.CreateUtf8("This is our ^|understanding^||"); + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out sequence, (byte)'|', (byte)'^', advancePastDelimiter: false)); + Assert.Equal(expected, sequence.ToArray()); + Assert.True(reader.IsNext((byte)'|')); + Assert.Equal(29, reader.Consumed); + + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out sequence, (byte)'|', (byte)'^', advancePastDelimiter: true)); + Assert.Equal(expected, sequence.ToArray()); + Assert.True(reader.End); + Assert.Equal(30, reader.Consumed); + + // All delimiters skipped + bytes = SequenceFactory.CreateUtf8("This is our ^|understanding^|"); + reader = new SequenceReader(bytes); + Assert.False(reader.TryReadTo(out sequence, (byte)'|', (byte)'^', advancePastDelimiter: false)); + Assert.Equal(0, reader.Consumed); + + reader = new SequenceReader(bytes); + Assert.False(reader.TryReadTo(out sequence, (byte)'|', (byte)'^', advancePastDelimiter: true)); + Assert.Equal(0, reader.Consumed); + + bytes = SequenceFactory.CreateUtf8("abc^|de|"); + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out sequence, (byte)'|', (byte)'^', advancePastDelimiter: true)); + Assert.Equal(Encoding.UTF8.GetBytes("abc^|de"), sequence.ToArray()); + Assert.True(reader.End); + Assert.Equal(8, reader.Consumed); + + // Escape leads + bytes = SequenceFactory.CreateUtf8("^|a|b"); + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out sequence, (byte)'|', (byte)'^', advancePastDelimiter: true)); + Assert.Equal(Encoding.UTF8.GetBytes("^|a"), sequence.ToArray()); + Assert.True(reader.IsNext((byte)'b')); + Assert.Equal(4, reader.Consumed); + + // Delimiter starts second segment. + bytes = SequenceFactory.CreateUtf8("^", "|a|b"); + reader = new SequenceReader(bytes); + Assert.True(reader.TryReadTo(out sequence, (byte)'|', (byte)'^', advancePastDelimiter: true)); + Assert.Equal(Encoding.UTF8.GetBytes("^|a"), sequence.ToArray()); + Assert.True(reader.IsNext((byte)'b')); + Assert.Equal(4, reader.Consumed); + } + } +} diff --git a/src/System.Memory/tests/SequenceReader/SpanLiteralExtensions.cs b/src/System.Memory/tests/SequenceReader/SpanLiteralExtensions.cs new file mode 100644 index 000000000000..5f750f986ffe --- /dev/null +++ b/src/System.Memory/tests/SequenceReader/SpanLiteralExtensions.cs @@ -0,0 +1,72 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Text; + +namespace System.Memory.Tests.SequenceReader +{ + public class SpanLiteralExtensions + { + internal static void AppendAsLiteral(ReadOnlySpan span, StringBuilder sb) + { + for (int i = 0; i < span.Length; i++) + { + AppendCharLiteral((char)span[i], sb); + } + } + + internal static void AppendCharLiteral(char c, StringBuilder sb) + { + switch (c) + { + case '\'': + sb.Append(@"\'"); + break; + case '\"': + sb.Append("\\\""); + break; + case '\\': + sb.Append(@"\\"); + break; + case '\0': + sb.Append(@"\0"); + break; + case '\a': + sb.Append(@"\a"); + break; + case '\b': + sb.Append(@"\b"); + break; + case '\f': + sb.Append(@"\f"); + break; + case '\n': + sb.Append(@"\n"); + break; + case '\r': + sb.Append(@"\r"); + break; + case '\t': + sb.Append(@"\t"); + break; + case '\v': + sb.Append(@"\v"); + break; + default: + // ASCII printable character + if (!char.IsControl(c)) + { + sb.Append(c); + // As UTF16 escaped character + } + else + { + sb.Append(@"\u"); + sb.Append(((int)c).ToString("x4")); + } + break; + } + } + } +} diff --git a/src/System.Memory/tests/System.Memory.Tests.csproj b/src/System.Memory/tests/System.Memory.Tests.csproj index a8753e5a5cbd..86e4a12a68aa 100644 --- a/src/System.Memory/tests/System.Memory.Tests.csproj +++ b/src/System.Memory/tests/System.Memory.Tests.csproj @@ -18,6 +18,17 @@ + + + + + + + + + + + From 0910977f3f3ec22d2dadb840194da31f2a6caf4e Mon Sep 17 00:00:00 2001 From: Jeremy Kuhne Date: Wed, 7 Nov 2018 11:49:45 -0800 Subject: [PATCH 02/10] Address feedback, fix non coreapp test builds. --- src/System.Memory/ref/System.Memory.cs | 77 +++++++++---------- src/System.Memory/src/System.Memory.csproj | 8 +- .../Buffers/ReadOnlySequence_helpers.cs | 31 ++++---- .../src/System/Buffers/SequenceReader.cs | 45 +++++------ .../SequenceReaderExtensions_binary.cs | 25 +++--- .../System/Buffers/SequenceReader_search.cs | 74 ++++++++---------- .../tests/SequenceReader/BinaryExtensions.cs | 10 +-- .../tests/System.Memory.Tests.csproj | 6 +- 8 files changed, 135 insertions(+), 141 deletions(-) diff --git a/src/System.Memory/ref/System.Memory.cs b/src/System.Memory/ref/System.Memory.cs index 794a01bf8c2e..db11013888bd 100644 --- a/src/System.Memory/ref/System.Memory.cs +++ b/src/System.Memory/ref/System.Memory.cs @@ -97,14 +97,10 @@ public static void Reverse(this System.Span span) { } private readonly object _dummy; private readonly int _dummyPrimitive; public SequencePosition(object @object, int integer) { throw null; } - [System.ComponentModel.EditorBrowsableAttribute((System.ComponentModel.EditorBrowsableState)(1))] public override bool Equals(object obj) { throw null; } public bool Equals(System.SequencePosition other) { throw null; } - [System.ComponentModel.EditorBrowsableAttribute((System.ComponentModel.EditorBrowsableState)(1))] public override int GetHashCode() { throw null; } - [System.ComponentModel.EditorBrowsableAttribute((System.ComponentModel.EditorBrowsableState)(1))] public int GetInteger() { throw null; } - [System.ComponentModel.EditorBrowsableAttribute((System.ComponentModel.EditorBrowsableState)(1))] public object GetObject() { throw null; } } } @@ -184,51 +180,50 @@ public partial struct Enumerator public bool MoveNext() { throw null; } } } - public ref partial struct SequenceReader where T : unmanaged, System.IEquatable + public static partial class SequenceReaderExtensions { - public SequenceReader(System.Buffers.ReadOnlySequence buffer) { } - public bool IsLastSegment { get { throw null; } } - public bool End { get { throw null; } } - public ReadOnlySequence Sequence { get { throw null; } } - public SequencePosition Position { get { throw null; } } - public ReadOnlySpan CurrentSpan { get { throw null; } } - public int CurrentSpanIndex { get { throw null; } } - public ReadOnlySpan UnreadSpan { get { throw null; } } + public static bool TryReadBigEndian(this ref System.Buffers.SequenceReader reader, out short value) { throw null; } + public static bool TryReadBigEndian(this ref System.Buffers.SequenceReader reader, out int value) { throw null; } + public static bool TryReadBigEndian(this ref System.Buffers.SequenceReader reader, out long value) { throw null; } + public static bool TryReadLittleEndian(this ref System.Buffers.SequenceReader reader, out short value) { throw null; } + public static bool TryReadLittleEndian(this ref System.Buffers.SequenceReader reader, out int value) { throw null; } + public static bool TryReadLittleEndian(this ref System.Buffers.SequenceReader reader, out long value) { throw null; } + public static bool TryRead(this ref System.Buffers.SequenceReader reader, out T value) where T : struct { throw null; } + } + public ref partial struct SequenceReader where T : struct, System.IEquatable + { + public SequenceReader(System.Buffers.ReadOnlySequence buffer) { throw null; } public long Consumed { get { throw null; } } - public long Remaining { get { throw null; } } + public System.ReadOnlySpan CurrentSpan { get { throw null; } } + public int CurrentSpanIndex { get { throw null; } } + public bool End { get { throw null; } } public long Length { get { throw null; } } - public ReadOnlySpan Peek(Span copyBuffer) { throw null; } - public bool TryPeek(out T value) { throw null; } - public void Advance(long count) { throw null; } - public void Rewind(long count) { throw null; } - public bool TryRead(out T value) { throw null; } - public bool TryReadTo(out ReadOnlySpan span, T delimiter) { throw null; } - public bool TryReadTo(out ReadOnlySpan span, T delimiter, bool advancePastDelimiter) { throw null; } - public bool TryReadTo(out ReadOnlySpan span, T delimiter, T delimiterEscape, bool advancePastDelimiter = true) { throw null; } - public bool TryReadTo(out ReadOnlySequence sequence, T delimiter, bool advancePastDelimiter = true) { throw null; } - public bool TryReadTo(out ReadOnlySequence sequence, T delimiter, T delimiterEscape, bool advancePastDelimiter = true) { throw null; } - public bool TryReadToAny(out ReadOnlySpan span, ReadOnlySpan delimiters, bool advancePastDelimiter = true) { throw null; } - public bool TryReadToAny(out ReadOnlySequence sequence, ReadOnlySpan delimiters, bool advancePastDelimiter = true) { throw null; } - public bool TryReadTo(out ReadOnlySequence sequence, ReadOnlySpan delimiter, bool advancePastDelimiter = true) { throw null; } - public bool TryAdvanceTo(T delimiter, bool advancePastDelimiter = true) { throw null; } - public bool TryAdvanceToAny(ReadOnlySpan delimiters, bool advancePastDelimiter = true) { throw null; } + public System.SequencePosition Position { get { throw null; } } + public long Remaining { get { throw null; } } + public System.Buffers.ReadOnlySequence Sequence { get { throw null; } } + public System.ReadOnlySpan UnreadSpan { get { throw null; } } + public void Advance(long count) { } public long AdvancePast(T value) { throw null; } + public long AdvancePastAny(System.ReadOnlySpan values) { throw null; } public long AdvancePastAny(T value0, T value1) { throw null; } public long AdvancePastAny(T value0, T value1, T value2) { throw null; } public long AdvancePastAny(T value0, T value1, T value2, T value3) { throw null; } - public long AdvancePastAny(ReadOnlySpan values) { throw null; } + public bool IsNext(System.ReadOnlySpan next, bool advancePast = false) { throw null; } public bool IsNext(T next, bool advancePast = false) { throw null; } - public bool IsNext(ReadOnlySpan next, bool advancePast = false) { throw null; } - } - public static partial class SequenceReaderExtensions - { - public static bool TryRead(ref this SequenceReader reader, out T value) where T : unmanaged { throw null; } - public static bool TryReadInt16LittleEndian(ref this SequenceReader reader, out short value) { throw null; } - public static bool TryReadInt16BigEndian(ref this SequenceReader reader, out short value) { throw null; } - public static bool TryReadInt32LittleEndian(ref this SequenceReader reader, out int value) { throw null; } - public static bool TryReadInt32BigEndian(ref this SequenceReader reader, out int value) { throw null; } - public static bool TryReadInt64LittleEndian(ref this SequenceReader reader, out long value) { throw null; } - public static bool TryReadInt64BigEndian(ref this SequenceReader reader, out long value) { throw null; } + public System.ReadOnlySpan Peek(System.Span copyBuffer) { throw null; } + public void Rewind(long count) { } + public bool TryAdvanceTo(T delimiter, bool advancePastDelimiter = true) { throw null; } + public bool TryAdvanceToAny(System.ReadOnlySpan delimiters, bool advancePastDelimiter = true) { throw null; } + public bool TryPeek(out T value) { throw null; } + public bool TryRead(out T value) { throw null; } + public bool TryReadTo(out System.Buffers.ReadOnlySequence sequence, System.ReadOnlySpan delimiter, bool advancePastDelimiter = true) { throw null; } + public bool TryReadTo(out System.Buffers.ReadOnlySequence sequence, T delimiter, bool advancePastDelimiter = true) { throw null; } + public bool TryReadTo(out System.Buffers.ReadOnlySequence sequence, T delimiter, T delimiterEscape, bool advancePastDelimiter = true) { throw null; } + public bool TryReadTo(out System.ReadOnlySpan span, T delimiter) { throw null; } + public bool TryReadTo(out System.ReadOnlySpan span, T delimiter, bool advancePastDelimiter) { throw null; } + public bool TryReadTo(out System.ReadOnlySpan span, T delimiter, T delimiterEscape, bool advancePastDelimiter = true) { throw null; } + public bool TryReadToAny(out System.Buffers.ReadOnlySequence sequence, System.ReadOnlySpan delimiters, bool advancePastDelimiter = true) { throw null; } + public bool TryReadToAny(out System.ReadOnlySpan span, System.ReadOnlySpan delimiters, bool advancePastDelimiter = true) { throw null; } } public readonly partial struct StandardFormat : System.IEquatable { diff --git a/src/System.Memory/src/System.Memory.csproj b/src/System.Memory/src/System.Memory.csproj index baf32b646edd..20f19f7179b1 100644 --- a/src/System.Memory/src/System.Memory.csproj +++ b/src/System.Memory/src/System.Memory.csproj @@ -1,4 +1,4 @@ - + {4BBC8F69-D03E-4432-AA8A-D458FA5B235A} true @@ -8,9 +8,6 @@ netcoreapp-Unix-Debug;netcoreapp-Unix-Release;netcoreapp-Windows_NT-Debug;netcoreapp-Windows_NT-Release;netcoreappaot-Windows_NT-Debug;netcoreappaot-Windows_NT-Release;uap-Windows_NT-Debug;uap-Windows_NT-Release - - - @@ -23,6 +20,9 @@ + + + diff --git a/src/System.Memory/src/System/Buffers/ReadOnlySequence_helpers.cs b/src/System.Memory/src/System/Buffers/ReadOnlySequence_helpers.cs index 8855788466ed..2a95e09e8fa8 100644 --- a/src/System.Memory/src/System/Buffers/ReadOnlySequence_helpers.cs +++ b/src/System.Memory/src/System/Buffers/ReadOnlySequence_helpers.cs @@ -4,6 +4,7 @@ using System.Diagnostics; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using Internal.Runtime.CompilerServices; namespace System.Buffers @@ -486,17 +487,17 @@ internal void GetFirstSpan(out ReadOnlySpan first, out SequencePosition next) { SequencePosition end = End; int endIndex = end.GetInteger(); - bool isMultiSegment = startObject != end.GetObject(); + bool hasMultipleSegments = startObject != end.GetObject(); - // A == 0 && B == 0 means SequenceType.MultiSegment if (startIndex >= 0) { - if (endIndex >= 0) // SequenceType.MultiSegment + if (endIndex >= 0) { + // Positive start and end index == ReadOnlySequenceSegment ReadOnlySequenceSegment segment = (ReadOnlySequenceSegment)startObject; next = new SequencePosition(segment.Next, 0); first = segment.Memory.Span; - if (isMultiSegment) + if (hasMultipleSegments) { first = first.Slice(startIndex); } @@ -507,7 +508,8 @@ internal void GetFirstSpan(out ReadOnlySpan first, out SequencePosition next) } else { - if (isMultiSegment) + // Positive start and negative end index == T[] + if (hasMultipleSegments) ThrowHelper.ThrowInvalidOperationException_EndPositionNotReached(); first = new ReadOnlySpan((T[])startObject, startIndex, (endIndex & ReadOnlySequence.IndexBitMask) - startIndex); @@ -515,30 +517,29 @@ internal void GetFirstSpan(out ReadOnlySpan first, out SequencePosition next) } else { - first = GetFirstSpanSlow(startObject, startIndex, endIndex, isMultiSegment); + first = GetFirstSpanSlow(startObject, startIndex, endIndex, hasMultipleSegments); } } } [MethodImpl(MethodImplOptions.NoInlining)] - private static ReadOnlySpan GetFirstSpanSlow(object startObject, int startIndex, int endIndex, bool isMultiSegment) + private static ReadOnlySpan GetFirstSpanSlow(object startObject, int startIndex, int endIndex, bool hasMultipleSegments) { - Debug.Assert(startIndex < 0 || endIndex < 0); - if (isMultiSegment) + Debug.Assert(startIndex < 0); + if (hasMultipleSegments) ThrowHelper.ThrowInvalidOperationException_EndPositionNotReached(); // The type == char check here is redundant. However, we still have it to allow // the JIT to see when that the code is unreachable and eliminate it. - // A == 1 && B == 1 means SequenceType.String if (typeof(T) == typeof(char) && endIndex < 0) { - var memory = (ReadOnlyMemory)(object)((string)startObject).AsMemory(); - - // No need to remove the FlagBitMask since (endIndex - startIndex) == (endIndex & ReadOnlySequence.IndexBitMask) - (startIndex & ReadOnlySequence.IndexBitMask) - return memory.Span.Slice(startIndex & ReadOnlySequence.IndexBitMask, endIndex - startIndex); + // Negative start and negative end index == string + ReadOnlySpan spanOfChar = ((string)startObject).AsSpan(startIndex & ReadOnlySequence.IndexBitMask, endIndex - startIndex); + return MemoryMarshal.CreateSpan(ref Unsafe.As(ref MemoryMarshal.GetReference(spanOfChar)), spanOfChar.Length); } - else // endIndex >= 0, A == 1 && B == 0 means SequenceType.MemoryManager + else { + // Negative start and positive end index == MemoryManager startIndex &= ReadOnlySequence.IndexBitMask; return ((MemoryManager)startObject).Memory.Span.Slice(startIndex, endIndex - startIndex); } diff --git a/src/System.Memory/src/System/Buffers/SequenceReader.cs b/src/System.Memory/src/System/Buffers/SequenceReader.cs index 565ed3e22b1c..0a14ec6260cc 100644 --- a/src/System.Memory/src/System/Buffers/SequenceReader.cs +++ b/src/System.Memory/src/System/Buffers/SequenceReader.cs @@ -18,30 +18,25 @@ namespace System.Buffers /// Create a over the given . /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public SequenceReader(ReadOnlySequence buffer) + public SequenceReader(ReadOnlySequence sequence) { CurrentSpanIndex = 0; Consumed = 0; - Sequence = buffer; - _currentPosition = buffer.Start; + Sequence = sequence; + _currentPosition = sequence.Start; _length = -1; - buffer.GetFirstSpan(out ReadOnlySpan first, out _nextPosition); + sequence.GetFirstSpan(out ReadOnlySpan first, out _nextPosition); CurrentSpan = first; _moreData = first.Length > 0; - if (!_moreData && !buffer.IsSingleSegment) + if (!_moreData && !sequence.IsSingleSegment) { _moreData = true; GetNextSpan(); } } - /// - /// Return true if we're in the last segment - /// - public bool IsLastSegment => _nextPosition.GetObject() == null; - /// /// True when there is no more data in the . /// @@ -59,7 +54,7 @@ public SequencePosition Position => Sequence.GetPosition(CurrentSpanIndex, _currentPosition); /// - /// The current segment in the . + /// The current segment in the as a span. /// public ReadOnlySpan CurrentSpan { get; private set; } @@ -96,6 +91,7 @@ public long Length { if (_length < 0) { + // Cache the length _length = Sequence.Length; } return _length; @@ -149,7 +145,7 @@ public bool TryRead(out T value) } /// - /// Move the reader back the specified number of positions. + /// Move the reader back the specified number of items. /// [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Rewind(long count) @@ -168,7 +164,7 @@ public void Rewind(long count) } else { - // Current segment doesn't have enough space, scan backward through segments + // Current segment doesn't have enough data, scan backward through segments RetreatToPreviousSpan(Consumed); } } @@ -194,7 +190,7 @@ private void ResetReader() if (memory.Length == 0) { CurrentSpan = default; - // No space in the first span, move to one with space + // No data in the first span, move to one with data GetNextSpan(); } else @@ -204,14 +200,14 @@ private void ResetReader() } else { - // No space in any spans and at end of sequence + // No data in any spans and at end of sequence _moreData = false; CurrentSpan = default; } } /// - /// Get the next segment with available space, if any. + /// Get the next segment with available data, if any. /// private void GetNextSpan() { @@ -239,7 +235,7 @@ private void GetNextSpan() } /// - /// Move the reader ahead the specified number of positions. + /// Move the reader ahead the specified number of items. /// [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Advance(long count) @@ -312,24 +308,26 @@ private void AdvanceToNextSpan(long count) GetNextSpan(); if (count == 0) + { break; + } } if (count != 0) { - // Not enough space left- adjust for where we actually ended and throw + // Not enough data left- adjust for where we actually ended and throw Consumed -= count; ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.count); } } /// - /// Peek forward the number of positions in (at most), copying into + /// Peek forward the number of items in (at most), copying into /// if needed. /// /// /// Temporary buffer to copy into if there isn't a contiguous span within the existing data to return. - /// Also describes the maximum count of positions to peek. + /// Also describes the maximum count of items to peek. /// /// /// Span over the peeked data. The length may be shorter than if there @@ -350,20 +348,23 @@ public ReadOnlySpan Peek(Span copyBuffer) internal ReadOnlySpan PeekSlow(Span destination) { ReadOnlySpan firstSpan = UnreadSpan; + Debug.Assert(firstSpan.Length < destination.Length); firstSpan.CopyTo(destination); int copied = firstSpan.Length; SequencePosition next = _nextPosition; while (Sequence.TryGet(ref next, out ReadOnlyMemory nextSegment, true)) { - ReadOnlySpan nextSpan = nextSegment.Span; - if (nextSpan.Length > 0) + if (nextSegment.Length > 0) { + ReadOnlySpan nextSpan = nextSegment.Span; int toCopy = Math.Min(nextSpan.Length, destination.Length - copied); nextSpan.Slice(0, toCopy).CopyTo(destination.Slice(copied)); copied += toCopy; if (copied >= destination.Length) + { break; + } } } diff --git a/src/System.Memory/src/System/Buffers/SequenceReaderExtensions_binary.cs b/src/System.Memory/src/System/Buffers/SequenceReaderExtensions_binary.cs index f53132d4f78c..c4a32a043daa 100644 --- a/src/System.Memory/src/System/Buffers/SequenceReaderExtensions_binary.cs +++ b/src/System.Memory/src/System/Buffers/SequenceReaderExtensions_binary.cs @@ -12,13 +12,16 @@ namespace System.Buffers public static partial class SequenceReaderExtensions { /// - /// Try to read the given type out of the buffer if possible. + /// Try to read the given type out of the buffer if possible. Warning: this is dangerous to use with arbitrary + /// structs- see remarks for full details. /// /// - /// The read is unaligned. + /// IMPORTANT: The read is a straight copy of bits. If a struct depends on specific state of it's members to + /// behave correctly this can lead to exceptions, etc. If reading endian specific integers, use the explicit + /// overloads such as /// /// - /// True if successful. will be default if failed. + /// True if successful. will be default if failed (due to lack of space). /// public static unsafe bool TryRead(ref this SequenceReader reader, out T value) where T : unmanaged { @@ -36,8 +39,8 @@ private static unsafe bool TryReadSlow(ref SequenceReader reader, out T Debug.Assert(reader.UnreadSpan.Length < sizeof(T)); // Not enough data in the current segment, try to peek for the data we need. - byte* buffer = stackalloc byte[sizeof(T)]; - Span tempSpan = new Span(buffer, sizeof(T)); + T buffer = default; + Span tempSpan = new Span(&buffer, sizeof(T)); if (reader.Peek(tempSpan).Length < sizeof(T)) { @@ -54,7 +57,7 @@ private static unsafe bool TryReadSlow(ref SequenceReader reader, out T /// Reads an as little endian. /// /// False if there wasn't enough data for an . - public static bool TryReadInt16LittleEndian(ref this SequenceReader reader, out short value) + public static bool TryReadLittleEndian(ref this SequenceReader reader, out short value) { if (BitConverter.IsLittleEndian) { @@ -68,7 +71,7 @@ public static bool TryReadInt16LittleEndian(ref this SequenceReader reader /// Reads an as big endian. /// /// False if there wasn't enough data for an . - public static bool TryReadInt16BigEndian(ref this SequenceReader reader, out short value) + public static bool TryReadBigEndian(ref this SequenceReader reader, out short value) { if (!BitConverter.IsLittleEndian) { @@ -93,7 +96,7 @@ private static bool TryReadReverseEndianness(ref SequenceReader reader, ou /// Reads an as little endian. /// /// False if there wasn't enough data for an . - public static bool TryReadInt32LittleEndian(ref this SequenceReader reader, out int value) + public static bool TryReadLittleEndian(ref this SequenceReader reader, out int value) { if (BitConverter.IsLittleEndian) { @@ -107,7 +110,7 @@ public static bool TryReadInt32LittleEndian(ref this SequenceReader reader /// Reads an as big endian. /// /// False if there wasn't enough data for an . - public static bool TryReadInt32BigEndian(ref this SequenceReader reader, out int value) + public static bool TryReadBigEndian(ref this SequenceReader reader, out int value) { if (!BitConverter.IsLittleEndian) { @@ -132,7 +135,7 @@ private static bool TryReadReverseEndianness(ref SequenceReader reader, ou /// Reads an as little endian. /// /// False if there wasn't enough data for an . - public static bool TryReadInt64LittleEndian(ref this SequenceReader reader, out long value) + public static bool TryReadLittleEndian(ref this SequenceReader reader, out long value) { if (BitConverter.IsLittleEndian) { @@ -146,7 +149,7 @@ public static bool TryReadInt64LittleEndian(ref this SequenceReader reader /// Reads an as big endian. /// /// False if there wasn't enough data for an . - public static bool TryReadInt64BigEndian(ref this SequenceReader reader, out long value) + public static bool TryReadBigEndian(ref this SequenceReader reader, out long value) { if (!BitConverter.IsLittleEndian) { diff --git a/src/System.Memory/src/System/Buffers/SequenceReader_search.cs b/src/System.Memory/src/System/Buffers/SequenceReader_search.cs index 7a265473e444..0b326b870d11 100644 --- a/src/System.Memory/src/System/Buffers/SequenceReader_search.cs +++ b/src/System.Memory/src/System/Buffers/SequenceReader_search.cs @@ -11,7 +11,7 @@ namespace System.Buffers { /// /// Try to read everything up to the given . The reader will be positioned - /// after the first occurence of when successful. + /// after the first occurrence of when successful. /// /// The read data, if any. /// The delimiter to look for. @@ -65,11 +65,11 @@ private bool TryReadToSlow(out ReadOnlySpan span, T delimiter, bool advancePa /// /// Try to read everything up to the given , ignoring delimiters that are - /// preceeded by . + /// preceded by . /// /// The read data, if any. /// The delimiter to look for. - /// If found prior to it will skip that occurence. + /// If found prior to it will skip that occurrence. /// True to move past the if found. /// True if the was found. public bool TryReadTo(out ReadOnlySpan span, T delimiter, T delimiterEscape, bool advancePastDelimiter = true) @@ -80,7 +80,7 @@ public bool TryReadTo(out ReadOnlySpan span, T delimiter, T delimiterEscape, if ((index > 0 && !remaining[index - 1].Equals(delimiterEscape)) || index == 0) { span = remaining.Slice(0, index); - Advance(index + (advancePastDelimiter ? 1 : 0)); + AdvanceCurrentSpan(index + (advancePastDelimiter ? 1 : 0)); return true; } @@ -141,16 +141,17 @@ private bool TryReadToSlow(out ReadOnlySequence sequence, T delimiter, T deli if ((escapeCount & 1) != 0) { - priorEscape = false; - // We're in the escaped state, so skip this delimiter + // An odd escape count means we're currently escaped, + // skip the delimiter and reset escaped state. Advance(index + 1); + priorEscape = false; remaining = UnreadSpan; goto Continue; } } // Found the delimiter. Move to it, slice, then move past it. - Advance(index); + AdvanceCurrentSpan(index); sequence = Sequence.Slice(copy.Position, Position); if (advancePastDelimiter) @@ -185,7 +186,7 @@ private bool TryReadToSlow(out ReadOnlySequence sequence, T delimiter, T deli } // Nothing in the current span, move to the end, checking for the skip delimiter - Advance(remaining.Length); + AdvanceCurrentSpan(remaining.Length); remaining = CurrentSpan; Continue: @@ -212,12 +213,13 @@ public bool TryReadTo(out ReadOnlySequence sequence, T delimiter, bool advanc private bool TryReadToInternal(out ReadOnlySequence sequence, T delimiter, bool advancePastDelimiter, int skip = 0) { + Debug.Assert(skip >= 0); SequenceReader copy = this; if (skip > 0) Advance(skip); ReadOnlySpan remaining = UnreadSpan; - while (!End) + while (_moreData) { int index = remaining.IndexOf(delimiter); if (index != -1) @@ -225,7 +227,7 @@ private bool TryReadToInternal(out ReadOnlySequence sequence, T delimiter, bo // Found the delimiter. Move to it, slice, then move past it. if (index > 0) { - Advance(index); + AdvanceCurrentSpan(index); } sequence = Sequence.Slice(copy.Position, Position); @@ -236,7 +238,7 @@ private bool TryReadToInternal(out ReadOnlySequence sequence, T delimiter, bo return true; } - Advance(remaining.Length); + AdvanceCurrentSpan(remaining.Length); remaining = CurrentSpan; } @@ -248,11 +250,11 @@ private bool TryReadToInternal(out ReadOnlySequence sequence, T delimiter, bo /// /// Try to read everything up to the given , ignoring delimiters that are - /// preceeded by . + /// preceded by . /// /// The read data, if any. /// The delimiter to look for. - /// If found prior to it will skip that occurence. + /// If found prior to it will skip that occurrence. /// True to move past the if found. /// True if the was found. public bool TryReadTo(out ReadOnlySequence sequence, T delimiter, T delimiterEscape, bool advancePastDelimiter = true) @@ -262,7 +264,7 @@ public bool TryReadTo(out ReadOnlySequence sequence, T delimiter, T delimiter ReadOnlySpan remaining = UnreadSpan; bool priorEscape = false; - while (!End) + while (_moreData) { int index = remaining.IndexOf(delimiter); if (index != -1) @@ -290,9 +292,9 @@ public bool TryReadTo(out ReadOnlySequence sequence, T delimiter, T delimiter } priorEscape = false; - if (escapeCount % 2 != 0) + if ((escapeCount & 1) != 0) { - // We're in the escaped state, so skip this delimiter + // Odd escape count means we're in the escaped state, so skip this delimiter Advance(index + 1); remaining = UnreadSpan; continue; @@ -390,7 +392,7 @@ private bool TryReadToAnyInternal(out ReadOnlySequence sequence, ReadOnlySpan SequenceReader copy = this; if (skip > 0) Advance(skip); - ReadOnlySpan remaining = CurrentSpanIndex == 0 ? CurrentSpan : UnreadSpan; + ReadOnlySpan remaining = UnreadSpan; while (!End) { @@ -403,7 +405,7 @@ private bool TryReadToAnyInternal(out ReadOnlySequence sequence, ReadOnlySpan // Found one of the delimiters. Move to it, slice, then move past it. if (index > 0) { - Advance(index); + AdvanceCurrentSpan(index); } sequence = Sequence.Slice(copy.Position, Position); @@ -425,13 +427,13 @@ private bool TryReadToAnyInternal(out ReadOnlySequence sequence, ReadOnlySpan } /// - /// Try to read data until the given sequence. + /// Try to read data until the given . /// /// The read data, if any. /// The multi (T) delimiter. - /// True to move past the sequence if found. + /// True to move past the if found. /// True if the was found. - public unsafe bool TryReadTo(out ReadOnlySequence sequence, ReadOnlySpan delimiter, bool advancePastDelimiter = true) + public bool TryReadTo(out ReadOnlySequence sequence, ReadOnlySpan delimiter, bool advancePastDelimiter = true) { if (delimiter.Length == 0) { @@ -441,17 +443,6 @@ public unsafe bool TryReadTo(out ReadOnlySequence sequence, ReadOnlySpan d SequenceReader copy = this; - Span peekBuffer; - if (delimiter.Length * sizeof(T) < 512) - { - T* t = stackalloc T[delimiter.Length]; - peekBuffer = new Span(t, delimiter.Length); - } - else - { - peekBuffer = new Span(new T[delimiter.Length]); - } - bool advanced = false; while (!End) { @@ -466,12 +457,14 @@ public unsafe bool TryReadTo(out ReadOnlySequence sequence, ReadOnlySpan d return true; } - ReadOnlySpan next = Peek(peekBuffer); - if (next.SequenceEqual(delimiter)) + if (IsNext(delimiter)) { - //TODO: Figure out a faster way to do this, potentially by avoiding the Advance in the previous TryReadTo call + // Probably a faster way to do this, potentially by avoiding the Advance in the previous TryReadTo call if (advanced) + { sequence = copy.Sequence.Slice(copy.Consumed, Consumed - copy.Consumed); + } + if (advancePastDelimiter) { Advance(delimiter.Length); @@ -514,14 +507,14 @@ public bool TryAdvanceTo(T delimiter, bool advancePastDelimiter = true) /// /// The delimiters to search for. /// True to move past the first found instance of any of the given . - /// True if any of the given was found. + /// True if any of the given were found. public bool TryAdvanceToAny(ReadOnlySpan delimiters, bool advancePastDelimiter = true) { ReadOnlySpan remaining = UnreadSpan; int index = remaining.IndexOfAny(delimiters); if (index != -1) { - AdvanceCurrentSpan(advancePastDelimiter ? index + 1 : index); + AdvanceCurrentSpan(index + (advancePastDelimiter ? 1 : 0)); return true; } @@ -706,7 +699,7 @@ public long AdvancePastAny(T value0, T value1) /// /// Check to see if the given value is next. /// - /// The value to compare the next position to. + /// The value to compare the next items to. /// Move past the value if found. [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool IsNext(T next, bool advancePast = false) @@ -714,8 +707,7 @@ public bool IsNext(T next, bool advancePast = false) if (End) return false; - ReadOnlySpan unread = UnreadSpan; - if (unread[0].Equals(next)) + if (CurrentSpan[CurrentSpanIndex].Equals(next)) { if (advancePast) { @@ -729,7 +721,7 @@ public bool IsNext(T next, bool advancePast = false) /// /// Check to see if the given values are next. /// - /// The sequence to compare the next position to. + /// The span to compare the next items to. /// Move past the values if found. [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool IsNext(ReadOnlySpan next, bool advancePast = false) diff --git a/src/System.Memory/tests/SequenceReader/BinaryExtensions.cs b/src/System.Memory/tests/SequenceReader/BinaryExtensions.cs index a6d9d0ed0f57..851715ce7286 100644 --- a/src/System.Memory/tests/SequenceReader/BinaryExtensions.cs +++ b/src/System.Memory/tests/SequenceReader/BinaryExtensions.cs @@ -50,19 +50,19 @@ public void MultiSegmentBytesReaderNumbers() Assert.True(reader.TryRead(out int intValue)); Assert.Equal(BitConverter.ToInt32(new byte[] { 0, 1, 0, 2 }), intValue); - Assert.True(reader.TryReadInt32BigEndian(out intValue)); + Assert.True(reader.TryReadBigEndian(out intValue)); Assert.Equal(BitConverter.ToInt32(new byte[] { 4, 3, 2, 1 }), intValue); - Assert.True(reader.TryReadInt64LittleEndian(out long longValue)); + Assert.True(reader.TryReadLittleEndian(out long longValue)); Assert.Equal(BitConverter.ToInt64(new byte[] { 5, 6, 7, 8, 9, 0, 1, 2 }), longValue); - Assert.True(reader.TryReadInt64BigEndian(out longValue)); + Assert.True(reader.TryReadBigEndian(out longValue)); Assert.Equal(BitConverter.ToInt64(new byte[] { 0, 9, 8, 7, 6, 5, 4, 3 }), longValue); - Assert.True(reader.TryReadInt16LittleEndian(out short shortValue)); + Assert.True(reader.TryReadLittleEndian(out short shortValue)); Assert.Equal(BitConverter.ToInt16(new byte[] { 1, 2 }), shortValue); - Assert.True(reader.TryReadInt16BigEndian(out shortValue)); + Assert.True(reader.TryReadBigEndian(out shortValue)); Assert.Equal(BitConverter.ToInt16(new byte[] { 4, 3 }), shortValue); } } diff --git a/src/System.Memory/tests/System.Memory.Tests.csproj b/src/System.Memory/tests/System.Memory.Tests.csproj index 86e4a12a68aa..9b755c2daba1 100644 --- a/src/System.Memory/tests/System.Memory.Tests.csproj +++ b/src/System.Memory/tests/System.Memory.Tests.csproj @@ -16,8 +16,7 @@ - - + @@ -29,6 +28,9 @@ + + + From 021075d1cc8a4d4e2e5a14ef6b58684b1b395aa0 Mon Sep 17 00:00:00 2001 From: Jeremy Kuhne Date: Wed, 7 Nov 2018 13:57:42 -0800 Subject: [PATCH 03/10] Replace Peek() with TryCopyTo() --- src/System.Memory/ref/System.Memory.cs | 2 +- .../src/System/Buffers/SequenceReader.cs | 29 ++++++++--------- .../SequenceReaderExtensions_binary.cs | 6 ++-- .../tests/SequenceReader/BasicTests.cs | 32 ++----------------- 4 files changed, 20 insertions(+), 49 deletions(-) diff --git a/src/System.Memory/ref/System.Memory.cs b/src/System.Memory/ref/System.Memory.cs index db11013888bd..fd53788facf3 100644 --- a/src/System.Memory/ref/System.Memory.cs +++ b/src/System.Memory/ref/System.Memory.cs @@ -210,10 +210,10 @@ public void Advance(long count) { } public long AdvancePastAny(T value0, T value1, T value2, T value3) { throw null; } public bool IsNext(System.ReadOnlySpan next, bool advancePast = false) { throw null; } public bool IsNext(T next, bool advancePast = false) { throw null; } - public System.ReadOnlySpan Peek(System.Span copyBuffer) { throw null; } public void Rewind(long count) { } public bool TryAdvanceTo(T delimiter, bool advancePastDelimiter = true) { throw null; } public bool TryAdvanceToAny(System.ReadOnlySpan delimiters, bool advancePastDelimiter = true) { throw null; } + public bool TryCopyTo(System.Span destination) { throw null; } public bool TryPeek(out T value) { throw null; } public bool TryRead(out T value) { throw null; } public bool TryReadTo(out System.Buffers.ReadOnlySequence sequence, System.ReadOnlySpan delimiter, bool advancePastDelimiter = true) { throw null; } diff --git a/src/System.Memory/src/System/Buffers/SequenceReader.cs b/src/System.Memory/src/System/Buffers/SequenceReader.cs index 0a14ec6260cc..f156582a8475 100644 --- a/src/System.Memory/src/System/Buffers/SequenceReader.cs +++ b/src/System.Memory/src/System/Buffers/SequenceReader.cs @@ -322,31 +322,28 @@ private void AdvanceToNextSpan(long count) } /// - /// Peek forward the number of items in (at most), copying into - /// if needed. + /// Copies data from the current to the given span. /// - /// - /// Temporary buffer to copy into if there isn't a contiguous span within the existing data to return. - /// Also describes the maximum count of items to peek. - /// - /// - /// Span over the peeked data. The length may be shorter than if there - /// is not enough data left to fill the requested length. - /// + /// Destination to copy to. + /// True if there is enough data to copy to the . [MethodImpl(MethodImplOptions.AggressiveInlining)] - public ReadOnlySpan Peek(Span copyBuffer) + public bool TryCopyTo(Span destination) { ReadOnlySpan firstSpan = UnreadSpan; - if (firstSpan.Length >= copyBuffer.Length) + if (firstSpan.Length >= destination.Length) { - return firstSpan.Slice(0, copyBuffer.Length); + firstSpan.Slice(0, destination.Length).CopyTo(destination); + return true; } - return PeekSlow(copyBuffer); + return TryCopyMultisegment(destination); } - internal ReadOnlySpan PeekSlow(Span destination) + internal bool TryCopyMultisegment(Span destination) { + if (Remaining < destination.Length) + return false; + ReadOnlySpan firstSpan = UnreadSpan; Debug.Assert(firstSpan.Length < destination.Length); firstSpan.CopyTo(destination); @@ -368,7 +365,7 @@ internal ReadOnlySpan PeekSlow(Span destination) } } - return destination.Slice(0, copied); + return true; } } } diff --git a/src/System.Memory/src/System/Buffers/SequenceReaderExtensions_binary.cs b/src/System.Memory/src/System/Buffers/SequenceReaderExtensions_binary.cs index c4a32a043daa..7bbe9ad50341 100644 --- a/src/System.Memory/src/System/Buffers/SequenceReaderExtensions_binary.cs +++ b/src/System.Memory/src/System/Buffers/SequenceReaderExtensions_binary.cs @@ -27,14 +27,14 @@ public static unsafe bool TryRead(ref this SequenceReader reader, out T { ReadOnlySpan span = reader.UnreadSpan; if (span.Length < sizeof(T)) - return TryReadSlow(ref reader, out value); + return TryReadMultisegment(ref reader, out value); value = Unsafe.ReadUnaligned(ref MemoryMarshal.GetReference(span)); reader.Advance(sizeof(T)); return true; } - private static unsafe bool TryReadSlow(ref SequenceReader reader, out T value) where T : unmanaged + private static unsafe bool TryReadMultisegment(ref SequenceReader reader, out T value) where T : unmanaged { Debug.Assert(reader.UnreadSpan.Length < sizeof(T)); @@ -42,7 +42,7 @@ private static unsafe bool TryReadSlow(ref SequenceReader reader, out T T buffer = default; Span tempSpan = new Span(&buffer, sizeof(T)); - if (reader.Peek(tempSpan).Length < sizeof(T)) + if (!reader.TryCopyTo(tempSpan)) { value = default; return false; diff --git a/src/System.Memory/tests/SequenceReader/BasicTests.cs b/src/System.Memory/tests/SequenceReader/BasicTests.cs index d0c48c048450..3986dc64ea9b 100644 --- a/src/System.Memory/tests/SequenceReader/BasicTests.cs +++ b/src/System.Memory/tests/SequenceReader/BasicTests.cs @@ -513,32 +513,6 @@ public void AdvanceTo_AdvancePast() } } - [Fact] - public void CopyToLargerBufferWorks() - { - T[] content = (T[])_inputData.Clone(); - - Span buffer = new T[content.Length + 1]; - SequenceReader reader = new SequenceReader(Factory.CreateWithContent(content)); - - // this loop skips more and more items in the reader - for (int i = 0; i < content.Length; i++) - { - int copied = reader.Peek(buffer).Length; - Assert.Equal(content.Length - i, copied); - Assert.True(buffer.Slice(0, copied).SequenceEqual(content.AsSpan(i))); - - // make sure that nothing more got written, i.e. tail is empty - for (int r = copied; r < buffer.Length; r++) - { - Assert.Equal(default, buffer[r]); - } - - reader.Advance(1); - buffer.Clear(); - } - } - [Fact] public void CopyToSmallerBufferWorks() { @@ -555,10 +529,10 @@ public void CopyToSmallerBufferWorks() { Span bufferSlice = buffer.Slice(0, j); bufferSlice.Clear(); - ReadOnlySpan peeked = reader.Peek(bufferSlice); - Assert.Equal(Math.Min(bufferSlice.Length, content.Length - i), peeked.Length); + Assert.True(reader.TryCopyTo(bufferSlice)); + Assert.Equal(Math.Min(bufferSlice.Length, content.Length - i), bufferSlice.Length); - Assert.True(peeked.SequenceEqual(content.AsSpan(i, j))); + Assert.True(bufferSlice.SequenceEqual(content.AsSpan(i, j))); } reader.Advance(1); From d287dafcb21e8284472440aa56664ee48b92eae3 Mon Sep 17 00:00:00 2001 From: Jeremy Kuhne Date: Thu, 8 Nov 2018 10:11:06 -0800 Subject: [PATCH 04/10] Address more feedback. Removing TryReadTo without advancePastDelmiter as we don't have the same pattern yet for the other overloads. --- src/System.Memory/ref/System.Memory.cs | 10 ++++--- .../src/System/Buffers/SequenceReader.cs | 2 ++ .../System/Buffers/SequenceReader_search.cs | 26 +++---------------- 3 files changed, 11 insertions(+), 27 deletions(-) diff --git a/src/System.Memory/ref/System.Memory.cs b/src/System.Memory/ref/System.Memory.cs index fd53788facf3..dea68152e2d1 100644 --- a/src/System.Memory/ref/System.Memory.cs +++ b/src/System.Memory/ref/System.Memory.cs @@ -99,8 +99,11 @@ public static void Reverse(this System.Span span) { } public SequencePosition(object @object, int integer) { throw null; } public override bool Equals(object obj) { throw null; } public bool Equals(System.SequencePosition other) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute((System.ComponentModel.EditorBrowsableState)(1))] public override int GetHashCode() { throw null; } + [System.ComponentModel.EditorBrowsableAttribute((System.ComponentModel.EditorBrowsableState)(1))] public int GetInteger() { throw null; } + [System.ComponentModel.EditorBrowsableAttribute((System.ComponentModel.EditorBrowsableState)(1))] public object GetObject() { throw null; } } } @@ -188,9 +191,9 @@ public static partial class SequenceReaderExtensions public static bool TryReadLittleEndian(this ref System.Buffers.SequenceReader reader, out short value) { throw null; } public static bool TryReadLittleEndian(this ref System.Buffers.SequenceReader reader, out int value) { throw null; } public static bool TryReadLittleEndian(this ref System.Buffers.SequenceReader reader, out long value) { throw null; } - public static bool TryRead(this ref System.Buffers.SequenceReader reader, out T value) where T : struct { throw null; } + public static bool TryRead(this ref System.Buffers.SequenceReader reader, out T value) where T : unmanaged { throw null; } } - public ref partial struct SequenceReader where T : struct, System.IEquatable + public ref partial struct SequenceReader where T : unmanaged, System.IEquatable { public SequenceReader(System.Buffers.ReadOnlySequence buffer) { throw null; } public long Consumed { get { throw null; } } @@ -219,8 +222,7 @@ public void Rewind(long count) { } public bool TryReadTo(out System.Buffers.ReadOnlySequence sequence, System.ReadOnlySpan delimiter, bool advancePastDelimiter = true) { throw null; } public bool TryReadTo(out System.Buffers.ReadOnlySequence sequence, T delimiter, bool advancePastDelimiter = true) { throw null; } public bool TryReadTo(out System.Buffers.ReadOnlySequence sequence, T delimiter, T delimiterEscape, bool advancePastDelimiter = true) { throw null; } - public bool TryReadTo(out System.ReadOnlySpan span, T delimiter) { throw null; } - public bool TryReadTo(out System.ReadOnlySpan span, T delimiter, bool advancePastDelimiter) { throw null; } + public bool TryReadTo(out System.ReadOnlySpan span, T delimiter, bool advancePastDelimiter = true) { throw null; } public bool TryReadTo(out System.ReadOnlySpan span, T delimiter, T delimiterEscape, bool advancePastDelimiter = true) { throw null; } public bool TryReadToAny(out System.Buffers.ReadOnlySequence sequence, System.ReadOnlySpan delimiters, bool advancePastDelimiter = true) { throw null; } public bool TryReadToAny(out System.ReadOnlySpan span, System.ReadOnlySpan delimiters, bool advancePastDelimiter = true) { throw null; } diff --git a/src/System.Memory/src/System/Buffers/SequenceReader.cs b/src/System.Memory/src/System/Buffers/SequenceReader.cs index f156582a8475..52a24e3ae0b2 100644 --- a/src/System.Memory/src/System/Buffers/SequenceReader.cs +++ b/src/System.Memory/src/System/Buffers/SequenceReader.cs @@ -301,6 +301,8 @@ private void AdvanceToNextSpan(long count) break; } + // As there may not be any further segments we need to + // push the current index to the end of the span. CurrentSpanIndex += remaining; count -= remaining; Debug.Assert(count >= 0); diff --git a/src/System.Memory/src/System/Buffers/SequenceReader_search.cs b/src/System.Memory/src/System/Buffers/SequenceReader_search.cs index 0b326b870d11..9b1eb1c68996 100644 --- a/src/System.Memory/src/System/Buffers/SequenceReader_search.cs +++ b/src/System.Memory/src/System/Buffers/SequenceReader_search.cs @@ -10,33 +10,13 @@ namespace System.Buffers public ref partial struct SequenceReader where T : unmanaged, IEquatable { /// - /// Try to read everything up to the given . The reader will be positioned - /// after the first occurrence of when successful. + /// Try to read everything up to the given . /// /// The read data, if any. /// The delimiter to look for. + /// True to move past the if found. /// True if the was found. - public bool TryReadTo(out ReadOnlySpan span, T delimiter) - { - ReadOnlySpan remaining = UnreadSpan; - int index = remaining.IndexOf(delimiter); - - if (index != -1) - { - span = remaining.Slice(0, index); - AdvanceCurrentSpan(index + 1); - return true; - } - - return TryReadToMultisegment(out span, delimiter); - } - - private bool TryReadToMultisegment(out ReadOnlySpan span, T delimiter) - { - return TryReadToSlow(out span, delimiter, advancePastDelimiter: true); - } - - public bool TryReadTo(out ReadOnlySpan span, T delimiter, bool advancePastDelimiter) + public bool TryReadTo(out ReadOnlySpan span, T delimiter, bool advancePastDelimiter = true) { ReadOnlySpan remaining = UnreadSpan; int index = remaining.IndexOf(delimiter); From 4d7568444f220a1a60262c13498a06b4d9f45944 Mon Sep 17 00:00:00 2001 From: Jeremy Kuhne Date: Thu, 8 Nov 2018 10:12:25 -0800 Subject: [PATCH 05/10] Rename files to not use underscore. --- src/System.Memory/src/System.Memory.csproj | 6 +++--- ...dOnlySequence_helpers.cs => ReadOnlySequence.helpers.cs} | 0 .../{SequenceReader_search.cs => SequenceReader.search.cs} | 0 ...ensions_binary.cs => SequenceReaderExtensions.binary.cs} | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename src/System.Memory/src/System/Buffers/{ReadOnlySequence_helpers.cs => ReadOnlySequence.helpers.cs} (100%) rename src/System.Memory/src/System/Buffers/{SequenceReader_search.cs => SequenceReader.search.cs} (100%) rename src/System.Memory/src/System/Buffers/{SequenceReaderExtensions_binary.cs => SequenceReaderExtensions.binary.cs} (100%) diff --git a/src/System.Memory/src/System.Memory.csproj b/src/System.Memory/src/System.Memory.csproj index 20f19f7179b1..bf782f5fb7c7 100644 --- a/src/System.Memory/src/System.Memory.csproj +++ b/src/System.Memory/src/System.Memory.csproj @@ -19,10 +19,10 @@ - + - - + + diff --git a/src/System.Memory/src/System/Buffers/ReadOnlySequence_helpers.cs b/src/System.Memory/src/System/Buffers/ReadOnlySequence.helpers.cs similarity index 100% rename from src/System.Memory/src/System/Buffers/ReadOnlySequence_helpers.cs rename to src/System.Memory/src/System/Buffers/ReadOnlySequence.helpers.cs diff --git a/src/System.Memory/src/System/Buffers/SequenceReader_search.cs b/src/System.Memory/src/System/Buffers/SequenceReader.search.cs similarity index 100% rename from src/System.Memory/src/System/Buffers/SequenceReader_search.cs rename to src/System.Memory/src/System/Buffers/SequenceReader.search.cs diff --git a/src/System.Memory/src/System/Buffers/SequenceReaderExtensions_binary.cs b/src/System.Memory/src/System/Buffers/SequenceReaderExtensions.binary.cs similarity index 100% rename from src/System.Memory/src/System/Buffers/SequenceReaderExtensions_binary.cs rename to src/System.Memory/src/System/Buffers/SequenceReaderExtensions.binary.cs From 70ef6d9410aaa6561749796c894c3cbe55928831 Mon Sep 17 00:00:00 2001 From: Jeremy Kuhne Date: Thu, 8 Nov 2018 10:28:16 -0800 Subject: [PATCH 06/10] Fix uap builds --- pkg/Microsoft.Private.PackageBaseline/packageIndex.json | 2 +- src/System.Memory/tests/System.Memory.Tests.csproj | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/Microsoft.Private.PackageBaseline/packageIndex.json b/pkg/Microsoft.Private.PackageBaseline/packageIndex.json index c023d5e2cf42..44e5e82c90f2 100644 --- a/pkg/Microsoft.Private.PackageBaseline/packageIndex.json +++ b/pkg/Microsoft.Private.PackageBaseline/packageIndex.json @@ -2546,7 +2546,7 @@ "netcoreapp3.0": "4.2.0.0", "monoandroid10": "Any", "monotouch10": "Any", - "uap10.0.16300": "4.1.0.0", + "uap10.0.16300": "4.2.0.0", "xamarinios10": "Any", "xamarinmac20": "Any", "xamarintvos10": "Any", diff --git a/src/System.Memory/tests/System.Memory.Tests.csproj b/src/System.Memory/tests/System.Memory.Tests.csproj index 9b755c2daba1..8c03849c4e1d 100644 --- a/src/System.Memory/tests/System.Memory.Tests.csproj +++ b/src/System.Memory/tests/System.Memory.Tests.csproj @@ -7,7 +7,7 @@ true - + @@ -15,8 +15,6 @@ - - From bb408c87cb147c23183b81beba9b7a5d4d0b06af Mon Sep 17 00:00:00 2001 From: Jeremy Kuhne Date: Thu, 8 Nov 2018 15:06:58 -0800 Subject: [PATCH 07/10] Move TryRead to SequenceMarshal and other feedback. --- .../packageIndex.json | 3 +-- src/System.Memory/ref/System.Memory.cs | 2 +- .../src/System/Buffers/SequenceReader.search.cs | 2 +- .../Buffers/SequenceReaderExtensions.binary.cs | 4 +++- .../Runtime/InteropServices/SequenceMarshal.cs | 17 +++++++++++++++++ .../tests/SequenceReader/BinaryExtensions.cs | 3 ++- 6 files changed, 25 insertions(+), 6 deletions(-) diff --git a/pkg/Microsoft.Private.PackageBaseline/packageIndex.json b/pkg/Microsoft.Private.PackageBaseline/packageIndex.json index 44e5e82c90f2..c76c5cd87c19 100644 --- a/pkg/Microsoft.Private.PackageBaseline/packageIndex.json +++ b/pkg/Microsoft.Private.PackageBaseline/packageIndex.json @@ -2554,8 +2554,7 @@ }, "AssemblyVersionInPackageVersion": { "4.0.1.0": "4.5.0", - "4.1.0.0": "4.6.0", - "4.2.0.0": "4.7.0" + "4.1.0.0": "4.6.0" } }, "System.Messaging": { diff --git a/src/System.Memory/ref/System.Memory.cs b/src/System.Memory/ref/System.Memory.cs index dea68152e2d1..93ed84648ef8 100644 --- a/src/System.Memory/ref/System.Memory.cs +++ b/src/System.Memory/ref/System.Memory.cs @@ -191,7 +191,6 @@ public static partial class SequenceReaderExtensions public static bool TryReadLittleEndian(this ref System.Buffers.SequenceReader reader, out short value) { throw null; } public static bool TryReadLittleEndian(this ref System.Buffers.SequenceReader reader, out int value) { throw null; } public static bool TryReadLittleEndian(this ref System.Buffers.SequenceReader reader, out long value) { throw null; } - public static bool TryRead(this ref System.Buffers.SequenceReader reader, out T value) where T : unmanaged { throw null; } } public ref partial struct SequenceReader where T : unmanaged, System.IEquatable { @@ -425,6 +424,7 @@ public static void Write(System.Span destination, ref T value) where T public static partial class SequenceMarshal { public static bool TryGetArray(System.Buffers.ReadOnlySequence sequence, out System.ArraySegment segment) { throw null; } + public static bool TryRead(ref System.Buffers.SequenceReader reader, out T value) where T : unmanaged { throw null; } public static bool TryGetReadOnlyMemory(System.Buffers.ReadOnlySequence sequence, out System.ReadOnlyMemory memory) { throw null; } public static bool TryGetReadOnlySequenceSegment(System.Buffers.ReadOnlySequence sequence, out System.Buffers.ReadOnlySequenceSegment startSegment, out int startIndex, out System.Buffers.ReadOnlySequenceSegment endSegment, out int endIndex) { throw null; } } diff --git a/src/System.Memory/src/System/Buffers/SequenceReader.search.cs b/src/System.Memory/src/System/Buffers/SequenceReader.search.cs index 9b1eb1c68996..771b39490b7e 100644 --- a/src/System.Memory/src/System/Buffers/SequenceReader.search.cs +++ b/src/System.Memory/src/System/Buffers/SequenceReader.search.cs @@ -407,7 +407,7 @@ private bool TryReadToAnyInternal(out ReadOnlySequence sequence, ReadOnlySpan } /// - /// Try to read data until the given . + /// Try to read data until the entire given matches. /// /// The read data, if any. /// The multi (T) delimiter. diff --git a/src/System.Memory/src/System/Buffers/SequenceReaderExtensions.binary.cs b/src/System.Memory/src/System/Buffers/SequenceReaderExtensions.binary.cs index 7bbe9ad50341..7dc5b84f4d29 100644 --- a/src/System.Memory/src/System/Buffers/SequenceReaderExtensions.binary.cs +++ b/src/System.Memory/src/System/Buffers/SequenceReaderExtensions.binary.cs @@ -4,6 +4,7 @@ using System.Buffers.Binary; using System.Diagnostics; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using Internal.Runtime.CompilerServices; @@ -23,7 +24,8 @@ public static partial class SequenceReaderExtensions /// /// True if successful. will be default if failed (due to lack of space). /// - public static unsafe bool TryRead(ref this SequenceReader reader, out T value) where T : unmanaged + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static unsafe bool TryRead(ref this SequenceReader reader, out T value) where T : unmanaged { ReadOnlySpan span = reader.UnreadSpan; if (span.Length < sizeof(T)) diff --git a/src/System.Memory/src/System/Runtime/InteropServices/SequenceMarshal.cs b/src/System.Memory/src/System/Runtime/InteropServices/SequenceMarshal.cs index 8335a4144bef..ded0b44d9d98 100644 --- a/src/System.Memory/src/System/Runtime/InteropServices/SequenceMarshal.cs +++ b/src/System.Memory/src/System/Runtime/InteropServices/SequenceMarshal.cs @@ -57,5 +57,22 @@ internal static bool TryGetString(ReadOnlySequence sequence, out string te { return sequence.TryGetString(out text, out start, out length); } + + /// + /// Try to read the given type out of the buffer if possible. Warning: this is dangerous to use with arbitrary + /// structs- see remarks for full details. + /// + /// + /// IMPORTANT: The read is a straight copy of bits. If a struct depends on specific state of it's members to + /// behave correctly this can lead to exceptions, etc. If reading endian specific integers, use the explicit + /// overloads such as + /// + /// + /// True if successful. will be default if failed (due to lack of space). + /// + public static bool TryRead(ref SequenceReader reader, out T value) where T : unmanaged + { + return reader.TryRead(out value); + } } } diff --git a/src/System.Memory/tests/SequenceReader/BinaryExtensions.cs b/src/System.Memory/tests/SequenceReader/BinaryExtensions.cs index 851715ce7286..f1a89ae48d23 100644 --- a/src/System.Memory/tests/SequenceReader/BinaryExtensions.cs +++ b/src/System.Memory/tests/SequenceReader/BinaryExtensions.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using System.Buffers; +using System.Runtime.InteropServices; using Xunit; namespace System.Memory.Tests.SequenceReader @@ -47,7 +48,7 @@ public void MultiSegmentBytesReaderNumbers() Assert.Equal(6, span[0]); Assert.Equal(7, span[1]); - Assert.True(reader.TryRead(out int intValue)); + Assert.True(SequenceMarshal.TryRead(ref reader, out int intValue)); Assert.Equal(BitConverter.ToInt32(new byte[] { 0, 1, 0, 2 }), intValue); Assert.True(reader.TryReadBigEndian(out intValue)); From 00643261f7866de8e89f7f1cc603bd5a1cbc81aa Mon Sep 17 00:00:00 2001 From: Jeremy Kuhne Date: Thu, 8 Nov 2018 16:05:21 -0800 Subject: [PATCH 08/10] Tweak file name casing (part1) --- src/System.Memory/src/System.Memory.csproj | 6 +++--- ...OnlySequence.helpers.cs => ReadOnlySequence.hHelpers.cs} | 0 .../{SequenceReader.search.cs => SequenceReader.sSearch.cs} | 0 ...nsions.binary.cs => SequenceReaderExtensions.bBinary.cs} | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename src/System.Memory/src/System/Buffers/{ReadOnlySequence.helpers.cs => ReadOnlySequence.hHelpers.cs} (100%) rename src/System.Memory/src/System/Buffers/{SequenceReader.search.cs => SequenceReader.sSearch.cs} (100%) rename src/System.Memory/src/System/Buffers/{SequenceReaderExtensions.binary.cs => SequenceReaderExtensions.bBinary.cs} (100%) diff --git a/src/System.Memory/src/System.Memory.csproj b/src/System.Memory/src/System.Memory.csproj index bf782f5fb7c7..e6ee86b4d397 100644 --- a/src/System.Memory/src/System.Memory.csproj +++ b/src/System.Memory/src/System.Memory.csproj @@ -19,10 +19,10 @@ - + - - + + diff --git a/src/System.Memory/src/System/Buffers/ReadOnlySequence.helpers.cs b/src/System.Memory/src/System/Buffers/ReadOnlySequence.hHelpers.cs similarity index 100% rename from src/System.Memory/src/System/Buffers/ReadOnlySequence.helpers.cs rename to src/System.Memory/src/System/Buffers/ReadOnlySequence.hHelpers.cs diff --git a/src/System.Memory/src/System/Buffers/SequenceReader.search.cs b/src/System.Memory/src/System/Buffers/SequenceReader.sSearch.cs similarity index 100% rename from src/System.Memory/src/System/Buffers/SequenceReader.search.cs rename to src/System.Memory/src/System/Buffers/SequenceReader.sSearch.cs diff --git a/src/System.Memory/src/System/Buffers/SequenceReaderExtensions.binary.cs b/src/System.Memory/src/System/Buffers/SequenceReaderExtensions.bBinary.cs similarity index 100% rename from src/System.Memory/src/System/Buffers/SequenceReaderExtensions.binary.cs rename to src/System.Memory/src/System/Buffers/SequenceReaderExtensions.bBinary.cs From 10443a097bd2e676caff09f3f2d8340c99ebe14b Mon Sep 17 00:00:00 2001 From: Jeremy Kuhne Date: Thu, 8 Nov 2018 16:06:18 -0800 Subject: [PATCH 09/10] Change casing pt 2. --- src/System.Memory/src/System.Memory.csproj | 6 +++--- ...OnlySequence.hHelpers.cs => ReadOnlySequence.Helpers.cs} | 0 .../{SequenceReader.sSearch.cs => SequenceReader.Search.cs} | 0 ...nsions.bBinary.cs => SequenceReaderExtensions.Binary.cs} | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename src/System.Memory/src/System/Buffers/{ReadOnlySequence.hHelpers.cs => ReadOnlySequence.Helpers.cs} (100%) rename src/System.Memory/src/System/Buffers/{SequenceReader.sSearch.cs => SequenceReader.Search.cs} (100%) rename src/System.Memory/src/System/Buffers/{SequenceReaderExtensions.bBinary.cs => SequenceReaderExtensions.Binary.cs} (100%) diff --git a/src/System.Memory/src/System.Memory.csproj b/src/System.Memory/src/System.Memory.csproj index e6ee86b4d397..866a8a4d985c 100644 --- a/src/System.Memory/src/System.Memory.csproj +++ b/src/System.Memory/src/System.Memory.csproj @@ -19,10 +19,10 @@ - + - - + + diff --git a/src/System.Memory/src/System/Buffers/ReadOnlySequence.hHelpers.cs b/src/System.Memory/src/System/Buffers/ReadOnlySequence.Helpers.cs similarity index 100% rename from src/System.Memory/src/System/Buffers/ReadOnlySequence.hHelpers.cs rename to src/System.Memory/src/System/Buffers/ReadOnlySequence.Helpers.cs diff --git a/src/System.Memory/src/System/Buffers/SequenceReader.sSearch.cs b/src/System.Memory/src/System/Buffers/SequenceReader.Search.cs similarity index 100% rename from src/System.Memory/src/System/Buffers/SequenceReader.sSearch.cs rename to src/System.Memory/src/System/Buffers/SequenceReader.Search.cs diff --git a/src/System.Memory/src/System/Buffers/SequenceReaderExtensions.bBinary.cs b/src/System.Memory/src/System/Buffers/SequenceReaderExtensions.Binary.cs similarity index 100% rename from src/System.Memory/src/System/Buffers/SequenceReaderExtensions.bBinary.cs rename to src/System.Memory/src/System/Buffers/SequenceReaderExtensions.Binary.cs From b221ba08aeeec2662be2a37a46b2bc05e7a62bc8 Mon Sep 17 00:00:00 2001 From: Jeremy Kuhne Date: Thu, 8 Nov 2018 16:08:54 -0800 Subject: [PATCH 10/10] Change package common types --- pkg/descriptions.json | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/descriptions.json b/pkg/descriptions.json index a103b5e8e72e..f1ec48f83ce2 100644 --- a/pkg/descriptions.json +++ b/pkg/descriptions.json @@ -1050,6 +1050,7 @@ "System.ReadOnlyMemory", "System.Buffers.MemoryPool", "System.Buffers.ReadOnlySequence", + "System.Buffers.SequenceReader", "System.Buffers.Text.Utf8Parser", "System.Buffers.Text.Utf8Formatter" ]