Fixes for NonBacktracking NFA mode

src/libraries/System.Text.RegularExpressions/src/System.Text.RegularExpressions.csproj

@@ -70,6 +70,7 @@
     <Compile Include="System\Text\RegularExpressions\Symbolic\MintermGenerator.cs" />
     <Compile Include="System\Text\RegularExpressions\Symbolic\RegexNodeConverter.cs" />
     <Compile Include="System\Text\RegularExpressions\Symbolic\SparseIntMap.cs" />
+    <Compile Include="System\Text\RegularExpressions\Symbolic\StateFlags.cs" />
     <Compile Include="System\Text\RegularExpressions\Symbolic\SymbolicMatch.cs" />
     <Compile Include="System\Text\RegularExpressions\Symbolic\SymbolicRegexBuilder.cs" />
     <Compile Include="System\Text\RegularExpressions\Symbolic\SymbolicRegexNode.cs" />

src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/MatchingState.cs

@@ -104,6 +104,43 @@ internal bool IsNullableFor(uint nextCharKind)
             return Node.IsNullableFor(context);
         }
 
+        /// <summary>
+        /// Builds a <see cref="StateFlags"/> with the relevant flags set.
+        /// </summary>
+        /// <param name="solver">a solver for <typeparamref name="TSet"/></param>
+        /// <param name="isInitial">whether this state is an initial state</param>
+        /// <returns>the flags for this matching state</returns>
+        internal StateFlags BuildStateFlags(ISolver<TSet> solver, bool isInitial)
+        {
+            StateFlags info = 0;
+
+            if (isInitial)
+            {
+                info |= StateFlags.IsInitialFlag;
+            }
+
+            if (IsDeadend(solver))
+            {
+                info |= StateFlags.IsDeadendFlag;
+            }
+
+            if (Node.CanBeNullable)
+            {
+                info |= StateFlags.CanBeNullableFlag;
+                if (Node.IsNullable)
+                {
+                    info |= StateFlags.IsNullableFlag;
+                }
+            }
+
+            if (Node.Kind != SymbolicRegexNodeKind.DisableBacktrackingSimulation)
+            {
+                info |= StateFlags.SimulatesBacktrackingFlag;
+            }
+
+            return info;
+        }
+
         public override bool Equals(object? obj) =>
             obj is MatchingState<TSet> s && PrevCharKind == s.PrevCharKind && Node.Equals(s.Node);
 

src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/StateFlags.cs

@@ -0,0 +1,36 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Runtime.CompilerServices;
+
+namespace System.Text.RegularExpressions.Symbolic
+{
+    /// <summary>
+    /// These flags provide context-independent information available for every state. They provide a fast way to evaluate
+    /// conditions in the inner matching loops of <see cref="SymbolicRegexMatcher{TSet}"/>. The matcher caches one of these
+    /// for every state, for which they are created by <see cref="MatchingState{TSet}.BuildStateFlags(ISolver{TSet}, bool)"/>.
+    /// In DFA mode the cached flags are used directly, while in NFA mode the <see cref="SymbolicRegexMatcher{TSet}.NfaStateHandler"/>
+    /// handles aggregating the flags in the state set.
+    /// </summary>
+    [Flags]
+    internal enum StateFlags : byte
+    {
+        IsInitialFlag = 1,
+        IsDeadendFlag = 2,
+        IsNullableFlag = 4,
+        CanBeNullableFlag = 8,
+        SimulatesBacktrackingFlag = 16,
+    }
+
+    /// <summary>
+    /// These extension methods for <see cref="StateFlags"/> make checking for the presence of flags more concise.
+    /// </summary>
+    internal static class StateFlagsExtensions
+    {
+        internal static bool IsInitial(this StateFlags info) => (info & StateFlags.IsInitialFlag) != 0;
+        internal static bool IsDeadend(this StateFlags info) => (info & StateFlags.IsDeadendFlag) != 0;
+        internal static bool IsNullable(this StateFlags info) => (info & StateFlags.IsNullableFlag) != 0;
+        internal static bool CanBeNullable(this StateFlags info) => (info & StateFlags.CanBeNullableFlag) != 0;
+        internal static bool SimulatesBacktracking(this StateFlags info) => (info & StateFlags.SimulatesBacktrackingFlag) != 0;
+    }
+}

src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicRegexMatcher.Automata.cs

@@ -40,17 +40,7 @@ internal sealed partial class SymbolicRegexMatcher<TSet>
         /// Maps state IDs to context-independent information for all states in <see cref="_stateArray"/>.
         /// The first valid entry is at index 1.
         /// </summary>
-        private ContextIndependentState[] _stateInfo;
-
-        /// <summary>Context-independent information available for every state.</summary>
-        [Flags]
-        private enum ContextIndependentState : byte
-        {
-            IsInitial = 1,
-            IsDeadend = 2,
-            IsNullable = 4,
-            CanBeNullable = 8,
-        }
+        private StateFlags[] _stateFlagsArray;
 
         /// <summary>
         /// The transition function for DFA mode.
@@ -152,19 +142,6 @@ private Span<int> GetDeltasFor(MatchingState<TSet> state)
             return _nfaDelta.AsSpan(nfaState << _mintermsLog, numMinterms);
         }
 
-        /// <summary>Get context-independent information for the given state.</summary>
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private (bool IsInitial, bool IsDeadend, bool IsNullable, bool CanBeNullable) GetStateInfo(int stateId)
-        {
-            Debug.Assert(stateId > 0);
-
-            ContextIndependentState info = _stateInfo[stateId];
-            return ((info & ContextIndependentState.IsInitial) != 0,
-                    (info & ContextIndependentState.IsDeadend) != 0,
-                    (info & ContextIndependentState.IsNullable) != 0,
-                    (info & ContextIndependentState.CanBeNullable) != 0);
-        }
-
         /// <summary>
         /// Create a state with given node and previous character context.
         /// </summary>
@@ -202,43 +179,13 @@ private MatchingState<TSet> GetOrCreateState_NoLock(SymbolicRegexNode<TSet> node
                     int newsize = _stateArray.Length * 2;
                     ArrayResizeAndVolatilePublish(ref _stateArray, newsize);
                     ArrayResizeAndVolatilePublish(ref _dfaDelta, newsize << _mintermsLog);
-                    ArrayResizeAndVolatilePublish(ref _stateInfo, newsize);
+                    ArrayResizeAndVolatilePublish(ref _stateFlagsArray, newsize);
                 }
                 _stateArray[state.Id] = state;
-                _stateInfo[state.Id] = BuildStateInfo(state.Id, isInitialState, state.IsDeadend(Solver), state.Node.IsNullable, state.Node.CanBeNullable);
+                _stateFlagsArray[state.Id] = state.BuildStateFlags(Solver, isInitialState);
             }
 
             return state;
-
-            // Assign the context-independent information for the given state
-            static ContextIndependentState BuildStateInfo(int stateId, bool isInitial, bool isDeadend, bool isNullable, bool canBeNullable)
-            {
-                Debug.Assert(stateId > 0);
-                Debug.Assert(!isNullable || canBeNullable);
-
-                ContextIndependentState info = 0;
-
-                if (isInitial)
-                {
-                    info |= ContextIndependentState.IsInitial;
-                }
-
-                if (isDeadend)
-                {
-                    info |= ContextIndependentState.IsDeadend;
-                }
-
-                if (canBeNullable)
-                {
-                    info |= ContextIndependentState.CanBeNullable;
-                    if (isNullable)
-                    {
-                        info |= ContextIndependentState.IsNullable;
-                    }
-                }
-
-                return info;
-            }
         }
 
         /// <summary>

src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicRegexMatcher.Dgml.cs

@@ -35,7 +35,7 @@ public override void SaveDGML(TextWriter writer, int maxLabelLength)
                     string nodeDgmlView = $"{(info == string.Empty ? info : $"Previous: {info}
")}{(deriv == string.Empty ? "()" : deriv)}";
 
                     writer.WriteLine("        <Node Id=\"{0}\" Label=\"{0}\" Category=\"State\" Group=\"Collapsed\" StateInfo=\"{1}\">", state.Id, nodeDgmlView);
-                    if (GetStateInfo(state.Id).IsInitial)
+                    if (_stateFlagsArray[state.Id].IsInitial())
                     {
                         writer.WriteLine("            <Category Ref=\"InitialState\" />");
                     }

src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicRegexMatcher.Sample.cs

@@ -79,11 +79,11 @@ public override IEnumerable<string> SampleMatches(int k, int randomseed)
 
                         // Gather the possible endings for satisfying nullability
                         possibleEndings.Clear();
-                        if (SymbolicRegexMatcher<TSet>.NfaStateHandler.CanBeNullable(this, in statesWrapper))
+                        StateFlags flags = SymbolicRegexMatcher<TSet>.NfaStateHandler.GetStateFlags(this, in statesWrapper);
+                        if (flags.CanBeNullable())
                         {
                             // Unconditionally final state or end of the input due to \Z anchor for example
-                            if (SymbolicRegexMatcher<TSet>.NfaStateHandler.IsNullable(this, in statesWrapper) ||
-                                SymbolicRegexMatcher<TSet>.NfaStateHandler.IsNullableFor(this, in statesWrapper, CharKind.BeginningEnd))
+                            if (flags.IsNullable() || SymbolicRegexMatcher<TSet>.NfaStateHandler.IsNullableFor(this, in statesWrapper, CharKind.BeginningEnd))
                             {
                                 possibleEndings.Add("");
                             }