diff --git a/src/libraries/System.Text.RegularExpressions/gen/Resources/Strings.resx b/src/libraries/System.Text.RegularExpressions/gen/Resources/Strings.resx index 8c2c008f7b7806..4edd9a7dc7dbb7 100644 --- a/src/libraries/System.Text.RegularExpressions/gen/Resources/Strings.resx +++ b/src/libraries/System.Text.RegularExpressions/gen/Resources/Strings.resx @@ -303,4 +303,7 @@ Unterminated (?#...) comment. + + Regex replacements with substitutions of groups are not supported with RegexOptions.NonBacktracking. + diff --git a/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.cs.xlf b/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.cs.xlf index 898a8b49d7fa00..5c933786208a19 100644 --- a/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.cs.xlf +++ b/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.cs.xlf @@ -192,6 +192,11 @@ Výsledek nelze volat pro shodu, která se nezdařila. + + Regex replacements with substitutions of groups are not supported with RegexOptions.NonBacktracking. + Regex replacements with substitutions of groups are not supported with RegexOptions.NonBacktracking. + + Collection is read-only. Kolekce je jen pro čtení. diff --git a/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.de.xlf b/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.de.xlf index e20c53d2614c76..821165f07545cd 100644 --- a/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.de.xlf +++ b/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.de.xlf @@ -192,6 +192,11 @@ Das Ergebnis kann nicht für eine fehlgeschlagene Übereinstimmung aufgerufen werden. + + Regex replacements with substitutions of groups are not supported with RegexOptions.NonBacktracking. + Regex replacements with substitutions of groups are not supported with RegexOptions.NonBacktracking. + + Collection is read-only. Die Sammlung ist schreibgeschützt. diff --git a/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.es.xlf b/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.es.xlf index 503face0253a4e..f0f919acc6e5bd 100644 --- a/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.es.xlf +++ b/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.es.xlf @@ -192,6 +192,11 @@ No se puede llamar al resultado si no se encuentra ninguna coincidencia. + + Regex replacements with substitutions of groups are not supported with RegexOptions.NonBacktracking. + Regex replacements with substitutions of groups are not supported with RegexOptions.NonBacktracking. + + Collection is read-only. La colección es de sólo lectura. diff --git a/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.fr.xlf b/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.fr.xlf index efb0bb67f03002..0a9bb527192bd5 100644 --- a/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.fr.xlf +++ b/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.fr.xlf @@ -192,6 +192,11 @@ Le résultat ne peut pas être appelé sur un Match ayant échoué. + + Regex replacements with substitutions of groups are not supported with RegexOptions.NonBacktracking. + Regex replacements with substitutions of groups are not supported with RegexOptions.NonBacktracking. + + Collection is read-only. La collection est en lecture seule. diff --git a/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.it.xlf b/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.it.xlf index b4b166fc3bd702..30d0b81bd4b420 100644 --- a/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.it.xlf +++ b/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.it.xlf @@ -192,6 +192,11 @@ Impossibile chiamare Result su un Match non riuscito. + + Regex replacements with substitutions of groups are not supported with RegexOptions.NonBacktracking. + Regex replacements with substitutions of groups are not supported with RegexOptions.NonBacktracking. + + Collection is read-only. La raccolta è di sola lettura. diff --git a/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.ja.xlf b/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.ja.xlf index 8e392f02b4bb40..fe5540dc87d23a 100644 --- a/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.ja.xlf +++ b/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.ja.xlf @@ -192,6 +192,11 @@ 失敗した Match で Result を呼び出すことはできません。 + + Regex replacements with substitutions of groups are not supported with RegexOptions.NonBacktracking. + Regex replacements with substitutions of groups are not supported with RegexOptions.NonBacktracking. + + Collection is read-only. コレクションは読み取り専用です。 diff --git a/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.ko.xlf b/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.ko.xlf index 08578afc7a53a1..6654610f3f303c 100644 --- a/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.ko.xlf +++ b/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.ko.xlf @@ -192,6 +192,11 @@ 실패한 Match에서 결과를 호출할 수 없습니다. + + Regex replacements with substitutions of groups are not supported with RegexOptions.NonBacktracking. + Regex replacements with substitutions of groups are not supported with RegexOptions.NonBacktracking. + + Collection is read-only. 읽기 전용 컬렉션입니다. diff --git a/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.pl.xlf b/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.pl.xlf index 474f5be3212a82..f84553cffe9b58 100644 --- a/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.pl.xlf +++ b/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.pl.xlf @@ -192,6 +192,11 @@ Nie można wywołać wyniku błędnego dopasowania. + + Regex replacements with substitutions of groups are not supported with RegexOptions.NonBacktracking. + Regex replacements with substitutions of groups are not supported with RegexOptions.NonBacktracking. + + Collection is read-only. Kolekcja jest tylko do odczytu. diff --git a/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.pt-BR.xlf b/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.pt-BR.xlf index b3f2489ecf7236..ee284582bf53e2 100644 --- a/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.pt-BR.xlf +++ b/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.pt-BR.xlf @@ -192,6 +192,11 @@ Não é possível chamar resultado quando há falha na correspondência. + + Regex replacements with substitutions of groups are not supported with RegexOptions.NonBacktracking. + Regex replacements with substitutions of groups are not supported with RegexOptions.NonBacktracking. + + Collection is read-only. A coleção é somente leitura. diff --git a/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.ru.xlf b/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.ru.xlf index 7939800aa28a5b..99b880cc7b40ad 100644 --- a/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.ru.xlf +++ b/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.ru.xlf @@ -192,6 +192,11 @@ Вызов результата невозможен при сбойном соответствии. + + Regex replacements with substitutions of groups are not supported with RegexOptions.NonBacktracking. + Regex replacements with substitutions of groups are not supported with RegexOptions.NonBacktracking. + + Collection is read-only. Данная коллекция предназначена только для чтения. diff --git a/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.tr.xlf b/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.tr.xlf index 567b9974605a9e..bb6c8e1863a10c 100644 --- a/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.tr.xlf +++ b/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.tr.xlf @@ -192,6 +192,11 @@ Sonuç, başarısız Eşleştirmede çağrılamaz. + + Regex replacements with substitutions of groups are not supported with RegexOptions.NonBacktracking. + Regex replacements with substitutions of groups are not supported with RegexOptions.NonBacktracking. + + Collection is read-only. Koleksiyon salt okunur. diff --git a/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.zh-Hans.xlf b/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.zh-Hans.xlf index 793005782d3ff3..a17a2102fd7d33 100644 --- a/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.zh-Hans.xlf +++ b/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.zh-Hans.xlf @@ -192,6 +192,11 @@ 不能对失败的匹配调用结果。 + + Regex replacements with substitutions of groups are not supported with RegexOptions.NonBacktracking. + Regex replacements with substitutions of groups are not supported with RegexOptions.NonBacktracking. + + Collection is read-only. 集合是只读的。 diff --git a/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.zh-Hant.xlf b/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.zh-Hant.xlf index b50bd83c543e43..6c552441ce0e9f 100644 --- a/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.zh-Hant.xlf +++ b/src/libraries/System.Text.RegularExpressions/gen/Resources/xlf/Strings.zh-Hant.xlf @@ -192,6 +192,11 @@ 無法在已失敗的對應 (Match) 上呼叫結果。 + + Regex replacements with substitutions of groups are not supported with RegexOptions.NonBacktracking. + Regex replacements with substitutions of groups are not supported with RegexOptions.NonBacktracking. + + Collection is read-only. 集合是唯讀的。 diff --git a/src/libraries/System.Text.RegularExpressions/ref/System.Text.RegularExpressions.cs b/src/libraries/System.Text.RegularExpressions/ref/System.Text.RegularExpressions.cs index c1fbfcad03a9d1..9c0062d5b810c8 100644 --- a/src/libraries/System.Text.RegularExpressions/ref/System.Text.RegularExpressions.cs +++ b/src/libraries/System.Text.RegularExpressions/ref/System.Text.RegularExpressions.cs @@ -255,6 +255,7 @@ public enum RegexOptions RightToLeft = 64, ECMAScript = 256, CultureInvariant = 512, + NonBacktracking = 1024, } public enum RegexParseError { diff --git a/src/libraries/System.Text.RegularExpressions/src/Resources/Strings.resx b/src/libraries/System.Text.RegularExpressions/src/Resources/Strings.resx index 9d3c5d417f9eb8..4f4fef7a861e7e 100644 --- a/src/libraries/System.Text.RegularExpressions/src/Resources/Strings.resx +++ b/src/libraries/System.Text.RegularExpressions/src/Resources/Strings.resx @@ -1,4 +1,5 @@ - + + @@ -153,9 +154,6 @@ Collection is read-only. - - This operation is only allowed once per object. - This platform does not support writing compiled regular expressions to an assembly. Use RegexGeneratorAttribute with the regular expression source generator instead. @@ -225,4 +223,37 @@ Unterminated (?#...) comment. + + Regex replacements with substitutions of groups are not supported with RegexOptions.NonBacktracking. + + + RegexOptions.NonBacktracking is not supported in conjunction with RegexOptions.{0}. + + + RegexOptions.NonBacktracking is not supported in conjunction with expressions containing: '{0}'. + + + backreference (\\ number) + + + captured group conditional (?( name ) yes-pattern | no-pattern ) or (?( number ) yes-pattern| no-pattern ) + + + positive lookahead (?= pattern) or positive lookbehind (?<= pattern) + + + negative lookahead (?! pattern) or negative lookbehind (?<! pattern) + + + contiguous matches (\\G) + + + atomic subexpressions (?> pattern) + + + test conditional (?( test-pattern ) yes-pattern | no-pattern ) + + + balancing group (?<name1-name2>subexpression) or (?'name1-name2' subexpression) + diff --git a/src/libraries/System.Text.RegularExpressions/src/System.Text.RegularExpressions.csproj b/src/libraries/System.Text.RegularExpressions/src/System.Text.RegularExpressions.csproj index a8f8b7bd4f69f1..4da04ae480e510 100644 --- a/src/libraries/System.Text.RegularExpressions/src/System.Text.RegularExpressions.csproj +++ b/src/libraries/System.Text.RegularExpressions/src/System.Text.RegularExpressions.csproj @@ -5,9 +5,9 @@ enable - + @@ -18,6 +18,7 @@ + @@ -41,11 +42,52 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -60,7 +102,7 @@ - + diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Match.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Match.cs index e5022d109f9557..b7f2f032e78a82 100644 --- a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Match.cs +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Match.cs @@ -121,7 +121,7 @@ public virtual string Result(string replacement) } // Gets the weakly cached replacement helper or creates one if there isn't one already. - RegexReplacement repl = RegexReplacement.GetOrCreate(regex._replref!, replacement, regex.caps!, regex.capsize, regex.capnames!, regex.roptions); + RegexReplacement repl = RegexReplacement.GetOrCreate(regex.RegexReplacementWeakReference, replacement, regex.caps!, regex.capsize, regex.capnames!, regex.roptions); SegmentStringBuilder segments = SegmentStringBuilder.Create(); repl.ReplacementImpl(ref segments, this); return segments.ToString(); diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Regex.Debug.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Regex.Debug.cs new file mode 100644 index 00000000000000..865c5d096bad54 --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Regex.Debug.cs @@ -0,0 +1,68 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#if DEBUG +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Text.RegularExpressions.Symbolic; +using System.Text.RegularExpressions.Symbolic.Unicode; + +namespace System.Text.RegularExpressions +{ + public partial class Regex + { + /// True if the regex has debugging enabled. + [ExcludeFromCodeCoverage(Justification = "Debug only")] + internal bool IsDebug => (roptions & RegexOptions.Debug) != 0; + + /// Unwind the regex and save the resulting state graph in DGML + /// roughly the maximum number of states, 0 means no bound + /// if true then hide state info + /// if true then pretend that there is a .* at the beginning + /// if true then unwind the regex backwards (addDotStar is then ignored) + /// if true then compute and save only general DFA info + /// dgml output is written here + /// maximum length of labels in nodes anything over that length is indicated with .. + /// if true creates NFA instead of DFA + [ExcludeFromCodeCoverage(Justification = "Debug only")] + internal void SaveDGML(TextWriter writer, int bound, bool hideStateInfo, bool addDotStar, bool inReverse, bool onlyDFAinfo, int maxLabelLength, bool asNFA) + { + if (factory is not SymbolicRegexRunnerFactory srmFactory) + { + throw new NotSupportedException(); + } + + srmFactory._runner._matcher.SaveDGML(writer, bound, hideStateInfo, addDotStar, inReverse, onlyDFAinfo, maxLabelLength, asNFA); + } + + /// + /// Generates two files IgnoreCaseRelation.cs and UnicodeCategoryRanges.cs for the namespace System.Text.RegularExpressions.Symbolic.Unicode + /// in the given directory path. Only avaliable in DEBUG mode. + /// + [ExcludeFromCodeCoverage(Justification = "Debug only")] + internal static void GenerateUnicodeTables(string path) + { + IgnoreCaseRelationGenerator.Generate("System.Text.RegularExpressions.Symbolic.Unicode", "IgnoreCaseRelation", path); + UnicodeCategoryRangesGenerator.Generate("System.Text.RegularExpressions.Symbolic.Unicode", "UnicodeCategoryRanges", path); + } + + /// + /// Generates up to k random strings matched by the regex + /// + /// upper bound on the number of generated strings + /// random seed for the generator, 0 means no random seed + /// if true then generate inputs that do not match + /// + [ExcludeFromCodeCoverage(Justification = "Debug only")] + internal Collections.Generic.IEnumerable GenerateRandomMembers(int k, int randomseed, bool negative) + { + if (factory is not SymbolicRegexRunnerFactory srmFactory) + { + throw new NotSupportedException(); + } + + return srmFactory._runner._matcher.GenerateRandomMembers(k, randomseed, negative); + } + } +} +#endif diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Regex.Replace.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Regex.Replace.cs index b3dfbcb4ac5749..1b048fc6875061 100644 --- a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Regex.Replace.cs +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Regex.Replace.cs @@ -77,7 +77,7 @@ public string Replace(string input, string replacement, int count, int startat) // Gets the weakly cached replacement helper or creates one if there isn't one already, // then uses it to perform the replace. return - RegexReplacement.GetOrCreate(_replref!, replacement, caps!, capsize, capnames!, roptions). + RegexReplacement.GetOrCreate(RegexReplacementWeakReference, replacement, caps!, capsize, capnames!, roptions). Replace(this, input, count, startat); } diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Regex.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Regex.cs index 69130ec6c03466..1fc73e49a2b5bc 100644 --- a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Regex.cs +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Regex.cs @@ -9,6 +9,7 @@ using System.Reflection.Emit; using System.Runtime.CompilerServices; using System.Runtime.Serialization; +using System.Text.RegularExpressions.Symbolic; using System.Threading; namespace System.Text.RegularExpressions @@ -19,20 +20,19 @@ namespace System.Text.RegularExpressions /// public partial class Regex : ISerializable { - internal const int MaxOptionShift = 10; + internal const int MaxOptionShift = 11; protected internal string? pattern; // The string pattern provided protected internal RegexOptions roptions; // the top-level options from the options string - protected internal RegexRunnerFactory? factory; + protected internal RegexRunnerFactory? factory; // Factory used to create runner instances for executing the regex protected internal Hashtable? caps; // if captures are sparse, this is the hashtable capnum->index protected internal Hashtable? capnames; // if named captures are used, this maps names->index protected internal string[]? capslist; // if captures are sparse or named captures are used, this is the sorted list of names protected internal int capsize; // the size of the capture array - internal WeakReference? _replref; // cached parsed replacement pattern + private WeakReference? _replref; // cached parsed replacement pattern private volatile RegexRunner? _runner; // cached runner private RegexCode? _code; // if interpreted, this is the code for RegexInterpreter - private bool _refsInitialized; protected Regex() { @@ -64,28 +64,40 @@ internal Regex(string pattern, CultureInfo? culture) { // Call Init directly rather than delegating to a Regex ctor that takes // options to enable linking / tree shaking to remove the Regex compiler - // if it may not be used. + // and NonBacktracking implementation if it's not used. Init(pattern, RegexOptions.None, s_defaultMatchTimeout, culture); } internal Regex(string pattern, RegexOptions options, TimeSpan matchTimeout, CultureInfo? culture) { + culture ??= GetTargetCulture(options); Init(pattern, options, matchTimeout, culture); - // if the compile option is set, then compile the code - if (RuntimeFeature.IsDynamicCodeCompiled && UseOptionC()) + if ((options & RegexOptions.NonBacktracking) != 0) { - factory = Compile(pattern, _code!, options, matchTimeout != InfiniteMatchTimeout); + // If we're in non-backtracking mode, create the appropriate factory. + factory = SymbolicRegexRunner.CreateFactory(_code, options, matchTimeout, culture); + _code = null; + } + else if (RuntimeFeature.IsDynamicCodeCompiled && UseOptionC()) + { + // If the compile option is set and compilation is supported, then compile the code. + factory = Compile(pattern, _code, options, matchTimeout != InfiniteMatchTimeout); _code = null; } } + /// Gets the culture to use based on the specified options. + private static CultureInfo GetTargetCulture(RegexOptions options) => + (options & RegexOptions.CultureInvariant) != 0 ? CultureInfo.InvariantCulture : CultureInfo.CurrentCulture; + /// Initializes the instance. /// /// This is separated out of the constructor so that an app only using 'new Regex(pattern)' /// rather than 'new Regex(pattern, options)' can avoid statically referencing the Regex /// compiler, such that a tree shaker / linker can trim it away if it's not otherwise used. /// + [MemberNotNull(nameof(_code))] private void Init(string pattern, RegexOptions options, TimeSpan matchTimeout, CultureInfo? culture) { ValidatePattern(pattern); @@ -93,8 +105,9 @@ private void Init(string pattern, RegexOptions options, TimeSpan matchTimeout, C ValidateMatchTimeout(matchTimeout); this.pattern = pattern; - roptions = options; internalMatchTimeout = matchTimeout; + roptions = options; + culture ??= GetTargetCulture(options); #if DEBUG if (IsDebug) @@ -104,16 +117,27 @@ private void Init(string pattern, RegexOptions options, TimeSpan matchTimeout, C #endif // Parse the input - RegexTree tree = RegexParser.Parse(pattern, roptions, culture ?? ((options & RegexOptions.CultureInvariant) != 0 ? CultureInfo.InvariantCulture : CultureInfo.CurrentCulture)); + RegexTree tree = RegexParser.Parse(pattern, roptions, culture); - // Extract the relevant information - capnames = tree.CapNames; - capslist = tree.CapsList; + // Generate the RegexCode from the node tree. This is required for interpreting, + // and is used as input into RegexOptions.Compiled and RegexOptions.NonBacktracking. _code = RegexWriter.Write(tree); - caps = _code.Caps; - capsize = _code.CapSize; - InitializeReferences(); + if ((options & RegexOptions.NonBacktracking) != 0) + { + // NonBacktracking doesn't support captures (other than the implicit top-level capture). + capnames = null; + capslist = null; + caps = null; + capsize = 1; + } + else + { + capnames = tree.CapNames; + capslist = tree.CapsList; + caps = _code.Caps; + capsize = _code.CapSize; + } } internal static void ValidatePattern(string pattern) @@ -128,7 +152,7 @@ internal static void ValidateOptions(RegexOptions options) { if (((((uint)options) >> MaxOptionShift) != 0) || ((options & RegexOptions.ECMAScript) != 0 && - (options & ~(RegexOptions.ECMAScript | RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.Compiled | + (options & ~(RegexOptions.ECMAScript | RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.Compiled | RegexOptions.NonBacktracking | #if DEBUG RegexOptions.Debug | #endif @@ -352,15 +376,17 @@ public int GroupNumberFromName(string name) } } + /// A weak reference to a regex replacement, lazily initialized. + internal WeakReference RegexReplacementWeakReference => + _replref ?? + Interlocked.CompareExchange(ref _replref, new WeakReference(null), null) ?? + _replref; + protected void InitializeReferences() { - if (_refsInitialized) - { - ThrowHelper.ThrowNotSupportedException(ExceptionResource.OnlyAllowedOnce); - } - - _replref = new WeakReference(null); - _refsInitialized = true; + // This method no longer has anything to initialize. It continues to exist + // purely for API compat, as it was originally shipped as protected, with + // assemblies generated by Regex.CompileToAssembly calling it. } /// Internal worker called by the public APIs @@ -375,7 +401,7 @@ protected void InitializeReferences() ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.length, ExceptionResource.LengthNotNegative); } - RegexRunner runner = RentRunner(); + RegexRunner runner = Interlocked.Exchange(ref _runner, null) ?? CreateRunner(); try { // Do the scan starting at the requested position @@ -387,47 +413,33 @@ protected void InitializeReferences() } finally { - ReturnRunner(runner); + _runner = runner; } } internal void Run(string input, int startat, ref TState state, MatchCallback callback, bool reuseMatchObject) { Debug.Assert((uint)startat <= (uint)input.Length); - RegexRunner runner = RentRunner(); + RegexRunner runner = Interlocked.Exchange(ref _runner, null) ?? CreateRunner(); try { - runner.Scan(this, input, startat, ref state, callback, reuseMatchObject, internalMatchTimeout); + runner.ScanInternal(this, input, startat, ref state, callback, reuseMatchObject, internalMatchTimeout); } finally { - ReturnRunner(runner); + _runner = runner; } } - /// Gets a runner from the cache, or creates a new one. - [MethodImpl(MethodImplOptions.AggressiveInlining)] // factored out to be used by only two call sites - private RegexRunner RentRunner() => - Interlocked.Exchange(ref _runner, null) ?? // use a cached runner if there is one - (factory != null ? factory.CreateInstance() : // use the compiled RegexRunner factory if there is one - new RegexInterpreter(_code!, UseOptionInvariant() ? CultureInfo.InvariantCulture : CultureInfo.CurrentCulture)); - - /// Release the runner back to the cache. - internal void ReturnRunner(RegexRunner runner) => _runner = runner; + /// Creates a new runner instance. + private RegexRunner CreateRunner() => + factory?.CreateInstance() ?? + new RegexInterpreter(_code!, GetTargetCulture(roptions)); /// True if the option was set. protected bool UseOptionC() => (roptions & RegexOptions.Compiled) != 0; /// True if the option was set. protected internal bool UseOptionR() => (roptions & RegexOptions.RightToLeft) != 0; - - /// True if the option was set. - internal bool UseOptionInvariant() => (roptions & RegexOptions.CultureInvariant) != 0; - -#if DEBUG - /// True if the regex has debugging enabled. - [ExcludeFromCodeCoverage(Justification = "Debug only")] - internal bool IsDebug => (roptions & RegexOptions.Debug) != 0; -#endif } } diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexCharClass.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexCharClass.cs index fb99b0b6300c0b..9e0afe80d01ac6 100644 --- a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexCharClass.cs +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexCharClass.cs @@ -35,7 +35,7 @@ internal sealed partial class RegexCharClass private const string NullCharString = "\0"; private const char NullChar = '\0'; - private const char LastChar = '\uFFFF'; + internal const char LastChar = '\uFFFF'; private const short SpaceConst = 100; private const short NotSpaceConst = -100; @@ -1267,16 +1267,17 @@ private static RegexCharClass ParseRecursive(string charClass, int start) /// /// Constructs the string representation of the class. /// - public string ToStringClass() + public string ToStringClass(RegexOptions options = RegexOptions.None) { + bool isNonBacktracking = (options & RegexOptions.NonBacktracking) != 0; var vsb = new ValueStringBuilder(stackalloc char[256]); - ToStringClass(ref vsb); + ToStringClass(isNonBacktracking, ref vsb); return vsb.ToString(); } - private void ToStringClass(ref ValueStringBuilder vsb) + private void ToStringClass(bool isNonBacktracking, ref ValueStringBuilder vsb) { - Canonicalize(); + Canonicalize(isNonBacktracking); int initialLength = vsb.Length; int categoriesLength = _categories?.Length ?? 0; @@ -1302,7 +1303,7 @@ private void ToStringClass(ref ValueStringBuilder vsb) // Update the range length. The ValueStringBuilder may have already had some // contents (if this is a subtactor), so we need to offset by the initial length. - vsb[initialLength + SetLengthIndex] = (char)((vsb.Length - initialLength) - SetStartIndex); + vsb[initialLength + SetLengthIndex] = (char)(vsb.Length - initialLength - SetStartIndex); // Append categories if (categoriesLength != 0) @@ -1314,13 +1315,13 @@ private void ToStringClass(ref ValueStringBuilder vsb) } // Append a subtractor if there is one. - _subtractor?.ToStringClass(ref vsb); + _subtractor?.ToStringClass(isNonBacktracking, ref vsb); } /// /// Logic to reduce a character class to a unique, sorted form. /// - private void Canonicalize() + private void Canonicalize(bool isNonBacktracking) { List? rangelist = _rangelist; if (rangelist != null) @@ -1376,7 +1377,10 @@ private void Canonicalize() // If the class now represents a single negated character, but does so by including every // other character, invert it to produce a normalized form recognized by IsSingletonInverse. - if (!_negate && _subtractor is null && (_categories is null || _categories.Length == 0)) + if (!isNonBacktracking && // do not produce the IsSingletonInverse transformation in NonBacktracking mode + !_negate && + _subtractor is null && + (_categories is null || _categories.Length == 0)) { if (rangelist.Count == 2) { diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexNode.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexNode.cs index 4de1c21514b179..282483e8d90657 100644 --- a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexNode.cs +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexNode.cs @@ -152,10 +152,7 @@ public RegexNode(int type, RegexOptions options, int m, int n) N = n; } - public bool UseOptionR() - { - return (Options & RegexOptions.RightToLeft) != 0; - } + public bool UseOptionR() => (Options & RegexOptions.RightToLeft) != 0; public RegexNode ReverseLeft() { @@ -172,7 +169,7 @@ public RegexNode ReverseLeft() /// private void MakeRep(int type, int min, int max) { - Type += (type - One); + Type += type - One; M = min; N = max; } @@ -384,6 +381,13 @@ private static void EliminateEndingBacktracking(RegexNode node, uint maxDepth) return; } + // RegexOptions.NonBacktracking doesn't support atomic groups, so when that option + // is set we don't want to create atomic groups where they weren't explicitly authored. + if ((node.Options & RegexOptions.NonBacktracking) != 0) + { + return; + } + // Walk the tree starting from the provided node. while (true) { @@ -560,6 +564,13 @@ private RegexNode ReduceGroup() /// private RegexNode ReduceAtomic() { + // RegexOptions.NonBacktracking doesn't support atomic groups, so when that option + // is set we don't want to create atomic groups where they weren't explicitly authored. + if ((Options & RegexOptions.NonBacktracking) != 0) + { + return this; + } + Debug.Assert(Type == Atomic); Debug.Assert(ChildCount() == 1); @@ -967,7 +978,7 @@ void ReduceSingleLetterAndNestedAlternations() } prev.Type = Set; - prev.Str = prevCharClass.ToStringClass(); + prev.Str = prevCharClass.ToStringClass(Options); } else if (at.Type == Nothing) { @@ -1442,6 +1453,13 @@ static bool CanCombineCounts(int nodeMin, int nodeMax, int nextMin, int nextMax) /// private void ReduceConcatenationWithAutoAtomic() { + // RegexOptions.NonBacktracking doesn't support atomic groups, so when that option + // is set we don't want to create atomic groups where they weren't explicitly authored. + if ((Options & RegexOptions.NonBacktracking) != 0) + { + return; + } + Debug.Assert(Type == Concatenate); Debug.Assert((Options & RegexOptions.RightToLeft) == 0); Debug.Assert(Children is List); @@ -1660,7 +1678,7 @@ private static bool CanBeMadeAtomic(RegexNode node, RegexNode subsequent, uint m case End: case EndZ when !RegexCharClass.CharInClass('\n', node.Str!): case Eol when !RegexCharClass.CharInClass('\n', node.Str!): - case Boundary when node.Str == RegexCharClass.WordClass || node.Str == RegexCharClass.DigitClass: // TODO: Expand these with a more inclusive overlap check that considers categories + case Boundary when node.Str == RegexCharClass.WordClass || node.Str == RegexCharClass.DigitClass: case NonBoundary when node.Str == RegexCharClass.NotWordClass || node.Str == RegexCharClass.NotDigitClass: case ECMABoundary when node.Str == RegexCharClass.ECMAWordClass || node.Str == RegexCharClass.ECMADigitClass: case NonECMABoundary when node.Str == RegexCharClass.NotECMAWordClass || node.Str == RegexCharClass.NotDigitClass: diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexOptions.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexOptions.cs index d39b2d557ab0e8..47b7eed807dc7b 100644 --- a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexOptions.cs +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexOptions.cs @@ -11,20 +11,61 @@ namespace System.Text.RegularExpressions #endif enum RegexOptions { + /// Use default behavior. None = 0x0000, + + /// Use case-insensitive matching. IgnoreCase = 0x0001, // "i" + + /// + /// Use multiline mode, where ^ and $ match the beginning and end of each line + /// (instead of the beginning and end of the input string). + /// Multiline = 0x0002, // "m" + + /// + /// Do not capture unnamed groups. The only valid captures are explicitly named + /// or numbered groups of the form (?<name> subexpression). + /// ExplicitCapture = 0x0004, // "n" + + /// Compile the regular expression to Microsoft intermediate language (MSIL). Compiled = 0x0008, // "c" + + /// + /// Use single-line mode, where the period (.) matches every character (instead of every character except \n). + /// Singleline = 0x0010, // "s" + + /// Exclude unescaped white space from the pattern, and enable comments after a number sign (#). IgnorePatternWhitespace = 0x0020, // "x" + + /// Change the search direction. Search moves from right to left instead of from left to right. RightToLeft = 0x0040, // "r" + #if DEBUG + /// Enable Regex debugging. Debug = 0x0080, // "d" #endif + + /// Enable ECMAScript-compliant behavior for the expression. ECMAScript = 0x0100, // "e" + + /// Ignore cultural differences in language. CultureInvariant = 0x0200, + /// + /// Enable matching using an approach that avoids backtracking and guarantees linear-time processing + /// in the length of the input. + /// + /// + /// Certain features aren't available when this option is set, including balancing groups, + /// backreferences, positive and negative lookaheads and lookbehinds, and atomic groups. + /// Capture groups are also ignored, such that the only capture available is that for + /// the top-level match. + /// + NonBacktracking = 0x0400, + // RegexCompiler internally uses 0x80000000 for its own internal purposes. // If such a value ever needs to be added publicly, RegexCompiler will need // to be changed to avoid it. diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexParser.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexParser.cs index 2e0ff1467eb65e..48f610caf303ba 100644 --- a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexParser.cs +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexParser.cs @@ -62,17 +62,17 @@ private RegexParser(string pattern, RegexOptions options, CultureInfo culture, H _capnames = capnames; _optionsStack = new ValueListBuilder(optionSpan); - _stack = default; - _group = default; - _alternation = default; - _concatenation = default; - _unit = default; + _stack = null; + _group = null; + _alternation = null; + _concatenation = null; + _unit = null; _currentPos = 0; - _autocap = default; - _capcount = default; - _captop = default; - _capnumlist = default; - _capnamelist = default; + _autocap = 0; + _capcount = 0; + _captop = 0; + _capnumlist = null; + _capnamelist = null; _ignoreNextParen = false; } @@ -240,10 +240,7 @@ private void Reset(RegexOptions options) _stack = null; } - public void Dispose() - { - _optionsStack.Dispose(); - } + public void Dispose() => _optionsStack.Dispose(); /* * The main parsing function. @@ -322,7 +319,7 @@ private RegexNode ScanRegex() goto ContinueOuterScan; case '[': - AddUnitSet(ScanCharClass(UseOptionI(), scanOnly: false)!.ToStringClass()); + AddUnitSet(ScanCharClass(UseOptionI(), scanOnly: false)!.ToStringClass(_options)); break; case '(': @@ -534,7 +531,19 @@ private RegexNode ScanReplacement() { if (RightCharMoveRight() == '$') { - AddUnitNode(ScanDollar()); + RegexNode node = ScanDollar(); + + // NonBacktracking does not support capture groups, so any replacement patterns that refer to + // groups are unsupported. However, the replacement patterns that refer to the left/right portion + // or all of the input as well as referring to group 0 (i.e. the whole match) are supported. + if ((_options & RegexOptions.NonBacktracking) != 0 && + node.Type == RegexNode.Ref && + node.M is not (0 or RegexReplacement.LeftPortion or RegexReplacement.RightPortion or RegexReplacement.WholeString)) + { + throw new NotSupportedException(SR.NotSupported_NonBacktrackingAndReplacementsWithSubstitutionsOfGroups); + } + + AddUnitNode(node); } AddConcatenate(); @@ -1206,7 +1215,7 @@ private void ScanBlank() cc.AddLowercase(_culture); } - return new RegexNode(RegexNode.Set, _options, cc.ToStringClass()); + return new RegexNode(RegexNode.Set, _options, cc.ToStringClass(_options)); default: return ScanBasicBackslash(scanOnly); @@ -1390,6 +1399,7 @@ private RegexNode ScanDollar() int capnum = -1; int newcapnum = ch - '0'; MoveRight(); + CheckUnsupportedNonBacktrackingNumericRef(newcapnum); if (IsCaptureSlot(newcapnum)) { capnum = newcapnum; @@ -1407,6 +1417,7 @@ private RegexNode ScanDollar() newcapnum = newcapnum * 10 + digit; MoveRight(); + CheckUnsupportedNonBacktrackingNumericRef(newcapnum); if (IsCaptureSlot(newcapnum)) { capnum = newcapnum; @@ -1424,6 +1435,7 @@ private RegexNode ScanDollar() int capnum = ScanDecimal(); if (!angled || CharsRight() > 0 && RightCharMoveRight() == '}') { + CheckUnsupportedNonBacktrackingNumericRef(capnum); if (IsCaptureSlot(capnum)) { return new RegexNode(RegexNode.Ref, _options, capnum); @@ -1434,9 +1446,19 @@ private RegexNode ScanDollar() else if (angled && RegexCharClass.IsWordChar(ch)) { string capname = ScanCapname(); - if (CharsRight() > 0 && RightCharMoveRight() == '}' && IsCaptureName(capname)) + if (CharsRight() > 0 && RightCharMoveRight() == '}') { - return new RegexNode(RegexNode.Ref, _options, CaptureSlotFromName(capname)); + // Throw unconditionally for non-backtracking, even if not a valid capture name, + // as information to determine whether a name is valid or not isn't tracked. + if ((_options & RegexOptions.NonBacktracking) != 0) + { + throw new NotSupportedException(SR.NotSupported_NonBacktrackingAndReplacementsWithSubstitutionsOfGroups); + } + + if (IsCaptureName(capname)) + { + return new RegexNode(RegexNode.Ref, _options, CaptureSlotFromName(capname)); + } } } else if (!angled) @@ -1483,6 +1505,16 @@ private RegexNode ScanDollar() return new RegexNode(RegexNode.One, _options, '$'); } + /// Throws on unsupported capture references for NonBacktracking in replacement patterns. + private void CheckUnsupportedNonBacktrackingNumericRef(int capnum) + { + // Throw for non-backtracking on non-zero group, even if not a valid capture number, as information to determine whether a name is valid or not isn't tracked + if ((_options & RegexOptions.NonBacktracking) != 0 && capnum != 0) + { + throw new NotSupportedException(SR.NotSupported_NonBacktrackingAndReplacementsWithSubstitutionsOfGroups); + } + } + /* * Scans a capture name: consumes word chars */ @@ -2271,22 +2303,13 @@ private void AddUnitNotone(char ch) } /// Sets the current unit to a single set node - private void AddUnitSet(string cc) - { - _unit = new RegexNode(RegexNode.Set, _options, cc); - } + private void AddUnitSet(string cc) => _unit = new RegexNode(RegexNode.Set, _options, cc); /// Sets the current unit to a subtree - private void AddUnitNode(RegexNode node) - { - _unit = node; - } + private void AddUnitNode(RegexNode node) => _unit = node; /// Sets the current unit to an assertion of the specified type - private void AddUnitType(int type) - { - _unit = new RegexNode(type, _options); - } + private void AddUnitType(int type) => _unit = new RegexNode(type, _options); /// Finish the current group (in response to a ')' or end) private void AddGroup() diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexReplacement.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexReplacement.cs index 3a27f5852df00a..814f05e5aeb119 100644 --- a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexReplacement.cs +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexReplacement.cs @@ -106,12 +106,11 @@ public RegexReplacement(string rep, RegexNode concat, Hashtable _caps) /// Either returns a weakly cached RegexReplacement helper or creates one and caches it. /// /// - public static RegexReplacement GetOrCreate(WeakReference replRef, string replacement, Hashtable caps, + public static RegexReplacement GetOrCreate(WeakReference replRef, string replacement, Hashtable caps, int capsize, Hashtable capnames, RegexOptions roptions) { - RegexReplacement? repl; - if (!replRef.TryGetTarget(out repl) || !repl.Pattern.Equals(replacement)) + if (!replRef.TryGetTarget(out RegexReplacement? repl) || !repl.Pattern.Equals(replacement)) { repl = RegexParser.ParseReplacement(replacement, roptions, caps, capsize, capnames); replRef.SetTarget(repl); diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexRunner.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexRunner.cs index 3006cbc26cf8d3..dbab82e7cfd4a2 100644 --- a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexRunner.cs +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexRunner.cs @@ -57,6 +57,9 @@ public abstract class RegexRunner protected internal Match? runmatch; // result object protected internal Regex? runregex; // regex object + // TODO: Expose something as protected internal: https://github.com/dotnet/runtime/issues/59629 + private protected bool quick; // false if match details matter, true if only the fact that match occurred matters + private int _timeout; // timeout in milliseconds (needed for actual) private bool _ignoreTimeout; private int _timeoutOccursAt; @@ -68,7 +71,7 @@ public abstract class RegexRunner private const int TimeoutCheckFrequency = 1000; private int _timeoutChecksToSkip; - protected internal RegexRunner() { } + protected RegexRunner() { } /// /// Scans the string to find the first match. Uses the Match object @@ -82,11 +85,13 @@ protected internal RegexRunner() { } /// and we could use a separate method Skip() that will quickly scan past /// any characters that we know can't match. /// - protected internal Match? Scan(Regex regex, string text, int textbeg, int textend, int textstart, int prevlen, bool quick) => + protected Match? Scan(Regex regex, string text, int textbeg, int textend, int textstart, int prevlen, bool quick) => Scan(regex, text, textbeg, textend, textstart, prevlen, quick, regex.MatchTimeout); protected internal Match? Scan(Regex regex, string text, int textbeg, int textend, int textstart, int prevlen, bool quick, TimeSpan timeout) { + this.quick = quick; + // Handle timeout argument _timeout = -1; // (int)Regex.InfiniteMatchTimeout.TotalMilliseconds bool ignoreTimeout = _ignoreTimeout = Regex.InfiniteMatchTimeout == timeout; @@ -218,8 +223,10 @@ protected internal RegexRunner() { } /// This optionally repeatedly hands out the same Match instance, updated with new information. /// should be set to false if the Match object is handed out to user code. /// - internal void Scan(Regex regex, string text, int textstart, ref TState state, MatchCallback callback, bool reuseMatchObject, TimeSpan timeout) + internal void ScanInternal(Regex regex, string text, int textstart, ref TState state, MatchCallback callback, bool reuseMatchObject, TimeSpan timeout) { + quick = false; + // Handle timeout argument _timeout = -1; // (int)Regex.InfiniteMatchTimeout.TotalMilliseconds bool ignoreTimeout = _ignoreTimeout = Regex.InfiniteMatchTimeout == timeout; diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexRunnerFactory.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexRunnerFactory.cs index 5d3a84299c30ac..93860f20cdf477 100644 --- a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexRunnerFactory.cs +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexRunnerFactory.cs @@ -1,8 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -// This RegexRunnerFactory class is a base class for compiled regex code. - namespace System.Text.RegularExpressions { public abstract class RegexRunnerFactory diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Algebras/BDD.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Algebras/BDD.cs new file mode 100644 index 00000000000000..60640e1741e8f5 --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Algebras/BDD.cs @@ -0,0 +1,501 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; + +namespace System.Text.RegularExpressions.Symbolic +{ + /// + /// Represents nodes in a Binary Decision Diagram (BDD), which compactly represent sets of integers. All non-leaf + /// nodes have an Ordinal, which indicates the position of the bit the node relates to (0 for the least significant + /// bit), and two children, One and Zero, for the cases of the current bit being 1 or 0, respectively. An integer + /// belongs to the set represented by the BDD if the path from the root following the branches that correspond to + /// the bits of the integer leads to the True leaf. This class also supports multi-terminal BDDs (MTBDD), i.e. ones where + /// the leaves are something other than True or False, which are used for representing classifiers. + /// + internal sealed class BDD : IComparable + { + /// + /// The ordinal for the True special value. + /// + private const int TrueOrdinal = -2; + + /// + /// The ordinal for the False special value. + /// + private const int FalseOrdinal = -1; + + /// + /// The unique BDD leaf that represents the full set or true. + /// + public static readonly BDD True = new BDD(TrueOrdinal, null, null); + + /// + /// The unique BDD leaf that represents the empty set or false. + /// + public static readonly BDD False = new BDD(FalseOrdinal, null, null); + + /// + /// The encoding of the set for lower ordinals for the case when the current bit is 1. + /// The value is null iff IsLeaf is true. + /// + public readonly BDD? One; + + /// + /// The encoding of the set for lower ordinals for the case when the current bit is 0. + /// The value is null iff IsLeaf is true. + /// + public readonly BDD? Zero; + + /// + /// Ordinal of this bit if nonleaf else MTBDD terminal value when nonnegative + /// + public readonly int Ordinal; + + /// + /// Preassigned hashcode value that respects equivalence: equivalent BDDs have equal hashcodes + /// + private readonly int _hashcode; + + /// + /// Representation of False for serialization. + /// + private static readonly long[] s_falseRepresentation = new long[] { 0 }; + + /// + /// Representation of True for serialization. + /// + private static readonly long[] s_trueRepresentation = new long[] { 1 }; + + internal BDD(int ordinal, BDD? one, BDD? zero) + { + One = one; + Zero = zero; + Ordinal = ordinal; + + // Precompute a hashchode value that respects BDD equivalence. + // Two equivalent BDDs will always have the same hashcode + // that is independent of object id values of the BDD objects. + _hashcode = HashCode.Combine(ordinal, one, zero); + } + + /// + /// True iff the node is a terminal (One and Zero are both null). + /// True and False are terminals. + /// + [MemberNotNullWhen(false, nameof(One))] + [MemberNotNullWhen(false, nameof(Zero))] + public bool IsLeaf + { + get + { + if (One is null) + { + Debug.Assert(Zero is null); + return true; + } + + Debug.Assert(Zero is not null); + return false; + } + } + + /// + /// True iff the BDD is True. + /// + public bool IsFull => this == True; + + /// + /// True iff the BDD is False. + /// + public bool IsEmpty => this == False; + + /// + /// Gets the lexicographically minimum bitvector in this BDD as a ulong. + /// The BDD must be nonempty. + /// + public ulong GetMin() + { + BDD set = this; + Debug.Assert(!set.IsEmpty); + + if (set.IsFull) + return 0; + + // starting from all 0, bits will be flipped to 1 as necessary + ulong res = 0; + + // follow the minimum path throught the branches to a True leaf + while (!set.IsLeaf) + { + if (set.Zero.IsEmpty) //the bit must be set to 1 + { + // the bit must be set to 1 when the zero branch is False + res |= (ulong)1 << set.Ordinal; + // if zero is empty then by the way BDDs are constructed one is not + set = set.One; + } + else + { + // otherwise, leaving the bit as 0 gives the smaller bitvector + set = set.Zero; + } + } + + return res; + } + + /// + /// O(1) operation that returns the precomputed hashcode. + /// + public override int GetHashCode() => _hashcode; + + /// + /// A shallow equality check that holds if ordinals are identical and one's are identical and zero's are identical. + /// This equality is used in the _bddCache lookup. + /// + public override bool Equals(object? obj) => + obj is BDD bdd && + (this == bdd || (Ordinal == bdd.Ordinal && One == bdd.One && Zero == bdd.Zero)); + + /// + /// Returns a topologically sorted array of all the nodes (other than True or False) in this BDD + /// such that, all MTBDD leaves (other than True or False) appear first in the array + /// and all nonterminals with smaller ordinal appear before nodes with larger ordinal. + /// So this BDD itself (if different from True or False) appears last. + /// In the case of True or False returns the empty array. + /// + public BDD[] TopologicalSort() + { + if (IsFull || IsEmpty) + return Array.Empty(); + + if (IsLeaf) + return new BDD[] { this }; + + // Order the nodes according to their ordinals into the nonterminals array + var nonterminals = new List[Ordinal + 1]; + var sorted = new List(); + var toVisit = new Stack(); + var visited = new HashSet(); + + toVisit.Push(this); + + while (toVisit.Count > 0) + { + BDD node = toVisit.Pop(); + // True and False are not included in the result + if (node.IsFull || node.IsEmpty) + continue; + + if (node.IsLeaf) + { + // MTBDD terminals can be directly added to the sorted nodes, since they have no children that + // would come first in the topological ordering. + sorted.Add(node); + } + else + { + // Non-terminals are grouped by their ordinal so that they can be sorted into a topological order. + (nonterminals[node.Ordinal] ??= new List()).Add(node); + + if (visited.Add(node.Zero)) + toVisit.Push(node.Zero); + + if (visited.Add(node.One)) + toVisit.Push(node.One); + } + } + + // Flush the grouped non-terminals into the sorted nodes from smallest to highest ordinal. The highest + // ordinal is guaranteed to have only one node, which places the root of the BDD at the end. + for (int i = 0; i < nonterminals.Length; i++) + { + if (nonterminals[i] != null) + { + sorted.AddRange(nonterminals[i]); + } + } + + return sorted.ToArray(); + } + + #region Serialization + /// + /// Serialize this BDD in a flat ulong array. The BDD may have at most 2^k ordinals and 2^n nodes, such that k+2n < 64 + /// BDD.False is represented by return value ulong[]{0}. + /// BDD.True is represented by return value ulong[]{1}. + /// Serializer uses more compacted representations when fewer bits are needed, which is reflected in the first + /// two numbers of the return value. MTBDD terminals are represented by negated numbers as -id. + /// + public long[] Serialize() + { + if (IsEmpty) + return s_falseRepresentation; + + if (IsFull) + return s_trueRepresentation; + + if (IsLeaf) + return new long[] { 0, 0, -Ordinal }; + + BDD[] nodes = TopologicalSort(); + + Debug.Assert(nodes[nodes.Length - 1] == this); + Debug.Assert(nodes.Length <= (1 << 24)); + + // As few bits as possible are used to for ordinals and node identifiers for compact serialization. + // Use at least a nibble (4 bits) to represent the ordinal and count how many are needed. + int ordinal_bits = 4; + while (Ordinal >= (1 << ordinal_bits)) + { + ordinal_bits += 1; + } + + // Use at least 2 bits to represent the node identifier and count how many are needed + int node_bits = 2; + while (nodes.Length >= (1 << node_bits)) + { + node_bits += 1; + } + + // Reserve space for all nodes plus 2 extra: index 0 and 1 are reserved for False and True + long[] res = new long[nodes.Length + 2]; + res[0] = ordinal_bits; + res[1] = node_bits; + + //use the following bit layout + BitLayout(ordinal_bits, node_bits, out int zero_node_shift, out int one_node_shift, out int ordinal_shift); + + //here we know that bdd is neither False nor True + //but it could still be a MTBDD leaf if both children are null + var idmap = new Dictionary + { + [True] = 1, + [False] = 0 + }; + + // Give all nodes ascending identifiers and produce their serializations into the result + for (int i = 0; i < nodes.Length; i++) + { + BDD node = nodes[i]; + idmap[node] = i + 2; + + if (node.IsLeaf) + { + // This is MTBDD leaf. Negating it should make it less than or equal to zero, as True and False are + // excluded here and MTBDD Ordinals are required to be non-negative. + res[i + 2] = -node.Ordinal; + } + else + { + // Combine ordinal and child identifiers according to the bit layout + long v = (((long)node.Ordinal) << ordinal_shift) | (idmap[node.One] << one_node_shift) | (idmap[node.Zero] << zero_node_shift); + Debug.Assert(v >= 0); + res[i + 2] = v; // children ids are well-defined due to the topological order of nodes + } + } + return res; + } + + /// + /// Recreates a BDD from a ulong array that has been created using Serialize. + /// Is executed using a lock on algebra (if algebra != null) in a single thread mode. + /// If no algebra is given (algebra is null) then creates the BDD without using a BDD algebra -- + /// which implies that all BDD nodes other than True and False are new BDD objects + /// that have not been internalized or cached. + /// + public static BDD Deserialize(long[] arcs, BDDAlgebra algebra) + { + if (arcs.Length == 1) + { + return arcs[0] == 0 ? False : True; + } + + // the number of bits used for ordinals and node identifiers are stored in the first two values + int k = arcs.Length; + int ordinal_bits = (int)arcs[0]; + int node_bits = (int)arcs[1]; + + // create bit masks for the sizes of ordinals and node identifiers + long ordinal_mask = (1 << ordinal_bits) - 1; + long node_mask = (1 << node_bits) - 1; + BitLayout(ordinal_bits, node_bits, out int zero_node_shift, out int one_node_shift, out int ordinal_shift); + + // store BDD nodes by their id when they are created + BDD[] nodes = new BDD[k]; + nodes[0] = False; + nodes[1] = True; + + for (int i = 2; i < k; i++) + { + long arc = arcs[i]; + if (arc <= 0) + { + // this is an MTBDD leaf. Its ordinal was serialized negated + nodes[i] = algebra.GetOrCreateBDD((int)-arc, null, null); + } + else + { + // reconstruct the ordinal and child identifiers for a non-terminal + int ord = (int)((arc >> ordinal_shift) & ordinal_mask); + int oneId = (int)((arc >> one_node_shift) & node_mask); + int zeroId = (int)((arc >> zero_node_shift) & node_mask); + + // the BDD nodes for the children are guaranteed to exist already due to the topological order + nodes[i] = algebra.GetOrCreateBDD(ord, nodes[oneId], nodes[zeroId]); + } + } + + //the result is the final BDD in the nodes array + return nodes[k - 1]; + } + + /// + /// Use this bit layout in the serialization + /// + private static void BitLayout(int ordinal_bits, int node_bits, out int zero_node_shift, out int one_node_shift, out int ordinal_shift) + { + //this bit layout seems to work best: zero,one,ord + zero_node_shift = ordinal_bits + node_bits; + one_node_shift = ordinal_bits; + ordinal_shift = 0; + } + #endregion + + /// + /// Finds the terminal for the input in a Multi-Terminal-BDD. + /// Bits of the input are used to determine the path in the BDD. + /// Returns -1 if False is reached and -2 if True is reached, + /// else returns the MTBDD terminal number that is reached. + /// If this is a nonterminal, Find does not care about input bits > Ordinal. + /// + public int Find(int input) => + IsLeaf ? Ordinal : + (input & (1 << Ordinal)) == 0 ? Zero.Find(input) : + One.Find(input); + + /// + /// Finds the terminal for the input in a Multi-Terminal-BDD. + /// Bits of the input are used to determine the path in the BDD. + /// Returns -1 if False is reached and 0 if True is reached, + /// else returns the MTBDD terminal number that is reached. + /// If this is a nonterminal, Find does not care about input bits > Ordinal. + /// + public int Find(ulong input) => + IsLeaf ? Ordinal : + (input & ((ulong)1 << Ordinal)) == 0 ? Zero.Find(input) : + One.Find(input); + + /// + /// Assumes BDD is not MTBDD and returns true iff it contains the input. + /// (Otherwise use BDD.Find if this is if fact a MTBDD.) + /// + public bool Contains(int input) => Find(input) == TrueOrdinal; //-2 is the Ordinal of BDD.True + + /// + /// Returns true if the only other terminal besides False is a MTBDD terminal that is different from True. + /// If this is the case, outputs that terminal. + /// + public bool IsEssentiallyBoolean([NotNullWhen(true)] out BDD? terminalActingAsTrue) + { + if (IsFull || IsEmpty) + { + terminalActingAsTrue = null; + return false; + } + + if (IsLeaf) + { + terminalActingAsTrue = this; + return true; + } + + var toVisit = new Stack(); + var visited = new HashSet(); + + toVisit.Push(this); + + // this will hold the unique MTBDD leaf + BDD? leaf = null; + + while (toVisit.Count > 0) + { + BDD node = toVisit.Pop(); + if (node.IsEmpty) + continue; + + if (node.IsFull) + { + //contains the True leaf + terminalActingAsTrue = null; + return false; + } + + if (node.IsLeaf) + { + if (leaf is null) + { + // remember the first MTBDD leaf seen + leaf = node; + } + else if (leaf != node) + { + // found two different MTBDD leaves + terminalActingAsTrue = null; + return false; + } + } + else + { + if (visited.Add(node.Zero)) + toVisit.Push(node.Zero); + + if (visited.Add(node.One)) + toVisit.Push(node.One); + } + } + + Debug.Assert(leaf is not null, "this should never happen because there must exist another leaf besides False"); + // found an MTBDD leaf and didn't find any other (non-False) leaves + terminalActingAsTrue = leaf; + return true; + } + + /// + /// All terminals precede all nonterminals. Compares Ordinals for terminals. + /// Compare non-terminals by comparing their minimal elements. + /// If minimal elements are the same, compare Ordinals. + /// This provides a total order for terminals. + /// + public int CompareTo(object? obj) + { + if (obj is not BDD bdd) + { + return -1; + } + + if (IsLeaf) + { + return + !bdd.IsLeaf || Ordinal < bdd.Ordinal ? -1 : + Ordinal == bdd.Ordinal ? 0 : + 1; + } + + if (bdd.IsLeaf) + { + return 1; + } + + ulong min = GetMin(); + ulong bdd_min = bdd.GetMin(); + return + min < bdd_min ? -1 : + bdd_min < min ? 1 : + Ordinal.CompareTo(bdd.Ordinal); + } + } +} diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Algebras/BDDAlgebra.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Algebras/BDDAlgebra.cs new file mode 100644 index 00000000000000..5096efe35ce6df --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Algebras/BDDAlgebra.cs @@ -0,0 +1,511 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; + +namespace System.Text.RegularExpressions.Symbolic +{ + /// + /// Boolean operations over BDDs. + /// + internal enum BoolOp + { + Or, + And, + Xor, + Not + } + + /// + /// Boolean algebra for Binary Decision Diagrams. Boolean operations on BDDs are cached for efficiency. The + /// IBooleanAlgebra interface implemented by this class is thread safe. + /// TBD: policy for clearing/reducing the caches when they grow too large. + /// Ultimately, the caches are crucial for efficiency, not for correctness. + /// + internal abstract class BDDAlgebra : IBooleanAlgebra + { + /// + /// Operation cache for Boolean operations over BDDs. + /// + private readonly ConcurrentDictionary<(BoolOp op, BDD a, BDD? b), BDD> _opCache = new(); + + /// + /// Internalize the creation of BDDs so that two BDDs with same ordinal and identical children are the same object. + /// The algorithms do not rely on 100% internalization + /// (they could but this would make it difficult (or near impossible) to clear caches. + /// Allowing distinct but equivalent BDDs is also a tradeoff between efficiency and flexibility. + /// + private readonly ConcurrentDictionary<(int ordinal, BDD? one, BDD? zero), BDD> _bddCache = new(); + + /// + /// Generator for minterms. + /// + private readonly MintermGenerator _mintermGen; + + /// + /// Construct a solver for BDDs. + /// + public BDDAlgebra() => _mintermGen = new MintermGenerator(this); + + /// + /// Create a BDD with given ordinal and given one and zero child. + /// Returns the BDD from the cache if it already exists. + /// + public BDD GetOrCreateBDD(int ordinal, BDD? one, BDD? zero) => + _bddCache.GetOrAdd((ordinal, one, zero), static key => new BDD(key.ordinal, key.one, key.zero)); + + #region IBooleanAlgebra members + + /// + /// Make the union of a and b + /// + public BDD Or(BDD a, BDD b) => ApplyBinaryOp(BoolOp.Or, a, b); + + /// + /// Make the intersection of a and b + /// + public BDD And(BDD a, BDD b) => ApplyBinaryOp(BoolOp.And, a, b); + + /// + /// Complement a + /// + public BDD Not(BDD a) => + a == False ? True : + a == True ? False : + _opCache.GetOrAdd((BoolOp.Not, a, null), static (key, algebra) => key.a.IsLeaf ? + algebra.GetOrCreateBDD(algebra.CombineTerminals(BoolOp.Not, key.a.Ordinal, 0), null, null) : // multi-terminal case + algebra.GetOrCreateBDD(key.a.Ordinal, algebra.Not(key.a.One), algebra.Not(key.a.Zero)), + this); + + /// + /// Applies the binary Boolean operation op and constructs the BDD recursively from a and b. + /// + /// given binary Boolean operation + /// first BDD + /// second BDD + /// + private BDD ApplyBinaryOp(BoolOp op, BDD a, BDD b) + { + // Handle base cases + #region the cases when one of a or b is True or False or when a == b + switch (op) + { + case BoolOp.Or: + if (a == False) + return b; + if (b == False) + return a; + if (a == True || b == True) + return True; + if (a == b) + return a; + break; + + case BoolOp.And: + if (a == True) + return b; + if (b == True) + return a; + if (a == False || b == False) + return False; + if (a == b) + return a; + break; + + case BoolOp.Xor: + if (a == False) + return b; + if (b == False) + return a; + if (a == b) + return False; + if (a == True) + return Not(b); + if (b == True) + return Not(a); + break; + + default: + Debug.Fail("Unhandled binary BoolOp case"); + break; + } + #endregion + + // Order operands by hash code to increase cache hits + if (a.GetHashCode() > b.GetHashCode()) + { + BDD tmp = a; + a = b; + b = tmp; + } + + return _opCache.GetOrAdd((op, a, b), static (key, algebra) => + { + Debug.Assert(key.b is not null, "Validated it was non-null prior to calling GetOrAdd"); + + if (key.a.IsLeaf && key.b.IsLeaf) + { + // Multi-terminal case, we know here that a is neither True nor False + int ord = algebra.CombineTerminals(key.op, key.a.Ordinal, key.b.Ordinal); + return algebra.GetOrCreateBDD(ord, null, null); + } + + if (key.a.IsLeaf || key.b!.Ordinal > key.a.Ordinal) + { + Debug.Assert(!key.b.IsLeaf); + BDD t = algebra.ApplyBinaryOp(key.op, key.a, key.b.One); + BDD f = algebra.ApplyBinaryOp(key.op, key.a, key.b.Zero); + return t == f ? t : algebra.GetOrCreateBDD(key.b.Ordinal, t, f); + } + + if (key.b.IsLeaf || key.a.Ordinal > key.b.Ordinal) + { + Debug.Assert(!key.a.IsLeaf); + BDD t = algebra.ApplyBinaryOp(key.op, key.a.One, key.b); + BDD f = algebra.ApplyBinaryOp(key.op, key.a.Zero, key.b); + return t == f ? t : algebra.GetOrCreateBDD(key.a.Ordinal, t, f); + } + + { + Debug.Assert(!key.a.IsLeaf); + Debug.Assert(!key.b.IsLeaf); + BDD t = algebra.ApplyBinaryOp(key.op, key.a.One, key.b.One); + BDD f = algebra.ApplyBinaryOp(key.op, key.a.Zero, key.b.Zero); + return t == f ? t : algebra.GetOrCreateBDD(key.a.Ordinal, t, f); + } + }, this); + } + + /// + /// Intersect all sets in the enumeration + /// + public BDD And(IEnumerable sets) + { + BDD res = True; + foreach (BDD bdd in sets) + { + res = And(res, bdd); + } + return res; + } + + /// + /// Take the union of all sets in the enumeration + /// + public BDD Or(IEnumerable sets) + { + BDD res = False; + foreach (BDD bdd in sets) + { + res = Or(res, bdd); + } + return res; + } + + /// + /// Gets the full set. + /// + public BDD True => BDD.True; + + /// + /// Gets the empty set. + /// + public BDD False => BDD.False; + + /// + /// Returns true if the set is nonempty. + /// + public bool IsSatisfiable(BDD set) => set != False; + + /// + /// Returns true if a and b represent equivalent BDDs. + /// + public bool AreEquivalent(BDD a, BDD b) => Xor(a, b) == False; + + #endregion + + /// + /// Make the XOR of a and b + /// + internal BDD Xor(BDD a, BDD b) => ApplyBinaryOp(BoolOp.Xor, a, b); + + #region bit-shift operations + + /// + /// Shift all elements k bits to the right. + /// For example if set denotes {*0000,*1110,*1111} then + /// ShiftRight(set) denotes {*000,*111} where * denotes any prefix of 0's or 1's. + /// + public BDD ShiftRight(BDD set, int k) + { + Debug.Assert(k >= 0); + return set.IsLeaf ? set : ShiftLeftImpl(new Dictionary<(BDD set, int k), BDD>(), set, 0 - k); + } + + /// + /// Shift all elements k bits to the left. + /// For example if k=1 and set denotes {*0000,*1111} then + /// ShiftLeft(set) denotes {*00000,*00001,*11110,*11111} where * denotes any prefix of 0's or 1's. + /// + public BDD ShiftLeft(BDD set, int k) + { + Debug.Assert(k >= 0); + return set.IsLeaf ? set : ShiftLeftImpl(new Dictionary<(BDD set, int k), BDD>(), set, k); + } + + /// + /// Uses shiftCache to avoid recomputations in shared BDDs (which are DAGs). + /// + private BDD ShiftLeftImpl(Dictionary<(BDD set, int k), BDD> shiftCache, BDD set, int k) + { + if (set.IsLeaf || k == 0) + return set; + + int ordinal = set.Ordinal + k; + + if (ordinal < 0) + return True; //this arises if k is negative + + if (!shiftCache.TryGetValue((set, k), out BDD? res)) + { + BDD zero = ShiftLeftImpl(shiftCache, set.Zero, k); + BDD one = ShiftLeftImpl(shiftCache, set.One, k); + + res = (zero == one) ? + zero : + GetOrCreateBDD((ushort)ordinal, one, zero); + shiftCache[(set, k)] = res; + } + return res; + } + + #endregion + + /// + /// Generate all non-overlapping Boolean combinations of a set of BDDs. + /// + /// the BDDs to create the minterms for + /// BDDs for the minterm + public List GenerateMinterms(params BDD[] sets) => _mintermGen.GenerateMinterms(sets); + + /// + /// Make a set containing all integers whose bits up to maxBit equal n. + /// + /// the given integer + /// bits above maxBit are unspecified + /// + public BDD CreateSetFrom(uint n, int maxBit) => CreateSetFromRange(n, n, maxBit); + + /// + /// Make the set containing all values greater than or equal to m and less than or equal to n when considering bits between 0 and maxBit. + /// + /// lower bound + /// upper bound + /// bits above maxBit are unspecified + public BDD CreateSetFromRange(uint lower, uint upper, int maxBit) + { + Debug.Assert(0 <= maxBit && maxBit <= 31, "maxBit must be between 0 and 31"); + + if (upper < lower) + return False; + + // Filter out bits greater than maxBit + if (maxBit < 31) + { + uint filter = (1u << (maxBit + 1)) - 1; + lower &= filter; + upper &= filter; + } + + return CreateSetFromRangeImpl(lower, upper, maxBit); + } + + private BDD CreateSetFromRangeImpl(uint lower, uint upper, int maxBit) + { + // Mask with 1 at position of maxBit + uint mask = 1u << maxBit; + + if (mask == 1) // Base case for least significant bit + { + return + upper == 0 ? GetOrCreateBDD(maxBit, False, True) : // lower must also be 0 + lower == 1 ? GetOrCreateBDD(maxBit, True, False) : // upper must also be 1 + True; // Otherwise both 0 and 1 are included + } + + // Check if range includes all numbers up to bit + if (lower == 0 && upper == ((mask << 1) - 1)) + { + return True; + } + + // Mask out the highest bit for the first and last elements in the range + uint lowerMasked = lower & mask; + uint upperMasked = upper & mask; + + if (upperMasked == 0) + { + // Highest value in range doesn't have maxBit set, so the one branch is empty + BDD zero = CreateSetFromRangeImpl(lower, upper, maxBit - 1); + return GetOrCreateBDD(maxBit, False, zero); + } + else if (lowerMasked == mask) + { + // Lowest value in range has maxBit set, so the zero branch is empty + BDD one = CreateSetFromRangeImpl(lower & ~mask, upper & ~mask, maxBit - 1); + return GetOrCreateBDD(maxBit, one, False); + } + else // Otherwise the range straddles (1< + /// Convert the set into an equivalent array of uint ranges. + /// Bits above maxBit are ignored. + /// The ranges are nonoverlapping and ordered. + /// + public static (uint, uint)[] ToRanges(BDD set, int maxBit) => BDDRangeConverter.ToRanges(set, maxBit); + + #region domain size and min computation + + /// + /// Calculate the number of elements in the set. Returns 0 when set is full and maxBit is 63. + /// + /// the given set + /// bits above maxBit are ignored + /// the cardinality of the set + public virtual ulong ComputeDomainSize(BDD set, int maxBit) + { + if (maxBit < set.Ordinal) + throw new ArgumentOutOfRangeException(nameof(maxBit)); + + if (set == False) + return 0UL; + + if (set == True) + return 1UL << maxBit << 1; // e.g. if maxBit is 15 then the return value is 1 << 16, i.e., 2^16 + + if (set.IsLeaf) + throw new NotSupportedException(); // multi-terminal case is not supported + + ulong res = ComputeDomainSizeImpl(new Dictionary(), set); + if (maxBit > set.Ordinal) + { + res = (1UL << (maxBit - set.Ordinal)) * res; + } + + return res; + } + + /// + /// Caches previously calculated values in sizeCache so that computations are not repeated inside a BDD for the same sub-BDD. + /// Thus the number of internal calls is propotional to the number of nodes of the BDD, that could otherwise be exponential in the worst case. + /// + /// previously computed sizes + /// given set to compute size of + /// + private ulong ComputeDomainSizeImpl(Dictionary sizeCache, BDD set) + { + if (!sizeCache.TryGetValue(set, out ulong size)) + { + if (set.IsLeaf) + throw new NotSupportedException(); //multi-terminal case is not supported + + ulong sizeL; + ulong sizeR; + if (set.Zero.IsEmpty) + { + sizeL = 0; + sizeR = set.One.IsFull ? + (uint)1 << set.Ordinal : + ((uint)1 << (set.Ordinal - 1 - set.One.Ordinal)) * ComputeDomainSizeImpl(sizeCache, set.One); + } + else if (set.Zero.IsFull) + { + sizeL = 1UL << set.Ordinal; + sizeR = set.One.IsEmpty ? + 0UL : + (1UL << (set.Ordinal - 1 - set.One.Ordinal)) * ComputeDomainSizeImpl(sizeCache, set.One); + } + else + { + sizeL = (1UL << (set.Ordinal - 1 - set.Zero.Ordinal)) * ComputeDomainSizeImpl(sizeCache, set.Zero); + sizeR = + set.One == False ? 0UL : + set.One == True ? 1UL << set.Ordinal : + (1UL << (set.Ordinal - 1 - set.One.Ordinal)) * ComputeDomainSizeImpl(sizeCache, set.One); + } + + size = sizeL + sizeR; + sizeCache[set] = size; + } + return size; + } + + /// + /// Get the lexicographically minimum bitvector in the set as a ulong. + /// Assumes that the set is nonempty and that the ordinal of the BDD is at most 63. + /// + /// the given nonempty set + /// the lexicographically smallest bitvector in the set + public ulong GetMin(BDD set) => set.GetMin(); + + #endregion + + /// + /// Any two BDDs that are equivalent are isomorphic and have the same hashcode. + /// + public bool HashCodesRespectEquivalence => true; + + /// + /// Two equivalent BDDs need not be identical + /// + public bool IsExtensional => false; + + /// + /// The returned integer must be nonegative + /// and will act as the combined terminal in a multi-terminal BDD. + /// May throw NotSupportedException. + /// + public abstract int CombineTerminals(BoolOp op, int terminal1, int terminal2); + + /// + /// Replace the True node in the BDD by a non-Boolean terminal. + /// Locks the algebra for single threaded use. + /// Observe that the Ordinal of False is -1 and the Ordinal of True is -2. + /// + public BDD ReplaceTrue(BDD bdd, int terminal) + { + Debug.Assert(terminal >= 0); + + BDD leaf = GetOrCreateBDD(terminal, null, null); + return ReplaceTrueImpl(bdd, leaf, new Dictionary()); + } + + private BDD ReplaceTrueImpl(BDD bdd, BDD leaf, Dictionary cache) + { + if (bdd == True) + return leaf; + + if (bdd.IsLeaf) + return bdd; + + if (!cache.TryGetValue(bdd, out BDD? res)) + { + BDD one = ReplaceTrueImpl(bdd.One, leaf, cache); + BDD zero = ReplaceTrueImpl(bdd.Zero, leaf, cache); + res = GetOrCreateBDD(bdd.Ordinal, one, zero); + cache[bdd] = res; + } + return res; + } + } +} diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Algebras/BDDRangeConverter.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Algebras/BDDRangeConverter.cs new file mode 100644 index 00000000000000..d6ff1d2b6aa30c --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Algebras/BDDRangeConverter.cs @@ -0,0 +1,230 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics; + +namespace System.Text.RegularExpressions.Symbolic +{ + /// + /// Converts sets of integers expressed as BDDs into non-overlapping and ordered arrays of ranges. + /// + internal sealed class BDDRangeConverter + { + /// + /// Cache for all range conversions. Having this cache is required to avoid exponential running time. + /// + private readonly Dictionary _rangeCache = new Dictionary(); + + private BDDRangeConverter() { } + + /// + /// Convert the set into an equivalent array of ranges. + /// The ranges are nonoverlapping and ordered. + /// + public static (uint, uint)[] ToRanges(BDD set, int maxBit) + { + Debug.Assert(0 <= maxBit && maxBit <= 31, "maxBit must be between 0 and 31"); + + if (set.IsEmpty) + return Array.Empty<(uint, uint)>(); + + if (set.IsFull) + return new[] { (0u, ((uint)1 << maxBit << 1) - 1) }; //note: maxBit could be 31 + + var rc = new BDDRangeConverter(); + return rc.LiftRanges(maxBit + 1, maxBit - set.Ordinal, rc.ToRangesFromOrdinal(set)); + } + + /// + /// Extends a set of ranges to include more significant bits. The new bits are allowed to be anything and all + /// combinations of the new bits are included. + /// e.g. if toBits = 6 and newBits = 2 and ranges = (in binary form) {[0000 1010, 0000 1110]} i.e. [x0A,x0E] + /// then res = {[0000 1010, 0000 1110], [0001 1010, 0001 1110], + /// [0010 1010, 0010 1110], [0011 1010, 0011 1110]}, + /// + private (uint, uint)[] LiftRanges(int toBits, int newBits, (uint, uint)[] ranges) + { + // nothing happens if no new bits are added + if (newBits == 0) + return ranges; + + int fromBits = toBits - newBits; + + // Iterate through all combinations of the new bits + var result = new (uint, uint)[(1 << newBits) * ranges.Length]; + int resultPos = 0; + for (uint i = 0; i < (1 << newBits); i++) + { + // Shift the prefix to be past the existing range of bits, and + // generate each range with this prefix added. + uint prefix = i << fromBits; + foreach ((uint, uint) range in ranges) + { + result[resultPos++] = (range.Item1 | prefix, range.Item2 | prefix); + } + } + + // lifted ranges can wrap around like this [0...][...2^fromBits-1][2^fromBits...][...2^(fromBits+1)-1] + uint maximal = ((uint)1 << fromBits) - 1; + if (ranges[0].Item1 == 0 && ranges[ranges.Length - 1].Item2 == maximal) + { + // merge consequtive ranges, we know that res has at least two elements here + var merged = new List<(uint, uint)>(); + uint from = result[0].Item1; + uint to = result[0].Item2; + for (int i = 1; i < result.Length; i++) + { + if (to == result[i].Item1 - 1) + { + // merge into previous instead of adding a new range + to = result[i].Item2; + } + else + { + merged.Add((from, to)); + from = result[i].Item1; + to = result[i].Item2; + } + } + merged.Add((from, to)); + result = merged.ToArray(); + } + + return result; + } + + private (uint, uint)[] ToRangesFromOrdinal(BDD set) + { + if (!_rangeCache.TryGetValue(set, out (uint, uint)[]? ranges)) + { + Debug.Assert(!set.IsLeaf); + + int b = set.Ordinal; + uint mask = (uint)1 << b; + if (set.Zero.IsEmpty) + { + #region 0-case is empty + if (set.One.IsFull) + { + ranges = new[] { (mask, (mask << 1) - 1) }; + } + else //1-case is neither full nor empty + { + (uint, uint)[] ranges1 = LiftRanges(b, b - set.One.Ordinal - 1, ToRangesFromOrdinal(set.One)); + ranges = new (uint, uint)[ranges1.Length]; + for (int i = 0; i < ranges1.Length; i++) + { + ranges[i] = (ranges1[i].Item1 | mask, ranges1[i].Item2 | mask); + } + } + #endregion + } + else if (set.Zero.IsFull) + { + #region 0-case is full + if (set.One.IsEmpty) + { + ranges = new[] { (0u, mask - 1) }; + } + else + { + (uint, uint)[] rangesR = LiftRanges(b, b - set.One.Ordinal - 1, ToRangesFromOrdinal(set.One)); + (uint, uint) range = rangesR[0]; + if (range.Item1 == 0) + { + ranges = new (uint, uint)[rangesR.Length]; + ranges[0] = (0, range.Item2 | mask); + for (int i = 1; i < rangesR.Length; i++) + { + ranges[i] = (rangesR[i].Item1 | mask, rangesR[i].Item2 | mask); + } + } + else + { + ranges = new (uint, uint)[rangesR.Length + 1]; + ranges[0] = (0, mask - 1); + for (int i = 0; i < rangesR.Length; i++) + { + ranges[i + 1] = (rangesR[i].Item1 | mask, rangesR[i].Item2 | mask); + } + } + } + #endregion + } + else + { + #region 0-case is neither full nor empty + (uint, uint)[] rangesL = LiftRanges(b, b - set.Zero.Ordinal - 1, ToRangesFromOrdinal(set.Zero)); + (uint, uint) last = rangesL[rangesL.Length - 1]; + + if (set.One.IsEmpty) + { + ranges = rangesL; + } + + else if (set.One.IsFull) + { + var ranges1 = new List<(uint, uint)>(); + for (int i = 0; i < rangesL.Length - 1; i++) + { + ranges1.Add(rangesL[i]); + } + + if (last.Item2 == (mask - 1)) + { + ranges1.Add((last.Item1, (mask << 1) - 1)); + } + else + { + ranges1.Add(last); + ranges1.Add((mask, (mask << 1) - 1)); + } + ranges = ranges1.ToArray(); + } + else //general case: neither 0-case, not 1-case is full or empty + { + (uint, uint)[] rangesR0 = ToRangesFromOrdinal(set.One); + + (uint, uint)[] rangesR = LiftRanges(b, b - set.One.Ordinal - 1, rangesR0); + + (uint, uint) first = rangesR[0]; + + if (last.Item2 == (mask - 1) && first.Item1 == 0) //merge together the last and first ranges + { + ranges = new (uint, uint)[rangesL.Length + rangesR.Length - 1]; + for (int i = 0; i < rangesL.Length - 1; i++) + { + ranges[i] = rangesL[i]; + } + + ranges[rangesL.Length - 1] = (last.Item1, first.Item2 | mask); + for (int i = 1; i < rangesR.Length; i++) + { + ranges[rangesL.Length - 1 + i] = (rangesR[i].Item1 | mask, rangesR[i].Item2 | mask); + } + } + else + { + ranges = new (uint, uint)[rangesL.Length + rangesR.Length]; + for (int i = 0; i < rangesL.Length; i++) + { + ranges[i] = rangesL[i]; + } + + for (int i = 0; i < rangesR.Length; i++) + { + ranges[rangesL.Length + i] = (rangesR[i].Item1 | mask, rangesR[i].Item2 | mask); + } + } + + } + #endregion + } + _rangeCache[set] = ranges; + } + + return ranges; + } + } +} diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Algebras/BV.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Algebras/BV.cs new file mode 100644 index 00000000000000..586bf1a9981473 --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Algebras/BV.cs @@ -0,0 +1,206 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; + +namespace System.Text.RegularExpressions.Symbolic +{ + /// + /// Represents a bitvector of arbitrary length (i.e. number of bits). + /// + internal sealed class BV : IComparable + { + /// + /// Stores the bits in an array of 64-bit integers. If Length is not evenly divisible by 64 then the remaining + /// bits are in the least significant bits of the last element. + /// + private readonly ulong[] _blocks; + + /// + /// Number of bits. + /// + internal readonly int Length; + + /// + /// Cache for the lazily computed hash code. + /// + private int? _hashcode; + + /// + /// Returns true iff the i'th bit is 1 + /// + internal bool this[int i] + { + get + { + Debug.Assert(i >= 0 && i < Length); + int k = i / 64; + int j = i % 64; + return (_blocks[k] & (1ul << j)) != 0; + } + private set + { + Debug.Assert(i >= 0 && i < Length); + int k = i / 64; + int j = i % 64; + if (value) + { + //set the j'th bit of the k'th block to 1 + _blocks[k] |= 1ul << j; + } + else + { + //set the j'th bit of the k'th block to 0 + _blocks[k] &= ~(1ul << j); + } + } + } + + private BV(int K) + { + Length = K; + _blocks = new ulong[((K - 1) / 64) + 1]; + } + + private BV(int K, ulong[] blocks) + { + Length = K; + _blocks = blocks; + } + + /// + /// Constructs a bitvector of K bits initially all 0. + /// + public static BV CreateFalse(int K) => new(K); + + /// + /// Constructs a bitvector of K bits initially all 1. + /// + public static BV CreateTrue(int K) => ~CreateFalse(K); + + /// + /// Returns the bitvector of length K with its i'th bit set to 1 all other bits are 0. + /// + public static BV CreateSingleBit(int K, int i) + { + BV bv = new BV(K); + bv[i] = true; + return bv; + } + + /// + /// Bitwise AND + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static BV operator &(BV x, BV y) + { + Debug.Assert(x.Length == y.Length); + + var blocks = new ulong[x._blocks.Length]; + // produce new blocks as the bitwise AND of the arguments' blocks + for (int i = 0; i < blocks.Length; i++) + { + blocks[i] = x._blocks[i] & y._blocks[i]; + } + return new BV(x.Length, blocks); + } + + /// + /// Bitwise OR + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static BV operator |(BV x, BV y) + { + Debug.Assert(x.Length == y.Length); + + var blocks = new ulong[x._blocks.Length]; + // produce new blocks as the bitwise OR of the arguments' blocks + for (int i = 0; i < blocks.Length; i++) + { + blocks[i] = x._blocks[i] | y._blocks[i]; + } + return new BV(x.Length, blocks); + } + + /// + /// Bitwise XOR + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static BV operator ^(BV x, BV y) + { + Debug.Assert(x.Length == y.Length); + + var blocks = new ulong[x._blocks.Length]; + // produce new blocks as the bitwise XOR of the arguments' blocks + for (int i = 0; i < blocks.Length; i++) + { + blocks[i] = x._blocks[i] ^ y._blocks[i]; + } + return new BV(x.Length, blocks); + } + + /// + /// Bitwise NOT + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static BV operator ~(BV x) + { + var blocks = new ulong[x._blocks.Length]; + for (int i = 0; i < blocks.Length; i++) + { + blocks[i] = ~x._blocks[i]; + } + + int j = x.Length % 64; + if (j > 0) + { + // the number of bits is not a precise multiple of 64 + // reset the extra bits in the last block to 0 + int last = (x.Length - 1) / 64; + blocks[last] &= (1ul << j) - 1; + } + + return new BV(x.Length, blocks); + } + + public override bool Equals([NotNullWhen(true)] object? obj) => CompareTo(obj) == 0; + + public override int GetHashCode() + { + // if the hash code hasn't been calculated yet, do so before returning it + if (_hashcode == null) + { + _hashcode = Length.GetHashCode(); + for (int i = 0; i < _blocks.Length; i++) + { + _hashcode = HashCode.Combine(_hashcode, _blocks[i].GetHashCode()); + } + } + return (int)_hashcode; + } + + public int CompareTo(object? obj) + { + if (obj is not BV that) + return 1; + + if (Length != that.Length) + return Length.CompareTo(that.Length); + + // compare all blocks starting from the last one (i.e. most significant) + for (int i = _blocks.Length - 1; i >= 0; i--) + { + if (_blocks[i] < that._blocks[i]) + return -1; + + if (_blocks[i] > that._blocks[i]) + return 1; + } + + //all blocks were equal + return 0; + } + } +} diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Algebras/BV64Algebra.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Algebras/BV64Algebra.cs new file mode 100644 index 00000000000000..269d5f765977f1 --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Algebras/BV64Algebra.cs @@ -0,0 +1,171 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics; +using System.Runtime.CompilerServices; + +namespace System.Text.RegularExpressions.Symbolic +{ + /// + /// Bit vector algebra for up to 64 bits that uses an ulong directly as the term representation, unlike the more + /// general BVAlgebra that uses an array of them. This simplifies the operations making the algebra more efficient. + /// + internal sealed class BV64Algebra : BVAlgebraBase, ICharAlgebra + { + private readonly MintermGenerator _mintermGenerator; + private readonly ulong _false; + private readonly ulong _true; + + /// + /// Return the number of characters belonging to the minterms in the given set. + /// + public ulong ComputeDomainSize(ulong set) + { + ulong size = 0; + for (int i = 0; i < _bits; i++) + { + // if the bit is set then include the corresponding minterm's cardinality + if (IsSatisfiable(set & ((ulong)1 << i))) + { + size += _cardinalities[i]; + } + } + + return size; + } + + public BV64Algebra(CharSetSolver solver, BDD[] minterms) : + base(new MintermClassifier(solver, minterms), solver.ComputeDomainSizes(minterms), minterms) + { + Debug.Assert(minterms.Length <= 64); + _mintermGenerator = new MintermGenerator(this); + _false = 0; + _true = _bits == 64 ? ulong.MaxValue : ulong.MaxValue >> (64 - _bits); + } + + public bool IsExtensional => true; + public bool HashCodesRespectEquivalence => true; + + public CharSetSolver CharSetProvider => throw new NotSupportedException(); + + ulong IBooleanAlgebra.False => _false; + + ulong IBooleanAlgebra.True => _true; + + public bool AreEquivalent(ulong predicate1, ulong predicate2) => predicate1 == predicate2; + + public List GenerateMinterms(params ulong[] constraints) => _mintermGenerator.GenerateMinterms(constraints); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool IsSatisfiable(ulong predicate) => predicate != _false; + + public ulong And(IEnumerable predicates) => throw new NotSupportedException(); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public ulong And(ulong predicate1, ulong predicate2) => predicate1 & predicate2; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public ulong Not(ulong predicate) => _true & ~predicate; //NOTE: must filter off unused bits + + public ulong Or(IEnumerable predicates) + { + ulong res = _false; + foreach (ulong p in predicates) + { + res |= p; + + // short circuit the evaluation on true, since 1|x=1 + if (res == _true) + { + return _true; + } + } + + return res; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public ulong Or(ulong predicate1, ulong predicate2) => predicate1 | predicate2; + + public ulong RangeConstraint(char lower, char upper, bool caseInsensitive = false, string? culture = null) => throw new NotSupportedException(); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public ulong CharConstraint(char c, bool caseInsensitive = false, string? culture = null) + { + Debug.Assert(!caseInsensitive); + return ((ulong)1) << _classifier.GetMintermID(c); + } + + /// + /// Assumes that set is a union of some minterms (or empty). + /// If null then 0 is returned. + /// + public ulong ConvertFromCharSet(BDDAlgebra alg, BDD? set) + { + ulong res = _false; + + if (set is not null) + { + for (int i = 0; i < _bits; i++) + { + Debug.Assert(_partition is not null); + + // set the i'th bit if the i'th minterm is in the set + if (alg.IsSatisfiable(alg.And(_partition[i], set))) + { + res |= (ulong)1 << i; + } + } + } + + return res; + } + + public BDD ConvertToCharSet(ICharAlgebra solver, ulong pred) + { + Debug.Assert(_partition is not null); + + // the result will be the union of all minterms in the set + BDD res = BDD.False; + if (pred != _false) + { + for (int i = 0; i < _bits; i++) + { + // include the i'th minterm in the union if the i'th bit is set + if ((pred & ((ulong)1 << i)) != _false) + { + res = solver.Or(res, _partition[i]); + } + } + } + + return res; + } + + /// + /// Return an array of bitvectors representing each of the minterms. + /// + public ulong[] GetMinterms() + { + ulong[] minterms = new ulong[_bits]; + for (int i = 0; i < _bits; i++) + { + minterms[i] = (ulong)1 << i; + } + + return minterms; + } + + public IEnumerable GenerateAllCharacters(ulong set) => throw new NotSupportedException(); + + /// Pretty print the bitvector bv as the character set it represents. + public string PrettyPrint(ulong bv) + { + ICharAlgebra bddalgebra = SymbolicRegexRunner.s_unicode._solver; + Debug.Assert(_partition is not null && bddalgebra is not null); + + return bddalgebra.PrettyPrint(ConvertToCharSet(bddalgebra, bv)); + } + } +} diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Algebras/BVAlgebra.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Algebras/BVAlgebra.cs new file mode 100644 index 00000000000000..af33f6a8a6b942 --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Algebras/BVAlgebra.cs @@ -0,0 +1,177 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; + +namespace System.Text.RegularExpressions.Symbolic +{ + /// + /// Base class for bitvector algebras, which represent sets as bitvectors indexed by the elements. An element is in + /// the set if the corresponding bit is set. + /// + /// These bitvector algebras are used to represent sets of minterms, and thus represent sets of characters + /// indirectly. However, the bitvector algebras are aware of this indirection in that the cardinalities of sets + /// count the characters rather than the minterms. For example, the cardinality of a bitvector "110" where the bits + /// correspond to minterms [a-c], [0-9] and [^a-c0-9] is 13 rather than 2. + /// + internal abstract class BVAlgebraBase + { + internal readonly MintermClassifier _classifier; + protected readonly ulong[] _cardinalities; + protected readonly int _bits; + protected readonly BDD[]? _partition; + + internal BVAlgebraBase(MintermClassifier classifier, ulong[] cardinalities, BDD[]? partition) + { + _classifier = classifier; + _cardinalities = cardinalities; + _bits = cardinalities.Length; + _partition = partition; + } + } + + /// + /// Bit vector algebra + /// + internal sealed class BVAlgebra : BVAlgebraBase, ICharAlgebra + { + private readonly MintermGenerator _mintermGenerator; + internal BV[] _minterms; + + public ulong ComputeDomainSize(BV set) + { + ulong size = 0; + for (int i = 0; i < _bits; i++) + { + // if the bit is set then add the minterm's size + if (set[i]) + { + size += _cardinalities[i]; + } + } + + return size; + } + + public BVAlgebra(CharSetSolver solver, BDD[] minterms) : + base(new MintermClassifier(solver, minterms), solver.ComputeDomainSizes(minterms), minterms) + { + _mintermGenerator = new MintermGenerator(this); + False = BV.CreateFalse(_bits); + True = BV.CreateTrue(_bits); + + var singleBitVectors = new BV[_bits]; + for (int i = 0; i < singleBitVectors.Length; i++) + { + singleBitVectors[i] = BV.CreateSingleBit(_bits, i); + } + _minterms = singleBitVectors; + } + + public BV False { get; } + public BV True { get; } + + public bool IsExtensional => true; + public bool HashCodesRespectEquivalence => true; + public CharSetSolver CharSetProvider => throw new NotSupportedException(); + public bool AreEquivalent(BV predicate1, BV predicate2) => predicate1.Equals(predicate2); + public List GenerateMinterms(params BV[] constraints) => _mintermGenerator.GenerateMinterms(constraints); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool IsSatisfiable(BV predicate) => !predicate.Equals(False); + + public BV And(IEnumerable predicates) => throw new NotSupportedException(); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public BV And(BV predicate1, BV predicate2) => predicate1 & predicate2; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public BV Not(BV predicate) => ~predicate; + + public BV Or(IEnumerable predicates) + { + BV res = False; + foreach (BV p in predicates) + { + res |= p; + } + + return res; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public BV Or(BV predicate1, BV predicate2) => predicate1 | predicate2; + + public BV RangeConstraint(char lower, char upper, bool caseInsensitive = false, string? culture = null) => throw new NotSupportedException(); + + public BV CharConstraint(char c, bool caseInsensitive = false, string? culture = null) + { + Debug.Assert(!caseInsensitive); + int i = _classifier.GetMintermID(c); + return _minterms[i]; + } + + /// + /// Assumes that set is a union of some minterms (or empty). + /// If null then null is returned. + /// + [return: NotNullIfNotNull("set")] + public BV? ConvertFromCharSet(BDDAlgebra alg, BDD set) + { + if (set == null) + return null; + + Debug.Assert(_partition is not null); + + BV res = False; + for (int i = 0; i < _bits; i++) + { + BDD bdd_i = _partition[i]; + BDD conj = alg.And(bdd_i, set); + if (alg.IsSatisfiable(conj)) + { + res |= _minterms[i]; + } + } + + return res; + } + + public BDD ConvertToCharSet(ICharAlgebra solver, BV pred) + { + Debug.Assert(_partition is not null); + + // the result will be the union of all minterms in the set + BDD res = solver.False; + if (!pred.Equals(False)) + { + for (int i = 0; i < _bits; i++) + { + // include the i'th minterm in the union if the i'th bit is set + if (pred[i]) + { + res = solver.Or(res, _partition[i]); + } + } + } + + return res; + } + + public BV[] GetMinterms() => _minterms; + public IEnumerable GenerateAllCharacters(BV set) => throw new NotSupportedException(); + + /// Pretty print the bitvector bv as the character set it represents. + public string PrettyPrint(BV bv) + { + //accesses the shared BDD solver + ICharAlgebra bddalgebra = SymbolicRegexRunner.s_unicode._solver; + Debug.Assert(_partition is not null && bddalgebra is not null); + + return bddalgebra.PrettyPrint(ConvertToCharSet(bddalgebra, bv)); + } + } +} diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Algebras/CharSetSolver.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Algebras/CharSetSolver.cs new file mode 100644 index 00000000000000..344ba84680d9d6 --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Algebras/CharSetSolver.cs @@ -0,0 +1,414 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; + +namespace System.Text.RegularExpressions.Symbolic +{ + /// + /// Provides functionality to build character sets, to perform boolean operations over character sets, + /// and to construct an SFA over character sets from a regex. + /// Character sets are represented by bitvector sets. + /// + internal sealed class CharSetSolver : BDDAlgebra, ICharAlgebra + { + /// BDDs for all characters for fast lookup. + private readonly BDD[] _charPredTable = new BDD[char.MaxValue + 1]; + private readonly Unicode.IgnoreCaseTransformer _ignoreCase; + internal readonly BDD _nonAscii; + + /// + /// Construct the solver. + /// + public CharSetSolver() + { + _nonAscii = CreateCharSetFromRange('\x80', '\uFFFF'); + _ignoreCase = new Unicode.IgnoreCaseTransformer(this); + } + + public BDD ApplyIgnoreCase(BDD set, string? culture = null) => _ignoreCase.Apply(set, culture); + + /// + /// Make a character predicate for the given character c. + /// + public BDD CharConstraint(char c, bool ignoreCase = false, string? culture = null) + { + if (ignoreCase) + { + return _ignoreCase.Apply(c, culture); + } + else + { + //individual character BDDs are always fixed + return _charPredTable[c] ??= CreateSetFrom(c, 15); + } + } + + /// + /// Make a CharSet from all the characters in the range from m to n. + /// Returns the empty set if n is less than m + /// + public BDD CreateCharSetFromRange(char m, char n) => + m == n ? CharConstraint(m) : + CreateSetFromRange(m, n, 15); + + /// + /// Make a character set that is the union of the character sets of the given ranges. + /// + public BDD CreateCharSetFromRanges(IEnumerable<(uint, uint)> ranges) + { + BDD res = False; + foreach ((uint, uint) range in ranges) + { + res = Or(res, CreateSetFromRange(range.Item1, range.Item2, 15)); + } + + return res; + } + + /// + /// Make a character set of all the characters in the interval from c to d. + /// If ignoreCase is true ignore cases for upper and lower case characters by including both versions. + /// + public BDD RangeConstraint(char c, char d, bool ignoreCase = false, string? culture = null) + { + if (c == d) + { + return CharConstraint(c, ignoreCase, culture); + } + + BDD res = CreateSetFromRange(c, d, 15); + if (ignoreCase) + { + res = _ignoreCase.Apply(res, culture); + } + + return res; + } + + /// + /// Make a BDD encoding of k least significant bits of all the integers in the ranges + /// + internal BDD CreateBddForIntRanges(IEnumerable ranges) + { + BDD bdd = False; + foreach (int[] range in ranges) + { + bdd = Or(bdd, CreateSetFromRange((uint)range[0], (uint)range[1], 15)); + } + + return bdd; + } + + /// + /// Identity function, returns s. + /// + public BDD ConvertFromCharSet(BDDAlgebra _, BDD s) => s; + + /// + /// Returns this character set solver. + /// + public CharSetSolver CharSetProvider => this; + + public IEnumerable GenerateAllCharacters(BDD bvSet, bool inReverseOrder = false) + { + foreach (uint c in GenerateAllElements(bvSet, inReverseOrder)) + { + yield return (char)c; + } + } + + public IEnumerable GenerateAllCharacters(BDD set) => GenerateAllCharacters(set, false); + + /// Calculate the number of elements in the set. + /// the given set + /// the cardinality of the set + public ulong ComputeDomainSize(BDD set) => ComputeDomainSize(set, 15); + + /// Calculate the number of elements in multiple sets. + /// The sets + /// An array of the cardinality of the sets. + public ulong[] ComputeDomainSizes(BDD[] sets) + { + var results = new ulong[sets.Length]; + for (int i = 0; i < sets.Length; i++) + { + results[i] = ComputeDomainSize(sets[i]); + } + return results; + } + + /// + /// Returns true iff the set contains exactly one element. + /// + /// the given set + /// true iff the set is a singleton + public bool IsSingleton(BDD set) => ComputeDomainSize(set, 15) == 1; + + /// + /// Convert the set into an equivalent array of ranges. The ranges are nonoverlapping and ordered. + /// + public (uint, uint)[] ToRanges(BDD set) => ToRanges(set, 15); + + private IEnumerable GenerateAllCharactersInOrder(BDD set) + { + foreach ((uint, uint) range in ToRanges(set)) + { + for (uint i = range.Item1; i <= range.Item2; i++) + { + yield return i; + } + } + } + + private IEnumerable GenerateAllCharactersInReverseOrder(BDD set) + { + (uint, uint)[] ranges = ToRanges(set); + for (int j = ranges.Length - 1; j >= 0; j--) + { + for (uint i = ranges[j].Item2; i >= ranges[j].Item1; i--) + { + yield return (char)i; + } + } + } + + /// + /// Generate all characters that are members of the set in alphabetical order, smallest first, provided that inReverseOrder is false. + /// + /// the given set + /// if true the members are generated in reverse alphabetical order with the largest first, otherwise in alphabetical order + /// enumeration of all characters in the set, the enumeration is empty if the set is empty + private IEnumerable GenerateAllElements(BDD set, bool inReverseOrder) => + set == False ? Array.Empty() : + inReverseOrder ? GenerateAllCharactersInReverseOrder(set) : + GenerateAllCharactersInOrder(set); + + public BDD ConvertToCharSet(ICharAlgebra _, BDD pred) => pred; + + public BDD[]? GetMinterms() => null; + + public string PrettyPrint(BDD pred) + { + if (pred.IsEmpty) + return "[]"; + + //check if pred is full, show this case with a dot + if (pred.IsFull) + return "."; + + // try to optimize representation involving common direct use of \d \w and \s to avoid blowup of ranges + BDD digit = SymbolicRegexRunner.s_unicode.CategoryCondition(8); + if (pred == SymbolicRegexRunner.s_unicode.WordLetterCondition) + return @"\w"; + if (pred == SymbolicRegexRunner.s_unicode.WhiteSpaceCondition) + return @"\s"; + if (pred == digit) + return @"\d"; + if (pred == Not(SymbolicRegexRunner.s_unicode.WordLetterCondition)) + return @"\W"; + if (pred == Not(SymbolicRegexRunner.s_unicode.WhiteSpaceCondition)) + return @"\S"; + if (pred == Not(digit)) + return @"\D"; + + (uint, uint)[] ranges = ToRanges(pred); + + if (IsSingletonRange(ranges)) + return Escape((char)ranges[0].Item1); + + #region if too many ranges try to optimize the representation using \d \w etc. + if (SymbolicRegexRunner.s_unicode != null && ranges.Length > 10) + { + BDD w = SymbolicRegexRunner.s_unicode.WordLetterCondition; + BDD W = Not(w); + BDD d = SymbolicRegexRunner.s_unicode.CategoryCondition(8); + BDD D = Not(d); + BDD asciiDigit = CreateCharSetFromRange('0', '9'); + BDD nonasciiDigit = And(d, Not(asciiDigit)); + BDD s = SymbolicRegexRunner.s_unicode.WhiteSpaceCondition; + BDD S = Not(s); + BDD wD = And(w, D); + BDD SW = And(S, W); + //s, d, wD, SW are the 4 main large minterms + //note: s|SW = W, d|wD = w + // + // Venn Diagram: s and w do not overlap, and d is contained in w + // ------------------------------------------------ + // | | + // | SW ------------(w)-------- | + // | -------- | | | + // | | | | ---------- | | + // | | s | | wD | | | | + // | | | | | d | | | + // | -------- | | | | | + // | | ---------- | | + // | ----------------------- | + // ------------------------------------------------ + // + //------------------------------------------------------------------- + // singletons + //--- + if (Or(s, pred) == s) + return $"[^\\S{RepresentSet(And(s, Not(pred)))}]"; + //--- + if (Or(d, pred) == d) + return $"[^\\D{RepresentSet(And(d, Not(pred)))}]"; + //--- + if (Or(wD, pred) == wD) + return $"[\\w-[\\d{RepresentSet(And(wD, Not(pred)))}]]"; + //--- + if (Or(SW, pred) == SW) + return $"[^\\s\\w{RepresentSet(And(SW, Not(pred)))}]"; + //------------------------------------------------------------------- + // unions of two + // s|SW + if (Or(W, pred) == W) + { + string? repr1 = null; + if (And(s, pred) == s) + { + //pred contains all of \s and is contained in \W + repr1 = $"[\\s{RepresentSet(And(S, pred))}]"; + } + + //the more common case is that pred is not \w and not some specific non-word character such as ':' + string repr2 = $"[^\\w{RepresentSet(And(W, Not(pred)))}]"; + return repr1 != null && repr1.Length < repr2.Length ? repr1 : repr2; + } + //--- + // s|d + BDD s_or_d = Or(s, d); + if (pred == s_or_d) + return "[\\s\\d]"; + + if (Or(s_or_d, pred) == s_or_d) + { + //check first if this is purely ascii range for digits + return And(pred, s).Equals(s) && And(pred, nonasciiDigit).IsEmpty ? + $"[\\s{RepresentRanges(ToRanges(And(pred, asciiDigit)), checkSingletonComlement: false)}]" : + $"[\\s\\d-[{RepresentSet(And(s_or_d, Not(pred)))}]]"; + } + //--- + // s|wD + BDD s_or_wD = Or(s, wD); + if (Or(s_or_wD, pred) == s_or_wD) + return $"[\\s\\w-[\\d{RepresentSet(And(s_or_wD, Not(pred)))}]]"; + //--- + // d|wD + if (Or(w, pred) == w) + return $"[\\w-[{RepresentSet(And(w, Not(pred)))}]]"; + //--- + // d|SW + BDD d_or_SW = Or(d, SW); + if (pred == d_or_SW) + return "\\d|[^\\s\\w]"; + if (Or(d_or_SW, pred) == d_or_SW) + return $"[\\d-[{RepresentSet(And(d, Not(pred)))}]]|[^\\s\\w{RepresentSet(And(SW, Not(pred)))}]"; + // wD|SW = S&D + BDD SD = Or(wD, SW); + if (Or(SD, pred) == SD) + return $"[^\\s\\d{RepresentSet(And(SD, Not(pred)))}]"; + //------------------------------------------------------------------- + //unions of three + // s|SW|wD = D + if (Or(D, pred) == D) + return $"[^\\d{RepresentSet(And(D, Not(pred)))}]"; + // SW|wD|d = S + if (Or(S, pred) == S) + return $"[^\\s{RepresentSet(And(S, Not(pred)))}]"; + // s|SW|d = complement of wD = W|d + BDD W_or_d = Not(wD); + if (Or(W_or_d, pred) == W_or_d) + return $"[\\W\\d-[{RepresentSet(And(W_or_d, Not(pred)))}]]"; + // s|wD|d = s|w + BDD s_or_w = Or(s, w); + if (Or(s_or_w, pred) == s_or_w) + return $"[\\s\\w-[{RepresentSet(And(s_or_w, Not(pred)))}]]"; + //------------------------------------------------------------------- + //touches all four minterms, typically happens as the fallback arc in .* extension + } + #endregion + + // Represent either the ranges or its complement, if the complement representation is more compact. + string ranges_repr = $"[{RepresentRanges(ranges, checkSingletonComlement: false)}]"; + string ranges_compl_repr = $"[^{RepresentRanges(ToRanges(Not(pred)), checkSingletonComlement: false)}]"; + return ranges_repr.Length <= ranges_compl_repr.Length ? ranges_repr : ranges_compl_repr; + } + + private string RepresentSet(BDD set) => + set.IsEmpty ? "" : RepresentRanges(ToRanges(set)); + + private static string RepresentRanges((uint, uint)[] ranges, bool checkSingletonComlement = true) + { + //check if ranges represents a complement of a singleton + if (checkSingletonComlement && ranges.Length == 2 && + ranges[0].Item1 == 0 && ranges[1].Item2 == 0xFFFF && + ranges[0].Item2 + 2 == ranges[1].Item1) + { + return "^" + Escape((char)(ranges[0].Item2 + 1)); + } + + StringBuilder sb = new(); + for (int i = 0; i < ranges.Length; i++) + { + if (ranges[i].Item1 == ranges[i].Item2) + { + sb.Append(Escape((char)ranges[i].Item1)); + } + else if (ranges[i].Item2 == ranges[i].Item1 + 1) + { + sb.Append(Escape((char)ranges[i].Item1)); + sb.Append(Escape((char)ranges[i].Item2)); + } + else + { + sb.Append(Escape((char)ranges[i].Item1)); + sb.Append('-'); + sb.Append(Escape((char)ranges[i].Item2)); + } + } + return sb.ToString(); + } + + /// Make an escaped string from a character. + /// The character to escape. + private static string Escape(char c) + { + uint code = c; + return c switch + { + '.' => @"\.", + '[' => @"\[", + ']' => @"\]", + '(' => @"\(", + ')' => @"\)", + '{' => @"\{", + '}' => @"\}", + '?' => @"\?", + '+' => @"\+", + '*' => @"\*", + '|' => @"\|", + '\\' => @"\\", + '^' => @"\^", + '$' => @"\$", + '-' => @"\-", + ':' => @"\:", + '\"' => "\\\"", + '\0' => @"\0", + '\t' => @"\t", + '\r' => @"\r", + '\v' => @"\v", + '\f' => @"\f", + '\n' => @"\n", + _ when code is >= 0x20 and <= 0x7E => c.ToString(), + _ when code <= 0xFF => $"\\x{code:X2}", + _ => $"\\u{code:X4}", + }; + } + + private static bool IsSingletonRange((uint, uint)[] ranges) => ranges.Length == 1 && ranges[0].Item1 == ranges[0].Item2; + + public override int CombineTerminals(BoolOp op, int terminal1, int terminal2) => throw new NotSupportedException(); + } +} diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Algebras/IBooleanAlgebra.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Algebras/IBooleanAlgebra.cs new file mode 100644 index 00000000000000..336871a5aa2ec3 --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Algebras/IBooleanAlgebra.cs @@ -0,0 +1,85 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; + +namespace System.Text.RegularExpressions.Symbolic +{ + /// + /// Generic Boolean Algebra solver. + /// Provides operations for conjunction, disjunction, and negation. + /// Allows to decide if a predicate is satisfiable and if two predicates are equivalent. + /// + /// predicates + internal interface IBooleanAlgebra + { + /// + /// Top element of the Boolean algebra, corresponds to the value true. + /// + T True { get; } + + /// + /// Bottom element of the Boolean algebra, corresponds to the value false. + /// + T False { get; } + + /// + /// Make a conjunction of predicate1 and predicate2. + /// + T And(T predicate1, T predicate2); + + /// + /// Make a conjunction of all the predicates in the enumeration. + /// Returns True if the enumeration is empty. + /// + T And(IEnumerable predicates); + + /// + /// Make a disjunction of predicate1 and predicate2. + /// + T Or(T predicate1, T predicate2); + + /// + /// Make a disjunction of all the predicates in the enumeration. + /// Must return False if the enumeration is empty. + /// + T Or(IEnumerable predicates); + + /// + /// Negate the predicate. + /// + T Not(T predicate); + + /// + /// Returns true iff the predicate is satisfiable. + /// + bool IsSatisfiable(T predicate); + + /// + /// Returns true iff predicate1 is equivalent to predicate2. + /// + bool AreEquivalent(T predicate1, T predicate2); + + /// + /// True means then if two predicates are equivalent then their hashcodes are equal. + /// This is a weak form of extensionality. + /// + bool HashCodesRespectEquivalence { get; } + + /// + /// True means that if two predicates are equivalent then they are identical. + /// + bool IsExtensional { get; } + + /// + /// Given an array of constraints {c_1, c_2, ..., c_n} where n>=0. + /// Enumerate all satisfiable Boolean combinations Tuple({b_1, b_2, ..., b_n}, c) + /// where c is satisfisable and equivalent to c'_1 & c'_2 & ... & c'_n, + /// where c'_i = c_i if b_i = true and c'_i is Not(c_i) otherwise. + /// If n=0 return Tuple({},True) + /// + /// array of constraints + /// constraints that are satisfiable + List GenerateMinterms(params T[] constraints); + } +} diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Algebras/ICharAlgebra.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Algebras/ICharAlgebra.cs new file mode 100644 index 00000000000000..a0f6a87e35d75a --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Algebras/ICharAlgebra.cs @@ -0,0 +1,66 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; + +namespace System.Text.RegularExpressions.Symbolic +{ + /// + /// Extends ICharAlgebra with character predicate solving and predicate pretty printing. + /// + /// predicates + internal interface ICharAlgebra : IBooleanAlgebra + { + /// + /// Make a constraint describing the set of all characters between a (inclusive) and b (inclusive). + /// Add both uppercase and lowercase elelements if caseInsensitive is true using the given culture + /// or the current culture when the given culture is null. + /// + T RangeConstraint(char lower, char upper, bool caseInsensitive = false, string? culture = null); + + /// + /// Make a constraint describing a singleton set containing the character c, or + /// a set containing also the upper and lowercase versions of c if caseInsensitive is true. + /// + /// if true include both the uppercase and the lowercase versions of the given character + /// the given character + /// given culture, if null then the current culture is assumed + T CharConstraint(char c, bool caseInsensitive = false, string? culture = null); + + /// + /// Make a term that encodes the given character set. + /// + T ConvertFromCharSet(BDDAlgebra bddAlg, BDD set); + + /// + /// Compute the number of elements in the set + /// + ulong ComputeDomainSize(T set); + + /// + /// Enumerate all characters in the set + /// + /// given set + IEnumerable GenerateAllCharacters(T set); + + /// + /// Convert a predicate into a set of characters. + /// + BDD ConvertToCharSet(ICharAlgebra bddalg, T pred); + + /// + /// Gets the underlying character set solver. + /// + CharSetSolver CharSetProvider { get; } + + /// + /// Returns the minterms (a partition of the full domain). + /// + T[]? GetMinterms(); + + /// + /// Pretty print the character predicate + /// + string PrettyPrint(T pred); + } +} diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Algebras/MintermGenerator.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Algebras/MintermGenerator.cs new file mode 100644 index 00000000000000..67fdcf10e864f5 --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Algebras/MintermGenerator.cs @@ -0,0 +1,249 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; + +namespace System.Text.RegularExpressions.Symbolic +{ + /// Provides a generic implementation for minterm generation over a given Boolean Algebra. + /// type of predicates + /// + /// The minterms for a set of predicates are all their non-overlapping, satisfiable Boolean combinations. For example, + /// if the predicates are [0-9] and [0-4], then there are three minterms: [0-4], [5-9] and [^0-9]. Notably, there is no + /// minterm corresponding to "[0-9] and not [0-4]", since that is unsatisfiable. + /// + internal sealed class MintermGenerator where TPredicate : notnull + { + private readonly IBooleanAlgebra _algebra; + + /// Constructs a minterm generator for a given Boolean Algebra. + /// given Boolean Algebra + public MintermGenerator(IBooleanAlgebra algebra) + { + // check that we can rely on equivalent predicates having the same hashcode, which EquivClass assumes + Debug.Assert(algebra.HashCodesRespectEquivalence); + + _algebra = algebra; + } + + /// + /// Given an array of predidates {p_1, p_2, ..., p_n} where n>=0, + /// enumerate all satisfiable Boolean combinations Tuple({b_1, b_2, ..., b_n}, p) + /// where p is satisfiable and equivalent to p'_1 & p'_2 & ... & p'_n, + /// where p'_i = p_i if b_i = true and p'_i is Not(p_i). Otherwise, if n=0 + /// return Tuple({},True). + /// + /// array of predicates + /// all minterms of the given predicate sequence + public List GenerateMinterms(params TPredicate[] preds) + { + if (preds.Length == 0) + { + return new List { _algebra.True }; + } + + // The minterms will be solved using non-equivalent predicates, i.e., the equivalence classes of preds. The + // following code maps each predicate to an equivalence class and also stores for each equivalence class the + // predicates belonging to it, so that a valuation for the original predicates may be reconstructed. + + var tree = new PartitionTree(_algebra); + + var seen = new HashSet(); + for (int i = 0; i < preds.Length; i++) + { + // Use a wrapper that overloads Equals to be logical equivalence as the key + if (seen.Add(new EquivalenceClass(_algebra, preds[i]))) + { + // Push each equivalence class into the partition tree + tree.Refine(preds[i]); + } + } + + // Return all minterms as the leaves of the partition tree + return tree.GetLeafPredicates(); + } + + /// Wraps a predicate as an equivalence class object whose Equals method is equivalence checking. + private readonly struct EquivalenceClass + { + private readonly TPredicate _set; + private readonly IBooleanAlgebra _algebra; + + internal EquivalenceClass(IBooleanAlgebra algebra, TPredicate set) + { + _set = set; + _algebra = algebra; + } + + public override int GetHashCode() => _set.GetHashCode(); + + public override bool Equals([NotNullWhen(true)] object? obj) => obj is EquivalenceClass ec && _algebra.AreEquivalent(_set, ec._set); + } + + /// A partition tree for efficiently solving minterms. + /// + /// Predicates are pushed into the tree with Refine(), which creates leaves in the tree for all satisfiable + /// and non-overlapping combinations with any previously pushed predicates. At the end of the process the + /// minterms can be read from the paths to the leaves of the tree. + /// + /// The valuations of the predicates are represented as follows. Given a path a^-1, a^0, a^1, ..., a^n, predicate + /// p^i is true in the corresponding minterm if and only if a^i is the left child of a^i-1. + /// + /// This class assumes that all predicates passed to Refine() are non-equivalent. + /// + private sealed class PartitionTree + { + internal readonly TPredicate _pred; + private readonly IBooleanAlgebra _solver; + private PartitionTree? _left; + private PartitionTree? _right; // complement + + /// Create the root of the partition tree. + /// Nodes below this will be indexed starting from 0. The initial predicate is true. + internal PartitionTree(IBooleanAlgebra solver) : this(solver, solver.True, null, null) { } + + private PartitionTree(IBooleanAlgebra solver, TPredicate pred, PartitionTree? left, PartitionTree? right) + { + _solver = solver; + _pred = pred; + _left = left; + _right = right; + } + + internal void Refine(TPredicate other) + { + if (_left is null && _right is null) + { + // If this is a leaf node create left and/or right children for the new predicate + TPredicate thisAndOther = _solver.And(_pred, other); + if (_solver.IsSatisfiable(thisAndOther)) + { + // The predicates overlap, now check if this is contained in other + TPredicate thisMinusOther = _solver.And(_pred, _solver.Not(other)); + if (_solver.IsSatisfiable(thisMinusOther)) + { + // This is not contained in other, both children are needed + _left = new PartitionTree(_solver, thisAndOther, null, null); + + // The right child corresponds to a conjunction with a negation, which matches thisMinusOther + _right = new PartitionTree(_solver, thisMinusOther, null, null); + } + else // [[this]] subset of [[other]] + { + // Other implies this, so populate the left child with this + _left = new PartitionTree(_solver, _pred, null, null); + } + } + else // [[this]] subset of [[not(other)]] + { + // negation of other implies this, so populate the right child with this + _right = new PartitionTree(_solver, _pred, null, null); //other must be false + } + } + else if (_left is null) + { + // No choice has to be made here, refine the single child that exists + _right!.Refine(other); + } + else if (_right is null) + { + // No choice has to be made here, refine the single child that exists + _left!.Refine(other); + } + else + { + TPredicate thisAndOther = _solver.And(_pred, other); + if (_solver.IsSatisfiable(thisAndOther)) + { + // Other is satisfiable in this subtree + TPredicate thisMinusOther = _solver.And(_pred, _solver.Not(other)); + if (_solver.IsSatisfiable(thisMinusOther)) + { + // But other does not imply this whole subtree, refine both children + _left.Refine(other); + _right.Refine(other); + } + else // [[this]] subset of [[other]] + { + // And other implies the whole subtree, include it in all minterms under here + _left.ExtendLeft(); + _right.ExtendLeft(); + } + } + else // [[this]] subset of [[not(other)]] + { + // Other is not satisfiable in this subtree, include its negation in all minterms under here + _left.ExtendRight(); + _right.ExtendRight(); + } + } + } + + /// + /// Include the next predicate in all minterms under this node. Assumes the next predicate implies the predicate + /// of this node. + /// + private void ExtendLeft() + { + if (_left is null && _right is null) + { + _left = new PartitionTree(_solver, _pred, null, null); + } + else + { + _left?.ExtendLeft(); + _right?.ExtendLeft(); + } + } + + /// + /// Include the negation of next predicate in all minterms under this node. Assumes the negation of the next + /// predicate implies the predicate of this node. + /// + private void ExtendRight() + { + if (_left is null && _right is null) + { + _right = new PartitionTree(_solver, _pred, null, null); + } + else + { + _left?.ExtendRight(); + _right?.ExtendRight(); + } + } + + /// Get the predicates from all of the leaves in the tree. + internal List GetLeafPredicates() + { + var leaves = new List(); + + var stack = new Stack(); + stack.Push(this); + while (stack.TryPop(out PartitionTree? node)) + { + if (node._left is null && node._right is null) + { + leaves.Add(node._pred); + } + else + { + if (node._left is not null) + { + stack.Push(node._left); + } + + if (node._right is not null) + { + stack.Push(node._right); + } + } + } + + return leaves; + } + } + } +} diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/BooleanClassifier.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/BooleanClassifier.cs new file mode 100644 index 00000000000000..b10fdd174efb72 --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/BooleanClassifier.cs @@ -0,0 +1,62 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.CompilerServices; + +namespace System.Text.RegularExpressions.Symbolic +{ + /// Classifies characters as true or false based on a supplied . + /// + /// The classification is determined entirely by the BDD used to construct the classifier, and in fact + /// simply calling Contains on the BDD instead of using the classifier would suffice from a correctness + /// perspective. The classifier as a wrapper for the BDD is valuable in order to optimize for ASCII, as + /// it precomputes the results for ASCII inputs and stores them in a separate table, only falling back + /// to using the BDD for non-ASCII. + /// + internal sealed class BooleanClassifier + { + /// Lookup table used for ASCII characters. + private readonly bool[] _ascii; + /// BDD used for non-ASCII characters. + private readonly BDD _nonAscii; + + /// Create a Boolean classifier. + /// Character algebra (the algebra is not stored in the classifier) + /// Elements that map to true. + public BooleanClassifier(CharSetSolver solver, BDD bdd) + { + // We want to optimize for ASCII, so query the BDD for each ASCII character in + // order to precompute a lookup table we'll use at match time. + var ascii = new bool[128]; + for (int i = 0; i < ascii.Length; i++) + { + ascii[i] = bdd.Contains(i); + } + + // At this point, we'll never consult the BDD for ASCII characters, so as an + // optimization we can remove them from the BDD in hopes of simplifying it and making + // it faster to query for the non-ASCII characters we will use it for. However, while + // this is typically an optimization, it isn't always: the act of removing some + // characters from the BDD can actually make the branching more complicated. The + // extreme case of this is when the BDD is True, meaning everything maps to True, which + // is as simple a BDD as you can get. In such a case, even though it's rare, this would + // definitively be a deoptimization, so we avoid doing so. Other trivial cases are handled + // by And itself, e.g. if the BDD == False, then And will just return False. + if (!bdd.IsFull) + { + bdd = solver.And(solver._nonAscii, bdd); + } + + _ascii = ascii; + _nonAscii = bdd; + } + + /// Gets whether the specified character is classified as true. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool IsTrue(char c) + { + bool[] ascii = _ascii; + return c < ascii.Length ? ascii[c] : _nonAscii.Contains(c); + } + } +} diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/CharKind.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/CharKind.cs new file mode 100644 index 00000000000000..39f95920da31b3 --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/CharKind.cs @@ -0,0 +1,44 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace System.Text.RegularExpressions.Symbolic +{ + internal static class CharKind + { + /// Number of kinds of chars. + internal const int CharKindCount = 5; + + /// All characters other than those in the four other kinds. + internal const uint General = 0; + + /// Start or Stop of input (bit 0 is 1) + internal const uint StartStop = 1; + + /// New line character (\n) (bit 1 is 1) + internal const uint Newline = 2; + + /// Last \n or first \n in reverse mode (both Newline and StartStop bits are 1) + internal const uint NewLineS = 3; + + /// Word letter (bit 2 is 1) + internal const uint WordLetter = 4; + + /// Gets the previous character kind from a context + internal static uint Prev(uint context) => context & 0x7; + + /// Gets the next character kind from a context + internal static uint Next(uint context) => context >> 3; + + /// Creates the context of the previous and the next character kinds. + internal static uint Context(uint prevKind, uint nextKind) => (nextKind << 3) | prevKind; + + internal static string DescribePrev(uint i) => i switch + { + StartStop => @"\A", + Newline => @"\n", + NewLineS => @"\A\n", + WordLetter => @"\w", + _ => string.Empty, + }; + } +} diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/DfaMatchingState.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/DfaMatchingState.cs new file mode 100644 index 00000000000000..254a7fd94407d2 --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/DfaMatchingState.cs @@ -0,0 +1,138 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Net; + +namespace System.Text.RegularExpressions.Symbolic +{ + /// Captures a state of a DFA explored during matching. + internal sealed class DfaMatchingState where T : notnull + { + internal DfaMatchingState(SymbolicRegexNode node, uint prevCharKind) + { + Node = node; + PrevCharKind = prevCharKind; + } + + internal SymbolicRegexNode Node { get; } + internal uint PrevCharKind { get; } + + internal int Id { get; set; } + internal bool IsInitialState { get; set; } + + /// State is lazy + internal bool IsLazy => Node._info.IsLazy; + + /// This is a deadend state + internal bool IsDeadend => Node.IsNothing; + + /// The node must be nullable here + internal int WatchDog + { + get + { + if (Node._kind == SymbolicRegexKind.WatchDog) + { + return Node._lower; + } + + if (Node._kind == SymbolicRegexKind.Or) + { + Debug.Assert(Node._alts is not null); + return Node._alts._watchdog; + } + + return -1; + } + } + + /// If true then the state is a dead-end, rejects all inputs. + internal bool IsNothing => Node.IsNothing; + + /// If true then state starts with a ^ or $ or \A or \z or \Z + internal bool StartsWithLineAnchor => Node._info.StartsWithLineAnchor; + + /// + /// Compute the target state for the given input minterm. + /// If is False this means that this is \n and it is the last character of the input. + /// + /// minterm corresponding to some input character or False corresponding to last \n + internal DfaMatchingState Next(T minterm) + { + ICharAlgebra alg = Node._builder._solver; + T wordLetterPredicate = Node._builder._wordLetterPredicateForAnchors; + T newLinePredicate = Node._builder._newLinePredicate; + + // minterm == solver.False is used to represent the very last \n + uint nextCharKind = 0; + if (alg.False.Equals(minterm)) + { + nextCharKind = CharKind.NewLineS; + minterm = newLinePredicate; + } + else if (newLinePredicate.Equals(minterm)) + { + // If the previous state was the start state, mark this as the very FIRST \n. + // Essentially, this looks the same as the very last \n and is used to nullify + // rev(\Z) in the conext of a reversed automaton. + nextCharKind = PrevCharKind == CharKind.StartStop ? + CharKind.NewLineS : + CharKind.Newline; + } + else if (alg.IsSatisfiable(alg.And(wordLetterPredicate, minterm))) + { + nextCharKind = CharKind.WordLetter; + } + + // Combined character context + uint context = CharKind.Context(PrevCharKind, nextCharKind); + + // Compute the derivative of the node for the given context + SymbolicRegexNode derivative = Node.MkDerivative(minterm, context); + + // nextCharKind will be the PrevCharKind of the target state + // use an existing state instead if one exists already + // otherwise create a new new id for it + return Node._builder.MkState(derivative, nextCharKind); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal bool IsNullable(uint nextCharKind) + { + Debug.Assert(nextCharKind is 0 or CharKind.StartStop or CharKind.Newline or CharKind.WordLetter or CharKind.NewLineS); + uint context = CharKind.Context(PrevCharKind, nextCharKind); + return Node.IsNullableFor(context); + } + + public override bool Equals(object? obj) => + obj is DfaMatchingState s && PrevCharKind == s.PrevCharKind && Node.Equals(s.Node); + + public override int GetHashCode() => (PrevCharKind, Node).GetHashCode(); + + public override string ToString() => + PrevCharKind == 0 ? Node.ToString() : + $"({CharKind.DescribePrev(PrevCharKind)},{Node})"; + + internal string DgmlView + { + get + { + string info = CharKind.DescribePrev(PrevCharKind); + if (info != string.Empty) + { + info = $"Previous: {info} "; + } + + string deriv = WebUtility.HtmlEncode(Node.ToString()); + if (deriv == string.Empty) + { + deriv = "()"; + } + + return $"{info}{deriv}"; + } + } + } +} diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Dgml/DgmlWriter.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Dgml/DgmlWriter.cs new file mode 100644 index 00000000000000..cbd134be453635 --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Dgml/DgmlWriter.cs @@ -0,0 +1,241 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#if DEBUG +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; + +namespace System.Text.RegularExpressions.Symbolic.DGML +{ + internal sealed class DgmlWriter + { + private readonly int _maxDgmlTransitionLabelLength; + private readonly TextWriter _tw; + private readonly bool _hideStateInfo; + private readonly bool _onlyDFAinfo; + + internal DgmlWriter(TextWriter tw, bool hideStateInfo, int maxDgmlTransitionLabelLength = -1, bool onlyDFAinfo = false) + { + _maxDgmlTransitionLabelLength = maxDgmlTransitionLabelLength; + _tw = tw; + _hideStateInfo = hideStateInfo; + _onlyDFAinfo = onlyDFAinfo; + } + + /// + /// Write the automaton in dgml format into the textwriter. + /// + public void Write(IAutomaton fa) + { + var nonEpsilonMoves = new Dictionary<(int, int), List>(); + var epsilonmoves = new List>(); + + var nonEpsilonStates = new HashSet(); + + foreach (Move move in fa.GetMoves()) + { + if (move.IsEpsilon) + { + epsilonmoves.Add(move); + } + else + { + nonEpsilonStates.Add(move.SourceState); + var p = (move.SourceState, move.TargetState); + if (!nonEpsilonMoves.TryGetValue(p, out List? rules)) + { + rules = new List(); + nonEpsilonMoves[p] = rules; + } + + Debug.Assert(move.Label is not null); + rules.Add(move.Label); + } + } + + _tw.WriteLine(""); + _tw.WriteLine(""); + _tw.WriteLine(""); + _tw.WriteLine("", GetDFAInfo(fa)); + _tw.WriteLine("", GetDFAInfo(fa)); + if (_onlyDFAinfo) + { + _tw.WriteLine(""); + } + else + { + foreach (int state in fa.GetStates()) + { + _tw.WriteLine("", state, _hideStateInfo ? "Collapsed" : "Expanded", GetStateInfo(fa, state)); + if (state == fa.InitialState) + { + _tw.WriteLine(""); + } + if (fa.IsFinalState(state)) + { + _tw.WriteLine(""); + } + _tw.WriteLine(""); + _tw.WriteLine("", state, GetStateInfo(fa, state)); + } + _tw.WriteLine(""); + _tw.WriteLine(""); + _tw.WriteLine("", fa.InitialState, fa.DescribeStartLabel()); + _tw.WriteLine(""); + + foreach (Move move in epsilonmoves) + { + _tw.WriteLine("", move.SourceState, move.TargetState); + } + + foreach (KeyValuePair<(int, int), List> move in nonEpsilonMoves) + { + _tw.WriteLine(GetNonFinalRuleInfo(fa, move.Key.Item1, move.Key.Item2, move.Value)); + } + + foreach (int state in fa.GetStates()) + { + _tw.WriteLine("", state); + } + + _tw.WriteLine(""); + WriteCategoriesAndStyles(); + } + _tw.WriteLine(""); + } + + private string GetDFAInfo(IAutomaton fa) + { + StringBuilder sb = new(); + sb.Append("|Q|="); + sb.Append(fa.StateCount); + sb.Append(" "); + sb.Append('|'); + sb.Append(DeltaCapital); + sb.Append("|="); + sb.Append(fa.TransitionCount); + sb.Append(" "); + sb.Append('|'); + sb.Append(SigmalCapital); + sb.Append("|="); + sb.Append(fa.Alphabet.Length); + sb.Append(" "); + sb.Append(SigmalCapital); + sb.Append('='); + for (int i = 0; i < fa.Alphabet.Length; i++) + { + if (i > 0) + sb.Append(','); + sb.Append(fa.DescribeLabel(fa.Alphabet[i])); + } + return sb.ToString(); + } + + private const string DeltaCapital = "Δ"; + private const string SigmalCapital = "Σ"; + + private static string GetStateInfo(IAutomaton fa, int state) + { + StringBuilder sb = new(); + sb.Append(fa.DescribeState(state)); + return sb.ToString(); + } + + private string GetNonFinalRuleInfo(IAutomaton aut, int source, int target, List rules) + { + string lab = ""; + string info = ""; + for (int i = 0; i < rules.Count; i++) + { + lab += (lab == "" ? "" : ",\n ") + aut.DescribeLabel(rules[i]); + } + + int lab_length = lab.Length; + if (_maxDgmlTransitionLabelLength >= 0 && lab_length > _maxDgmlTransitionLabelLength) + { + info += $" FullLabel = \"{lab}\""; + lab = string.Concat(lab.AsSpan(0, _maxDgmlTransitionLabelLength), ".."); + } + + return $""; + } + + private void WriteCategoriesAndStyles() + { + _tw.WriteLine(""); + _tw.WriteLine(""); + _tw.WriteLine(""); + _tw.WriteLine(""); + _tw.WriteLine(""); + _tw.WriteLine(""); + _tw.WriteLine(""); + _tw.WriteLine(""); + _tw.WriteLine(""); + _tw.WriteLine(""); + _tw.WriteLine(""); + _tw.WriteLine(""); + _tw.WriteLine(""); + _tw.WriteLine(""); + _tw.WriteLine(""); + //_tw.WriteLine(""); + //_tw.WriteLine(""); + _tw.WriteLine(""); + _tw.WriteLine(""); + _tw.WriteLine(""); + _tw.WriteLine(""); + _tw.WriteLine(""); + _tw.WriteLine(""); + _tw.WriteLine(""); + _tw.WriteLine(""); + } + } +} +#endif diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Dgml/IAutomaton.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Dgml/IAutomaton.cs new file mode 100644 index 00000000000000..f6237967905f49 --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Dgml/IAutomaton.cs @@ -0,0 +1,66 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#if DEBUG +using System.Collections.Generic; + +namespace System.Text.RegularExpressions.Symbolic.DGML +{ + /// + /// For accessing the key components of an automaton. + /// + /// type of labels in moves + internal interface IAutomaton + { + /// + /// Enumerates all moves of the automaton. + /// + IEnumerable> GetMoves(); + + /// + /// Enumerates all states of the automaton. + /// + IEnumerable GetStates(); + + /// + /// Returns the minterm partition of the alphabet. + /// + TLabel[] Alphabet { get; } + + /// + /// Provides a description of the state for visualization purposes. + /// + string DescribeState(int state); + + /// + /// Provides a description of the label for visualization purposes. + /// + string DescribeLabel(TLabel lab); + + /// + /// Provides a description of the start label for visualization purposes. + /// + string DescribeStartLabel(); + + /// + /// The initial state of the automaton. + /// + int InitialState { get; } + + /// + /// The number of states of the automaton. + /// + int StateCount { get; } + + /// + /// The number of transitions of the automaton. + /// + int TransitionCount { get; } + + /// + /// Returns true iff the state is a final state. + /// + bool IsFinalState(int state); + } +} +#endif diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Dgml/Move.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Dgml/Move.cs new file mode 100644 index 00000000000000..e25e3b720a9017 --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Dgml/Move.cs @@ -0,0 +1,77 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#if DEBUG +using System.Diagnostics.CodeAnalysis; + +namespace System.Text.RegularExpressions.Symbolic.DGML +{ + /// + /// Represents a move of a symbolic finite automaton. + /// The value default(L) is reserved to represent the label of an epsilon move. + /// Thus if S is a reference type the label of an epsilon move is null. + /// + /// the type of the labels on moves + internal sealed class Move + { + /// + /// Source state of the move + /// + public readonly int SourceState; + /// + /// Target state of the move + /// + public readonly int TargetState; + /// + /// Label of the move + /// + public readonly TLabel? Label; + + /// + /// Transition of an automaton. + /// + /// source state of the transition + /// target state of the transition + /// label of the transition + public Move(int sourceState, int targetState, TLabel? lab) + { + SourceState = sourceState; + TargetState = targetState; + Label = lab; + } + + /// + /// Creates a move. Creates an epsilon move if label is default(L). + /// + public static Move Create(int sourceState, int targetState, TLabel condition) => new Move(sourceState, targetState, condition); + + /// + /// Creates an epsilon move. Same as Create(sourceState, targetState, default(L)). + /// + public static Move Epsilon(int sourceState, int targetState) => new Move(sourceState, targetState, default); + + /// + /// Returns true if label equals default(S). + /// + public bool IsEpsilon => Equals(Label, default(TLabel)); + + /// + /// Returns true if the source state and the target state are identical + /// + public bool IsSelfLoop => SourceState == TargetState; + + /// + /// Returns true if obj is a move with the same source state, target state, and label. + /// + public override bool Equals([NotNullWhen(false)] object? obj) => + obj is Move t && + t.SourceState == SourceState && + t.TargetState == TargetState && + (t.Label is null ? Label is null : t.Label.Equals(Label)); + + public override int GetHashCode() => (SourceState, Label, TargetState).GetHashCode(); + + public override string ToString() => $"({SourceState},{(Equals(Label, default(TLabel)) ? "" : Label + ",")}{TargetState})"; + } +} +#endif diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Dgml/RegexAutomaton.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Dgml/RegexAutomaton.cs new file mode 100644 index 00000000000000..56273393331812 --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Dgml/RegexAutomaton.cs @@ -0,0 +1,140 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#if DEBUG +using System.Collections.Generic; +using System.Diagnostics; + +namespace System.Text.RegularExpressions.Symbolic.DGML +{ + /// + /// Used by DgmlWriter to unwind a regex into a DFA up to a bound that limits the number of states + /// + internal sealed class RegexAutomaton : IAutomaton<(SymbolicRegexNode?, T)> where T : notnull + { + private readonly DfaMatchingState _q0; + private readonly List _states = new(); + private readonly HashSet _stateSet = new(); + private readonly List?, T)>> _moves = new(); + private readonly SymbolicRegexBuilder _builder; + private SymbolicNFA? _nfa; + + internal RegexAutomaton(SymbolicRegexMatcher srm, int bound, bool addDotStar, bool inReverse, bool asNFA) + { + _builder = srm._builder; + uint startId = inReverse ? + (srm._reversePattern._info.StartsWithLineAnchor ? CharKind.StartStop : 0) : + (srm._pattern._info.StartsWithLineAnchor ? CharKind.StartStop : 0); + + //inReverse only matters if Ar contains some line anchor + _q0 = _builder.MkState(inReverse ? srm._reversePattern : (addDotStar ? srm._dotStarredPattern : srm._pattern), startId); + + if (asNFA) + { + _nfa = _q0.Node.Explore(bound); + for (int q = 0; q < _nfa.StateCount; q++) + { + _states.Add(q); + foreach ((T, SymbolicRegexNode?, int) branch in _nfa.EnumeratePaths(q)) + _moves.Add(Move<(SymbolicRegexNode?, T)>.Create(q, branch.Item3, (branch.Item2, branch.Item1))); + } + } + else + { + Dictionary<(int, int), T> normalizedmoves = new(); + Stack> stack = new(); + stack.Push(_q0); + _states.Add(_q0.Id); + _stateSet.Add(_q0.Id); + + T[]? partition = _builder._solver.GetMinterms(); + Debug.Assert(partition is not null); + //unwind until the stack is empty or the bound has been reached + while (stack.Count > 0 && (bound <= 0 || _states.Count < bound)) + { + DfaMatchingState q = stack.Pop(); + foreach (T c in partition) + { + DfaMatchingState p = q.Next(c); + + // check that p is not a dead-end + if (!p.IsNothing) + { + if (_stateSet.Add(p.Id)) + { + stack.Push(p); + _states.Add(p.Id); + } + + var qp = (q.Id, p.Id); + normalizedmoves[qp] = normalizedmoves.ContainsKey(qp) ? + _builder._solver.Or(normalizedmoves[qp], c) : + c; + } + } + } + + foreach (KeyValuePair<(int, int), T> entry in normalizedmoves) + _moves.Add(Move<(SymbolicRegexNode?, T)>.Create(entry.Key.Item1, entry.Key.Item2, (null, entry.Value))); + } + } + + public (SymbolicRegexNode?, T)[] Alphabet + { + get + { + T[]? alphabet = _builder._solver.GetMinterms(); + Debug.Assert(alphabet is not null); + var results = new (SymbolicRegexNode?, T)[alphabet.Length]; + for (int i = 0; i < alphabet.Length; i++) + { + results[i] = (null, alphabet[i]); + } + return results; + } + } + + public int InitialState => _nfa is not null ? 0 : _q0.Id; + + public int StateCount => _states.Count; + + public int TransitionCount => _moves.Count; + + public string DescribeLabel((SymbolicRegexNode?, T) lab) => + lab.Item1 is null ? Net.WebUtility.HtmlEncode(_builder._solver.PrettyPrint(lab.Item2)) : + // Conditional nullability based on anchors + Net.WebUtility.HtmlEncode($"{lab.Item1}/{_builder._solver.PrettyPrint(lab.Item2)}"); + + public string DescribeStartLabel() => ""; + + public string DescribeState(int state) + { + if (_nfa is not null) + { + Debug.Assert(state < _nfa.StateCount); + var str = Net.WebUtility.HtmlEncode(_nfa.GetNode(state).ToString()); + return _nfa.IsUnexplored(state) ? $"Unexplored:{str}" : str; + } + + Debug.Assert(_builder._statearray is not null); + return _builder._statearray[state].DgmlView; + } + + public IEnumerable GetStates() => _states; + + public bool IsFinalState(int state) + { + if (_nfa is not null) + { + Debug.Assert(state < _nfa.StateCount); + return _nfa.CanBeNullable(state); + } + + Debug.Assert(_builder._statearray is not null && state < _builder._statearray.Length); + return _builder._statearray[state].Node.CanBeNullable; + } + + public IEnumerable?, T)>> GetMoves() => _moves; + } +} +#endif diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/MintermClassifier.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/MintermClassifier.cs new file mode 100644 index 00000000000000..e42576421e3ce2 --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/MintermClassifier.cs @@ -0,0 +1,100 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Runtime.CompilerServices; + +namespace System.Text.RegularExpressions.Symbolic +{ + /// Classifies characters as their corresponding minterm IDs. + /// + /// Minterms are a mechanism for compressing the input character space, or "alphabet", + /// by creating an equivalence class for all characters treated the same. For example, + /// in the expression "[0-9]*", the 10 digits 0 through 9 are all treated the same as each + /// other, and every other of the 65,526 characters are treated the same as each other, + /// so there are two minterms, one for the digits, and one for everything else. Minterms + /// are computed in such a way that every character maps to one and only one minterm. + /// While in the limit there could be one minterm per character, in practice the number + /// of minterms for any reasonable expression is way less, and in fact is typically + /// less than 64. + /// + internal sealed class MintermClassifier + { + /// An array used when there's a single minterm, in order to map every ASCII character to it trivially. + private static readonly int[] AllAsciiIsZeroMintermArray = new int[128]; + + /// Array providing fast mapping from an ASCII character (the array index) to its corresponding minterm ID. + private readonly int[] _ascii; + /// A multi-terminal BDD for mapping any non-ASCII character to its associated minterm ID. + /// + /// The use of a multi-terminal BDD here is an implementation detail. Should we decide its important to optimize non-ASCII inputs further, + /// or to consolidate the mechanism with the other engines, an alternatie lookup algorithm / data structure could be employed. + /// + private readonly BDD _nonAscii; + + /// Create a classifier that maps a character to the ID of its associated minterm. + /// Character algebra + /// A BDD for classifying all characters (ASCII and non-ASCII) to their corresponding minterm IDs. + public MintermClassifier(CharSetSolver solver, BDD[] minterms) + { + Debug.Assert(minterms.Length > 0, "Requires at least"); + + if (minterms.Length == 1) + { + // With only a single minterm, the mapping is trivial: everything maps to it (ID 0). + // For ASCII, use an array containing all zeros. For non-ASCII, use a BDD that maps everything to 0. + _ascii = AllAsciiIsZeroMintermArray; + _nonAscii = solver.ReplaceTrue(BDD.True, 0); + return; + } + + // Create a multi-terminal BDD for mapping any character to its associated minterm. + BDD anyCharacterToMintermId = BDD.False; + for (int i = 0; i < minterms.Length; i++) + { + // Each supplied minterm BDD decides whether a given character maps to it or not. + // We need to combine all of those into a multi-terminal BDD that decides which + // minterm a character maps to. To do that, we take each minterm BDD and replace + // its True result with the ID of the minterm, such that a character that would + // have returned True for that BDD now returns the minterm ID. + BDD charToTargetMintermId = solver.ReplaceTrue(minterms[i], i); + + // Now union this BDD with the multi-terminal BDD we've built up thus far. Unioning + // is valid because every character belongs to exactly one minterm and thus will + // only map to an ID instead of False in exactly one of the input BDDs. + anyCharacterToMintermId = solver.Or(anyCharacterToMintermId, charToTargetMintermId); + } + + // Now that we have our mapping that supports any input character, we want to optimize for + // ASCII inputs. Rather than forcing every input ASCII character to consult the BDD at match + // time, we precompute a lookup table, where each ASCII character can be used to index into the + // array to determine the ID for its corresponding minterm. + var ascii = new int[128]; + for (int i = 0; i < ascii.Length; i++) + { + ascii[i] = anyCharacterToMintermId.Find(i); + } + _ascii = ascii; + + // We can also further optimize the BDD in two ways: + // 1. We can now remove the ASCII characters from it, as we'll always consult the lookup table first + // for ASCII inputs and thus will never use the BDD for them. While optional (skipping this step will not + // affect correctness), removing the ASCII values from the BDD reduces the size of the multi-terminal BDD. + // 2. We can check if every character now maps to the same minterm ID (the same terminal in the + // multi-terminal BDD). This can be relatively common after (1) above is applied, as many + // patterns don't distinguish between any non-ASCII characters (e.g. "[0-9]*"). If every character + // in the BDD now maps to the same minterm, we can replace the BDD with a much simpler/faster/smaller one. + BDD nonAsciiBDD = solver.And(anyCharacterToMintermId, solver._nonAscii); + nonAsciiBDD = nonAsciiBDD.IsEssentiallyBoolean(out BDD? singleTerminalBDD) ? singleTerminalBDD : nonAsciiBDD; + _nonAscii = nonAsciiBDD; + } + + /// Gets the ID of the minterm associated with the specified character. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int GetMintermID(int c) + { + int[] ascii = _ascii; + return (uint)c < ascii.Length ? ascii[c] : _nonAscii.Find(c); + } + } +} diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/RegexNodeToSymbolicConverter.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/RegexNodeToSymbolicConverter.cs new file mode 100644 index 00000000000000..7b88e04f29c969 --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/RegexNodeToSymbolicConverter.cs @@ -0,0 +1,510 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.Runtime.CompilerServices; + +namespace System.Text.RegularExpressions.Symbolic +{ + /// Provides functionality to convert s to corresponding s. + internal sealed class RegexNodeToSymbolicConverter + { + internal readonly Unicode.UnicodeCategoryTheory _categorizer; + internal readonly SymbolicRegexBuilder _builder; + private readonly CultureInfo _culture; + private readonly Dictionary<(bool, string), BDD> _createConditionFromSet_Cache = new(); + + /// Constructs a regex to symbolic finite automata converter + public RegexNodeToSymbolicConverter(Unicode.UnicodeCategoryTheory categorizer, CultureInfo culture) + { + _categorizer = categorizer; + _culture = culture; + Solver = categorizer._solver; + _builder = new SymbolicRegexBuilder(Solver); + } + + /// The character solver associated with the regex converter + public ICharAlgebra Solver { get; } + + private BDD CreateConditionFromSet(bool ignoreCase, string set) + { + (bool ignoreCase, string set) key = (ignoreCase, set); + if (!_createConditionFromSet_Cache.TryGetValue(key, out BDD? result)) + { + _createConditionFromSet_Cache[key] = result = Compute(ignoreCase, set); + } + + return result; + + BDD Compute(bool ignoreCase, string set) + { + // Char at position 0 is 1 iff the set is negated + bool negate = RegexCharClass.IsNegated(set); + + // The set is divided into three pieces: ranges, conditions, subtraction + + // Handle ranges: + // Following are conditions over characters in the set. + // These will become disjuncts of a single disjunction + // or conjuncts of a conjunction in case negate is true. + // Negation is pushed in when the conditions are created. + List conditions = new List(); + foreach ((char first, char last) in ComputeRanges(set)) + { + BDD cond = Solver.RangeConstraint(first, last, ignoreCase, _culture.Name); + conditions.Add(negate ? Solver.Not(cond) : cond); + } + + // Handle categories: + int setLength = set[RegexCharClass.SetLengthIndex]; + int catLength = set[RegexCharClass.CategoryLengthIndex]; + int catStart = setLength + RegexCharClass.SetStartIndex; + int j = catStart; + while (j < catStart + catLength) + { + // Singleton categories are stored as unicode characters whose code is 1 + the + // unicode category code as a short. Thus -1 is applied to extract the actual + // code of the category. The category itself may be negated, e.g. \D instead of \d. + short catCode = (short)set[j++]; + if (catCode != 0) + { + // Note that double negation cancels out the negation of the category. + BDD cond = MapCategoryCodeToCondition(Math.Abs(catCode) - 1); + conditions.Add(catCode < 0 ^ negate ? Solver.Not(cond) : cond); + } + else + { + // Special case for a whole group G of categories surrounded by 0's. + // Essentially 0 C1 C2 ... Cn 0 ==> G = (C1 | C2 | ... | Cn) + catCode = (short)set[j++]; + if (catCode == 0) + { + continue; //empty set of categories + } + + // Collect individual category codes into this set + var catCodes = new HashSet(); + + // If the first catCode is negated, the group as a whole is negated + bool negGroup = catCode < 0; + + while (catCode != 0) + { + catCodes.Add(Math.Abs(catCode) - 1); + catCode = (short)set[j++]; + } + + // C1 | C2 | ... | Cn + BDD catCondDisj = MapCategoryCodeSetToCondition(catCodes); + + BDD catGroupCond = negate ^ negGroup ? Solver.Not(catCondDisj) : catCondDisj; + conditions.Add(catGroupCond); + } + } + + // Handle subtraction + BDD? subtractorCond = null; + if (set.Length > j) + { + // The set has a subtractor-set at the end. + // All characters in the subtractor-set are excluded from the set. + // Note that the subtractor sets may be nested, e.g. in r=[a-z-[b-g-[cd]]] + // the subtractor set [b-g-[cd]] has itself a subtractor set [cd]. + // Thus r is the set of characters between a..z except b,e,f,g + subtractorCond = CreateConditionFromSet(ignoreCase, set.Substring(j)); + } + + // If there are no ranges and no groups then there are no conditions. + // This situation arises for SingleLine regegex option and . + // and means that all characters are accepted. + BDD moveCond = conditions.Count == 0 ? + (negate ? Solver.False : Solver.True) : + (negate ? Solver.And(conditions) : Solver.Or(conditions)); + + // Subtlety of regex sematics: + // The subtractor is not within the scope of the negation (if there is a negation). + // Thus the negated subtractor is conjuncted with moveCond after the negation has been + // performed above. + if (subtractorCond is not null) + { + moveCond = Solver.And(moveCond, Solver.Not(subtractorCond)); + } + + return moveCond; + + static List<(char First, char Last)> ComputeRanges(string set) + { + int setLength = set[RegexCharClass.SetLengthIndex]; + + var ranges = new List<(char, char)>(setLength); + int i = RegexCharClass.SetStartIndex; + int end = i + setLength; + while (i < end) + { + char first = set[i]; + i++; + + char last = i < end ? + (char)(set[i] - 1) : + RegexCharClass.LastChar; + i++; + + ranges.Add((first, last)); + } + + return ranges; + } + + BDD MapCategoryCodeSetToCondition(HashSet catCodes) + { + // TBD: perhaps other common cases should be specialized similarly + // check first if all word character category combinations are covered + // which is the most common case, then use the combined predicate \w + // rather than a disjunction of the component category predicates + // the word character class \w covers categories 0,1,2,3,4,8,18 + BDD? catCond = null; + if (catCodes.Contains(0) && catCodes.Contains(1) && catCodes.Contains(2) && catCodes.Contains(3) && + catCodes.Contains(4) && catCodes.Contains(8) && catCodes.Contains(18)) + { + catCodes.Remove(0); + catCodes.Remove(1); + catCodes.Remove(2); + catCodes.Remove(3); + catCodes.Remove(4); + catCodes.Remove(8); + catCodes.Remove(18); + catCond = _categorizer.WordLetterCondition; + } + + foreach (int cat in catCodes) + { + BDD cond = MapCategoryCodeToCondition(cat); + catCond = catCond is null ? cond : Solver.Or(catCond, cond); + } + + Debug.Assert(catCodes.Count != 0); + return catCond!; + } + + BDD MapCategoryCodeToCondition(int code) => + code switch + { + 99 => _categorizer.WhiteSpaceCondition, // whitespace has special code 99 + < 0 or > 29 => throw new ArgumentOutOfRangeException(nameof(code), "Must be in the range 0..29 or equal to 99"), // TODO-NONBACKTRACKING: Remove message or put it into the .resx + _ => _categorizer.CategoryCondition(code) + }; + } + } + + public SymbolicRegexNode Convert(RegexNode node, bool topLevel) + { + // Guard against stack overflow due to deep recursion + if (!RuntimeHelpers.TryEnsureSufficientExecutionStack()) + { + RegexNode localNode = node; + bool localTopLevel = topLevel; + return StackHelper.CallOnEmptyStack(() => Convert(localNode, localTopLevel)); + } + + switch (node.Type) + { + case RegexNode.Alternate: + { + var nested = new SymbolicRegexNode[node.ChildCount()]; + for (int i = 0; i < nested.Length; i++) + { + nested[i] = Convert(node.Child(i), topLevel); + } + return _builder.MkOr(nested); + } + + case RegexNode.Beginning: + return _builder._startAnchor; + + case RegexNode.Bol: + EnsureNewlinePredicateInitialized(); + return _builder._bolAnchor; + + case RegexNode.Capture when node.N == -1: + return Convert(node.Child(0), topLevel); // treat as non-capturing group (...) + + case RegexNode.Concatenate: + { + List nested = FlattenNestedConcatenations(node); + var converted = new SymbolicRegexNode[nested.Count]; + for (int i = 0; i < converted.Length; i++) + { + converted[i] = Convert(nested[i], topLevel: false); + } + return _builder.MkConcat(converted, topLevel); + } + + case RegexNode.Empty: + case RegexNode.UpdateBumpalong: // optional directive that behaves the same as Empty + return _builder._epsilon; + + case RegexNode.End: // \z anchor + return _builder._endAnchor; + + case RegexNode.EndZ: // \Z anchor + EnsureNewlinePredicateInitialized(); + return _builder._endAnchorZ; + + case RegexNode.Eol: + EnsureNewlinePredicateInitialized(); + return _builder._eolAnchor; + + case RegexNode.Loop: + return _builder.MkLoop(Convert(node.Child(0), topLevel: false), isLazy: false, node.M, node.N); + + case RegexNode.Lazyloop: + return _builder.MkLoop(Convert(node.Child(0), topLevel: false), isLazy: true, node.M, node.N); + + case RegexNode.Multi: + return ConvertMulti(node, topLevel); + + case RegexNode.Notone: + return _builder.MkSingleton(Solver.Not(Solver.CharConstraint(node.Ch, (node.Options & RegexOptions.IgnoreCase) != 0, _culture.Name))); + + case RegexNode.Notoneloop: + case RegexNode.Notonelazy: + return ConvertNotoneloop(node, node.Type == RegexNode.Notonelazy); + + case RegexNode.One: + return _builder.MkSingleton(Solver.CharConstraint(node.Ch, (node.Options & RegexOptions.IgnoreCase) != 0, _culture.Name)); + + case RegexNode.Oneloop: + case RegexNode.Onelazy: + return ConvertOneloop(node, node.Type == RegexNode.Onelazy); + + case RegexNode.Set: + return ConvertSet(node); + + case RegexNode.Setloop: + case RegexNode.Setlazy: + return ConvertSetloop(node, node.Type == RegexNode.Setlazy); + + // TBD: ECMA case intersect predicate with ascii range ? + case RegexNode.Boundary: + case RegexNode.ECMABoundary: + EnsureWordLetterPredicateInitialized(); + return _builder._wbAnchor; + + // TBD: ECMA case intersect predicate with ascii range ? + case RegexNode.NonBoundary: + case RegexNode.NonECMABoundary: + EnsureWordLetterPredicateInitialized(); + return _builder._nwbAnchor; + + case RegexNode.Nothing: + return _builder._nothing; + +#if DEBUG + case RegexNode.Testgroup: + // Try to extract the special case representing complement or intersection + if (IsComplementedNode(node)) + { + return _builder.MkNot(Convert(node.Child(0), topLevel: false)); + } + + if (TryGetIntersection(node, out List? conjuncts)) + { + var nested = new SymbolicRegexNode[conjuncts.Count]; + for (int i = 0; i < nested.Length; i++) + { + nested[i] = Convert(conjuncts[i], topLevel: false); + } + return _builder.MkAnd(nested); + } + + goto default; +#endif + + default: + throw new NotSupportedException(SR.Format(SR.NotSupported_NonBacktrackingConflictingExpression, node.Type switch + { + RegexNode.Capture => SR.ExpressionDescription_BalancingGroup, + RegexNode.Testgroup => SR.ExpressionDescription_IfThenElse, + RegexNode.Ref => SR.ExpressionDescription_Backreference, + RegexNode.Testref => SR.ExpressionDescription_Conditional, + RegexNode.Require => SR.ExpressionDescription_PositiveLookaround, + RegexNode.Prevent => SR.ExpressionDescription_NegativeLookaround, + RegexNode.Start => SR.ExpressionDescription_ContiguousMatches, + RegexNode.Atomic or + RegexNode.Setloopatomic or + RegexNode.Oneloopatomic or + RegexNode.Notoneloopatomic => SR.ExpressionDescription_AtomicSubexpressions, + _ => UnexpectedNodeType(node) + })); + + static string UnexpectedNodeType(RegexNode node) + { + // The default should never arise, since other node types are either supported + // or have been removed (e.g. Group) from the final parse tree. + string description = $"Unexpected node type ({nameof(RegexNode)}:{node.Type})"; + Debug.Fail(description); + return description; + } + } + + void EnsureNewlinePredicateInitialized() + { + // Update the \n predicate in the builder if it has not been updated already + if (_builder._newLinePredicate.Equals(_builder._solver.False)) + { + _builder._newLinePredicate = _builder._solver.CharConstraint('\n'); + } + } + + void EnsureWordLetterPredicateInitialized() + { + // Update the word letter predicate based on the Unicode definition of it if it was not updated already + if (_builder._wordLetterPredicateForAnchors.Equals(_builder._solver.False)) + { + // Use the predicate including joiner and non joiner + _builder._wordLetterPredicateForAnchors = _categorizer.WordLetterConditionForAnchors; + } + } + + List FlattenNestedConcatenations(RegexNode concat) + { + var results = new List(); + + var todo = new Stack(); + todo.Push(concat); + + while (todo.TryPop(out RegexNode? node)) + { + if (node.Type == RegexNode.Concatenate) + { + // Flatten nested concatenations + for (int i = node.ChildCount() - 1; i >= 0; i--) + { + todo.Push(node.Child(i)); + } + } + else if (node.Type == RegexNode.Capture) + { + if (node.N == -1) + { + // Unwrap nonbalancing capture groups + todo.Push(node.Child(0)); + } + else + { + // Balancing groups are not supported + throw new NotSupportedException(SR.Format(SR.NotSupported_NonBacktrackingConflictingExpression, SR.ExpressionDescription_BalancingGroup)); + } + } + else + { + results.Add(node); + } + } + + return results; + } + + SymbolicRegexNode ConvertMulti(RegexNode node, bool topLevel) + { + Debug.Assert(node.Type == RegexNode.Multi); + + string? sequence = node.Str; + Debug.Assert(sequence is not null); + + bool ignoreCase = (node.Options & RegexOptions.IgnoreCase) != 0; + + var conds = new BDD[sequence.Length]; + for (int i = 0; i < conds.Length; i++) + { + conds[i] = Solver.CharConstraint(sequence[i], ignoreCase, _culture.Name); + } + + return _builder.MkSequence(conds, topLevel); + } + + SymbolicRegexNode ConvertOneloop(RegexNode node, bool isLazy) + { + Debug.Assert(node.Type is RegexNode.Oneloop or RegexNode.Onelazy); + + bool ignoreCase = (node.Options & RegexOptions.IgnoreCase) != 0; + BDD cond = Solver.CharConstraint(node.Ch, ignoreCase, _culture.Name); + + SymbolicRegexNode body = _builder.MkSingleton(cond); + SymbolicRegexNode loop = _builder.MkLoop(body, isLazy, node.M, node.N); + return loop; + } + + SymbolicRegexNode ConvertNotoneloop(RegexNode node, bool isLazy) + { + Debug.Assert(node.Type is RegexNode.Notoneloop or RegexNode.Notonelazy); + + bool ignoreCase = (node.Options & RegexOptions.IgnoreCase) != 0; + BDD cond = Solver.Not(Solver.CharConstraint(node.Ch, ignoreCase, _culture.Name)); + + SymbolicRegexNode body = _builder.MkSingleton(cond); + SymbolicRegexNode loop = _builder.MkLoop(body, isLazy, node.M, node.N); + return loop; + } + + SymbolicRegexNode ConvertSet(RegexNode node) + { + Debug.Assert(node.Type == RegexNode.Set); + + string? set = node.Str; + Debug.Assert(set is not null); + + BDD moveCond = CreateConditionFromSet((node.Options & RegexOptions.IgnoreCase) != 0, set); + + return _builder.MkSingleton(moveCond); + } + + SymbolicRegexNode ConvertSetloop(RegexNode node, bool isLazy) + { + Debug.Assert(node.Type is RegexNode.Setloop or RegexNode.Setlazy); + + string? set = node.Str; + Debug.Assert(set is not null); + + BDD moveCond = CreateConditionFromSet((node.Options & RegexOptions.IgnoreCase) != 0, set); + + SymbolicRegexNode body = _builder.MkSingleton(moveCond); + return _builder.MkLoop(body, isLazy, node.M, node.N); + } + +#if DEBUG + // TODO-NONBACKTRACKING: recognizing strictly only [] (RegexNode.Nothing), for example [0-[0]] would not be recognized + bool IsNothing(RegexNode node) => node.Type == RegexNode.Nothing || (node.Type == RegexNode.Set && ConvertSet(node).IsNothing); + + bool IsDotStar(RegexNode node) => node.Type == RegexNode.Setloop && Convert(node, topLevel: false).IsAnyStar; + + bool IsIntersect(RegexNode node) => node.Type == RegexNode.Testgroup && node.ChildCount() > 2 && IsNothing(node.Child(2)); + + bool TryGetIntersection(RegexNode node, [Diagnostics.CodeAnalysis.NotNullWhen(true)] out List? conjuncts) + { + if (!IsIntersect(node)) + { + conjuncts = null; + return false; + } + + conjuncts = new(); + conjuncts.Add(node.Child(0)); + node = node.Child(1); + while (IsIntersect(node)) + { + conjuncts.Add(node.Child(0)); + node = node.Child(1); + } + + conjuncts.Add(node); + return true; + } + + bool IsComplementedNode(RegexNode node) => IsNothing(node.Child(1)) && IsDotStar(node.Child(2)); +#endif + } + } +} diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/StackHelper.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/StackHelper.cs new file mode 100644 index 00000000000000..254c0d5e28dfff --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/StackHelper.cs @@ -0,0 +1,31 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Threading; +using System.Threading.Tasks; + +namespace System.Text.RegularExpressions.Symbolic +{ + /// Provides tools for avoiding stack overflows. + internal static class StackHelper + { + // Queues the supplied delegate to the thread pool, then block waiting for it to complete. + // It does so in a way that prevents task inlining (which would defeat the purpose) but that + // also plays nicely with the thread pool's sync-over-async aggressive thread injection policies. + + /// Calls the provided function on the stack of a different thread pool thread. + /// The return type of the function. + /// The function to invoke. + public static T CallOnEmptyStack(Func func) => + Task.Run(func) + .ContinueWith(t => t.GetAwaiter().GetResult(), CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default) + .GetAwaiter().GetResult(); + + /// Calls the provided action on the stack of a different thread pool thread. + /// The action to invoke. + public static void CallOnEmptyStack(Action action) => + Task.Run(action) + .ContinueWith(t => t.GetAwaiter().GetResult(), CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default) + .GetAwaiter().GetResult(); + } +} diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicMatch.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicMatch.cs new file mode 100644 index 00000000000000..c6754b8577b0dc --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicMatch.cs @@ -0,0 +1,37 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace System.Text.RegularExpressions.Symbolic +{ + internal readonly struct SymbolicMatch + { + /// Indicates failure to find a match. + internal static SymbolicMatch NoMatch => new SymbolicMatch(-1, -1); + + /// Indicates a match was found but without meaningful details about where. + internal static SymbolicMatch QuickMatch => new SymbolicMatch(0, 0); + + public SymbolicMatch(int index, int length) + { + Index = index; + Length = length; + } + + public int Index { get; } + public int Length { get; } + public bool Success => Index >= 0; + + public static bool operator ==(SymbolicMatch left, SymbolicMatch right) => + left.Index == right.Index && left.Length == right.Length; + + public static bool operator !=(SymbolicMatch left, SymbolicMatch right) => + !(left == right); + + public override bool Equals([NotNullWhen(true)] object? obj) => + obj is SymbolicMatch other && this == other; + + public override int GetHashCode() => HashCode.Combine(Index, Length); + } +} diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicNFA.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicNFA.cs new file mode 100644 index 00000000000000..aaf1d57a4eae06 --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicNFA.cs @@ -0,0 +1,391 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics; + +namespace System.Text.RegularExpressions.Symbolic +{ + /// Represents the exploration of a symbolic regex as a symbolic NFA + internal sealed class SymbolicNFA where S : notnull + { + private readonly IBooleanAlgebra _solver; + private readonly Transition[] _transitionFunction; + private readonly SymbolicRegexNode[] _finalCondition; + private readonly HashSet _unexplored; + private readonly SymbolicRegexNode[] _nodes; + + private const int DeadendState = -1; + private const int UnexploredState = -2; + + /// If true then some states have not been explored + public bool IsIncomplete => _unexplored.Count > 0; + + private SymbolicNFA(IBooleanAlgebra solver, Transition[] transitionFunction, HashSet unexplored, SymbolicRegexNode[] nodes) + { + Debug.Assert(transitionFunction.Length > 0 && nodes.Length == transitionFunction.Length); + _solver = solver; + _transitionFunction = transitionFunction; + _finalCondition = new SymbolicRegexNode[nodes.Length]; + for (int i = 0; i < nodes.Length; i++) + { + _finalCondition[i] = nodes[i].ExtractNullabilityTest(); + } + _unexplored = unexplored; + _nodes = nodes; + } + + /// Total number of states, 0 is the initial state, states are numbered from 0 to StateCount-1 + public int StateCount => _transitionFunction.Length; + + /// If true then the state has not been explored + public bool IsUnexplored(int state) => _transitionFunction[state]._leaf == UnexploredState; + + /// If true then the state has no outgoing transitions + public bool IsDeadend(int state) => _transitionFunction[state]._leaf == DeadendState; + + /// If true then the state involves lazy loops or has no loops + public bool IsLazy(int state) => _nodes[state].IsLazy; + + /// Returns true if the state is nullable in the given context + public bool IsFinal(int state, uint context) => _finalCondition[state].IsNullableFor(context); + + /// Returns true if the state is nullable for some context + public bool CanBeNullable(int state) => _finalCondition[state].CanBeNullable; + + /// Returns true if the state is nullable for all contexts + public bool IsNullable(int state) => _finalCondition[state].IsNullable; + + /// Gets the underlying node of the state + public SymbolicRegexNode GetNode(int state) => _nodes[state]; + + /// Enumerates all target states from the given source state + /// must be a an integer between 0 and StateCount-1 + /// must be a value that acts as a minterm for the transitions emanating from the source state + /// reflects the immediate surrounding of the input and is used to determine nullability of anchors + public IEnumerable EnumerateTargetStates(int sourceState, S input, uint context) + { + Debug.Assert(sourceState >= 0 && sourceState < _transitionFunction.Length); + + // First operate in a mode assuming no Union happens by finding the target leaf state if one exists + Transition transition = _transitionFunction[sourceState]; + while (transition._kind != TransitionRegexKind.Union) + { + switch (transition._kind) + { + case TransitionRegexKind.Leaf: + // deadend and unexplored are negative + if (transition._leaf >= 0) + { + Debug.Assert(transition._leaf < _transitionFunction.Length); + yield return transition._leaf; + } + // The single target (or no target) state was found, so exit the whole enumeration + yield break; + + case TransitionRegexKind.Conditional: + Debug.Assert(transition._test is not null && transition._first is not null && transition._second is not null); + // Branch according to the input condition in relation to the test condition + if (_solver.IsSatisfiable(_solver.And(input, transition._test))) + { + // in a conditional transition input must be exclusive + Debug.Assert(!_solver.IsSatisfiable(_solver.And(input, _solver.Not(transition._test)))); + transition = transition._first; + } + else + { + transition = transition._second; + } + break; + + default: + Debug.Assert(transition._kind == TransitionRegexKind.Lookaround && transition._look is not null && transition._first is not null && transition._second is not null); + // Branch according to nullability of the lookaround condition in the given context + transition = transition._look.IsNullableFor(context) ? + transition._first : + transition._second; + break; + } + } + + // Continue operating in a mode where several target states can be yielded + Debug.Assert(transition._first is not null && transition._second is not null); + Stack todo = new(); + todo.Push(transition._second); + todo.Push(transition._first); + while (todo.TryPop(out Transition? top)) + { + switch (transition._kind) + { + case TransitionRegexKind.Leaf: + // dead-end + if (transition._leaf >= 0) + { + Debug.Assert(transition._leaf < _transitionFunction.Length); + yield return transition._leaf; + } + break; + + case TransitionRegexKind.Conditional: + Debug.Assert(transition._test is not null && transition._first is not null && transition._second is not null); + // Branch according to the input condition in relation to the test condition + if (_solver.IsSatisfiable(_solver.And(input, transition._test))) + { + // in a conditional transition input must be exclusive + Debug.Assert(!_solver.IsSatisfiable(_solver.And(input, _solver.Not(transition._test)))); + todo.Push(transition._first); + } + else + { + todo.Push(transition._second); + } + break; + + case TransitionRegexKind.Lookaround: + Debug.Assert(transition._look is not null && transition._first is not null && transition._second is not null); + // Branch according to nullability of the lookaround condition in the given context + todo.Push(transition._look.IsNullableFor(context) ? transition._first : transition._second); + break; + + + default: + Debug.Assert(transition._kind == TransitionRegexKind.Union && transition._first is not null && transition._second is not null); + todo.Push(transition._second); + todo.Push(transition._first); + break; + } + } + } + + public IEnumerable<(S, SymbolicRegexNode?, int)> EnumeratePaths(int sourceState) => + _transitionFunction[sourceState].EnumeratePaths(_solver, _solver.True); + + /// + /// TODO: Explore an unexplored state on transition further. + /// + public void ExploreState(int state) => new NotImplementedException(); + + public static SymbolicNFA Explore(SymbolicRegexNode root, int bound) + { + (Dictionary, Transition> cache, + Dictionary, int> statemap, + List> nodes, + Stack front) workState = (new(), new(), new(), new()); + + workState.nodes.Add(root); + workState.statemap[root] = 0; + workState.front.Push(0); + + Dictionary transitions = new(); + Stack front = new(); + + while (workState.front.Count > 0) + { + Debug.Assert(front.Count == 0); + + // Work Breadth-First in layers, swap front with workState.front + Stack tmp = front; + front = workState.front; + workState.front = tmp; + + // Process all the states in front first + // Any new states detected in Convert are added to workState.front + while (front.Count > 0 && (bound <= 0 || workState.nodes.Count < bound)) + { + int q = front.Pop(); + + // If q was on the front it must be associated with a node but not have a transition yet + Debug.Assert(q >= 0 && q < workState.nodes.Count && !transitions.ContainsKey(q)); + transitions[q] = Convert(workState.nodes[q].MkDerivative(), workState); + } + + if (front.Count > 0) + { + // The state bound was reached without completing the exploration so exit the loop + break; + } + } + + SymbolicRegexNode[] nodes_array = workState.nodes.ToArray(); + + // All states are numbered from 0 to nodes.Count-1 + Transition[] transition_array = new Transition[nodes_array.Length]; + foreach (var entry in transitions) + { + transition_array[entry.Key] = entry.Value; + } + + HashSet unexplored = new(front); + unexplored.UnionWith(workState.front); + foreach (int q in unexplored) + { + transition_array[q] = Transition.s_unexplored; + } + + // At this point no entry can be null in the transition array + Debug.Assert(Array.TrueForAll(transition_array, tr => tr is not null)); + + var nfa = new SymbolicNFA(root._builder._solver, transition_array, unexplored, nodes_array); + return nfa; + } + + private static Transition Convert(TransitionRegex tregex, + (Dictionary, Transition> cache, + Dictionary, int> statemap, + List> nodes, + Stack front) args) + { + Transition? transition; + if (args.cache.TryGetValue(tregex, out transition)) + { + return transition; + } + + Stack<(TransitionRegex, bool)> work = new(); + work.Push((tregex, false)); + + while (work.TryPop(out (TransitionRegex, bool) top)) + { + TransitionRegex tr = top.Item1; + bool wasPushedSecondTime = top.Item2; + if (wasPushedSecondTime) + { + Debug.Assert(tr._kind != TransitionRegexKind.Leaf && tr._first is not null && tr._second is not null); + transition = new Transition(kind: tr._kind, + test: tr._test, + look: tr._node, + first: args.cache[tr._first], + second: args.cache[tr._second]); + args.cache[tr] = transition; + } + else + { + switch (tr._kind) + { + case TransitionRegexKind.Leaf: + Debug.Assert(tr._node is not null); + + if (tr._node.IsNothing) + { + args.cache[tr] = Transition.s_deadend; + } + else + { + int state; + if (!args.statemap.TryGetValue(tr._node, out state)) + { + state = args.nodes.Count; + args.nodes.Add(tr._node); + args.statemap[tr._node] = state; + args.front.Push(state); + } + transition = new Transition(kind: TransitionRegexKind.Leaf, leaf: state); + args.cache[tr] = transition; + } + break; + + default: + Debug.Assert(tr._first is not null && tr._second is not null); + + // Push the tr for the second time + work.Push((tr, true)); + + // Push the branches also, unless they have been computed already + if (!args.cache.ContainsKey(tr._second)) + { + work.Push((tr._second, false)); + } + + if (!args.cache.ContainsKey(tr._first)) + { + work.Push((tr._first, false)); + } + + break; + } + } + } + + return args.cache[tregex]; + } + + /// Representation of transitions inside the parent class + private class Transition + { + public readonly TransitionRegexKind _kind; + public readonly int _leaf; + public readonly S? _test; + public readonly SymbolicRegexNode? _look; + public readonly Transition? _first; + public readonly Transition? _second; + + public static readonly Transition s_deadend = new Transition(TransitionRegexKind.Leaf, leaf: DeadendState); + public static readonly Transition s_unexplored = new Transition(TransitionRegexKind.Leaf, leaf: UnexploredState); + + internal Transition(TransitionRegexKind kind, int leaf = 0, S? test = default(S), SymbolicRegexNode? look = null, Transition? first = null, Transition? second = null) + { + _kind = kind; + _leaf = leaf; + _test = test; + _look = look; + _first = first; + _second = second; + } + + /// Enumerates all the paths in this transition excluding paths to dead-ends (and unexplored states if any) + internal IEnumerable<(S, SymbolicRegexNode?, int)> EnumeratePaths(IBooleanAlgebra solver, S pathCondition) + { + switch (_kind) + { + case TransitionRegexKind.Leaf: + // Omit any path that leads to a deadend or is unexplored + if (_leaf >= 0) + { + yield return (pathCondition, null, _leaf); + } + break; + + case TransitionRegexKind.Union: + Debug.Assert(_first is not null && _second is not null); + foreach ((S, SymbolicRegexNode?, int) path in _first.EnumeratePaths(solver, pathCondition)) + { + yield return path; + } + foreach ((S, SymbolicRegexNode?, int) path in _second.EnumeratePaths(solver, pathCondition)) + { + yield return path; + } + break; + + case TransitionRegexKind.Conditional: + Debug.Assert(_test is not null && _first is not null && _second is not null); + foreach ((S, SymbolicRegexNode?, int) path in _first.EnumeratePaths(solver, solver.And(pathCondition, _test))) + { + yield return path; + } + foreach ((S, SymbolicRegexNode?, int) path in _second.EnumeratePaths(solver, solver.And(pathCondition, solver.Not(_test)))) + { + yield return path; + } + break; + + default: + Debug.Assert(_kind is TransitionRegexKind.Lookaround && _look is not null && _first is not null && _second is not null); + foreach ((S, SymbolicRegexNode?, int) path in _first.EnumeratePaths(solver, pathCondition)) + { + SymbolicRegexNode nullabilityTest = path.Item2 is null ? _look : _look._builder.MkAnd(path.Item2, _look); + yield return (path.Item1, nullabilityTest, path.Item3); + } + foreach ((S, SymbolicRegexNode?, int) path in _second.EnumeratePaths(solver, pathCondition)) + { + // Complement the nullability test + SymbolicRegexNode nullabilityTest = path.Item2 is null ? _look._builder.MkNot(_look) : _look._builder.MkAnd(path.Item2, _look._builder.MkNot(_look)); + yield return (path.Item1, nullabilityTest, path.Item3); + } + break; + } + } + } + } +} diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicRegexBuilder.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicRegexBuilder.cs new file mode 100644 index 00000000000000..2e01a330bb84e7 --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicRegexBuilder.cs @@ -0,0 +1,418 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics; + +namespace System.Text.RegularExpressions.Symbolic +{ + /// + /// Builder of symbolic regexes over TElement. + /// TElement is the type of elements of an effective Boolean algebra. + /// Used to convert .NET regexes to symbolic regexes. + /// + internal sealed class SymbolicRegexBuilder where TElement : notnull + { + internal readonly ICharAlgebra _solver; + + internal readonly SymbolicRegexNode _epsilon; + internal readonly SymbolicRegexNode _nothing; + internal readonly SymbolicRegexNode _startAnchor; + internal readonly SymbolicRegexNode _endAnchor; + internal readonly SymbolicRegexNode _endAnchorZ; + internal readonly SymbolicRegexNode _endAnchorZRev; + internal readonly SymbolicRegexNode _bolAnchor; + internal readonly SymbolicRegexNode _eolAnchor; + internal readonly SymbolicRegexNode _anyChar; + internal readonly SymbolicRegexNode _anyStar; + internal readonly SymbolicRegexNode _wbAnchor; + internal readonly SymbolicRegexNode _nwbAnchor; + internal readonly SymbolicRegexSet _fullSet; + internal readonly SymbolicRegexSet _emptySet; + internal readonly SymbolicRegexNode _eagerEmptyLoop; + + internal TElement _wordLetterPredicateForAnchors; + internal TElement _newLinePredicate; + + /// Partition of the input space of predicates. + internal TElement[]? _minterms; + + private readonly Dictionary> _singletonCache = new(); + + // states that have been created + internal HashSet> _stateCache = new(); + + internal readonly Dictionary<(SymbolicRegexKind, + SymbolicRegexNode?, // _left + SymbolicRegexNode?, // _right + int, int, TElement?, // _lower, _upper, _set + SymbolicRegexSet?, + SymbolicRegexInfo), SymbolicRegexNode> _nodeCache = new(); + + internal readonly Dictionary<(TransitionRegexKind, // _kind + TElement?, // _test + TransitionRegex?, // _first + TransitionRegex?, // _second + SymbolicRegexNode?), // _leaf + TransitionRegex> _trCache = new(); + + /// + /// Maps state ids to states, initial capacity is 1024 states. + /// Each time more states are needed the length is increased by 1024. + /// + internal DfaMatchingState[]? _statearray; + internal DfaMatchingState[]? _delta; + private const int InitialStateLimit = 1024; + + /// + /// is the smallest k s.t. 2^k >= minterms.Length + 1 + /// + internal int _mintermsCount; + + /// + /// If true then delta is used in a mode where + /// each target state represents a set of states. + /// + internal bool _antimirov; + + /// Create a new symbolic regex builder. + internal SymbolicRegexBuilder(ICharAlgebra solver) + { + // Solver must be set first, else it will cause null reference exception in the following + _solver = solver; + _epsilon = SymbolicRegexNode.MkEpsilon(this); + _startAnchor = SymbolicRegexNode.MkStartAnchor(this); + _endAnchor = SymbolicRegexNode.MkEndAnchor(this); + _endAnchorZ = SymbolicRegexNode.MkEndAnchorZ(this); + _endAnchorZRev = SymbolicRegexNode.MkEndAnchorZRev(this); + _eolAnchor = SymbolicRegexNode.MkEolAnchor(this); + _bolAnchor = SymbolicRegexNode.MkBolAnchor(this); + _wbAnchor = SymbolicRegexNode.MkWBAnchor(this); + _nwbAnchor = SymbolicRegexNode.MkNWBAnchor(this); + _emptySet = SymbolicRegexSet.CreateEmpty(this); + _fullSet = SymbolicRegexSet.CreateFull(this); + _eagerEmptyLoop = SymbolicRegexNode.MkEagerEmptyLoop(this, _epsilon); + + // minterms = null if partition of the solver is undefined and returned as null + _minterms = solver.GetMinterms(); + if (_minterms == null) + { + _mintermsCount = -1; + } + else + { + _statearray = new DfaMatchingState[InitialStateLimit]; + + // the extra slot with id minterms.Length is reserved for \Z (last occurrence of \n) + int mintermsCount = 1; + while (_minterms.Length >= (1 << mintermsCount)) + { + mintermsCount++; + } + _mintermsCount = mintermsCount; + _delta = new DfaMatchingState[InitialStateLimit << _mintermsCount]; + } + + // initialized to False but updated later to the actual condition ony if \b or \B occurs anywhere in the regex + // this implies that if a regex never uses \b or \B then the character context will never + // update the previous character context to distinguish word and nonword letters + _wordLetterPredicateForAnchors = solver.False; + + // initialized to False but updated later to the actual condition of \n ony if a line anchor occurs anywhere in the regex + // this implies that if a regex never uses a line anchor then the character context will never + // update the previous character context to mark that the previous caharcter was \n + _newLinePredicate = solver.False; + _nothing = SymbolicRegexNode.MkFalse(this); + _anyChar = SymbolicRegexNode.MkTrue(this); + _anyStar = SymbolicRegexNode.MkStar(this, _anyChar); + + // --- initialize singletonCache --- + _singletonCache[_solver.False] = _nothing; + _singletonCache[_solver.True] = _anyChar; + } + + /// + /// Make a disjunction of given regexes, simplify by eliminating any regex that accepts no inputs + /// + internal SymbolicRegexNode MkOr(params SymbolicRegexNode[] regexes) => + SymbolicRegexNode.MkOr(this, regexes); + + /// + /// Make a conjunction of given regexes, simplify by eliminating regexes that accept everything + /// + internal SymbolicRegexNode MkAnd(params SymbolicRegexNode[] regexes) => + SymbolicRegexNode.MkAnd(this, regexes); + + /// + /// Make a disjunction of given regexes, simplify by eliminating any regex that accepts no inputs + /// + internal SymbolicRegexNode MkOr(SymbolicRegexSet alts) => + alts.IsNothing ? _nothing : + alts.IsEverything ? _anyStar : + alts.IsSingleton ? alts.GetSingletonElement() : + SymbolicRegexNode.MkOr(this, alts); + + internal SymbolicRegexNode MkOr2(SymbolicRegexNode x, SymbolicRegexNode y) => + x == _anyStar || y == _anyStar ? _anyStar : + x == _nothing ? y : + y == _nothing ? x : + SymbolicRegexNode.MkOr(this, x, y); + + /// + /// Make a conjunction of given regexes, simplify by eliminating any regex that accepts all inputs, + /// returns the empty regex if the regex accepts nothing + /// + internal SymbolicRegexNode MkAnd(SymbolicRegexSet alts) => + alts.IsNothing ? _nothing : + alts.IsEverything ? _anyStar : + alts.IsSingleton ? alts.GetSingletonElement() : + SymbolicRegexNode.MkAnd(this, alts); + + /// + /// Make a concatenation of given regexes, if any regex is nothing then return nothing, eliminate + /// intermediate epsilons, if toplevel and length is fixed, add watchdog at the end + /// + internal SymbolicRegexNode MkConcat(SymbolicRegexNode[] regexes, bool topLevel) + { + if (regexes.Length == 0) + return _epsilon; + + SymbolicRegexNode sr = _epsilon; + int length = CalculateFixedLength(regexes); + if (topLevel && length >= 0) + sr = MkWatchDog(length); + + //exclude epsilons from the concatenation + for (int i = regexes.Length - 1; i >= 0; i--) + { + if (regexes[i] == _nothing) + return _nothing; + + sr = SymbolicRegexNode.MkConcat(this, regexes[i], sr); + } + + return sr; + } + + internal SymbolicRegexNode MkConcat(SymbolicRegexNode left, SymbolicRegexNode right) => SymbolicRegexNode.MkConcat(this, left, right); + + private int CalculateFixedLength(SymbolicRegexNode[] regexes) + { + int length = 0; + for (int i = 0; i < regexes.Length; i++) + { + int k = regexes[i].GetFixedLength(); + if (k < 0) + { + return -1; + } + + length += k; + } + + return length; + } + + + /// + /// Make loop regex + /// + internal SymbolicRegexNode MkLoop(SymbolicRegexNode regex, bool isLazy, int lower = 0, int upper = int.MaxValue) + { + if (lower == 1 && upper == 1) + { + return regex; + } + + if (lower == 0 && upper == 0) + { + return isLazy ? _epsilon : _eagerEmptyLoop; + } + + if (!isLazy && lower == 0 && upper == int.MaxValue && regex._kind == SymbolicRegexKind.Singleton) + { + Debug.Assert(regex._set is not null); + if (_solver.AreEquivalent(_solver.True, regex._set)) + { + return _anyStar; + } + } + + return SymbolicRegexNode.MkLoop(this, regex, lower, upper, isLazy); + } + + /// + /// Make a singleton sequence regex + /// + internal SymbolicRegexNode MkSingleton(TElement set) + { + if (!_singletonCache.TryGetValue(set, out SymbolicRegexNode? res)) + { + _singletonCache[set] = res = SymbolicRegexNode.MkSingleton(this, set); + } + + return res; + } + + /// + /// Make end of sequence marker + /// + internal SymbolicRegexNode MkWatchDog(int length) => SymbolicRegexNode.MkWatchDog(this, length); + + /// + /// Make a sequence regex, i.e., a concatenation of singletons, with a watchdog at the end + /// + internal SymbolicRegexNode MkSequence(TElement[] seq, bool topLevel) + { + int k = seq.Length; + if (k == 0) + { + return _epsilon; + } + else if (k == 1) + { + return topLevel ? + SymbolicRegexNode.MkConcat(this, MkSingleton(seq[0]), MkWatchDog(1)) : + MkSingleton(seq[0]); + } + else + { + var singletons = new SymbolicRegexNode[seq.Length]; + for (int i =0; i < singletons.Length; i++) + { + singletons[i] = MkSingleton(seq[i]); + } + return MkConcat(singletons, topLevel); + } + } + + /// + /// Make a complemented node + /// + /// node to be complemented + /// + internal SymbolicRegexNode MkNot(SymbolicRegexNode node) => SymbolicRegexNode.MkNot(this, node); + + internal SymbolicRegexNode Transform(SymbolicRegexNode sr, SymbolicRegexBuilder builderT, Func predicateTransformer) where T : notnull + { + switch (sr._kind) + { + case SymbolicRegexKind.StartAnchor: + return builderT._startAnchor; + + case SymbolicRegexKind.EndAnchor: + return builderT._endAnchor; + + case SymbolicRegexKind.EndAnchorZ: + return builderT._endAnchorZ; + + case SymbolicRegexKind.EndAnchorZRev: + return builderT._endAnchorZRev; + + case SymbolicRegexKind.BOLAnchor: + return builderT._bolAnchor; + + case SymbolicRegexKind.EOLAnchor: + return builderT._eolAnchor; + + case SymbolicRegexKind.WBAnchor: + return builderT._wbAnchor; + + case SymbolicRegexKind.NWBAnchor: + return builderT._nwbAnchor; + + case SymbolicRegexKind.WatchDog: + return builderT.MkWatchDog(sr._lower); + + case SymbolicRegexKind.Epsilon: + return builderT._epsilon; + + case SymbolicRegexKind.Singleton: + Debug.Assert(sr._set is not null); + return builderT.MkSingleton(predicateTransformer(sr._set)); + + case SymbolicRegexKind.Loop: + Debug.Assert(sr._left is not null); + return builderT.MkLoop(Transform(sr._left, builderT, predicateTransformer), sr.IsLazy, sr._lower, sr._upper); + + case SymbolicRegexKind.Or: + Debug.Assert(sr._alts is not null); + return builderT.MkOr(sr._alts.Transform(builderT, predicateTransformer)); + + case SymbolicRegexKind.And: + Debug.Assert(sr._alts is not null); + return builderT.MkAnd(sr._alts.Transform(builderT, predicateTransformer)); + + case SymbolicRegexKind.Concat: + { + List> sr_elems = sr.ToList(); + SymbolicRegexNode[] sr_elems_trasformed = new SymbolicRegexNode[sr_elems.Count]; + for (int i = 0; i < sr_elems.Count; i++) + { + sr_elems_trasformed[i] = Transform(sr_elems[i], builderT, predicateTransformer); + } + return builderT.MkConcat(sr_elems_trasformed, false); + } + + default: + Debug.Assert(sr._kind == SymbolicRegexKind.Not); + Debug.Assert(sr._left is not null); + return builderT.MkNot(Transform(sr._left, builderT, predicateTransformer)); + } + } + + /// + /// Make a state with given node and previous character context + /// + public DfaMatchingState MkState(SymbolicRegexNode node, uint prevCharKind, bool antimirov = false) + { + //first prune the anchors in the node + TElement WLpred = _wordLetterPredicateForAnchors; + TElement startSet = node.GetStartSet(); + + //true if the startset of the node overlaps with some wordletter or the node can be nullable + bool contWithWL = node.CanBeNullable || _solver.IsSatisfiable(_solver.And(WLpred, startSet)); + + //true if the startset of the node overlaps with some nonwordletter or the node can be nullable + bool contWithNWL = node.CanBeNullable || _solver.IsSatisfiable(_solver.And(_solver.Not(WLpred), startSet)); + SymbolicRegexNode pruned_node = node.PruneAnchors(prevCharKind, contWithWL, contWithNWL); + var s = new DfaMatchingState(pruned_node, prevCharKind); + if (!_stateCache.TryGetValue(s, out DfaMatchingState? state)) + { + // do not cache set of states as states in antimirov mode + if (antimirov && pruned_node.Kind == SymbolicRegexKind.Or) + { + s.Id = -1; // mark the Id as invalid + state = s; + } + else + { + state = MakeNewState(s); + } + } + + return state; + } + + private DfaMatchingState MakeNewState(DfaMatchingState state) + { + lock (this) + { + state.Id = _stateCache.Count; + int k = state.GetHashCode(); + _stateCache.Add(state); + + Debug.Assert(_statearray is not null); + + if (state.Id == _statearray.Length) + { + int newsize = _statearray.Length + 1024; + Array.Resize(ref _statearray, newsize); + Array.Resize(ref _delta, newsize << _mintermsCount); + } + _statearray[state.Id] = state; + return state; + } + } + } +} diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicRegexInfo.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicRegexInfo.cs new file mode 100644 index 00000000000000..3bd803e47f3388 --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicRegexInfo.cs @@ -0,0 +1,205 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace System.Text.RegularExpressions.Symbolic +{ + /// Misc information of structural properties of a that is computed bottom up. + internal readonly struct SymbolicRegexInfo + { + private const uint IsAlwaysNullableMask = 1; + private const uint StartsWithLineAnchorMask = 2; + private const uint IsLazyMask = 4; + private const uint CanBeNullableMask = 8; + private const uint ContainsSomeAnchorMask = 16; + private const uint ContainsLineAnchorMask = 32; + private const uint ContainsSomeCharacterMask = 64; + private const uint StartsWithBoundaryAnchorMask = 128; + + private readonly uint _info; + + private SymbolicRegexInfo(uint i) => _info = i; + + /// Optimized lookup array for most common combinations. + /// Most common cases will be 0 (no anchors and not nullable) and 1 (no anchors and nullable) + private static readonly SymbolicRegexInfo[] s_infos = CreateSymbolicRegexInfos(); + + private static SymbolicRegexInfo[] CreateSymbolicRegexInfos() + { + var infos = new SymbolicRegexInfo[128]; + for (uint i = 0; i < infos.Length; i++) + { + infos[i] = new SymbolicRegexInfo(i); + } + return infos; + } + + private static SymbolicRegexInfo Mk(uint i) + { + SymbolicRegexInfo[] infos = s_infos; + return i < infos.Length ? + infos[i] : + new SymbolicRegexInfo(i); + } + + internal static SymbolicRegexInfo Mk(bool isAlwaysNullable = false, bool canBeNullable = false, bool startsWithLineAnchor = false, + bool startsWithBoundaryAnchor = false, bool containsSomeAnchor = false, + bool containsLineAnchor = false, bool containsSomeCharacter = false, bool isLazy = true) + { + uint i = 0; + + if (canBeNullable || isAlwaysNullable) + { + i |= CanBeNullableMask; + + if (isAlwaysNullable) + { + i |= IsAlwaysNullableMask; + } + } + + if (startsWithLineAnchor || containsLineAnchor || startsWithBoundaryAnchor || containsSomeAnchor) + { + i |= ContainsSomeAnchorMask; + + if (startsWithLineAnchor || containsLineAnchor) + { + i |= ContainsLineAnchorMask; + + if (startsWithLineAnchor) + { + i |= StartsWithLineAnchorMask; + } + } + + if (startsWithBoundaryAnchor) + { + i |= StartsWithBoundaryAnchorMask; + } + } + + if (containsSomeCharacter) + { + i |= ContainsSomeCharacterMask; + } + + if (isLazy) + { + i |= IsLazyMask; + } + + return Mk(i); + } + + public bool IsNullable => (_info & IsAlwaysNullableMask) != 0; + + public bool CanBeNullable => (_info & CanBeNullableMask) != 0; + + public bool StartsWithSomeAnchor => (_info & (StartsWithLineAnchorMask | StartsWithBoundaryAnchorMask)) != 0; + + public bool StartsWithLineAnchor => (_info & StartsWithLineAnchorMask) != 0; + + public bool StartsWithBoundaryAnchor => (_info & StartsWithBoundaryAnchorMask) != 0; + + public bool ContainsSomeAnchor => (_info & ContainsSomeAnchorMask) != 0; + + public bool ContainsLineAnchor => (_info & ContainsLineAnchorMask) != 0; + + public bool ContainsSomeCharacter => (_info & ContainsSomeCharacterMask) != 0; + + public bool IsLazy => (_info & IsLazyMask) != 0; + + public static SymbolicRegexInfo Or(SymbolicRegexInfo[] infos) + { + uint isLazy = IsLazyMask; + uint i = 0; + + for (int j = 0; j < infos.Length; j++) + { + // Disjunction is lazy if ALL of its members are lazy + isLazy &= infos[j]._info; + i |= infos[j]._info; + } + + i = (i & ~IsLazyMask) | isLazy; + return Mk(i); + } + + public static SymbolicRegexInfo And(params SymbolicRegexInfo[] infos) + { + uint isLazy = IsLazyMask; + uint isNullable = IsAlwaysNullableMask | CanBeNullableMask; + uint i = 0; + + foreach (SymbolicRegexInfo info in infos) + { + //nullability and lazyness are conjunctive while other properties are disjunctive + isLazy &= info._info; + isNullable &= info._info; + i |= info._info; + } + + i = (i & ~IsLazyMask) | isLazy; + i = (i & ~(IsAlwaysNullableMask | CanBeNullableMask)) | isNullable; + return Mk(i); + } + + public static SymbolicRegexInfo Concat(SymbolicRegexInfo left_info, SymbolicRegexInfo right_info) + { + bool isNullable = left_info.IsNullable && right_info.IsNullable; + bool canBeNullable = left_info.CanBeNullable && right_info.CanBeNullable; + bool isLazy = left_info.IsLazy && right_info.IsLazy; + + bool startsWithLineAnchor = left_info.StartsWithLineAnchor || (left_info.CanBeNullable && right_info.StartsWithLineAnchor); + bool startsWithBoundaryAnchor = left_info.StartsWithBoundaryAnchor || (left_info.CanBeNullable && right_info.StartsWithBoundaryAnchor); + bool containsSomeAnchor = left_info.ContainsSomeAnchor || right_info.ContainsSomeAnchor; + bool containsLineAnchor = left_info.ContainsLineAnchor || right_info.ContainsLineAnchor; + bool containsSomeCharacter = left_info.ContainsSomeCharacter || right_info.ContainsSomeCharacter; + + return Mk(isNullable, canBeNullable, startsWithLineAnchor, startsWithBoundaryAnchor, containsSomeAnchor, containsLineAnchor, containsSomeCharacter, isLazy); + } + + public static SymbolicRegexInfo Loop(SymbolicRegexInfo body_info, int lowerBound, bool isLazy) + { + // Inherit anchor visibility from the loop body + uint i = body_info._info; + + // The loop is nullable if either the body is nullable or if the lower boud is 0 + i |= lowerBound == 0 ? (IsAlwaysNullableMask | CanBeNullableMask) : 0; + + // The loop is lazy iff it is marked lazy + if (isLazy) + { + i |= IsLazyMask; + } + else + { + i &= ~IsLazyMask; + } + + return Mk(i); + } + + public static SymbolicRegexInfo Not(SymbolicRegexInfo info) => + // Nullability is complemented, all other properties remain the same + // The following rules are used to determine nullability of Not(node): + // Observe that this is used as an over-approximation, actual nullability is checked dynamically based on given context. + // - If node is never nullable (for any context, info.CanBeNullable=false) then Not(node) is always nullable + // - If node is always nullable (info.IsNullable=true) then Not(node) can never be nullable + // For example \B.CanBeNullable=true and \B.IsNullable=false + // and ~(\B).CanBeNullable=true and ~(\B).IsNullable=false + Mk(isAlwaysNullable: !info.CanBeNullable, + canBeNullable: !info.IsNullable, + startsWithLineAnchor: info.StartsWithLineAnchor, + startsWithBoundaryAnchor: info.StartsWithBoundaryAnchor, + containsSomeAnchor: info.ContainsSomeAnchor, + containsLineAnchor: info.ContainsLineAnchor, + containsSomeCharacter: info.ContainsSomeCharacter, + isLazy: info.IsLazy); + + public override bool Equals(object? obj) => obj is SymbolicRegexInfo i && i._info == _info; + + public override int GetHashCode() => _info.GetHashCode(); + + public override string ToString() => _info.ToString("X"); + } +} diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicRegexKind.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicRegexKind.cs new file mode 100644 index 00000000000000..f012d0559e9e8a --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicRegexKind.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace System.Text.RegularExpressions.Symbolic +{ + /// Kinds of symbolic regexes + internal enum SymbolicRegexKind + { + StartAnchor = 1, + EndAnchor = 2, + Epsilon = 4, + Singleton = 8, + Or = 0x10, + Concat = 0x20, + Loop = 0x40, + Not = 0x80, + And = 0x100, + WatchDog = 0x200, + BOLAnchor = 0x400, + EOLAnchor = 0x800, + WBAnchor = 0x1000, + NWBAnchor = 0x2000, + EndAnchorZ = 0x4000, + /// Anchor for very first line or start-line after very first \n arises as the reverse of EndAnchorZ + EndAnchorZRev = 0x8000, + } +} diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicRegexMatcher.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicRegexMatcher.cs new file mode 100644 index 00000000000000..ac56fd972d0cf1 --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicRegexMatcher.cs @@ -0,0 +1,903 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Runtime.CompilerServices; +using System.Threading; + +namespace System.Text.RegularExpressions.Symbolic +{ + /// Represents a regex matching engine that performs regex matching using symbolic derivatives. + internal abstract class SymbolicRegexMatcher + { + /// Returns the next match index and length in the input string. + /// Whether to return once we know there's a match without determining where exactly it matched. + /// The input string. + /// The start position in the input. + /// The end position in the input. + public abstract SymbolicMatch FindMatch(bool isMatch, string input, int startat, int endat); + +#if DEBUG + /// Unwind the regex of the matcher and save the resulting state graph in DGML + /// roughly the maximum number of states, 0 means no bound + /// if true then hide state info + /// if true then pretend that there is a .* at the beginning + /// if true then unwind the regex backwards (addDotStar is then ignored) + /// if true then compute and save only genral DFA info + /// dgml output is written here + /// maximum length of labels in nodes anything over that length is indicated with .. + /// if true creates NFA instead of DFA + public abstract void SaveDGML(TextWriter writer, int bound, bool hideStateInfo, bool addDotStar, bool inReverse, bool onlyDFAinfo, int maxLabelLength, bool asNFA); + + + /// + /// Generates up to k random strings matched by the regex + /// + /// upper bound on the number of generated strings + /// random seed for the generator, 0 means no random seed + /// if true then generate inputs that do not match + /// + public abstract IEnumerable GenerateRandomMembers(int k, int randomseed, bool negative); +#endif + } + + /// Represents a regex matching engine that performs regex matching using symbolic derivatives. + /// Character set type. + internal sealed class SymbolicRegexMatcher : SymbolicRegexMatcher where TSetType : notnull + { + /// Maximum number of states before switching over to Antimirov mode. + /// + /// "Brzozowski" is by default used for state graphs that represent the DFA nodes for the regex. + /// In this mode, for the singular state we're currently in, we can evaluate the next character and determine + /// the singular next state to be in. Some regular expressions, however, can result in really, really large DFA + /// state graphs. Instead of falling over with large representations, after this (somewhat arbitrary) threshold, + /// the implementation switches to "Antimirov" mode. In this mode, which can be thought of as NFA-based instead + /// of DFA-based, we can be in any number of states at the same time, represented as a + /// that's the union of all such states; transitioning based on the next character is then handled by finding + /// all possible states we might transition to from each of the states in the current set, and producing a new set + /// that's the union of all of those. The matching engine switches dynamically from Brzozowski to Antimirov once + /// it trips over this threshold in the size of the state graph, which may be produced lazily. + /// + private const int AntimirovThreshold = 10_000; + + /// Wiggle room around the exact size of the state graph before we switch to Antimirov. + /// + /// The inner loop of the matching engine wants to be as streamlined as possible, and in Brzozowski mode + /// having to check at every iteration whether we need to switch to Antimirov is a perf bottleneck. As such, + /// the inner loop is allowed to run for up to this many transitions without regard for the size of the + /// graph, which could cause to be exceeded by this limit. Once + /// we've hit number of transitions in the inner loop, we + /// force ourselves back out to the outerloop, where we can check the graph size and other things like timeouts. + /// + private const int AntimirovThresholdLeeway = 1_000; + + /// Sentinel value used internally by the matcher to indicate no match exists. + private const int NoMatchExists = -2; + + /// Builder used to create s while matching. + /// + /// The builder servers two purposes: + /// 1. For Brzozowski, we build up the DFA state space lazily, which means we need to be able to + /// produce new s as we match. + /// 2. For Antimirov, currently the list of states we're currently in is represented as a + /// that's a union of all current states. Augmenting that list requires building new nodes. + /// The builder maintains a cache of nodes, and requests for it to make new ones might return existing ones from the cache. + /// The separation of a matcher and a builder is somewhat arbitrary; they could potentially be combined. + /// + internal readonly SymbolicRegexBuilder _builder; + + /// Maps each character into a partition id in the range 0..K-1. + private readonly MintermClassifier _partitions; + + /// prefixed with [0-0xFFFF]* + /// + /// The matching engine first uses to find whether there is a match + /// and where that match might end. Prepending the .* prefix onto the original pattern provides the DFA + /// with the ability to continue to process input characters even if those characters aren't part of + /// the match. If Regex.IsMatch is used, nothing further is needed beyond this prefixed pattern. If, however, + /// other matching operations are performed that require knowing the exact start and end of the match, + /// the engine then needs to process the pattern in reverse to find where the match actually started; + /// for that, it uses the and walks backwards through the input characters + /// from where left off. At this point we know that there was a match, + /// where it started, and where it could have ended, but that ending point could be influenced by the + /// selection of the starting point. So, to find the actual ending point, the original + /// is then used from that starting point to walk forward through the input characters again to find the + /// actual end point used for the match. + /// + internal readonly SymbolicRegexNode _dotStarredPattern; + + /// The original regex pattern. + internal readonly SymbolicRegexNode _pattern; + + /// The reverse of . + /// + /// Determining that there is a match and where the match ends requires only . + /// But from there determining where the match began requires reversing the pattern and running + /// the matcher again, starting from the ending position. This caches + /// that reversed pattern used for extracting match start. + /// + internal readonly SymbolicRegexNode _reversePattern; + + /// true iff timeout checking is enabled. + private readonly bool _checkTimeout; + + /// Timeout in milliseconds. This is only used if is true. + private readonly int _timeout; + + /// Classifier used to say whether a particular character can start a match for . + internal readonly BooleanClassifier _startSetClassifier; + + /// Predicate over characters that make some progress + private readonly TSetType _startSet; + + /// Maximum allowed size of . + private const int StartSetArrayMaxSize = 5; + + /// String of at most many characters + private readonly char[] _startSetArray; + + /// Number of elements in + private readonly int _startSetSize; + + /// If nonempty then has that fixed prefix + private readonly string _prefix; + + /// Non-null when is nonempty + private readonly RegexBoyerMoore? _prefixBoyerMoore; + + /// If true then the fixed prefix of is idependent of case + private readonly bool _isPrefixCaseInsensitive; + + /// Cached skip states from the initial state of for the 5 possible previous character kinds. + private readonly DfaMatchingState?[] _prefixSkipStates = new DfaMatchingState[CharKind.CharKindCount]; + /// Cached skip states from the initial state of Ar for the 5 possible previous character kinds. + private readonly DfaMatchingState?[] _reversePrefixSkipStates = new DfaMatchingState[CharKind.CharKindCount]; + + private readonly string _reversePrefix; + + private readonly DfaMatchingState[] _initialStates = new DfaMatchingState[CharKind.CharKindCount]; + private readonly DfaMatchingState[] _dotstarredInitialStates = new DfaMatchingState[CharKind.CharKindCount]; + private readonly DfaMatchingState[] _reverseInitialStates = new DfaMatchingState[CharKind.CharKindCount]; + + private readonly uint[] _asciiCharKinds = new uint[128]; + + internal readonly CultureInfo _culture; + + private DfaMatchingState GetSkipState(uint prevCharKind) => + Volatile.Read(ref _prefixSkipStates[prevCharKind]) ?? + Interlocked.CompareExchange(ref _prefixSkipStates[prevCharKind], DeltaPlus(_prefix, _dotstarredInitialStates[prevCharKind]), null) ?? + _prefixSkipStates[prevCharKind]!; + + private DfaMatchingState GetReverseSkipState(uint prevCharKind) => + Volatile.Read(ref _reversePrefixSkipStates[prevCharKind]) ?? + Interlocked.CompareExchange(ref _reversePrefixSkipStates[prevCharKind], DeltaPlus(_reversePrefix, _reverseInitialStates[prevCharKind]), null) ?? + _reversePrefixSkipStates[prevCharKind]!; + + /// Get the minterm of . + /// character code + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private TSetType GetMinterm(int c) + { + Debug.Assert(_builder._minterms is not null); + return _builder._minterms[_partitions.GetMintermID(c)]; + } + + /// Constructs matcher for given symbolic regex. + internal SymbolicRegexMatcher(SymbolicRegexNode sr, CharSetSolver css, BDD[] minterms, TimeSpan matchTimeout, CultureInfo culture) + { + _pattern = sr; + _builder = sr._builder; + + _checkTimeout = Regex.InfiniteMatchTimeout != matchTimeout; + _timeout = (int)(matchTimeout.TotalMilliseconds + 0.5); // Round up, so it will be at least 1ms + _culture = culture; + + Debug.Assert(_builder._solver is BV64Algebra or BVAlgebra or CharSetSolver, $"Unsupported algebra: {_builder._solver}"); + _partitions = _builder._solver switch + { + BV64Algebra bv64 => bv64._classifier, + BVAlgebra bv => bv._classifier, + _ => new MintermClassifier((CharSetSolver)(object)_builder._solver, minterms), + }; + + _dotStarredPattern = _builder.MkConcat(_builder._anyStar, _pattern); + _reversePattern = _pattern.Reverse(); + ConfigureRegexes(); + + _startSet = _pattern.GetStartSet(); + if (!_builder._solver.IsSatisfiable(_startSet) || _pattern.CanBeNullable) + { + // If the startset is empty make it full instead by including all characters + // this is to ensure that startset is nonempty -- as an invariant assumed by operations using it + // + // Also, if A can be nullable then effectively disable use of startset by making it true + // because it may force search of next character in startset and fail to recognize an empty match + // because (by definition) an empty match has no start character. + // + // For example (this is also a unit test): + // for pattern "\B\W*?" or "\B\W*" or "\B\W?" and input "e.g:abc" there is an empty match in position 5 + // but startset \W will force search beyond position 5 and fails to find that match + _startSet = _builder._solver.True; + } + + _startSetSize = (int)_builder._solver.ComputeDomainSize(_startSet); + + BDD startbdd = _builder._solver.ConvertToCharSet(css, _startSet); + _startSetClassifier = new BooleanClassifier(css, startbdd); + + //store the start characters in the A_startset_array if there are not too many characters + _startSetArray = _startSetSize <= StartSetArrayMaxSize ? + new List(css.GenerateAllCharacters(startbdd)).ToArray() : + Array.Empty(); + + _prefix = _pattern.GetFixedPrefix(css, culture.Name, out _isPrefixCaseInsensitive); + _reversePrefix = _reversePattern.GetFixedPrefix(css, culture.Name, out _); + + _prefixBoyerMoore = InitializePrefixBoyerMoore(); + + if (_pattern._info.ContainsSomeAnchor) + { + for (int i = 0; i < 128; i++) + { + TSetType predicate2; + uint charKind; + + if (i == '\n') + { + predicate2 = _builder._newLinePredicate; + charKind = CharKind.Newline; + } + else + { + predicate2 = _builder._wordLetterPredicateForAnchors; + charKind = CharKind.WordLetter; + } + + _asciiCharKinds[i] = _builder._solver.And(GetMinterm(i), predicate2).Equals(_builder._solver.False) ? 0 : charKind; + } + } + } + + private RegexBoyerMoore? InitializePrefixBoyerMoore() + { + if (_prefix != string.Empty && _prefix.Length <= RegexBoyerMoore.MaxLimit && _prefix.Length > 1) + { + // RegexBoyerMoore expects the prefix to be lower case when case is ignored. + // Use the culture of the matcher. + string prefix = _isPrefixCaseInsensitive ? _prefix.ToLower(_culture) : _prefix; + return new RegexBoyerMoore(prefix, _isPrefixCaseInsensitive, rightToLeft: false, _culture); + } + + return null; + } + + private void ConfigureRegexes() + { + void Configure(uint i) + { + _initialStates[i] = _builder.MkState(_pattern, i); + + // Used to detect if initial state was reentered, then startset can be triggered + // but observe that the behavior from the state may ultimately depend on the previous + // input char e.g. possibly causing nullability of \b or \B or of a start-of-line anchor, + // in that sense there can be several "versions" (not more than StateCount) of the initial state. + _dotstarredInitialStates[i] = _builder.MkState(_dotStarredPattern, i); + _dotstarredInitialStates[i].IsInitialState = true; + + _reverseInitialStates[i] = _builder.MkState(_reversePattern, i); + } + + // Create initial states for A, A1 and Ar. + if (!_pattern._info.ContainsSomeAnchor) + { + // Only the default previous character kind 0 is ever going to be used for all initial states. + // _A1q0[0] is recognized as special initial state. + // This information is used for search optimization based on start set and prefix of A. + Configure(0); + } + else + { + for (uint i = 0; i < CharKind.CharKindCount; i++) + { + Configure(i); + } + } + } + + /// Return the state after the given string from the given state . + private DfaMatchingState DeltaPlus(string pattern, DfaMatchingState state) where TTransition : struct, ITransition + { + for (int i = 0; i < pattern.Length; i++) + { + state = Delta(pattern, i, state); + } + + return state; + } + + /// Interface for transitions used by the method. + private interface ITransition + { + /// Find the next state given the current state and next character. + DfaMatchingState TakeTransition(SymbolicRegexMatcher matcher, DfaMatchingState currentState, int mintermId, TSetType minterm); + } + + /// Compute the target state for the source state and input[i] character. + /// input string + /// The index into at which the target character lives. + /// The source state + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private DfaMatchingState Delta(string input, int i, DfaMatchingState sourceState) where TTransition : struct, ITransition + { + TSetType[]? minterms = _builder._minterms; + Debug.Assert(minterms is not null); + + int c = input[i]; + + int mintermId = c == '\n' && i == input.Length - 1 && sourceState.StartsWithLineAnchor ? + minterms.Length : // mintermId = minterms.Length represents \Z (last \n) + _partitions.GetMintermID(c); + + TSetType minterm = (uint)mintermId < minterms.Length ? + minterms[mintermId] : + _builder._solver.False; // minterm=False represents \Z + + return default(TTransition).TakeTransition(this, sourceState, mintermId, minterm); + } + + /// Transition for Brzozowski-style derivatives (i.e. a DFA). + private readonly struct BrzozowskiTransition : ITransition + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public DfaMatchingState TakeTransition( + SymbolicRegexMatcher matcher, DfaMatchingState currentState, int mintermId, TSetType minterm) + { + SymbolicRegexBuilder builder = matcher._builder; + Debug.Assert(builder._delta is not null); + + int offset = (currentState.Id << builder._mintermsCount) | mintermId; + return + builder._delta[offset] ?? + matcher.CreateNewTransition(currentState, minterm, offset); + } + } + + /// Transition for Antimirov-style derivatives (i.e. an NFA). + private readonly struct AntimirovTransition : ITransition + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public DfaMatchingState TakeTransition( + SymbolicRegexMatcher matcher, DfaMatchingState currentStates, int mintermId, TSetType minterm) + { + if (currentStates.Node.Kind != SymbolicRegexKind.Or) + { + // Fall back to Brzozowski when the state is not a disjunction. + return default(BrzozowskiTransition).TakeTransition(matcher, currentStates, mintermId, minterm); + } + + SymbolicRegexBuilder builder = matcher._builder; + Debug.Assert(builder._delta is not null); + + SymbolicRegexNode union = builder._nothing; + uint kind = 0; + + // Produce the new list of states from the current list, considering transitions from members one at a time. + Debug.Assert(currentStates.Node._alts is not null); + foreach (SymbolicRegexNode oneState in currentStates.Node._alts) + { + DfaMatchingState nextStates = builder.MkState(oneState, currentStates.PrevCharKind); + + int offset = (nextStates.Id << builder._mintermsCount) | mintermId; + DfaMatchingState p = builder._delta[offset] ?? matcher.CreateNewTransition(nextStates, minterm, offset); + + // Observe that if p.Node is an Or it will be flattened. + union = builder.MkOr2(union, p.Node); + + // kind is just the kind of the partition. + kind = p.PrevCharKind; + } + + return builder.MkState(union, kind, true); + } + } + + /// Critical region for defining a new transition + private DfaMatchingState CreateNewTransition(DfaMatchingState state, TSetType minterm, int offset) + { + Debug.Assert(_builder._delta is not null); + lock (this) + { + // check if meanwhile delta[offset] has become defined possibly by another thread + DfaMatchingState p = _builder._delta[offset]; + if (p is null) + { + // this is the only place in code where the Next method is called in the matcher + _builder._delta[offset] = p = state.Next(minterm); + + // switch to antimirov mode if the maximum bound has been reached + if (p.Id == AntimirovThreshold) + { + _builder._antimirov = true; + } + } + + return p; + } + } + + private void DoCheckTimeout(int timeoutOccursAt) + { + // This code is identical to RegexRunner.DoCheckTimeout(), + // with the exception of check skipping. RegexRunner calls + // DoCheckTimeout potentially on every iteration of a loop, + // whereas this calls it only once per transition. + + int currentMillis = Environment.TickCount; + + if (currentMillis < timeoutOccursAt) + return; + + if (0 > timeoutOccursAt && 0 < currentMillis) + return; + + //regex pattern is in general not available in srm and + //the input is not available here but could be passed as argument to DoCheckTimeout + throw new RegexMatchTimeoutException(string.Empty, string.Empty, TimeSpan.FromMilliseconds(_timeout)); + } + + /// Find a match. + /// Whether to return once we know there's a match without determining where exactly it matched. + /// input string + /// the position to start search in the input string + /// the next position after the end position in the input + public override SymbolicMatch FindMatch(bool isMatch, string input, int startat, int k) + { + int timeoutOccursAt = 0; + if (_checkTimeout) + { + // Using Environment.TickCount for efficiency instead of Stopwatch -- as in the non-DFA case. + timeoutOccursAt = Environment.TickCount + (int)(_timeout + 0.5); + } + + if (startat == k) + { + //covers the special case when the remaining input suffix + //where a match is sought is empty (for example when the input is empty) + //in this case the only possible match is an empty match + uint prevKind = GetCharKind(input, startat - 1); + uint nextKind = GetCharKind(input, startat); + + bool emptyMatchExists = _pattern.IsNullableFor(CharKind.Context(prevKind, nextKind)); + return + !emptyMatchExists ? SymbolicMatch.NoMatch : + new SymbolicMatch(startat, 0); + } + + // Find the first accepting state. Initial start position in the input is i == 0. + int i = startat; + + // May return -1 as a legitimate value when the initial state is nullable and startat == 0. + // Returns NoMatchExists when there is no match. + i = FindFinalStatePosition(input, k, i, timeoutOccursAt, out int i_q0_A1, out int watchdog); + + if (i == NoMatchExists) + { + return SymbolicMatch.NoMatch; + } + + if (isMatch) + { + // this means success -- the original call was IsMatch + return SymbolicMatch.QuickMatch; + } + + int i_start; + int i_end; + + if (watchdog >= 0) + { + i_start = i - watchdog + 1; + i_end = i; + } + else + { + if (i < startat) + { + Debug.Assert(i == startat - 1); + i_start = startat; + } + else + { + // Walk in reverse to locate the start position of the match + i_start = FindStartPosition(input, i, i_q0_A1); + } + + i_end = FindEndPosition(input, k, i_start); + } + + return new SymbolicMatch(i_start, i_end + 1 - i_start); + } + + /// Find match end position using A, end position is known to exist. + /// input array + /// inclusive start position + /// exclusive end position + /// + private int FindEndPosition(string input, int exclusiveEnd, int i) + { + int i_end = exclusiveEnd; + + // Pick the correct start state based on previous character kind. + uint prevCharKind = GetCharKind(input, i - 1); + DfaMatchingState state = _initialStates[prevCharKind]; + + if (state.IsNullable(GetCharKind(input, i))) + { + // Empty match exists because the initial state is accepting. + i_end = i - 1; + + // Stop here if q is lazy. + if (state.IsLazy) + { + return i_end; + } + } + + while (i < exclusiveEnd) + { + int j = Math.Min(exclusiveEnd, i + AntimirovThresholdLeeway); + bool done = _builder._antimirov ? + FindEndPositionDeltas(input, ref i, j, ref state, ref i_end) : + FindEndPositionDeltas(input, ref i, j, ref state, ref i_end); + + if (done) + { + break; + } + } + + Debug.Assert(i_end != exclusiveEnd); + return i_end; + } + + // Inner loop for FindEndPosition parameterized by an ITransition type. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool FindEndPositionDeltas(string input, ref int i, int j, ref DfaMatchingState q, ref int i_end) where TTransition : struct, ITransition + { + do + { + q = Delta(input, i, q); + + if (q.IsNullable(GetCharKind(input, i + 1))) + { + // Accepting state has been reached. Record the position. + i_end = i; + + // Stop here if q is lazy. + if (q.IsLazy) + { + return true; + } + } + else if (q.IsDeadend) + { + // Nonaccepting sink state (deadend) has been reached in A. + // So the match ended when the last i_end was updated. + return true; + } + + i++; + } + while (i < j); + + return false; + } + + /// Walk back in reverse using Ar to find the start position of match, start position is known to exist. + /// the input string + /// position to start walking back from, i points at the last character of the match + /// do not pass this boundary when walking back + /// + private int FindStartPosition(string input, int i, int match_start_boundary) + { + // Fetch the correct start state for Ar. + // This depends on previous character --- which, because going backwards, is character number i+1. + uint prevKind = GetCharKind(input, i + 1); + DfaMatchingState q = _reverseInitialStates[prevKind]; + + // Ar may have a fixed prefix sequence + if (_reversePrefix.Length > 0) + { + //skip past the prefix portion of Ar + q = GetReverseSkipState(prevKind); + i -= _reversePrefix.Length; + } + + if (i == -1) + { + Debug.Assert(q.IsNullable(GetCharKind(input, i)), "we reached the beginning of the input, thus the state q must be accepting"); + return 0; + } + + int last_start = -1; + if (q.IsNullable(GetCharKind(input, i))) + { + // The whole prefix of Ar was in reverse a prefix of A, + // for example when the pattern of A is concrete word such as "abc" + last_start = i + 1; + } + + //walk back to the accepting state of Ar + while (i >= match_start_boundary) + { + int j = Math.Max(match_start_boundary, i - AntimirovThresholdLeeway); + bool done = _builder._antimirov ? + FindStartPositionDeltas(input, ref i, j, ref q, ref last_start) : + FindStartPositionDeltas(input, ref i, j, ref q, ref last_start); + + if (done) + { + break; + } + } + + Debug.Assert(last_start != -1); + return last_start; + } + + // Inner loop for FindStartPosition parameterized by an ITransition type. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool FindStartPositionDeltas(string input, ref int i, int j, ref DfaMatchingState q, ref int last_start) where TTransition : struct, ITransition + { + do + { + q = Delta(input, i, q); + + // Reached a deadend state, thus the earliest match start point must have occurred already. + if (q.IsNothing) + { + return true; + } + + if (q.IsNullable(GetCharKind(input, i - 1))) + { + // Earliest start point so far. This must happen at some point + // or else A1 would not have reached a final state after match_start_boundary. + last_start = i; + } + + i -= 1; + } + while (i > j); + + return false; + } + + /// Returns NoMatchExists if no match exists. Returns -1 when i=0 and the initial state is nullable. + /// given input string + /// input length or bounded input length + /// start position + /// The time at which timeout occurs, if timeouts are being checked. + /// last position the initial state of was visited + /// length of match when positive + private int FindFinalStatePosition(string input, int k, int i, int timeoutOccursAt, out int initialStateIndex, out int watchdog) + { + // Get the correct start state of A1, which in general depends on the previous character kind in the input. + uint prevCharKindId = GetCharKind(input, i - 1); + DfaMatchingState q = _dotstarredInitialStates[prevCharKindId]; + initialStateIndex = i; + + if (q.IsNothing) + { + // If q is nothing then it is a deadend from the beginning this happens for example when the original + // regex started with start anchor and prevCharKindId is not Start + watchdog = -1; + return NoMatchExists; + } + + if (q.IsNullable(GetCharKind(input, i))) + { + // The initial state is nullable in this context so at least an empty match exists. + // The last position of the match is i-1 because the match is empty. + // This value is -1 if i == 0. + watchdog = -1; + return i - 1; + } + + watchdog = -1; + + // Search for a match end position within input[i..k-1] + while (i < k) + { + if (q.IsInitialState) + { + // i_q0_A1 is the most recent position in the input when A1 is in the initial state + initialStateIndex = i; + + if (_prefixBoyerMoore != null) + { + // Stay in the initial state if the prefix does not match. + // Thus advance the current position to the first position where the prefix does match. + i = _prefixBoyerMoore.Scan(input, i, 0, input.Length); + + if (i == -1) // Scan returns -1 when a matching position does not exist + { + watchdog = -1; + return -2; + } + + // Compute the end state for the A prefix. + // Skip directly to the resulting state + // --- i.e. do the loop --- + // for (int j = 0; j < prefix.Length; j++) + // q = Delta(prefix[j], q, out regex); + // --- + q = GetSkipState(q.PrevCharKind); + + // skip the prefix + i += _prefix.Length; + + // here i points at the next character (the character immediately following the prefix) + if (q.IsNullable(GetCharKind(input, i))) + { + // Return the last position of the match + watchdog = q.WatchDog; + return i - 1; + } + + if (i == k) + { + // no match was found + return -2; + } + } + else + { + // we are still in the initial state, when the prefix is empty + // find the first position i that matches with some character in the start set + i = IndexOfStartSet(input, i); + + if (i == -1) + { + // no match was found + return NoMatchExists; + } + + initialStateIndex = i; + + // the start state must be updated + // to reflect the kind of the previous character + // when anchors are not used, q will remain the same state + q = _dotstarredInitialStates[GetCharKind(input, i - 1)]; + if (q.IsNothing) + { + return NoMatchExists; + } + } + } + + int result; + int j = Math.Min(k, i + AntimirovThresholdLeeway); + bool done = _builder._antimirov ? + FindFinalStatePositionDeltas(input, j, ref i, ref q, ref watchdog, out result) : + FindFinalStatePositionDeltas(input, j, ref i, ref q, ref watchdog, out result); + + if (done) + { + return result; + } + + if (_checkTimeout) + { + DoCheckTimeout(timeoutOccursAt); + } + } + + //no match was found + return NoMatchExists; + } + + /// Inner loop for FindFinalStatePosition parameterized by an ITransition type. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool FindFinalStatePositionDeltas(string input, int j, ref int i, ref DfaMatchingState q, ref int watchdog, out int result) where TTransition : struct, ITransition + { + do + { + // Make the transition based on input[i]. + q = Delta(input, i, q); + + if (q.IsNullable(GetCharKind(input, i + 1))) + { + watchdog = q.WatchDog; + result = i; + return true; + } + + if (q.IsNothing) + { + // q is a deadend state so any further search is meaningless + result = NoMatchExists; + return true; + } + + // continue from the next character + i++; + } + while (i < j && !q.IsInitialState); + + result = 0; + return false; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private uint GetCharKind(string input, int i) + { + if (!_pattern._info.ContainsSomeAnchor) + { + // The previous character kind is irrelevant when anchors are not used. + return CharKind.General; + } + + if (i == -1 || i == input.Length) + { + return CharKind.StartStop; + } + + char nextChar = input[i]; + if (nextChar == '\n') + { + return + _builder._newLinePredicate.Equals(_builder._solver.False) ? 0 : // ignore \n + i == 0 || i == input.Length - 1 ? CharKind.NewLineS : // very first or very last \n. Detection of very first \n is needed for rev(\Z). + CharKind.Newline; + } + + uint[] asciiCharKinds = _asciiCharKinds; + return + nextChar < asciiCharKinds.Length ? asciiCharKinds[nextChar] : + _builder._solver.And(GetMinterm(nextChar), _builder._wordLetterPredicateForAnchors).Equals(_builder._solver.False) ? 0 : //apply the wordletter predicate to compute the kind of the next character + CharKind.WordLetter; + } + + /// + /// Find first occurrence of startset element in input starting from index i. + /// Startset here is assumed to consist of a few characters. + /// + /// input string to search in + /// the start index in input to search from + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private int IndexOfStartSet(string input, int i) + { + if (_startSetSize <= StartSetArrayMaxSize) + { + return input.IndexOfAny(_startSetArray, i); + } + + for (int j = i; j < input.Length; j++) + { + if (_startSetClassifier.IsTrue(input[j])) + { + return j; + } + } + + return -1; + } + +#if DEBUG + public override void SaveDGML(TextWriter writer, int bound, bool hideStateInfo, bool addDotStar, bool inReverse, bool onlyDFAinfo, int maxLabelLength, bool asNFA) + { + var graph = new DGML.RegexAutomaton(this, bound, addDotStar, inReverse, asNFA); + var dgml = new DGML.DgmlWriter(writer, hideStateInfo, maxLabelLength, onlyDFAinfo); + dgml.Write(graph); + } + + public override IEnumerable GenerateRandomMembers(int k, int randomseed, bool negative) => + new SymbolicRegexSampler(_pattern, randomseed, negative).GenerateRandomMembers(k); +#endif + } +} diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicRegexNode.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicRegexNode.cs new file mode 100644 index 00000000000000..fbd1cbda6ee753 --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicRegexNode.cs @@ -0,0 +1,1740 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; + +namespace System.Text.RegularExpressions.Symbolic +{ + /// Represents an AST node of a symbolic regex. + internal sealed class SymbolicRegexNode where S : notnull + { + internal const string EmptyCharClass = "[]"; + + internal readonly SymbolicRegexBuilder _builder; + internal readonly SymbolicRegexKind _kind; + internal readonly int _lower; + internal readonly int _upper; + internal readonly S? _set; + internal readonly SymbolicRegexNode? _left; + internal readonly SymbolicRegexNode? _right; + internal readonly SymbolicRegexSet? _alts; + + private Dictionary? _nullabilityCache; + + private S _startSet; + + /// AST node of a symbolic regex + /// the builder + /// what kind of node + /// left child + /// right child + /// lower bound of a loop + /// upper boubd of a loop + /// singelton set + /// alternatives set of a disjunction or conjunction + /// misc flags including laziness + private SymbolicRegexNode(SymbolicRegexBuilder builder, SymbolicRegexKind kind, SymbolicRegexNode? left, SymbolicRegexNode? right, int lower, int upper, S? set, SymbolicRegexSet? alts, SymbolicRegexInfo info) + { + _builder = builder; + _kind = kind; + _left = left; + _right = right; + _lower = lower; + _upper = upper; + _set = set; + _alts = alts; + _info = info; + _hashcode = ComputeHashCode(); + _startSet = ComputeStartSet(); + } + + private bool _isInternalizedUnion; + + /// Create a new node or retrieve one from the builder _nodeCache + private static SymbolicRegexNode Create(SymbolicRegexBuilder builder, SymbolicRegexKind kind, SymbolicRegexNode? left, SymbolicRegexNode? right, int lower, int upper, S? set, SymbolicRegexSet? alts, SymbolicRegexInfo info) + { + SymbolicRegexNode? node; + var key = (kind, left, right, lower, upper, set, alts, info); + if (!builder._nodeCache.TryGetValue(key, out node)) + { + // Do not internalize top level Or-nodes or else Antimirov mode will become ineffective + if (kind == SymbolicRegexKind.Or) + { + node = new(builder, kind, left, right, lower, upper, set, alts, info); + return node; + } + + left = left == null || left._kind != SymbolicRegexKind.Or || left._isInternalizedUnion ? left : Internalize(left); + right = right == null || right._kind != SymbolicRegexKind.Or || right._isInternalizedUnion ? right : Internalize(right); + + node = new(builder, kind, left, right, lower, upper, set, alts, info); + builder._nodeCache[key] = node; + } + + Debug.Assert(node is not null); + return node; + } + + /// Internalize an Or-node that is not yet internalized + private static SymbolicRegexNode Internalize(SymbolicRegexNode node) + { + Debug.Assert(node._kind == SymbolicRegexKind.Or && !node._isInternalizedUnion); + + (SymbolicRegexKind, SymbolicRegexNode?, SymbolicRegexNode?, int, int, S?, SymbolicRegexSet?, SymbolicRegexInfo) node_key = + (SymbolicRegexKind.Or, null, null, -1, -1, default(S), node._alts, node._info); + SymbolicRegexNode? node1; + if (node._builder._nodeCache.TryGetValue(node_key, out node1)) + { + Debug.Assert(node1 is not null && node1._isInternalizedUnion); + return node1; + } + else + { + node._isInternalizedUnion = true; + node._builder._nodeCache[node_key] = node; + return node; + } + } + + /// True if this node only involves lazy loops + internal bool IsLazy => _info.IsLazy; + + /// True if this node accepts the empty string unconditionally. + internal bool IsNullable => _info.IsNullable; + + /// True if this node can potentially accept the empty string depending on anchors and immediate context. + internal bool CanBeNullable + { + get + { + Debug.Assert(_info.CanBeNullable || !_info.IsNullable); + return _info.CanBeNullable; + } + } + + internal SymbolicRegexInfo _info; + + private readonly int _hashcode; + + + /// Converts a concatenation into an array, returns a non-concatenation in a singleton array. + public List> ToList() + { + var list = new List>(); + AppendToList(this, list); + return list; + + static void AppendToList(SymbolicRegexNode concat, List> list) + { + SymbolicRegexNode node = concat; + while (node._kind == SymbolicRegexKind.Concat) + { + Debug.Assert(node._left is not null && node._right is not null); + if (node._left._kind == SymbolicRegexKind.Concat) + { + AppendToList(node._left, list); + } + else + { + list.Add(node._left); + } + node = node._right; + } + + list.Add(node); + } + } + + /// + /// Relative nullability that takes into account the immediate character context + /// in order to resolve nullability of anchors + /// + /// kind info for previous and next characters + internal bool IsNullableFor(uint context) + { + if (!_info.StartsWithSomeAnchor) + return IsNullable; + + if (!_info.CanBeNullable) + return false; + + // Initialize the nullability cache for this node. + _nullabilityCache ??= new Dictionary(); + + if (!_nullabilityCache.TryGetValue(context, out bool is_nullable)) + { + switch (_kind) + { + case SymbolicRegexKind.Loop: + Debug.Assert(_left is not null); + is_nullable = _lower == 0 || _left.IsNullableFor(context); + break; + + case SymbolicRegexKind.Concat: + Debug.Assert(_left is not null && _right is not null); + is_nullable = _left.IsNullableFor(context) && _right.IsNullableFor(context); + break; + + case SymbolicRegexKind.Or: + case SymbolicRegexKind.And: + Debug.Assert(_alts is not null); + is_nullable = _alts.IsNullableFor(context); + break; + + case SymbolicRegexKind.Not: + Debug.Assert(_left is not null); + is_nullable = !_left.IsNullableFor(context); + break; + + case SymbolicRegexKind.StartAnchor: + is_nullable = CharKind.Prev(context) == CharKind.StartStop; + break; + + case SymbolicRegexKind.EndAnchor: + is_nullable = CharKind.Next(context) == CharKind.StartStop; + break; + + case SymbolicRegexKind.BOLAnchor: + // Beg-Of-Line anchor is nullable when the previous character is Newline or Start + // note: at least one of the bits must be 1, but both could also be 1 in case of very first newline + is_nullable = (CharKind.Prev(context) & CharKind.NewLineS) != 0; + break; + + case SymbolicRegexKind.EOLAnchor: + // End-Of-Line anchor is nullable when the next character is Newline or Stop + // note: at least one of the bits must be 1, but both could also be 1 in case of \Z + is_nullable = (CharKind.Next(context) & CharKind.NewLineS) != 0; + break; + + case SymbolicRegexKind.WBAnchor: + // test that prev char is word letter iff next is not not word letter + is_nullable = ((CharKind.Prev(context) & CharKind.WordLetter) ^ (CharKind.Next(context) & CharKind.WordLetter)) != 0; + break; + + case SymbolicRegexKind.NWBAnchor: + // test that prev char is word letter iff next is word letter + is_nullable = ((CharKind.Prev(context) & CharKind.WordLetter) ^ (CharKind.Next(context) & CharKind.WordLetter)) == 0; + break; + + case SymbolicRegexKind.EndAnchorZ: + // \Z anchor is nullable when the next character is either the last Newline or Stop + // note: CharKind.NewLineS == CharKind.Newline|CharKind.StartStop + is_nullable = (CharKind.Next(context) & CharKind.StartStop) != 0; + break; + + default: //SymbolicRegexKind.EndAnchorZRev: + // EndAnchorZRev (rev(\Z)) anchor is nullable when the prev character is either the first Newline or Start + // note: CharKind.NewLineS == CharKind.Newline|CharKind.StartStop + Debug.Assert(_kind == SymbolicRegexKind.EndAnchorZRev); + is_nullable = (CharKind.Prev(context) & CharKind.StartStop) != 0; + break; + } + + _nullabilityCache[context] = is_nullable; + } + + return is_nullable; + } + + /// Returns true if this is equivalent to .* (the node must be eager also) + public bool IsAnyStar + { + get + { + if (IsStar) + { + Debug.Assert(_left is not null); + if (_left._kind == SymbolicRegexKind.Singleton) + { + Debug.Assert(_left._set is not null); + return !IsLazy && _builder._solver.True.Equals(_left._set); + } + } + + return false; + } + } + + /// Returns true if this is equivalent to .+ (the node must be eager also) + public bool IsAnyPlus + { + get + { + if (IsPlus) + { + Debug.Assert(_left is not null); + if (_left._kind == SymbolicRegexKind.Singleton) + { + Debug.Assert(_left._set is not null); + return !IsLazy && _builder._solver.True.Equals(_left._set); + } + } + + return false; + } + } + + /// Returns true if this is equivalent to [\0-\xFFFF] + public bool IsAnyChar + { + get + { + if (_kind == SymbolicRegexKind.Singleton) + { + Debug.Assert(_set is not null); + return _builder._solver.AreEquivalent(_builder._solver.True, _set); + } + + return false; + } + } + + /// Returns true if this is equivalent to [0-[0]] + public bool IsNothing + { + get + { + if (_kind == SymbolicRegexKind.Singleton) + { + Debug.Assert(_set is not null); + return !_builder._solver.IsSatisfiable(_set); + } + + return false; + } + } + + /// Returns true iff this is a loop whose lower bound is 0 and upper bound is max + public bool IsStar => _lower == 0 && _upper == int.MaxValue; + + /// Returns true iff this is a loop whose lower bound is 0 and upper bound is 1 + public bool IsMaybe => _lower == 0 && _upper == 1; + + /// Returns true if this is Epsilon + public bool IsEpsilon => _kind == SymbolicRegexKind.Epsilon; + + /// Gets the kind of the regex + internal SymbolicRegexKind Kind => _kind; + + /// + /// Returns true iff this is a loop whose lower bound is 1 and upper bound is max + /// + public bool IsPlus => _lower == 1 && _upper == int.MaxValue; + + #region called only once, in the constructor of SymbolicRegexBuilder + + internal static SymbolicRegexNode MkFalse(SymbolicRegexBuilder builder) => + Create(builder, SymbolicRegexKind.Singleton, null, null, -1, -1, builder._solver.False, null, SymbolicRegexInfo.Mk()); + + internal static SymbolicRegexNode MkTrue(SymbolicRegexBuilder builder) => + Create(builder, SymbolicRegexKind.Singleton, null, null, -1, -1, builder._solver.True, null, SymbolicRegexInfo.Mk(containsSomeCharacter: true)); + + internal static SymbolicRegexNode MkWatchDog(SymbolicRegexBuilder builder, int length) => + Create(builder, SymbolicRegexKind.WatchDog, null, null, length, -1, default, null, SymbolicRegexInfo.Mk(isAlwaysNullable: true)); + + internal static SymbolicRegexNode MkEpsilon(SymbolicRegexBuilder builder) => + Create(builder, SymbolicRegexKind.Epsilon, null, null, -1, -1, default, null, SymbolicRegexInfo.Mk(isAlwaysNullable: true)); + + internal static SymbolicRegexNode MkEagerEmptyLoop(SymbolicRegexBuilder builder, SymbolicRegexNode body) => + Create(builder, SymbolicRegexKind.Loop, body, null, 0, 0, default, null, SymbolicRegexInfo.Mk(isAlwaysNullable: true, isLazy: false)); + + internal static SymbolicRegexNode MkStartAnchor(SymbolicRegexBuilder builder) => + Create(builder, SymbolicRegexKind.StartAnchor, null, null, -1, -1, default, null, SymbolicRegexInfo.Mk(startsWithLineAnchor: true, canBeNullable: true)); + + internal static SymbolicRegexNode MkEndAnchor(SymbolicRegexBuilder builder) => + Create(builder, SymbolicRegexKind.EndAnchor, null, null, -1, -1, default, null, SymbolicRegexInfo.Mk(startsWithLineAnchor: true, canBeNullable: true)); + + internal static SymbolicRegexNode MkEndAnchorZ(SymbolicRegexBuilder builder) => + Create(builder, SymbolicRegexKind.EndAnchorZ, null, null, -1, -1, default, null, SymbolicRegexInfo.Mk(startsWithLineAnchor: true, canBeNullable: true)); + + internal static SymbolicRegexNode MkEndAnchorZRev(SymbolicRegexBuilder builder) => + Create(builder, SymbolicRegexKind.EndAnchorZRev, null, null, -1, -1, default, null, SymbolicRegexInfo.Mk(startsWithLineAnchor: true, canBeNullable: true)); + + internal static SymbolicRegexNode MkEolAnchor(SymbolicRegexBuilder builder) => + Create(builder, SymbolicRegexKind.EOLAnchor, null, null, -1, -1, default, null, SymbolicRegexInfo.Mk(startsWithLineAnchor: true, canBeNullable: true)); + + internal static SymbolicRegexNode MkBolAnchor(SymbolicRegexBuilder builder) => + Create(builder, SymbolicRegexKind.BOLAnchor, null, null, -1, -1, default, null, SymbolicRegexInfo.Mk(startsWithLineAnchor: true, canBeNullable: true)); + + internal static SymbolicRegexNode MkWBAnchor(SymbolicRegexBuilder builder) => + Create(builder, SymbolicRegexKind.WBAnchor, null, null, -1, -1, default, null, SymbolicRegexInfo.Mk(startsWithBoundaryAnchor: true, canBeNullable: true)); + + internal static SymbolicRegexNode MkNWBAnchor(SymbolicRegexBuilder builder) => + Create(builder, SymbolicRegexKind.NWBAnchor, null, null, -1, -1, default, null, SymbolicRegexInfo.Mk(startsWithBoundaryAnchor: true, canBeNullable: true)); + + internal static SymbolicRegexNode MkStar(SymbolicRegexBuilder builder, SymbolicRegexNode body) => + Create(builder, SymbolicRegexKind.Loop, body, null, 0, int.MaxValue, default, null, SymbolicRegexInfo.Loop(body._info, 0, false)); + + internal static SymbolicRegexNode MkPlus(SymbolicRegexBuilder builder, SymbolicRegexNode body) => + Create(builder, SymbolicRegexKind.Loop, body, null, 1, int.MaxValue, default, null, SymbolicRegexInfo.Loop(body._info, 1, false)); + + #endregion + + internal static SymbolicRegexNode MkSingleton(SymbolicRegexBuilder builder, S set) => + Create(builder, SymbolicRegexKind.Singleton, null, null, -1, -1, set, null, SymbolicRegexInfo.Mk(containsSomeCharacter: !set.Equals(builder._solver.False))); + + internal static SymbolicRegexNode MkLoop(SymbolicRegexBuilder builder, SymbolicRegexNode body, int lower, int upper, bool isLazy) + { + Debug.Assert(lower >= 0 && lower <= upper); + return Create(builder, SymbolicRegexKind.Loop, body, null, lower, upper, default, null, SymbolicRegexInfo.Loop(body._info, lower, isLazy)); + } + + internal static SymbolicRegexNode MkOr(SymbolicRegexBuilder builder, params SymbolicRegexNode[] disjuncts) => + MkCollection(builder, SymbolicRegexKind.Or, SymbolicRegexSet.CreateMulti(builder, disjuncts, SymbolicRegexKind.Or), SymbolicRegexInfo.Or(GetInfos(disjuncts))); + + internal static SymbolicRegexNode MkOr(SymbolicRegexBuilder builder, SymbolicRegexSet disjuncts) + { + Debug.Assert(disjuncts._kind == SymbolicRegexKind.Or || disjuncts.IsEverything); + return MkCollection(builder, SymbolicRegexKind.Or, disjuncts, SymbolicRegexInfo.Or(GetInfos(disjuncts))); + } + + internal static SymbolicRegexNode MkAnd(SymbolicRegexBuilder builder, params SymbolicRegexNode[] conjuncts) => + MkCollection(builder, SymbolicRegexKind.And, SymbolicRegexSet.CreateMulti(builder, conjuncts, SymbolicRegexKind.And), SymbolicRegexInfo.And(GetInfos(conjuncts))); + + internal static SymbolicRegexNode MkAnd(SymbolicRegexBuilder builder, SymbolicRegexSet conjuncts) + { + Debug.Assert(conjuncts.IsNothing || conjuncts._kind == SymbolicRegexKind.And); + return MkCollection(builder, SymbolicRegexKind.And, conjuncts, SymbolicRegexInfo.And(GetInfos(conjuncts))); + } + + private static SymbolicRegexNode MkCollection(SymbolicRegexBuilder builder, SymbolicRegexKind kind, SymbolicRegexSet alts, SymbolicRegexInfo info) => + alts.IsNothing ? builder._nothing : + alts.IsEverything ? builder._anyStar : + alts.IsSingleton ? alts.GetSingletonElement() : + Create(builder, kind, null, null, -1, -1, default, alts, info); + + private static SymbolicRegexInfo[] GetInfos(SymbolicRegexNode[] nodes) + { + var infos = new SymbolicRegexInfo[nodes.Length]; + for (int i = 0; i < nodes.Length; i++) + { + infos[i] = nodes[i]._info; + } + return infos; + } + + private static SymbolicRegexInfo[] GetInfos(SymbolicRegexSet nodes) + { + var infos = new SymbolicRegexInfo[nodes.Count]; + int i = 0; + foreach (SymbolicRegexNode node in nodes) + { + Debug.Assert(i < nodes.Count); + infos[i++] = node._info; + } + Debug.Assert(i == nodes.Count); + return infos; + } + + /// + /// Make a concatenation of given regexes, if any regex is nothing then return nothing, eliminate + /// intermediate epsilons. Keep the concatenation flat, assuming both right and left are flat. + /// + internal static SymbolicRegexNode MkConcat(SymbolicRegexBuilder builder, SymbolicRegexNode left, SymbolicRegexNode right) + { + if (left == builder._nothing || right == builder._nothing) + return builder._nothing; + + if (left.IsEpsilon) + return right; + + if (right.IsEpsilon) + return left; + + if (left._kind != SymbolicRegexKind.Concat) + { + return Create(builder, SymbolicRegexKind.Concat, left, right, -1, -1, default, null, SymbolicRegexInfo.Concat(left._info, right._info)); + } + + SymbolicRegexNode concat = right; + List> left_elems = left.ToList(); + for (int i = left_elems.Count - 1; i >= 0; i--) + { + concat = Create(builder, SymbolicRegexKind.Concat, left_elems[i], concat, -1, -1, default, null, SymbolicRegexInfo.Concat(left_elems[i]._info, concat._info)); + } + return concat; + } + + internal static SymbolicRegexNode MkNot(SymbolicRegexBuilder builder, SymbolicRegexNode root) + { + // Instead of just creating a negated root node + // Convert ~root to Negation Normal Form (NNF) by using deMorgan's laws and push ~ to the leaves + // This may avoid rather large overhead (such case was discovered with unit test PasswordSearchDual) + // Do this transformation in-line without recursion, to avoid any chance of deep recursion + // OBSERVE: NNF[node] represents the Negation Normal Form of ~node + Dictionary, SymbolicRegexNode> NNF = new(); + Stack<(SymbolicRegexNode, bool)> todo = new(); + todo.Push((root, false)); + while (todo.Count > 0) + { + (SymbolicRegexNode, bool) top = todo.Pop(); + bool secondTimePushed = top.Item2; + SymbolicRegexNode node = top.Item1; + if (secondTimePushed) + { + Debug.Assert((node._kind == SymbolicRegexKind.Or || node._kind == SymbolicRegexKind.And) && node._alts is not null); + // Here all members of _alts have been processed + List> alts_nnf = new(); + foreach (SymbolicRegexNode elem in node._alts) + { + alts_nnf.Add(NNF[elem]); + } + // Using deMorgan's laws, flip the kind: Or becomes And, And becomes Or + SymbolicRegexNode node_nnf = node._kind == SymbolicRegexKind.Or ? MkAnd(builder, alts_nnf.ToArray()) : MkOr(builder, alts_nnf.ToArray()); + NNF[node] = node_nnf; + } + else + { + switch (node._kind) + { + case SymbolicRegexKind.Not: + Debug.Assert(node._left is not null); + // Here we assume that top._left is already in NNF, double negation is cancelled out + NNF[node] = node._left; + break; + + case SymbolicRegexKind.Or or SymbolicRegexKind.And: + Debug.Assert(node._alts is not null); + // Push the node for the second time + todo.Push((node, true)); + // Compute the negation normal form of all the members + // Their computation is actually the same independent from being inside an 'Or' or 'And' node + foreach (var elem in node._alts) + { + todo.Push((elem, false)); + } + break; + + case SymbolicRegexKind.Epsilon: + // ~() = .+ + NNF[node] = SymbolicRegexNode.MkPlus(builder, builder._anyChar); + break; + + case SymbolicRegexKind.Singleton: + Debug.Assert(node._set is not null); + // ~[] = .* + if (node.IsNothing) + { + NNF[node] = builder._anyStar; + break; + } + goto default; + + case SymbolicRegexKind.Loop: + Debug.Assert(node._left is not null); + // ~(.*) = [] and ~(.+) = () + if (node.IsAnyStar) + { + NNF[node] = builder._nothing; + break; + } + else if (node.IsPlus && node._left.IsAnyChar) + { + NNF[node] = builder._epsilon; + break; + } + goto default; + + default: + // In all other cases construct the complement + NNF[node] = Create(builder, SymbolicRegexKind.Not, node, null, -1, -1, default, null, SymbolicRegexInfo.Not(node._info)); + break; + } + } + } + return NNF[root]; + } + + /// + /// Transform the symbolic regex so that all singletons have been intersected with the given predicate pred. + /// + public SymbolicRegexNode Restrict(S pred) + { + switch (_kind) + { + case SymbolicRegexKind.StartAnchor: + case SymbolicRegexKind.EndAnchor: + case SymbolicRegexKind.BOLAnchor: + case SymbolicRegexKind.EOLAnchor: + case SymbolicRegexKind.Epsilon: + case SymbolicRegexKind.WatchDog: + case SymbolicRegexKind.WBAnchor: + case SymbolicRegexKind.NWBAnchor: + case SymbolicRegexKind.EndAnchorZ: + case SymbolicRegexKind.EndAnchorZRev: + return this; + + case SymbolicRegexKind.Singleton: + { + Debug.Assert(_set is not null); + S newset = _builder._solver.And(_set, pred); + return _set.Equals(newset) ? this : _builder.MkSingleton(newset); + } + + case SymbolicRegexKind.Loop: + { + Debug.Assert(_left is not null); + SymbolicRegexNode body = _left.Restrict(pred); + return body == _left ? this : _builder.MkLoop(body, IsLazy, _lower, _upper); + } + + case SymbolicRegexKind.Concat: + { + Debug.Assert(_left is not null && _right is not null); + SymbolicRegexNode first = _left.Restrict(pred); + SymbolicRegexNode second = _right.Restrict(pred); + return first == _left && second == _right ? this : _builder.MkConcat(first, second); + } + + case SymbolicRegexKind.Or: + { + Debug.Assert(_alts is not null); + SymbolicRegexSet choices = _alts.Restrict(pred); + return _builder.MkOr(choices); + } + + case SymbolicRegexKind.And: + { + Debug.Assert(_alts is not null); + SymbolicRegexSet conjuncts = _alts.Restrict(pred); + return _builder.MkAnd(conjuncts); + } + + default: + { + Debug.Assert(_kind == SymbolicRegexKind.Not, $"{nameof(Restrict)}:{_kind}"); + Debug.Assert(_left is not null); + SymbolicRegexNode restricted = _left.Restrict(pred); + return _builder.MkNot(restricted); + } + } + } + + /// + /// Returns the fixed matching length of the regex or -1 if the regex does not have a fixed matching length. + /// + public int GetFixedLength() + { + // Guard against stack overflow due to deep recursion. + if (!RuntimeHelpers.TryEnsureSufficientExecutionStack()) + { + SymbolicRegexNode thisRef = this; + return StackHelper.CallOnEmptyStack(() => thisRef.GetFixedLength()); + } + + switch (_kind) + { + case SymbolicRegexKind.WatchDog: + case SymbolicRegexKind.Epsilon: + case SymbolicRegexKind.BOLAnchor: + case SymbolicRegexKind.EOLAnchor: + case SymbolicRegexKind.EndAnchor: + case SymbolicRegexKind.StartAnchor: + case SymbolicRegexKind.EndAnchorZ: + case SymbolicRegexKind.EndAnchorZRev: + case SymbolicRegexKind.WBAnchor: + case SymbolicRegexKind.NWBAnchor: + return 0; + + case SymbolicRegexKind.Singleton: + return 1; + + case SymbolicRegexKind.Loop: + { + Debug.Assert(_left is not null); + if (_lower == _upper) + { + int body_length = _left.GetFixedLength(); + if (body_length >= 0) + { + return _lower * body_length; + } + } + break; + } + + case SymbolicRegexKind.Concat: + { + Debug.Assert(_left is not null && _right is not null); + int left_length = _left.GetFixedLength(); + if (left_length >= 0) + { + int right_length = _right.GetFixedLength(); + if (right_length >= 0) + { + return left_length + right_length; + } + } + break; + } + + case SymbolicRegexKind.Or: + Debug.Assert(_alts is not null); + return _alts.GetFixedLength(); + } + + return -1; + } + + private Dictionary<(S, uint), SymbolicRegexNode>? _MkDerivative_Cache; + /// + /// Takes the derivative of the symbolic regex wrt elem. + /// Assumes that elem is either a minterm wrt the predicates of the whole regex or a singleton set. + /// + /// given element wrt which the derivative is taken + /// immediately surrounding character context that affects nullability of anchors + /// + internal SymbolicRegexNode MkDerivative(S elem, uint context) + { + // Guard against stack overflow due to deep recursion + if (!RuntimeHelpers.TryEnsureSufficientExecutionStack()) + { + S localElem = elem; + uint localContext = context; + return StackHelper.CallOnEmptyStack(() => MkDerivative(localElem, localContext)); + } + + if (this == _builder._anyStar || this == _builder._nothing) + { + return this; + } + + if (Kind == SymbolicRegexKind.Or && !_isInternalizedUnion && !_builder._antimirov) + { + // Internalize the node before proceeding + // this node could end up being internalized or replaced by + // an already previously created object (!= this) + SymbolicRegexNode this_internalized = Internalize(this); + Debug.Assert(this_internalized._isInternalizedUnion); + if (this_internalized != this) + { + return this_internalized.MkDerivative(elem, context); + } + } + + _MkDerivative_Cache ??= new(); + (S, uint) key = (elem, context); + SymbolicRegexNode? deriv; + if (_MkDerivative_Cache.TryGetValue(key, out deriv)) + { + Debug.Assert(deriv != null); + return deriv; + } + + switch (_kind) + { + case SymbolicRegexKind.Singleton: + Debug.Assert(_set is not null); + deriv = _builder._solver.IsSatisfiable(_builder._solver.And(elem, _set)) ? + _builder._epsilon : + _builder._nothing; + break; + + case SymbolicRegexKind.Loop: + { + #region d(a, R*) = d(a,R)R* + Debug.Assert(_left is not null); + SymbolicRegexNode step = _left.MkDerivative(elem, context); + + if (step == _builder._nothing || _upper == 0) + { + deriv = _builder._nothing; + break; + } + + if (IsStar) + { + deriv = _builder.MkConcat(step, this); + break; + } + + if (IsPlus) + { + SymbolicRegexNode star = _builder.MkLoop(_left, IsLazy); + deriv = _builder.MkConcat(step, star); + break; + } + + int newupper = _upper == int.MaxValue ? int.MaxValue : _upper - 1; + int newlower = _lower == 0 ? 0 : _lower - 1; + SymbolicRegexNode rest = _builder.MkLoop(_left, IsLazy, newlower, newupper); + deriv = _builder.MkConcat(step, rest); + break; + #endregion + } + + case SymbolicRegexKind.Concat: + { + #region d(a, AB) = d(a,A)B | (if A nullable then d(a,B)) + Debug.Assert(_left is not null && _right is not null); + SymbolicRegexNode leftd = _left.MkDerivative(elem, context); + SymbolicRegexNode first = _builder._nothing; + if (_builder._antimirov && leftd._kind == SymbolicRegexKind.Or) + { + // push concatenations into the union + Debug.Assert(leftd._alts is not null); + foreach (SymbolicRegexNode d in leftd._alts) + { + first = _builder.MkOr(first, _builder.MkConcat(d, _right)); + } + } + else + { + first = _builder.MkConcat(leftd, _right); + } + + if (_left.IsNullableFor(context)) + { + SymbolicRegexNode second = _right.MkDerivative(elem, context); + deriv = _builder.MkOr2(first, second); + break; + } + + deriv = first; + break; + #endregion + } + + case SymbolicRegexKind.Or: + { + #region d(a,A|B) = d(a,A)|d(a,B) + Debug.Assert(_alts is not null && _alts._kind == SymbolicRegexKind.Or); + SymbolicRegexSet alts_deriv = _alts.CreateDerivative(elem, context); + // At this point alts_deriv can be the empty conjunction denoting .* + deriv = _builder.MkOr(alts_deriv); + break; + #endregion + } + + case SymbolicRegexKind.And: + { + #region d(a,A & B) = d(a,A) & d(a,B) + Debug.Assert(_alts is not null && _alts._kind == SymbolicRegexKind.And); + SymbolicRegexSet alts_deriv = _alts.CreateDerivative(elem, context); + // At this point alts_deriv can be the empty disjunction denoting nothing + deriv = _builder.MkAnd(alts_deriv); + break; + #endregion + } + + case SymbolicRegexKind.Not: + { + #region d(a,~(A)) = ~(d(a,A)) + Debug.Assert(_left is not null); + SymbolicRegexNode leftD = _left.MkDerivative(elem, context); + deriv = _builder.MkNot(leftD); + break; + #endregion + } + + default: + deriv = _builder._nothing; + break; + } + + _MkDerivative_Cache[key] = deriv; + return deriv; + } + + private TransitionRegex? _transitionRegex; + /// Computes the symbolic derivative as a transition regex + internal TransitionRegex MkDerivative() + { + if (_transitionRegex is not null) + { + return _transitionRegex; + } + + if (IsNothing || IsEpsilon) + { + _transitionRegex = TransitionRegex.Leaf(_builder._nothing); + return _transitionRegex; + } + + if (IsAnyStar || IsAnyPlus) + { + _transitionRegex = TransitionRegex.Leaf(_builder._anyStar); + return _transitionRegex; + } + + switch (_kind) + { + case SymbolicRegexKind.Singleton: + Debug.Assert(_set is not null); + _transitionRegex = TransitionRegex.Conditional(_set, TransitionRegex.Leaf(_builder._epsilon), TransitionRegex.Leaf(_builder._nothing)); + break; + + case SymbolicRegexKind.Concat: + Debug.Assert(_left is not null && _right is not null); + TransitionRegex mainTransition = _left.MkDerivative().Concat(_right); + + if (!_left.CanBeNullable) + { + // If _left is never nullable + _transitionRegex = mainTransition; + } + else if (_left.IsNullable) + { + // If _left is unconditionally nullable + _transitionRegex = mainTransition | _right.MkDerivative(); + } + else + { + // The left side contains anchors and can be nullable in some context + // Extract the nullability as the lookaround condition + SymbolicRegexNode leftNullabilityTest = _left.ExtractNullabilityTest(); + _transitionRegex = TransitionRegex.Lookaround(leftNullabilityTest, mainTransition | _right.MkDerivative(), mainTransition); + } + break; + + case SymbolicRegexKind.Loop: + // d(R*) = d(R+) = d(R)R* + Debug.Assert(_left is not null); + Debug.Assert(_upper > 0); + TransitionRegex step = _left.MkDerivative(); + + if (IsStar || IsPlus) + { + _transitionRegex = step.Concat(_builder.MkLoop(_left, IsLazy)); + } + else + { + int newupper = _upper == int.MaxValue ? int.MaxValue : _upper - 1; + int newlower = _lower == 0 ? 0 : _lower - 1; + SymbolicRegexNode rest = _builder.MkLoop(_left, IsLazy, newlower, newupper); + _transitionRegex = step.Concat(rest); + } + break; + + case SymbolicRegexKind.Or: + Debug.Assert(_alts is not null); + _transitionRegex = TransitionRegex.Leaf(_builder._nothing); + foreach (SymbolicRegexNode elem in _alts) + { + _transitionRegex = _transitionRegex | elem.MkDerivative(); + } + break; + + case SymbolicRegexKind.And: + Debug.Assert(_alts is not null); + _transitionRegex = TransitionRegex.Leaf(_builder._anyStar); + foreach (SymbolicRegexNode elem in _alts) + { + _transitionRegex = _transitionRegex & elem.MkDerivative(); + } + break; + + case SymbolicRegexKind.Not: + Debug.Assert(_left is not null); + _transitionRegex = ~_left.MkDerivative(); + break; + + default: + _transitionRegex = TransitionRegex.Leaf(_builder._nothing); + break; + } + return _transitionRegex; + } + + /// + /// Computes the closure of MkDerivative, by exploring all the leaves + /// of the transition regex until no more new leaves are found. + /// Converts the resulting transition system into a symbolic NFA. + /// If the exploration remains incomplete due to the given state bound + /// being reached then the InComplete property of the constructed NFA is true. + /// + internal SymbolicNFA Explore(int bound) => SymbolicNFA.Explore(this, bound); + + /// Extracts the nullability test as a Boolean combination of anchors + public SymbolicRegexNode ExtractNullabilityTest() + { + if (IsNullable) + { + return _builder._anyStar; + } + + if (!CanBeNullable) + { + return _builder._nothing; + } + + switch (_kind) + { + case SymbolicRegexKind.StartAnchor: + case SymbolicRegexKind.EndAnchor: + case SymbolicRegexKind.BOLAnchor: + case SymbolicRegexKind.EOLAnchor: + case SymbolicRegexKind.WBAnchor: + case SymbolicRegexKind.NWBAnchor: + case SymbolicRegexKind.EndAnchorZ: + case SymbolicRegexKind.EndAnchorZRev: + return this; + case SymbolicRegexKind.Concat: + Debug.Assert(_left is not null && _right is not null); + return _builder.MkAnd(_left.ExtractNullabilityTest(), _right.ExtractNullabilityTest()); + case SymbolicRegexKind.Or: + Debug.Assert(_alts is not null); + SymbolicRegexNode disjunction = _builder._nothing; + foreach (SymbolicRegexNode elem in _alts) + { + disjunction = _builder.MkOr(disjunction, elem.ExtractNullabilityTest()); + } + return disjunction; + case SymbolicRegexKind.And: + Debug.Assert(_alts is not null); + SymbolicRegexNode conjunction = _builder._anyStar; + foreach (SymbolicRegexNode elem in _alts) + { + conjunction = _builder.MkAnd(conjunction, elem.ExtractNullabilityTest()); + } + return conjunction; + case SymbolicRegexKind.Loop: + Debug.Assert(_left is not null); + return _left.ExtractNullabilityTest(); + default: + // All remaining cases could not be nullable or were trivially nullable + // Sigleton cannot be nullable and Epsilon and Watchdog are trivially nullable + Debug.Assert(_kind == SymbolicRegexKind.Not && _left is not null); + return _builder.MkNot(_left.ExtractNullabilityTest()); + } + } + + public override int GetHashCode() + { + return _hashcode; + } + + private int ComputeHashCode() + { + switch (_kind) + { + case SymbolicRegexKind.EndAnchor: + case SymbolicRegexKind.StartAnchor: + case SymbolicRegexKind.BOLAnchor: + case SymbolicRegexKind.EOLAnchor: + case SymbolicRegexKind.Epsilon: + case SymbolicRegexKind.WBAnchor: + case SymbolicRegexKind.NWBAnchor: + case SymbolicRegexKind.EndAnchorZ: + case SymbolicRegexKind.EndAnchorZRev: + return HashCode.Combine(_kind, _info); + + case SymbolicRegexKind.WatchDog: + return HashCode.Combine(_kind, _lower); + + case SymbolicRegexKind.Loop: + return HashCode.Combine(_kind, _left, _lower, _upper, _info); + + case SymbolicRegexKind.Or or SymbolicRegexKind.And: + return HashCode.Combine(_kind, _alts, _info); + + case SymbolicRegexKind.Concat: + return HashCode.Combine(_left, _right, _info); + + case SymbolicRegexKind.Singleton: + return HashCode.Combine(_kind, _set); + + default: + Debug.Assert(_kind == SymbolicRegexKind.Not); + return HashCode.Combine(_kind, _left, _info); + }; + } + + public override bool Equals([NotNullWhen(true)] object? obj) + { + if (obj is not SymbolicRegexNode that) + { + return false; + } + + if (this == that) + { + return true; + } + + if (_kind != that._kind) + { + return false; + } + + if (_kind == SymbolicRegexKind.Or) + { + if (_isInternalizedUnion && that._isInternalizedUnion) + { + // Internalized nodes that are not identical are not equal + return false; + } + + // Check equality of the sets of regexes + Debug.Assert(_alts is not null && that._alts is not null); + return _alts.Equals(that._alts); + } + + return false; + } + + private void ToStringForLoop(StringBuilder sb) + { + if (_kind == SymbolicRegexKind.Singleton) + { + ToString(sb); + } + else + { + sb.Append('('); + ToString(sb); + sb.Append(')'); + } + } + + public override string ToString() + { + StringBuilder sb = new(); + ToString(sb); + return sb.ToString(); + } + + internal void ToString(StringBuilder sb) + { + // Guard against stack overflow due to deep recursion + if (!RuntimeHelpers.TryEnsureSufficientExecutionStack()) + { + StringBuilder localSb = sb; + StackHelper.CallOnEmptyStack(() => ToString(localSb)); + return; + } + + switch (_kind) + { + case SymbolicRegexKind.EndAnchor: + sb.Append("\\z"); + return; + + case SymbolicRegexKind.StartAnchor: + sb.Append("\\A"); + return; + + case SymbolicRegexKind.BOLAnchor: + sb.Append('^'); + return; + + case SymbolicRegexKind.EOLAnchor: + sb.Append('$'); + return; + + case SymbolicRegexKind.Epsilon: + case SymbolicRegexKind.WatchDog: + return; + + case SymbolicRegexKind.WBAnchor: + sb.Append("\\b"); + return; + + case SymbolicRegexKind.NWBAnchor: + sb.Append("\\B"); + return; + + case SymbolicRegexKind.EndAnchorZ: + sb.Append("\\Z"); + return; + + case SymbolicRegexKind.EndAnchorZRev: + sb.Append("\\a"); + return; + + case SymbolicRegexKind.Or: + case SymbolicRegexKind.And: + Debug.Assert(_alts is not null); + _alts.ToString(sb); + return; + + case SymbolicRegexKind.Concat: + Debug.Assert(_left is not null && _right is not null); + _left.ToString(sb); + _right.ToString(sb); + return; + + case SymbolicRegexKind.Singleton: + Debug.Assert(_set is not null); + sb.Append(_builder._solver.PrettyPrint(_set)); + return; + + case SymbolicRegexKind.Loop: + Debug.Assert(_left is not null); + if (IsAnyStar) + { + sb.Append(".*"); + } + else if (IsMaybe) + { + _left.ToStringForLoop(sb); + sb.Append('?'); + } + else if (IsStar) + { + _left.ToStringForLoop(sb); + sb.Append('*'); + if (IsLazy) + { + sb.Append('?'); + } + } + else if (IsPlus) + { + _left.ToStringForLoop(sb); + sb.Append('+'); + if (IsLazy) + { + sb.Append('?'); + } + } + else if (_lower == 0 && _upper == 0) + { + sb.Append("()"); + } + else if (!IsBoundedLoop) + { + _left.ToStringForLoop(sb); + sb.Append('{'); + sb.Append(_lower); + sb.Append(",}"); + if (IsLazy) + sb.Append('?'); + } + else if (_lower == _upper) + { + _left.ToStringForLoop(sb); + sb.Append('{'); + sb.Append(_lower); + sb.Append('}'); + if (IsLazy) + sb.Append('?'); + } + else + { + _left.ToStringForLoop(sb); + sb.Append('{'); + sb.Append(_lower); + sb.Append(','); + sb.Append(_upper); + sb.Append('}'); + if (IsLazy) + sb.Append('?'); + } + return; + + default: + // Using the operator ~ for complement + Debug.Assert(_kind == SymbolicRegexKind.Not); + Debug.Assert(_left is not null); + sb.Append("~("); + _left.ToString(sb); + sb.Append(')'); + return; + } + } + + /// + /// Returns the set of all predicates that occur in the regex or + /// the set containing True if there are no precidates in the regex, e.g., if the regex is "^" + /// + public HashSet GetPredicates() + { + var predicates = new HashSet(); + CollectPredicates_helper(predicates); + if (predicates.Count == 0) + { + predicates.Add(_builder._solver.True); + } + return predicates; + } + + /// + /// Collects all predicates that occur in the regex into the given set predicates + /// + private void CollectPredicates_helper(HashSet predicates) + { + switch (_kind) + { + case SymbolicRegexKind.BOLAnchor: + case SymbolicRegexKind.EOLAnchor: + case SymbolicRegexKind.EndAnchorZ: + case SymbolicRegexKind.EndAnchorZRev: + predicates.Add(_builder._newLinePredicate); + return; + + case SymbolicRegexKind.StartAnchor: + case SymbolicRegexKind.EndAnchor: + case SymbolicRegexKind.Epsilon: + case SymbolicRegexKind.WatchDog: + return; + + case SymbolicRegexKind.Singleton: + Debug.Assert(_set is not null); + predicates.Add(_set); + return; + + case SymbolicRegexKind.Loop: + Debug.Assert(_left is not null); + _left.CollectPredicates_helper(predicates); + return; + + case SymbolicRegexKind.Or: + case SymbolicRegexKind.And: + Debug.Assert(_alts is not null); + foreach (SymbolicRegexNode sr in _alts) + { + sr.CollectPredicates_helper(predicates); + } + return; + + case SymbolicRegexKind.Concat: + // avoid deep nested recursion over long concat nodes + SymbolicRegexNode conc = this; + while (conc._kind == SymbolicRegexKind.Concat) + { + Debug.Assert(conc._left is not null && conc._right is not null); + conc._left.CollectPredicates_helper(predicates); + conc = conc._right; + } + conc.CollectPredicates_helper(predicates); + return; + + case SymbolicRegexKind.Not: + Debug.Assert(_left is not null); + _left.CollectPredicates_helper(predicates); + return; + + case SymbolicRegexKind.NWBAnchor: + case SymbolicRegexKind.WBAnchor: + predicates.Add(_builder._wordLetterPredicateForAnchors); + return; + + default: + Debug.Fail($"{nameof(CollectPredicates_helper)}:{_kind}"); + break; + } + } + + /// + /// Compute all the minterms from the predicates in this regex. + /// If S implements IComparable then sort the result in increasing order. + /// + public S[] ComputeMinterms() + { + Debug.Assert(typeof(S).IsAssignableTo(typeof(IComparable))); + + HashSet predicates = GetPredicates(); + Debug.Assert(predicates.Count != 0); + + S[] predicatesArray = new S[predicates.Count]; + int i = 0; + foreach (S s in predicates) + { + predicatesArray[i++] = s; + } + Debug.Assert(i == predicatesArray.Length); + + List mt = _builder._solver.GenerateMinterms(predicatesArray); + mt.Sort(); + return mt.ToArray(); + } + + /// + /// Create the reverse of this regex + /// + public SymbolicRegexNode Reverse() + { + switch (_kind) + { + case SymbolicRegexKind.Loop: + Debug.Assert(_left is not null); + return _builder.MkLoop(_left.Reverse(), IsLazy, _lower, _upper); + + case SymbolicRegexKind.Concat: + { + Debug.Assert(_left is not null && _right is not null); + SymbolicRegexNode rev = _left.Reverse(); + SymbolicRegexNode rest = _right; + while (rest._kind == SymbolicRegexKind.Concat) + { + Debug.Assert(rest._left is not null && rest._right is not null); + SymbolicRegexNode rev1 = rest._left.Reverse(); + rev = _builder.MkConcat(rev1, rev); + rest = rest._right; + } + SymbolicRegexNode restr = rest.Reverse(); + rev = _builder.MkConcat(restr, rev); + return rev; + } + + case SymbolicRegexKind.Or: + Debug.Assert(_alts is not null); + return _builder.MkOr(_alts.Reverse()); + + case SymbolicRegexKind.And: + Debug.Assert(_alts is not null); + return _builder.MkAnd(_alts.Reverse()); + + case SymbolicRegexKind.Not: + Debug.Assert(_left is not null); + return _builder.MkNot(_left.Reverse()); + + case SymbolicRegexKind.WatchDog: + // Watchdogs are omitted in reverse + return _builder._epsilon; + + case SymbolicRegexKind.StartAnchor: + // The reverse of StartAnchor is EndAnchor + return _builder._endAnchor; + + case SymbolicRegexKind.EndAnchor: + return _builder._startAnchor; + + case SymbolicRegexKind.BOLAnchor: + // The reverse of BOLanchor is EOLanchor + return _builder._eolAnchor; + + case SymbolicRegexKind.EOLAnchor: + return _builder._bolAnchor; + + case SymbolicRegexKind.EndAnchorZ: + // The reversal of the \Z anchor + return _builder._endAnchorZRev; + + case SymbolicRegexKind.EndAnchorZRev: + // This can potentially only happen if a reversed regex is reversed again. + // Thus, this case is unreachable here, but included for completeness. + return _builder._endAnchorZ; + + default: + // Remaining cases map to themselves: + // SymbolicRegexKind.Epsilon + // SymbolicRegexKind.Singleton + // SymbolicRegexKind.WBAnchor + // SymbolicRegexKind.NWBAnchor + return this; + } + } + + internal bool StartsWithLoop(int upperBoundLowestValue = 1) + { + switch (_kind) + { + case SymbolicRegexKind.Loop: + return (_upper < int.MaxValue) && (_upper > upperBoundLowestValue); + + case SymbolicRegexKind.Concat: + Debug.Assert(_left is not null && _right is not null); + return _left.StartsWithLoop(upperBoundLowestValue) || (_left.IsNullable && _right.StartsWithLoop(upperBoundLowestValue)); + + case SymbolicRegexKind.Or: + Debug.Assert(_alts is not null); + return _alts.StartsWithLoop(upperBoundLowestValue); + + default: + return false; + }; + } + + /// + /// Gets the string prefix that the regex must match or the empty string if such a prefix does not exist. + /// Sets ignoreCase = true when the prefix works under case-insensitivity. + /// For example if the input prefix is "---" it sets ignoreCase=false, + /// if the prefix is "---[aA][bB]" it returns "---AB" and sets ignoreCase=true + /// + internal string GetFixedPrefix(CharSetSolver css, string culture, out bool ignoreCase) + { + ignoreCase = false; + StringBuilder prefix = new(); + bool doneWithoutIgnoreCase = false; + bool doneWithIgnoreCase = false; + foreach (S x in GetPrefixSequence()) + { + BDD bdd = _builder._solver.ConvertToCharSet(css, x); + char character = (char)bdd.GetMin(); + // Check if the prefix extends without ignore case: the set is a single character + if (!doneWithoutIgnoreCase && !css.IsSingleton(bdd)) + { + doneWithoutIgnoreCase = true; + } + if (!doneWithIgnoreCase) + { + // Check if the prefix extends with ignore case: ignoring case doesn't change the set + if (css.ApplyIgnoreCase(css.CharConstraint(character), culture).Equals(bdd)) + { + // Turn ignoreCase on when the prefix extends only under ignore case + if (doneWithoutIgnoreCase) + { + ignoreCase = true; + } + } + else + { + doneWithIgnoreCase = true; + } + } + // Append the character when the prefix extends in either of the ways + if (!doneWithoutIgnoreCase || !doneWithIgnoreCase) + prefix.Append(character); + else + break; + } + return prefix.ToString(); + } + + private IEnumerable GetPrefixSequence() + { + List> paths = new(); + HashSet> nextPaths = new(); + + paths.Add(this); + while (true) + { + bool done = false; + Debug.Assert(paths.Count > 0, "The generator should have ended when any path fails to extend."); + // Generate the next set from one path + S next; + if (!GetNextPrefixSet(ref paths, ref nextPaths, ref done, out next)) + { + // A path didn't have a next set as supported by this algorithm + yield break; + } + if (!_builder._solver.IsSatisfiable(next)) + { + yield break; + } + while (paths.Count > 0) + { + // For all other paths check that they produce the same set + S newSet; + if (!GetNextPrefixSet(ref paths, ref nextPaths, ref done, out newSet) || !newSet.Equals(next)) + { + // Either a path didn't have a next set as supported by this algorithm, or the next set was not equal + yield break; + } + } + // At this point all paths generated equal next sets + yield return next; + if (done) + { + // Some path had no continuation, end the prefix + yield break; + } + else + { + Debug.Assert(paths.Count == 0, "Not all paths were considered for next set."); + paths.AddRange(nextPaths); + nextPaths.Clear(); + } + } + } + + private bool GetNextPrefixSet(ref List> paths, ref HashSet> nextPaths, ref bool done, out S set) + { + while (paths.Count > 0) + { + SymbolicRegexNode node = paths[paths.Count - 1]; + paths.RemoveAt(paths.Count - 1); + switch (node._kind) + { + case SymbolicRegexKind.Singleton: + Debug.Assert(node._set is not null); + set = node._set; + done = true; // No continuation, done after the next set + return true; + case SymbolicRegexKind.Concat: + Debug.Assert(node._left is not null && node._right is not null); + if (!node._left.CanBeNullable) + { + if (node._left.GetFixedLength() == 1) + { + set = node._left.GetStartSet(); + // Left side had just one character, can use just right side as path + nextPaths.Add(node._right); + return true; + } + else + { + // Left side may need multiple steps to get through. However, it is safe + // (though not complete) to forget the right side and just expand the path + // for the left side. + paths.Add(node._left); + break; + } + } + else + { + // Left side may be nullable, can't extend the prefix + set = _builder._solver.False; // Not going to be used + return false; + } + case SymbolicRegexKind.Or: + case SymbolicRegexKind.And: + Debug.Assert(node._alts is not null); + // Handle alternatives as separate paths + paths.AddRange(node._alts); + break; + default: + set = _builder._solver.False; // Not going to be used + return false; // Cut prefix immediately for unhandled node + } + } + set = _builder._solver.False; // Not going to be used + return false; + } + + /// Get the predicate that covers all elements that make some progress. + internal S GetStartSet() => _startSet; + + /// Compute the predicate that covers all elements that make some progress. + private S ComputeStartSet() + { + switch (_kind) + { + // Anchors and () do not contribute to the startset + case SymbolicRegexKind.Epsilon: + case SymbolicRegexKind.WatchDog: + case SymbolicRegexKind.EndAnchor: + case SymbolicRegexKind.StartAnchor: + case SymbolicRegexKind.WBAnchor: + case SymbolicRegexKind.NWBAnchor: + case SymbolicRegexKind.EOLAnchor: + case SymbolicRegexKind.EndAnchorZ: + case SymbolicRegexKind.EndAnchorZRev: + case SymbolicRegexKind.BOLAnchor: + return _builder._solver.False; + + case SymbolicRegexKind.Singleton: + Debug.Assert(_set is not null); + return _set; + + case SymbolicRegexKind.Loop: + Debug.Assert(_left is not null); + return _left._startSet; + + case SymbolicRegexKind.Concat: + { + Debug.Assert(_left is not null && _right is not null); + S startSet = _left.CanBeNullable ? _builder._solver.Or(_left._startSet, _right._startSet) : _left._startSet; + return startSet; + } + + case SymbolicRegexKind.Or: + { + Debug.Assert(_alts is not null); + S startSet = _builder._solver.False; + foreach (SymbolicRegexNode alt in _alts) + { + startSet = _builder._solver.Or(startSet, alt._startSet); + } + return startSet; + } + + case SymbolicRegexKind.And: + { + Debug.Assert(_alts is not null); + S startSet = _builder._solver.True; + foreach (SymbolicRegexNode alt in _alts) + { + startSet = _builder._solver.And(startSet, alt._startSet); + } + return startSet; + } + + default: + Debug.Assert(_kind == SymbolicRegexKind.Not); + return _builder._solver.True; + } + } + + /// + /// Returns true if this is a loop with an upper bound + /// + public bool IsBoundedLoop => _kind == SymbolicRegexKind.Loop && _upper < int.MaxValue; + + /// + /// Replace anchors that are infeasible by [] wrt the given previous character kind and what continuation is possible. + /// + /// previous character kind + /// if true the continuation can start with wordletter or stop + /// if true the continuation can start with nonwordletter or stop + internal SymbolicRegexNode PruneAnchors(uint prevKind, bool contWithWL, bool contWithNWL) + { + // Guard against stack overflow due to deep recursion + if (!RuntimeHelpers.TryEnsureSufficientExecutionStack()) + { + uint localPrevKind = prevKind; + bool localContWithWL = contWithWL; + bool localContWithNWL = contWithNWL; + return StackHelper.CallOnEmptyStack(() => PruneAnchors(localPrevKind, localContWithWL, localContWithNWL)); + } + + if (!_info.StartsWithSomeAnchor) + return this; + + switch (_kind) + { + case SymbolicRegexKind.StartAnchor: + return prevKind == CharKind.StartStop ? + this : + _builder._nothing; //start anchor is only nullable if the previous character is Start + + case SymbolicRegexKind.EndAnchorZRev: + return ((prevKind & CharKind.StartStop) != 0) ? + this : + _builder._nothing; //rev(\Z) is only nullable if the previous characters is Start or the very first \n + + case SymbolicRegexKind.WBAnchor: + return (prevKind == CharKind.WordLetter ? contWithNWL : contWithWL) ? + this : + // \b is impossible when the previous character is \w but no continuation matches \W + // or the previous character is \W but no continuation matches \w + _builder._nothing; + + case SymbolicRegexKind.NWBAnchor: + return (prevKind == CharKind.WordLetter ? contWithWL : contWithNWL) ? + this : + // \B is impossible when the previous character is \w but no continuation matches \w + // or the previous character is \W but no continuation matches \W + _builder._nothing; + + case SymbolicRegexKind.Loop: + Debug.Assert(_left is not null); + SymbolicRegexNode body = _left.PruneAnchors(prevKind, contWithWL, contWithNWL); + return body == _left ? + this : + MkLoop(_builder, body, _lower, _upper, IsLazy); + + case SymbolicRegexKind.Concat: + Debug.Assert(_left is not null && _right is not null); + SymbolicRegexNode left1 = _left.PruneAnchors(prevKind, contWithWL, contWithNWL); + SymbolicRegexNode right1 = _left.IsNullable ? _right.PruneAnchors(prevKind, contWithWL, contWithNWL) : _right; + + Debug.Assert(left1 is not null && right1 is not null); + return left1 == _left && right1 == _right ? + this : + MkConcat(_builder, left1, right1); + + case SymbolicRegexKind.Or: + { + Debug.Assert(_alts != null); + var elements = new SymbolicRegexNode[_alts.Count]; + int i = 0; + foreach (SymbolicRegexNode alt in _alts) + { + elements[i++] = alt.PruneAnchors(prevKind, contWithWL, contWithNWL); + } + Debug.Assert(i == elements.Length); + return MkOr(_builder, elements); + } + + default: + return this; + } + } + } +} diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicRegexRunner.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicRegexRunner.cs new file mode 100644 index 00000000000000..39ecdba2dce0b4 --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicRegexRunner.cs @@ -0,0 +1,104 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Globalization; +using System.Text.RegularExpressions.Symbolic.Unicode; + +namespace System.Text.RegularExpressions.Symbolic +{ + internal sealed class SymbolicRegexRunner : RegexRunner + { + /// The unicode component, including the BDD algebra. + internal static readonly UnicodeCategoryTheory s_unicode = new UnicodeCategoryTheory(new CharSetSolver()); + /// The matching engine. + internal readonly SymbolicRegexMatcher _matcher; + /// Minimum length computed + private readonly int _minRequiredLength; + + private SymbolicRegexRunner(RegexCode code, TimeSpan matchTimeout, CultureInfo culture) + { + var converter = new RegexNodeToSymbolicConverter(s_unicode, culture); + var solver = (CharSetSolver)s_unicode._solver; + SymbolicRegexNode root = converter.Convert(code.Tree.Root, topLevel: true); + + _minRequiredLength = code.Tree.MinRequiredLength; + + BDD[] minterms = root.ComputeMinterms(); + if (minterms.Length > 64) + { + // Use BV to represent a predicate + var algBV = new BVAlgebra(solver, minterms); + var builderBV = new SymbolicRegexBuilder(algBV); + + // The default constructor sets the following predicates to False; this update happens after the fact. + // It depends on whether anchors where used in the regex whether the predicates are actually different from False. + builderBV._wordLetterPredicateForAnchors = algBV.ConvertFromCharSet(solver, converter._builder._wordLetterPredicateForAnchors); + builderBV._newLinePredicate = algBV.ConvertFromCharSet(solver, converter._builder._newLinePredicate); + + //Convert the BDD based AST to BV based AST + SymbolicRegexNode rootBV = converter._builder.Transform(root, builderBV, bdd => builderBV._solver.ConvertFromCharSet(solver, bdd)); + _matcher = new SymbolicRegexMatcher(rootBV, solver, minterms, matchTimeout, culture); + } + else + { + // Use ulong to represent a predicate + var alg64 = new BV64Algebra(solver, minterms); + var builder64 = new SymbolicRegexBuilder(alg64) + { + // The default constructor sets the following predicates to False, this update happens after the fact + // It depends on whether anchors where used in the regex whether the predicates are actually different from False + _wordLetterPredicateForAnchors = alg64.ConvertFromCharSet(solver, converter._builder._wordLetterPredicateForAnchors), + _newLinePredicate = alg64.ConvertFromCharSet(solver, converter._builder._newLinePredicate) + }; + + // Convert the BDD-based AST to ulong-based AST + SymbolicRegexNode root64 = converter._builder.Transform(root, builder64, bdd => builder64._solver.ConvertFromCharSet(solver, bdd)); + _matcher = new SymbolicRegexMatcher(root64, solver, minterms, matchTimeout, culture); + } + } + + public static RegexRunnerFactory CreateFactory(RegexCode code, RegexOptions options, TimeSpan matchTimeout, CultureInfo culture) + { + // RightToLeft and ECMAScript are currently not supported in conjunction with NonBacktracking. + if ((options & (RegexOptions.RightToLeft | RegexOptions.ECMAScript)) != 0) + { + throw new NotSupportedException( + SR.Format(SR.NotSupported_NonBacktrackingConflictingOption, + (options & RegexOptions.RightToLeft) != 0 ? nameof(RegexOptions.RightToLeft) : nameof(RegexOptions.ECMAScript))); + } + + return new SymbolicRegexRunnerFactory(new SymbolicRegexRunner(code, matchTimeout, culture)); + } + protected override void InitTrackCount() { } // nop, no backtracking + + protected override bool FindFirstChar() => + // The real logic is all in Go. Here we simply validate if there's enough text remaining to possibly match. + runtextpos <= runtextend - _minRequiredLength; + + protected override void Go() + { + // Perform the match. + SymbolicMatch pos = _matcher.FindMatch(quick, runtext!, runtextpos, runtextend); + if (pos.Success) + { + // If we successfully matched, capture the match, and then jump the current position to the end of the match. + int start = pos.Index; + int end = start + pos.Length; + Capture(0, start, end); + runtextpos = end; + } + else + { + // If we failed to find a match in the entire remainder of the input, skip the current position to the end. + // The calling scan loop will then exit. + runtextpos = runtextend; + } + } + + private sealed class SymbolicRegex : Regex + { + public SymbolicRegex(SymbolicRegexRunner runner) => + factory = new SymbolicRegexRunnerFactory(runner); + } + } +} diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicRegexRunnerFactory.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicRegexRunnerFactory.cs new file mode 100644 index 00000000000000..2e0c5c5f557aa2 --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicRegexRunnerFactory.cs @@ -0,0 +1,23 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace System.Text.RegularExpressions.Symbolic +{ + /// for symbolic regexes. + internal sealed class SymbolicRegexRunnerFactory : RegexRunnerFactory + { + /// Shared runner instance. + internal readonly SymbolicRegexRunner _runner; + + /// Initializes the factory. + /// Shared runner instance. + public SymbolicRegexRunnerFactory(SymbolicRegexRunner runner) => _runner = runner; + + /// Creates a object. + /// + /// instances are observably immutable and thread-safe. + /// All calls to return the same shared instance. + /// + protected internal override RegexRunner CreateInstance() => _runner; + } +} diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicRegexSampler.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicRegexSampler.cs new file mode 100644 index 00000000000000..b6e761726b7007 --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicRegexSampler.cs @@ -0,0 +1,237 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#if DEBUG +using System.Collections.Generic; +using System.Diagnostics; + +namespace System.Text.RegularExpressions.Symbolic +{ + internal class SymbolicRegexSampler where S : notnull + { + private Random _random; + private SymbolicRegexNode _root; + /// The used random seed + public int RandomSeed { get; private set; } + private BDD _asciiWordCharacters; + private BDD _asciiNonWordCharacters; // omits all characters before ' ' + private BDD _ascii; // omits all characters before ' ' + private ICharAlgebra _solver; + + public SymbolicRegexSampler(SymbolicRegexNode root, int randomseed, bool negative) + { + _root = negative ? root._builder.MkNot(root) : root; + // Treat 0 as no seed and instead choose a random seed randomly + RandomSeed = randomseed == 0 ? new Random().Next() : randomseed; + _random = new Random(RandomSeed); + _solver = root._builder._solver; + ICharAlgebra bddSolver = SymbolicRegexRunner.s_unicode._solver; + _asciiWordCharacters = bddSolver.Or(new BDD[] { + bddSolver.RangeConstraint('A', 'Z'), + bddSolver.RangeConstraint('a', 'z'), + bddSolver.CharConstraint('_'), + bddSolver.RangeConstraint('0', '9')}); + // Visible ASCII range for input character generation + _ascii = bddSolver.RangeConstraint('\x20', '\x7E'); + _asciiNonWordCharacters = bddSolver.And(_ascii, bddSolver.Not(_asciiWordCharacters)); + } + + /// Generates up to k random strings accepted by the regex + public IEnumerable GenerateRandomMembers(int k) + { + ICharAlgebra bddSolver = SymbolicRegexRunner.s_unicode._solver; + for (int i = 0; i < k; i++) + { + // Holds the generated input so far + StringBuilder input_so_far = new(); + + // Initially there is no previous character + // Here one could also consider previous characters for example for \b, \B, and ^ anchors + // and initialize input_so_far accordingly + uint prevCharKind = CharKind.StartStop; + + // This flag is set to false in the unlikely situation that generation ends up in a dead-end + bool generationSucceeded = true; + + // Current set of states reached initially contains just the root + List> states = new(); + states.Add(_root); + + // Used for end suffixes + List possible_endings = new(); + + List> nextStates = new(); + + while (true) + { + Debug.Assert(states.Count > 0); + + if (CanBeFinal(states)) + { + // Unconditionally final state or end of the input due to \Z anchor for example + if (IsFinal(states) || IsFinal(states, CharKind.Context(prevCharKind, CharKind.StartStop))) + { + possible_endings.Add(""); + } + + // End of line due to end-of-line anchor + if (IsFinal(states, CharKind.Context(prevCharKind, CharKind.Newline))) + { + possible_endings.Add("\n"); + } + + // Related to wordborder due to \b or \B + if (IsFinal(states, CharKind.Context(prevCharKind, CharKind.WordLetter))) + { + possible_endings.Add(ChooseChar(_asciiWordCharacters).ToString()); + } + + // Related to wordborder due to \b or \B + if (IsFinal(states, CharKind.Context(prevCharKind, CharKind.General))) + { + possible_endings.Add(ChooseChar(_asciiNonWordCharacters).ToString()); + } + } + + // Choose to stop here based on a coin-toss + if (possible_endings.Count > 0 && ChooseRandomlyTrueOrFalse()) + { + //Choose some suffix that allows some anchor (if any) to be nullable + input_so_far.Append(Choose(possible_endings)); + break; + } + + SymbolicRegexNode state = Choose(states); + char c = '\0'; + uint cKind = 0; + // Observe that state.MkDerivative() can be a deadend + List<(S, SymbolicRegexNode?, SymbolicRegexNode)> paths = new(state.MkDerivative().EnumeratePaths(_solver.True)); + if (paths.Count > 0) + { + (S, SymbolicRegexNode?, SymbolicRegexNode) path = Choose(paths); + // Consider a random path from some random state in states and + // select a random member of the predicate on that path + c = ChooseChar(ToBDD(path.Item1)); + + // Map the character back into the corresponding character constraint of the solver + S c_pred = _solver.CharConstraint(c); + + // Determine the character kind of c + cKind = IsNewline(c_pred) ? CharKind.Newline : (IsWordchar(c_pred) ? CharKind.WordLetter : CharKind.General); + + // Construct the combined context of previous and c kind + uint context = CharKind.Context(prevCharKind, cKind); + + // Step into the next set of states + nextStates.AddRange(Step(states, c_pred, context)); + } + + // In the case that there are no next states: stop here + if (nextStates.Count == 0) + { + if (possible_endings.Count > 0) + { + input_so_far.Append(Choose(possible_endings)); + } + else + { + // Ending up here is unlikely but possible for example for infeasible patterns such as @"no\bway" + // or due to poor choice of c -- no anchor is enabled -- so this is a deadend + generationSucceeded = false; + } + break; + } + + input_so_far.Append(c); + states.Clear(); + possible_endings.Clear(); + List> tmp = states; + states = nextStates; + nextStates = tmp; + prevCharKind = cKind; + } + + if (generationSucceeded) + { + yield return input_so_far.ToString(); + } + } + } + + private IEnumerable> Step(List> states, S pred, uint context) + { + HashSet> seen = new(); + foreach (SymbolicRegexNode state in states) + { + foreach ((S, SymbolicRegexNode?, SymbolicRegexNode) path in state.MkDerivative().EnumeratePaths(pred)) + { + // Either there are no anchors or else check that the anchors are nullable in the given context + if (path.Item2 is null || path.Item2.IsNullableFor(context)) + { + // Omit repetitions from the enumeration + if (seen.Add(path.Item3)) + { + yield return path.Item3; + } + } + } + } + } + + private BDD ToBDD(S pred) => _solver.ConvertToCharSet(SymbolicRegexRunner.s_unicode._solver, pred); + private T Choose(IList elems) => elems[_random.Next(elems.Count)]; + private T Choose(IEnumerable elems) + { + List list = new List(elems); + return list[_random.Next(list.Count)]; + } + private char ChooseChar((uint, uint) pair) => (char)_random.Next((int)pair.Item1, (int)pair.Item2 + 1); + private char ChooseChar(BDD bdd) + { + Debug.Assert(!bdd.IsEmpty); + // Select characters from the visible ASCII range whenever possible + BDD bdd1 = SymbolicRegexRunner.s_unicode._solver.And(bdd, _ascii); + return ChooseChar(Choose(((CharSetSolver)SymbolicRegexRunner.s_unicode._solver).ToRanges(bdd1.IsEmpty ? bdd : bdd1))); + } + private bool ChooseRandomlyTrueOrFalse() => _random.Next(100) < 50; + /// Returns true if some state is unconditionally final + private bool IsFinal(IEnumerable> states) + { + foreach (SymbolicRegexNode state in states) + { + if (state.IsNullable) + { + return true; + } + } + return false; + } + /// Returns true if some state can be final + private bool CanBeFinal(IEnumerable> states) + { + foreach (SymbolicRegexNode state in states) + { + if (state.CanBeNullable) + { + return true; + } + } + return false; + } + /// Returns true if some state is final in the given context + private bool IsFinal(IEnumerable> states, uint context) + { + foreach (SymbolicRegexNode state in states) + { + if (state.IsNullableFor(context)) + { + return true; + } + } + return false; + } + private bool IsWordchar(S pred) => _solver.IsSatisfiable(_solver.And(pred, _root._builder._wordLetterPredicateForAnchors)); + private bool IsNewline(S pred) => _solver.IsSatisfiable(_solver.And(pred, _root._builder._newLinePredicate)); + } +} +#endif diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicRegexSet.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicRegexSet.cs new file mode 100644 index 00000000000000..da28c7cc43476f --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicRegexSet.cs @@ -0,0 +1,594 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; + +namespace System.Text.RegularExpressions.Symbolic +{ + /// Represents a set of symbolic regexes that is either a disjunction or a conjunction + internal sealed class SymbolicRegexSet : IEnumerable> where S : notnull + { + internal readonly SymbolicRegexBuilder _builder; + + private readonly HashSet> _set; + + /// + /// Symbolic regex A{0,k}?B is stored as (A,B,true) -> k -- lazy + /// Symbolic regex A{0,k}? is stored as (A,(),true) -> k -- lazy + /// Symbolic regex A{0,k}B is stored as (A,B,false) -> k -- eager + /// Symbolic regex A{0,k} is stored as (A,(),false) -> k -- eager + /// + private readonly Dictionary<(SymbolicRegexNode, SymbolicRegexNode, bool), int> _loops; + + /// the union (or intersection) of all singletons in the collection if any or null if none + private readonly SymbolicRegexNode? _singleton; + + internal readonly SymbolicRegexKind _kind; + + private int _hashCode; + + /// If >= 0 then the maximal length of a watchdog in the set + internal int _watchdog = -1; + + private SymbolicRegexSet(SymbolicRegexBuilder builder, SymbolicRegexKind kind, HashSet>? set, Dictionary<(SymbolicRegexNode, SymbolicRegexNode, bool), int>? loops, SymbolicRegexNode? singleton) + { + Debug.Assert(kind is SymbolicRegexKind.And or SymbolicRegexKind.Or); + Debug.Assert((set is null) == (loops is null)); + + _builder = builder; + _kind = kind; + _set = set ?? new HashSet>(); + _loops = loops ?? new Dictionary<(SymbolicRegexNode, SymbolicRegexNode, bool), int>(); + _singleton = singleton; + } + + /// Denotes the empty conjunction + public bool IsEverything => _kind == SymbolicRegexKind.And && _set.Count == 0 && _loops.Count == 0 && _singleton == null; + + /// Denotes the empty disjunction + public bool IsNothing => _kind == SymbolicRegexKind.Or && _set.Count == 0 && _loops.Count == 0 && _singleton == null; + + /// How many elements are there in this set + public int Count => _set.Count + _loops.Count + (_singleton == null ? 0 : 1); + + /// True iff the set is a singleton + public bool IsSingleton => Count == 1; + + internal static SymbolicRegexSet CreateFull(SymbolicRegexBuilder builder) => new SymbolicRegexSet(builder, SymbolicRegexKind.And, null, null, null); + + internal static SymbolicRegexSet CreateEmpty(SymbolicRegexBuilder builder) => new SymbolicRegexSet(builder, SymbolicRegexKind.Or, null, null, null); + + internal static SymbolicRegexSet CreateMulti(SymbolicRegexBuilder builder, IEnumerable> elems, SymbolicRegexKind kind) + { + // Loops contains the actual multi-set part of the collection + var loops = new Dictionary<(SymbolicRegexNode, SymbolicRegexNode, bool), int>(); + + // Other represents a normal set + var other = new HashSet>(); + + // Combination of singletons (when not null) + SymbolicRegexNode? singleton = null; + + int watchdog = -1; + + foreach (SymbolicRegexNode elem in elems) + { + // Keep track of the maximal watchdog if this is a disjunction + // this means for example if the regex is abc(3)|bc(2) and + // the input is xxxabcyyy then two watchdogs will occur (3) and (2) + // after reading c and the maximal one is taken + // in a conjuctive setting this is undefined and the watchdog remains -1 + if (kind == SymbolicRegexKind.Or && + elem._kind == SymbolicRegexKind.WatchDog && elem._lower > watchdog) + { + watchdog = elem._lower; + } + + #region start foreach + if (elem == builder._anyStar) + { + // .* is the absorbing element for disjunction + if (kind == SymbolicRegexKind.Or) + { + return builder._fullSet; + } + } + else if (elem == builder._nothing) + { + // [] is the absorbing element for conjunction + if (kind == SymbolicRegexKind.And) + { + return builder._emptySet; + } + } + else + { + switch (elem._kind) + { + case SymbolicRegexKind.And: + case SymbolicRegexKind.Or: + Debug.Assert(elem._alts is not null); + if (kind == elem._kind) + { + // Flatten the inner set + foreach (SymbolicRegexNode alt in elem._alts) + { + if (alt._kind == SymbolicRegexKind.Loop && alt._lower == 0) + { + AddLoopElement(builder, loops, other, alt, builder._epsilon, kind); + } + else + { + if (alt._kind == SymbolicRegexKind.Concat && alt._left!._kind == SymbolicRegexKind.Loop && alt._left._lower == 0) + { + Debug.Assert(alt._right is not null); + AddLoopElement(builder, loops, other, alt._left, alt._right, kind); + } + else + { + if (alt._kind == SymbolicRegexKind.Singleton) + { + Debug.Assert(alt._set is not null); + if (singleton is null) + { + singleton = alt; + } + else + { + Debug.Assert(singleton._kind == SymbolicRegexKind.Singleton && singleton._set is not null); + // Join the predicates either by Intersecting or Unioning + // which at the character predicate level translates to conjunction or disjunction in the underlying character algebra + S pred = kind == SymbolicRegexKind.Or ? builder._solver.Or(singleton._set, alt._set) : builder._solver.And(singleton._set, alt._set); + singleton = SymbolicRegexNode.MkSingleton(builder, pred); + } + } + else + { + other.Add(alt); + } + } + } + } + } + else + { + other.Add(elem); + } + break; + + case SymbolicRegexKind.Loop: + if (elem._lower == 0) + { + AddLoopElement(builder, loops, other, elem, builder._epsilon, kind); + } + else + { + other.Add(elem); + } + break; + + case SymbolicRegexKind.Concat: + Debug.Assert(elem._left is not null && elem._right is not null); + if (elem._kind == SymbolicRegexKind.Concat && elem._left._kind == SymbolicRegexKind.Loop && elem._left._lower == 0) + { + AddLoopElement(builder, loops, other, elem._left, elem._right, kind); + } + else + { + other.Add(elem); + } + break; + + case SymbolicRegexKind.Singleton: + Debug.Assert(elem._set is not null); + if (singleton is null) + { + singleton = elem; + } + else + { + Debug.Assert(singleton._kind == SymbolicRegexKind.Singleton && singleton._set is not null); + // Join the predicates either by Intersecting or Unioning + // which at the character predicate level translates to conjunction or disjunction in the underlying character algebra + S pred = kind == SymbolicRegexKind.Or ? builder._solver.Or(singleton._set, elem._set) : builder._solver.And(singleton._set, elem._set); + singleton = SymbolicRegexNode.MkSingleton(builder, pred); + } + break; + + default: + other.Add(elem); + break; + } + } + #endregion + } + + // This optimization is only valid for a conjunction/intersection + if (kind == SymbolicRegexKind.And && singleton is not null && singleton.Equals(builder._solver.False)) + { + return builder._emptySet; + } + + // The following is only valid for a disjunction/union + if (kind == SymbolicRegexKind.Or) + { + // If any element of other is covered in loops then omit it + var others1 = new HashSet>(); + foreach (SymbolicRegexNode sr in other) + { + // If there is an element A{0,m} then A is not needed because + // it is included by the loop due to the upper bound m > 0 + if (loops.ContainsKey((sr, builder._epsilon, false))) + { + others1.Add(sr); + } + } + + foreach (KeyValuePair<(SymbolicRegexNode, SymbolicRegexNode, bool), int> pair in loops) + { + // If there is an element A{0,m}B then B is not needed because + // it is included by the concatenation due to the lower bound 0 + if (other.Contains(pair.Key.Item2)) + { + others1.Add(pair.Key.Item2); + } + } + + other.ExceptWith(others1); + } + + return + other.Count != 0 || loops.Count != 0 || singleton is not null ? new SymbolicRegexSet(builder, kind, other, loops, singleton) { _watchdog = watchdog } : + kind == SymbolicRegexKind.Or ? builder._emptySet : + builder._fullSet; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void AddLoopElement( + SymbolicRegexBuilder builder, + Dictionary<(SymbolicRegexNode, SymbolicRegexNode, bool), int> loops, + HashSet> other, SymbolicRegexNode loop, + SymbolicRegexNode rest, + SymbolicRegexKind kind) + { + if (loop._upper == 0 && rest.IsEpsilon) + { + // In a set treat a loop with upper=lower=0 and no rest (no continuation after the loop) + // as () independent of whether it is lazy or eager + other.Add(builder._epsilon); + } + else + { + Debug.Assert(loop._left is not null); + var key = (loop._left, rest, loop.IsLazy); + if (!loops.TryGetValue(key, out int count) || + (kind == SymbolicRegexKind.Or ? count < loop._upper : count > loop._upper)) // If disjunction then map to the maximum of the upper bounds else to the minimum + { + loops[key] = loop._upper; + } + } + } + } + + public SymbolicRegexSet Restrict(S pred) + { + return CreateMulti(_builder, RestrictElements(pred), _kind); + + IEnumerable> RestrictElements(S pred) + { + foreach (SymbolicRegexNode elem in this) + { + yield return elem.Restrict(pred); + } + } + } + + internal bool IsNullableFor(uint context) + { + Enumerator e = GetEnumerator(); + + if (_kind == SymbolicRegexKind.Or) + { + // Some element must be nullable + while (e.MoveNext()) + { + if (e.Current.IsNullableFor(context)) + { + return true; + } + } + + return false; + } + else + { + Debug.Assert(_kind == SymbolicRegexKind.And); + + // All elements must be nullable + while (e.MoveNext()) + { + if (!e.Current.IsNullableFor(context)) + { + return false; + } + } + + return true; + } + } + + public override int GetHashCode() + { + if (_hashCode == 0) + { + int hashCode = _kind.GetHashCode(); + + if (_singleton is not null) + { + hashCode ^= _singleton.GetHashCode(); + } + + foreach (SymbolicRegexNode n in _set) + { + hashCode ^= n.GetHashCode(); + } + + foreach (KeyValuePair<(SymbolicRegexNode, SymbolicRegexNode, bool), int> entry in _loops) + { + hashCode ^= entry.Key.GetHashCode() + entry.Value.GetHashCode(); + } + + _hashCode = hashCode; + } + + return _hashCode; + } + + public override bool Equals([NotNullWhen(true)] object? obj) + { + if (obj is not SymbolicRegexSet that || + _kind != that._kind || + _singleton is null && that._singleton is not null || + _singleton is not null && !_singleton.Equals(that._singleton) || + _set.Count != that._set.Count || + _loops.Count != that._loops.Count || + (_set.Count > 0 && !_set.SetEquals(that._set))) + { + return false; + } + + foreach (KeyValuePair<(SymbolicRegexNode, SymbolicRegexNode, bool), int> c in _loops) + { + if (!that._loops.TryGetValue(c.Key, out int count) || !count.Equals(c.Value)) + { + return false; + } + } + + return true; + } + + public void ToString(StringBuilder sb) + { + if (IsNothing) + { + sb.Append(SymbolicRegexNode.EmptyCharClass); + } + else if (IsEverything) + { + sb.Append(".*"); + } + else + { + Enumerator enumerator = GetEnumerator(); + bool nonempty = enumerator.MoveNext(); + Debug.Assert(nonempty, "Collection must be nonempty because IsNothing is false and IsEverything is false"); + SymbolicRegexNode node = enumerator.Current; + if (!enumerator.MoveNext()) + { + // The collection only has one element + node.ToString(sb); + } + else + { + // Union of two or more elements + sb.Append('('); + // Append the first two elements + node.ToString(sb); + // Using the operator & for intersection + char op = _kind == SymbolicRegexKind.Or ? '|' : '&'; + sb.Append(op); + enumerator.Current.ToString(sb); + while (enumerator.MoveNext()) + { + // Append all the remaining elements + sb.Append(op); + enumerator.Current.ToString(sb); + } + sb.Append(')'); + } + } + } + + internal SymbolicRegexSet CreateDerivative(S elem, uint context) + { + return CreateMulti(_builder, MkDerivativesOfElems(elem, context), _kind); + + IEnumerable> MkDerivativesOfElems(S elem, uint context) + { + foreach (SymbolicRegexNode s in this) + { + yield return s.MkDerivative(elem, context); + } + } + } + + internal SymbolicRegexSet Transform(SymbolicRegexBuilder builderT, Func predicateTransformer) where T : notnull + { + return SymbolicRegexSet.CreateMulti(builderT, TransformElements(builderT, predicateTransformer), _kind); + + IEnumerable> TransformElements(SymbolicRegexBuilder builderT, Func predicateTransformer) + { + foreach (SymbolicRegexNode sr in this) + { + yield return _builder.Transform(sr, builderT, predicateTransformer); + } + } + } + + internal SymbolicRegexNode GetSingletonElement() + { + Debug.Assert(IsSingleton); + + Enumerator e = GetEnumerator(); + bool success = e.MoveNext(); + Debug.Assert(success); + return e.Current; + } + + internal SymbolicRegexSet Reverse() + { + return CreateMulti(_builder, ReverseElements(), _kind); + + IEnumerable> ReverseElements() + { + foreach (SymbolicRegexNode n in this) + { + yield return n.Reverse(); + } + } + } + + internal bool StartsWithLoop(int upperBoundLowestValue) + { + foreach (SymbolicRegexNode n in this) + { + if (n.StartsWithLoop(upperBoundLowestValue)) + { + return true; + } + } + + return false; + } + + public Enumerator GetEnumerator() => new Enumerator(this); + + IEnumerator> IEnumerable>.GetEnumerator() => new Enumerator(this); + + IEnumerator IEnumerable.GetEnumerator() => new Enumerator(this); + + internal int GetFixedLength() + { + if (_loops.Count > 0) + { + return -1; + } + + int length = -1; + foreach (SymbolicRegexNode node in _set) + { + int nodeLength = node.GetFixedLength(); + + if (nodeLength == -1) + { + return -1; + } + else if (length == -1) + { + length = nodeLength; + } + else if (length != nodeLength) + { + return -1; + } + } + + if (_singleton is not null && length != 1) + { + if (length == -1) + { + length = 1; + } + else + { + length = -1; + } + } + + return length; + } + + /// Enumerates all symbolic regexes in the set + internal struct Enumerator : IEnumerator> + { + private readonly SymbolicRegexSet _set; + private int _state; // 0 = return singleton, 1 == iterate set, 2 == iterate loops, 3 == done + private SymbolicRegexNode? _current; + private HashSet>.Enumerator _setEnumerator; + private Dictionary<(SymbolicRegexNode, SymbolicRegexNode, bool), int>.Enumerator _loopsEnumerator; + + internal Enumerator(SymbolicRegexSet symbolicRegexSet) + { + _state = symbolicRegexSet._singleton is null ? 1 : 0; + _set = symbolicRegexSet; + _setEnumerator = symbolicRegexSet._set.GetEnumerator(); + _loopsEnumerator = symbolicRegexSet._loops.GetEnumerator(); + _current = null; + } + + public SymbolicRegexNode Current => _current!; + + object IEnumerator.Current => Current; + + public void Dispose() + { + _state = 3; + _setEnumerator.Dispose(); + _loopsEnumerator.Dispose(); + } + + public bool MoveNext() + { + switch (_state) + { + case 0: + Debug.Assert(_set._singleton is not null); + _current = _set._singleton; + _state = 1; + return true; + + case 1: + if (_setEnumerator.MoveNext()) + { + _current = _setEnumerator.Current; + return true; + } + _state = 2; + goto case 2; + + case 2: + if (_loopsEnumerator.MoveNext()) + { + // Recreate the symbolic regex from (body,rest)->k to body{0,k}rest + (SymbolicRegexNode body, SymbolicRegexNode rest, bool isLazy) = _loopsEnumerator.Current.Key; + int upper = _loopsEnumerator.Current.Value; + _current = _set._builder.MkConcat(_set._builder.MkLoop(body, isLazy, 0, upper), rest); + return true; + } + _state = 3; + goto default; + + default: + _current = null!; + return false; + } + } + + public void Reset() => throw new NotSupportedException(); + } + } +} diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/TransitionRegex.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/TransitionRegex.cs new file mode 100644 index 00000000000000..9d472c3c62b26b --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/TransitionRegex.cs @@ -0,0 +1,435 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics; + +namespace System.Text.RegularExpressions.Symbolic +{ + /// Represents a symbolic derivative created from a symbolic regex without using minterms + internal class TransitionRegex : IEnumerable<(S, SymbolicRegexNode?, SymbolicRegexNode)> where S : notnull + { + public readonly SymbolicRegexBuilder _builder; + public readonly TransitionRegexKind _kind; + public readonly S? _test; + public readonly TransitionRegex? _first; + public readonly TransitionRegex? _second; + public readonly SymbolicRegexNode? _node; + + private readonly int _hashCode; + + public bool IsNothing + { + get + { + if (_kind == TransitionRegexKind.Leaf) + { + Debug.Assert(_node != null); + return _node.IsNothing; + } + return false; + } + } + + public bool IsAnyStar + { + get + { + if (_kind == TransitionRegexKind.Leaf) + { + Debug.Assert(_node != null); + return _node.IsAnyStar; + } + return false; + } + } + + private TransitionRegex(SymbolicRegexBuilder builder, TransitionRegexKind kind, S? test, TransitionRegex? first, TransitionRegex? second, SymbolicRegexNode? node) + { + Debug.Assert(builder is not null); + Debug.Assert( + kind is TransitionRegexKind.Leaf && node is not null && Equals(test, default(S)) && first is null && second is null || + kind is TransitionRegexKind.Conditional && test is not null && first is not null && second is not null && node is null || + kind is TransitionRegexKind.Union && Equals(test, default(S)) && first is not null && second is not null && node is null || + kind is TransitionRegexKind.Lookaround && Equals(test, default(S)) && first is not null && second is not null && node is not null); + _builder = builder; + _kind = kind; + _test = test; + _first = first; + _second = second; + _node = node; + _hashCode = HashCode.Combine(kind, test, first, second, node); + } + + private static TransitionRegex Create(SymbolicRegexBuilder builder, TransitionRegexKind kind, S? test, TransitionRegex? one, TransitionRegex? two, SymbolicRegexNode? node) + { + // Keep transition regexes internalized using the builder + (TransitionRegexKind, S?, TransitionRegex?, TransitionRegex?, SymbolicRegexNode?) key = (kind, test, one, two, node); + TransitionRegex? tr; + if (!builder._trCache.TryGetValue(key, out tr)) + { + tr = new TransitionRegex(builder, kind, test, one, two, node); + builder._trCache[key] = tr; + } + return tr; + } + + public TransitionRegex Complement() + { + switch (_kind) + { + case TransitionRegexKind.Leaf: + Debug.Assert(_node is not null); + // Complement is propagated to the leaf + return Create(_builder, _kind, default(S), null, null, _node._builder.MkNot(_node)); + case TransitionRegexKind.Union: + Debug.Assert(_first is not null && _second is not null); + // Apply deMorgan's laws + return Intersect(_first.Complement(), _second.Complement()); + default: + Debug.Assert(_first is not null && _second is not null); + // Both Conditional and Nullability obey the same laws of propagation of complement + return Create(_builder, _kind, _test, _first.Complement(), _second.Complement(), _node); + } + } + + public static TransitionRegex Leaf(SymbolicRegexNode node) => + Create(node._builder, TransitionRegexKind.Leaf, default(S), null, null, node); + + /// Concatenate a node at the end of this transition regex + public TransitionRegex Concat(SymbolicRegexNode node) + { + switch (_kind) + { + case TransitionRegexKind.Leaf: + Debug.Assert(_node is not null); + return Create(_builder, _kind, default(S), null, null, _node._builder.MkConcat(_node, node)); + default: + // All other three cases are disjunctive and obey the same laws of propagation of complement + Debug.Assert(_first is not null && _second is not null); + return Create(_builder, _kind, _test, _first.Concat(node), _second.Concat(node), _node); + } + } + + private static TransitionRegex Intersect(TransitionRegex one, TransitionRegex two) + { + // Apply standard simplifications + // [] & t = [], t & .* = t + if (one.IsNothing || two.IsAnyStar || one == two) + { + return one; + } + + // t & [] = [], .* & t = t + if (two.IsNothing || one.IsAnyStar) + { + return two; + } + + return one.IntersectWith(two, one._builder._solver.True); + } + + private TransitionRegex IntersectWith(TransitionRegex that, S pathIn) + { + Debug.Assert(_builder._solver.IsSatisfiable(pathIn)); + + #region Conditional + // Intersect when this is a Conditional + if (_kind == TransitionRegexKind.Conditional) + { + Debug.Assert(_test is not null && _first is not null && _second is not null); + S thenPath = _builder._solver.And(pathIn, _test); + S elsePath = _builder._solver.And(pathIn, _builder._solver.Not(_test)); + if (!_builder._solver.IsSatisfiable(thenPath)) + { + // then case being infeasible implies that elsePath must be satisfiable + return _second.IntersectWith(that, elsePath); + } + if (!_builder._solver.IsSatisfiable(elsePath)) + { + // else case is infeasible + return _first.IntersectWith(that, thenPath); + } + TransitionRegex thencase = _first.IntersectWith(that, thenPath); + TransitionRegex elsecase = _second.IntersectWith(that, elsePath); + if (thencase == elsecase) + { + // Both branches result in the same thing, so the test can be omitted + return thencase; + } + return Create(_builder, TransitionRegexKind.Conditional, _test, thencase, elsecase, null); + } + + // Swap the order of this and that if that is a Conditional + if (that._kind == TransitionRegexKind.Conditional) + { + return that.IntersectWith(this, pathIn); + } + #endregion + + #region Union + // Intersect when this is a Union + // Use the following law of distributivity: (A|B)&C = A&C|B&C + if (_kind == TransitionRegexKind.Union) + { + Debug.Assert(_first is not null && _second is not null); + return Union(_first.IntersectWith(that, pathIn), _second.IntersectWith(that, pathIn)); + } + + // Swap the order of this and that if that is a Union + if (that._kind == TransitionRegexKind.Union) + { + return that.IntersectWith(this, pathIn); + } + #endregion + + #region Nullability + if (_kind == TransitionRegexKind.Lookaround) + { + Debug.Assert(_node is not null && _first is not null && _second is not null); + return Lookaround(_node, _first.IntersectWith(that, pathIn), _second.IntersectWith(that, pathIn)); + } + + if (that._kind == TransitionRegexKind.Lookaround) + { + Debug.Assert(that._node is not null && that._first is not null && that._second is not null); + return Lookaround(that._node, that._first.IntersectWith(this, pathIn), that._second.IntersectWith(this, pathIn)); + } + #endregion + + // Propagate intersection to the leaves + Debug.Assert(_kind is TransitionRegexKind.Leaf && that._kind is TransitionRegexKind.Leaf && _node is not null && that._node is not null); + return Leaf(_builder.MkAnd(_node, that._node)); + } + + private static TransitionRegex Union(TransitionRegex one, TransitionRegex two) + { + // Apply common simplifications, always trying to push the operations into the leaves or to eliminate redundant branches + if (one.IsNothing || two.IsAnyStar || one == two) + { + return two; + } + + if (two.IsNothing || one.IsAnyStar) + { + return one; + } + + if (one._kind == TransitionRegexKind.Conditional && two._kind == TransitionRegexKind.Conditional) + { + Debug.Assert(one._test is not null && one._first is not null && one._second is not null); + Debug.Assert(two._test is not null && two._first is not null && two._second is not null); + + // if(psi, t1, t2) | if(psi, s1, s2) = if(psi, t1|s1, t2|s2) + if (one._test.Equals(two._test)) + { + return Conditional(one._test, Union(one._first, two._first), Union(one._second, two._second)); + } + + // if(psi, t, []) | if(phi, t, []) = if(psi or phi, t, []) + if (one._second.IsNothing && two._second.IsNothing && one._first.Equals(two._first)) + { + return Conditional(one._builder._solver.Or(one._test, two._test), one._first, one._second); + } + } + + // TODO-NONBACKTRACKING: keep the representation of Union in right-associative form ordered by hashcode "as a list" + // so that in a Union, _first is never a union and _first._hashcode is less than _second._hashcode (if _second is not a Union) + // and if _second is a union then _first._hashcode is less than _second._first._hashcode, etc. + // This will help to maintain a canonical representation of two equivalent unions and avoid equivalent unions being nonequal + return Create(one._builder, TransitionRegexKind.Union, default(S), one, two, null); + } + + public static TransitionRegex Conditional(S test, TransitionRegex thencase, TransitionRegex elsecase) => + (thencase == elsecase || thencase._builder._solver.True.Equals(test)) ? thencase : + thencase._builder._solver.False.Equals(test) ? elsecase : + Create(thencase._builder, TransitionRegexKind.Conditional, test, thencase, elsecase, null); + + public static TransitionRegex Lookaround(SymbolicRegexNode nullabilityTest, TransitionRegex thencase, TransitionRegex elsecase) => + (thencase == elsecase) ? thencase : Create(thencase._builder, TransitionRegexKind.Lookaround, default(S), thencase, elsecase, nullabilityTest); + + /// Intersection of transition regexes + public static TransitionRegex operator &(TransitionRegex one, TransitionRegex two) => Intersect(one, two); + + /// Union of transition regexes + public static TransitionRegex operator |(TransitionRegex one, TransitionRegex two) => Union(one, two); + + /// Complement of transition regex + public static TransitionRegex operator ~(TransitionRegex tr) => tr.Complement(); + + public override int GetHashCode() => _hashCode; + + public override string ToString() + { + switch (_kind) + { + case TransitionRegexKind.Leaf: + return $"{_node}"; + case TransitionRegexKind.Union: + return $"{_first}|{_second}"; + case TransitionRegexKind.Conditional: + return $"if({_test},{_first},{_second})"; + default: + return $"if(IsNull({_node}),{_first},{_second})"; + } + } + + public IEnumerator<(S, SymbolicRegexNode?, SymbolicRegexNode)> GetEnumerator() => EnumeratePaths(_builder._solver.True).GetEnumerator(); + IEnumerator IEnumerable.GetEnumerator() => EnumeratePaths(_builder._solver.True).GetEnumerator(); + + /// Enumerates all the paths in this transition regex excluding dead-end paths + public IEnumerable<(S, SymbolicRegexNode?, SymbolicRegexNode)> EnumeratePaths(S pathCondition) + { + switch (_kind) + { + case TransitionRegexKind.Leaf: + Debug.Assert(_node is not null); + // Omit any path that leads to a deadend + if (!_node.IsNothing) + { + yield return (pathCondition, null, _node); + } + break; + + case TransitionRegexKind.Union: + Debug.Assert(_first is not null && _second is not null); + foreach (var path in _first.EnumeratePaths(pathCondition)) + { + yield return path; + } + foreach (var path in _second.EnumeratePaths(pathCondition)) + { + yield return path; + } + break; + + case TransitionRegexKind.Conditional: + Debug.Assert(_test is not null && _first is not null && _second is not null); + foreach (var path in _first.EnumeratePaths(_builder._solver.And(pathCondition, _test))) + { + yield return path; + } + foreach (var path in _second.EnumeratePaths(_builder._solver.And(pathCondition, _builder._solver.Not(_test)))) + { + yield return path; + } + break; + + default: + Debug.Assert(_kind is TransitionRegexKind.Lookaround && _node is not null && _first is not null && _second is not null); + foreach (var path in _first.EnumeratePaths(pathCondition)) + { + SymbolicRegexNode nullabilityTest = path.Item2 is null ? _node : _builder.MkAnd(path.Item2, _node); + yield return (path.Item1, nullabilityTest, path.Item3); + } + foreach (var path in _second.EnumeratePaths(pathCondition)) + { + // Complement the nullability test + SymbolicRegexNode nullabilityTest = path.Item2 is null ? _builder.MkNot(_node) : _builder.MkAnd(path.Item2, _builder.MkNot(_node)); + yield return (path.Item1, nullabilityTest, path.Item3); + } + break; + } + } + + /// Enumerate all distinct leaves that are not nothing. + public IEnumerable> EnumerateLeaves() + { + Stack> todo = new(); + HashSet> done = new(); + todo.Push(this); + while (todo.Count > 0) + { + TransitionRegex top = todo.Pop(); + switch (top._kind) + { + case TransitionRegexKind.Leaf: + // Omit any leaf that is nothing or has already been yielded + if (!top.IsNothing && done.Add(top)) + { + yield return top; + } + break; + + default: + Debug.Assert(top._first is not null && top._second is not null); + // In general the structure is a DAG so avoid yielding duplicates + if (done.Add(top._second)) + { + todo.Push(top._second); + } + if (done.Add(top._first)) + { + todo.Push(top._first); + } + break; + } + } + } + + public SymbolicRegexNode Transition(S minterm, uint context) + { + // Collect the union of all target leaves + SymbolicRegexNode target = _builder._nothing; + Stack> todo = new(); + todo.Push(this); + while (todo.Count > 0) + { + TransitionRegex top = todo.Pop(); + switch (top._kind) + { + case TransitionRegexKind.Leaf: + Debug.Assert(top._node is not null); + target = _builder.MkOr2(target, top._node); + break; + + case TransitionRegexKind.Conditional: + Debug.Assert(top._test is not null && top._first is not null && top._second is not null); + if (_builder._solver.IsSatisfiable(_builder._solver.And(minterm, top._test))) + { + if (!top._first.IsNothing) + { + todo.Push(top._first); + } + } + else + { + if (!top._second.IsNothing) + { + todo.Push(top._second); + } + } + break; + + case TransitionRegexKind.Union: + // Observe that without Union Transition returns excatly one of the leaves + Debug.Assert(top._first is not null && top._second is not null); + todo.Push(top._second); + todo.Push(top._first); + break; + + default: + Debug.Assert(top._kind is TransitionRegexKind.Lookaround && top._node is not null && top._first is not null && top._second is not null); + // Branch according to the result of nullability + if (top._node.IsNullableFor(context)) + { + if (!top._first.IsNothing) + { + todo.Push(top._first); + } + } + else + { + if (!top._second.IsNothing) + { + todo.Push(top._second); + } + } + break; + } + } + return target; + } + } +} diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/TransitionRegexKind.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/TransitionRegexKind.cs new file mode 100644 index 00000000000000..6be3c169a94093 --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/TransitionRegexKind.cs @@ -0,0 +1,14 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace System.Text.RegularExpressions.Symbolic +{ + /// Kinds of transition regexes. Transition regexes maintain a DNF form that pushes all intersections and complements to the leaves. + internal enum TransitionRegexKind + { + Leaf, + Conditional, + Union, + Lookaround + } +} diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Unicode/GeneratorHelper.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Unicode/GeneratorHelper.cs new file mode 100644 index 00000000000000..871eb15e2dc16b --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Unicode/GeneratorHelper.cs @@ -0,0 +1,22 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.IO; + +namespace System.Text.RegularExpressions.Symbolic.Unicode +{ +#if DEBUG + internal static class GeneratorHelper + { + public static void WriteInt64ArrayInitSyntax(StreamWriter sw, long[] values) + { + sw.Write("new long[] {"); + for (int i = 0; i < values.Length; i++) + { + sw.Write($" 0x{values[i]:X}, "); + } + sw.Write("}"); + } + } +#endif +} diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Unicode/IgnoreCaseRelation.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Unicode/IgnoreCaseRelation.cs new file mode 100644 index 00000000000000..458aabd55c6524 --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Unicode/IgnoreCaseRelation.cs @@ -0,0 +1,14 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// This is a programmatically generated file from Regex.GenerateUnicodeTables. +// It provides serialized BDD Unicode category definitions for System.Environment.Version = 7.0.0 + +namespace System.Text.RegularExpressions.Symbolic.Unicode +{ + internal static class IgnoreCaseRelation + { + /// Serialized BDD for mapping characters to their case-ignoring equivalence classes in the default (en-US) culture. + public static readonly long[] IgnoreCaseEnUsSerializedBDD = new long[] { 0x5, 0xE, 0x80000, 0x20, 0x41, 0x180001, 0x100001, 0x61, 0x21, 0x80001, 0x180041, 0x180021, 0x80041, 0x200002, 0x280002, 0x300002, 0xE2, 0x82, 0xA2, 0xC2, 0x380002, 0x400002, 0x102, 0x122, 0x480002, 0x3800C2, 0x500002, 0x142, 0x580002, 0x4000C2, 0x182, 0x1A3, 0x1C3, 0x1E3, 0x800003, 0x880003, 0x900003, 0x980003, 0xA00003, 0x680003, 0x700003, 0x780003, 0x203, 0x223, 0x243, 0x263, 0x283, 0xA80003, 0x2C3, 0x2E3, 0x2A3, 0x303, 0xB00003, 0xB80003, 0xC00003, 0x323, 0x343, 0x363, 0xD80003, 0x8001E3, 0xC80003, 0xA00263, 0x900263, 0x780203, 0xD00003, 0x6801C3, 0xE00003, 0x3A3, 0x800303, 0xF00003, 0x3E4, 0x404, 0x424, 0x444, 0x464, 0x484, 0x4A4, 0x4C4, 0x4E4, 0x504, 0x524, 0x1500004, 0x1580004, 0x1600004, 0x1680004, 0x1700004, 0xF80004, 0x1000004, 0x1080004, 0x1100004, 0x1180004, 0x1200004, 0x1280004, 0x1300004, 0x1380004, 0x1400004, 0x544, 0x564, 0x584, 0x5A4, 0x5C4, 0x1480004, 0x1780004, 0x604, 0x624, 0x644, 0x664, 0x684, 0x6A4, 0x1980004, 0x1A00004, 0x1A80004, 0x1B00004, 0x5E4, 0x6C4, 0x1B80004, 0x6E4, 0x704, 0x1800004, 0x1880004, 0x1900004, 0x1C80004, 0x744, 0x1C00004, 0x1D80004, 0x1200544, 0x1280564, 0x1300584, 0x13805A4, 0x14005C4, 0x14803E4, 0x1E00004, 0x7A4, 0x1F00004, 0x700004, 0x780004, 0x1C4, 0x1E4, 0x1E80004, 0x800004, 0x880004, 0x900004, 0x980004, 0xA00004, 0x680004, 0x204, 0x224, 0x244, 0x264, 0x284, 0x1A4, 0x1F80004, 0x1D00004, 0x2000004, 0x824, 0x764, 0x1500444, 0x844, 0x784, 0x804, 0x1500524, 0x2180004, 0x2200004, 0x2280004, 0x724, 0x2300005, 0x8C5, 0x2380005, 0x8E5, 0x2400005, 0x905, 0x2480005, 0x925, 0x2500005, 0x945, 0x2580005, 0x965, 0x2600005, 0x985, 0x2680005, 0x9A5, 0x2700005, 0x9C5, 0x2780005, 0x9E5, 0x2800005, 0xA05, 0x2880005, 0xA25, 0x2900005, 0xA45, 0x2980005, 0xA65, 0x2A00005, 0xA85, 0x2A80005, 0xAA5, 0x2B00005, 0xAC5, 0x2B80005, 0xAE5, 0x2C00005, 0xB05, 0x2C80005, 0xB25, 0x2D00005, 0xB45, 0x2D80005, 0xB65, 0x2E00005, 0xB85, 0x2E80005, 0xBA5, 0x2F00005, 0xBC5, 0x2F80005, 0xBE5, 0xC05, 0xC25, 0xC45, 0xC65, 0xC85, 0xCA5, 0x3000005, 0x3080005, 0x3100005, 0x3180005, 0x3200005, 0x3280005, 0x3300005, 0xCE5, 0xD05, 0xD25, 0xD45, 0xD65, 0xD85, 0xDA5, 0xDC5, 0xDE5, 0xCC5, 0xE05, 0x3380005, 0x3400005, 0x3480005, 0x3500005, 0x3580005, 0x3880005, 0x3900005, 0x3980005, 0x3700005, 0x3780005, 0x3800005, 0xE85, 0xEA5, 0xEC5, 0xEE5, 0xF05, 0x3600005, 0x3B00005, 0x3B80005, 0x3C00005, 0x3680005, 0xE25, 0xF25, 0xE65, 0xE45, 0xF45, 0xF65, 0xF85, 0x2880C05, 0x2900C25, 0x2980C45, 0x2A00C65, 0x2A80C85, 0x2B008C5, 0x2B808E5, 0x2C00905, 0x2C80925, 0x2D00945, 0x2D80965, 0x2E00985, 0x2E809A5, 0x2F009C5, 0x2F809E5, 0x3280A05, 0x2880AE5, 0x2900B05, 0x2980B25, 0x2A00B45, 0x2A80B65, 0x2B00B85, 0x2B80BA5, 0x2C00BC5, 0x2C80BE5, 0x2D00CA5, 0x3E80005, 0x3F00005, 0x3F80005, 0x4000005, 0x4080005, 0x4100005, 0x4180005, 0x545, 0x565, 0x585, 0x5A5, 0x5C5, 0x3E5, 0x405, 0x425, 0x445, 0x465, 0x485, 0x4A5, 0x4C5, 0x4E5, 0x505, 0x525, 0x1085, 0x10A5, 0x10C5, 0x10E5, 0x4400005, 0x4480005, 0x4500005, 0x1105, 0x1125, 0x1165, 0x1185, 0x11A5, 0x11C5, 0x11E5, 0x1205, 0x4880005, 0x4900005, 0x4980005, 0x4A00005, 0x4A80005, 0x4B00005, 0x4580005, 0x4600005, 0x4680005, 0x4700005, 0x4780005, 0x4800005, 0x4300005, 0x4380005, 0x1225, 0x1245, 0x1265, 0x1285, 0x12A5, 0x12C5, 0x4B80005, 0x3C80005, 0x3D80005, 0x3E00005, 0x4C00005, 0x4C80005, 0x1345, 0x1365, 0x4E00005, 0x2B00A25, 0x2B80A45, 0x2C00A65, 0x2D00A85, 0x13A5, 0x3A00005, 0x3A80005, 0x4D80005, 0x3D00005, 0x4F00005, 0x4F80005, 0x5000005, 0x5080005, 0x5100005, 0x5180005, 0x13C5, 0x1405, 0x1485, 0x53014A6, 0x54014E6, 0x5501526, 0x5601566, 0x57015A6, 0x58015E6, 0x5901626, 0x5A01666, 0x5B016A6, 0x5C016E6, 0x5D01726, 0x5E01766, 0x5F017A6, 0x60017E6, 0x6101826, 0x6201866, 0x63018A6, 0x64018E6, 0x6501926, 0x6601966, 0x67019A6, 0x68019E6, 0x6901A26, 0x6A01A66, 0x6B01AA6, 0x6C01AE6, 0x6C80006, 0x1786, 0x6D00006, 0x17C6, 0x6D80006, 0x1806, 0x6E00006, 0x1846, 0x6E80006, 0x1886, 0x5300006, 0x18C6, 0x5400006, 0x1906, 0x5500006, 0x1946, 0x5600006, 0x1986, 0x5700006, 0x19C6, 0x5800006, 0x1A06, 0x5900006, 0x1A46, 0x5A00006, 0x1A86, 0x5B00006, 0x1AC6, 0x5C00006, 0x1B06, 0x5D00006, 0x1BC6, 0x5E00006, 0x1BE6, 0x5F00006, 0x1C06, 0x6000006, 0x1C26, 0x6100006, 0x1C46, 0x6200006, 0x1C66, 0x6300006, 0x14A6, 0x6400006, 0x14E6, 0x6500006, 0x1526, 0x6600006, 0x1566, 0x6700006, 0x15A6, 0x6800006, 0x15E6, 0x6900006, 0x1626, 0x6A00006, 0x1666, 0x6B00006, 0x16A6, 0x6C00006, 0x16E6, 0x6F00006, 0x1726, 0x6F80006, 0x1766, 0x7000006, 0x17A6, 0x7080006, 0x17E6, 0x7100006, 0x1826, 0x7180006, 0x1866, 0x5280006, 0x18A6, 0x5380006, 0x18E6, 0x5480006, 0x1926, 0x5580006, 0x1966, 0x5680006, 0x19A6, 0x5780006, 0x19E6, 0x5880006, 0x1A26, 0x5980006, 0x1A66, 0x5A80006, 0x1AA6, 0x5B80006, 0x1AE6, 0x5C80006, 0x1C86, 0x5D80006, 0x5E80006, 0x5F80006, 0x6080006, 0x6180006, 0x6280006, 0x6380006, 0x6480006, 0x6580006, 0x6680006, 0x6780006, 0x6880006, 0x6980006, 0x6A80006, 0x6B80006, 0x7200006, 0x1B26, 0x1B46, 0x1B66, 0x1B86, 0x1BA6, 0x14C6, 0x1506, 0x1546, 0x1586, 0x15C6, 0x1606, 0x1646, 0x1686, 0x16C6, 0x1706, 0x1746, 0x5881A26, 0x1CA6, 0x7300006, 0x7380006, 0x7400006, 0x7480006, 0x7500006, 0x7580006, 0x7600006, 0x7680006, 0x7700006, 0x7780006, 0x7800006, 0x7880006, 0x7900006, 0x7980006, 0x7A00006, 0x7A80006, 0x7B00006, 0x7B80006, 0x7C00006, 0x7C80006, 0x7D00006, 0x7280006, 0x7D80006, 0x1CC6, 0x1F86, 0x1FA6, 0x1FC6, 0x1FE6, 0x2006, 0x1D86, 0x1DA6, 0x1DC6, 0x1DE6, 0x1E06, 0x1E26, 0x1E46, 0x1E66, 0x1E86, 0x1EA6, 0x2026, 0x1EC6, 0x1EE6, 0x2046, 0x2066, 0x2086, 0x20A6, 0x1F26, 0x1F46, 0x1F66, 0x8300006, 0x7F00006, 0x7F80006, 0x8000006, 0x8080006, 0x8100006, 0x8180006, 0x8200006, 0x8280006, 0x20C6, 0x20E6, 0x2106, 0x8480006, 0x2146, 0x2166, 0x2186, 0x5F01C06, 0x6001C26, 0x6101C46, 0x6201C66, 0x63014A6, 0x64014E6, 0x6501526, 0x6601566, 0x67015A6, 0x68015E6, 0x6901626, 0x6A01666, 0x6B016A6, 0x6C016E6, 0x6F01726, 0x6F81766, 0x70017A6, 0x70817E6, 0x7101826, 0x7181866, 0x52818A6, 0x53818E6, 0x5481926, 0x5581966, 0x56819A6, 0x57819E6, 0x5981A66, 0x5A81AA6, 0x5B81AE6, 0x5C81C86, 0x8680006, 0x8700006, 0x8780006, 0x8800006, 0x8880006, 0x8900006, 0x8980006, 0x8A00006, 0x8A80006, 0x8B00006, 0x8B80006, 0x8C00006, 0x8C80006, 0x8D00006, 0x8D80006, 0x8E00006, 0x23A6, 0x23C6, 0x23E6, 0x2406, 0x2426, 0x2446, 0x2466, 0x2486, 0x24A6, 0x24C6, 0x24E6, 0x2506, 0x2526, 0x2546, 0x2566, 0x2586, 0x6C814E6, 0x6D01526, 0x6D81566, 0x6E015A6, 0x6E815E6, 0x5301626, 0x5401666, 0x55016A6, 0x56016E6, 0x5701726, 0x9680006, 0x25C6, 0x25E6, 0x2606, 0x2626, 0x2646, 0x2666, 0x2686, 0x26A6, 0x26C6, 0x26E6, 0x2706, 0x2726, 0x2746, 0x2766, 0x2786, 0x27A6, 0x5B017A6, 0xB66, 0xAA6, 0x27C6, 0x27E6, 0x2806, 0x2826, 0x2846, 0x2866, 0x2886, 0x9F00006, 0xA280006, 0xA300006, 0xA380006, 0xA400006, 0xA480006, 0xA500006, 0xA580006, 0xA600006, 0xA000006, 0xA080006, 0xA680006, 0xA700006, 0xA780006, 0xA800006, 0xA880006, 0xA900006, 0xA100006, 0xA180006, 0xA980006, 0xAA00006, 0xAA80006, 0xAB00006, 0xAB80006, 0xAC00006, 0xAC80006, 0xAD00006, 0x28E6, 0x2906, 0x2926, 0x2946, 0x2966, 0x2986, 0x29A6, 0x29E6, 0x2A26, 0x2AA6, 0x2AC6, 0x2AE6, 0x2B06, 0x2B26, 0x2B46, 0xAD80006, 0xAE00006, 0xAE80006, 0xAF00006, 0xAF80006, 0xB000006, 0x1CE6, 0x1D06, 0x1D26, 0x1D46, 0x1D66, 0x2126, 0x2BA6, 0x2BC6, 0x2BE6, 0x2C06, 0x28A6, 0x28C6, 0x21A6, 0x21C6, 0x21E6, 0x2206, 0x2226, 0x2246, 0x2266, 0x2286, 0x22A6, 0x22C6, 0x22E6, 0x2306, 0x2326, 0x2346, 0x2366, 0x2386, 0x6C81786, 0x6D017C6, 0x6D81806, 0x6E01846, 0x6E81886, 0x53018C6, 0x5401906, 0x5501946, 0x5601986, 0x57019C6, 0x5801A06, 0x5901A46, 0x5A01A86, 0x5B01AC6, 0x5C01B06, 0x2C26, 0x2C46, 0x1F06, 0x2C66, 0x2C86, 0x2CA6, 0x25A6, 0x2CC6, 0x5D81BE6, 0x5E81C06, 0x5F81C26, 0x6081C46, 0x6181C66, 0x62814A6, 0x63814E6, 0x6481526, 0x6581566, 0x66815A6, 0x67815E6, 0x6881626, 0x6981666, 0x6A816A6, 0x6B816E6, 0x7201726, 0x6F01C86, 0x3000006, 0x3080006, 0x3100006, 0x3180006, 0x3200006, 0x2300006, 0x2380006, 0x2400006, 0x2480006, 0x2500006, 0x2580006, 0x2600006, 0x2680006, 0x2700006, 0x2780006, 0x2800006, 0x2CE6, 0x2D06, 0x2401646, 0x2D26, 0x5D817A6, 0x5E817E6, 0xB500006, 0xB580006, 0xB600006, 0xB680006, 0x7E00006, 0x6C81586, 0x57815C6, 0x2DC6, 0x5E81C26, 0x2DE6, 0x2E06, 0x2E26, 0x2E46, 0x2E66, 0x2E86, 0x2EA6, 0x2EC6, 0x2EE6, 0x2F06, 0x8500006, 0xBC80006, 0xBD00006, 0x8600006, 0xBA00006, 0xB200006, 0x2F66, 0x6C81C86, 0xBD80006, 0x7E80006, 0xAE6, 0xC26, 0xC46, 0xC66, 0xC86, 0x8C6, 0x8E6, 0x906, 0x946, 0x966, 0x986, 0x9A6, 0x9C6, 0x9E6, 0xA06, 0xA26, 0xA46, 0xA66, 0xA86, 0xAC6, 0xB06, 0xB26, 0xB46, 0xB86, 0xBA6, 0xBC6, 0xBE6, 0xCA6, 0x926, 0xBE00007, 0xBE80007, 0xBF00007, 0xBF80007, 0xC000007, 0xC080007, 0xC100007, 0xC180007, 0xC200007, 0xC280007, 0xC300007, 0xC380007, 0xC400007, 0xC480007, 0xC500007, 0xC580007, 0xC600007, 0xC680007, 0xC700007, 0xC780007, 0xC800007, 0xC880007, 0xC900007, 0xC980007, 0xCA00007, 0xCA80007, 0x32C7, 0x32E7, 0x3307, 0x3327, 0x3347, 0x3367, 0x3387, 0x33A7, 0x33C7, 0x33E7, 0x3407, 0x3427, 0x3447, 0x3467, 0x3487, 0x34A7, 0x34C7, 0x34E7, 0x3507, 0x3527, 0x3547, 0x3567, 0x3587, 0x35A7, 0x35C7, 0x35E7, 0x3607, 0x3627, 0x3647, 0x3667, 0x3687, 0x36A7, 0x36C7, 0x36E7, 0x3707, 0x3727, 0x3747, 0x3767, 0x3787, 0x37A7, 0x37C7, 0x37E7, 0x3807, 0x3827, 0x3847, 0x3867, 0x3887, 0x38A7, 0x38C7, 0x38E7, 0x3907, 0x3927, 0x3947, 0x3967, 0x3987, 0x39A7, 0x39C7, 0x39E7, 0x3A07, 0x3A27, 0x3A47, 0x3A67, 0x3A87, 0x3AA7, 0x3AC7, 0x3AE7, 0x3B07, 0x3B27, 0x3B47, 0x3B67, 0x3B87, 0x3BA7, 0x3BC7, 0x3BE7, 0x3C07, 0x3C27, 0x3C47, 0x3C67, 0x3C87, 0x3CA7, 0x3CC7, 0x3CE7, 0x3D07, 0x3D27, 0x3D47, 0x3D67, 0x3D87, 0x3DA7, 0x3DC7, 0x3DE7, 0x3E07, 0x3E27, 0x3E47, 0x3E67, 0x3E87, 0x3EA7, 0x3EC7, 0x3EE7, 0x3F07, 0x3F27, 0x3F47, 0x3F67, 0x3F87, 0x3FA7, 0x3FC7, 0x3FE7, 0x4007, 0x4027, 0x4047, 0x4067, 0x4087, 0x40A7, 0x10300007, 0x10380007, 0x10400007, 0x10480007, 0x10500007, 0x10580007, 0x10600007, 0x10680007, 0x10700007, 0x10780007, 0x10800007, 0x10880007, 0x10900007, 0x10980007, 0x10A00007, 0x10A80007, 0xE780007, 0x42C7, 0x42E7, 0x4307, 0x4327, 0x4347, 0x4367, 0x4387, 0x43A7, 0xD080007, 0xCE80007, 0xD980007, 0xDE80007, 0xD480007, 0x43C7, 0x43E7, 0x4407, 0x4427, 0x4447, 0x4467, 0x4487, 0x44A7, 0x44C7, 0x44E7, 0x4507, 0x4527, 0xD580007, 0x4547, 0x4567, 0x4587, 0x45A7, 0x45C7, 0x11780007, 0x11800007, 0x11880007, 0x11900007, 0x11980007, 0x11A00007, 0x11A80007, 0x11B00007, 0x11B80007, 0x11C00007, 0x11C80007, 0x11D00007, 0x11D80007, 0x11E00007, 0x11E80007, 0x11F00007, 0x11F80007, 0x12000007, 0x12080007, 0x12100007, 0x12180007, 0x12200007, 0x12280007, 0x12300007, 0x12380007, 0x10B80007, 0x12400007, 0x10C00007, 0x10C80007, 0x10D00007, 0x10D80007, 0x10E00007, 0x10E80007, 0x12480007, 0x12500007, 0x12580007, 0x12600007, 0x10F00007, 0x10F80007, 0x11000007, 0x11080007, 0x49A7, 0x49C7, 0x49E7, 0x4A07, 0x4A27, 0xDD00007, 0xE300007, 0xE500007, 0xE600007, 0xE700007, 0xE800007, 0xE900007, 0xEA00007, 0xEB00007, 0xEC00007, 0xED00007, 0xEE00007, 0xEF00007, 0xF000007, 0xF100007, 0xF200007, 0xF300007, 0xF400007, 0xF500007, 0xF600007, 0xF700007, 0xF800007, 0xF900007, 0xFA00007, 0xFB00007, 0xFB80007, 0xFC00007, 0xFC80007, 0xFD00007, 0xFD80007, 0xFE00007, 0xFE80007, 0xFF00007, 0xFF80007, 0x10000007, 0x10080007, 0x10100007, 0x10180007, 0x10200007, 0x10280007, 0x4A47, 0x4A67, 0x4A87, 0x4707, 0x4727, 0x4747, 0x4767, 0x4787, 0x47A7, 0x47C7, 0x47E7, 0x4807, 0x4827, 0x4847, 0x4867, 0x4887, 0x48A7, 0x48C7, 0x48E7, 0x4907, 0x4927, 0x4AA7, 0x4947, 0x4967, 0x4987, 0xFA80007, 0xCB00007, 0x12B00007, 0x12900007, 0xE880007, 0xCB80007, 0xEA80007, 0xCC80007, 0xCD80007, 0xE980007, 0x12A00007, 0x12B80007, 0x12C00007, 0xCC00007, 0xD000007, 0xD680007, 0xD780007, 0xD880007, 0xCF80007, 0x12C80007, 0x12D00007, 0x12D80007, 0x12E00007, 0x12E80007, 0x12F00007, 0x12F80007, 0x13000007, 0x13080007, 0x13100007, 0x13180007, 0x13200007, 0x13280007, 0x13300007, 0x13380007, 0x13400007, 0x13480007, 0x13500007, 0x13580007, 0x13600007, 0x13680007, 0x13700007, 0x13780007, 0x13800007, 0x13880007, 0x13900007, 0x10B00007, 0x13980007, 0x13A00007, 0x13A80007, 0x13B00007, 0x13B80007, 0x13C00007, 0x13C80007, 0x13D00007, 0x13D80007, 0x13E00007, 0x13E80007, 0x13F00007, 0x13F80007, 0x14000007, 0x14080007, 0x14100007, 0x14180007, 0x14200007, 0x14280007, 0x14300007, 0x50E7, 0x5107, 0x5127, 0x5147, 0x5167, 0x5187, 0x51A7, 0x51C7, 0x51E7, 0x5207, 0x5227, 0x5247, 0x5267, 0x5287, 0x52A7, 0x52C7, 0x52E7, 0x5307, 0x5327, 0x5347, 0x5367, 0x5387, 0x53A7, 0x53C7, 0x53E7, 0x5407, 0x5427, 0x15100007, 0x15180007, 0x15200007, 0x15280007, 0x15300007, 0x15380007, 0x15400007, 0x15480007, 0x15500007, 0x15580007, 0x15600007, 0x15680007, 0x15700007, 0x15780007, 0x15800007, 0x15880007, 0x15900007, 0xDF00007, 0x5667, 0xE000007, 0x15A00007, 0xE400007, 0x31A7, 0x56A7, 0x10404147, 0x10484167, 0x56C7, 0x105033E7, 0x10583427, 0x56E7, 0x5707, 0x107037E7, 0x10783827, 0x5727, 0x5747, 0x5767, 0x10803BE7, 0x10883C27, 0x10903C67, 0x10983CA7, 0x5787, 0x10A033C7, 0x10A83407, 0x57A7, 0x57C7, 0x57E7, 0x5807, 0x5827, 0x5847, 0x5867, 0x5887, 0x58A7, 0x58C7, 0x58E7, 0x5907, 0x5927, 0x5947, 0x5967, 0x5987, 0x59A7, 0x59C7, 0x59E7, 0x5A07, 0x5A27, 0x5A47, 0x5A67, 0x5A87, 0x5AA7, 0x5AC7, 0x16B80007, 0x16C00007, 0x16C80007, 0x16D00007, 0x16D80007, 0x16E00007, 0x15B80007, 0x15C00007, 0x16E80007, 0x16F00007, 0x16F80007, 0x15C80007, 0x17000007, 0x17080007, 0x17100007, 0x17180007, 0x17200007, 0x17280007, 0x17300007, 0x17380007, 0x17400007, 0x17480007, 0x17500007, 0x17580007, 0x15E80007, 0x15F00007, 0x15F80007, 0x16000007, 0x16080007, 0x16100007, 0x16180007, 0x16200007, 0x16280007, 0x16300007, 0x16480007, 0x16500007, 0x16580007, 0x16600007, 0x16680007, 0x16700007, 0x16780007, 0x16800007, 0x16880007, 0x16900007, 0x16980007, 0x16A00007, 0x16A80007, 0x16B00007, 0x45E7, 0x5D87, 0x5DA7, 0x5DC7, 0x5DE7, 0x5E07, 0x5E27, 0x4647, 0x4667, 0x4687, 0x46A7, 0x46C7, 0x46E7, 0x17600007, 0x17680007, 0x17700007, 0x17780007, 0x17800007, 0x17880007, 0x12A80007, 0x11100007, 0x11180007, 0x11200007, 0x11280007, 0x11300007, 0x11380007, 0x12680007, 0x11400007, 0x11480007, 0x12700007, 0x12780007, 0x12800007, 0x12880007, 0x11580007, 0x11600007, 0x11680007, 0x11700007, 0x40C7, 0x40E7, 0x4107, 0x4167, 0x4187, 0x41A7, 0x41C7, 0x41E7, 0x4207, 0x4227, 0x4247, 0x4267, 0x4287, 0x42A7, 0x5E47, 0x5E67, 0x5E87, 0x5EA7, 0x5EC7, 0x5EE7, 0xE483FE7, 0xE584007, 0xE684027, 0xE784047, 0xE884067, 0xE984087, 0xEA840A7, 0x17C00007, 0x17C80007, 0x17D00007, 0x17D80007, 0x17E00007, 0x17E80007, 0x17F00007, 0x17F80007, 0x18000007, 0x18080007, 0x18100007, 0x18180007, 0x18200007, 0x18280007, 0x18300007, 0x18380007, 0x18400007, 0x18480007, 0x18500007, 0x18580007, 0x18600007, 0x18680007, 0x18700007, 0x18780007, 0x18800007, 0x18880007, 0x18900007, 0x18980007, 0x18A00007, 0x18A80007, 0x18B00007, 0x62E7, 0x6307, 0x6327, 0x6347, 0x6367, 0x6387, 0x63A7, 0x63C7, 0x18F80007, 0x19000007, 0x19080007, 0x19100007, 0x19180007, 0x19200007, 0x19280007, 0x19300007, 0x19380007, 0x19400007, 0x19480007, 0x19500007, 0x19580007, 0x19600007, 0x19680007, 0x19700007, 0x19780007, 0x19800007, 0x19880007, 0x19900007, 0x19980007, 0x19A00007, 0x19A80007, 0x19B00007, 0x19B80007, 0x19C00007, 0x19C80007, 0x19D00007, 0x19D80007, 0x19E00007, 0x19E80007, 0x19F00007, 0x19F80007, 0x104040C7, 0x104840E7, 0x10504107, 0x6807, 0x6827, 0x6847, 0x10304247, 0x6867, 0x6887, 0x68A7, 0x1827, 0x3167, 0x3187, 0x31C7, 0x31E7, 0x3207, 0x3227, 0x3247, 0x3267, 0x32A7, 0x65E7, 0x6607, 0x6627, 0x6647, 0x6667, 0x6687, 0x66A7, 0x66C7, 0x6707, 0x6727, 0x6747, 0x6767, 0x6787, 0x67A7, 0x67C7, 0x68C7, 0x68E7, 0x6907, 0x6927, 0xF583F27, 0xF683F87, 0xD180007, 0xD280007, 0xDA80007, 0xDF80007, 0xE180007, 0xE380007, 0xE480007, 0xE680007, 0xF780007, 0x18F00007, 0xCD00007, 0x1A500007, 0x6967, 0x6987, 0x69A7, 0x69C7, 0x69E7, 0x6A07, 0x6A27, 0x6A47, 0x6A67, 0x6A87, 0x6AA7, 0x6AC7, 0x6AE7, 0x6B07, 0x6B27, 0x6B47, 0x6B67, 0x6B87, 0x6BA7, 0x6BC7, 0x1AF80007, 0x18D80007, 0x18E00007, 0x18E80007, 0x1B000007, 0x1B080007, 0x1B100007, 0xDA00007, 0x1B180007, 0x6C87, 0x6CA7, 0x6CC7, 0x6CE7, 0x6D07, 0x6D27, 0x6D47, 0x6D67, 0x6D87, 0x6DA7, 0x6DC7, 0x6DE7, 0x6E07, 0x6E27, 0x6E47, 0x6E67, 0x6E87, 0x6EA7, 0x5687, 0x6EC7, 0x6C67, 0x6EE7, 0x6F07, 0x6F27, 0x6F47, 0x6F67, 0x6F87, 0x6FA7, 0x6FC7, 0x1B400007, 0x1B480007, 0x1B500007, 0x1BF80007, 0x1B580007, 0x1B600007, 0x1B680007, 0x1B700007, 0x1B780007, 0x1B800007, 0x1B880007, 0x1B900007, 0x1B980007, 0x1BA00007, 0x1BA80007, 0x1BB00007, 0x1BB80007, 0x1BC00007, 0x1BC80007, 0x15980007, 0x1BD00007, 0x1BD80007, 0x1BE00007, 0x1BE80007, 0x7008, 0x7028, 0x7048, 0x7068, 0x7088, 0x70A8, 0x70C8, 0x70E8, 0x7108, 0x7128, 0x7148, 0x7168, 0x7188, 0x71A8, 0x71C8, 0x71E8, 0x7208, 0x7228, 0x7248, 0x7268, 0x7288, 0x72A8, 0x72C8, 0x72E8, 0x7308, 0x7328, 0x7348, 0x7368, 0x7388, 0x73A8, 0x73C8, 0x73E8, 0x7408, 0x7428, 0x7448, 0x7468, 0x7488, 0x74A8, 0x74C8, 0x74E8, 0x7508, 0x7528, 0x7548, 0x7568, 0x7588, 0x75A8, 0x75C8, 0x75E8, 0x7608, 0x7628, 0x7648, 0x7668, 0x7688, 0x76A8, 0x76C8, 0x76E8, 0x7708, 0x7728, 0x7748, 0x7768, 0x7788, 0x77A8, 0x77C8, 0x77E8, 0x7808, 0x7828, 0x7848, 0x7868, 0x7888, 0x78A8, 0x78C8, 0x78E8, 0x7908, 0x7928, 0x7948, 0x7968, 0x7988, 0x79A8, 0x79C8, 0x79E8, 0x7A08, 0x7A28, 0x7A48, 0x7A68, 0x7A88, 0x7AA8, 0x7AC8, 0x7AE8, 0x7B08, 0x7B28, 0x7B48, 0x7B68, 0x7B88, 0x7BA8, 0x7BC8, 0x7BE8, 0x7C08, 0x7C28, 0x7C48, 0x7C68, 0x7C88, 0x7CA8, 0x7CC8, 0x7CE8, 0x7D08, 0x7D28, 0x7D48, 0x7D68, 0x7D88, 0x7DA8, 0x7DC8, 0x7DE8, 0x7E08, 0x7E28, 0x7E48, 0x7E68, 0x7E88, 0x7EA8, 0x7EC8, 0x7EE8, 0x7F08, 0x7F28, 0x7F48, 0x7F68, 0x7F88, 0x7FA8, 0x7FC8, 0x7FE8, 0x8008, 0x8028, 0x8048, 0x8068, 0x8088, 0x80A8, 0x80C8, 0x80E8, 0x8108, 0x8128, 0x8148, 0x8168, 0x8188, 0x81A8, 0x81C8, 0x81E8, 0x8208, 0x8228, 0x8248, 0x8268, 0x8288, 0x82A8, 0x82C8, 0x82E8, 0x8308, 0x8328, 0x8348, 0x20380008, 0x8368, 0x8388, 0x83A8, 0x83C8, 0x83E8, 0x8408, 0x8428, 0x8448, 0x1EF00008, 0x20100008, 0x1EE00008, 0x21180008, 0x21200008, 0x21280008, 0x21300008, 0x21380008, 0x8508, 0x8528, 0x8548, 0x8568, 0x8588, 0x85A8, 0x85C8, 0x85E8, 0x8608, 0x8628, 0x8648, 0x8668, 0x21A00008, 0x86A8, 0x86C8, 0x86E8, 0x8708, 0x8728, 0x8748, 0x8768, 0x8788, 0x87A8, 0x87C8, 0x87E8, 0x8808, 0x8828, 0x8848, 0x8868, 0x8888, 0x88A8, 0x88C8, 0x88E8, 0x8908, 0x8928, 0x8948, 0x8968, 0x8988, 0x89A8, 0x89C8, 0x89E8, 0x8A08, 0x8A28, 0x8A48, 0x8A68, 0x8A88, 0x8AA8, 0x8AC8, 0x8AE8, 0x8B08, 0x8B28, 0x8B48, 0x8B68, 0x8B88, 0x8BA8, 0x8BC8, 0x8BE8, 0x8C08, 0x8C28, 0x8C48, 0x21780008, 0x21800008, 0x21880008, 0x23180008, 0x21900008, 0x21980008, 0x23200008, 0x23280008, 0x23300008, 0x23380008, 0x21B00008, 0x21B80008, 0x21C00008, 0x21C80008, 0x21F00008, 0x21F80008, 0x22000008, 0x22080008, 0x22100008, 0x22180008, 0x22200008, 0x22280008, 0x22300008, 0x22380008, 0x22400008, 0x22480008, 0x22500008, 0x22580008, 0x22600008, 0x22680008, 0x22700008, 0x22780008, 0x22800008, 0x22880008, 0x22900008, 0x22980008, 0x22A00008, 0x8D08, 0x1EF80008, 0x8D28, 0x1F580008, 0x8D48, 0x1F780008, 0x8D68, 0x1F880008, 0x8D88, 0x1F980008, 0x8DA8, 0x1FA80008, 0x8DC8, 0x1FB80008, 0x8DE8, 0x1FC80008, 0x8E08, 0x1CD00008, 0x8E28, 0x1CE00008, 0x8E48, 0x1CF00008, 0x8E68, 0x1D000008, 0x8E88, 0x1D100008, 0x8EA8, 0x1D200008, 0x8EC8, 0x1D300008, 0x8EE8, 0x1D400008, 0x8F08, 0x1D500008, 0x8F28, 0x1D600008, 0x8F48, 0x1D700008, 0x8F68, 0x1D800008, 0x8F88, 0x1D900008, 0x8FA8, 0x1DA00008, 0x8FC8, 0x1DB00008, 0x8FE8, 0x1DC00008, 0x9008, 0x1DD00008, 0x9028, 0x1DE00008, 0x9048, 0x1DF00008, 0x9068, 0x1E000008, 0x9088, 0x1E100008, 0x90A8, 0x1E200008, 0x90C8, 0x1E300008, 0x90E8, 0x1E400008, 0x9108, 0x1E500008, 0x9128, 0x1E600008, 0x9148, 0x1E700008, 0x9168, 0x1E800008, 0x9188, 0x1E900008, 0x91A8, 0x1EA00008, 0x91C8, 0x1EB00008, 0x91E8, 0x1EC00008, 0x24800008, 0x24880008, 0x24900008, 0x24980008, 0x24A00008, 0x24A80008, 0x24B00008, 0x24B80008, 0x24C00008, 0x24C80008, 0x24D00008, 0x24D80008, 0x24E00008, 0x24E80008, 0x24F00008, 0x24F80008, 0x25000008, 0x25080008, 0x25100008, 0x20E00008, 0x25180008, 0x20E80008, 0x20F00008, 0x20F80008, 0x21000008, 0x21080008, 0x21100008, 0x25200008, 0x25280008, 0x25300008, 0x25380008, 0x25400008, 0x21400008, 0x21480008, 0x21500008, 0x21580008, 0x21600008, 0x21680008, 0x21700008, 0x20500008, 0x25480008, 0x20580008, 0x25500008, 0x25580008, 0x25600008, 0x20C80008, 0x25680008, 0x25700008, 0x25780008, 0x25800008, 0x20C00008, 0x25880008, 0x25900008, 0x25980008, 0x25A00008, 0x25A80008, 0x25B00008, 0x25B80008, 0x25C00008, 0x20600008, 0x25C80008, 0x25D00008, 0x25D80008, 0x25E00008, 0x25E80008, 0x25F00008, 0x25F80008, 0x26000008, 0x26080008, 0x26100008, 0x26180008, 0x26200008, 0x26280008, 0x26300008, 0x26380008, 0x26400008, 0x26480008, 0x26500008, 0x26580008, 0x26600008, 0x26680008, 0x26700008, 0x26780008, 0x26800008, 0x26880008, 0x26900008, 0x26980008, 0x26A00008, 0x26A80008, 0x26B00008, 0x26B80008, 0x26C00008, 0x26C80008, 0x26D00008, 0x26D80008, 0x26E00008, 0x26E80008, 0x26F00008, 0x26F80008, 0x27000008, 0x27080008, 0x27100008, 0x27180008, 0x27200008, 0x27280008, 0x27300008, 0x27380008, 0x27400008, 0x27480008, 0x27500008, 0x27580008, 0x27600008, 0x27680008, 0x27700008, 0x27780008, 0x27800008, 0x27880008, 0x27900008, 0x27980008, 0x27A00008, 0x27A80008, 0x27B00008, 0x27B80008, 0x27C00008, 0x27C80008, 0x27D00008, 0x27D80008, 0x27E00008, 0x27E80008, 0x27F00008, 0x27F80008, 0x28000008, 0x28080008, 0x28100008, 0x28180008, 0x28200008, 0xA0A8, 0xA0C8, 0xA0E8, 0xA108, 0xA128, 0xA148, 0xA168, 0xA188, 0xA1A8, 0xA1C8, 0xA1E8, 0xA208, 0xA228, 0xA248, 0xA268, 0xA288, 0xA2A8, 0xA2C8, 0xA2E8, 0x28C00008, 0xA328, 0x28D00008, 0xA368, 0xA388, 0xA3A8, 0xA3C8, 0xA3E8, 0x4188, 0x41A8, 0xA408, 0xA428, 0xA448, 0xA468, 0xA488, 0xA4A8, 0xA4C8, 0xA4E8, 0xA508, 0xA528, 0xA548, 0xA568, 0xA588, 0xA5A8, 0xA5C8, 0xA5E8, 0xA608, 0xA628, 0xA648, 0xA668, 0xA688, 0xA6A8, 0xA6C8, 0xA6E8, 0xA708, 0xA728, 0xA748, 0xA768, 0xA788, 0xA7A8, 0xA7C8, 0xA7E8, 0xA808, 0xA828, 0xA848, 0xA868, 0xA888, 0xA8A8, 0xA8C8, 0xA8E8, 0xA908, 0xA928, 0xA948, 0xA968, 0xA988, 0xA9A8, 0xA9C8, 0xA9E8, 0xAA08, 0xAA28, 0xAA48, 0xAA68, 0xAA88, 0xAAA8, 0xAAC8, 0xAAE8, 0xAB08, 0xAB28, 0xAB48, 0xAB68, 0xAB88, 0xABA8, 0xABC8, 0xABE8, 0xAC08, 0xAC28, 0xAC48, 0xAC68, 0xAC88, 0xACA8, 0xACC8, 0xACE8, 0xAD08, 0xAD28, 0xAD48, 0xAD68, 0xAD88, 0xADA8, 0xADC8, 0xADE8, 0xAE08, 0xAE28, 0xAE48, 0xAE68, 0xAE88, 0xAEA8, 0xAEC8, 0xAEE8, 0xAF08, 0xAF28, 0xAF48, 0x2BD80008, 0x2BE00008, 0x2BE80008, 0x2BF00008, 0x2BF80008, 0x2C000008, 0x2C080008, 0x2C100008, 0x2C180008, 0x2C200008, 0x2C280008, 0x2C300008, 0x2C380008, 0x1DD80008, 0x21D00008, 0x2C400008, 0x2C480008, 0x2C500008, 0x2C580008, 0x2C600008, 0x2C680008, 0x21E80008, 0x22A80008, 0x22B00008, 0x22B80008, 0x22C00008, 0x22C80008, 0x22D00008, 0x22D80008, 0x2C700008, 0x22E00008, 0x22E80008, 0x22F00008, 0x22F80008, 0x23000008, 0x23080008, 0x23100008, 0x2C780008, 0x2C800008, 0x2C880008, 0x2C900008, 0x2C980008, 0x2CA00008, 0x2CA80008, 0x2CB00008, 0x2CB80008, 0x2CC00008, 0x2CC80008, 0x2CD00008, 0x2CD80008, 0x2CE00008, 0x2CE80008, 0x2CF00008, 0x2CF80008, 0x2D000008, 0x2D080008, 0x2D100008, 0x2D180008, 0x2D200008, 0x2D280008, 0x2D300008, 0x2D380008, 0x2D400008, 0x2D480008, 0x2D500008, 0x2D580008, 0x2D600008, 0x2D680008, 0x1CD80008, 0x1CE80008, 0x1CF80008, 0x1D080008, 0x1D180008, 0x1D280008, 0x1D380008, 0x1D480008, 0x1D580008, 0x1D680008, 0x1D780008, 0x1D880008, 0x1D980008, 0x1DA80008, 0x1DB80008, 0x1DC80008, 0x1ED00008, 0x1DE80008, 0x1DF80008, 0x1F000008, 0x1E080008, 0x1F100008, 0x1E180008, 0x1F200008, 0x1E280008, 0x1F300008, 0x1E380008, 0x1F400008, 0x1E480008, 0x1F500008, 0x1E580008, 0x1F600008, 0x1E680008, 0x1F700008, 0x1E780008, 0x1F800008, 0x1E880008, 0x1F900008, 0x1E980008, 0x1FA00008, 0x1EA80008, 0x1FB00008, 0x1EB80008, 0x1FC00008, 0x1EC80008, 0xB5C8, 0xB5E8, 0xB608, 0xB628, 0xB648, 0xB668, 0xB688, 0xB6A8, 0xB6C8, 0xB6E8, 0xB708, 0xB728, 0xB748, 0xB768, 0xB788, 0xB7A8, 0xB7C8, 0xB7E8, 0xB808, 0xB828, 0xB848, 0xB868, 0xB888, 0xB8A8, 0xB8C8, 0xB8E8, 0xB908, 0xB928, 0xB948, 0xB968, 0xB988, 0xB9A8, 0xB9C8, 0xB9E8, 0xBA08, 0xBA28, 0xBA48, 0xBA68, 0xBA88, 0xBAA8, 0xBAC8, 0xBAE8, 0xBB08, 0xBB28, 0xB1E8, 0xB208, 0xB228, 0xB248, 0xB268, 0xB288, 0xB2A8, 0xB2C8, 0xB2E8, 0xB308, 0xB328, 0xB348, 0xB368, 0xB388, 0xB3A8, 0xB3C8, 0xB3E8, 0x2ED00008, 0x2ED80008, 0x2EE00008, 0x2EE80008, 0x2EF00008, 0x2EF80008, 0x2F000008, 0x2F080008, 0x2F100008, 0x2F180008, 0x2F200008, 0x2F280008, 0x2F300008, 0x2F380008, 0x2F400008, 0x2F480008, 0x2F500008, 0x2F580008, 0x2F600008, 0x2F680008, 0x2F700008, 0x2F780008, 0x2F800008, 0x2F880008, 0x1C580008, 0x1C600008, 0x1C680008, 0x1C700008, 0x1C780008, 0x1C800008, 0x1C880008, 0x1C900008, 0x1C980008, 0x1CA00008, 0x1CA80008, 0x1CB00008, 0x1CB80008, 0x1CC00008, 0x1CC80008, 0x2F900008, 0x2F980008, 0x2FA00008, 0x2FA80008, 0x2FB00008, 0x2FB80008, 0x2FC00008, 0x2FC80008, 0x2FD00008, 0x2FD80008, 0x2FE00008, 0x2FE80008, 0x2FF00008, 0x2FF80008, 0x30000008, 0x30080008, 0x30100008, 0xC068, 0xC088, 0xC0A8, 0xAFA8, 0xC0C8, 0xC0E8, 0xC108, 0xC128, 0xB048, 0xB068, 0xB088, 0xB0A8, 0xB0C8, 0xB0E8, 0x9268, 0x9288, 0x92A8, 0x92C8, 0x92E8, 0x9308, 0xC148, 0xC168, 0xC188, 0xC1A8, 0xC1C8, 0xC1E8, 0xC208, 0xC228, 0xC248, 0xC268, 0xC288, 0xC2A8, 0xC2C8, 0xC2E8, 0xC308, 0xC328, 0xC348, 0xC368, 0xC388, 0xC3A8, 0xC3C8, 0xC3E8, 0xC408, 0xC428, 0xC448, 0xC468, 0xC488, 0xC4A8, 0xC4C8, 0xC4E8, 0xC508, 0xC528, 0xB168, 0x9588, 0xB1A8, 0x31500008, 0x1FF00008, 0x1FF80008, 0x31580008, 0x20080008, 0x20300008, 0x20480008, 0x20A00008, 0x20B80008, 0x31600008, 0x31680008, 0x31700008, 0x31780008, 0x31800008, 0x31880008, 0x31900008, 0x31980008, 0x20D00008, 0x31A00008, 0x31A80008, 0x31B00008, 0x31B80008, 0x23780008, 0xAF68, 0xAF88, 0xAFC8, 0xC708, 0xC728, 0xB008, 0xC748, 0xC768, 0xC788, 0xC7A8, 0xC7C8, 0xC7E8, 0xC808, 0xC828, 0xC848, 0xBB68, 0xC868, 0xC888, 0xC8A8, 0xC8C8, 0xC8E8, 0xC908, 0x94E8, 0xC928, 0xC948, 0xC968, 0xC988, 0x2D008228, 0xB188, 0xC9A8, 0xC9C8, 0xC9E8, 0xC6A8, 0xCA08, 0xCA28, 0xC6E8, 0xCA48, 0x32A0CA68, 0x32A80008, 0x32B00008, 0x32B80008, 0x32C00008, 0x32C80008, 0x32D00008, 0x32D80008, 0x32E00008, 0x32E80008, 0x32F00008, 0x32F80008, 0x33000008, 0x33080008, 0x33100008, 0x33180008, 0x33200008, 0x33280008, 0x33300008, 0x33380008, 0x33400008, 0x33480008, 0x33500008, 0x33580008, 0x33600008, 0x33680008, 0x33700008, 0x33780008, 0x33800008, 0x33880008, 0x33900008, 0x33980008, 0x33A00008, 0x33A80008, 0x33B00008, 0x33B80008, 0x33C00008, 0x33C80008, 0x33D00008, 0x33D80008, 0x33E00008, 0x33E80008, 0x33F00008, 0x33F80008, 0x34000008, 0x34080008, 0x34100008, 0x34180008, 0x34200008, 0x34280008, 0x34300008, 0x34380008, 0x34400008, 0x34480008, 0xD149, 0xD169, 0xD189, 0xD1A9, 0xD1C9, 0xD1E9, 0xD209, 0xD229, 0xD249, 0xD269, 0xD289, 0xD2A9, 0xD2C9, 0xD2E9, 0xD309, 0xD329, 0xD349, 0xD369, 0xD389, 0xD3A9, 0xD3C9, 0xD3E9, 0xD409, 0xD429, 0xD449, 0xD469, 0xD489, 0xD4A9, 0xD4C9, 0xD4E9, 0xD509, 0xD529, 0xD549, 0xD569, 0xD589, 0xD5A9, 0xD5C9, 0xD5E9, 0xD609, 0xD629, 0xD649, 0xD669, 0xD689, 0xD6A9, 0xD6C9, 0xD6E9, 0xD709, 0xD729, 0xD749, 0xD769, 0xD789, 0xD7A9, 0xD7C9, 0xD7E9, 0xD809, 0xD829, 0xD849, 0xD869, 0xD889, 0xD8A9, 0xD8C9, 0xD8E9, 0xD909, 0xD929, 0xD949, 0xD969, 0xD989, 0xD9A9, 0xD9C9, 0xD9E9, 0xDA09, 0xDA29, 0xDA49, 0xDA69, 0xDA89, 0xDAA9, 0xDAC9, 0xDAE9, 0xDB09, 0xDB29, 0xDB49, 0xDB69, 0xDB89, 0xDBA9, 0xDBC9, 0xDBE9, 0xDC09, 0xDC29, 0xDC49, 0xDC69, 0xDC89, 0xDCA9, 0xDCC9, 0xDCE9, 0xDD09, 0xDD29, 0xDD49, 0xDD69, 0xDD89, 0xDDA9, 0xDDC9, 0xDDE9, 0xDE09, 0xDE29, 0xDE49, 0xDE69, 0xDE89, 0xDEA9, 0xDEC9, 0xDEE9, 0xDF09, 0xDF29, 0xDF49, 0xDF69, 0xDF89, 0xDFA9, 0xDFC9, 0xDFE9, 0xE009, 0xE029, 0xE049, 0xE069, 0xE089, 0xE0A9, 0xE0C9, 0xE0E9, 0xE109, 0xE129, 0xE149, 0xE169, 0xE189, 0xE1A9, 0xE1C9, 0xE1E9, 0xE209, 0xE229, 0xE249, 0xE269, 0xE289, 0xE2A9, 0xE2C9, 0xE2E9, 0xE309, 0xE329, 0xE349, 0xE369, 0xE389, 0xE3A9, 0xE3C9, 0xE3E9, 0xE409, 0xE429, 0xE449, 0xE469, 0xE489, 0x38280009, 0xE4A9, 0xE4C9, 0xE4E9, 0xE509, 0xE529, 0xE549, 0xE569, 0xE589, 0xE5A9, 0xE5C9, 0xE5E9, 0xE609, 0xE629, 0xE649, 0xE669, 0xE689, 0xE6A9, 0xE6C9, 0xE6E9, 0xE709, 0xE729, 0xE749, 0xE769, 0xE789, 0xE7A9, 0xE7C9, 0xE7E9, 0xE809, 0xE829, 0xE849, 0xE869, 0xE889, 0xE8A9, 0xE8C9, 0xE8E9, 0xE909, 0x38D00009, 0xE929, 0xE949, 0xE969, 0xE989, 0xE9A9, 0xE9C9, 0xE9E9, 0xEA09, 0xEA29, 0xEA49, 0xEA69, 0xEA89, 0xEAA9, 0xEAC9, 0xEAE9, 0xEB09, 0xEB29, 0xEB49, 0xEB69, 0xEB89, 0xEBA9, 0xEBC9, 0xEBE9, 0xEC09, 0xEC29, 0xEC49, 0xEC69, 0xEC89, 0xECA9, 0xECC9, 0xECE9, 0xED09, 0xED29, 0xED49, 0xED69, 0xED89, 0xEDA9, 0xEDC9, 0xEDE9, 0xEE09, 0xEE29, 0xEE49, 0xEE69, 0xEE89, 0xEEA9, 0xEEC9, 0xEEE9, 0xEF09, 0xEF29, 0xEF49, 0xEF69, 0xEF89, 0xEFA9, 0xEFC9, 0xEFE9, 0xF009, 0xF029, 0xF049, 0xF069, 0xF089, 0xF0A9, 0xF0C9, 0xF0E9, 0xF109, 0xF129, 0xF149, 0xF169, 0xF189, 0xF1A9, 0xF1C9, 0xF1E9, 0xF209, 0xF229, 0xF249, 0xF269, 0xF289, 0xF2A9, 0x3CB00009, 0x3CB80009, 0x3CC00009, 0x3CC80009, 0x3CD00009, 0x3CD80009, 0x3CE00009, 0x3CE80009, 0x3CF00009, 0x3CF80009, 0x3D000009, 0x3D080009, 0x3D100009, 0x3D180009, 0x3D200009, 0x3D280009, 0x3D300009, 0x3D380009, 0x3D400009, 0x3D480009, 0x3D500009, 0x3D580009, 0x3D600009, 0x3D680009, 0x3D700009, 0x3D780009, 0x3D800009, 0x3D880009, 0x3D900009, 0x3D980009, 0x3DA00009, 0x3DA80009, 0x3DB00009, 0x3DB80009, 0x3DC00009, 0x3DC80009, 0x3DD00009, 0x3DD80009, 0x3DE00009, 0x3DE80009, 0x3DF00009, 0x3DF80009, 0x3E000009, 0x3E080009, 0x3E100009, 0x3E180009, 0x3E200009, 0x3E280009, 0x3E300009, 0x3E380009, 0x3E400009, 0x3E480009, 0x3E500009, 0x3E580009, 0x3E600009, 0x3E680009, 0x3E700009, 0x3E780009, 0x3E800009, 0x3E880009, 0x3E900009, 0x3E980009, 0x3EA00009, 0x3EA80009, 0x3EB00009, 0x3EB80009, 0x3EC00009, 0x3EC80009, 0x3ED00009, 0x3ED80009, 0x3EE00009, 0x3EE80009, 0x3EF00009, 0x3EF80009, 0x3F000009, 0x3F080009, 0x3F100009, 0x3F180009, 0x3F200009, 0x3F280009, 0x3F300009, 0x3F380009, 0x3F400009, 0x3F480009, 0x3F500009, 0x3F580009, 0x3F600009, 0x3F680009, 0x3F700009, 0x3F780009, 0x3F800009, 0x3F880009, 0x3F900009, 0x3F980009, 0x3FA00009, 0x3FA80009, 0x3FB00009, 0x3FB80009, 0x3FC00009, 0x3FC80009, 0x3FD00009, 0x3FD80009, 0x3FE00009, 0x3FE80009, 0x3FF00009, 0x3FF80009, 0x40000009, 0x40080009, 0x40100009, 0x40180009, 0x40200009, 0x40280009, 0x40300009, 0x40380009, 0x40400009, 0x40480009, 0x40500009, 0x40580009, 0x40600009, 0x3B880009, 0x3B900009, 0x3B980009, 0x3BA00009, 0x3BA80009, 0x3BB00009, 0x3BB80009, 0x3BC00009, 0x3BC80009, 0x3BD00009, 0x3BD80009, 0x3BE00009, 0x3BE80009, 0x3BF00009, 0x40680009, 0x101C9, 0x40780009, 0x10209, 0x40880009, 0x40900009, 0x40980009, 0x10289, 0x40A80009, 0x102C9, 0x40B80009, 0x10309, 0x40C80009, 0x10349, 0x40D80009, 0x40E00009, 0x40E80009, 0x39A80009, 0x103C9, 0x3A100009, 0x103E9, 0x41000009, 0x10429, 0x41100009, 0x38B00009, 0x41180009, 0x10489, 0x3C280009, 0x41280009, 0x41300009, 0x41380009, 0x41400009, 0x41480009, 0x41500009, 0x41580009, 0x41600009, 0x41680009, 0x41700009, 0x41780009, 0x41800009, 0x41880009, 0x41900009, 0x41980009, 0x41A00009, 0x41A80009, 0x41B00009, 0x41B80009, 0x41C00009, 0x41C80009, 0x41D00009, 0x41D80009, 0x41E00009, 0x41E80009, 0x41F00009, 0x41F80009, 0x42000009, 0x42080009, 0x42100009, 0x42180009, 0x42200009, 0x42280009, 0x42300009, 0x42380009, 0x42400009, 0x42480009, 0x42500009, 0x42580009, 0x42600009, 0x42680009, 0x42700009, 0x42780009, 0x42800009, 0x42880009, 0x42900009, 0x42980009, 0x42A00009, 0x42A80009, 0x42B00009, 0x42B80009, 0x42C00009, 0x42C80009, 0x42D00009, 0x42D80009, 0x42E00009, 0x42E80009, 0x42F00009, 0x42F80009, 0x43000009, 0x43080009, 0x43100009, 0x43180009, 0x43200009, 0x43280009, 0x43300009, 0x43380009, 0x43400009, 0x43480009, 0x43500009, 0x43580009, 0x43600009, 0x43680009, 0x43700009, 0x43780009, 0x43800009, 0x43880009, 0x43900009, 0x43980009, 0x43A00009, 0x43A80009, 0x43B00009, 0x43B80009, 0x43C00009, 0x43C80009, 0x43D00009, 0x43D80009, 0x43E00009, 0x43E80009, 0x43F00009, 0x43F80009, 0x44000009, 0x44080009, 0x44100009, 0x44180009, 0x44200009, 0x110A9, 0x110C9, 0x110E9, 0x11109, 0x11129, 0x11149, 0x11169, 0x11189, 0x111A9, 0x111C9, 0x111E9, 0x11209, 0x11229, 0x11249, 0x11269, 0x11289, 0x112A9, 0x112C9, 0x112E9, 0x11309, 0x11329, 0x11349, 0x11369, 0x11389, 0x113A9, 0x113C9, 0x113E9, 0x11409, 0x11429, 0x11449, 0x11469, 0x11489, 0x114A9, 0x114C9, 0x114E9, 0x11509, 0x11529, 0x11549, 0x11569, 0x11589, 0x115A9, 0x115C9, 0x115E9, 0x11609, 0x11629, 0x11649, 0x11669, 0x11689, 0x116A9, 0x116C9, 0x116E9, 0x11709, 0x11729, 0x11749, 0x11769, 0x11789, 0x117A9, 0x117C9, 0x117E9, 0x11809, 0x11829, 0x11849, 0x11869, 0x11889, 0x118A9, 0x118C9, 0x118E9, 0x11909, 0x11929, 0x11949, 0x11969, 0x11989, 0x119A9, 0x119C9, 0x119E9, 0x11A09, 0x11A29, 0x11A49, 0x11A69, 0x11A89, 0x11AA9, 0x11AC9, 0x11AE9, 0x11B09, 0x11B29, 0x11B49, 0x11B69, 0x11B89, 0x11BA9, 0x11BC9, 0x11BE9, 0x11C09, 0x11C29, 0x11C49, 0x11C69, 0x11C89, 0x11CA9, 0x11CC9, 0x11CE9, 0x11D09, 0x11D29, 0x11D49, 0x11D69, 0xFCC9, 0x11D89, 0x11DA9, 0x11DC9, 0x11DE9, 0x11E09, 0x11E29, 0x11E49, 0xFD29, 0xFD49, 0xFD69, 0xFD89, 0xFDA9, 0xFDC9, 0xFDE9, 0xFE09, 0xFE29, 0xFE49, 0xFE69, 0xFE89, 0xFEA9, 0xFEC9, 0xFEE9, 0xFF09, 0xFF29, 0xFF49, 0xFF69, 0xFF89, 0xFFA9, 0xFFC9, 0xFFE9, 0x10009, 0x10029, 0x10049, 0x10069, 0x10089, 0x100A9, 0x100C9, 0x100E9, 0x10109, 0x10129, 0x10149, 0x47980009, 0x11E89, 0x11EA9, 0x11EC9, 0x11EE9, 0x11F09, 0x11F29, 0x10249, 0x11F49, 0x11F69, 0x11F89, 0x11FA9, 0x11FC9, 0x11FE9, 0x12009, 0x12029, 0x12049, 0x12069, 0x12089, 0x120A9, 0x120C9, 0x120E9, 0x12109, 0x12129, 0x12149, 0x12169, 0x12189, 0x121A9, 0x121C9, 0x121E9, 0x12209, 0x12229, 0x12249, 0x12269, 0x12289, 0x122A9, 0x122C9, 0x122E9, 0x12309, 0x12329, 0x12349, 0x12369, 0x48E00009, 0x48E80009, 0x48F00009, 0x48F80009, 0x49000009, 0x49080009, 0x49100009, 0x49180009, 0x49200009, 0x49280009, 0x49300009, 0x49380009, 0x49400009, 0x49480009, 0x49500009, 0x49580009, 0x49600009, 0x49680009, 0x49700009, 0x49780009, 0x49800009, 0x49880009, 0x49900009, 0x49980009, 0x49A00009, 0x49A80009, 0x49B00009, 0x49B80009, 0x49C00009, 0x49C80009, 0x49D00009, 0x39800009, 0x49D80009, 0x39700009, 0x49E00009, 0x49E80009, 0x49F00009, 0x49F80009, 0x4A000009, 0x4A080009, 0x4A100009, 0x4A180009, 0x4A200009, 0x4A280009, 0x4A300009, 0x4A380009, 0x4A400009, 0x4A480009, 0x4A500009, 0x4A580009, 0x4A600009, 0x4A680009, 0x4A700009, 0x4A780009, 0x4A800009, 0x4A880009, 0x4A900009, 0x4A980009, 0x4AA00009, 0x4AA80009, 0x4AB00009, 0x12AE9, 0x12B09, 0x12B29, 0x12B49, 0x12B69, 0x12B89, 0x4AE80009, 0x4AF00009, 0x4AF80009, 0x4B000009, 0x4B080009, 0x4B100009, 0x4B180009, 0x4B200009, 0x4B280009, 0x4B300009, 0x4B380009, 0x4B400009, 0x4B480009, 0x4B500009, 0x4B580009, 0x4B600009, 0x4B680009, 0x4B700009, 0x4B780009, 0x4B800009, 0x4B880009, 0x4B900009, 0x4B980009, 0x4BA00009, 0x4BA80009, 0x4BB00009, 0x4BB80009, 0x4BC00009, 0x4BC80009, 0x4BD00009, 0x4BD80009, 0x4BE00009, 0x4BE80009, 0x4BF00009, 0x4BF80009, 0x4C000009, 0x4C080009, 0x4C100009, 0x3B500009, 0x3B580009, 0x3B600009, 0x3B680009, 0x3B700009, 0x3B780009, 0x3B800009, 0x4C180009, 0x4C200009, 0x4C280009, 0x4C300009, 0x4C380009, 0x4C400009, 0x4C480009, 0x4C500009, 0x4C580009, 0x4C600009, 0x4C680009, 0x4C700009, 0x4C780009, 0x4C800009, 0x4C880009, 0x4C900009, 0x4C980009, 0x47300009, 0x47380009, 0x47400009, 0x47480009, 0x47500009, 0x47580009, 0x47600009, 0x47680009, 0x47700009, 0x47780009, 0x47800009, 0x47880009, 0x47900009, 0x4CA00009, 0x4CA80009, 0x4CB00009, 0x4CB80009, 0x4CC00009, 0x4CC80009, 0x4CD00009, 0x4CD80009, 0x47A00009, 0x47A80009, 0x47B00009, 0x47B80009, 0x47C00009, 0x47C80009, 0x47D00009, 0x47D80009, 0x3BF80009, 0x3C000009, 0x3C080009, 0x3C100009, 0x3C180009, 0x3C200009, 0x4CE00009, 0x4CE80009, 0x4CF00009, 0x4CF80009, 0x4D000009, 0x4D080009, 0x4D100009, 0x4D180009, 0x4D200009, 0x4D280009, 0x4D300009, 0x4D380009, 0x4D400009, 0x4D480009, 0x4D500009, 0x4D580009, 0x4D600009, 0x4D680009, 0x4D700009, 0x4D780009, 0x4D800009, 0x4D880009, 0x4D900009, 0x4D980009, 0x4DA00009, 0x4DA80009, 0x4DB00009, 0x4DB80009, 0x4DC00009, 0x4DC80009, 0x4DD00009, 0x4DD80009, 0x4DE00009, 0x4DE80009, 0x4DF00009, 0x4DF80009, 0x4E000009, 0x4E080009, 0x4E100009, 0x4E180009, 0x4E200009, 0x4E280009, 0x4E300009, 0x4E380009, 0x4E400009, 0x4E480009, 0x4E500009, 0x4E580009, 0x13989, 0x139A9, 0x139C9, 0x139E9, 0x13A09, 0x13A29, 0x13A49, 0x13A69, 0x13A89, 0x13AA9, 0x13AC9, 0x13AE9, 0x13B09, 0x13B29, 0x13B49, 0x13B69, 0x13B89, 0x13BA9, 0x13BC9, 0x13BE9, 0x13C09, 0x13C29, 0x13C49, 0x13C69, 0x13C89, 0x13CA9, 0x13CC9, 0x13CE9, 0x13D09, 0x13D29, 0x13D49, 0x13D69, 0x13D89, 0x13DA9, 0x13DC9, 0x13DE9, 0x13E09, 0x13E29, 0x13E49, 0x13E69, 0x13E89, 0x13EA9, 0x13EC9, 0x13EE9, 0x13F09, 0x13F29, 0x13F49, 0x13F69, 0x13F89, 0x13FA9, 0x13FC9, 0x13FE9, 0x14009, 0x14029, 0x14049, 0x35A12A29, 0x14069, 0x35F14089, 0x360140A9, 0x140C9, 0x363140E9, 0x36814109, 0x36B14129, 0x37214149, 0x37414169, 0x375102A9, 0x37B14189, 0x37A141A9, 0x37D10449, 0x37E141C9, 0x381141E9, 0x38214209, 0x38494229, 0x38514249, 0x38694269, 0x38914289, 0x389942A9, 0x142C9, 0x377142E9, 0x14309, 0x37314329, 0x50D00009, 0x50D80009, 0x4E780009, 0x50E00009, 0x50E80009, 0x50F00009, 0x50F80009, 0x51000009, 0x4EA00009, 0x4EA80009, 0x4EB00009, 0x4EB80009, 0x4EC00009, 0x4EC80009, 0x4ED00009, 0x4ED80009, 0x4EE00009, 0x51080009, 0x51100009, 0x51180009, 0x51200009, 0x51280009, 0x51300009, 0x51380009, 0x51400009, 0x51480009, 0x51500009, 0x51580009, 0x51600009, 0x39480009, 0x39580009, 0x51680009, 0x51700009, 0x51780009, 0x51800009, 0x51880009, 0x39C00009, 0x39C80009, 0x39D00009, 0x39F00009, 0x51900009, 0x3A180009, 0x51980009, 0x3A280009, 0x3A300009, 0x51A00009, 0x3A480009, 0x3A500009, 0x51A80009, 0x50000009, 0x51B00009, 0x50080009, 0x50100009, 0x3A580009, 0x3A600009, 0x3A680009, 0x3A700009, 0x3A780009, 0x3A800009, 0x3A880009, 0x3A900009, 0x3A980009, 0x3AA00009, 0x3AA80009, 0x3AB00009, 0x3AB80009, 0x3AC00009, 0x3AC80009, 0x3AD00009, 0x3AD80009, 0x3AE00009, 0x3AE80009, 0x51B80009, 0x51C00009, 0x51C80009, 0x51D00009, 0x51D80009, 0x51E00009, 0x51E80009, 0x51F00009, 0x3B380009, 0x3B400009, 0x3B480009, 0x51F80009, 0x52000009, 0x52080009, 0x52100009, 0x52180009, 0x52200009, 0x52280009, 0x52300009, 0x52380009, 0x52400009, 0x52480009, 0x52500009, 0x52580009, 0x52600009, 0x52680009, 0x52700009, 0x52780009, 0x52800009, 0x52880009, 0x52900009, 0x52980009, 0x52A00009, 0x52A80009, 0x52B00009, 0x52B80009, 0x52C00009, 0x52C80009, 0x52D00009, 0x52D80009, 0x52E00009, 0x52E80009, 0x52F00009, 0x52F80009, 0x53000009, 0x53080009, 0x53100009, 0x53180009, 0x53200009, 0x53280009, 0x53300009, 0x53380009, 0x53400009, 0x53480009, 0x53500009, 0x53580009, 0x53600009, 0x53680009, 0x53700009, 0x53780009, 0x53800009, 0x53880009, 0x53900009, 0x53980009, 0x53A00009, 0x14EAA, 0x14ECA, 0x14EEA, 0x14F0A, 0x14F2A, 0x14F4A, 0x14F6A, 0x14F8A, 0x14FAA, 0x14FCA, 0x14FEA, 0x1500A, 0x1502A, 0x1504A, 0x1506A, 0x1508A, 0x150AA, 0x150CA, 0x150EA, 0x1510A, 0x1512A, 0x1514A, 0x1516A, 0x1518A, 0x151AA, 0x151CA, 0x5478000A, 0x5480000A, 0x5488000A, 0x5490000A, 0x5498000A, 0x54A0000A, 0x54A8000A, 0x54B0000A, 0x54B8000A, 0x54C0000A, 0x54C8000A, 0x54D0000A, 0x54D8000A, 0x54E0000A, 0x54E8000A, 0x54F0000A, 0x54F8000A, 0x5500000A, 0x5508000A, 0x5510000A, 0x5518000A, 0x5520000A, 0x5528000A, 0x5530000A, 0x5538000A, 0x5540000A, 0x5548000A, 0x5550000A, 0x5558000A, 0x5560000A, 0x5568000A, 0x5570000A, 0x5578000A, 0x5580000A, 0x5588000A, 0x5590000A, 0x5598000A, 0x55A0000A, 0x55A8000A, 0x55B0000A, 0x55B8000A, 0x55C0000A, 0x55C8000A, 0x55D0000A, 0x55D8000A, 0x55E0000A, 0x55E8000A, 0x55F0000A, 0x55F8000A, 0x5600000A, 0x5608000A, 0x5610000A, 0x5618000A, 0x5620000A, 0x5628000A, 0x5630000A, 0x5638000A, 0x5640000A, 0x5648000A, 0x5650000A, 0x5658000A, 0x5660000A, 0x5668000A, 0x5670000A, 0x5678000A, 0x5680000A, 0x5688000A, 0x5690000A, 0x5698000A, 0x56A0000A, 0x56A8000A, 0x56B0000A, 0x56B8000A, 0x56C0000A, 0x56C8000A, 0x56D0000A, 0x56D8000A, 0x56E0000A, 0x56E8000A, 0x56F0000A, 0x56F8000A, 0x5700000A, 0x5708000A, 0x5710000A, 0x5718000A, 0x5720000A, 0x5728000A, 0x5730000A, 0x5738000A, 0x5740000A, 0x5748000A, 0x5750000A, 0x5758000A, 0x5760000A, 0x5768000A, 0x5770000A, 0x5778000A, 0x5780000A, 0x5788000A, 0x5790000A, 0x5798000A, 0x57A0000A, 0x57A8000A, 0x57B0000A, 0x57B8000A, 0x57C0000A, 0x57C8000A, 0x57D0000A, 0x57D8000A, 0x57E0000A, 0x57E8000A, 0x57F0000A, 0x57F8000A, 0x5800000A, 0x5808000A, 0x5810000A, 0x5818000A, 0x5820000A, 0x5828000A, 0x5830000A, 0x5838000A, 0x5840000A, 0x5848000A, 0x5850000A, 0x5858000A, 0x5860000A, 0x5868000A, 0x5870000A, 0x5878000A, 0x154EA, 0x15C4A, 0x1620A, 0x15C8A, 0x5888000A, 0x1624A, 0x1626A, 0x1628A, 0x162AA, 0x162CA, 0x162EA, 0x1630A, 0x1632A, 0x1552A, 0x58D0000A, 0x1556A, 0x58D8000A, 0x155AA, 0x58E0000A, 0x1562A, 0x58E8000A, 0x1566A, 0x58F0000A, 0x156AA, 0x58F8000A, 0x156EA, 0x5900000A, 0x1572A, 0x5908000A, 0x1644A, 0x1646A, 0x1648A, 0x164AA, 0x164CA, 0x164EA, 0x1650A, 0x1652A, 0x1654A, 0x1656A, 0x1658A, 0x165AA, 0x15E2A, 0x5970000A, 0x165EA, 0x1660A, 0x1662A, 0x1664A, 0x1666A, 0x1668A, 0x1602A, 0x166AA, 0x166CA, 0x166EA, 0x1670A, 0x1672A, 0x1674A, 0x1676A, 0x1678A, 0x167AA, 0x167CA, 0x167EA, 0x1680A, 0x1682A, 0x1684A, 0x1686A, 0x1688A, 0x168AA, 0x168CA, 0x168EA, 0x1690A, 0x1692A, 0x1694A, 0x1696A, 0x1698A, 0x169AA, 0x169CA, 0x169EA, 0x16A0A, 0x16A2A, 0x16A4A, 0x16A6A, 0x16A8A, 0x16AAA, 0x16ACA, 0x16AEA, 0x16B0A, 0x16B2A, 0x16B4A, 0x16B6A, 0x16B8A, 0x16BAA, 0x16BCA, 0x16BEA, 0x16C0A, 0x16C2A, 0x16C4A, 0x16C6A, 0x16C8A, 0x16CAA, 0x16CCA, 0x16CEA, 0x16D0A, 0x16D2A, 0x16D4A, 0x16D6A, 0x16D8A, 0x16DAA, 0x16DCA, 0x16DEA, 0x16E0A, 0x16E2A, 0x16E4A, 0x16E6A, 0x16E8A, 0x16EAA, 0x16ECA, 0x16EEA, 0x16F0A, 0x16F2A, 0x16F4A, 0x16F6A, 0x16F8A, 0x16FAA, 0x16FCA, 0x16FEA, 0x1700A, 0x1702A, 0x1704A, 0x1706A, 0x5C20000A, 0x170AA, 0x5C30000A, 0x170EA, 0x5C40000A, 0x1712A, 0x5C50000A, 0x1716A, 0x5C60000A, 0x171AA, 0x5C70000A, 0x171EA, 0x5C80000A, 0x1722A, 0x5C90000A, 0x1726A, 0x5CA0000A, 0x172AA, 0x5CB0000A, 0x172EA, 0x5CC0000A, 0x1732A, 0x5CD0000A, 0x1736A, 0x5CE0000A, 0x173AA, 0x5CF0000A, 0x173EA, 0x5D00000A, 0x1742A, 0x5D10000A, 0x1746A, 0x5D20000A, 0x174AA, 0x5D30000A, 0x174EA, 0x5D40000A, 0x1752A, 0x5D50000A, 0x1756A, 0x5D60000A, 0x175AA, 0x5D70000A, 0x175EA, 0x5D80000A, 0x1762A, 0x5D90000A, 0x1766A, 0x5DA0000A, 0x176AA, 0x5DB0000A, 0x176EA, 0x5DC0000A, 0x1772A, 0x5DD0000A, 0x1776A, 0x5DE0000A, 0x177AA, 0x5DF0000A, 0x177EA, 0x5E00000A, 0x1782A, 0x5E10000A, 0x1786A, 0x5E20000A, 0x178AA, 0x5E30000A, 0x178EA, 0x5E40000A, 0x1792A, 0x5E50000A, 0x1796A, 0x5E60000A, 0x179AA, 0x5E70000A, 0x179EA, 0x5E80000A, 0x17A2A, 0x5E90000A, 0x17A6A, 0x17A8A, 0x17AAA, 0x17ACA, 0x17AEA, 0x17B0A, 0x17B2A, 0x17B4A, 0x17B6A, 0x17B8A, 0x17BAA, 0x17BCA, 0x17BEA, 0x17C0A, 0x17C2A, 0x17C4A, 0x17C6A, 0x17C8A, 0x17CAA, 0x17CCA, 0x17CEA, 0x17D0A, 0x17D2A, 0x17D4A, 0x17D6A, 0x17D8A, 0x17DAA, 0x17DCA, 0x17DEA, 0x17E0A, 0x17E2A, 0x17E4A, 0x17E6A, 0x17E8A, 0x17EAA, 0x17ECA, 0x17EEA, 0x17F0A, 0x17F2A, 0x17F4A, 0x17F6A, 0x17F8A, 0x17FAA, 0x17FCA, 0x17FEA, 0x1800A, 0x1802A, 0x1804A, 0x1806A, 0x1808A, 0x180AA, 0x180CA, 0x180EA, 0x1810A, 0x6048000A, 0x1814A, 0x6058000A, 0x1818A, 0x181AA, 0x181CA, 0x6078000A, 0x1820A, 0x6088000A, 0x1824A, 0x6098000A, 0x1828A, 0x60A8000A, 0x182CA, 0x182EA, 0x1830A, 0x1832A, 0x60D0000A, 0x1836A, 0x60E0000A, 0x183AA, 0x60F0000A, 0x183EA, 0x1840A, 0x1842A, 0x6110000A, 0x1846A, 0x1848A, 0x184AA, 0x184CA, 0x184EA, 0x1850A, 0x1852A, 0x1854A, 0x1856A, 0x1858A, 0x185AA, 0x185CA, 0x185EA, 0x1860A, 0x1862A, 0x1864A, 0x1866A, 0x1868A, 0x186AA, 0x186CA, 0x186EA, 0x1870A, 0x1872A, 0x1874A, 0x1876A, 0x1878A, 0x187AA, 0x187CA, 0x187EA, 0x1880A, 0x1882A, 0x1884A, 0x1886A, 0x1888A, 0x188AA, 0x188CA, 0x188EA, 0x1890A, 0x1892A, 0x1894A, 0x1896A, 0x1898A, 0x189AA, 0x189CA, 0x189EA, 0x18A0A, 0x18A2A, 0x18A4A, 0x18A6A, 0x18A8A, 0x18AAA, 0x18ACA, 0x18AEA, 0x18B0A, 0x18B2A, 0x18B4A, 0x18B6A, 0x18B8A, 0x18BAA, 0x18BCA, 0x18BEA, 0x18C0A, 0x18C2A, 0x18C4A, 0x18C6A, 0x18C8A, 0x18CAA, 0x18CCA, 0x18CEA, 0x18D0A, 0x18D2A, 0x18D4A, 0x18D6A, 0x18D8A, 0x6368000A, 0x6370000A, 0x6378000A, 0x6380000A, 0x6388000A, 0x6390000A, 0x6398000A, 0x63A0000A, 0x63A8000A, 0x63B0000A, 0x63B8000A, 0x63C0000A, 0x63C8000A, 0x63D0000A, 0x63D8000A, 0x63E0000A, 0x63E8000A, 0x63F0000A, 0x63F8000A, 0x6400000A, 0x6408000A, 0x6410000A, 0x6418000A, 0x6420000A, 0x190AA, 0x190CA, 0x190EA, 0x1910A, 0x1912A, 0x1914A, 0x1916A, 0x1918A, 0x191AA, 0x191CA, 0x191EA, 0x1920A, 0x1922A, 0x1924A, 0x1926A, 0x1928A, 0x192AA, 0x192CA, 0x192EA, 0x1930A, 0x1932A, 0x1934A, 0x1936A, 0x1938A, 0x193AA, 0x193CA, 0x193EA, 0x1940A, 0x1942A, 0x1944A, 0x1946A, 0x1948A, 0x194AA, 0x194CA, 0x194EA, 0x1950A, 0x1952A, 0x1954A, 0x1956A, 0x1958A, 0x195AA, 0x195CA, 0x195EA, 0x1960A, 0x1962A, 0x1964A, 0x1966A, 0x1968A, 0x196AA, 0x196CA, 0x196EA, 0x1970A, 0x1972A, 0x1974A, 0x1976A, 0x1978A, 0x197AA, 0x197CA, 0x197EA, 0x1980A, 0x1982A, 0x1984A, 0x1986A, 0x1988A, 0x198AA, 0x198CA, 0x198EA, 0x1990A, 0x1992A, 0x1994A, 0x1996A, 0x1998A, 0x199AA, 0x199CA, 0x199EA, 0x19A0A, 0x19A2A, 0x19A4A, 0x19A6A, 0x19A8A, 0x19AAA, 0x19ACA, 0x19AEA, 0x19B0A, 0x19B2A, 0x19B4A, 0x19B6A, 0x19B8A, 0x19BAA, 0x19BCA, 0x19BEA, 0x19C0A, 0x19C2A, 0x19C4A, 0x19C6A, 0x19C8A, 0x19CAA, 0x19CCA, 0x19CEA, 0x19D0A, 0x19D2A, 0x19D4A, 0x19D6A, 0x19D8A, 0x19DAA, 0x19DCA, 0x19DEA, 0x19E0A, 0x19E2A, 0x19E4A, 0x19E6A, 0x19E8A, 0x19EAA, 0x19ECA, 0x19EEA, 0x19F0A, 0x19F2A, 0x19F4A, 0x19F6A, 0x19F8A, 0x19FAA, 0x19FCA, 0x19FEA, 0x1A00A, 0x1A02A, 0x1A04A, 0x1A06A, 0x1A08A, 0x1A0AA, 0x1A0CA, 0x1A0EA, 0x1A10A, 0x1A12A, 0x1A14A, 0x1A16A, 0x1A18A, 0x1A1AA, 0x1A1CA, 0x1A1EA, 0x1A20A, 0x1A22A, 0x1A24A, 0x1A26A, 0x1A28A, 0x1638A, 0x68A8000A, 0x1A2CA, 0x1A2EA, 0x1A30A, 0x1A32A, 0x1A34A, 0x1A36A, 0x1A38A, 0x1A3AA, 0x1A3CA, 0x1A3EA, 0x1A40A, 0x1A42A, 0x1A44A, 0x1A46A, 0x1A48A, 0x1A4AA, 0x1A4CA, 0x1A4EA, 0x1A50A, 0x1A52A, 0x1A54A, 0x1A56A, 0x1A58A, 0x1A5AA, 0x1A5CA, 0x1A5EA, 0x1A60A, 0x1A62A, 0x1A64A, 0x1A66A, 0x1A68A, 0x1A6AA, 0x1A6CA, 0x1A6EA, 0x1A70A, 0x1A72A, 0x1A74A, 0x1A76A, 0x1A78A, 0x1A7AA, 0x1A7CA, 0x1728A, 0x69F8000A, 0x172CA, 0x6A00000A, 0x1730A, 0x6A08000A, 0x173CA, 0x6A10000A, 0x1740A, 0x6A18000A, 0x1744A, 0x6A20000A, 0x1748A, 0x6A28000A, 0x174CA, 0x6A30000A, 0x1750A, 0x6A38000A, 0x1754A, 0x6A40000A, 0x1758A, 0x6A48000A, 0x175CA, 0x6A50000A, 0x1760A, 0x6A58000A, 0x1764A, 0x6A60000A, 0x1768A, 0x6A68000A, 0x176CA, 0x6A70000A, 0x1770A, 0x6A78000A, 0x1774A, 0x6A80000A, 0x1778A, 0x6A88000A, 0x177CA, 0x6A90000A, 0x1780A, 0x6A98000A, 0x1784A, 0x6AA0000A, 0x1788A, 0x6AA8000A, 0x178CA, 0x6AB0000A, 0x1790A, 0x6AB8000A, 0x1794A, 0x6AC0000A, 0x1798A, 0x6AC8000A, 0x179CA, 0x6AD0000A, 0x17A0A, 0x6AD8000A, 0x17A4A, 0x6AE0000A, 0x1ABAA, 0x1ABCA, 0x6AF8000A, 0x1AC0A, 0x6B08000A, 0x1AC4A, 0x6B18000A, 0x1AC8A, 0x6B28000A, 0x1ACCA, 0x6B38000A, 0x1AD0A, 0x6B48000A, 0x1AD4A, 0x6B58000A, 0x1AD8A, 0x6B68000A, 0x1ADCA, 0x6B78000A, 0x1AE0A, 0x6B88000A, 0x1AE4A, 0x6B98000A, 0x1AE8A, 0x6BA8000A, 0x1AECA, 0x6BB8000A, 0x1AF0A, 0x6BC8000A, 0x1AF4A, 0x6BD8000A, 0x6BE0000A, 0x6BE8000A, 0x6BF0000A, 0x6BF8000A, 0x6C00000A, 0x6C08000A, 0x1B04A, 0x1B06A, 0x1B08A, 0x1B0AA, 0x1B0CA, 0x1B0EA, 0x1B10A, 0x1B12A, 0x1B14A, 0x1B16A, 0x1B18A, 0x1B1AA, 0x1B1CA, 0x1B1EA, 0x1B20A, 0x1B22A, 0x1B24A, 0x1B26A, 0x1B28A, 0x1B2AA, 0x1B2CA, 0x1B2EA, 0x1B30A, 0x1B32A, 0x1B34A, 0x1B36A, 0x1B38A, 0x1B3AA, 0x1B3CA, 0x1B3EA, 0x1B40A, 0x1B42A, 0x1B44A, 0x1B46A, 0x1B48A, 0x1B4AA, 0x1B4CA, 0x1B4EA, 0x1B50A, 0x1B52A, 0x1B54A, 0x1B56A, 0x1B58A, 0x1B5AA, 0x1B5CA, 0x1B5EA, 0x1B60A, 0x1B62A, 0x1B64A, 0x1B66A, 0x1B68A, 0x1B6AA, 0x1B6CA, 0x1B6EA, 0x1B70A, 0x1B72A, 0x1B74A, 0x1B76A, 0x1B78A, 0x1B7AA, 0x1B7CA, 0x1B7EA, 0x1B80A, 0x1B82A, 0x1B84A, 0x1B86A, 0x1B88A, 0x1B8AA, 0x1B8CA, 0x1B8EA, 0x1B90A, 0x1B92A, 0x1B94A, 0x1B96A, 0x1B98A, 0x1B9AA, 0x1B9CA, 0x1B9EA, 0x1BA0A, 0x1BA2A, 0x1BA4A, 0x1BA6A, 0x1BA8A, 0x1BAAA, 0x1BACA, 0x1BAEA, 0x1BB0A, 0x1BB2A, 0x1BB4A, 0x1BB6A, 0x1BB8A, 0x1BBAA, 0x1BBCA, 0x1BBEA, 0x1BC0A, 0x1BC2A, 0x1BC4A, 0x1BC6A, 0x1BC8A, 0x1BCAA, 0x1BCCA, 0x1BCEA, 0x1BD0A, 0x1BD2A, 0x1BD4A, 0x1BD6A, 0x1BD8A, 0x1BDAA, 0x1BDCA, 0x1BDEA, 0x1BE0A, 0x1BE2A, 0x1BE4A, 0x1BE6A, 0x1BE8A, 0x1BEAA, 0x1BECA, 0x1BEEA, 0x1BF0A, 0x1BF2A, 0x1BF4A, 0x1BF6A, 0x1BF8A, 0x1BFAA, 0x1BFCA, 0x1BFEA, 0x1C00A, 0x1C02A, 0x1C04A, 0x1C06A, 0x1C08A, 0x1C0AA, 0x1C0CA, 0x1C0EA, 0x1C10A, 0x1C12A, 0x1C14A, 0x1C16A, 0x1C18A, 0x1C1AA, 0x1C1CA, 0x1C1EA, 0x1C20A, 0x1C22A, 0x1C24A, 0x7098000A, 0x70A0000A, 0x70A8000A, 0x70B0000A, 0x70B8000A, 0x70C0000A, 0x70C8000A, 0x70D0000A, 0x70D8000A, 0x70E0000A, 0x70E8000A, 0x70F0000A, 0x70F8000A, 0x7100000A, 0x7108000A, 0x7110000A, 0x7118000A, 0x7120000A, 0x7128000A, 0x7130000A, 0x7138000A, 0x7140000A, 0x7148000A, 0x7150000A, 0x7158000A, 0x7160000A, 0x7168000A, 0x7170000A, 0x7178000A, 0x7180000A, 0x7188000A, 0x7190000A, 0x7198000A, 0x71A0000A, 0x71A8000A, 0x71B0000A, 0x71B8000A, 0x71C0000A, 0x71C8000A, 0x71D0000A, 0x71D8000A, 0x71E0000A, 0x71E8000A, 0x71F0000A, 0x71F8000A, 0x7200000A, 0x7208000A, 0x7210000A, 0x7218000A, 0x7220000A, 0x7228000A, 0x7230000A, 0x7238000A, 0x7240000A, 0x7248000A, 0x7250000A, 0x7258000A, 0x7260000A, 0x7268000A, 0x7270000A, 0x7278000A, 0x7280000A, 0x7288000A, 0x7290000A, 0x7298000A, 0x72A0000A, 0x72A8000A, 0x72B0000A, 0x72B8000A, 0x72C0000A, 0x72C8000A, 0x72D0000A, 0x72D8000A, 0x72E0000A, 0x72E8000A, 0x72F0000A, 0x5BD8000A, 0x5BE0000A, 0x5BE8000A, 0x5BF0000A, 0x5BF8000A, 0x72F8000A, 0x7300000A, 0x7308000A, 0x7310000A, 0x6928000A, 0x6930000A, 0x6938000A, 0x6940000A, 0x6948000A, 0x6950000A, 0x6958000A, 0x6960000A, 0x6968000A, 0x7318000A, 0x6978000A, 0x6980000A, 0x6988000A, 0x6990000A, 0x6998000A, 0x69A0000A, 0x69A8000A, 0x69B0000A, 0x69B8000A, 0x69C0000A, 0x69C8000A, 0x69D0000A, 0x69D8000A, 0x69E0000A, 0x69E8000A, 0x69F0000A, 0x7320000A, 0x7328000A, 0x7330000A, 0x7338000A, 0x7340000A, 0x7348000A, 0x7350000A, 0x7358000A, 0x7360000A, 0x7368000A, 0x7370000A, 0x7378000A, 0x7380000A, 0x7388000A, 0x7390000A, 0x7398000A, 0x73A0000A, 0x73A8000A, 0x73B0000A, 0x73B8000A, 0x73C0000A, 0x73C8000A, 0x73D0000A, 0x73D8000A, 0x73E0000A, 0x73E8000A, 0x73F0000A, 0x73F8000A, 0x7400000A, 0x7408000A, 0x7410000A, 0x7418000A, 0x7420000A, 0x7428000A, 0x7430000A, 0x7438000A, 0x7440000A, 0x7448000A, 0x7450000A, 0x7458000A, 0x7460000A, 0x7468000A, 0x7470000A, 0x7478000A, 0x7480000A, 0x7488000A, 0x7490000A, 0x7498000A, 0x74A0000A, 0x74A8000A, 0x74B0000A, 0x74B8000A, 0x74C0000A, 0x74C8000A, 0x74D0000A, 0x74D8000A, 0x74E0000A, 0x74E8000A, 0x74F0000A, 0x74F8000A, 0x7500000A, 0x7508000A, 0x7510000A, 0x7518000A, 0x7520000A, 0x7528000A, 0x7530000A, 0x7538000A, 0x7540000A, 0x7548000A, 0x7550000A, 0x7558000A, 0x7560000A, 0x7568000A, 0x7570000A, 0x7578000A, 0x7580000A, 0x7588000A, 0x7590000A, 0x7598000A, 0x75A0000A, 0x75A8000A, 0x75B0000A, 0x75B8000A, 0x6D40000A, 0x6D48000A, 0x6D50000A, 0x6D58000A, 0x6D60000A, 0x6D68000A, 0x6D70000A, 0x6D78000A, 0x6D80000A, 0x6D88000A, 0x6D90000A, 0x6D98000A, 0x6DA0000A, 0x6DA8000A, 0x6DB0000A, 0x6DB8000A, 0x6DC0000A, 0x6DC8000A, 0x6DD0000A, 0x6DD8000A, 0x6DE0000A, 0x6DE8000A, 0x6DF0000A, 0x6DF8000A, 0x75C0000A, 0x75C8000A, 0x75D0000A, 0x75D8000A, 0x75E0000A, 0x75E8000A, 0x75F0000A, 0x75F8000A, 0x7600000A, 0x7608000A, 0x7610000A, 0x7618000A, 0x7620000A, 0x7628000A, 0x7630000A, 0x7638000A, 0x7640000A, 0x7648000A, 0x7650000A, 0x7658000A, 0x7660000A, 0x7668000A, 0x7670000A, 0x7678000A, 0x7680000A, 0x7688000A, 0x7690000A, 0x7698000A, 0x76A0000A, 0x76A8000A, 0x76B0000A, 0x76B8000A, 0x76C0000A, 0x76C8000A, 0x76D0000A, 0x76D8000A, 0x76E0000A, 0x76E8000A, 0x76F0000A, 0x76F8000A, 0x7700000A, 0x7708000A, 0x7710000A, 0x7718000A, 0x7720000A, 0x7728000A, 0x7730000A, 0x7738000A, 0x7740000A, 0x7748000A, 0x7750000A, 0x7758000A, 0x7760000A, 0x1DDAB, 0x1DDCB, 0x1DDEB, 0x1DE0B, 0x1DE2B, 0x1DE4B, 0x1DE6B, 0x1DE8B, 0x1DEAB, 0x1DECB, 0x1DEEB, 0x1DF0B, 0x1DF2B, 0x1DF4B, 0x1DF6B, 0x1DF8B, 0x1DFAB, 0x1DFCB, 0x1DFEB, 0x1E00B, 0x1E02B, 0x1E04B, 0x1E06B, 0x1E08B, 0x1E0AB, 0x1E0CB, 0x1E0EB, 0x7840000B, 0x1E12B, 0x7850000B, 0x1E16B, 0x7860000B, 0x1E1AB, 0x7870000B, 0x1E1EB, 0x7880000B, 0x1E22B, 0x7890000B, 0x1E26B, 0x78A0000B, 0x1E2AB, 0x78B0000B, 0x1E2EB, 0x78C0000B, 0x1E32B, 0x78D0000B, 0x1E36B, 0x78E0000B, 0x1E3AB, 0x78F0000B, 0x1E3EB, 0x7900000B, 0x1E42B, 0x7910000B, 0x1E46B, 0x7920000B, 0x1E4AB, 0x7930000B, 0x1E4EB, 0x7940000B, 0x1E52B, 0x7950000B, 0x1E56B, 0x7960000B, 0x1E5AB, 0x7970000B, 0x1E5EB, 0x7980000B, 0x1E62B, 0x7990000B, 0x1E66B, 0x79A0000B, 0x1E6AB, 0x79B0000B, 0x1E6EB, 0x79C0000B, 0x1E72B, 0x79D0000B, 0x1E76B, 0x79E0000B, 0x1E7AB, 0x79F0000B, 0x1E7EB, 0x7A00000B, 0x1E82B, 0x7A10000B, 0x1E86B, 0x7A20000B, 0x1E8AB, 0x7A30000B, 0x1E8EB, 0x7A40000B, 0x1E92B, 0x7A50000B, 0x1E96B, 0x7A60000B, 0x1E9AB, 0x7A70000B, 0x1E9EB, 0x7A80000B, 0x1EA2B, 0x7A90000B, 0x1EA6B, 0x7AA0000B, 0x1EAAB, 0x7AB0000B, 0x1EAEB, 0x7AC0000B, 0x1EB2B, 0x7AD0000B, 0x1EB6B, 0x7AE0000B, 0x1EBAB, 0x7AF0000B, 0x1EBEB, 0x7B00000B, 0x1EC2B, 0x7B10000B, 0x1EC6B, 0x7B20000B, 0x1ECAB, 0x7B30000B, 0x1ECEB, 0x7838000B, 0x1ED0B, 0x7848000B, 0x1ED2B, 0x7858000B, 0x1ED4B, 0x7868000B, 0x1ED6B, 0x7878000B, 0x1ED8B, 0x7888000B, 0x1EDAB, 0x7898000B, 0x1EDCB, 0x78A8000B, 0x1EDEB, 0x78B8000B, 0x1EE0B, 0x78C8000B, 0x1EE2B, 0x78D8000B, 0x1EE4B, 0x78E8000B, 0x1EE6B, 0x78F8000B, 0x1EE8B, 0x7908000B, 0x1EEAB, 0x7918000B, 0x1EECB, 0x7928000B, 0x1EEEB, 0x7938000B, 0x1EF0B, 0x7948000B, 0x1EF2B, 0x7958000B, 0x1EF4B, 0x7968000B, 0x1EF6B, 0x7978000B, 0x1EF8B, 0x7988000B, 0x1EFAB, 0x7998000B, 0x1EFCB, 0x79A8000B, 0x1EFEB, 0x79B8000B, 0x1F00B, 0x79C8000B, 0x1F02B, 0x79D8000B, 0x1F04B, 0x79E8000B, 0x1F06B, 0x79F8000B, 0x1F08B, 0x7A08000B, 0x1F0AB, 0x7A18000B, 0x1F0CB, 0x7A28000B, 0x7C41F0EB, 0x7C48000B, 0x1F14B, 0x7C58000B, 0x7C60000B, 0x7C68000B, 0x7C70000B, 0x7C78000B, 0x7C80000B, 0x7C88000B, 0x7C90000B, 0x7C98000B, 0x7CA0000B, 0x7CA8000B, 0x7CB0000B, 0x7CB8000B, 0x7CC0000B, 0x7CC8000B, 0x7CD0000B, 0x7CD8000B, 0x7CE0000B, 0x7CE8000B, 0x7CF0000B, 0x7CF8000B, 0x7D00000B, 0x7D08000B, 0x7D10000B, 0x7D18000B, 0x7D20000B, 0x7D28000B, 0x7D30000B, 0x7D38000B, 0x7D40000B, 0x7D48000B, 0x7D50000B, 0x7D58000B, 0x7D60000B, 0x7D68000B, 0x7D70000B, 0x7D78000B, 0x7D80000B, 0x7D88000B, 0x7D90000B, 0x7D98000B, 0x7DA0000B, 0x7DA8000B, 0x7DB0000B, 0x7DB8000B, 0x7DC0000B, 0x7DC8000B, 0x1F74B, 0x7DD8000B, 0x7DE0000B, 0x7DE8000B, 0x7DF0000B, 0x7DF8000B, 0x7E00000B, 0x7E08000B, 0x7E10000B, 0x7E18000B, 0x7E20000B, 0x7E28000B, 0x7E30000B, 0x7E38000B, 0x7E40000B, 0x7E48000B, 0x7E50000B, 0x7E58000B, 0x7E60000B, 0x7E68000B, 0x7E70000B, 0x7E78000B, 0x7E80000B, 0x7E88000B, 0x7E90000B, 0x7E98000B, 0x7EA0000B, 0x7EA8000B, 0x7EB0000B, 0x7EB8000B, 0x7EC0000B, 0x7EC8000B, 0x7ED0000B, 0x7ED8000B, 0x7EE0000B, 0x7EE8000B, 0x7EF0000B, 0x7EF8000B, 0x7F00000B, 0x7F08000B, 0x7F10000B, 0x7F18000B, 0x7F20000B, 0x7F28000B, 0x7F30000B, 0x7F38000B, 0x7F40000B, 0x7F48000B, 0x7F50000B, 0x7F58000B, 0x7F60000B, 0x7F68000B, 0x7F70000B, 0x7F78000B, 0x7F80000B, 0x7F88000B, 0x7F90000B, 0x7F98000B, 0x7FA0000B, 0x7FA8000B, 0x7FB0000B, 0x7FB8000B, 0x7FC0000B, 0x7FC8000B, 0x7FD0000B, 0x7FD8000B, 0x7FE0000B, 0x7FE8000B, 0x7FF0000B, 0x7FF8000B, 0x8000000B, 0x8008000B, 0x8010000B, 0x8018000B, 0x8020000B, 0x8028000B, 0x8030000B, 0x8038000B, 0x2010B, 0x8048000B, 0x2014B, 0x8058000B, 0x2018B, 0x8068000B, 0x201CB, 0x8078000B, 0x2020B, 0x8088000B, 0x2024B, 0x8098000B, 0x2028B, 0x80A8000B, 0x202CB, 0x80B8000B, 0x2030B, 0x80C8000B, 0x2034B, 0x80D8000B, 0x2038B, 0x80E8000B, 0x203CB, 0x80F8000B, 0x2040B, 0x8108000B, 0x2044B, 0x8118000B, 0x2048B, 0x8128000B, 0x204CB, 0x8138000B, 0x2050B, 0x8148000B, 0x2054B, 0x8158000B, 0x2058B, 0x8168000B, 0x205CB, 0x8178000B, 0x2060B, 0x8188000B, 0x2064B, 0x8198000B, 0x2068B, 0x81A8000B, 0x206CB, 0x81B8000B, 0x2070B, 0x81C8000B, 0x2074B, 0x81D8000B, 0x2078B, 0x81E8000B, 0x207CB, 0x81F8000B, 0x2080B, 0x8208000B, 0x2084B, 0x8218000B, 0x2088B, 0x8228000B, 0x208CB, 0x8238000B, 0x2090B, 0x8248000B, 0x2094B, 0x8258000B, 0x2098B, 0x8268000B, 0x209CB, 0x8278000B, 0x20A0B, 0x8288000B, 0x20A4B, 0x8298000B, 0x20A8B, 0x82A8000B, 0x20ACB, 0x82B8000B, 0x20B0B, 0x20B2B, 0x20B4B, 0x20B6B, 0x20B8B, 0x20BAB, 0x20BCB, 0x20BEB, 0x20C0B, 0x20C2B, 0x20C4B, 0x20C6B, 0x20C8B, 0x20CAB, 0x20CCB, 0x20CEB, 0x20D0B, 0x20D2B, 0x20D4B, 0x20D6B, 0x20D8B, 0x20DAB, 0x20DCB, 0x20DEB, 0x20E0B, 0x20E2B, 0x20E4B, 0x20E6B, 0x20E8B, 0x20EAB, 0x20ECB, 0x20EEB, 0x20F0B, 0x20F2B, 0x20F4B, 0x20F6B, 0x20F8B, 0x20FAB, 0x20FCB, 0x20FEB, 0x2100B, 0x2102B, 0x2104B, 0x2106B, 0x2108B, 0x210AB, 0x210CB, 0x210EB, 0x2110B, 0x2112B, 0x2114B, 0x2116B, 0x2118B, 0x211AB, 0x8470000B, 0x211EB, 0x8480000B, 0x2122B, 0x2124B, 0x2126B, 0x84A0000B, 0x212AB, 0x84B0000B, 0x212EB, 0x84C0000B, 0x2132B, 0x84D0000B, 0x2136B, 0x2138B, 0x213AB, 0x213CB, 0x84F8000B, 0x2140B, 0x8508000B, 0x2144B, 0x8518000B, 0x2148B, 0x214AB, 0x214CB, 0x8538000B, 0x2150B, 0x2152B, 0x2154B, 0x2156B, 0x2158B, 0x215AB, 0x215CB, 0x215EB, 0x2160B, 0x2162B, 0x2164B, 0x2166B, 0x2168B, 0x216AB, 0x216CB, 0x216EB, 0x2170B, 0x2172B, 0x2174B, 0x2176B, 0x2178B, 0x217AB, 0x217CB, 0x217EB, 0x2180B, 0x2182B, 0x2184B, 0x2186B, 0x2188B, 0x218AB, 0x218CB, 0x218EB, 0x2190B, 0x2192B, 0x2194B, 0x2196B, 0x2198B, 0x219AB, 0x219CB, 0x219EB, 0x21A0B, 0x21A2B, 0x21A4B, 0x21A6B, 0x21A8B, 0x21AAB, 0x21ACB, 0x21AEB, 0x86C0000B, 0x86C8000B, 0x86D0000B, 0x86D8000B, 0x86E0000B, 0x86E8000B, 0x86F0000B, 0x86F8000B, 0x8700000B, 0x8708000B, 0x8710000B, 0x8718000B, 0x8720000B, 0x8728000B, 0x8730000B, 0x8738000B, 0x8740000B, 0x8748000B, 0x8750000B, 0x8758000B, 0x8760000B, 0x8768000B, 0x8770000B, 0x8778000B, 0x8780000B, 0x8788000B, 0x8790000B, 0x8798000B, 0x87A0000B, 0x87A8000B, 0x87B0000B, 0x87B8000B, 0x87C0000B, 0x87C8000B, 0x87D0000B, 0x87D8000B, 0x87E0000B, 0x87E8000B, 0x87F0000B, 0x87F8000B, 0x8800000B, 0x8808000B, 0x8810000B, 0x8818000B, 0x8820000B, 0x8828000B, 0x8830000B, 0x8838000B, 0x8840000B, 0x8848000B, 0x2214B, 0x2216B, 0x2218B, 0x221AB, 0x221CB, 0x221EB, 0x2220B, 0x2222B, 0x2224B, 0x2226B, 0x2228B, 0x222AB, 0x222CB, 0x222EB, 0x2230B, 0x2232B, 0x2234B, 0x2236B, 0x2238B, 0x223AB, 0x223CB, 0x223EB, 0x2240B, 0x2242B, 0x2244B, 0x2246B, 0x2248B, 0x224AB, 0x224CB, 0x224EB, 0x2250B, 0x2252B, 0x2254B, 0x2256B, 0x2258B, 0x225AB, 0x225CB, 0x225EB, 0x2260B, 0x2262B, 0x2264B, 0x2266B, 0x2268B, 0x226AB, 0x226CB, 0x226EB, 0x2270B, 0x2272B, 0x2274B, 0x2276B, 0x2278B, 0x227AB, 0x227CB, 0x227EB, 0x2280B, 0x2282B, 0x2284B, 0x2286B, 0x2288B, 0x228AB, 0x228CB, 0x228EB, 0x2290B, 0x2292B, 0x2294B, 0x2296B, 0x2298B, 0x229AB, 0x229CB, 0x229EB, 0x22A0B, 0x22A2B, 0x22A4B, 0x22A6B, 0x22A8B, 0x22AAB, 0x22ACB, 0x22AEB, 0x22B0B, 0x22B2B, 0x22B4B, 0x22B6B, 0x22B8B, 0x22BAB, 0x22BCB, 0x22BEB, 0x22C0B, 0x22C2B, 0x22C4B, 0x22C6B, 0x22C8B, 0x22CAB, 0x22CCB, 0x22CEB, 0x22D0B, 0x22D2B, 0x22D4B, 0x22D6B, 0x22D8B, 0x22DAB, 0x22DCB, 0x22DEB, 0x22E0B, 0x22E2B, 0x22E4B, 0x22E6B, 0x22E8B, 0x22EAB, 0x22ECB, 0x22EEB, 0x22F0B, 0x22F2B, 0x22F4B, 0x22F6B, 0x22F8B, 0x22FAB, 0x22FCB, 0x22FEB, 0x2300B, 0x2302B, 0x2304B, 0x2306B, 0x2308B, 0x230AB, 0x230CB, 0x230EB, 0x2310B, 0x2312B, 0x2314B, 0x2316B, 0x2318B, 0x231AB, 0x231CB, 0x231EB, 0x2320B, 0x2322B, 0x2324B, 0x2326B, 0x2328B, 0x232AB, 0x232CB, 0x232EB, 0x2330B, 0x2332B, 0x2334B, 0x8CD8000B, 0x1FCCB, 0x1FCEB, 0x1FD0B, 0x1FD2B, 0x1FD4B, 0x1FD6B, 0x1FD8B, 0x1FDAB, 0x1FDCB, 0x1FDEB, 0x1FE0B, 0x2338B, 0x233AB, 0x233CB, 0x233EB, 0x2340B, 0x2342B, 0x2344B, 0x2346B, 0x2348B, 0x1FE2B, 0x1FE4B, 0x1FE6B, 0x1FE8B, 0x1FEAB, 0x1FECB, 0x1FEEB, 0x1FF0B, 0x1FF2B, 0x1FF4B, 0x1FF6B, 0x1FF8B, 0x1FFAB, 0x1FFCB, 0x1FFEB, 0x2000B, 0x2002B, 0x2004B, 0x2006B, 0x2008B, 0x200AB, 0x200CB, 0x200EB, 0x234AB, 0x234CB, 0x234EB, 0x2350B, 0x2352B, 0x2354B, 0x2356B, 0x2358B, 0x235AB, 0x235CB, 0x235EB, 0x2360B, 0x2362B, 0x2364B, 0x2366B, 0x2368B, 0x236AB, 0x236CB, 0x236EB, 0x2370B, 0x2372B, 0x2374B, 0x2376B, 0x2378B, 0x237AB, 0x237CB, 0x237EB, 0x2380B, 0x2382B, 0x2384B, 0x2386B, 0x2388B, 0x8E3238AB, 0x8E4238EB, 0x8E52392B, 0x8E62396B, 0x8E7239AB, 0x8E8239EB, 0x8E923A2B, 0x8EA23A6B, 0x8EB23AAB, 0x8EC23AEB, 0x8ED23B2B, 0x8EE23B6B, 0x8EF23BAB, 0x8F023BEB, 0x8F123C2B, 0x8F223C6B, 0x8F323CAB, 0x8F423CEB, 0x8F523D2B, 0x8F623D6B, 0x8F723DAB, 0x8F823DEB, 0x8F923E2B, 0x8FA23E6B, 0x8FB23EAB, 0x8FC23EEB, 0x8FD23F2B, 0x8FE23F6B, 0x8FF23FAB, 0x90023FEB, 0x8CDA402B, 0x901A404B, 0x902A408B, 0x903A40CB, 0x904A410B, 0x905A414B, 0x906A418B, 0x907A41CB, 0x908A420B, 0x909A424B, 0x90AA428B, 0x90BA42CB, 0x90CA430B, 0x90DA434B, 0x90EA438B, 0x90FA43CB, 0x9100000B, 0x9108000B, 0x9110000B, 0x9118000B, 0x9120000B, 0x9128000B, 0x9130000B, 0x9138000B, 0x9140000B, 0x9148000B, 0x9150000B, 0x9158000B, 0x9160000B, 0x9168000B, 0x9170000B, 0x9178000B, 0x9180000B, 0x9188000B, 0x9190000B, 0x9198000B, 0x91A0000B, 0x91A8000B, 0x91B0000B, 0x91B8000B, 0x91C0000B, 0x91C8000B, 0x91D0000B, 0x91D8000B, 0x91E0000B, 0x91E8000B, 0x91F0000B, 0x91F8000B, 0x9200000B, 0x9208000B, 0x9210000B, 0x9218000B, 0x9220000B, 0x9228000B, 0x9230000B, 0x9238000B, 0x9240000B, 0x9248000B, 0x9250000B, 0x9258000B, 0x9260000B, 0x9268000B, 0x9270000B, 0x9278000B, 0x9280000B, 0x9288000B, 0x9290000B, 0x9298000B, 0x92A0000B, 0x92A8000B, 0x92B0000B, 0x92B8000B, 0x92C0000B, 0x92C8000B, 0x92D0000B, 0x92D8000B, 0x92E0000B, 0x92E8000B, 0x92F0000B, 0x92F8000B, 0x9300000B, 0x9308000B, 0x9310000B, 0x9318000B, 0x9320000B, 0x9328000B, 0x9330000B, 0x9338000B, 0x9340000B, 0x9348000B, 0x82C0000B, 0x9350000B, 0x9358000B, 0x9360000B, 0x9368000B, 0x9370000B, 0x9378000B, 0x9380000B, 0x82D8000B, 0x82E0000B, 0x82E8000B, 0x82F0000B, 0x82F8000B, 0x8300000B, 0x8308000B, 0x8310000B, 0x8318000B, 0x8320000B, 0x9388000B, 0x9390000B, 0x9398000B, 0x93A0000B, 0x93A8000B, 0x93B0000B, 0x93B8000B, 0x93C0000B, 0x8368000B, 0x8370000B, 0x8378000B, 0x8380000B, 0x8388000B, 0x8390000B, 0x8398000B, 0x83A0000B, 0x83A8000B, 0x83B0000B, 0x83B8000B, 0x83C0000B, 0x83C8000B, 0x83D0000B, 0x83D8000B, 0x83E0000B, 0x83E8000B, 0x83F0000B, 0x83F8000B, 0x8400000B, 0x8408000B, 0x8410000B, 0x8418000B, 0x8420000B, 0x8428000B, 0x8430000B, 0x8438000B, 0x8460000B, 0x93C8000B, 0x93D0000B, 0x93D8000B, 0x93E0000B, 0x93E8000B, 0x93F0000B, 0x8490000B, 0x93F8000B, 0x9400000B, 0x9408000B, 0x9410000B, 0x9418000B, 0x9420000B, 0x9428000B, 0x9430000B, 0x8540000B, 0x9438000B, 0x9440000B, 0x9448000B, 0x9450000B, 0x9458000B, 0x9460000B, 0x9468000B, 0x9470000B, 0x9478000B, 0x9480000B, 0x9488000B, 0x9490000B, 0x9498000B, 0x94A0000B, 0x94A8000B, 0x94B0000B, 0x94B8000B, 0x94C0000B, 0x94C8000B, 0x94D0000B, 0x94D8000B, 0x94E0000B, 0x94E8000B, 0x94F0000B, 0x94F8000B, 0x9500000B, 0x9508000B, 0x9510000B, 0x9518000B, 0x9520000B, 0x9528000B, 0x9530000B, 0x9538000B, 0x9540000B, 0x9548000B, 0x9550000B, 0x9558000B, 0x9560000B, 0x9568000B, 0x9570000B, 0x9578000B, 0x9580000B, 0x9588000B, 0x9590000B, 0x9598000B, 0x95A0000B, 0x95A8000B, 0x95B0000B, 0x95B8000B, 0x95C0000B, 0x95C8000B, 0x95D0000B, 0x95D8000B, 0x95E0000B, 0x95E8000B, 0x95F0000B, 0x95F8000B, 0x9600000B, 0x9608000B, 0x9610000B, 0x9618000B, 0x9620000B, 0x9628000B, 0x9630000B, 0x9638000B, 0x9640000B, 0x9648000B, 0x9650000B, 0x9658000B, 0x9660000B, 0x9668000B, 0x9670000B, 0x9678000B, 0x9680000B, 0x9688000B, 0x9690000B, 0x9698000B, 0x96A0000B, 0x96A8000B, 0x96B0000B, 0x96B8000B, 0x96C0000B, 0x96C8000B, 0x96D0000B, 0x96D8000B, 0x96E0000B, 0x96E8000B, 0x96F0000B, 0x96F8000B, 0x9700000B, 0x9708000B, 0x9710000B, 0x9718000B, 0x9720000B, 0x9728000B, 0x9730000B, 0x9738000B, 0x9740000B, 0x9748000B, 0x9750000B, 0x9758000B, 0x9760000B, 0x9768000B, 0x9770000B, 0x9778000B, 0x9780000B, 0x9788000B, 0x9790000B, 0x9798000B, 0x97A0000B, 0x97A8000B, 0x97B0000B, 0x97B8000B, 0x97C0000B, 0x97C8000B, 0x97D0000B, 0x97D8000B, 0x97E0000B, 0x97E8000B, 0x97F0000B, 0x97F8000B, 0x9800000B, 0x9808000B, 0x9810000B, 0x9818000B, 0x9820000B, 0x9828000B, 0x9830000B, 0x9838000B, 0x9840000B, 0x9848000B, 0x9850000B, 0x9858000B, 0x9860000B, 0x9868000B, 0x9870000B, 0x9878000B, 0x9880000B, 0x9888000B, 0x9890000B, 0x9898000B, 0x98A0000B, 0x98A8000B, 0x98B0000B, 0x98B8000B, 0x98C0000B, 0x98C8000B, 0x98D0000B, 0x98D8000B, 0x98E0000B, 0x98E8000B, 0x98F0000B, 0x98F8000B, 0x9900000B, 0x9908000B, 0x9910000B, 0x9918000B, 0x9920000B, 0x9928000B, 0x9930000B, 0x9938000B, 0x9940000B, 0x9948000B, 0x9950000B, 0x9958000B, 0x9960000B, 0x9968000B, 0x9970000B, 0x9978000B, 0x9980000B, 0x9988000B, 0x9990000B, 0x9998000B, 0x99A0000B, 0x99A8000B, 0x99B0000B, 0x99B8000B, 0x99C0000B, 0x99C8000B, 0x99D0000B, 0x99D8000B, 0x99E0000B, 0x99E8000B, 0x99F0000B, 0x99F8000B, 0x9A00000B, 0x9A08000B, 0x9A10000B, 0x9A18000B, 0x9A20000B, 0x9A28000B, 0x9A30000B, 0x9A38000B, 0x9A40000B, 0x9A48000B, 0x9A50000B, 0x9A58000B, 0x9A60000B, 0x9A68000B, 0x9A70000B, 0x9A78000B, 0x9A80000B, 0x9A88000B, 0x9A90000B, 0x9A98000B, 0x9AA0000B, 0x9AA8000B, 0x9AB0000B, 0x9AB8000B, 0x9AC0000B, 0x9AC8000B, 0x9AD0000B, 0x9AD8000B, 0x9AE0000B, 0x9AE8000B, 0x9AF0000B, 0x9AF8000B, 0x9B00000B, 0x9B08000B, 0x9B10000B, 0x9B18000B, 0x9B20000B, 0x9B28000B, 0x9B30000B, 0x9B38000B, 0x9B40000B, 0x9B48000B, 0x9B50000B, 0x9B58000B, 0x9B60000B, 0x9B68000B, 0x9B70000B, 0x9B78000B, 0x9B80000B, 0x9B88000B, 0x9B90000B, 0x9B98000B, 0x9BA0000B, 0x9BA8000B, 0x9BB0000B, 0x9BB8000B, 0x9BC0000B, 0x9BC8000B, 0x9BD0000B, 0x9BD8000B, 0x9BE0000B, 0x9BE8000B, 0x9BF0000B, 0x9BF8000B, 0x9C00000B, 0x9C08000B, 0x9C10000B, 0x9C18000B, 0x9C20000B, 0x9C28000B, 0x9C30000B, 0x9C38000B, 0x9C40000B, 0x9C48000B, 0x9C50000B, 0x9C58000B, 0x9C60000B, 0x9C68000B, 0x9C70000B, 0x9C78000B, 0x9C80000B, 0x9C88000B, 0x9C90000B, 0x9C98000B, 0x9CA0000B, 0x9CA8000B, 0x9CB0000B, 0x9CB8000B, 0x9CC0000B, 0x9CC8000B, 0x9CD0000B, 0x9CD8000B, 0x9CE0000B, 0x9CE8000B, 0x9CF0000B, 0x9CF8000B, 0x9D00000B, 0x9D08000B, 0x9D10000B, 0x9D18000B, 0x9D20000B, 0x9D28000B, 0x9D30000B, 0x9D38000B, 0x9D40000B, 0x9D48000B, 0x9D50000B, 0x9D58000B, 0x9D60000B, 0x9D68000B, 0x9D70000B, 0x9D78000B, 0x9D80000B, 0x9D88000B, 0x9D90000B, 0x9D98000B, 0x9DA0000B, 0x9DA8000B, 0x9DB0000B, 0x9DB8000B, 0x9DC0000B, 0x9DC8000B, 0x9DD0000B, 0x9DD8000B, 0x9DE0000B, 0x9DE8000B, 0x9DF0000B, 0x9DF8000B, 0x9E00000B, 0x9E08000B, 0x9E10000B, 0x9E18000B, 0x9E20000B, 0x9E28000B, 0x9E30000B, 0x278EC, 0x2790C, 0x2792C, 0x2794C, 0x2796C, 0x2798C, 0x279AC, 0x279CC, 0x279EC, 0x27A0C, 0x27A2C, 0x27A4C, 0x27A6C, 0x27A8C, 0x27AAC, 0x27ACC, 0x27AEC, 0x27B0C, 0x27B2C, 0x27B4C, 0x27B6C, 0x27B8C, 0x27BAC, 0x27BCC, 0x27BEC, 0x27C0C, 0x9F08000C, 0x27C4C, 0x9F18000C, 0x27C8C, 0x9F28000C, 0x27CCC, 0x9F38000C, 0x27D0C, 0x9F48000C, 0x27D4C, 0x9F58000C, 0x27D8C, 0x9F68000C, 0x27DCC, 0x9F78000C, 0x27E0C, 0x9F88000C, 0x27E4C, 0x9F98000C, 0x27E8C, 0x9FA8000C, 0x27ECC, 0x9FB8000C, 0x27F0C, 0x9FC8000C, 0x27F4C, 0x9FD8000C, 0x27F8C, 0x9FE8000C, 0x27FCC, 0x9FF8000C, 0x2800C, 0xA008000C, 0x2804C, 0xA018000C, 0x2808C, 0xA028000C, 0x280CC, 0xA038000C, 0x2810C, 0xA048000C, 0x2814C, 0xA058000C, 0x2818C, 0xA068000C, 0x281CC, 0xA078000C, 0x2820C, 0xA088000C, 0x2824C, 0xA098000C, 0x2828C, 0xA0A8000C, 0x282CC, 0xA0B8000C, 0x2830C, 0xA0C8000C, 0x2834C, 0xA0D8000C, 0x2838C, 0xA0E8000C, 0x283CC, 0xA0F8000C, 0x2840C, 0xA108000C, 0x2844C, 0xA118000C, 0x2848C, 0xA128000C, 0x284CC, 0xA138000C, 0x2850C, 0xA148000C, 0x2854C, 0xA158000C, 0x2858C, 0xA168000C, 0x285CC, 0xA178000C, 0x2860C, 0xA188000C, 0x2864C, 0xA198000C, 0x2868C, 0xA1A8000C, 0x286CC, 0xA1B8000C, 0x2870C, 0xA1C8000C, 0x2874C, 0xA1D8000C, 0x2878C, 0xA1E8000C, 0x287CC, 0xA1F8000C, 0x2880C, 0xA208000C, 0x2884C, 0xA218000C, 0x2888C, 0xA228000C, 0x288CC, 0xA238000C, 0x2890C, 0xA248000C, 0x2894C, 0xA258000C, 0x2898C, 0xA268000C, 0x289CC, 0xA278000C, 0x28A0C, 0xA288000C, 0x28A4C, 0xA298000C, 0x28A8C, 0xA2A8000C, 0x28ACC, 0xA2B8000C, 0x28B0C, 0xA2C8000C, 0x28B4C, 0xA2D8000C, 0x28B8C, 0xA2E8000C, 0x28BCC, 0xA2F8000C, 0x28C0C, 0xA308000C, 0x28C4C, 0xA318000C, 0x28C8C, 0xA328000C, 0x28CCC, 0xA338000C, 0x28D0C, 0xA348000C, 0x28D4C, 0xA358000C, 0x28D8C, 0xA368000C, 0x28DCC, 0xA378000C, 0x28E0C, 0xA388000C, 0x28E4C, 0xA398000C, 0x28E8C, 0xA3A8000C, 0x28ECC, 0xA3B8000C, 0x28F0C, 0xA3C8000C, 0x28F4C, 0xA3D8000C, 0x28F8C, 0xA3E8000C, 0x28FCC, 0xA3F8000C, 0x2900C, 0xA408000C, 0xA410000C, 0x2906C, 0xA420000C, 0xA428000C, 0xA430000C, 0xA438000C, 0xA440000C, 0xA448000C, 0xA450000C, 0xA458000C, 0xA460000C, 0xA468000C, 0xA470000C, 0xA478000C, 0xA480000C, 0xA488000C, 0xA490000C, 0xA498000C, 0xA4A0000C, 0xA4A8000C, 0xA4B0000C, 0xA4B8000C, 0xA4C0000C, 0xA4C8000C, 0xA4D0000C, 0xA4D8000C, 0xA4E0000C, 0xA4E8000C, 0xA4F0000C, 0xA4F8000C, 0xA500000C, 0xA508000C, 0xA510000C, 0xA518000C, 0xA520000C, 0xA528000C, 0xA530000C, 0xA538000C, 0xA540000C, 0xA548000C, 0xA550000C, 0xA558000C, 0xA560000C, 0xA568000C, 0xA570000C, 0xA578000C, 0xA580000C, 0xA588000C, 0xA590000C, 0x2966C, 0xA5A0000C, 0xA5A8000C, 0xA5B0000C, 0xA5B8000C, 0xA5C0000C, 0xA5C8000C, 0xA5D0000C, 0xA5D8000C, 0xA5E0000C, 0xA5E8000C, 0xA5F0000C, 0xA5F8000C, 0xA600000C, 0xA608000C, 0xA610000C, 0xA618000C, 0xA620000C, 0xA628000C, 0xA630000C, 0xA638000C, 0xA640000C, 0xA648000C, 0xA650000C, 0xA658000C, 0xA660000C, 0xA668000C, 0xA670000C, 0xA678000C, 0xA680000C, 0xA688000C, 0xA690000C, 0xA698000C, 0xA6A0000C, 0xA6A8000C, 0xA6B0000C, 0xA6B8000C, 0xA6C0000C, 0xA6C8000C, 0xA6D0000C, 0xA6D8000C, 0xA6E0000C, 0xA6E8000C, 0xA6F0000C, 0xA6F8000C, 0xA700000C, 0xA708000C, 0xA710000C, 0xA718000C, 0xA720000C, 0xA728000C, 0xA730000C, 0xA738000C, 0xA740000C, 0xA748000C, 0xA750000C, 0xA758000C, 0xA760000C, 0xA768000C, 0xA770000C, 0xA778000C, 0xA780000C, 0xA788000C, 0xA790000C, 0xA798000C, 0xA7A0000C, 0xA7A8000C, 0xA7B0000C, 0xA7B8000C, 0xA7C0000C, 0xA7C8000C, 0xA7D0000C, 0xA7D8000C, 0xA7E0000C, 0xA7E8000C, 0xA7F0000C, 0xA7F8000C, 0xA800000C, 0xA808000C, 0x2A04C, 0xA818000C, 0x2A08C, 0xA828000C, 0x2A0CC, 0xA838000C, 0x2A10C, 0xA848000C, 0x2A14C, 0xA858000C, 0x2A18C, 0xA868000C, 0x2A1CC, 0xA878000C, 0x2A20C, 0xA888000C, 0x2A24C, 0xA898000C, 0x2A28C, 0xA8A8000C, 0x2A2CC, 0xA8B8000C, 0x2A30C, 0xA8C8000C, 0x2A34C, 0xA8D8000C, 0x2A38C, 0xA8E8000C, 0x2A3CC, 0xA8F8000C, 0x2A40C, 0xA908000C, 0x2A44C, 0xA918000C, 0x2A48C, 0xA928000C, 0x2A4CC, 0xA938000C, 0x2A50C, 0xA948000C, 0x2A54C, 0xA958000C, 0x2A58C, 0xA968000C, 0x2A5CC, 0xA978000C, 0x2A60C, 0xA988000C, 0x2A64C, 0xA998000C, 0x2A68C, 0xA9A8000C, 0x2A6CC, 0xA9B8000C, 0x2A70C, 0xA9C8000C, 0x2A74C, 0xA9D8000C, 0x2A78C, 0xA9E8000C, 0x2A7CC, 0xA9F8000C, 0x2A80C, 0xAA08000C, 0x2A84C, 0xAA18000C, 0x2A88C, 0xAA28000C, 0x2A8CC, 0xAA38000C, 0x2A90C, 0xAA48000C, 0x2A94C, 0xAA58000C, 0x2A98C, 0xAA68000C, 0x2A9CC, 0xAA78000C, 0x2AA0C, 0xAA88000C, 0xAA90000C, 0xAA98000C, 0xAAA0000C, 0xAAA8000C, 0xAAB0000C, 0xAAB8000C, 0xAAC0000C, 0xAAC8000C, 0xAAD0000C, 0xAAD8000C, 0xAAE0000C, 0xAAE8000C, 0xAAF0000C, 0xAAF8000C, 0xAB00000C, 0xAB08000C, 0xAB10000C, 0xAB18000C, 0xAB20000C, 0xAB28000C, 0xAB30000C, 0xAB38000C, 0xAB40000C, 0xAB48000C, 0xAB50000C, 0xAB58000C, 0xAB60000C, 0xAB68000C, 0xAB70000C, 0xAB78000C, 0xAB80000C, 0xAB88000C, 0xAB90000C, 0xAB98000C, 0xABA0000C, 0xABA8000C, 0xABB0000C, 0xABB8000C, 0xABC0000C, 0xABC8000C, 0xABD0000C, 0xABD8000C, 0xABE0000C, 0xABE8000C, 0xABF0000C, 0xABF8000C, 0xAC00000C, 0xAC08000C, 0xAC10000C, 0xAC18000C, 0xAC20000C, 0xAC28000C, 0xAC30000C, 0xAC38000C, 0xAC40000C, 0xAC48000C, 0xAC50000C, 0xAC58000C, 0xAC60000C, 0xAC68000C, 0xAC70000C, 0xAC78000C, 0xAC80000C, 0xAC88000C, 0xAC90000C, 0xAC98000C, 0xACA0000C, 0xACA8000C, 0xACB0000C, 0xACB8000C, 0xACC0000C, 0xACC8000C, 0xACD0000C, 0xACD8000C, 0xACE0000C, 0xACE8000C, 0x2B3CC, 0xACF8000C, 0xAD00000C, 0xAD08000C, 0xAD10000C, 0xAD18000C, 0xAD20000C, 0xAD28000C, 0xAD30000C, 0xAD38000C, 0xAD40000C, 0xAD48000C, 0xAD50000C, 0xAD58000C, 0xAD60000C, 0xAD68000C, 0xAD70000C, 0xAD78000C, 0xAD80000C, 0xAD88000C, 0xAD90000C, 0xAD98000C, 0xADA0000C, 0xADA8000C, 0xADB0000C, 0xADB8000C, 0xADC0000C, 0xADC8000C, 0xADD0000C, 0xADD8000C, 0xADE0000C, 0xADE8000C, 0xADF0000C, 0xADF8000C, 0xAE00000C, 0xAE08000C, 0xAE10000C, 0xAE18000C, 0xAE20000C, 0xAE28000C, 0xAE30000C, 0xAE38000C, 0xAE40000C, 0xAE48000C, 0xAE50000C, 0xAE58000C, 0xAE60000C, 0xAE68000C, 0xAE70000C, 0xAE78000C, 0xAE80000C, 0xAE88000C, 0xAE90000C, 0xAE98000C, 0xAEA0000C, 0xAEA8000C, 0xAEB0000C, 0xAEB8000C, 0xAEC0000C, 0xAEC8000C, 0xAED0000C, 0xAED8000C, 0xAEE0000C, 0xAEE8000C, 0xAEF0000C, 0xAEF8000C, 0xAF00000C, 0xAF08000C, 0xAF10000C, 0xAF18000C, 0xAF20000C, 0xAF28000C, 0xAF30000C, 0xAF38000C, 0xAF40000C, 0xAF48000C, 0xAF50000C, 0xAF58000C, 0xAF60000C, 0xAF68000C, 0xAF70000C, 0xAF78000C, 0xAF80000C, 0xAF88000C, 0xAF90000C, 0xAF98000C, 0xAFA0000C, 0xAFA8000C, 0xAFB0000C, 0xAFB8000C, 0xAFC0000C, 0xAFC8000C, 0xAFD0000C, 0xAFD8000C, 0xAFE0000C, 0xAFE8000C, 0xAFF0000C, 0xAFF8000C, 0xB000000C, 0xB008000C, 0xB010000C, 0x2C06C, 0x2C08C, 0x2C0AC, 0x2C0CC, 0x2C0EC, 0x2C10C, 0x2C12C, 0x2C14C, 0x2C16C, 0x2C18C, 0x2C1AC, 0x2C1CC, 0x2C1EC, 0x2C20C, 0x2C22C, 0x2C24C, 0x2C26C, 0x2C28C, 0x2C2AC, 0x2C2CC, 0x2C2EC, 0x2C30C, 0x2C32C, 0x2C34C, 0x2C36C, 0x2C38C, 0x2C3AC, 0x2C3CC, 0x2C3EC, 0x2C40C, 0x2C42C, 0x2C44C, 0x2C46C, 0x2C48C, 0x2C4AC, 0x2C4CC, 0x2C4EC, 0x2C50C, 0x2C52C, 0x2C54C, 0x2C56C, 0x2C58C, 0x2C5AC, 0x2C5CC, 0x2C5EC, 0x2C60C, 0x2C62C, 0x2C64C, 0x2C66C, 0x2C68C, 0x2C6AC, 0x2C6CC, 0x2C6EC, 0x2C70C, 0x2C72C, 0x2C74C, 0x2C76C, 0x2C78C, 0x2C7AC, 0x2C7CC, 0x2C7EC, 0x2C80C, 0x2C82C, 0x2C84C, 0x2C86C, 0x2C88C, 0x2C8AC, 0x2C8CC, 0x2C8EC, 0x2C90C, 0x2C92C, 0x2C94C, 0x2C96C, 0x2C98C, 0x2C9AC, 0x2C9CC, 0x2C9EC, 0x2CA0C, 0x2CA2C, 0x2CA4C, 0x2CA6C, 0x2CA8C, 0x2CAAC, 0x2CACC, 0x2CAEC, 0x2CB0C, 0x2CB2C, 0x2CB4C, 0x2CB6C, 0x2CB8C, 0x2CBAC, 0x2CBCC, 0x2CBEC, 0x2CC0C, 0x2CC2C, 0x2CC4C, 0x2CC6C, 0x2CC8C, 0x2CCAC, 0x2CCCC, 0x2CCEC, 0x2CD0C, 0x2CD2C, 0x2CD4C, 0x2CD6C, 0x2CD8C, 0x2CDAC, 0x2CDCC, 0x2CDEC, 0x2CE0C, 0x2CE2C, 0x2CE4C, 0x2CE6C, 0x2CE8C, 0x2CEAC, 0x2CECC, 0x2CEEC, 0x2CF0C, 0x2CF2C, 0x2CF4C, 0x2CF6C, 0x2CF8C, 0x2CFAC, 0x2CFCC, 0x2CFEC, 0x2D00C, 0x2D02C, 0x2D04C, 0x2D06C, 0x2D08C, 0x2D0AC, 0x2D0CC, 0x2D0EC, 0x2D10C, 0x2D12C, 0x2D14C, 0x2D16C, 0x2D18C, 0x2D1AC, 0x2D1CC, 0x2D1EC, 0x2D20C, 0x2D22C, 0x2D24C, 0xB4A2D26C, 0x2D2AC, 0x2D2CC, 0x2D2EC, 0x2D30C, 0x2D32C, 0x2D34C, 0x2D36C, 0x2D38C, 0x2D3AC, 0x2D3CC, 0x2D3EC, 0x2D40C, 0x2D42C, 0x2D44C, 0x2D46C, 0x2D48C, 0x2D4AC, 0x2D4CC, 0x2D4EC, 0x2D50C, 0x2D52C, 0x2D54C, 0x2D56C, 0x2D58C, 0x2D5AC, 0x2D5CC, 0x2D5EC, 0x2D60C, 0x2D62C, 0x2D64C, 0x2D66C, 0x2D68C, 0x2D6AC, 0x2D6CC, 0x2D6EC, 0x2D70C, 0x2D72C, 0x2D74C, 0x2D76C, 0x2D78C, 0x2D7AC, 0x2D7CC, 0x2D7EC, 0x2D80C, 0x2D82C, 0x2D84C, 0x2D86C, 0x2D88C, 0x2D8AC, 0x2D8CC, 0x2D8EC, 0x2D90C, 0x2D92C, 0x2D94C, 0x2D96C, 0x2D98C, 0x2D9AC, 0x2D9CC, 0x2D9EC, 0x2DA0C, 0x2DA2C, 0x2DA4C, 0x2DA6C, 0x2DA8C, 0x2DAAC, 0x2DACC, 0x2DAEC, 0x2DB0C, 0x2DB2C, 0x2DB4C, 0x2DB6C, 0x2DB8C, 0x2DBAC, 0x2DBCC, 0x2DBEC, 0x2DC0C, 0x2DC2C, 0x2DC4C, 0x2DC6C, 0x2DC8C, 0x2DCAC, 0x2DCCC, 0x2DCEC, 0x2DD0C, 0x2DD2C, 0x2DD4C, 0x2DD6C, 0x2DD8C, 0x2DDAC, 0x2DDCC, 0x2DDEC, 0x2DE0C, 0x2DE2C, 0x2DE4C, 0x2DE6C, 0x2DE8C, 0x2DEAC, 0x2DECC, 0x2DEEC, 0x2DF0C, 0x2DF2C, 0x2DF4C, 0x2DF6C, 0x2DF8C, 0x2DFAC, 0x2DFCC, 0x2DFEC, 0x2E00C, 0x2E02C, 0x2E04C, 0x2E06C, 0x2E08C, 0x2E0AC, 0x2E0CC, 0x2E0EC, 0x2E10C, 0x2E12C, 0x2E14C, 0x2E16C, 0x2E18C, 0x2E1AC, 0x2E1CC, 0x2E1EC, 0x2E20C, 0x2E22C, 0x2E24C, 0x2E26C, 0xB8A0000C, 0xB8A8000C, 0xB8B0000C, 0xB8B8000C, 0xB8C0000C, 0xB8C8000C, 0xB8D0000C, 0xB8D8000C, 0xB8E0000C, 0xB8E8000C, 0xB8F0000C, 0xB8F8000C, 0xB900000C, 0xB908000C, 0xB910000C, 0xB918000C, 0xB920000C, 0xB928000C, 0xB930000C, 0xB938000C, 0xB940000C, 0xB948000C, 0xB950000C, 0xB958000C, 0xB960000C, 0xB968000C, 0xB970000C, 0xB978000C, 0xB980000C, 0xB988000C, 0xB990000C, 0xB998000C, 0xB9A0000C, 0xB9A8000C, 0xB9B0000C, 0xB9B8000C, 0xB9C0000C, 0xB9C8000C, 0xB9D0000C, 0xB9D8000C, 0xB9E0000C, 0xB9E8000C, 0xB9F0000C, 0xB9F8000C, 0xBA00000C, 0xBA08000C, 0xBA10000C, 0xBA18000C, 0xBA20000C, 0xBA28000C, 0xBA30000C, 0xBA38000C, 0xBA40000C, 0xBA48000C, 0xBA50000C, 0xBA58000C, 0xBA60000C, 0xBA68000C, 0xBA70000C, 0xBA78000C, 0xBA80000C, 0xBA88000C, 0xBA90000C, 0xBA98000C, 0xBAA0000C, 0xBAA8000C, 0xBAB0000C, 0xBAB8000C, 0xBAC0000C, 0xBAC8000C, 0xBAD0000C, 0xBAD8000C, 0xBAE0000C, 0xBAE8000C, 0xBAF0000C, 0xBAF8000C, 0xBB00000C, 0xBB08000C, 0xBB10000C, 0xBB18000C, 0xBB20000C, 0xBB28000C, 0xBB30000C, 0xBB38000C, 0xBB40000C, 0xBB48000C, 0xBB50000C, 0xBB58000C, 0xBB60000C, 0xBB68000C, 0xBB70000C, 0xBB78000C, 0xBB80000C, 0xBB88000C, 0xBB90000C, 0xBB98000C, 0xBBA0000C, 0xBBA8000C, 0xBBB0000C, 0xBBB8000C, 0xBBC0000C, 0xBBC8000C, 0xBBD0000C, 0xBBD8000C, 0xBBE0000C, 0xBBE8000C, 0xBBF0000C, 0xBBF8000C, 0xBC00000C, 0xBC08000C, 0xBC10000C, 0xBC18000C, 0xBC20000C, 0xBC28000C, 0xBC30000C, 0xBC38000C, 0xBC40000C, 0xBC48000C, 0xBC50000C, 0xBC58000C, 0xBC60000C, 0xBC68000C, 0xBC70000C, 0xBC78000C, 0xBC80000C, 0xBC88000C, 0xBC90000C, 0xBC98000C, 0xBCA0000C, 0xBCA8000C, 0xBCB0000C, 0xBCB8000C, 0xBCC0000C, 0xBCC8000C, 0xBCD0000C, 0xBCD8000C, 0xBCE0000C, 0xBCE8000C, 0xBCF0000C, 0xBCF8000C, 0xBD00000C, 0xBD08000C, 0xBD10000C, 0xBD18000C, 0xBD20000C, 0xBD28000C, 0xBD30000C, 0xBD38000C, 0xBD40000C, 0xBD48000C, 0xBD50000C, 0xBD58000C, 0xBD60000C, 0xBD68000C, 0xBD70000C, 0xBD78000C, 0xBD80000C, 0xBD88000C, 0xBD90000C, 0xBD98000C, 0xBDA0000C, 0xBDA8000C, 0xBDB0000C, 0xBDB8000C, 0xBDC0000C, 0xBDC8000C, 0xBDD0000C, 0xBDD8000C, 0xBDE0000C, 0xBDE8000C, 0xBDF0000C, 0xBDF8000C, 0xBE00000C, 0xBE08000C, 0xBE10000C, 0xBE18000C, 0xBE20000C, 0xBE28000C, 0xBE30000C, 0xBE38000C, 0xBE40000C, 0xBE48000C, 0xBE50000C, 0xBE58000C, 0xBE60000C, 0xBE68000C, 0xBE70000C, 0xBE78000C, 0xBE80000C, 0xBE88000C, 0xBE90000C, 0xBE98000C, 0xBEA0000C, 0xBEA8000C, 0xBEB0000C, 0xBEB8000C, 0xBEC0000C, 0xBEC8000C, 0xBED0000C, 0xBED8000C, 0xBEE0000C, 0xBEE8000C, 0xBEF0000C, 0xBEF8000C, 0xBF00000C, 0xBF08000C, 0xBF10000C, 0xBF18000C, 0xBF20000C, 0xBF28000C, 0xBF30000C, 0xBF38000C, 0xBF40000C, 0xBF48000C, 0xBF50000C, 0xBF58000C, 0xBF60000C, 0xBF68000C, 0xBF70000C, 0xBF78000C, 0xBF80000C, 0xBF88000C, 0xBF90000C, 0xBF98000C, 0xBFA0000C, 0xBFA8000C, 0xBFB0000C, 0xBFB8000C, 0xBFC0000C, 0xBFC8000C, 0xBFD0000C, 0xBFD8000C, 0xBFE0000C, 0xBFE8000C, 0xBFF0000C, 0xBFF8000C, 0xC000000C, 0xC008000C, 0xC010000C, 0xC018000C, 0xC020000C, 0xC028000C, 0xC030000C, 0xC038000C, 0xC040000C, 0xC048000C, 0xC050000C, 0xC058000C, 0xC060000C, 0xC068000C, 0xC070000C, 0xC078000C, 0xC080000C, 0xC088000C, 0xC090000C, 0xC098000C, 0xC0A0000C, 0xC0A8000C, 0xC0B0000C, 0xC0B8000C, 0xC0C0000C, 0xC0C8000C, 0xC0D0000C, 0xC0D8000C, 0xC0E0000C, 0xC0E8000C, 0xC0F0000C, 0xC0F8000C, 0xC100000C, 0xC108000C, 0xC110000C, 0xC118000C, 0xC120000C, 0xC128000C, 0xC130000C, 0xC138000C, 0xC140000C, 0xC148000C, 0xC150000C, 0xC158000C, 0xC160000C, 0xC168000C, 0xC170000C, 0xC178000C, 0xC180000C, 0xC188000C, 0xC190000C, 0xC198000C, 0xC1A0000C, 0xC1A8000C, 0xC1B0000C, 0xC1B8000C, 0xC1C0000C, 0xC1C8000C, 0xC1D0000C, 0xC1D8000C, 0xC1E0000C, 0xC1E8000C, 0xC1F0000C, 0xC1F8000C, 0xC200000C, 0xC208000C, 0xC210000C, 0xC218000C, 0xC220000C, 0xC228000C, 0xC230000C, 0xC238000C, 0xC240000C, 0xC248000C, 0xC250000C, 0xC258000C, 0xC260000C, 0xC268000C, 0xC270000C, 0xC278000C, 0xC280000C, 0xC288000C, 0xC290000C, 0xC298000C, 0xC2A0000C, 0xC2A8000C, 0xC2B0000C, 0xC2B8000C, 0xC2C0000C, 0xC2C8000C, 0xC2D0000C, 0xC2D8000C, 0xC2E0000C, 0xC2E8000C, 0xC2F0000C, 0xC2F8000C, 0xC300000C, 0xC308000C, 0xC310000C, 0xC318000C, 0xC320000C, 0xC328000C, 0xC330000C, 0xC338000C, 0xC340000C, 0xC348000C, 0xC350000C, 0xC358000C, 0xC360000C, 0xC368000C, 0xC370000C, 0xC378000C, 0xC380000C, 0xC388000C, 0xC390000C, 0xC398000C, 0xC3A0000C, 0xC3A8000C, 0xC3B0000C, 0xC3B8000C, 0xC3C0000C, 0xC3C8000C, 0xC3D0000C, 0xC3D8000C, 0xC3E0000C, 0xC3E8000C, 0xC3F0000C, 0xC3F8000C, 0xC400000C, 0xC408000C, 0xC410000C, 0xC418000C, 0xC420000C, 0xC428000C, 0xC430000C, 0xC438000C, 0xC440000C, 0xC448000C, 0xC450000C, 0xC458000C, 0xC460000C, 0xC468000C, 0xC470000C, 0xC478000C, 0xC480000C, 0xC488000C, 0xC490000C, 0xC498000C, 0xC4A0000C, 0xC4A8000C, 0xC4B0000C, 0xC4B8000C, 0xC4C0000C, 0xC4C8000C, 0xC4D0000C, 0xC4D8000C, 0xC4E0000C, 0xC4E8000C, 0xC4F0000C, 0xC4F8000C, 0xC500000C, 0xC508000C, 0xC510000C, 0xC518000C, 0xC520000C, 0xC528000C, 0xC530000C, 0xC538000C, 0xC540000C, 0xC548000C, 0xC550000C, 0xC558000C, 0xC560000C, 0xC568000C, 0xC570000C, 0xC578000C, 0xC580000C, 0xC588000C, 0xC590000C, 0xC598000C, 0xC5A0000C, 0xC5A8000C, 0xC5B0000C, 0xC5B8000C, 0xC5C0000C, 0xC5C8000C, 0xC5D0000C, 0xC5D8000C, 0xC5E0000C, 0xC5E8000C, 0xC5F0000C, 0xC5F8000C, 0xC600000C, 0xC608000C, 0xC610000C, 0xC618000C, 0xC620000C, 0xC628000C, 0xC630000C, 0xC638000C, 0xC640000C, 0xC648000C, 0xC650000C, 0xC658000C, 0xC660000C, 0xC668000C, 0xC670000C, 0xC678000C, 0xC680000C, 0xC688000C, 0xC690000C, 0xC698000C, 0xC6A0000C, 0xC6A8000C, 0xC6B0000C, 0xC6B8000C, 0xC6C0000C, 0xC6C8000C, 0xC6D0000C, 0xC6D8000C, 0xC6E0000C, 0xC6E8000C, 0x31BCD, 0x31BED, 0x31C0D, 0x31C2D, 0x31C4D, 0x31C6D, 0x31C8D, 0x31CAD, 0x31CCD, 0x31CED, 0x31D0D, 0x31D2D, 0x31D4D, 0x31D6D, 0x31D8D, 0x31DAD, 0x31DCD, 0x31DED, 0x31E0D, 0x31E2D, 0x31E4D, 0x31E6D, 0x31E8D, 0x31EAD, 0x31ECD, 0x31EED, 0x31F0D, 0xC7C8000D, 0x31F4D, 0xC7D8000D, 0x31F8D, 0xC7E8000D, 0x31FCD, 0xC7F8000D, 0x3200D, 0xC808000D, 0x3204D, 0xC818000D, 0x3208D, 0xC828000D, 0x320CD, 0xC838000D, 0x3210D, 0xC848000D, 0x3214D, 0xC858000D, 0x3218D, 0xC868000D, 0x321CD, 0xC878000D, 0x3220D, 0xC888000D, 0x3224D, 0xC898000D, 0x3228D, 0xC8A8000D, 0x322CD, 0xC8B8000D, 0x3230D, 0xC8C8000D, 0x3234D, 0xC8D8000D, 0x3238D, 0xC8E8000D, 0x323CD, 0xC8F8000D, 0x3240D, 0xC908000D, 0x3244D, 0xC918000D, 0x3248D, 0xC928000D, 0x324CD, 0xC938000D, 0x3250D, 0xC948000D, 0x3254D, 0xC958000D, 0x3258D, 0xC968000D, 0x325CD, 0xC978000D, 0x3260D, 0xC988000D, 0x3264D, 0xC998000D, 0x3268D, 0xC9A8000D, 0x326CD, 0xC9B8000D, 0x3270D, 0xC9C8000D, 0x3274D, 0xC9D8000D, 0x3278D, 0xC9E8000D, 0x327CD, 0xC9F8000D, 0x3280D, 0xCA08000D, 0x3284D, 0xCA18000D, 0x3288D, 0xCA28000D, 0x328CD, 0xCA38000D, 0x3290D, 0xCA48000D, 0x3294D, 0xCA58000D, 0x3298D, 0xCA68000D, 0x329CD, 0xCA78000D, 0x32A0D, 0xCA88000D, 0x32A4D, 0xCA98000D, 0x32A8D, 0xCAA8000D, 0x32ACD, 0xCAB8000D, 0x32B0D, 0xCAC8000D, 0x32B4D, 0xCAD8000D, 0x32B8D, 0xCAE8000D, 0x32BCD, 0xCAF8000D, 0x32C0D, 0xCB08000D, 0x32C4D, 0xCB18000D, 0x32C8D, 0xCB28000D, 0x32CCD, 0xCB38000D, 0x32D0D, 0xCB48000D, 0x32D4D, 0xCB58000D, 0x32D8D, 0xCB68000D, 0x32DCD, 0xCB78000D, 0x32E0D, 0xCB88000D, 0x32E4D, 0xCB98000D, 0x32E8D, 0xCBA8000D, 0x32ECD, 0xCBB8000D, 0x32F0D, 0xCBC8000D, 0x32F4D, 0xCBD8000D, 0x32F8D, 0xCBE8000D, 0x32FCD, 0xCBF8000D, 0x3300D, 0xCC08000D, 0x3304D, 0xCC18000D, 0x3308D, 0xCC28000D, 0x330CD, 0xCC38000D, 0x3310D, 0xCC48000D, 0x3314D, 0xCC58000D, 0x3318D, 0xCC68000D, 0x331CD, 0xCC78000D, 0x3320D, 0xCC88000D, 0x3324D, 0xCC98000D, 0x3328D, 0xCCA8000D, 0x332CD, 0xCCB8000D, 0x3330D, 0x3332D, 0xCCD0000D, 0x3336D, 0xCCE0000D, 0x333AD, 0x333CD, 0x333ED, 0x3340D, 0x3342D, 0x3344D, 0x3346D, 0x3348D, 0x334AD, 0xCD30000D, 0x334ED, 0xCD40000D, 0x3352D, 0xCD50000D, 0x3356D, 0xCD60000D, 0x335AD, 0xCD70000D, 0x335ED, 0xCD80000D, 0x3362D, 0xCD90000D, 0x3366D, 0xCDA0000D, 0x336AD, 0x336CD, 0x336ED, 0x3370D, 0x3372D, 0x3374D, 0x3376D, 0x3378D, 0x337AD, 0x337CD, 0x337ED, 0x3380D, 0x3382D, 0xCE10000D, 0x3386D, 0x3388D, 0x338AD, 0x338CD, 0x338ED, 0x3390D, 0x3392D, 0xCE50000D, 0x3396D, 0x3398D, 0x339AD, 0x339CD, 0x339ED, 0x33A0D, 0x33A2D, 0x33A4D, 0x33A6D, 0x33A8D, 0x33AAD, 0x33ACD, 0x33AED, 0x33B0D, 0x33B2D, 0x33B4D, 0x33B6D, 0x33B8D, 0x33BAD, 0x33BCD, 0x33BED, 0x33C0D, 0x33C2D, 0x33C4D, 0x33C6D, 0x33C8D, 0x33CAD, 0x33CCD, 0x33CED, 0x33D0D, 0x33D2D, 0x33D4D, 0x33D6D, 0x33D8D, 0x33DAD, 0x33DCD, 0x33DED, 0x33E0D, 0x33E2D, 0x33E4D, 0x33E6D, 0x33E8D, 0x33EAD, 0x33ECD, 0x33EED, 0x33F0D, 0x33F2D, 0x33F4D, 0x33F6D, 0x33F8D, 0x33FAD, 0x33FCD, 0x33FED, 0x3400D, 0x3402D, 0x3404D, 0x3406D, 0x3408D, 0x340AD, 0x340CD, 0x340ED, 0x3410D, 0x3412D, 0x3414D, 0x3416D, 0x3418D, 0x341AD, 0x341CD, 0x341ED, 0x3420D, 0x3422D, 0x3424D, 0x3426D, 0x3428D, 0x342AD, 0x342CD, 0x342ED, 0xD0CB430D, 0xD0DB434D, 0xD0EB438D, 0xD0FB43CD, 0xD10B440D, 0xD11B444D, 0xD12B448D, 0xD13B44CD, 0xD14B450D, 0xD15B454D, 0xD16B458D, 0xD17B45CD, 0xD18B460D, 0xD19B464D, 0xD1AB468D, 0xD1BB46CD, 0xD1CB470D, 0xD1DB474D, 0xD1EB478D, 0xD1FB47CD, 0xD20B480D, 0xD21B484D, 0xD22B488D, 0xD23B48CD, 0xD24B490D, 0xD25B494D, 0xD26B498D, 0xD27B49CD, 0xD28B4A0D, 0xD29B4A4D, 0xD2AB4A8D, 0xD2BB4ACD, 0xD2CB4B0D, 0xD2DB4B4D, 0xD2EB4B8D, 0xD2FB4BCD, 0xD30B4C0D, 0xD31B4C4D, 0xD32B4C8D, 0xD33B4CCD, 0x34D0D, 0x34D2D, 0x34D4D, 0x34D6D, 0x34D8D, 0x34DAD, 0x34DCD, 0x34DED, 0x34E0D, 0x34E2D, 0x34E4D, 0x34E6D, 0x34E8D, 0x34EAD, 0x34ECD, 0x34EED, 0x34F0D, 0x34F2D, 0x34F4D, 0x34F6D, 0x34F8D, 0x34FAD, 0x34FCD, 0x34FED, 0x3500D, 0x3502D, 0x3504D, 0x3506D, 0x3508D, 0x350AD, 0x350CD, 0x350ED, 0x3510D, 0x3512D, 0x3514D, 0x3516D, 0x3518D, 0x351AD, 0x351CD, 0x351ED, 0x3520D, 0x3522D, 0x3524D, 0x3526D, 0x3528D, 0x352AD, 0x352CD, 0x352ED, 0x3530D, 0x3532D, 0x3534D, 0x3536D, 0x3538D, 0xD4F353AD, 0xD50353ED, 0x3542D, 0x3544D, 0xD523546D, 0xD53354AD, 0xD54354ED, 0xD553552D, 0x3556D, 0x3558D, 0x355AD, 0xD57B55CD, 0xD58B560D, 0xD59B564D, 0xD5AB568D, 0xD5BB56CD, 0x3570D, 0x3572D, 0x3574D, 0x3576D, 0x3578D, 0x357AD, 0x357CD, 0x357ED, 0x3580D, 0x3582D, 0x3584D, 0x3586D, 0x3588D, 0x358AD, 0x358CD, 0x358ED, 0x3590D, 0x3592D, 0x3594D, 0x3596D, 0x3598D, 0x359AD, 0x359CD, 0x359ED, 0x35A0D, 0x35A2D, 0x35A4D, 0x35A6D, 0x35A8D, 0x35AAD, 0x35ACD, 0x35AED, 0x35B0D, 0x35B2D, 0x35B4D, 0x35B6D, 0x35B8D, 0x35BAD, 0x35BCD, 0x35BED, 0x35C0D, 0x35C2D, 0x35C4D, 0x35C6D, 0x35C8D, 0x35CAD, 0x35CCD, 0x35CED, 0x35D0D, 0x35D2D, 0x35D4D, 0x35D6D, 0x35D8D, 0x35DAD, 0x35DCD, 0x35DED, 0x35E0D, 0x35E2D, 0x35E4D, 0x35E6D, 0x35E8D, 0x35EAD, 0x35ECD, 0x35EED, 0x35F0D, 0x35F2D, 0x35F4D, 0x35F6D, 0x35F8D, 0x35FAD, 0x35FCD, 0x35FED, 0x3600D, 0x3602D, 0x3604D, 0x3606D, 0x3608D, 0x360AD, 0x360CD, 0x360ED, 0x3610D, 0x3612D, 0x3614D, 0x3616D, 0x3618D, 0x361AD, 0x361CD, 0x361ED, 0x3620D, 0x3622D, 0x3624D, 0x3626D, 0xD8AB628D, 0xD8BB62CD, 0xD8CB630D, 0xD8D0000D, 0xD8D8000D, 0xD8E0000D, 0xD8E8000D, 0xD8F0000D, 0xD8F8000D, 0xD900000D, 0xD908000D, 0xD910000D, 0xD918000D, 0xD920000D, 0xD928000D, 0xD930000D, 0xD938000D, 0xD940000D, 0xD948000D, 0xD950000D, 0xD958000D, 0xD960000D, 0xD968000D, 0xD970000D, 0xD978000D, 0xD980000D, 0xD988000D, 0xD990000D, 0xD998000D, 0xD9A0000D, 0xD9A8000D, 0xD9B0000D, 0xD9B8000D, 0xD9C0000D, 0xD9C8000D, 0xD9D0000D, 0xD9D8000D, 0xD9E0000D, 0xD9E8000D, 0xD9F0000D, 0xD9F8000D, 0xDA00000D, 0xDA08000D, 0xDA10000D, 0xDA18000D, 0xDA20000D, 0xDA28000D, 0xDA30000D, 0xDA38000D, 0xDA40000D, 0xDA48000D, 0xDA50000D, 0xDA58000D, 0xDA60000D, 0xDA68000D, 0xDA70000D, 0xDA78000D, 0xDA80000D, 0xDA88000D, 0xDA90000D, 0xDA98000D, 0xDAA0000D, 0xDAA8000D, 0xDAB0000D, 0xDAB8000D, 0xDAC0000D, 0xDAC8000D, 0xDAD0000D, 0xDAD8000D, 0xDAE0000D, 0xDAE8000D, 0xDAF0000D, 0xDAF8000D, 0xDB00000D, 0xDB08000D, 0xDB10000D, 0xDB18000D, 0xDB20000D, 0xDB28000D, 0xDB30000D, 0xDB38000D, 0xDB40000D, 0xDB48000D, 0xDB50000D, 0xDB58000D, 0xDB60000D, 0xDB68000D, 0xDB70000D, 0xDB78000D, 0xDB80000D, 0xDB88000D, 0xDB90000D, 0xDB98000D, 0xDBA0000D, 0xDBA8000D, 0xDBB0000D, 0xDBB8000D, 0xDBC0000D, 0xDBC8000D, 0xDBD0000D, 0xDBD8000D, 0xDBE0000D, 0xDBE8000D, 0xDBF0000D, 0xDBF8000D, 0xDC00000D, 0xDC08000D, 0xDC10000D, 0xDC18000D, 0xDC20000D, 0xDC28000D, 0xDC30000D, 0xDC38000D, 0xDC40000D, 0xDC48000D, 0xDC50000D, 0xDC58000D, 0xDC60000D, 0xDC68000D, 0xDC70000D, 0xDC78000D, 0xDC80000D, 0xDC88000D, 0xDC90000D, 0xDC98000D, 0xDCA0000D, 0xDCA8000D, 0xDCB0000D, 0xDCB8000D, 0xDCC0000D, 0xDCC8000D, 0xDCD0000D, 0xDCD8000D, 0xDCE0000D, 0xDCE8000D, 0xDCF0000D, 0xDCF8000D, 0xDD00000D, 0xDD08000D, 0xDD10000D, 0xDD18000D, 0xDD20000D, 0xDD28000D, 0xDD30000D, 0xDD38000D, 0xDD40000D, 0xDD48000D, 0xDD50000D, 0xDD58000D, 0xDD60000D, 0xDD68000D, 0xDD70000D, 0xDD78000D, 0xDD80000D, 0xDD88000D, 0xDD90000D, 0xDD98000D, 0xDDA0000D, 0xDDA8000D, 0xDDB0000D, 0xDDB8000D, 0xDDC0000D, 0xDDC8000D, 0xDDD0000D, 0xDDD8000D, 0xDDE0000D, 0xDDE8000D, 0xDDF0000D, 0xDDF8000D, 0xDE00000D, 0xDE08000D, 0xDE10000D, 0xDE18000D, 0xDE20000D, 0xDE28000D, 0xDE30000D, 0xDE38000D, 0xDE40000D, 0xDE48000D, 0xDE50000D, 0xDE58000D, 0xDE60000D, 0xDE68000D, 0xDE70000D, 0xDE78000D, 0xDE80000D, 0xDE88000D, 0xDE90000D, 0xDE98000D, 0xDEA0000D, 0xDEA8000D, 0xDEB0000D, 0xDEB8000D, 0xDEC0000D, 0xDEC8000D, 0xDED0000D, 0xDED8000D, 0xDEE0000D, 0xDEE8000D, 0xDEF0000D, 0xDEF8000D, 0xDF00000D, 0xDF08000D, 0xDF10000D, 0xDF18000D, 0xDF20000D, 0xDF28000D, 0xDF30000D, 0xDF38000D, 0xDF40000D, 0xDF48000D, 0xDF50000D, 0xDF58000D, 0xDF60000D, 0xDF68000D, 0xDF70000D, 0xDF78000D, 0xDF80000D, 0xDF88000D, 0xDF90000D, 0xDF98000D, 0xDFA0000D, 0xDFA8000D, 0xDFB0000D, 0xDFB8000D, 0xDFC0000D, 0xDFC8000D, 0xDFD0000D, 0xDFD8000D, 0xDFE0000D, 0xDFE8000D, 0xDFF0000D, 0xDFF8000D, 0xE000000D, 0xE008000D, 0xE010000D, 0xE018000D, 0xE020000D, 0xE028000D, 0xE030000D, 0xE038000D, 0xE040000D, 0xE048000D, 0xE050000D, 0xE058000D, 0xE060000D, 0xE068000D, 0xE070000D, 0xE078000D, 0xE080000D, 0xE088000D, 0xE090000D, 0xE098000D, 0xE0A0000D, 0xE0A8000D, 0xE0B0000D, 0xE0B8000D, 0xE0C0000D, 0xE0C8000D, 0xE0D0000D, 0xE0D8000D, 0xE0E0000D, 0xE0E8000D, 0xE0F0000D, 0xE0F8000D, 0xE100000D, 0xE108000D, 0xE110000D, 0xE118000D, 0xE120000D, 0xE128000D, 0xE130000D, 0xE138000D, 0xE140000D, 0xE148000D, 0xE150000D, 0xE158000D, 0xE160000D, 0xE168000D, 0xE170000D, 0xE178000D, 0xE180000D, 0xE188000D, 0xE190000D, 0xE198000D, 0xE1A0000D, 0xE1A8000D, 0xE1B0000D, 0xE1B8000D, 0xE1C0000D, 0xE1C8000D, 0xE1D0000D, 0xE1D8000D, 0xE1E0000D, 0xE1E8000D, 0xE1F0000D, 0xE1F8000D, 0xE200000D, 0xE208000D, 0xE210000D, 0xE218000D, 0xE220000D, 0xE228000D, 0xE230000D, 0xE238000D, 0xE240000D, 0xE248000D, 0xE250000D, 0xE258000D, 0xE260000D, 0xE268000D, 0xE270000D, 0xE278000D, 0xE280000D, 0xE288000D, 0xE290000D, 0xE298000D, 0xE2A0000D, 0xE2A8000D, 0xE2B0000D, 0xE2B8000D, 0xE2C0000D, 0xE2C8000D, 0xE2D0000D, 0xE2D8000D, 0xE2E0000D, 0xE2E8000D, 0xE2F0000D, 0xE2F8000D, 0xE300000D, 0xE308000D, 0xE310000D, 0xE318000D, 0xE320000D, 0xE328000D, 0xE330000D, 0xE338000D, 0xE340000D, 0xE348000D, 0xE350000D, 0xE358000D, 0xE360000D, 0xE368000D, 0xE370000D, 0xE378000D, 0xE380000D, 0xE388000D, 0xE390000D, 0xE398000D, 0xE3A0000D, 0xE3A8000D, 0xE3B0000D, 0xE3B8000D, 0xE3C0000D, 0xE3C8000D, 0xE3D0000D, 0xE3D8000D, 0xE3E0000D, 0xE3E8000D, 0xE3F0000D, 0xE3F8000D, 0xE400000D, 0xE408000D, 0xE410000D, 0xE418000D, 0xE420000D, 0xE428000D, 0xE430000D, 0xE438000D, 0xE440000D, 0xE448000D, 0xE450000D, 0xE458000D, 0xE460000D, 0xE468000D, 0xE470000D, 0xE478000D, 0xE480000D, 0xE488000D, 0xE490000D, 0xE498000D, 0xE4A0000D, 0xE4A8000D, 0xE4B0000D, 0xE4B8000D, 0xE4C0000D, 0xE4C8000D, 0xE4D0000D, 0xE4D8000D, 0xE4E0000D, 0xE4E8000D, 0xE4F0000D, 0xE4F8000D, 0xE500000D, 0xE508000D, 0xE510000D, 0xE518000D, 0xE520000D, 0xE528000D, 0xE530000D, 0xE538000D, 0xE540000D, 0xE548000D, 0xE550000D, 0xE558000D, 0xE560000D, 0xE568000D, 0xE570000D, 0xE578000D, 0xE580000D, 0xE588000D, 0xE590000D, 0xE598000D, 0xE5A0000D, 0xE5A8000D, 0xE5B0000D, 0xE5B8000D, 0xE5C0000D, 0xE5C8000D, 0xE5D0000D, 0xE5D8000D, 0xE5E0000D, 0xE5E8000D, 0xE5F0000D, 0xE5F8000D, 0xE600000D, 0xE608000D, 0xE610000D, 0xE618000D, 0xE620000D, 0xE628000D, 0xE630000D, 0xE638000D, 0xE640000D, 0xE648000D, 0xE650000D, 0xE658000D, 0xE660000D, 0xE668000D, 0xE670000D, 0xE678000D, 0xE680000D, 0xE688000D, 0xE690000D, 0xE698000D, 0xE6A0000D, 0xE6A8000D, 0xE6B0000D, 0xE6B8000D, 0xE6C0000D, 0xE6C8000D, 0xE6D0000D, 0xE6D8000D, 0xE6E0000D, 0xE6E8000D, 0xE6F0000D, 0xE6F8000D, 0xE700000D, 0xE708000D, 0xE710000D, 0xE718000D, 0xE720000D, 0xE728000D, 0xE730000D, 0xE738000D, 0xE740000D, 0xE748000D, 0xE750000D, 0xE758000D, 0xE760000D, 0xE768000D, 0xE770000D, 0xE778000D, 0xE780000D, 0xE788000D, 0xE790000D, 0xE798000D, 0xE7A0000D, 0xE7A8000D, 0xE7B0000D, 0xE7B8000D, 0xE7C0000D, 0xE7C8000D, 0xE7D0000D, 0xE7D8000D, 0xE7E0000D, 0xE7E8000D, 0xE7F0000D, 0xE7F8000D, 0xE800000D, 0xE808000D, 0xE810000D, 0xE818000D, 0xE820000D, 0xE828000D, 0xE830000D, 0xE838000D, 0xE840000D, 0xE848000D, 0xE850000D, 0xE858000D, 0xE860000D, 0xE868000D, 0xE870000D, 0xE878000D, 0xE880000D, 0xE888000D, 0xE890000D, 0xE898000D, 0xE8A0000D, 0xE8A8000D, 0xE8B0000D, 0xE8B8000D, 0xE8C0000D, 0xE8C8000D, 0xE8D0000D, 0xE8D8000D, 0xE8E0000D, 0xE8E8000D, 0xE8F0000D, 0xE8F8000D, 0xE900000D, 0xE908000D, 0xE910000D, 0xE918000D, 0xE920000D, 0xE928000D, 0xE930000D, 0xE938000D, 0xE940000D, 0xE948000D, 0xE950000D, 0xE958000D, 0xE960000D, 0xE968000D, 0xE970000D, 0xE978000D, 0xE980000D, 0xE988000D, 0xE990000D, 0xE998000D, 0xE9A0000D, 0xE9A8000D, 0xE9B0000D, 0xE9B8000D, 0xE9C0000D, 0xE9C8000D, 0xE9D0000D, 0xE9D8000D, 0xE9E0000D, 0xE9E8000D, 0xE9F0000D, 0xE9F8000D, 0xEA00000D, 0xEA08000D, 0xEA10000D, 0xEA18000D, 0xEA20000D, 0xEA28000D, 0xEA30000D, 0xEA38000D, 0xEA40000D, 0xEA48000D, 0xEA50000D, 0xEA58000D, 0xEA60000D, 0xEA68000D, 0xEA70000D, 0xEA78000D, 0xEA80000D, 0xEA88000D, 0xEA90000D, 0xEA98000D, 0xEAA0000D, 0xEAA8000D, 0xEAB0000D, 0xEAB8000D, 0xEAC0000D, 0xEAC8000D, 0xEAD0000D, 0xEAD8000D, 0xEAE0000D, 0xEAE8000D, 0xEAF0000D, 0xEAF8000D, 0xEB00000D, 0xEB08000D, 0xEB10000D, 0xEB18000D, 0xEB20000D, 0xEB28000D, 0xEB30000D, 0xEB38000D, 0xEB40000D, 0xEB48000D, 0xEB50000D, 0xEB58000D, 0xEB60000D, 0xEB68000D, 0xEB70000D, 0xEB78000D, 0xEB80000D, 0xEB88000D, 0xEB90000D, 0xEB98000D, 0xEBA0000D, 0xEBA8000D, 0xEBB0000D, 0xEBB8000D, 0xEBC0000D, 0xEBC8000D, 0xEBD0000D, 0xEBD8000D, 0xEBE0000D, 0xEBE8000D, 0xEBF0000D, 0xEBF8000D, 0xEC00000D, 0xEC08000D, 0xEC10000D, 0xEC18000D, 0xEC20000D, 0xEC28000D, 0xEC30000D, 0xEC38000D, 0xEC40000D, 0xEC48000D, 0xEC50000D, 0xEC58000D, 0xEC60000D, 0xEC68000D, 0xEC70000D, 0xEC78000D, 0xEC80000D, 0xEC88000D, 0xEC90000D, 0xEC98000D, 0xECA0000D, 0xECA8000D, 0xECB0000D, 0xECB8000D, 0xECC0000D, 0xECC8000D, 0xECD0000D, 0xECD8000D, 0xECE0000D, 0xECE8000D, 0xECF0000D, 0xECF8000D, 0xED00000D, 0xED08000D, 0xED10000D, 0xED18000D, 0xED20000D, 0xED28000D, 0xED30000D, 0xED38000D, 0xED40000D, 0xED48000D, 0xED50000D, 0xED58000D, 0xED60000D, 0xED68000D, 0xED70000D, 0xED78000D, 0xED80000D, 0xED88000D, 0xED90000D, 0xED98000D, 0xEDA0000D, 0xEDA8000D, 0xEDB0000D, 0xEDB8000D, 0xEDC0000D, 0xEDC8000D, 0xEDD0000D, 0xEDD8000D, 0xEDE0000D, 0xEDE8000D, 0xEDF0000D, 0xEDF8000D, 0xEE00000D, 0xEE08000D, 0xEE10000D, 0xEE18000D, 0xEE20000D, 0xEE28000D, 0xEE30000D, 0xEE38000D, 0xEE40000D, 0xEE48000D, 0xEE50000D, 0xEE58000D, 0xEE60000D, 0xEE68000D, 0xEE70000D, 0xEE78000D, 0xEE80000D, 0xEE88000D, 0xEE90000D, 0xEE98000D, 0xEEA0000D, 0xEEA8000D, 0xEEB0000D, 0xEEB8000D, 0xEEC0000D, 0xEEC8000D, 0xEED0000D, 0xEED8000D, 0xEEE0000D, 0xEEE8000D, 0xEEF0000D, 0xEEF8000D, 0xEF00000D, 0xEF08000D, 0xEF10000D, 0xEF18000D, 0xEF20000D, 0xEF28000D, 0xEF30000D, 0xEF38000D, 0xEF40000D, 0xEF48000D, 0xEF50000D, 0xEF58000D, 0xEF60000D, 0xEF68000D, 0xEF70000D, 0xEF78000D, 0xEF80000D, 0xEF88000D, 0xEF90000D, 0xEF98000D, 0x3BE8E, 0x3BEAE, 0x3BECE, 0x3BEEE, 0x3BF0E, 0x3BF2E, 0x3BF4E, 0x3BF6E, 0x3BF8E, 0x3BFAE, 0x3BFCE, 0x3BFEE, 0x3C00E, 0x3C02E, 0x3C04E, 0x3C06E, 0x3C08E, 0x3C0AE, 0x3C0CE, 0x3C0EE, 0x3C10E, 0x3C12E, 0x3C14E, 0x3C16E, 0x3C18E, 0x3C1AE, 0xF070000E, 0xF078000E, 0xF080000E, 0xF088000E, 0xF090000E, 0xF098000E, 0xF0A0000E, 0xF0A8000E, 0xF0B0000E, 0xF0B8000E, 0xF0C0000E, 0xF0C8000E, 0xF0D0000E, 0xF0D8000E, 0xF0E0000E, 0xF0E8000E, 0xF0F0000E, 0xF0F8000E, 0xF100000E, 0xF108000E, 0xF110000E, 0xF118000E, 0xF120000E, 0xF128000E, 0xF130000E, 0xF138000E, 0xF140000E, 0xF148000E, 0xF150000E, 0xF158000E, 0xF160000E, 0xF168000E, 0xF170000E, 0xF178000E, 0xF180000E, 0xF188000E, 0xF190000E, 0xF198000E, 0xF1A0000E, 0xF1A8000E, 0xF1B0000E, 0xF1B8000E, 0xF1C0000E, 0xF1C8000E, 0xF1D0000E, 0xF1D8000E, 0xF1E0000E, 0xF1E8000E, 0xF1F0000E, 0xF1F8000E, 0xF200000E, 0xF208000E, 0xF210000E, 0xF218000E, 0xF220000E, 0xF228000E, 0xF230000E, 0xF238000E, 0xF240000E, 0xF248000E, 0xF250000E, 0xF258000E, 0xF260000E, 0xF268000E, 0xF270000E, 0xF278000E, 0xF280000E, 0xF288000E, 0xF290000E, 0xF298000E, 0xF2A0000E, 0xF2A8000E, 0xF2B0000E, 0xF2B8000E, 0xF2C0000E, 0xF2C8000E, 0xF2D0000E, 0xF2D8000E, 0xF2E0000E, 0xF2E8000E, 0xF2F0000E, 0xF2F8000E, 0xF300000E, 0xF308000E, 0xF310000E, 0xF318000E, 0xF320000E, 0xF328000E, 0xF330000E, 0xF338000E, 0xF340000E, 0xF348000E, 0xF350000E, 0xF358000E, 0xF360000E, 0xF368000E, 0xF370000E, 0xF378000E, 0xF380000E, 0xF388000E, 0xF390000E, 0xF398000E, 0xF3A0000E, 0xF3A8000E, 0xF3B0000E, 0xF3B8000E, 0xF3C0000E, 0xF3C8000E, 0xF3D0000E, 0xF3D8000E, 0xF3E0000E, 0xF3E8000E, 0xF3F0000E, 0xF3F8000E, 0xF400000E, 0xF408000E, 0xF410000E, 0xF418000E, 0xF420000E, 0xF428000E, 0xF430000E, 0xF438000E, 0xF440000E, 0xF448000E, 0xF450000E, 0xF458000E, 0xF460000E, 0xF468000E, 0xF470000E, 0xF478000E, 0xF480000E, 0xF488000E, 0xF490000E, 0xF498000E, 0xF4A0000E, 0xF4A8000E, 0xF4B0000E, 0xF4B8000E, 0xF4C0000E, 0xF4C8000E, 0xF4D0000E, 0xF4D8000E, 0xF4E0000E, 0xF4E8000E, 0xF4F0000E, 0xF4F8000E, 0xF500000E, 0xF508000E, 0xF510000E, 0xF518000E, 0xF520000E, 0xF528000E, 0xF530000E, 0xF538000E, 0xF540000E, 0xF548000E, 0xF550000E, 0xF558000E, 0xF560000E, 0xF568000E, 0xF570000E, 0xF578000E, 0xF580000E, 0xF588000E, 0xF590000E, 0xF598000E, 0xF5A0000E, 0xF5A8000E, 0xF5B0000E, 0xF5B8000E, 0xF5C0000E, 0xF5C8000E, 0xF5D0000E, 0xF5D8000E, 0xF5E0000E, 0xF5E8000E, 0xF5F0000E, 0xF5F8000E, 0xF600000E, 0xF608000E, 0xF610000E, 0xF618000E, 0xF620000E, 0xF628000E, 0xF630000E, 0xF638000E, 0xF640000E, 0xF648000E, 0xF650000E, 0xF658000E, 0xF660000E, 0xF668000E, 0xF670000E, 0xF678000E, 0xF680000E, 0xF688000E, 0xF690000E, 0xF698000E, 0xF6A0000E, 0xF6A8000E, 0xF6B0000E, 0xF6B8000E, 0xF6C0000E, 0xF6C8000E, 0xF6D0000E, 0xF6D8000E, 0xF6E0000E, 0xF6E8000E, 0xF6F0000E, 0xF6F8000E, 0xF700000E, 0xF708000E, 0xF710000E, 0xF718000E, 0xF720000E, 0xF728000E, 0xF730000E, 0xF738000E, 0xF740000E, 0xF748000E, 0xF750000E, 0xF758000E, 0xF760000E, 0xF768000E, 0xF770000E, 0xF778000E, 0xF780000E, 0xF788000E, 0xF790000E, 0xF798000E, 0xF7A0000E, 0xF7A8000E, 0xF7B0000E, 0xF7B8000E, 0xF7C0000E, 0xF7C8000E, 0xF7D0000E, 0xF7D8000E, 0xF7E0000E, 0xF7E8000E, 0xF7F0000E, 0xF7F8000E, 0xF800000E, 0xF808000E, 0xF810000E, 0xF818000E, 0xF820000E, 0xF828000E, 0xF830000E, 0xF838000E, 0xF840000E, 0xF848000E, 0xF850000E, 0xF858000E, 0xF860000E, 0xF868000E, 0xF870000E, 0xF878000E, 0xF880000E, 0xF888000E, 0xF890000E, 0xF898000E, 0xF8A0000E, 0xF8A8000E, 0xF8B0000E, 0xF8B8000E, 0xF8C0000E, 0xF8C8000E, 0xF8D0000E, 0xF8D8000E, 0xF8E0000E, 0xF8E8000E, 0xF8F0000E, 0xF8F8000E, 0xF900000E, 0xF908000E, 0xF910000E, 0xF918000E, 0xF920000E, 0xF928000E, 0xF930000E, 0xF938000E, 0xF940000E, 0xF948000E, 0xF950000E, 0xF958000E, 0xF960000E, 0xF968000E, 0xF970000E, 0xF978000E, 0xF980000E, 0xF988000E, 0xF990000E, 0xF998000E, 0xF9A0000E, 0xF9A8000E, 0xF9B0000E, 0xF9B8000E, 0xF9C0000E, 0xF9C8000E, 0xF9D0000E, 0xF9D8000E, 0xF9E0000E, 0xF9E8000E, 0xF9F0000E, 0xF9F8000E, 0xFA00000E, 0xFA08000E, 0xFA10000E, 0xFA18000E, 0xFA20000E, 0xFA28000E, 0xFA30000E, 0xFA38000E, 0xFA40000E, 0xFA48000E, 0xFA50000E, 0xFA58000E, 0xFA60000E, 0xFA68000E, 0xFA70000E, 0xFA78000E, 0xFA80000E, 0xFA88000E, 0xFA90000E, 0xFA98000E, 0xFAA0000E, 0xFAA8000E, 0xFAB0000E, 0xFAB8000E, 0xFAC0000E, 0xFAC8000E, 0xFAD0000E, 0xFAD8000E, 0xFAE0000E, 0xFAE8000E, 0xFAF0000E, 0xFAF8000E, 0xFB00000E, 0xFB08000E, 0xFB10000E, 0xFB18000E, 0xFB20000E, 0xFB28000E, 0xFB30000E, 0xFB38000E, 0xFB40000E, 0xFB48000E, 0xFB50000E, 0xFB58000E, 0xFB60000E, 0xFB68000E, 0xFB70000E, 0xFB78000E, 0xFB80000E, 0xFB88000E, 0xFB90000E, 0xFB98000E, 0xFBA0000E, 0xFBA8000E, 0xFBB0000E, 0xFBB8000E, 0xFBC0000E, 0xFBC8000E, 0xFBD0000E, 0xFBD8000E, 0xFBE0000E, 0xFBE8000E, 0xFBF0000E, 0xFBF8000E, 0xFC00000E, 0xFC08000E, 0xFC10000E, 0xFC18000E, 0xFC20000E, 0xFC28000E, 0xFC30000E, 0xFC38000E, 0xFC40000E, 0xFC48000E, 0xFC50000E, 0xFC58000E, 0xFC60000E, 0xFC68000E, 0xFC70000E, 0xFC78000E, 0xFC80000E, 0xFC88000E, 0xFC90000E, 0xFC98000E, 0xFCA0000E, 0xFCA8000E, 0xFCB0000E, 0xFCB8000E, 0xFCC0000E, 0xFCC8000E, 0xFCD0000E, 0xFCD8000E, 0xFCE0000E, 0xFCE8000E, 0xFCF0000E, 0xFCF8000E, 0xFD00000E, 0xFD08000E, 0xFD10000E, 0xFD18000E, 0xFD20000E, 0xFD28000E, 0xFD30000E, 0xFD38000E, 0xFD40000E, 0xFD48000E, 0xFD50000E, 0xFD58000E, 0xFD60000E, 0xFD68000E, 0xFD70000E, 0xFD78000E, 0xFD80000E, 0xFD88000E, 0xFD90000E, 0xFD98000E, 0xFDA0000E, 0xFDA8000E, 0xFDB0000E, 0xFDB8000E, 0xFDC0000E, 0xFDC8000E, 0xFDD0000E, 0xFDD8000E, 0xFDE0000E, 0xFDE8000E, 0xFDF0000E, 0xFDF8000E, 0xFE00000E, 0xFE08000E, 0xFE10000E, 0xFE18000E, 0xFE20000E, 0xFE28000E, 0xFE30000E, 0xFE38000E, 0xFE40000E, 0xFE48000E, 0xFE50000E, 0xFE58000E, 0xFE60000E, 0xFE68000E, 0xFE70000E, 0xFE78000E, 0xFE80000E, 0xFE88000E, 0xFE90000E, 0xFE98000E, 0xFEA0000E, 0xFEA8000E, 0xFEB0000E, 0xFEB8000E, 0xFEC0000E, 0xFEC8000E, 0xFED0000E, 0xFED8000E, 0xFEE0000E, 0xFEE8000E, 0xFEF0000E, 0xFEF8000E, 0xFF00000E, 0xFF08000E, 0xFF10000E, 0xFF18000E, 0xFF20000E, 0xFF28000E, 0xFF30000E, 0xFF38000E, 0xFF40000E, 0xFF48000E, 0xFF50000E, 0xFF58000E, 0xFF60000E, 0xFF68000E, 0xFF70000E, 0xFF78000E, 0xFF80000E, 0xFF88000E, 0xFF90000E, 0xFF98000E, 0xFFA0000E, 0xFFA8000E, 0xFFB0000E, 0xFFB8000E, 0xFFC0000E, 0xFFC8000E, 0xFFD0000E, 0xFFD8000E, 0xFFE0000E, 0xFFE8000E, 0xFFF0000E, 0xFFF8000E, 0x10000000E, 0x10008000E, 0x10010000E, 0x10018000E, 0x10020000E, 0x10028000E, 0x10030000E, 0x10038000E, 0x10040000E, 0x10048000E, 0x10050000E, 0x10058000E, 0x10060000E, 0x10068000E, 0x10070000E, 0x10078000E, 0x10080000E, 0x10088000E, 0x10090000E, 0x10098000E, 0x100A0000E, 0x100A8000E, 0x100B0000E, 0x100B8000E, 0x100C0000E, 0x100C8000E, 0x100D0000E, 0x100D8000E, 0x100E0000E, 0x100E8000E, 0x100F0000E, 0x100F8000E, 0x10100000E, 0x10108000E, 0x10110000E, 0x10118000E, 0x10120000E, 0x10128000E, 0x10130000E, 0x10138000E, 0x10140000E, 0x10148000E, 0x10150000E, 0x10158000E, 0x10160000E, 0x10168000E, 0x10170000E, 0x10178000E, 0x10180000E, 0x10188000E, 0x10190000E, 0x10198000E, 0x101A0000E, 0x101A8000E, 0x101B0000E, 0x101B8000E, 0x101C0000E, 0x101C8000E, 0x101D0000E, 0x101D8000E, 0x101E0000E, 0x101E8000E, 0x101F0000E, 0x101F8000E, 0x10200000E, 0x10208000E, 0x10210000E, 0x10218000E, 0x10220000E, 0x10228000E, 0x10230000E, 0x10238000E, 0x10240000E, 0x10248000E, 0x10250000E, 0x10258000E, 0x10260000E, 0x10268000E, 0x10270000E, 0x10278000E, 0x10280000E, 0x10288000E, 0x10290000E, 0x10298000E, 0x102A0000E, 0x102A8000E, 0x102B0000E, 0x102B8000E, 0x102C0000E, 0x102C8000E, 0x102D0000E, 0x102D8000E, 0x102E0000E, 0x102E8000E, 0x102F0000E, 0x102F8000E, 0x10300000E, 0x10308000E, 0x10310000E, 0x10318000E, 0x10320000E, 0x10328000E, 0x10330000E, 0x10338000E, 0x10340000E, 0x10348000E, 0x10350000E, 0x10358000E, 0x10360000E, 0x10368000E, 0x10370000E, 0x10378000E, 0x10380000E, 0x10388000E, 0x10390000E, 0x10398000E, 0x103A0000E, 0x103A8000E, 0x103B0000E, 0x103B8000E, 0x103C0000E, 0x103C8000E, 0x103D0000E, 0x103D8000E, 0x103E0000E, 0x103E8000E, 0x103F0000E, 0x103F8000E, 0x10400000E, 0x10408000E, 0x10410000E, 0x10418000E, 0x10420000E, 0x10428000E, 0x10430000E, 0x10438000E, 0x10440000E, 0x10448000E, 0x10450000E, 0x10458000E, 0x10460000E, 0x10468000E, 0x10470000E, 0x10478000E, 0x10480000E, 0x10488000E, 0x10490000E, 0x10498000E, 0x104A0000E, 0x104A8000E, 0x104B0000E, 0x104B8000E, 0x104C0000E, 0x104C8000E, 0x104D0000E, 0x104D8000E, 0x104E0000E, 0x104E8000E, 0x104F0000E, 0x104F8000E, 0x10500000E, 0x10508000E, 0x10510000E, 0x10518000E, 0x10520000E, 0x10528000E, 0x10530000E, 0x10538000E, 0x10540000E, 0x10548000E, 0x10550000E, 0x10558000E, 0x10560000E, 0x10568000E, 0x10570000E, 0x10578000E, 0x10580000E, 0x10588000E, 0x10590000E, 0x10598000E, 0x105A0000E, 0x105A8000E, 0x105B0000E, 0x105B8000E, 0x105C0000E, 0x105C8000E, 0x105D0000E, 0x105D8000E, 0x105E0000E, 0x105E8000E, 0x105F0000E, 0x105F8000E, 0x10600000E, 0x10608000E, 0x10610000E, 0x10618000E, 0x10620000E, 0x10628000E, 0x10630000E, 0x10638000E, 0x10640000E, 0x10648000E, 0x10650000E, 0x10658000E, 0x10660000E, 0x10668000E, 0x10670000E, 0x10678000E, 0x10680000E, 0x10688000E, 0x10690000E, 0x10698000E, 0x106A0000E, 0x106A8000E, 0x106B0000E, 0x106B8000E, 0x106C0000E, 0x106C8000E, 0x106D0000E, 0x106D8000E, 0x106E0000E, 0x106E8000E, 0x106F0000E, 0x106F8000E, 0x10700000E, 0x10708000E, 0x10710000E, 0x10718000E, 0x10720000E, 0x10728000E, 0x10730000E, 0x10738000E, 0x10740000E, 0x10748000E, 0x10750000E, 0x10758000E, 0x10760000E, 0x10768000E, 0x10770000E, 0x10778000E, 0x10780000E, 0x10788000E, 0x10790000E, 0x10798000E, 0x107A0000E, 0x107A8000E, 0x107B0000E, 0x107B8000E, 0x107C0000E, 0x107C8000E, 0x107D0000E, 0x107D8000E, 0x107E0000E, 0x107E8000E, 0x107F0000E, 0x107F8000E, 0x10800000E, 0x10808000E, 0x10810000E, 0x10818000E, 0x10820000E, 0x10828000E, 0x10830000E, 0x10838000E, 0x10840000E, 0x10848000E, 0x10850000E, 0x10858000E, 0x10860000E, 0x10868000E, 0x10870000E, 0x10878000E, 0x10880000E, 0x10888000E, 0x10890000E, 0x10898000E, 0x108A0000E, 0x108A8000E, 0x108B0000E, 0x108B8000E, 0x108C0000E, 0x108C8000E, 0x108D0000E, 0x108D8000E, 0x108E0000E, 0x108E8000E, 0x108F0000E, 0x108F8000E, 0x10900000E, 0x10908000E, 0x10910000E, 0x10918000E, 0x10920000E, 0x10928000E, 0x10930000E, 0x10938000E, 0x10940000E, 0x10948000E, 0x10950000E, 0x10958000E, 0x10960000E, 0x10968000E, 0x10970000E, 0x10978000E, 0x10980000E, 0x10988000E, 0x10990000E, 0x10998000E, 0x109A0000E, 0x109A8000E, 0x109B0000E, 0x109B8000E, 0x109C0000E, 0x109C8000E, 0x109D0000E, 0x109D8000E, 0x109E0000E, 0x109E8000E, 0x109F0000E, 0x109F8000E, 0x10A00000E, 0x10A08000E, 0x10A10000E, 0x10A18000E, 0x10A20000E, 0x10A28000E, 0x10A30000E, 0x10A38000E, 0x10A40000E, 0x10A48000E, 0x10A50000E, 0x10A58000E, 0x10A60000E, 0x10A68000E, 0x10A70000E, 0x10A78000E, 0x10A80000E, 0x10A88000E, 0x10A90000E, 0x10A98000E, 0x10AA0000E, 0x10AA8000E, 0x10AB0000E, 0x10AB8000E, 0x10AC0000E, 0x10AC8000E, 0x10AD0000E, 0x10AD8000E, 0x10AE0000E, 0x10AE8000E, 0x10AF0000E, 0x10AF8000E, 0x10B00000E, 0x10B08000E, 0x10B10000E, 0x10B18000E, 0x10B20000E, 0x10B28000E, 0x10B30000E, 0x10B38000E, 0x10B40000E, 0x10B48000E, 0x10B50000E, 0x10B58000E, 0x10B60000E, 0x10B68000E, 0x10B70000E, 0x10B78000E, 0x10B80000E, 0x10B88000E, 0x10B90000E, 0x10B98000E, 0x10BA0000E, 0x10BA8000E, 0x10BB0000E, 0x10BB8000E, 0x10BC0000E, 0x10BC8000E, 0x10BD0000E, 0x10BD8000E, 0x10BE0000E, 0x10BE8000E, 0x10BF0000E, 0x10BF8000E, 0x10C00000E, 0x10C08000E, 0x10C10000E, 0x10C18000E, 0x10C20000E, 0x10C28000E, 0x10C30000E, 0x10C38000E, 0x10C40000E, 0x10C48000E, 0x10C50000E, 0x10C58000E, 0x10C60000E, 0x10C68000E, 0x10C70000E, 0x10C78000E, 0x10C80000E, 0x10C88000E, 0x10C90000E, 0x10C98000E, 0x10CA0000E, 0x10CA8000E, 0x10CB0000E, 0x10CB8000E, 0x10CC0000E, 0x10CC8000E, 0x10CD0000E, 0x10CD8000E, 0x10CE0000E, 0x10CE8000E, 0x10CF0000E, 0x10CF8000E, 0x10D00000E, 0x10D08000E, 0x10D10000E, 0x10D18000E, 0x10D20000E, 0x10D28000E, 0x10D30000E, 0x10D38000E, 0x10D40000E, 0x10D48000E, 0x10D50000E, 0x10D58000E, 0x10D60000E, 0x10D68000E, 0x10D70000E, 0x10D78000E, 0x10D80000E, 0x10D88000E, 0x10D90000E, 0x10D98000E, 0x10DA0000E, 0x10DA8000E, 0x10DB0000E, 0x10DB8000E, 0x10DC0000E, 0x10DC8000E, 0x10DD0000E, 0x10DD8000E, 0x10DE0000E, 0x10DE8000E, 0x10DF0000E, 0x10DF8000E, 0x10E00000E, 0x10E08000E, 0x10E10000E, 0x10E18000E, 0x10E20000E, 0x10E28000E, 0x10E30000E, 0x10E38000E, 0x10E40000E, 0x10E48000E, 0x10E50000E, 0x10E58000E, 0x10E60000E, 0x10E68000E, 0x10E70000E, 0x10E78000E, 0x10E80000E, 0x10E88000E, 0x10E90000E, 0x10E98000E, 0x10EA0000E, 0x10EA8000E, 0x10EB0000E, 0x10EB8000E, 0x10EC0000E, 0x10EC8000E, 0x10ED0000E, 0x10ED8000E, 0x10EE0000E, 0x10EE8000E, 0x10EF0000E, 0x10EF8000E, 0x10F00000E, 0x10F08000E, 0x10F10000E, 0x10F18000E, 0x10F20000E, 0x10F28000E, 0x10F30000E, 0x10F38000E, 0x10F40000E, 0x10F48000E, 0x10F50000E, 0x10F58000E, 0x10F60000E, 0x10F68000E, 0x10F70000E, 0x10F78000E, 0x10F80000E, 0x10F88000E, 0x10F90000E, 0x10F98000E, 0x10FA0000E, 0x10FA8000E, 0x10FB0000E, 0x10FB8000E, 0x10FC0000E, 0x10FC8000E, 0x10FD0000E, 0x10FD8000E, 0x10FE0000E, 0x10FE8000E, 0x10FF0000E, 0x10FF8000E, 0x11000000E, 0x11008000E, 0x11010000E, 0x11018000E, 0x11020000E, 0x11028000E, 0x11030000E, 0x11038000E, 0x11040000E, 0x11048000E, 0x11050000E, 0x11058000E, 0x11060000E, 0x11068000E, 0x11070000E, 0x11078000E, 0x11080000E, 0x11088000E, 0x11090000E, 0x11098000E, 0x110A0000E, 0x110A8000E, 0x110B0000E, 0x110B8000E, 0x110C0000E, 0x110C8000E, 0x110D0000E, 0x110D8000E, 0x110E0000E, 0x110E8000E, 0x110F0000E, 0x110F8000E, 0x11100000E, 0x11108000E, 0x11110000E, 0x11118000E, 0x11120000E, 0x11128000E, 0x11130000E, 0x11138000E, 0x11140000E, 0x11148000E, 0x11150000E, 0x11158000E, 0x11160000E, 0x11168000E, 0x11170000E, 0x11178000E, 0x11180000E, 0x11188000E, 0x11190000E, 0x11198000E, 0x111A0000E, 0x111A8000E, 0x111B0000E, 0x111B8000E, 0x111C0000E, 0x111C8000E, 0x111D0000E, 0x111D8000E, 0x111E0000E, 0x111E8000E, 0x111F0000E, 0x111F8000E, 0x11200000E, 0x11208000E, 0x11210000E, 0x11218000E, 0x11220000E, 0x11228000E, 0x11230000E, 0x11238000E, 0x11240000E, 0x11248000E, 0x11250000E, 0x11258000E, 0x11260000E, 0x11268000E, 0x11270000E, 0x11278000E, 0x11280000E, 0x11288000E, 0x11290000E, 0x11298000E, 0x112A0000E, 0x112A8000E, 0x112B0000E, 0x112B8000E, 0x112C0000E, 0x112C8000E, 0x112D0000E, 0x112D8000E, 0x112E0000E, 0x112E8000E, 0x112F0000E, 0x112F8000E, 0x11300000E, 0x11308000E, 0x11310000E, 0x11318000E, 0x11320000E, 0x11328000E, 0x11330000E, 0x11338000E, 0x11340000E, 0x11348000E, 0x11350000E, 0x11358000E, 0x11360000E, 0x11368000E, 0x11370000E, 0x11378000E, 0x11380000E, 0x11388000E, 0x11390000E, 0x11398000E, 0x113A0000E, 0x113A8000E, 0x113B0000E, 0x113B8000E, 0x113C0000E, 0x113C8000E, 0x113D0000E, 0x113D8000E, 0x113E0000E, 0x113E8000E, 0x113F0000E, 0x113F8000E, 0x11400000E, 0x11408000E, 0x11410000E, 0x11418000E, 0x11420000E, 0x11428000E, 0x11430000E, 0x11438000E, 0x11440000E, 0x11448000E, 0x11450000E, 0x11458000E, 0x11460000E, 0x11468000E, 0x11470000E, 0x11478000E, 0x11480000E, 0x11488000E, 0x11490000E, 0x11498000E, 0x114A0000E, 0x114A8000E, 0x114B0000E, 0x114B8000E, 0x114C0000E, 0x114C8000E, 0x114D0000E, 0x114D8000E, 0x114E0000E, 0x114E8000E, 0x114F0000E, 0x114F8000E, 0x11500000E, 0x11508000E, 0x11510000E, 0x11518000E, 0x11520000E, 0x11528000E, 0x11530000E, 0x11538000E, 0x11540000E, 0x11548000E, 0x11550000E, 0x11558000E, 0x11560000E, 0x11568000E, 0x11570000E, 0x11578000E, 0x11580000E, 0x11588000E, 0x11590000E, 0x11598000E, 0x115A0000E, 0x115A8000E, 0x115B0000E, 0x115B8000E, 0x115C0000E, 0x115C8000E, 0x115D0000E, 0x115D8000E, 0x115E0000E, 0x115E8000E, 0x115F0000E, 0x115F8000E, 0x11600000E, 0x11608000E, 0x11610000E, 0x11618000E, 0x11620000E, 0x11628000E, 0x11630000E, 0x11638000E, 0x11640000E, 0x11648000E, 0x11650000E, 0x11658000E, 0x11660000E, 0x11668000E, 0x11670000E, 0x11678000E, 0x11680000E, 0x11688000E, 0x11690000E, 0x11698000E, 0x45A8F, 0x45AAF, 0x45ACF, 0x45AEF, 0x45B0F, 0x45B2F, 0x45B4F, 0x45B6F, 0x45B8F, 0x45BAF, 0x45BCF, 0x45BEF, 0x45C0F, 0x45C2F, 0x45C4F, 0x45C6F, 0x45C8F, 0x45CAF, 0x45CCF, 0x45CEF, 0x45D0F, 0x45D2F, 0x45D4F, 0x45D6F, 0x45D8F, 0x45DAF, 0x1177C5DCF, 0x1178C5E0F, 0x1179C5E4F, 0x117AC5E8F, 0x117BC5ECF, 0x117CC5F0F, 0x117DC5F4F, 0x117EC5F8F, 0x117FC5FCF, 0x1180C600F, 0x1181C604F, 0x1182C608F, 0x1183C60CF, 0x1184C610F, 0x1185C614F, 0x1186C618F, 0x1187C61CF, 0x1188C620F, 0x1189C624F, 0x118AC628F, 0x118BC62CF, 0x118CC630F, 0x118DC634F, 0x118EC638F, 0x118FC63CF, 0x1190C640F, 0x1191C644F, 0x1192C648F, 0x1193C64CF, 0x1194C650F, 0x1195C654F, 0x1196C658F, 0x1197C65CF, 0x1198C660F, 0x1199C664F, 0x119AC668F, 0x119BC66CF, 0x119CC670F, 0x119DC674F, 0x119EC678F, 0x119FC67CF, 0x11A0C680F, 0x11A1C684F, 0x11A2C688F, 0x11A3C68CF, 0x11A4C690F, 0x11A5C694F, 0x11A6C698F, 0x11A7C69CF, 0x11A8C6A0F, 0x11A9C6A4F, 0x11AAC6A8F, 0x11ABC6ACF, 0x11ACC6B0F, 0x11ADC6B4F, 0x11AEC6B8F, 0x11AFC6BCF, 0x11B0C6C0F, 0x11B1C6C4F, 0x11B2C6C8F, 0x11B3C6CCF, 0x11B4C6D0F, 0x11B5C6D4F, 0x11B6C6D8F, 0x11B7C6DCF, 0x11B8C6E0F, 0x11B9C6E4F, 0x11BAC6E8F, 0x11BBC6ECF, 0x11BCC6F0F, 0x11BDC6F4F, 0x11BEC6F8F, 0x11BFC6FCF, 0x11C0C700F, 0x11C1C704F, 0x11C2C708F, 0x11C3C70CF, 0x11C4C710F, 0x11C5C714F, 0x11C6C718F, 0x471CF, 0x11C8471EF, 0x11C94722F, 0x4726F, 0x4728F, 0x472AF, 0x472CF, 0x472EF, 0x4730F, 0x4732F, 0x4734F, 0x11CE4736F, 0x11CF473AF, 0x11D0473EF, 0x11D14742F, 0x11D24746F, 0x11D3474AF, 0x11D4474EF, 0x11D54752F, 0x4756F, 0x4758F, 0x475AF, 0x475CF, 0x475EF, 0x4760F, 0x4762F, 0x4764F, 0x4766F, 0x4768F, 0x476AF, 0x476CF, 0x11DC476EF, 0x4772F, 0x4774F, 0x4776F, 0x4778F, 0x477AF, 0x477CF, 0x11E0477EF, 0x4782F, 0x4784F, 0x4786F, 0x4788F, 0x478AF, 0x478CF, 0x478EF, 0x4790F, 0x4792F, 0x4794F, 0x4796F, 0x4798F, 0x479AF, 0x479CF, 0x479EF, 0x47A0F, 0x47A2F, 0x47A4F, 0x47A6F, 0x47A8F, 0x47AAF, 0x47ACF, 0x47AEF, 0x47B0F, 0x47B2F, 0x47B4F, 0x47B6F, 0x47B8F, 0x47BAF, 0x47BCF, 0x47BEF, 0x47C0F, 0x47C2F, 0x47C4F, 0x47C6F, 0x47C8F, 0x47CAF, 0x47CCF, 0x47CEF, 0x47D0F, 0x47D2F, 0x47D4F, 0x47D6F, 0x47D8F, 0x47DAF, 0x47DCF, 0x47DEF, 0x47E0F, 0x47E2F, 0x47E4F, 0x47E6F, 0x47E8F, 0x47EAF, 0x47ECF, 0x47EEF, 0x47F0F, 0x47F2F, 0x47F4F, 0x47F6F, 0x47F8F, 0x47FAF, 0x47FCF, 0x47FEF, 0x4800F, 0x4802F, 0x4804F, 0x4806F, 0x4808F, 0x480AF, 0x480CF, 0x480EF, 0x4810F, 0x4812F, 0x4814F, 0x4816F, 0x4818F, 0x481AF, 0x12070000F, 0x12078000F, 0x12080000F, 0x12088000F, 0x12090000F, 0x12098000F, 0x120A0000F, 0x120A8000F, 0x120B0000F, 0x120B8000F, 0x120C0000F, 0x120C8000F, 0x120D0000F, 0x120D8000F, 0x120E0000F, 0x120E8000F, 0x120F0000F, 0x120F8000F, 0x12100000F, 0x12108000F, 0x12110000F, 0x12118000F, 0x12120000F, 0x12128000F, 0x12130000F, 0x12138000F, 0x12140000F, 0x12148000F, 0x12150000F, 0x12158000F, 0x12160000F, 0x12168000F, 0x12170000F, 0x12178000F, 0x12180000F, 0x12188000F, 0x12190000F, 0x12198000F, 0x121A0000F, 0x121A8000F, 0x121B0000F, 0x121B8000F, 0x121C0000F, 0x121C8000F, 0x121D0000F, 0x121D8000F, 0x121E0000F, 0x121E8000F, 0x121F0000F, 0x121F8000F, 0x12200000F, 0x12208000F, 0x12210000F, 0x12218000F, 0x12220000F, 0x12228000F, 0x12230000F, 0x12238000F, 0x12240000F, 0x12248000F, 0x12250000F, 0x12258000F, 0x12260000F, 0x12268000F, 0x12270000F, 0x12278000F, 0x12280000F, 0x12288000F, 0x12290000F, 0x12298000F, 0x122A0000F, 0x122A8000F, 0x122B0000F, 0x122B8000F, 0x122C0000F, 0x122C8000F, 0x122D0000F, 0x122D8000F, 0x122E0000F, 0x122E8000F, 0x122F0000F, 0x122F8000F, 0x12300000F, 0x12308000F, 0x12310000F, 0x12318000F, 0x12320000F, 0x12328000F, 0x12330000F, 0x12338000F, 0x12340000F, 0x12348000F, 0x12350000F, 0x12358000F, 0x12360000F, 0x12368000F, 0x12370000F, 0x12378000F, 0x12380000F, 0x12388000F, 0x12390000F, 0x12398000F, 0x123A0000F, 0x123A8000F, 0x123B0000F, 0x123B8000F, 0x123C0000F, 0x123C8000F, 0x123D0000F, 0x123D8000F, 0x123E0000F, 0x123E8000F, 0x123F0000F, 0x123F8000F, 0x12400000F, 0x12408000F, 0x12410000F, 0x12418000F, 0x12420000F, 0x12428000F, 0x12430000F, 0x12438000F, 0x12440000F, 0x12448000F, 0x12450000F, 0x12458000F, 0x12460000F, 0x12468000F, 0x12470000F, 0x12478000F, 0x12480000F, 0x12488000F, 0x12490000F, 0x12498000F, 0x124A0000F, 0x124A8000F, 0x124B0000F, 0x124B8000F, 0x124C0000F, 0x124C8000F, 0x124D0000F, 0x124D8000F, 0x124E0000F, 0x124E8000F, 0x124F0000F, 0x124F8000F, 0x12500000F, 0x12508000F, 0x12510000F, 0x12518000F, 0x12520000F, 0x12528000F, 0x12530000F, 0x12538000F, 0x12540000F, 0x12548000F, 0x12550000F, 0x12558000F, 0x12560000F, 0x12568000F, 0x12570000F, 0x12578000F, 0x12580000F, 0x12588000F, 0x12590000F, 0x12598000F, 0x125A0000F, 0x125A8000F, 0x125B0000F, 0x125B8000F, 0x125C0000F, 0x125C8000F, 0x125D0000F, 0x125D8000F, 0x125E0000F, 0x125E8000F, 0x125F0000F, 0x125F8000F, 0x12600000F, 0x12608000F, 0x12610000F, 0x12618000F, 0x12620000F, 0x12628000F, 0x12630000F, 0x12638000F, 0x12640000F, 0x12648000F, 0x12650000F, 0x12658000F, 0x12660000F, 0x12668000F, 0x12670000F, 0x12678000F, 0x12680000F, 0x12688000F, 0x12690000F, 0x12698000F, 0x126A0000F, 0x126A8000F, 0x126B0000F, 0x126B8000F, 0x126C0000F, 0x126C8000F, 0x126D0000F, 0x126D8000F, 0x126E0000F, 0x126E8000F, 0x126F0000F, 0x126F8000F, 0x12700000F, 0x12708000F, 0x12710000F, 0x12718000F, 0x12720000F, 0x12728000F, 0x12730000F, 0x12738000F, 0x12740000F, 0x12748000F, 0x12750000F, 0x12758000F, 0x12760000F, 0x12768000F, 0x12770000F, 0x12778000F, 0x12780000F, 0x12788000F, 0x12790000F, 0x12798000F, 0x127A0000F, 0x127A8000F, 0x127B0000F, 0x127B8000F, 0x127C0000F, 0x127C8000F, 0x127D0000F, 0x127D8000F, 0x127E0000F, 0x127E8000F, 0x127F0000F, 0x127F8000F, 0x12800000F, 0x12808000F, 0x12810000F, 0x12818000F, 0x12820000F, 0x12828000F, 0x12830000F, 0x12838000F, 0x12840000F, 0x12848000F, 0x12850000F, 0x12858000F, 0x12860000F, 0x12868000F, 0x12870000F, 0x12878000F, 0x12880000F, 0x12888000F, 0x12890000F, 0x12898000F, 0x128A0000F, 0x128A8000F, 0x128B0000F, 0x128B8000F, 0x128C0000F, 0x128C8000F, 0x128D0000F, 0x128D8000F, 0x128E0000F, 0x128E8000F, 0x128F0000F, 0x128F8000F, 0x12900000F, 0x12908000F, 0x12910000F, 0x12918000F, 0x12920000F, 0x12928000F, 0x12930000F, 0x12938000F, 0x12940000F, 0x12948000F, 0x12950000F, 0x12958000F, 0x12960000F, 0x12968000F, 0x12970000F, 0x12978000F, 0x12980000F, 0x12988000F, 0x12990000F, 0x12998000F, 0x129A0000F, 0x129A8000F, 0x129B0000F, 0x129B8000F, 0x129C0000F, 0x129C8000F, 0x129D0000F, 0x129D8000F, 0x129E0000F, 0x129E8000F, 0x129F0000F, 0x129F8000F, 0x12A00000F, 0x12A08000F, 0x12A10000F, 0x12A18000F, 0x12A20000F, 0x12A28000F, 0x12A30000F, 0x12A38000F, 0x12A40000F, 0x12A48000F, 0x12A50000F, 0x12A58000F, 0x12A60000F, 0x12A68000F, 0x12A70000F, 0x12A78000F, 0x12A80000F, 0x12A88000F, 0x12A90000F, 0x12A98000F, 0x12AA0000F, 0x12AA8000F, 0x12AB0000F, 0x12AB8000F, 0x12AC0000F, 0x12AC8000F, 0x12AD0000F, 0x12AD8000F, 0x12AE0000F, 0x12AE8000F, 0x12AF0000F, 0x12AF8000F, 0x12B00000F, 0x12B08000F, 0x12B10000F, 0x12B18000F, 0x12B20000F, 0x12B28000F, 0x12B30000F, 0x12B38000F, 0x12B40000F, 0x12B48000F, 0x12B50000F, 0x12B58000F, 0x12B60000F, 0x12B68000F, 0x12B70000F, 0x12B78000F, 0x12B80000F, 0x12B88000F, 0x12B90000F, 0x12B98000F, 0x12BA0000F, 0x12BA8000F, 0x12BB0000F, 0x12BB8000F, 0x12BC0000F, 0x12BC8000F, 0x12BD0000F, 0x12BD8000F, 0x12BE0000F, 0x12BE8000F, 0x12BF0000F, 0x12BF8000F, 0x12C00000F, 0x12C08000F, 0x12C10000F, 0x12C18000F, 0x12C20000F, 0x12C28000F, 0x12C30000F, 0x12C38000F, 0x12C40000F, 0x12C48000F, 0x12C50000F, 0x12C58000F, 0x12C60000F, 0x12C68000F, 0x12C70000F, 0x12C78000F, 0x12C80000F, 0x12C88000F, 0x12C90000F, 0x12C98000F, 0x12CA0000F, 0x12CA8000F, 0x12CB0000F, 0x12CB8000F, 0x12CC0000F, 0x12CC8000F, 0x12CD0000F, 0x12CD8000F, 0x12CE0000F, 0x12CE8000F, 0x12CF0000F, 0x12CF8000F, 0x12D00000F, 0x12D08000F, 0x12D10000F, 0x12D18000F, 0x12D20000F, 0x12D28000F, 0x12D30000F, 0x12D38000F, 0x12D40000F, 0x12D48000F, 0x12D50000F, 0x12D58000F, 0x12D60000F, 0x12D68000F, 0x12D70000F, 0x12D78000F, 0x12D80000F, 0x12D88000F, 0x12D90000F, 0x12D98000F, 0x12DA0000F, 0x12DA8000F, 0x12DB0000F, 0x12DB8000F, 0x12DC0000F, 0x12DC8000F, 0x12DD0000F, 0x12DD8000F, 0x12DE0000F, 0x12DE8000F, 0x12DF0000F, 0x12DF8000F, 0x12E00000F, 0x12E08000F, 0x12E10000F, 0x12E18000F, 0x12E20000F, 0x12E28000F, 0x12E30000F, 0x12E38000F, 0x12E40000F, 0x12E48000F, 0x12E50000F, 0x12E58000F, 0x12E60000F, 0x12E68000F, 0x12E70000F, 0x12E78000F, 0x12E80000F, 0x12E88000F, 0x12E90000F, 0x12E98000F, 0x12EA0000F, 0x12EA8000F, 0x12EB0000F, 0x12EB8000F, 0x12EC0000F, 0x12EC8000F, 0x12ED0000F, 0x12ED8000F, 0x12EE0000F, 0x12EE8000F, 0x12EF0000F, 0x12EF8000F, 0x12F00000F, 0x12F08000F, 0x12F10000F, 0x12F18000F, 0x12F20000F, 0x12F28000F, 0x12F30000F, 0x12F38000F, 0x12F40000F, 0x12F48000F, 0x12F50000F, 0x12F58000F, 0x12F60000F, 0x12F68000F, 0x12F70000F, 0x12F78000F, 0x12F80000F, 0x12F88000F, 0x12F90000F, 0x12F98000F, 0x12FA0000F, 0x12FA8000F, 0x12FB0000F, 0x12FB8000F, 0x12FC0000F, 0x12FC8000F, 0x12FD0000F, 0x12FD8000F, 0x12FE0000F, 0x12FE8000F, 0x12FF0000F, 0x12FF8000F, 0x13000000F, 0x13008000F, 0x13010000F, 0x13018000F, 0x13020000F, 0x13028000F, 0x13030000F, 0x13038000F, 0x13040000F, 0x13048000F, 0x13050000F, 0x13058000F, 0x13060000F, 0x13068000F, 0x13070000F, 0x13078000F, 0x13080000F, 0x13088000F, 0x13090000F, 0x13098000F, 0x130A0000F, 0x130A8000F, 0x130B0000F, 0x130B8000F, 0x130C0000F, 0x130C8000F, 0x130D0000F, 0x130D8000F, 0x130E0000F, 0x130E8000F, 0x130F0000F, 0x130F8000F, 0x13100000F, 0x13108000F, 0x13110000F, 0x13118000F, 0x13120000F, 0x13128000F, 0x13130000F, 0x13138000F, 0x13140000F, 0x13148000F, 0x13150000F, 0x13158000F, 0x13160000F, 0x13168000F, 0x13170000F, 0x13178000F, 0x13180000F, 0x13188000F, 0x13190000F, 0x13198000F, 0x131A0000F, 0x131A8000F, 0x131B0000F, 0x131B8000F, 0x131C0000F, 0x131C8000F, 0x131D0000F, 0x131D8000F, 0x131E0000F, 0x131E8000F, 0x131F0000F, 0x131F8000F, 0x13200000F, 0x13208000F, 0x13210000F, 0x13218000F, 0x13220000F, 0x13228000F, 0x13230000F, 0x13238000F, 0x13240000F, 0x13248000F, 0x13250000F, 0x13258000F, 0x13260000F, 0x13268000F, 0x13270000F, 0x13278000F, 0x13280000F, 0x13288000F, 0x13290000F, 0x13298000F, 0x132A0000F, 0x132A8000F, 0x132B0000F, 0x132B8000F, 0x132C0000F, 0x132C8000F, 0x132D0000F, 0x132D8000F, 0x132E0000F, 0x132E8000F, 0x132F0000F, 0x132F8000F, 0x13300000F, 0x13308000F, 0x13310000F, 0x13318000F, 0x13320000F, 0x13328000F, 0x13330000F, 0x13338000F, 0x13340000F, 0x13348000F, 0x13350000F, 0x13358000F, 0x13360000F, 0x13368000F, 0x13370000F, 0x13378000F, 0x13380000F, 0x13388000F, 0x13390000F, 0x13398000F, 0x133A0000F, 0x133A8000F, 0x133B0000F, 0x133B8000F, 0x133C0000F, 0x133C8000F, 0x133D0000F, 0x133D8000F, 0x133E0000F, 0x133E8000F, 0x133F0000F, 0x133F8000F, 0x13400000F, 0x13408000F, 0x13410000F, 0x13418000F, 0x13420000F, 0x13428000F, 0x13430000F, 0x13438000F, 0x13440000F, 0x13448000F, 0x13450000F, 0x13458000F, 0x13460000F, 0x13468000F, 0x13470000F, 0x13478000F, 0x13480000F, 0x13488000F, 0x13490000F, 0x13498000F, 0x134A0000F, 0x134A8000F, 0x134B0000F, 0x134B8000F, 0x134C0000F, 0x134C8000F, 0x134D0000F, 0x134D8000F, 0x134E0000F, 0x134E8000F, 0x134F0000F, 0x134F8000F, 0x13500000F, 0x13508000F, 0x13510000F, 0x13518000F, 0x13520000F, 0x13528000F, 0x13530000F, 0x13538000F, 0x13540000F, 0x13548000F, 0x13550000F, 0x13558000F, 0x13560000F, 0x13568000F, 0x13570000F, 0x13578000F, 0x13580000F, 0x13588000F, 0x13590000F, 0x13598000F, 0x135A0000F, 0x135A8000F, 0x135B0000F, 0x135B8000F, 0x135C0000F, 0x135C8000F, 0x135D0000F, 0x135D8000F, 0x135E0000F, 0x135E8000F, 0x135F0000F, 0x135F8000F, 0x13600000F, 0x13608000F, 0x13610000F, 0x13618000F, 0x13620000F, 0x13628000F, 0x13630000F, 0x13638000F, 0x13640000F, 0x13648000F, 0x13650000F, 0x13658000F, 0x13660000F, 0x13668000F, 0x13670000F, 0x13678000F, 0x13680000F, 0x13688000F, 0x13690000F, 0x13698000F, 0x136A0000F, 0x136A8000F, 0x136B0000F, 0x136B8000F, 0x136C0000F, 0x136C8000F, 0x136D0000F, 0x136D8000F, 0x136E0000F, 0x136E8000F, 0x136F0000F, 0x136F8000F, 0x13700000F, 0x13708000F, 0x13710000F, 0x13718000F, 0x13720000F, 0x13728000F, 0x13730000F, 0x13738000F, 0x13740000F, 0x13748000F, 0x13750000F, 0x13758000F, 0x13760000F, 0x13768000F, 0x13770000F, 0x13778000F, 0x13780000F, 0x13788000F, 0x13790000F, 0x13798000F, 0x137A0000F, 0x137A8000F, 0x137B0000F, 0x137B8000F, 0x137C0000F, 0x137C8000F, 0x137D0000F, 0x137D8000F, 0x137E0000F, 0x137E8000F, 0x137F0000F, 0x137F8000F, 0x13800000F, 0x13808000F, 0x13810000F, 0x13818000F, 0x13820000F, 0x13828000F, 0x13830000F, 0x13838000F, 0x13840000F, 0x13848000F, 0x13850000F, 0x13858000F, 0x13860000F, 0x13868000F, 0x13870000F, 0x13878000F, 0x13880000F, 0x13888000F, 0x13890000F, 0x13898000F, 0x138A0000F, 0x138A8000F, 0x138B0000F, 0x138B8000F, 0x138C0000F, 0x138C8000F, 0x138D0000F, 0x138D8000F, 0x138E0000F, 0x138E8000F, 0x138F0000F, 0x138F8000F, 0x13900000F, 0x13908000F, 0x13910000F, 0x13918000F, 0x13920000F, 0x13928000F, 0x13930000F, 0x13938000F, 0x13940000F, 0x13948000F, 0x13950000F, 0x13958000F, 0x13960000F, 0x13968000F, 0x13970000F, 0x13978000F, 0x13980000F, 0x13988000F, 0x13990000F, 0x13998000F, 0x139A0000F, 0x139A8000F, 0x139B0000F, 0x139B8000F, 0x139C0000F, 0x139C8000F, 0x139D0000F, 0x139D8000F, 0x139E0000F, 0x139E8000F, 0x139F0000F, 0x139F8000F, 0x13A00000F, 0x13A08000F, 0x13A10000F, 0x13A18000F, 0x13A20000F, 0x13A28000F, 0x13A30000F, 0x13A38000F, 0x13A40000F, 0x13A48000F, 0x13A50000F, 0x13A58000F, 0x13A60000F, 0x13A68000F, 0x13A70000F, 0x13A78000F, 0x13A80000F, 0x13A88000F, 0x13A90000F, 0x13A98000F, 0x13AA0000F, 0x13AA8000F, 0x13AB0000F, 0x13AB8000F, 0x13AC0000F, 0x13AC8000F, 0x13AD0000F, 0x13AD8000F, 0x13AE0000F, 0x13AE8000F, 0x13AF0000F, 0x13AF8000F, 0x13B00000F, 0x13B08000F, 0x13B10000F, 0x13B18000F, 0x13B20000F, 0x13B28000F, 0x13B30000F, 0x13B38000F, 0x13B40000F, 0x13B48000F, 0x13B50000F, 0x13B58000F, 0x13B60000F, 0x13B68000F, 0x13B70000F, 0x13B78000F, 0x13B80000F, 0x13B88000F, 0x13B90000F, 0x13B98000F, 0x13BA0000F, 0x13BA8000F, 0x13BB0000F, 0x13BB8000F, 0x13BC0000F, 0x13BC8000F, 0x13BD0000F, 0x13BD8000F, 0x13BE0000F, 0x13BE8000F, 0x13BF0000F, 0x13BF8000F, 0x13C00000F, 0x13C08000F, 0x13C10000F, 0x13C18000F, 0x13C20000F, 0x13C28000F, 0x13C30000F, 0x13C38000F, 0x13C40000F, 0x13C48000F, 0x13C50000F, 0x13C58000F, 0x13C60000F, 0x13C68000F, 0x13C70000F, 0x13C78000F, 0x13C80000F, 0x13C88000F, 0x13C90000F, 0x13C98000F, 0x13CA0000F, 0x13CA8000F, 0x13CB0000F, 0x13CB8000F, 0x13CC0000F, 0x13CC8000F, 0x13CD0000F, 0x13CD8000F, 0x13CE0000F, 0x13CE8000F, 0x13CF0000F, 0x13CF8000F, 0x13D00000F, 0x13D08000F, 0x13D10000F, 0x13D18000F, 0x13D20000F, 0x13D28000F, 0x13D30000F, 0x13D38000F, 0x13D40000F, 0x13D48000F, 0x13D50000F, 0x13D58000F, 0x13D60000F, 0x13D68000F, 0x13D70000F, 0x13D78000F, 0x13D80000F, 0x13D88000F, 0x13D90000F, 0x13D98000F, 0x13DA00010, 0x13DB4F6B0, 0x13DC4F6F0, 0x13DD4F730, 0x13DE4F770, 0x13DF4F7B0, 0x13E04F7F0, 0x13E14F830, 0x13E24F870, 0x13E34F8B0, 0x13E44F8F0, 0x13E54F930, 0x13E64F970, 0x4F9B0, 0x13E7CF9D0, 0x13E8CFA10, 0x13E9CFA50, 0x13EACFA90, 0x13EBCFAD0, 0x13ECCFB10, 0x13EDCFB50, 0x13EECFB90, 0x13EFCFBD0, 0x13F0CFC10, 0x13F1CFC50, 0x13F2CFC90, 0x13F3CFCD0, 0x13F4CFD10, 0x13F5CFD50, 0x13F6CFD90, 0x13F7CFDD0, 0x13F8CFE10, 0x13F9CFE50, 0x13FACFE90, 0x13FBCFED0, 0x13FCCFF10, 0x13FDCFF50, 0x13FECFF90, 0x13FFCFFD0, 0x1400D0010, 0x1401D0050, 0x1402D0090, 0x1403D00D0, 0x1404D0110, 0x1405D0150, 0x1406D0190, 0x1407D01D0, 0x1408D0210, 0x1409D0250, 0x140AD0290, 0x140BD02D0, 0x140CD0310, 0x140DD0350, 0x140ED0390, 0x503D0, 0x140F80010, 0x1410D0410, 0x1414D03D0, 0x1415D0550, 0x141600010, 0x1417505B0, 0x1418505F0, 0x141080010, 0x141F507B0, 0x507D0, 0x1422D0890, 0x1423508B0, 0x508D0, 0x51250, 0x51270, 0x144AD1290, 0x144BD12D0, 0x144CD1310, 0x144DD1350, 0x144ED1390, 0x144FD13D0, 0x1450D1410, 0x1451D1450, 0x1452D1490, 0x1453D14D0, 0x1454D1510, 0x1455D1550, 0x1456D1590, 0x1457D15D0, 0x1458D1610, 0x1459D1650, 0x145AD1690, 0x145BD16D0, 0x145CD1710, 0x145D80010, 0x145E51770, 0x51790, 0x147851DF0, 0x147880010, 0x51E30, 0x147980010, 0x147AD1E90, 0x147BD1ED0, 0x147C51EF0, 0x147CD1F10, 0x147D51F30, 0x147E51F70, 0x147F51FB0, 0x148000010, 0x148152030, 0x148252070, 0x1483520B0, 0x1484520F0, 0x148552130, 0x148652170, 0x1487521B0, 0x1488521F0, 0x148952230, 0x148A52270, 0x148B522B0, 0x148C522F0, 0x148D52330, 0x148E52370, 0x148F523B0, 0x1490523F0, 0x149152430, 0x149252470, 0x1493524B0, 0x1494524F0, 0x149552530, 0x149652570, 0x1497525B0, 0x1498525F0, 0x149952630, 0x149A52670, 0x149B526B0, 0x149C526F0, 0x149D52730, 0x149E52770, 0x149F527B0, 0x14A0527F0, 0x14A152830, 0x14A252870, 0x14A3528B0, 0x14A4528F0, 0x14A480010, 0x52930, 0x14A5D2950, 0x14A6D2990, 0x14A7D29D0, 0x14A8D2A10, 0x14A9D2A50, 0x14AAD2A90, 0x14ABD2AD0, 0x14ACD2B10, 0x14AD00010, 0x14AE52B70, 0x14AE80010, 0x14AF00010, 0x14B052BF0, 0x14B152C30, 0x52BD0, 0x14B180010, 0x14B2D2C90, 0x14B3D2CD0, 0x52C70, 0x14B4D2D10, 0x14B5D2D50, 0x14B600010, 0x14B752DB0, 0x14B852DF0, 0x52D90, 0x14B880010, 0x14B9D2E50, 0x14BAD2E90, 0x52E30, 0x14BBD2ED0, 0x14BCD2F10, 0x14BDD2F50, 0x14BED2F90, 0x14BFD2FD0, 0x14C0D3010, 0x14C1D3050, 0x14C2D3090, 0x14C3D30D0, 0x14C4D3110, 0x14C5D3150, 0x14C6D3190, 0x14C7D31D0, 0x14C8D3210, 0x14C9D3250, 0x14CAD3290, 0x532D0, 0x532F0, 0x53310, 0x53330, 0x14CDD3350, 0x14CED3390, 0x14CFD33D0, 0x14D0D3410, 0x14D1D3450, 0x14D2D3490, 0x14D3D34D0, 0x14D4D3510, 0x14D5D3550, 0x14D6D3590, 0x14D7D35D0, 0x14D8D3610, 0x14D9D3650, 0x14DAD3690, 0x14DBD36D0, 0x14DCD3710, 0x14DDD3750, 0x14DED3790, 0x14F700010, 0x51FB0, 0x50890, 0x151DD4750, 0x54790, 0x151E80010, 0x151FD47D0, 0x1520D4810, 0x1521D4850, 0x1522D4890, 0x1523D48D0, 0x1524D4910, 0x1525D4950, 0x1526D4990, 0x1527D49D0, 0x1528D4A10, 0x1529D4A50, 0x152AD4A90, 0x152BD4AD0, 0x152CD4B10, 0x152DD4B50, 0x152ED4B90, 0x152FD4BD0, 0x1530D4C10, 0x1531D4C50, 0x1532D4C90, 0x1533D4CD0, 0x1534D4D10, 0x1535D4D50, 0x1536D4D90, 0x153700010, 0x153854DF0, 0x153954E30, 0x153A54E70, 0x153B54EB0, 0x153C54EF0, 0x153D54F30, 0x153E54F70, 0x153F54FB0, 0x154054FF0, 0x154155030, 0x154255070, 0x1543550B0, 0x1544550F0, 0x154555130, 0x154655170, 0x1547551B0, 0x1548551F0, 0x154955230, 0x55270, 0x1562D5890, 0x1563558B0, 0x1563D58D0, 0x1564558F0, 0x1564D5910, 0x156555930, 0x1565D5950, 0x156255970, 0x157CD5F10, 0x157DD5F50, 0x157ED5F90, 0x157FD5FD0, 0x1580D6010, 0x1581D6050, 0x1582D6090, 0x1583D60D0, 0x1584D6110, 0x1585D6150, 0x1586D6190, 0x1587D61D0, 0x1588D6210, 0x1589D6250, 0x158AD6290, 0x158BD62D0, 0x158CD6310, 0x158DD6350, 0x158ED6390, 0x158FD63D0, 0x1590D6410, 0x1591D6450, 0x1592D6490, 0x1593D64D0, 0x1594D6510, 0x56550, 0x1596D6590, 0x565B0, 0x159700010, 0x1596565F0, 0x56790, 0x159ED6790, 0x159FD67D0, 0x15A0D6810, 0x15A152BB0, 0x15A256870, 0x15A3568B0, 0x568F0, 0x15A4D6910, 0x15A5D6950, 0x15A6D6990, 0x15A7D69D0, 0x159756A10, 0x15A956A30, 0x15AA56A70, 0x15AB56AB0, 0x56AF0, 0x15ACD6B10, 0x15ADD6B50, 0x159F567B0, 0x159F80010, 0x15AC00010, 0x15AD56B30, 0x15AD80010, 0x565F0, 0x141580010, 0x50530, 0x15AF80010, 0x15B000010, 0x15B156C30, 0x15B256C70, 0x50550, 0x141056CB0, 0x15B300010, 0x51F90, 0x56CF0, 0x15B400010, 0x51EB0, 0x56D30, 0x141680010, 0x141651FD0, 0x15B5D6D50, 0x141800010, 0x507B0, 0x56D90, 0x15B6D05D0, 0x141780010, 0x56DD0, 0x56DF0, 0x15B8D6E10, 0x15B900010, 0x1479D6E70, 0x147A51ED0, 0x15B1D6C10, 0x15BD56F30, 0x1477D6F50, 0x147D51E10, 0x15BE56F70, 0x147DD6F90, 0x15C300010, 0x15CE57370, 0x573D0, 0x15D4D7510, 0x15D557530, 0x15D5D7550, 0x15D657570, 0x15D6D7590, 0x15D7575B0, 0x15D7D75D0, 0x15D8575F0, 0x15D8D7610, 0x15D9D7650, 0x57370, 0x15DB56BF0, 0x15DBD76D0, 0x15B0D76F0, 0x15DC56C50, 0x15B257710, 0x15DD56CB0, 0x15B357750, 0x15C356CF0, 0x15B4D6D10, 0x15BD80010, 0x15B556D70, 0x15B657390, 0x15DFD6DB0, 0x15B7577F0, 0x15D456DF0, 0x15E000010, 0x15B857810, 0x15E0D6E30, 0x15B957830, 0x15BCD6E70, 0x15E200010, 0x15E2D7890, 0x15E3578B0, 0x15E3D78D0, 0x15EF80010, 0x15F057BF0, 0x15F0D7C10, 0x15F157C30, 0x15F1D7C50, 0x15F257C70, 0x15F2D7C90, 0x15F357CB0, 0x57CD0, 0x15F500010, 0x1601D78F0, 0x1602D8090, 0x1603D80D0, 0x1604D8110, 0x160500010, 0x160658170, 0x1607581B0, 0x1608581F0, 0x160958230, 0x160A58270, 0x160B582B0, 0x160C582F0, 0x160D58330, 0x160DD2B70, 0x160ED8390, 0x160FD83D0, 0x1601D3DD0, 0x161000010, 0x161158430, 0x161258470, 0x1613584B0, 0x1614584F0, 0x161558530, 0x161658570, 0x1617585B0, 0x1617D2B90, 0x161857D50, 0x161958630, 0x161A58670, 0x161B586B0, 0x586F0, 0x161CD8711, 0x161DD8751, 0x161ED8791, 0x161FD87D1, 0x1620D8811, 0x1621D8851, 0x1622D8891, 0x1623D88D1, 0x1624D8911, 0x1625D8951, 0x1626D8991, 0x1627D89D1, 0x1628D8A11, 0x1629D8A51, 0x162AD8A91, 0x162BD8AD1, 0x162CD8B11, 0x162DD8B51, 0x162ED8B91, 0x162FD8BD1, 0x1630D8C11, 0x1631D8C51, 0x1632D8C91, 0x1633D8CD1, 0x1634D8D11, 0x1635D8D51, 0x1636D8D91, 0x58DD1, 0x163858DF1, 0x50451, 0x141250471, 0x1413504B1, 0x1414504F1, 0x163958E31, 0x163A58E71, 0x1418D8EB1, 0x1419D0651, 0x141AD0691, 0x141BD06D1, 0x141CD0711, 0x163B50751, 0x141E50771, 0x163B80011, 0x58F11, 0x1420507F1, 0x142150831, 0x163CD0871, 0x163DD8F51, 0x1424508F1, 0x142550931, 0x142650971, 0x1427509B1, 0x1428509F1, 0x142950A31, 0x142A50A71, 0x142B50AB1, 0x142C50AF1, 0x142D50B31, 0x142E50B71, 0x142F50BB1, 0x143050BF1, 0x143150C31, 0x143250C71, 0x50CB1, 0x1433D0CD1, 0x1434D0D11, 0x1435D0D51, 0x50D91, 0x143750DB1, 0x143850DF1, 0x143950E31, 0x143A50E71, 0x143B50EB1, 0x143C50EF1, 0x143D50F31, 0x143D80011, 0x143ED0F91, 0x143FD0FD1, 0x1440D1011, 0x1441D1051, 0x1442D1091, 0x1443D10D1, 0x1444D1111, 0x1445D1151, 0x1446D1191, 0x1447D11D1, 0x1448D1211, 0x163E00011, 0x163F58FB1, 0x164058FF1, 0x164159031, 0x164259071, 0x1643590B1, 0x1644590F1, 0x164559131, 0x164659171, 0x1647591B1, 0x1648591F1, 0x51751, 0x164959231, 0x59271, 0x145F517B1, 0x1460517F1, 0x146151831, 0x146251871, 0x1463518B1, 0x1464518F1, 0x146551931, 0x146651971, 0x1467519B1, 0x1468519F1, 0x146951A31, 0x146A51A71, 0x146B51AB1, 0x146C51AF1, 0x146D51B31, 0x146E51B71, 0x146F51BB1, 0x147051BF1, 0x147151C31, 0x147251C71, 0x147351CB1, 0x147451CF1, 0x147551D31, 0x147651D71, 0x147751DB1, 0x59291, 0x164B592B1, 0x164BD1E51, 0x164CD9311, 0x164DD9351, 0x164ED9391, 0x147FD93D1, 0x1650593F1, 0x165159431, 0x165259471, 0x1653594B1, 0x1654594F1, 0x165559531, 0x165659571, 0x1657595B1, 0x1658595F1, 0x165959631, 0x165A59671, 0x165B596B1, 0x165B80011, 0x165CD9711, 0x165DD9751, 0x165ED9791, 0x165FD97D1, 0x1660D9811, 0x1661D9851, 0x165C596F1, 0x165D59731, 0x165E59771, 0x165F597B1, 0x1660597F1, 0x166159831, 0x59871, 0x166200011, 0x598B1, 0x1663D98D1, 0x1664D9911, 0x1665D9951, 0x1666D9991, 0x599D1, 0x599F1, 0x59A11, 0x166880011, 0x1669D9A51, 0x59A91, 0x166A80011, 0x166BD9AD1, 0x166C00011, 0x166B80011, 0x166D59B31, 0x166D00011, 0x166D80011, 0x166ED9B91, 0x59BD1, 0x166F80011, 0x1670D9C11, 0x1670D9C51, 0x167259C71, 0x167359CB1, 0x167459CF1, 0x167559D31, 0x167659D71, 0x167759DB1, 0x166900011, 0x1669D9AD1, 0x166E59B31, 0x167059BB1, 0x167859DF1, 0x167959E31, 0x167A59E71, 0x167B59EB1, 0x167B80011, 0x167CD9F11, 0x167DD9F51, 0x167ED9F91, 0x167FD9FD1, 0x1680DA011, 0x168100011, 0x16825A071, 0x16835A0B1, 0x16845A0F1, 0x14DFD37D1, 0x14E0D3811, 0x14E1D3851, 0x14E2D3891, 0x14E3D38D1, 0x14E4D3911, 0x14E5D3951, 0x14E6D3991, 0x14E7D39D1, 0x14E8D3A11, 0x14E9D3A51, 0x14EAD3A91, 0x14EBD3AD1, 0x14ECD3B11, 0x14EDD3B51, 0x14EED3B91, 0x14EFD3BD1, 0x14F0D3C11, 0x14F1D3C51, 0x14F2D3C91, 0x14F3D3CD1, 0x14F4D3D11, 0x14F5D3D51, 0x14F6D3D91, 0x5A131, 0x14F780011, 0x14F8D3E11, 0x14F9D3E51, 0x14FAD3E91, 0x14FBD3ED1, 0x14FCD3F11, 0x14FDD3F51, 0x14FED3F91, 0x14FFD3FD1, 0x1500D4011, 0x1501D4051, 0x1502D4091, 0x1503D40D1, 0x1504D4111, 0x1505D4151, 0x1506D4191, 0x1507D41D1, 0x1508D4211, 0x1509D4251, 0x150AD4291, 0x150BD42D1, 0x150CD4311, 0x150DD4351, 0x150ED4391, 0x150FD43D1, 0x1510D4411, 0x1511D4451, 0x1512D4491, 0x1513D44D1, 0x1514D4511, 0x1515D4551, 0x1516D4591, 0x1517D45D1, 0x1518D4611, 0x1519D4651, 0x151AD4691, 0x151BD46D1, 0x151CD4711, 0x58DF1, 0x168500011, 0x168580011, 0x1686DA191, 0x1687DA1D1, 0x1688DA211, 0x1689DA251, 0x168ADA291, 0x168BDA2D1, 0x168CDA311, 0x168DDA351, 0x168EDA391, 0x168FDA3D1, 0x1690DA411, 0x1691DA451, 0x169200011, 0x16935A4B1, 0x16945A4F1, 0x16955A531, 0x16965A571, 0x16975A5B1, 0x16985A5F1, 0x16995A631, 0x169A5A671, 0x169B5A6B1, 0x169C5A6F1, 0x169D5A731, 0x154AD5291, 0x154BD52D1, 0x154CD5311, 0x154DD5351, 0x154ED5391, 0x154FD53D1, 0x1550D5411, 0x1551D5451, 0x1552D5491, 0x1553D54D1, 0x1554D5511, 0x1555D5551, 0x1556D5591, 0x1557D55D1, 0x1558D5611, 0x1559D5651, 0x155AD5691, 0x155BD56D1, 0x155CD5711, 0x155DD5751, 0x155ED5791, 0x155FD57D1, 0x1560D5811, 0x1561D5851, 0x169E5A771, 0x169F5A7B1, 0x16A05A7F1, 0x16A15A831, 0x1566D5991, 0x1567D59D1, 0x1568D5A11, 0x1569D5A51, 0x156AD5A91, 0x156BD5AD1, 0x156CD5B11, 0x156DD5B51, 0x156ED5B91, 0x156FD5BD1, 0x1570D5C11, 0x1571D5C51, 0x1572D5C91, 0x55CD1, 0x157380011, 0x1574D5D11, 0x1575D5D51, 0x1576D5D91, 0x1577D5DD1, 0x1578D5E11, 0x1579D5E51, 0x157AD5E91, 0x157BD5ED1, 0x16A25A871, 0x16A35A8B1, 0x16A45A8F1, 0x16A55A931, 0x16A65A971, 0x16A75A9B1, 0x16A85A9F1, 0x16A95AA31, 0x16AA5AA71, 0x16AB5AAB1, 0x16AC5AAF1, 0x16AD5AB31, 0x16AE5AB71, 0x16AED6571, 0x16AFDABD1, 0x5AC11, 0x1598D6611, 0x1599D6651, 0x159AD6691, 0x159BD66D1, 0x159CD6711, 0x159DD6751, 0x5AC31, 0x16B1DAC51, 0x16B2DAC91, 0x16B3DACD1, 0x16B4DAD11, 0x16B5DAD51, 0x16B6DAD91, 0x16B7DADD1, 0x16B8DAE11, 0x16B9DAE51, 0x16BADAE91, 0x16BBDAED1, 0x5AF11, 0x16ADDAF31, 0x5AB91, 0x56B91, 0x15AF56BB1, 0x16BDDAF51, 0x5AF91, 0x16BE80011, 0x16BFDAFD1, 0x5B011, 0x16C15B031, 0x16C180011, 0x16C200011, 0x16C35B0B1, 0x16C45B0F1, 0x16C55B131, 0x16C65B171, 0x16C75B1B1, 0x16C780011, 0x16C8DB211, 0x16C9DB251, 0x16CADB291, 0x15BAD6E91, 0x15BBD6ED1, 0x16CB56F11, 0x16CC5B2F1, 0x16CD5B331, 0x5B371, 0x15BF56FB1, 0x15C056FF1, 0x15C157031, 0x15C257071, 0x16CE570B1, 0x15C4570F1, 0x15C557131, 0x15C657171, 0x15C7571B1, 0x15C8571F1, 0x15C957231, 0x15CA57271, 0x15CB572B1, 0x15CC572F1, 0x15CD57331, 0x15CEDB3B1, 0x16CF573D1, 0x15D0573F1, 0x15D157431, 0x15D257471, 0x15D3574B1, 0x16CFD74F1, 0x16D0DB411, 0x16D1DB451, 0x16D2DB491, 0x16D3DB4D1, 0x15D957631, 0x15D9DB511, 0x15DA5B531, 0x15DA80011, 0x16D5DB551, 0x16D6DB591, 0x15DCDB5D1, 0x16D780011, 0x15DDDB611, 0x15DED7791, 0x16D95B631, 0x15DF5B671, 0x16DADB691, 0x16DBDB6D1, 0x16DCDB711, 0x16DDDB751, 0x15E15B791, 0x16DED7871, 0x16DFDB7D1, 0x16E0DB811, 0x15E4D7911, 0x15E5D7951, 0x15E6D7991, 0x15E7D79D1, 0x15E8D7A11, 0x15E9D7A51, 0x15EAD7A91, 0x15EBD7AD1, 0x15ECD7B11, 0x15EDD7B51, 0x15EED7B91, 0x16E157BD1, 0x16E25B871, 0x16E35B8B1, 0x16E45B8F1, 0x16E55B931, 0x15F457CF1, 0x16E5D7D31, 0x15F657D71, 0x15F757DB1, 0x15F857DF1, 0x15F957E31, 0x15FA57E71, 0x15FB57EB1, 0x15FC57EF1, 0x15FD57F31, 0x15FE57F71, 0x15FF57FB1, 0x160057FF1, 0x160158031, 0x16E6DB991, 0x16E7DB9D1, 0x16E8DBA11, 0x16E9DBA51, 0x16EADBA91, 0x16EBDBAD1, 0x16ECDBB11, 0x16EDDBB51, 0x16E6DBB91, 0x16EF5BBB1, 0x16F05BBF1, 0x16F15BC31, 0x16F25BC71, 0x16F35BCB1, 0x16F45BCF1, 0x16F55BD31, 0x16F580012, 0x16F6DBD92, 0x16F7DBDD2, 0x16F8DBE12, 0x16F9DBE52, 0x16FADBE92, 0x16FBDBED2, 0x16FCDBF12, 0x16FDDBF52, 0x16FEDBF92, 0x16FFDBFD2, 0x1700DC012, 0x1701DC052, 0x1702DC092, 0x170300012, 0x17045C0F2, 0x17055C132, 0x17065C172, 0x17075C1B2, 0x17085C1F2, 0x17095C232, 0x170A5C272, 0x170B5C2B2, 0x170C5C2F2, 0x170D5C332, 0x170E5C372, 0x170F5C3B2, 0x17105C3F2, 0x17115C432, 0x17125C472, 0x17135C4B2, 0x17145C4F2, 0x17155C532, 0x17165C572, 0x17175C5B2, 0x171780012, 0x1718DC612, 0x1719DC652, 0x171ADC692, 0x171BDC6D2, 0x171CDC712, 0x171DDC752, 0x171EDC792, 0x171FDC7D2, 0x1720DC812, 0x5C852, 0x17225C872, 0x17235C8B2, 0x17245C8F2, 0x17255C932, 0x17265C972, 0x172680012, 0x1727DC9D2, 0x172800012, 0x17295CA32, 0x172A5CA72, 0x172B5CAB2, 0x172C5CAF2, 0x172D5CB32, 0x172E5CB72, 0x172F5CBB2, 0x17305CBF2, 0x17315CC32, 0x17325CC72, 0x17335CCB2, 0x17345CCF2, 0x5CD32, 0x1735DCD52, 0x1736DCD92, 0x1737DCDD2, 0x1738DCE12, 0x1739DCE52, 0x173ADCE92, 0x173BDCED2, 0x173CDCF12, 0x173DDCF52, 0x173E00012, 0x173F5CFB2, 0x17405CFF2, 0x17415D032, 0x17425D072, 0x17435D0B2, 0x17445D0F2, 0x5D132, 0x1745DD152, 0x1746DD192, 0x1747DD1D2, 0x5D212, 0x174800012, 0x174880012, 0x5D252, 0x174A5D272, 0x174A80012, 0x174BDD2D2, 0x174CDD312, 0x174D00012, 0x174D80012, 0x174EDD392, 0x174F00012, 0x17505D3F2, 0x175080012, 0x1751DD452, 0x1752DD492, 0x1753DD4D2, 0x1754DD512, 0x1755DD552, 0x1756DD592, 0x1757DD5D2, 0x1758DD612, 0x1759DD652, 0x175ADD692, 0x175BDD6D2, 0x175CDD712, 0x175DDD752, 0x175EDD792, 0x175FDD7D2, 0x1760DD812, 0x1761DD852, 0x1762DD892, 0x1763DD8D2, 0x1764DD912, 0x1765DD952, 0x1766DD992, 0x1767DD9D2, 0x1768DDA12, 0x5DA52, 0x176A5DA72, 0x176B5DAB2, 0x176C5DAF2, 0x176D5DB32, 0x176E5DB72, 0x176F5DBB2, 0x17705DBF2, 0x17715DC32, 0x17725DC72, 0x17735DCB2, 0x17745DCF2, 0x17755DD32, 0x17765DD72, 0x17775DDB2, 0x17785DDF2, 0x17795DE32, 0x177A5DE72, 0x177B5DEB2, 0x177C5DEF2, 0x5DF32, 0x177DDDF52, 0x177EDDF92, 0x177FDDFD2, 0x1780DE012, 0x1781DE052, 0x1782DE092, 0x1783DE0D2, 0x1784DE112, 0x1785DE152, 0x1786DE192, 0x1787DE1D2, 0x1788DE212, 0x1789DE252, 0x178ADE292, 0x178BDE2D2, 0x178CDE312, 0x178DDE352, 0x178EDE392, 0x178FDE3D2, 0x1790DE412, 0x1791DE452, 0x1792DE492, 0x1793DE4D2, 0x1794DE512, 0x1795DE552, 0x1796DE592, 0x1797DE5D2, 0x1798DE612, 0x1799DE652, 0x179ADE692, 0x179BDE6D2, 0x179CDE712, 0x179DDE752, 0x179EDE792, 0x179F00012, 0x17A05E7F2, 0x17A15E832, 0x17A25E872, 0x17A35E8B2, 0x17A45E8F2, 0x17A55E932, 0x17A65E972, 0x17A75E9B2, 0x17A85E9F2, 0x17A95EA32, 0x17AA5EA72, 0x17AB5EAB2, 0x17AC5EAF2, 0x17AD5EB32, 0x17AE5EB72, 0x5EBB2, 0x17AFDEBD2, 0x17B0DEC12, 0x17B1DEC52, 0x17B2DEC92, 0x17AFDECD2, 0x17B45ECF2, 0x5ED32, 0x17B5DED52, 0x17B6DED92, 0x5EDD2, 0x17B780012, 0x17B8DEE12, 0x17B9DEE52, 0x5EE92, 0x17BB5EEB2, 0x17BC5EEF2, 0x17BD5EF32, 0x17BE5EF72, 0x17BF5EFB2, 0x17C05EFF2, 0x17C15F032, 0x17C25F072, 0x17C280012, 0x17C3DF0D2, 0x17C4DF112, 0x17C5DF152, 0x17C6DF192, 0x17C7DF1D2, 0x17C8DF212, 0x17C9DF252, 0x17CADF292, 0x17CBDF2D2, 0x17CCDF312, 0x17CDDF352, 0x17CEDF392, 0x17CFDF3D2, 0x5F412, 0x17D15F432, 0x17D25F472, 0x17D35F4B2, 0x17D45F4F2, 0x17D55F532, 0x17D65F572, 0x17D75F5B2, 0x17D85F5F2, 0x17D95F632, 0x17DA5F672, 0x17DB5F6B2, 0x17DC5F6F2, 0x17DD5F732, 0x17DE5F772, 0x17DF5F7B2, 0x17E05F7F2, 0x17E15F832, 0x17E25F872, 0x17E35F8B2, 0x17E45F8F2, 0x17E55F932, 0x17E65F972, 0x17E75F9B2, 0x17E85F9F2, 0x17E95FA32, 0x17EA5FA72, 0x17EB5FAB2, 0x17EC5FAF2, 0x17E95FB32, 0x17ED00012, 0x17EE5FB72, 0x17EF5FBB2, 0x17F05FBF2, 0x17F15FC33, 0x17F25FC73, 0x17F35FCB3, 0x17F45FCF3, 0x17F55FD33, 0x17F65FD73, 0x17F75FDB3, 0x17F780013, 0x17F800013, 0x17F95FE33, 0x17FA5FE73, 0x17FB5FEB3, 0x17FC5FEF3, 0x5FF33, 0x17FDDFF53, 0x17FEDFF93, 0x17FFDFFD3, 0x1800E0013, 0x1801E0053, 0x1802E0093, 0x1803E00D3, 0x1804E0113, 0x1805E0153, 0x1806E0193, 0x1807E01D3, 0x1808E0213, 0x1809E0253, 0x180A00013, 0x180B602B3, 0x180C602F3, 0x180D60333, 0x180E60373, 0x180F603B3, 0x1810603F3, 0x181160433, 0x181260473, 0x1813604B3, 0x1814604F3, 0x181560533, 0x181660573, 0x1817605B3, 0x1818605F3, 0x181960633, 0x181A60673, 0x181A80013, 0x181BE06D3, 0x60713, 0x181C80013, 0x181DE0753, 0x181EE0793, 0x181FE07D3, 0x1820E0813, 0x1821E0853, 0x1822E0893, 0x182560933, 0x1829E0A53, 0x182AE0A93, 0x182BE0AD3, 0x182CE0B13, 0x182DE0B53, 0x182EE0B93, 0x182FE0BD3, 0x1830E0C13, 0x1831E0C53, 0x1832E0C93, 0x1833E0CD3, 0x1834E0D13, 0x1835E0D53, 0x1836E0D93, 0x1837E0DD3, 0x1838E0E13, 0x60E53, 0x60E73, 0x183AE0E93, 0x183BE0ED3, 0x183CE0F13, 0x183D80013, 0x183EE0F93, 0x183FE0FD3, 0x1840E1013, 0x1841E1053, 0x1842E1093, 0x1843E10D3, 0x1844E1113, 0x1845E1153, 0x1846E1193, 0x1847E11D3, 0x1848E1213, 0x1849E1253, 0x184AE1293, 0x184BE12D3, 0x184CE1313, 0x184DE1353, 0x184EE1393, 0x184FE13D3, 0x1850E1413, 0x1851E1453, 0x1852E1493, 0x1853E14D3, 0x1854E1513, 0x1853E1553, 0x185661573, 0x1857615B3, 0x1858615F3, 0x185961633, 0x185A61673, 0x185B616B3, 0x185C616F3, 0x185D61733, 0x185E61773, 0x185F617B3, 0x1860617F3, 0x186161833, 0x186261873, 0x1863618B3, 0x1864618F3, 0x186561933, 0x186661973, 0x1867619B3, 0x1868619F3, 0x186961A33, 0x186A61A73, 0x186B61AB3, 0x186C61AF3, 0x186D61B33, 0x186E61B73, 0x186F61BB3, 0x187061BF3, 0x187161C33, 0x187261C73, 0x187361CB3, 0x187261CF3, 0x1874E1D13, 0x1875E1D53, 0x1876E1D94, 0x1877E1DD4, 0x1878E1E14, 0x61E54, 0x61E74, 0x187A00014, 0x187B61EB4, 0x187C61EF4, 0x187D61F34, 0x187E61F74, 0x187F61FB4, 0x188061FF4, 0x188080014, 0x1881E2054, 0x188200014, 0x1883620B4, 0x1884620F4, 0x188562134, 0x188662174, 0x1887621B4, 0x1888621F4, 0x188962234, 0x1888E2274, 0x1889E2254, 0x188A00014, 0x188B622B4, 0x622F4, 0x188C00014, 0x188D00014, 0x188E62374, 0x188F623B4, 0x1890623F4, 0x182362434, 0x1824608F4, 0x1825E2454, 0x1826E0994, 0x1827E09D4, 0x1828E0A14, 0x189262474, 0x1893624B4, 0x1894624F4, 0x189562534, 0x189662574, 0x1897625B4, 0x1898625F4, 0x189962634, 0x189980014, 0x62694, 0x189B626B4, 0x626F4, 0x187760F54, 0x187861DF4, 0x187961E34, 0x1882626F4, 0x189C00014, 0x189D62734, 0x189CE2714, 0x189DE2754, 0x189EE2794, 0x189FE27D4, 0x18A0E2814, 0x18A1E2854, 0x18A2E2894, 0x18A3E28D4, 0x18A4E2914, 0x18A4E2954, 0x18A462954, 0x18A662974, 0x18A7629B4, 0x18A8629F4, 0x18A8E29F4, 0x62A54, 0x18AA62A74, 0x18AB62AB4, 0x18AC62AF4, 0x18AD62B34, 0x18AE62B74, 0x18AF62BB4, 0x18B062BF4, 0x18B162C34, 0x18B262C74, 0x18B362CB4, 0x18B462CF4, 0x18B562D34, 0x18B662D74, 0x18B762DB4, 0x18B762DF4, 0x18B8E2E14, 0x18B900015, 0x62E55, 0x18BA62E75, 0x18BB62EB5, 0x18BB80015, 0x18BCE2F15, 0x18BDE2F55, 0x62F95, 0x18BE80015, 0x18BFE2FD5, 0x18C0E3015, 0x18C1E3055, 0x18C2E3095, 0x18C3E30D5, 0x18C4E3115, 0x18C5E3155, 0x63195, 0x18C680015, 0x18C762335, 0x631F5, 0x18C8E3215, 0x18C9E3255, 0x18CAE3295, 0x18CBE32D5, 0x18CCE3315, 0x18CDE3355, 0x18CEE3395, 0x18CFE33D5, 0x18D000015, 0x63435, 0x18D1E3455, 0x18D2E3495, 0x634D5, 0x18D3E3455, 0x63035, 0x18D400015, 0x18D563535, 0x18D663575, 0x18D7635B5, 0x18D8635F5, 0x18D963635, 0x18DA63675, 0x18DB636B5, 0x18DC636F5, 0x63735, 0x18DD00015, 0x18DE63775, 0x18DF637B5, 0x18E0637F5, 0x18E163835, 0x18E263875, 0x18E3638B5, 0x18E4638F5, 0x18E5E3956, 0x18E600016, 0x639B6, 0x18E7E39D6, 0x18E8E3A16, 0x18E900016, 0x63A76, 0x18EA00016, 0x18EB63AB6, 0x18EC63AF6, 0x18ED63B36, 0x18ED80016, 0x18EEE3B96, 0x18EFE3BD6, 0x18F0E3C16, 0x18F1E3C56, 0x18F2E3C96, 0x18F300016, 0x63CF6, 0x18F400016, 0x18F563D36, 0x18F663D76, 0x18F680016, 0x18F7E3DD6, 0x18F8E3E16, 0x18F9E3E56, 0x18FAE3E96, 0x63ED6, 0x18FB80016, 0x18FCE3F16, 0x18FDE3F56, 0x18FEE3F96, 0x63FD6, 0x63936, 0x18FF80017, 0x1900E4017, 0x1901E4057, 0x1902E4097, 0x190300017, 0x1904640F7, 0x64137, 0x1905E4157, 0x1906E4197, 0x1907E41D7, 0x1908E4217, 0x64257, 0x64277, 0x64297, 0x190B642B7, 0x190C642F7, 0x190D64337, 0x190E64377, 0x190F643B7, 0x1910643F7, 0x64438, 0x64458, 0x191264478, 0x1913644B8, 0x191380018, 0x64518, 0x191564538, 0x191664578, 0x645B8, 0x191700018, 0x1918645F8, 0x191964638, 0x191A64678, 0x646B9, 0x646D9, 0x646F9, 0x191C00019, 0x191C80019, 0x191D00019, 0x191E64779, 0x191F647B9, 0x191F80019, 0x1920E4819, 0x6485A, 0x19218001A, 0x6489A, 0x648BA, 0x1923E48DA, 0x6491A, 0x19248001A, 0x1925E495A, 0x6499B, 0x1927649BB, 0x1928649FB, 0x192964A3B, 0x19298001B, 0x64A9C, 0x192A8001C, 0x192B0001C, 0x192C64AFC, 0x64B3D, 0x64B5D, 0x192E64B7D, 0x192F64BBE, 0x192F8001E, 0x1930E4C1F, }; + } +} diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Unicode/IgnoreCaseRelationGenerator.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Unicode/IgnoreCaseRelationGenerator.cs new file mode 100644 index 00000000000000..22375801f442b6 --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Unicode/IgnoreCaseRelationGenerator.cs @@ -0,0 +1,112 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.IO; + +namespace System.Text.RegularExpressions.Symbolic.Unicode +{ +#if DEBUG + internal static class IgnoreCaseRelationGenerator + { + private const string DefaultCultureName = "en-US"; + + public static void Generate(string namespacename, string classname, string path) + { + Debug.Assert(namespacename != null); + Debug.Assert(classname != null); + Debug.Assert(path != null); + + using StreamWriter sw = new StreamWriter($"{Path.Combine(path, classname)}.cs"); + sw.WriteLine( +$@"// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// This is a programmatically generated file from Regex.GenerateUnicodeTables. +// It provides serialized BDD Unicode category definitions for System.Environment.Version = {Environment.Version} + +namespace {namespacename} +{{ + internal static class {classname} + {{"); + WriteIgnoreCaseBDD(sw); + sw.WriteLine($@" }} +}}"); + } + + private static void WriteIgnoreCaseBDD(StreamWriter sw) + { + sw.WriteLine(" /// Serialized BDD for mapping characters to their case-ignoring equivalence classes in the default (en-US) culture."); + + var solver = new CharSetSolver(); + Dictionary ignoreCase = ComputeIgnoreCaseDictionary(solver, new CultureInfo(DefaultCultureName)); + BDD ignorecase = solver.False; + foreach (KeyValuePair kv in ignoreCase) + { + BDD a = solver.CreateCharSetFromRange(kv.Key, kv.Key); + BDD b = kv.Value; + ignorecase = solver.Or(ignorecase, solver.And(solver.ShiftLeft(a, 16), b)); + } + + sw.Write(" public static readonly long[] IgnoreCaseEnUsSerializedBDD = "); + GeneratorHelper.WriteInt64ArrayInitSyntax(sw, ignorecase.Serialize()); + sw.WriteLine(";"); + } + + private static Dictionary ComputeIgnoreCaseDictionary(CharSetSolver solver, CultureInfo culture) + { + CultureInfo originalCulture = CultureInfo.CurrentCulture; + try + { + CultureInfo.CurrentCulture = culture; + + var ignoreCase = new Dictionary(); + + for (uint i = 0; i <= 0xFFFF; i++) + { + char c = (char)i; + char cUpper = char.ToUpper(c); + char cLower = char.ToLower(c); + + if (cUpper == cLower) + { + continue; + } + + // c may be different from both cUpper as well as cLower. + // Make sure that the regex engine considers c as being equivalent to cUpper and cLower, else ignore c. + // In some cases c != cU but the regex engine does not consider the chacarters equivalent wrt the ignore-case option. + if (Regex.IsMatch($"{cUpper}{cLower}", $"^(?i:\\u{i:X4}\\u{i:X4})$")) + { + BDD equiv = solver.False; + + if (ignoreCase.ContainsKey(c)) + equiv = solver.Or(equiv, ignoreCase[c]); + + if (ignoreCase.ContainsKey(cUpper)) + equiv = solver.Or(equiv, ignoreCase[cUpper]); + + if (ignoreCase.ContainsKey(cLower)) + equiv = solver.Or(equiv, ignoreCase[cLower]); + + // Make sure all characters are included initially or when some is still missing + equiv = solver.Or(equiv, solver.Or(solver.CreateCharSetFromRange(c, c), solver.Or(solver.CreateCharSetFromRange(cUpper, cUpper), solver.CreateCharSetFromRange(cLower, cLower)))); + + // Update all the members with their case-invariance equivalence classes + foreach (char d in solver.GenerateAllCharacters(equiv)) + ignoreCase[d] = equiv; + } + } + + return ignoreCase; + } + finally + { + CultureInfo.CurrentCulture = originalCulture; + } + } + }; +#endif +} diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Unicode/IgnoreCaseTransformer.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Unicode/IgnoreCaseTransformer.cs new file mode 100644 index 00000000000000..945fdeae5cffa5 --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Unicode/IgnoreCaseTransformer.cs @@ -0,0 +1,237 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.Threading; + +namespace System.Text.RegularExpressions.Symbolic.Unicode +{ + internal sealed class IgnoreCaseTransformer + { + private const char Turkish_I_WithDot = '\u0130'; + private const char Turkish_i_WithoutDot = '\u0131'; + private const char KelvinSign = '\u212A'; + + private readonly CharSetSolver _solver; + private readonly BDD _i_Invariant; + private readonly BDD _i_Default; + private readonly BDD _i_Turkish; + private readonly BDD _I_Turkish; + + private volatile IgnoreCaseRelation? _relationDefault; + private volatile IgnoreCaseRelation? _relationInvariant; + private volatile IgnoreCaseRelation? _relationTurkish; + + /// Maps each char c to the case-insensitive set of c that is culture-independent (for non-null entries). + private readonly BDD?[] _cultureIndependentChars = new BDD[char.MaxValue + 1]; + + private sealed class IgnoreCaseRelation + { + public IgnoreCaseRelation(BDD instance, BDD instanceDomain) + { + Instance = instance; + InstanceDomain = instanceDomain; + } + + public BDD Instance { get; } + public BDD InstanceDomain { get; } + } + + public IgnoreCaseTransformer(CharSetSolver solver) + { + _solver = solver; + _i_Invariant = solver.Or(_solver.CharConstraint('i'), solver.CharConstraint('I')); + _i_Default = solver.Or(_i_Invariant, solver.CharConstraint(Turkish_I_WithDot)); + _i_Turkish = solver.Or(solver.CharConstraint('i'), solver.CharConstraint(Turkish_I_WithDot)); + _I_Turkish = solver.Or(solver.CharConstraint('I'), solver.CharConstraint(Turkish_i_WithoutDot)); + } + + /// + /// Get the set of CI-equivalent characters to c. + /// This operation depends on culture for i, I, '\u0130', and '\u0131'; + /// culture="" means InvariantCulture while culture=null means to use the current culture. + /// + public BDD Apply(char c, string? culture = null) + { + if (Volatile.Read(ref _cultureIndependentChars[c]) is BDD bdd) + { + return bdd; + } + + culture ??= CultureInfo.CurrentCulture.Name; + switch (c) + { + // Do not cache in _cultureIndependentChars values that are culture-dependent + + case 'i': + return + culture == string.Empty ? _i_Invariant : + IsTurkishAlphabet(culture) ? _i_Turkish : + _i_Default; // for all other cultures, case-sensitivity is the same as for en-US + + case 'I': + return + culture == string.Empty ? _i_Invariant : + IsTurkishAlphabet(culture) ? _I_Turkish : // different from 'i' above + _i_Default; + + case Turkish_I_WithDot: + return + culture == string.Empty ? _solver.CharConstraint(Turkish_I_WithDot) : + IsTurkishAlphabet(culture) ? _i_Turkish : + _i_Default; + + case Turkish_i_WithoutDot: + return + IsTurkishAlphabet(culture) ? _I_Turkish : + _solver.CharConstraint(Turkish_i_WithoutDot); + + case 'k': + case 'K': + case KelvinSign: + Volatile.Write(ref _cultureIndependentChars[c], _solver.Or(_solver.Or(_solver.CharConstraint('k'), _solver.CharConstraint('K')), _solver.CharConstraint(KelvinSign))); + return _cultureIndependentChars[c]!; + + // Cache in _cultureIndependentChars entries that are culture-independent. + // BDDs are idempotent, so while we use volatile to ensure proper adherence + // to ECMA's memory model, we don't need Interlocked.CompareExchange. + + case <= '\x7F': + // For ASCII range other than letters i, I, k, and K, the case-conversion is independent of culture and does + // not include case-insensitive-equivalent non-ASCII. + Volatile.Write(ref _cultureIndependentChars[c], _solver.Or(_solver.CharConstraint(char.ToLower(c)), _solver.CharConstraint(char.ToUpper(c)))); + return _cultureIndependentChars[c]!; + + default: + // Bring in the full transfomation relation, but here it does not actually depend on culture + // so it is safe to store the result for c. + Volatile.Write(ref _cultureIndependentChars[c], Apply(_solver.CharConstraint(c))); + return _cultureIndependentChars[c]!; + } + } + + /// + /// For all letters in the bdd add their lower and upper case equivalents. + /// This operation depends on culture for i, I, '\u0130', and '\u0131'; + /// culture="" means InvariantCulture while culture=null means to use the current culture. + /// + public BDD Apply(BDD bdd, string? culture = null) + { + // First get the culture specific relation + IgnoreCaseRelation relation = GetIgnoreCaseRelation(culture); + + if (_solver.And(relation.InstanceDomain, bdd).IsEmpty) + { + // No elements need to be added + return bdd; + } + + // Compute the set of all characters that are equivalent to some element in bdd. + // restr is the relation restricted to the relevant characters in bdd. + // This conjunction works because bdd is unspecified for bits > 15. + BDD restr = _solver.And(bdd, relation.Instance); + + // ShiftRight essentially produces the LHS of the relation (char X char) that restr represents. + BDD ignorecase = _solver.ShiftRight(restr, 16); + + // The final set is the union of all the characters. + return _solver.Or(ignorecase, bdd); + } + + /// Gets the transformation relation based on the current culture. + /// culture == "" means InvariantCulture. culture == null means to use the current culture. + private IgnoreCaseRelation GetIgnoreCaseRelation(string? culture = null) + { + culture ??= CultureInfo.CurrentCulture.Name; + + if (culture == string.Empty) + { + return _relationInvariant ?? CreateIgnoreCaseRelationInvariant(); + } + + if (IsTurkishAlphabet(culture)) + { + return _relationTurkish ?? CreateIgnoreCaseRelationTurkish(); + } + + // All other cultures are equivalent to the default culture wrt case-sensitivity. + return _relationDefault ?? EnsureDefault(); + } + + [MemberNotNull(nameof(_relationDefault))] + private IgnoreCaseRelation EnsureDefault() + { + // Deserialize the table for the default culture. + if (_relationDefault is null) + { + BDD instance = BDD.Deserialize(Unicode.IgnoreCaseRelation.IgnoreCaseEnUsSerializedBDD, _solver); + BDD instanceDomain = _solver.ShiftRight(instance, 16); // represents the set of all case-sensitive characters in the default culture. + _relationDefault = new IgnoreCaseRelation(instance, instanceDomain); + } + + return _relationDefault; + } + + private IgnoreCaseRelation CreateIgnoreCaseRelationInvariant() + { + EnsureDefault(); + + // Compute the invariant table based off of default. + // In the default (en-US) culture: Turkish_I_withDot = i = I + // In the invariant culture: i = I, while Turkish_I_withDot is case-insensitive + BDD tr_I_withdot_BDD = _solver.CharConstraint(Turkish_I_WithDot); + + // Since Turkish_I_withDot is case-insensitive in invariant culture, remove it from the default (en-US culture) table. + BDD inv_table = _solver.And(_relationDefault.Instance, _solver.Not(tr_I_withdot_BDD)); + + // Next, remove Turkish_I_withDot from the RHS of the relation. + // This also effectively removes Turkish_I_withDot from the equivalence sets of 'i' and 'I'. + BDD instance = _solver.And(inv_table, _solver.Not(_solver.ShiftLeft(tr_I_withdot_BDD, 16))); + + // Remove Turkish_I_withDot from the domain of casesensitive characters in the default case + BDD instanceDomain = _solver.And(instance, _solver.Not(tr_I_withdot_BDD)); + + _relationInvariant = new IgnoreCaseRelation(instance, instanceDomain); + return _relationInvariant; + } + + private IgnoreCaseRelation CreateIgnoreCaseRelationTurkish() + { + EnsureDefault(); + + // Compute the tr table based off of default. + // In the default (en-US) culture: Turkish_I_withDot = i = I + // In the tr culture: i = Turkish_I_withDot, I = Turkish_i_withoutDot + BDD tr_I_withdot_BDD = _solver.CharConstraint(Turkish_I_WithDot); + BDD tr_i_withoutdot_BDD = _solver.CharConstraint(Turkish_i_WithoutDot); + BDD i_BDD = _solver.CharConstraint('i'); + BDD I_BDD = _solver.CharConstraint('I'); + + // First remove all i's from the default table from the LHS and from the RHS. + // Note that Turkish_i_withoutDot is not in the default table because it is case-insensitive in the en-US culture. + BDD iDefault = _solver.Or(i_BDD, _solver.Or(I_BDD, tr_I_withdot_BDD)); + BDD tr_table = _solver.And(_relationDefault.Instance, _solver.Not(iDefault)); + tr_table = _solver.And(tr_table, _solver.Not(_solver.ShiftLeft(iDefault, 16))); + + BDD i_tr = _solver.Or(i_BDD, tr_I_withdot_BDD); + BDD I_tr = _solver.Or(I_BDD, tr_i_withoutdot_BDD); + + // The Cartesian product i_tr X i_tr. + BDD i_trXi_tr = _solver.And(_solver.ShiftLeft(i_tr, 16), i_tr); + + // The Cartesian product I_tr X I_tr. + BDD I_trXI_tr = _solver.And(_solver.ShiftLeft(I_tr, 16), I_tr); + + // Update the table with the new entries, and add Turkish_i_withoutDot also into the domain of case-sensitive characters. + BDD instance = _solver.Or(tr_table, _solver.Or(i_trXi_tr, I_trXI_tr)); + BDD instanceDomain = _solver.Or(_relationDefault.InstanceDomain, tr_i_withoutdot_BDD); + + _relationTurkish = new IgnoreCaseRelation(instance, instanceDomain); + return _relationTurkish; + } + + private static bool IsTurkishAlphabet(string culture) => + culture is "az" or "az-Cyrl" or "az-Cyrl-AZ" or "az-Latn" or "az-Latn-AZ" or "tr" or "tr-CY" or "tr-TR"; + } +} diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Unicode/UnicodeCategoryRanges.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Unicode/UnicodeCategoryRanges.cs new file mode 100644 index 00000000000000..caf22c757fd22b --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Unicode/UnicodeCategoryRanges.cs @@ -0,0 +1,82 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// This is a programmatically generated file from Regex.GenerateUnicodeTables. +// It provides serialized BDD Unicode category definitions for System.Environment.Version = 7.0.0 + +namespace System.Text.RegularExpressions.Symbolic.Unicode +{ + internal static class UnicodeCategoryRanges + { + /// Serialized BDD representations of all the Unicode categories. + public static readonly long[][] AllCategoriesSerializedBDD = new long[][] + { + // UppercaseLetter(0): + new long[] { 0x4, 0x9, 0x2000, 0x10, 0x2021, 0x6011, 0x6001, 0x21, 0x4011, 0x31, 0x6021, 0x4001, 0x11, 0x4031, 0x2001, 0x2031, 0x8002, 0xA012, 0x62, 0xC002, 0xE012, 0x2022, 0x10042, 0x4072, 0x12062, 0x60A2, 0xE022, 0x4002, 0x40B2, 0xE002, 0xC2, 0x4062, 0x6052, 0x100D2, 0x2042, 0x12002, 0x2002, 0x100E2, 0x22, 0xA0E2, 0x8062, 0x120E2, 0xE092, 0x20B2, 0x20E2, 0x72, 0x8052, 0x20F2, 0x6062, 0x1E032, 0x16002, 0x14052, 0xD2, 0x180B2, 0x92, 0x1E012, 0x8082, 0x6042, 0x180A2, 0xC082, 0x6072, 0xE032, 0xA032, 0x4082, 0x160F2, 0x1E082, 0xA0C2, 0x14082, 0x1E062, 0x2103, 0x22013, 0x24003, 0x28133, 0x2A023, 0x4163, 0x2E023, 0x4183, 0x193, 0x34023, 0x41B3, 0x41C3, 0x3A003, 0x36183, 0x3E1E3, 0x42203, 0x2223, 0x46003, 0x481E3, 0x4C253, 0x50273, 0x54293, 0x2B3, 0x243, 0x13, 0x33, 0x2C3, 0x382D3, 0x22E3, 0x58003, 0x5E123, 0x44003, 0x62303, 0x641A3, 0x68333, 0x6A023, 0x363, 0x6E243, 0x5A383, 0x2E363, 0x72023, 0x363A3, 0x76023, 0x63C3, 0x683D3, 0x7C0B3, 0x7E3B3, 0x82403, 0x86423, 0x4443, 0x61A3, 0x4033, 0x8C454, 0x474, 0x90004, 0x94494, 0x984B4, 0x44D4, 0x44F4, 0xA0004, 0xA4514, 0xA8534, 0xAA004, 0xAC004, 0x8E004, 0xB0574, 0xB4594, 0xB65C4, 0x5C4, 0xBA004, 0xBE5E4, 0xBA5F4, 0x4604, 0x2614, 0x14, 0x2624, 0xC6004, 0x2644, 0x4464, 0xCA024, 0xCC024, 0x2004, 0x4674, 0xD2684, 0xD4004, 0xD6464, 0x6C4, 0xDA004, 0x46E4, 0x46F4, 0xE2704, 0xE6724, 0xEA744, 0x4764, 0xEE024, 0x4784, 0x2224, 0x795, 0xF67A5, 0xFA7C5, 0x47E5, 0x4E5, 0xFE005, 0x4805, 0x4815, 0x825, 0x2835, 0x108005, 0x10A005, 0x10E865, 0xB8885, 0x895, 0x1168A5, 0x1185D5, 0x11A025, 0x11E8E5, 0x2905, 0x15, 0x122005, 0x124005, 0x4935, 0x128025, 0x12A025, 0x25, 0x2965, 0x130975, 0x134995, 0x9B5, 0x138005, 0x49D5, 0x13E9E5, 0x142A05, 0x146A25, 0x4A45, 0x14A005, 0xF2005, 0x14C006, 0x150A76, 0x154A96, 0x156006, 0xAC6, 0x4AD6, 0x15EAE6, 0x160006, 0x164B16, 0x168B36, 0x16CB56, 0x16E026, 0x170006, 0x174B96, 0x174BB6, 0x17ABC6, 0x17EBE6, 0x182C06, 0x186C26, 0xC46, 0x18CC56, 0x190C76, 0x194C96, 0xCB6, 0xCC6, 0x19A007, 0x19ECE7, 0x1A2D07, 0x1A6D27, 0x1AAD47, 0x1AED67, 0x4D87, 0xD97, 0xDA7, 0xDB7, 0x1B8007, 0x1BCDD7, 0x1C0DF7, 0x1C2007, 0x1C6E27, 0x1CAE47, 0xE68, 0x1D0E78, 0x1D2008, 0xEA8, 0x1D8EB8, 0x1DA008, 0xEE8, 0x1DE008, 0x1E2F08, 0x1E6F28, 0x1EAF48, 0xF69, 0xF79, 0x1F0009, 0x1F2009, 0x1F6FA9, 0x1FAFC9, 0x1FC009, 0x200FF9, 0x101A, 0x102A, 0x103A, 0x20800A, 0x105A, 0x20C00A, 0x21107A, 0x109B, 0x21400B, 0x2190BB, 0x21D0DB, 0x21E00B, 0x110C, 0x22200C, 0x22400C, 0x22913C, 0x115D, 0x116D, 0x23117D, 0x23519E, 0x23600E, 0x23B1CF, }, + // LowercaseLetter(1): + new long[] { 0x4, 0x9, 0x2000, 0x10, 0x2021, 0x6011, 0x31, 0x4001, 0x21, 0x6001, 0x2031, 0x4031, 0x4011, 0x6021, 0x2001, 0x11, 0x8002, 0xA012, 0xC012, 0x2042, 0xE002, 0x10002, 0x82, 0x4002, 0xC002, 0x32, 0x12062, 0x60A2, 0x22, 0x40B2, 0x14032, 0xC032, 0x6002, 0x6092, 0x92, 0x20A2, 0x6072, 0x2002, 0x60C2, 0x4072, 0x120D2, 0x72, 0x1C082, 0xF2, 0x120E2, 0x62, 0x100F2, 0x1E0C2, 0x20F2, 0x20C2, 0x20E2, 0x6052, 0x8052, 0x40C2, 0x10022, 0x12002, 0x16072, 0x2092, 0x1C052, 0x12, 0xC0E2, 0x6062, 0x2052, 0x4062, 0x1C0B2, 0x6012, 0x16092, 0x40A2, 0x12022, 0x1A052, 0xE022, 0x18092, 0xA082, 0x10092, 0xE0E2, 0x100C2, 0x2103, 0x22013, 0x24003, 0x26003, 0x2143, 0x2C153, 0x30173, 0x32033, 0x61A3, 0x36033, 0x61C3, 0x221D3, 0x3C033, 0x3E033, 0x6203, 0x6213, 0x46223, 0x30003, 0x481C3, 0x4C253, 0x50273, 0x2133, 0x52003, 0x562A3, 0x522C3, 0x2D3, 0x2E3, 0x5E003, 0x2003, 0x60003, 0x62163, 0x2323, 0x64003, 0x66233, 0x2113, 0x123, 0x2253, 0x323, 0x2343, 0x6C353, 0x6E1F3, 0x72383, 0x74033, 0x3B3, 0x28003, 0x7A3C3, 0x7C013, 0x7E033, 0x82403, 0x84033, 0x4433, 0x2C443, 0x8C453, 0x42473, 0x92483, 0x844A3, 0x64B3, 0x41E3, 0x6023, 0x26013, 0x44003, 0x9A4C4, 0x9E4E4, 0xA0014, 0x24C4, 0x14, 0x514, 0xA4004, 0xA8534, 0xAC554, 0x6574, 0xB2584, 0x65A4, 0xB6004, 0xB8004, 0xBC5D4, 0xC05F4, 0x2614, 0xC4004, 0xC6004, 0xCA644, 0xCC5D4, 0xD0674, 0xCE694, 0xD06A4, 0xD06B4, 0xD8684, 0xD06C4, 0x66D4, 0xDE6E4, 0xE0004, 0xA0004, 0x714, 0x2724, 0x6004, 0xE6034, 0xE8034, 0x6754, 0xC2764, 0xEE014, 0x784, 0x794, 0x2004, 0x27A4, 0xF6014, 0x67C4, 0x67D4, 0xFE7E4, 0x102804, 0x106824, 0x6844, 0x10A034, 0x6864, 0x2874, 0x654, 0x884, 0x112005, 0x114005, 0x1188B5, 0x8D5, 0x11E8E5, 0x122905, 0x6925, 0x935, 0x128005, 0x6955, 0x2965, 0x6975, 0x132985, 0x134005, 0x136005, 0x13A9C5, 0x13E9E5, 0xD0A05, 0x144A15, 0x146685, 0x148035, 0x118005, 0xA55, 0x2A65, 0x14E005, 0xA85, 0x11AA95, 0x15, 0x6AA5, 0x156035, 0x158035, 0x2035, 0x15CAD5, 0x160AF5, 0xB15, 0x166B25, 0x168015, 0x6B55, 0x16EB65, 0x172B85, 0x176BA5, 0x6BC5, 0x17CBD5, 0xBF5, 0x895, 0xC06, 0x182006, 0x2006, 0x186C26, 0x18AC46, 0x18EC66, 0x190006, 0xC96, 0x194006, 0x6CB6, 0x186CC6, 0x19A006, 0x19ECE6, 0x1A2D06, 0x1A6D26, 0x1A8036, 0x1AA006, 0x1AED66, 0x1B0006, 0xD96, 0xDA6, 0x1B8DB6, 0x1BCDD6, 0x186DF6, 0x1C2E06, 0xE26, 0x1C6006, 0x1CAE46, 0x1CEE66, 0x1D2E86, 0x1D6EA6, 0xEC6, 0x1DA007, 0x1DC007, 0x1E0EF7, 0x1E4F17, 0x1E8F37, 0x1EA007, 0x1EEF67, 0x1F2F87, 0x1F6FA7, 0x6FC7, 0x1FCFD7, 0xFF7, 0x1007, 0x1017, 0x204FF7, 0x209037, 0x20D057, 0x211077, 0x215097, 0x2190B7, 0x10D8, 0x10E8, 0x10F8, 0x223108, 0x227128, 0x1148, 0x22D158, 0x231178, 0x1198, 0x234008, 0x2391B8, 0x23D1D8, 0x2411F8, 0x1219, 0x1229, 0x1239, 0x1249, 0x24A009, 0x24C009, 0x251279, 0x255299, 0x256009, 0x25B2C9, 0x25F2EA, 0x26000A, 0x131A, 0x132A, 0x26600A, 0x134A, 0x26A00A, 0x26F36A, 0x138B, 0x27539B, 0x2793BB, 0x27D3DB, 0x27E00B, 0x140C, 0x28200C, 0x28400C, 0x28943C, 0x145D, 0x146D, 0x29147D, 0x29549E, 0x29600E, 0x29B4CF, }, + // TitlecaseLetter(2): + new long[] { 0x4, 0x6, 0x400, 0x10, 0x801, 0x21, 0x831, 0xC01, 0x42, 0x1402, 0x1802, 0x72, 0x83, 0x13, 0x2403, 0x2CA3, 0xC4, 0x3004, 0x34C4, 0xE4, 0x3C04, 0x4505, 0x3525, 0x5135, 0x5956, 0x176, 0x187, 0x197, 0x1A8, 0x1B8, 0x1C9, 0x7409, 0x1EA, 0x7C0A, 0x20B, 0x840B, 0x8E2C, 0x900D, 0x940E, 0x980F, }, + // ModifierLetter(3): + new long[] { 0x4, 0x9, 0x2000, 0x10, 0x11, 0x4001, 0x6001, 0x31, 0x21, 0x2001, 0x2021, 0x6021, 0x6011, 0x42, 0xA002, 0xC002, 0x12, 0xE052, 0x62, 0x82, 0x72, 0x12002, 0x92, 0x52, 0x2092, 0xA2, 0xB2, 0xE002, 0x18092, 0x2052, 0xE012, 0x14002, 0x10002, 0x22, 0x12042, 0xD3, 0x1C003, 0xF3, 0x103, 0x22003, 0x123, 0x26003, 0x143, 0x153, 0xE3, 0x28013, 0x163, 0x173, 0x183, 0x24003, 0x193, 0x1A3, 0x381B3, 0x21D3, 0x1E143, 0x1E3, 0x21F3, 0x28003, 0x36003, 0x1E003, 0x2E0E3, 0x203, 0x2C203, 0x34003, 0x2E203, 0x3A213, 0x2A003, 0x44013, 0x234, 0x244, 0x4A004, 0x264, 0x274, 0x284, 0x52004, 0x54004, 0x2B4, 0x58004, 0x2D4, 0x2E4, 0x2A4, 0x5E004, 0x304, 0x314, 0x324, 0x334, 0x344, 0x62004, 0x354, 0x364, 0x374, 0x702C4, 0x4C014, 0x72004, 0x74004, 0x394, 0x2F4, 0x3B4, 0x78004, 0x3D4, 0x3E4, 0x7E004, 0x48004, 0x254, 0x404, 0x82004, 0x86424, 0x14, 0x88005, 0x455, 0x8E465, 0x92485, 0x964A5, 0x4C5, 0x9A005, 0x9C005, 0x9E005, 0x505, 0xA2005, 0x525, 0xA6005, 0x545, 0xAA005, 0xAE565, 0x4B5, 0x4F5, 0xB0005, 0x595, 0xB4015, 0x25B5, 0x5C5, 0x5D5, 0xBC005, 0xBE005, 0x605, 0x94005, 0x615, 0xC6625, 0x645, 0x655, 0xCC005, 0xCE005, 0x685, 0xD4695, 0x6B5, 0xD8006, 0x6D6, 0x6E6, 0x6F6, 0x706, 0xE4716, 0xE66D6, 0xE8006, 0xEC756, 0x776, 0xF0006, 0xF4796, 0xF6006, 0xF8006, 0x7C6, 0x7D6, 0xFC006, 0x7F6, 0x100006, 0x104816, 0x106006, 0x846, 0x856, 0x866, 0x876, 0x886, 0x112006, 0x8A6, 0x8B6, 0x8C6, 0x8D6, 0x8E6, 0x1208F6, 0x124917, 0x126007, 0x124947, 0x957, 0x12E967, 0x132987, 0x9A7, 0x136007, 0x13A9C7, 0x13C007, 0x13E007, 0x140007, 0x144A17, 0x148A37, 0x134007, 0xA57, 0x14C007, 0xA77, 0xA87, 0x154007, 0x156007, 0xAC7, 0x15CAD7, 0x15E007, 0x160007, 0xB17, 0xB28, 0x168B38, 0xB58, 0x16EB68, 0x170008, 0x172008, 0x174008, 0x176008, 0x17ABC8, 0x17C008, 0x180BF8, 0x182008, 0x184008, 0xC38, 0x188008, 0x152008, 0x18CC58, 0x190C78, 0xC98, 0x196CA8, 0xCC9, 0x19CCD9, 0x1A0CF9, 0x1A2009, 0x1A4009, 0x1A8D39, 0x1AA009, 0x1AC009, 0x1B0D79, 0xD99, 0x1B4009, 0xDB9, 0x1B8009, 0x1BCDD9, 0xDF9, 0xE0A, 0x1C200A, 0x1C6E2A, 0x1C800A, 0xE5A, 0x1CC00A, 0x1D0E7A, 0x1D4E9A, 0x1D8EBA, 0x1DCEDA, 0xEFB, 0x1E2F0B, 0x1E400B, 0x1E8F3B, 0x1ECF5B, 0x1F0F7B, 0xF9C, 0x1F400C, 0x1F8FBC, 0x1FCFDC, 0xFFD, 0x100D, 0x20501D, 0x20903E, 0x20A00E, 0x20F06F, }, + // OtherLetter(4): + new long[] { 0x4, 0xA, 0x4000, 0x10, 0x8001, 0x11, 0x4021, 0x4001, 0xC011, 0x8011, 0x31, 0x4031, 0x21, 0xC021, 0xC001, 0x8031, 0x14042, 0x14012, 0x4062, 0x4072, 0x20012, 0x52, 0x4042, 0x4092, 0x4002, 0x28012, 0x1C002, 0x2C092, 0x4022, 0x24012, 0x32, 0x18002, 0x20062, 0x30002, 0x28042, 0x8002, 0x20072, 0x380D2, 0x30052, 0x18012, 0x10002, 0x14062, 0x40F2, 0x12, 0x280D2, 0xA2, 0xC2, 0x14002, 0x82, 0x180A2, 0x42, 0x40D2, 0x20092, 0x24072, 0x10082, 0x14072, 0xE2, 0x24002, 0x4032, 0x34092, 0x40E2, 0x2C012, 0x62, 0x180E2, 0x34002, 0x1C0E2, 0x4082, 0x18052, 0x28082, 0x38002, 0xB2, 0x24082, 0x3C012, 0x40B2, 0x20022, 0x240D2, 0x8052, 0x140F2, 0x28002, 0x44103, 0x4123, 0x4133, 0x50013, 0x54013, 0x4163, 0x5C013, 0x4183, 0x4003, 0x44013, 0x64013, 0x68003, 0x6C013, 0x481C3, 0x41D3, 0x1E3, 0x41A3, 0x48193, 0x60003, 0x7C003, 0x80003, 0x84003, 0x41F3, 0x223, 0x8C003, 0x94243, 0x80263, 0x9C183, 0x4283, 0x293, 0xA8013, 0xAC013, 0x48003, 0x4C003, 0x113, 0x442C3, 0x2C273, 0xB4193, 0x2D3, 0x2E3, 0x2F3, 0xC0013, 0x4313, 0x482D3, 0x323, 0xB8003, 0xC0283, 0xCC213, 0x343, 0x303, 0xA0153, 0xC0183, 0x58003, 0x4233, 0x9C013, 0xA0003, 0x4173, 0x50283, 0x42A3, 0xD4013, 0x48013, 0xD4123, 0x4353, 0x48353, 0x682E3, 0xD8013, 0x94153, 0x4C373, 0x163, 0x4143, 0x2B3, 0xE4383, 0xE8013, 0xEC273, 0xE4003, 0xF43C3, 0x48113, 0xF82D3, 0x43F3, 0x74013, 0xAC173, 0x100003, 0x108413, 0xD8173, 0x1F3, 0x4413, 0xC0173, 0x88433, 0xD43B3, 0x110433, 0x114003, 0x463, 0x11C413, 0x120013, 0xC02A3, 0x453, 0xF4013, 0xC0493, 0x40003, 0x4A3, 0x12C1A3, 0xC0313, 0x68323, 0x130413, 0xD82A3, 0xA0013, 0x4D3, 0x153, 0xF0003, 0x501F3, 0xC8003, 0x4E3, 0x213, 0x444F4, 0x4504, 0x4514, 0x14C524, 0x4544, 0x554, 0x564, 0x15C004, 0x4584, 0x14, 0x594, 0x45A4, 0x16C014, 0x1745C4, 0x5E4, 0x45F4, 0x144014, 0x4564, 0x180014, 0x184014, 0x188004, 0x48004, 0x80634, 0x194644, 0x19C664, 0x4684, 0x4694, 0x1A8004, 0x1AC004, 0x1B46C4, 0x4624, 0x1B8014, 0x1BC004, 0x1C0014, 0x1C4014, 0x724, 0x4614, 0x160014, 0x1CC014, 0x744, 0x1D4004, 0x1C0004, 0x1D8004, 0x1DC004, 0x158014, 0x4574, 0x150004, 0x4554, 0x140004, 0x4524, 0x1E0014, 0x4794, 0x148014, 0x47A4, 0x7B4, 0x1F0004, 0x46F4, 0x7D4, 0x1FC7E4, 0x200714, 0x184004, 0x204714, 0x164014, 0x208004, 0x4834, 0x17C004, 0x144834, 0x4704, 0x210014, 0x1B4014, 0x46B4, 0x214014, 0x21C864, 0x21C5A4, 0x194884, 0x148654, 0x224014, 0x4004, 0x4654, 0x48A4, 0x2288B4, 0x48C4, 0x234014, 0x2348E4, 0x23C004, 0x244904, 0x924, 0x194754, 0x24C004, 0x250014, 0x218004, 0x20C954, 0x25C964, 0x260014, 0x4994, 0x49A4, 0x1E09B4, 0x168714, 0x1D89C4, 0x49D4, 0x27C9E4, 0x168A04, 0x764, 0x174A14, 0x2889E4, 0x168004, 0xA34, 0x174A44, 0x2949E4, 0x864, 0x2985F4, 0x2A0A74, 0x168A94, 0xAA4, 0x174AB4, 0x2B4AC4, 0x168AE4, 0x2C0AF4, 0xB14, 0xB24, 0x174B34, 0x2D0AC4, 0x168B54, 0x1D8AA4, 0x174B64, 0x2DCAC4, 0x168584, 0xB84, 0x4A44, 0x194004, 0x1C0A94, 0x200014, 0x9E4, 0x2E8B94, 0x4BB4, 0x2E8524, 0x2F0004, 0x1E4624, 0xBD4, 0xBE4, 0x300005, 0x308C15, 0xC35, 0x4C45, 0xC55, 0x31CC65, 0x320015, 0x324015, 0x4C25, 0x328015, 0x4CB5, 0x330015, 0x338CD5, 0x33C005, 0x4D05, 0x348D15, 0x4D35, 0x4D45, 0x358D55, 0x360D75, 0x4D95, 0x36CDA5, 0x4DC5, 0xDD5, 0x37CDE5, 0x380C45, 0x388E15, 0xE35, 0x394E45, 0x4E45, 0x398D45, 0xE75, 0x3A0005, 0x4E95, 0x15, 0xEA5, 0x3B0EB5, 0x324ED5, 0x3B8005, 0x3BC015, 0xC95, 0x3C0015, 0x3C8F15, 0x4F35, 0x3D4F45, 0x3D0015, 0x3DCF65, 0x48005, 0x3E0125, 0x4C75, 0xF95, 0xFA5, 0x3ECED5, 0x4FC5, 0x3F8FD5, 0x3FC005, 0x3C8E45, 0x400005, 0x3E0015, 0x404005, 0x4EC5, 0x325025, 0x304005, 0x5035, 0x415045, 0x41D065, 0x3D8005, 0x32D085, 0x424CB5, 0x50A5, 0x42C015, 0x50C5, 0x434005, 0x438005, 0x43C015, 0x440015, 0x449115, 0x44C015, 0x450005, 0x459155, 0x5175, 0x460005, 0x464EE5, 0x468005, 0x46C005, 0x4751C5, 0x3A4005, 0x3D11E5, 0x380005, 0x4811F5, 0x489215, 0x491235, 0x499255, 0x4A1275, 0x4A9295, 0x4B12B5, 0x4B4005, 0x4BD2E5, 0x4C5305, 0x4CD325, 0x4B5345, 0x4D5325, 0x4DD365, 0x4E5385, 0x4ED3A5, 0x4F53C5, 0x4FD3E5, 0x37D405, 0x31C005, 0xEF5, 0x419415, 0x40C005, 0x389415, 0x5425, 0x50C015, 0x5110D5, 0x519455, 0x521475, 0x325495, 0x528005, 0x3F0005, 0x14B5, 0xBF5, 0x5354C6, 0x14E6, 0x54F6, 0x1506, 0x549516, 0x551536, 0x559556, 0x561576, 0x5596, 0x55A6, 0x5715B6, 0x15D6, 0x578006, 0x5815F6, 0x589616, 0x591636, 0x599656, 0x5A1676, 0x5A9696, 0x5B16B6, 0x5B96D6, 0x5C16F6, 0x5C9716, 0x5CC016, 0x4006, 0x5D5746, 0x5D8016, 0x5E1776, 0x5E9796, 0x5F17B6, 0x5D17D6, 0x5F8006, 0x17F6, 0x605806, 0x6096D6, 0x611836, 0x619856, 0x621876, 0x629896, 0x6318B6, 0x5B98D6, 0x5A58E6, 0x6418F6, 0x649916, 0x5936, 0x5E4016, 0x650006, 0x659956, 0x661976, 0x5996, 0x668006, 0x6719B6, 0x674006, 0x67D9E6, 0x685A06, 0x68DA26, 0x695A46, 0x69DA66, 0x6A5A86, 0x6ADAA6, 0x6B5AC6, 0x6BDAE6, 0x6C5B06, 0x6CDB26, 0x6D5B46, 0x6DDB66, 0x6E5B86, 0x6EDBA6, 0x6F5BC6, 0x6FDBE6, 0x5C06, 0x5B9C16, 0x1C26, 0x70C006, 0x715C46, 0x718006, 0x721C77, 0x729C97, 0x731CB7, 0x739CD7, 0x741CF7, 0x5D17, 0x74DD27, 0x755D47, 0x75DD67, 0x765D87, 0x1DA7, 0x771DB7, 0x5DD7, 0x778017, 0x5C97, 0x5DF7, 0x785E07, 0x78DE27, 0x795E47, 0x798007, 0x7A1E77, 0x7A9E97, 0x7AC007, 0x7B5EC7, 0x7BDEE7, 0x7C5F07, 0x7CDF27, 0x7D0017, 0x7D9F57, 0x7E1F77, 0x7E9F97, 0x7F1FB7, 0x7F9FD7, 0x801FF7, 0x80A017, 0x812037, 0x81A057, 0x822077, 0x82A097, 0x8320B7, 0x83A0D7, 0x20F7, 0x2107, 0x2117, 0x2127, 0x852138, 0x6158, 0x85E168, 0x18, 0x6188, 0x86A198, 0x8721B8, 0x87A1D8, 0x87C018, 0x880018, 0x6218, 0x6228, 0x892238, 0x2258, 0x2268, 0x89C008, 0x8A6288, 0x8AE2A8, 0x8B62C8, 0x8B8018, 0x8C22F8, 0x8C4018, 0x8CE328, 0x8D6348, 0x8DE368, 0x8E6388, 0x8EE3A8, 0x23C8, 0x8F4008, 0x8FE3E8, 0x906409, 0x90E429, 0x6449, 0x91A459, 0x922479, 0x924019, 0x64A9, 0x92C019, 0x930009, 0x934009, 0x938009, 0x93C009, 0x946509, 0x94E529, 0x956549, 0x95E569, 0x966589, 0x96E5A9, 0x9765C9, 0x97E5EA, 0x660A, 0x98401A, 0x98E62A, 0x664A, 0x665A, 0x99801A, 0x267A, 0x9A000A, 0x9AA69A, 0x9B26BA, 0x9BA6DA, 0x9C26FA, 0x271B, 0x9C800B, 0x9D273B, 0x675B, 0x676B, 0x9DC01B, 0x9E678B, 0x9EE7AB, 0x9F67CB, 0x27EC, 0x67FC, 0xA0001C, 0x681C, 0xA0801C, 0xA1283C, 0xA1A85C, 0xA2287D, 0xA2A89D, 0xA2C01D, 0xA368CD, 0xA3E8EE, 0xA4690E, 0xA4E92F, }, + // NonSpacingMark(5): + new long[] { 0x4, 0xA, 0x4000, 0x10, 0x21, 0xC001, 0x8001, 0x4001, 0x11, 0x8031, 0x8011, 0x31, 0xC021, 0x4021, 0xC011, 0x4031, 0x42, 0x18052, 0x52, 0x72, 0x14002, 0x18082, 0x28092, 0x62, 0x2C002, 0x380D2, 0x2C082, 0x34002, 0x1C002, 0xB2, 0x4072, 0x82, 0xC2, 0x12, 0x30002, 0x20072, 0x18002, 0x140E2, 0x4062, 0x3C012, 0x28012, 0x34012, 0x4082, 0x1C032, 0x3C072, 0x4002, 0x2C012, 0x10002, 0x34062, 0xA2, 0x40F2, 0x4092, 0x80E2, 0x40D2, 0x24002, 0x38002, 0x340B2, 0x2C072, 0x38012, 0xF2, 0x20002, 0x20062, 0xE2, 0x100C2, 0x38062, 0x1C082, 0x40A2, 0x32, 0x14012, 0x20022, 0x2C062, 0x92, 0x28072, 0x20012, 0x380F2, 0x1C052, 0x24072, 0x300B2, 0x10062, 0x38072, 0x40E2, 0x300F2, 0x103, 0x48113, 0x40003, 0x133, 0x50003, 0x58153, 0x173, 0x60173, 0x30003, 0x193, 0x48003, 0x68073, 0x6C003, 0x70003, 0x74013, 0x7C1E3, 0x701D3, 0x4C003, 0x80173, 0x10183, 0x1F3, 0x841E3, 0x1D3, 0x223, 0x233, 0x90003, 0x94013, 0x4263, 0x4273, 0x5C1C3, 0xA0113, 0xA4013, 0xA8003, 0x213, 0x402B3, 0x8C2C3, 0xB4003, 0x2E3, 0xBC003, 0xC4303, 0x4323, 0xB8333, 0xD0263, 0x40353, 0x74363, 0xBC373, 0xE0243, 0x143, 0x80003, 0x393, 0xB4123, 0x403A3, 0xEC1E3, 0xF0003, 0xF4003, 0x3E3, 0x123, 0xFC123, 0x100003, 0x413, 0xA00C3, 0x43A3, 0x1083E3, 0xE8353, 0x10C143, 0x1C3, 0x1E3, 0x110263, 0x74353, 0x1101B3, 0x114003, 0x2F3, 0x100123, 0x463, 0x40133, 0x473, 0x54483, 0x18003, 0x90123, 0x493, 0x128113, 0x88003, 0x12C003, 0x1344C3, 0xE8003, 0xE8113, 0x4E3, 0x9C013, 0xB8013, 0x373, 0x1404F3, 0x7C273, 0x7C013, 0x41B3, 0xCC483, 0x7C333, 0x144003, 0xE8013, 0xB8003, 0x4004, 0x524, 0x14C004, 0x154544, 0x158004, 0x574, 0x584, 0x164004, 0x16C5A4, 0x170004, 0x5D4, 0x178004, 0x1805F4, 0x184004, 0x4624, 0x18C004, 0x190004, 0x194004, 0x5F4, 0x664, 0x1A0674, 0x694, 0x1A8004, 0x684, 0x1A05F4, 0x1B06B4, 0x6D4, 0x46E4, 0x1C06F4, 0x714, 0x1CC724, 0x1D05F4, 0x1D4004, 0x17C004, 0x1DC764, 0x1E0004, 0x794, 0x1D8004, 0x1AC004, 0x7A4, 0x1F07B4, 0x7D4, 0x7E4, 0x2007F4, 0x204004, 0x208004, 0x20C004, 0x214844, 0x864, 0x874, 0x884, 0x894, 0x22C8A4, 0x1AC8C4, 0x8D4, 0x2248E4, 0x150004, 0x46D4, 0x2408F4, 0x914, 0x924, 0x934, 0x250004, 0x954, 0x258004, 0x974, 0x264984, 0x21C004, 0x268004, 0x9B4, 0x270004, 0x9D4, 0x278824, 0x27C004, 0x280004, 0x268824, 0x21CA14, 0x288004, 0x28C004, 0xA44, 0x294564, 0x21C524, 0x29CA64, 0xA84, 0x2A4014, 0xAA4, 0xAB4, 0x2B0004, 0xAD4, 0x1DC844, 0x2B86B4, 0x2BC004, 0x14, 0x564, 0x2C0004, 0xB14, 0x6B4, 0x1DC014, 0xAF4, 0x2C8004, 0x47A4, 0xB34, 0x2D0004, 0x2D8005, 0xB75, 0x2E4B85, 0xBA5, 0x2F0BB5, 0xBD5, 0xBE5, 0x300BF5, 0x304005, 0xC25, 0x310C35, 0x318C55, 0xC75, 0x320005, 0xC95, 0x328005, 0xCB5, 0x15, 0xCC5, 0xCD5, 0x33CCE5, 0x4D05, 0x348D15, 0xD35, 0xD45, 0x358D55, 0x360D75, 0x368D95, 0x36C005, 0xDC5, 0x378DD5, 0x37C005, 0xE05, 0x388E15, 0x38C005, 0x390005, 0xE55, 0x3A0005, 0x3A4005, 0x3ACEA5, 0xEC5, 0x3B4005, 0x3BCEE5, 0xF05, 0x3C8F15, 0x3CC005, 0xF45, 0x3D4005, 0xF65, 0x3DC005, 0x2E4005, 0x3E4F85, 0x358FA5, 0x3ECF85, 0x2E4FC5, 0x3F4F85, 0x3F8C85, 0x3FC005, 0x360005, 0x400F85, 0x409015, 0x40CBB5, 0x415045, 0x3E5065, 0x2E4BB5, 0x41CF85, 0x301085, 0x429095, 0x42C005, 0x4350C5, 0x10E5, 0x10F5, 0x440005, 0x449115, 0x451135, 0x459155, 0x45C005, 0x460005, 0x469195, 0x46C005, 0x4B55, 0x2D4006, 0x470006, 0x11D6, 0x47D1E6, 0x485206, 0x48D226, 0x495246, 0x1266, 0x49C006, 0x4A5286, 0x12A6, 0x4AC006, 0x4B0006, 0x12D6, 0x12E6, 0x12F6, 0x1306, 0x1316, 0x1326, 0x4CC006, 0x4D5346, 0x4DD366, 0x4E5386, 0x4ED3A6, 0x4F0006, 0x4F4006, 0x4F8006, 0x5013F6, 0x39CE66, 0x1416, 0x508006, 0x511436, 0x519456, 0x521476, 0x529496, 0x5314B6, 0x5394D6, 0x5414F6, 0x549516, 0x551536, 0x559556, 0x549576, 0x565586, 0x5655A6, 0x5715B6, 0x5795D6, 0x15F6, 0x585606, 0x58D626, 0x595646, 0x1666, 0x5A1676, 0x5A9696, 0x5AC006, 0x56C6, 0x5B4007, 0x5B8007, 0x16F7, 0x5C5707, 0x5CD727, 0x5D5747, 0x5DD767, 0x5E5787, 0x5ED7A7, 0x17C7, 0x17D7, 0x17E7, 0x6017F7, 0x609817, 0x611837, 0x614007, 0x61D867, 0x625887, 0x628007, 0x6318B7, 0x6398D7, 0x6418F7, 0x649917, 0x651937, 0x659957, 0x661977, 0x669997, 0x6719B7, 0x6799D7, 0x6819F7, 0x1A17, 0x1A27, 0x68C007, 0x690008, 0x1A58, 0x69DA68, 0x6A5A88, 0x6A8008, 0x6AC008, 0x6B5AC8, 0x6B8008, 0x6C1AF8, 0x6C9B18, 0x6D1B38, 0x1B58, 0x1B68, 0x6DC008, 0x6E5B88, 0x6EDBA8, 0x6F5BC8, 0x6FDBE8, 0x705C08, 0x70DC28, 0x1C48, 0x1C59, 0x1C69, 0x721C79, 0x1C99, 0x728009, 0x72C009, 0x730009, 0x734009, 0x73DCE9, 0x1D09, 0x749D19, 0x751D39, 0x759D59, 0x761D79, 0x1D99, 0x76DDAA, 0x77000A, 0x1DDA, 0x77800A, 0x1DFA, 0x78000A, 0x789E1A, 0x791E3A, 0x799E5A, 0x7A1E7A, 0x1E9B, 0x7ADEAB, 0x7B000B, 0x7B9EDB, 0x7C1EFB, 0x7C9F1B, 0x1F3C, 0x7D000C, 0x7D9F5C, 0x7E1F7C, 0x1F9D, 0x1FAD, 0x7F1FBD, 0x7F9FDE, 0x7FC00E, 0x80600F, }, + // SpacingCombiningMark(6): + new long[] { 0x4, 0x9, 0x2000, 0x10, 0x4001, 0x6021, 0x4011, 0x31, 0x6001, 0x11, 0x4031, 0x2001, 0x21, 0x2031, 0x6011, 0x2021, 0xA042, 0xE062, 0x82, 0xE092, 0xE082, 0x14042, 0x72, 0x8002, 0xB2, 0xE002, 0x12002, 0x2002, 0x12, 0x16002, 0xE0A2, 0x92, 0x10002, 0x12022, 0x18002, 0x10092, 0x1A042, 0xE0E2, 0x42, 0x1E002, 0xE2, 0x6042, 0x32, 0xA002, 0x1A012, 0x1C002, 0xE0F2, 0x20D2, 0xC2, 0x12042, 0x20A2, 0x20B2, 0x120A2, 0xE042, 0xC042, 0x8092, 0x52, 0xC002, 0x1C042, 0xA092, 0x8072, 0x1C062, 0x22103, 0x24003, 0x133, 0x143, 0x123, 0x2A003, 0x163, 0x2E003, 0x30093, 0x32003, 0x34003, 0x36003, 0x38013, 0x3A003, 0x3C003, 0x1F3, 0x2C003, 0x40003, 0x30003, 0x381B3, 0x2C213, 0x46223, 0x44003, 0x48003, 0x24253, 0x4C003, 0x4E003, 0x52283, 0x54003, 0x2B3, 0x58173, 0x5C2D3, 0x5E173, 0x601F3, 0x313, 0x3C323, 0x68333, 0x3E003, 0x40153, 0x353, 0x3A013, 0x6E363, 0x70003, 0x64393, 0x303, 0x74003, 0x5A003, 0x76363, 0x78153, 0x2E293, 0x2E3D3, 0x7C004, 0x803F4, 0x414, 0x84004, 0x88434, 0x8A004, 0x464, 0x8E004, 0x484, 0x92004, 0x4A4, 0x96004, 0x98004, 0x9A004, 0x9E4E4, 0xA2504, 0xA4484, 0xA6004, 0xA8004, 0xAA004, 0x564, 0xAE004, 0xB2584, 0x5A4, 0x5B4, 0xBA5C4, 0xBC004, 0x5F4, 0xC2604, 0xC4004, 0x634, 0xCA644, 0x444, 0x4D4, 0x88664, 0x90004, 0xCE4E4, 0xD2684, 0x6A4, 0xD6004, 0xD8004, 0xDA4E4, 0xDC4E4, 0xDE004, 0xE0004, 0x404, 0x715, 0x725, 0xE8735, 0x755, 0xEC005, 0xF0775, 0xF2005, 0xF4005, 0xF87B5, 0x7D5, 0x7E5, 0x7F5, 0x805, 0x815, 0x106825, 0x108005, 0x10C855, 0x110875, 0x112005, 0x8A5, 0x116005, 0x8C5, 0x11A005, 0x11E8E5, 0x905, 0x915, 0x925, 0x126795, 0x128005, 0x12A005, 0x128925, 0x12C005, 0x128975, 0x130005, 0x132005, 0x134005, 0x136005, 0x138005, 0xF0925, 0x13A005, 0xF09E5, 0x9F6, 0xA06, 0x144A16, 0x148A36, 0xA56, 0x14EA66, 0x150006, 0x152006, 0xAA6, 0x156006, 0x15AAC6, 0x15EAE6, 0x162B06, 0x164006, 0x168B36, 0x16A006, 0x16EB66, 0x172B86, 0x176BA6, 0x17ABC6, 0x17EBE6, 0x182C06, 0x172C26, 0x17EC36, 0x18AC46, 0x18AA36, 0x17AC36, 0x18EC66, 0xC87, 0x194C97, 0x198CB7, 0x19CCD7, 0x19E007, 0x1A2D07, 0x1A6D27, 0x1A8007, 0x1AA007, 0xD67, 0x1B0D77, 0x1B2007, 0x1B6DA7, 0x1BADC7, 0x1BEDE7, 0x1C2E07, 0x1C6E27, 0x1CAE48, 0x1CEE68, 0x1D0008, 0x1D2008, 0x1D6EA8, 0xEC8, 0xED8, 0x1DC008, 0xEF8, 0x1E2F08, 0x1E6F28, 0xF48, 0x1ECF59, 0x1EE009, 0x1F0009, 0x1F4F99, 0xFB9, 0x1F8009, 0x1FCFD9, 0x200FF9, 0x20200A, 0x20400A, 0x20903A, 0x20D05A, 0x21107A, 0x109B, 0x21400B, 0x2190BB, 0x10DB, 0x21C00C, 0x10FC, 0x22310C, 0x112D, 0x22913D, 0x22A00E, 0x22C00E, 0x23117F, }, + // EnclosingMark(7): + new long[] { 0x4, 0x6, 0x400, 0x10, 0x421, 0x801, 0x811, 0xC11, 0x21, 0x401, 0x1002, 0x1852, 0x72, 0x82, 0x2402, 0x2803, 0x2C03, 0xC3, 0xD3, 0xE3, 0xF4, 0x4004, 0x114, 0x124, 0x4C04, 0x145, 0x5955, 0x175, 0x6005, 0x196, 0x1A6, 0x6C06, 0x7006, 0x7407, 0x1E7, 0x1F7, 0x207, 0x8408, 0x8808, 0x8C08, 0x9008, 0x259, 0x9809, 0x279, 0xA009, 0x29A, 0xA80A, 0xAC0A, 0x2CA, 0xB40B, 0xB80B, 0x2FB, 0xC00B, 0xC40C, 0xC80C, 0xD33C, 0x35D, 0xDF6D, 0xE00E, 0xE40E, 0xEFAF, }, + // DecimalDigitNumber(8): + new long[] { 0x4, 0x7, 0x801, 0x11, 0x1002, 0x32, 0x843, 0x2813, 0x64, 0x3004, 0x3804, 0x4005, 0x85, 0x4805, 0x95, 0x3005, 0x5005, 0xA5, 0x5806, 0xC6, 0xB6, 0x86, 0x6806, 0x7006, 0xF6, 0x6006, 0x7806, 0x106, 0xE6, 0xD6, 0x116, 0x9007, 0x137, 0xA007, 0xB157, 0x147, 0xB807, 0xC007, 0xA197, 0x1A7, 0xD947, 0x1C7, 0xE927, 0x1D7, 0xE137, 0xC807, 0x1F8, 0x10A08, 0x11A28, 0x12008, 0x12808, 0x13A68, 0xFA88, 0x298, 0x15008, 0xA248, 0x1E8, 0x162B8, 0x16808, 0x2E9, 0x182F9, 0x319, 0x19009, 0x1A339, 0x359, 0x1B009, 0xF379, 0x1C1E9, 0x399, 0x1D009, 0x3BA, 0x1E00A, 0x3DA, 0x1FBEA, 0x20C0A, 0x21C2A, 0x22C4A, 0x46B, 0x2447B, 0x2549B, 0x264BB, 0x4DC, 0x2700C, 0x284FC, 0x51D, 0x52D, 0x2980D, 0x2AD4E, 0x2B00E, 0x2C57F, }, + // LetterNumber(9): + new long[] { 0x4, 0x7, 0x800, 0x10, 0x11, 0x821, 0x801, 0x1811, 0x31, 0x1001, 0x42, 0x2802, 0x3002, 0x3812, 0x82, 0x4802, 0x2872, 0x5013, 0xB3, 0x68C3, 0x7003, 0x80F3, 0x7803, 0xA3, 0x8804, 0x9924, 0xA004, 0xA804, 0xB964, 0x185, 0xD195, 0xD805, 0x15, 0x1C5, 0x1D6, 0xF006, 0xF806, 0x206, 0x216, 0x227, 0x11807, 0x12A47, 0x267, 0x13808, 0x14008, 0x298, 0x15008, 0x2B9, 0x16009, 0x16809, 0x2E9, 0x2FA, 0x1800A, 0x1880A, 0x32A, 0x1980B, 0x1A00B, 0x1A80B, 0x1B00B, 0x1B80C, 0x1CB8C, 0x3AC, 0x3BD, 0x1EBCD, 0x1F00E, 0x1F80E, 0x20C0F, }, + // OtherNumber(10): + new long[] { 0x4, 0x8, 0x10, 0x1000, 0x1001, 0x2011, 0x11, 0x2001, 0x3001, 0x31, 0x1031, 0x1042, 0x5012, 0x4002, 0x6042, 0x72, 0x1002, 0x62, 0x6012, 0x7002, 0x8012, 0x9002, 0x1082, 0x8002, 0x10A2, 0xA002, 0x12, 0x70A2, 0x6002, 0xB003, 0xC013, 0x10D3, 0x13, 0xE003, 0xF3, 0x10003, 0x11013, 0x123, 0x1103, 0x133, 0x140D3, 0x153, 0x1163, 0xC3, 0x1173, 0x183, 0x19003, 0x12003, 0x1A0D3, 0x1C1B3, 0x1D4, 0x1E4, 0x1F004, 0x201E4, 0x214, 0x224, 0x1234, 0x244, 0x25014, 0x1264, 0x27004, 0x14, 0x284, 0x294, 0x1F4, 0x2B2A4, 0x25234, 0x2C4, 0x2D4, 0x2E4, 0x2F4, 0x304, 0x314, 0x325, 0x34335, 0x35005, 0x345, 0x36005, 0x375, 0x38005, 0x395, 0x3A5, 0x3B005, 0x15, 0x3C005, 0x3D005, 0x34005, 0x3E5, 0x3F005, 0x405, 0x415, 0x425, 0x44435, 0x445, 0x455, 0x465, 0x475, 0x485, 0x49006, 0x4A006, 0x4C4B6, 0x4D006, 0x4E6, 0x4F006, 0x506, 0x52516, 0x536, 0x54006, 0x556, 0x56006, 0x576, 0x586, 0x596, 0x5A6, 0x5B006, 0x5C6, 0x5D6, 0x5E6, 0x5F6, 0x606, 0x61006, 0x62007, 0x64637, 0x657, 0x667, 0x68677, 0x6A697, 0x6C6B7, 0x6E6D7, 0x6F7, 0x707, 0x71007, 0x72007, 0x73007, 0x74007, 0x76757, 0x777, 0x787, 0x79008, 0x7A008, 0x7B8, 0x7C008, 0x7D8, 0x7E008, 0x807F8, 0x818, 0x828, 0x838, 0x848, 0x86858, 0x878, 0x888, 0x89008, 0x8A009, 0x8C8B9, 0x8D009, 0x8F8E9, 0x90009, 0x91009, 0x929, 0x939, 0x95949, 0x97969, 0x98009, 0x9900A, 0x9A00A, 0x9BA, 0x9D9CA, 0x9E00A, 0xA09FA, 0xA2A1A, 0xA300A, 0xA4B, 0xA500B, 0xA7A6B, 0xA9A8B, 0xABAAB, 0xAC00C, 0xAEADC, 0xB0AFC, 0xB1D, 0xB3B2D, 0xB400E, 0xB500E, 0xB7B6F, }, + // SpaceSeparator(11): + new long[] { 0x4, 0x6, 0x400, 0x10, 0x801, 0x31, 0x421, 0x1002, 0x52, 0x1802, 0x1C03, 0x83, 0x493, 0x2804, 0xB4, 0x2C04, 0x3004, 0x3405, 0x3805, 0x40F5, 0xD5, 0x4406, 0x4D26, 0x5006, 0x5407, 0x5807, 0x157, 0x6008, 0x6408, 0x6808, 0x5C08, 0x6C09, 0x7009, 0x1D9, 0x7809, 0x7C0A, 0x800A, 0x21A, 0x880A, 0x8C0B, 0x900B, 0x940B, 0x980B, 0xA27C, 0xAA9C, 0xB2BD, 0xB40E, 0xB80F, }, + // LineSeparator(12): + new long[] { 0x4, 0x5, 0x200, 0x401, 0x602, 0x43, 0xA04, 0x65, 0xE06, 0x1007, 0x1208, 0x1409, 0x160A, 0x180B, 0x1A0C, 0xED, 0x1E0E, 0x200F, }, + // ParagraphSeparator(13): + new long[] { 0x4, 0x5, 0x10, 0x401, 0x602, 0x43, 0xA04, 0x65, 0xE06, 0x1007, 0x1208, 0x1409, 0x160A, 0x180B, 0x1A0C, 0xED, 0x1E0E, 0x200F, }, + // Control(14): + new long[] { 0x4, 0x5, 0x10, 0x21, 0x32, 0x43, 0x54, 0x205, 0x65, 0xE06, 0xE86, 0x1497, 0x1608, 0x1809, 0x1A0A, 0x1C0B, 0x1E0C, 0x200D, 0x220E, 0x240F, }, + // Format(15): + new long[] { 0x4, 0x7, 0x10, 0x800, 0x1011, 0x21, 0x1811, 0x831, 0x11, 0x31, 0x1001, 0x1801, 0x801, 0x2002, 0x52, 0x862, 0x4072, 0x2812, 0x92, 0x4802, 0xA2, 0xB2, 0x8C2, 0xD3, 0xE3, 0x7813, 0x103, 0x113, 0x123, 0x9803, 0x143, 0x153, 0xB003, 0x174, 0x184, 0xC804, 0xD004, 0xD804, 0xE004, 0xE804, 0xC004, 0x1E4, 0x101F4, 0xF004, 0x215, 0x225, 0x235, 0x12A45, 0x13005, 0x275, 0x14005, 0x14805, 0x15005, 0x2B5, 0x2C6, 0x2D6, 0x17AE6, 0x18006, 0x316, 0x19006, 0x336, 0x1A006, 0x1A806, 0x367, 0x377, 0x1C007, 0x1C807, 0x3A7, 0x1D807, 0x1EBC7, 0x3E7, 0x203F8, 0x20808, 0x21008, 0x21808, 0x22C48, 0x23008, 0x479, 0x24009, 0x24809, 0x25009, 0x4B9, 0x26009, 0x4DA, 0x2700A, 0x2780A, 0x2800A, 0x2951A, 0x53B, 0x2A00B, 0x55B, 0x2BD6B, 0x58C, 0x2C80C, 0x2DDAC, 0x5CD, 0x2F5DD, 0x5FE, 0x3000E, 0x3161F, }, + // Surrogate(16): + new long[] { 0x4, 0x3, 0x1B, 0x2C, 0x18D, 0x4E, 0x5F, }, + // PrivateUse(17): + new long[] { 0x4, 0x4, 0x108, 0x209, 0x30A, 0x14B, 0x15C, 0x6D, 0x7E, 0x8F, }, + // ConnectorPunctuation(18): + new long[] { 0x4, 0x6, 0x10, 0x400, 0x21, 0x811, 0xC01, 0x42, 0x52, 0x1062, 0x62, 0x1802, 0x73, 0x83, 0x2403, 0x2803, 0x2C03, 0xC4, 0x3404, 0xE4, 0x40F4, 0x115, 0x4805, 0x135, 0x5005, 0x4405, 0x5406, 0x5D66, 0x5586, 0x196, 0x6807, 0x6C07, 0x7007, 0x7407, 0x7DE8, 0x8008, 0x8408, 0x229, 0x8C09, 0x9009, 0x25A, 0x980A, 0x9C0A, 0x28B, 0xA40B, 0xA80B, 0x2BC, 0xB00C, 0xB40C, 0x2ED, 0xC2FD, 0x31E, 0xC80E, 0xD33F, }, + // DashPunctuation(19): + new long[] { 0x4, 0x7, 0x10, 0x800, 0x1001, 0x21, 0x1801, 0x1031, 0x11, 0x31, 0x801, 0x42, 0x2802, 0x3002, 0x3802, 0x62, 0x4002, 0x4802, 0x52, 0x8A2, 0x92, 0xB3, 0x6003, 0xD3, 0x7003, 0x6803, 0xF3, 0x103, 0x9113, 0x9803, 0xA003, 0x143, 0x113, 0xA804, 0xB004, 0x174, 0x184, 0xC804, 0x194, 0x1A4, 0x1B4, 0x1C4, 0x1D4, 0xF004, 0x1F4, 0x10004, 0x10805, 0x11A25, 0x245, 0x255, 0x13A65, 0x12805, 0x14A85, 0x15005, 0x15805, 0x16AC5, 0x215, 0x17006, 0x182F6, 0x18806, 0x19006, 0x1A336, 0x1A806, 0x1B006, 0x19806, 0x1B806, 0x1C006, 0x1C807, 0x1D007, 0x1E3B7, 0x1E807, 0x1F007, 0x1F807, 0x20007, 0x417, 0x21007, 0x22438, 0x22808, 0x23008, 0x23808, 0x24008, 0x24808, 0x4A8, 0x25808, 0x4C9, 0x26809, 0x4E9, 0x27809, 0x28009, 0x28809, 0x29009, 0x29809, 0x54A, 0x2A80A, 0x56A, 0x2B80A, 0x2C00A, 0x59A, 0x2DDAA, 0x5CB, 0x2E80B, 0x2FDEB, 0x30E0B, 0x3100B, 0x63C, 0x32E4C, 0x33E6C, 0x68D, 0x3569D, 0x6BE, 0x3600E, 0x376DF, }, + // OpenPunctuation(20): + new long[] { 0x4, 0x8, 0x1000, 0x10, 0x21, 0x31, 0x2001, 0x3001, 0x4002, 0x5002, 0x6002, 0x3072, 0x3052, 0x32, 0x52, 0x2072, 0x6022, 0x4022, 0x62, 0x2002, 0x5032, 0x42, 0x72, 0x2062, 0x7002, 0x4062, 0x8003, 0x53, 0x93, 0xA3, 0xB3, 0xC003, 0xD033, 0xE003, 0xE3, 0x100F3, 0x23, 0x110A3, 0x123, 0x133, 0x3003, 0x14033, 0x15023, 0x16003, 0x17003, 0x183, 0x163, 0x43, 0x193, 0x1A004, 0x1B4, 0x1C4, 0x1D004, 0x1F1E4, 0x204, 0x214, 0x224, 0x24234, 0x25004, 0x264, 0x274, 0x29284, 0x2A004, 0x2B004, 0x242C4, 0x2D004, 0x27004, 0x2E004, 0x2E4, 0x2F4, 0x304, 0x32315, 0x34335, 0x35005, 0x37365, 0x385, 0x39005, 0x31005, 0x3A5, 0x3C3B5, 0x3D005, 0x3F3E5, 0x405, 0x42415, 0x43005, 0x3F445, 0x45005, 0x33005, 0x465, 0x345, 0x48476, 0x4A496, 0x4B006, 0x4C006, 0x4E4D6, 0x504F6, 0x516, 0x526, 0x53006, 0x54006, 0x56556, 0x57006, 0x58006, 0x59336, 0x5A007, 0x5B007, 0x5C007, 0x5D007, 0x5E007, 0x5F7, 0x61607, 0x62007, 0x64637, 0x657, 0x66007, 0x67007, 0x69688, 0x6A8, 0x6B008, 0x6C008, 0x6D8, 0x6E8, 0x6F8, 0x70008, 0x71008, 0x728, 0x73008, 0x75749, 0x76009, 0x779, 0x78009, 0x799, 0x7B7A9, 0x7C9, 0x7D9, 0x7E009, 0x7FA, 0x8000A, 0x8281A, 0x8483A, 0x85A, 0x86A, 0x8700A, 0x88B, 0x8900B, 0x8B8AB, 0x8C00B, 0x8E8DB, 0x8FC, 0x9190C, 0x9392C, 0x94D, 0x9695D, 0x97E, 0x9800E, 0x9A99F, }, + // ClosePunctuation(21): + new long[] { 0x4, 0x8, 0x10, 0x1000, 0x3021, 0x2001, 0x31, 0x3001, 0x11, 0x21, 0x4002, 0x52, 0x5002, 0x6032, 0x7002, 0x3072, 0x62, 0x2082, 0x5022, 0x9022, 0x2002, 0x32, 0x92, 0x2052, 0x6002, 0x72, 0x9052, 0xA003, 0xB3, 0xC3, 0xD3, 0xF0E3, 0x10033, 0xE3, 0x103, 0x12113, 0x23, 0x130C3, 0x143, 0x30E3, 0x15033, 0x16023, 0x10003, 0x17003, 0x183, 0x193, 0x1A3, 0x1B004, 0x1C4, 0x1D004, 0x1F1E4, 0x204, 0x214, 0x224, 0x24234, 0x25004, 0x264, 0x28274, 0x29004, 0x2A004, 0x242B4, 0x2C004, 0x26004, 0x22004, 0x2D4, 0x2E4, 0x302F5, 0x31305, 0x32005, 0x34335, 0x355, 0x36005, 0x375, 0x38305, 0x39005, 0x3B3A5, 0x3C5, 0x3E3D5, 0x3F005, 0x3B355, 0x40005, 0x415, 0x315, 0x43426, 0x45446, 0x46006, 0x47006, 0x48006, 0x4A496, 0x4B6, 0x4C6, 0x4D006, 0x4E006, 0x4F6, 0x50006, 0x51006, 0x52306, 0x53007, 0x54007, 0x55007, 0x56007, 0x57007, 0x587, 0x5A597, 0x5B007, 0x5D5C7, 0x5E7, 0x5F007, 0x60007, 0x62618, 0x638, 0x64008, 0x65008, 0x668, 0x678, 0x688, 0x69008, 0x6A008, 0x6B8, 0x6C008, 0x6E6D9, 0x6F009, 0x709, 0x71009, 0x729, 0x74739, 0x759, 0x769, 0x77009, 0x78A, 0x7900A, 0x7B7AA, 0x7D7CA, 0x7EA, 0x7FA, 0x8000A, 0x81B, 0x8200B, 0x8483B, 0x8500B, 0x8786B, 0x88C, 0x8A89C, 0x8C8BC, 0x8DD, 0x8F8ED, 0x90E, 0x9100E, 0x9392F, }, + // InitialQuotePunctuation(22): + new long[] { 0x4, 0x6, 0x400, 0x10, 0x801, 0xC01, 0x21, 0x831, 0x31, 0x1002, 0x42, 0x1442, 0x1842, 0x1402, 0x2002, 0x2403, 0xA3, 0x30B3, 0xD3, 0x73, 0xE3, 0x3C04, 0x4504, 0x124, 0x134, 0x5004, 0x5955, 0x6175, 0x195, 0x6806, 0x6C06, 0x7006, 0x7407, 0x7807, 0x1F7, 0x8008, 0x8408, 0x8808, 0x239, 0x9009, 0x9409, 0x26A, 0x9C0A, 0xA00A, 0xAA9B, 0xAC0B, 0xB00C, 0xB40C, 0xBEED, 0xC00E, 0xC40F, }, + // FinalQuotePunctuation(23): + new long[] { 0x4, 0x6, 0x10, 0x400, 0x801, 0x31, 0x21, 0x1002, 0x42, 0x1442, 0x1842, 0x1402, 0x1802, 0x1C03, 0x83, 0x2893, 0xB3, 0x43, 0xC3, 0x3404, 0x3CE4, 0x104, 0x114, 0x124, 0x5135, 0x5955, 0x175, 0x6006, 0x6406, 0x6806, 0x6C07, 0x7007, 0x1D7, 0x7808, 0x7C08, 0x8008, 0x219, 0x8809, 0x8C09, 0x24A, 0x940A, 0x980A, 0xA27B, 0xA40B, 0xA80C, 0xAC0C, 0xB6CD, 0xB80E, 0xBC0F, }, + // OtherPunctuation(24): + new long[] { 0x4, 0x9, 0x10, 0x2000, 0x2001, 0x4001, 0x6001, 0x21, 0x11, 0x6011, 0x31, 0x4011, 0x2031, 0x4031, 0x2021, 0x6021, 0xA042, 0x62, 0xC002, 0x10072, 0x14092, 0x12002, 0x8002, 0x72, 0x18012, 0x16062, 0xD2, 0xA002, 0x20C2, 0xE002, 0x82, 0x12, 0x2042, 0x16012, 0x18062, 0x10012, 0xA2, 0xB2, 0x52, 0x16002, 0x14002, 0x4012, 0x8012, 0x100C2, 0x1C082, 0x1E082, 0x8082, 0x16092, 0x1C0B2, 0x16072, 0x1E0C2, 0xE012, 0x42, 0x18002, 0x20E2, 0xC2, 0xE042, 0x1A002, 0x2062, 0x10002, 0x10042, 0xE082, 0x1A042, 0xE062, 0x1E0A2, 0xA072, 0x20003, 0x113, 0x24003, 0x133, 0x16143, 0x2C153, 0x30173, 0x34193, 0x381B3, 0x1D3, 0x2C003, 0x1E3, 0x1F3, 0x42203, 0x173, 0x223, 0x3E003, 0x46003, 0x3A243, 0x253, 0x263, 0x4E003, 0x50003, 0x52013, 0x22A3, 0x2B3, 0x382C3, 0x5C2D3, 0x2F3, 0x601C3, 0x62013, 0x2323, 0x2003, 0x3C003, 0x3A003, 0x333, 0x233, 0x38203, 0x68003, 0x6C353, 0x6E353, 0x34003, 0x383, 0x243, 0x2123, 0x74393, 0x4C003, 0x44003, 0x3E013, 0x3B3, 0x22003, 0x2E003, 0x48003, 0x21C3, 0x2E163, 0x2203, 0x3C3, 0x3D3, 0x3E3, 0x7E003, 0x80003, 0x1B3, 0x3C173, 0x82003, 0x84004, 0x88434, 0x8C454, 0x8E004, 0x92484, 0x444, 0x4A4, 0x96004, 0x4C4, 0x4D4, 0x4E4, 0x9E4D4, 0x504, 0x9A004, 0x514, 0x524, 0x534, 0x544, 0xAA004, 0x4B4, 0x564, 0xAE004, 0xB2584, 0xB65A4, 0xBA5C4, 0x5E4, 0xC05F4, 0xC4614, 0x634, 0xC4644, 0x654, 0x88004, 0x664, 0xCE004, 0xD0004, 0xD2004, 0x6A4, 0x6B4, 0xD8004, 0xDA004, 0xDC004, 0xCC004, 0x6F4, 0xE0004, 0xE4714, 0xA0734, 0x744, 0xE8004, 0x754, 0x764, 0xD0444, 0x6D4, 0x774, 0x784, 0xF2004, 0xF4004, 0xF87B4, 0x7D4, 0xFC004, 0xFE004, 0xEA004, 0x102804, 0x825, 0x108835, 0x10C855, 0x110875, 0x895, 0x1168A5, 0x118005, 0x11A005, 0x11C005, 0x8F5, 0x11E905, 0x915, 0x925, 0x935, 0x128005, 0x8B5, 0x955, 0x12E965, 0x130005, 0x134995, 0x875, 0x9B5, 0x138005, 0x13C9D5, 0x13E005, 0xA05, 0x8C5, 0x144A15, 0xA35, 0x116005, 0x148005, 0x14A005, 0x14C005, 0xA75, 0xA85, 0xA95, 0xAA5, 0x156005, 0x158005, 0x15A005, 0x15C005, 0x15E005, 0xB05, 0x162005, 0xB25, 0xB35, 0x965, 0xB45, 0x16A005, 0xB65, 0xB75, 0x170005, 0x160005, 0xB95, 0x174005, 0x178BB5, 0x17A005, 0x144005, 0x17C005, 0xB55, 0xBF5, 0x106005, 0x845, 0x182C06, 0x186C26, 0xC46, 0xC56, 0xC66, 0xC76, 0x192C86, 0xCA6, 0xCB6, 0xCC6, 0x19CCD6, 0xCF6, 0xD06, 0x1A2006, 0x1A6D26, 0xD46, 0xD56, 0x1AED66, 0xD86, 0x1B2CF6, 0xDA6, 0xDB6, 0x1B8006, 0x1BA006, 0xDE6, 0x1BE006, 0xE06, 0x1C2006, 0xE26, 0xE36, 0xE46, 0xE56, 0x1CEE66, 0x1D0006, 0xE96, 0xEA6, 0x1D6006, 0xEC6, 0xED6, 0xEE6, 0xEF6, 0x1E2F06, 0xF26, 0x1E6006, 0xF46, 0x1ECF56, 0x1F0F76, 0xF96, 0x1F4006, 0xFB6, 0x1F8006, 0x1FCFD6, 0x1FE007, 0x200007, 0x1017, 0x207027, 0x20B047, 0x20F067, 0x213087, 0x10A7, 0x2190B7, 0x21A007, 0x21C007, 0x10F7, 0x220007, 0x225117, 0x229137, 0x22D157, 0x22E007, 0x230007, 0x235197, 0x2391B7, 0x23A007, 0x23D0B7, 0x2411F7, 0x242007, 0x1227, 0x249237, 0x24B0E7, 0x24F267, 0x250007, 0x255297, 0x2592B7, 0x25D2D7, 0x2612F7, 0x265317, 0x269338, 0x26D358, 0x271378, 0x272008, 0x274008, 0x276008, 0x278008, 0x27D3D8, 0x27E008, 0x280008, 0x285418, 0x289438, 0x28D458, 0x1478, 0x290008, 0x295498, 0x2994B8, 0x29A008, 0x29F4E8, 0x2A3508, 0x1528, 0x1538, 0x2A8008, 0x1559, 0x2AF569, 0x2B3589, 0x2B4009, 0x2B95B9, 0x2BA009, 0x2BC009, 0x2C15F9, 0x1619, 0x2C7629, 0x2CB649, 0x2CF669, 0x2D3689, 0x2D76A9, 0x16CA, 0x2DA00A, 0x16EA, 0x2DE00A, 0x170A, 0x2E200A, 0x2E772A, 0x2EB74A, 0x2EF76A, 0x2F378A, 0x17AB, 0x2F97BB, 0x2FA00B, 0x2FF7EB, 0x30380B, 0x30782B, 0x184C, 0x30A00C, 0x30F86C, 0x31388C, 0x18AD, 0x18BD, 0x31B8CD, 0x31F8EE, 0x32000E, 0x32591F, }, + // MathSymbol(25): + new long[] { 0x4, 0x8, 0x1000, 0x10, 0x2001, 0x3011, 0x21, 0x1021, 0x31, 0x3001, 0x2031, 0x11, 0x1001, 0x5042, 0x6002, 0x22, 0x72, 0x8002, 0x6072, 0x9002, 0x1042, 0x10A2, 0x10B2, 0x12, 0x5012, 0x7002, 0x10C2, 0x82, 0xC002, 0x1002, 0x8012, 0x42, 0x6042, 0xB2, 0x62, 0xA062, 0xB002, 0x4002, 0xB042, 0xE0D3, 0xF3, 0x103, 0x113, 0x12003, 0x133, 0x15143, 0x1163, 0x1173, 0x183, 0x19003, 0x1A003, 0x15013, 0x1B3, 0x13, 0x13003, 0x1B003, 0x1C003, 0x173, 0x1D003, 0x1E3, 0x1F3, 0x17013, 0x20003, 0x213, 0x23223, 0x14243, 0x14113, 0x253, 0x263, 0xE003, 0x1F003, 0x21253, 0x22003, 0x27004, 0x284, 0x2A294, 0x2B004, 0x2C004, 0x2D004, 0x14, 0x12E4, 0x12F4, 0x31304, 0x32014, 0x33014, 0x34004, 0x354, 0x36004, 0x374, 0x38004, 0x394, 0x13A4, 0x3B4, 0x3C4, 0x3D4, 0x3F3E4, 0x40004, 0x414, 0x42004, 0x434, 0x44004, 0x444, 0x46454, 0x47004, 0x484, 0x3C364, 0x495, 0x4A005, 0x4B005, 0x4C5, 0x4D5, 0x4E005, 0x4F5, 0x51505, 0x52015, 0x54535, 0x555, 0x57565, 0x585, 0x5A595, 0x5C5B5, 0x5D5, 0x595, 0x5F5E5, 0x61605, 0x62005, 0x63005, 0x64005, 0x66655, 0x67005, 0x685, 0x695, 0x4A5, 0x4B5, 0x6A6, 0x6C6B6, 0x6D6, 0x6E006, 0x706F6, 0x72716, 0x736, 0x746, 0x76756, 0x78776, 0x7A796, 0x7C7B6, 0x7E7D6, 0x7F006, 0x806, 0x81006, 0x826, 0x83586, 0x85846, 0x87867, 0x88007, 0x89007, 0x8A007, 0x18B7, 0x8C7, 0x8D007, 0x8E7, 0x908F7, 0x92917, 0x94937, 0x95007, 0x967, 0x98977, 0x9A998, 0x9B8, 0x19C8, 0x9D8, 0x9F9E8, 0xA08, 0x1A18, 0xA3A28, 0xA4008, 0xA58, 0xA6008, 0xA79, 0xA89, 0xAAA99, 0xACAB9, 0xAEAD9, 0xAF9, 0xB1B09, 0xB3B2A, 0xB400A, 0xB6B5A, 0xB8B7A, 0xB9B, 0xBBBAB, 0xBC00B, 0xBDC, 0xBE00C, 0xBF00C, 0xC0D, 0xC2C1D, 0xC3E, 0xC400E, 0xC6C5F, }, + // CurrencySymbol(26): + new long[] { 0x4, 0x8, 0x1000, 0x10, 0x3021, 0x1001, 0x2001, 0x3001, 0x31, 0x11, 0x5042, 0x62, 0x7002, 0x6002, 0x8002, 0x82, 0x9002, 0x92, 0x9052, 0xA003, 0xB003, 0xC3, 0xB3, 0xD3, 0xE3, 0xF3, 0xC003, 0x100E3, 0x113, 0x12003, 0x13004, 0x14004, 0x15004, 0x164, 0x174, 0x184, 0x194, 0x154, 0x1A4, 0x1B4, 0x1C4, 0x18004, 0x19004, 0x1D004, 0x1E5, 0x1F005, 0x205, 0x215, 0x225, 0x15, 0x23005, 0x245, 0x255, 0x265, 0x275, 0x285, 0x29005, 0x2A005, 0x2B5, 0x1F5, 0x2C6, 0x2D006, 0x2E6, 0x2F6, 0x30006, 0x31006, 0x326, 0x33006, 0x346, 0x356, 0x366, 0x376, 0x38006, 0x39006, 0x3A006, 0x3B006, 0x3D3C7, 0x3E007, 0x3F7, 0x40007, 0x417, 0x427, 0x43007, 0x447, 0x457, 0x467, 0x477, 0x48007, 0x497, 0x4B4A7, 0x4D4C8, 0x4E8, 0x4F008, 0x50008, 0x518, 0x52008, 0x54538, 0x558, 0x57568, 0x588, 0x59008, 0x5B5A9, 0x5C009, 0x5D009, 0x5E9, 0x5F9, 0x61609, 0x63629, 0x64009, 0x65A, 0x6600A, 0x6700A, 0x68A, 0x6A69A, 0x6C6BA, 0x6DB, 0x6EB, 0x6F00B, 0x7000B, 0x7271B, 0x73C, 0x7400C, 0x7500C, 0x7776C, 0x78D, 0x79D, 0x7B7AD, 0x7D7CE, 0x7E00E, 0x807FF, }, + // ModifierSymbol(27): + new long[] { 0x4, 0x8, 0x10, 0x1000, 0x21, 0x3001, 0x31, 0x1001, 0x11, 0x2031, 0x1031, 0x2011, 0x2001, 0x4002, 0x5002, 0x62, 0x7002, 0x8012, 0x8002, 0x9002, 0x10A2, 0x4052, 0x92, 0xB2, 0x22, 0x72, 0xC2, 0x1022, 0x8072, 0x52, 0x5042, 0xD003, 0xE003, 0xF3, 0x10003, 0x11013, 0x123, 0xD3, 0x133, 0x14003, 0x153, 0x163, 0x173, 0x10173, 0x183, 0x19003, 0x1A003, 0x171B3, 0x1C003, 0x1D0E3, 0x1E3, 0x1F004, 0x20004, 0x214, 0x22004, 0x234, 0x24004, 0x254, 0x26004, 0x1274, 0x284, 0x2A294, 0x2B2A4, 0x2C4, 0x2D004, 0x2E4, 0x2F014, 0x30234, 0x32314, 0x335, 0x34005, 0x355, 0x36005, 0x375, 0x39385, 0x3A005, 0x3B365, 0x3C005, 0x3E3D5, 0x3F5, 0x40005, 0x415, 0x43425, 0x445, 0x35345, 0x456, 0x47466, 0x49486, 0x4A6, 0x4B006, 0x4C006, 0x4D006, 0x4F4E6, 0x50006, 0x516, 0x526, 0x53006, 0x546, 0x56557, 0x577, 0x58007, 0x5A597, 0x5B7, 0x5C7, 0x5E5D7, 0x5F7, 0x61607, 0x628, 0x638, 0x648, 0x658, 0x66008, 0x678, 0x69688, 0x6A008, 0x6B9, 0x6C9, 0x6D9, 0x6E9, 0x6F009, 0x709, 0x72719, 0x7473A, 0x7500A, 0x76A, 0x7700A, 0x78A, 0x7900A, 0x7AB, 0x7C7BB, 0x7D00B, 0x7EB, 0x7F00B, 0x80C, 0x8100C, 0x82C, 0x8483C, 0x85D, 0x86D, 0x8887D, 0x8A89E, 0x8B00E, 0x8D8CF, }, + // OtherSymbol(28): + new long[] { 0x4, 0x9, 0x2000, 0x10, 0x2001, 0x6021, 0x4001, 0x6001, 0x31, 0x11, 0x2021, 0x2031, 0x6011, 0x4011, 0x4031, 0x21, 0x42, 0xC052, 0x62, 0x72, 0x8002, 0x82, 0xE002, 0x92, 0x2002, 0x20A2, 0xC002, 0x12012, 0x8092, 0x12002, 0x2042, 0x16012, 0x14002, 0xC2, 0x20B2, 0x2092, 0x52, 0x12, 0x1A012, 0x20C2, 0x1C012, 0x160C2, 0xA0B2, 0x8012, 0x1E0B2, 0xE0F2, 0x2032, 0xD2, 0x160A2, 0x2062, 0x18012, 0x20D2, 0x22, 0x100C2, 0x18002, 0x4002, 0x10012, 0x1E002, 0xF2, 0x103, 0x24113, 0x133, 0x2A143, 0x2E163, 0x183, 0x32003, 0x34003, 0x1B3, 0x2003, 0x2193, 0x30003, 0x38013, 0x3A003, 0x24003, 0x2183, 0x3C003, 0x21F3, 0x28003, 0x42203, 0x44013, 0x46013, 0x48213, 0x4A013, 0x4C013, 0x32013, 0x2143, 0x253, 0x2203, 0x36013, 0x2273, 0x36283, 0x2253, 0x52013, 0x21E3, 0x54223, 0x422B3, 0x1D3, 0x2C3, 0x5C2D3, 0x5E173, 0x60143, 0x4A313, 0x64203, 0x173, 0x421A3, 0x44333, 0x681A3, 0x6A1B3, 0x6C003, 0x163, 0x153, 0x70373, 0x393, 0x74003, 0x243, 0x3A3, 0x72003, 0x742D3, 0x783B4, 0x3D4, 0x3E4, 0x803F4, 0x82004, 0x14, 0x2424, 0x86014, 0x88424, 0x2454, 0x8C004, 0x474, 0x84174, 0x92484, 0x4A4, 0x24B4, 0x2464, 0x24C4, 0x4D4, 0x9C004, 0x24F4, 0x2504, 0xA2014, 0x2004, 0x524, 0x88004, 0x8A014, 0x2444, 0xA6014, 0x2544, 0xAA004, 0x564, 0xAE004, 0xB0014, 0x24A4, 0x2574, 0x2594, 0xB4014, 0xB6014, 0xBA5C4, 0xBC014, 0xC05F4, 0xC2004, 0xC4604, 0xC8634, 0xCC654, 0x674, 0x84004, 0x7A004, 0x554, 0xD2684, 0x6A4, 0xD86B4, 0x6D4, 0xDC004, 0x6E4, 0x6F4, 0x424, 0x704, 0x714, 0xDA724, 0x734, 0xCE004, 0xE4004, 0xE8004, 0xEA424, 0x765, 0x775, 0x785, 0x795, 0xF4005, 0xF6015, 0xFA7C5, 0xFC015, 0xFE7D5, 0x2805, 0x102005, 0x106825, 0x10A845, 0x2865, 0x10E015, 0x110005, 0x895, 0x114015, 0x1188B5, 0x28D5, 0x11C015, 0x28F5, 0x2905, 0x124915, 0x2935, 0x2945, 0x12A015, 0x12C005, 0x27A5, 0x130975, 0x1328E5, 0x29A5, 0x1389B5, 0x13A865, 0x13E9E5, 0x140005, 0x144A15, 0xA35, 0x148015, 0x14A005, 0xA65, 0x14E005, 0x148005, 0x150005, 0xA45, 0x154A95, 0x158AB5, 0xAD5, 0xAE5, 0xAF5, 0xB05, 0xB15, 0x166B25, 0x168005, 0x16A005, 0x16C005, 0xB75, 0xB86, 0xB96, 0xBA6, 0x176006, 0x17ABC6, 0x16, 0x17C016, 0x180BF6, 0x184C16, 0x186006, 0x2C46, 0x18CC56, 0xC76, 0xC86, 0x192016, 0x196CA6, 0x198006, 0x2CD6, 0x2CE6, 0x1A0CF6, 0x1A4D16, 0x1A8D36, 0x1ACD56, 0x1B0D76, 0x1B4D96, 0x1B8DB6, 0xDD6, 0xDE6, 0xDF6, 0xE06, 0x1C2006, 0x1C4006, 0x1C8E36, 0x1CA006, 0xE66, 0xE76, 0xE86, 0xE96, 0xEA6, 0xEB6, 0xEC6, 0x1DA006, 0x1DC006, 0x1DE006, 0x1E0006, 0xF17, 0xF27, 0x1E6007, 0x1E8007, 0xF57, 0xF67, 0x1F0F77, 0xF97, 0x1F4007, 0x2FB7, 0x1FAFC7, 0xFE7, 0x200FF7, 0x205017, 0x206017, 0x3047, 0x20D057, 0x211077, 0x215097, 0x216007, 0x21B0C7, 0x21C007, 0x10F7, 0x1107, 0x225117, 0x226007, 0x228007, 0x22D157, 0x1177, 0x1187, 0x235197, 0x11B7, 0x11C7, 0x11D7, 0x11E8, 0x11F8, 0x240008, 0x242008, 0x244008, 0x1238, 0x248018, 0x24D258, 0x251278, 0x252008, 0x12A8, 0x2008, 0x2592B8, 0x25D2D8, 0x12F8, 0x1308, 0x1318, 0x1328, 0x266008, 0x1348, 0x26A008, 0x1368, 0x271378, 0x1398, 0x13A8, 0x2793B8, 0x27D3D8, 0x27E008, 0x283409, 0x287429, 0x288009, 0x28A009, 0x28F469, 0x293489, 0x2974A9, 0x29B4C9, 0x29F4E9, 0x2A3509, 0x1529, 0x2A9539, 0x2AD559, 0x2B1579, 0x2B5599, 0x2B6009, 0x15CA, 0x2BA00A, 0x15EA, 0x15FA, 0x2C000A, 0x2C561A, 0x2C963A, 0x2CA00A, 0x2CF66A, 0x2D368A, 0x2D76AA, 0x16CB, 0x2DD6DB, 0x16FB, 0x2E000B, 0x2E571B, 0x2E973B, 0x2ED75B, 0x177C, 0x2F000C, 0x2F200C, 0x2F77AC, 0x2FB7CC, 0x17ED, 0x17FD, 0x30000D, 0x30581D, 0x30983E, 0x30D85E, 0x31187F, }, + // OtherNotAssigned(29): + new long[] { 0x4, 0xA, 0x4000, 0x10, 0x11, 0x8001, 0x31, 0xC011, 0x4001, 0xC021, 0xC001, 0x4021, 0x21, 0x8011, 0x8031, 0x4031, 0x14042, 0x62, 0x20072, 0x20002, 0x14002, 0x92, 0xA2, 0x12, 0x18002, 0x10012, 0x42, 0x2C002, 0x300A2, 0x32, 0x4052, 0x14062, 0x18012, 0x10002, 0x10082, 0xC2, 0x72, 0x40B2, 0x4082, 0x30002, 0x1C012, 0x40D2, 0x82, 0x200A2, 0x52, 0x18052, 0x10052, 0x4002, 0x18082, 0x1C002, 0x28012, 0x28042, 0x22, 0x380A2, 0x180B2, 0x3C0B2, 0xD2, 0x28002, 0x38012, 0x40F2, 0x40E2, 0x1C0B2, 0x2C072, 0x20052, 0x40C2, 0x24042, 0x28052, 0x24002, 0x30042, 0x14032, 0x34012, 0x100A2, 0x280E2, 0x24062, 0xC082, 0xF2, 0x180A2, 0x4103, 0x4C123, 0x113, 0x50003, 0x153, 0x58003, 0x44173, 0x60003, 0x193, 0x1A3, 0x13, 0x4C003, 0x6C003, 0x64013, 0x70003, 0x441D3, 0x6C1E3, 0x44013, 0x173, 0x441B3, 0x5C013, 0x7C013, 0x41B3, 0x80013, 0x213, 0x223, 0x233, 0x243, 0x5C253, 0x68263, 0x78003, 0x4C203, 0x44143, 0x68013, 0x273, 0xA0253, 0x253, 0x8C293, 0x5C143, 0x8C003, 0xA8003, 0x203, 0xA0013, 0x84003, 0xAC113, 0xA82C3, 0x23, 0x2D3, 0x283, 0x2E3, 0x1B3, 0x5C2F3, 0x2A3, 0x303, 0x90013, 0xC4003, 0xC8013, 0x163, 0xCC003, 0x44003, 0xCC113, 0x333, 0x44333, 0x143, 0x741A3, 0xD0003, 0xD4183, 0x363, 0x48013, 0x98003, 0x74003, 0x44373, 0x9C383, 0x44133, 0x14003, 0xBC003, 0x58393, 0xE4003, 0xE8013, 0xF03B3, 0x58333, 0xB0213, 0x393, 0x94003, 0xF0203, 0xF4013, 0xC0333, 0xF8303, 0xCC353, 0xFC303, 0x78403, 0x54413, 0x108213, 0x10C003, 0x14153, 0x64143, 0x8C443, 0x9C003, 0x14233, 0x118453, 0xD8413, 0x120473, 0x14363, 0x94403, 0x54493, 0x128213, 0xB0153, 0x4B3, 0xBC4C3, 0x133, 0x444D4, 0x4C4E4, 0x4F4, 0x140004, 0x514, 0x14C524, 0x544, 0x554, 0x4564, 0x15C014, 0x584, 0x4004, 0x168594, 0x16C004, 0x5C4, 0x1785D4, 0x158004, 0x5F4, 0x180004, 0x184004, 0x158554, 0x17C004, 0x44004, 0x7C624, 0x5E4, 0x190634, 0x158654, 0x19C664, 0x684, 0x694, 0x1A8554, 0x574, 0x1A0554, 0x46B4, 0x1B0014, 0x17C014, 0x178004, 0x1A0004, 0x13C004, 0x504, 0x1AC004, 0x6D4, 0x45F4, 0x6E4, 0x614, 0x6F4, 0x644, 0x1C4704, 0x1C8004, 0x734, 0x744, 0x754, 0x1D8014, 0x178014, 0x774, 0x13C684, 0x148784, 0x794, 0x1487A4, 0x524, 0x564, 0x687B4, 0x1A4, 0x764, 0x15C004, 0x7C4, 0x1F4004, 0x1F8004, 0x7F4, 0x804, 0x1DC014, 0x814, 0x154824, 0x158834, 0x210004, 0x1D8004, 0x7D4, 0x13C554, 0x218854, 0x218834, 0x874, 0x21C884, 0x894, 0x228004, 0x2288B4, 0x218764, 0x864, 0x8C4, 0x1A0504, 0x230004, 0x234654, 0x238564, 0x23C004, 0x904, 0x248914, 0x250934, 0x954, 0x25C964, 0x1D4004, 0x264984, 0x2189A4, 0x1D49B4, 0x2749C4, 0x27C9E4, 0x1D4A04, 0x274A14, 0x27C7E4, 0x248764, 0x28CA24, 0x290664, 0x298A54, 0x1D4574, 0x2A0A74, 0x27CA94, 0x2ACAA4, 0x1D4AC4, 0x2B4774, 0x2BCAE4, 0x2485E4, 0x2C4B04, 0x27CB24, 0x2CCAA4, 0x1D44F4, 0x2D4B44, 0x27CB64, 0x2DCAA4, 0x15C594, 0xB84, 0x1584F4, 0x5A4, 0x1F0004, 0x19C004, 0x240834, 0x1F0504, 0x2B8004, 0x2E4004, 0xBA4, 0x2F0BB5, 0xBD5, 0x2F8005, 0xBF5, 0x304C05, 0x308005, 0x310C35, 0x314005, 0x318005, 0x31C005, 0x320005, 0x328C95, 0x308015, 0xCB5, 0x334CC5, 0xCE5, 0xCF5, 0xD05, 0x348D15, 0x350D35, 0x354005, 0xD35, 0x358BD5, 0x360D75, 0x364005, 0xDA5, 0xDB5, 0x374DC5, 0xDE5, 0x37C005, 0x380005, 0xD75, 0x2F4005, 0x384005, 0x38CE25, 0x390005, 0x398E55, 0xE75, 0x3A0005, 0x3A4015, 0x44005, 0x34C115, 0xEA5, 0xEB5, 0xEC5, 0xE15, 0x3B4005, 0xEE5, 0x3BC005, 0xF05, 0x340005, 0x318F15, 0x3C8005, 0xF35, 0x3D4F45, 0xF65, 0x3E0F75, 0x3E4005, 0xE85, 0x3ECFA5, 0x3F4FC5, 0x3F8005, 0xFF5, 0x1005, 0x340D75, 0x404015, 0x154CB5, 0x2F4BF5, 0x408005, 0x40C005, 0x415045, 0x2F45F5, 0xE65, 0x1065, 0x1075, 0x420005, 0x3DC555, 0x39D095, 0x428D35, 0x35C005, 0xF95, 0x3F0D75, 0x42C005, 0x430005, 0x4390D5, 0x43C005, 0x3AC005, 0x440015, 0x449115, 0x451135, 0x454015, 0x45D165, 0x330015, 0x2F9185, 0x469195, 0x4711B5, 0x4791D5, 0x47C005, 0x485205, 0x47D225, 0x491235, 0x47D255, 0x49D265, 0x4A5285, 0x4AD2A5, 0x4B52C5, 0x4BD2E5, 0x4C12C5, 0x4C9315, 0x4D1335, 0x4D9355, 0x4E1375, 0x4E4005, 0x4F65, 0x4E8EF5, 0x13B5, 0xFC5, 0x13C5, 0x4F4005, 0x4F8005, 0x444005, 0x3ED3F5, 0x500005, 0xE25, 0x509415, 0x1435, 0x515446, 0x518006, 0x1476, 0x525486, 0x52D4A6, 0x14C6, 0x14D6, 0x53D4E6, 0x1506, 0x1516, 0x54D526, 0x1546, 0x559556, 0x1576, 0x565586, 0x15A6, 0x15B6, 0x15C6, 0x5795D6, 0x15F6, 0x15D6, 0x580006, 0x589616, 0x1636, 0x590006, 0x595536, 0x598006, 0x59C006, 0x1466, 0x1686, 0x5A9696, 0x16B6, 0x5B56C6, 0x5BD6E6, 0x1706, 0x5C5646, 0x5C8006, 0x1736, 0x5D5746, 0x5D8006, 0x5E1776, 0x1796, 0x5ED7A6, 0x5F57C6, 0x17E6, 0x6017F6, 0x609816, 0x1836, 0x1846, 0x619856, 0x621876, 0x555896, 0x62D8A6, 0x6358C6, 0x63D8E6, 0x1906, 0x649916, 0x64D8E6, 0x525946, 0x659956, 0x661976, 0x1996, 0x19A6, 0x6719B6, 0x19D6, 0x67D9E6, 0x685A06, 0x68DA26, 0x695A46, 0x69DA66, 0x6A5A86, 0x6ADAA6, 0x6B5AC6, 0x6BDAE6, 0x6C5B06, 0x6CDB26, 0x6D5B46, 0x6DDB66, 0x6E5B86, 0x6EDBA6, 0x6F0006, 0x6F9BD6, 0x6FD676, 0x700006, 0x1C16, 0x70DC27, 0x715C47, 0x71DC67, 0x725C87, 0x72DCA7, 0x1CC7, 0x739CD7, 0x741CF7, 0x749D17, 0x751D37, 0x1D57, 0x75DD67, 0x1D87, 0x1D97, 0x768007, 0x771DB7, 0x779DD7, 0x1DF7, 0x785E07, 0x78DE27, 0x795E47, 0x79DE67, 0x7A0007, 0x1E97, 0x7ADEA7, 0x7B5EC7, 0x1EE7, 0x7C1EF7, 0x7C9F17, 0x7D1F37, 0x7D9F57, 0x7E1F77, 0x7E9F97, 0x1FB7, 0x7F5FC7, 0x7FDFE7, 0x2007, 0x80A017, 0x812037, 0x81A057, 0x822077, 0x82A097, 0x8320B7, 0x20D7, 0x83E0E7, 0x846107, 0x848007, 0x852137, 0x85A157, 0x862178, 0x2198, 0x86E1A8, 0x21C8, 0x87A1D8, 0x8821F8, 0x88A218, 0x88C008, 0x2248, 0x894008, 0x89E268, 0x8A6288, 0x8AE2A8, 0x22C8, 0x8B4008, 0x8BE2E8, 0x2308, 0x8CA318, 0x8D2338, 0x8DA358, 0x8E2378, 0x8EA398, 0x8EC008, 0x8F63C8, 0x8FE3E8, 0x906408, 0x90E428, 0x916448, 0x2468, 0x2478, 0x926489, 0x24A9, 0x24B9, 0x9364C9, 0x93E4E9, 0x2509, 0x94A519, 0x952539, 0x2559, 0x958009, 0x95C009, 0x966589, 0x96E5A9, 0x25C9, 0x97A5D9, 0x9825F9, 0x98A619, 0x992639, 0x2659, 0x99E66A, 0x268A, 0x9A400A, 0x26AA, 0x26BA, 0x9B000A, 0x9BA6DA, 0x9C26FA, 0x9CA71A, 0x9D273A, 0x9DA75A, 0x9E277A, 0x279B, 0x9E800B, 0x9F27BB, 0x27DB, 0x9F800B, 0xA027FB, 0xA0A81B, 0xA1283B, 0x285C, 0x286C, 0xA1C00C, 0x288C, 0xA2A89C, 0xA328BC, 0xA3A8DD, 0xA428FD, 0xA4A91D, 0xA5293E, 0xA5400E, 0xA5E96F, }, + }; + + /// Serialized BDD representation of the set of all whitespace characters. + public static readonly long[] WhitespaceSerializedBDD = new long[] { 0x4, 0x6, 0x400, 0x10, 0x801, 0x31, 0x401, 0x421, 0xC01, 0xC11, 0x1002, 0x52, 0x1852, 0x1C02, 0x82, 0x2462, 0x2803, 0xB3, 0xC3, 0x4D3, 0x3803, 0xF3, 0x4004, 0x114, 0x4804, 0x4C04, 0x5004, 0x5404, 0x5805, 0x5C05, 0x6585, 0x6965, 0x6D65, 0x7006, 0x79D6, 0x7C06, 0x8006, 0x8407, 0x8807, 0x217, 0x9237, 0x9408, 0x9808, 0x9C08, 0xA008, 0xA409, 0xA809, 0x2B9, 0xB009, 0xB40A, 0xB80A, 0x2FA, 0xC00A, 0xC40B, 0xC80B, 0xCC0B, 0xD00B, 0xDB5C, 0xE37C, 0xEB9D, 0xEC0E, 0xF00F, }; + + /// Serialized BDD representation of the set of all word characters + public static readonly long[] WordCharactersSerializedBDD = new long[] { 0x4, 0xA, 0x4000, 0x10, 0x8001, 0x11, 0x4021, 0xC011, 0x31, 0x4001, 0x8011, 0x4031, 0xC001, 0x21, 0xC021, 0x8031, 0x14042, 0x14012, 0x4062, 0x52, 0x18002, 0x1C012, 0x18082, 0x24002, 0x4042, 0x40A2, 0x72, 0x20042, 0x4002, 0x4092, 0x20012, 0x2C0A2, 0x4022, 0x28012, 0x100C2, 0x180C2, 0x18012, 0x1C062, 0x14022, 0x18092, 0x20092, 0x340A2, 0x82, 0x4052, 0x20072, 0x92, 0x42, 0x180E2, 0x3C012, 0x40B2, 0x12, 0x38072, 0x1C092, 0x14092, 0xE2, 0xC2, 0x240D2, 0x24012, 0x280B2, 0x22, 0x40C2, 0x34082, 0x10002, 0x30072, 0x30002, 0x140A2, 0x2C012, 0x18072, 0x24072, 0x24032, 0x28092, 0x34072, 0x18042, 0x4072, 0x40F2, 0x8072, 0x3C002, 0x1C002, 0x34002, 0x4082, 0xD2, 0x28002, 0x240C2, 0x240E2, 0x32, 0x4032, 0x380A2, 0x62, 0x1C042, 0x38002, 0x240B2, 0x10052, 0x10012, 0x18052, 0x14072, 0xB2, 0x28072, 0x1C0B2, 0x1C022, 0x3C092, 0x38082, 0x24042, 0x280E2, 0x8052, 0x340C2, 0x24052, 0x380B2, 0x180A2, 0x140B2, 0x44103, 0x4123, 0x4C013, 0x4143, 0x54013, 0x4163, 0x4173, 0x4183, 0x64013, 0x1A3, 0x6C003, 0x41C3, 0x4003, 0x44013, 0x41D3, 0x78013, 0x5C003, 0x7C013, 0x48203, 0x4213, 0x781A3, 0x48003, 0x481E3, 0x70003, 0x8C223, 0x4243, 0x94003, 0x98003, 0x4273, 0x283, 0x50003, 0x48293, 0x38003, 0x2A3, 0xAC093, 0x90013, 0x42C3, 0xB4003, 0xBC2E3, 0x68013, 0x44143, 0x4303, 0xA8013, 0x74003, 0xC8313, 0x48333, 0xD01B3, 0x353, 0xD8003, 0x42A3, 0xC4373, 0x601E3, 0x68383, 0x4393, 0xEC3A3, 0xF0343, 0xF4113, 0xF8003, 0xFC013, 0x1002A3, 0xB8003, 0x104183, 0xAC1C3, 0x64203, 0x4033, 0x4423, 0x48143, 0x84013, 0x4433, 0x43E3, 0x4443, 0xAC003, 0x70323, 0x48453, 0x118423, 0x108013, 0x1E3, 0x11C1C3, 0x64483, 0x124013, 0x4313, 0xA8003, 0x784A3, 0x12C183, 0x80123, 0x44C3, 0x60003, 0x1384D3, 0x583E3, 0x13C2D3, 0x140153, 0xC41D3, 0x144003, 0x4193, 0x543E3, 0x44A3, 0x118013, 0x48013, 0x118123, 0x4463, 0x48463, 0x4523, 0xBC363, 0x14C133, 0x74013, 0x84333, 0x140003, 0x4153, 0x54123, 0x150403, 0x173, 0x801D3, 0x154013, 0x158243, 0x484E3, 0x1083C3, 0x48113, 0x113, 0xAC013, 0x15C2A3, 0x160363, 0xE4193, 0x164003, 0x503, 0x1402D3, 0x1245A3, 0x110193, 0xD8143, 0x16C463, 0x170193, 0xF8373, 0x6C5D3, 0x118563, 0x1785D3, 0x100003, 0xD85F3, 0x160373, 0x1805A3, 0xC0013, 0xFC4A3, 0x153, 0x184223, 0x180093, 0xCC313, 0x100623, 0x190633, 0x198653, 0xCC163, 0x5C3B3, 0x5F3, 0x19C093, 0x1104A3, 0x54223, 0x74683, 0x1244A3, 0x4693, 0x1A8003, 0x48403, 0x108113, 0xC4013, 0x1406B3, 0x646C3, 0x603B3, 0x1A4013, 0xDC4E3, 0x4E3, 0x446D4, 0x46E4, 0x1BC014, 0x1C4704, 0x1C4724, 0x734, 0x4744, 0x754, 0x1D8004, 0x4774, 0x4004, 0x784, 0x1E4004, 0x47A4, 0x14, 0x47B4, 0x7C4, 0x47D4, 0x1F8014, 0x2007F4, 0x208814, 0x4734, 0x1EC014, 0x4784, 0x20C014, 0x210014, 0x214734, 0x1CC014, 0x4864, 0x48014, 0x94874, 0x224884, 0x22C8A4, 0x48C4, 0x1D0734, 0x1B88D4, 0x238734, 0x48F4, 0x240014, 0x4914, 0x248734, 0x4844, 0x1E8014, 0x24C004, 0x944, 0x254004, 0x258014, 0x974, 0x2607D4, 0x4994, 0x1E0004, 0x1D0014, 0x1D0004, 0x1B8004, 0x4714, 0x250014, 0x1C4014, 0x49A4, 0x2709B4, 0x274004, 0x238004, 0x4824, 0x2788E4, 0x27C014, 0x280844, 0x1B8014, 0x1DC004, 0x284004, 0x28CA24, 0x294A44, 0x29CA64, 0x744, 0xA84, 0x298A94, 0x8E4, 0x1D0AA4, 0x2A8AB4, 0x4AC4, 0x74AD4, 0x41D4, 0x4AE4, 0x2C0AF4, 0x904, 0x4B14, 0x2C8014, 0x2CC014, 0x2D4B44, 0x2D87D4, 0x2DC014, 0x2E0014, 0x2E4844, 0x2E8734, 0x4BB4, 0x2F0014, 0x298004, 0x2F8BD4, 0x300BF4, 0x4C14, 0x4C24, 0x1E0014, 0x1ECC34, 0x314C44, 0x4984, 0x1C0014, 0x4B24, 0x228734, 0x1CC004, 0x31CC64, 0x4C84, 0x328C94, 0x4C34, 0x328C34, 0x1C0CB4, 0x330014, 0x4CD4, 0x334CE4, 0x4CF4, 0x340014, 0x340D14, 0x34CD24, 0x350014, 0x1CCD54, 0x2C4D64, 0x35C004, 0x1D4D84, 0x1D0D94, 0x360014, 0x1CCDA4, 0x298DB4, 0x370A24, 0x3747B4, 0x378014, 0x1B8734, 0x4704, 0x1BC004, 0x37C884, 0x4E04, 0x29CE14, 0x38CE24, 0x394E44, 0x398B04, 0x38CE74, 0x3A4E84, 0x200EA4, 0x3ACB04, 0x38C004, 0x3B4EC4, 0x200B34, 0x3B8B04, 0x3BCA64, 0x3C0734, 0x3C8F14, 0x38CF34, 0x3D4F44, 0x200F64, 0x3E0F74, 0x38CF94, 0x3E8A64, 0x200FB4, 0x3F0B84, 0x1BC984, 0x3F8FD4, 0x200FF4, 0x400F74, 0x38D014, 0x395024, 0x201034, 0x410F74, 0x38C714, 0x414014, 0x4894, 0x1E47C4, 0x4754, 0x1C0004, 0x1EC004, 0x5064, 0x2C4014, 0x200CC4, 0x5074, 0x420014, 0x704, 0x1C88B4, 0x424014, 0x4BD4, 0x2C8714, 0x50A4, 0x42C014, 0x50C4, 0x434014, 0x50E4, 0x43C004, 0x4407D4, 0x4CE4, 0x449114, 0x44C005, 0x5145, 0x459155, 0x461175, 0x5195, 0x46D1A5, 0x4751C5, 0x47D1E5, 0x480015, 0x484015, 0x5225, 0x48C015, 0x5245, 0x494015, 0x49D265, 0x4A0005, 0x5295, 0x4AD2A5, 0x52C5, 0x52D5, 0x4BD2E5, 0x4C5305, 0x4CD325, 0x4D5345, 0x5365, 0x4DD145, 0x4E5385, 0x491195, 0x4ED3A5, 0x4F53C5, 0x53C5, 0x53E5, 0x5013F5, 0x504015, 0x5093D5, 0x5435, 0x5445, 0x519455, 0x485225, 0x51C005, 0x1215, 0x520015, 0x529495, 0x52F5, 0x5314B5, 0x52C015, 0x5394D5, 0x14F5, 0x48015, 0x540125, 0x5515, 0x5525, 0x5535, 0x450015, 0x5545, 0x554005, 0x558005, 0x561575, 0x569595, 0x568005, 0x5715B5, 0x15D5, 0x57D5E5, 0x5605, 0x585225, 0x588015, 0x5635, 0x595645, 0x59D665, 0x5A1225, 0x5695, 0x56A5, 0x5B16B5, 0x5B96D5, 0x5C16F5, 0x5C4005, 0x1CD725, 0x5D1735, 0x5D4015, 0x1CC005, 0x5765, 0x455775, 0x451785, 0x5795, 0x57A5, 0x57B5, 0x5F0015, 0x5F97D5, 0x57F5, 0x4F1805, 0x609815, 0x5835, 0x458015, 0x5845, 0x5625, 0x474015, 0x598005, 0x614015, 0x618015, 0x621875, 0x624015, 0x5492F5, 0x628015, 0x6318B5, 0x58D5, 0x638005, 0x63D195, 0x645905, 0x64D925, 0x650005, 0x659955, 0x65C005, 0x52D985, 0x669995, 0x6719B5, 0x6799D5, 0x67D225, 0x685A05, 0x68DA25, 0x695A45, 0x69DA65, 0x6A1995, 0x6A9A95, 0x6B1AB5, 0x6B9AD5, 0x6C1AF5, 0x6C9B15, 0x6D1B35, 0x6D9B55, 0x6E1B75, 0x6E9B95, 0x6F1BB5, 0x4E5BD5, 0x6F96C5, 0x1BF5, 0x4A9C05, 0x5C15, 0x5C25, 0x70C015, 0x715C45, 0x5C65, 0x71C015, 0x725C85, 0x72DCA5, 0x730015, 0x5495, 0x734015, 0x5CE5, 0x73D6C5, 0x5D05, 0x749D15, 0x1D45, 0x45D165, 0x1185, 0x759D56, 0x761D76, 0x5D96, 0x76DDA6, 0x775DC6, 0x77DDE6, 0x785E06, 0x78DE26, 0x5E46, 0x5E56, 0x79DE66, 0x5E86, 0x7A9E96, 0x5EB6, 0x7B5EC6, 0x7BDEE6, 0x795F06, 0x7C9F16, 0x7D1F36, 0x7D9F56, 0x7DC016, 0x5F86, 0x7E9F96, 0x7F1FB6, 0x4006, 0x7F9FD6, 0x7FC016, 0x806006, 0x80E026, 0x810006, 0x81A056, 0x822076, 0x6096, 0x82E0A6, 0x830006, 0x83A0D6, 0x8420F6, 0x84A116, 0x852136, 0x85A156, 0x6176, 0x866186, 0x86E1A6, 0x8761C6, 0x87E1E6, 0x886206, 0x88E226, 0x896246, 0x89E266, 0x8A6286, 0x8AE2A6, 0x8B62C6, 0x8BE2E6, 0x8C6306, 0x6326, 0x808016, 0x8D2336, 0x8DA356, 0x8E2376, 0x6396, 0x8EE3A6, 0x8F63C6, 0x8FE3E6, 0x906406, 0x90E426, 0x916446, 0x91E466, 0x926486, 0x92E4A6, 0x9364C6, 0x93E4E6, 0x946506, 0x94E526, 0x956546, 0x95E566, 0x966586, 0x96E5A6, 0x9765C6, 0x7865E6, 0x77A5F6, 0x6606, 0x98A616, 0x992636, 0x99A656, 0x99C016, 0x9A6686, 0x66A6, 0x66B6, 0x9B1D36, 0x9BA6D6, 0x9C26F7, 0x9CA717, 0x9D2737, 0x9DA757, 0x9E2777, 0x6797, 0x9EE7A7, 0x9F67C7, 0x9FE7E7, 0xA06807, 0xA0E827, 0xA16847, 0x6867, 0x6717, 0x6877, 0xA26887, 0xA2E8A7, 0xA30007, 0xA3A8D7, 0xA428F7, 0xA4A917, 0xA52937, 0xA5A957, 0x6977, 0xA66987, 0xA6E9A7, 0xA769C7, 0xA7E9E7, 0xA86A07, 0xA8EA27, 0xA96A47, 0xA98017, 0xAA2A77, 0xAAAA97, 0xAB2AB7, 0xABAAD7, 0xAC2AF7, 0xACAB17, 0xAD2B37, 0xADAB57, 0xAE2B77, 0xAEAB97, 0xAF2BB7, 0xAFABD7, 0xB02BF7, 0xB0AC17, 0x6C37, 0xB16C47, 0x6C67, 0xB22C77, 0xB2AC98, 0x6CB8, 0xB36CC8, 0x18, 0x6CE8, 0xB42CF8, 0xB4AD18, 0xB52D38, 0xB54018, 0x6D68, 0x6D78, 0xB66D88, 0xB68008, 0xB72DB8, 0xB7ADD8, 0x6DF8, 0xB86E08, 0xB8EE28, 0xB96E48, 0xB9EE68, 0xBA0018, 0xBAAE98, 0xBAC018, 0xBB6EC8, 0xBBEEE8, 0xBC6F08, 0xBCEF28, 0xBD6F48, 0xBDEF68, 0xBE6F88, 0xBE8018, 0xBF2FB9, 0xBFAFD9, 0x6FF9, 0xC07009, 0xC0F029, 0x7049, 0xC14019, 0xC18009, 0xC23079, 0xC24009, 0xC2F0A9, 0xC370C9, 0xC3F0E9, 0xC47109, 0xC4F129, 0xC57149, 0xC5F169, 0xC67189, 0xC6F1AA, 0x71CA, 0xC7401A, 0x71EA, 0x71FA, 0x720A, 0xC8401A, 0x322A, 0xC8C00A, 0xC9724A, 0xC9F26A, 0xCA728A, 0xCAF2AA, 0x32CB, 0xCB400B, 0xCBF2EB, 0x730B, 0x731B, 0xCC801B, 0xCD333B, 0xCDB35B, 0xCE337B, 0x339C, 0x73AC, 0xCEC01C, 0x73CC, 0xCF401C, 0xCFF3EC, 0xD0740C, 0xD0F42D, 0xD1744D, 0xD1801D, 0xD2347D, 0xD2B49E, 0xD334BE, 0xD3B4DF, }; + } +} diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Unicode/UnicodeCategoryRangesGenerator.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Unicode/UnicodeCategoryRangesGenerator.cs new file mode 100644 index 00000000000000..b705f88247d6cc --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Unicode/UnicodeCategoryRangesGenerator.cs @@ -0,0 +1,125 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.IO; + +namespace System.Text.RegularExpressions.Symbolic.Unicode +{ +#if DEBUG + /// Utility for generating unicode category ranges and corresponing binary decision diagrams. + internal static class UnicodeCategoryRangesGenerator + { + /// Generator for BDD Unicode category definitions. + /// namespace for the class + /// name of the class + /// path where the file classname.cs is written + public static void Generate(string namespacename, string classname, string path) + { + Debug.Assert(namespacename != null); + Debug.Assert(classname != null); + Debug.Assert(path != null); + + using StreamWriter sw = new StreamWriter($"{Path.Combine(path, classname)}.cs"); + sw.WriteLine( +$@"// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// This is a programmatically generated file from Regex.GenerateUnicodeTables. +// It provides serialized BDD Unicode category definitions for System.Environment.Version = {Environment.Version} + +namespace {namespacename} +{{ + internal static class {classname} + {{"); + WriteSerializedBDDs(sw); + sw.WriteLine($@" }} +}}"); + } + + private static void WriteSerializedBDDs(StreamWriter sw) + { + int maxChar = 0xFFFF; + var catMap = new Dictionary(); + foreach (UnicodeCategory c in Enum.GetValues()) + { + catMap[c] = new Ranges(); + } + + Ranges whitespace = new Ranges(); + Ranges wordcharacter = new Ranges(); + Regex whitespaceRegex = new(@"\s"); + Regex wordcharRegex = new(@"\w"); + for (int i = 0; i <= maxChar; i++) + { + char ch = (char)i; + catMap[char.GetUnicodeCategory(ch)].Add(i); + + if (whitespaceRegex.IsMatch(ch.ToString())) + whitespace.Add(i); + + if (wordcharRegex.IsMatch(ch.ToString())) + wordcharacter.Add(i); + } + + //generate bdd reprs for each of the category ranges + BDD[] catBDDs = new BDD[catMap.Count]; + CharSetSolver bddb = new CharSetSolver(); + for (int c = 0; c < catBDDs.Length; c++) + catBDDs[c] = bddb.CreateBddForIntRanges(catMap[(UnicodeCategory)c].ranges); + + BDD whitespaceBdd = bddb.CreateBddForIntRanges(whitespace.ranges); + + BDD wordCharBdd = bddb.CreateBddForIntRanges(wordcharacter.ranges); + + sw.WriteLine(" /// Serialized BDD representations of all the Unicode categories."); + sw.WriteLine(" public static readonly long[][] AllCategoriesSerializedBDD = new long[][]"); + sw.WriteLine(" {"); + for (int i = 0; i < catBDDs.Length; i++) + { + sw.WriteLine(" // {0}({1}):", (UnicodeCategory)i, i); + sw.Write(" "); + GeneratorHelper.WriteInt64ArrayInitSyntax(sw, catBDDs[i].Serialize()); + sw.WriteLine(","); + } + sw.WriteLine(" };"); + sw.WriteLine(); + + sw.WriteLine(" /// Serialized BDD representation of the set of all whitespace characters."); + sw.Write($" public static readonly long[] WhitespaceSerializedBDD = "); + GeneratorHelper.WriteInt64ArrayInitSyntax(sw, whitespaceBdd.Serialize()); + sw.WriteLine(";"); + sw.WriteLine(); + + sw.WriteLine(" /// Serialized BDD representation of the set of all word characters"); + sw.Write($" public static readonly long[] WordCharactersSerializedBDD = "); + GeneratorHelper.WriteInt64ArrayInitSyntax(sw, wordCharBdd.Serialize()); + sw.WriteLine(";"); + } + } + + /// Used internally for creating a collection of ranges for serialization. + internal sealed class Ranges + { + public readonly List ranges = new List(); + + public void Add(int n) + { + for (int i = 0; i < ranges.Count; i++) + { + if (ranges[i][1] == (n - 1)) + { + ranges[i][1] = n; + return; + } + } + + ranges.Add(new int[] { n, n }); + } + + public int Count => ranges.Count; + } +#endif +} diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Unicode/UnicodeCategoryTheory.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Unicode/UnicodeCategoryTheory.cs new file mode 100644 index 00000000000000..f167d563c20b42 --- /dev/null +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/Unicode/UnicodeCategoryTheory.cs @@ -0,0 +1,74 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace System.Text.RegularExpressions.Symbolic.Unicode +{ + internal sealed class UnicodeCategoryTheory where TPredicate : class + { + internal readonly ICharAlgebra _solver; + private readonly TPredicate[] _catConditions = new TPredicate[30]; + + private TPredicate? _whiteSpaceCondition; + private TPredicate? _wordLetterCondition; + private TPredicate? _wordLetterConditionForAnchors; + + public UnicodeCategoryTheory(ICharAlgebra solver) => _solver = solver; + + public TPredicate CategoryCondition(int i) + { + if (_catConditions[i] is not TPredicate condition) + { + BDD bdd = BDD.Deserialize(UnicodeCategoryRanges.AllCategoriesSerializedBDD[i], _solver.CharSetProvider); + _catConditions[i] = condition = _solver.ConvertFromCharSet(_solver.CharSetProvider, bdd); + } + + return condition; + } + + public TPredicate WhiteSpaceCondition + { + get + { + if (_whiteSpaceCondition is not TPredicate condition) + { + BDD bdd = BDD.Deserialize(UnicodeCategoryRanges.WhitespaceSerializedBDD, _solver.CharSetProvider); + _whiteSpaceCondition = condition = _solver.ConvertFromCharSet(_solver.CharSetProvider, bdd); + } + + return condition; + } + } + + public TPredicate WordLetterCondition + { + get + { + if (_wordLetterCondition is not TPredicate condition) + { + BDD bdd = BDD.Deserialize(UnicodeCategoryRanges.WordCharactersSerializedBDD, _solver.CharSetProvider); + _wordLetterCondition = condition = _solver.ConvertFromCharSet(_solver.CharSetProvider, bdd); + } + + return condition; + } + } + + public TPredicate WordLetterConditionForAnchors + { + get + { + if (_wordLetterConditionForAnchors is not TPredicate condition) + { + // Create the condition from WordLetterCondition together with the characters + // \u200C (zero width non joiner) and \u200D (zero width joiner) that are treated + // as if they were word characters in the context of the anchors \b and \B + BDD extra_bdd = _solver.CharSetProvider.CreateCharSetFromRange('\u200C', '\u200D'); + TPredicate extra_pred = _solver.ConvertFromCharSet(_solver.CharSetProvider, extra_bdd); + _wordLetterConditionForAnchors = condition = _solver.Or(WordLetterCondition, extra_pred); + } + + return condition; + } + } + } +} diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/ThrowHelper.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/ThrowHelper.cs index cd7d260882e733..7d7f3281216f6b 100644 --- a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/ThrowHelper.cs +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/ThrowHelper.cs @@ -57,7 +57,6 @@ internal static void ThrowNotSupportedException(ExceptionResource resource) => ExceptionResource.BeginIndexNotNegative => SR.BeginIndexNotNegative, ExceptionResource.CountTooSmall => SR.CountTooSmall, ExceptionResource.LengthNotNegative => SR.LengthNotNegative, - ExceptionResource.OnlyAllowedOnce => SR.OnlyAllowedOnce, ExceptionResource.ReplacementError => SR.ReplacementError, _ => null }; @@ -90,7 +89,6 @@ internal enum ExceptionResource BeginIndexNotNegative, CountTooSmall, LengthNotNegative, - OnlyAllowedOnce, ReplacementError, } } diff --git a/src/libraries/System.Text.RegularExpressions/tests/AttRegexTests.cs b/src/libraries/System.Text.RegularExpressions/tests/AttRegexTests.cs index 24d27c049641cb..aba276d49b3806 100644 --- a/src/libraries/System.Text.RegularExpressions/tests/AttRegexTests.cs +++ b/src/libraries/System.Text.RegularExpressions/tests/AttRegexTests.cs @@ -118,8 +118,8 @@ public class AttRegexTests [InlineData("aba|bab", "baaabbbaba", "(6,9)")] [InlineData("(aa|aaa)*|(a|aaaaa)", "aa", "(0,2)(0,2)")] [InlineData("(a.|.a.)*|(a|.a...)", "aa", "(0,2)(0,2)")] - [InlineData("ab|a", "xabc", "(1,3)")] - [InlineData("ab|a", "xxabc", "(2,4)")] + [InlineData("ab|a", "xabc", "(1,3)", "(1,2)")] + [InlineData("ab|a", "xxabc", "(2,4)", "(2,3)")] [InlineData("(?i)(Ab|cD)*", "aBcD", "(0,4)(2,4)")] [InlineData("[^-]", "--a", "(2,3)")] [InlineData("[a-]*", "--a", "(0,3)")] @@ -285,18 +285,18 @@ public class AttRegexTests [InlineData("X(.?){6,8}Y", "X1234567Y", "(0,9)(8,8)")] // was "(0,9)(7,8)" [InlineData("X(.?){7,8}Y", "X1234567Y", "(0,9)(8,8)")] // was "(0,9)(7,8)" [InlineData("X(.?){8,8}Y", "X1234567Y", "(0,9)(8,8)")] - [InlineData("(a|ab|c|bcd){0,}(d*)", "ababcd", "(0,1)(1,1)")] // was "(0,6)(3,6)(6,6)" - [InlineData("(a|ab|c|bcd){1,}(d*)", "ababcd", "(0,1)(1,1)")] // was "(0,6)(3,6)(6,6)" + [InlineData("(a|ab|c|bcd){0,}(d*)", "ababcd", "(0,1)(1,1)", "(0,6)")] // was "(0,6)(3,6)(6,6)" + [InlineData("(a|ab|c|bcd){1,}(d*)", "ababcd", "(0,1)(1,1)", "(0,6)")] // was "(0,6)(3,6)(6,6)" [InlineData("(a|ab|c|bcd){2,}(d*)", "ababcd", "(0,6)(3,6)(6,6)")] [InlineData("(a|ab|c|bcd){3,}(d*)", "ababcd", "(0,6)(3,6)(6,6)")] [InlineData("(a|ab|c|bcd){4,}(d*)", "ababcd", "NOMATCH")] - [InlineData("(a|ab|c|bcd){0,10}(d*)", "ababcd", "(0,1)(1,1)")] // was "(0,6)(3,6)(6,6)" - [InlineData("(a|ab|c|bcd){1,10}(d*)", "ababcd", "(0,1)(1,1)")] // was "(0,6)(3,6)(6,6)" + [InlineData("(a|ab|c|bcd){0,10}(d*)", "ababcd", "(0,1)(1,1)", "(0,6)")] // was "(0,6)(3,6)(6,6)" + [InlineData("(a|ab|c|bcd){1,10}(d*)", "ababcd", "(0,1)(1,1)", "(0,6)")] // was "(0,6)(3,6)(6,6)" [InlineData("(a|ab|c|bcd){2,10}(d*)", "ababcd", "(0,6)(3,6)(6,6)")] [InlineData("(a|ab|c|bcd){3,10}(d*)", "ababcd", "(0,6)(3,6)(6,6)")] [InlineData("(a|ab|c|bcd){4,10}(d*)", "ababcd", "NOMATCH")] - [InlineData("(a|ab|c|bcd)*(d*)", "ababcd", "(0,1)(1,1)")] // was "(0,6)(3,6)(6,6)" - [InlineData("(a|ab|c|bcd)+(d*)", "ababcd", "(0,1)(1,1)")] // was "(0,6)(3,6)(6,6)" + [InlineData("(a|ab|c|bcd)*(d*)", "ababcd", "(0,1)(1,1)", "(0,6)")] // was "(0,6)(3,6)(6,6)" + [InlineData("(a|ab|c|bcd)+(d*)", "ababcd", "(0,1)(1,1)", "(0,6)")] // was "(0,6)(3,6)(6,6)" [InlineData("(ab|a|c|bcd){0,}(d*)", "ababcd", "(0,6)(4,5)(5,6)")] // was "(0,6)(3,6)(6,6)" [InlineData("(ab|a|c|bcd){1,}(d*)", "ababcd", "(0,6)(4,5)(5,6)")] // was "(0,6)(3,6)(6,6)" [InlineData("(ab|a|c|bcd){2,}(d*)", "ababcd", "(0,6)(4,5)(5,6)")] // was "(0,6)(3,6)(6,6)" @@ -354,11 +354,11 @@ public class AttRegexTests [InlineData("(a)*?", "aaa", "(0,0)")] [InlineData("(a*?)*?", "aaa", "(0,0)")] [InlineData("(a*)*(x)", "x", "(0,1)(0,0)(0,1)")] - [InlineData("(a*)*(x)(\\1)", "x", "(0,1)(0,0)(0,1)(1,1)")] - [InlineData("(a*)*(x)(\\1)", "ax", "(0,2)(1,1)(1,2)(2,2)")] - [InlineData("(a*)*(x)(\\1)", "axa", "(0,2)(1,1)(1,2)(2,2)")] // was "(0,3)(0,1)(1,2)(2,3)" - [InlineData("(a*)*(x)(\\1)(x)", "axax", "(0,4)(0,1)(1,2)(2,3)(3,4)")] - [InlineData("(a*)*(x)(\\1)(x)", "axxa", "(0,3)(1,1)(1,2)(2,2)(2,3)")] + [InlineData("(a*)*(x)(\\1)", "x", "(0,1)(0,0)(0,1)(1,1)", "NONBACKTRACKINGINCOMPATIBLE")] + [InlineData("(a*)*(x)(\\1)", "ax", "(0,2)(1,1)(1,2)(2,2)", "NONBACKTRACKINGINCOMPATIBLE")] + [InlineData("(a*)*(x)(\\1)", "axa", "(0,2)(1,1)(1,2)(2,2)", "NONBACKTRACKINGINCOMPATIBLE")] // was "(0,3)(0,1)(1,2)(2,3)" + [InlineData("(a*)*(x)(\\1)(x)", "axax", "(0,4)(0,1)(1,2)(2,3)(3,4)", "NONBACKTRACKINGINCOMPATIBLE")] + [InlineData("(a*)*(x)(\\1)(x)", "axxa", "(0,3)(1,1)(1,2)(2,2)(2,3)", "NONBACKTRACKINGINCOMPATIBLE")] [InlineData("(a*)*(x)", "ax", "(0,2)(1,1)(1,2)")] [InlineData("(a*)*(x)", "axa", "(0,2)(1,1)(1,2)")] // was "(0,2)(0,1)(1,2)" [InlineData("(a*)+(x)", "x", "(0,1)(0,0)(0,1)")] @@ -367,7 +367,7 @@ public class AttRegexTests [InlineData("(a*){2}(x)", "x", "(0,1)(0,0)(0,1)")] [InlineData("(a*){2}(x)", "ax", "(0,2)(1,1)(1,2)")] [InlineData("(a*){2}(x)", "axa", "(0,2)(1,1)(1,2)")] - public async Task Test(string pattern, string input, string captures) + public async Task Test(string pattern, string input, string captures, string nonBacktrackingCaptures = null) { if (input == "NULL") { @@ -376,44 +376,60 @@ public async Task Test(string pattern, string input, string captures) foreach (RegexEngine engine in RegexHelpers.AvailableEngines) { - if (captures == "BADBR") + foreach (RegexOptions options in new[] { RegexOptions.None, RegexOptions.Multiline }) { - await Assert.ThrowsAnyAsync(async () => (await RegexHelpers.GetRegexAsync(engine, pattern)).IsMatch(input)); - return; - } + bool nonBacktracking = engine == RegexEngine.NonBacktracking; + string expected = nonBacktracking && nonBacktrackingCaptures != null ? + nonBacktrackingCaptures : // nonBacktrackingCaptures value overrides the expected result in NonBacktracking mode + captures; - Regex r = await RegexHelpers.GetRegexAsync(engine, pattern); + if (expected == "BADBR") + { + await Assert.ThrowsAnyAsync(async () => await RegexHelpers.GetRegexAsync(engine, pattern, options)); + return; + } - if (captures == "NOMATCH") - { - Assert.False(r.IsMatch(input)); - return; - } + if (nonBacktracking && nonBacktrackingCaptures == "NONBACKTRACKINGINCOMPATIBLE") + { + // In particular: backreferences are not supported in NonBacktracking mode + await Assert.ThrowsAnyAsync(() => RegexHelpers.GetRegexAsync(engine, pattern, options)); + return; + } - Match match = r.Match(input); - Assert.True(match.Success); + Regex r = await RegexHelpers.GetRegexAsync(engine, pattern, options); - var expected = new HashSet<(int start, int end)>( - captures - .Split(new[] { '(', ')' }, StringSplitOptions.RemoveEmptyEntries) - .Select(s => s.Split(',')) - .Select(s => (start: int.Parse(s[0]), end: int.Parse(s[1]))) - .Distinct() - .OrderBy(c => c.start) - .ThenBy(c => c.end)); + if (expected == "NOMATCH") + { + Assert.False(r.IsMatch(input)); + return; + } - var actual = new HashSet<(int start, int end)>( - match.Groups - .Cast() - .Select(g => (start: g.Index, end: g.Index + g.Length)) - .Distinct() - .OrderBy(g => g.start) - .ThenBy(g => g.end)); + Match match = r.Match(input); + Assert.True(match.Success); - // The .NET implementation sometimes has extra captures beyond what the original data specifies, so we assert a subset. - if (!expected.IsSubsetOf(actual)) - { - throw new Xunit.Sdk.XunitException($"Actual: {string.Join(", ", actual)}{Environment.NewLine}Expected: {string.Join(", ", expected)}"); + var expectedSet = new HashSet<(int start, int end)>( + expected + .Split(new[] { '(', ')' }, StringSplitOptions.RemoveEmptyEntries) + .Select(s => s.Split(',')) + .Select(s => (start: int.Parse(s[0]), end: int.Parse(s[1]))) + .Distinct() + .OrderBy(c => c.start) + .ThenBy(c => c.end)); + + var actualSet = new HashSet<(int start, int end)>( + match.Groups + .Cast() + .Select(g => (start: g.Index, end: g.Index + g.Length)) + .Distinct() + .OrderBy(g => g.start) + .ThenBy(g => g.end)); + + // NonBacktracking mode only provides the top-level match. + // The .NET implementation sometimes has extra captures beyond what the original data specifies, so we assert a subset. + if (nonBacktracking ? !actualSet.IsSubsetOf(expectedSet) : !expectedSet.IsSubsetOf(actualSet)) + { + throw new Xunit.Sdk.XunitException($"Actual: {string.Join(", ", actualSet)}{Environment.NewLine}Expected: {string.Join(", ", expected)}"); + } } } } diff --git a/src/libraries/System.Text.RegularExpressions/tests/MatchCollectionTests.cs b/src/libraries/System.Text.RegularExpressions/tests/MatchCollectionTests.cs index 477a1647e5a685..61ba020a519815 100644 --- a/src/libraries/System.Text.RegularExpressions/tests/MatchCollectionTests.cs +++ b/src/libraries/System.Text.RegularExpressions/tests/MatchCollectionTests.cs @@ -2,16 +2,28 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Collections; +using System.Collections.Generic; using Xunit; namespace System.Text.RegularExpressions.Tests { public static partial class MatchCollectionTests { - [Fact] - public static void GetEnumerator() + public static IEnumerable NoneCompiledBacktracking() + { + yield return new object[] { RegexOptions.None }; + yield return new object[] { RegexOptions.Compiled }; + if (PlatformDetection.IsNetCore) + { + yield return new object[] { RegexHelpers.RegexOptionNonBacktracking }; + } + } + + [Theory] + [MemberData(nameof(NoneCompiledBacktracking))] + public static void GetEnumerator(RegexOptions options) { - Regex regex = new Regex("e"); + Regex regex = new Regex("e", options); MatchCollection matches = regex.Matches("dotnet"); IEnumerator enumerator = matches.GetEnumerator(); for (int i = 0; i < 2; i++) @@ -28,10 +40,11 @@ public static void GetEnumerator() } } - [Fact] - public static void GetEnumerator_Invalid() + [Theory] + [MemberData(nameof(NoneCompiledBacktracking))] + public static void GetEnumerator_Invalid(RegexOptions options) { - Regex regex = new Regex("e"); + Regex regex = new Regex("e", options); MatchCollection matches = regex.Matches("dotnet"); IEnumerator enumerator = matches.GetEnumerator(); @@ -54,19 +67,21 @@ public static void Item_Get() Assert.Equal("t", collection[1].ToString()); } - [Fact] - public static void Item_Get_InvalidIndex_ThrowsArgumentOutOfRangeException() + [Theory] + [MemberData(nameof(NoneCompiledBacktracking))] + public static void Item_Get_InvalidIndex_ThrowsArgumentOutOfRangeException(RegexOptions options) { - Regex regex = new Regex("e"); + Regex regex = new Regex("e", options); MatchCollection matches = regex.Matches("dotnet"); AssertExtensions.Throws("i", () => matches[-1]); AssertExtensions.Throws("i", () => matches[matches.Count]); } - [Fact] - public static void ICollection_Properties() + [Theory] + [MemberData(nameof(NoneCompiledBacktracking))] + public static void ICollection_Properties(RegexOptions options) { - Regex regex = new Regex("e"); + Regex regex = new Regex("e", options); MatchCollection matches = regex.Matches("dotnet"); ICollection collection = matches; @@ -76,31 +91,34 @@ public static void ICollection_Properties() } [Theory] - [InlineData(0)] - [InlineData(5)] - public static void ICollection_CopyTo(int index) + [MemberData(nameof(NoneCompiledBacktracking))] + public static void ICollection_CopyTo(RegexOptions options) { - Regex regex = new Regex("e"); - MatchCollection matches = regex.Matches("dotnet"); - ICollection collection = matches; + foreach (int index in new[] { 0, 5 }) + { + Regex regex = new Regex("e", options); + MatchCollection matches = regex.Matches("dotnet"); + ICollection collection = matches; - Match[] copy = new Match[collection.Count + index]; - collection.CopyTo(copy, index); + Match[] copy = new Match[collection.Count + index]; + collection.CopyTo(copy, index); - for (int i = 0; i < index; i++) - { - Assert.Null(copy[i]); - } - for (int i = index; i < copy.Length; i++) - { - Assert.Same(matches[i - index], copy[i]); + for (int i = 0; i < index; i++) + { + Assert.Null(copy[i]); + } + for (int i = index; i < copy.Length; i++) + { + Assert.Same(matches[i - index], copy[i]); + } } } - [Fact] - public static void ICollection_CopyTo_Invalid() + [Theory] + [MemberData(nameof(NoneCompiledBacktracking))] + public static void ICollection_CopyTo_Invalid(RegexOptions options) { - Regex regex = new Regex("e"); + Regex regex = new Regex("e", options); ICollection collection = regex.Matches("dotnet"); // Array is null diff --git a/src/libraries/System.Text.RegularExpressions/tests/MonoRegexTests.cs b/src/libraries/System.Text.RegularExpressions/tests/MonoRegexTests.cs index dcfdbe7d7b40c0..6623449da15012 100644 --- a/src/libraries/System.Text.RegularExpressions/tests/MonoRegexTests.cs +++ b/src/libraries/System.Text.RegularExpressions/tests/MonoRegexTests.cs @@ -60,8 +60,32 @@ public static IEnumerable RegexTestCasesWithOptions() { foreach (object[] obj in RegexTestCases()) { - yield return new object[] { engine, obj[0], obj[1], obj[2], obj[3] }; - yield return new object[] { engine, obj[0], RegexOptions.CultureInvariant | (RegexOptions)obj[1], obj[2], obj[3] }; + string expected_result = (string)obj[3]; + + if (RegexHelpers.IsNonBacktracking(engine)) + { + // For the NonBacktracking option, skip the tests that use nonsupported options + // or if the optional 5th arguments says "NONBACKTRACKINGINCOMPATIBLE" that applies + // when the following are being used in the pattern: + // backreferences, (negative/positive) lookahead, (negative/positive) lookbehind, atomic, \G + if ((((RegexOptions)obj[1] & (RegexOptions.ECMAScript | RegexOptions.RightToLeft)) != 0) || + (obj.Length > 4 && obj[4].Equals("NONBACKTRACKINGINCOMPATIBLE"))) + { + continue; + } + + // Add the NonBacktracking specific tests + // group i values for i>0 are not generated in NonBacktracking mode and are removed + // -- thus the string beyond the first ')' when the test succeeds is removed in obj[3] + int j = expected_result.IndexOf(')'); + if (j > 0) + { + expected_result = expected_result.Substring(0, j + 1); + } + } + + yield return new object[] { engine, obj[0], obj[1], obj[2], expected_result }; + yield return new object[] { engine, obj[0], RegexOptions.CultureInvariant | (RegexOptions)obj[1], obj[2], expected_result }; } } } @@ -220,7 +244,7 @@ public static IEnumerable RegexTestCases() yield return new object[] { @"(bc+d$|ef*g.|h?i(j|k))", RegexOptions.None, "bcdd", "Fail." }; yield return new object[] { @"(bc+d$|ef*g.|h?i(j|k))", RegexOptions.None, "reffgz", "Pass. Group[0]=(1,5) Group[1]=(1,5) Group[2]=" }; yield return new object[] { @"((((((((((a))))))))))", RegexOptions.None, "a", "Pass. Group[0]=(0,1) Group[1]=(0,1) Group[2]=(0,1) Group[3]=(0,1) Group[4]=(0,1) Group[5]=(0,1) Group[6]=(0,1) Group[7]=(0,1) Group[8]=(0,1) Group[9]=(0,1) Group[10]=(0,1)" }; - yield return new object[] { @"((((((((((a))))))))))\10", RegexOptions.None, "aa", "Pass. Group[0]=(0,2) Group[1]=(0,1) Group[2]=(0,1) Group[3]=(0,1) Group[4]=(0,1) Group[5]=(0,1) Group[6]=(0,1) Group[7]=(0,1) Group[8]=(0,1) Group[9]=(0,1) Group[10]=(0,1)" }; + yield return new object[] { @"((((((((((a))))))))))\10", RegexOptions.None, "aa", "Pass. Group[0]=(0,2) Group[1]=(0,1) Group[2]=(0,1) Group[3]=(0,1) Group[4]=(0,1) Group[5]=(0,1) Group[6]=(0,1) Group[7]=(0,1) Group[8]=(0,1) Group[9]=(0,1) Group[10]=(0,1)", "NONBACKTRACKINGINCOMPATIBLE" }; yield return new object[] { @"((((((((((a))))))))))!", RegexOptions.None, "aa", "Fail." }; yield return new object[] { @"((((((((((a))))))))))!", RegexOptions.None, "a!", "Pass. Group[0]=(0,2) Group[1]=(0,1) Group[2]=(0,1) Group[3]=(0,1) Group[4]=(0,1) Group[5]=(0,1) Group[6]=(0,1) Group[7]=(0,1) Group[8]=(0,1) Group[9]=(0,1) Group[10]=(0,1)" }; yield return new object[] { @"(((((((((a)))))))))", RegexOptions.None, "a", "Pass. Group[0]=(0,1) Group[1]=(0,1) Group[2]=(0,1) Group[3]=(0,1) Group[4]=(0,1) Group[5]=(0,1) Group[6]=(0,1) Group[7]=(0,1) Group[8]=(0,1) Group[9]=(0,1)" }; @@ -232,18 +256,18 @@ public static IEnumerable RegexTestCases() yield return new object[] { @"abcd", RegexOptions.None, "abcd", "Pass. Group[0]=(0,4)" }; yield return new object[] { @"a(bc)d", RegexOptions.None, "abcd", "Pass. Group[0]=(0,4) Group[1]=(1,2)" }; yield return new object[] { @"a[-]?c", RegexOptions.None, "ac", "Pass. Group[0]=(0,2)" }; - yield return new object[] { @"(abc)\1", RegexOptions.None, "abcabc", "Pass. Group[0]=(0,6) Group[1]=(0,3)" }; - yield return new object[] { @"([a-c]*)\1", RegexOptions.None, "abcabc", "Pass. Group[0]=(0,6) Group[1]=(0,3)" }; + yield return new object[] { @"(abc)\1", RegexOptions.None, "abcabc", "Pass. Group[0]=(0,6) Group[1]=(0,3)", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"([a-c]*)\1", RegexOptions.None, "abcabc", "Pass. Group[0]=(0,6) Group[1]=(0,3)", "NONBACKTRACKINGINCOMPATIBLE" }; yield return new object[] { @"\1", RegexOptions.None, "-", "Error." }; yield return new object[] { @"\2", RegexOptions.None, "-", "Error." }; - yield return new object[] { @"(a)|\1", RegexOptions.None, "a", "Pass. Group[0]=(0,1) Group[1]=(0,1)" }; - yield return new object[] { @"(a)|\1", RegexOptions.None, "x", "Fail." }; + yield return new object[] { @"(a)|\1", RegexOptions.None, "a", "Pass. Group[0]=(0,1) Group[1]=(0,1)", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"(a)|\1", RegexOptions.None, "x", "Fail.", "NONBACKTRACKINGINCOMPATIBLE" }; yield return new object[] { @"(a)|\2", RegexOptions.None, "-", "Error." }; - yield return new object[] { @"(([a-c])b*?\2)*", RegexOptions.None, "ababbbcbc", "Pass. Group[0]=(0,5) Group[1]=(0,3)(3,2) Group[2]=(0,1)(3,1)" }; - yield return new object[] { @"(([a-c])b*?\2){3}", RegexOptions.None, "ababbbcbc", "Pass. Group[0]=(0,9) Group[1]=(0,3)(3,3)(6,3) Group[2]=(0,1)(3,1)(6,1)" }; - yield return new object[] { @"((\3|b)\2(a)x)+", RegexOptions.None, "aaxabxbaxbbx", "Fail." }; - yield return new object[] { @"((\3|b)\2(a)x)+", RegexOptions.None, "aaaxabaxbaaxbbax", "Pass. Group[0]=(12,4) Group[1]=(12,4) Group[2]=(12,1) Group[3]=(14,1)" }; - yield return new object[] { @"((\3|b)\2(a)){2,}", RegexOptions.None, "bbaababbabaaaaabbaaaabba", "Pass. Group[0]=(15,9) Group[1]=(15,3)(18,3)(21,3) Group[2]=(15,1)(18,1)(21,1) Group[3]=(17,1)(20,1)(23,1)" }; + yield return new object[] { @"(([a-c])b*?\2)*", RegexOptions.None, "ababbbcbc", "Pass. Group[0]=(0,5) Group[1]=(0,3)(3,2) Group[2]=(0,1)(3,1)", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"(([a-c])b*?\2){3}", RegexOptions.None, "ababbbcbc", "Pass. Group[0]=(0,9) Group[1]=(0,3)(3,3)(6,3) Group[2]=(0,1)(3,1)(6,1)", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"((\3|b)\2(a)x)+", RegexOptions.None, "aaxabxbaxbbx", "Fail.", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"((\3|b)\2(a)x)+", RegexOptions.None, "aaaxabaxbaaxbbax", "Pass. Group[0]=(12,4) Group[1]=(12,4) Group[2]=(12,1) Group[3]=(14,1)", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"((\3|b)\2(a)){2,}", RegexOptions.None, "bbaababbabaaaaabbaaaabba", "Pass. Group[0]=(15,9) Group[1]=(15,3)(18,3)(21,3) Group[2]=(15,1)(18,1)(21,1) Group[3]=(17,1)(20,1)(23,1)", "NONBACKTRACKINGINCOMPATIBLE" }; yield return new object[] { @"abc", RegexOptions.IgnoreCase, "ABC", "Pass. Group[0]=(0,3)" }; yield return new object[] { @"abc", RegexOptions.IgnoreCase, "XBC", "Fail." }; yield return new object[] { @"abc", RegexOptions.IgnoreCase, "AXC", "Fail." }; @@ -360,7 +384,7 @@ public static IEnumerable RegexTestCases() yield return new object[] { @"(bc+d$|ef*g.|h?i(j|k))", RegexOptions.IgnoreCase, "BCDD", "Fail." }; yield return new object[] { @"(bc+d$|ef*g.|h?i(j|k))", RegexOptions.IgnoreCase, "REFFGZ", "Pass. Group[0]=(1,5) Group[1]=(1,5) Group[2]=" }; yield return new object[] { @"((((((((((a))))))))))", RegexOptions.IgnoreCase, "A", "Pass. Group[0]=(0,1) Group[1]=(0,1) Group[2]=(0,1) Group[3]=(0,1) Group[4]=(0,1) Group[5]=(0,1) Group[6]=(0,1) Group[7]=(0,1) Group[8]=(0,1) Group[9]=(0,1) Group[10]=(0,1)" }; - yield return new object[] { @"((((((((((a))))))))))\10", RegexOptions.IgnoreCase, "AA", "Pass. Group[0]=(0,2) Group[1]=(0,1) Group[2]=(0,1) Group[3]=(0,1) Group[4]=(0,1) Group[5]=(0,1) Group[6]=(0,1) Group[7]=(0,1) Group[8]=(0,1) Group[9]=(0,1) Group[10]=(0,1)" }; + yield return new object[] { @"((((((((((a))))))))))\10", RegexOptions.IgnoreCase, "AA", "Pass. Group[0]=(0,2) Group[1]=(0,1) Group[2]=(0,1) Group[3]=(0,1) Group[4]=(0,1) Group[5]=(0,1) Group[6]=(0,1) Group[7]=(0,1) Group[8]=(0,1) Group[9]=(0,1) Group[10]=(0,1)", "NONBACKTRACKINGINCOMPATIBLE" }; yield return new object[] { @"((((((((((a))))))))))!", RegexOptions.IgnoreCase, "AA", "Fail." }; yield return new object[] { @"((((((((((a))))))))))!", RegexOptions.IgnoreCase, "A!", "Pass. Group[0]=(0,2) Group[1]=(0,1) Group[2]=(0,1) Group[3]=(0,1) Group[4]=(0,1) Group[5]=(0,1) Group[6]=(0,1) Group[7]=(0,1) Group[8]=(0,1) Group[9]=(0,1) Group[10]=(0,1)" }; yield return new object[] { @"(((((((((a)))))))))", RegexOptions.IgnoreCase, "A", "Pass. Group[0]=(0,1) Group[1]=(0,1) Group[2]=(0,1) Group[3]=(0,1) Group[4]=(0,1) Group[5]=(0,1) Group[6]=(0,1) Group[7]=(0,1) Group[8]=(0,1) Group[9]=(0,1)" }; @@ -374,11 +398,11 @@ public static IEnumerable RegexTestCases() yield return new object[] { @"abcd", RegexOptions.IgnoreCase, "ABCD", "Pass. Group[0]=(0,4)" }; yield return new object[] { @"a(bc)d", RegexOptions.IgnoreCase, "ABCD", "Pass. Group[0]=(0,4) Group[1]=(1,2)" }; yield return new object[] { @"a[-]?c", RegexOptions.IgnoreCase, "AC", "Pass. Group[0]=(0,2)" }; - yield return new object[] { @"(abc)\1", RegexOptions.IgnoreCase, "ABCABC", "Pass. Group[0]=(0,6) Group[1]=(0,3)" }; - yield return new object[] { @"([a-c]*)\1", RegexOptions.IgnoreCase, "ABCABC", "Pass. Group[0]=(0,6) Group[1]=(0,3)" }; - yield return new object[] { @"a(?!b).", RegexOptions.None, "abad", "Pass. Group[0]=(2,2)" }; - yield return new object[] { @"a(?=d).", RegexOptions.None, "abad", "Pass. Group[0]=(2,2)" }; - yield return new object[] { @"a(?=c|d).", RegexOptions.None, "abad", "Pass. Group[0]=(2,2)" }; + yield return new object[] { @"(abc)\1", RegexOptions.IgnoreCase, "ABCABC", "Pass. Group[0]=(0,6) Group[1]=(0,3)", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"([a-c]*)\1", RegexOptions.IgnoreCase, "ABCABC", "Pass. Group[0]=(0,6) Group[1]=(0,3)", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"a(?!b).", RegexOptions.None, "abad", "Pass. Group[0]=(2,2)", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"a(?=d).", RegexOptions.None, "abad", "Pass. Group[0]=(2,2)", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"a(?=c|d).", RegexOptions.None, "abad", "Pass. Group[0]=(2,2)", "NONBACKTRACKINGINCOMPATIBLE" }; yield return new object[] { @"a(?:b|c|d)(.)", RegexOptions.None, "ace", "Pass. Group[0]=(0,3) Group[1]=(2,1)" }; yield return new object[] { @"a(?:b|c|d)*(.)", RegexOptions.None, "ace", "Pass. Group[0]=(0,3) Group[1]=(2,1)" }; yield return new object[] { @"a(?:b|c|d)+?(.)", RegexOptions.None, "ace", "Pass. Group[0]=(0,3) Group[1]=(2,1)" }; @@ -399,26 +423,26 @@ public static IEnumerable RegexTestCases() yield return new object[] { @"^(.+)?B", RegexOptions.None, "AB", "Pass. Group[0]=(0,2) Group[1]=(0,1)" }; yield return new object[] { @"^([^a-z])|(\^)$", RegexOptions.None, ".", "Pass. Group[0]=(0,1) Group[1]=(0,1) Group[2]=" }; yield return new object[] { @"^[<>]&", RegexOptions.None, "<&OUT", "Pass. Group[0]=(0,2)" }; - yield return new object[] { @"^(a\1?){4}$", RegexOptions.None, "aaaaaaaaaa", "Pass. Group[0]=(0,10) Group[1]=(0,1)(1,2)(3,3)(6,4)" }; - yield return new object[] { @"^(a\1?){4}$", RegexOptions.None, "aaaaaaaaa", "Fail." }; - yield return new object[] { @"^(a\1?){4}$", RegexOptions.None, "aaaaaaaaaaa", "Fail." }; - yield return new object[] { @"^(a(?(1)\1)){4}$", RegexOptions.None, "aaaaaaaaaa", "Pass. Group[0]=(0,10) Group[1]=(0,1)(1,2)(3,3)(6,4)" }; - yield return new object[] { @"^(a(?(1)\1)){4}$", RegexOptions.None, "aaaaaaaaa", "Fail." }; - yield return new object[] { @"^(a(?(1)\1)){4}$", RegexOptions.None, "aaaaaaaaaaa", "Fail." }; + yield return new object[] { @"^(a\1?){4}$", RegexOptions.None, "aaaaaaaaaa", "Pass. Group[0]=(0,10) Group[1]=(0,1)(1,2)(3,3)(6,4)", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"^(a\1?){4}$", RegexOptions.None, "aaaaaaaaa", "Fail.", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"^(a\1?){4}$", RegexOptions.None, "aaaaaaaaaaa", "Fail.", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"^(a(?(1)\1)){4}$", RegexOptions.None, "aaaaaaaaaa", "Pass. Group[0]=(0,10) Group[1]=(0,1)(1,2)(3,3)(6,4)", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"^(a(?(1)\1)){4}$", RegexOptions.None, "aaaaaaaaa", "Fail.", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"^(a(?(1)\1)){4}$", RegexOptions.None, "aaaaaaaaaaa", "Fail.", "NONBACKTRACKINGINCOMPATIBLE" }; yield return new object[] { @"((a{4})+)", RegexOptions.None, "aaaaaaaaa", "Pass. Group[0]=(0,8) Group[1]=(0,8) Group[2]=(0,4)(4,4)" }; yield return new object[] { @"(((aa){2})+)", RegexOptions.None, "aaaaaaaaaa", "Pass. Group[0]=(0,8) Group[1]=(0,8) Group[2]=(0,4)(4,4) Group[3]=(0,2)(2,2)(4,2)(6,2)" }; yield return new object[] { @"(((a{2}){2})+)", RegexOptions.None, "aaaaaaaaaa", "Pass. Group[0]=(0,8) Group[1]=(0,8) Group[2]=(0,4)(4,4) Group[3]=(0,2)(2,2)(4,2)(6,2)" }; yield return new object[] { @"(?:(f)(o)(o)|(b)(a)(r))*", RegexOptions.None, "foobar", "Pass. Group[0]=(0,6) Group[1]=(0,1) Group[2]=(1,1) Group[3]=(2,1) Group[4]=(3,1) Group[5]=(4,1) Group[6]=(5,1)" }; - yield return new object[] { @"(?<=a)b", RegexOptions.None, "ab", "Pass. Group[0]=(1,1)" }; - yield return new object[] { @"(?<=a)b", RegexOptions.None, "cb", "Fail." }; - yield return new object[] { @"(?<=a)b", RegexOptions.None, "b", "Fail." }; - yield return new object[] { @"(? RegexTestCases() yield return new object[] { @"((?s-i:a.))b", RegexOptions.IgnoreCase, "B\nB", "Fail." }; yield return new object[] { @"(?:c|d)(?:)(?:a(?:)(?:b)(?:b(?:))(?:b(?:)(?:b)))", RegexOptions.None, "cabbbb", "Pass. Group[0]=(0,6)" }; yield return new object[] { @"(?:c|d)(?:)(?:aaaaaaaa(?:)(?:bbbbbbbb)(?:bbbbbbbb(?:))(?:bbbbbbbb(?:)(?:bbbbbbbb)))", RegexOptions.None, "caaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "Pass. Group[0]=(0,41)" }; - yield return new object[] { @"(ab)\d\1", RegexOptions.IgnoreCase, "Ab4ab", "Pass. Group[0]=(0,5) Group[1]=(0,2)" }; - yield return new object[] { @"(ab)\d\1", RegexOptions.IgnoreCase, "ab4Ab", "Pass. Group[0]=(0,5) Group[1]=(0,2)" }; + yield return new object[] { @"(ab)\d\1", RegexOptions.IgnoreCase, "Ab4ab", "Pass. Group[0]=(0,5) Group[1]=(0,2)", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"(ab)\d\1", RegexOptions.IgnoreCase, "ab4Ab", "Pass. Group[0]=(0,5) Group[1]=(0,2)", "NONBACKTRACKINGINCOMPATIBLE" }; yield return new object[] { @"foo\w*\d{4}baz", RegexOptions.None, "foobar1234baz", "Pass. Group[0]=(0,13)" }; yield return new object[] { @"x(~~)*(?:(?:F)?)?", RegexOptions.None, "x~~", "Pass. Group[0]=(0,3) Group[1]=(1,2)" }; yield return new object[] { @"^a(?#xxx){3}c", RegexOptions.None, "aaac", "Pass. Group[0]=(0,4)" }; - yield return new object[] { @"(? RegexTestCases() yield return new object[] { @"(?m)^(b)", RegexOptions.None, "a\nb\n", "Pass. Group[0]=(2,1) Group[1]=(2,1)" }; yield return new object[] { @"((?m)^b)", RegexOptions.None, "a\nb\n", "Pass. Group[0]=(2,1) Group[1]=(2,1)" }; yield return new object[] { @"\n((?m)^b)", RegexOptions.None, "a\nb\n", "Pass. Group[0]=(1,2) Group[1]=(2,1)" }; - yield return new object[] { @"((?s).)c(?!.)", RegexOptions.None, "a\nb\nc\n", "Pass. Group[0]=(3,2) Group[1]=(3,1)" }; - yield return new object[] { @"((?s)b.)c(?!.)", RegexOptions.None, "a\nb\nc\n", "Pass. Group[0]=(2,3) Group[1]=(2,2)" }; + yield return new object[] { @"((?s).)c(?!.)", RegexOptions.None, "a\nb\nc\n", "Pass. Group[0]=(3,2) Group[1]=(3,1)", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"((?s)b.)c(?!.)", RegexOptions.None, "a\nb\nc\n", "Pass. Group[0]=(2,3) Group[1]=(2,2)", "NONBACKTRACKINGINCOMPATIBLE" }; yield return new object[] { @"^b", RegexOptions.None, "a\nb\nc\n", "Fail." }; yield return new object[] { @"()^b", RegexOptions.None, "a\nb\nc\n", "Fail." }; yield return new object[] { @"((?m)^b)", RegexOptions.None, "a\nb\nc\n", "Pass. Group[0]=(2,1) Group[1]=(2,1)" }; - yield return new object[] { @"(x)?(?(1)a|b)", RegexOptions.None, "a", "Fail." }; - yield return new object[] { @"(x)?(?(1)b|a)", RegexOptions.None, "a", "Pass. Group[0]=(0,1) Group[1]=" }; - yield return new object[] { @"()?(?(1)b|a)", RegexOptions.None, "a", "Pass. Group[0]=(0,1) Group[1]=" }; - yield return new object[] { @"()(?(1)b|a)", RegexOptions.None, "a", "Fail." }; - yield return new object[] { @"()?(?(1)a|b)", RegexOptions.None, "a", "Pass. Group[0]=(0,1) Group[1]=(0,0)" }; - yield return new object[] { @"^(\()?blah(?(1)(\)))$", RegexOptions.None, "(blah)", "Pass. Group[0]=(0,6) Group[1]=(0,1) Group[2]=(5,1)" }; - yield return new object[] { @"^(\()?blah(?(1)(\)))$", RegexOptions.None, "blah", "Pass. Group[0]=(0,4) Group[1]= Group[2]=" }; - yield return new object[] { @"^(\()?blah(?(1)(\)))$", RegexOptions.None, "blah)", "Fail." }; - yield return new object[] { @"^(\()?blah(?(1)(\)))$", RegexOptions.None, "(blah", "Fail." }; - yield return new object[] { @"^(\(+)?blah(?(1)(\)))$", RegexOptions.None, "(blah)", "Pass. Group[0]=(0,6) Group[1]=(0,1) Group[2]=(5,1)" }; - yield return new object[] { @"^(\(+)?blah(?(1)(\)))$", RegexOptions.None, "blah", "Pass. Group[0]=(0,4) Group[1]= Group[2]=" }; - yield return new object[] { @"^(\(+)?blah(?(1)(\)))$", RegexOptions.None, "blah)", "Fail." }; - yield return new object[] { @"^(\(+)?blah(?(1)(\)))$", RegexOptions.None, "(blah", "Fail." }; + yield return new object[] { @"(x)?(?(1)a|b)", RegexOptions.None, "a", "Fail.", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"(x)?(?(1)b|a)", RegexOptions.None, "a", "Pass. Group[0]=(0,1) Group[1]=", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"()?(?(1)b|a)", RegexOptions.None, "a", "Pass. Group[0]=(0,1) Group[1]=", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"()(?(1)b|a)", RegexOptions.None, "a", "Fail.", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"()?(?(1)a|b)", RegexOptions.None, "a", "Pass. Group[0]=(0,1) Group[1]=(0,0)", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"^(\()?blah(?(1)(\)))$", RegexOptions.None, "(blah)", "Pass. Group[0]=(0,6) Group[1]=(0,1) Group[2]=(5,1)", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"^(\()?blah(?(1)(\)))$", RegexOptions.None, "blah", "Pass. Group[0]=(0,4) Group[1]= Group[2]=", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"^(\()?blah(?(1)(\)))$", RegexOptions.None, "blah)", "Fail.", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"^(\()?blah(?(1)(\)))$", RegexOptions.None, "(blah", "Fail.", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"^(\(+)?blah(?(1)(\)))$", RegexOptions.None, "(blah)", "Pass. Group[0]=(0,6) Group[1]=(0,1) Group[2]=(5,1)", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"^(\(+)?blah(?(1)(\)))$", RegexOptions.None, "blah", "Pass. Group[0]=(0,4) Group[1]= Group[2]=", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"^(\(+)?blah(?(1)(\)))$", RegexOptions.None, "blah)", "Fail.", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"^(\(+)?blah(?(1)(\)))$", RegexOptions.None, "(blah", "Fail.", "NONBACKTRACKINGINCOMPATIBLE" }; yield return new object[] { @"(?(1)a|b|c)", RegexOptions.None, "a", "Error." }; - yield return new object[] { @"(?(?!a)a|b)", RegexOptions.None, "a", "Fail." }; - yield return new object[] { @"(?(?!a)b|a)", RegexOptions.None, "a", "Pass. Group[0]=(0,1)" }; - yield return new object[] { @"(?(?=a)b|a)", RegexOptions.None, "a", "Fail." }; - yield return new object[] { @"(?(?=a)a|b)", RegexOptions.None, "a", "Pass. Group[0]=(0,1)" }; - yield return new object[] { @"(?=(a+?))(\1ab)", RegexOptions.None, "aaab", "Pass. Group[0]=(1,3) Group[1]=(1,1) Group[2]=(1,3)" }; - yield return new object[] { @"^(?=(a+?))\1ab", RegexOptions.None, "aaab", "Fail." }; + yield return new object[] { @"(?(?!a)a|b)", RegexOptions.None, "a", "Fail.", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"(?(?!a)b|a)", RegexOptions.None, "a", "Pass. Group[0]=(0,1)", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"(?(?=a)b|a)", RegexOptions.None, "a", "Fail.", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"(?(?=a)a|b)", RegexOptions.None, "a", "Pass. Group[0]=(0,1)", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"(?=(a+?))(\1ab)", RegexOptions.None, "aaab", "Pass. Group[0]=(1,3) Group[1]=(1,1) Group[2]=(1,3)", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"^(?=(a+?))\1ab", RegexOptions.None, "aaab", "Fail.", "NONBACKTRACKINGINCOMPATIBLE" }; yield return new object[] { @"(\w+:)+", RegexOptions.None, "one:", "Pass. Group[0]=(0,4) Group[1]=(0,4)" }; - yield return new object[] { @"$(?<=^(a))", RegexOptions.None, "a", "Pass. Group[0]=(1,0) Group[1]=(0,1)" }; + yield return new object[] { @"$(?<=^(a))", RegexOptions.None, "a", "Pass. Group[0]=(1,0) Group[1]=(0,1)", "NONBACKTRACKINGINCOMPATIBLE" }; yield return new object[] { @"([\w:]+::)?(\w+)$", RegexOptions.None, "abcd:", "Fail." }; yield return new object[] { @"([\w:]+::)?(\w+)$", RegexOptions.None, "abcd", "Pass. Group[0]=(0,4) Group[1]= Group[2]=(0,4)" }; yield return new object[] { @"([\w:]+::)?(\w+)$", RegexOptions.None, "xy:z:::abcd", "Pass. Group[0]=(0,11) Group[1]=(0,7) Group[2]=(7,4)" }; yield return new object[] { @"^[^bcd]*(c+)", RegexOptions.None, "aexycd", "Pass. Group[0]=(0,5) Group[1]=(4,1)" }; yield return new object[] { @"(a*)b+", RegexOptions.None, "caab", "Pass. Group[0]=(1,3) Group[1]=(1,2)" }; yield return new object[] { @"(>a+)ab", RegexOptions.None, "aaab", "Fail." }; - yield return new object[] { @"(?>a+)b", RegexOptions.None, "aaab", "Pass. Group[0]=(0,4)" }; + yield return new object[] { @"(?>a+)b", RegexOptions.None, "aaab", "Pass. Group[0]=(0,4)", "NONBACKTRACKINGINCOMPATIBLE" }; yield return new object[] { @"([[:]+)", RegexOptions.None, "a:[b]:", "Pass. Group[0]=(1,2) Group[1]=(1,2)" }; yield return new object[] { @"([[=]+)", RegexOptions.None, "a=[b]=", "Pass. Group[0]=(1,2) Group[1]=(1,2)" }; yield return new object[] { @"([[.]+)", RegexOptions.None, "a.[b].", "Pass. Group[0]=(1,2) Group[1]=(1,2)" }; yield return new object[] { @"[a[:]b[:c]", RegexOptions.None, "abc", "Pass. Group[0]=(0,3)" }; - yield return new object[] { @"((?>a+)b)", RegexOptions.None, "aaab", "Pass. Group[0]=(0,4) Group[1]=(0,4)" }; - yield return new object[] { @"(?>(a+))b", RegexOptions.None, "aaab", "Pass. Group[0]=(0,4) Group[1]=(0,3)" }; - yield return new object[] { @"((?>[^()]+)|\([^()]*\))+", RegexOptions.None, "((abc(ade)ufh()()x", "Pass. Group[0]=(2,16) Group[1]=(2,3)(5,5)(10,3)(13,2)(15,2)(17,1)" }; - yield return new object[] { @"(?<=x+)", RegexOptions.None, "xxxxy", "Pass. Group[0]=(1,0)" }; + yield return new object[] { @"((?>a+)b)", RegexOptions.None, "aaab", "Pass. Group[0]=(0,4) Group[1]=(0,4)", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"(?>(a+))b", RegexOptions.None, "aaab", "Pass. Group[0]=(0,4) Group[1]=(0,3)", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"((?>[^()]+)|\([^()]*\))+", RegexOptions.None, "((abc(ade)ufh()()x", "Pass. Group[0]=(2,16) Group[1]=(2,3)(5,5)(10,3)(13,2)(15,2)(17,1)", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"(?<=x+)", RegexOptions.None, "xxxxy", "Pass. Group[0]=(1,0)", "NONBACKTRACKINGINCOMPATIBLE" }; yield return new object[] { @"a{37,17}", RegexOptions.None, "-", "Error." }; yield return new object[] { @"\Z", RegexOptions.None, "a\nb\n", "Pass. Group[0]=(3,0)" }; yield return new object[] { @"\z", RegexOptions.None, "a\nb\n", "Pass. Group[0]=(4,0)" }; @@ -714,7 +738,7 @@ public static IEnumerable RegexTestCases() yield return new object[] { @"abb$", RegexOptions.Multiline, "b\nca", "Fail." }; yield return new object[] { @"(^|x)(c)", RegexOptions.None, "ca", "Pass. Group[0]=(0,1) Group[1]=(0,0) Group[2]=(0,1)" }; yield return new object[] { @"a*abc?xyz+pqr{3}ab{2,}xy{4,5}pq{0,6}AB{0,}zz", RegexOptions.None, "x", "Fail." }; - yield return new object[] { @"round\(((?>[^()]+))\)", RegexOptions.None, "_I(round(xs * sz),1)", "Pass. Group[0]=(3,14) Group[1]=(9,7)" }; + yield return new object[] { @"round\(((?>[^()]+))\)", RegexOptions.None, "_I(round(xs * sz),1)", "Pass. Group[0]=(3,14) Group[1]=(9,7)", "NONBACKTRACKINGINCOMPATIBLE" }; yield return new object[] { @"foo.bart", RegexOptions.None, "foo.bart", "Pass. Group[0]=(0,8)" }; yield return new object[] { @"^d[x][x][x]", RegexOptions.Multiline, "abcd\ndxxx", "Pass. Group[0]=(5,4)" }; yield return new object[] { @".X(.+)+X", RegexOptions.None, "bbbbXcXaaaaaaaa", "Pass. Group[0]=(3,4) Group[1]=(5,1)" }; @@ -738,7 +762,7 @@ public static IEnumerable RegexTestCases() yield return new object[] { @"tt+$", RegexOptions.None, "xxxtt", "Pass. Group[0]=(3,2)" }; yield return new object[] { @"([\d-z]+)", RegexOptions.None, "a0-za", "Pass. Group[0]=(1,3) Group[1]=(1,3)" }; yield return new object[] { @"([\d-\s]+)", RegexOptions.None, "a0- z", "Pass. Group[0]=(1,3) Group[1]=(1,3)" }; - yield return new object[] { @"\GX.*X", RegexOptions.None, "aaaXbX", "Fail." }; + yield return new object[] { @"\GX.*X", RegexOptions.None, "aaaXbX", "Fail.", "NONBACKTRACKINGINCOMPATIBLE" }; yield return new object[] { @"(\d+\.\d+)", RegexOptions.None, "3.1415926", "Pass. Group[0]=(0,9) Group[1]=(0,9)" }; yield return new object[] { @"(\ba.{0,10}br)", RegexOptions.None, "have a web browser", "Pass. Group[0]=(5,8) Group[1]=(5,8)" }; yield return new object[] { @"\.c(pp|xx|c)?$", RegexOptions.IgnoreCase, "Changes", "Fail." }; @@ -748,7 +772,7 @@ public static IEnumerable RegexTestCases() yield return new object[] { @"^\S\s+aa$", RegexOptions.Multiline, "\nx aa", "Pass. Group[0]=(1,4)" }; yield return new object[] { @"(^|a)b", RegexOptions.None, "ab", "Pass. Group[0]=(0,2) Group[1]=(0,1)" }; yield return new object[] { @"^([ab]*?)(b)?(c)$", RegexOptions.None, "abac", "Pass. Group[0]=(0,4) Group[1]=(0,3) Group[2]= Group[3]=(3,1)" }; - yield return new object[] { @"(\w)?(abc)\1b", RegexOptions.None, "abcab", "Fail." }; + yield return new object[] { @"(\w)?(abc)\1b", RegexOptions.None, "abcab", "Fail.", "NONBACKTRACKINGINCOMPATIBLE" }; yield return new object[] { @"^(?:.,){2}c", RegexOptions.None, "a,b,c", "Pass. Group[0]=(0,5)" }; yield return new object[] { @"^(.,){2}c", RegexOptions.None, "a,b,c", "Pass. Group[0]=(0,5) Group[1]=(0,2)(2,2)" }; yield return new object[] { @"^(?:[^,]*,){2}c", RegexOptions.None, "a,b,c", "Pass. Group[0]=(0,5)" }; @@ -766,14 +790,14 @@ public static IEnumerable RegexTestCases() yield return new object[] { @"^([^,]{0,3},){3,}d", RegexOptions.None, "aaa,b,c,d", "Pass. Group[0]=(0,9) Group[1]=(0,4)(4,2)(6,2)" }; yield return new object[] { @"^([^,]{0,3},){0,3}d", RegexOptions.None, "aaa,b,c,d", "Pass. Group[0]=(0,9) Group[1]=(0,4)(4,2)(6,2)" }; yield return new object[] { @"(?i)", RegexOptions.None, "", "Pass. Group[0]=(0,0)" }; - yield return new object[] { @"(?!\A)x", RegexOptions.Multiline, "a\nxb\n", "Pass. Group[0]=(2,1)" }; + yield return new object[] { @"(?!\A)x", RegexOptions.Multiline, "a\nxb\n", "Pass. Group[0]=(2,1)", "NONBACKTRACKINGINCOMPATIBLE" }; yield return new object[] { @"^(a(b)?)+$", RegexOptions.None, "aba", "Pass. Group[0]=(0,3) Group[1]=(0,2)(2,1) Group[2]=(1,1)" }; yield return new object[] { @"^(aa(bb)?)+$", RegexOptions.None, "aabbaa", "Pass. Group[0]=(0,6) Group[1]=(0,4)(4,2) Group[2]=(2,2)" }; yield return new object[] { @"^.{9}abc.*\n", RegexOptions.Multiline, "123\nabcabcabcabc\n", "Pass. Group[0]=(4,13)" }; yield return new object[] { @"^(a)?a$", RegexOptions.None, "a", "Pass. Group[0]=(0,1) Group[1]=" }; - yield return new object[] { @"^(a)?(?(1)a|b)+$", RegexOptions.None, "a", "Fail." }; - yield return new object[] { @"^(a\1?)(a\1?)(a\2?)(a\3?)$", RegexOptions.None, "aaaaaa", "Pass. Group[0]=(0,6) Group[1]=(0,1) Group[2]=(1,2) Group[3]=(3,1) Group[4]=(4,2)" }; - yield return new object[] { @"^(a\1?){4}$", RegexOptions.None, "aaaaaa", "Pass. Group[0]=(0,6) Group[1]=(0,1)(1,2)(3,1)(4,2)" }; + yield return new object[] { @"^(a)?(?(1)a|b)+$", RegexOptions.None, "a", "Fail.", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"^(a\1?)(a\1?)(a\2?)(a\3?)$", RegexOptions.None, "aaaaaa", "Pass. Group[0]=(0,6) Group[1]=(0,1) Group[2]=(1,2) Group[3]=(3,1) Group[4]=(4,2)", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"^(a\1?){4}$", RegexOptions.None, "aaaaaa", "Pass. Group[0]=(0,6) Group[1]=(0,1)(1,2)(3,1)(4,2)", "NONBACKTRACKINGINCOMPATIBLE" }; yield return new object[] { @"^(0+)?(?:x(1))?", RegexOptions.None, "x1", "Pass. Group[0]=(0,2) Group[1]= Group[2]=(1,1)" }; yield return new object[] { @"^([0-9a-fA-F]+)(?:x([0-9a-fA-F]+)?)(?:x([0-9a-fA-F]+))?", RegexOptions.None, "012cxx0190", "Pass. Group[0]=(0,10) Group[1]=(0,4) Group[2]= Group[3]=(6,4)" }; yield return new object[] { @"^(b+?|a){1,2}c", RegexOptions.None, "bbbac", "Pass. Group[0]=(0,5) Group[1]=(0,3)(3,1)" }; @@ -1005,18 +1029,18 @@ public static IEnumerable RegexTestCases() yield return new object[] { @"((\3|b)\2(a)x)+", RegexOptions.RightToLeft, "aaxabxbaxbbx", "Fail." }; yield return new object[] { @"((\3|b)\2(a)x)+", RegexOptions.RightToLeft, "aaaxabaxbaaxbbax", "Fail." }; yield return new object[] { @"((\3|b)\2(a)){2,}", RegexOptions.RightToLeft, "bbaababbabaaaaabbaaaabba", "Fail." }; - yield return new object[] { @"\((?>[^()]+|\((?)|\)(?<-depth>))*(?(depth)(?!))\)", RegexOptions.None, "((a(b))c)", "Pass. Group[0]=(0,9) Group[1]=" }; - yield return new object[] { @"^\((?>[^()]+|\((?)|\)(?<-depth>))*(?(depth)(?!))\)$", RegexOptions.None, "((a(b))c)", "Pass. Group[0]=(0,9) Group[1]=" }; - yield return new object[] { @"^\((?>[^()]+|\((?)|\)(?<-depth>))*(?(depth)(?!))\)$", RegexOptions.None, "((a(b))c", "Fail." }; - yield return new object[] { @"^\((?>[^()]+|\((?)|\)(?<-depth>))*(?(depth)(?!))\)$", RegexOptions.None, "())", "Fail." }; - yield return new object[] { @"(((?\()[^()]*)+((?\))[^()]*)+)+(?(foo)(?!))", RegexOptions.None, "((a(b))c)", "Pass. Group[0]=(0,9) Group[1]=(0,9) Group[2]=(0,1)(1,2)(3,2) Group[3]=(5,1)(6,2)(8,1) Group[4]= Group[5]=(4,1)(2,4)(1,7)" }; - yield return new object[] { @"^(((?\()[^()]*)+((?\))[^()]*)+)+(?(foo)(?!))$", RegexOptions.None, "((a(b))c)", "Pass. Group[0]=(0,9) Group[1]=(0,9) Group[2]=(0,1)(1,2)(3,2) Group[3]=(5,1)(6,2)(8,1) Group[4]= Group[5]=(4,1)(2,4)(1,7)" }; - yield return new object[] { @"(((?\()[^()]*)+((?\))[^()]*)+)+(?(foo)(?!))", RegexOptions.None, "x(a((b)))b)x", "Pass. Group[0]=(1,9) Group[1]=(1,9) Group[2]=(1,2)(3,1)(4,2) Group[3]=(6,1)(7,1)(8,2) Group[4]= Group[5]=(5,1)(4,3)(2,6)" }; - yield return new object[] { @"(((?\()[^()]*)+((?\))[^()]*)+)+(?(foo)(?!))", RegexOptions.None, "x((a((b)))x", "Pass. Group[0]=(2,9) Group[1]=(2,9) Group[2]=(2,2)(4,1)(5,2) Group[3]=(7,1)(8,1)(9,2) Group[4]= Group[5]=(6,1)(5,3)(3,6)" }; - yield return new object[] { @"^(((?\()[^()]*)+((?\))[^()]*)+)+(?(foo)(?!))$", RegexOptions.None, "((a(b))c","Fail." }; - yield return new object[] { @"^(((?\()[^()]*)+((?\))[^()]*)+)+(?(foo)(?!))$", RegexOptions.None, "((a(b))c))","Fail." }; - yield return new object[] { @"^(((?\()[^()]*)+((?\))[^()]*)+)+(?(foo)(?!))$", RegexOptions.None, ")(","Fail." }; - yield return new object[] { @"^(((?\()[^()]*)+((?\))[^()]*)+)+(?(foo)(?!))$", RegexOptions.None, "((a((b))c)","Fail." }; + yield return new object[] { @"\((?>[^()]+|\((?)|\)(?<-depth>))*(?(depth)(?!))\)", RegexOptions.None, "((a(b))c)", "Pass. Group[0]=(0,9) Group[1]=", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"^\((?>[^()]+|\((?)|\)(?<-depth>))*(?(depth)(?!))\)$", RegexOptions.None, "((a(b))c)", "Pass. Group[0]=(0,9) Group[1]=", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"^\((?>[^()]+|\((?)|\)(?<-depth>))*(?(depth)(?!))\)$", RegexOptions.None, "((a(b))c", "Fail.", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"^\((?>[^()]+|\((?)|\)(?<-depth>))*(?(depth)(?!))\)$", RegexOptions.None, "())", "Fail.", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"(((?\()[^()]*)+((?\))[^()]*)+)+(?(foo)(?!))", RegexOptions.None, "((a(b))c)", "Pass. Group[0]=(0,9) Group[1]=(0,9) Group[2]=(0,1)(1,2)(3,2) Group[3]=(5,1)(6,2)(8,1) Group[4]= Group[5]=(4,1)(2,4)(1,7)", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"^(((?\()[^()]*)+((?\))[^()]*)+)+(?(foo)(?!))$", RegexOptions.None, "((a(b))c)", "Pass. Group[0]=(0,9) Group[1]=(0,9) Group[2]=(0,1)(1,2)(3,2) Group[3]=(5,1)(6,2)(8,1) Group[4]= Group[5]=(4,1)(2,4)(1,7)", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"(((?\()[^()]*)+((?\))[^()]*)+)+(?(foo)(?!))", RegexOptions.None, "x(a((b)))b)x", "Pass. Group[0]=(1,9) Group[1]=(1,9) Group[2]=(1,2)(3,1)(4,2) Group[3]=(6,1)(7,1)(8,2) Group[4]= Group[5]=(5,1)(4,3)(2,6)", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"(((?\()[^()]*)+((?\))[^()]*)+)+(?(foo)(?!))", RegexOptions.None, "x((a((b)))x", "Pass. Group[0]=(2,9) Group[1]=(2,9) Group[2]=(2,2)(4,1)(5,2) Group[3]=(7,1)(8,1)(9,2) Group[4]= Group[5]=(6,1)(5,3)(3,6)", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"^(((?\()[^()]*)+((?\))[^()]*)+)+(?(foo)(?!))$", RegexOptions.None, "((a(b))c","Fail.", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"^(((?\()[^()]*)+((?\))[^()]*)+)+(?(foo)(?!))$", RegexOptions.None, "((a(b))c))","Fail.", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"^(((?\()[^()]*)+((?\))[^()]*)+)+(?(foo)(?!))$", RegexOptions.None, ")(","Fail.", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"^(((?\()[^()]*)+((?\))[^()]*)+)+(?(foo)(?!))$", RegexOptions.None, "((a((b))c)","Fail.", "NONBACKTRACKINGINCOMPATIBLE" }; yield return new object[] { @"^((\[(?[^\]]+)\])|(?[^\.\[\]]+))$", RegexOptions.IgnoreCase | RegexOptions.ExplicitCapture, "[n]", "Pass. Group[0]=(0,3) Group[1]=(1,1)" }; yield return new object[] { @"^((\[(?[^\]]+)\])|(?[^\.\[\]]+))$", RegexOptions.IgnoreCase | RegexOptions.ExplicitCapture, "n", "Pass. Group[0]=(0,1) Group[1]=(0,1)" }; yield return new object[] { @"^((\[(?[^\]]+)\])|(?[^\.\[\]]+))$", RegexOptions.IgnoreCase | RegexOptions.ExplicitCapture, "n[i]e", "Fail." }; @@ -1050,55 +1074,55 @@ public static IEnumerable RegexTestCases() yield return new object[] { @"(a)(?<1>b)(c)", RegexOptions.ExplicitCapture, "abc", "Pass. Group[0]=(0,3) Group[1]=(1,1)" }; yield return new object[] { @"(a)(?<2>b)(c)", RegexOptions.None, "abc", "Pass. Group[0]=(0,3) Group[1]=(0,1) Group[2]=(1,1)(2,1)" }; yield return new object[] { @"(a)(?b)(c)", RegexOptions.ExplicitCapture, "abc", "Pass. Group[0]=(0,3) Group[1]=(1,1)" }; - yield return new object[] { @"(F)(2)(3)(4)(5)(6)(7)(8)(9)(10)(L)\11", RegexOptions.None, "F2345678910LL", "Pass. Group[0]=(0,13) Group[1]=(0,1) Group[2]=(1,1) Group[3]=(2,1) Group[4]=(3,1) Group[5]=(4,1) Group[6]=(5,1) Group[7]=(6,1) Group[8]=(7,1) Group[9]=(8,1) Group[10]=(9,2) Group[11]=(11,1)" }; - yield return new object[] { @"(F)(2)(3)(4)(5)(6)(7)(8)(9)(10)(L)\11", RegexOptions.ExplicitCapture, "F2345678910LL", "Fail." }; - yield return new object[] { @"(F)(2)(3)(4)(5)(6)(?7)(8)(9)(10)(L)\1", RegexOptions.None, "F2345678910L71", "Fail." }; - yield return new object[] { @"(F)(2)(3)(4)(5)(6)(7)(8)(9)(10)(L)\11", RegexOptions.None, "F2345678910LF1", "Fail." }; - yield return new object[] { @"(F)(2)(3)(4)(5)(6)(?7)(8)(9)(10)(L)\11", RegexOptions.None, "F2345678910L71", "Pass. Group[0]=(0,13) Group[1]=(0,1) Group[2]=(1,1) Group[3]=(2,1) Group[4]=(3,1) Group[5]=(4,1) Group[6]=(5,1) Group[7]=(7,1) Group[8]=(8,1) Group[9]=(9,2) Group[10]=(11,1) Group[11]=(6,1)" }; - yield return new object[] { @"(F)(2)(3)(?4)(5)(6)(?'S'7)(8)(9)(10)(L)\10", RegexOptions.None, "F2345678910L71", "Pass. Group[0]=(0,13) Group[1]=(0,1) Group[2]=(1,1) Group[3]=(2,1) Group[4]=(4,1) Group[5]=(5,1) Group[6]=(7,1) Group[7]=(8,1) Group[8]=(9,2) Group[9]=(11,1) Group[10]=(3,1)(6,1)" }; - yield return new object[] { @"(F)(2)(3)(?4)(5)(6)(?'S'7)(8)(9)(10)(L)\10", RegexOptions.ExplicitCapture, "F2345678910L70", "Fail." }; - yield return new object[] { @"(F)(2)(3)(?4)(5)(6)(?'S'7)(8)(9)(10)(L)\1", RegexOptions.ExplicitCapture, "F2345678910L70", "Pass. Group[0]=(0,13) Group[1]=(3,1)(6,1)" }; - yield return new object[] { @"(?n:(F)(2)(3)(?4)(5)(6)(?'S'7)(8)(9)(10)(L)\1)", RegexOptions.None, "F2345678910L70", "Pass. Group[0]=(0,13) Group[1]=(3,1)(6,1)" }; - yield return new object[] { @"(F)(2)(3)(?4)(5)(6)(?'S'7)(8)(9)(10)(L)(?(10)\10)", RegexOptions.None, "F2345678910L70","Pass. Group[0]=(0,13) Group[1]=(0,1) Group[2]=(1,1) Group[3]=(2,1) Group[4]=(4,1) Group[5]=(5,1) Group[6]=(7,1) Group[7]=(8,1) Group[8]=(9,2) Group[9]=(11,1) Group[10]=(3,1)(6,1)" }; - yield return new object[] { @"(F)(2)(3)(?4)(5)(6)(?'S'7)(8)(9)(10)(L)(?(S)|\10)", RegexOptions.None, "F2345678910L70","Pass. Group[0]=(0,12) Group[1]=(0,1) Group[2]=(1,1) Group[3]=(2,1) Group[4]=(4,1) Group[5]=(5,1) Group[6]=(7,1) Group[7]=(8,1) Group[8]=(9,2) Group[9]=(11,1) Group[10]=(3,1)(6,1)" }; - yield return new object[] { @"(F)(2)(3)(?4)(5)(6)(?'S'7)(8)(9)(10)(L)(?(7)|\10)", RegexOptions.None, "F2345678910L70","Pass. Group[0]=(0,12) Group[1]=(0,1) Group[2]=(1,1) Group[3]=(2,1) Group[4]=(4,1) Group[5]=(5,1) Group[6]=(7,1) Group[7]=(8,1) Group[8]=(9,2) Group[9]=(11,1) Group[10]=(3,1)(6,1)" }; - yield return new object[] { @"(F)(2)(3)(?4)(5)(6)(?'S'7)(8)(9)(10)(L)(?(K)|\10)", RegexOptions.None, "F2345678910L70","Pass. Group[0]=(0,13) Group[1]=(0,1) Group[2]=(1,1) Group[3]=(2,1) Group[4]=(4,1) Group[5]=(5,1) Group[6]=(7,1) Group[7]=(8,1) Group[8]=(9,2) Group[9]=(11,1) Group[10]=(3,1)(6,1)" }; + yield return new object[] { @"(F)(2)(3)(4)(5)(6)(7)(8)(9)(10)(L)\11", RegexOptions.None, "F2345678910LL", "Pass. Group[0]=(0,13) Group[1]=(0,1) Group[2]=(1,1) Group[3]=(2,1) Group[4]=(3,1) Group[5]=(4,1) Group[6]=(5,1) Group[7]=(6,1) Group[8]=(7,1) Group[9]=(8,1) Group[10]=(9,2) Group[11]=(11,1)", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"(F)(2)(3)(4)(5)(6)(7)(8)(9)(10)(L)\11", RegexOptions.ExplicitCapture, "F2345678910LL", "Fail.", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"(F)(2)(3)(4)(5)(6)(?7)(8)(9)(10)(L)\1", RegexOptions.None, "F2345678910L71", "Fail.", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"(F)(2)(3)(4)(5)(6)(7)(8)(9)(10)(L)\11", RegexOptions.None, "F2345678910LF1", "Fail.", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"(F)(2)(3)(4)(5)(6)(?7)(8)(9)(10)(L)\11", RegexOptions.None, "F2345678910L71", "Pass. Group[0]=(0,13) Group[1]=(0,1) Group[2]=(1,1) Group[3]=(2,1) Group[4]=(3,1) Group[5]=(4,1) Group[6]=(5,1) Group[7]=(7,1) Group[8]=(8,1) Group[9]=(9,2) Group[10]=(11,1) Group[11]=(6,1)", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"(F)(2)(3)(?4)(5)(6)(?'S'7)(8)(9)(10)(L)\10", RegexOptions.None, "F2345678910L71", "Pass. Group[0]=(0,13) Group[1]=(0,1) Group[2]=(1,1) Group[3]=(2,1) Group[4]=(4,1) Group[5]=(5,1) Group[6]=(7,1) Group[7]=(8,1) Group[8]=(9,2) Group[9]=(11,1) Group[10]=(3,1)(6,1)", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"(F)(2)(3)(?4)(5)(6)(?'S'7)(8)(9)(10)(L)\10", RegexOptions.ExplicitCapture, "F2345678910L70", "Fail.", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"(F)(2)(3)(?4)(5)(6)(?'S'7)(8)(9)(10)(L)\1", RegexOptions.ExplicitCapture, "F2345678910L70", "Pass. Group[0]=(0,13) Group[1]=(3,1)(6,1)", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"(?n:(F)(2)(3)(?4)(5)(6)(?'S'7)(8)(9)(10)(L)\1)", RegexOptions.None, "F2345678910L70", "Pass. Group[0]=(0,13) Group[1]=(3,1)(6,1)", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"(F)(2)(3)(?4)(5)(6)(?'S'7)(8)(9)(10)(L)(?(10)\10)", RegexOptions.None, "F2345678910L70","Pass. Group[0]=(0,13) Group[1]=(0,1) Group[2]=(1,1) Group[3]=(2,1) Group[4]=(4,1) Group[5]=(5,1) Group[6]=(7,1) Group[7]=(8,1) Group[8]=(9,2) Group[9]=(11,1) Group[10]=(3,1)(6,1)", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"(F)(2)(3)(?4)(5)(6)(?'S'7)(8)(9)(10)(L)(?(S)|\10)", RegexOptions.None, "F2345678910L70","Pass. Group[0]=(0,12) Group[1]=(0,1) Group[2]=(1,1) Group[3]=(2,1) Group[4]=(4,1) Group[5]=(5,1) Group[6]=(7,1) Group[7]=(8,1) Group[8]=(9,2) Group[9]=(11,1) Group[10]=(3,1)(6,1)", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"(F)(2)(3)(?4)(5)(6)(?'S'7)(8)(9)(10)(L)(?(7)|\10)", RegexOptions.None, "F2345678910L70","Pass. Group[0]=(0,12) Group[1]=(0,1) Group[2]=(1,1) Group[3]=(2,1) Group[4]=(4,1) Group[5]=(5,1) Group[6]=(7,1) Group[7]=(8,1) Group[8]=(9,2) Group[9]=(11,1) Group[10]=(3,1)(6,1)", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"(F)(2)(3)(?4)(5)(6)(?'S'7)(8)(9)(10)(L)(?(K)|\10)", RegexOptions.None, "F2345678910L70","Pass. Group[0]=(0,13) Group[1]=(0,1) Group[2]=(1,1) Group[3]=(2,1) Group[4]=(4,1) Group[5]=(5,1) Group[6]=(7,1) Group[7]=(8,1) Group[8]=(9,2) Group[9]=(11,1) Group[10]=(3,1)(6,1)", "NONBACKTRACKINGINCOMPATIBLE" }; yield return new object[] { @"\P{IsHebrew}", RegexOptions.None, "\u05D0a", "Pass. Group[0]=(1,1)" }; yield return new object[] { @"\p{IsHebrew}", RegexOptions.None, "abc\u05D0def", "Pass. Group[0]=(3,1)" }; - yield return new object[] { @"(?<=a+)(?:a)*bc", RegexOptions.None, "aabc", "Pass. Group[0]=(1,3)" }; - yield return new object[] { @"(?<=a*)(?:a)*bc", RegexOptions.None, "aabc", "Pass. Group[0]=(0,4)" }; - yield return new object[] { @"(?<=a{1,5})(?:a)*bc", RegexOptions.None, "aabc", "Pass. Group[0]=(1,3)" }; - yield return new object[] { @"(?<=a{1})(?:a)*bc", RegexOptions.None, "aabc", "Pass. Group[0]=(1,3)" }; - yield return new object[] { @"(?<=a{1,})(?:a)*bc", RegexOptions.None, "aabc", "Pass. Group[0]=(1,3)" }; - yield return new object[] { @"(?<=a+?)(?:a)*bc", RegexOptions.None, "aabc", "Pass. Group[0]=(1,3)" }; - yield return new object[] { @"(?<=a*?)(?:a)*bc", RegexOptions.None, "aabc", "Pass. Group[0]=(0,4)" }; - yield return new object[] { @"(?<=a{1,5}?)(?:a)*bc", RegexOptions.None, "aabc", "Pass. Group[0]=(1,3)" }; - yield return new object[] { @"(?<=a{1}?)(?:a)*bc", RegexOptions.None, "aabc", "Pass. Group[0]=(1,3)" }; - yield return new object[] { @"(?b)(?'1'c)", RegexOptions.ExplicitCapture, "abc", "Pass. Group[0]=(0,3) Group[1]=(1,1)(2,1)" }; - yield return new object[] { @"(?>a*).", RegexOptions.ExplicitCapture, "aaaa", "Fail." }; - yield return new object[] { @"(?ab)c\1", RegexOptions.None, "abcabc", "Pass. Group[0]=(0,5) Group[1]=(0,2)" }; + yield return new object[] { @"(?>a*).", RegexOptions.ExplicitCapture, "aaaa", "Fail.", "NONBACKTRACKINGINCOMPATIBLE" }; + yield return new object[] { @"(?ab)c\1", RegexOptions.None, "abcabc", "Pass. Group[0]=(0,5) Group[1]=(0,2)", "NONBACKTRACKINGINCOMPATIBLE" }; yield return new object[] { @"\1", RegexOptions.ECMAScript, "-", "Fail." }; yield return new object[] { @"\2", RegexOptions.ECMAScript, "-", "Fail." }; yield return new object[] { @"(a)|\2", RegexOptions.ECMAScript, "-", "Fail." }; diff --git a/src/libraries/System.Text.RegularExpressions/tests/Regex.Ctor.Tests.cs b/src/libraries/System.Text.RegularExpressions/tests/Regex.Ctor.Tests.cs index a8ac0cf8f735f9..d6ba970561eff1 100644 --- a/src/libraries/System.Text.RegularExpressions/tests/Regex.Ctor.Tests.cs +++ b/src/libraries/System.Text.RegularExpressions/tests/Regex.Ctor.Tests.cs @@ -19,6 +19,11 @@ public class RegexConstructorTests { public static IEnumerable Ctor_TestData() { + if (PlatformDetection.IsNetCore) + { + yield return new object[] { "foo", RegexHelpers.RegexOptionNonBacktracking, Regex.InfiniteMatchTimeout }; + yield return new object[] { "foo", RegexHelpers.RegexOptionNonBacktracking, new TimeSpan(1) }; + } yield return new object[] { "foo", RegexOptions.None, Regex.InfiniteMatchTimeout }; yield return new object[] { "foo", RegexOptions.RightToLeft, Regex.InfiniteMatchTimeout }; yield return new object[] { "foo", RegexOptions.Compiled, Regex.InfiniteMatchTimeout }; @@ -55,24 +60,34 @@ public static void Ctor(string pattern, RegexOptions options, TimeSpan matchTime Assert.Equal(matchTimeout, regex3.MatchTimeout); } + public static IEnumerable NoneCompiledBacktracking() + { + yield return new object[] { RegexOptions.None }; + yield return new object[] { RegexOptions.Compiled }; + if (PlatformDetection.IsNetCore) + { + yield return new object[] { RegexHelpers.RegexOptionNonBacktracking }; + } + } + [Theory] - [InlineData(RegexOptions.None)] - [InlineData(RegexOptions.Compiled)] + [MemberData(nameof(NoneCompiledBacktracking))] public void CtorDebugInvoke(RegexOptions options) { Regex r; - r = new Regex("[abc]def(ghi|jkl)", options | (RegexOptions)0x80 /*RegexOptions.Debug*/); + r = new Regex("[abc]def(ghi|jkl)", options | RegexHelpers.RegexOptionDebug); Assert.False(r.Match("a").Success); Assert.True(r.Match("adefghi").Success); - Assert.Equal("123456789", r.Replace("123adefghi789", "456")); + string repl = r.Replace("123adefghi78bdefjkl9", "###"); + Assert.Equal("123###78###9", repl); - r = new Regex("(ghi|jkl)*ghi", options | (RegexOptions)0x80 /*RegexOptions.Debug*/); + r = new Regex("(ghi|jkl)*ghi", options | RegexHelpers.RegexOptionDebug); Assert.False(r.Match("jkl").Success); Assert.True(r.Match("ghi").Success); Assert.Equal("123456789", r.Replace("123ghi789", "456")); - r = new Regex("(ghi|jkl)*ghi", options | (RegexOptions)0x80 /*RegexOptions.Debug*/, TimeSpan.FromDays(1)); + r = new Regex("(ghi|jkl)*ghi", options | RegexHelpers.RegexOptionDebug, TimeSpan.FromDays(1)); Assert.False(r.Match("jkl").Success); Assert.True(r.Match("ghi").Success); Assert.Equal("123456789", r.Replace("123ghi789", "456")); @@ -81,6 +96,21 @@ public void CtorDebugInvoke(RegexOptions options) [Fact] public static void Ctor_Invalid() { + if (PlatformDetection.IsNetCore) + { + // NonBacktracking option is not supported together with these other options + Assert.Throws(() => new Regex("abc", RegexOptions.ECMAScript | RegexHelpers.RegexOptionNonBacktracking)); + Assert.Throws(() => new Regex("abc", RegexOptions.RightToLeft | RegexHelpers.RegexOptionNonBacktracking)); + + // NonBacktracking option is not supported for these constructs + Assert.Throws(() => new Regex("(?=a)", RegexHelpers.RegexOptionNonBacktracking)); + Assert.Throws(() => new Regex("(?!a)", RegexHelpers.RegexOptionNonBacktracking)); + Assert.Throws(() => new Regex("(?<=a)", RegexHelpers.RegexOptionNonBacktracking)); + Assert.Throws(() => new Regex("(?(() => new Regex(@"(?(0)ab)", RegexHelpers.RegexOptionNonBacktracking)); + Assert.Throws(() => new Regex(@"([ab])\1", RegexHelpers.RegexOptionNonBacktracking)); + } + // Pattern is null AssertExtensions.Throws("pattern", () => new Regex(null)); AssertExtensions.Throws("pattern", () => new Regex(null, RegexOptions.None)); @@ -90,8 +120,13 @@ public static void Ctor_Invalid() AssertExtensions.Throws("options", () => new Regex("foo", (RegexOptions)(-1))); AssertExtensions.Throws("options", () => new Regex("foo", (RegexOptions)(-1), new TimeSpan())); - AssertExtensions.Throws("options", () => new Regex("foo", (RegexOptions)0x400)); - AssertExtensions.Throws("options", () => new Regex("foo", (RegexOptions)0x400, new TimeSpan())); + AssertExtensions.Throws("options", () => new Regex("foo", (RegexOptions)0x800)); + AssertExtensions.Throws("options", () => new Regex("foo", (RegexOptions)0x800, new TimeSpan())); + if (PlatformDetection.IsNetFramework) + { + AssertExtensions.Throws("options", () => new Regex("foo", RegexHelpers.RegexOptionNonBacktracking)); + AssertExtensions.Throws("options", () => new Regex("foo", RegexHelpers.RegexOptionNonBacktracking, new TimeSpan())); + } AssertExtensions.Throws("options", () => new Regex("foo", RegexOptions.ECMAScript | RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.CultureInvariant | RegexOptions.RightToLeft)); AssertExtensions.Throws("options", () => new Regex("foo", RegexOptions.ECMAScript | RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.CultureInvariant | RegexOptions.ExplicitCapture)); @@ -104,6 +139,44 @@ public static void Ctor_Invalid() AssertExtensions.Throws("matchTimeout", () => new Regex("foo", RegexOptions.None, TimeSpan.FromMilliseconds(int.MaxValue))); } + /// + /// Nonsupported cases for the NonBacktracking option + /// + public static IEnumerable Ctor_Invalid_NonBacktracking_Data() + { + yield return new object[] { @"(?cat)", RegexOptions.None, "balancing group" }; + yield return new object[] { @"(?cat)\w+(?dog)", RegexOptions.None, "balancing group"}; + yield return new object[] { @"(?cat)\w+(?dog)", RegexOptions.None, "balancing group" }; + yield return new object[] { @"abc", RegexOptions.RightToLeft, "RightToLeft" }; + yield return new object[] { @"abc", RegexOptions.ECMAScript, "ECMAScript" }; + yield return new object[] { @"^(a)?(?(1)a|b)+$", RegexOptions.None, "conditional" }; + yield return new object[] { @"(abc)\1", RegexOptions.None, "backreference" }; + yield return new object[] { @"a(?=d).", RegexOptions.None, "positive lookahead" }; + yield return new object[] { @"a(?!b).", RegexOptions.None, "negative lookahead" }; + yield return new object[] { @"(?<=a)b", RegexOptions.None, "positive lookbehind" }; + yield return new object[] { @"(?(abc)*).", RegexOptions.None, "atomic" }; + yield return new object[] { @"\G(\w+\s?\w*),?", RegexOptions.None, "contiguous" }; + yield return new object[] { @"(?>a*).", RegexOptions.None, "atomic" }; + yield return new object[] { @"(?(A)B|C)", RegexOptions.None, "conditional" }; + } + [SkipOnTargetFramework(TargetFrameworkMonikers.NetFramework, "Doesn't support NonBacktracking")] + [Theory] + [MemberData(nameof(Ctor_Invalid_NonBacktracking_Data))] + public void Ctor_Invalid_NonBacktracking(string pattern, RegexOptions options, string expected_word_in_error_message) + { + string actual = string.Empty; + try + { + new Regex(pattern, options | RegexHelpers.RegexOptionNonBacktracking); + } + catch (NotSupportedException e) + { + actual = e.Message; + } + Assert.Contains(expected_word_in_error_message, actual); + } + [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] public static void StaticCtor_InvalidTimeoutObject_ExceptionThrown() { @@ -129,7 +202,15 @@ public void InitializeReferences_OnlyInvokedOnce() { var r = new DerivedRegex(); r.InitializeReferences(); - Assert.Throws(() => r.InitializeReferences()); + if (PlatformDetection.IsNetFramework) + { + Assert.Throws(() => r.InitializeReferences()); + } + else + { + // As of .NET 7, this method is a nop. + r.InitializeReferences(); + } } [Fact] diff --git a/src/libraries/System.Text.RegularExpressions/tests/Regex.GetGroupNames.Tests.cs b/src/libraries/System.Text.RegularExpressions/tests/Regex.GetGroupNames.Tests.cs index 65f81b34cf6acd..38d2b72f3e0b6a 100644 --- a/src/libraries/System.Text.RegularExpressions/tests/Regex.GetGroupNames.Tests.cs +++ b/src/libraries/System.Text.RegularExpressions/tests/Regex.GetGroupNames.Tests.cs @@ -8,11 +8,20 @@ namespace System.Text.RegularExpressions.Tests { public class GetGroupNamesTests { + public static IEnumerable GetGroupNames_MemberData() + { + yield return new object[] { "(?\\S+)\\s(?\\S+)", RegexOptions.None, new string[] { "0", "first_name", "last_name" } }; + if (PlatformDetection.IsNetCore) + { + yield return new object[] { "(?\\S+)\\s(?\\S+)", RegexHelpers.RegexOptionNonBacktracking, new string[] { "0" } }; + } + } + [Theory] - [InlineData("(?\\S+)\\s(?\\S+)", new string[] { "0", "first_name", "last_name" })] - public void GetGroupNames(string pattern, string[] expectedGroupNames) + [MemberData(nameof(GetGroupNames_MemberData))] + public void GetGroupNames(string pattern, RegexOptions options, string[] expectedGroupNames) { - Regex regex = new Regex(pattern); + Regex regex = new Regex(pattern, options); Assert.Equal(expectedGroupNames, regex.GetGroupNames()); } @@ -113,13 +122,34 @@ public static IEnumerable GroupNamesAndNumbers_TestData() new int[] { 0, 15 }, new string[] { "Ryan Byington", "Byington" } }; + + if (PlatformDetection.IsNetCore) + { + yield return new object[] + { + "(?'15'\\S+)\\s(?'15'\\S+)", "Ryan Byington", + new string[] { "0" }, + new int[] { 0 }, + new string[] { "Ryan Byington" }, + RegexHelpers.RegexOptionNonBacktracking + }; + + yield return new object[] + { + "(?\\S+)\\s(?\\S+)", "Ryan Byington", + new string[] { "0" }, + new int[] { 0 }, + new string[] { "Ryan Byington" }, + RegexHelpers.RegexOptionNonBacktracking + }; + } } [Theory] [MemberData(nameof(GroupNamesAndNumbers_TestData))] - public void GroupNamesAndNumbers(string pattern, string input, string[] expectedNames, int[] expectedNumbers, string[] expectedGroups) + public void GroupNamesAndNumbers(string pattern, string input, string[] expectedNames, int[] expectedNumbers, string[] expectedGroups, RegexOptions options = RegexOptions.None) { - Regex regex = new Regex(pattern); + Regex regex = new Regex(pattern, options); Match match = regex.Match(input); Assert.True(match.Success); @@ -143,25 +173,49 @@ public void GroupNamesAndNumbers(string pattern, string input, string[] expected } } + public static IEnumerable GroupNameFromNumber_InvalidIndex_ReturnsEmptyString_MemberData() + { + yield return new object[] { "foo", 1 }; + yield return new object[] { "foo", -1 }; + yield return new object[] { "(?\\S+)\\s(?\\S+)", -1 }; + yield return new object[] { "(?\\S+)\\s(?\\S+)", 3 }; + yield return new object[] { @"((?<256>abc)\d+)?(?<16>xyz)(.*)", -1 }; + + if (PlatformDetection.IsNetCore) + { + yield return new object[] { "(f)(oo)", 1, RegexHelpers.RegexOptionNonBacktracking }; + yield return new object[] { "(f)(oo)", -1, RegexHelpers.RegexOptionNonBacktracking }; + yield return new object[] { "(f)(oo)", 2, RegexHelpers.RegexOptionNonBacktracking }; + } + } + [Theory] - [InlineData("foo", 1)] - [InlineData("foo", -1)] - [InlineData("(?\\S+)\\s(?\\S+)", -1)] - [InlineData("(?\\S+)\\s(?\\S+)", 3)] - [InlineData(@"((?<256>abc)\d+)?(?<16>xyz)(.*)", -1)] - public void GroupNameFromNumber_InvalidIndex_ReturnsEmptyString(string pattern, int index) + [MemberData(nameof(GroupNameFromNumber_InvalidIndex_ReturnsEmptyString_MemberData))] + public void GroupNameFromNumber_InvalidIndex_ReturnsEmptyString(string pattern, int index, RegexOptions options = RegexOptions.None) + { + Assert.Same(string.Empty, new Regex(pattern, options).GroupNameFromNumber(index)); + } + + public static IEnumerable GroupNumberFromName_InvalidName_ReturnsMinusOne_MemberData() { - Assert.Same(string.Empty, new Regex(pattern).GroupNameFromNumber(index)); + yield return new object[] { "foo", "no-such-name" }; + yield return new object[] { "foo", "1" }; + yield return new object[] { "(?\\S+)\\s(?\\S+)", "no-such-name" }; + yield return new object[] { "(?\\S+)\\s(?\\S+)", "FIRST_NAME" }; + + if (PlatformDetection.IsNetCore) + { + yield return new object[] { "(f)(oo)", "no-such-name", RegexHelpers.RegexOptionNonBacktracking }; + yield return new object[] { "(f)(oo)", "1", RegexHelpers.RegexOptionNonBacktracking }; + yield return new object[] { "(f)(oo)", "2", RegexHelpers.RegexOptionNonBacktracking }; + } } [Theory] - [InlineData("foo", "no-such-name")] - [InlineData("foo", "1")] - [InlineData("(?\\S+)\\s(?\\S+)", "no-such-name")] - [InlineData("(?\\S+)\\s(?\\S+)", "FIRST_NAME")] - public void GroupNumberFromName_InvalidName_ReturnsMinusOne(string pattern, string name) + [MemberData(nameof(GroupNumberFromName_InvalidName_ReturnsMinusOne_MemberData))] + public void GroupNumberFromName_InvalidName_ReturnsMinusOne(string pattern, string name, RegexOptions options = RegexOptions.None) { - Assert.Equal(-1, new Regex(pattern).GroupNumberFromName(name)); + Assert.Equal(-1, new Regex(pattern, options).GroupNumberFromName(name)); } [Fact] diff --git a/src/libraries/System.Text.RegularExpressions/tests/Regex.Groups.Tests.cs b/src/libraries/System.Text.RegularExpressions/tests/Regex.Groups.Tests.cs index a668331ce14774..5b43fab690865f 100644 --- a/src/libraries/System.Text.RegularExpressions/tests/Regex.Groups.Tests.cs +++ b/src/libraries/System.Text.RegularExpressions/tests/Regex.Groups.Tests.cs @@ -503,12 +503,13 @@ public static IEnumerable Groups_Basic_TestData() yield return new object[] { null, @"^(([d-f]*)|([c-e]*))$", "dddeeeccceee", RegexOptions.None, new string[] { "dddeeeccceee", "dddeeeccceee", "", "dddeeeccceee" } }; yield return new object[] { null, @"^(([c-e]*)|([d-f]*))$", "dddeeeccceee", RegexOptions.None, new string[] { "dddeeeccceee", "dddeeeccceee", "dddeeeccceee", "" } }; - yield return new object[] { null, @"(([a-d]*)|([a-z]*))", "aaabbbcccdddeeefff", RegexOptions.None, new string[] { "aaabbbcccddd", "aaabbbcccddd", "aaabbbcccddd", "" } }; - yield return new object[] { null, @"(([d-f]*)|([c-e]*))", "dddeeeccceee", RegexOptions.None, new string[] { "dddeee", "dddeee", "dddeee", "" } }; + // Different match in NonBackTracking when order of alternations does not matter + yield return new object[] { null, @"(([a-d]*)|([a-z]*))", "aaabbbcccdddeeefff", RegexOptions.None, new string[] { "aaabbbcccddd", "aaabbbcccddd", "aaabbbcccddd", "" }, "aaabbbcccdddeeefff" }; // <-- Nonbacktracking match same as for "(([a-z]*)|([a-d]*))" + yield return new object[] { null, @"(([d-f]*)|([c-e]*))", "dddeeeccceee", RegexOptions.None, new string[] { "dddeee", "dddeee", "dddeee", "" }, "dddeeeccceee" }; // <-- Nonbacktracking match same as for "(([c-e]*)|([d-f]*))" yield return new object[] { null, @"(([c-e]*)|([d-f]*))", "dddeeeccceee", RegexOptions.None, new string[] { "dddeeeccceee", "dddeeeccceee", "dddeeeccceee", "" } }; - yield return new object[] { null, @"(([a-d]*)|(.*))", "aaabbbcccdddeeefff", RegexOptions.None, new string[] { "aaabbbcccddd", "aaabbbcccddd", "aaabbbcccddd", "" } }; - yield return new object[] { null, @"(([d-f]*)|(.*))", "dddeeeccceee", RegexOptions.None, new string[] { "dddeee", "dddeee", "dddeee", "" } }; + yield return new object[] { null, @"(([a-d]*)|(.*))", "aaabbbcccdddeeefff", RegexOptions.None, new string[] { "aaabbbcccddd", "aaabbbcccddd", "aaabbbcccddd", "" }, "aaabbbcccdddeeefff" }; // <-- Nonbacktracking match same as for ".*" + yield return new object[] { null, @"(([d-f]*)|(.*))", "dddeeeccceee", RegexOptions.None, new string[] { "dddeee", "dddeee", "dddeee", "" }, "dddeeeccceee" }; // <-- Nonbacktracking match same as for ".*" yield return new object[] { null, @"(([c-e]*)|(.*))", "dddeeeccceee", RegexOptions.None, new string[] { "dddeeeccceee", "dddeeeccceee", "dddeeeccceee", "" } }; // \p{Pi} (Punctuation Initial quote) \p{Pf} (Punctuation Final quote) @@ -770,7 +771,8 @@ public static IEnumerable Groups_Basic_TestData() yield return new object[] { null, @"(?s).*(?-s)[1a]", "\n\n\n\n1", RegexOptions.None, new string[] { "\n\n\n\n1" } }; yield return new object[] { null, @".*|.*|.*", "", RegexOptions.None, new string[] { "" } }; yield return new object[] { null, @".*123|abc", "abc\n123", RegexOptions.None, new string[] { "abc" } }; - yield return new object[] { null, @".*123|abc", "abc\n123", RegexOptions.Singleline, new string[] { "abc\n123" } }; + yield return new object[] { null, @".*123|abc", "abc\n123", RegexOptions.Singleline, new string[] { "abc\n123" }, "abc" }; // <-- Nonbacktracking match same as for "abc|.*123" + yield return new object[] { null, @"abc|.*123", "abc\n123", RegexOptions.Singleline, new string[] { "abc" } }; yield return new object[] { null, @".*", "\n", RegexOptions.None, new string[] { "" } }; yield return new object[] { null, @".*\n", "\n", RegexOptions.None, new string[] { "\n" } }; yield return new object[] { null, @".*", "\n", RegexOptions.Singleline, new string[] { "\n" } }; @@ -778,7 +780,8 @@ public static IEnumerable Groups_Basic_TestData() yield return new object[] { null, @".*", "abc", RegexOptions.None, new string[] { "abc" } }; yield return new object[] { null, @".*abc", "abc", RegexOptions.None, new string[] { "abc" } }; yield return new object[] { null, @".*abc|ghi", "ghi", RegexOptions.None, new string[] { "ghi" } }; - yield return new object[] { null, @".*abc|.*ghi", "abcghi", RegexOptions.None, new string[] { "abc" } }; + yield return new object[] { null, @".*abc|.*ghi", "abcghi", RegexOptions.None, new string[] { "abc" }, "abcghi" }; // <-- Nonbacktracking match same as for ".*ghi|.*abc" + yield return new object[] { null, @".*ghi|.*abc", "abcghi", RegexOptions.None, new string[] { "abcghi" } }; yield return new object[] { null, @".*abc|.*ghi", "bcghi", RegexOptions.None, new string[] { "bcghi" } }; yield return new object[] { null, @".*abc|.+c", " \n \n bc", RegexOptions.None, new string[] { " bc" } }; yield return new object[] { null, @".*abc", "12345 abc", RegexOptions.None, new string[] { "12345 abc" } }; @@ -906,7 +909,7 @@ public static IEnumerable Groups_CustomCulture_TestData_AzeriLatin() [MemberData(nameof(Groups_CustomCulture_TestData_AzeriLatin))] [ActiveIssue("https://github.com/dotnet/runtime/issues/56407", TestPlatforms.Android)] [ActiveIssue("https://github.com/dotnet/runtime/issues/36900", TestPlatforms.iOS | TestPlatforms.tvOS | TestPlatforms.MacCatalyst)] - public async Task Groups(string cultureName, string pattern, string input, RegexOptions options, string[] expectedGroups) + public async Task Groups(string cultureName, string pattern, string input, RegexOptions options, string[] expectedGroups, string altMatch = null) { if (cultureName is null) { @@ -918,29 +921,55 @@ public async Task Groups(string cultureName, string pattern, string input, Regex { foreach (RegexEngine engine in RegexHelpers.AvailableEngines) { - await GroupsAsync(engine, pattern, input, options, expectedGroups); + // Alternative altMatch when order of alternations matters in backtracking but order does not matter in NonBacktracking mode + // Also in NonBacktracking there is only a single top-level match, which is expectedGroups[0] when altMatch is null + string[] expected = engine == RegexEngine.NonBacktracking ? + new string[] { altMatch ?? expectedGroups[0] } : + expectedGroups; + + await GroupsAsync(engine, pattern, input, options, expected); } } static async Task GroupsAsync(RegexEngine engine, string pattern, string input, RegexOptions options, string[] expectedGroups) { - Regex regex = await RegexHelpers.GetRegexAsync(engine, pattern, options); + if (engine == RegexEngine.NonBacktracking && pattern.Contains("?(cat)")) + { + // General if-then-else construct is not supported and uses the ?(cat) condition in the tests + // TODO-NONBACKTRACKING: The constructor will throw NotSupportedException so this check will become obsolete + return; + } + + Regex regex; + try + { + regex = await RegexHelpers.GetRegexAsync(engine, pattern, options); + } + catch (NotSupportedException) when (RegexHelpers.IsNonBacktracking(engine)) + { + // Some constructs are not supported in NonBacktracking mode, such as: if-then-else, lookaround, and backreferences + return; + } Match match = regex.Match(input); - Assert.True(match.Success); - Assert.Equal(expectedGroups.Length, match.Groups.Count); + Assert.True(match.Success); Assert.Equal(expectedGroups[0], match.Value); - int[] groupNumbers = regex.GetGroupNumbers(); - string[] groupNames = regex.GetGroupNames(); - for (int i = 0; i < expectedGroups.Length; i++) + if (!RegexHelpers.IsNonBacktracking(engine)) { - Assert.Equal(expectedGroups[i], match.Groups[groupNumbers[i]].Value); - Assert.Equal(match.Groups[groupNumbers[i]], match.Groups[groupNames[i]]); - - Assert.Equal(groupNumbers[i], regex.GroupNumberFromName(groupNames[i])); - Assert.Equal(groupNames[i], regex.GroupNameFromNumber(groupNumbers[i])); + Assert.Equal(expectedGroups.Length, match.Groups.Count); + + int[] groupNumbers = regex.GetGroupNumbers(); + string[] groupNames = regex.GetGroupNames(); + for (int i = 0; i < expectedGroups.Length; i++) + { + Assert.Equal(expectedGroups[i], match.Groups[groupNumbers[i]].Value); + Assert.Equal(match.Groups[groupNumbers[i]], match.Groups[groupNames[i]]); + + Assert.Equal(groupNumbers[i], regex.GroupNumberFromName(groupNames[i])); + Assert.Equal(groupNames[i], regex.GroupNameFromNumber(groupNumbers[i])); + } } } } diff --git a/src/libraries/System.Text.RegularExpressions/tests/Regex.KnownPattern.Tests.cs b/src/libraries/System.Text.RegularExpressions/tests/Regex.KnownPattern.Tests.cs index d43a29f774b516..7aefe06de78b7b 100644 --- a/src/libraries/System.Text.RegularExpressions/tests/Regex.KnownPattern.Tests.cs +++ b/src/libraries/System.Text.RegularExpressions/tests/Regex.KnownPattern.Tests.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Collections.Generic; using System.Globalization; using System.Linq; using System.Threading.Tasks; @@ -35,18 +36,27 @@ public async Task Docs_Examples_ScanningHrefs(RegexEngine engine) Match m = r.Match(InputString); Assert.True(m.Success); - Assert.Equal("http://msdn2.microsoft.com", m.Groups[1].ToString()); - Assert.Equal(43, m.Groups[1].Index); + if (!RegexHelpers.IsNonBacktracking(engine)) + { + Assert.Equal("http://msdn2.microsoft.com", m.Groups[1].ToString()); + Assert.Equal(43, m.Groups[1].Index); + } m = m.NextMatch(); Assert.True(m.Success); - Assert.Equal("http://www.microsoft.com", m.Groups[1].ToString()); - Assert.Equal(102, m.Groups[1].Index); + if (!RegexHelpers.IsNonBacktracking(engine)) + { + Assert.Equal("http://www.microsoft.com", m.Groups[1].ToString()); + Assert.Equal(102, m.Groups[1].Index); + } m = m.NextMatch(); Assert.True(m.Success); - Assert.Equal("http://blogs.msdn.com/bclteam", m.Groups[1].ToString()); - Assert.Equal(176, m.Groups[1].Index); + if (!RegexHelpers.IsNonBacktracking(engine)) + { + Assert.Equal("http://blogs.msdn.com/bclteam", m.Groups[1].ToString()); + Assert.Equal(176, m.Groups[1].Index); + } m = m.NextMatch(); Assert.False(m.Success); @@ -57,6 +67,12 @@ public async Task Docs_Examples_ScanningHrefs(RegexEngine engine) [MemberData(nameof(RegexHelpers.AvailableEngines_MemberData), MemberType = typeof(RegexHelpers))] public async Task Docs_Examples_MDYtoDMY(RegexEngine engine) { + if (RegexHelpers.IsNonBacktracking(engine)) + { + // named group replacements not supported + return; + } + Regex r = await RegexHelpers.GetRegexAsync(engine, @"\b(?\d{1,2})/(?\d{1,2})/(?\d{2,4})\b"); string dt = new DateTime(2020, 1, 8, 0, 0, 0, DateTimeKind.Utc).ToString("d", DateTimeFormatInfo.InvariantInfo); @@ -71,31 +87,45 @@ public async Task Docs_Examples_ExtractProtocolPort(RegexEngine engine) Regex r = await RegexHelpers.GetRegexAsync(engine, @"^(?\w+)://[^/]+?(?:\d+)?/"); Match m = r.Match("http://www.contoso.com:8080/letters/readme.html"); Assert.True(m.Success); - Assert.Equal("http:8080", m.Result("${proto}${port}")); + if (!RegexHelpers.IsNonBacktracking(engine)) + { + Assert.Equal("http:8080", m.Result("${proto}${port}")); + } } - // https://docs.microsoft.com/en-us/dotnet/standard/base-types/how-to-verify-that-strings-are-in-valid-email-format - [Theory] - [InlineData("david.jones@proseware.com", true)] - [InlineData("d.j@server1.proseware.com", true)] - [InlineData("jones@ms1.proseware.com", true)] - [InlineData("j.@server1.proseware.com", false)] - [InlineData("j@proseware.com9", true)] - [InlineData("js#internal@proseware.com", true)] - [InlineData("j_9@[129.126.118.1]", true)] - [InlineData("j..s@proseware.com", false)] - [InlineData("js*@proseware.com", false)] - [InlineData("js@proseware..com", false)] - [InlineData("js@proseware.com9", true)] - [InlineData("j.s@server1.proseware.com", true)] - [InlineData("\"j\\\"s\\\"\"@proseware.com", true)] - [InlineData("js@contoso.\u4E2D\u56FD", true)] - public async Task Docs_Examples_ValidateEmail(string email, bool expectedIsValid) + public static IEnumerable Docs_Examples_ValidateEmail_TestData() { foreach (RegexEngine engine in RegexHelpers.AvailableEngines) { - Assert.Equal(expectedIsValid, await IsValidEmailAsync(email, engine)); + if (RegexHelpers.IsNonBacktracking(engine)) + { + // The email pattern uses conditional test-patterns so NonBacktracking is not supported here + continue; + } + + yield return new object[] { engine, "david.jones@proseware.com", true }; + yield return new object[] { engine, "d.j@server1.proseware.com", true }; + yield return new object[] { engine, "jones@ms1.proseware.com", true }; + yield return new object[] { engine, "j.@server1.proseware.com", false }; + yield return new object[] { engine, "j@proseware.com9", true }; + yield return new object[] { engine, "js#internal@proseware.com", true }; + yield return new object[] { engine, "j_9@[129.126.118.1]", true }; + yield return new object[] { engine, "j..s@proseware.com", false }; + yield return new object[] { engine, "js*@proseware.com", false }; + yield return new object[] { engine, "js@proseware..com", false }; + yield return new object[] { engine, "js@proseware.com9", true }; + yield return new object[] { engine, "j.s@server1.proseware.com", true }; + yield return new object[] { engine, "\"j\\\"s\\\"\"@proseware.com", true }; + yield return new object[] { engine, "js@contoso.\u4E2D\u56FD", true }; } + } + + // https://docs.microsoft.com/en-us/dotnet/standard/base-types/how-to-verify-that-strings-are-in-valid-email-format + [Theory] + [MemberData(nameof(Docs_Examples_ValidateEmail_TestData))] + public async Task Docs_Examples_ValidateEmail(RegexEngine engine, string email, bool expectedIsValid) + { + Assert.Equal(expectedIsValid, await IsValidEmailAsync(email, engine)); async Task IsValidEmailAsync(string email, RegexEngine engine) { @@ -152,6 +182,12 @@ async Task IsValidEmailAsync(string email, RegexEngine engine) [MemberData(nameof(RegexHelpers.AvailableEngines_MemberData), MemberType = typeof(RegexHelpers))] public async Task Docs_GroupingConstructs_MatchedSubexpression(RegexEngine engine) { + if (RegexHelpers.IsNonBacktracking(engine)) + { + // backreferences not supported + return; + } + const string Pattern = @"(\w+)\s(\1)"; const string Input = "He said that that was the the correct answer."; @@ -178,6 +214,12 @@ public async Task Docs_GroupingConstructs_MatchedSubexpression(RegexEngine engin [MemberData(nameof(RegexHelpers.AvailableEngines_MemberData), MemberType = typeof(RegexHelpers))] public async Task Docs_GroupingConstructs_NamedMatchedSubexpression1(RegexEngine engine) { + if (RegexHelpers.IsNonBacktracking(engine)) + { + // subcaptures not supported + return; + } + const string Pattern = @"(?\w+)\s\k\W(?\w+)"; const string Input = "He said that that was the the correct answer."; @@ -204,6 +246,12 @@ public async Task Docs_GroupingConstructs_NamedMatchedSubexpression1(RegexEngine [MemberData(nameof(RegexHelpers.AvailableEngines_MemberData), MemberType = typeof(RegexHelpers))] public async Task Docs_GroupingConstructs_NamedMatchedSubexpression2(RegexEngine engine) { + if (RegexHelpers.IsNonBacktracking(engine)) + { + // subcaptures not supported + return; + } + const string Pattern = @"\D+(?\d+)\D+(?\d+)?"; string[] inputs = { "abc123def456", "abc123def" }; @@ -245,6 +293,12 @@ public async Task Docs_GroupingConstructs_NamedMatchedSubexpression2(RegexEngine [MemberData(nameof(RegexHelpers.AvailableEngines_MemberData), MemberType = typeof(RegexHelpers))] public async Task Docs_GroupingConstructs_BalancingGroups(RegexEngine engine) { + if (RegexHelpers.IsNonBacktracking(engine)) + { + // balancing groups not supported + return; + } + const string Pattern = "^[^<>]*" + "(" + @@ -345,6 +399,12 @@ public async Task Docs_GroupingConstructs_GroupOptions(RegexEngine engine) [MemberData(nameof(RegexHelpers.AvailableEngines_MemberData), MemberType = typeof(RegexHelpers))] public async Task Docs_GroupingConstructs_ZeroWidthPositiveLookaheadAssertions(RegexEngine engine) { + if (RegexHelpers.IsNonBacktracking(engine)) + { + // lookaheads not supported + return; + } + const string Pattern = @"\b\w+(?=\sis\b)"; Match match; @@ -370,6 +430,12 @@ public async Task Docs_GroupingConstructs_ZeroWidthPositiveLookaheadAssertions(R [MemberData(nameof(RegexHelpers.AvailableEngines_MemberData), MemberType = typeof(RegexHelpers))] public async Task Docs_GroupingConstructs_ZeroWidthNegativeLookaheadAssertions(RegexEngine engine) { + if (RegexHelpers.IsNonBacktracking(engine)) + { + // lookaheads not supported + return; + } + const string Pattern = @"\b(?!un)\w+\b"; const string Input = "unite one unethical ethics use untie ultimate"; @@ -387,6 +453,12 @@ public async Task Docs_GroupingConstructs_ZeroWidthNegativeLookaheadAssertions(R [MemberData(nameof(RegexHelpers.AvailableEngines_MemberData), MemberType = typeof(RegexHelpers))] public async Task Docs_GroupingConstructs_ZeroWidthPositiveLookbehindAssertions(RegexEngine engine) { + if (RegexHelpers.IsNonBacktracking(engine)) + { + // lookbehinds not supported + return; + } + const string Pattern = @"(?<=\b20)\d{2}\b"; const string Input = "2010 1999 1861 2140 2009"; @@ -402,6 +474,12 @@ public async Task Docs_GroupingConstructs_ZeroWidthPositiveLookbehindAssertions( [MemberData(nameof(RegexHelpers.AvailableEngines_MemberData), MemberType = typeof(RegexHelpers))] public async Task Docs_GroupingConstructs_ZeroWidthNegativeLookbehindAssertions(RegexEngine engine) { + if (RegexHelpers.IsNonBacktracking(engine)) + { + // lookbehinds not supported + return; + } + const string Pattern = @"(?(\w)\1+).\b"); string[] inputs = { "aaad", "aaaa" }; @@ -450,6 +534,12 @@ public async Task Docs_GroupingConstructs_NonbacktrackingSubexpressions(RegexEng [MemberData(nameof(RegexHelpers.AvailableEngines_MemberData), MemberType = typeof(RegexHelpers))] public async Task Docs_GroupingConstructs_GroupCaptureRelationship(RegexEngine engine) { + if (RegexHelpers.IsNonBacktracking(engine)) + { + // subcaptures not supported + return; + } + const string Pattern = @"(\b(\w+)\W+)+"; const string Input = "This is a short sentence."; @@ -493,6 +583,12 @@ public async Task Docs_GroupingConstructs_GroupCaptureRelationship(RegexEngine e [MemberData(nameof(RegexHelpers.AvailableEngines_MemberData), MemberType = typeof(RegexHelpers))] public async Task Docs_Capture_Sentences(RegexEngine engine) { + if (RegexHelpers.IsNonBacktracking(engine)) + { + // subcaptures not supported + return; + } + const string Pattern = @"((\w+)[\s.])+"; const string Input = "Yes. This dog is very friendly."; @@ -545,6 +641,12 @@ public async Task Docs_Capture_Sentences(RegexEngine engine) [MemberData(nameof(RegexHelpers.AvailableEngines_MemberData), MemberType = typeof(RegexHelpers))] public async Task Docs_Capture_ProductNumber(RegexEngine engine) { + if (RegexHelpers.IsNonBacktracking(engine)) + { + // subcaptures not supported + return; + } + const string Pattern = @"^([a-z]+)(\d+)?\.([a-z]+(\d)*)$"; string[] values = { "AC10", "Za203.CYM", "XYZ.CoA", "ABC.x170" }; @@ -660,7 +762,10 @@ public async Task Docs_Backtracking_WithOptionalQuantifiersOrAlternationConstruc Assert.True(m.Success); Assert.Equal("Essential services are provided by regular expres", m.Value); Assert.Equal(0, m.Index); - Assert.Equal(47, m.Groups[1].Index); + if (!RegexHelpers.IsNonBacktracking(engine)) + { + Assert.Equal(47, m.Groups[1].Index); + } Assert.False(m.NextMatch().Success); } @@ -680,8 +785,13 @@ public async Task Docs_Backtracking_WithNestedOptionalQuantifiers_ExcessiveBackt [MemberData(nameof(RegexHelpers.AvailableEngines_MemberData), MemberType = typeof(RegexHelpers))] public async Task Docs_Backtracking_WithNestedOptionalQuantifiers_BacktrackingEliminated(RegexEngine engine) { - Regex r = await RegexHelpers.GetRegexAsync(engine, "^((?>[0-9a-fA-F]{1,4}:)*(?>[0-9a-fA-F]{1,4}))*(::)$"); - Assert.False(r.IsMatch("b51:4:1DB:9EE1:5:27d60:f44:D4:cd:E:5:0A5:4a:D24:41Ad:")); + const string Input = "b51:4:1DB:9EE1:5:27d60:f44:D4:cd:E:5:0A5:4a:D24:41Ad:"; + + Regex r = await RegexHelpers.GetRegexAsync(engine, engine == RegexEngine.NonBacktracking ? + "^(([0-9a-fA-F]{1,4}:)*([0-9a-fA-F]{1,4}))*(::)$" : // Using RegexOptions.NonBacktracking to avoid backtracking + "^((?>[0-9a-fA-F]{1,4}:)*(?>[0-9a-fA-F]{1,4}))*(::)$"); // Using atomic to avoid backtracking + + Assert.False(r.IsMatch(Input)); } // https://docs.microsoft.com/en-us/dotnet/standard/base-types/backtracking-in-regular-expressions#lookbehind-assertions @@ -689,6 +799,12 @@ public async Task Docs_Backtracking_WithNestedOptionalQuantifiers_BacktrackingEl [MemberData(nameof(RegexHelpers.AvailableEngines_MemberData), MemberType = typeof(RegexHelpers))] public async Task Docs_Backtracking_LookbehindAssertions(RegexEngine engine) { + if (RegexHelpers.IsNonBacktracking(engine)) + { + // lookbehinds not supported + return; + } + const string Input = "test@contoso.com"; Regex rPattern = await RegexHelpers.GetRegexAsync(engine, @"^[0-9A-Z]([-.\w]*[0-9A-Z])?@", RegexOptions.IgnoreCase); @@ -713,6 +829,12 @@ public async Task Docs_Backtracking_LookaheadAssertions_ExcessiveBacktracking(Re [MemberData(nameof(RegexHelpers.AvailableEngines_MemberData), MemberType = typeof(RegexHelpers))] public async Task Docs_Backtracking_LookaheadAssertions_BacktrackingEliminated(RegexEngine engine) { + if (RegexHelpers.IsNonBacktracking(engine)) + { + // lookaheads not supported + return; + } + Regex r = await RegexHelpers.GetRegexAsync(engine, @"^((?=[A-Z])\w+\.)*[A-Z]\w*$", RegexOptions.IgnoreCase); Assert.False(r.IsMatch("aaaaaaaaaaaaaaaaaaaaaa.")); } @@ -727,12 +849,18 @@ public async Task Docs_EngineCapabilities_LazyQuantifiers(RegexEngine engine) Regex rGreedy = await RegexHelpers.GetRegexAsync(engine, @".+(\d+)\."); Match match = rGreedy.Match(Input); Assert.True(match.Success); - Assert.Equal("5", match.Groups[1].Value); + if (!RegexHelpers.IsNonBacktracking(engine)) + { + Assert.Equal("5", match.Groups[1].Value); + } Regex rLazy = await RegexHelpers.GetRegexAsync(engine, @".+?(\d+)\."); match = rLazy.Match(Input); Assert.True(match.Success); - Assert.Equal("107325", match.Groups[1].Value); + if (!RegexHelpers.IsNonBacktracking(engine)) + { + Assert.Equal("107325", match.Groups[1].Value); + } } // https://docs.microsoft.com/en-us/dotnet/standard/base-types/details-of-regular-expression-behavior#net-framework-engine-capabilities @@ -740,6 +868,12 @@ public async Task Docs_EngineCapabilities_LazyQuantifiers(RegexEngine engine) [MemberData(nameof(RegexHelpers.AvailableEngines_MemberData), MemberType = typeof(RegexHelpers))] public async Task Docs_EngineCapabilities_PositiveLookahead(RegexEngine engine) { + if (RegexHelpers.IsNonBacktracking(engine)) + { + // lookaheads not supported + return; + } + const string Pattern = @"\b[A-Z]+\b(?=\P{P})"; const string Input = "If so, what comes next?"; @@ -757,6 +891,12 @@ public async Task Docs_EngineCapabilities_PositiveLookahead(RegexEngine engine) [MemberData(nameof(RegexHelpers.AvailableEngines_MemberData), MemberType = typeof(RegexHelpers))] public async Task Docs_EngineCapabilities_NegativeLookahead(RegexEngine engine) { + if (RegexHelpers.IsNonBacktracking(engine)) + { + // lookaheads not supported + return; + } + const string Pattern = @"\b(?!non)\w+\b"; const string Input = "Nonsense is not always non-functional."; @@ -775,6 +915,12 @@ public async Task Docs_EngineCapabilities_NegativeLookahead(RegexEngine engine) [MemberData(nameof(RegexHelpers.AvailableEngines_MemberData), MemberType = typeof(RegexHelpers))] public async Task Docs_EngineCapabilities_ConditionalEvaluation(RegexEngine engine) { + if (RegexHelpers.IsNonBacktracking(engine)) + { + // conditionals not supported + return; + } + const string Pattern = @"\b(?(\d{2}-)\d{2}-\d{7}|\d{3}-\d{2}-\d{4})\b"; const string Input = "01-9999999 020-333333 777-88-9999"; @@ -795,6 +941,12 @@ public async Task Docs_EngineCapabilities_ConditionalEvaluation(RegexEngine engi [MemberData(nameof(RegexHelpers.AvailableEngines_MemberData), MemberType = typeof(RegexHelpers))] public async Task Docs_EngineCapabilities_RightToLeftMatching(RegexEngine engine) { + if (RegexHelpers.IsNonBacktracking(engine)) + { + // RightToLeft not supported + return; + } + const string GreedyPattern = @".+(\d+)\."; const string Input = "This sentence ends with the number 107325."; @@ -817,6 +969,12 @@ public async Task Docs_EngineCapabilities_RightToLeftMatching(RegexEngine engine [MemberData(nameof(RegexHelpers.AvailableEngines_MemberData), MemberType = typeof(RegexHelpers))] public async Task Docs_EngineCapabilities_PositiveNegativeLookbehind(RegexEngine engine) { + if (RegexHelpers.IsNonBacktracking(engine)) + { + // lookbehinds not supported + return; + } + const string Pattern = @"^[A-Z0-9]([-!#$%&'.*+/=?^`{}|~\w])*(?<=[A-Z0-9])$"; Regex r = await RegexHelpers.GetRegexAsync(engine, Pattern, RegexOptions.IgnoreCase); @@ -833,6 +991,12 @@ public async Task Docs_EngineCapabilities_PositiveNegativeLookbehind(RegexEngine [MemberData(nameof(RegexHelpers.AvailableEngines_MemberData), MemberType = typeof(RegexHelpers))] public async Task Docs_InlineOptions(RegexEngine engine) { + if (RegexHelpers.IsNonBacktracking(engine)) + { + // subcaptures not supported + return; + } + const string Input = "double dare double Double a Drooling dog The Dreaded Deep"; var actual = new StringBuilder(); @@ -888,14 +1052,20 @@ public async Task Docs_InlineComment(RegexEngine engine) Match match = r.Match(Input); Assert.True(match.Success); Assert.Equal("Drooling dog", match.Value); - Assert.Equal(2, match.Groups.Count); - Assert.Equal("Drooling", match.Groups[1].Value); + if (!RegexHelpers.IsNonBacktracking(engine)) + { + Assert.Equal(2, match.Groups.Count); + Assert.Equal("Drooling", match.Groups[1].Value); + } match = match.NextMatch(); Assert.True(match.Success); Assert.Equal("Dreaded Deep", match.Value); - Assert.Equal(2, match.Groups.Count); - Assert.Equal("Dreaded", match.Groups[1].Value); + if (!RegexHelpers.IsNonBacktracking(engine)) + { + Assert.Equal(2, match.Groups.Count); + Assert.Equal("Dreaded", match.Groups[1].Value); + } Assert.False(match.NextMatch().Success); } @@ -918,6 +1088,12 @@ public async Task Docs_EndOfLineComment(RegexEngine engine) [MemberData(nameof(RegexHelpers.AvailableEngines_MemberData), MemberType = typeof(RegexHelpers))] public async Task Docs_Anchors_ContiguousMatches(RegexEngine engine) { + if (RegexHelpers.IsNonBacktracking(engine)) + { + // contiguous matches (\G) not supported + return; + } + const string Input = "capybara,squirrel,chipmunk,porcupine"; const string Pattern = @"\G(\w+\s?\w*),?"; string[] expected = new[] { "capybara", "squirrel", "chipmunk", "porcupine" }; @@ -957,8 +1133,11 @@ public async Task RealWorld_ExtractResourceUri(string url, string expected) Regex r = await RegexHelpers.GetRegexAsync(engine, @"/providers/(.+?)\?"); Match m = r.Match(url); Assert.True(m.Success); - Assert.Equal(2, m.Groups.Count); - Assert.Equal(expected, m.Groups[1].Value); + if (!RegexHelpers.IsNonBacktracking(engine)) + { + Assert.Equal(2, m.Groups.Count); + Assert.Equal(expected, m.Groups[1].Value); + } } } @@ -1045,7 +1224,10 @@ public async Task RealWorld_ValueParse(string value, string expected) Regex r = await RegexHelpers.GetRegexAsync(engine, @"(?-?\d+(\.\d+)?)"); Match m = r.Match(value); Assert.True(m.Success); - Assert.Equal(expected, m.Groups["value"].Value); + if (!RegexHelpers.IsNonBacktracking(engine)) // named capture groups unsupported + { + Assert.Equal(expected, m.Groups["value"].Value); + } } } @@ -1059,7 +1241,10 @@ public async Task RealWorld_FirebirdVersionString(string value, string expected) Regex r = await RegexHelpers.GetRegexAsync(engine, @"\w{2}-\w(\d+\.\d+\.\d+\.\d+)"); Match m = r.Match(value); Assert.True(m.Success); - Assert.Equal(expected, m.Groups[1].Value); + if (!RegexHelpers.IsNonBacktracking(engine)) + { + Assert.Equal(expected, m.Groups[1].Value); + } } } @@ -1075,10 +1260,63 @@ public async Task RealWorld_ExternalEntryPoint(string value, string a, string b, Regex r = await RegexHelpers.GetRegexAsync(engine, @"^(.+)!(.+)\.([^.]+)$"); Match m = r.Match(value); Assert.True(m.Success); - Assert.Equal(a, m.Groups[1].Value); - Assert.Equal(b, m.Groups[2].Value); - Assert.Equal(c, m.Groups[3].Value); + if (!RegexHelpers.IsNonBacktracking(engine)) // subcaptures aren't supported + { + Assert.Equal(a, m.Groups[1].Value); + Assert.Equal(b, m.Groups[2].Value); + Assert.Equal(c, m.Groups[3].Value); + } } } + + /// + /// Test that these well-known patterns that are hard for backtracking engines + /// are not a problem with NonBacktracking. + /// + [SkipOnTargetFramework(TargetFrameworkMonikers.NetFramework, "Doesn't support NonBacktracking")] + [Theory] + [InlineData("((?:0*)+?(?:.*)+?)?", "0a", 2)] + [InlineData("(?:(?:0?)+?(?:a?)+?)?", "0a", 2)] + [InlineData(@"(?i:(\()((?\w+(\.\w+)*)(,(?\w+(\.\w+)*)*)?)(\)))", "some.text(this.is,the.match)", 1)] + private void DifficultForBacktracking(string pattern, string input, int matchcount) + { + var regex = new Regex(pattern, RegexHelpers.RegexOptionNonBacktracking); + List matches = new List(); + var match = regex.Match(input); + while (match.Success) + { + matches.Add(match); + match = match.NextMatch(); + } + Assert.Equal(matchcount, matches.Count); + } + + /// + /// Another difficult pattern in backtracking that is fast in NonBacktracking. + /// + [SkipOnTargetFramework(TargetFrameworkMonikers.NetFramework, "Doesn't support NonBacktracking")] + [Theory] + [InlineData(RegexOptions.None, 1)] + [InlineData(RegexOptions.Compiled, 1)] + public void TerminationInNonBacktrackingVsBackTracking(RegexOptions options, int sec) + { + string input = " 123456789 123456789 123456789 123456789 123456789"; + TimeSpan ts = new TimeSpan(0, 0, sec); + for (int i = 0; i < 12; i++) + { + input += input; + } + + // The input has 2^12 * 50 = 204800 characters + string rawregex = @"[\\/]?[^\\/]*?(heythere|hej)[^\\/]*?$"; + Regex reC = new Regex(rawregex, options, ts); + Regex re = new Regex(rawregex, RegexHelpers.RegexOptionNonBacktracking, ts); + + // It takes over 4min with backtracking, so test that 1sec timeout happens + Assert.Throws(() => { reC.Match(input); }); + + // NonBacktracking needs way less than 1s + Assert.False(re.Match(input).Success); + } } } diff --git a/src/libraries/System.Text.RegularExpressions/tests/Regex.Match.Tests.cs b/src/libraries/System.Text.RegularExpressions/tests/Regex.Match.Tests.cs index bc4f615b9878f3..c262db55b0ee8f 100644 --- a/src/libraries/System.Text.RegularExpressions/tests/Regex.Match.Tests.cs +++ b/src/libraries/System.Text.RegularExpressions/tests/Regex.Match.Tests.cs @@ -10,471 +10,540 @@ using System.Tests; using Microsoft.DotNet.RemoteExecutor; using Xunit; +using Microsoft.DotNet.XUnitExtensions; namespace System.Text.RegularExpressions.Tests { public class RegexMatchTests { + public static IEnumerable Match_Basic_TestData_LazyLoops() + { + var allOptions = new List() { RegexOptions.Singleline, RegexOptions.Compiled | RegexOptions.Singleline }; + if (PlatformDetection.IsNetCore) + { + allOptions.Add(RegexHelpers.RegexOptionNonBacktracking | RegexOptions.Singleline); + } + + foreach (RegexOptions options in allOptions) + { + yield return new object[] { @"\W.*?\D", "seq 012 of 3 digits", options, 0, 19, true, " 012 " }; + yield return new object[] { @"\W.+?\D", "seq 012 of 3 digits", options, 0, 19, true, " 012 " }; + yield return new object[] { @"\W.{1,7}?\D", "seq 012 of 3 digits", options, 0, 19, true, " 012 " }; + yield return new object[] { @"\W.{1,2}?\D", "seq 012 of 3 digits", options, 0, 19, true, " of" }; + yield return new object[] { @"\W.*?\b", "digits:0123456789", options, 0, 17, true, ":" }; + yield return new object[] { @"\B.*?\B", "e.g:abc", options, 0, 7, true, "" }; + yield return new object[] { @"\B\W+?", "e.g:abc", options, 0, 7, false, "" }; + yield return new object[] { @"\B\W*?", "e.g:abc", options, 0, 7, true, "" }; + + // While not lazy loops themselves, variants of the prior case that should give same results here + yield return new object[] { @"\B\W*", "e.g:abc", options, 0, 7, true, "" }; + yield return new object[] { @"\B\W?", "e.g:abc", options, 0, 7, true, "" }; + + //mixed lazy and eager counting + yield return new object[] { "z(a{0,5}|a{0,10}?)", "xyzaaaaaaaaaxyz", options, 0, 15, true, "zaaaaa" }; + } + } + public static IEnumerable Match_Basic_TestData() { - // pattern, input, options, beginning, length, expectedSuccess, expectedValue - yield return new object[] { @"H#", "#H#", RegexOptions.IgnoreCase, 0, 3, true, "H#" }; // https://github.com/dotnet/runtime/issues/39390 - yield return new object[] { @"H#", "#H#", RegexOptions.None, 0, 3, true, "H#" }; + foreach (RegexEngine engine in RegexHelpers.AvailableEngines) + { + // pattern, input, options, beginning, length, expectedSuccess, expectedValue + yield return new object[] { engine, @"H#", "#H#", RegexOptions.IgnoreCase, 0, 3, true, "H#" }; // https://github.com/dotnet/runtime/issues/39390 + yield return new object[] { engine, @"H#", "#H#", RegexOptions.None, 0, 3, true, "H#" }; - // Testing octal sequence matches: "\\060(\\061)?\\061" - // Octal \061 is ASCII 49 ('1') - yield return new object[] { @"\060(\061)?\061", "011", RegexOptions.None, 0, 3, true, "011" }; + // Testing octal sequence matches: "\\060(\\061)?\\061" + // Octal \061 is ASCII 49 ('1') + yield return new object[] { engine, @"\060(\061)?\061", "011", RegexOptions.None, 0, 3, true, "011" }; - // Testing hexadecimal sequence matches: "(\\x30\\x31\\x32)" - // Hex \x31 is ASCII 49 ('1') - yield return new object[] { @"(\x30\x31\x32)", "012", RegexOptions.None, 0, 3, true, "012" }; + // Testing hexadecimal sequence matches: "(\\x30\\x31\\x32)" + // Hex \x31 is ASCII 49 ('1') + yield return new object[] { engine, @"(\x30\x31\x32)", "012", RegexOptions.None, 0, 3, true, "012" }; - // Testing control character escapes???: "2", "(\u0032)" - yield return new object[] { "(\u0034)", "4", RegexOptions.None, 0, 1, true, "4", }; + // Testing control character escapes???: "2", "(\u0032)" + yield return new object[] { engine, "(\u0034)", "4", RegexOptions.None, 0, 1, true, "4", }; - // Using long loop prefix - yield return new object[] { @"a{10}", new string('a', 10), RegexOptions.None, 0, 10, true, new string('a', 10) }; - yield return new object[] { @"a{100}", new string('a', 100), RegexOptions.None, 0, 100, true, new string('a', 100) }; + // Using long loop prefix + yield return new object[] { engine, @"a{10}", new string('a', 10), RegexOptions.None, 0, 10, true, new string('a', 10) }; + yield return new object[] { engine, @"a{100}", new string('a', 100), RegexOptions.None, 0, 100, true, new string('a', 100) }; - yield return new object[] { @"a{10}b", new string('a', 10) + "bc", RegexOptions.None, 0, 12, true, new string('a', 10) + "b" }; - yield return new object[] { @"a{100}b", new string('a', 100) + "bc", RegexOptions.None, 0, 102, true, new string('a', 100) + "b" }; + yield return new object[] { engine, @"a{10}b", new string('a', 10) + "bc", RegexOptions.None, 0, 12, true, new string('a', 10) + "b" }; + yield return new object[] { engine, @"a{100}b", new string('a', 100) + "bc", RegexOptions.None, 0, 102, true, new string('a', 100) + "b" }; - yield return new object[] { @"a{11}b", new string('a', 10) + "bc", RegexOptions.None, 0, 12, false, string.Empty }; - yield return new object[] { @"a{101}b", new string('a', 100) + "bc", RegexOptions.None, 0, 102, false, string.Empty }; + yield return new object[] { engine, @"a{11}b", new string('a', 10) + "bc", RegexOptions.None, 0, 12, false, string.Empty }; + yield return new object[] { engine, @"a{101}b", new string('a', 100) + "bc", RegexOptions.None, 0, 102, false, string.Empty }; - yield return new object[] { @"a{1,3}b", "bc", RegexOptions.None, 0, 2, false, string.Empty }; - yield return new object[] { @"a{1,3}b", "abc", RegexOptions.None, 0, 3, true, "ab" }; - yield return new object[] { @"a{1,3}b", "aaabc", RegexOptions.None, 0, 5, true, "aaab" }; - yield return new object[] { @"a{1,3}b", "aaaabc", RegexOptions.None, 0, 6, true, "aaab" }; + yield return new object[] { engine, @"a{1,3}b", "bc", RegexOptions.None, 0, 2, false, string.Empty }; + yield return new object[] { engine, @"a{1,3}b", "abc", RegexOptions.None, 0, 3, true, "ab" }; + yield return new object[] { engine, @"a{1,3}b", "aaabc", RegexOptions.None, 0, 5, true, "aaab" }; + yield return new object[] { engine, @"a{1,3}b", "aaaabc", RegexOptions.None, 0, 6, true, "aaab" }; - yield return new object[] { @"a{2,}b", "abc", RegexOptions.None, 0, 3, false, string.Empty }; - yield return new object[] { @"a{2,}b", "aabc", RegexOptions.None, 0, 4, true, "aab" }; + yield return new object[] { engine, @"a{2,}b", "abc", RegexOptions.None, 0, 3, false, string.Empty }; + yield return new object[] { engine, @"a{2,}b", "aabc", RegexOptions.None, 0, 4, true, "aab" }; - // {,n} is treated as a literal rather than {0,n} as it should be - yield return new object[] { @"a{,3}b", "a{,3}bc", RegexOptions.None, 0, 6, true, "a{,3}b" }; - yield return new object[] { @"a{,3}b", "aaabc", RegexOptions.None, 0, 5, false, string.Empty }; + // {,n} is treated as a literal rather than {0,n} as it should be + yield return new object[] { engine, @"a{,3}b", "a{,3}bc", RegexOptions.None, 0, 6, true, "a{,3}b" }; + yield return new object[] { engine, @"a{,3}b", "aaabc", RegexOptions.None, 0, 5, false, string.Empty }; - // Using [a-z], \s, \w: Actual - "([a-zA-Z]+)\\s(\\w+)" - yield return new object[] { @"([a-zA-Z]+)\s(\w+)", "David Bau", RegexOptions.None, 0, 9, true, "David Bau" }; + // Using [a-z], \s, \w: Actual - "([a-zA-Z]+)\\s(\\w+)" + yield return new object[] { engine, @"([a-zA-Z]+)\s(\w+)", "David Bau", RegexOptions.None, 0, 9, true, "David Bau" }; - // \\S, \\d, \\D, \\W: Actual - "(\\S+):\\W(\\d+)\\s(\\D+)" - yield return new object[] { @"(\S+):\W(\d+)\s(\D+)", "Price: 5 dollars", RegexOptions.None, 0, 16, true, "Price: 5 dollars" }; + // \\S, \\d, \\D, \\W: Actual - "(\\S+):\\W(\\d+)\\s(\\D+)" + yield return new object[] { engine, @"(\S+):\W(\d+)\s(\D+)", "Price: 5 dollars", RegexOptions.None, 0, 16, true, "Price: 5 dollars" }; - // \\S, \\d, \\D, \\W: Actual - "[^0-9]+(\\d+)" - yield return new object[] { @"[^0-9]+(\d+)", "Price: 30 dollars", RegexOptions.None, 0, 17, true, "Price: 30" }; + // \\S, \\d, \\D, \\W: Actual - "[^0-9]+(\\d+)" + yield return new object[] { engine, @"[^0-9]+(\d+)", "Price: 30 dollars", RegexOptions.None, 0, 17, true, "Price: 30" }; - // Zero-width negative lookahead assertion: Actual - "abc(?!XXX)\\w+" - yield return new object[] { @"abc(?!XXX)\w+", "abcXXXdef", RegexOptions.None, 0, 9, false, string.Empty }; + if (!RegexHelpers.IsNonBacktracking(engine)) + { + // Zero-width negative lookahead assertion: Actual - "abc(?!XXX)\\w+" + yield return new object[] { engine, @"abc(?!XXX)\w+", "abcXXXdef", RegexOptions.None, 0, 9, false, string.Empty }; - // Zero-width positive lookbehind assertion: Actual - "(\\w){6}(?<=XXX)def" - yield return new object[] { @"(\w){6}(?<=XXX)def", "abcXXXdef", RegexOptions.None, 0, 9, true, "abcXXXdef" }; + // Zero-width positive lookbehind assertion: Actual - "(\\w){6}(?<=XXX)def" + yield return new object[] { engine, @"(\w){6}(?<=XXX)def", "abcXXXdef", RegexOptions.None, 0, 9, true, "abcXXXdef" }; - // Zero-width negative lookbehind assertion: Actual - "(\\w){6}(?[0-9]+)3" - // The last 3 causes the match to fail, since the non backtracking subexpression does not give up the last digit it matched - // for it to be a success. For a correct match, remove the last character, '3' from the pattern - yield return new object[] { "[^0-9]+(?>[0-9]+)3", "abc123", RegexOptions.None, 0, 6, false, string.Empty }; - yield return new object[] { "[^0-9]+(?>[0-9]+)", "abc123", RegexOptions.None, 0, 6, true, "abc123" }; + // Nonbacktracking subexpression: Actual - "[^0-9]+(?>[0-9]+)3" + // The last 3 causes the match to fail, since the non backtracking subexpression does not give up the last digit it matched + // for it to be a success. For a correct match, remove the last character, '3' from the pattern + yield return new object[] { engine, "[^0-9]+(?>[0-9]+)3", "abc123", RegexOptions.None, 0, 6, false, string.Empty }; + yield return new object[] { engine, "[^0-9]+(?>[0-9]+)", "abc123", RegexOptions.None, 0, 6, true, "abc123" }; + } - // More nonbacktracking expressions - foreach (RegexOptions options in new[] { RegexOptions.None, RegexOptions.IgnoreCase }) - { - string Case(string s) => (options & RegexOptions.IgnoreCase) != 0 ? s.ToUpper() : s; - - yield return new object[] { Case("(?>[0-9]+)abc"), "abc12345abc", options, 3, 8, true, "12345abc" }; - yield return new object[] { Case("(?>(?>[0-9]+))abc"), "abc12345abc", options, 3, 8, true, "12345abc" }; - yield return new object[] { Case("(?>[0-9]*)abc"), "abc12345abc", options, 3, 8, true, "12345abc" }; - yield return new object[] { Case("(?>[^z]+)z"), "zzzzxyxyxyz123", options, 4, 9, true, "xyxyxyz" }; - yield return new object[] { Case("(?>(?>[^z]+))z"), "zzzzxyxyxyz123", options, 4, 9, true, "xyxyxyz" }; - yield return new object[] { Case("(?>[^z]*)z123"), "zzzzxyxyxyz123", options, 4, 10, true, "xyxyxyz123" }; - yield return new object[] { Case("(?>a+)123"), "aa1234", options, 0, 5, true, "aa123" }; - yield return new object[] { Case("(?>a*)123"), "aa1234", options, 0, 5, true, "aa123" }; - yield return new object[] { Case("(?>(?>a*))123"), "aa1234", options, 0, 5, true, "aa123" }; - yield return new object[] { Case("(?>a+?)a"), "aaaaa", options, 0, 2, true, "aa" }; - yield return new object[] { Case("(?>a*?)a"), "aaaaa", options, 0, 1, true, "a" }; - yield return new object[] { Case("(?>hi|hello|hey)hi"), "hellohi", options, 0, 0, false, string.Empty }; - yield return new object[] { Case("(?:hi|hello|hey)hi"), "hellohi", options, 0, 7, true, "hellohi" }; // allow backtracking and it succeeds - yield return new object[] { Case("(?>hi|hello|hey)hi"), "hihi", options, 0, 4, true, "hihi" }; - yield return new object[] { Case(@"a[^wyz]*w"), "abczw", RegexOptions.IgnoreCase, 0, 0, false, string.Empty }; - } + // More nonbacktracking expressions + foreach (RegexOptions options in new[] { RegexOptions.None, RegexOptions.IgnoreCase }) + { + string Case(string s) => (options & RegexOptions.IgnoreCase) != 0 ? s.ToUpper() : s; - // Loops at beginning of expressions - yield return new object[] { @"a+", "aaa", RegexOptions.None, 0, 3, true, "aaa" }; - yield return new object[] { @"a+\d+", "a1", RegexOptions.None, 0, 2, true, "a1" }; - yield return new object[] { @".+\d+", "a1", RegexOptions.None, 0, 2, true, "a1" }; - yield return new object[] { ".+\nabc", "a\nabc", RegexOptions.None, 0, 5, true, "a\nabc" }; - yield return new object[] { @"\d+", "abcd123efg", RegexOptions.None, 0, 10, true, "123" }; - yield return new object[] { @"\d+\d+", "abcd123efg", RegexOptions.None, 0, 10, true, "123" }; - yield return new object[] { @"\w+123\w+", "abcd123efg", RegexOptions.None, 0, 10, true, "abcd123efg" }; - yield return new object[] { @"\d+\w+", "abcd123efg", RegexOptions.None, 0, 10, true, "123efg" }; - yield return new object[] { @"\w+@\w+.com", "abc@def.com", RegexOptions.None, 0, 11, true, "abc@def.com" }; - yield return new object[] { @"\w{3,}@\w+.com", "abc@def.com", RegexOptions.None, 0, 11, true, "abc@def.com" }; - yield return new object[] { @"\w{4,}@\w+.com", "abc@def.com", RegexOptions.None, 0, 11, false, string.Empty }; - yield return new object[] { @"\w{2,5}@\w+.com", "abc@def.com", RegexOptions.None, 0, 11, true, "abc@def.com" }; - yield return new object[] { @"\w{3}@\w+.com", "abc@def.com", RegexOptions.None, 0, 11, true, "abc@def.com" }; - yield return new object[] { @"\w{0,3}@\w+.com", "abc@def.com", RegexOptions.None, 0, 11, true, "abc@def.com" }; - yield return new object[] { @"\w{0,2}c@\w+.com", "abc@def.com", RegexOptions.None, 0, 11, true, "abc@def.com" }; - yield return new object[] { @"\w*@\w+.com", "abc@def.com", RegexOptions.None, 0, 11, true, "abc@def.com" }; - yield return new object[] { @"(\w+)@\w+.com", "abc@def.com", RegexOptions.None, 0, 11, true, "abc@def.com" }; - yield return new object[] { @"((\w+))@\w+.com", "abc@def.com", RegexOptions.None, 0, 11, true, "abc@def.com" }; - yield return new object[] { @"(\w+)c@\w+.com", "abc@def.comabcdef", RegexOptions.None, 0, 17, true, "abc@def.com" }; - yield return new object[] { @"(\w+)c@\w+.com\1", "abc@def.comabcdef", RegexOptions.None, 0, 17, true, "abc@def.comab" }; - yield return new object[] { @"(\w+)@def.com\1", "abc@def.comab", RegexOptions.None, 0, 13, false, string.Empty }; - yield return new object[] { @"(\w+)@def.com\1", "abc@def.combc", RegexOptions.None, 0, 13, true, "bc@def.combc" }; - yield return new object[] { @"(\w*)@def.com\1", "abc@def.com", RegexOptions.None, 0, 11, true, "@def.com" }; - yield return new object[] { @"\w+(?\w+)(?\w+)(?[0-9]+)abc"), "abc12345abc", options, 3, 8, true, "12345abc" }; + yield return new object[] { engine, Case("(?>(?>[0-9]+))abc"), "abc12345abc", options, 3, 8, true, "12345abc" }; + yield return new object[] { engine, Case("(?>[0-9]*)abc"), "abc12345abc", options, 3, 8, true, "12345abc" }; + yield return new object[] { engine, Case("(?>[^z]+)z"), "zzzzxyxyxyz123", options, 4, 9, true, "xyxyxyz" }; + yield return new object[] { engine, Case("(?>(?>[^z]+))z"), "zzzzxyxyxyz123", options, 4, 9, true, "xyxyxyz" }; + yield return new object[] { engine, Case("(?>[^z]*)z123"), "zzzzxyxyxyz123", options, 4, 10, true, "xyxyxyz123" }; + yield return new object[] { engine, Case("(?>a+)123"), "aa1234", options, 0, 5, true, "aa123" }; + yield return new object[] { engine, Case("(?>a*)123"), "aa1234", options, 0, 5, true, "aa123" }; + yield return new object[] { engine, Case("(?>(?>a*))123"), "aa1234", options, 0, 5, true, "aa123" }; + yield return new object[] { engine, Case("(?>a+?)a"), "aaaaa", options, 0, 2, true, "aa" }; + yield return new object[] { engine, Case("(?>a*?)a"), "aaaaa", options, 0, 1, true, "a" }; + yield return new object[] { engine, Case("(?>hi|hello|hey)hi"), "hellohi", options, 0, 0, false, string.Empty }; + yield return new object[] { engine, Case("(?>hi|hello|hey)hi"), "hihi", options, 0, 4, true, "hihi" }; + } + } - // Using beginning/end of string chars \A, \Z: Actual - "\\Aaaa\\w+zzz\\Z" - yield return new object[] { @"\Aaaa\w+zzz\Z", "aaaasdfajsdlfjzzza", RegexOptions.None, 0, 18, false, string.Empty }; + // Loops at beginning of expressions + yield return new object[] { engine, @"a+", "aaa", RegexOptions.None, 0, 3, true, "aaa" }; + yield return new object[] { engine, @"a+\d+", "a1", RegexOptions.None, 0, 2, true, "a1" }; + yield return new object[] { engine, @".+\d+", "a1", RegexOptions.None, 0, 2, true, "a1" }; + yield return new object[] { engine, ".+\nabc", "a\nabc", RegexOptions.None, 0, 5, true, "a\nabc" }; + yield return new object[] { engine, @"\d+", "abcd123efg", RegexOptions.None, 0, 10, true, "123" }; + yield return new object[] { engine, @"\d+\d+", "abcd123efg", RegexOptions.None, 0, 10, true, "123" }; + yield return new object[] { engine, @"\w+123\w+", "abcd123efg", RegexOptions.None, 0, 10, true, "abcd123efg" }; + yield return new object[] { engine, @"\d+\w+", "abcd123efg", RegexOptions.None, 0, 10, true, "123efg" }; + yield return new object[] { engine, @"\w+@\w+.com", "abc@def.com", RegexOptions.None, 0, 11, true, "abc@def.com" }; + yield return new object[] { engine, @"\w{3,}@\w+.com", "abc@def.com", RegexOptions.None, 0, 11, true, "abc@def.com" }; + yield return new object[] { engine, @"\w{4,}@\w+.com", "abc@def.com", RegexOptions.None, 0, 11, false, string.Empty }; + yield return new object[] { engine, @"\w{2,5}@\w+.com", "abc@def.com", RegexOptions.None, 0, 11, true, "abc@def.com" }; + yield return new object[] { engine, @"\w{3}@\w+.com", "abc@def.com", RegexOptions.None, 0, 11, true, "abc@def.com" }; + yield return new object[] { engine, @"\w{0,3}@\w+.com", "abc@def.com", RegexOptions.None, 0, 11, true, "abc@def.com" }; + yield return new object[] { engine, @"\w{0,2}c@\w+.com", "abc@def.com", RegexOptions.None, 0, 11, true, "abc@def.com" }; + yield return new object[] { engine, @"\w*@\w+.com", "abc@def.com", RegexOptions.None, 0, 11, true, "abc@def.com" }; + yield return new object[] { engine, @"(\w+)@\w+.com", "abc@def.com", RegexOptions.None, 0, 11, true, "abc@def.com" }; + yield return new object[] { engine, @"((\w+))@\w+.com", "abc@def.com", RegexOptions.None, 0, 11, true, "abc@def.com" }; + yield return new object[] { engine, @"(\w+)c@\w+.com", "abc@def.comabcdef", RegexOptions.None, 0, 17, true, "abc@def.com" }; + if (!RegexHelpers.IsNonBacktracking(engine)) + { + yield return new object[] { engine, @"(\w+)c@\w+.com\1", "abc@def.comabcdef", RegexOptions.None, 0, 17, true, "abc@def.comab" }; + yield return new object[] { engine, @"(\w+)@def.com\1", "abc@def.comab", RegexOptions.None, 0, 13, false, string.Empty }; + yield return new object[] { engine, @"(\w+)@def.com\1", "abc@def.combc", RegexOptions.None, 0, 13, true, "bc@def.combc" }; + yield return new object[] { engine, @"(\w*)@def.com\1", "abc@def.com", RegexOptions.None, 0, 11, true, "@def.com" }; + yield return new object[] { engine, @"\w+(?\w+)(?\w+)(?\\w)\\" - yield return new object[] { @"(?\w)\", "aa", RegexOptions.None, 0, 2, true, "aa" }; + // Using beginning/end of string chars \A, \Z: Actual - "\\Aaaa\\w+zzz\\Z" + yield return new object[] { engine, @"\A(line2\n)line3\Z", "line2\nline3\n", RegexOptions.Multiline, 0, 12, true, "line2\nline3" }; - // Actual - "(?<43>\\w)\\43" - yield return new object[] { @"(?<43>\w)\43", "aa", RegexOptions.None, 0, 2, true, "aa" }; + // Using beginning/end of string chars ^: Actual - "^b" + yield return new object[] { engine, "^b", "ab", RegexOptions.None, 0, 2, false, string.Empty }; - // Actual - "abc(?(1)111|222)" - yield return new object[] { "(abbc)(?(1)111|222)", "abbc222", RegexOptions.None, 0, 7, false, string.Empty }; + if (!RegexHelpers.IsNonBacktracking(engine)) + { + // Actual - "(?\\w)\\" + yield return new object[] { engine, @"(?\w)\", "aa", RegexOptions.None, 0, 2, true, "aa" }; - // "x" option. Removes unescaped whitespace from the pattern: Actual - " ([^/]+) ","x" - yield return new object[] { " ((.)+) #comment ", "abc", RegexOptions.IgnorePatternWhitespace, 0, 3, true, "abc" }; + // Actual - "(?<43>\\w)\\43" + yield return new object[] { engine, @"(?<43>\w)\43", "aa", RegexOptions.None, 0, 2, true, "aa" }; - // "x" option. Removes unescaped whitespace from the pattern. : Actual - "\x20([^/]+)\x20","x" - yield return new object[] { "\x20([^/]+)\x20\x20\x20\x20\x20\x20\x20", " abc ", RegexOptions.IgnorePatternWhitespace, 0, 10, true, " abc " }; + // Actual - "abc(?(1)111|222)" + yield return new object[] { engine, "(abbc)(?(1)111|222)", "abbc222", RegexOptions.None, 0, 7, false, string.Empty }; + } - // Turning on case insensitive option in mid-pattern : Actual - "aaa(?i:match this)bbb" - if ("i".ToUpper() == "I") - { - yield return new object[] { "aaa(?i:match this)bbb", "aaaMaTcH ThIsbbb", RegexOptions.None, 0, 16, true, "aaaMaTcH ThIsbbb" }; - } + // "x" option. Removes unescaped whitespace from the pattern: Actual - " ([^/]+) ","x" + yield return new object[] { engine, " ((.)+) #comment ", "abc", RegexOptions.IgnorePatternWhitespace, 0, 3, true, "abc" }; - // Turning off case insensitive option in mid-pattern : Actual - "aaa(?-i:match this)bbb", "i" - yield return new object[] { "aAa(?-i:match this)bbb", "AaAmatch thisBBb", RegexOptions.IgnoreCase, 0, 16, true, "AaAmatch thisBBb" }; - - // Turning on/off all the options at once : Actual - "aaa(?imnsx-imnsx:match this)bbb", "i" - yield return new object[] { "aaa(?imnsx-imnsx:match this)bbb", "AaAmatcH thisBBb", RegexOptions.IgnoreCase, 0, 16, false, string.Empty }; - - // Actual - "aaa(?#ignore this completely)bbb" - yield return new object[] { "aAa(?#ignore this completely)bbb", "aAabbb", RegexOptions.None, 0, 6, true, "aAabbb" }; - - // Trying empty string: Actual "[a-z0-9]+", "" - yield return new object[] { "[a-z0-9]+", "", RegexOptions.None, 0, 0, false, string.Empty }; - - // Numbering pattern slots: "(?<1>\\d{3})(?<2>\\d{3})(?<3>\\d{4})" - yield return new object[] { @"(?<1>\d{3})(?<2>\d{3})(?<3>\d{4})", "8885551111", RegexOptions.None, 0, 10, true, "8885551111" }; - yield return new object[] { @"(?<1>\d{3})(?<2>\d{3})(?<3>\d{4})", "Invalid string", RegexOptions.None, 0, 14, false, string.Empty }; - - // Not naming pattern slots at all: "^(cat|chat)" - yield return new object[] { "^(cat|chat)", "cats are bad", RegexOptions.None, 0, 12, true, "cat" }; - - yield return new object[] { "abc", "abc", RegexOptions.None, 0, 3, true, "abc" }; - yield return new object[] { "abc", "aBc", RegexOptions.None, 0, 3, false, string.Empty }; - yield return new object[] { "abc", "aBc", RegexOptions.IgnoreCase, 0, 3, true, "aBc" }; - yield return new object[] { @"abc.*def", "abcghiDEF", RegexOptions.IgnoreCase, 0, 9, true, "abcghiDEF" }; - - // Using *, +, ?, {}: Actual - "a+\\.?b*\\.?c{2}" - yield return new object[] { @"a+\.?b*\.+c{2}", "ab.cc", RegexOptions.None, 0, 5, true, "ab.cc" }; - yield return new object[] { @"[^a]+\.[^z]+", "zzzzz", RegexOptions.None, 0, 5, false, string.Empty }; - - // RightToLeft - yield return new object[] { @"\s+\d+", "sdf 12sad", RegexOptions.RightToLeft, 0, 9, true, " 12" }; - yield return new object[] { @"\s+\d+", " asdf12 ", RegexOptions.RightToLeft, 0, 6, false, string.Empty }; - yield return new object[] { "aaa", "aaabbb", RegexOptions.None, 3, 3, false, string.Empty }; - yield return new object[] { "abc|def", "123def456", RegexOptions.RightToLeft | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant, 0, 9, true, "def" }; - - yield return new object[] { @"foo\d+", "0123456789foo4567890foo ", RegexOptions.RightToLeft, 10, 3, false, string.Empty }; - yield return new object[] { @"foo\d+", "0123456789foo4567890foo ", RegexOptions.RightToLeft, 11, 21, false, string.Empty }; - - // IgnoreCase - yield return new object[] { "AAA", "aaabbb", RegexOptions.IgnoreCase, 0, 6, true, "aaa" }; - yield return new object[] { @"\p{Lu}", "1bc", RegexOptions.IgnoreCase, 0, 3, true, "b" }; - yield return new object[] { @"\p{Ll}", "1bc", RegexOptions.IgnoreCase, 0, 3, true, "b" }; - yield return new object[] { @"\p{Lt}", "1bc", RegexOptions.IgnoreCase, 0, 3, true, "b" }; - yield return new object[] { @"\p{Lo}", "1bc", RegexOptions.IgnoreCase, 0, 3, false, string.Empty }; - - // "\D+" - yield return new object[] { @"\D+", "12321", RegexOptions.None, 0, 5, false, string.Empty }; - - // Groups - yield return new object[] { "(?\\S+)\\s(?\\S+)", "David Bau", RegexOptions.None, 0, 9, true, "David Bau" }; - - // "^b" - yield return new object[] { "^b", "abc", RegexOptions.None, 0, 3, false, string.Empty }; - - // RightToLeft - yield return new object[] { @"foo\d+", "0123456789foo4567890foo ", RegexOptions.RightToLeft, 0, 32, true, "foo4567890" }; - yield return new object[] { @"foo\d+", "0123456789foo4567890foo ", RegexOptions.RightToLeft, 10, 22, true, "foo4567890" }; - yield return new object[] { @"foo\d+", "0123456789foo4567890foo ", RegexOptions.RightToLeft, 10, 4, true, "foo4" }; - - // Trim leading and trailing whitespace - yield return new object[] { @"\s*(.*?)\s*$", " Hello World ", RegexOptions.None, 0, 13, true, " Hello World " }; - - // < in group - yield return new object[] { @"(?cat)\w+(?dog)", "cat_Hello_World_dog", RegexOptions.None, 0, 19, false, string.Empty }; - - // Atomic Zero-Width Assertions \A \Z \z \G \b \B - yield return new object[] { @"\A(cat)\s+(dog)", "cat \n\n\ncat dog", RegexOptions.None, 0, 20, false, string.Empty }; - yield return new object[] { @"\A(cat)\s+(dog)", "cat \n\n\ncat dog", RegexOptions.Multiline, 0, 20, false, string.Empty }; - yield return new object[] { @"\A(cat)\s+(dog)", "cat \n\n\ncat dog", RegexOptions.ECMAScript, 0, 20, false, string.Empty }; - - yield return new object[] { @"(cat)\s+(dog)\Z", "cat dog\n\n\ncat", RegexOptions.None, 0, 15, false, string.Empty }; - yield return new object[] { @"(cat)\s+(dog)\Z", "cat dog\n\n\ncat ", RegexOptions.Multiline, 0, 20, false, string.Empty }; - yield return new object[] { @"(cat)\s+(dog)\Z", "cat dog\n\n\ncat ", RegexOptions.ECMAScript, 0, 20, false, string.Empty }; - - yield return new object[] { @"(cat)\s+(dog)\z", "cat dog\n\n\ncat", RegexOptions.None, 0, 15, false, string.Empty }; - yield return new object[] { @"(cat)\s+(dog)\z", "cat dog\n\n\ncat ", RegexOptions.Multiline, 0, 20, false, string.Empty }; - yield return new object[] { @"(cat)\s+(dog)\z", "cat dog\n\n\ncat ", RegexOptions.ECMAScript, 0, 20, false, string.Empty }; - yield return new object[] { @"(cat)\s+(dog)\z", "cat \n\n\n dog\n", RegexOptions.None, 0, 16, false, string.Empty }; - yield return new object[] { @"(cat)\s+(dog)\z", "cat \n\n\n dog\n", RegexOptions.Multiline, 0, 16, false, string.Empty }; - yield return new object[] { @"(cat)\s+(dog)\z", "cat \n\n\n dog\n", RegexOptions.ECMAScript, 0, 16, false, string.Empty }; - - yield return new object[] { @"\b@cat", "123START123;@catEND", RegexOptions.None, 0, 19, false, string.Empty }; - yield return new object[] { @"\b Match_Basic_TestData_NetCore() - { - // Unicode symbols in character ranges. These are chars whose lowercase values cannot be found by using the offsets specified in s_lcTable. - yield return new object[] { @"^(?i:[\u00D7-\u00D8])$", '\u00F7'.ToString(), RegexOptions.IgnoreCase | RegexOptions.CultureInvariant, 0, 1, false, "" }; - yield return new object[] { @"^(?i:[\u00C0-\u00DE])$", '\u00F7'.ToString(), RegexOptions.IgnoreCase | RegexOptions.CultureInvariant, 0, 1, false, "" }; - yield return new object[] { @"^(?i:[\u00C0-\u00DE])$", ((char)('\u00C0' + 32)).ToString(), RegexOptions.IgnoreCase | RegexOptions.CultureInvariant, 0, 1, true, ((char)('\u00C0' + 32)).ToString() }; - yield return new object[] { @"^(?i:[\u00C0-\u00DE])$", ((char)('\u00DE' + 32)).ToString(), RegexOptions.IgnoreCase | RegexOptions.CultureInvariant, 0, 1, true, ((char)('\u00DE' + 32)).ToString() }; - yield return new object[] { @"^(?i:[\u0391-\u03AB])$", ((char)('\u03A2' + 32)).ToString(), RegexOptions.IgnoreCase | RegexOptions.CultureInvariant, 0, 1, false, "" }; - yield return new object[] { @"^(?i:[\u0391-\u03AB])$", ((char)('\u0391' + 32)).ToString(), RegexOptions.IgnoreCase | RegexOptions.CultureInvariant, 0, 1, true, ((char)('\u0391' + 32)).ToString() }; - yield return new object[] { @"^(?i:[\u0391-\u03AB])$", ((char)('\u03AB' + 32)).ToString(), RegexOptions.IgnoreCase | RegexOptions.CultureInvariant, 0, 1, true, ((char)('\u03AB' + 32)).ToString() }; - yield return new object[] { @"^(?i:[\u1F18-\u1F1F])$", ((char)('\u1F1F' - 8)).ToString(), RegexOptions.IgnoreCase | RegexOptions.CultureInvariant, 0, 1, false, "" }; - yield return new object[] { @"^(?i:[\u1F18-\u1F1F])$", ((char)('\u1F18' - 8)).ToString(), RegexOptions.IgnoreCase | RegexOptions.CultureInvariant, 0, 1, true, ((char)('\u1F18' - 8)).ToString() }; - yield return new object[] { @"^(?i:[\u10A0-\u10C5])$", ((char)('\u10A0' + 7264)).ToString(), RegexOptions.IgnoreCase | RegexOptions.CultureInvariant, 0, 1, true, ((char)('\u10A0' + 7264)).ToString() }; - yield return new object[] { @"^(?i:[\u10A0-\u10C5])$", ((char)('\u1F1F' + 48)).ToString(), RegexOptions.IgnoreCase | RegexOptions.CultureInvariant, 0, 1, false, "" }; - yield return new object[] { @"^(?i:[\u24B6-\u24D0])$", ((char)('\u24D0' + 26)).ToString(), RegexOptions.IgnoreCase | RegexOptions.CultureInvariant, 0, 1, false, "" }; - yield return new object[] { @"^(?i:[\u24B6-\u24D0])$", ((char)('\u24CF' + 26)).ToString(), RegexOptions.IgnoreCase | RegexOptions.CultureInvariant, 0, 1, true, ((char)('\u24CF' + 26)).ToString() }; - } + // Actual - "aaa(?#ignore this completely)bbb" + yield return new object[] { engine, "aAa(?#ignore this completely)bbb", "aAabbb", RegexOptions.None, 0, 6, true, "aAabbb" }; - public static IEnumerable Match_Basic_TestData_WithEngine() => - RegexHelpers.PrependEngines(Match_Basic_TestData()); + // Trying empty string: Actual "[a-z0-9]+", "" + yield return new object[] { engine, "[a-z0-9]+", "", RegexOptions.None, 0, 0, false, string.Empty }; - public static IEnumerable Match_Basic_TestData_NetCore_WithEngine() => - RegexHelpers.PrependEngines(Match_Basic_TestData()); + // Numbering pattern slots: "(?<1>\\d{3})(?<2>\\d{3})(?<3>\\d{4})" + yield return new object[] { engine, @"(?<1>\d{3})(?<2>\d{3})(?<3>\d{4})", "8885551111", RegexOptions.None, 0, 10, true, "8885551111" }; + yield return new object[] { engine, @"(?<1>\d{3})(?<2>\d{3})(?<3>\d{4})", "Invalid string", RegexOptions.None, 0, 14, false, string.Empty }; - [SkipOnTargetFramework(TargetFrameworkMonikers.NetFramework)] - [Theory] - [ActiveIssue("https://github.com/dotnet/runtime/issues/36149")] - [MemberData(nameof(Match_Basic_TestData_NetCore_WithEngine))] - public async Task Match_NetCore(RegexEngine engine, string pattern, string input, RegexOptions options, int beginning, int length, bool expectedSuccess, string expectedValue) - { - await Match(engine, pattern, input, options, beginning, length, expectedSuccess, expectedValue); + // Not naming pattern slots at all: "^(cat|chat)" + yield return new object[] { engine, "^(cat|chat)", "cats are bad", RegexOptions.None, 0, 12, true, "cat" }; + + yield return new object[] { engine, "abc", "abc", RegexOptions.None, 0, 3, true, "abc" }; + yield return new object[] { engine, "abc", "aBc", RegexOptions.None, 0, 3, false, string.Empty }; + yield return new object[] { engine, "abc", "aBc", RegexOptions.IgnoreCase, 0, 3, true, "aBc" }; + yield return new object[] { engine, @"abc.*def", "abcghiDEF", RegexOptions.IgnoreCase, 0, 9, true, "abcghiDEF" }; + + // Using *, +, ?, {}: Actual - "a+\\.?b*\\.?c{2}" + yield return new object[] { engine, @"a+\.?b*\.+c{2}", "ab.cc", RegexOptions.None, 0, 5, true, "ab.cc" }; + yield return new object[] { engine, @"[^a]+\.[^z]+", "zzzzz", RegexOptions.None, 0, 5, false, string.Empty }; + + // IgnoreCase + yield return new object[] { engine, "AAA", "aaabbb", RegexOptions.IgnoreCase, 0, 6, true, "aaa" }; + yield return new object[] { engine, @"\p{Lu}", "1bc", RegexOptions.IgnoreCase, 0, 3, true, "b" }; + yield return new object[] { engine, @"\p{Ll}", "1bc", RegexOptions.IgnoreCase, 0, 3, true, "b" }; + yield return new object[] { engine, @"\p{Lt}", "1bc", RegexOptions.IgnoreCase, 0, 3, true, "b" }; + yield return new object[] { engine, @"\p{Lo}", "1bc", RegexOptions.IgnoreCase, 0, 3, false, string.Empty }; + + // "\D+" + yield return new object[] { engine, @"\D+", "12321", RegexOptions.None, 0, 5, false, string.Empty }; + + // Groups + yield return new object[] { engine, "(?\\S+)\\s(?\\S+)", "David Bau", RegexOptions.None, 0, 9, true, "David Bau" }; + + // "^b" + yield return new object[] { engine, "^b", "abc", RegexOptions.None, 0, 3, false, string.Empty }; + + // Trim leading and trailing whitespace + yield return new object[] { engine, @"\s*(.*?)\s*$", " Hello World ", RegexOptions.None, 0, 13, true, " Hello World " }; + + if (!RegexHelpers.IsNonBacktracking(engine)) + { + // Throws NotSupported with NonBacktracking engine because of the balancing group dog-0 + yield return new object[] { engine, @"(?cat)\w+(?dog)", "cat_Hello_World_dog", RegexOptions.None, 0, 19, false, string.Empty }; + } + + // Atomic Zero-Width Assertions \A \Z \z \b \B + yield return new object[] { engine, @"\A(cat)\s+(dog)", "cat \n\n\ncat dog", RegexOptions.None, 0, 20, false, string.Empty }; + yield return new object[] { engine, @"\A(cat)\s+(dog)", "cat \n\n\ncat dog", RegexOptions.Multiline, 0, 20, false, string.Empty }; + if (!RegexHelpers.IsNonBacktracking(engine)) + { + yield return new object[] { engine, @"\A(cat)\s+(dog)", "cat \n\n\ncat dog", RegexOptions.ECMAScript, 0, 20, false, string.Empty }; + } + + yield return new object[] { engine, @"(cat)\s+(dog)\Z", "cat dog\n\n\ncat", RegexOptions.None, 0, 15, false, string.Empty }; + yield return new object[] { engine, @"(cat)\s+(dog)\Z", "cat dog\n\n\ncat ", RegexOptions.Multiline, 0, 20, false, string.Empty }; + if (!RegexHelpers.IsNonBacktracking(engine)) + { + yield return new object[] { engine, @"(cat)\s+(dog)\Z", "cat dog\n\n\ncat ", RegexOptions.ECMAScript, 0, 20, false, string.Empty }; + } + + yield return new object[] { engine, @"(cat)\s+(dog)\z", "cat dog\n\n\ncat", RegexOptions.None, 0, 15, false, string.Empty }; + yield return new object[] { engine, @"(cat)\s+(dog)\z", "cat dog\n\n\ncat ", RegexOptions.Multiline, 0, 20, false, string.Empty }; + if (!RegexHelpers.IsNonBacktracking(engine)) + { + yield return new object[] { engine, @"(cat)\s+(dog)\z", "cat dog\n\n\ncat ", RegexOptions.ECMAScript, 0, 20, false, string.Empty }; + } + yield return new object[] { engine, @"(cat)\s+(dog)\z", "cat \n\n\n dog\n", RegexOptions.None, 0, 16, false, string.Empty }; + yield return new object[] { engine, @"(cat)\s+(dog)\z", "cat \n\n\n dog\n", RegexOptions.Multiline, 0, 16, false, string.Empty }; + if (!RegexHelpers.IsNonBacktracking(engine)) + { + yield return new object[] { engine, @"(cat)\s+(dog)\z", "cat \n\n\n dog\n", RegexOptions.ECMAScript, 0, 16, false, string.Empty }; + } + + yield return new object[] { engine, @"\b@cat", "123START123;@catEND", RegexOptions.None, 0, 19, false, string.Empty }; + yield return new object[] { engine, @"\b= 1); + if (!RegexHelpers.IsNonBacktracking(engine)) + { + Assert.Equal(expectedSuccess, match.Groups[0].Success); + RegexAssert.Equal(expectedValue, match.Groups[0]); + } + } } public static IEnumerable Match_VaryingLengthStrings_MemberData() @@ -513,7 +603,7 @@ public static IEnumerable Match_VaryingLengthStrings_MemberData() } [SkipOnTargetFramework(TargetFrameworkMonikers.NetFramework, "Takes several minutes on .NET Framework")] - [Theory] + [ConditionalTheory] [MemberData(nameof(Match_VaryingLengthStrings_MemberData))] public async Task Match_VaryingLengthStrings(RegexEngine engine, RegexOptions options) { @@ -532,21 +622,16 @@ public async Task Match_VaryingLengthStrings(RegexEngine engine, RegexOptions op } } - private static void VerifyMatch(Match match, bool expectedSuccess, string expectedValue) - { - Assert.Equal(expectedSuccess, match.Success); - RegexAssert.Equal(expectedValue, match); - - // Groups can never be empty - Assert.True(match.Groups.Count >= 1); - Assert.Equal(expectedSuccess, match.Groups[0].Success); - RegexAssert.Equal(expectedValue, match.Groups[0]); - } - public static IEnumerable Match_DeepNesting_MemberData() { foreach (RegexEngine engine in RegexHelpers.AvailableEngines) { + if (RegexHelpers.IsNonBacktracking(engine)) + { + // expression uses atomic group + continue; + } + yield return new object[] { engine, 1 }; yield return new object[] { engine, 10 }; yield return new object[] { engine, 100 }; @@ -581,10 +666,36 @@ public async Task Match_Timeout(RegexEngine engine) RegexAssert.Equal("a", match); } + /// + /// Test that timeout exception is being thrown. + /// + [Theory] + [MemberData(nameof(RegexHelpers.AvailableEngines_MemberData), MemberType = typeof(RegexHelpers))] + private async Task Match_TestThatTimeoutHappens(RegexEngine engine) + { + var rnd = new Random(42); + var chars = new char[1_000_000]; + for (int i = 0; i < chars.Length; i++) + { + byte b = (byte)rnd.Next(0, 256); + chars[i] = b < 200 ? 'a' : (char)b; + } + string input = new string(chars); + + Regex re = await RegexHelpers.GetRegexAsync(engine, @"a.{20}$", RegexOptions.None, TimeSpan.FromMilliseconds(10)); + Assert.Throws(() => { re.Match(input); }); + } + [Theory] [MemberData(nameof(RegexHelpers.AvailableEngines_MemberData), MemberType = typeof(RegexHelpers))] public async Task Match_Timeout_Throws(RegexEngine engine) { + if (RegexHelpers.IsNonBacktracking(engine)) + { + // test relies on backtracking taking a long time + return; + } + const string Pattern = @"^([0-9a-zA-Z]([-.\w]*[0-9a-zA-Z])*@(([0-9a-zA-Z])+([-\w]*[0-9a-zA-Z])*\.)+[a-zA-Z]{2,9})$"; string input = new string('a', 50) + "@a.a"; @@ -595,9 +706,9 @@ public async Task Match_Timeout_Throws(RegexEngine engine) // TODO: Figure out what to do with default timeouts for source generated regexes [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] [InlineData(RegexOptions.None)] - [InlineData(RegexOptions.None | (RegexOptions)0x80 /* Debug */)] + [InlineData(RegexOptions.None | RegexHelpers.RegexOptionDebug)] [InlineData(RegexOptions.Compiled)] - [InlineData(RegexOptions.Compiled | (RegexOptions)0x80 /* Debug */)] + [InlineData(RegexOptions.Compiled | RegexHelpers.RegexOptionDebug)] public void Match_DefaultTimeout_Throws(RegexOptions options) { RemoteExecutor.Invoke(optionsString => @@ -631,9 +742,9 @@ public void Match_DefaultTimeout_Throws(RegexOptions options) // TODO: Figure out what to do with default timeouts for source generated regexes [Theory] [InlineData(RegexOptions.None)] - [InlineData(RegexOptions.None | (RegexOptions)0x80 /* Debug */)] + [InlineData(RegexOptions.None | RegexHelpers.RegexOptionDebug)] [InlineData(RegexOptions.Compiled)] - [InlineData(RegexOptions.Compiled | (RegexOptions)0x80 /* Debug */)] + [InlineData(RegexOptions.Compiled | RegexHelpers.RegexOptionDebug)] public void Match_CachedPattern_NewTimeoutApplies(RegexOptions options) { const string PatternLeadingToLotsOfBacktracking = @"^(\w+\s?)*$"; @@ -670,287 +781,314 @@ public async Task Match_Timeout_Repetition_Throws(RegexEngine engine) public static IEnumerable Match_Advanced_TestData() { - // \B special character escape: ".*\\B(SUCCESS)\\B.*" - yield return new object[] - { - @".*\B(SUCCESS)\B.*", "adfadsfSUCCESSadsfadsf", RegexOptions.None, 0, 22, - new CaptureData[] - { - new CaptureData("adfadsfSUCCESSadsfadsf", 0, 22), - new CaptureData("SUCCESS", 7, 7) - } - }; - - // Using |, (), ^, $, .: Actual - "^aaa(bb.+)(d|c)$" - yield return new object[] - { - "^aaa(bb.+)(d|c)$", "aaabb.cc", RegexOptions.None, 0, 8, - new CaptureData[] - { - new CaptureData("aaabb.cc", 0, 8), - new CaptureData("bb.c", 3, 4), - new CaptureData("c", 7, 1) - } - }; - - // Using greedy quantifiers: Actual - "(a+)(b*)(c?)" - yield return new object[] - { - "(a+)(b*)(c?)", "aaabbbccc", RegexOptions.None, 0, 9, - new CaptureData[] - { - new CaptureData("aaabbbc", 0, 7), - new CaptureData("aaa", 0, 3), - new CaptureData("bbb", 3, 3), - new CaptureData("c", 6, 1) - } - }; - - // Using lazy quantifiers: Actual - "(d+?)(e*?)(f??)" - // Interesting match from this pattern and input. If needed to go to the end of the string change the ? to + in the last lazy quantifier - yield return new object[] - { - "(d+?)(e*?)(f??)", "dddeeefff", RegexOptions.None, 0, 9, - new CaptureData[] - { - new CaptureData("d", 0, 1), - new CaptureData("d", 0, 1), - new CaptureData(string.Empty, 1, 0), - new CaptureData(string.Empty, 1, 0) - } - }; - - // Noncapturing group : Actual - "(a+)(?:b*)(ccc)" - yield return new object[] + foreach (RegexEngine engine in RegexHelpers.AvailableEngines) { - "(a+)(?:b*)(ccc)", "aaabbbccc", RegexOptions.None, 0, 9, - new CaptureData[] + // \B special character escape: ".*\\B(SUCCESS)\\B.*" + yield return new object[] { - new CaptureData("aaabbbccc", 0, 9), - new CaptureData("aaa", 0, 3), - new CaptureData("ccc", 6, 3), - } - }; + engine, + @".*\B(SUCCESS)\B.*", "adfadsfSUCCESSadsfadsf", RegexOptions.None, 0, 22, + new CaptureData[] + { + new CaptureData("adfadsfSUCCESSadsfadsf", 0, 22), + new CaptureData("SUCCESS", 7, 7) + } + }; - // Zero-width positive lookahead assertion: Actual - "abc(?=XXX)\\w+" - yield return new object[] - { - @"abc(?=XXX)\w+", "abcXXXdef", RegexOptions.None, 0, 9, - new CaptureData[] + // Using |, (), ^, $, .: Actual - "^aaa(bb.+)(d|c)$" + yield return new object[] { - new CaptureData("abcXXXdef", 0, 9) - } - }; - - // Backreferences : Actual - "(\\w)\\1" - yield return new object[] - { - @"(\w)\1", "aa", RegexOptions.None, 0, 2, - new CaptureData[] + engine, + "^aaa(bb.+)(d|c)$", "aaabb.cc", RegexOptions.None, 0, 8, + new CaptureData[] + { + new CaptureData("aaabb.cc", 0, 8), + new CaptureData("bb.c", 3, 4), + new CaptureData("c", 7, 1) + } + }; + + // Using greedy quantifiers: Actual - "(a+)(b*)(c?)" + yield return new object[] { - new CaptureData("aa", 0, 2), - new CaptureData("a", 0, 1), - } - }; - - // Alternation constructs: Actual - "(111|aaa)" - yield return new object[] - { - "(111|aaa)", "aaa", RegexOptions.None, 0, 3, - new CaptureData[] + engine, + "(a+)(b*)(c?)", "aaabbbccc", RegexOptions.None, 0, 9, + new CaptureData[] + { + new CaptureData("aaabbbc", 0, 7), + new CaptureData("aaa", 0, 3), + new CaptureData("bbb", 3, 3), + new CaptureData("c", 6, 1) + } + }; + + // Using lazy quantifiers: Actual - "(d+?)(e*?)(f??)" + // Interesting match from this pattern and input. If needed to go to the end of the string change the ? to + in the last lazy quantifier + yield return new object[] { - new CaptureData("aaa", 0, 3), - new CaptureData("aaa", 0, 3) - } - }; - - // Actual - "(?<1>\\d+)abc(?(1)222|111)" - yield return new object[] - { - @"(?\d+)abc(?(MyDigits)222|111)", "111abc222", RegexOptions.None, 0, 9, - new CaptureData[] + engine, + "(d+?)(e*?)(f??)", "dddeeefff", RegexOptions.None, 0, 9, + new CaptureData[] + { + new CaptureData("d", 0, 1), + new CaptureData("d", 0, 1), + new CaptureData(string.Empty, 1, 0), + new CaptureData(string.Empty, 1, 0) + } + }; + + // Noncapturing group : Actual - "(a+)(?:b*)(ccc)" + yield return new object[] { - new CaptureData("111abc222", 0, 9), - new CaptureData("111", 0, 3) - } - }; - - // Using "n" Regex option. Only explicitly named groups should be captured: Actual - "([0-9]*)\\s(?[a-z_A-Z]+)", "n" - yield return new object[] - { - @"([0-9]*)\s(?[a-z_A-Z]+)", "200 dollars", RegexOptions.ExplicitCapture, 0, 11, - new CaptureData[] + engine, + "(a+)(?:b*)(ccc)", "aaabbbccc", RegexOptions.None, 0, 9, + new CaptureData[] + { + new CaptureData("aaabbbccc", 0, 9), + new CaptureData("aaa", 0, 3), + new CaptureData("ccc", 6, 3), + } + }; + + // Alternation constructs: Actual - "(111|aaa)" + yield return new object[] { - new CaptureData("200 dollars", 0, 11), - new CaptureData("dollars", 4, 7) - } - }; + engine, + "(111|aaa)", "aaa", RegexOptions.None, 0, 3, + new CaptureData[] + { + new CaptureData("aaa", 0, 3), + new CaptureData("aaa", 0, 3) + } + }; - // Single line mode "s". Includes new line character: Actual - "([^/]+)","s" - yield return new object[] - { - "(.*)", "abc\nsfc", RegexOptions.Singleline, 0, 7, - new CaptureData[] + // Using "n" Regex option. Only explicitly named groups should be captured: Actual - "([0-9]*)\\s(?[a-z_A-Z]+)", "n" + yield return new object[] { - new CaptureData("abc\nsfc", 0, 7), - new CaptureData("abc\nsfc", 0, 7), - } - }; + engine, + @"([0-9]*)\s(?[a-z_A-Z]+)", "200 dollars", RegexOptions.ExplicitCapture, 0, 11, + new CaptureData[] + { + new CaptureData("200 dollars", 0, 11), + new CaptureData("dollars", 4, 7) + } + }; - // "([0-9]+(\\.[0-9]+){3})" - yield return new object[] - { - @"([0-9]+(\.[0-9]+){3})", "209.25.0.111", RegexOptions.None, 0, 12, - new CaptureData[] + // Single line mode "s". Includes new line character: Actual - "([^/]+)","s" + yield return new object[] { - new CaptureData("209.25.0.111", 0, 12), - new CaptureData("209.25.0.111", 0, 12), - new CaptureData(".111", 8, 4, new CaptureData[] + engine, + "(.*)", "abc\nsfc", RegexOptions.Singleline, 0, 7, + new CaptureData[] { - new CaptureData(".25", 3, 3), - new CaptureData(".0", 6, 2), - new CaptureData(".111", 8, 4), - }), - } - }; + new CaptureData("abc\nsfc", 0, 7), + new CaptureData("abc\nsfc", 0, 7), + } + }; - // Groups and captures - yield return new object[] - { - @"(?a*)(?b*)(?c*)", "aaabbccccccccccaaaabc", RegexOptions.None, 0, 21, - new CaptureData[] + // "([0-9]+(\\.[0-9]+){3})" + yield return new object[] { - new CaptureData("aaabbcccccccccc", 0, 15), - new CaptureData("aaa", 0, 3), - new CaptureData("bb", 3, 2), - new CaptureData("cccccccccc", 5, 10) - } - }; - - yield return new object[] - { - @"(?A*)(?B*)(?C*)", "aaabbccccccccccaaaabc", RegexOptions.IgnoreCase, 0, 21, - new CaptureData[] + engine, + @"([0-9]+(\.[0-9]+){3})", "209.25.0.111", RegexOptions.None, 0, 12, + new CaptureData[] + { + new CaptureData("209.25.0.111", 0, 12), + new CaptureData("209.25.0.111", 0, 12), + new CaptureData(".111", 8, 4, new CaptureData[] + { + new CaptureData(".25", 3, 3), + new CaptureData(".0", 6, 2), + new CaptureData(".111", 8, 4), + }), + } + }; + + // Groups and captures + yield return new object[] { - new CaptureData("aaabbcccccccccc", 0, 15), - new CaptureData("aaa", 0, 3), - new CaptureData("bb", 3, 2), - new CaptureData("cccccccccc", 5, 10) - } - }; - - // Using |, (), ^, $, .: Actual - "^aaa(bb.+)(d|c)$" - yield return new object[] - { - "^aaa(bb.+)(d|c)$", "aaabb.cc", RegexOptions.None, 0, 8, - new CaptureData[] + engine, + @"(?a*)(?b*)(?c*)", "aaabbccccccccccaaaabc", RegexOptions.None, 0, 21, + new CaptureData[] + { + new CaptureData("aaabbcccccccccc", 0, 15), + new CaptureData("aaa", 0, 3), + new CaptureData("bb", 3, 2), + new CaptureData("cccccccccc", 5, 10) + } + }; + + yield return new object[] { - new CaptureData("aaabb.cc", 0, 8), - new CaptureData("bb.c", 3, 4), - new CaptureData("c", 7, 1) - } - }; - - // Actual - ".*\\b(\\w+)\\b" - yield return new object[] - { - @".*\b(\w+)\b", "XSP_TEST_FAILURE SUCCESS", RegexOptions.None, 0, 24, - new CaptureData[] + engine, + @"(?A*)(?B*)(?C*)", "aaabbccccccccccaaaabc", RegexOptions.IgnoreCase, 0, 21, + new CaptureData[] + { + new CaptureData("aaabbcccccccccc", 0, 15), + new CaptureData("aaa", 0, 3), + new CaptureData("bb", 3, 2), + new CaptureData("cccccccccc", 5, 10) + } + }; + + // Using |, (), ^, $, .: Actual - "^aaa(bb.+)(d|c)$" + yield return new object[] { - new CaptureData("XSP_TEST_FAILURE SUCCESS", 0, 24), - new CaptureData("SUCCESS", 17, 7) - } - }; - - // Multiline - yield return new object[] - { - "(line2$\n)line3", "line1\nline2\nline3\n\nline4", RegexOptions.Multiline, 0, 24, - new CaptureData[] + engine, + "^aaa(bb.+)(d|c)$", "aaabb.cc", RegexOptions.None, 0, 8, + new CaptureData[] + { + new CaptureData("aaabb.cc", 0, 8), + new CaptureData("bb.c", 3, 4), + new CaptureData("c", 7, 1) + } + }; + + // Actual - ".*\\b(\\w+)\\b" + yield return new object[] { - new CaptureData("line2\nline3", 6, 11), - new CaptureData("line2\n", 6, 6) - } - }; + engine, + @".*\b(\w+)\b", "XSP_TEST_FAILURE SUCCESS", RegexOptions.None, 0, 24, + new CaptureData[] + { + new CaptureData("XSP_TEST_FAILURE SUCCESS", 0, 24), + new CaptureData("SUCCESS", 17, 7) + } + }; - // Multiline - yield return new object[] - { - "(line2\n^)line3", "line1\nline2\nline3\n\nline4", RegexOptions.Multiline, 0, 24, - new CaptureData[] + // Multiline + yield return new object[] { - new CaptureData("line2\nline3", 6, 11), - new CaptureData("line2\n", 6, 6) - } - }; + engine, + "(line2$\n)line3", "line1\nline2\nline3\n\nline4", RegexOptions.Multiline, 0, 24, + new CaptureData[] + { + new CaptureData("line2\nline3", 6, 11), + new CaptureData("line2\n", 6, 6) + } + }; - // Multiline - yield return new object[] - { - "(line3\n$\n)line4", "line1\nline2\nline3\n\nline4", RegexOptions.Multiline, 0, 24, - new CaptureData[] + // Multiline + yield return new object[] { - new CaptureData("line3\n\nline4", 12, 12), - new CaptureData("line3\n\n", 12, 7) - } - }; + engine, + "(line2\n^)line3", "line1\nline2\nline3\n\nline4", RegexOptions.Multiline, 0, 24, + new CaptureData[] + { + new CaptureData("line2\nline3", 6, 11), + new CaptureData("line2\n", 6, 6) + } + }; - // Multiline - yield return new object[] - { - "(line3\n^\n)line4", "line1\nline2\nline3\n\nline4", RegexOptions.Multiline, 0, 24, - new CaptureData[] + // Multiline + yield return new object[] { - new CaptureData("line3\n\nline4", 12, 12), - new CaptureData("line3\n\n", 12, 7) - } - }; + engine, + "(line3\n$\n)line4", "line1\nline2\nline3\n\nline4", RegexOptions.Multiline, 0, 24, + new CaptureData[] + { + new CaptureData("line3\n\nline4", 12, 12), + new CaptureData("line3\n\n", 12, 7) + } + }; - // Multiline - yield return new object[] - { - "(line2$\n^)line3", "line1\nline2\nline3\n\nline4", RegexOptions.Multiline, 0, 24, - new CaptureData[] + // Multiline + yield return new object[] { - new CaptureData("line2\nline3", 6, 11), - new CaptureData("line2\n", 6, 6) - } - }; + engine, + "(line3\n^\n)line4", "line1\nline2\nline3\n\nline4", RegexOptions.Multiline, 0, 24, + new CaptureData[] + { + new CaptureData("line3\n\nline4", 12, 12), + new CaptureData("line3\n\n", 12, 7) + } + }; - // RightToLeft - yield return new object[] - { - "aaa", "aaabbb", RegexOptions.RightToLeft, 3, 3, - new CaptureData[] + // Multiline + yield return new object[] { - new CaptureData("aaa", 0, 3) - } - }; + engine, + "(line2$\n^)line3", "line1\nline2\nline3\n\nline4", RegexOptions.Multiline, 0, 24, + new CaptureData[] + { + new CaptureData("line2\nline3", 6, 11), + new CaptureData("line2\n", 6, 6) + } + }; - // RightToLeft with anchor - yield return new object[] - { - "^aaa", "aaabbb", RegexOptions.RightToLeft, 3, 3, - new CaptureData[] - { - new CaptureData("aaa", 0, 3) - } - }; - yield return new object[] - { - "bbb$", "aaabbb", RegexOptions.RightToLeft, 0, 3, - new CaptureData[] + if (!RegexHelpers.IsNonBacktracking(engine)) { - new CaptureData("bbb", 0, 3) + // Zero-width positive lookahead assertion: Actual - "abc(?=XXX)\\w+" + yield return new object[] + { + engine, + @"abc(?=XXX)\w+", "abcXXXdef", RegexOptions.None, 0, 9, + new CaptureData[] + { + new CaptureData("abcXXXdef", 0, 9) + } + }; + + // Backreferences : Actual - "(\\w)\\1" + yield return new object[] + { + engine, + @"(\w)\1", "aa", RegexOptions.None, 0, 2, + new CaptureData[] + { + new CaptureData("aa", 0, 2), + new CaptureData("a", 0, 1), + } + }; + + // Actual - "(?<1>\\d+)abc(?(1)222|111)" + yield return new object[] + { + engine, + @"(?\d+)abc(?(MyDigits)222|111)", "111abc222", RegexOptions.None, 0, 9, + new CaptureData[] + { + new CaptureData("111abc222", 0, 9), + new CaptureData("111", 0, 3) + } + }; + + // RightToLeft + yield return new object[] + { + engine, + "aaa", "aaabbb", RegexOptions.RightToLeft, 3, 3, + new CaptureData[] + { + new CaptureData("aaa", 0, 3) + } + }; + + // RightToLeft with anchor + yield return new object[] + { + engine, + "^aaa", "aaabbb", RegexOptions.RightToLeft, 3, 3, + new CaptureData[] + { + new CaptureData("aaa", 0, 3) + } + }; + yield return new object[] + { + engine, + "bbb$", "aaabbb", RegexOptions.RightToLeft, 0, 3, + new CaptureData[] + { + new CaptureData("bbb", 0, 3) + } + }; } - }; + } } - public static IEnumerable Match_Advanced_TestData_WithEngine() => - RegexHelpers.PrependEngines(Match_Advanced_TestData()); - [Theory] - [MemberData(nameof(Match_Advanced_TestData_WithEngine))] + [MemberData(nameof(Match_Advanced_TestData))] public async Task Match_Advanced(RegexEngine engine, string pattern, string input, RegexOptions options, int beginning, int length, CaptureData[] expected) { bool isDefaultStart = RegexHelpers.IsDefaultStart(input, options, beginning); @@ -961,8 +1099,8 @@ public async Task Match_Advanced(RegexEngine engine, string pattern, string inpu if (isDefaultStart && isDefaultCount) { // Use Match(string) or Match(string, string, RegexOptions) - VerifyMatch(r.Match(input), true, expected); - VerifyMatch(Regex.Match(input, pattern, options), true, expected); + VerifyMatch(r.Match(input)); + VerifyMatch(Regex.Match(input, pattern, options)); Assert.True(Regex.IsMatch(input, pattern, options)); } @@ -970,13 +1108,50 @@ public async Task Match_Advanced(RegexEngine engine, string pattern, string inpu if (beginning + length == input.Length) { // Use Match(string, int) - VerifyMatch(r.Match(input, beginning), true, expected); + VerifyMatch(r.Match(input, beginning)); } if ((options & RegexOptions.RightToLeft) == 0) { // Use Match(string, int, int) - VerifyMatch(r.Match(input, beginning, length), true, expected); + VerifyMatch(r.Match(input, beginning, length)); + } + + void VerifyMatch(Match match) + { + Assert.True(match.Success); + + RegexAssert.Equal(expected[0].Value, match); + Assert.Equal(expected[0].Index, match.Index); + Assert.Equal(expected[0].Length, match.Length); + + if (RegexHelpers.IsNonBacktracking(engine)) + { + return; + } + + Assert.Equal(1, match.Captures.Count); + RegexAssert.Equal(expected[0].Value, match.Captures[0]); + Assert.Equal(expected[0].Index, match.Captures[0].Index); + Assert.Equal(expected[0].Length, match.Captures[0].Length); + + Assert.Equal(expected.Length, match.Groups.Count); + for (int i = 0; i < match.Groups.Count; i++) + { + Assert.True(match.Groups[i].Success); + + RegexAssert.Equal(expected[i].Value, match.Groups[i]); + Assert.Equal(expected[i].Index, match.Groups[i].Index); + Assert.Equal(expected[i].Length, match.Groups[i].Length); + + Assert.Equal(expected[i].Captures.Length, match.Groups[i].Captures.Count); + for (int j = 0; j < match.Groups[i].Captures.Count; j++) + { + RegexAssert.Equal(expected[i].Captures[j].Value, match.Groups[i].Captures[j]); + Assert.Equal(expected[i].Captures[j].Index, match.Groups[i].Captures[j].Index); + Assert.Equal(expected[i].Captures[j].Length, match.Groups[i].Captures[j].Length); + } + } } } @@ -984,6 +1159,11 @@ public static IEnumerable Match_StartatDiffersFromBeginning_MemberData { foreach (RegexEngine engine in RegexHelpers.AvailableEngines) { + if (RegexHelpers.IsNonBacktracking(engine)) + { + continue; + } + foreach (RegexOptions options in new[] { RegexOptions.None, RegexOptions.Singleline, RegexOptions.Multiline }) { // Anchors @@ -1012,38 +1192,6 @@ public async Task Match_StartatDiffersFromBeginning(RegexEngine engine, string p Assert.Equal(expectedSuccessBeginning, r.Match(input, startat, input.Length - startat).Success); } - private static void VerifyMatch(Match match, bool expectedSuccess, CaptureData[] expected) - { - Assert.Equal(expectedSuccess, match.Success); - - RegexAssert.Equal(expected[0].Value, match); - Assert.Equal(expected[0].Index, match.Index); - Assert.Equal(expected[0].Length, match.Length); - - Assert.Equal(1, match.Captures.Count); - RegexAssert.Equal(expected[0].Value, match.Captures[0]); - Assert.Equal(expected[0].Index, match.Captures[0].Index); - Assert.Equal(expected[0].Length, match.Captures[0].Length); - - Assert.Equal(expected.Length, match.Groups.Count); - for (int i = 0; i < match.Groups.Count; i++) - { - Assert.Equal(expectedSuccess, match.Groups[i].Success); - - RegexAssert.Equal(expected[i].Value, match.Groups[i]); - Assert.Equal(expected[i].Index, match.Groups[i].Index); - Assert.Equal(expected[i].Length, match.Groups[i].Length); - - Assert.Equal(expected[i].Captures.Length, match.Groups[i].Captures.Count); - for (int j = 0; j < match.Groups[i].Captures.Count; j++) - { - RegexAssert.Equal(expected[i].Captures[j].Value, match.Groups[i].Captures[j]); - Assert.Equal(expected[i].Captures[j].Index, match.Groups[i].Captures[j].Index); - Assert.Equal(expected[i].Captures[j].Length, match.Groups[i].Captures[j].Length); - } - } - } - [Theory] [InlineData(@"(?<1>\d{1,2})/(?<2>\d{1,2})/(?<3>\d{2,4})\s(?