Add data versioning and properly count linear segments

Mapperator.ConsoleApp/Mapperator.ConsoleApp.csproj

@@ -7,16 +7,16 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="CommandLineParser" Version="2.8.0" />
+    <PackageReference Include="CommandLineParser" Version="2.9.1" />
     <PackageReference Include="MappingTools.Core" Version="0.6.2" />
     <PackageReference Include="Microsoft.Win32.Registry" Version="5.0.0" />
     <PackageReference Include="OsuParsers" Version="1.7.2" />
-    <PackageReference Include="Parquet.Net" Version="5.0.1" />
-    <PackageReference Include="ppy.osu.Game" Version="2024.1104.0" />
-    <PackageReference Include="ppy.osu.Game.Rulesets.Catch" Version="2024.1104.0" />
-    <PackageReference Include="ppy.osu.Game.Rulesets.Mania" Version="2024.1104.0" />
-    <PackageReference Include="ppy.osu.Game.Rulesets.Osu" Version="2024.1104.0" />
-    <PackageReference Include="ppy.osu.Game.Rulesets.Taiko" Version="2024.1104.0" />
+    <PackageReference Include="Parquet.Net" Version="5.1.1" />
+    <PackageReference Include="ppy.osu.Game" Version="2025.424.0" />
+    <PackageReference Include="ppy.osu.Game.Rulesets.Catch" Version="2025.424.0" />
+    <PackageReference Include="ppy.osu.Game.Rulesets.Mania" Version="2025.424.0" />
+    <PackageReference Include="ppy.osu.Game.Rulesets.Osu" Version="2025.424.0" />
+    <PackageReference Include="ppy.osu.Game.Rulesets.Taiko" Version="2025.424.0" />
   </ItemGroup>
 
   <ItemGroup>

Mapperator.ConsoleApp/Program.cs

@@ -19,17 +19,17 @@ private static int Main(string[] args) {
                     Dataset.DatasetOptions,
                     Dataset2.DatasetOptions2
                 >(args)
-              .MapResult(
-                  (Count.CountOptions opts) => Count.DoDataCount(opts),
-                (Extract.ExtractOptions opts) => Extract.DoDataExtraction(opts),
-                (Build.BuildOptions opts) => Build.DoBuildGraph(opts),
-                (Convert.ConvertOptions opts) => Convert.DoMapConvert(opts),
-                (Search.SearchOptions opts) => Search.DoPatternSearch(opts),
-                (Analyze.AnalyzeOptions opts) => Analyze.DoVisualSpacingExtract(opts),
-                (Extract2.Extract2Options opts) => Extract2.DoDataExtraction(opts),
-                (Dataset.DatasetOptions opts) => Dataset.DoDataExtraction(opts),
-                (Dataset2.DatasetOptions2 opts) => Dataset2.DoDataExtraction2(opts),
-                _ => 1);
+                .MapResult(
+                    (Count.CountOptions opts) => Count.DoDataCount(opts),
+                    (Extract.ExtractOptions opts) => Extract.DoDataExtraction(opts),
+                    (Build.BuildOptions opts) => Build.DoBuildGraph(opts),
+                    (Convert.ConvertOptions opts) => Convert.DoMapConvert(opts),
+                    (Search.SearchOptions opts) => Search.DoPatternSearch(opts),
+                    (Analyze.AnalyzeOptions opts) => Analyze.DoVisualSpacingExtract(opts),
+                    (Extract2.Extract2Options opts) => Extract2.DoDataExtraction(opts),
+                    (Dataset.DatasetOptions opts) => Dataset.DoDataExtraction(opts),
+                    (Dataset2.DatasetOptions2 opts) => Dataset2.DoDataExtraction2(opts),
+                    _ => 1);
         }
     }
 }

Mapperator.ConsoleApp/Verbs/Build.cs

@@ -23,7 +23,7 @@ public static int DoBuildGraph(BuildOptions opts) {
         if (opts.OutputStructName is null) throw new ArgumentNullException(nameof(opts));
         if (opts.DataPath is null) throw new ArgumentNullException(nameof(opts));
 
-        var trainData = DataSerializer.DeserializeBeatmapData(File.ReadLines(Path.ChangeExtension(opts.DataPath, ".txt")));
+        var (_, trainData) = DataSerializer.DeserializeBeatmapData(File.ReadAllLines(Path.ChangeExtension(opts.DataPath, ".txt")));
         var data = new RhythmDistanceTrieStructure();
 
         if (data is not ISerializable sData) {

Mapperator.ConsoleApp/Verbs/Convert.cs

@@ -49,9 +49,9 @@ public static int DoMapConvert(ConvertOptions opts) {
         stopwatch.Start();
 
         Console.WriteLine(Strings.Program_DoMapConvert_Extracting_data___);
-        var trainData = DataSerializer.DeserializeBeatmapData(File.ReadLines(Path.ChangeExtension(opts.DataPath, ".txt")));
+        var (trainVersion, trainData) = DataSerializer.DeserializeBeatmapData(File.ReadAllLines(Path.ChangeExtension(opts.DataPath, ".txt")));
         var map = new BeatmapEditor(Path.ChangeExtension(opts.InputBeatmapPath, ".osu")).ReadFile();
-        var input = new DataExtractor().ExtractBeatmapData(map).ToArray();
+        var input = new DataExtractor(trainVersion).ExtractBeatmapData(map).ToArray();
 
         // TODO: add options to automatically add distance spacing
         // TODO: also add options for ignoring angles, nc, or slider attributes
@@ -64,7 +64,7 @@ public static int DoMapConvert(ConvertOptions opts) {
         if (opts.SpacingBeatmapPath is not null) {
             Console.WriteLine(Strings.Program_DoMapConvert_Converting_spacing_to_reference_beatmap___);
             var spacingMap = new BeatmapEditor(Path.ChangeExtension(opts.SpacingBeatmapPath, ".osu")).ReadFile();
-            var spacingMapData = new DataExtractor().ExtractBeatmapData(spacingMap).ToArray();
+            var spacingMapData = new DataExtractor(trainVersion).ExtractBeatmapData(spacingMap).ToArray();
             input = TransferSpacing(spacingMapData, input);
         }
 

Mapperator.ConsoleApp/Verbs/Extract.cs

@@ -21,7 +21,7 @@ public static int DoDataExtraction(ExtractOptions opts) {
         File.WriteAllLines(Path.ChangeExtension(opts.OutputName, ".txt"),
             DataSerializer.SerializeBeatmapData(DbManager.GetFilteredAndRead(opts)
                 .SelectMany(b => mirrors.Select(m => extractor.ExtractBeatmapData(b, m)))
-            ));
+            ).Prepend(DataSerializer.CurrentHeader));
 
         return 0;
     }

Mapperator.DemoApp/Mapperator.DemoApp.Game.Tests/Mapperator.DemoApp.Game.Tests.csproj

@@ -8,8 +8,8 @@
     <ProjectReference Include="..\Mapperator.DemoApp.Game\Mapperator.DemoApp.Game.csproj" />
   </ItemGroup>
   <ItemGroup Label="Package References">
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.0.0" />
-    <PackageReference Include="NUnit3TestAdapter" Version="4.0.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.0" />
+    <PackageReference Include="NUnit3TestAdapter" Version="5.1.0-alpha.2" />
   </ItemGroup>
   <ItemGroup>
     <None Remove="Resources\input.osu" />

Mapperator.DemoApp/Mapperator.DemoApp.Game/MainScreen.cs

@@ -222,7 +222,7 @@ private void OnBeatmapChange(ValueChangedEvent<Beatmap> obj)
 
             pos.MaxValue = obj.NewValue.HitObjects.Count - length - 1;
 
-            pattern = new DataExtractor().ExtractBeatmapData(obj.NewValue).ToArray();
+            pattern = new DataExtractor(1).ExtractBeatmapData(obj.NewValue).ToArray();
             matcher = new TrieDataMatcher(dataStruct, pattern);
             filter = new OnScreenFilter();
             judge = new SuperJudge(pattern);

Mapperator.DemoApp/Mapperator.DemoApp.Game/MapDataStore.cs

@@ -1,3 +1,4 @@
+using System;
 using System.Collections;
 using System.Collections.Generic;
 using System.IO;
@@ -36,15 +37,21 @@ public IEnumerable<IEnumerable<MapDataPoint>> Get(string name)
         using Stream stream = store.GetStream(name);
         if (stream is null) return null;
         using StreamReader reader = new StreamReader(stream);
-        return DataSerializer.DeserializeBeatmapData(iterateLines(reader).ToArray());
+        var (version, data) = DataSerializer.DeserializeBeatmapData(iterateLines(reader).ToArray());
+        if (version != 1)
+            throw new NotImplementedException($"Data version {version} is not currently supported in MapDataStore");
+        return data;
     }
 
     public Task<IEnumerable<IEnumerable<MapDataPoint>>> GetAsync(string name, CancellationToken cancellationToken = new())
     {
         using Stream stream = store.GetStream(name);
         if (stream is null) return null;
         using StreamReader reader = new StreamReader(stream);
-        return Task.FromResult(DataSerializer.DeserializeBeatmapData(iterateLines(reader)));
+        var (version, data) = DataSerializer.DeserializeBeatmapData(iterateLines(reader));
+        if (version != 1)
+            throw new NotImplementedException($"Data version {version} is not currently supported in MapDataStore");
+        return Task.FromResult(data);
     }
 
     public Stream GetStream(string name)

Mapperator.DemoApp/Mapperator.DemoApp.Game/Mapperator.DemoApp.Game.csproj

@@ -7,7 +7,7 @@
     <ProjectReference Include="..\Mapperator.DemoApp.Resources\Mapperator.DemoApp.Resources.csproj" />
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="MappingTools.Core" Version="0.6.0" />
-    <PackageReference Include="ppy.osu.Framework" Version="2022.916.1" />
+    <PackageReference Include="MappingTools.Core" Version="0.6.2" />
+    <PackageReference Include="ppy.osu.Framework" Version="2025.512.0" />
   </ItemGroup>
 </Project>

Mapperator.Tests/Mapperator.Tests.csproj

@@ -7,10 +7,13 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.11.0" />
-    <PackageReference Include="NUnit" Version="3.13.2" />
-    <PackageReference Include="NUnit3TestAdapter" Version="4.0.0" />
-    <PackageReference Include="coverlet.collector" Version="3.1.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.0" />
+    <PackageReference Include="NUnit" Version="4.3.2" />
+    <PackageReference Include="NUnit3TestAdapter" Version="5.1.0-alpha.2" />
+    <PackageReference Include="coverlet.collector" Version="6.0.4">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
   </ItemGroup>
 
   <ItemGroup>

Mapperator.Tests/Matching/DataStructures/RhythmDistanceTrieStructureTests.cs

@@ -39,27 +39,27 @@ public void TestQuery() {
 
                     var result = data.Trie.RetrieveSubstrings(query).ToList();
 
-                    Assert.IsTrue(result.Count > 0);
+                    Assert.That(result.Count, Is.GreaterThan(0));
                     foreach (var wordPosition in result) {
                         //var rhythmString = dataRhythmStrings[wordPosition.Value];
                         //Console.WriteLine(string.Join('-', Enumerable.Range(0, searchLength).Select(o => rhythmString.Span[wordPosition.CharPosition + o])));
                         //Console.WriteLine(string.Join('-', query.ToArray()));
-                        Assert.IsTrue(data.WordPositionInRange(wordPosition));
-                        Assert.IsTrue(data.WordPositionInRange(wordPosition, searchLength - 1));
-                        Assert.AreEqual(searchLength, GetMatchLength(wordPosition, query));
+                        Assert.That(data.WordPositionInRange(wordPosition));
+                        Assert.That(data.WordPositionInRange(wordPosition, searchLength - 1));
+                        Assert.That(searchLength, Is.EqualTo(GetMatchLength(wordPosition, query)));
                     }
 
                     var (min, max) = RhythmDistanceTrieStructure.ToDistanceRange(query, 10);
                     var rangeResult = data.Trie.RetrieveSubstringsRange(min, max).ToList();
 
-                    Assert.IsTrue(rangeResult.Count > 0);
+                    Assert.That(rangeResult.Count > 0);
                     foreach (var wordPosition in rangeResult) {
                         //var rhythmString = dataRhythmStrings[wordPosition.Value];
                         //Console.WriteLine(string.Join('-', Enumerable.Range(0, searchLength).Select(o => rhythmString.Span[wordPosition.CharPosition + o])));
                         //Console.WriteLine(string.Join('-', query.ToArray()));
-                        Assert.IsTrue(data.WordPositionInRange(wordPosition));
-                        Assert.IsTrue(data.WordPositionInRange(wordPosition, searchLength - 1));
-                        Assert.AreEqual(searchLength, GetMatchLengthRange(wordPosition, min.Span, max.Span));
+                        Assert.That(data.WordPositionInRange(wordPosition));
+                        Assert.That(data.WordPositionInRange(wordPosition, searchLength - 1));
+                        Assert.That(searchLength, Is.EqualTo(GetMatchLengthRange(wordPosition, min.Span, max.Span)));
                     }
                 }
             }

Mapperator.Tests/Matching/DataStructures/RhythmDistanceTrieTests.cs

@@ -37,13 +37,13 @@ public void TestRetrieveSubstringsDynamicLengthAndDistanceRange() {
 
         var result = trie.RetrieveSubstringsDynamicLengthAndDistanceRange(query, new RhythmDistanceTrie.MinLengthProvider(1)).ToArray();
 
-        Assert.AreEqual(1, result.Length);
+        Assert.That(1, Is.EqualTo(result.Length));
         var (pos, length, minMult, maxMult) = result[0];
         Console.WriteLine(result[0]);
-        Assert.AreEqual(0, pos.Value);
-        Assert.AreEqual(0, pos.CharPosition);
-        Assert.AreEqual(7, length);
-        Assert.IsTrue(minMult < 1 && 1 < maxMult);
+        Assert.That(0, Is.EqualTo(pos.Value));
+        Assert.That(0, Is.EqualTo(pos.CharPosition));
+        Assert.That(7, Is.EqualTo(length));
+        Assert.That(minMult < 1 && 1 < maxMult);
     }
 
     [Test]
@@ -60,45 +60,45 @@ public void TestDynamicLengthRetrieveSubstringsDynamicLengthAndDistanceRange() {
         var result = trie.RetrieveSubstringsDynamicLengthAndDistanceRange(query, new RhythmDistanceTrie.MinLengthProvider(1))
             .OrderBy(o => o.Item1.CharPosition).ToArray();
 
-        Assert.AreEqual(5, result.Length);
+        Assert.That(5, Is.EqualTo(result.Length));
         var (pos, length, minMult, maxMult) = result[0];
-        Assert.AreEqual(1, pos.CharPosition);
-        Assert.AreEqual(6, length);
-        Assert.IsTrue(minMult < 1 && 1 < maxMult);
+        Assert.That(1, Is.EqualTo(pos.CharPosition));
+        Assert.That(6, Is.EqualTo(length));
+        Assert.That(minMult < 1 && 1 < maxMult);
         (pos, length, minMult, maxMult) = result[1];
-        Assert.AreEqual(2, pos.CharPosition);
-        Assert.AreEqual(2, length);
-        Assert.IsTrue(minMult < 0.6 && 0.6 < maxMult);
+        Assert.That(2, Is.EqualTo(pos.CharPosition));
+        Assert.That(2, Is.EqualTo(length));
+        Assert.That(minMult < 0.6 && 0.6 < maxMult);
         (pos, length, minMult, maxMult) = result[2];
-        Assert.AreEqual(3, pos.CharPosition);
-        Assert.AreEqual(1, length);
-        Assert.IsTrue(minMult < 0.5 && 0.5 < maxMult);
+        Assert.That(3, Is.EqualTo(pos.CharPosition));
+        Assert.That(1, Is.EqualTo(length));
+        Assert.That(minMult < 0.5 && 0.5 < maxMult);
         (pos, length, minMult, maxMult) = result[3];
-        Assert.AreEqual(5, pos.CharPosition);
-        Assert.AreEqual(2, length);
-        Assert.IsTrue(minMult < 1 && 1 < maxMult);
+        Assert.That(5, Is.EqualTo(pos.CharPosition));
+        Assert.That(2, Is.EqualTo(length));
+        Assert.That(minMult < 1 && 1 < maxMult);
         (pos, length, minMult, maxMult) = result[4];
-        Assert.AreEqual(6, pos.CharPosition);
-        Assert.AreEqual(1, length);
-        Assert.IsTrue(minMult < 1 && 1 < maxMult);
+        Assert.That(6, Is.EqualTo(pos.CharPosition));
+        Assert.That(1, Is.EqualTo(length));
+        Assert.That(minMult < 1 && 1 < maxMult);
 
         result = trie.RetrieveSubstringsDynamicLengthAndDistanceRange(query, new RhythmDistanceTrie.MinLengthProvider(2))
             .OrderBy(o => o.Item1.CharPosition).ToArray();
 
-        Assert.AreEqual(3, result.Length);
+        Assert.That(3, Is.EqualTo(result.Length));
         (pos, length, _, _) = result[0];
-        Assert.AreEqual(1, pos.CharPosition);
-        Assert.AreEqual(6, length);
+        Assert.That(1, Is.EqualTo(pos.CharPosition));
+        Assert.That(6, Is.EqualTo(length));
         (pos, length, _, _) = result[1];
-        Assert.AreEqual(2, pos.CharPosition);
-        Assert.AreEqual(2, length);
+        Assert.That(2, Is.EqualTo(pos.CharPosition));
+        Assert.That(2, Is.EqualTo(length));
         (pos, length, _, _) = result[2];
-        Assert.AreEqual(5, pos.CharPosition);
-        Assert.AreEqual(2, length);
+        Assert.That(5, Is.EqualTo(pos.CharPosition));
+        Assert.That(2, Is.EqualTo(length));
 
         result = trie.RetrieveSubstringsDynamicLengthAndDistanceRange(query, new RhythmDistanceTrie.MinLengthProvider(3)).ToArray();
 
-        Assert.AreEqual(1, result.Length);
+        Assert.That(1, Is.EqualTo(result.Length));
     }
 
     [Test]
@@ -115,13 +115,13 @@ public void TestDynamicDistanceRetrieveSubstringsDynamicLengthAndDistanceRange()
 
         var result = trie.RetrieveSubstringsDynamicLengthAndDistanceRange(query, new RhythmDistanceTrie.MinLengthProvider(1)).ToArray();
 
-        Assert.AreEqual(1, result.Length);
+        Assert.That(1, Is.EqualTo(result.Length));
         var (pos, length, minMult, maxMult) = result[0];
         Console.WriteLine(result[0]);
-        Assert.AreEqual(0, pos.Value);
-        Assert.AreEqual(0, pos.CharPosition);
-        Assert.AreEqual(7, length);
-        Assert.IsTrue(minMult < 0.5 && 0.5 < maxMult);
+        Assert.That(0, Is.EqualTo(pos.Value));
+        Assert.That(0, Is.EqualTo(pos.CharPosition));
+        Assert.That(7, Is.EqualTo(length));
+        Assert.That(minMult < 0.5 && 0.5 < maxMult);
     }
 
     [Test]

Mapperator.Tests/Matching/Matchers/TrieDataMatcherTests.cs

@@ -27,8 +27,8 @@ public void TestQuery() {
         foreach (var match in result) {
             Console.WriteLine(match.Sequence.Length);
             Console.WriteLine(string.Join('-', RhythmDistanceTrieStructure.ToRhythmString(match.Sequence.Span).ToArray()));
-            //Assert.IsTrue(WordPositionInRange(wordPosition));
-            //Assert.IsTrue(WordPositionInRange(wordPosition, searchLength));
+            //Assert.That(WordPositionInRange(wordPosition));
+            //Assert.That(WordPositionInRange(wordPosition, searchLength));
             //Assert.AreEqual(searchLength, GetMatchLength(wordPosition, query));
         }
     }

Mapperator.Tests/Matching/TrieTests.cs

@@ -1,6 +1,7 @@
 using NUnit.Framework;
 using System;
 using System.Linq;
+using NUnit.Framework.Legacy;
 using TrieNet.Ukkonen;
 
 namespace Mapperator.Tests.Matching {

Mapperator/DataExtractor.cs

@@ -10,11 +10,13 @@
 namespace Mapperator {
     public class DataExtractor {
         private readonly HitObjectEncoder encoder;
+        private readonly int dataVersion;
 
-        public DataExtractor() : this(new HitObjectEncoder()) { }
+        public DataExtractor(int dataVersion = DataSerializer.CurrentDataVersion) : this(new HitObjectEncoder(), dataVersion) { }
 
-        public DataExtractor(HitObjectEncoder encoder) {
+        public DataExtractor(HitObjectEncoder encoder, int dataVersion) {
             this.encoder = encoder;
+            this.dataVersion = dataVersion;
         }
 
         public IEnumerable<MapDataPoint> ExtractBeatmapData(IBeatmap beatmap, bool mirror = false) {
@@ -43,9 +45,17 @@ public IEnumerable<MapDataPoint> ExtractBeatmapData(IEnumerable<HitObject> hitob
                         var segments = 0;
                         var controlPoints = path.ControlPoints;
 
-                        for (var i = 0; i < controlPoints.Count; i++) {
-                            if (i == controlPoints.Count - 1 || controlPoints[i] == controlPoints[i + 1] && i != controlPoints.Count - 2) {
-                                segments++;
+                        if (dataVersion >= 2 && slider.SliderType == PathType.Linear) {
+                            for (var i = 0; i < controlPoints.Count - 1; i++) {
+                                if (controlPoints[i] != controlPoints[i + 1]) {
+                                    segments++;
+                                }
+                            }
+                        } else {
+                            for (var i = 0; i < controlPoints.Count; i++) {
+                                if (i == controlPoints.Count - 1 || controlPoints[i] == controlPoints[i + 1] && i != controlPoints.Count - 2) {
+                                    segments++;
+                                }
                             }
                         }