diff --git a/src/tools/ResultsComparer/CommandLineOptions.cs b/src/tools/ResultsComparer/CommandLineOptions.cs new file mode 100644 index 00000000000..61e3d7f5c51 --- /dev/null +++ b/src/tools/ResultsComparer/CommandLineOptions.cs @@ -0,0 +1,46 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; +using System.IO; +using CommandLine; +using CommandLine.Text; + +namespace ResultsComparer +{ + public class CommandLineOptions + { + [Option("base", HelpText = "Path to the folder/file with base results.")] + public string BasePath { get; set; } + + [Option("diff", HelpText = "Path to the folder/file with diff results.")] + public string DiffPath { get; set; } + + [Option("threshold", Required = true, HelpText = "Threshold for Statistical Test. Examples: 5%, 10ms, 100ns, 1s.")] + public string StatisticalTestThreshold { get; set; } + + [Option("noise", HelpText = "Noise threshold for Statistical Test. The difference for 1.0ns and 1.1ns is 10%, but it's just a noise. Examples: 0.5ns 1ns.", Default = "0.3ns" )] + public string NoiseThreshold { get; set; } + + [Option("top", HelpText = "Filter the diff to top/bottom N results. Optional.")] + public int? TopCount { get; set; } + + [Option("csv", HelpText = "Path to exported CSV results. Optional.")] + public FileInfo CsvPath { get; set; } + + [Usage(ApplicationAlias = "")] + public static IEnumerable Examples + { + get + { + yield return new Example(@"Compare the results stored in 'C:\results\win' (base) vs 'C:\results\unix' (diff) using 5% threshold.", + new CommandLineOptions { BasePath = @"C:\results\win", DiffPath = @"C:\results\unix", StatisticalTestThreshold = "5%" }); + yield return new Example(@"Compare the results stored in 'C:\results\win' (base) vs 'C:\results\unix' (diff) using 5% threshold and show only top/bottom 10 results.", + new CommandLineOptions { BasePath = @"C:\results\win", DiffPath = @"C:\results\unix", StatisticalTestThreshold = "5%", TopCount = 10 }); + yield return new Example(@"Compare the results stored in 'C:\results\win' (base) vs 'C:\results\unix' (diff) using 5% threshold and 0.5ns noise filter.", + new CommandLineOptions { BasePath = @"C:\results\win", DiffPath = @"C:\results\unix", StatisticalTestThreshold = "5%", NoiseThreshold = "0.5ns" }); + } + } + } +} \ No newline at end of file diff --git a/src/tools/ResultsComparer/DataTransferContracts.cs b/src/tools/ResultsComparer/DataTransferContracts.cs new file mode 100644 index 00000000000..e68f7971c80 --- /dev/null +++ b/src/tools/ResultsComparer/DataTransferContracts.cs @@ -0,0 +1,133 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +// + +using System.Collections.Generic; +using System.Linq; + +namespace DataTransferContracts // generated with http://json2csharp.com/# +{ + public class ChronometerFrequency + { + public int Hertz { get; set; } + } + + public class HostEnvironmentInfo + { + public string BenchmarkDotNetCaption { get; set; } + public string BenchmarkDotNetVersion { get; set; } + public string OsVersion { get; set; } + public string ProcessorName { get; set; } + public int? PhysicalProcessorCount { get; set; } + public int? PhysicalCoreCount { get; set; } + public int? LogicalCoreCount { get; set; } + public string RuntimeVersion { get; set; } + public string Architecture { get; set; } + public bool? HasAttachedDebugger { get; set; } + public bool? HasRyuJit { get; set; } + public string Configuration { get; set; } + public string JitModules { get; set; } + public string DotNetCliVersion { get; set; } + public ChronometerFrequency ChronometerFrequency { get; set; } + public string HardwareTimerKind { get; set; } + } + + public class ConfidenceInterval + { + public int N { get; set; } + public double Mean { get; set; } + public double StandardError { get; set; } + public int Level { get; set; } + public double Margin { get; set; } + public double Lower { get; set; } + public double Upper { get; set; } + } + + public class Percentiles + { + public double P0 { get; set; } + public double P25 { get; set; } + public double P50 { get; set; } + public double P67 { get; set; } + public double P80 { get; set; } + public double P85 { get; set; } + public double P90 { get; set; } + public double P95 { get; set; } + public double P100 { get; set; } + } + + public class Statistics + { + public int N { get; set; } + public double Min { get; set; } + public double LowerFence { get; set; } + public double Q1 { get; set; } + public double Median { get; set; } + public double Mean { get; set; } + public double Q3 { get; set; } + public double UpperFence { get; set; } + public double Max { get; set; } + public double InterquartileRange { get; set; } + public List LowerOutliers { get; set; } + public List UpperOutliers { get; set; } + public List AllOutliers { get; set; } + public double StandardError { get; set; } + public double Variance { get; set; } + public double StandardDeviation { get; set; } + public double Skewness { get; set; } + public double Kurtosis { get; set; } + public ConfidenceInterval ConfidenceInterval { get; set; } + public Percentiles Percentiles { get; set; } + } + + public class Memory + { + public int Gen0Collections { get; set; } + public int Gen1Collections { get; set; } + public int Gen2Collections { get; set; } + public int TotalOperations { get; set; } + public long BytesAllocatedPerOperation { get; set; } + } + + public class Measurement + { + public string IterationStage { get; set; } + public int LaunchIndex { get; set; } + public int IterationIndex { get; set; } + public long Operations { get; set; } + public double Nanoseconds { get; set; } + } + + public class Benchmark + { + public string DisplayInfo { get; set; } + public object Namespace { get; set; } + public string Type { get; set; } + public string Method { get; set; } + public string MethodTitle { get; set; } + public string Parameters { get; set; } + public string FullName { get; set; } + public Statistics Statistics { get; set; } + public Memory Memory { get; set; } + public List Measurements { get; set; } + + /// + /// this method was not auto-generated by a tool, it was added manually + /// + /// an array of the actual workload results (not warmup, not pilot) + internal double[] GetOriginalValues() + => Measurements + .Where(measurement => measurement.IterationStage == "Result") + .Select(measurement => measurement.Nanoseconds / measurement.Operations) + .ToArray(); + } + + public class BdnResult + { + public string Title { get; set; } + public HostEnvironmentInfo HostEnvironmentInfo { get; set; } + public List Benchmarks { get; set; } + } +} \ No newline at end of file diff --git a/src/tools/ResultsComparer/NuGet.Config b/src/tools/ResultsComparer/NuGet.Config new file mode 100644 index 00000000000..5985e549509 --- /dev/null +++ b/src/tools/ResultsComparer/NuGet.Config @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/tools/ResultsComparer/Program.cs b/src/tools/ResultsComparer/Program.cs new file mode 100644 index 00000000000..7e9b165510f --- /dev/null +++ b/src/tools/ResultsComparer/Program.cs @@ -0,0 +1,183 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Threading; +using BenchmarkDotNet.Mathematics; +using BenchmarkDotNet.Mathematics.StatisticalTesting; +using CommandLine; +using DataTransferContracts; +using MarkdownLog; +using Newtonsoft.Json; + +namespace ResultsComparer +{ + public class Program + { + private const string FullBdnJsonFileExtension = "full.json"; + + public static void Main(string[] args) + { + // we print a lot of numbers here and we want to make it always in invariant way + Thread.CurrentThread.CurrentCulture = CultureInfo.InvariantCulture; + + Parser.Default.ParseArguments(args).WithParsed(Compare); + } + + private static void Compare(CommandLineOptions args) + { + if (!Threshold.TryParse(args.StatisticalTestThreshold, out var testThreshold)) + { + Console.WriteLine($"Invalid Threshold {args.StatisticalTestThreshold}. Examples: 5%, 10ms, 100ns, 1s."); + return; + } + if (!Threshold.TryParse(args.NoiseThreshold, out var noiseThreshold)) + { + Console.WriteLine($"Invalid Noise Threshold {args.NoiseThreshold}. Examples: 0.3ns 1ns."); + return; + } + + var notSame = GetNotSameResults(args, testThreshold, noiseThreshold).ToArray(); + + PrintTable(notSame, EquivalenceTestConclusion.Slower, args); + PrintTable(notSame, EquivalenceTestConclusion.Faster, args); + + ExportToCsv(notSame, args.CsvPath); + } + + private static IEnumerable<(string id, Benchmark baseResult, Benchmark diffResult, EquivalenceTestConclusion conclusion)> GetNotSameResults(CommandLineOptions args, Threshold testThreshold, Threshold noiseThreshold) + { + foreach (var pair in ReadResults(args) + .Where(result => result.baseResult.Statistics != null && result.diffResult.Statistics != null)) // failures + { + var baseValues = pair.baseResult.GetOriginalValues(); + var diffValues = pair.diffResult.GetOriginalValues(); + + var userTresholdResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, testThreshold); + if (userTresholdResult.Conclusion == EquivalenceTestConclusion.Same) + continue; + + var noiseResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, noiseThreshold); + if (noiseResult.Conclusion == EquivalenceTestConclusion.Same) + continue; + + yield return (pair.id, pair.baseResult, pair.diffResult, userTresholdResult.Conclusion); + } + } + + private static void PrintTable((string id, Benchmark baseResult, Benchmark diffResult, EquivalenceTestConclusion conclusion)[] notSame, EquivalenceTestConclusion conclusion, CommandLineOptions args) + { + var data = notSame + .Where(result => result.conclusion == conclusion) + .OrderByDescending(result => GetRatio(conclusion, result.baseResult, result.diffResult)) + .Take(args.TopCount ?? int.MaxValue) + .Select(result => new { + Id = result.id.Length > 80 ? result.id.Substring(0, 80) : result.id, + DisplayValue = GetRatio(conclusion, result.baseResult, result.diffResult), + BaseMedian = result.baseResult.Statistics.Median, + DiffMedian = result.diffResult.Statistics.Median, + Modality = GetModalInfo(result.baseResult) ?? GetModalInfo(result.diffResult) + }) + .ToArray(); + + var table = data.ToMarkdownTable().WithHeaders(conclusion.ToString(), conclusion == EquivalenceTestConclusion.Faster ? "base/diff" : "diff/base", "Base Median (ns)", "Diff Median (ns)", "Modality"); + + foreach (var line in table.ToMarkdown().Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries)) + Console.WriteLine($"| {line.TrimStart()}|"); // the table starts with \t and does not end with '|' and it looks bad so we fix it + + Console.WriteLine(); + } + + private static IEnumerable<(string id, Benchmark baseResult, Benchmark diffResult)> ReadResults(CommandLineOptions args) + { + var baseFiles = GetFilesToParse(args.BasePath); + var diffFiles = GetFilesToParse(args.DiffPath); + + if (!baseFiles.Any() || !diffFiles.Any()) + throw new ArgumentException($"Provided paths contained no {FullBdnJsonFileExtension} files."); + + var baseResults = baseFiles.Select(ReadFromFile); + var diffResults = diffFiles.Select(ReadFromFile); + + var benchmarkIdToDiffResults = diffResults.SelectMany(result => result.Benchmarks).ToDictionary(benchmarkResult => benchmarkResult.FullName, benchmarkResult => benchmarkResult); + + return baseResults + .SelectMany(result => result.Benchmarks) + .ToDictionary(benchmarkResult => benchmarkResult.FullName, benchmarkResult => benchmarkResult) // we use ToDictionary to make sure the results have unique IDs + .Where(baseResult => benchmarkIdToDiffResults.ContainsKey(baseResult.Key)) + .Select(baseResult => (baseResult.Key, baseResult.Value, benchmarkIdToDiffResults[baseResult.Key])); + } + + private static void ExportToCsv((string id, Benchmark baseResult, Benchmark diffResult, EquivalenceTestConclusion conclusion)[] notSame, FileInfo csvPath) + { + if (csvPath == null) + return; + + if (csvPath.Exists) + csvPath.Delete(); + + using (var textWriter = csvPath.CreateText()) + { + foreach (var result in notSame) + { + textWriter.WriteLine($"\"{result.id.Replace("\"", "\"\"")}\";base;{result.conclusion};{string.Join(';', result.baseResult.GetOriginalValues())}"); + textWriter.WriteLine($"\"{result.id.Replace("\"", "\"\"")}\";diff;{result.conclusion};{string.Join(';', result.diffResult.GetOriginalValues())}"); + } + } + + Console.WriteLine($"CSV results exported to {csvPath.FullName}"); + } + + private static string[] GetFilesToParse(string path) + { + if (Directory.Exists(path)) + return Directory.GetFiles(path, $"*{FullBdnJsonFileExtension}", SearchOption.AllDirectories); + else if (File.Exists(path) || !path.EndsWith(FullBdnJsonFileExtension)) + return new[] { path }; + else + throw new FileNotFoundException($"Provided path does NOT exist or is not a {path} file", path); + } + + // code and magic values taken from BenchmarkDotNet.Analysers.MultimodalDistributionAnalyzer + // See http://www.brendangregg.com/FrequencyTrails/modes.html + private static string GetModalInfo(Benchmark benchmark) + { + if (benchmark.Statistics.N < 12) // not enough data to tell + return null; + + double mValue = MathHelper.CalculateMValue(new BenchmarkDotNet.Mathematics.Statistics(benchmark.GetOriginalValues())); + if (mValue > 4.2) + return "multimodal"; + else if (mValue > 3.2) + return "bimodal"; + else if (mValue > 2.8) + return "several?"; + + return null; + } + + private static double GetRatio(EquivalenceTestConclusion conclusion, Benchmark baseResult, Benchmark diffResult) + => conclusion == EquivalenceTestConclusion.Faster + ? baseResult.Statistics.Median / diffResult.Statistics.Median + : diffResult.Statistics.Median / baseResult.Statistics.Median; + + private static BdnResult ReadFromFile(string resultFilePath) + { + try + { + return JsonConvert.DeserializeObject(File.ReadAllText(resultFilePath)); + } + catch (JsonSerializationException) + { + Console.WriteLine($"Exception while reading the {resultFilePath} file."); + + throw; + } + } + } +} \ No newline at end of file diff --git a/src/tools/ResultsComparer/README.md b/src/tools/ResultsComparer/README.md new file mode 100644 index 00000000000..63d922e5c10 Binary files /dev/null and b/src/tools/ResultsComparer/README.md differ diff --git a/src/tools/ResultsComparer/ResultsComparer.csproj b/src/tools/ResultsComparer/ResultsComparer.csproj new file mode 100644 index 00000000000..1305bf30dc1 --- /dev/null +++ b/src/tools/ResultsComparer/ResultsComparer.csproj @@ -0,0 +1,13 @@ + + + Exe + netcoreapp2.1 + latest + + + + + + + + \ No newline at end of file diff --git a/src/tools/ResultsComparer/ResultsComparer.sln b/src/tools/ResultsComparer/ResultsComparer.sln new file mode 100644 index 00000000000..951a4d0fb5d --- /dev/null +++ b/src/tools/ResultsComparer/ResultsComparer.sln @@ -0,0 +1,16 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ResultsComparer", "ResultsComparer.csproj", "{00859394-44F8-466B-8624-41578CA94009}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {00859394-44F8-466B-8624-41578CA94009}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {00859394-44F8-466B-8624-41578CA94009}.Debug|Any CPU.Build.0 = Debug|Any CPU + {00859394-44F8-466B-8624-41578CA94009}.Release|Any CPU.ActiveCfg = Release|Any CPU + {00859394-44F8-466B-8624-41578CA94009}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection +EndGlobal