From 174aafa9be6764dc8cf5e4253cc79f48a361dbfd Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Mon, 3 Dec 2018 12:12:27 +0100
Subject: [PATCH 01/11] Results Comparer

---
 .../ResultsComparer/CommandLineOptions.cs     |  42 ++++++
 .../ResultsComparer/DataTransferContracts.cs  | 129 ++++++++++++++++
 src/tools/ResultsComparer/NuGet.Config        |  14 ++
 src/tools/ResultsComparer/Program.cs          | 139 ++++++++++++++++++
 src/tools/ResultsComparer/README.md           | Bin 0 -> 4134 bytes
 .../ResultsComparer/ResultsComparer.csproj    |  12 ++
 src/tools/ResultsComparer/ResultsComparer.sln |  16 ++
 7 files changed, 352 insertions(+)
 create mode 100644 src/tools/ResultsComparer/CommandLineOptions.cs
 create mode 100644 src/tools/ResultsComparer/DataTransferContracts.cs
 create mode 100644 src/tools/ResultsComparer/NuGet.Config
 create mode 100644 src/tools/ResultsComparer/Program.cs
 create mode 100644 src/tools/ResultsComparer/README.md
 create mode 100644 src/tools/ResultsComparer/ResultsComparer.csproj
 create mode 100644 src/tools/ResultsComparer/ResultsComparer.sln

diff --git a/src/tools/ResultsComparer/CommandLineOptions.cs b/src/tools/ResultsComparer/CommandLineOptions.cs
new file mode 100644
index 00000000000..d3a3e6a580b
--- /dev/null
+++ b/src/tools/ResultsComparer/CommandLineOptions.cs
@@ -0,0 +1,42 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Generic;
+using CommandLine;
+using CommandLine.Text;
+
+namespace ResultsComparer
+{
+    public class CommandLineOptions
+    {
+        [Option("base", HelpText = "Path to the folder/file with base results.")]
+        public string BasePath { get; set; }
+
+        [Option("diff", HelpText = "Path to the folder/file with diff results.")]
+        public string DiffPath { get; set; }
+
+        [Option("merged", HelpText = "Path to the folder/file with results merged for multiple jobs in the same file.")]
+        public string MergedPath { get; set; }
+
+        [Option("treshold", Required = true, HelpText = "Threshold for Statistical Test. Examples: 5%, 10ms, 100ns, 1s.")]
+        public string StatisticalTestThreshold { get; set; }
+
+        [Option("top", HelpText = "Filter the diff to top/bottom N results. Optional.")]
+        public int? TopCount { get; set; }
+
+        [Usage(ApplicationAlias = "")]
+        public static IEnumerable<Example> Examples
+        {
+            get
+            {
+                yield return new Example(@"Compare the results stored in 'C:\results\win' (base) vs 'C:\results\unix' (diff) using 5% threshold.",
+                    new CommandLineOptions { BasePath = @"C:\results\win", DiffPath = @"C:\results\unix", StatisticalTestThreshold = "5%" });
+                yield return new Example(@"Compare the results stored in 'C:\results\21_vs_22' (multiple jobs per single benchmark run using 1ms threshold.",
+                    new CommandLineOptions { MergedPath = @"C:\results\21_vs_22", StatisticalTestThreshold = "1ms" });
+                yield return new Example(@"Compare the results stored in 'C:\results\win' (base) vs 'C:\results\unix' (diff) using 5% threshold and show only top/bottom 10 results.",
+                    new CommandLineOptions { BasePath = @"C:\results\win", DiffPath = @"C:\results\unix", StatisticalTestThreshold = "5%", TopCount = 10 });
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/src/tools/ResultsComparer/DataTransferContracts.cs b/src/tools/ResultsComparer/DataTransferContracts.cs
new file mode 100644
index 00000000000..6c3eb52b888
--- /dev/null
+++ b/src/tools/ResultsComparer/DataTransferContracts.cs
@@ -0,0 +1,129 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+// <auto-generated />
+
+using System.Collections.Generic;
+using System.Linq;
+
+namespace DataTransferContracts // generated with http://json2csharp.com/#
+{
+    public class ChronometerFrequency
+    {
+        public int Hertz { get; set; }
+    }
+
+    public class HostEnvironmentInfo
+    {
+        public string BenchmarkDotNetCaption { get; set; }
+        public string BenchmarkDotNetVersion { get; set; }
+        public string OsVersion { get; set; }
+        public string ProcessorName { get; set; }
+        public int PhysicalProcessorCount { get; set; }
+        public int PhysicalCoreCount { get; set; }
+        public int LogicalCoreCount { get; set; }
+        public string RuntimeVersion { get; set; }
+        public string Architecture { get; set; }
+        public bool HasAttachedDebugger { get; set; }
+        public bool HasRyuJit { get; set; }
+        public string Configuration { get; set; }
+        public string JitModules { get; set; }
+        public string DotNetCliVersion { get; set; }
+        public ChronometerFrequency ChronometerFrequency { get; set; }
+        public string HardwareTimerKind { get; set; }
+    }
+
+    public class ConfidenceInterval
+    {
+        public int N { get; set; }
+        public double Mean { get; set; }
+        public double StandardError { get; set; }
+        public int Level { get; set; }
+        public double Margin { get; set; }
+        public double Lower { get; set; }
+        public double Upper { get; set; }
+    }
+
+    public class Percentiles
+    {
+        public double P0 { get; set; }
+        public double P25 { get; set; }
+        public double P50 { get; set; }
+        public double P67 { get; set; }
+        public double P80 { get; set; }
+        public double P85 { get; set; }
+        public double P90 { get; set; }
+        public double P95 { get; set; }
+        public double P100 { get; set; }
+    }
+
+    public class Statistics
+    {
+        public int N { get; set; }
+        public double Min { get; set; }
+        public double LowerFence { get; set; }
+        public double Q1 { get; set; }
+        public double Median { get; set; }
+        public double Mean { get; set; }
+        public double Q3 { get; set; }
+        public double UpperFence { get; set; }
+        public double Max { get; set; }
+        public double InterquartileRange { get; set; }
+        public List<double> LowerOutliers { get; set; }
+        public List<double> UpperOutliers { get; set; }
+        public List<double> AllOutliers { get; set; }
+        public double StandardError { get; set; }
+        public double Variance { get; set; }
+        public double StandardDeviation { get; set; }
+        public double Skewness { get; set; }
+        public double Kurtosis { get; set; }
+        public ConfidenceInterval ConfidenceInterval { get; set; }
+        public Percentiles Percentiles { get; set; }
+    }
+
+    public class Memory
+    {
+        public int Gen0Collections { get; set; }
+        public int Gen1Collections { get; set; }
+        public int Gen2Collections { get; set; }
+        public int TotalOperations { get; set; }
+        public long BytesAllocatedPerOperation { get; set; }
+    }
+
+    public class Measurement
+    {
+        public string IterationStage { get; set; }
+        public int LaunchIndex { get; set; }
+        public int IterationIndex { get; set; }
+        public long Operations { get; set; }
+        public double Nanoseconds { get; set; }
+    }
+
+    public class Benchmark
+    {
+        public string DisplayInfo { get; set; }
+        public object Namespace { get; set; }
+        public string Type { get; set; }
+        public string Method { get; set; }
+        public string MethodTitle { get; set; }
+        public string Parameters { get; set; }
+        public string FullName { get; set; }
+        public Statistics Statistics { get; set; }
+        public Memory Memory { get; set; }
+        public List<Measurement> Measurements { get; set; }
+
+        internal double[] GetOriginalValues()
+            => Measurements
+                .Where(measurement => measurement.IterationStage == "Result")
+                .Select(measurement => measurement.Nanoseconds / measurement.Operations)
+                .ToArray();
+    }
+
+    public class BdnResult
+    {
+        public string Title { get; set; }
+        public HostEnvironmentInfo HostEnvironmentInfo { get; set; }
+        public List<Benchmark> Benchmarks { get; set; }
+    }
+}
\ No newline at end of file
diff --git a/src/tools/ResultsComparer/NuGet.Config b/src/tools/ResultsComparer/NuGet.Config
new file mode 100644
index 00000000000..5985e549509
--- /dev/null
+++ b/src/tools/ResultsComparer/NuGet.Config
@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <solution>
+    <add key="disableSourceControlIntegration" value="true" />
+  </solution>
+  <packageSources>
+    <!--To inherit the global NuGet package sources remove the <clear/> line below -->
+    <clear />
+
+    <add key="api.nuget.org" value="https://api.nuget.org/v3/index.json" />
+    <add key="benchmarkdotnet-ci-feed" value="https://ci.appveyor.com/nuget/benchmarkdotnet" />
+    <add key="dotnet-core" value="https://dotnet.myget.org/F/dotnet-core/api/v3/index.json" />
+  </packageSources>
+</configuration>
\ No newline at end of file
diff --git a/src/tools/ResultsComparer/Program.cs b/src/tools/ResultsComparer/Program.cs
new file mode 100644
index 00000000000..7f7008ade34
--- /dev/null
+++ b/src/tools/ResultsComparer/Program.cs
@@ -0,0 +1,139 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using BenchmarkDotNet.Mathematics;
+using BenchmarkDotNet.Mathematics.StatisticalTesting;
+using CommandLine;
+using DataTransferContracts;
+using Newtonsoft.Json;
+
+namespace ResultsComparer
+{
+    public class Program
+    {
+        private const string FullBdnJsonFileExtension = "full.json";
+
+        private static readonly Threshold NoiseThreshold = Threshold.Create(ThresholdUnit.Nanoseconds, 0.3);
+
+        public static void Main(string[] args) => Parser.Default.ParseArguments<CommandLineOptions>(args).WithParsed(Compare);
+
+        private static void Compare(CommandLineOptions args)
+        {
+            if (!Threshold.TryParse(args.StatisticalTestThreshold, out var userProvidedThreshold))
+            {
+                Console.WriteLine($"Invalid Threshold {args.StatisticalTestThreshold}. Examples: 5%, 10ms, 100ns, 1s.");
+                return;
+            }
+
+            var notSame = GetNotSameResults(args, userProvidedThreshold).ToArray();
+            var takeCount = args.TopCount.HasValue ? args.TopCount.Value : int.MaxValue;
+
+            Console.WriteLine("SLOWER:");
+            foreach (var result in notSame.Where(result => result.conclusion == EquivalenceTestConclusion.Slower).OrderBy(pair => pair.ratio).Take(takeCount))
+                PrintResult(result);
+
+            Console.WriteLine();
+            Console.WriteLine("FASTER:");
+            foreach (var result in notSame.Where(result => result.conclusion == EquivalenceTestConclusion.Faster).OrderByDescending(pair => pair.ratio).Take(takeCount))
+                PrintResult(result);
+        }
+
+        private static IEnumerable<(string id, Benchmark baseResult, Benchmark diffResult, EquivalenceTestConclusion conclusion, double ratio)> GetNotSameResults(CommandLineOptions args, Threshold userProvidedThreshold)
+        {
+            foreach (var pair in ReadResults(args))
+            {
+                var baseValues = pair.baseResult.GetOriginalValues();
+                var diffValues = pair.diffResult.GetOriginalValues();
+
+                var userTresholdResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, userProvidedThreshold);
+                if (userTresholdResult.Conclusion == EquivalenceTestConclusion.Same)
+                    continue;
+
+                // the difference for 1.0ns and 1.1ns is 10%, but it's just a noise. So we test against 0.3ns to filter out the nano-benchmarks noise
+                var noiseResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, NoiseThreshold);
+                if (noiseResult.Conclusion == EquivalenceTestConclusion.Same)
+                    continue;
+
+                var ratio = (1.0 - pair.diffResult.Statistics.Median / pair.baseResult.Statistics.Median);
+
+                yield return (pair.id, pair.baseResult, pair.diffResult, userTresholdResult.Conclusion, ratio);
+            }
+        }
+
+        private static IEnumerable<(string id, Benchmark baseResult, Benchmark diffResult)> ReadResults(CommandLineOptions args)
+        {
+            // this is the case where BDN run one benchmark for multiple jobs, for example for two runtimes using -r netcoreapp2.1 netcoreapp2.2
+            if (!string.IsNullOrEmpty(args.MergedPath))
+            {
+                return GetFilesToParse(args.MergedPath)
+                    .Select(resultFile => JsonConvert.DeserializeObject<BdnResult>(File.ReadAllText(resultFile)))
+                    .SelectMany(result => result.Benchmarks)
+                    .GroupBy(result => result.FullName)
+                        .SelectMany(sameKey => sameKey
+                            .Select(group => (group.FullName, sameKey.First(), group)) // first is always the base
+                            .Skip(1)); // we skip the first entry in the group, we want to do 1st vs 2nd, 1st vs 3rd etc..
+            }
+            else if(!string.IsNullOrEmpty(args.BasePath) && !string.IsNullOrEmpty(args.DiffPath))
+            {
+                var baseFiles = GetFilesToParse(args.BasePath);
+                var diffFiles = GetFilesToParse(args.DiffPath);
+
+                if (!baseFiles.Any() || !diffFiles.Any())
+                    throw new ArgumentException($"Provided paths contained no {FullBdnJsonFileExtension} files.");
+
+                var baseResults = baseFiles.Select(resultFile => JsonConvert.DeserializeObject<BdnResult>(File.ReadAllText(resultFile)));
+                var diffResults = diffFiles.Select(resultFile => JsonConvert.DeserializeObject<BdnResult>(File.ReadAllText(resultFile)));
+
+                var benchmarkIdToDiffResults = diffResults.SelectMany(result => result.Benchmarks).ToDictionary(benchmarkResult => benchmarkResult.FullName, benchmarkResult => benchmarkResult);
+
+                return baseResults
+                    .SelectMany(result => result.Benchmarks)
+                    .ToDictionary(benchmarkResult => benchmarkResult.FullName, benchmarkResult => benchmarkResult) // we use ToDictionary to make sure the results have unique IDs
+                    .Where(baseResult => benchmarkIdToDiffResults.ContainsKey(baseResult.Key))
+                    .Select(baseResult => (baseResult.Key, baseResult.Value, benchmarkIdToDiffResults[baseResult.Key]));
+            }
+            else
+            {
+                Console.WriteLine("Invalid parameters, use --help to find out how to use this app");
+
+                return Array.Empty<(string id, Benchmark baseResult, Benchmark diffResult)>();
+            }
+        }
+
+        private static string[] GetFilesToParse(string path)
+        {
+            if (Directory.Exists(path))
+                return Directory.GetFiles(path, $"*{FullBdnJsonFileExtension}", SearchOption.AllDirectories);
+            else if (File.Exists(path) || !path.EndsWith(FullBdnJsonFileExtension))
+                return new[] { path };
+            else
+                throw new FileNotFoundException($"Provided path does NOT exist or is not a {path} file", path);
+        }
+
+        private static void PrintResult((string id, Benchmark baseResult, Benchmark diffResult, EquivalenceTestConclusion conclusion, double ratio) result)
+            => Console.WriteLine($"{result.ratio:0.00%} {result.id} {GetModalInfo(result.baseResult) ?? GetModalInfo(result.diffResult)}");
+
+        // code and magic values taken from BenchmarkDotNet.Analysers.MultimodalDistributionAnalyzer
+        // See http://www.brendangregg.com/FrequencyTrails/modes.html
+        private static string GetModalInfo(Benchmark benchmark)
+        {
+            if (benchmark.Statistics.N < 12) // not enough data to tell
+                return null;
+
+            double mValue = MathHelper.CalculateMValue(new BenchmarkDotNet.Mathematics.Statistics(benchmark.GetOriginalValues()));
+            if (mValue > 4.2)
+                return "[multimodal]";
+            else if (mValue > 3.2)
+                return "[bimodal]";
+            else if (mValue > 2.8)
+                return "[can have several modes]";
+
+            return null;
+        }
+    }
+}
\ No newline at end of file
diff --git a/src/tools/ResultsComparer/README.md b/src/tools/ResultsComparer/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..6c07eaa4fdb5f34a11d6a1f49e877d2d7b1fd614
GIT binary patch
literal 4134
zcmbuCYfl?j5QgV-rT&LiP$Y=32?k2jZ(OUYA&Ictd}u0czy=b7Enk52$G3f-nLXaM
zu~Um`<=yo;XYMoa%>MKDie+|cZ5!Iyx;D}?vjgkdt@W*C1MLj#d;89R4ZF4*KQYvk
z6J70RqU!1D>aC@FSMLvgs-w4oR@+)1+E=|@XeDumQ_t?S+Ov-CeVy&uy*OWr;!-Q;
zdftiVS~TeLMQdnO=vtlV_KUQDf6F;SnaQGI&It{trMK-Tr+uaMu!=JSNrwlvOiRja
zRlKgOsvWq8ZAd{P+P1Vd(he5i=GI>O?v+-Dwl<MB%Q^Y!QqnuR;DO~>^xo=DU6%h*
z=a8BFd@t^q{U*J}PBng}tONTXE?6>fuiv;m7rKvgZ<Muov>$9+r$hRlvgpKDo@eLC
zrQ#{9?nqusUTiCqh{_kYu!um_-r12gbdp@fTbb1qo6U(n{Hx`DAb<BfGc(1dCu=hK
zsjZ8+MQ%)IUnUO^OBvF%imr3%L`F45Rnd<M87X@b58~JrPvW;B8*cOu%^vg?=Msmm
zN07bgTtHJYGUTOvc)q;E;oEXXxfa*C{>Tdp(K<9td&DNHx-OfrA5U>lb<>c|tWYUO
z_ULg8q`;$@?bu6O*K^ZeE5gIc`)kD{{Toi|`n|jicaS3LlI*-v49N~^<Y}CUXnzu=
zh(oGOEW+=FXCnSVv!NLG#FKxwI@MI3;f<$JB2x8|ZCF-DH)JGP)6|)JC$g6Oqn43$
zAn$ydk@U%Pr>J#wiF#Xfeq2HUPJ@9+xN;er@;$5b;+k-bN`Qho#OqzhvWAf8gI0)I
zfx1m`Ys&6h`4f+TNh8rj6%QmG+sd_**^8wteUZ}@aau%Iwm4DEW;A-g%sED+!xD53
z8+haQO4mELqV4a*HN_Fi=yP<SzAmB|J+tssQ*<5e(+R8gS^j75X%^8JA16Fg$D>Q`
zduEsW-l2Gmq!)grt{QpIn&m<7i4IL|w3G`VQ1ntbgH0W+N3T!0hkZdsreYO2gS7+2
zzKHfb-_WrNRHfhDiw6}Gu@AgOuczIdu9eW_HrMG!@&h$LOa8euDfb+35nhpD)EgSN
zMAM((|DwH`Vz4IKl;^1vJopOPj>U_r7~EfWQb?TBG!k<NHBKdfTOe5Uk&ubcQ_aOn
z%DMSHrHILjtw^s^_X4#Nm7b`lktgMN8MkJ!EvBNPE|})7yfW(cRv7S6n0}gf#)@qT
z4c2uB)92;w(0-P8vA8O<?CZ%iL2vJRl?}9aCi<#)k}pN(XZDA86!4?SDxyGs1qbG=
zp6B$?{&XEH(qvsz$uj!<Dh<YxLAFs{F?XKXn+4jO>qN*y`y_8$@@lrbg#G&-9lF<7
z)i05uPo8<+&RUQEE84-9+7i4^Jcp_7u6)QGJrd_#SyFg()vNk1odMIf6*(?CahgWn
zYLEOOd!kdP6JQasV-BP9vI`#XO3NcneHHz<Z!4DReb?WQ?MJQb`TfvW_I0Xahfe#(
z?=?Ht%CY!ya?^LW#6KoUd~vMxy1Z88-j+1qQLO$u?~dI!cha3`Uwf@w>itn3+L;sU
zRrvt>=yZ4|P&zUfEUiyGuqCTP$LSuz)B}ovG%+F30gCU9z|5#o@*rXMo#IwZCh05S
zLMPB;eoe=;4i6?4P@737x-paFw`K8NHFB?jvJb9HLaWrn<{>u-9CZyo2V#|ZC0*xX
z{@VYrCRIPZEM;vVHuFl$D=f`}na<z=up%Z%Y-}p7RYf-D+caIAxX*5M61>^aFHP#@
z9r_2^m^gzN@LbMQ<rofg@9}Lw?2^yH?=p`5E{pgEnwHCSJ3SLa7o9Ys5c4v*!8|+G
S=^}p$3?p0cWPE=wj@iF*@~-v(

literal 0
HcmV?d00001

diff --git a/src/tools/ResultsComparer/ResultsComparer.csproj b/src/tools/ResultsComparer/ResultsComparer.csproj
new file mode 100644
index 00000000000..87f980a6423
--- /dev/null
+++ b/src/tools/ResultsComparer/ResultsComparer.csproj
@@ -0,0 +1,12 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>netcoreapp2.1</TargetFramework>
+    <LangVersion>latest</LangVersion>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="CommandLineParser" Version="2.3.0" />
+    <PackageReference Include="Newtonsoft.Json" Version="12.0.1" />
+    <PackageReference Include="BenchmarkDotNet" Version="0.11.3.889" />
+  </ItemGroup>
+</Project>
\ No newline at end of file
diff --git a/src/tools/ResultsComparer/ResultsComparer.sln b/src/tools/ResultsComparer/ResultsComparer.sln
new file mode 100644
index 00000000000..951a4d0fb5d
--- /dev/null
+++ b/src/tools/ResultsComparer/ResultsComparer.sln
@@ -0,0 +1,16 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ResultsComparer", "ResultsComparer.csproj", "{00859394-44F8-466B-8624-41578CA94009}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{00859394-44F8-466B-8624-41578CA94009}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{00859394-44F8-466B-8624-41578CA94009}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{00859394-44F8-466B-8624-41578CA94009}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{00859394-44F8-466B-8624-41578CA94009}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+EndGlobal

From ee39c1a11cec9889b87b8f5bde41fdfe9502e062 Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Tue, 4 Dec 2018 22:17:28 +0100
Subject: [PATCH 02/11] code review fixes

---
 .../ResultsComparer/CommandLineOptions.cs     |   2 +-
 .../ResultsComparer/DataTransferContracts.cs  |  10 ++++-----
 src/tools/ResultsComparer/Program.cs          |  20 +++++++++++++++---
 src/tools/ResultsComparer/README.md           | Bin 4134 -> 4172 bytes
 4 files changed, 23 insertions(+), 9 deletions(-)

diff --git a/src/tools/ResultsComparer/CommandLineOptions.cs b/src/tools/ResultsComparer/CommandLineOptions.cs
index d3a3e6a580b..c37b5a7bcaf 100644
--- a/src/tools/ResultsComparer/CommandLineOptions.cs
+++ b/src/tools/ResultsComparer/CommandLineOptions.cs
@@ -19,7 +19,7 @@ public class CommandLineOptions
         [Option("merged", HelpText = "Path to the folder/file with results merged for multiple jobs in the same file.")]
         public string MergedPath { get; set; }
 
-        [Option("treshold", Required = true, HelpText = "Threshold for Statistical Test. Examples: 5%, 10ms, 100ns, 1s.")]
+        [Option("threshold", Required = true, HelpText = "Threshold for Statistical Test. Examples: 5%, 10ms, 100ns, 1s.")]
         public string StatisticalTestThreshold { get; set; }
 
         [Option("top", HelpText = "Filter the diff to top/bottom N results. Optional.")]
diff --git a/src/tools/ResultsComparer/DataTransferContracts.cs b/src/tools/ResultsComparer/DataTransferContracts.cs
index 6c3eb52b888..3f02cbf8d90 100644
--- a/src/tools/ResultsComparer/DataTransferContracts.cs
+++ b/src/tools/ResultsComparer/DataTransferContracts.cs
@@ -20,13 +20,13 @@ public class HostEnvironmentInfo
         public string BenchmarkDotNetVersion { get; set; }
         public string OsVersion { get; set; }
         public string ProcessorName { get; set; }
-        public int PhysicalProcessorCount { get; set; }
-        public int PhysicalCoreCount { get; set; }
-        public int LogicalCoreCount { get; set; }
+        public int? PhysicalProcessorCount { get; set; }
+        public int? PhysicalCoreCount { get; set; }
+        public int? LogicalCoreCount { get; set; }
         public string RuntimeVersion { get; set; }
         public string Architecture { get; set; }
-        public bool HasAttachedDebugger { get; set; }
-        public bool HasRyuJit { get; set; }
+        public bool? HasAttachedDebugger { get; set; }
+        public bool? HasRyuJit { get; set; }
         public string Configuration { get; set; }
         public string JitModules { get; set; }
         public string DotNetCliVersion { get; set; }
diff --git a/src/tools/ResultsComparer/Program.cs b/src/tools/ResultsComparer/Program.cs
index 7f7008ade34..1a00d616dde 100644
--- a/src/tools/ResultsComparer/Program.cs
+++ b/src/tools/ResultsComparer/Program.cs
@@ -71,7 +71,7 @@ private static void Compare(CommandLineOptions args)
             if (!string.IsNullOrEmpty(args.MergedPath))
             {
                 return GetFilesToParse(args.MergedPath)
-                    .Select(resultFile => JsonConvert.DeserializeObject<BdnResult>(File.ReadAllText(resultFile)))
+                    .Select(ReadFromFile)
                     .SelectMany(result => result.Benchmarks)
                     .GroupBy(result => result.FullName)
                         .SelectMany(sameKey => sameKey
@@ -86,8 +86,8 @@ private static void Compare(CommandLineOptions args)
                 if (!baseFiles.Any() || !diffFiles.Any())
                     throw new ArgumentException($"Provided paths contained no {FullBdnJsonFileExtension} files.");
 
-                var baseResults = baseFiles.Select(resultFile => JsonConvert.DeserializeObject<BdnResult>(File.ReadAllText(resultFile)));
-                var diffResults = diffFiles.Select(resultFile => JsonConvert.DeserializeObject<BdnResult>(File.ReadAllText(resultFile)));
+                var baseResults = baseFiles.Select(ReadFromFile);
+                var diffResults = diffFiles.Select(ReadFromFile);
 
                 var benchmarkIdToDiffResults = diffResults.SelectMany(result => result.Benchmarks).ToDictionary(benchmarkResult => benchmarkResult.FullName, benchmarkResult => benchmarkResult);
 
@@ -135,5 +135,19 @@ private static string GetModalInfo(Benchmark benchmark)
 
             return null;
         }
+
+        private static BdnResult ReadFromFile(string resultFilePath)
+        {
+            try
+            {
+                return JsonConvert.DeserializeObject<BdnResult>(File.ReadAllText(resultFilePath));
+            }
+            catch (JsonSerializationException)
+            {
+                Console.WriteLine($"Exception while reading the {resultFilePath} file.");
+
+                throw;
+            }
+        }
     }
 }
\ No newline at end of file
diff --git a/src/tools/ResultsComparer/README.md b/src/tools/ResultsComparer/README.md
index 6c07eaa4fdb5f34a11d6a1f49e877d2d7b1fd614..b820edf95868ce7c096f77f34a168e91b0b038e9 100644
GIT binary patch
delta 42
ycmZ3ca7JMR2Mc4yW_Fg1tilQm<qVk&B@7u13Jk>zr3}do857q_Z$8AC!Uh2FrwdU4

delta 17
ZcmX@3uuNeC2g_zImJO_%?{Vg_0RTAd1@iy^


From feb22c89437855dddd2495af359eb0251d658aea Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Wed, 5 Dec 2018 18:19:55 +0100
Subject: [PATCH 03/11] don't try to parse results for benchmarks which has
 failed to execute

---
 src/tools/ResultsComparer/Program.cs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/tools/ResultsComparer/Program.cs b/src/tools/ResultsComparer/Program.cs
index 1a00d616dde..a0311484e13 100644
--- a/src/tools/ResultsComparer/Program.cs
+++ b/src/tools/ResultsComparer/Program.cs
@@ -45,7 +45,8 @@ private static void Compare(CommandLineOptions args)
 
         private static IEnumerable<(string id, Benchmark baseResult, Benchmark diffResult, EquivalenceTestConclusion conclusion, double ratio)> GetNotSameResults(CommandLineOptions args, Threshold userProvidedThreshold)
         {
-            foreach (var pair in ReadResults(args))
+            foreach (var pair in ReadResults(args)
+                .Where(result => result.baseResult.Statistics != null && result.diffResult.Statistics != null)) // failures
             {
                 var baseValues = pair.baseResult.GetOriginalValues();
                 var diffValues = pair.diffResult.GetOriginalValues();

From 22d4852805dcd41db1eaa2d337b367f4dca22762 Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Wed, 5 Dec 2018 22:40:46 +0100
Subject: [PATCH 04/11] print nice tables

---
 src/tools/ResultsComparer/Program.cs          | 55 ++++++++++++-------
 .../ResultsComparer/ResultsComparer.csproj    |  1 +
 2 files changed, 37 insertions(+), 19 deletions(-)

diff --git a/src/tools/ResultsComparer/Program.cs b/src/tools/ResultsComparer/Program.cs
index a0311484e13..1bc140c180c 100644
--- a/src/tools/ResultsComparer/Program.cs
+++ b/src/tools/ResultsComparer/Program.cs
@@ -10,6 +10,7 @@
 using BenchmarkDotNet.Mathematics.StatisticalTesting;
 using CommandLine;
 using DataTransferContracts;
+using MarkdownLog;
 using Newtonsoft.Json;
 
 namespace ResultsComparer
@@ -31,19 +32,12 @@ private static void Compare(CommandLineOptions args)
             }
 
             var notSame = GetNotSameResults(args, userProvidedThreshold).ToArray();
-            var takeCount = args.TopCount.HasValue ? args.TopCount.Value : int.MaxValue;
 
-            Console.WriteLine("SLOWER:");
-            foreach (var result in notSame.Where(result => result.conclusion == EquivalenceTestConclusion.Slower).OrderBy(pair => pair.ratio).Take(takeCount))
-                PrintResult(result);
-
-            Console.WriteLine();
-            Console.WriteLine("FASTER:");
-            foreach (var result in notSame.Where(result => result.conclusion == EquivalenceTestConclusion.Faster).OrderByDescending(pair => pair.ratio).Take(takeCount))
-                PrintResult(result);
+            PrintTable(notSame, EquivalenceTestConclusion.Slower, args);
+            PrintTable(notSame, EquivalenceTestConclusion.Faster, args);
         }
 
-        private static IEnumerable<(string id, Benchmark baseResult, Benchmark diffResult, EquivalenceTestConclusion conclusion, double ratio)> GetNotSameResults(CommandLineOptions args, Threshold userProvidedThreshold)
+        private static IEnumerable<(string id, Benchmark baseResult, Benchmark diffResult, EquivalenceTestConclusion conclusion)> GetNotSameResults(CommandLineOptions args, Threshold userProvidedThreshold)
         {
             foreach (var pair in ReadResults(args)
                 .Where(result => result.baseResult.Statistics != null && result.diffResult.Statistics != null)) // failures
@@ -60,12 +54,33 @@ private static void Compare(CommandLineOptions args)
                 if (noiseResult.Conclusion == EquivalenceTestConclusion.Same)
                     continue;
 
-                var ratio = (1.0 - pair.diffResult.Statistics.Median / pair.baseResult.Statistics.Median);
-
-                yield return (pair.id, pair.baseResult, pair.diffResult, userTresholdResult.Conclusion, ratio);
+                yield return (pair.id, pair.baseResult, pair.diffResult, userTresholdResult.Conclusion);
             }
         }
 
+        private static void PrintTable((string id, Benchmark baseResult, Benchmark diffResult, EquivalenceTestConclusion conclusion)[] notSame, EquivalenceTestConclusion conclusion, CommandLineOptions args)
+        {
+            var data = notSame
+                .Where(result => result.conclusion == conclusion)
+                .OrderByDescending(result => GetRatio(conclusion, result.baseResult, result.diffResult))
+                .Take(args.TopCount ?? int.MaxValue)
+                .Select(result => new {
+                    Id = result.id.Length > 100 ? result.id.Substring(0, 100) : result.id,
+                    DisplayValue = GetRatio(conclusion, result.baseResult, result.diffResult),              
+                    BaseMedian = result.baseResult.Statistics.Median,
+                    DiffMedian = result.diffResult.Statistics.Median,
+                    Modality = GetModalInfo(result.baseResult) ?? GetModalInfo(result.diffResult)
+                })
+                .ToArray();
+
+            var table = data.ToMarkdownTable().WithHeaders(conclusion.ToString(), conclusion == EquivalenceTestConclusion.Faster ? "base/diff" : "diff/base", "Base Median (ns)", "Diff Median (ns)", "Modality");
+
+            foreach (var line in table.ToMarkdown().Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries))
+                Console.WriteLine($"| {line.TrimStart()}|"); // the table starts with \t and does not end with '|' and it looks bad so we fix it
+
+            Console.WriteLine();
+        }
+
         private static IEnumerable<(string id, Benchmark baseResult, Benchmark diffResult)> ReadResults(CommandLineOptions args)
         {
             // this is the case where BDN run one benchmark for multiple jobs, for example for two runtimes using -r netcoreapp2.1 netcoreapp2.2
@@ -116,9 +131,6 @@ private static string[] GetFilesToParse(string path)
                 throw new FileNotFoundException($"Provided path does NOT exist or is not a {path} file", path);
         }
 
-        private static void PrintResult((string id, Benchmark baseResult, Benchmark diffResult, EquivalenceTestConclusion conclusion, double ratio) result)
-            => Console.WriteLine($"{result.ratio:0.00%} {result.id} {GetModalInfo(result.baseResult) ?? GetModalInfo(result.diffResult)}");
-
         // code and magic values taken from BenchmarkDotNet.Analysers.MultimodalDistributionAnalyzer
         // See http://www.brendangregg.com/FrequencyTrails/modes.html
         private static string GetModalInfo(Benchmark benchmark)
@@ -128,15 +140,20 @@ private static string GetModalInfo(Benchmark benchmark)
 
             double mValue = MathHelper.CalculateMValue(new BenchmarkDotNet.Mathematics.Statistics(benchmark.GetOriginalValues()));
             if (mValue > 4.2)
-                return "[multimodal]";
+                return "multimodal";
             else if (mValue > 3.2)
-                return "[bimodal]";
+                return "bimodal";
             else if (mValue > 2.8)
-                return "[can have several modes]";
+                return "can have several modes";
 
             return null;
         }
 
+        private static double GetRatio(EquivalenceTestConclusion conclusion, Benchmark baseResult, Benchmark diffResult)
+            => conclusion == EquivalenceTestConclusion.Faster
+                ? baseResult.Statistics.Median / diffResult.Statistics.Median
+                : diffResult.Statistics.Median / baseResult.Statistics.Median;
+
         private static BdnResult ReadFromFile(string resultFilePath)
         {
             try
diff --git a/src/tools/ResultsComparer/ResultsComparer.csproj b/src/tools/ResultsComparer/ResultsComparer.csproj
index 87f980a6423..1305bf30dc1 100644
--- a/src/tools/ResultsComparer/ResultsComparer.csproj
+++ b/src/tools/ResultsComparer/ResultsComparer.csproj
@@ -6,6 +6,7 @@
   </PropertyGroup>
   <ItemGroup>
     <PackageReference Include="CommandLineParser" Version="2.3.0" />
+    <PackageReference Include="MarkdownLog.NS20" Version="0.10.1" />
     <PackageReference Include="Newtonsoft.Json" Version="12.0.1" />
     <PackageReference Include="BenchmarkDotNet" Version="0.11.3.889" />
   </ItemGroup>

From 8b072d7beb8949e4cd7c9914bdb0574364cf2313 Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Wed, 5 Dec 2018 23:13:00 +0100
Subject: [PATCH 05/11] allow the user to specify noise threshold

---
 .../ResultsComparer/CommandLineOptions.cs      |  3 +++
 src/tools/ResultsComparer/Program.cs           | 18 ++++++++++--------
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/src/tools/ResultsComparer/CommandLineOptions.cs b/src/tools/ResultsComparer/CommandLineOptions.cs
index c37b5a7bcaf..c0edf4c7425 100644
--- a/src/tools/ResultsComparer/CommandLineOptions.cs
+++ b/src/tools/ResultsComparer/CommandLineOptions.cs
@@ -22,6 +22,9 @@ public class CommandLineOptions
         [Option("threshold", Required = true, HelpText = "Threshold for Statistical Test. Examples: 5%, 10ms, 100ns, 1s.")]
         public string StatisticalTestThreshold { get; set; }
 
+        [Option("noise", HelpText = "Noise threshold for Statistical Test. The difference for 1.0ns and 1.1ns is 10%, but it's just a noise. Examples: 0.5ns 1ns.", Default = "0.3ns" )]
+        public string NoiseThreshold { get; set; }
+
         [Option("top", HelpText = "Filter the diff to top/bottom N results. Optional.")]
         public int? TopCount { get; set; }
 
diff --git a/src/tools/ResultsComparer/Program.cs b/src/tools/ResultsComparer/Program.cs
index 1bc140c180c..b4e79dbcb1c 100644
--- a/src/tools/ResultsComparer/Program.cs
+++ b/src/tools/ResultsComparer/Program.cs
@@ -19,25 +19,28 @@ public class Program
     {
         private const string FullBdnJsonFileExtension = "full.json";
 
-        private static readonly Threshold NoiseThreshold = Threshold.Create(ThresholdUnit.Nanoseconds, 0.3);
-
         public static void Main(string[] args) => Parser.Default.ParseArguments<CommandLineOptions>(args).WithParsed(Compare);
 
         private static void Compare(CommandLineOptions args)
         {
-            if (!Threshold.TryParse(args.StatisticalTestThreshold, out var userProvidedThreshold))
+            if (!Threshold.TryParse(args.StatisticalTestThreshold, out var testThreshold))
             {
                 Console.WriteLine($"Invalid Threshold {args.StatisticalTestThreshold}. Examples: 5%, 10ms, 100ns, 1s.");
                 return;
             }
+            if (!Threshold.TryParse(args.NoiseThreshold, out var noiseThreshold))
+            {
+                Console.WriteLine($"Invalid Noise Threshold {args.NoiseThreshold}. Examples: 0.3ns 1ns.");
+                return;
+            }
 
-            var notSame = GetNotSameResults(args, userProvidedThreshold).ToArray();
+            var notSame = GetNotSameResults(args, testThreshold, noiseThreshold).ToArray();
 
             PrintTable(notSame, EquivalenceTestConclusion.Slower, args);
             PrintTable(notSame, EquivalenceTestConclusion.Faster, args);
         }
 
-        private static IEnumerable<(string id, Benchmark baseResult, Benchmark diffResult, EquivalenceTestConclusion conclusion)> GetNotSameResults(CommandLineOptions args, Threshold userProvidedThreshold)
+        private static IEnumerable<(string id, Benchmark baseResult, Benchmark diffResult, EquivalenceTestConclusion conclusion)> GetNotSameResults(CommandLineOptions args, Threshold testThreshold, Threshold noiseThreshold)
         {
             foreach (var pair in ReadResults(args)
                 .Where(result => result.baseResult.Statistics != null && result.diffResult.Statistics != null)) // failures
@@ -45,12 +48,11 @@ private static void Compare(CommandLineOptions args)
                 var baseValues = pair.baseResult.GetOriginalValues();
                 var diffValues = pair.diffResult.GetOriginalValues();
 
-                var userTresholdResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, userProvidedThreshold);
+                var userTresholdResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, testThreshold);
                 if (userTresholdResult.Conclusion == EquivalenceTestConclusion.Same)
                     continue;
 
-                // the difference for 1.0ns and 1.1ns is 10%, but it's just a noise. So we test against 0.3ns to filter out the nano-benchmarks noise
-                var noiseResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, NoiseThreshold);
+                var noiseResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, noiseThreshold);
                 if (noiseResult.Conclusion == EquivalenceTestConclusion.Same)
                     continue;
 

From f0ab62b09bbdb69ae6d55632f34165355952f0b9 Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Wed, 5 Dec 2018 23:28:32 +0100
Subject: [PATCH 06/11] always use invariant culture

---
 src/tools/ResultsComparer/Program.cs | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/tools/ResultsComparer/Program.cs b/src/tools/ResultsComparer/Program.cs
index b4e79dbcb1c..3d6159533dc 100644
--- a/src/tools/ResultsComparer/Program.cs
+++ b/src/tools/ResultsComparer/Program.cs
@@ -4,8 +4,10 @@
 
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using System.IO;
 using System.Linq;
+using System.Threading;
 using BenchmarkDotNet.Mathematics;
 using BenchmarkDotNet.Mathematics.StatisticalTesting;
 using CommandLine;
@@ -19,7 +21,13 @@ public class Program
     {
         private const string FullBdnJsonFileExtension = "full.json";
 
-        public static void Main(string[] args) => Parser.Default.ParseArguments<CommandLineOptions>(args).WithParsed(Compare);
+        public static void Main(string[] args)
+        {
+            // we print a lot of numbers here and we want to make it always in invariant way
+            Thread.CurrentThread.CurrentCulture = CultureInfo.InvariantCulture;
+
+            Parser.Default.ParseArguments<CommandLineOptions>(args).WithParsed(Compare);
+        }
 
         private static void Compare(CommandLineOptions args)
         {

From 562b1c350aebad719b608f10c4059f8cd312fe57 Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Wed, 5 Dec 2018 23:31:30 +0100
Subject: [PATCH 07/11] keep it simple

---
 .../ResultsComparer/CommandLineOptions.cs     |  7 +--
 src/tools/ResultsComparer/Program.cs          | 44 +++++--------------
 2 files changed, 14 insertions(+), 37 deletions(-)

diff --git a/src/tools/ResultsComparer/CommandLineOptions.cs b/src/tools/ResultsComparer/CommandLineOptions.cs
index c0edf4c7425..649a342693c 100644
--- a/src/tools/ResultsComparer/CommandLineOptions.cs
+++ b/src/tools/ResultsComparer/CommandLineOptions.cs
@@ -16,9 +16,6 @@ public class CommandLineOptions
         [Option("diff", HelpText = "Path to the folder/file with diff results.")]
         public string DiffPath { get; set; }
 
-        [Option("merged", HelpText = "Path to the folder/file with results merged for multiple jobs in the same file.")]
-        public string MergedPath { get; set; }
-
         [Option("threshold", Required = true, HelpText = "Threshold for Statistical Test. Examples: 5%, 10ms, 100ns, 1s.")]
         public string StatisticalTestThreshold { get; set; }
 
@@ -35,10 +32,10 @@ public static IEnumerable<Example> Examples
             {
                 yield return new Example(@"Compare the results stored in 'C:\results\win' (base) vs 'C:\results\unix' (diff) using 5% threshold.",
                     new CommandLineOptions { BasePath = @"C:\results\win", DiffPath = @"C:\results\unix", StatisticalTestThreshold = "5%" });
-                yield return new Example(@"Compare the results stored in 'C:\results\21_vs_22' (multiple jobs per single benchmark run using 1ms threshold.",
-                    new CommandLineOptions { MergedPath = @"C:\results\21_vs_22", StatisticalTestThreshold = "1ms" });
                 yield return new Example(@"Compare the results stored in 'C:\results\win' (base) vs 'C:\results\unix' (diff) using 5% threshold and show only top/bottom 10 results.",
                     new CommandLineOptions { BasePath = @"C:\results\win", DiffPath = @"C:\results\unix", StatisticalTestThreshold = "5%", TopCount = 10 });
+                yield return new Example(@"Compare the results stored in 'C:\results\win' (base) vs 'C:\results\unix' (diff) using 5% threshold and 0.5ns noise filter.",
+                    new CommandLineOptions { BasePath = @"C:\results\win", DiffPath = @"C:\results\unix", StatisticalTestThreshold = "5%", NoiseThreshold = "0.5ns" });
             }
         }
     }
diff --git a/src/tools/ResultsComparer/Program.cs b/src/tools/ResultsComparer/Program.cs
index 3d6159533dc..009b9188723 100644
--- a/src/tools/ResultsComparer/Program.cs
+++ b/src/tools/ResultsComparer/Program.cs
@@ -93,42 +93,22 @@ private static void PrintTable((string id, Benchmark baseResult, Benchmark diffR
 
         private static IEnumerable<(string id, Benchmark baseResult, Benchmark diffResult)> ReadResults(CommandLineOptions args)
         {
-            // this is the case where BDN run one benchmark for multiple jobs, for example for two runtimes using -r netcoreapp2.1 netcoreapp2.2
-            if (!string.IsNullOrEmpty(args.MergedPath))
-            {
-                return GetFilesToParse(args.MergedPath)
-                    .Select(ReadFromFile)
-                    .SelectMany(result => result.Benchmarks)
-                    .GroupBy(result => result.FullName)
-                        .SelectMany(sameKey => sameKey
-                            .Select(group => (group.FullName, sameKey.First(), group)) // first is always the base
-                            .Skip(1)); // we skip the first entry in the group, we want to do 1st vs 2nd, 1st vs 3rd etc..
-            }
-            else if(!string.IsNullOrEmpty(args.BasePath) && !string.IsNullOrEmpty(args.DiffPath))
-            {
-                var baseFiles = GetFilesToParse(args.BasePath);
-                var diffFiles = GetFilesToParse(args.DiffPath);
-
-                if (!baseFiles.Any() || !diffFiles.Any())
-                    throw new ArgumentException($"Provided paths contained no {FullBdnJsonFileExtension} files.");
+            var baseFiles = GetFilesToParse(args.BasePath);
+            var diffFiles = GetFilesToParse(args.DiffPath);
 
-                var baseResults = baseFiles.Select(ReadFromFile);
-                var diffResults = diffFiles.Select(ReadFromFile);
+            if (!baseFiles.Any() || !diffFiles.Any())
+                throw new ArgumentException($"Provided paths contained no {FullBdnJsonFileExtension} files.");
 
-                var benchmarkIdToDiffResults = diffResults.SelectMany(result => result.Benchmarks).ToDictionary(benchmarkResult => benchmarkResult.FullName, benchmarkResult => benchmarkResult);
+            var baseResults = baseFiles.Select(ReadFromFile);
+            var diffResults = diffFiles.Select(ReadFromFile);
 
-                return baseResults
-                    .SelectMany(result => result.Benchmarks)
-                    .ToDictionary(benchmarkResult => benchmarkResult.FullName, benchmarkResult => benchmarkResult) // we use ToDictionary to make sure the results have unique IDs
-                    .Where(baseResult => benchmarkIdToDiffResults.ContainsKey(baseResult.Key))
-                    .Select(baseResult => (baseResult.Key, baseResult.Value, benchmarkIdToDiffResults[baseResult.Key]));
-            }
-            else
-            {
-                Console.WriteLine("Invalid parameters, use --help to find out how to use this app");
+            var benchmarkIdToDiffResults = diffResults.SelectMany(result => result.Benchmarks).ToDictionary(benchmarkResult => benchmarkResult.FullName, benchmarkResult => benchmarkResult);
 
-                return Array.Empty<(string id, Benchmark baseResult, Benchmark diffResult)>();
-            }
+            return baseResults
+                .SelectMany(result => result.Benchmarks)
+                .ToDictionary(benchmarkResult => benchmarkResult.FullName, benchmarkResult => benchmarkResult) // we use ToDictionary to make sure the results have unique IDs
+                .Where(baseResult => benchmarkIdToDiffResults.ContainsKey(baseResult.Key))
+                .Select(baseResult => (baseResult.Key, baseResult.Value, benchmarkIdToDiffResults[baseResult.Key]));
         }
 
         private static string[] GetFilesToParse(string path)

From 3ed0007f830e1447859ac58de31c25948e514cb9 Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Wed, 5 Dec 2018 23:41:30 +0100
Subject: [PATCH 08/11] update docs

---
 src/tools/ResultsComparer/README.md | Bin 4172 -> 4426 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)

diff --git a/src/tools/ResultsComparer/README.md b/src/tools/ResultsComparer/README.md
index b820edf95868ce7c096f77f34a168e91b0b038e9..dee77c6883fccf59e8892f2314c871aea5b79ab6 100644
GIT binary patch
literal 4426
zcmdUzT~Av_5QgWvQvbsW6qTaJV4RRNcj2Q_q>`ZGsuxWa8*Br`V9Or~mGax$zR!;L
z>^Zh^i7#4JN7(1=nVp?^=baDp*Pojf>?iBm*rwLEiN1p!+Q4pYXl)zm&B(sCui{_R
zuIxHWjP;E~U+;r<4RrN&*VboW_n)FvPj@4&cC|jXKXiAYm82O>1H04ez<T-|N_Jo$
zrMaUW9j%<}`%ZhVv<F*mw1!35)>>iP8(D$=cGSquAdki~O;}hxy=$-H-fgYBSICTX
zIy&&BdQz}0>AJL<-k{xm;}lNB+P2mvdV|NeGjFfr+e@vE?c3tKdCdu@j!y6CLI<AX
z(K|bfyUXYQB00`XIDeG(U_Z;QY1}nk^Q=QVl@>f1MbxjOKNtF(&d?}Zac}$fOwxY(
zYq97guB_+hB%TUVc-_-^ZK2o|lZeVq^e{)DW^e7-nv#U8pcQOSv8fmK@U%C>0sg*@
z%nXXlK;8u5)YV1YoEug41tB~xF{EWrb)Cy57}e6Qr}`s9CSp(0K^*(iN&I%?!?m8U
z?33<XE@|k;2=cDV1uTV;eqQ3k`sXDLtIxqUx8rK&0VleTi|>ez=k=Mq0s*k%Gi6Rw
zUb8~39NT@2q}K{C4ffog*mwG_+pc0fF79^~mGm?&GW(ryMmx2_GZTJZDvI!fELn^c
zF&!3h@<^oI#3PW;*f;}x&yJ!zkWQX%CDjtsKx0*$?j#EanHhNnZp@?mWe7Is>`m3-
zB6^WxQ5V{dU#>{3)gz|PXGh$zr&f?@F!LMP9_)9m5H<FbAIYbOqPQ#V&u7~G6eP2?
z)X27U(p9Ju#BLgG2CE=zYu1SSa}__6RO%}*rQ9!wS&CgX8X?;4b%2bfIuf}<vF-g_
z<le_@%+-_FdIft(ep;%lNP`k5T*fo%;xentqMSMdwS^tv0E_#vJ2X|>Pqji8WCU8$
z){@`1;wROBDnHR4uZts{j&J2^6l~*Bo^H(cW!^uuuk3N!HJRa<Dg<^vnQ8RQaZj+>
zec;aP*|u^BFS@F2+pz+qII`0x2YUKY7g6*nBqP<*uAbgAThzo;pBomll1byf(2-^c
z=8glssZJ6{(W18z*Th@rWmN~=JgrlI&SgVcGu6J#bLSrZ`7W4>S8xVzM~Z!p_NHx0
zf0^~mDh()pj&eR*`HT^?^6y^Qkk~_=^lliUZ3o5ENyO(`&oFgMpUgfkbtDv+aDrw9
zP+Tr`BAhzfPnVnsug{T3oDT2*yUTr8JpWZhU~#>=vc42M69-O(+eirhD16RUZN1Zu
z#i;|~HBjZai8IkqxFy+|RzS1AM27g#NsslmtMA-VJ)d`D>@v@YLcZuQ`>J1#OkS_v
zeJ`m!U8|((nk8%YLfV!~WdzRT8GQ<CYZU~NzaNxQ)GM#j$Z&saAA|!F`AE~?NY|G1
z&}B9XEgNwbxrnR3sxJopF{$|7Zv~IDe<1KowQ#xBNp4?Kb$j^_QZFL{58>npBp&=A
zKO6-r)cP0kOqsk>WVd_sf68gMxSDZ1t)F#|(^byR$1u~G*N$Bv-}5VmjBVJ_0!Lt2
zV<p2H${T*~yjUr<C;oYkH5Sfr{$z%|i8b20G#xDUK&Han1Im*qS(3W?{B%afp@WH<
zicao!WN)z5=kTk4qKd{ZtS!xL=<wh4Owc|LEdMV7otj$4oJrT^7kSo|>|Ff+0-fs1
IJs~Im23<*%8vp<R

literal 4172
zcmbuC+fEx-6o%JzrM|-`C=x{21Ouh%9ml9@NFq#|i>AT`Y#=e%@&QO6zRCZsHQO^b
zc4ASDJTu;Vuk*hS^Yy<a%k0=%Hn5R(Y^Y~ud)BoZ>sizK+UeWR_LKik?8>hF#6V9@
zbhMv|s;jG`x2EnLz2Ez(w%+<$ZE1aAU-WjamBblNUAxt4*V?-Gbhc}E;(Q^B3$2{#
zc`KSL(V)v`t)Wq&Yjv#K@6rPPP3H(@CW{6+Cp4Ut-m>4E_J!8ND$evJ9Uj;+DJipM
z@w&9CcHkbiAq9zO+tk`nJ6L>^TYKfZms%az%2?hs=j5jgNpI_d2bN>eJF7c&S^fu|
zLuT^xow#TAr}P>*)#!<`_Uyg5U`gM-e(m<0>psf8QP$$Yez8rR4(U5W(UGq_&Ca1q
z#Zy?_mb|9C*b<V6%4fH*h(Oie+JT+uBv{2;nbj1V^|3zutL1(GzdMf1OmXSTnoNFb
z=^}1{jY;%n^6;R9kcL%sok=GU)eu!hKQd$}^dug{u_K<uZ%sB_>mQok>n+YD4jqpm
zdy`l|QxF;Q5+0r|FLC(39CQmku5t^I7&boMBRY}Sby<Z6@D=xzIVZB36>{ak9z2p!
zEAVM%TlU;m^<1}?it%9l{!&p%{|4jCekVV}omvr@2|h0sMesqEJdPAG?TzCUkx03T
zMR>mOO&}m>))eKgc=GQ?ry4>U-gq1*VpT8MhGpgYhKwX@k~(wm1a3hf*^Hz;dFSJl
zq>qlCBHPg=GH%}aQK=4anktBdOP8@O-?KU^uBnod3s8`Wc)jD*>_oNby;g`?QFj~S
z){xyd@+TglDh)*wdEA$DY%6C^X3rL~^jS_<#AzN~+2TYs%>9@Ujo#05juGjw1f9bM
z-uS)H_13Lu`8#n<afC8D9X+V0izr6dEIic^U0eI~!m53e|Ji$tB0A&4m`Cb)^vPXE
zcDeKIi^ouU;aBqNB=1_&Jm@~rqsfh?us{upZVG3rQ(NoN?GyH}FQ`B&R)HC;?JM?0
zv}gH-o>kOUI^LalkTDVasJQ6$xS!LvQZ>2Bb-I@PsGFZ8|4f>cyAG8RUV$+34UL<k
z>5V(UyuF%Yup-)o^VAFOeFd~*@w_ai@-I3oB+h9biAjVUCljb!RIunIArqgcoQsu&
zxgU2Gun1PBt5gxUC0o+&v3rVqifWJ~Ojf6q2j$u}%{gYLB6pbiE<Jnd_C|H$gX;cq
z-ajk0p^CApJ9U2+wg>iF5x^@|Rn(rI%p6Pr9nZwR_D)4#6;Ci#WJhLy>uk(?1@ed#
z7z_?fV_nDn!2WR^E7D|Dv&$m-{2>kSE{G%pV>&&xS97#E(}|FW_DSBR<kf6@0sD77
zI&`=%%5fq?FFkc^Pg{@wE84-9+5)_f+*4G5j(o_pJrw6{SyFg(*)#uNouTe+Dso(O
z>ol{x(H{5#J<-k62{1_PnAYgd>{7G0rR9NU!-{^~Hx<kDzU}Xa_N!KQ{J!riyE;{|
zeW!ip_nIAQ<xu=Mx$e6g;ve%SzBtr+U0y4&w;|296ssTO?$CX6E8QcP$dwDdKgdH{
zGh)3gA7CHd5bs1K59C5wAA4X!R)vm}eTK=1N(XvlilR#tUnNmjBS*nOs_VCkTQR+)
zZ-sN6K#$ou9Wy;Vm};o(OheJJnK!>Li?6St;!1^C);t2ARMAqOn^oPZ(vjEH@Th8K
zUP;$kePQK)Sd+4!ewR=iRh^lp=^2)0#!P4M0JS2fOl)i@u2n@gX5ch|9J$Y~b&{E8
zO}{kHmv`tfpfPc#V!(4TPnBah$i2rGMA_%m?{XdeOBV5^G%1$}cyexrF1l?*A!cf@
W!OT0-=>k7R4FfHBGQQI1$Lt#}y|zC9


From e2dc48dc248fc34b7f8cac21feb2d38e9a0e5214 Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Wed, 5 Dec 2018 23:59:23 +0100
Subject: [PATCH 09/11] allow to export the results to CSV

---
 .../ResultsComparer/CommandLineOptions.cs     |   4 ++++
 src/tools/ResultsComparer/Program.cs          |  22 ++++++++++++++++++
 src/tools/ResultsComparer/README.md           | Bin 4426 -> 4532 bytes
 3 files changed, 26 insertions(+)

diff --git a/src/tools/ResultsComparer/CommandLineOptions.cs b/src/tools/ResultsComparer/CommandLineOptions.cs
index 649a342693c..61e3d7f5c51 100644
--- a/src/tools/ResultsComparer/CommandLineOptions.cs
+++ b/src/tools/ResultsComparer/CommandLineOptions.cs
@@ -3,6 +3,7 @@
 // See the LICENSE file in the project root for more information.
 
 using System.Collections.Generic;
+using System.IO;
 using CommandLine;
 using CommandLine.Text;
 
@@ -25,6 +26,9 @@ public class CommandLineOptions
         [Option("top", HelpText = "Filter the diff to top/bottom N results. Optional.")]
         public int? TopCount { get; set; }
 
+        [Option("csv", HelpText = "Path to exported CSV results. Optional.")]
+        public FileInfo CsvPath { get; set; }
+
         [Usage(ApplicationAlias = "")]
         public static IEnumerable<Example> Examples
         {
diff --git a/src/tools/ResultsComparer/Program.cs b/src/tools/ResultsComparer/Program.cs
index 009b9188723..c9645ca6eb4 100644
--- a/src/tools/ResultsComparer/Program.cs
+++ b/src/tools/ResultsComparer/Program.cs
@@ -46,6 +46,8 @@ private static void Compare(CommandLineOptions args)
 
             PrintTable(notSame, EquivalenceTestConclusion.Slower, args);
             PrintTable(notSame, EquivalenceTestConclusion.Faster, args);
+
+            ExportToCsv(notSame, args.CsvPath);
         }
 
         private static IEnumerable<(string id, Benchmark baseResult, Benchmark diffResult, EquivalenceTestConclusion conclusion)> GetNotSameResults(CommandLineOptions args, Threshold testThreshold, Threshold noiseThreshold)
@@ -111,6 +113,26 @@ private static void PrintTable((string id, Benchmark baseResult, Benchmark diffR
                 .Select(baseResult => (baseResult.Key, baseResult.Value, benchmarkIdToDiffResults[baseResult.Key]));
         }
 
+        private static void ExportToCsv((string id, Benchmark baseResult, Benchmark diffResult, EquivalenceTestConclusion conclusion)[] notSame, FileInfo csvPath)
+        {
+            if (csvPath == null)
+                return;
+
+            if (csvPath.Exists)
+                csvPath.Delete();
+
+            using (var textWriter = csvPath.CreateText())
+            {
+                foreach (var result in notSame)
+                {
+                    textWriter.WriteLine($"\"{result.id.Replace("\"", "\"\"")}\";base;{result.conclusion};{string.Join(';', result.baseResult.GetOriginalValues())}");
+                    textWriter.WriteLine($"\"{result.id.Replace("\"", "\"\"")}\";diff;{result.conclusion};{string.Join(';', result.diffResult.GetOriginalValues())}");
+                }
+            }
+
+            Console.WriteLine($"CSV results exported to {csvPath.FullName}");
+        }
+
         private static string[] GetFilesToParse(string path)
         {
             if (Directory.Exists(path))
diff --git a/src/tools/ResultsComparer/README.md b/src/tools/ResultsComparer/README.md
index dee77c6883fccf59e8892f2314c871aea5b79ab6..63d922e5c10e285a719e00990f4436f43c5e9b39 100644
GIT binary patch
delta 112
zcmX@5v_*MC4_mSpg91YWgDwyzGZZtF0oe*bwgN)|Ln1>7Lk5sl!jKOnQyD6NV)+b3
xKz=Gi3Q)wEA($ZyNEZS5#SEnkIY60W20fsNKTu@}Lncsl9?+DW&8=)lxd8RS7N7tC

delta 12
Tcmdm@d`f9U58LK%Z2PzXC3*#Y


From 23433d9698758db407e495bae9b86e4e26d07f56 Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Sat, 15 Dec 2018 11:51:05 +0100
Subject: [PATCH 10/11] add missing comment

---
 src/tools/ResultsComparer/DataTransferContracts.cs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/tools/ResultsComparer/DataTransferContracts.cs b/src/tools/ResultsComparer/DataTransferContracts.cs
index 3f02cbf8d90..e68f7971c80 100644
--- a/src/tools/ResultsComparer/DataTransferContracts.cs
+++ b/src/tools/ResultsComparer/DataTransferContracts.cs
@@ -113,6 +113,10 @@ public class Benchmark
         public Memory Memory { get; set; }
         public List<Measurement> Measurements { get; set; }
 
+        /// <summary>
+        /// this method was not auto-generated by a tool, it was added manually
+        /// </summary>
+        /// <returns>an array of the actual workload results (not warmup, not pilot)</returns>
         internal double[] GetOriginalValues()
             => Measurements
                 .Where(measurement => measurement.IterationStage == "Result")

From de4fee622182d0b6c91dcaf4bab6d003458daf97 Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Sat, 15 Dec 2018 11:51:17 +0100
Subject: [PATCH 11/11] shorten the output

---
 src/tools/ResultsComparer/Program.cs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/tools/ResultsComparer/Program.cs b/src/tools/ResultsComparer/Program.cs
index c9645ca6eb4..7e9b165510f 100644
--- a/src/tools/ResultsComparer/Program.cs
+++ b/src/tools/ResultsComparer/Program.cs
@@ -77,7 +77,7 @@ private static void PrintTable((string id, Benchmark baseResult, Benchmark diffR
                 .OrderByDescending(result => GetRatio(conclusion, result.baseResult, result.diffResult))
                 .Take(args.TopCount ?? int.MaxValue)
                 .Select(result => new {
-                    Id = result.id.Length > 100 ? result.id.Substring(0, 100) : result.id,
+                    Id = result.id.Length > 80 ? result.id.Substring(0, 80) : result.id,
                     DisplayValue = GetRatio(conclusion, result.baseResult, result.diffResult),              
                     BaseMedian = result.baseResult.Statistics.Median,
                     DiffMedian = result.diffResult.Statistics.Median,
@@ -156,7 +156,7 @@ private static string GetModalInfo(Benchmark benchmark)
             else if (mValue > 3.2)
                 return "bimodal";
             else if (mValue > 2.8)
-                return "can have several modes";
+                return "several?";
 
             return null;
         }