built-in accurate and cross platform Memory Diagnoser

BenchmarkDotNet.sln

@@ -37,7 +37,7 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Integration
 EndProject
 Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "BenchmarkDotNet.Samples.FSharp", "samples\BenchmarkDotNet.Samples.FSharp\BenchmarkDotNet.Samples.FSharp.fsproj", "{A329F00E-4B9D-4BC6-B688-92698D773CBF}"
 	ProjectSection(ProjectDependencies) = postProject
-		{95F5D645-19E3-432F-95D4-C5EA374DD15B} = {95F5D645-19E3-432F-95D4-C5EA374DD15B}
+		{AF1E6F8A-5C63-465F-96F4-5E5F183A33B9} = {AF1E6F8A-5C63-465F-96F4-5E5F183A33B9}
 	EndProjectSection
 EndProject
 Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "BenchmarkDotNet.IntegrationTests.FSharp", "tests\BenchmarkDotNet.IntegrationTests.FSharp\BenchmarkDotNet.IntegrationTests.FSharp.fsproj", "{367FAFE1-A1C8-4AA1-9334-F4762E128DBB}"

docs/guide/Configs/Diagnosers.md

@@ -1,25 +1,22 @@
 # Diagnosers
 
-A **diagnoser** can attach to your benchmark and get some useful info. There is a separated package with diagnosers for Windows (`BenchmarkDotNet.Diagnostics.Windows`):
-
-[![NuGet](https://img.shields.io/nuget/v/BenchmarkDotNet.svg)](https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/)
-
+A **diagnoser** can attach to your benchmark and get some useful info.
 
 The current Diagnosers are:
 
-- GC and Memory Allocation (`MemoryDiagnoser`)
-- JIT Inlining Events (`InliningDiagnoser`)
+- GC and Memory Allocation (`MemoryDiagnoser`) which is cross platform, built-in and enabled by default.
+- JIT Inlining Events (`InliningDiagnoser`). You can find this diagnoser in a separated package with diagnosers for Windows (`BenchmarkDotNet.Diagnostics.Windows`): [![NuGet](https://img.shields.io/nuget/v/BenchmarkDotNet.svg)](https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/)
 
 
 ## Examples
 
-Below is a sample output from the `GC and Memory Allocation` diagnoser, note the extra columns on the right-hand side ("Gen 0", "Gen 1", "Gen 2" and "Bytes Allocated/Op"):
-
-    Method |  Lookup |     Median |    StdDev | Scaled |    Gen 0 | Gen 1 | Gen 2 | Bytes Allocated/Op |
----------- |-------- |----------- |---------- |------- |--------- |------ |------ |------------------- |
-      LINQ | Testing | 49.1154 ns | 0.5301 ns |   2.48 | 1,526.00 |     - |     - |              25.21 |
- Iterative | Testing | 19.8040 ns | 0.0456 ns |   1.00 |        - |     - |     - |               0.00 |
+Below is a sample output from the `GC and Memory Allocation` diagnoser, note the extra columns on the right-hand side ("Gen 0", "Gen 1", "Gen 2" and "Allocated"):
 
+```
+           Method |        Mean |     StdErr |      StdDev |      Median |  Gen 0 | Allocated |
+----------------- |------------ |----------- |------------ |------------ |------- |---------- |
+ 'new byte[10kB]' | 884.4896 ns | 46.3528 ns | 245.2762 ns | 776.4237 ns | 0.1183 |     10 kB |
+ ```
 
 A config example:
 
@@ -28,8 +25,21 @@ private class Config : ManualConfig
 {
     public Config()
     {
-        Add(new MemoryDiagnoser());
+        Add(MemoryDiagnoser.Default);
         Add(new InliningDiagnoser());
     }
 }
 ```
+
+You can also use one of the following attributes (apply it on a class that contains Benchmarks):
+```cs
+[MemoryDiagnoser]
+[InliningDiagnoser]
+```
+
+## Restrictions
+
+* Mono currently [does not](http://stackoverflow.com/questions/40234948/how-to-get-the-number-of-allocated-bytes-in-mono) expose any api to get the number of allocated bytes. That's why our Mono users will get `?` in Allocated column.
+* In order to get the number of allocated bytes in cross platform way we are using `GC.GetAllocatedBytesForCurrentThread` which recently got [exposed](https://github.com/dotnet/corefx/pull/12489) for netcoreapp1.1. That's why BenchmarkDotNet does not support netcoreapp1.0 from version 0.10.1.
+* In order to not affect main results we perform a separate run if any diagnoser is used. That's why it might take more time to execute benchmarks.
+* MemoryDiagnoser is `100%` accurate about allocated memory when using Job.ShortRun or longer job.

docs/guide/Contributing/Building.md

@@ -2,10 +2,10 @@
 
 For building the BenchmarkDotNet source-code, the following elements are required:
 
-* [Visual Studio 2015 Update **3**](http://go.microsoft.com/fwlink/?LinkId=691129)
+* [Visual Studio 2015 Update **3**](https://go.microsoft.com/fwlink/?LinkId=691978)
 * [Latest NuGet Manager extension for Visual Studio](https://dist.nuget.org/visualstudio-2015-vsix/v3.5.0-beta/NuGet.Tools.vsix)
-* [.NET Core SDK](https://go.microsoft.com/fwlink/?LinkID=809122)
-* [.NET Core Tooling Preview 2 for Visual Studio 2015](https://go.microsoft.com/fwlink/?LinkId=817245)
+* [.NET Core SDK **1.1**](https://go.microsoft.com/fwlink/?LinkID=835014)
+* [.NET Core 1.0.1 Tooling Preview 2 for Visual Studio 2015](https://go.microsoft.com/fwlink/?LinkID=827546)
 * Internet connection and disk space to download all the required packages
 
 If your build fails because some packages are not available, let say F#, then just disable these project and hope for nuget server to work later on ;)

samples/BenchmarkDotNet.Samples.FSharp.Core/project.json

@@ -13,7 +13,7 @@
     "Microsoft.FSharp.Core.netcore": "1.0.0-*",
     "Microsoft.NETCore.App": {
       "type": "platform",
-      "version": "1.0.0"
+      "version": "1.1.0"
     },
     "BenchmarkDotNet": {
       "target": "project"
@@ -31,7 +31,7 @@
     }
   },
   "frameworks": {
-    "netcoreapp1.0": {
+    "netcoreapp1.1": {
       "imports": [
         "portable-net45+win8",
         "dnxcore50",

samples/BenchmarkDotNet.Samples/Intro/IntroGcMode.cs

@@ -8,41 +8,27 @@ namespace BenchmarkDotNet.Samples.Intro
 {
     [Config(typeof(Config))]
     [OrderProvider(SummaryOrderPolicy.FastestToSlowest)]
+    [MemoryDiagnoser]
     public class IntroGcMode
     {
         private class Config : ManualConfig
         {
             public Config()
             {
-                Add(new Job("ServerForce")
-                {
-                    Env = { Gc = { Server = true, Force = true } }
-                });
-                Add(new Job("Server")
-                {
-                    Env = { Gc = { Server = true, Force = false } }
-                });
-                Add(new Job("Workstation")
-                {
-                    Env = { Gc = { Server = false, Force = false } }
-                });
-                Add(new Job("WorkstationForce")
-                {
-                    Env = { Gc = { Server = false, Force = true } }
-                });
-#if !CORE
-                Add(new Diagnostics.Windows.MemoryDiagnoser());
-#endif
+                Add(Job.MediumRun.WithGcServer(true).WithGcForce(true).WithId("ServerForce"));
+                Add(Job.MediumRun.WithGcServer(true).WithGcForce(false).WithId("Server"));
+                Add(Job.MediumRun.WithGcServer(false).WithGcForce(true).WithId("Workstation"));
+                Add(Job.MediumRun.WithGcServer(false).WithGcForce(false).WithId("WorkstationForce"));
             }
         }
 
-        [Benchmark(Description = "new byte[10KB]")]
+        [Benchmark(Description = "new byte[10kB]")]
         public byte[] Allocate()
         {
             return new byte[10000];
         }
 
-        [Benchmark(Description = "stackalloc byte[10KB]")]
+        [Benchmark(Description = "stackalloc byte[10kB]")]
         public unsafe void AllocateWithStackalloc()
         {
             var array = stackalloc byte[10000];

samples/BenchmarkDotNet.Samples/project.json

@@ -33,14 +33,14 @@
         }
       }
     },
-    "netcoreapp1.0": {
+    "netcoreapp1.1": {
       "buildOptions": {
         "define": [ "CORE" ]
       },
       "dependencies": {
         "Microsoft.NETCore.App": {
           "type": "platform",
-          "version": "1.0.0"
+          "version": "1.1.0"
         },
         "System.ComponentModel.EventBasedAsync": "4.0.11"
       }

src/BenchmarkDotNet.Diagnostics.Windows/Configs/MemoryDiagnoserAttribute.cs → src/BenchmarkDotNet.Core/Attributes/MemoryDiagnoserAttribute.cs

@@ -1,15 +1,16 @@
 using System;
 using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Diagnosers;
 
-namespace BenchmarkDotNet.Diagnostics.Windows.Configs
+namespace BenchmarkDotNet.Attributes
 {
     public class MemoryDiagnoserAttribute : Attribute, IConfigSource
     {
         public IConfig Config { get; }
 
         public MemoryDiagnoserAttribute()
         {
-            Config = ManualConfig.CreateEmpty().With(new MemoryDiagnoser());
+            Config = ManualConfig.CreateEmpty().With(MemoryDiagnoser.Default);
         }
     }
 }

src/BenchmarkDotNet.Core/Configs/DefaultConfig.cs

@@ -60,7 +60,10 @@ public IEnumerable<IValidator> GetValidators()
 
         public bool KeepBenchmarkFiles => false;
 
-        public IEnumerable<IDiagnoser> GetDiagnosers() => Enumerable.Empty<IDiagnoser>();
+        public IEnumerable<IDiagnoser> GetDiagnosers()
+        {
+            yield return MemoryDiagnoser.Default;
+        }
 
         // Make the Diagnosers lazy-loaded, so they are only instantiated if neededs
         public static readonly Lazy<IDiagnoser[]> LazyLoadedDiagnosers =
@@ -86,7 +89,7 @@ private static IDiagnoser[] LoadDiagnosers()
                 {
                     return new[]
                     {
-                        GetDiagnoser(loadedAssembly, "BenchmarkDotNet.Diagnostics.Windows.MemoryDiagnoser"),
+                        MemoryDiagnoser.Default,
                         GetDiagnoser(loadedAssembly, "BenchmarkDotNet.Diagnostics.Windows.InliningDiagnoser"),
                     };
                 }

src/BenchmarkDotNet.Core/Diagnosers/CompositeDiagnoser.cs

@@ -14,7 +14,7 @@ public class CompositeDiagnoser : IDiagnoser
 
         public CompositeDiagnoser(params IDiagnoser[] diagnosers)
         {
-            this.diagnosers = diagnosers;
+            this.diagnosers = diagnosers.Distinct().ToArray();
         }
 
         public IColumnProvider GetColumnProvider() 

src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs

@@ -0,0 +1,114 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using BenchmarkDotNet.Columns;
+using BenchmarkDotNet.Environments;
+using BenchmarkDotNet.Loggers;
+using BenchmarkDotNet.Reports;
+using BenchmarkDotNet.Running;
+using BenchmarkDotNet.Engines;
+using BenchmarkDotNet.Extensions;
+using BenchmarkDotNet.Portability;
+
+namespace BenchmarkDotNet.Diagnosers
+{
+    public class MemoryDiagnoser : IDiagnoser
+    {
+        private const int Gen0 = 0, Gen1 = 1, Gen2 = 2;
+
+        public static readonly MemoryDiagnoser Default = new MemoryDiagnoser();
+
+        private readonly Dictionary<Benchmark, GcStats> results = new Dictionary<Benchmark, GcStats>();
+
+        public IColumnProvider GetColumnProvider() => new SimpleColumnProvider(
+            new GCCollectionColumn(results, Gen0),
+            new GCCollectionColumn(results, Gen1),
+            new GCCollectionColumn(results, Gen2),
+            new AllocationColumn(results));
+
+        // the following methods are left empty on purpose
+        // the action takes places in other process, and the values are gathered by Engine
+        public void BeforeAnythingElse(Process process, Benchmark benchmark) { }
+        public void AfterSetup(Process process, Benchmark benchmark) { }
+        public void BeforeCleanup() { }
+
+        public void DisplayResults(ILogger logger)
+            => logger.WriteInfo("Note: the Gen 0/1/2/ Measurements are per 1k Operations");
+
+        public void ProcessResults(Benchmark benchmark, BenchmarkReport report) 
+            => results.Add(benchmark, report.GcStats);
+
+        public class AllocationColumn : IColumn
+        {
+            private readonly Dictionary<Benchmark, GcStats> results;
+
+            public AllocationColumn(Dictionary<Benchmark, GcStats> results)
+            {
+                this.results = results;
+            }
+
+            public string Id => nameof(AllocationColumn);
+            public string ColumnName => "Allocated";
+            public bool IsDefault(Summary summary, Benchmark benchmark) => false;
+            public bool IsAvailable(Summary summary) => true;
+            public bool AlwaysShow => true;
+            public ColumnCategory Category => ColumnCategory.Diagnoser;
+            public int PriorityInCategory => 0;
+
+            public string GetValue(Summary summary, Benchmark benchmark)
+            {
+                if (RuntimeInformation.IsMono())
+                    return "?";
+                if (!results.ContainsKey(benchmark))
+                    return "N/A";
+
+                return results[benchmark].BytesAllocatedPerOperation.ToFormattedBytes();
+            }
+        }
+
+        public class GCCollectionColumn : IColumn
+        {
+            private readonly Dictionary<Benchmark, GcStats> results;
+            private readonly int generation;
+
+            public GCCollectionColumn(Dictionary<Benchmark, GcStats> results, int generation)
+            {
+                this.results = results;
+                this.generation = generation;
+            }
+
+            public bool IsDefault(Summary summary, Benchmark benchmark) => true;
+            public string Id => $"{nameof(GCCollectionColumn)}{generation}";
+            public string ColumnName => $"Gen {generation}";
+
+            public bool AlwaysShow => generation == Gen0; // Gen 0 must always be visible
+            public ColumnCategory Category => ColumnCategory.Diagnoser;
+            public int PriorityInCategory => 0;
+
+            public bool IsAvailable(Summary summary)
+                => generation == Gen0
+                    || summary
+                        .Reports
+                        .Any(report => generation == Gen1 
+                            ? report.GcStats.Gen1Collections != 0 
+                            : report.GcStats.Gen2Collections != 0);
+
+            public string GetValue(Summary summary, Benchmark benchmark)
+            {
+                if (results.ContainsKey(benchmark))
+                {
+                    var gcStats = results[benchmark];
+                    var value = generation == Gen0 ? gcStats.Gen0Collections : 
+                                generation == Gen1 ? gcStats.Gen1Collections : gcStats.Gen2Collections;
+
+                    if (value == 0)
+                        return "-"; // make zero more obvious
+
+                    return ((value / (double)gcStats.TotalOperations) * 1000).ToString("#0.0000", HostEnvironmentInfo.MainCultureInfo);
+                }
+                return "N/A";
+            }
+        }
+    }
+}