diff --git a/BenchmarkDotNet.sln b/BenchmarkDotNet.sln index faad78bfca..b8722fdb99 100644 --- a/BenchmarkDotNet.sln +++ b/BenchmarkDotNet.sln @@ -37,7 +37,7 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Integration EndProject Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "BenchmarkDotNet.Samples.FSharp", "samples\BenchmarkDotNet.Samples.FSharp\BenchmarkDotNet.Samples.FSharp.fsproj", "{A329F00E-4B9D-4BC6-B688-92698D773CBF}" ProjectSection(ProjectDependencies) = postProject - {95F5D645-19E3-432F-95D4-C5EA374DD15B} = {95F5D645-19E3-432F-95D4-C5EA374DD15B} + {AF1E6F8A-5C63-465F-96F4-5E5F183A33B9} = {AF1E6F8A-5C63-465F-96F4-5E5F183A33B9} EndProjectSection EndProject Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "BenchmarkDotNet.IntegrationTests.FSharp", "tests\BenchmarkDotNet.IntegrationTests.FSharp\BenchmarkDotNet.IntegrationTests.FSharp.fsproj", "{367FAFE1-A1C8-4AA1-9334-F4762E128DBB}" diff --git a/docs/guide/Configs/Diagnosers.md b/docs/guide/Configs/Diagnosers.md index 3befa86b21..70eee38fb1 100644 --- a/docs/guide/Configs/Diagnosers.md +++ b/docs/guide/Configs/Diagnosers.md @@ -1,25 +1,22 @@ # Diagnosers -A **diagnoser** can attach to your benchmark and get some useful info. There is a separated package with diagnosers for Windows (`BenchmarkDotNet.Diagnostics.Windows`): - -[![NuGet](https://img.shields.io/nuget/v/BenchmarkDotNet.svg)](https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/) - +A **diagnoser** can attach to your benchmark and get some useful info. The current Diagnosers are: -- GC and Memory Allocation (`MemoryDiagnoser`) -- JIT Inlining Events (`InliningDiagnoser`) +- GC and Memory Allocation (`MemoryDiagnoser`) which is cross platform, built-in and enabled by default. +- JIT Inlining Events (`InliningDiagnoser`). You can find this diagnoser in a separated package with diagnosers for Windows (`BenchmarkDotNet.Diagnostics.Windows`): [![NuGet](https://img.shields.io/nuget/v/BenchmarkDotNet.svg)](https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/) ## Examples -Below is a sample output from the `GC and Memory Allocation` diagnoser, note the extra columns on the right-hand side ("Gen 0", "Gen 1", "Gen 2" and "Bytes Allocated/Op"): - - Method | Lookup | Median | StdDev | Scaled | Gen 0 | Gen 1 | Gen 2 | Bytes Allocated/Op | ----------- |-------- |----------- |---------- |------- |--------- |------ |------ |------------------- | - LINQ | Testing | 49.1154 ns | 0.5301 ns | 2.48 | 1,526.00 | - | - | 25.21 | - Iterative | Testing | 19.8040 ns | 0.0456 ns | 1.00 | - | - | - | 0.00 | +Below is a sample output from the `GC and Memory Allocation` diagnoser, note the extra columns on the right-hand side ("Gen 0", "Gen 1", "Gen 2" and "Allocated"): +``` + Method | Mean | StdErr | StdDev | Median | Gen 0 | Allocated | +----------------- |------------ |----------- |------------ |------------ |------- |---------- | + 'new byte[10kB]' | 884.4896 ns | 46.3528 ns | 245.2762 ns | 776.4237 ns | 0.1183 | 10 kB | + ``` A config example: @@ -28,8 +25,21 @@ private class Config : ManualConfig { public Config() { - Add(new MemoryDiagnoser()); + Add(MemoryDiagnoser.Default); Add(new InliningDiagnoser()); } } ``` + +You can also use one of the following attributes (apply it on a class that contains Benchmarks): +```cs +[MemoryDiagnoser] +[InliningDiagnoser] +``` + +## Restrictions + +* Mono currently [does not](http://stackoverflow.com/questions/40234948/how-to-get-the-number-of-allocated-bytes-in-mono) expose any api to get the number of allocated bytes. That's why our Mono users will get `?` in Allocated column. +* In order to get the number of allocated bytes in cross platform way we are using `GC.GetAllocatedBytesForCurrentThread` which recently got [exposed](https://github.com/dotnet/corefx/pull/12489) for netcoreapp1.1. That's why BenchmarkDotNet does not support netcoreapp1.0 from version 0.10.1. +* In order to not affect main results we perform a separate run if any diagnoser is used. That's why it might take more time to execute benchmarks. +* MemoryDiagnoser is `100%` accurate about allocated memory when using Job.ShortRun or longer job. diff --git a/docs/guide/Contributing/Building.md b/docs/guide/Contributing/Building.md index 0bc3e37e1e..5e6de7b22c 100644 --- a/docs/guide/Contributing/Building.md +++ b/docs/guide/Contributing/Building.md @@ -2,10 +2,10 @@ For building the BenchmarkDotNet source-code, the following elements are required: -* [Visual Studio 2015 Update **3**](http://go.microsoft.com/fwlink/?LinkId=691129) +* [Visual Studio 2015 Update **3**](https://go.microsoft.com/fwlink/?LinkId=691978) * [Latest NuGet Manager extension for Visual Studio](https://dist.nuget.org/visualstudio-2015-vsix/v3.5.0-beta/NuGet.Tools.vsix) -* [.NET Core SDK](https://go.microsoft.com/fwlink/?LinkID=809122) -* [.NET Core Tooling Preview 2 for Visual Studio 2015](https://go.microsoft.com/fwlink/?LinkId=817245) +* [.NET Core SDK **1.1**](https://go.microsoft.com/fwlink/?LinkID=835014) +* [.NET Core 1.0.1 Tooling Preview 2 for Visual Studio 2015](https://go.microsoft.com/fwlink/?LinkID=827546) * Internet connection and disk space to download all the required packages If your build fails because some packages are not available, let say F#, then just disable these project and hope for nuget server to work later on ;) diff --git a/samples/BenchmarkDotNet.Samples.FSharp.Core/project.json b/samples/BenchmarkDotNet.Samples.FSharp.Core/project.json index ee777a923e..8a097798d4 100644 --- a/samples/BenchmarkDotNet.Samples.FSharp.Core/project.json +++ b/samples/BenchmarkDotNet.Samples.FSharp.Core/project.json @@ -13,7 +13,7 @@ "Microsoft.FSharp.Core.netcore": "1.0.0-*", "Microsoft.NETCore.App": { "type": "platform", - "version": "1.0.0" + "version": "1.1.0" }, "BenchmarkDotNet": { "target": "project" @@ -31,7 +31,7 @@ } }, "frameworks": { - "netcoreapp1.0": { + "netcoreapp1.1": { "imports": [ "portable-net45+win8", "dnxcore50", diff --git a/samples/BenchmarkDotNet.Samples/Intro/IntroGcMode.cs b/samples/BenchmarkDotNet.Samples/Intro/IntroGcMode.cs index 8c9a72a1c9..e3107b4eeb 100644 --- a/samples/BenchmarkDotNet.Samples/Intro/IntroGcMode.cs +++ b/samples/BenchmarkDotNet.Samples/Intro/IntroGcMode.cs @@ -8,41 +8,27 @@ namespace BenchmarkDotNet.Samples.Intro { [Config(typeof(Config))] [OrderProvider(SummaryOrderPolicy.FastestToSlowest)] + [MemoryDiagnoser] public class IntroGcMode { private class Config : ManualConfig { public Config() { - Add(new Job("ServerForce") - { - Env = { Gc = { Server = true, Force = true } } - }); - Add(new Job("Server") - { - Env = { Gc = { Server = true, Force = false } } - }); - Add(new Job("Workstation") - { - Env = { Gc = { Server = false, Force = false } } - }); - Add(new Job("WorkstationForce") - { - Env = { Gc = { Server = false, Force = true } } - }); -#if !CORE - Add(new Diagnostics.Windows.MemoryDiagnoser()); -#endif + Add(Job.MediumRun.WithGcServer(true).WithGcForce(true).WithId("ServerForce")); + Add(Job.MediumRun.WithGcServer(true).WithGcForce(false).WithId("Server")); + Add(Job.MediumRun.WithGcServer(false).WithGcForce(true).WithId("Workstation")); + Add(Job.MediumRun.WithGcServer(false).WithGcForce(false).WithId("WorkstationForce")); } } - [Benchmark(Description = "new byte[10KB]")] + [Benchmark(Description = "new byte[10kB]")] public byte[] Allocate() { return new byte[10000]; } - [Benchmark(Description = "stackalloc byte[10KB]")] + [Benchmark(Description = "stackalloc byte[10kB]")] public unsafe void AllocateWithStackalloc() { var array = stackalloc byte[10000]; diff --git a/samples/BenchmarkDotNet.Samples/project.json b/samples/BenchmarkDotNet.Samples/project.json index 83c169f291..c6eb661ecd 100644 --- a/samples/BenchmarkDotNet.Samples/project.json +++ b/samples/BenchmarkDotNet.Samples/project.json @@ -33,14 +33,14 @@ } } }, - "netcoreapp1.0": { + "netcoreapp1.1": { "buildOptions": { "define": [ "CORE" ] }, "dependencies": { "Microsoft.NETCore.App": { "type": "platform", - "version": "1.0.0" + "version": "1.1.0" }, "System.ComponentModel.EventBasedAsync": "4.0.11" } diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/Configs/MemoryDiagnoserAttribute.cs b/src/BenchmarkDotNet.Core/Attributes/MemoryDiagnoserAttribute.cs similarity index 60% rename from src/BenchmarkDotNet.Diagnostics.Windows/Configs/MemoryDiagnoserAttribute.cs rename to src/BenchmarkDotNet.Core/Attributes/MemoryDiagnoserAttribute.cs index aa03c926cf..a0a5c85697 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/Configs/MemoryDiagnoserAttribute.cs +++ b/src/BenchmarkDotNet.Core/Attributes/MemoryDiagnoserAttribute.cs @@ -1,7 +1,8 @@ using System; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Diagnosers; -namespace BenchmarkDotNet.Diagnostics.Windows.Configs +namespace BenchmarkDotNet.Attributes { public class MemoryDiagnoserAttribute : Attribute, IConfigSource { @@ -9,7 +10,7 @@ public class MemoryDiagnoserAttribute : Attribute, IConfigSource public MemoryDiagnoserAttribute() { - Config = ManualConfig.CreateEmpty().With(new MemoryDiagnoser()); + Config = ManualConfig.CreateEmpty().With(MemoryDiagnoser.Default); } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet.Core/Configs/DefaultConfig.cs b/src/BenchmarkDotNet.Core/Configs/DefaultConfig.cs index 8e6efe60ed..a58c319e5a 100644 --- a/src/BenchmarkDotNet.Core/Configs/DefaultConfig.cs +++ b/src/BenchmarkDotNet.Core/Configs/DefaultConfig.cs @@ -60,7 +60,10 @@ public IEnumerable GetValidators() public bool KeepBenchmarkFiles => false; - public IEnumerable GetDiagnosers() => Enumerable.Empty(); + public IEnumerable GetDiagnosers() + { + yield return MemoryDiagnoser.Default; + } // Make the Diagnosers lazy-loaded, so they are only instantiated if neededs public static readonly Lazy LazyLoadedDiagnosers = @@ -86,7 +89,7 @@ private static IDiagnoser[] LoadDiagnosers() { return new[] { - GetDiagnoser(loadedAssembly, "BenchmarkDotNet.Diagnostics.Windows.MemoryDiagnoser"), + MemoryDiagnoser.Default, GetDiagnoser(loadedAssembly, "BenchmarkDotNet.Diagnostics.Windows.InliningDiagnoser"), }; } diff --git a/src/BenchmarkDotNet.Core/Diagnosers/CompositeDiagnoser.cs b/src/BenchmarkDotNet.Core/Diagnosers/CompositeDiagnoser.cs index 464fce63ce..9172d8f6ae 100644 --- a/src/BenchmarkDotNet.Core/Diagnosers/CompositeDiagnoser.cs +++ b/src/BenchmarkDotNet.Core/Diagnosers/CompositeDiagnoser.cs @@ -14,7 +14,7 @@ public class CompositeDiagnoser : IDiagnoser public CompositeDiagnoser(params IDiagnoser[] diagnosers) { - this.diagnosers = diagnosers; + this.diagnosers = diagnosers.Distinct().ToArray(); } public IColumnProvider GetColumnProvider() diff --git a/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs b/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs new file mode 100644 index 0000000000..7cdac68f99 --- /dev/null +++ b/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs @@ -0,0 +1,114 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Portability; + +namespace BenchmarkDotNet.Diagnosers +{ + public class MemoryDiagnoser : IDiagnoser + { + private const int Gen0 = 0, Gen1 = 1, Gen2 = 2; + + public static readonly MemoryDiagnoser Default = new MemoryDiagnoser(); + + private readonly Dictionary results = new Dictionary(); + + public IColumnProvider GetColumnProvider() => new SimpleColumnProvider( + new GCCollectionColumn(results, Gen0), + new GCCollectionColumn(results, Gen1), + new GCCollectionColumn(results, Gen2), + new AllocationColumn(results)); + + // the following methods are left empty on purpose + // the action takes places in other process, and the values are gathered by Engine + public void BeforeAnythingElse(Process process, Benchmark benchmark) { } + public void AfterSetup(Process process, Benchmark benchmark) { } + public void BeforeCleanup() { } + + public void DisplayResults(ILogger logger) + => logger.WriteInfo("Note: the Gen 0/1/2/ Measurements are per 1k Operations"); + + public void ProcessResults(Benchmark benchmark, BenchmarkReport report) + => results.Add(benchmark, report.GcStats); + + public class AllocationColumn : IColumn + { + private readonly Dictionary results; + + public AllocationColumn(Dictionary results) + { + this.results = results; + } + + public string Id => nameof(AllocationColumn); + public string ColumnName => "Allocated"; + public bool IsDefault(Summary summary, Benchmark benchmark) => false; + public bool IsAvailable(Summary summary) => true; + public bool AlwaysShow => true; + public ColumnCategory Category => ColumnCategory.Diagnoser; + public int PriorityInCategory => 0; + + public string GetValue(Summary summary, Benchmark benchmark) + { + if (RuntimeInformation.IsMono()) + return "?"; + if (!results.ContainsKey(benchmark)) + return "N/A"; + + return results[benchmark].BytesAllocatedPerOperation.ToFormattedBytes(); + } + } + + public class GCCollectionColumn : IColumn + { + private readonly Dictionary results; + private readonly int generation; + + public GCCollectionColumn(Dictionary results, int generation) + { + this.results = results; + this.generation = generation; + } + + public bool IsDefault(Summary summary, Benchmark benchmark) => true; + public string Id => $"{nameof(GCCollectionColumn)}{generation}"; + public string ColumnName => $"Gen {generation}"; + + public bool AlwaysShow => generation == Gen0; // Gen 0 must always be visible + public ColumnCategory Category => ColumnCategory.Diagnoser; + public int PriorityInCategory => 0; + + public bool IsAvailable(Summary summary) + => generation == Gen0 + || summary + .Reports + .Any(report => generation == Gen1 + ? report.GcStats.Gen1Collections != 0 + : report.GcStats.Gen2Collections != 0); + + public string GetValue(Summary summary, Benchmark benchmark) + { + if (results.ContainsKey(benchmark)) + { + var gcStats = results[benchmark]; + var value = generation == Gen0 ? gcStats.Gen0Collections : + generation == Gen1 ? gcStats.Gen1Collections : gcStats.Gen2Collections; + + if (value == 0) + return "-"; // make zero more obvious + + return ((value / (double)gcStats.TotalOperations) * 1000).ToString("#0.0000", HostEnvironmentInfo.MainCultureInfo); + } + return "N/A"; + } + } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet.Core/Engines/Engine.cs b/src/BenchmarkDotNet.Core/Engines/Engine.cs index 1365750c56..9da1316ee6 100644 --- a/src/BenchmarkDotNet.Core/Engines/Engine.cs +++ b/src/BenchmarkDotNet.Core/Engines/Engine.cs @@ -3,6 +3,7 @@ using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Horology; using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Portability; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; using JetBrains.Annotations; @@ -35,6 +36,7 @@ public class Engine : IEngine private readonly EngineWarmupStage warmupStage; private readonly EngineTargetStage targetStage; private bool isJitted, isPreAllocated; + private int forcedFullGarbageCollections; internal Engine(Action idleAction, Action mainAction, Job targetJob, Action setupAction, Action cleanupAction, long operationsPerInvoke, bool isDiagnoserAttached) { @@ -98,12 +100,19 @@ public RunResults Run() warmupStage.RunMain(invokeCount, UnrollFactor); } + // we enable monitoring after pilot & warmup, just to ignore the memory allocated by these runs + EnableMonitoring(); + var initialGcStats = GcStats.ReadInitial(IsDiagnoserAttached); + var main = targetStage.RunMain(invokeCount, UnrollFactor); + var finalGcStats = GcStats.ReadFinal(IsDiagnoserAttached); + var forcedCollections = GcStats.FromForced(forcedFullGarbageCollections); + var workGcHasDone = finalGcStats - forcedCollections - initialGcStats; + bool removeOutliers = TargetJob.ResolveValue(AccuracyMode.RemoveOutliersCharacteristic, Resolver); - return new RunResults(idle, main, removeOutliers); - } + return new RunResults(idle, main, removeOutliers, workGcHasDone); } public Measurement RunIteration(IterationData data) { @@ -137,11 +146,13 @@ private void GcCollect() ForceGcCollect(); } - private static void ForceGcCollect() + private void ForceGcCollect() { GC.Collect(); GC.WaitForPendingFinalizers(); GC.Collect(); + + forcedFullGarbageCollections += 2; } public void WriteLine(string text) @@ -158,6 +169,18 @@ public void WriteLine() Console.WriteLine(); } + private void EnableMonitoring() + { + if(!IsDiagnoserAttached) // it could affect the results, we do this in separate, diagnostics-only run + return; +#if CLASSIC + if(RuntimeInformation.IsMono()) // Monitoring is not available in Mono, see http://stackoverflow.com/questions/40234948/how-to-get-the-number-of-allocated-bytes-in-mono + return; + + AppDomain.MonitoringIsEnabled = true; +#endif + } + private void EnsureNothingIsPrintedWhenDiagnoserIsAttached() { if (IsDiagnoserAttached) diff --git a/src/BenchmarkDotNet.Core/Engines/EngineTargetStage.cs b/src/BenchmarkDotNet.Core/Engines/EngineTargetStage.cs index b0949e3a7d..69ca4e6e05 100644 --- a/src/BenchmarkDotNet.Core/Engines/EngineTargetStage.cs +++ b/src/BenchmarkDotNet.Core/Engines/EngineTargetStage.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Mathematics; using BenchmarkDotNet.Reports; @@ -15,12 +16,15 @@ public class EngineTargetStage : EngineStage private readonly int? targetCount; private readonly double maxStdErrRelative; private readonly bool removeOutliers; + private readonly MeasurementsPool measurementsPool; + public EngineTargetStage(IEngine engine) : base(engine) { targetCount = engine.TargetJob.ResolveValueAsNullable(RunMode.TargetCountCharacteristic); maxStdErrRelative = engine.TargetJob.ResolveValue(AccuracyMode.MaxStdErrRelativeCharacteristic, engine.Resolver); removeOutliers = engine.TargetJob.ResolveValue(AccuracyMode.RemoveOutliersCharacteristic, engine.Resolver); + measurementsPool = MeasurementsPool.PreAllocate(10, MaxIterationCount, targetCount); } public IReadOnlyList RunIdle(long invokeCount, int unrollFactor) @@ -36,8 +40,8 @@ internal IReadOnlyList Run(long invokeCount, IterationMode iteratio private List RunAuto(long invokeCount, IterationMode iterationMode, int unrollFactor) { - var measurements = new List(MaxIterationCount); - var measurementsForStatistics = new List(MaxIterationCount); + var measurements = measurementsPool.Next(); + var measurementsForStatistics = measurementsPool.Next(); int iterationCounter = 0; bool isIdle = iterationMode.IsIdle(); @@ -65,7 +69,7 @@ private List RunAuto(long invokeCount, IterationMode iterationMode, private List RunSpecific(long invokeCount, IterationMode iterationMode, int iterationCount, int unrollFactor) { - var measurements = new List(MaxIterationCount); + var measurements = measurementsPool.Next(); for (int i = 0; i < iterationCount; i++) measurements.Add(RunIteration(iterationMode, i + 1, invokeCount, unrollFactor)); diff --git a/src/BenchmarkDotNet.Core/Engines/GcStats.cs b/src/BenchmarkDotNet.Core/Engines/GcStats.cs new file mode 100644 index 0000000000..a1eeeb70aa --- /dev/null +++ b/src/BenchmarkDotNet.Core/Engines/GcStats.cs @@ -0,0 +1,138 @@ +using System; +using System.Reflection; +using BenchmarkDotNet.Portability; + +namespace BenchmarkDotNet.Engines +{ + public struct GcStats + { + internal const string ResultsLinePrefix = "GC: "; + + private static readonly Func getAllocatedBytesForCurrentThread = GetAllocatedBytesForCurrentThread(); + + private GcStats(int gen0Collections, int gen1Collections, int gen2Collections, long allocatedBytes, long totalOperations) + { + Gen0Collections = gen0Collections; + Gen1Collections = gen1Collections; + Gen2Collections = gen2Collections; + AllocatedBytes = allocatedBytes; + TotalOperations = totalOperations; + } + + // did not use array here just to avoid heap allocation + public int Gen0Collections { get; } + public int Gen1Collections { get; } + public int Gen2Collections { get; } + public long AllocatedBytes { get; } + public long TotalOperations { get; } + + public long BytesAllocatedPerOperation => AllocatedBytes / TotalOperations; + + public static GcStats operator +(GcStats left, GcStats right) + { + return new GcStats( + left.Gen0Collections + right.Gen0Collections, + left.Gen1Collections + right.Gen1Collections, + left.Gen2Collections + right.Gen2Collections, + left.AllocatedBytes + right.AllocatedBytes, + left.TotalOperations + right.TotalOperations); + } + + public static GcStats operator -(GcStats left, GcStats right) + { + return new GcStats( + Math.Max(0, left.Gen0Collections - right.Gen0Collections), + Math.Max(0, left.Gen1Collections - right.Gen1Collections), + Math.Max(0, left.Gen2Collections - right.Gen2Collections), + Math.Max(0, left.AllocatedBytes - right.AllocatedBytes), + Math.Max(0, left.TotalOperations - right.TotalOperations)); + } + + public GcStats WithTotalOperations(long totalOperationsCount) + => this + new GcStats(0, 0, 0, 0, totalOperationsCount); + + internal static GcStats ReadInitial(bool isDiagnosticsEnabled) + { + // this will force GC.Collect, so we want to do this before collecting collections counts + long allocatedBytes = GetAllocatedBytes(isDiagnosticsEnabled); + + return new GcStats( + GC.CollectionCount(0), + GC.CollectionCount(1), + GC.CollectionCount(2), + allocatedBytes, + 0); + } + + internal static GcStats ReadFinal(bool isDiagnosticsEnabled) + { + return new GcStats( + GC.CollectionCount(0), + GC.CollectionCount(1), + GC.CollectionCount(2), + + // this will force GC.Collect, so we want to do this after collecting collections counts + // to exclude this single full forced collection from results + GetAllocatedBytes(isDiagnosticsEnabled), + 0); + } + + public static GcStats FromForced(int forcedFullGarbageCollections) + => new GcStats(forcedFullGarbageCollections, forcedFullGarbageCollections, forcedFullGarbageCollections, 0, 0); + + private static long GetAllocatedBytes(bool isDiagnosticsEnabled) + { + if (!isDiagnosticsEnabled + || RuntimeInformation.IsMono()) // Monitoring is not available in Mono, see http://stackoverflow.com/questions/40234948/how-to-get-the-number-of-allocated-bytes- + return 0; + + // "This instance Int64 property returns the number of bytes that have been allocated by a specific + // AppDomain. The number is accurate as of the last garbage collection." - CLR via C# + // so we enforce GC.Collect here just to make sure we get accurate results + GC.Collect(); +#if CORE + return getAllocatedBytesForCurrentThread.Invoke(); +#elif CLASSIC + + return AppDomain.CurrentDomain.MonitoringTotalAllocatedMemorySize; +#endif + } + + private static Func GetAllocatedBytesForCurrentThread() + { + // for some versions of .NET Core this method is internal, + // for some public and for others public and exposed ;) + var method = typeof(GC).GetTypeInfo().GetMethod("GetAllocatedBytesForCurrentThread", + BindingFlags.Public | BindingFlags.Static) + ?? typeof(GC).GetTypeInfo().GetMethod("GetAllocatedBytesForCurrentThread", + BindingFlags.NonPublic | BindingFlags.Static); + + return () => (long)method.Invoke(null, null); + } + + internal string ToOutputLine() + => $"{ResultsLinePrefix} {Gen0Collections} {Gen1Collections} {Gen2Collections} {AllocatedBytes} {TotalOperations}"; + + internal static GcStats Parse(string line) + { + if(!line.StartsWith(ResultsLinePrefix)) + throw new NotSupportedException($"Line must start with {ResultsLinePrefix}"); + + int gen0, gen1, gen2; + long allocatedBytes, totalOperationsCount; + var measurementSplit = line.Remove(0, ResultsLinePrefix.Length).Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); + if (!int.TryParse(measurementSplit[0], out gen0) + || !int.TryParse(measurementSplit[1], out gen1) + || !int.TryParse(measurementSplit[2], out gen2) + || !long.TryParse(measurementSplit[3], out allocatedBytes) + || !long.TryParse(measurementSplit[4], out totalOperationsCount)) + { + throw new NotSupportedException("Invalid string"); + } + + return new GcStats(gen0, gen1, gen2, allocatedBytes, totalOperationsCount); + } + + public override string ToString() => ToOutputLine(); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet.Core/Engines/MeasurementsPool.cs b/src/BenchmarkDotNet.Core/Engines/MeasurementsPool.cs new file mode 100644 index 0000000000..0dadab0950 --- /dev/null +++ b/src/BenchmarkDotNet.Core/Engines/MeasurementsPool.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; +using BenchmarkDotNet.Reports; + +namespace BenchmarkDotNet.Engines +{ + internal class MeasurementsPool + { + private readonly Stack> preAllocatedListsOfMeasurements; + + private MeasurementsPool(int capacity) + { + preAllocatedListsOfMeasurements = new Stack>(capacity); + } + + internal List Next() => preAllocatedListsOfMeasurements.Pop(); + + internal static MeasurementsPool PreAllocate(int capacity, int maxLength, int? configuredLength) + { + var pool = new MeasurementsPool(capacity); + var maxSize = configuredLength.HasValue ? Math.Max(configuredLength.Value, maxLength) : maxLength; + + for (int i = 0; i < capacity; i++) + { + pool.preAllocatedListsOfMeasurements.Push(new List(maxSize)); + } + + return pool; + } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet.Core/Engines/RunResults.cs b/src/BenchmarkDotNet.Core/Engines/RunResults.cs index f5c6c0772f..1d2fc36fb0 100644 --- a/src/BenchmarkDotNet.Core/Engines/RunResults.cs +++ b/src/BenchmarkDotNet.Core/Engines/RunResults.cs @@ -10,6 +10,7 @@ namespace BenchmarkDotNet.Engines public struct RunResults { private readonly bool removeOutliers; + private readonly long totalOperationsCount; [CanBeNull] public IReadOnlyList Idle { get; } @@ -17,11 +18,22 @@ public struct RunResults [NotNull] public IReadOnlyList Main { get; } - public RunResults([CanBeNull] IReadOnlyList idle, [NotNull] IReadOnlyList main, bool removeOutliers) + public GcStats GCStats { get; } + + public RunResults( + [CanBeNull] IReadOnlyList idle, [NotNull] IReadOnlyList main, bool removeOutliers, GcStats gcStats) { this.removeOutliers = removeOutliers; Idle = idle; Main = main; + GCStats = gcStats; + + totalOperationsCount = 0; + foreach (var measurement in Main) + { + if (!measurement.IterationMode.IsIdle()) + totalOperationsCount += measurement.Operations; + } } // TODO: rewrite without allocations @@ -52,6 +64,8 @@ public void Print() { foreach (var measurement in GetMeasurements()) WriteLine(measurement.ToOutputLine()); + + WriteLine(GCStats.WithTotalOperations(totalOperationsCount).ToOutputLine()); WriteLine(); } diff --git a/src/BenchmarkDotNet.Core/Extensions/CommonExtensions.cs b/src/BenchmarkDotNet.Core/Extensions/CommonExtensions.cs index 0630f2fbb8..6c67e24293 100644 --- a/src/BenchmarkDotNet.Core/Extensions/CommonExtensions.cs +++ b/src/BenchmarkDotNet.Core/Extensions/CommonExtensions.cs @@ -8,6 +8,10 @@ namespace BenchmarkDotNet.Extensions { internal static class CommonExtensions { + const int BytesInKiloByte = 1000; // 1000 vs 1024 thing.. + + static readonly string[] SizeSuffixes = { "B", "kB", "MB", "GB", "TB" }; + public static List ToSortedList(this IEnumerable values) { var list = new List(); @@ -24,6 +28,18 @@ public static string ToTimeStr(this double value, TimeUnit unit = null, int unit return $"{unitValue.ToStr("N4")} {unitName}"; } + internal static string ToFormattedBytes(this long bytes) + { + int i; + double dblSByte = bytes; + for (i = 0; i < SizeSuffixes.Length && bytes >= BytesInKiloByte; i++, bytes /= BytesInKiloByte) + { + dblSByte = bytes / (double)BytesInKiloByte; + } + + return string.Format(HostEnvironmentInfo.MainCultureInfo, "{0:0.##} {1}", dblSByte, SizeSuffixes[i]); + } + public static string ToStr(this double value, string format = "0.##") { // Here we should manually create an object[] for string.Format diff --git a/src/BenchmarkDotNet.Core/Loggers/AsynchronousProcessOutputLogger.cs b/src/BenchmarkDotNet.Core/Loggers/AsyncErrorOutputLogger.cs similarity index 67% rename from src/BenchmarkDotNet.Core/Loggers/AsynchronousProcessOutputLogger.cs rename to src/BenchmarkDotNet.Core/Loggers/AsyncErrorOutputLogger.cs index 97a99eacb2..7ba604f3e4 100644 --- a/src/BenchmarkDotNet.Core/Loggers/AsynchronousProcessOutputLogger.cs +++ b/src/BenchmarkDotNet.Core/Loggers/AsyncErrorOutputLogger.cs @@ -3,12 +3,12 @@ namespace BenchmarkDotNet.Loggers { - internal class AsynchronousProcessOutputLogger : IDisposable + internal class AsyncErrorOutputLogger : IDisposable { private readonly Process process; private readonly ILogger logger; - public AsynchronousProcessOutputLogger(ILogger logger, Process process) + public AsyncErrorOutputLogger(ILogger logger, Process process) { if (process.StartInfo.UseShellExecute) { @@ -18,10 +18,6 @@ public AsynchronousProcessOutputLogger(ILogger logger, Process process) this.logger = logger; this.process = process; - if (process.StartInfo.RedirectStandardOutput) - { - this.process.OutputDataReceived += ProcessOnOutputDataReceived; - } if (process.StartInfo.RedirectStandardError) { this.process.ErrorDataReceived += ProcessOnErrorDataReceived; @@ -30,15 +26,9 @@ public AsynchronousProcessOutputLogger(ILogger logger, Process process) public void Dispose() { - process.OutputDataReceived -= ProcessOnOutputDataReceived; process.ErrorDataReceived -= ProcessOnErrorDataReceived; } - private void ProcessOnOutputDataReceived(object sender, DataReceivedEventArgs dataReceivedEventArgs) - { - logger.WriteLine(LogKind.Default, dataReceivedEventArgs.Data); - } - private void ProcessOnErrorDataReceived(object sender, DataReceivedEventArgs dataReceivedEventArgs) { if (!string.IsNullOrEmpty(dataReceivedEventArgs.Data)) // happened often and added unnecessary blank line to output diff --git a/src/BenchmarkDotNet.Core/Portability/RuntimeInformation.cs b/src/BenchmarkDotNet.Core/Portability/RuntimeInformation.cs index 33dc759762..0c78181b56 100644 --- a/src/BenchmarkDotNet.Core/Portability/RuntimeInformation.cs +++ b/src/BenchmarkDotNet.Core/Portability/RuntimeInformation.cs @@ -19,6 +19,8 @@ namespace BenchmarkDotNet.Portability { public class RuntimeInformation { + private static readonly bool isMono = Type.GetType("Mono.Runtime") != null; // it allocates a lot of memory, we need to check it once in order to keep Enging non-allocating! + private const string Debug = "DEBUG"; private const string Release = "RELEASE"; internal const string Unknown = "?"; @@ -39,7 +41,7 @@ internal static bool IsWindows() #endif } - private static bool IsMono() => Type.GetType("Mono.Runtime") != null; + internal static bool IsMono() => isMono; internal static string GetOsVersion() { diff --git a/src/BenchmarkDotNet.Core/Reports/BenchmarkReport.cs b/src/BenchmarkDotNet.Core/Reports/BenchmarkReport.cs index 33952d8954..68a130abcf 100644 --- a/src/BenchmarkDotNet.Core/Reports/BenchmarkReport.cs +++ b/src/BenchmarkDotNet.Core/Reports/BenchmarkReport.cs @@ -12,6 +12,7 @@ public sealed class BenchmarkReport { public Benchmark Benchmark { get; } public IReadOnlyList AllMeasurements { get; } + public GcStats GcStats { get; } public GenerateResult GenerateResult { get; } public BuildResult BuildResult { get; } @@ -30,13 +31,15 @@ public BenchmarkReport( GenerateResult generateResult, BuildResult buildResult, IReadOnlyList executeResults, - IReadOnlyList allMeasurements) + IReadOnlyList allMeasurements, + GcStats gcStats) { Benchmark = benchmark; GenerateResult = generateResult; BuildResult = buildResult; ExecuteResults = executeResults ?? new ExecuteResult[0]; AllMeasurements = allMeasurements ?? new Measurement[0]; + GcStats = gcStats; } public override string ToString() => $"{Benchmark.DisplayInfo}, {AllMeasurements.Count} runs"; diff --git a/src/BenchmarkDotNet.Core/Reports/Measurement.cs b/src/BenchmarkDotNet.Core/Reports/Measurement.cs index 1b07e8edef..ba66767469 100644 --- a/src/BenchmarkDotNet.Core/Reports/Measurement.cs +++ b/src/BenchmarkDotNet.Core/Reports/Measurement.cs @@ -66,6 +66,9 @@ public Measurement(int launchIndex, IterationMode iterationMode, int iterationIn /// An instance of if parsed successfully. Null in case of any trouble. public static Measurement Parse(ILogger logger, string line, int processIndex) { + if (line != null && line.StartsWith(GcStats.ResultsLinePrefix)) + return Error; + try { var lineSplit = line.Split(new[] { ':' }, StringSplitOptions.RemoveEmptyEntries); diff --git a/src/BenchmarkDotNet.Core/Running/BenchmarkRunnerCore.cs b/src/BenchmarkDotNet.Core/Running/BenchmarkRunnerCore.cs index f4d29e315d..c1dda70fa1 100644 --- a/src/BenchmarkDotNet.Core/Running/BenchmarkRunnerCore.cs +++ b/src/BenchmarkDotNet.Core/Running/BenchmarkRunnerCore.cs @@ -155,22 +155,26 @@ public static BenchmarkReport Run(Benchmark benchmark, ILogger logger, IConfig c try { if (!generateResult.IsGenerateSuccess) - return new BenchmarkReport(benchmark, generateResult, null, null, null); + return new BenchmarkReport(benchmark, generateResult, null, null, null, default(GcStats)); var buildResult = Build(logger, toolchain, generateResult, benchmark, resolver); if (!buildResult.IsBuildSuccess) - return new BenchmarkReport(benchmark, generateResult, buildResult, null, null); + return new BenchmarkReport(benchmark, generateResult, buildResult, null, null, default(GcStats)); - List executeResults = Execute(logger, benchmark, toolchain, buildResult, config, resolver); + var executeResults = Execute(logger, benchmark, toolchain, buildResult, config, resolver); var runs = new List(); + GcStats gcStats = default(GcStats); for (int index = 0; index < executeResults.Count; index++) { var executeResult = executeResults[index]; runs.AddRange(executeResult.Data.Select(line => Measurement.Parse(logger, line, index + 1)).Where(r => r.IterationMode != IterationMode.Unknown)); + + if (executeResult.Data.Any()) + gcStats = gcStats + GcStats.Parse(executeResult.Data.Last()); } - return new BenchmarkReport(benchmark, generateResult, buildResult, executeResults, runs); + return new BenchmarkReport(benchmark, generateResult, buildResult, executeResults, runs, gcStats); } finally { @@ -256,7 +260,7 @@ private static List Execute(ILogger logger, Benchmark benchmark, if (!measurements.Any()) { // Something went wrong during the benchmark, don't bother doing more runs - logger.WriteLineError($"No more Benchmark runs will be launched as NO measurements were obtained from the previous run!"); + logger.WriteLineError("No more Benchmark runs will be launched as NO measurements were obtained from the previous run!"); break; } @@ -280,7 +284,8 @@ private static List Execute(ILogger logger, Benchmark benchmark, var executeResult = toolchain.Executor.Execute(buildResult, benchmark, logger, resolver, compositeDiagnoser); var allRuns = executeResult.Data.Select(line => Measurement.Parse(logger, line, 0)).Where(r => r.IterationMode != IterationMode.Unknown).ToList(); - var report = new BenchmarkReport(benchmark, null, null, new[] { executeResult }, allRuns); + var gcStats = GcStats.Parse(executeResult.Data.Last()); + var report = new BenchmarkReport(benchmark, null, null, new[] { executeResult }, allRuns, gcStats); compositeDiagnoser.ProcessResults(benchmark, report); if (!executeResult.FoundExecutable) diff --git a/src/BenchmarkDotNet.Core/Toolchains/Core/CoreToolchain.cs b/src/BenchmarkDotNet.Core/Toolchains/Core/CoreToolchain.cs index 19b1bce5c8..0d9a359e0b 100644 --- a/src/BenchmarkDotNet.Core/Toolchains/Core/CoreToolchain.cs +++ b/src/BenchmarkDotNet.Core/Toolchains/Core/CoreToolchain.cs @@ -1,7 +1,5 @@ using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Environments; -using BenchmarkDotNet.Extensions; -using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Portability; @@ -12,7 +10,7 @@ namespace BenchmarkDotNet.Toolchains.Core { public class CoreToolchain : Toolchain { - private const string TargetFrameworkMoniker = "netcoreapp1.0"; + private const string TargetFrameworkMoniker = "netcoreapp1.1"; public static readonly IToolchain Instance = new CoreToolchain(); diff --git a/src/BenchmarkDotNet.Core/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs b/src/BenchmarkDotNet.Core/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs index 186fa10b2b..7ee9d28673 100644 --- a/src/BenchmarkDotNet.Core/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs +++ b/src/BenchmarkDotNet.Core/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs @@ -36,12 +36,13 @@ internal static bool ExecuteCommand(string commandWithArguments, string workingD { using (var process = new Process { StartInfo = BuildStartInfo(workingDirectory, commandWithArguments) }) { - using (new AsynchronousProcessOutputLogger(logger, process)) + using (new AsyncErrorOutputLogger(logger, process)) { process.Start(); // don't forget to call, otherwise logger will not get any events process.BeginErrorReadLine(); + process.BeginOutputReadLine(); process.WaitForExit((int)timeout.TotalMilliseconds); @@ -59,7 +60,7 @@ private static ProcessStartInfo BuildStartInfo(string workingDirectory, string a Arguments = arguments, UseShellExecute = false, CreateNoWindow = true, - RedirectStandardOutput = true, // we redirect it but never call process.BeginOutputReadLine() in order to ignore it + RedirectStandardOutput = true, RedirectStandardError = true }; } diff --git a/src/BenchmarkDotNet.Core/Toolchains/Executor.cs b/src/BenchmarkDotNet.Core/Toolchains/Executor.cs index c96e626f8b..edc5365cf1 100644 --- a/src/BenchmarkDotNet.Core/Toolchains/Executor.cs +++ b/src/BenchmarkDotNet.Core/Toolchains/Executor.cs @@ -48,7 +48,7 @@ private ExecuteResult Execute(Benchmark benchmark, ILogger logger, string exeNam { var loggerWithDiagnoser = new SynchronousProcessOutputLoggerWithDiagnoser(logger, process, diagnoser, benchmark); - return Execute(process, benchmark, loggerWithDiagnoser, diagnoser, logger); + return Execute(process, benchmark, loggerWithDiagnoser, logger); } } finally @@ -57,8 +57,7 @@ private ExecuteResult Execute(Benchmark benchmark, ILogger logger, string exeNam } } - private ExecuteResult Execute(Process process, Benchmark benchmark, SynchronousProcessOutputLoggerWithDiagnoser loggerWithDiagnoser, - IDiagnoser compositeDiagnoser, ILogger logger) + private ExecuteResult Execute(Process process, Benchmark benchmark, SynchronousProcessOutputLoggerWithDiagnoser loggerWithDiagnoser, ILogger logger) { consoleHandler.SetProcess(process); diff --git a/src/BenchmarkDotNet.Core/project.json b/src/BenchmarkDotNet.Core/project.json index 3cf4471cac..673fc09f1c 100644 --- a/src/BenchmarkDotNet.Core/project.json +++ b/src/BenchmarkDotNet.Core/project.json @@ -51,34 +51,18 @@ "System.Threading.Tasks.Extensions": "4.0.0" } }, - "netstandard1.5": { + "netcoreapp1.1": { "buildOptions": { "define": [ "CORE" ] }, "dependencies": { - "System.Linq": "4.1.0", - "System.Resources.ResourceManager": "4.0.1", - "Microsoft.CSharp": "4.0.1", - "Microsoft.Win32.Primitives": "4.0.1", - "System.Console": "4.0.0", - "System.Text.RegularExpressions": "4.1.0", - "System.Threading": "4.0.11", - "System.Reflection": "4.1.0", - "System.Reflection.Primitives": "4.0.1", - "System.Reflection.TypeExtensions": "4.1.0", - "System.Threading.Tasks.Extensions": "4.0.0", - "System.Threading.Thread": "4.0.0", - "System.Diagnostics.Process": "4.1.0", - "System.IO.FileSystem": "4.0.1", - "System.Runtime.InteropServices.RuntimeInformation": "4.0.0", + "Microsoft.NETCore.App": { + "type": "platform", + "version": "1.1.0" + }, "System.Runtime.Serialization.Primitives": "4.1.1", - "System.Diagnostics.Tools": "4.0.1", - "System.Runtime.InteropServices": "4.1.0", "Microsoft.DotNet.InternalAbstractions": "1.0.0", - "System.Reflection.Extensions": "4.0.1", - "System.Diagnostics.Debug": "4.0.11", - "System.Xml.XPath.XmlDocument": "4.0.1", - "System.Collections.Concurrent": "4.0.12" + "System.Xml.XPath.XmlDocument": "4.0.1" } } } diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/MemoryDiagnoser.cs b/src/BenchmarkDotNet.Diagnostics.Windows/MemoryDiagnoser.cs index 76d20979e8..eebb6fd995 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/MemoryDiagnoser.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/MemoryDiagnoser.cs @@ -1,4 +1,5 @@ -using BenchmarkDotNet.Columns; +using System; +using BenchmarkDotNet.Columns; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Reports; @@ -13,9 +14,9 @@ namespace BenchmarkDotNet.Diagnostics.Windows { + [Obsolete("Please use our new BenchmarkDotNet.Diagnosers.MemoryDiagnoser", true)] public class MemoryDiagnoser : EtwDiagnoser, IDiagnoser { - private readonly List output = new List(); private readonly Dictionary results = new Dictionary(); public IColumnProvider GetColumnProvider() => new SimpleColumnProvider( @@ -40,11 +41,7 @@ public void ProcessResults(Benchmark benchmark, BenchmarkReport report) results.Add(benchmark, stats); } - public void DisplayResults(ILogger logger) - { - foreach (var line in output) - logger.Write(line.Kind, line.Text); - } + public void DisplayResults(ILogger logger) { } private Stats ProcessEtwEvents(long totalOperations) { diff --git a/src/BenchmarkDotNet/project.json b/src/BenchmarkDotNet/project.json index 85984ca5f3..2b4c070d3d 100644 --- a/src/BenchmarkDotNet/project.json +++ b/src/BenchmarkDotNet/project.json @@ -48,7 +48,7 @@ } } }, - "netstandard1.5": { + "netcoreapp1.1": { "buildOptions": { "define": [ "CORE" ] }, diff --git a/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs b/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs index 0034bb43e3..ec5ff8fbaf 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs @@ -72,7 +72,8 @@ public RunResults Run() return new RunResults( new List() { default(Measurement) }, new List() { default(Measurement) }, - false); + false, + default(GcStats)); } public Job TargetJob { get; } diff --git a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs index 57c8ad5d55..1830523fde 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs @@ -1,171 +1,139 @@ -#if !CORE -using System; +using System; using System.Collections.Generic; using System.Globalization; using System.Linq; -using System.Runtime.CompilerServices; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Diagnosers; -using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; -using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; using BenchmarkDotNet.Tests.Loggers; using Xunit; using Xunit.Abstractions; -using BenchmarkDotNet.Reports; -using BenchmarkDotNet.Diagnostics.Windows; namespace BenchmarkDotNet.IntegrationTests { - public class NewVsStackalloc + public class MemoryDiagnoserTests { - [Benchmark] - public void New() => Consume(new byte[100]); + private readonly ITestOutputHelper output; - [Benchmark] - public unsafe void Stackalloc() + public MemoryDiagnoserTests(ITestOutputHelper outputHelper) { - var bytes = stackalloc byte[100]; - Consume(bytes); + output = outputHelper; } - [MethodImpl(MethodImplOptions.NoInlining)] - private void Consume(T input) { } - - [MethodImpl(MethodImplOptions.NoInlining)] - private unsafe void Consume(byte* input) { } - } - - public class AllocatingSetupAndCleanup - { - private List list; - - [Setup] - public void AllocatingSetUp() => AllocateUntilGcWakesUp(); - - [Benchmark] - public void AllocateNothing() { } - - [Cleanup] - public void AllocatingCleanUp() => AllocateUntilGcWakesUp(); - - private void AllocateUntilGcWakesUp() + public class AccurateAllocations { - int initialCollectionCount = GC.CollectionCount(0); - - while (initialCollectionCount == GC.CollectionCount(0)) - { - list = Enumerable.Range(0, 100).ToList(); - } + [Benchmark]public void Empty() { } + [Benchmark]public byte[] EightBytes() => new byte[8]; + [Benchmark]public byte[] SixtyFourBytes() => new byte[64]; } - } - - [KeepBenchmarkFiles()] - public class NoAllocationsAtAll - { - [Benchmark] - public void EmptyMethod() { } - } - - // this class is not compiled for CORE because it is using Diagnosers that currently do not support Core - public class MemoryDiagnoserTests - { - private readonly ITestOutputHelper output; - public MemoryDiagnoserTests(ITestOutputHelper outputHelper) + [Fact] + public void MemoryDiagnoserIsAccurate() { - output = outputHelper; + long objectAllocationOverhead = IntPtr.Size * 3; // pointer to method table + object header word + pointer to the object + AssertAllocations(typeof(AccurateAllocations), 200, new Dictionary> + { + { "Empty", allocatedBytes => allocatedBytes == 0 }, + { "EightBytes", allocatedBytes => allocatedBytes == 8 + objectAllocationOverhead }, + { "SixtyFourBytes", allocatedBytes => allocatedBytes == 64 + objectAllocationOverhead }, + }); } - [Fact(Skip = "Temporarily suppressed, see https://github.com/dotnet/BenchmarkDotNet/issues/208")] - public void MemoryDiagnoserTracksHeapMemoryAllocation() + public class AllocatingSetupAndCleanup { - var memoryDiagnoser = new MemoryDiagnoser(); - var config = CreateConfig(memoryDiagnoser, 50); - var benchmarks = BenchmarkConverter.TypeToBenchmarks(typeof(NewVsStackalloc), config); + private List list; - var summary = BenchmarkRunner.Run((Benchmark[])benchmarks, config); + [Benchmark]public void AllocateNothing() { } - var gcCollectionColumns = GetColumns(memoryDiagnoser).ToArray(); - var stackallocBenchmarks = benchmarks.Where(benchmark => benchmark.DisplayInfo.Contains("Stackalloc")); - var newArrayBenchmarks = benchmarks.Where(benchmark => benchmark.DisplayInfo.Contains("New")); + [Setup]public void AllocatingSetUp() => AllocateUntilGcWakesUp(); + [Cleanup]public void AllocatingCleanUp() => AllocateUntilGcWakesUp(); - const int gen0Index = 0; - - foreach (var benchmark in stackallocBenchmarks) + private void AllocateUntilGcWakesUp() { - var gen0Collections = gcCollectionColumns[gen0Index].GetValue(summary, benchmark); + int initialCollectionCount = GC.CollectionCount(0); - Assert.Equal("-", gen0Collections); + while (initialCollectionCount == GC.CollectionCount(0)) + list = Enumerable.Range(0, 100).ToList(); } + } - foreach (var benchmark in newArrayBenchmarks) + [Fact] + public void MemoryDiagnoserDoesNotIncludeAllocationsFromSetupAndCleanup() + { + AssertAllocations(typeof(AllocatingSetupAndCleanup), 100, new Dictionary> { - var gen0Str = gcCollectionColumns[gen0Index].GetValue(summary, benchmark); - - AssertParsed(gen0Str, gen0Value => gen0Value > 0); - } + { "AllocateNothing", allocatedBytes => allocatedBytes == 0 } + }); } - [Fact(Skip = "Temporarily suppressed, see https://github.com/dotnet/BenchmarkDotNet/issues/208")] - public void MemoryDiagnoserDoesNotIncludeAllocationsFromSetupAndCleanup() + public class NoAllocationsAtAll { - AssertZeroAllocations(typeof(AllocatingSetupAndCleanup), "AllocateNothing", targetCount: 50); + [Benchmark]public void EmptyMethod() { } } - [Fact(Skip = "Temporarily suppressed, see https://github.com/dotnet/BenchmarkDotNet/issues/208")] + [Fact] public void EngineShouldNotInterfereAllocationResults() { - AssertZeroAllocations(typeof(NoAllocationsAtAll), "EmptyMethod", targetCount: 5000); // we need a lot of iterations to be sure!! + AssertAllocations(typeof(NoAllocationsAtAll), 100, new Dictionary> + { + { "EmptyMethod", allocatedBytes => allocatedBytes == 0 } + }); } - public void AssertZeroAllocations(Type benchmarkType, string benchmarkMethodName, int targetCount) + private void AssertAllocations(Type benchmarkType, int targetCount, + Dictionary> benchmarksAllocationsValidators) { - var memoryDiagnoser = new MemoryDiagnoser(); + var memoryDiagnoser = MemoryDiagnoser.Default; var config = CreateConfig(memoryDiagnoser, targetCount); var benchmarks = BenchmarkConverter.TypeToBenchmarks(benchmarkType, config); var summary = BenchmarkRunner.Run((Benchmark[])benchmarks, config); - var allocationColumn = GetColumns(memoryDiagnoser).Single(); - var allocateNothingBenchmarks = benchmarks.Where(benchmark => benchmark.DisplayInfo.Contains(benchmarkMethodName)); + var allocationColumn = GetColumns(memoryDiagnoser, summary).Single(); - foreach (var benchmark in allocateNothingBenchmarks) + foreach (var benchmarkAllocationsValidator in benchmarksAllocationsValidators) { - var allocations = allocationColumn.GetValue(summary, benchmark); + var allocatingBenchmarks = benchmarks.Where(benchmark => benchmark.DisplayInfo.Contains(benchmarkAllocationsValidator.Key)); + + foreach (var benchmark in allocatingBenchmarks) + { + var allocations = allocationColumn.GetValue(summary, benchmark); - AssertParsed(allocations, allocatedBytes => allocatedBytes == 0); + AssertParsed(allocations, benchmarkAllocationsValidator.Value); + } } } private IConfig CreateConfig(IDiagnoser diagnoser, int targetCount) { return ManualConfig.CreateEmpty() - .With( - new Job(Job.Dry) - { - Run = { LaunchCount = 1, WarmupCount = 1, TargetCount = targetCount }, - Env = { Gc = { Force = false }} - }) - .With(DefaultConfig.Instance.GetLoggers().ToArray()) - .With(DefaultColumnProviders.Instance) - .With(diagnoser) - .With(new OutputLogger(output)); + .With( + Job.Dry + .WithLaunchCount(1) + .WithWarmupCount(1) + .WithTargetCount(targetCount) + .WithInvocationCount(100) + .WithGcForce(false)) + .With(DefaultConfig.Instance.GetLoggers().ToArray()) + .With(DefaultColumnProviders.Instance) + .With(diagnoser) + .With(new OutputLogger(output)); } - private static T[] GetColumns(MemoryDiagnoser memoryDiagnoser) - => memoryDiagnoser.GetColumnProvider().GetColumns(null).OfType().ToArray(); + private static T[] GetColumns(MemoryDiagnoser memoryDiagnoser, Summary summary) + => memoryDiagnoser.GetColumnProvider().GetColumns(summary).OfType().ToArray(); - private static void AssertParsed(string text, Predicate condition) + private static void AssertParsed(string text, Predicate condition) { - double value; - if (double.TryParse(text, NumberStyles.Number, HostEnvironmentInfo.MainCultureInfo, out value)) + long value; + if (long.TryParse(text.Replace(" B", string.Empty), NumberStyles.Number, HostEnvironmentInfo.MainCultureInfo, out value)) { - Assert.True(condition(value)); + Assert.True(condition(value), $"Failed for value {value}"); } else { @@ -173,5 +141,4 @@ private static void AssertParsed(string text, Predicate condition) } } } -} -#endif \ No newline at end of file +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/project.json b/tests/BenchmarkDotNet.IntegrationTests/project.json index 525d7d205f..3150dc52e4 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/project.json +++ b/tests/BenchmarkDotNet.IntegrationTests/project.json @@ -21,7 +21,7 @@ "allowUnsafe": true }, "frameworks": { - "netcoreapp1.0": { + "netcoreapp1.1": { "buildOptions": { "define": [ "CORE" ] }, @@ -33,7 +33,7 @@ "dependencies": { "Microsoft.NETCore.App": { "type": "platform", - "version": "1.0.0" + "version": "1.1.0" }, "System.ComponentModel.EventBasedAsync": "4.0.11" } diff --git a/tests/BenchmarkDotNet.Tests/Engine/EngineResultStageTests.cs b/tests/BenchmarkDotNet.Tests/Engine/EngineResultStageTests.cs index 9d83901a06..f81d3e068c 100644 --- a/tests/BenchmarkDotNet.Tests/Engine/EngineResultStageTests.cs +++ b/tests/BenchmarkDotNet.Tests/Engine/EngineResultStageTests.cs @@ -25,7 +25,7 @@ public void OutliersTest() private static void CheckResults(int expectedResultCount, List measurements, bool removeOutliers) { - Assert.Equal(expectedResultCount, new RunResults(null, measurements, removeOutliers).GetMeasurements().Count()); + Assert.Equal(expectedResultCount, new RunResults(null, measurements, removeOutliers, default(GcStats)).GetMeasurements().Count()); } private static void Add(List measurements, int time) diff --git a/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs b/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs index 5377cc4cbd..368e9146a3 100644 --- a/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs +++ b/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Runtime; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Engines; @@ -42,7 +43,7 @@ private static BenchmarkReport CreateReport(IConfig config) { new Measurement(1, IterationMode.Result, 1, 1, 1) }; - return new BenchmarkReport(benchmark, buildResult, buildResult, new List { executeResult }, measurements); + return new BenchmarkReport(benchmark, buildResult, buildResult, new List { executeResult }, measurements, default(GcStats)); } public class MockBenchmarkClass diff --git a/tests/BenchmarkDotNet.Tests/project.json b/tests/BenchmarkDotNet.Tests/project.json index e5c5bfb8c0..45216578e2 100644 --- a/tests/BenchmarkDotNet.Tests/project.json +++ b/tests/BenchmarkDotNet.Tests/project.json @@ -20,7 +20,7 @@ "copyToOutput": [ "xunit.runner.json" ] }, "frameworks": { - "netcoreapp1.0": { + "netcoreapp1.1": { "imports": [ "dnxcore50", "portable-net45+win8", @@ -29,7 +29,7 @@ "dependencies": { "Microsoft.NETCore.App": { "type": "platform", - "version": "1.0.0" + "version": "1.1.0" }, "System.ComponentModel.EventBasedAsync": "4.0.11" } diff --git a/tests/runCoreTests.cmd b/tests/runCoreTests.cmd index 24c97af269..91992ad945 100644 --- a/tests/runCoreTests.cmd +++ b/tests/runCoreTests.cmd @@ -15,7 +15,7 @@ echo ----------------------------- echo Running Core tests echo ----------------------------- -call dotnet test "BenchmarkDotNet.IntegrationTests/" --configuration Release --framework netcoreapp1.0 2> failedCoreTests.txt +call dotnet test "BenchmarkDotNet.IntegrationTests/" --configuration Release --framework netcoreapp1.1 2> failedCoreTests.txt if NOT %ERRORLEVEL% == 0 ( echo CORE tests has failed