From 23f3b29b4cf0c13f49f47609b26b32a30d10289e Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Sat, 15 Oct 2016 22:39:16 +0200 Subject: [PATCH 01/11] built-in accurate and cross platform Memory Diagnoser, fixes #186, fixes #200 --- BenchmarkDotNet.sln | 2 +- .../Intro/IntroGcMode.cs | 4 +- .../Attributes}/MemoryDiagnoserAttribute.cs | 5 +- .../Configs/DefaultConfig.cs | 5 +- .../Diagnosers/CompositeDiagnoser.cs | 2 +- .../Diagnosers/MemoryDiagnoser.cs | 110 ++++++++++++ src/BenchmarkDotNet.Core/Engines/Engine.cs | 24 ++- src/BenchmarkDotNet.Core/Engines/GcStats.cs | 120 +++++++++++++ .../Engines/RunResults.cs | 9 +- ...hronousProcessOutputLoggerWithDiagnoser.cs | 2 - .../Reports/BenchmarkReport.cs | 5 +- .../Reports/Measurement.cs | 3 + .../Running/BenchmarkRunnerCore.cs | 17 +- .../MemoryDiagnoser.cs | 11 +- .../CustomEngineTests.cs | 3 +- .../MemoryDiagnoserTests.cs | 169 ++++++++---------- .../Mocks/MockFactory.cs | 3 +- 17 files changed, 370 insertions(+), 124 deletions(-) rename src/{BenchmarkDotNet.Diagnostics.Windows/Configs => BenchmarkDotNet.Core/Attributes}/MemoryDiagnoserAttribute.cs (60%) create mode 100644 src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs create mode 100644 src/BenchmarkDotNet.Core/Engines/GcStats.cs diff --git a/BenchmarkDotNet.sln b/BenchmarkDotNet.sln index faad78bfca..b8722fdb99 100644 --- a/BenchmarkDotNet.sln +++ b/BenchmarkDotNet.sln @@ -37,7 +37,7 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Integration EndProject Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "BenchmarkDotNet.Samples.FSharp", "samples\BenchmarkDotNet.Samples.FSharp\BenchmarkDotNet.Samples.FSharp.fsproj", "{A329F00E-4B9D-4BC6-B688-92698D773CBF}" ProjectSection(ProjectDependencies) = postProject - {95F5D645-19E3-432F-95D4-C5EA374DD15B} = {95F5D645-19E3-432F-95D4-C5EA374DD15B} + {AF1E6F8A-5C63-465F-96F4-5E5F183A33B9} = {AF1E6F8A-5C63-465F-96F4-5E5F183A33B9} EndProjectSection EndProject Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "BenchmarkDotNet.IntegrationTests.FSharp", "tests\BenchmarkDotNet.IntegrationTests.FSharp\BenchmarkDotNet.IntegrationTests.FSharp.fsproj", "{367FAFE1-A1C8-4AA1-9334-F4762E128DBB}" diff --git a/samples/BenchmarkDotNet.Samples/Intro/IntroGcMode.cs b/samples/BenchmarkDotNet.Samples/Intro/IntroGcMode.cs index 33982dbbbf..3a0beacccb 100644 --- a/samples/BenchmarkDotNet.Samples/Intro/IntroGcMode.cs +++ b/samples/BenchmarkDotNet.Samples/Intro/IntroGcMode.cs @@ -8,6 +8,7 @@ namespace BenchmarkDotNet.Samples.Intro { [Config(typeof(Config))] [OrderProvider(SummaryOrderPolicy.FastestToSlowest)] + [MemoryDiagnoser] public class IntroGcMode { private class Config : ManualConfig @@ -18,9 +19,6 @@ public Config() Add(Job.MediumRun.WithGcServer(true).WithGcForce(false).WithId("Server")); Add(Job.MediumRun.WithGcServer(false).WithGcForce(true).WithId("Workstation")); Add(Job.MediumRun.WithGcServer(false).WithGcForce(false).WithId("WorkstationForce")); -#if !CORE - Add(new Diagnostics.Windows.MemoryDiagnoser()); -#endif } } diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/Configs/MemoryDiagnoserAttribute.cs b/src/BenchmarkDotNet.Core/Attributes/MemoryDiagnoserAttribute.cs similarity index 60% rename from src/BenchmarkDotNet.Diagnostics.Windows/Configs/MemoryDiagnoserAttribute.cs rename to src/BenchmarkDotNet.Core/Attributes/MemoryDiagnoserAttribute.cs index aa03c926cf..a0a5c85697 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/Configs/MemoryDiagnoserAttribute.cs +++ b/src/BenchmarkDotNet.Core/Attributes/MemoryDiagnoserAttribute.cs @@ -1,7 +1,8 @@ using System; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Diagnosers; -namespace BenchmarkDotNet.Diagnostics.Windows.Configs +namespace BenchmarkDotNet.Attributes { public class MemoryDiagnoserAttribute : Attribute, IConfigSource { @@ -9,7 +10,7 @@ public class MemoryDiagnoserAttribute : Attribute, IConfigSource public MemoryDiagnoserAttribute() { - Config = ManualConfig.CreateEmpty().With(new MemoryDiagnoser()); + Config = ManualConfig.CreateEmpty().With(MemoryDiagnoser.Default); } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet.Core/Configs/DefaultConfig.cs b/src/BenchmarkDotNet.Core/Configs/DefaultConfig.cs index f63e851a8f..284e6e51dc 100644 --- a/src/BenchmarkDotNet.Core/Configs/DefaultConfig.cs +++ b/src/BenchmarkDotNet.Core/Configs/DefaultConfig.cs @@ -59,7 +59,10 @@ public IEnumerable GetValidators() public bool KeepBenchmarkFiles => false; - public IEnumerable GetDiagnosers() => Enumerable.Empty(); + public IEnumerable GetDiagnosers() + { + yield return MemoryDiagnoser.Default; + } // Make the Diagnosers lazy-loaded, so they are only instantiated if neededs public static readonly Lazy LazyLoadedDiagnosers = diff --git a/src/BenchmarkDotNet.Core/Diagnosers/CompositeDiagnoser.cs b/src/BenchmarkDotNet.Core/Diagnosers/CompositeDiagnoser.cs index 464fce63ce..9172d8f6ae 100644 --- a/src/BenchmarkDotNet.Core/Diagnosers/CompositeDiagnoser.cs +++ b/src/BenchmarkDotNet.Core/Diagnosers/CompositeDiagnoser.cs @@ -14,7 +14,7 @@ public class CompositeDiagnoser : IDiagnoser public CompositeDiagnoser(params IDiagnoser[] diagnosers) { - this.diagnosers = diagnosers; + this.diagnosers = diagnosers.Distinct().ToArray(); } public IColumnProvider GetColumnProvider() diff --git a/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs b/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs new file mode 100644 index 0000000000..33edd67320 --- /dev/null +++ b/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs @@ -0,0 +1,110 @@ +using System.Collections.Generic; +using System.Diagnostics; +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using System.Linq; +using BenchmarkDotNet.Engines; + +namespace BenchmarkDotNet.Diagnosers +{ + public class MemoryDiagnoser : IDiagnoser + { + public static readonly MemoryDiagnoser Default = new MemoryDiagnoser(); + + private readonly Dictionary results = new Dictionary(); + + public IColumnProvider GetColumnProvider() => new SimpleColumnProvider( + new GCCollectionColumn(results, 0), + new GCCollectionColumn(results, 1), + new GCCollectionColumn(results, 2), + new AllocationColumn(results)); + + // the methods are left empty on purpose + // the action takes places in other process, and the values are gathered by Engine + public void BeforeAnythingElse(Process process, Benchmark benchmark) { } + public void AfterSetup(Process process, Benchmark benchmark) { } + public void BeforeCleanup() { } + + public void ProcessResults(Benchmark benchmark, BenchmarkReport report) + { + results.Add(benchmark, report.GcStats); + } + + public void DisplayResults(ILogger logger) { } + + public class AllocationColumn : IColumn + { + private readonly Dictionary results; + + public AllocationColumn(Dictionary results) + { + this.results = results; + } + + public string Id => nameof(AllocationColumn); + public string ColumnName => "Bytes Allocated/Op"; + public bool IsDefault(Summary summary, Benchmark benchmark) => false; + public bool IsAvailable(Summary summary) => true; + public bool AlwaysShow => true; + public ColumnCategory Category => ColumnCategory.Diagnoser; + public int PriorityInCategory => 0; + + public string GetValue(Summary summary, Benchmark benchmark) + { +#if !CORE + if (results.ContainsKey(benchmark)) + { + var result = results[benchmark]; + // TODO scale this based on the minimum value in the column, i.e. use B/KB/MB as appropriate + return (result.AllocatedBytes / result.TotalOperations).ToString("N2", HostEnvironmentInfo.MainCultureInfo); + } + return "N/A"; +#else + return "?"; +#endif + } + } + + public class GCCollectionColumn : IColumn + { + private Dictionary results; + private int generation; + // TODO also need to find a sensible way of including this in the column name? + private long opsPerGCCount; + + public GCCollectionColumn(Dictionary results, int generation) + { + ColumnName = $"Gen {generation}"; + this.results = results; + this.generation = generation; + opsPerGCCount = results.Any() ? results.Min(r => r.Value.TotalOperations) : 1; + } + + public bool IsDefault(Summary summary, Benchmark benchmark) => true; + public string Id => $"{nameof(GCCollectionColumn)}{generation}"; + public string ColumnName { get; } + public bool IsAvailable(Summary summary) => true; + public bool AlwaysShow => true; + public ColumnCategory Category => ColumnCategory.Diagnoser; + public int PriorityInCategory => 0; + + public string GetValue(Summary summary, Benchmark benchmark) + { + if (results.ContainsKey(benchmark)) + { + var result = results[benchmark]; + var value = generation == 0 ? result.Gen0Collections : + generation == 1 ? result.Gen1Collections : result.Gen2Collections; + + if (value == 0) + return "-"; // make zero more obvious + return (value / (double)result.TotalOperations * opsPerGCCount).ToString("N2", HostEnvironmentInfo.MainCultureInfo); + } + return "N/A"; + } + } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet.Core/Engines/Engine.cs b/src/BenchmarkDotNet.Core/Engines/Engine.cs index 2a86eed96a..277bec0448 100644 --- a/src/BenchmarkDotNet.Core/Engines/Engine.cs +++ b/src/BenchmarkDotNet.Core/Engines/Engine.cs @@ -35,6 +35,7 @@ public class Engine : IEngine private readonly EngineWarmupStage warmupStage; private readonly EngineTargetStage targetStage; private bool isJitted, isPreAllocated; + private int forcedFullGarbageCollections; internal Engine(Action idleAction, Action mainAction, Job targetJob, Action setupAction, Action cleanupAction, long operationsPerInvoke, bool isDiagnoserAttached) { @@ -98,9 +99,17 @@ public RunResults Run() warmupStage.RunMain(invokeCount, UnrollFactor); } + // we enable monitoring after pilot & warmup, just to ignore the memory allocated by these runs + EnableMonitoring(); + var initialGcStats = GcStats.ReadInitial(IsDiagnoserAttached); + var main = targetStage.RunMain(invokeCount, UnrollFactor); - return new RunResults(idle, main); + var finalGcStats = GcStats.ReadFinal(IsDiagnoserAttached); + var forcedCollections = GcStats.FromForced(forcedFullGarbageCollections); + var workGcHasDone = finalGcStats - forcedCollections - initialGcStats; + + return new RunResults(idle, main, workGcHasDone); } public Measurement RunIteration(IterationData data) @@ -135,11 +144,13 @@ private void GcCollect() ForceGcCollect(); } - private static void ForceGcCollect() + private void ForceGcCollect() { GC.Collect(); GC.WaitForPendingFinalizers(); GC.Collect(); + + forcedFullGarbageCollections += 2; } public void WriteLine(string text) @@ -156,6 +167,15 @@ public void WriteLine() Console.WriteLine(); } + private void EnableMonitoring() + { + if(!IsDiagnoserAttached) // it could affect the results, we do this in separate, diagnostics-only run + return; +#if CLASSIC + AppDomain.MonitoringIsEnabled = true; +#endif + } + private void EnsureNothingIsPrintedWhenDiagnoserIsAttached() { if (IsDiagnoserAttached) diff --git a/src/BenchmarkDotNet.Core/Engines/GcStats.cs b/src/BenchmarkDotNet.Core/Engines/GcStats.cs new file mode 100644 index 0000000000..f28310a372 --- /dev/null +++ b/src/BenchmarkDotNet.Core/Engines/GcStats.cs @@ -0,0 +1,120 @@ +using System; + +namespace BenchmarkDotNet.Engines +{ + public struct GcStats + { + internal const string ResultsLinePrefix = "GC: "; + + private GcStats(int gen0Collections, int gen1Collections, int gen2Collections, long allocatedBytes, long totalOperations) + { + Gen0Collections = gen0Collections; + Gen1Collections = gen1Collections; + Gen2Collections = gen2Collections; + AllocatedBytes = allocatedBytes; + TotalOperations = totalOperations; + } + + // did not use array here just to avoid heap allocation + public int Gen0Collections { get; } + public int Gen1Collections { get; } + public int Gen2Collections { get; } + public long AllocatedBytes { get; } + public long TotalOperations { get; } + + public static GcStats operator +(GcStats left, GcStats right) + { + return new GcStats( + left.Gen0Collections + right.Gen0Collections, + left.Gen1Collections + right.Gen1Collections, + left.Gen2Collections + right.Gen2Collections, + left.AllocatedBytes + right.AllocatedBytes, + left.TotalOperations + right.TotalOperations); + } + + public static GcStats operator -(GcStats left, GcStats right) + { + return new GcStats( + Math.Max(0, left.Gen0Collections - right.Gen0Collections), + Math.Max(0, left.Gen1Collections - right.Gen1Collections), + Math.Max(0, left.Gen2Collections - right.Gen2Collections), + Math.Max(0, left.AllocatedBytes - right.AllocatedBytes), + Math.Max(0, left.TotalOperations - right.TotalOperations)); + } + + public GcStats WithTotalOperations(long totalOperationsCount) + => this + new GcStats(0, 0, 0, 0, totalOperationsCount); + + internal static GcStats ReadInitial(bool isDiagnosticsEnabled) + { + // this might force GC.Collect, so we want to do this before collecting collections counts + long allocatedBytes = GetAllocatedBytes(isDiagnosticsEnabled); + + return new GcStats( + GC.CollectionCount(0), + GC.CollectionCount(1), + GC.CollectionCount(2), + allocatedBytes, + 0); + } + + internal static GcStats ReadFinal(bool isDiagnosticsEnabled) + { + return new GcStats( + GC.CollectionCount(0), + GC.CollectionCount(1), + GC.CollectionCount(2), + + // this might force GC.Collect, so we want to do this after collecting collections counts + // to exclude this single full forced collection from results + GetAllocatedBytes(isDiagnosticsEnabled), + 0); + } + + public static GcStats FromForced(int forcedFullGarbageCollections) + => new GcStats(forcedFullGarbageCollections, forcedFullGarbageCollections, forcedFullGarbageCollections, 0, 0); + + private static long GetAllocatedBytes(bool isDiagnosticsEnabled) + { +#if NETCOREAPP11 // when MS releases new version of .NET Runtime to nuget.org + return GC.GetAllocatedBytesForCurrentThread(); // https://github.com/dotnet/corefx/pull/12489 +#elif CLASSIC + if (!isDiagnosticsEnabled) + return 0; + + // "This instance Int64 property returns the number of bytes that have been allocated by a specific + // AppDomain. The number is accurate as of the last garbage collection." - CLR via C# + // so we enforce GC.Collect here just to make sure we get accurate results + GC.Collect(); + return AppDomain.CurrentDomain.MonitoringTotalAllocatedMemorySize; +#else + return 0; // currently for .NET Core +#endif + } + + internal string ToOutputLine() + => $"{ResultsLinePrefix} {Gen0Collections} {Gen1Collections} {Gen2Collections} {AllocatedBytes} {TotalOperations}"; + + internal static GcStats Parse(string line) + { + if(!line.StartsWith(ResultsLinePrefix)) + throw new NotSupportedException($"Line must start with {ResultsLinePrefix}"); + + int gen0, gen1, gen2; + long allocatedBytes, totalOperationsCount; + var measurementSplit = line.Remove(0, ResultsLinePrefix.Length).Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); + if (!int.TryParse(measurementSplit[0], out gen0) + || !int.TryParse(measurementSplit[1], out gen1) + || !int.TryParse(measurementSplit[2], out gen2) + || !long.TryParse(measurementSplit[3], out allocatedBytes) + || !long.TryParse(measurementSplit[4], out totalOperationsCount)) + { + throw new NotSupportedException("Invalid string"); + } + + return new GcStats(gen0, gen1, gen2, allocatedBytes, totalOperationsCount); + } + + public override string ToString() => ToOutputLine(); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet.Core/Engines/RunResults.cs b/src/BenchmarkDotNet.Core/Engines/RunResults.cs index bf63ee846d..a7970e6a33 100644 --- a/src/BenchmarkDotNet.Core/Engines/RunResults.cs +++ b/src/BenchmarkDotNet.Core/Engines/RunResults.cs @@ -10,11 +10,13 @@ public struct RunResults { public List Idle { get; } public List Main { get; } + public GcStats GCStats { get; } - public RunResults(List idle, List main) + public RunResults(List idle, List main, GcStats gcStats) { Idle = idle; Main = main; + GCStats = gcStats; } public void Print() @@ -23,8 +25,12 @@ public void Print() // TODO: check if resulted measurements are too small (like < 0.1ns) double overhead = Idle == null ? 0.0 : new Statistics(Idle.Select(m => m.Nanoseconds)).Mean; int resultIndex = 0; + long totalOperationsCount = 0; foreach (var measurement in Main) { + if (!measurement.IterationMode.IsIdle()) + totalOperationsCount += measurement.Operations; + var resultMeasurement = new Measurement( measurement.LaunchIndex, IterationMode.Result, @@ -33,6 +39,7 @@ public void Print() Math.Max(0, measurement.Nanoseconds - overhead)); WriteLine(resultMeasurement.ToOutputLine()); } + WriteLine(GCStats.WithTotalOperations(totalOperationsCount).ToOutputLine()); WriteLine(); } diff --git a/src/BenchmarkDotNet.Core/Loggers/SynchronousProcessOutputLoggerWithDiagnoser.cs b/src/BenchmarkDotNet.Core/Loggers/SynchronousProcessOutputLoggerWithDiagnoser.cs index 56e798b006..7bfa8246ec 100644 --- a/src/BenchmarkDotNet.Core/Loggers/SynchronousProcessOutputLoggerWithDiagnoser.cs +++ b/src/BenchmarkDotNet.Core/Loggers/SynchronousProcessOutputLoggerWithDiagnoser.cs @@ -14,8 +14,6 @@ internal class SynchronousProcessOutputLoggerWithDiagnoser private readonly Benchmark benchmark; private readonly IDiagnoser diagnoser; - private bool diagnosticsAlreadyRun = false; - public SynchronousProcessOutputLoggerWithDiagnoser(ILogger logger, Process process, IDiagnoser diagnoser, Benchmark benchmark) { if (!process.StartInfo.RedirectStandardOutput) diff --git a/src/BenchmarkDotNet.Core/Reports/BenchmarkReport.cs b/src/BenchmarkDotNet.Core/Reports/BenchmarkReport.cs index 283724baa0..4150f5614a 100644 --- a/src/BenchmarkDotNet.Core/Reports/BenchmarkReport.cs +++ b/src/BenchmarkDotNet.Core/Reports/BenchmarkReport.cs @@ -12,6 +12,7 @@ public sealed class BenchmarkReport { public Benchmark Benchmark { get; } public IList AllMeasurements { get; } + public GcStats GcStats { get; } public GenerateResult GenerateResult { get; } public BuildResult BuildResult { get; } @@ -28,13 +29,15 @@ public BenchmarkReport( GenerateResult generateResult, BuildResult buildResult, IList executeResults, - IList allMeasurements) + IList allMeasurements, + GcStats gcStats) { Benchmark = benchmark; GenerateResult = generateResult; BuildResult = buildResult; ExecuteResults = executeResults ?? new ExecuteResult[0]; AllMeasurements = allMeasurements ?? new Measurement[0]; + GcStats = gcStats; } public override string ToString() => $"{Benchmark.DisplayInfo}, {AllMeasurements.Count} runs"; diff --git a/src/BenchmarkDotNet.Core/Reports/Measurement.cs b/src/BenchmarkDotNet.Core/Reports/Measurement.cs index 1b07e8edef..ba66767469 100644 --- a/src/BenchmarkDotNet.Core/Reports/Measurement.cs +++ b/src/BenchmarkDotNet.Core/Reports/Measurement.cs @@ -66,6 +66,9 @@ public Measurement(int launchIndex, IterationMode iterationMode, int iterationIn /// An instance of if parsed successfully. Null in case of any trouble. public static Measurement Parse(ILogger logger, string line, int processIndex) { + if (line != null && line.StartsWith(GcStats.ResultsLinePrefix)) + return Error; + try { var lineSplit = line.Split(new[] { ':' }, StringSplitOptions.RemoveEmptyEntries); diff --git a/src/BenchmarkDotNet.Core/Running/BenchmarkRunnerCore.cs b/src/BenchmarkDotNet.Core/Running/BenchmarkRunnerCore.cs index 4efb56fa02..0e215f67fa 100644 --- a/src/BenchmarkDotNet.Core/Running/BenchmarkRunnerCore.cs +++ b/src/BenchmarkDotNet.Core/Running/BenchmarkRunnerCore.cs @@ -156,22 +156,26 @@ private static BenchmarkReport Run(Benchmark benchmark, ILogger logger, IConfig try { if (!generateResult.IsGenerateSuccess) - return new BenchmarkReport(benchmark, generateResult, null, null, null); + return new BenchmarkReport(benchmark, generateResult, null, null, null, default(GcStats)); var buildResult = Build(logger, toolchain, generateResult, benchmark, resolver); if (!buildResult.IsBuildSuccess) - return new BenchmarkReport(benchmark, generateResult, buildResult, null, null); + return new BenchmarkReport(benchmark, generateResult, buildResult, null, null, default(GcStats)); - List executeResults = Execute(logger, benchmark, toolchain, buildResult, config, resolver); + var executeResults = Execute(logger, benchmark, toolchain, buildResult, config, resolver); var runs = new List(); + GcStats gcStats = default(GcStats); for (int index = 0; index < executeResults.Count; index++) { var executeResult = executeResults[index]; runs.AddRange(executeResult.Data.Select(line => Measurement.Parse(logger, line, index + 1)).Where(r => r.IterationMode != IterationMode.Unknown)); + + if (executeResult.Data.Any()) + gcStats = gcStats + GcStats.Parse(executeResult.Data.Last()); } - return new BenchmarkReport(benchmark, generateResult, buildResult, executeResults, runs); + return new BenchmarkReport(benchmark, generateResult, buildResult, executeResults, runs, gcStats); } finally { @@ -250,7 +254,7 @@ private static List Execute(ILogger logger, Benchmark benchmark, if (!measurements.Any()) { // Something went wrong during the benchmark, don't bother doing more runs - logger.WriteLineError($"No more Benchmark runs will be launched as NO measurements were obtained from the previous run!"); + logger.WriteLineError("No more Benchmark runs will be launched as NO measurements were obtained from the previous run!"); break; } @@ -274,7 +278,8 @@ private static List Execute(ILogger logger, Benchmark benchmark, var executeResult = toolchain.Executor.Execute(buildResult, benchmark, logger, resolver, compositeDiagnoser); var allRuns = executeResult.Data.Select(line => Measurement.Parse(logger, line, 0)).Where(r => r.IterationMode != IterationMode.Unknown).ToList(); - var report = new BenchmarkReport(benchmark, null, null, new[] { executeResult }, allRuns); + var gcStats = GcStats.Parse(executeResult.Data.Last()); + var report = new BenchmarkReport(benchmark, null, null, new[] { executeResult }, allRuns, gcStats); compositeDiagnoser.ProcessResults(benchmark, report); if (!executeResult.FoundExecutable) diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/MemoryDiagnoser.cs b/src/BenchmarkDotNet.Diagnostics.Windows/MemoryDiagnoser.cs index 76d20979e8..eebb6fd995 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/MemoryDiagnoser.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/MemoryDiagnoser.cs @@ -1,4 +1,5 @@ -using BenchmarkDotNet.Columns; +using System; +using BenchmarkDotNet.Columns; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Reports; @@ -13,9 +14,9 @@ namespace BenchmarkDotNet.Diagnostics.Windows { + [Obsolete("Please use our new BenchmarkDotNet.Diagnosers.MemoryDiagnoser", true)] public class MemoryDiagnoser : EtwDiagnoser, IDiagnoser { - private readonly List output = new List(); private readonly Dictionary results = new Dictionary(); public IColumnProvider GetColumnProvider() => new SimpleColumnProvider( @@ -40,11 +41,7 @@ public void ProcessResults(Benchmark benchmark, BenchmarkReport report) results.Add(benchmark, stats); } - public void DisplayResults(ILogger logger) - { - foreach (var line in output) - logger.Write(line.Kind, line.Text); - } + public void DisplayResults(ILogger logger) { } private Stats ProcessEtwEvents(long totalOperations) { diff --git a/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs b/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs index 52b5d9c3ab..599c49cc31 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs @@ -71,7 +71,8 @@ public RunResults Run() return new RunResults( new List() { default(Measurement) }, - new List() { default(Measurement) }); + new List() { default(Measurement) }, + default(GcStats)); } public Job TargetJob { get; } diff --git a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs index 7fa2f7f4aa..c96681bbf6 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs @@ -1,5 +1,4 @@ -#if !CORE -using System; +using System; using System.Collections.Generic; using System.Globalization; using System.Linq; @@ -17,144 +16,125 @@ using Xunit; using Xunit.Abstractions; using BenchmarkDotNet.Reports; -using BenchmarkDotNet.Diagnostics.Windows; namespace BenchmarkDotNet.IntegrationTests { - public class NewVsStackalloc + public class MemoryDiagnoserTests { - [Benchmark] - public void New() => Consume(new byte[100]); + private const string SkipAllocationsTests +#if CORE + = "Not supported for .NET Core yet"; +#else + = null; +#endif + + private readonly ITestOutputHelper output; - [Benchmark] - public unsafe void Stackalloc() + public MemoryDiagnoserTests(ITestOutputHelper outputHelper) { - var bytes = stackalloc byte[100]; - Consume(bytes); + output = outputHelper; } - [MethodImpl(MethodImplOptions.NoInlining)] - private void Consume(T input) { } - - [MethodImpl(MethodImplOptions.NoInlining)] - private unsafe void Consume(byte* input) { } - } - - public class AllocatingSetupAndCleanup - { - private List list; - - [Setup] - public void AllocatingSetUp() => AllocateUntilGcWakesUp(); - - [Benchmark] - public void AllocateNothing() { } - - [Cleanup] - public void AllocatingCleanUp() => AllocateUntilGcWakesUp(); - - private void AllocateUntilGcWakesUp() + public class AccurateAllocations { - int initialCollectionCount = GC.CollectionCount(0); - - while (initialCollectionCount == GC.CollectionCount(0)) - { - list = Enumerable.Range(0, 100).ToList(); - } + [Benchmark]public void Empty() { } + [Benchmark]public byte[] EightBytes() => new byte[8]; + [Benchmark]public byte[] SixtyFourBytes() => new byte[64]; + [Benchmark]public byte[] ThousandBytes() => new byte[1000]; } - } - [KeepBenchmarkFiles()] - public class NoAllocationsAtAll - { - [Benchmark] - public void EmptyMethod() { } - } - - // this class is not compiled for CORE because it is using Diagnosers that currently do not support Core - public class MemoryDiagnoserTests - { - private readonly ITestOutputHelper output; - - public MemoryDiagnoserTests(ITestOutputHelper outputHelper) + [Fact(Skip = SkipAllocationsTests)] + public void MemoryDiagnoserIsAccurate() { - output = outputHelper; + double objectAllocationOverhead = IntPtr.Size * 3; // pointer to method table + object header word + pointer to the object + AssertAllocations(typeof(AccurateAllocations), 100, new Dictionary> + { + { "Empty", allocatedBytes => allocatedBytes == 0 }, + { "EightBytes", allocatedBytes => allocatedBytes == 8 + objectAllocationOverhead }, + { "SixtyFourBytes", allocatedBytes => allocatedBytes == 64 + objectAllocationOverhead }, + { "ThousandBytes", allocatedBytes => allocatedBytes == 1000 + objectAllocationOverhead } + }); } - [Fact(Skip = "Temporarily suppressed, see https://github.com/PerfDotNet/BenchmarkDotNet/issues/208")] - public void MemoryDiagnoserTracksHeapMemoryAllocation() + public class AllocatingSetupAndCleanup { - var memoryDiagnoser = new MemoryDiagnoser(); - var config = CreateConfig(memoryDiagnoser, 50); - var benchmarks = BenchmarkConverter.TypeToBenchmarks(typeof(NewVsStackalloc), config); - - var summary = BenchmarkRunner.Run((Benchmark[])benchmarks, config); + private List list; - var gcCollectionColumns = GetColumns(memoryDiagnoser).ToArray(); - var stackallocBenchmarks = benchmarks.Where(benchmark => benchmark.DisplayInfo.Contains("Stackalloc")); - var newArrayBenchmarks = benchmarks.Where(benchmark => benchmark.DisplayInfo.Contains("New")); + [Benchmark]public void AllocateNothing() { } - const int gen0Index = 0; + [Setup]public void AllocatingSetUp() => AllocateUntilGcWakesUp(); + [Cleanup]public void AllocatingCleanUp() => AllocateUntilGcWakesUp(); - foreach (var benchmark in stackallocBenchmarks) + private void AllocateUntilGcWakesUp() { - var gen0Collections = gcCollectionColumns[gen0Index].GetValue(summary, benchmark); + int initialCollectionCount = GC.CollectionCount(0); - Assert.Equal("-", gen0Collections); + while (initialCollectionCount == GC.CollectionCount(0)) + list = Enumerable.Range(0, 100).ToList(); } + } - foreach (var benchmark in newArrayBenchmarks) + [Fact(Skip = SkipAllocationsTests)] + public void MemoryDiagnoserDoesNotIncludeAllocationsFromSetupAndCleanup() + { + AssertAllocations(typeof(AllocatingSetupAndCleanup), 5, new Dictionary> { - var gen0Str = gcCollectionColumns[gen0Index].GetValue(summary, benchmark); - - AssertParsed(gen0Str, gen0Value => gen0Value > 0); - } + { "AllocateNothing", allocatedBytes => allocatedBytes == 0 } + }); } - [Fact(Skip = "Temporarily suppressed, see https://github.com/PerfDotNet/BenchmarkDotNet/issues/208")] - public void MemoryDiagnoserDoesNotIncludeAllocationsFromSetupAndCleanup() + public class NoAllocationsAtAll { - AssertZeroAllocations(typeof(AllocatingSetupAndCleanup), "AllocateNothing", targetCount: 50); + [Benchmark]public void EmptyMethod() { } } - [Fact(Skip = "Temporarily suppressed, see https://github.com/PerfDotNet/BenchmarkDotNet/issues/208")] + [Fact(Skip = SkipAllocationsTests)] public void EngineShouldNotInterfereAllocationResults() { - AssertZeroAllocations(typeof(NoAllocationsAtAll), "EmptyMethod", targetCount: 5000); // we need a lot of iterations to be sure!! + AssertAllocations(typeof(NoAllocationsAtAll), 100, new Dictionary> + { + { "EmptyMethod", allocatedBytes => allocatedBytes == 0 } + }); } - public void AssertZeroAllocations(Type benchmarkType, string benchmarkMethodName, int targetCount) + private void AssertAllocations(Type benchmarkType, int targetCount, + Dictionary> benchmarksAllocationsValidators) { - var memoryDiagnoser = new MemoryDiagnoser(); + var memoryDiagnoser = MemoryDiagnoser.Default; var config = CreateConfig(memoryDiagnoser, targetCount); var benchmarks = BenchmarkConverter.TypeToBenchmarks(benchmarkType, config); var summary = BenchmarkRunner.Run((Benchmark[])benchmarks, config); var allocationColumn = GetColumns(memoryDiagnoser).Single(); - var allocateNothingBenchmarks = benchmarks.Where(benchmark => benchmark.DisplayInfo.Contains(benchmarkMethodName)); - foreach (var benchmark in allocateNothingBenchmarks) + foreach (var benchmarkAllocationsValidator in benchmarksAllocationsValidators) { - var allocations = allocationColumn.GetValue(summary, benchmark); + var allocatingBenchmarks = benchmarks.Where(benchmark => benchmark.DisplayInfo.Contains(benchmarkAllocationsValidator.Key)); + + foreach (var benchmark in allocatingBenchmarks) + { + var allocations = allocationColumn.GetValue(summary, benchmark); - AssertParsed(allocations, allocatedBytes => allocatedBytes == 0); + AssertParsed(allocations, benchmarkAllocationsValidator.Value); + } } } private IConfig CreateConfig(IDiagnoser diagnoser, int targetCount) { return ManualConfig.CreateEmpty() - .With( - Job.Dry - .WithLaunchCount(1) - .WithWarmupCount(1) - .WithTargetCount(targetCount) - .WithGcForce(false)) - .With(DefaultConfig.Instance.GetLoggers().ToArray()) - .With(DefaultColumnProviders.Instance) - .With(diagnoser) - .With(new OutputLogger(output)); + .With( + Job.Dry + .WithLaunchCount(1) + .WithWarmupCount(1) + .WithTargetCount(targetCount) + .WithInvocationCount(100) + .WithGcForce(false)) + .With(DefaultConfig.Instance.GetLoggers().ToArray()) + .With(DefaultColumnProviders.Instance) + .With(diagnoser) + .With(new OutputLogger(output)); } private static T[] GetColumns(MemoryDiagnoser memoryDiagnoser) @@ -165,7 +145,7 @@ private static void AssertParsed(string text, Predicate condition) double value; if (double.TryParse(text, NumberStyles.Number, HostEnvironmentInfo.MainCultureInfo, out value)) { - Assert.True(condition(value)); + Assert.True(condition(value), $"Failed for value {value}"); } else { @@ -173,5 +153,4 @@ private static void AssertParsed(string text, Predicate condition) } } } -} -#endif \ No newline at end of file +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs b/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs index 5377cc4cbd..368e9146a3 100644 --- a/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs +++ b/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Runtime; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Engines; @@ -42,7 +43,7 @@ private static BenchmarkReport CreateReport(IConfig config) { new Measurement(1, IterationMode.Result, 1, 1, 1) }; - return new BenchmarkReport(benchmark, buildResult, buildResult, new List { executeResult }, measurements); + return new BenchmarkReport(benchmark, buildResult, buildResult, new List { executeResult }, measurements, default(GcStats)); } public class MockBenchmarkClass From 4cabc202bcb5f76a2e417d6b20aedf5c23e12e3a Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 3 Nov 2016 15:02:13 +0100 Subject: [PATCH 02/11] don't try to use AppDomain's Monitoring in Mono since it's not implemented there --- src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs | 4 ++++ src/BenchmarkDotNet.Core/Engines/Engine.cs | 4 ++++ src/BenchmarkDotNet.Core/Engines/GcStats.cs | 3 ++- src/BenchmarkDotNet.Core/Portability/RuntimeInformation.cs | 4 +++- 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs b/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs index 33edd67320..a072c753ff 100644 --- a/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs +++ b/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs @@ -7,6 +7,7 @@ using BenchmarkDotNet.Running; using System.Linq; using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Portability; namespace BenchmarkDotNet.Diagnosers { @@ -55,6 +56,9 @@ public AllocationColumn(Dictionary results) public string GetValue(Summary summary, Benchmark benchmark) { #if !CORE + if (RuntimeInformation.IsMono()) + return "?"; + if (results.ContainsKey(benchmark)) { var result = results[benchmark]; diff --git a/src/BenchmarkDotNet.Core/Engines/Engine.cs b/src/BenchmarkDotNet.Core/Engines/Engine.cs index 277bec0448..8625803141 100644 --- a/src/BenchmarkDotNet.Core/Engines/Engine.cs +++ b/src/BenchmarkDotNet.Core/Engines/Engine.cs @@ -3,6 +3,7 @@ using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Horology; using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Portability; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; using JetBrains.Annotations; @@ -172,6 +173,9 @@ private void EnableMonitoring() if(!IsDiagnoserAttached) // it could affect the results, we do this in separate, diagnostics-only run return; #if CLASSIC + if(RuntimeInformation.IsMono()) // Monitoring is not available in Mono, see http://stackoverflow.com/questions/40234948/how-to-get-the-number-of-allocated-bytes-in-mono + return; + AppDomain.MonitoringIsEnabled = true; #endif } diff --git a/src/BenchmarkDotNet.Core/Engines/GcStats.cs b/src/BenchmarkDotNet.Core/Engines/GcStats.cs index f28310a372..95bcfd8c4c 100644 --- a/src/BenchmarkDotNet.Core/Engines/GcStats.cs +++ b/src/BenchmarkDotNet.Core/Engines/GcStats.cs @@ -1,4 +1,5 @@ using System; +using BenchmarkDotNet.Portability; namespace BenchmarkDotNet.Engines { @@ -79,7 +80,7 @@ private static long GetAllocatedBytes(bool isDiagnosticsEnabled) #if NETCOREAPP11 // when MS releases new version of .NET Runtime to nuget.org return GC.GetAllocatedBytesForCurrentThread(); // https://github.com/dotnet/corefx/pull/12489 #elif CLASSIC - if (!isDiagnosticsEnabled) + if (!isDiagnosticsEnabled || RuntimeInformation.IsMono()) // Monitoring is not available in Mono, see http://stackoverflow.com/questions/40234948/how-to-get-the-number-of-allocated-bytes- return 0; // "This instance Int64 property returns the number of bytes that have been allocated by a specific diff --git a/src/BenchmarkDotNet.Core/Portability/RuntimeInformation.cs b/src/BenchmarkDotNet.Core/Portability/RuntimeInformation.cs index 9715958d2c..b68867c8e0 100644 --- a/src/BenchmarkDotNet.Core/Portability/RuntimeInformation.cs +++ b/src/BenchmarkDotNet.Core/Portability/RuntimeInformation.cs @@ -19,6 +19,8 @@ namespace BenchmarkDotNet.Portability { public class RuntimeInformation { + private static readonly bool isMono = Type.GetType("Mono.Runtime") != null; // it allocates a lot of memory, we need to check it once in order to keep Enging non-allocating! + private const string Debug = "DEBUG"; private const string Release = "RELEASE"; internal const string Unknown = "?"; @@ -39,7 +41,7 @@ internal static bool IsWindows() #endif } - private static bool IsMono() => Type.GetType("Mono.Runtime") != null; + internal static bool IsMono() => isMono; internal static string GetOsVersion() { From 99c21e842ec925a51f98844959b9a49b2493e971 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 3 Nov 2016 16:38:11 +0100 Subject: [PATCH 03/11] scale GC collections count / op, makes MemoryDiagnoser output stable for benchmarks with different # of runs, fixes #133 --- .../Diagnosers/MemoryDiagnoser.cs | 48 ++++++++----------- src/BenchmarkDotNet.Core/Engines/GcStats.cs | 2 + 2 files changed, 21 insertions(+), 29 deletions(-) diff --git a/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs b/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs index a072c753ff..0e053a1e04 100644 --- a/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs +++ b/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs @@ -1,11 +1,11 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Diagnostics; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; -using System.Linq; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Portability; @@ -23,18 +23,15 @@ public class MemoryDiagnoser : IDiagnoser new GCCollectionColumn(results, 2), new AllocationColumn(results)); - // the methods are left empty on purpose + // the following methods are left empty on purpose // the action takes places in other process, and the values are gathered by Engine public void BeforeAnythingElse(Process process, Benchmark benchmark) { } public void AfterSetup(Process process, Benchmark benchmark) { } public void BeforeCleanup() { } - public void ProcessResults(Benchmark benchmark, BenchmarkReport report) - { - results.Add(benchmark, report.GcStats); - } + public void DisplayResults(ILogger logger) { } // no custom output - public void DisplayResults(ILogger logger) { } + public void ProcessResults(Benchmark benchmark, BenchmarkReport report) => results.Add(benchmark, report.GcStats); public class AllocationColumn : IColumn { @@ -55,41 +52,33 @@ public AllocationColumn(Dictionary results) public string GetValue(Summary summary, Benchmark benchmark) { -#if !CORE +#if CORE + return "?"; +#else if (RuntimeInformation.IsMono()) return "?"; + if (!results.ContainsKey(benchmark)) + return "N/A"; - if (results.ContainsKey(benchmark)) - { - var result = results[benchmark]; - // TODO scale this based on the minimum value in the column, i.e. use B/KB/MB as appropriate - return (result.AllocatedBytes / result.TotalOperations).ToString("N2", HostEnvironmentInfo.MainCultureInfo); - } - return "N/A"; -#else - return "?"; + return results[benchmark].BytesAllocatedPerOperation.ToString("N0", HostEnvironmentInfo.MainCultureInfo); #endif } } public class GCCollectionColumn : IColumn { - private Dictionary results; - private int generation; - // TODO also need to find a sensible way of including this in the column name? - private long opsPerGCCount; + private readonly Dictionary results; + private readonly int generation; public GCCollectionColumn(Dictionary results, int generation) { - ColumnName = $"Gen {generation}"; this.results = results; this.generation = generation; - opsPerGCCount = results.Any() ? results.Min(r => r.Value.TotalOperations) : 1; } public bool IsDefault(Summary summary, Benchmark benchmark) => true; public string Id => $"{nameof(GCCollectionColumn)}{generation}"; - public string ColumnName { get; } + public string ColumnName => $"Gen {generation}/op"; public bool IsAvailable(Summary summary) => true; public bool AlwaysShow => true; public ColumnCategory Category => ColumnCategory.Diagnoser; @@ -99,13 +88,14 @@ public string GetValue(Summary summary, Benchmark benchmark) { if (results.ContainsKey(benchmark)) { - var result = results[benchmark]; - var value = generation == 0 ? result.Gen0Collections : - generation == 1 ? result.Gen1Collections : result.Gen2Collections; + var gcStats = results[benchmark]; + var value = generation == 0 ? gcStats.Gen0Collections : + generation == 1 ? gcStats.Gen1Collections : gcStats.Gen2Collections; if (value == 0) return "-"; // make zero more obvious - return (value / (double)result.TotalOperations * opsPerGCCount).ToString("N2", HostEnvironmentInfo.MainCultureInfo); + + return (value / (double)gcStats.TotalOperations).ToString("N6", HostEnvironmentInfo.MainCultureInfo); } return "N/A"; } diff --git a/src/BenchmarkDotNet.Core/Engines/GcStats.cs b/src/BenchmarkDotNet.Core/Engines/GcStats.cs index 95bcfd8c4c..1494ee8463 100644 --- a/src/BenchmarkDotNet.Core/Engines/GcStats.cs +++ b/src/BenchmarkDotNet.Core/Engines/GcStats.cs @@ -23,6 +23,8 @@ private GcStats(int gen0Collections, int gen1Collections, int gen2Collections, l public long AllocatedBytes { get; } public long TotalOperations { get; } + public long BytesAllocatedPerOperation => AllocatedBytes / TotalOperations; + public static GcStats operator +(GcStats left, GcStats right) { return new GcStats( From e91255e2a6e5b3d0e683ee5cf1773cdbbbe3649f Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Fri, 4 Nov 2016 11:06:27 +0100 Subject: [PATCH 04/11] use per mille to make the Memory Diagnoser output more human-friendly + reduce the column's name length (everything is per operation now) --- src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs b/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs index 0e053a1e04..5730894527 100644 --- a/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs +++ b/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs @@ -43,7 +43,7 @@ public AllocationColumn(Dictionary results) } public string Id => nameof(AllocationColumn); - public string ColumnName => "Bytes Allocated/Op"; + public string ColumnName => "Bytes Allocated"; public bool IsDefault(Summary summary, Benchmark benchmark) => false; public bool IsAvailable(Summary summary) => true; public bool AlwaysShow => true; @@ -78,7 +78,7 @@ public GCCollectionColumn(Dictionary results, int generation public bool IsDefault(Summary summary, Benchmark benchmark) => true; public string Id => $"{nameof(GCCollectionColumn)}{generation}"; - public string ColumnName => $"Gen {generation}/op"; + public string ColumnName => $"Gen {generation}"; public bool IsAvailable(Summary summary) => true; public bool AlwaysShow => true; public ColumnCategory Category => ColumnCategory.Diagnoser; @@ -95,7 +95,7 @@ public string GetValue(Summary summary, Benchmark benchmark) if (value == 0) return "-"; // make zero more obvious - return (value / (double)gcStats.TotalOperations).ToString("N6", HostEnvironmentInfo.MainCultureInfo); + return (value / (double)gcStats.TotalOperations).ToString("#0.000‰", HostEnvironmentInfo.MainCultureInfo); } return "N/A"; } From ade1bea023aab7822373d158a8617131e371a117 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Sun, 6 Nov 2016 16:35:59 +0100 Subject: [PATCH 05/11] preallocate results list in more safe, but still ugly way --- .../Engines/EngineTargetStage.cs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/BenchmarkDotNet.Core/Engines/EngineTargetStage.cs b/src/BenchmarkDotNet.Core/Engines/EngineTargetStage.cs index 8c436238d8..63c183c71e 100644 --- a/src/BenchmarkDotNet.Core/Engines/EngineTargetStage.cs +++ b/src/BenchmarkDotNet.Core/Engines/EngineTargetStage.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Mathematics; using BenchmarkDotNet.Reports; @@ -15,12 +16,18 @@ public class EngineTargetStage : EngineStage private readonly int? targetCount; private readonly double maxStdErrRelative; private readonly bool removeOutliers; + private readonly Stack> preAllocatedListsOfMeasurements; public EngineTargetStage(IEngine engine) : base(engine) { targetCount = engine.TargetJob.ResolveValueAsNullable(RunMode.TargetCountCharacteristic); maxStdErrRelative = engine.TargetJob.ResolveValue(AccuracyMode.MaxStdErrRelativeCharacteristic, engine.Resolver); removeOutliers = engine.TargetJob.ResolveValue(AccuracyMode.RemoveOutliersCharacteristic, engine.Resolver); + + preAllocatedListsOfMeasurements = new Stack>(10); + var maxSize = targetCount.HasValue ? Math.Max(targetCount.Value, MaxIterationCount) : MaxIterationCount; + for (int i = 0; i < 10; i++) + preAllocatedListsOfMeasurements.Push(new List(maxSize)); } public List RunIdle(long invokeCount, int unrollFactor) @@ -36,8 +43,8 @@ internal List Run(long invokeCount, IterationMode iterationMode, bo private List RunAuto(long invokeCount, IterationMode iterationMode, int unrollFactor) { - var measurements = new List(MaxIterationCount); - var measurementsForStatistics = new List(MaxIterationCount); + var measurements = preAllocatedListsOfMeasurements.Pop(); + var measurementsForStatistics = preAllocatedListsOfMeasurements.Pop(); int iterationCounter = 0; bool isIdle = iterationMode.IsIdle(); @@ -65,7 +72,7 @@ private List RunAuto(long invokeCount, IterationMode iterationMode, private List RunSpecific(long invokeCount, IterationMode iterationMode, int iterationCount, int unrollFactor) { - var measurements = new List(MaxIterationCount); + var measurements = preAllocatedListsOfMeasurements.Pop(); for (int i = 0; i < iterationCount; i++) measurements.Add(RunIteration(iterationMode, i + 1, invokeCount, unrollFactor)); From 1022827bbba62e855a90c9c16e6af01830c380ea Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Mon, 7 Nov 2016 20:59:40 +0100 Subject: [PATCH 06/11] closed the ugly code in separate class --- .../Engines/EngineTargetStage.cs | 15 ++++----- .../Engines/MeasurementsPool.cs | 31 +++++++++++++++++++ 2 files changed, 37 insertions(+), 9 deletions(-) create mode 100644 src/BenchmarkDotNet.Core/Engines/MeasurementsPool.cs diff --git a/src/BenchmarkDotNet.Core/Engines/EngineTargetStage.cs b/src/BenchmarkDotNet.Core/Engines/EngineTargetStage.cs index 63c183c71e..94f04c2dc7 100644 --- a/src/BenchmarkDotNet.Core/Engines/EngineTargetStage.cs +++ b/src/BenchmarkDotNet.Core/Engines/EngineTargetStage.cs @@ -16,18 +16,15 @@ public class EngineTargetStage : EngineStage private readonly int? targetCount; private readonly double maxStdErrRelative; private readonly bool removeOutliers; - private readonly Stack> preAllocatedListsOfMeasurements; + private readonly MeasurementsPool measurementsPool; + public EngineTargetStage(IEngine engine) : base(engine) { targetCount = engine.TargetJob.ResolveValueAsNullable(RunMode.TargetCountCharacteristic); maxStdErrRelative = engine.TargetJob.ResolveValue(AccuracyMode.MaxStdErrRelativeCharacteristic, engine.Resolver); removeOutliers = engine.TargetJob.ResolveValue(AccuracyMode.RemoveOutliersCharacteristic, engine.Resolver); - - preAllocatedListsOfMeasurements = new Stack>(10); - var maxSize = targetCount.HasValue ? Math.Max(targetCount.Value, MaxIterationCount) : MaxIterationCount; - for (int i = 0; i < 10; i++) - preAllocatedListsOfMeasurements.Push(new List(maxSize)); + measurementsPool = MeasurementsPool.PreAllocate(10, MaxIterationCount, targetCount); } public List RunIdle(long invokeCount, int unrollFactor) @@ -43,8 +40,8 @@ internal List Run(long invokeCount, IterationMode iterationMode, bo private List RunAuto(long invokeCount, IterationMode iterationMode, int unrollFactor) { - var measurements = preAllocatedListsOfMeasurements.Pop(); - var measurementsForStatistics = preAllocatedListsOfMeasurements.Pop(); + var measurements = measurementsPool.Next(); + var measurementsForStatistics = measurementsPool.Next(); int iterationCounter = 0; bool isIdle = iterationMode.IsIdle(); @@ -72,7 +69,7 @@ private List RunAuto(long invokeCount, IterationMode iterationMode, private List RunSpecific(long invokeCount, IterationMode iterationMode, int iterationCount, int unrollFactor) { - var measurements = preAllocatedListsOfMeasurements.Pop(); + var measurements = measurementsPool.Next(); for (int i = 0; i < iterationCount; i++) measurements.Add(RunIteration(iterationMode, i + 1, invokeCount, unrollFactor)); diff --git a/src/BenchmarkDotNet.Core/Engines/MeasurementsPool.cs b/src/BenchmarkDotNet.Core/Engines/MeasurementsPool.cs new file mode 100644 index 0000000000..0dadab0950 --- /dev/null +++ b/src/BenchmarkDotNet.Core/Engines/MeasurementsPool.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; +using BenchmarkDotNet.Reports; + +namespace BenchmarkDotNet.Engines +{ + internal class MeasurementsPool + { + private readonly Stack> preAllocatedListsOfMeasurements; + + private MeasurementsPool(int capacity) + { + preAllocatedListsOfMeasurements = new Stack>(capacity); + } + + internal List Next() => preAllocatedListsOfMeasurements.Pop(); + + internal static MeasurementsPool PreAllocate(int capacity, int maxLength, int? configuredLength) + { + var pool = new MeasurementsPool(capacity); + var maxSize = configuredLength.HasValue ? Math.Max(configuredLength.Value, maxLength) : maxLength; + + for (int i = 0; i < capacity; i++) + { + pool.preAllocatedListsOfMeasurements.Push(new List(maxSize)); + } + + return pool; + } + } +} \ No newline at end of file From 1e2d381b6a7ba34d53140f28f4481d3d829b6260 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Sun, 13 Nov 2016 17:39:48 +0100 Subject: [PATCH 07/11] update to netcoreapp1.1 in order to get universal cross platform memory diagnoser --- .../project.json | 4 +-- samples/BenchmarkDotNet.Samples/project.json | 4 +-- .../Diagnosers/MemoryDiagnoser.cs | 4 --- src/BenchmarkDotNet.Core/Engines/GcStats.cs | 31 +++++++++++----- .../Toolchains/Core/CoreToolchain.cs | 4 +-- .../DotNetCli/DotNetCliCommandExecutor.cs | 1 + src/BenchmarkDotNet.Core/project.json | 28 ++++----------- src/BenchmarkDotNet/project.json | 2 +- .../MemoryDiagnoserTests.cs | 35 ++++++------------- .../project.json | 4 +-- tests/BenchmarkDotNet.Tests/project.json | 4 +-- tests/runCoreTests.cmd | 2 +- 12 files changed, 52 insertions(+), 71 deletions(-) diff --git a/samples/BenchmarkDotNet.Samples.FSharp.Core/project.json b/samples/BenchmarkDotNet.Samples.FSharp.Core/project.json index ee777a923e..73953744bc 100644 --- a/samples/BenchmarkDotNet.Samples.FSharp.Core/project.json +++ b/samples/BenchmarkDotNet.Samples.FSharp.Core/project.json @@ -13,7 +13,7 @@ "Microsoft.FSharp.Core.netcore": "1.0.0-*", "Microsoft.NETCore.App": { "type": "platform", - "version": "1.0.0" + "version": "1.1.0-preview1-001100-00" }, "BenchmarkDotNet": { "target": "project" @@ -31,7 +31,7 @@ } }, "frameworks": { - "netcoreapp1.0": { + "netcoreapp1.1": { "imports": [ "portable-net45+win8", "dnxcore50", diff --git a/samples/BenchmarkDotNet.Samples/project.json b/samples/BenchmarkDotNet.Samples/project.json index 83c169f291..445f9b827d 100644 --- a/samples/BenchmarkDotNet.Samples/project.json +++ b/samples/BenchmarkDotNet.Samples/project.json @@ -33,14 +33,14 @@ } } }, - "netcoreapp1.0": { + "netcoreapp1.1": { "buildOptions": { "define": [ "CORE" ] }, "dependencies": { "Microsoft.NETCore.App": { "type": "platform", - "version": "1.0.0" + "version": "1.1.0-preview1-001100-00" }, "System.ComponentModel.EventBasedAsync": "4.0.11" } diff --git a/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs b/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs index 5730894527..1821f120fa 100644 --- a/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs +++ b/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs @@ -52,16 +52,12 @@ public AllocationColumn(Dictionary results) public string GetValue(Summary summary, Benchmark benchmark) { -#if CORE - return "?"; -#else if (RuntimeInformation.IsMono()) return "?"; if (!results.ContainsKey(benchmark)) return "N/A"; return results[benchmark].BytesAllocatedPerOperation.ToString("N0", HostEnvironmentInfo.MainCultureInfo); -#endif } } diff --git a/src/BenchmarkDotNet.Core/Engines/GcStats.cs b/src/BenchmarkDotNet.Core/Engines/GcStats.cs index 1494ee8463..a1eeeb70aa 100644 --- a/src/BenchmarkDotNet.Core/Engines/GcStats.cs +++ b/src/BenchmarkDotNet.Core/Engines/GcStats.cs @@ -1,4 +1,5 @@ using System; +using System.Reflection; using BenchmarkDotNet.Portability; namespace BenchmarkDotNet.Engines @@ -7,6 +8,8 @@ public struct GcStats { internal const string ResultsLinePrefix = "GC: "; + private static readonly Func getAllocatedBytesForCurrentThread = GetAllocatedBytesForCurrentThread(); + private GcStats(int gen0Collections, int gen1Collections, int gen2Collections, long allocatedBytes, long totalOperations) { Gen0Collections = gen0Collections; @@ -50,7 +53,7 @@ public GcStats WithTotalOperations(long totalOperationsCount) internal static GcStats ReadInitial(bool isDiagnosticsEnabled) { - // this might force GC.Collect, so we want to do this before collecting collections counts + // this will force GC.Collect, so we want to do this before collecting collections counts long allocatedBytes = GetAllocatedBytes(isDiagnosticsEnabled); return new GcStats( @@ -68,7 +71,7 @@ internal static GcStats ReadFinal(bool isDiagnosticsEnabled) GC.CollectionCount(1), GC.CollectionCount(2), - // this might force GC.Collect, so we want to do this after collecting collections counts + // this will force GC.Collect, so we want to do this after collecting collections counts // to exclude this single full forced collection from results GetAllocatedBytes(isDiagnosticsEnabled), 0); @@ -79,22 +82,34 @@ public static GcStats FromForced(int forcedFullGarbageCollections) private static long GetAllocatedBytes(bool isDiagnosticsEnabled) { -#if NETCOREAPP11 // when MS releases new version of .NET Runtime to nuget.org - return GC.GetAllocatedBytesForCurrentThread(); // https://github.com/dotnet/corefx/pull/12489 -#elif CLASSIC - if (!isDiagnosticsEnabled || RuntimeInformation.IsMono()) // Monitoring is not available in Mono, see http://stackoverflow.com/questions/40234948/how-to-get-the-number-of-allocated-bytes- + if (!isDiagnosticsEnabled + || RuntimeInformation.IsMono()) // Monitoring is not available in Mono, see http://stackoverflow.com/questions/40234948/how-to-get-the-number-of-allocated-bytes- return 0; // "This instance Int64 property returns the number of bytes that have been allocated by a specific // AppDomain. The number is accurate as of the last garbage collection." - CLR via C# // so we enforce GC.Collect here just to make sure we get accurate results GC.Collect(); +#if CORE + return getAllocatedBytesForCurrentThread.Invoke(); +#elif CLASSIC + return AppDomain.CurrentDomain.MonitoringTotalAllocatedMemorySize; -#else - return 0; // currently for .NET Core #endif } + private static Func GetAllocatedBytesForCurrentThread() + { + // for some versions of .NET Core this method is internal, + // for some public and for others public and exposed ;) + var method = typeof(GC).GetTypeInfo().GetMethod("GetAllocatedBytesForCurrentThread", + BindingFlags.Public | BindingFlags.Static) + ?? typeof(GC).GetTypeInfo().GetMethod("GetAllocatedBytesForCurrentThread", + BindingFlags.NonPublic | BindingFlags.Static); + + return () => (long)method.Invoke(null, null); + } + internal string ToOutputLine() => $"{ResultsLinePrefix} {Gen0Collections} {Gen1Collections} {Gen2Collections} {AllocatedBytes} {TotalOperations}"; diff --git a/src/BenchmarkDotNet.Core/Toolchains/Core/CoreToolchain.cs b/src/BenchmarkDotNet.Core/Toolchains/Core/CoreToolchain.cs index 19b1bce5c8..0d9a359e0b 100644 --- a/src/BenchmarkDotNet.Core/Toolchains/Core/CoreToolchain.cs +++ b/src/BenchmarkDotNet.Core/Toolchains/Core/CoreToolchain.cs @@ -1,7 +1,5 @@ using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Environments; -using BenchmarkDotNet.Extensions; -using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Portability; @@ -12,7 +10,7 @@ namespace BenchmarkDotNet.Toolchains.Core { public class CoreToolchain : Toolchain { - private const string TargetFrameworkMoniker = "netcoreapp1.0"; + private const string TargetFrameworkMoniker = "netcoreapp1.1"; public static readonly IToolchain Instance = new CoreToolchain(); diff --git a/src/BenchmarkDotNet.Core/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs b/src/BenchmarkDotNet.Core/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs index 186fa10b2b..08114526ef 100644 --- a/src/BenchmarkDotNet.Core/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs +++ b/src/BenchmarkDotNet.Core/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs @@ -42,6 +42,7 @@ internal static bool ExecuteCommand(string commandWithArguments, string workingD // don't forget to call, otherwise logger will not get any events process.BeginErrorReadLine(); + process.BeginOutputReadLine(); process.WaitForExit((int)timeout.TotalMilliseconds); diff --git a/src/BenchmarkDotNet.Core/project.json b/src/BenchmarkDotNet.Core/project.json index cbe10a8d9e..993d2407b8 100644 --- a/src/BenchmarkDotNet.Core/project.json +++ b/src/BenchmarkDotNet.Core/project.json @@ -51,34 +51,18 @@ "System.Threading.Tasks.Extensions": "4.0.0" } }, - "netstandard1.5": { + "netcoreapp1.1": { "buildOptions": { "define": [ "CORE" ] }, "dependencies": { - "System.Linq": "4.1.0", - "System.Resources.ResourceManager": "4.0.1", - "Microsoft.CSharp": "4.0.1", - "Microsoft.Win32.Primitives": "4.0.1", - "System.Console": "4.0.0", - "System.Text.RegularExpressions": "4.1.0", - "System.Threading": "4.0.11", - "System.Reflection": "4.1.0", - "System.Reflection.Primitives": "4.0.1", - "System.Reflection.TypeExtensions": "4.1.0", - "System.Threading.Tasks.Extensions": "4.0.0", - "System.Threading.Thread": "4.0.0", - "System.Diagnostics.Process": "4.1.0", - "System.IO.FileSystem": "4.0.1", - "System.Runtime.InteropServices.RuntimeInformation": "4.0.0", + "Microsoft.NETCore.App": { + "type": "platform", + "version": "1.1.0-preview1-001100-00" + }, "System.Runtime.Serialization.Primitives": "4.1.1", - "System.Diagnostics.Tools": "4.0.1", - "System.Runtime.InteropServices": "4.1.0", "Microsoft.DotNet.InternalAbstractions": "1.0.0", - "System.Reflection.Extensions": "4.0.1", - "System.Diagnostics.Debug": "4.0.11", - "System.Xml.XPath.XmlDocument": "4.0.1", - "System.Collections.Concurrent": "4.0.12" + "System.Xml.XPath.XmlDocument": "4.0.1" } } } diff --git a/src/BenchmarkDotNet/project.json b/src/BenchmarkDotNet/project.json index 4f7361bbae..1aeadf287b 100644 --- a/src/BenchmarkDotNet/project.json +++ b/src/BenchmarkDotNet/project.json @@ -48,7 +48,7 @@ } } }, - "netstandard1.5": { + "netcoreapp1.1": { "buildOptions": { "define": [ "CORE" ] }, diff --git a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs index c96681bbf6..4ce34505a7 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs @@ -2,32 +2,21 @@ using System.Collections.Generic; using System.Globalization; using System.Linq; -using System.Runtime.CompilerServices; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Diagnosers; -using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; -using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Running; using BenchmarkDotNet.Tests.Loggers; using Xunit; using Xunit.Abstractions; -using BenchmarkDotNet.Reports; namespace BenchmarkDotNet.IntegrationTests { public class MemoryDiagnoserTests { - private const string SkipAllocationsTests -#if CORE - = "Not supported for .NET Core yet"; -#else - = null; -#endif - private readonly ITestOutputHelper output; public MemoryDiagnoserTests(ITestOutputHelper outputHelper) @@ -40,19 +29,17 @@ public class AccurateAllocations [Benchmark]public void Empty() { } [Benchmark]public byte[] EightBytes() => new byte[8]; [Benchmark]public byte[] SixtyFourBytes() => new byte[64]; - [Benchmark]public byte[] ThousandBytes() => new byte[1000]; } - [Fact(Skip = SkipAllocationsTests)] + [Fact] public void MemoryDiagnoserIsAccurate() { - double objectAllocationOverhead = IntPtr.Size * 3; // pointer to method table + object header word + pointer to the object - AssertAllocations(typeof(AccurateAllocations), 100, new Dictionary> + long objectAllocationOverhead = IntPtr.Size * 3; // pointer to method table + object header word + pointer to the object + AssertAllocations(typeof(AccurateAllocations), 200, new Dictionary> { { "Empty", allocatedBytes => allocatedBytes == 0 }, { "EightBytes", allocatedBytes => allocatedBytes == 8 + objectAllocationOverhead }, { "SixtyFourBytes", allocatedBytes => allocatedBytes == 64 + objectAllocationOverhead }, - { "ThousandBytes", allocatedBytes => allocatedBytes == 1000 + objectAllocationOverhead } }); } @@ -74,10 +61,10 @@ private void AllocateUntilGcWakesUp() } } - [Fact(Skip = SkipAllocationsTests)] + [Fact] public void MemoryDiagnoserDoesNotIncludeAllocationsFromSetupAndCleanup() { - AssertAllocations(typeof(AllocatingSetupAndCleanup), 5, new Dictionary> + AssertAllocations(typeof(AllocatingSetupAndCleanup), 100, new Dictionary> { { "AllocateNothing", allocatedBytes => allocatedBytes == 0 } }); @@ -88,17 +75,17 @@ public class NoAllocationsAtAll [Benchmark]public void EmptyMethod() { } } - [Fact(Skip = SkipAllocationsTests)] + [Fact] public void EngineShouldNotInterfereAllocationResults() { - AssertAllocations(typeof(NoAllocationsAtAll), 100, new Dictionary> + AssertAllocations(typeof(NoAllocationsAtAll), 100, new Dictionary> { { "EmptyMethod", allocatedBytes => allocatedBytes == 0 } }); } private void AssertAllocations(Type benchmarkType, int targetCount, - Dictionary> benchmarksAllocationsValidators) + Dictionary> benchmarksAllocationsValidators) { var memoryDiagnoser = MemoryDiagnoser.Default; var config = CreateConfig(memoryDiagnoser, targetCount); @@ -140,10 +127,10 @@ private IConfig CreateConfig(IDiagnoser diagnoser, int targetCount) private static T[] GetColumns(MemoryDiagnoser memoryDiagnoser) => memoryDiagnoser.GetColumnProvider().GetColumns(null).OfType().ToArray(); - private static void AssertParsed(string text, Predicate condition) + private static void AssertParsed(string text, Predicate condition) { - double value; - if (double.TryParse(text, NumberStyles.Number, HostEnvironmentInfo.MainCultureInfo, out value)) + long value; + if (long.TryParse(text, NumberStyles.Number, HostEnvironmentInfo.MainCultureInfo, out value)) { Assert.True(condition(value), $"Failed for value {value}"); } diff --git a/tests/BenchmarkDotNet.IntegrationTests/project.json b/tests/BenchmarkDotNet.IntegrationTests/project.json index 525d7d205f..433acdc57b 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/project.json +++ b/tests/BenchmarkDotNet.IntegrationTests/project.json @@ -21,7 +21,7 @@ "allowUnsafe": true }, "frameworks": { - "netcoreapp1.0": { + "netcoreapp1.1": { "buildOptions": { "define": [ "CORE" ] }, @@ -33,7 +33,7 @@ "dependencies": { "Microsoft.NETCore.App": { "type": "platform", - "version": "1.0.0" + "version": "1.1.0-preview1-001100-00" }, "System.ComponentModel.EventBasedAsync": "4.0.11" } diff --git a/tests/BenchmarkDotNet.Tests/project.json b/tests/BenchmarkDotNet.Tests/project.json index e5c5bfb8c0..1fdda3408c 100644 --- a/tests/BenchmarkDotNet.Tests/project.json +++ b/tests/BenchmarkDotNet.Tests/project.json @@ -20,7 +20,7 @@ "copyToOutput": [ "xunit.runner.json" ] }, "frameworks": { - "netcoreapp1.0": { + "netcoreapp1.1": { "imports": [ "dnxcore50", "portable-net45+win8", @@ -29,7 +29,7 @@ "dependencies": { "Microsoft.NETCore.App": { "type": "platform", - "version": "1.0.0" + "version": "1.1.0-preview1-001100-00" }, "System.ComponentModel.EventBasedAsync": "4.0.11" } diff --git a/tests/runCoreTests.cmd b/tests/runCoreTests.cmd index 24c97af269..91992ad945 100644 --- a/tests/runCoreTests.cmd +++ b/tests/runCoreTests.cmd @@ -15,7 +15,7 @@ echo ----------------------------- echo Running Core tests echo ----------------------------- -call dotnet test "BenchmarkDotNet.IntegrationTests/" --configuration Release --framework netcoreapp1.0 2> failedCoreTests.txt +call dotnet test "BenchmarkDotNet.IntegrationTests/" --configuration Release --framework netcoreapp1.1 2> failedCoreTests.txt if NOT %ERRORLEVEL% == 0 ( echo CORE tests has failed From e69e80b46b612918a156362d88843a37f123564d Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Sun, 13 Nov 2016 18:25:29 +0100 Subject: [PATCH 08/11] don't show Gen 1 and Gen 2 columns if empty for all benchmarks --- .../Diagnosers/MemoryDiagnoser.cs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs b/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs index 1821f120fa..1a10de4e1e 100644 --- a/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs +++ b/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.Linq; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Loggers; @@ -75,11 +76,18 @@ public GCCollectionColumn(Dictionary results, int generation public bool IsDefault(Summary summary, Benchmark benchmark) => true; public string Id => $"{nameof(GCCollectionColumn)}{generation}"; public string ColumnName => $"Gen {generation}"; - public bool IsAvailable(Summary summary) => true; - public bool AlwaysShow => true; + public bool AlwaysShow => false; public ColumnCategory Category => ColumnCategory.Diagnoser; public int PriorityInCategory => 0; + public bool IsAvailable(Summary summary) + => generation == 0 // Gen 0 must always be visible + || summary + .Reports + .Any(report => generation == 1 + ? report.GcStats.Gen1Collections != 0 + : report.GcStats.Gen2Collections != 0); + public string GetValue(Summary summary, Benchmark benchmark) { if (results.ContainsKey(benchmark)) From 2a529abfff9d72bdd4b11d82d1c918beaffaf0ff Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Wed, 16 Nov 2016 20:12:21 +0100 Subject: [PATCH 09/11] update to .NET Core 1.1, fixes #301 --- docs/guide/Contributing/Building.md | 6 +++--- .../project.json | 2 +- samples/BenchmarkDotNet.Samples/project.json | 2 +- ...ssOutputLogger.cs => AsyncErrorOutputLogger.cs} | 14 ++------------ .../DotNetCli/DotNetCliCommandExecutor.cs | 4 ++-- src/BenchmarkDotNet.Core/project.json | 2 +- .../BenchmarkDotNet.IntegrationTests/project.json | 2 +- tests/BenchmarkDotNet.Tests/project.json | 2 +- 8 files changed, 12 insertions(+), 22 deletions(-) rename src/BenchmarkDotNet.Core/Loggers/{AsynchronousProcessOutputLogger.cs => AsyncErrorOutputLogger.cs} (67%) diff --git a/docs/guide/Contributing/Building.md b/docs/guide/Contributing/Building.md index 0bc3e37e1e..5e6de7b22c 100644 --- a/docs/guide/Contributing/Building.md +++ b/docs/guide/Contributing/Building.md @@ -2,10 +2,10 @@ For building the BenchmarkDotNet source-code, the following elements are required: -* [Visual Studio 2015 Update **3**](http://go.microsoft.com/fwlink/?LinkId=691129) +* [Visual Studio 2015 Update **3**](https://go.microsoft.com/fwlink/?LinkId=691978) * [Latest NuGet Manager extension for Visual Studio](https://dist.nuget.org/visualstudio-2015-vsix/v3.5.0-beta/NuGet.Tools.vsix) -* [.NET Core SDK](https://go.microsoft.com/fwlink/?LinkID=809122) -* [.NET Core Tooling Preview 2 for Visual Studio 2015](https://go.microsoft.com/fwlink/?LinkId=817245) +* [.NET Core SDK **1.1**](https://go.microsoft.com/fwlink/?LinkID=835014) +* [.NET Core 1.0.1 Tooling Preview 2 for Visual Studio 2015](https://go.microsoft.com/fwlink/?LinkID=827546) * Internet connection and disk space to download all the required packages If your build fails because some packages are not available, let say F#, then just disable these project and hope for nuget server to work later on ;) diff --git a/samples/BenchmarkDotNet.Samples.FSharp.Core/project.json b/samples/BenchmarkDotNet.Samples.FSharp.Core/project.json index 73953744bc..8a097798d4 100644 --- a/samples/BenchmarkDotNet.Samples.FSharp.Core/project.json +++ b/samples/BenchmarkDotNet.Samples.FSharp.Core/project.json @@ -13,7 +13,7 @@ "Microsoft.FSharp.Core.netcore": "1.0.0-*", "Microsoft.NETCore.App": { "type": "platform", - "version": "1.1.0-preview1-001100-00" + "version": "1.1.0" }, "BenchmarkDotNet": { "target": "project" diff --git a/samples/BenchmarkDotNet.Samples/project.json b/samples/BenchmarkDotNet.Samples/project.json index 445f9b827d..c6eb661ecd 100644 --- a/samples/BenchmarkDotNet.Samples/project.json +++ b/samples/BenchmarkDotNet.Samples/project.json @@ -40,7 +40,7 @@ "dependencies": { "Microsoft.NETCore.App": { "type": "platform", - "version": "1.1.0-preview1-001100-00" + "version": "1.1.0" }, "System.ComponentModel.EventBasedAsync": "4.0.11" } diff --git a/src/BenchmarkDotNet.Core/Loggers/AsynchronousProcessOutputLogger.cs b/src/BenchmarkDotNet.Core/Loggers/AsyncErrorOutputLogger.cs similarity index 67% rename from src/BenchmarkDotNet.Core/Loggers/AsynchronousProcessOutputLogger.cs rename to src/BenchmarkDotNet.Core/Loggers/AsyncErrorOutputLogger.cs index 97a99eacb2..7ba604f3e4 100644 --- a/src/BenchmarkDotNet.Core/Loggers/AsynchronousProcessOutputLogger.cs +++ b/src/BenchmarkDotNet.Core/Loggers/AsyncErrorOutputLogger.cs @@ -3,12 +3,12 @@ namespace BenchmarkDotNet.Loggers { - internal class AsynchronousProcessOutputLogger : IDisposable + internal class AsyncErrorOutputLogger : IDisposable { private readonly Process process; private readonly ILogger logger; - public AsynchronousProcessOutputLogger(ILogger logger, Process process) + public AsyncErrorOutputLogger(ILogger logger, Process process) { if (process.StartInfo.UseShellExecute) { @@ -18,10 +18,6 @@ public AsynchronousProcessOutputLogger(ILogger logger, Process process) this.logger = logger; this.process = process; - if (process.StartInfo.RedirectStandardOutput) - { - this.process.OutputDataReceived += ProcessOnOutputDataReceived; - } if (process.StartInfo.RedirectStandardError) { this.process.ErrorDataReceived += ProcessOnErrorDataReceived; @@ -30,15 +26,9 @@ public AsynchronousProcessOutputLogger(ILogger logger, Process process) public void Dispose() { - process.OutputDataReceived -= ProcessOnOutputDataReceived; process.ErrorDataReceived -= ProcessOnErrorDataReceived; } - private void ProcessOnOutputDataReceived(object sender, DataReceivedEventArgs dataReceivedEventArgs) - { - logger.WriteLine(LogKind.Default, dataReceivedEventArgs.Data); - } - private void ProcessOnErrorDataReceived(object sender, DataReceivedEventArgs dataReceivedEventArgs) { if (!string.IsNullOrEmpty(dataReceivedEventArgs.Data)) // happened often and added unnecessary blank line to output diff --git a/src/BenchmarkDotNet.Core/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs b/src/BenchmarkDotNet.Core/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs index 08114526ef..7ee9d28673 100644 --- a/src/BenchmarkDotNet.Core/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs +++ b/src/BenchmarkDotNet.Core/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs @@ -36,7 +36,7 @@ internal static bool ExecuteCommand(string commandWithArguments, string workingD { using (var process = new Process { StartInfo = BuildStartInfo(workingDirectory, commandWithArguments) }) { - using (new AsynchronousProcessOutputLogger(logger, process)) + using (new AsyncErrorOutputLogger(logger, process)) { process.Start(); @@ -60,7 +60,7 @@ private static ProcessStartInfo BuildStartInfo(string workingDirectory, string a Arguments = arguments, UseShellExecute = false, CreateNoWindow = true, - RedirectStandardOutput = true, // we redirect it but never call process.BeginOutputReadLine() in order to ignore it + RedirectStandardOutput = true, RedirectStandardError = true }; } diff --git a/src/BenchmarkDotNet.Core/project.json b/src/BenchmarkDotNet.Core/project.json index 993d2407b8..5dd658fe50 100644 --- a/src/BenchmarkDotNet.Core/project.json +++ b/src/BenchmarkDotNet.Core/project.json @@ -58,7 +58,7 @@ "dependencies": { "Microsoft.NETCore.App": { "type": "platform", - "version": "1.1.0-preview1-001100-00" + "version": "1.1.0" }, "System.Runtime.Serialization.Primitives": "4.1.1", "Microsoft.DotNet.InternalAbstractions": "1.0.0", diff --git a/tests/BenchmarkDotNet.IntegrationTests/project.json b/tests/BenchmarkDotNet.IntegrationTests/project.json index 433acdc57b..3150dc52e4 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/project.json +++ b/tests/BenchmarkDotNet.IntegrationTests/project.json @@ -33,7 +33,7 @@ "dependencies": { "Microsoft.NETCore.App": { "type": "platform", - "version": "1.1.0-preview1-001100-00" + "version": "1.1.0" }, "System.ComponentModel.EventBasedAsync": "4.0.11" } diff --git a/tests/BenchmarkDotNet.Tests/project.json b/tests/BenchmarkDotNet.Tests/project.json index 1fdda3408c..45216578e2 100644 --- a/tests/BenchmarkDotNet.Tests/project.json +++ b/tests/BenchmarkDotNet.Tests/project.json @@ -29,7 +29,7 @@ "dependencies": { "Microsoft.NETCore.App": { "type": "platform", - "version": "1.1.0-preview1-001100-00" + "version": "1.1.0" }, "System.ComponentModel.EventBasedAsync": "4.0.11" } From e6ccee61de69c83c9ce5716819c48b9d844dc05e Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Fri, 18 Nov 2016 10:45:17 +0100 Subject: [PATCH 10/11] always show Gen 0 column, display Gen 0/1/2 per 1k op --- .../Diagnosers/MemoryDiagnoser.cs | 27 +++++++++++-------- .../MemoryDiagnoserTests.cs | 7 ++--- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs b/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs index 1a10de4e1e..ce885489f8 100644 --- a/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs +++ b/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs @@ -14,14 +14,16 @@ namespace BenchmarkDotNet.Diagnosers { public class MemoryDiagnoser : IDiagnoser { + private const int Gen0 = 0, Gen1 = 1, Gen2 = 2; + public static readonly MemoryDiagnoser Default = new MemoryDiagnoser(); private readonly Dictionary results = new Dictionary(); public IColumnProvider GetColumnProvider() => new SimpleColumnProvider( - new GCCollectionColumn(results, 0), - new GCCollectionColumn(results, 1), - new GCCollectionColumn(results, 2), + new GCCollectionColumn(results, Gen0), + new GCCollectionColumn(results, Gen1), + new GCCollectionColumn(results, Gen2), new AllocationColumn(results)); // the following methods are left empty on purpose @@ -30,9 +32,11 @@ public void BeforeAnythingElse(Process process, Benchmark benchmark) { } public void AfterSetup(Process process, Benchmark benchmark) { } public void BeforeCleanup() { } - public void DisplayResults(ILogger logger) { } // no custom output + public void DisplayResults(ILogger logger) + => logger.WriteInfo("Note: the Gen 0/1/2/ Measurements are per 1k Operations"); - public void ProcessResults(Benchmark benchmark, BenchmarkReport report) => results.Add(benchmark, report.GcStats); + public void ProcessResults(Benchmark benchmark, BenchmarkReport report) + => results.Add(benchmark, report.GcStats); public class AllocationColumn : IColumn { @@ -76,15 +80,16 @@ public GCCollectionColumn(Dictionary results, int generation public bool IsDefault(Summary summary, Benchmark benchmark) => true; public string Id => $"{nameof(GCCollectionColumn)}{generation}"; public string ColumnName => $"Gen {generation}"; - public bool AlwaysShow => false; + + public bool AlwaysShow => generation == Gen0; // Gen 0 must always be visible public ColumnCategory Category => ColumnCategory.Diagnoser; public int PriorityInCategory => 0; public bool IsAvailable(Summary summary) - => generation == 0 // Gen 0 must always be visible + => generation == Gen0 || summary .Reports - .Any(report => generation == 1 + .Any(report => generation == Gen1 ? report.GcStats.Gen1Collections != 0 : report.GcStats.Gen2Collections != 0); @@ -93,13 +98,13 @@ public string GetValue(Summary summary, Benchmark benchmark) if (results.ContainsKey(benchmark)) { var gcStats = results[benchmark]; - var value = generation == 0 ? gcStats.Gen0Collections : - generation == 1 ? gcStats.Gen1Collections : gcStats.Gen2Collections; + var value = generation == Gen0 ? gcStats.Gen0Collections : + generation == Gen1 ? gcStats.Gen1Collections : gcStats.Gen2Collections; if (value == 0) return "-"; // make zero more obvious - return (value / (double)gcStats.TotalOperations).ToString("#0.000‰", HostEnvironmentInfo.MainCultureInfo); + return ((value / (double)gcStats.TotalOperations) * 1000).ToString("#0.0000", HostEnvironmentInfo.MainCultureInfo); } return "N/A"; } diff --git a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs index 4ce34505a7..19a12a0268 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs @@ -8,6 +8,7 @@ using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; using BenchmarkDotNet.Tests.Loggers; using Xunit; @@ -93,7 +94,7 @@ private void AssertAllocations(Type benchmarkType, int targetCount, var summary = BenchmarkRunner.Run((Benchmark[])benchmarks, config); - var allocationColumn = GetColumns(memoryDiagnoser).Single(); + var allocationColumn = GetColumns(memoryDiagnoser, summary).Single(); foreach (var benchmarkAllocationsValidator in benchmarksAllocationsValidators) { @@ -124,8 +125,8 @@ private IConfig CreateConfig(IDiagnoser diagnoser, int targetCount) .With(new OutputLogger(output)); } - private static T[] GetColumns(MemoryDiagnoser memoryDiagnoser) - => memoryDiagnoser.GetColumnProvider().GetColumns(null).OfType().ToArray(); + private static T[] GetColumns(MemoryDiagnoser memoryDiagnoser, Summary summary) + => memoryDiagnoser.GetColumnProvider().GetColumns(summary).OfType().ToArray(); private static void AssertParsed(string text, Predicate condition) { From eae2cd5c24fcb7388df4b3fe69b32ba9c40d8c22 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Wed, 23 Nov 2016 17:39:08 +0100 Subject: [PATCH 11/11] added documentation and smarter bytes formatting --- docs/guide/Configs/Diagnosers.md | 36 ++++++++++++------- .../Intro/IntroGcMode.cs | 4 +-- .../Configs/DefaultConfig.cs | 2 +- .../Diagnosers/MemoryDiagnoser.cs | 5 +-- .../Extensions/CommonExtensions.cs | 16 +++++++++ .../Toolchains/Executor.cs | 5 ++- .../MemoryDiagnoserTests.cs | 2 +- 7 files changed, 48 insertions(+), 22 deletions(-) diff --git a/docs/guide/Configs/Diagnosers.md b/docs/guide/Configs/Diagnosers.md index 3befa86b21..70eee38fb1 100644 --- a/docs/guide/Configs/Diagnosers.md +++ b/docs/guide/Configs/Diagnosers.md @@ -1,25 +1,22 @@ # Diagnosers -A **diagnoser** can attach to your benchmark and get some useful info. There is a separated package with diagnosers for Windows (`BenchmarkDotNet.Diagnostics.Windows`): - -[![NuGet](https://img.shields.io/nuget/v/BenchmarkDotNet.svg)](https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/) - +A **diagnoser** can attach to your benchmark and get some useful info. The current Diagnosers are: -- GC and Memory Allocation (`MemoryDiagnoser`) -- JIT Inlining Events (`InliningDiagnoser`) +- GC and Memory Allocation (`MemoryDiagnoser`) which is cross platform, built-in and enabled by default. +- JIT Inlining Events (`InliningDiagnoser`). You can find this diagnoser in a separated package with diagnosers for Windows (`BenchmarkDotNet.Diagnostics.Windows`): [![NuGet](https://img.shields.io/nuget/v/BenchmarkDotNet.svg)](https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/) ## Examples -Below is a sample output from the `GC and Memory Allocation` diagnoser, note the extra columns on the right-hand side ("Gen 0", "Gen 1", "Gen 2" and "Bytes Allocated/Op"): - - Method | Lookup | Median | StdDev | Scaled | Gen 0 | Gen 1 | Gen 2 | Bytes Allocated/Op | ----------- |-------- |----------- |---------- |------- |--------- |------ |------ |------------------- | - LINQ | Testing | 49.1154 ns | 0.5301 ns | 2.48 | 1,526.00 | - | - | 25.21 | - Iterative | Testing | 19.8040 ns | 0.0456 ns | 1.00 | - | - | - | 0.00 | +Below is a sample output from the `GC and Memory Allocation` diagnoser, note the extra columns on the right-hand side ("Gen 0", "Gen 1", "Gen 2" and "Allocated"): +``` + Method | Mean | StdErr | StdDev | Median | Gen 0 | Allocated | +----------------- |------------ |----------- |------------ |------------ |------- |---------- | + 'new byte[10kB]' | 884.4896 ns | 46.3528 ns | 245.2762 ns | 776.4237 ns | 0.1183 | 10 kB | + ``` A config example: @@ -28,8 +25,21 @@ private class Config : ManualConfig { public Config() { - Add(new MemoryDiagnoser()); + Add(MemoryDiagnoser.Default); Add(new InliningDiagnoser()); } } ``` + +You can also use one of the following attributes (apply it on a class that contains Benchmarks): +```cs +[MemoryDiagnoser] +[InliningDiagnoser] +``` + +## Restrictions + +* Mono currently [does not](http://stackoverflow.com/questions/40234948/how-to-get-the-number-of-allocated-bytes-in-mono) expose any api to get the number of allocated bytes. That's why our Mono users will get `?` in Allocated column. +* In order to get the number of allocated bytes in cross platform way we are using `GC.GetAllocatedBytesForCurrentThread` which recently got [exposed](https://github.com/dotnet/corefx/pull/12489) for netcoreapp1.1. That's why BenchmarkDotNet does not support netcoreapp1.0 from version 0.10.1. +* In order to not affect main results we perform a separate run if any diagnoser is used. That's why it might take more time to execute benchmarks. +* MemoryDiagnoser is `100%` accurate about allocated memory when using Job.ShortRun or longer job. diff --git a/samples/BenchmarkDotNet.Samples/Intro/IntroGcMode.cs b/samples/BenchmarkDotNet.Samples/Intro/IntroGcMode.cs index 3a0beacccb..e3107b4eeb 100644 --- a/samples/BenchmarkDotNet.Samples/Intro/IntroGcMode.cs +++ b/samples/BenchmarkDotNet.Samples/Intro/IntroGcMode.cs @@ -22,13 +22,13 @@ public Config() } } - [Benchmark(Description = "new byte[10KB]")] + [Benchmark(Description = "new byte[10kB]")] public byte[] Allocate() { return new byte[10000]; } - [Benchmark(Description = "stackalloc byte[10KB]")] + [Benchmark(Description = "stackalloc byte[10kB]")] public unsafe void AllocateWithStackalloc() { var array = stackalloc byte[10000]; diff --git a/src/BenchmarkDotNet.Core/Configs/DefaultConfig.cs b/src/BenchmarkDotNet.Core/Configs/DefaultConfig.cs index 7f1a9b2417..a58c319e5a 100644 --- a/src/BenchmarkDotNet.Core/Configs/DefaultConfig.cs +++ b/src/BenchmarkDotNet.Core/Configs/DefaultConfig.cs @@ -89,7 +89,7 @@ private static IDiagnoser[] LoadDiagnosers() { return new[] { - GetDiagnoser(loadedAssembly, "BenchmarkDotNet.Diagnostics.Windows.MemoryDiagnoser"), + MemoryDiagnoser.Default, GetDiagnoser(loadedAssembly, "BenchmarkDotNet.Diagnostics.Windows.InliningDiagnoser"), }; } diff --git a/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs b/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs index ce885489f8..7cdac68f99 100644 --- a/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs +++ b/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs @@ -8,6 +8,7 @@ using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Portability; namespace BenchmarkDotNet.Diagnosers @@ -48,7 +49,7 @@ public AllocationColumn(Dictionary results) } public string Id => nameof(AllocationColumn); - public string ColumnName => "Bytes Allocated"; + public string ColumnName => "Allocated"; public bool IsDefault(Summary summary, Benchmark benchmark) => false; public bool IsAvailable(Summary summary) => true; public bool AlwaysShow => true; @@ -62,7 +63,7 @@ public string GetValue(Summary summary, Benchmark benchmark) if (!results.ContainsKey(benchmark)) return "N/A"; - return results[benchmark].BytesAllocatedPerOperation.ToString("N0", HostEnvironmentInfo.MainCultureInfo); + return results[benchmark].BytesAllocatedPerOperation.ToFormattedBytes(); } } diff --git a/src/BenchmarkDotNet.Core/Extensions/CommonExtensions.cs b/src/BenchmarkDotNet.Core/Extensions/CommonExtensions.cs index 379fc63148..c40fa4f6c9 100644 --- a/src/BenchmarkDotNet.Core/Extensions/CommonExtensions.cs +++ b/src/BenchmarkDotNet.Core/Extensions/CommonExtensions.cs @@ -8,6 +8,10 @@ namespace BenchmarkDotNet.Extensions { internal static class CommonExtensions { + const int BytesInKiloByte = 1000; // 1000 vs 1024 thing.. + + static readonly string[] SizeSuffixes = { "B", "kB", "MB", "GB", "TB" }; + public static List ToSortedList(this IEnumerable values) { var list = new List(); @@ -24,6 +28,18 @@ public static string ToTimeStr(this double value, TimeUnit unit = null, int unit return $"{unitValue.ToStr("N4")} {unitName}"; } + internal static string ToFormattedBytes(this long bytes) + { + int i; + double dblSByte = bytes; + for (i = 0; i < SizeSuffixes.Length && bytes >= BytesInKiloByte; i++, bytes /= BytesInKiloByte) + { + dblSByte = bytes / (double)BytesInKiloByte; + } + + return string.Format(HostEnvironmentInfo.MainCultureInfo, "{0:0.##} {1}", dblSByte, SizeSuffixes[i]); + } + public static string ToStr(this double value, string format = "0.##") { // Here we should manually create an object[] for string.Format diff --git a/src/BenchmarkDotNet.Core/Toolchains/Executor.cs b/src/BenchmarkDotNet.Core/Toolchains/Executor.cs index c96e626f8b..edc5365cf1 100644 --- a/src/BenchmarkDotNet.Core/Toolchains/Executor.cs +++ b/src/BenchmarkDotNet.Core/Toolchains/Executor.cs @@ -48,7 +48,7 @@ private ExecuteResult Execute(Benchmark benchmark, ILogger logger, string exeNam { var loggerWithDiagnoser = new SynchronousProcessOutputLoggerWithDiagnoser(logger, process, diagnoser, benchmark); - return Execute(process, benchmark, loggerWithDiagnoser, diagnoser, logger); + return Execute(process, benchmark, loggerWithDiagnoser, logger); } } finally @@ -57,8 +57,7 @@ private ExecuteResult Execute(Benchmark benchmark, ILogger logger, string exeNam } } - private ExecuteResult Execute(Process process, Benchmark benchmark, SynchronousProcessOutputLoggerWithDiagnoser loggerWithDiagnoser, - IDiagnoser compositeDiagnoser, ILogger logger) + private ExecuteResult Execute(Process process, Benchmark benchmark, SynchronousProcessOutputLoggerWithDiagnoser loggerWithDiagnoser, ILogger logger) { consoleHandler.SetProcess(process); diff --git a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs index 19a12a0268..1830523fde 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs @@ -131,7 +131,7 @@ private static T[] GetColumns(MemoryDiagnoser memoryDiagnoser, Summary summar private static void AssertParsed(string text, Predicate condition) { long value; - if (long.TryParse(text, NumberStyles.Number, HostEnvironmentInfo.MainCultureInfo, out value)) + if (long.TryParse(text.Replace(" B", string.Empty), NumberStyles.Number, HostEnvironmentInfo.MainCultureInfo, out value)) { Assert.True(condition(value), $"Failed for value {value}"); }