From 78ff1a8ba56efff56b1042f4b88a354534ba452a Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Tue, 24 Oct 2023 17:27:51 +0200 Subject: [PATCH 01/13] Port the OOM embed files fix --- .../BinaryLogger/BinaryLogger.cs | 16 +++- .../BinaryLogger/BuildEventArgsWriter.cs | 16 ++++ .../BinaryLogger/ProjectImportsCollector.cs | 95 ++++++++++--------- 3 files changed, 79 insertions(+), 48 deletions(-) diff --git a/src/StructuredLogger/BinaryLogger/BinaryLogger.cs b/src/StructuredLogger/BinaryLogger/BinaryLogger.cs index 6d8b85800..2b8c6dba3 100644 --- a/src/StructuredLogger/BinaryLogger/BinaryLogger.cs +++ b/src/StructuredLogger/BinaryLogger/BinaryLogger.cs @@ -222,9 +222,23 @@ public void Shutdown() if (projectImportsCollector != null) { + projectImportsCollector.Close(); + if (CollectProjectImports == ProjectImportsCollectionMode.Embed) { - eventArgsWriter.WriteBlob(BinaryLogRecordKind.ProjectImportArchive, projectImportsCollector.GetAllBytes()); + var archiveFilePath = projectImportsCollector.ArchiveFilePath; + + // It is possible that the archive couldn't be created for some reason. + // Only embed it if it actually exists. + if (File.Exists(archiveFilePath)) + { + using (FileStream fileStream = File.OpenRead(archiveFilePath)) + { + eventArgsWriter.WriteBlob(BinaryLogRecordKind.ProjectImportArchive, fileStream); + } + + File.Delete(archiveFilePath); + } } projectImportsCollector.Close(); diff --git a/src/StructuredLogger/BinaryLogger/BuildEventArgsWriter.cs b/src/StructuredLogger/BinaryLogger/BuildEventArgsWriter.cs index 7e83a77d7..cc588b673 100644 --- a/src/StructuredLogger/BinaryLogger/BuildEventArgsWriter.cs +++ b/src/StructuredLogger/BinaryLogger/BuildEventArgsWriter.cs @@ -214,6 +214,17 @@ public void WriteBlob(BinaryLogRecordKind kind, byte[] bytes) Write(bytes); } + public void WriteBlob(BinaryLogRecordKind kind, Stream stream) + { + // write the blob directly to the underlying writer, + // bypassing the memory stream + using var redirection = RedirectWritesToOriginalWriter(); + + Write(kind); + Write(stream.Length); + Write(stream); + } + /// /// Switches the binaryWriter used by the Write* methods to the direct underlying stream writer /// until the disposable is disposed. Useful to bypass the currentRecordWriter to write a string, @@ -1113,6 +1124,11 @@ private void Write(byte[] bytes) binaryWriter.Write(bytes); } + private void Write(Stream stream) + { + stream.CopyTo(binaryWriter.BaseStream); + } + private void Write(byte b) { binaryWriter.Write(b); diff --git a/src/StructuredLogger/BinaryLogger/ProjectImportsCollector.cs b/src/StructuredLogger/BinaryLogger/ProjectImportsCollector.cs index 5b2b1ae4c..ebc775a96 100644 --- a/src/StructuredLogger/BinaryLogger/ProjectImportsCollector.cs +++ b/src/StructuredLogger/BinaryLogger/ProjectImportsCollector.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.IO.Compression; using System.Text; @@ -15,30 +16,10 @@ namespace Microsoft.Build.Logging /// internal class ProjectImportsCollector { - private Stream _stream; - public byte[] GetAllBytes() - { - if (_stream == null) - { - return Array.Empty(); - } - else if (ArchiveFilePath == null) - { - var stream = _stream as MemoryStream; - // Before we can use the zip archive, it must be closed. - Close(false); - return stream.ToArray(); - } - else - { - Close(); - return File.ReadAllBytes(ArchiveFilePath); - } - } - + private Stream _fileStream; private ZipArchive _zipArchive; - private string ArchiveFilePath { get; set; } + public string ArchiveFilePath { get; } /// /// Avoid visiting each file more than once. @@ -50,33 +31,56 @@ public byte[] GetAllBytes() public ProjectImportsCollector(string logFilePath, bool createFile, string sourcesArchiveExtension = ".ProjectImports.zip") { - try + if (createFile) { - if (createFile) - { - ArchiveFilePath = Path.ChangeExtension(logFilePath, sourcesArchiveExtension); - _stream = new FileStream(ArchiveFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.Delete); - } - else + // Archive file will be stored alongside the binlog + ArchiveFilePath = Path.ChangeExtension(logFilePath, sourcesArchiveExtension); + } + else + { + + + string cacheDirectory = GetCacheDirectory(); + if (!Directory.Exists(cacheDirectory)) { - _stream = new MemoryStream(); + Directory.CreateDirectory(cacheDirectory); } - _zipArchive = new ZipArchive(_stream, ZipArchiveMode.Create, true); + + // Archive file will be temporarily stored in MSBuild cache folder and deleted when no longer needed + ArchiveFilePath = Path.Combine( + cacheDirectory, + Path.ChangeExtension( + Path.GetFileName(logFilePath), + sourcesArchiveExtension)); + } + + try + { + _fileStream = new FileStream(ArchiveFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.Delete); + _zipArchive = new ZipArchive(_fileStream, ZipArchiveMode.Create); } catch { // For some reason we weren't able to create a file for the archive. // Disable the file collector. - _stream = null; + _fileStream = null; _zipArchive = null; } } + private static string GetCacheDirectory() + { + string dir = Path.Combine(Path.GetTempPath(), + $"MSBuildTemp-{Environment.UserName}-{Process.GetCurrentProcess().Id}-{AppDomain.CurrentDomain.Id}"); + Directory.CreateDirectory(dir); + return dir; + } + public void AddFile(string filePath) { - if (filePath != null && _stream != null) + if (filePath != null && _fileStream != null) { - lock (_stream) + lock (_fileStream) { // enqueue the task to add a file and return quickly // to avoid holding up the current thread @@ -96,9 +100,9 @@ public void AddFile(string filePath) public void AddFileFromMemory(string filePath, string data) { - if (filePath != null && data != null && _stream != null) + if (filePath != null && data != null && _fileStream != null) { - lock (_stream) + lock (_fileStream) { // enqueue the task to add a file and return quickly // to avoid holding up the current thread @@ -128,8 +132,7 @@ private void AddFileCore(string filePath) return; } - var fileInfo = new FileInfo(filePath); - if (!fileInfo.Exists || fileInfo.Length == 0) + if (!File.Exists(filePath)) { _processedFiles.Add(filePath); return; @@ -143,11 +146,9 @@ private void AddFileCore(string filePath) return; } - using (Stream entryStream = OpenArchiveEntry(filePath)) - using (FileStream content = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete)) - { - content.CopyTo(entryStream); - } + using FileStream content = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete); + using Stream entryStream = OpenArchiveEntry(filePath); + content.CopyTo(entryStream); } /// @@ -195,7 +196,7 @@ private static string CalculateArchivePath(string filePath) return archivePath; } - public void Close(bool closeStream = true) + public void Close() { // wait for all pending file writes to complete _currentTask.Wait(); @@ -206,10 +207,10 @@ public void Close(bool closeStream = true) _zipArchive = null; } - if (closeStream && (_stream != null)) + if (_fileStream != null) { - _stream.Dispose(); - _stream = null; + _fileStream.Dispose(); + _fileStream = null; } } } From c2d11e94f6798f588eb6e1a0b842e2849a959ed3 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Wed, 25 Oct 2023 18:48:23 +0200 Subject: [PATCH 02/13] Port the missed BuildEventArgsWriter changes --- .../BinaryLogger/BuildEventArgsWriter.cs | 197 ++++++++++++------ src/StructuredLogger/StructuredLogger.cs | 10 +- 2 files changed, 135 insertions(+), 72 deletions(-) diff --git a/src/StructuredLogger/BinaryLogger/BuildEventArgsWriter.cs b/src/StructuredLogger/BinaryLogger/BuildEventArgsWriter.cs index cc588b673..d4b01ebfc 100644 --- a/src/StructuredLogger/BinaryLogger/BuildEventArgsWriter.cs +++ b/src/StructuredLogger/BinaryLogger/BuildEventArgsWriter.cs @@ -6,6 +6,7 @@ using System.Collections.Generic; using System.Globalization; using System.IO; +using System.Linq; using Microsoft.Build.Framework; using Microsoft.Build.Framework.Profiler; using Microsoft.Build.Internal; @@ -133,43 +134,47 @@ public void Write(BuildEventArgs e) currentRecordStream.SetLength(0); } -/* -Base types and inheritance ("EventArgs" suffix omitted): - -Build - Telemetry - LazyFormattedBuild - BuildMessage - CriticalBuildMessage - EnvironmentVariableRead - FileUsed - MetaprojectGenerated - ProjectImported - PropertyInitialValueSet - PropertyReassignment - TargetSkipped - TaskCommandLine - TaskParameter - UninitializedPropertyRead - AssemblyLoaded - BuildStatus - TaskStarted - TaskFinished - TargetStarted - TargetFinished - ProjectStarted - ProjectFinished - BuildStarted - BuildFinished - ProjectEvaluationStarted - ProjectEvaluationFinished - BuildError - BuildWarning - CustomBuild - ExternalProjectStarted - ExternalProjectFinished - -*/ + /* + Base types and inheritance ("EventArgs" suffix omitted): + + Build + Telemetry + LazyFormattedBuild + BuildMessage + CriticalBuildMessage + EnvironmentVariableRead + FileUsed + MetaprojectGenerated + ProjectImported + PropertyInitialValueSet + PropertyReassignment + TargetSkipped + TaskCommandLine + TaskParameter + UninitializedPropertyRead + AssemblyLoaded + ExtendedMessage + BuildStatus + TaskStarted + TaskFinished + TargetStarted + TargetFinished + ProjectStarted + ProjectFinished + BuildStarted + BuildFinished + ProjectEvaluationStarted + ProjectEvaluationFinished + BuildError + ExtendedBuildError + BuildWarning + ExtendedBuildWarning + CustomBuild + ExternalProjectStarted + ExternalProjectFinished + ExtendedCustomBuild + + */ private void WriteCore(BuildEventArgs e) { @@ -191,12 +196,31 @@ private void WriteCore(BuildEventArgs e) default: // convert all unrecognized objects to message // and just preserve the message - var buildMessageEventArgs = new BuildMessageEventArgs( - e.Message, - e.HelpKeyword, - e.SenderName, - MessageImportance.Normal, - e.Timestamp); + BuildMessageEventArgs buildMessageEventArgs; + if (e is IExtendedBuildEventArgs extendedData) + { + // For Extended events convert to ExtendedBuildMessageEventArgs + buildMessageEventArgs = new ExtendedBuildMessageEventArgs( + extendedData.ExtendedType, + e.Message, + e.HelpKeyword, + e.SenderName, + MessageImportance.Normal, + e.Timestamp) + { + ExtendedData = extendedData.ExtendedData, + ExtendedMetadata = extendedData.ExtendedMetadata, + }; + } + else + { + buildMessageEventArgs = new BuildMessageEventArgs( + e.Message, + e.HelpKeyword, + e.SenderName, + MessageImportance.Normal, + e.Timestamp); + } buildMessageEventArgs.BuildEventContext = e.BuildEventContext ?? BuildEventContext.Invalid; Write(buildMessageEventArgs); break; @@ -216,12 +240,17 @@ public void WriteBlob(BinaryLogRecordKind kind, byte[] bytes) public void WriteBlob(BinaryLogRecordKind kind, Stream stream) { + if (stream.Length > int.MaxValue) + { + throw new ArgumentOutOfRangeException(nameof(stream)); + } + // write the blob directly to the underlying writer, // bypassing the memory stream using var redirection = RedirectWritesToOriginalWriter(); Write(kind); - Write(stream.Length); + Write((int)stream.Length); Write(stream); } @@ -261,10 +290,17 @@ private void Write(BuildStartedEventArgs e) } else { - Write(0); + Write(e.BuildEnvironment?.Where(kvp => IsWellKnownEnvironmentDerivedProperty(kvp.Key))); } } + private static bool IsWellKnownEnvironmentDerivedProperty(string propertyName) + { + return propertyName.StartsWith("MSBUILD") || + propertyName.StartsWith("COMPLUS_") || + propertyName.StartsWith("DOTNET_"); + } + private void Write(BuildFinishedEventArgs e) { Write(BinaryLogRecordKind.BuildFinished); @@ -383,7 +419,9 @@ private void Write(TargetFinishedEventArgs e) private void Write(TaskStartedEventArgs e) { Write(BinaryLogRecordKind.TaskStarted); - WriteBuildEventArgsFields(e, writeMessage: false); + WriteBuildEventArgsFields(e, writeMessage: false, writeLineAndColumn: true); + Write(e.LineNumber); + Write(e.ColumnNumber); WriteDeduplicatedString(e.TaskName); WriteDeduplicatedString(e.ProjectFile); WriteDeduplicatedString(e.TaskFile); @@ -433,6 +471,7 @@ private void Write(BuildMessageEventArgs e) { switch (e) { + case ResponseFileUsedEventArgs responseFileUsed: Write(responseFileUsed); break; case TaskParameterEventArgs taskParameter: Write(taskParameter); break; case ProjectImportedEventArgs projectImported: Write(projectImported); break; case TargetSkippedEventArgs targetSkipped: Write(targetSkipped); break; @@ -442,8 +481,7 @@ private void Write(BuildMessageEventArgs e) case EnvironmentVariableReadEventArgs environmentVariableRead: Write(environmentVariableRead); break; case PropertyInitialValueSetEventArgs propertyInitialValueSet: Write(propertyInitialValueSet); break; case CriticalBuildMessageEventArgs criticalBuildMessage: Write(criticalBuildMessage); break; - case FileUsedEventArgs fileUsed: Write(fileUsed); break; - case AssemblyLoadBuildEventArgs assemblyLoaded: Write(assemblyLoaded); break; + case AssemblyLoadBuildEventArgs assemblyLoad: Write(assemblyLoad); break; default: // actual BuildMessageEventArgs Write(BinaryLogRecordKind.Message); WriteMessageFields(e, writeImportance: true); @@ -467,12 +505,24 @@ private void Write(TargetSkippedEventArgs e) WriteDeduplicatedString(e.TargetFile); WriteDeduplicatedString(e.TargetName); WriteDeduplicatedString(e.ParentTarget); - WriteDeduplicatedString(""); // Condition - WriteDeduplicatedString(""); // EvaluatedCondition - Write(true); // OriginallySucceeded + WriteDeduplicatedString(e.Condition); // Condition + WriteDeduplicatedString(e.EvaluatedCondition); // EvaluatedCondition + Write(e.OriginallySucceeded); // OriginallySucceeded Write((int)e.BuildReason); - Write((int)0); // SkipReason - Write(false); // OptionalBuildEventContext + Write((int)e.SkipReason); // SkipReason + binaryWriter.WriteOptionalBuildEventContext(e.OriginalBuildEventContext); // OptionalBuildEventContext + } + + private void Write(AssemblyLoadBuildEventArgs e) + { + Write(BinaryLogRecordKind.AssemblyLoad); + WriteMessageFields(e, writeMessage: false, writeImportance: false); + Write((int)e.LoadingContext); + WriteDeduplicatedString(e.LoadingInitiator); + WriteDeduplicatedString(e.AssemblyName); + WriteDeduplicatedString(e.AssemblyPath); + Write(e.MVID); + WriteDeduplicatedString(e.AppDomainDescriptor); } private void Write(CriticalBuildMessageEventArgs e) @@ -514,23 +564,11 @@ private void Write(EnvironmentVariableReadEventArgs e) WriteDeduplicatedString(e.EnvironmentVariableName); } - private void Write(FileUsedEventArgs e) + private void Write(ResponseFileUsedEventArgs e) { Write(BinaryLogRecordKind.FileUsed); - WriteMessageFields(e, writeImportance: false); - WriteDeduplicatedString(e.FilePath); - } - - private void Write(AssemblyLoadBuildEventArgs e) - { - Write(BinaryLogRecordKind.AssemblyLoad); - WriteMessageFields(e, writeMessage: false, writeImportance: false); - Write((int)e.LoadingContext); - WriteDeduplicatedString(e.LoadingInitiator); - WriteDeduplicatedString(e.AssemblyName); - WriteDeduplicatedString(e.AssemblyPath); - Write(e.MVID); - WriteDeduplicatedString(e.AppDomainDescriptor); + WriteMessageFields(e); + WriteDeduplicatedString(e.ResponseFilePath); } private void Write(TaskCommandLineEventArgs e) @@ -548,7 +586,8 @@ private void Write(TaskParameterEventArgs e) Write((int)e.Kind); WriteDeduplicatedString(e.ItemType); WriteTaskItemList(e.Items, e.LogItemMetadata); - if (e.Kind == TaskParameterMessageKind.AddItem) + if (e.Kind == TaskParameterMessageKind.AddItem + || e.Kind == TaskParameterMessageKind.TaskOutput) { CheckForFilesToEmbed(e.ItemType, e.Items); } @@ -598,6 +637,11 @@ private void WriteBaseFields(BuildEventArgs e, BuildEventArgsFieldFlags flags) { Write(e.Timestamp); } + + if ((flags & BuildEventArgsFieldFlags.Extended) != 0) + { + Write(e as IExtendedBuildEventArgs); + } } private void WriteMessageFields(BuildMessageEventArgs e, bool writeMessage = true, bool writeImportance = false) @@ -764,6 +808,11 @@ private static BuildEventArgsFieldFlags GetBuildEventArgsFieldFlags(BuildEventAr flags |= BuildEventArgsFieldFlags.Timestamp; } + if (e is IExtendedBuildEventArgs extendedData) + { + flags |= BuildEventArgsFieldFlags.Extended; + } + return flags; } @@ -1229,6 +1278,16 @@ private void Write(ProfiledLocation e) Write(e.InclusiveTime); } + private void Write(IExtendedBuildEventArgs extendedData) + { + if (extendedData?.ExtendedType != null) + { + WriteDeduplicatedString(extendedData.ExtendedType); + Write(extendedData.ExtendedMetadata); + WriteDeduplicatedString(extendedData.ExtendedData); + } + } + internal readonly struct HashKey : IEquatable { private readonly ulong value; diff --git a/src/StructuredLogger/StructuredLogger.cs b/src/StructuredLogger/StructuredLogger.cs index e88817d83..49862cd96 100644 --- a/src/StructuredLogger/StructuredLogger.cs +++ b/src/StructuredLogger/StructuredLogger.cs @@ -103,10 +103,14 @@ public override void Shutdown() if (projectImportsCollector != null) { - var bytes = projectImportsCollector.GetAllBytes(); - construction.Build.SourceFilesArchive = bytes; - projectImportsCollector.Close(); + var archiveFilePath = projectImportsCollector.ArchiveFilePath; + if (File.Exists(archiveFilePath)) + { + var bytes = File.ReadAllBytes(archiveFilePath); + construction.Build.SourceFilesArchive = bytes; + } + projectImportsCollector = null; } From 7cf6361ab8dff809406941c2c3be0b33fabb37c4 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Thu, 26 Oct 2023 15:21:47 +0200 Subject: [PATCH 03/13] Add tests for roundtrip equality of replayed logs --- .../BinaryLoggerTests.cs | 89 ++++++++++++- .../StructuredLogger.Tests.csproj | 1 + .../BinaryLogReplayEventSource.cs | 122 ++++++++++++++++++ .../BinaryLogger/BuildEventArgsReader.cs | 12 +- .../BinaryLogger/BuildEventArgsWriter.cs | 6 +- 5 files changed, 225 insertions(+), 5 deletions(-) create mode 100644 src/StructuredLogger/BinaryLogger/BinaryLogReplayEventSource.cs diff --git a/src/StructuredLogger.Tests/BinaryLoggerTests.cs b/src/StructuredLogger.Tests/BinaryLoggerTests.cs index e86b12a3c..6c4822f06 100644 --- a/src/StructuredLogger.Tests/BinaryLoggerTests.cs +++ b/src/StructuredLogger.Tests/BinaryLoggerTests.cs @@ -3,6 +3,8 @@ using System.IO; using System.Linq; using System.Runtime.CompilerServices; +using FluentAssertions; +using Microsoft.Build.Logging; using Microsoft.Build.Logging.StructuredLogger; using StructuredLogger.Tests; using Xunit; @@ -124,6 +126,91 @@ public void TestBinaryLoggerRoundtrip(bool useInMemoryProject) AssertEx.EqualOrDiff(File.ReadAllText(xml1), File.ReadAllText(GetTestFile("4.xml"))); } + [Fact] + public void TestReaderWriterRoundtripEquality() + { + var binLog = GetTestFile("1.binlog"); + var replayedBinlog = GetTestFile("1-replayed.binlog"); + + //need to have in this repo + var logReader = new Logging.StructuredLogger.BinaryLogReplayEventSource(); + + BinaryLogger outputBinlog = new BinaryLogger() + { + Parameters = replayedBinlog + }; + outputBinlog.Initialize(logReader); + logReader.Replay(binLog); + outputBinlog.Shutdown(); + + //assert here + AssertBinlogsHaveEqualContent(binLog, replayedBinlog); + + // TODO: removing for now - replaying embedded files is not supported + //// If this assertation complicates development - it can possibly be removed + //// The structured equality above should be enough. + //AssertFilesAreBinaryEqualAfterUnpack(binLog, replayedBinlog); + } + + private static void AssertFilesAreBinaryEqualAfterUnpack(string firstPath, string secondPath) + { + using var br1 = Logging.StructuredLogger.BinaryLogReplayEventSource.OpenReader(firstPath); + using var br2 = Logging.StructuredLogger.BinaryLogReplayEventSource.OpenReader(secondPath); + const int bufferSize = 4096; + + int readCount = 0; + while (br1.ReadBytes(bufferSize) is { Length: > 0 } bytes1) + { + var bytes2 = br2.ReadBytes(bufferSize); + + bytes1.Should().BeEquivalentTo(bytes2, + $"Buffers starting at position {readCount} differ. First:{Environment.NewLine}{string.Join(",", bytes1)}{Environment.NewLine}Second:{Environment.NewLine}{string.Join(",", bytes2)}"); + readCount += bufferSize; + } + + br2.ReadBytes(bufferSize).Length.Should().Be(0, "Second buffer contains bytes after first file end"); + } + + private static void AssertBinlogsHaveEqualContent(string firstPath, string secondPath) + { + using var reader1 = Logging.StructuredLogger.BinaryLogReplayEventSource.OpenBuildEventsReader(firstPath); + using var reader2 = Logging.StructuredLogger.BinaryLogReplayEventSource.OpenBuildEventsReader(secondPath); + + //Dictionary embedFiles1 = new(); + //Dictionary embedFiles2 = new(); + + //reader1.ArchiveFileEncountered += arg + // => AddArchiveFile(embedFiles1, arg); + //reader2.ArchiveFileEncountered += arg + // => AddArchiveFile(embedFiles2, arg); + + + int i = 0; + while (reader1.Read() is { } ev1) + { + i++; + var ev2 = reader2.Read(); + + ev1.Should().BeEquivalentTo(ev2, + $"Binlogs ({firstPath} and {secondPath}) should be equal at event {i}"); + } + // Read the second reader - to confirm there are no more events + // and to force the embedded files to be read. + reader2.Read().Should().BeNull($"Binlogs ({firstPath} and {secondPath}) are not equal - second has more events >{i + 1}"); + + //Assert.Equal(embedFiles1, embedFiles2); + + //void AddArchiveFile(Dictionary files, ArchiveFileEventArgs arg) + //{ + // ArchiveFile embedFile = arg.ObtainArchiveFile(); + // string content = embedFile.GetContent(); + // files.Add(embedFile.FullPath, content); + // arg.SetResult(embedFile.FullPath, content); + //} + } + + + private static string GetProperty(Logging.StructuredLogger.Build build) { var property = build.FindFirstDescendant().FindChild("Properties").FindChild(p => p.Name == "FrameworkSDKRoot").Value; @@ -134,4 +221,4 @@ public void Dispose() { } } -} \ No newline at end of file +} diff --git a/src/StructuredLogger.Tests/StructuredLogger.Tests.csproj b/src/StructuredLogger.Tests/StructuredLogger.Tests.csproj index b5aa3af9b..b43f5b10d 100644 --- a/src/StructuredLogger.Tests/StructuredLogger.Tests.csproj +++ b/src/StructuredLogger.Tests/StructuredLogger.Tests.csproj @@ -21,6 +21,7 @@ + diff --git a/src/StructuredLogger/BinaryLogger/BinaryLogReplayEventSource.cs b/src/StructuredLogger/BinaryLogger/BinaryLogReplayEventSource.cs new file mode 100644 index 000000000..fcea41085 --- /dev/null +++ b/src/StructuredLogger/BinaryLogger/BinaryLogReplayEventSource.cs @@ -0,0 +1,122 @@ +using System; +using System.Collections.Generic; +using System.IO.Compression; +using System.IO; +using System.Text; +using Microsoft.Build.BackEnd; +using Microsoft.Build.Shared; +using System.Threading; + +namespace Microsoft.Build.Logging.StructuredLogger +{ + /// + /// Provides a method to read a binary log file (*.binlog) and replay all stored BuildEventArgs + /// by implementing IEventSource and raising corresponding events. + /// + /// The class is public so that we can call it from MSBuild.exe when replaying a log file. + public sealed class BinaryLogReplayEventSource : EventArgsDispatcher + { + /// + /// Read the provided binary log file and raise corresponding events for each BuildEventArgs + /// + /// The full file path of the binary log file + public void Replay(string sourceFilePath) + { + Replay(sourceFilePath, CancellationToken.None); + } + + /// + /// Creates a for the provided binary log file. + /// Performs decompression and buffering in the optimal way. + /// Caller is responsible for disposing the returned reader. + /// + /// + /// BinaryReader of the given binlog file. + public static BinaryReader OpenReader(string sourceFilePath) + { + Stream? stream = null; + try + { + stream = new FileStream(sourceFilePath, FileMode.Open, FileAccess.Read, FileShare.Read); + var gzipStream = new GZipStream(stream, CompressionMode.Decompress, leaveOpen: false); + + // wrapping the GZipStream in a buffered stream significantly improves performance + // and the max throughput is reached with a 32K buffer. See details here: + // https://github.com/dotnet/runtime/issues/39233#issuecomment-745598847 + var bufferedStream = new BufferedStream(gzipStream, 32768); + return new BinaryReader(bufferedStream); + } + catch (Exception) + { + stream?.Dispose(); + throw; + } + } + + /// + /// Creates a for the provided binary log file. + /// Performs decompression and buffering in the optimal way. + /// + /// + /// BinaryReader of the given binlog file. + public static BuildEventArgsReader OpenBuildEventsReader(string sourceFilePath) + => OpenBuildEventsReader(OpenReader(sourceFilePath), true); + + /// + /// Creates a for the provided binary reader over binary log file. + /// Caller is responsible for disposing the returned reader. + /// + /// + /// Indicates whether the passed BinaryReader should be closed on disposing. + /// Unknown build events or unknown parts of known build events will be ignored if this is set to true. + /// BuildEventArgsReader over the given binlog file binary reader. + public static BuildEventArgsReader OpenBuildEventsReader( + BinaryReader binaryReader, + bool closeInput, + bool allowForwardCompatibility = true) + { + int fileFormatVersion = binaryReader.ReadInt32(); + int minimumReaderVersion = binaryReader.ReadInt32(); + + // the log file is written using a newer version of file format + // that we don't know how to read + if (fileFormatVersion > BinaryLogger.FileFormatVersion && + (!allowForwardCompatibility || minimumReaderVersion > BinaryLogger.FileFormatVersion)) + { + var text = $"The log file format version is {fileFormatVersion}, whereas this version of MSBuild only supports versions up to {BinaryLogger.FileFormatVersion}."; + throw new NotSupportedException(text); + } + + return new BuildEventArgsReader(binaryReader, fileFormatVersion) + { + CloseInput = closeInput, + }; + } + + /// + /// Read the provided binary log file and raise corresponding events for each BuildEventArgs + /// + /// The full file path of the binary log file + /// A indicating the replay should stop as soon as possible. + public void Replay(string sourceFilePath, CancellationToken cancellationToken) + { + using var binaryReader = OpenReader(sourceFilePath); + Replay(binaryReader, cancellationToken); + } + + /// + /// Read the provided binary log file and raise corresponding events for each BuildEventArgs + /// + /// The binary log content binary reader - caller is responsible for disposing. + /// A indicating the replay should stop as soon as possible. + public void Replay(BinaryReader binaryReader, CancellationToken cancellationToken) + { + using BuildEventArgsReader reader = OpenBuildEventsReader(binaryReader, false); + + while (!cancellationToken.IsCancellationRequested && reader.Read() is { } instance) + { + Dispatch(instance); + } + } + } +} diff --git a/src/StructuredLogger/BinaryLogger/BuildEventArgsReader.cs b/src/StructuredLogger/BinaryLogger/BuildEventArgsReader.cs index d10284990..6fefcdc92 100644 --- a/src/StructuredLogger/BinaryLogger/BuildEventArgsReader.cs +++ b/src/StructuredLogger/BinaryLogger/BuildEventArgsReader.cs @@ -21,7 +21,7 @@ namespace Microsoft.Build.Logging.StructuredLogger /// /// Deserializes and returns BuildEventArgs-derived objects from a BinaryReader /// - internal partial class BuildEventArgsReader : IDisposable + public partial class BuildEventArgsReader : IDisposable { private readonly BinaryReader binaryReader; private readonly int fileFormatVersion; @@ -59,6 +59,12 @@ public BuildEventArgsReader(BinaryReader binaryReader, int fileFormatVersion) this.fileFormatVersion = fileFormatVersion; } + /// + /// Directs whether the passed should be closed when this instance is disposed. + /// Defaults to "false". + /// + public bool CloseInput { private get; set; } = false; + public void Dispose() { if (stringStorage != null) @@ -66,6 +72,10 @@ public void Dispose() stringStorage.Dispose(); stringStorage = null; } + if (CloseInput) + { + binaryReader.Dispose(); + } } /// diff --git a/src/StructuredLogger/BinaryLogger/BuildEventArgsWriter.cs b/src/StructuredLogger/BinaryLogger/BuildEventArgsWriter.cs index d4b01ebfc..64e92c45e 100644 --- a/src/StructuredLogger/BinaryLogger/BuildEventArgsWriter.cs +++ b/src/StructuredLogger/BinaryLogger/BuildEventArgsWriter.cs @@ -471,7 +471,7 @@ private void Write(BuildMessageEventArgs e) { switch (e) { - case ResponseFileUsedEventArgs responseFileUsed: Write(responseFileUsed); break; + case FileUsedEventArgs responseFileUsed: Write(responseFileUsed); break; case TaskParameterEventArgs taskParameter: Write(taskParameter); break; case ProjectImportedEventArgs projectImported: Write(projectImported); break; case TargetSkippedEventArgs targetSkipped: Write(targetSkipped); break; @@ -564,11 +564,11 @@ private void Write(EnvironmentVariableReadEventArgs e) WriteDeduplicatedString(e.EnvironmentVariableName); } - private void Write(ResponseFileUsedEventArgs e) + private void Write(FileUsedEventArgs e) { Write(BinaryLogRecordKind.FileUsed); WriteMessageFields(e); - WriteDeduplicatedString(e.ResponseFilePath); + WriteDeduplicatedString(e.FilePath); } private void Write(TaskCommandLineEventArgs e) From fcd5a83e9d768521389a5c9e154e6165de0936a5 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Fri, 27 Oct 2023 15:13:52 +0200 Subject: [PATCH 04/13] Added support for embedded content replaying --- .../BinaryLoggerTests.cs | 34 ++- .../BinaryLogReplayEventSource.cs | 32 ++- .../BinaryLogger/BinaryLogger.cs | 48 ++-- .../BinaryLogger/BuildEventArgsReader.cs | 126 +++++++++-- .../Postprocessing/ArchiveFileEventArgs.cs | 19 ++ .../ArchiveFileEventArgsExtensions.cs | 23 ++ .../Postprocessing/CleanupScope.cs | 15 ++ .../Postprocessing/ConcatenatedReadStream.cs | 100 +++++++++ .../EmbeddedContentEventArgs.cs | 21 ++ .../IBuildEventArgsReaderNotifications.cs | 13 ++ .../IBuildEventStringsReader.cs | 20 ++ .../Postprocessing/IBuildFileReader.cs | 34 +++ .../Postprocessing/IEmbeddedContentSource.cs | 17 ++ .../StreamChunkOverreadException.cs | 22 ++ .../Postprocessing/StreamExtensions.cs | 118 ++++++++++ .../Postprocessing/StringReadEventArgs.cs | 35 +++ .../BinaryLogger/Postprocessing/SubStream.cs | 55 +++++ .../Postprocessing/TransparentReadStream.cs | 115 ++++++++++ .../BinaryLogger/ProjectImportsCollector.cs | 208 ++++++++++++------ .../ObjectModel/ArchiveFile.cs | 9 +- src/StructuredLogger/StructuredLogger.cs | 13 +- 21 files changed, 949 insertions(+), 128 deletions(-) create mode 100644 src/StructuredLogger/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs create mode 100644 src/StructuredLogger/BinaryLogger/Postprocessing/ArchiveFileEventArgsExtensions.cs create mode 100644 src/StructuredLogger/BinaryLogger/Postprocessing/CleanupScope.cs create mode 100644 src/StructuredLogger/BinaryLogger/Postprocessing/ConcatenatedReadStream.cs create mode 100644 src/StructuredLogger/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs create mode 100644 src/StructuredLogger/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs create mode 100644 src/StructuredLogger/BinaryLogger/Postprocessing/IBuildEventStringsReader.cs create mode 100644 src/StructuredLogger/BinaryLogger/Postprocessing/IBuildFileReader.cs create mode 100644 src/StructuredLogger/BinaryLogger/Postprocessing/IEmbeddedContentSource.cs create mode 100644 src/StructuredLogger/BinaryLogger/Postprocessing/StreamChunkOverreadException.cs create mode 100644 src/StructuredLogger/BinaryLogger/Postprocessing/StreamExtensions.cs create mode 100644 src/StructuredLogger/BinaryLogger/Postprocessing/StringReadEventArgs.cs create mode 100644 src/StructuredLogger/BinaryLogger/Postprocessing/SubStream.cs create mode 100644 src/StructuredLogger/BinaryLogger/Postprocessing/TransparentReadStream.cs diff --git a/src/StructuredLogger.Tests/BinaryLoggerTests.cs b/src/StructuredLogger.Tests/BinaryLoggerTests.cs index 6c4822f06..9f0de2114 100644 --- a/src/StructuredLogger.Tests/BinaryLoggerTests.cs +++ b/src/StructuredLogger.Tests/BinaryLoggerTests.cs @@ -131,13 +131,14 @@ public void TestReaderWriterRoundtripEquality() { var binLog = GetTestFile("1.binlog"); var replayedBinlog = GetTestFile("1-replayed.binlog"); + File.Delete(replayedBinlog); //need to have in this repo var logReader = new Logging.StructuredLogger.BinaryLogReplayEventSource(); BinaryLogger outputBinlog = new BinaryLogger() { - Parameters = replayedBinlog + Parameters = $"LogFile={replayedBinlog};OmitInitialInfo" }; outputBinlog.Initialize(logReader); logReader.Replay(binLog); @@ -146,7 +147,8 @@ public void TestReaderWriterRoundtripEquality() //assert here AssertBinlogsHaveEqualContent(binLog, replayedBinlog); - // TODO: removing for now - replaying embedded files is not supported + //TODO: temporarily disabling - as we do not have guarantee for binary equality of events + // there are few mismatches between null and empty - will be fixed along with porting in-flight changes in MSBuild (needs log version update) //// If this assertation complicates development - it can possibly be removed //// The structured equality above should be enough. //AssertFilesAreBinaryEqualAfterUnpack(binLog, replayedBinlog); @@ -176,13 +178,13 @@ private static void AssertBinlogsHaveEqualContent(string firstPath, string secon using var reader1 = Logging.StructuredLogger.BinaryLogReplayEventSource.OpenBuildEventsReader(firstPath); using var reader2 = Logging.StructuredLogger.BinaryLogReplayEventSource.OpenBuildEventsReader(secondPath); - //Dictionary embedFiles1 = new(); - //Dictionary embedFiles2 = new(); + Dictionary embedFiles1 = new(); + Dictionary embedFiles2 = new(); - //reader1.ArchiveFileEncountered += arg - // => AddArchiveFile(embedFiles1, arg); - //reader2.ArchiveFileEncountered += arg - // => AddArchiveFile(embedFiles2, arg); + reader1.ArchiveFileEncountered += arg + => AddArchiveFile(embedFiles1, arg); + reader2.ArchiveFileEncountered += arg + => AddArchiveFile(embedFiles2, arg); int i = 0; @@ -198,19 +200,15 @@ private static void AssertBinlogsHaveEqualContent(string firstPath, string secon // and to force the embedded files to be read. reader2.Read().Should().BeNull($"Binlogs ({firstPath} and {secondPath}) are not equal - second has more events >{i + 1}"); - //Assert.Equal(embedFiles1, embedFiles2); + Assert.Equal(embedFiles1, embedFiles2); + embedFiles1.Should().NotBeEmpty(); - //void AddArchiveFile(Dictionary files, ArchiveFileEventArgs arg) - //{ - // ArchiveFile embedFile = arg.ObtainArchiveFile(); - // string content = embedFile.GetContent(); - // files.Add(embedFile.FullPath, content); - // arg.SetResult(embedFile.FullPath, content); - //} + void AddArchiveFile(Dictionary files, ArchiveFileEventArgs arg) + { + files.Add(arg.ArchiveFile.FullPath, arg.ArchiveFile.Text); + } } - - private static string GetProperty(Logging.StructuredLogger.Build build) { var property = build.FindFirstDescendant().FindChild("Properties").FindChild(p => p.Name == "FrameworkSDKRoot").Value; diff --git a/src/StructuredLogger/BinaryLogger/BinaryLogReplayEventSource.cs b/src/StructuredLogger/BinaryLogger/BinaryLogReplayEventSource.cs index fcea41085..3c340a009 100644 --- a/src/StructuredLogger/BinaryLogger/BinaryLogReplayEventSource.cs +++ b/src/StructuredLogger/BinaryLogger/BinaryLogReplayEventSource.cs @@ -6,15 +6,25 @@ using Microsoft.Build.BackEnd; using Microsoft.Build.Shared; using System.Threading; +using Microsoft.Build.Framework; namespace Microsoft.Build.Logging.StructuredLogger { + /// + /// Interface for replaying a binary log file (*.binlog) + /// + internal interface IBinaryLogReplaySource : + IEventSource, + // IBuildEventStringsReader, + IEmbeddedContentSource + { } + /// /// Provides a method to read a binary log file (*.binlog) and replay all stored BuildEventArgs /// by implementing IEventSource and raising corresponding events. /// /// The class is public so that we can call it from MSBuild.exe when replaying a log file. - public sealed class BinaryLogReplayEventSource : EventArgsDispatcher + public sealed class BinaryLogReplayEventSource : EventArgsDispatcher, IBinaryLogReplaySource { /// /// Read the provided binary log file and raise corresponding events for each BuildEventArgs @@ -68,20 +78,16 @@ public static BuildEventArgsReader OpenBuildEventsReader(string sourceFilePath) /// /// /// Indicates whether the passed BinaryReader should be closed on disposing. - /// Unknown build events or unknown parts of known build events will be ignored if this is set to true. /// BuildEventArgsReader over the given binlog file binary reader. public static BuildEventArgsReader OpenBuildEventsReader( BinaryReader binaryReader, - bool closeInput, - bool allowForwardCompatibility = true) + bool closeInput) { int fileFormatVersion = binaryReader.ReadInt32(); - int minimumReaderVersion = binaryReader.ReadInt32(); // the log file is written using a newer version of file format // that we don't know how to read - if (fileFormatVersion > BinaryLogger.FileFormatVersion && - (!allowForwardCompatibility || minimumReaderVersion > BinaryLogger.FileFormatVersion)) + if (fileFormatVersion > BinaryLogger.FileFormatVersion) { var text = $"The log file format version is {fileFormatVersion}, whereas this version of MSBuild only supports versions up to {BinaryLogger.FileFormatVersion}."; throw new NotSupportedException(text); @@ -113,10 +119,22 @@ public void Replay(BinaryReader binaryReader, CancellationToken cancellationToke { using BuildEventArgsReader reader = OpenBuildEventsReader(binaryReader, false); + reader.EmbeddedContentRead += _embeddedContentRead; + while (!cancellationToken.IsCancellationRequested && reader.Read() is { } instance) { Dispatch(instance); } } + + private Action? _embeddedContentRead; + /// + event Action? IEmbeddedContentSource.EmbeddedContentRead + { + // Explicitly implemented event has to declare explicit add/remove accessors + // https://stackoverflow.com/a/2268472/2308106 + add => _embeddedContentRead += value; + remove => _embeddedContentRead -= value; + } } } diff --git a/src/StructuredLogger/BinaryLogger/BinaryLogger.cs b/src/StructuredLogger/BinaryLogger/BinaryLogger.cs index 2b8c6dba3..35624ff92 100644 --- a/src/StructuredLogger/BinaryLogger/BinaryLogger.cs +++ b/src/StructuredLogger/BinaryLogger/BinaryLogger.cs @@ -125,7 +125,8 @@ public void Initialize(IEventSource eventSource) Environment.SetEnvironmentVariable("MSBUILDLOGIMPORTS", "1"); bool logPropertiesAndItemsAfterEvaluation = true; - ProcessParameters(); + ProcessParameters(out bool omitInitialInfo); + var replayEventsSource = eventSource as IBinaryLogReplaySource; try { @@ -147,7 +148,7 @@ public void Initialize(IEventSource eventSource) stream = new FileStream(FilePath, FileMode.Create); - if (CollectProjectImports != ProjectImportsCollectionMode.None) + if (CollectProjectImports != ProjectImportsCollectionMode.None && replayEventsSource == null) { projectImportsCollector = new ProjectImportsCollector(FilePath, CollectProjectImports == ProjectImportsCollectionMode.ZipFile); } @@ -186,7 +187,24 @@ public void Initialize(IEventSource eventSource) binaryWriter.Write(FileFormatVersion); - LogInitialInfo(); + if (replayEventsSource != null) + { + if (CollectProjectImports == ProjectImportsCollectionMode.Embed) + { + replayEventsSource.EmbeddedContentRead += args => + eventArgsWriter.WriteBlob(args.ContentKind, args.ContentStream); + } + else if (CollectProjectImports == ProjectImportsCollectionMode.ZipFile) + { + replayEventsSource.EmbeddedContentRead += args => + ProjectImportsCollector.FlushBlobToFile(FilePath, args.ContentStream); + } + } + + if (!omitInitialInfo) + { + LogInitialInfo(); + } eventSource.AnyEventRaised += EventSource_AnyEventRaised; } @@ -226,22 +244,13 @@ public void Shutdown() if (CollectProjectImports == ProjectImportsCollectionMode.Embed) { - var archiveFilePath = projectImportsCollector.ArchiveFilePath; - - // It is possible that the archive couldn't be created for some reason. - // Only embed it if it actually exists. - if (File.Exists(archiveFilePath)) - { - using (FileStream fileStream = File.OpenRead(archiveFilePath)) - { - eventArgsWriter.WriteBlob(BinaryLogRecordKind.ProjectImportArchive, fileStream); - } + projectImportsCollector.ProcessResult( + streamToEmbed => eventArgsWriter.WriteBlob(BinaryLogRecordKind.ProjectImportArchive, streamToEmbed), + LogMessage); - File.Delete(archiveFilePath); - } + projectImportsCollector.DeleteArchive(); } - projectImportsCollector.Close(); projectImportsCollector = null; } @@ -299,13 +308,14 @@ private void CollectImports(BuildEventArgs e) /// /// /// - private void ProcessParameters() + private void ProcessParameters(out bool omitInitialInfo) { if (Parameters == null) { throw new LoggerException(ResourceUtilities.FormatResourceStringStripCodeAndKeyword("InvalidBinaryLoggerParameters", "")); } + omitInitialInfo = false; var parameters = Parameters.Split(MSBuildConstants.SemicolonChar, StringSplitOptions.RemoveEmptyEntries); foreach (var parameter in parameters) { @@ -321,6 +331,10 @@ private void ProcessParameters() { CollectProjectImports = ProjectImportsCollectionMode.ZipFile; } + else if (string.Equals(parameter, "OmitInitialInfo", StringComparison.OrdinalIgnoreCase)) + { + omitInitialInfo = true; + } else if (parameter.EndsWith(".binlog", StringComparison.OrdinalIgnoreCase)) { FilePath = parameter; diff --git a/src/StructuredLogger/BinaryLogger/BuildEventArgsReader.cs b/src/StructuredLogger/BinaryLogger/BuildEventArgsReader.cs index 6fefcdc92..ba71efcb2 100644 --- a/src/StructuredLogger/BinaryLogger/BuildEventArgsReader.cs +++ b/src/StructuredLogger/BinaryLogger/BuildEventArgsReader.cs @@ -5,6 +5,7 @@ using System.Collections; using System.Collections.Generic; using System.IO; +using System.IO.Compression; using System.Linq; using System.Reflection; using System.Runtime.InteropServices; @@ -21,7 +22,7 @@ namespace Microsoft.Build.Logging.StructuredLogger /// /// Deserializes and returns BuildEventArgs-derived objects from a BinaryReader /// - public partial class BuildEventArgsReader : IDisposable + public partial class BuildEventArgsReader : IBuildEventArgsReaderNotifications, IDisposable { private readonly BinaryReader binaryReader; private readonly int fileFormatVersion; @@ -78,6 +79,15 @@ public void Dispose() } } + /// + public event Action? StringReadDone; + + /// + internal event Action? EmbeddedContentRead; + + /// + public event Action? ArchiveFileEncountered; + /// /// Raised when the log reader encounters a binary blob embedded in the stream. /// The arguments include the blob kind and the byte buffer with the contents. @@ -213,10 +223,91 @@ private static bool IsAuxiliaryRecord(BinaryLogRecordKind recordKind) private void ReadBlob(BinaryLogRecordKind kind) { // Work around bad logs caused by https://github.com/dotnet/msbuild/pull/9022#discussion_r1271468212 - if (kind == BinaryLogRecordKind.ProjectImportArchive && fileFormatVersion == 16) + var canHaveCorruptedSize = kind == BinaryLogRecordKind.ProjectImportArchive && fileFormatVersion == 16; + Stream embeddedStream = SliceOfEmdeddedContent(canHaveCorruptedSize); + + if (ArchiveFileEncountered != null) + { + // We could create ZipArchive over the target stream, and write to that directly, + // however, binlog format needs to know stream size upfront - which is unknown, + // so we would need to write the size after - and that would require the target stream to be seekable (which it's not) + ProjectImportsCollector? projectImportsCollector = null; + + if (EmbeddedContentRead != null) + { + projectImportsCollector = + new ProjectImportsCollector(Path.GetRandomFileName(), false, runOnBackground: false); + } + + // We are intentionally not grace handling corrupt embedded stream + + using var zipArchive = new ZipArchive(embeddedStream, ZipArchiveMode.Read); + + foreach (var entry in zipArchive.Entries/*.OrderBy(e => e.LastWriteTime)*/) + { + var file = ArchiveFile.From(entry, adjustPath: false); + ArchiveFileEventArgs archiveFileEventArgs = new(file); + ArchiveFileEncountered(archiveFileEventArgs); + + if (projectImportsCollector != null) + { + var resultFile = archiveFileEventArgs.ArchiveFile; + + projectImportsCollector.AddFileFromMemory( + resultFile.FullPath, + resultFile.Text, + makePathAbsolute: false, + entryCreationStamp: entry.LastWriteTime); + } + } + + // Once embedded files are replayed one by one - we can send the resulting stream to subscriber + if (OnBlobRead != null || EmbeddedContentRead != null) + { + projectImportsCollector!.ProcessResult( + streamToEmbed => InvokeEmbeddedDataListeners(kind, streamToEmbed), + error => throw new InvalidDataException(error)); + projectImportsCollector.DeleteArchive(); + } + } + else if (OnBlobRead != null || EmbeddedContentRead != null) + { + InvokeEmbeddedDataListeners(kind, embeddedStream); + } + else + { + embeddedStream.SkipBytes(); + } + } + + private void InvokeEmbeddedDataListeners(BinaryLogRecordKind kind, Stream embeddedStream) + { + byte[] bytes = null; + // we need to materialize the stream into a byte array + if (OnBlobRead != null) + { + bytes = embeddedStream.ReadToEnd(); + OnBlobRead(kind, bytes); + } + + if (EmbeddedContentRead != null) + { + // the embed stream was already read - we need to simulate the stream + if (bytes != null) + { + embeddedStream = new MemoryStream(bytes, writable: false); + } + + EmbeddedContentRead(new EmbeddedContentEventArgs(kind, embeddedStream)); + } + } + + private Stream SliceOfEmdeddedContent(bool canHaveCorruptedSize) + { + // Work around bad logs caused by https://github.com/dotnet/msbuild/pull/9022#discussion_r1271468212 + if (canHaveCorruptedSize) { int length; - byte[] bytes; // We have to preread some bytes to figure out if the log is buggy, // so store bytes to backfill for the "real" read later. @@ -260,23 +351,22 @@ private void ReadBlob(BinaryLogRecordKind kind) prefixBytes = reader.ReadBytes(12 - bytesRead); } - bytes = binaryReader.ReadBytes(length - prefixBytes.Length); - - byte[] fullBytes = new byte[length]; - prefixBytes.CopyTo(fullBytes, 0); - bytes.CopyTo(fullBytes, prefixBytes.Length); - bytes = fullBytes; - - OnBlobRead?.Invoke(kind, bytes); + Stream prefixStream = new MemoryStream(prefixBytes); + Stream dataStream = binaryReader.BaseStream.Slice(length - prefixBytes.Length); + return prefixStream.Concat(dataStream); } else { int length = ReadInt32(); - byte[] bytes = binaryReader.ReadBytes(length); - OnBlobRead?.Invoke(kind, bytes); + return binaryReader.BaseStream.Slice(length); } } + private void SkipBytes(int count) + { + binaryReader.BaseStream.SkipBytes(count, true); + } + private readonly List<(int name, int value)> nameValues = new List<(int name, int value)>(4096); private void ReadNameValueList() @@ -1458,9 +1548,17 @@ private IEnumerable ReadTaskItemList() return list; } + private readonly StringReadEventArgs stringReadEventArgs = new StringReadEventArgs(string.Empty); private string ReadString() { - return binaryReader.ReadString(); + string text = binaryReader.ReadString(); + if (this.StringReadDone != null) + { + stringReadEventArgs.Reuse(text); + StringReadDone(stringReadEventArgs); + text = stringReadEventArgs.StringToBeUsed; + } + return text; } private string ReadOptionalString() diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs new file mode 100644 index 000000000..0d7421ab2 --- /dev/null +++ b/src/StructuredLogger/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs @@ -0,0 +1,19 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Microsoft.Build.Logging.StructuredLogger; + +namespace Microsoft.Build.Logging; + +/// +/// Event arguments for event. +/// +public sealed class ArchiveFileEventArgs : EventArgs +{ + public ArchiveFileEventArgs(ArchiveFile archiveFile) => + ArchiveFile = archiveFile; + + + public ArchiveFile ArchiveFile { get; set; } +} diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/ArchiveFileEventArgsExtensions.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/ArchiveFileEventArgsExtensions.cs new file mode 100644 index 000000000..9580f7b20 --- /dev/null +++ b/src/StructuredLogger/BinaryLogger/Postprocessing/ArchiveFileEventArgsExtensions.cs @@ -0,0 +1,23 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Microsoft.Build.Logging.StructuredLogger; + +namespace Microsoft.Build.Logging; + +public static class ArchiveFileEventArgsExtensions +{ + public static Action ToArchiveFileHandler(this Action stringHandler) + { + return args => + { + var pathArgs = new StringReadEventArgs(args.ArchiveFile.FullPath); + stringHandler(pathArgs); + var contentArgs = new StringReadEventArgs(args.ArchiveFile.Text); + stringHandler(contentArgs); + + args.ArchiveFile = new ArchiveFile(pathArgs.StringToBeUsed, contentArgs.StringToBeUsed); + }; + } +} diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/CleanupScope.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/CleanupScope.cs new file mode 100644 index 000000000..e44afc523 --- /dev/null +++ b/src/StructuredLogger/BinaryLogger/Postprocessing/CleanupScope.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Build.Logging; + +internal readonly struct CleanupScope : IDisposable +{ + private readonly Action _disposeAction; + + public CleanupScope(Action disposeAction) => _disposeAction = disposeAction; + + public void Dispose() => _disposeAction(); +} diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/ConcatenatedReadStream.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/ConcatenatedReadStream.cs new file mode 100644 index 000000000..1fecf03f3 --- /dev/null +++ b/src/StructuredLogger/BinaryLogger/Postprocessing/ConcatenatedReadStream.cs @@ -0,0 +1,100 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; + +namespace StructuredLogger.BinaryLogger.Postprocessing +{ + internal class ConcatenatedReadStream : Stream + { + private readonly Queue _streams; + private long _position; + + public ConcatenatedReadStream(IEnumerable streams) + => _streams = EnsureStreamsAreReadable(streams); + + public ConcatenatedReadStream(params Stream[] streams) + => _streams = EnsureStreamsAreReadable(streams); + + private static Queue EnsureStreamsAreReadable(IEnumerable streams) + { + var result = (streams is ICollection collection) ? new Queue(collection.Count) : new Queue(); + + foreach (Stream stream in streams) + { + if (!stream.CanRead) + { + throw new ArgumentException("All streams must be readable", nameof(streams)); + } + + if (stream is ConcatenatedReadStream concatenatedStream) + { + foreach (Stream subStream in concatenatedStream._streams) + { + result.Enqueue(subStream); + } + } + else + { + result.Enqueue(stream); + } + } + + return result; + } + + public override void Flush() + { + throw new NotSupportedException("ConcatenatedReadStream is forward-only read-only"); + } + + public override int Read(byte[] buffer, int offset, int count) + { + int totalBytesRead = 0; + + while (count > 0 && _streams.Count > 0) + { + int bytesRead = _streams.Peek().Read(buffer, offset, count); + if (bytesRead == 0) + { + _streams.Dequeue().Dispose(); + continue; + } + + totalBytesRead += bytesRead; + offset += bytesRead; + count -= bytesRead; + } + + _position += totalBytesRead; + return totalBytesRead; + } + + public override long Seek(long offset, SeekOrigin origin) + { + throw new NotSupportedException("ConcatenatedReadStream is forward-only read-only"); + } + + public override void SetLength(long value) + { + throw new NotSupportedException("ConcatenatedReadStream is forward-only read-only"); + } + + public override void Write(byte[] buffer, int offset, int count) + { + throw new NotSupportedException("ConcatenatedReadStream is forward-only read-only"); + } + + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => _streams.Sum(s => s.Length); + + public override long Position + { + get => _position; + set => throw new NotSupportedException("ConcatenatedReadStream is forward-only read-only"); + } + } +} diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs new file mode 100644 index 000000000..08b927bd6 --- /dev/null +++ b/src/StructuredLogger/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs @@ -0,0 +1,21 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using Microsoft.Build.Logging.StructuredLogger; + +namespace Microsoft.Build.Logging +{ + internal sealed class EmbeddedContentEventArgs : EventArgs + { + public EmbeddedContentEventArgs(BinaryLogRecordKind contentKind, Stream contentStream) + { + ContentKind = contentKind; + ContentStream = contentStream; + } + + public BinaryLogRecordKind ContentKind { get; } + public Stream ContentStream { get; } + } +} diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs new file mode 100644 index 000000000..1ad97a6ed --- /dev/null +++ b/src/StructuredLogger/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.Build.Logging +{ + /// + /// An interface for notifications from BuildEventArgsReader + /// + public interface IBuildEventArgsReaderNotifications : + IBuildEventStringsReader, + IBuildFileReader + { } +} diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/IBuildEventStringsReader.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/IBuildEventStringsReader.cs new file mode 100644 index 000000000..83d60f4fe --- /dev/null +++ b/src/StructuredLogger/BinaryLogger/Postprocessing/IBuildEventStringsReader.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Build.Logging +{ + /// + /// An interface for notifications about reading strings from the binary log. + /// + public interface IBuildEventStringsReader + { + /// + /// An event that allows the subscriber to be notified when a string is read from the binary log. + /// Subscriber may adjust the string by setting property. + /// The passed event arg can be reused and should not be stored. + /// + public event Action? StringReadDone; + } +} diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/IBuildFileReader.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/IBuildFileReader.cs new file mode 100644 index 000000000..8015083e5 --- /dev/null +++ b/src/StructuredLogger/BinaryLogger/Postprocessing/IBuildFileReader.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Build.Logging; + +public interface IBuildFileReader +{ + /// + /// An event that allows the caller to be notified when an embedded file is encountered in the binary log. + /// Subscribing to this event obligates the subscriber to read the file content and set the resulting content + /// via or . + /// When subscriber is OK with greedy reading entire content of the file, it can simplify subscribing to this event, + /// by using handler with same signature as handler for and wrapping it via + /// extension. + /// + /// + /// + /// private void OnStringReadDone(StringReadEventArgs e) + /// { + /// e.StringToBeUsed = e.StringToBeUsed.Replace("foo", "bar"); + /// } + /// + /// private void SubscribeToEvents() + /// { + /// reader.StringReadDone += OnStringReadDone; + /// reader.ArchiveFileEncountered += ((Action<StringReadEventArgs>)OnStringReadDone).ToArchiveFileHandler(); + /// } + /// + /// + /// + public event Action? ArchiveFileEncountered; +} diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/IEmbeddedContentSource.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/IEmbeddedContentSource.cs new file mode 100644 index 000000000..e867c5577 --- /dev/null +++ b/src/StructuredLogger/BinaryLogger/Postprocessing/IEmbeddedContentSource.cs @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Build.Logging +{ + internal interface IEmbeddedContentSource + { + /// + /// Raised when the log reader encounters a project import archive (embedded content) in the stream. + /// The subscriber must read the exactly given length of binary data from the stream - otherwise exception is raised. + /// If no subscriber is attached, the data is skipped. + /// + event Action EmbeddedContentRead; + } +} diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/StreamChunkOverreadException.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/StreamChunkOverreadException.cs new file mode 100644 index 000000000..4d4fe21b5 --- /dev/null +++ b/src/StructuredLogger/BinaryLogger/Postprocessing/StreamChunkOverreadException.cs @@ -0,0 +1,22 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Build.Logging +{ + public class StreamChunkOverReadException : Exception + { + public StreamChunkOverReadException() + { + } + + public StreamChunkOverReadException(string message) : base(message) + { + } + + public StreamChunkOverReadException(string message, Exception inner) : base(message, inner) + { + } + } +} diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/StreamExtensions.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/StreamExtensions.cs new file mode 100644 index 000000000..180f924bf --- /dev/null +++ b/src/StructuredLogger/BinaryLogger/Postprocessing/StreamExtensions.cs @@ -0,0 +1,118 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Buffers; +using System.Diagnostics; +using System.IO; +using Microsoft.Build.Shared; +using StructuredLogger.BinaryLogger.Postprocessing; + +namespace Microsoft.Build.Logging +{ + internal static class StreamExtensions + { + private static bool CheckIsSkipNeeded(long bytesCount) + { + if(bytesCount is < 0 or > int.MaxValue) + { + throw new ArgumentOutOfRangeException(nameof(bytesCount), $"Attempt to skip {bytesCount} bytes, only non-negative offset up to int.MaxValue is allowed."); + } + + return bytesCount > 0; + } + + public static long SkipBytes(this Stream stream) + => SkipBytes(stream, stream.Length, true); + + public static long SkipBytes(this Stream stream, long bytesCount) + => SkipBytes(stream, bytesCount, true); + + public static long SkipBytes(this Stream stream, long bytesCount, bool throwOnEndOfStream) + { + if (!CheckIsSkipNeeded(bytesCount)) + { + return 0; + } + + byte[] buffer = ArrayPool.Shared.Rent(4096); + using var _ = new CleanupScope(() => ArrayPool.Shared.Return(buffer)); + return SkipBytes(stream, bytesCount, throwOnEndOfStream, buffer); + } + + public static long SkipBytes(this Stream stream, long bytesCount, bool throwOnEndOfStream, byte[] buffer) + { + if (!CheckIsSkipNeeded(bytesCount)) + { + return 0; + } + + long totalRead = 0; + while (totalRead < bytesCount) + { + int read = stream.Read(buffer, 0, (int)Math.Min(bytesCount - totalRead, buffer.Length)); + if (read == 0) + { + if (throwOnEndOfStream) + { + throw new InvalidDataException("Unexpected end of stream."); + } + + return totalRead; + } + + totalRead += read; + } + + return totalRead; + } + + public static byte[] ReadToEnd(this Stream stream) + { + if (stream.TryGetLength(out long length)) + { + BinaryReader reader = new(stream); + return reader.ReadBytes((int)length); + } + + using var ms = new MemoryStream(); + stream.CopyTo(ms); + return ms.ToArray(); + } + + public static bool TryGetLength(this Stream stream, out long length) + { + try + { + length = stream.Length; + return true; + } + catch (NotSupportedException) + { + length = 0; + return false; + } + } + + public static Stream ToReadableSeekableStream(this Stream stream) + { + return TransparentReadStream.EnsureSeekableStream(stream); + } + + /// + /// Creates bounded read-only, forward-only view over an underlying stream. + /// + /// + /// + /// + public static Stream Slice(this Stream stream, long length) + { + return new SubStream(stream, length); + } + + public static Stream Concat(this Stream stream, Stream other) + { + return new ConcatenatedReadStream(stream, other); + } + } +} diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/StringReadEventArgs.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/StringReadEventArgs.cs new file mode 100644 index 000000000..977006015 --- /dev/null +++ b/src/StructuredLogger/BinaryLogger/Postprocessing/StringReadEventArgs.cs @@ -0,0 +1,35 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Build.Logging +{ + /// + /// An event args for callback. + /// + public sealed class StringReadEventArgs : EventArgs + { + /// + /// The original string that was read from the binary log. + /// + public string OriginalString { get; private set; } + + /// + /// The adjusted string (or the original string of none subscriber replaced it) that will be used by the reader. + /// + public string StringToBeUsed { get; set; } + + public StringReadEventArgs(string str) + { + OriginalString = str; + StringToBeUsed = str; + } + + internal void Reuse(string newValue) + { + OriginalString = newValue; + StringToBeUsed = newValue; + } + } +} diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/SubStream.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/SubStream.cs new file mode 100644 index 000000000..eb2713d24 --- /dev/null +++ b/src/StructuredLogger/BinaryLogger/Postprocessing/SubStream.cs @@ -0,0 +1,55 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using Microsoft.Build.Shared; + +namespace Microsoft.Build.Logging +{ + /// + /// Bounded read-only, forward-only view over an underlying stream. + /// + internal sealed class SubStream : Stream + { + // Do not Dispose/Close on Dispose/Close !! + private readonly Stream _stream; + private readonly long _length; + private long _position; + + public SubStream(Stream stream, long length) + { + _stream = stream; + _length = length; + + if (!stream.CanRead) + { + throw new NotSupportedException("Stream must be readable."); + } + } + + public bool IsAtEnd => _position >= _length; + + public override bool CanRead => true; + + public override bool CanSeek => false; + + public override bool CanWrite => false; + + public override long Length => _length; + + public override long Position { get => _position; set => throw new NotImplementedException(); } + + public override void Flush() { } + public override int Read(byte[] buffer, int offset, int count) + { + count = Math.Min((int)Math.Max(Length - _position, 0), count); + int read = _stream.Read(buffer, offset, count); + _position += read; + return read; + } + public override long Seek(long offset, SeekOrigin origin) => throw new NotImplementedException(); + public override void SetLength(long value) => throw new NotImplementedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotImplementedException(); + } +} diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/TransparentReadStream.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/TransparentReadStream.cs new file mode 100644 index 000000000..14f7eb2d1 --- /dev/null +++ b/src/StructuredLogger/BinaryLogger/Postprocessing/TransparentReadStream.cs @@ -0,0 +1,115 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using Microsoft.Build.Shared; + +namespace Microsoft.Build.Logging +{ + /// + /// A wrapper stream that allows position tracking and forward seeking. + /// + internal sealed class TransparentReadStream : Stream + { + private readonly Stream _stream; + private long _position; + private long _maxAllowedPosition = long.MaxValue; + + public static Stream EnsureSeekableStream(Stream stream) + { + if (stream.CanSeek) + { + return stream; + } + + if (!stream.CanRead) + { + throw new InvalidOperationException("Stream must be readable."); + } + + return new TransparentReadStream(stream); + } + + public static TransparentReadStream EnsureTransparentReadStream(Stream stream) + { + if (stream is TransparentReadStream transparentReadStream) + { + return transparentReadStream; + } + + if (!stream.CanRead) + { + throw new InvalidOperationException("Stream must be readable."); + } + + return new TransparentReadStream(stream); + } + + private TransparentReadStream(Stream stream) + { + _stream = stream; + } + + public int? BytesCountAllowedToRead + { + set { _maxAllowedPosition = value.HasValue ? _position + value.Value : long.MaxValue; } + } + + // if we haven't constrained the allowed read size - do not report it being unfinished either. + public int BytesCountAllowedToReadRemaining => + _maxAllowedPosition == long.MaxValue ? 0 : (int)(_maxAllowedPosition - _position); + + public override bool CanRead => _stream.CanRead; + public override bool CanSeek => true; + public override bool CanWrite => false; + public override long Length => _stream.Length; + public override long Position + { + get => _position; + set => this.SkipBytes(value - _position, true); + } + + public override void Flush() + { + _stream.Flush(); + } + + public override int Read(byte[] buffer, int offset, int count) + { + if (_position + count > _maxAllowedPosition) + { + throw new StreamChunkOverReadException( + $"Attempt to read {count} bytes, when only {_maxAllowedPosition - _position} are allowed to be read."); + } + + int cnt = _stream.Read(buffer, offset, count); + _position += cnt; + return cnt; + } + + public override long Seek(long offset, SeekOrigin origin) + { + if (origin != SeekOrigin.Current) + { + throw new NotSupportedException("Only seeking from SeekOrigin.Current is supported."); + } + + this.SkipBytes(offset, true); + + return _position; + } + + public override void SetLength(long value) + { + throw new NotSupportedException("Expanding stream is not supported."); + } + + public override void Write(byte[] buffer, int offset, int count) + { + throw new NotSupportedException("Writing is not supported."); + } + + public override void Close() => _stream.Close(); + } +} diff --git a/src/StructuredLogger/BinaryLogger/ProjectImportsCollector.cs b/src/StructuredLogger/BinaryLogger/ProjectImportsCollector.cs index ebc775a96..a8af61e2b 100644 --- a/src/StructuredLogger/BinaryLogger/ProjectImportsCollector.cs +++ b/src/StructuredLogger/BinaryLogger/ProjectImportsCollector.cs @@ -5,6 +5,7 @@ using System.IO.Compression; using System.Text; using System.Threading.Tasks; +using Microsoft.Build.Shared; namespace Microsoft.Build.Logging { @@ -18,8 +19,9 @@ internal class ProjectImportsCollector { private Stream _fileStream; private ZipArchive _zipArchive; - - public string ArchiveFilePath { get; } + private readonly string _archiveFilePath; + private readonly bool _runOnBackground; + private const string DefaultSourcesArchiveExtension = ".ProjectImports.zip"; /// /// Avoid visiting each file more than once. @@ -29,17 +31,33 @@ internal class ProjectImportsCollector // this will form a chain of file write tasks, running sequentially on a background thread private Task _currentTask = Task.CompletedTask; - public ProjectImportsCollector(string logFilePath, bool createFile, string sourcesArchiveExtension = ".ProjectImports.zip") + internal static void FlushBlobToFile( + string logFilePath, + Stream contentStream) + { + string archiveFilePath = GetArchiveFilePath(logFilePath, DefaultSourcesArchiveExtension); + + using var fileStream = new FileStream(archiveFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.Delete); + contentStream.CopyTo(fileStream); + } + + // Archive file will be stored alongside the binlog + private static string GetArchiveFilePath(string logFilePath, string sourcesArchiveExtension) + => Path.ChangeExtension(logFilePath, sourcesArchiveExtension); + + public ProjectImportsCollector( + string logFilePath, + bool createFile, + string sourcesArchiveExtension = DefaultSourcesArchiveExtension, + bool runOnBackground = true) { if (createFile) { // Archive file will be stored alongside the binlog - ArchiveFilePath = Path.ChangeExtension(logFilePath, sourcesArchiveExtension); + _archiveFilePath = GetArchiveFilePath(logFilePath, sourcesArchiveExtension); } else { - - string cacheDirectory = GetCacheDirectory(); if (!Directory.Exists(cacheDirectory)) { @@ -47,16 +65,16 @@ public ProjectImportsCollector(string logFilePath, bool createFile, string sourc } // Archive file will be temporarily stored in MSBuild cache folder and deleted when no longer needed - ArchiveFilePath = Path.Combine( + _archiveFilePath = Path.Combine( cacheDirectory, - Path.ChangeExtension( + GetArchiveFilePath( Path.GetFileName(logFilePath), sourcesArchiveExtension)); } try { - _fileStream = new FileStream(ArchiveFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.Delete); + _fileStream = new FileStream(_archiveFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.Delete); _zipArchive = new ZipArchive(_fileStream, ZipArchiveMode.Create); } catch @@ -66,6 +84,8 @@ public ProjectImportsCollector(string logFilePath, bool createFile, string sourc _fileStream = null; _zipArchive = null; } + + _runOnBackground = runOnBackground; } private static string GetCacheDirectory() @@ -76,47 +96,61 @@ private static string GetCacheDirectory() return dir; } - public void AddFile(string filePath) + public void AddFile(string? filePath) + { + AddFileHelper(filePath, AddFileCore); + } + + public void AddFileFromMemory( + string? filePath, + string data, + DateTimeOffset? entryCreationStamp = null, + bool makePathAbsolute = true) + { + AddFileHelper(filePath, path => + AddFileFromMemoryCore(path, data, makePathAbsolute, entryCreationStamp)); + } + + public void AddFileFromMemory( + string? filePath, + Stream data, + DateTimeOffset? entryCreationStamp = null, + bool makePathAbsolute = true) + { + AddFileHelper(filePath, path => AddFileFromMemoryCore(path, data, makePathAbsolute, entryCreationStamp)); + } + + private void AddFileHelper( + string? filePath, + Action addFileWorker) { if (filePath != null && _fileStream != null) { lock (_fileStream) { - // enqueue the task to add a file and return quickly - // to avoid holding up the current thread - _currentTask = _currentTask.ContinueWith(t => + if (_runOnBackground) + { + // enqueue the task to add a file and return quickly + // to avoid holding up the current thread + _currentTask = _currentTask.ContinueWith( + t => { TryAddFile(); }, + TaskScheduler.Default); + } + else { - try - { - AddFileCore(filePath); - } - catch - { - } - }, TaskScheduler.Default); + TryAddFile(); + } } } - } - public void AddFileFromMemory(string filePath, string data) - { - if (filePath != null && data != null && _fileStream != null) + void TryAddFile() { - lock (_fileStream) + try { - // enqueue the task to add a file and return quickly - // to avoid holding up the current thread - _currentTask = _currentTask.ContinueWith(t => - { - try - { - AddFileFromMemoryCore(filePath, data); - } - catch - { - } - }, TaskScheduler.Default); + addFileWorker(filePath); } + catch + { } } } @@ -127,61 +161,82 @@ public void AddFileFromMemory(string filePath, string data) private void AddFileCore(string filePath) { // quick check to avoid repeated disk access for Exists etc. - if (_processedFiles.Contains(filePath)) + if (!ShouldAddFile(ref filePath, true, true)) { return; } - if (!File.Exists(filePath)) + using FileStream content = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete); + AddFileData(filePath, content, null); + } + + /// + /// This method doesn't need locking/synchronization because it's only called + /// from a task that is chained linearly + /// + private void AddFileFromMemoryCore(string filePath, string data, bool makePathAbsolute, DateTimeOffset? entryCreationStamp) + { + // quick check to avoid repeated disk access for Exists etc. + if (!ShouldAddFile(ref filePath, false, makePathAbsolute)) { - _processedFiles.Add(filePath); return; } - filePath = Path.GetFullPath(filePath); + using var content = new MemoryStream(Encoding.UTF8.GetBytes(data)); + AddFileData(filePath, content, entryCreationStamp); + } - // if the file is already included, don't include it again - if (!_processedFiles.Add(filePath)) + private void AddFileFromMemoryCore(string filePath, Stream data, bool makePathAbsolute, DateTimeOffset? entryCreationStamp) + { + // quick check to avoid repeated disk access for Exists etc. + if (!ShouldAddFile(ref filePath, false, makePathAbsolute)) { return; } - using FileStream content = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete); - using Stream entryStream = OpenArchiveEntry(filePath); - content.CopyTo(entryStream); + AddFileData(filePath, data, entryCreationStamp); } - /// - /// This method doesn't need locking/synchronization because it's only called - /// from a task that is chained linearly - /// - private void AddFileFromMemoryCore(string filePath, string data) + private void AddFileData(string filePath, Stream data, DateTimeOffset? entryCreationStamp) + { + using Stream entryStream = OpenArchiveEntry(filePath, entryCreationStamp); + data.CopyTo(entryStream); + } + + private bool ShouldAddFile(ref string filePath, bool checkFileExistence, bool makeAbsolute) { // quick check to avoid repeated disk access for Exists etc. if (_processedFiles.Contains(filePath)) { - return; + return false; } - filePath = Path.GetFullPath(filePath); - - // if the file is already included, don't include it again - if (!_processedFiles.Add(filePath)) + if (checkFileExistence && !File.Exists(filePath)) { - return; + _processedFiles.Add(filePath); + return false; } - using (Stream entryStream = OpenArchiveEntry(filePath)) - using (var content = new MemoryStream(Encoding.UTF8.GetBytes(data))) + // Only make the path absolute if it's request. In the replay scenario, the file entries + // are read from zip archive - where ':' is stripped and path can then seem relative. + if (makeAbsolute) { - content.CopyTo(entryStream); + filePath = Path.GetFullPath(filePath); } + + // if the file is already included, don't include it again + return _processedFiles.Add(filePath); } - private Stream OpenArchiveEntry(string filePath) + private Stream OpenArchiveEntry(string filePath, DateTimeOffset? entryCreationStamp) { string archivePath = CalculateArchivePath(filePath); - var archiveEntry = _zipArchive.CreateEntry(archivePath); + var archiveEntry = _zipArchive!.CreateEntry(archivePath); + if (entryCreationStamp.HasValue) + { + archiveEntry.LastWriteTime = entryCreationStamp.Value; + } + return archiveEntry.Open(); } @@ -196,6 +251,27 @@ private static string CalculateArchivePath(string filePath) return archivePath; } + public void ProcessResult(Action consumeStream, Action onError) + { + Close(); + + // It is possible that the archive couldn't be created for some reason. + // Only embed it if it actually exists. + if (File.Exists(_archiveFilePath)) + { + using FileStream fileStream = File.OpenRead(_archiveFilePath); + + if (fileStream.Length > int.MaxValue) + { + onError("Imported files archive exceeded 2GB limit and it's not embedded."); + } + else + { + consumeStream(fileStream); + } + } + } + public void Close() { // wait for all pending file writes to complete @@ -213,5 +289,11 @@ public void Close() _fileStream = null; } } + + public void DeleteArchive() + { + Close(); + File.Delete(_archiveFilePath); + } } } diff --git a/src/StructuredLogger/ObjectModel/ArchiveFile.cs b/src/StructuredLogger/ObjectModel/ArchiveFile.cs index 10e4d95e2..e7f76678f 100644 --- a/src/StructuredLogger/ObjectModel/ArchiveFile.cs +++ b/src/StructuredLogger/ObjectModel/ArchiveFile.cs @@ -1,4 +1,4 @@ -using System.IO; +using System.IO; using System.IO.Compression; namespace Microsoft.Build.Logging.StructuredLogger @@ -15,8 +15,11 @@ public ArchiveFile(string fullPath, string text) public string Text { get; } public static ArchiveFile From(ZipArchiveEntry entry) + => From(entry, adjustPath: true); + + public static ArchiveFile From(ZipArchiveEntry entry, bool adjustPath) { - var filePath = CalculateArchivePath(entry.FullName); + var filePath = adjustPath ? CalculateArchivePath(entry.FullName) : entry.FullName; var text = GetText(entry); var file = new ArchiveFile(filePath, text); return file; @@ -51,4 +54,4 @@ public static string CalculateArchivePath(string filePath) return archivePath; } } -} \ No newline at end of file +} diff --git a/src/StructuredLogger/StructuredLogger.cs b/src/StructuredLogger/StructuredLogger.cs index 49862cd96..4d7fc1cad 100644 --- a/src/StructuredLogger/StructuredLogger.cs +++ b/src/StructuredLogger/StructuredLogger.cs @@ -2,6 +2,7 @@ using System.IO; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; +using static Microsoft.Build.Logging.StructuredLogger.BinaryLogger; namespace Microsoft.Build.Logging.StructuredLogger { @@ -104,13 +105,13 @@ public override void Shutdown() if (projectImportsCollector != null) { projectImportsCollector.Close(); - var archiveFilePath = projectImportsCollector.ArchiveFilePath; - if (File.Exists(archiveFilePath)) - { - var bytes = File.ReadAllBytes(archiveFilePath); - construction.Build.SourceFilesArchive = bytes; - } + + projectImportsCollector.ProcessResult( + streamToEmbed => construction.Build.SourceFilesArchive = streamToEmbed.ReadToEnd(), + _ => {}); + + projectImportsCollector.DeleteArchive(); projectImportsCollector = null; } From 8a3cd50f46800e6ddcc8250e4be4fc5e6e279d9b Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Fri, 27 Oct 2023 15:23:17 +0200 Subject: [PATCH 05/13] Mount string reading event --- .../BinaryLogger/BinaryLogReplayEventSource.cs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/StructuredLogger/BinaryLogger/BinaryLogReplayEventSource.cs b/src/StructuredLogger/BinaryLogger/BinaryLogReplayEventSource.cs index 3c340a009..3c3e58058 100644 --- a/src/StructuredLogger/BinaryLogger/BinaryLogReplayEventSource.cs +++ b/src/StructuredLogger/BinaryLogger/BinaryLogReplayEventSource.cs @@ -15,7 +15,7 @@ namespace Microsoft.Build.Logging.StructuredLogger /// internal interface IBinaryLogReplaySource : IEventSource, - // IBuildEventStringsReader, + IBuildEventStringsReader, IEmbeddedContentSource { } @@ -120,6 +120,7 @@ public void Replay(BinaryReader binaryReader, CancellationToken cancellationToke using BuildEventArgsReader reader = OpenBuildEventsReader(binaryReader, false); reader.EmbeddedContentRead += _embeddedContentRead; + reader.StringReadDone += _stringReadDone; while (!cancellationToken.IsCancellationRequested && reader.Read() is { } instance) { @@ -136,5 +137,13 @@ event Action? IEmbeddedContentSource.EmbeddedContentRe add => _embeddedContentRead += value; remove => _embeddedContentRead -= value; } + + private Action? _stringReadDone; + /// + event Action? IBuildEventStringsReader.StringReadDone + { + add => _stringReadDone += value; + remove => _stringReadDone -= value; + } } } From 7f67e8e5e800d40679a40134c705ea7166949e02 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Fri, 27 Oct 2023 16:29:54 +0200 Subject: [PATCH 06/13] Add Redacting PoC --- src/BinlogTool/Program.cs | 17 +++- .../Controls/RedactInputControl.xaml | 28 ++++++ .../Controls/RedactInputControl.xaml.cs | 43 +++++++++ src/StructuredLogViewer/MainWindow.xaml | 5 +- src/StructuredLogViewer/MainWindow.xaml.cs | 18 ++++ .../BinaryLogReplayEventSource.cs | 9 ++ .../Postprocessing/BinlogRedactor.cs | 91 +++++++++++++++++++ 7 files changed, 208 insertions(+), 3 deletions(-) create mode 100644 src/StructuredLogViewer/Controls/RedactInputControl.xaml create mode 100644 src/StructuredLogViewer/Controls/RedactInputControl.xaml.cs create mode 100644 src/StructuredLogger/BinaryLogger/Postprocessing/BinlogRedactor.cs diff --git a/src/BinlogTool/Program.cs b/src/BinlogTool/Program.cs index cbef55236..cf02a4437 100644 --- a/src/BinlogTool/Program.cs +++ b/src/BinlogTool/Program.cs @@ -2,6 +2,7 @@ using System.IO; using System.Linq; using Microsoft.Build.Logging.StructuredLogger; +using StructuredLogger.BinaryLogger.Postprocessing; namespace BinlogTool { @@ -16,7 +17,8 @@ binlogtool listtools input.binlog binlogtool savefiles input.binlog output_path binlogtool reconstruct input.binlog output_path binlogtool savestrings input.binlog output.txt - binlogtool search *.binlog search string"); + binlogtool search *.binlog search string + binlogtool redact input.binlog list of passwords to redact"); return; } @@ -71,6 +73,19 @@ binlogtool savestrings input.binlog output.txt return; } + if (firstArg == "redact") + { + if (args.Length < 3) + { + Console.Error.WriteLine("binlogtool redact input.binlog list of passwords to redact"); + return; + } + + var binlog = args[1]; + BinlogRedactor.RedactSecrets(binlog, args.Skip(2).ToArray()); + return; + } + Console.Error.WriteLine("Invalid arguments"); } diff --git a/src/StructuredLogViewer/Controls/RedactInputControl.xaml b/src/StructuredLogViewer/Controls/RedactInputControl.xaml new file mode 100644 index 000000000..8f2bd9044 --- /dev/null +++ b/src/StructuredLogViewer/Controls/RedactInputControl.xaml @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/src/StructuredLogViewer/Controls/RedactInputControl.xaml.cs b/src/StructuredLogViewer/Controls/RedactInputControl.xaml.cs new file mode 100644 index 000000000..54a0761a9 --- /dev/null +++ b/src/StructuredLogViewer/Controls/RedactInputControl.xaml.cs @@ -0,0 +1,43 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Windows; +using System.Windows.Controls; +using System.Windows.Data; +using System.Windows.Documents; +using System.Windows.Input; +using System.Windows.Media; +using System.Windows.Media.Imaging; +using System.Windows.Shapes; + +namespace StructuredLogViewer.Controls +{ + /// + /// Interaction logic for RedactInputControl.xaml + /// + public partial class RedactInputControl : Window + { + public RedactInputControl() + { + InitializeComponent(); + } + + private void btnDialogOk_Click(object sender, RoutedEventArgs e) + { + this.DialogResult = true; + } + + private void Window_ContentRendered(object sender, EventArgs e) + { + txtAnswer.SelectAll(); + txtAnswer.Focus(); + } + + public string Answer + { + get { return txtAnswer.Text; } + } + } +} diff --git a/src/StructuredLogViewer/MainWindow.xaml b/src/StructuredLogViewer/MainWindow.xaml index 41715ffde..e7ef22b09 100644 --- a/src/StructuredLogViewer/MainWindow.xaml +++ b/src/StructuredLogViewer/MainWindow.xaml @@ -1,4 +1,4 @@ - - + + diff --git a/src/StructuredLogViewer/MainWindow.xaml.cs b/src/StructuredLogViewer/MainWindow.xaml.cs index 83ce8eb09..6154bd857 100644 --- a/src/StructuredLogViewer/MainWindow.xaml.cs +++ b/src/StructuredLogViewer/MainWindow.xaml.cs @@ -13,6 +13,7 @@ using Microsoft.Build.Logging.StructuredLogger; using Microsoft.Win32; using Squirrel; +using StructuredLogger.BinaryLogger.Postprocessing; using StructuredLogViewer.Controls; namespace StructuredLogViewer @@ -374,11 +375,13 @@ private void SetContent(object content) { ReloadMenu.Visibility = logFilePath != null ? Visibility.Visible : Visibility.Collapsed; SaveAsMenu.Visibility = Visibility.Visible; + RedactSecretsMenu.Visibility = Visibility.Visible; } else { ReloadMenu.Visibility = Visibility.Collapsed; SaveAsMenu.Visibility = Visibility.Collapsed; + RedactSecretsMenu.Visibility = Visibility.Collapsed; } // If we had text inside search log control bring it back @@ -659,6 +662,16 @@ private void Reload() OpenLogFile(logFilePath); } + private void RedactSecrets() + { + RedactInputControl redactInputControl = new RedactInputControl(); + if (redactInputControl.ShowDialog() == true) + { + BinlogRedactor.RedactSecrets(logFilePath, redactInputControl.Answer.Split()); + OpenLogFile(logFilePath); + } + } + private void SaveAs() { if (currentBuild != null) @@ -814,6 +827,11 @@ private void SaveAs_Click(object sender, RoutedEventArgs e) SaveAs(); } + private void RedactSecrets_Click(object sender, RoutedEventArgs e) + { + RedactSecrets(); + } + private void HelpLink_Click(object sender, RoutedEventArgs e) { Process.Start(new ProcessStartInfo("https://github.com/KirillOsenkov/MSBuildStructuredLog") { UseShellExecute = true }); diff --git a/src/StructuredLogger/BinaryLogger/BinaryLogReplayEventSource.cs b/src/StructuredLogger/BinaryLogger/BinaryLogReplayEventSource.cs index 3c3e58058..3693a5576 100644 --- a/src/StructuredLogger/BinaryLogger/BinaryLogReplayEventSource.cs +++ b/src/StructuredLogger/BinaryLogger/BinaryLogReplayEventSource.cs @@ -16,6 +16,7 @@ namespace Microsoft.Build.Logging.StructuredLogger internal interface IBinaryLogReplaySource : IEventSource, IBuildEventStringsReader, + IBuildFileReader, IEmbeddedContentSource { } @@ -138,6 +139,14 @@ event Action? IEmbeddedContentSource.EmbeddedContentRe remove => _embeddedContentRead -= value; } + private Action? _archiveFileEncountered; + /// + event Action? IBuildFileReader.ArchiveFileEncountered + { + add => _archiveFileEncountered += value; + remove => _archiveFileEncountered -= value; + } + private Action? _stringReadDone; /// event Action? IBuildEventStringsReader.StringReadDone diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/BinlogRedactor.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/BinlogRedactor.cs new file mode 100644 index 000000000..1ff12d70a --- /dev/null +++ b/src/StructuredLogger/BinaryLogger/Postprocessing/BinlogRedactor.cs @@ -0,0 +1,91 @@ +using System; +using System.Linq; +using System.Collections.Generic; +using System.Text; +using Microsoft.Build.Logging; +using Microsoft.Build.Logging.StructuredLogger; +using System.IO; + +namespace StructuredLogger.BinaryLogger.Postprocessing +{ + public interface ISensitiveDataProcessor + { + /// + /// Processes the given text and if needed, replaces sensitive data with a placeholder. + /// + string ReplaceSensitiveData(string text); + } + + internal sealed class SimpleSensitiveDataProcessor : ISensitiveDataProcessor + { + public const string DefaultReplacementPattern = "*******"; + private readonly (string pwd, string replacement)[] _passwordsToRedact; + + public SimpleSensitiveDataProcessor(string[] passwordsToRedact, bool identifyReplacements) + { + _passwordsToRedact = passwordsToRedact.Select((pwd, cnt) => + (pwd, identifyReplacements ? $"REDACTED__PWD{(cnt + 1):00}" : DefaultReplacementPattern)) + .ToArray(); + } + + public string ReplaceSensitiveData(string text) + { + foreach ((string pwd, string replacement) pwd in _passwordsToRedact) + { + text = text.Replace(pwd.pwd, pwd.replacement); + } + + return text; + } + } + + public class BinlogRedactor + { + private readonly ISensitiveDataProcessor _sensitiveDataProcessor; + + public static void RedactSecrets(string binlogPath, string[] secrets) + { + var sensitivityProcessor = new SimpleSensitiveDataProcessor(secrets, true); + string outputFile = Path.GetFileName(Path.GetTempFileName()) + ".binlog"; + new BinlogRedactor(sensitivityProcessor).ProcessBinlog(binlogPath, outputFile, skipEmbeddedFiles: false); + File.Delete(binlogPath); + File.Move(outputFile, binlogPath); + } + + public BinlogRedactor(ISensitiveDataProcessor sensitiveDataProcessor) + { + _sensitiveDataProcessor = sensitiveDataProcessor; + } + + public void ProcessBinlog( + string inputFileName, + string outputFileName, + bool skipEmbeddedFiles) + { + BinaryLogReplayEventSource originalEventsSource = new BinaryLogReplayEventSource(); + + Microsoft.Build.Logging.StructuredLogger.BinaryLogger outputBinlog = new Microsoft.Build.Logging.StructuredLogger.BinaryLogger() + { + Parameters = $"LogFile={outputFileName};OmitInitialInfo", + }; + + ((IBuildEventStringsReader) originalEventsSource).StringReadDone += HandleStringRead; + if (!skipEmbeddedFiles) + { + ((IBuildFileReader)originalEventsSource).ArchiveFileEncountered += + ((Action)HandleStringRead).ToArchiveFileHandler(); + } + + outputBinlog.Initialize(originalEventsSource); + originalEventsSource.Replay(inputFileName); + outputBinlog.Shutdown(); + + // TODO: error handling + + void HandleStringRead(StringReadEventArgs args) + { + args.StringToBeUsed = _sensitiveDataProcessor.ReplaceSensitiveData(args.OriginalString); + } + } + } +} From c96f8255bf15a035910a2496fb49e98b73bb331a Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Wed, 1 Nov 2023 14:10:11 +0100 Subject: [PATCH 07/13] Apply code review suggestions --- src/BinlogTool/Program.cs | 4 +-- .../BinaryLoggerTests.cs | 23 ++++++++--------- .../StructuredLogger.Tests.csproj | 5 ++++ .../BinaryLogReplayEventSource.cs | 4 +-- .../BinaryLogger/BuildEventArgsReader.cs | 2 +- .../Postprocessing/BinlogRedactor.cs | 14 +++++------ .../BinaryLogger/ProjectImportsCollector.cs | 3 ++- src/StructuredLogger/ErrorReporting.cs | 15 ++--------- src/StructuredLogger/PathUtils.cs | 25 +++++++++++++++++++ src/StructuredLogger/StructuredLogger.csproj | 3 +++ 10 files changed, 60 insertions(+), 38 deletions(-) create mode 100644 src/StructuredLogger/PathUtils.cs diff --git a/src/BinlogTool/Program.cs b/src/BinlogTool/Program.cs index cf02a4437..3302e9b7f 100644 --- a/src/BinlogTool/Program.cs +++ b/src/BinlogTool/Program.cs @@ -18,7 +18,7 @@ binlogtool savefiles input.binlog output_path binlogtool reconstruct input.binlog output_path binlogtool savestrings input.binlog output.txt binlogtool search *.binlog search string - binlogtool redact input.binlog list of passwords to redact"); + binlogtool redact input.binlog list of secrets to redact"); return; } @@ -77,7 +77,7 @@ binlogtool search *.binlog search string { if (args.Length < 3) { - Console.Error.WriteLine("binlogtool redact input.binlog list of passwords to redact"); + Console.Error.WriteLine("binlogtool redact input.binlog list of secrets to redact"); return; } diff --git a/src/StructuredLogger.Tests/BinaryLoggerTests.cs b/src/StructuredLogger.Tests/BinaryLoggerTests.cs index 9f0de2114..a58690d17 100644 --- a/src/StructuredLogger.Tests/BinaryLoggerTests.cs +++ b/src/StructuredLogger.Tests/BinaryLoggerTests.cs @@ -57,16 +57,20 @@ public BinaryLoggerTests(ITestOutputHelper output) { this.output = output; } + private bool BuildProject(string projectFile, string binLog, bool useInMemoryProject) + { + File.Delete(binLog); + var binaryLogger = new BinaryLogger { Parameters = binLog }; + return useInMemoryProject + ? MSBuild.BuildProjectInMemory(projectFile, binaryLogger) + : MSBuild.BuildProjectFromFile(projectFile, binaryLogger); + } [Fact] public void TestBuildTreeStructureCount() { var binLog = GetTestFile("1.binlog"); - var binaryLogger = new BinaryLogger(); - binaryLogger.Parameters = binLog; - var buildSuccessful = MSBuild.BuildProjectFromFile(s_testProject, binaryLogger); - - Assert.True(buildSuccessful); + Assert.True(BuildProject(s_testProject, binLog, false)); var build = Serialization.Read(binLog); BuildAnalyzer.AnalyzeBuild(build); @@ -95,13 +99,7 @@ public void TestBuildTreeStructureCount() public void TestBinaryLoggerRoundtrip(bool useInMemoryProject) { var binLog = GetTestFile("1.binlog"); - var binaryLogger = new BinaryLogger(); - binaryLogger.Parameters = binLog; - var buildSuccessful = useInMemoryProject - ? MSBuild.BuildProjectInMemory(s_testProject, binaryLogger) - : MSBuild.BuildProjectFromFile(s_testProject, binaryLogger); - - Assert.True(buildSuccessful); + Assert.True(BuildProject(s_testProject, binLog, useInMemoryProject)); var build = Serialization.Read(binLog); var xml1 = GetTestFile("1.xml"); @@ -130,6 +128,7 @@ public void TestBinaryLoggerRoundtrip(bool useInMemoryProject) public void TestReaderWriterRoundtripEquality() { var binLog = GetTestFile("1.binlog"); + Assert.True(BuildProject(s_testProject, binLog, false)); var replayedBinlog = GetTestFile("1-replayed.binlog"); File.Delete(replayedBinlog); diff --git a/src/StructuredLogger.Tests/StructuredLogger.Tests.csproj b/src/StructuredLogger.Tests/StructuredLogger.Tests.csproj index b43f5b10d..11099a746 100644 --- a/src/StructuredLogger.Tests/StructuredLogger.Tests.csproj +++ b/src/StructuredLogger.Tests/StructuredLogger.Tests.csproj @@ -33,4 +33,9 @@ + + True + ..\StructuredLogger\key.snk + False + \ No newline at end of file diff --git a/src/StructuredLogger/BinaryLogger/BinaryLogReplayEventSource.cs b/src/StructuredLogger/BinaryLogger/BinaryLogReplayEventSource.cs index 3693a5576..85b0e5da9 100644 --- a/src/StructuredLogger/BinaryLogger/BinaryLogReplayEventSource.cs +++ b/src/StructuredLogger/BinaryLogger/BinaryLogReplayEventSource.cs @@ -70,7 +70,7 @@ public static BinaryReader OpenReader(string sourceFilePath) /// /// /// BinaryReader of the given binlog file. - public static BuildEventArgsReader OpenBuildEventsReader(string sourceFilePath) + internal static BuildEventArgsReader OpenBuildEventsReader(string sourceFilePath) => OpenBuildEventsReader(OpenReader(sourceFilePath), true); /// @@ -80,7 +80,7 @@ public static BuildEventArgsReader OpenBuildEventsReader(string sourceFilePath) /// /// Indicates whether the passed BinaryReader should be closed on disposing. /// BuildEventArgsReader over the given binlog file binary reader. - public static BuildEventArgsReader OpenBuildEventsReader( + internal static BuildEventArgsReader OpenBuildEventsReader( BinaryReader binaryReader, bool closeInput) { diff --git a/src/StructuredLogger/BinaryLogger/BuildEventArgsReader.cs b/src/StructuredLogger/BinaryLogger/BuildEventArgsReader.cs index ba71efcb2..dd07ece89 100644 --- a/src/StructuredLogger/BinaryLogger/BuildEventArgsReader.cs +++ b/src/StructuredLogger/BinaryLogger/BuildEventArgsReader.cs @@ -22,7 +22,7 @@ namespace Microsoft.Build.Logging.StructuredLogger /// /// Deserializes and returns BuildEventArgs-derived objects from a BinaryReader /// - public partial class BuildEventArgsReader : IBuildEventArgsReaderNotifications, IDisposable + internal partial class BuildEventArgsReader : IBuildEventArgsReaderNotifications, IDisposable { private readonly BinaryReader binaryReader; private readonly int fileFormatVersion; diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/BinlogRedactor.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/BinlogRedactor.cs index 1ff12d70a..e77e74931 100644 --- a/src/StructuredLogger/BinaryLogger/Postprocessing/BinlogRedactor.cs +++ b/src/StructuredLogger/BinaryLogger/Postprocessing/BinlogRedactor.cs @@ -19,20 +19,20 @@ public interface ISensitiveDataProcessor internal sealed class SimpleSensitiveDataProcessor : ISensitiveDataProcessor { public const string DefaultReplacementPattern = "*******"; - private readonly (string pwd, string replacement)[] _passwordsToRedact; + private readonly (string token, string replacement)[] _secretsToRedact; - public SimpleSensitiveDataProcessor(string[] passwordsToRedact, bool identifyReplacements) + public SimpleSensitiveDataProcessor(string[] secretsToRedact, bool identifyReplacements) { - _passwordsToRedact = passwordsToRedact.Select((pwd, cnt) => - (pwd, identifyReplacements ? $"REDACTED__PWD{(cnt + 1):00}" : DefaultReplacementPattern)) + _secretsToRedact = secretsToRedact.Select((secret, cnt) => + (token: secret, identifyReplacements ? $"REDACTED__TKN{(cnt + 1):00}" : DefaultReplacementPattern)) .ToArray(); } public string ReplaceSensitiveData(string text) { - foreach ((string pwd, string replacement) pwd in _passwordsToRedact) + foreach ((string token, string replacement) secret in _secretsToRedact) { - text = text.Replace(pwd.pwd, pwd.replacement); + text = text.Replace(secret.token, secret.replacement); } return text; @@ -46,7 +46,7 @@ public class BinlogRedactor public static void RedactSecrets(string binlogPath, string[] secrets) { var sensitivityProcessor = new SimpleSensitiveDataProcessor(secrets, true); - string outputFile = Path.GetFileName(Path.GetTempFileName()) + ".binlog"; + string outputFile = Path.Combine(PathUtils.TempPath, Path.GetFileName(Path.GetTempFileName()) + ".binlog"); new BinlogRedactor(sensitivityProcessor).ProcessBinlog(binlogPath, outputFile, skipEmbeddedFiles: false); File.Delete(binlogPath); File.Move(outputFile, binlogPath); diff --git a/src/StructuredLogger/BinaryLogger/ProjectImportsCollector.cs b/src/StructuredLogger/BinaryLogger/ProjectImportsCollector.cs index a8af61e2b..e85d6984f 100644 --- a/src/StructuredLogger/BinaryLogger/ProjectImportsCollector.cs +++ b/src/StructuredLogger/BinaryLogger/ProjectImportsCollector.cs @@ -6,6 +6,7 @@ using System.Text; using System.Threading.Tasks; using Microsoft.Build.Shared; +using StructuredLogger; namespace Microsoft.Build.Logging { @@ -90,7 +91,7 @@ public ProjectImportsCollector( private static string GetCacheDirectory() { - string dir = Path.Combine(Path.GetTempPath(), + string dir = Path.Combine(PathUtils.TempPath, $"MSBuildTemp-{Environment.UserName}-{Process.GetCurrentProcess().Id}-{AppDomain.CurrentDomain.Id}"); Directory.CreateDirectory(dir); return dir; diff --git a/src/StructuredLogger/ErrorReporting.cs b/src/StructuredLogger/ErrorReporting.cs index f83f13f46..9ad55da18 100644 --- a/src/StructuredLogger/ErrorReporting.cs +++ b/src/StructuredLogger/ErrorReporting.cs @@ -1,23 +1,12 @@ using System; using System.IO; +using StructuredLogger; namespace Microsoft.Build.Logging.StructuredLogger { public class ErrorReporting { - private static readonly string logFilePath = Path.Combine(GetRootPath(), "LoggerExceptions.txt"); - - private static string GetRootPath() - { -#if NETCORE - var path = Path.GetTempPath(); -#else - var path = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData); -#endif - - path = Path.Combine(path, "Microsoft", "MSBuildStructuredLog"); - return path; - } + private static readonly string logFilePath = Path.Combine(PathUtils.RootPath, "LoggerExceptions.txt"); public static void ReportException(Exception ex) { diff --git a/src/StructuredLogger/PathUtils.cs b/src/StructuredLogger/PathUtils.cs new file mode 100644 index 000000000..de25a3421 --- /dev/null +++ b/src/StructuredLogger/PathUtils.cs @@ -0,0 +1,25 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace StructuredLogger +{ + public static class PathUtils + { + public static readonly string RootPath = GetRootPath(); + public static readonly string TempPath = Path.Combine(RootPath, "Temp"); + + private static string GetRootPath() + { +#if NETCORE + var path = Path.GetTempPath(); +#else + var path = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData); +#endif + + path = Path.Combine(path, "Microsoft", "MSBuildStructuredLog"); + return path; + } + } +} diff --git a/src/StructuredLogger/StructuredLogger.csproj b/src/StructuredLogger/StructuredLogger.csproj index f1388227b..a6f31fb42 100644 --- a/src/StructuredLogger/StructuredLogger.csproj +++ b/src/StructuredLogger/StructuredLogger.csproj @@ -60,4 +60,7 @@ key.snk False + + + From 782c33ef0990e2786dae3c0a554b3e43d8b41335 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Thu, 2 Nov 2023 18:22:09 +0100 Subject: [PATCH 08/13] Improve the redacting UX --- .../Controls/RedactInputControl.xaml | 39 ++++-- .../Controls/RedactInputControl.xaml.cs | 50 +++++++- src/StructuredLogViewer/MainWindow.xaml.cs | 116 ++++++++++++++---- .../BinaryLogReplayEventSource.cs | 37 ++++-- .../Postprocessing/BinlogRedactor.cs | 64 +++++++++- 5 files changed, 252 insertions(+), 54 deletions(-) diff --git a/src/StructuredLogViewer/Controls/RedactInputControl.xaml b/src/StructuredLogViewer/Controls/RedactInputControl.xaml index 8f2bd9044..fd20bf412 100644 --- a/src/StructuredLogViewer/Controls/RedactInputControl.xaml +++ b/src/StructuredLogViewer/Controls/RedactInputControl.xaml @@ -6,23 +6,36 @@ xmlns:local="clr-namespace:StructuredLogViewer.Controls" mc:Ignorable="d" ContentRendered="Window_ContentRendered" - Title="Redact Binlog" SizeToContent="WidthAndHeight"> - - - - + Title="Redact Binlog" SizeToContent="WidthAndHeight" Height="280"> + - - + - - + + + + + + + + + + + + + + + + + + + + + + - - - - + diff --git a/src/StructuredLogViewer/Controls/RedactInputControl.xaml.cs b/src/StructuredLogViewer/Controls/RedactInputControl.xaml.cs index 54a0761a9..a8ca609f7 100644 --- a/src/StructuredLogViewer/Controls/RedactInputControl.xaml.cs +++ b/src/StructuredLogViewer/Controls/RedactInputControl.xaml.cs @@ -19,8 +19,19 @@ namespace StructuredLogViewer.Controls /// public partial class RedactInputControl : Window { - public RedactInputControl() + private readonly Func _getSaveAsDestination; + public string DestinationFile { get; private set; } + public bool RedactCommonCredentials { get; private set; } = false; + public bool RedactUsername { get; set; } = true; + public bool RedactEmbeddedFiles { get; set; } = true; + public string SecretsBlock { + get { return ChckbxCustomSecrets.IsChecked == true ? TxtSecrets.Text : null; } + } + + public RedactInputControl(Func getSaveAsDestination) + { + _getSaveAsDestination = getSaveAsDestination; InitializeComponent(); } @@ -29,15 +40,44 @@ private void btnDialogOk_Click(object sender, RoutedEventArgs e) this.DialogResult = true; } + private void btnSaveAs_Click(object sender, RoutedEventArgs e) + { + var destination = _getSaveAsDestination(); + if (destination != null) + { + this.DestinationFile = destination; + this.DialogResult = true; + } + } + private void Window_ContentRendered(object sender, EventArgs e) { - txtAnswer.SelectAll(); - txtAnswer.Focus(); + ChckbxUsername.IsChecked = RedactUsername; + ChckbxCommonCredentials.IsChecked = RedactCommonCredentials; + ChckbxEmbeddedFiles.IsChecked = RedactEmbeddedFiles; + + TxtSecrets.SelectAll(); + TxtSecrets.Focus(); + } + + private void ChckbxCustomSecrets_OnChanged(object sender, RoutedEventArgs e) + { + TxtSecrets.IsEnabled = ChckbxCustomSecrets.IsChecked == true; + } + + private void ChckbxUsername_OnChanged(object sender, RoutedEventArgs e) + { + RedactUsername = ChckbxUsername.IsChecked == true; + } + + private void ChckbxCommonCredentials_OnChanged(object sender, RoutedEventArgs e) + { + RedactCommonCredentials = ChckbxCommonCredentials.IsChecked == true; } - public string Answer + private void ChckbxEmbeddedFiles_OnChanged(object sender, RoutedEventArgs e) { - get { return txtAnswer.Text; } + RedactEmbeddedFiles = ChckbxEmbeddedFiles.IsChecked == true; } } } diff --git a/src/StructuredLogViewer/MainWindow.xaml.cs b/src/StructuredLogViewer/MainWindow.xaml.cs index 6154bd857..10093ae9d 100644 --- a/src/StructuredLogViewer/MainWindow.xaml.cs +++ b/src/StructuredLogViewer/MainWindow.xaml.cs @@ -5,6 +5,7 @@ using System.IO; using System.Linq; using System.Net.Http; +using System.Threading; using System.Windows; using System.Windows.Controls; using System.Windows.Input; @@ -357,7 +358,7 @@ private void UpdateRecentItemsMenu(WelcomeScreen welcomeScreen = null) private void SetContent(object content) { // We save build control to allow to bring back states to new one - if (mainContent.Content is BuildControl current) + if (mainContent.Content is BuildControl current) { lastSearchText = current.searchLogControl.SearchText; } @@ -662,41 +663,110 @@ private void Reload() OpenLogFile(logFilePath); } - private void RedactSecrets() + private async void RedactSecrets() { - RedactInputControl redactInputControl = new RedactInputControl(); + RedactInputControl redactInputControl = new RedactInputControl(GetSaveAsDestination); if (redactInputControl.ShowDialog() == true) { - BinlogRedactor.RedactSecrets(logFilePath, redactInputControl.Answer.Split()); - OpenLogFile(logFilePath); + List stringsToRedact = new(redactInputControl.SecretsBlock?.Split() ?? new string[] { }); + if(redactInputControl.RedactUsername) + { + // This will change as we have explicit username redactor + // (that can detect username in log produced on different machine) + stringsToRedact.Add(Environment.UserName); + } + + if (!stringsToRedact.Any()) + { + MessageBox.Show("No secrets to redact - no action will be performed"); + return; + } + + var progress = new BuildProgress(); + progress.Progress.Updated += update => + { + Dispatcher.InvokeAsync(() => + { + progress.Value = update.Ratio; + }, DispatcherPriority.Background); + }; + progress.ProgressText = "Performing the log redaction ..."; + SetContent(progress); + + string error = await System.Threading.Tasks.Task.Run(() => + { + try + { + BinlogRedactor.RedactSecrets( + logFilePath, + redactInputControl.DestinationFile, + stringsToRedact.ToArray(), + redactInputControl.RedactEmbeddedFiles, + progress.Progress); + } + catch(Exception e) + { + return e.ToString(); + } + + return null; + }); + + if (!string.IsNullOrEmpty(error)) + { + MessageBox.Show($"Redaction failed:{Environment.NewLine}{error}"); + SetContent(currentBuild); + } + else if (string.IsNullOrEmpty(redactInputControl.DestinationFile)) + { + // Reload + OpenLogFile(logFilePath); + } + else + { + SetContent(currentBuild); + AnnounceFileSaved(redactInputControl.DestinationFile); + } + } } + private string GetSaveAsDestination() + { + string currentFilePath = currentBuild.LogFilePath; + + var saveFileDialog = new SaveFileDialog(); + saveFileDialog.Filter = currentFilePath != null && currentFilePath.EndsWith(".binlog", StringComparison.OrdinalIgnoreCase) ? Serialization.BinlogFileDialogFilter : Serialization.FileDialogFilter; + saveFileDialog.Title = "Save log file as"; + saveFileDialog.CheckFileExists = false; + saveFileDialog.OverwritePrompt = true; + saveFileDialog.ValidateNames = true; + var result = saveFileDialog.ShowDialog(this); + + return result == true ? saveFileDialog.FileName : null; + } + + private void AnnounceFileSaved(string filePath) + { + Dispatcher.InvokeAsync(() => + { + currentBuild.UpdateBreadcrumb(new Message { Text = $"Saved {filePath}" }); + }); + SettingsService.AddRecentLogFile(filePath); + } + private void SaveAs() { if (currentBuild != null) { string currentFilePath = currentBuild.LogFilePath; - - var saveFileDialog = new SaveFileDialog(); - saveFileDialog.Filter = currentFilePath != null && currentFilePath.EndsWith(".binlog", StringComparison.OrdinalIgnoreCase) ? Serialization.BinlogFileDialogFilter : Serialization.FileDialogFilter; - saveFileDialog.Title = "Save log file as"; - saveFileDialog.CheckFileExists = false; - saveFileDialog.OverwritePrompt = true; - saveFileDialog.ValidateNames = true; - var result = saveFileDialog.ShowDialog(this); - if (result != true) - { - return; - } - - string newFilePath = saveFileDialog.FileName; + string newFilePath = GetSaveAsDestination(); if (string.IsNullOrEmpty(newFilePath) || string.Equals(currentFilePath, newFilePath, StringComparison.OrdinalIgnoreCase)) { return; } - logFilePath = saveFileDialog.FileName; + logFilePath = newFilePath; lock (inProgressOperationLock) { @@ -715,11 +785,7 @@ private void SaveAs() currentBuild.Build.LogFilePath = logFilePath; - Dispatcher.InvokeAsync(() => - { - currentBuild.UpdateBreadcrumb(new Message { Text = $"Saved {logFilePath}" }); - }); - SettingsService.AddRecentLogFile(logFilePath); + AnnounceFileSaved(logFilePath); } catch { diff --git a/src/StructuredLogger/BinaryLogger/BinaryLogReplayEventSource.cs b/src/StructuredLogger/BinaryLogger/BinaryLogReplayEventSource.cs index 85b0e5da9..e9aa2991e 100644 --- a/src/StructuredLogger/BinaryLogger/BinaryLogReplayEventSource.cs +++ b/src/StructuredLogger/BinaryLogger/BinaryLogReplayEventSource.cs @@ -36,6 +36,17 @@ public void Replay(string sourceFilePath) Replay(sourceFilePath, CancellationToken.None); } + /// + /// Read the provided binary log file opened as a stream and raise corresponding events for each BuildEventArgs + /// + /// Stream over the binlog content. + /// + public void Replay(Stream sourceFileStream, CancellationToken cancellationToken) + { + using var binaryReader = OpenReader(sourceFileStream); + Replay(binaryReader, cancellationToken); + } + /// /// Creates a for the provided binary log file. /// Performs decompression and buffering in the optimal way. @@ -49,13 +60,7 @@ public static BinaryReader OpenReader(string sourceFilePath) try { stream = new FileStream(sourceFilePath, FileMode.Open, FileAccess.Read, FileShare.Read); - var gzipStream = new GZipStream(stream, CompressionMode.Decompress, leaveOpen: false); - - // wrapping the GZipStream in a buffered stream significantly improves performance - // and the max throughput is reached with a 32K buffer. See details here: - // https://github.com/dotnet/runtime/issues/39233#issuecomment-745598847 - var bufferedStream = new BufferedStream(gzipStream, 32768); - return new BinaryReader(bufferedStream); + return OpenReader(stream); } catch (Exception) { @@ -64,6 +69,24 @@ public static BinaryReader OpenReader(string sourceFilePath) } } + /// + /// Creates a for the provided binary log file. + /// Performs decompression and buffering in the optimal way. + /// Caller is responsible for disposing the returned reader. + /// + /// Stream over the binlog file + /// BinaryReader of the given binlog file. + public static BinaryReader OpenReader(Stream sourceFileStream) + { + var gzipStream = new GZipStream(sourceFileStream, CompressionMode.Decompress, leaveOpen: false); + + // wrapping the GZipStream in a buffered stream significantly improves performance + // and the max throughput is reached with a 32K buffer. See details here: + // https://github.com/dotnet/runtime/issues/39233#issuecomment-745598847 + var bufferedStream = new BufferedStream(gzipStream, 32768); + return new BinaryReader(bufferedStream); + } + /// /// Creates a for the provided binary log file. /// Performs decompression and buffering in the optimal way. diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/BinlogRedactor.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/BinlogRedactor.cs index e77e74931..ab058b8c8 100644 --- a/src/StructuredLogger/BinaryLogger/Postprocessing/BinlogRedactor.cs +++ b/src/StructuredLogger/BinaryLogger/Postprocessing/BinlogRedactor.cs @@ -5,6 +5,7 @@ using Microsoft.Build.Logging; using Microsoft.Build.Logging.StructuredLogger; using System.IO; +using System.Threading; namespace StructuredLogger.BinaryLogger.Postprocessing { @@ -43,20 +44,49 @@ public class BinlogRedactor { private readonly ISensitiveDataProcessor _sensitiveDataProcessor; - public static void RedactSecrets(string binlogPath, string[] secrets) + public static void RedactSecrets( + string binlogPath, + string[] secrets) + => RedactSecrets(binlogPath, secrets, processEmbeddedFiles: true, progress: null); + + public static void RedactSecrets( + string binlogPath, + string[] secrets, + bool processEmbeddedFiles, + Progress progress) { - var sensitivityProcessor = new SimpleSensitiveDataProcessor(secrets, true); string outputFile = Path.Combine(PathUtils.TempPath, Path.GetFileName(Path.GetTempFileName()) + ".binlog"); - new BinlogRedactor(sensitivityProcessor).ProcessBinlog(binlogPath, outputFile, skipEmbeddedFiles: false); + RedactSecrets(binlogPath, outputFile, secrets, processEmbeddedFiles, progress); File.Delete(binlogPath); File.Move(outputFile, binlogPath); } + public static void RedactSecrets( + string binlogPath, + string outputFile, + string[] secrets, + bool processEmbeddedFiles, + Progress progress) + { + if (string.IsNullOrEmpty(outputFile) || + string.Equals(binlogPath, outputFile, StringComparison.OrdinalIgnoreCase)) + { + // This is in place redaction. + RedactSecrets(binlogPath, secrets, processEmbeddedFiles, progress); + return; + } + var sensitivityProcessor = new SimpleSensitiveDataProcessor(secrets, true); + new BinlogRedactor(sensitivityProcessor) { Progress = progress } + .ProcessBinlog(binlogPath, outputFile, skipEmbeddedFiles: !processEmbeddedFiles); + } + public BinlogRedactor(ISensitiveDataProcessor sensitiveDataProcessor) { _sensitiveDataProcessor = sensitiveDataProcessor; } + public Progress Progress { private get; set; } + public void ProcessBinlog( string inputFileName, string outputFileName, @@ -77,11 +107,37 @@ public void ProcessBinlog( } outputBinlog.Initialize(originalEventsSource); - originalEventsSource.Replay(inputFileName); + + var inputStream = new FileStream(inputFileName, FileMode.Open, FileAccess.Read, FileShare.Read); + + CancellationTokenSource cts = null; + if (Progress != null) + { + cts = new CancellationTokenSource(); + long streamLength = inputStream.Length; + System.Threading.Tasks.Task.Run(async () => + { + while (!cts.IsCancellationRequested) + { + await System.Threading.Tasks.Task.Delay(200, cts.Token); + Progress.Report((double)inputStream.Position / streamLength); + } + }, cts.Token); + } + + originalEventsSource.Replay(inputStream, CancellationToken.None); outputBinlog.Shutdown(); // TODO: error handling + if (Progress != null) + { + cts.Cancel(); + Progress.Report(1.0); + } + + ((IBuildEventStringsReader)originalEventsSource).StringReadDone -= HandleStringRead; + void HandleStringRead(StringReadEventArgs args) { args.StringToBeUsed = _sensitiveDataProcessor.ReplaceSensitiveData(args.OriginalString); From bdfbafe4f4f6f46614a9d01b2360357f9e75c50d Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Fri, 3 Nov 2023 10:49:43 +0100 Subject: [PATCH 09/13] Fixing after applying changes from main --- .../Controls/RedactInputControl.xaml.cs | 1 + .../BinaryLogger/BinaryLogReplayEventSource.cs | 1 + src/StructuredLogger/Progress.cs | 7 ++++++- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/StructuredLogViewer/Controls/RedactInputControl.xaml.cs b/src/StructuredLogViewer/Controls/RedactInputControl.xaml.cs index a8ca609f7..12e177d45 100644 --- a/src/StructuredLogViewer/Controls/RedactInputControl.xaml.cs +++ b/src/StructuredLogViewer/Controls/RedactInputControl.xaml.cs @@ -54,6 +54,7 @@ private void Window_ContentRendered(object sender, EventArgs e) { ChckbxUsername.IsChecked = RedactUsername; ChckbxCommonCredentials.IsChecked = RedactCommonCredentials; + TxtSecrets.IsEnabled = RedactCommonCredentials; ChckbxEmbeddedFiles.IsChecked = RedactEmbeddedFiles; TxtSecrets.SelectAll(); diff --git a/src/StructuredLogger/BinaryLogger/BinaryLogReplayEventSource.cs b/src/StructuredLogger/BinaryLogger/BinaryLogReplayEventSource.cs index e9aa2991e..dfb725a84 100644 --- a/src/StructuredLogger/BinaryLogger/BinaryLogReplayEventSource.cs +++ b/src/StructuredLogger/BinaryLogger/BinaryLogReplayEventSource.cs @@ -145,6 +145,7 @@ public void Replay(BinaryReader binaryReader, CancellationToken cancellationToke reader.EmbeddedContentRead += _embeddedContentRead; reader.StringReadDone += _stringReadDone; + reader.ArchiveFileEncountered += _archiveFileEncountered; while (!cancellationToken.IsCancellationRequested && reader.Read() is { } instance) { diff --git a/src/StructuredLogger/Progress.cs b/src/StructuredLogger/Progress.cs index 3c6fa8aed..40f0e8653 100644 --- a/src/StructuredLogger/Progress.cs +++ b/src/StructuredLogger/Progress.cs @@ -9,6 +9,11 @@ public class Progress : IProgress public event Action Updated; + public virtual void Report(double ratio) + { + Report(new ProgressUpdate { Ratio = ratio }); + } + public virtual void Report(ProgressUpdate progressUpdate) { Updated?.Invoke(progressUpdate); @@ -20,4 +25,4 @@ public struct ProgressUpdate public double Ratio; public int BufferLength; } -} \ No newline at end of file +} From 0c2cbe532ee0bde86fb597bf4db3c54a0af58fb1 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Fri, 3 Nov 2023 20:41:43 +0100 Subject: [PATCH 10/13] Removed temporary code, replaced with packages --- .../Controls/RedactInputControl.xaml | 5 +- .../Controls/RedactInputControl.xaml.cs | 12 +- src/StructuredLogViewer/MainWindow.xaml.cs | 26 ++-- .../BinaryLogger/BuildEventArgsReader.cs | 1 + .../Postprocessing/BinlogRedactor.cs | 109 ++++++++-------- .../Postprocessing/CleanupScope.cs | 15 --- .../Postprocessing/ConcatenatedReadStream.cs | 100 --------------- .../Postprocessing/StreamExtensions.cs | 118 ------------------ .../BinaryLogger/Postprocessing/SubStream.cs | 55 -------- .../Postprocessing/TransparentReadStream.cs | 115 ----------------- src/StructuredLogger/StructuredLogger.cs | 1 + src/StructuredLogger/StructuredLogger.csproj | 2 + 12 files changed, 86 insertions(+), 473 deletions(-) delete mode 100644 src/StructuredLogger/BinaryLogger/Postprocessing/CleanupScope.cs delete mode 100644 src/StructuredLogger/BinaryLogger/Postprocessing/ConcatenatedReadStream.cs delete mode 100644 src/StructuredLogger/BinaryLogger/Postprocessing/StreamExtensions.cs delete mode 100644 src/StructuredLogger/BinaryLogger/Postprocessing/SubStream.cs delete mode 100644 src/StructuredLogger/BinaryLogger/Postprocessing/TransparentReadStream.cs diff --git a/src/StructuredLogViewer/Controls/RedactInputControl.xaml b/src/StructuredLogViewer/Controls/RedactInputControl.xaml index fd20bf412..32f4415ae 100644 --- a/src/StructuredLogViewer/Controls/RedactInputControl.xaml +++ b/src/StructuredLogViewer/Controls/RedactInputControl.xaml @@ -6,7 +6,7 @@ xmlns:local="clr-namespace:StructuredLogViewer.Controls" mc:Ignorable="d" ContentRendered="Window_ContentRendered" - Title="Redact Binlog" SizeToContent="WidthAndHeight" Height="280"> + Title="Redact Binlog" SizeToContent="WidthAndHeight" Height="310"> @@ -15,7 +15,8 @@ - + + diff --git a/src/StructuredLogViewer/Controls/RedactInputControl.xaml.cs b/src/StructuredLogViewer/Controls/RedactInputControl.xaml.cs index 12e177d45..74ee75a9c 100644 --- a/src/StructuredLogViewer/Controls/RedactInputControl.xaml.cs +++ b/src/StructuredLogViewer/Controls/RedactInputControl.xaml.cs @@ -21,9 +21,10 @@ public partial class RedactInputControl : Window { private readonly Func _getSaveAsDestination; public string DestinationFile { get; private set; } - public bool RedactCommonCredentials { get; private set; } = false; + public bool RedactCommonCredentials { get; private set; } = true; public bool RedactUsername { get; set; } = true; public bool RedactEmbeddedFiles { get; set; } = true; + public bool DistinguishSecretsReplacements { get; set; } = true; public string SecretsBlock { get { return ChckbxCustomSecrets.IsChecked == true ? TxtSecrets.Text : null; } @@ -54,8 +55,10 @@ private void Window_ContentRendered(object sender, EventArgs e) { ChckbxUsername.IsChecked = RedactUsername; ChckbxCommonCredentials.IsChecked = RedactCommonCredentials; - TxtSecrets.IsEnabled = RedactCommonCredentials; + ChckbxCustomSecrets.IsChecked = false; + TxtSecrets.IsEnabled = false; ChckbxEmbeddedFiles.IsChecked = RedactEmbeddedFiles; + ChckbxDistinguishReplacements.IsChecked = DistinguishSecretsReplacements; TxtSecrets.SelectAll(); TxtSecrets.Focus(); @@ -80,5 +83,10 @@ private void ChckbxEmbeddedFiles_OnChanged(object sender, RoutedEventArgs e) { RedactEmbeddedFiles = ChckbxEmbeddedFiles.IsChecked == true; } + + private void ChckbxDistinguishReplacements_OnChanged(object sender, RoutedEventArgs e) + { + DistinguishSecretsReplacements = ChckbxDistinguishReplacements.IsChecked == true; + } } } diff --git a/src/StructuredLogViewer/MainWindow.xaml.cs b/src/StructuredLogViewer/MainWindow.xaml.cs index 10093ae9d..71aa22d4f 100644 --- a/src/StructuredLogViewer/MainWindow.xaml.cs +++ b/src/StructuredLogViewer/MainWindow.xaml.cs @@ -669,14 +669,11 @@ private async void RedactSecrets() if (redactInputControl.ShowDialog() == true) { List stringsToRedact = new(redactInputControl.SecretsBlock?.Split() ?? new string[] { }); - if(redactInputControl.RedactUsername) - { - // This will change as we have explicit username redactor - // (that can detect username in log produced on different machine) - stringsToRedact.Add(Environment.UserName); - } - if (!stringsToRedact.Any()) + if ( + !stringsToRedact.Any() && + !redactInputControl.RedactUsername && + !redactInputControl.RedactCommonCredentials) { MessageBox.Show("No secrets to redact - no action will be performed"); return; @@ -697,11 +694,18 @@ private async void RedactSecrets() { try { + BinlogRedactorOptions redactorOptions = new BinlogRedactorOptions(logFilePath) + { + OutputFileName = redactInputControl.DestinationFile, + ProcessEmbeddedFiles = redactInputControl.RedactEmbeddedFiles, + AutodetectUsername = redactInputControl.RedactUsername, + AutodetectCommonPatterns = redactInputControl.RedactCommonCredentials, + IdentifyReplacemenets = redactInputControl.DistinguishSecretsReplacements, + TokensToRedact = stringsToRedact.ToArray(), + }; + BinlogRedactor.RedactSecrets( - logFilePath, - redactInputControl.DestinationFile, - stringsToRedact.ToArray(), - redactInputControl.RedactEmbeddedFiles, + redactorOptions, progress.Progress); } catch(Exception e) diff --git a/src/StructuredLogger/BinaryLogger/BuildEventArgsReader.cs b/src/StructuredLogger/BinaryLogger/BuildEventArgsReader.cs index dd07ece89..b91efce5c 100644 --- a/src/StructuredLogger/BinaryLogger/BuildEventArgsReader.cs +++ b/src/StructuredLogger/BinaryLogger/BuildEventArgsReader.cs @@ -10,6 +10,7 @@ using System.Reflection; using System.Runtime.InteropServices; using System.Text; +using DotUtils.StreamUtils; using Microsoft.Build.BackEnd; using Microsoft.Build.Collections; using Microsoft.Build.Framework; diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/BinlogRedactor.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/BinlogRedactor.cs index ab058b8c8..4b30ee89b 100644 --- a/src/StructuredLogger/BinaryLogger/Postprocessing/BinlogRedactor.cs +++ b/src/StructuredLogger/BinaryLogger/Postprocessing/BinlogRedactor.cs @@ -6,83 +6,82 @@ using Microsoft.Build.Logging.StructuredLogger; using System.IO; using System.Threading; +using Microsoft.Build.SensitiveDataDetector; namespace StructuredLogger.BinaryLogger.Postprocessing { - public interface ISensitiveDataProcessor + public sealed class BinlogRedactorOptions { - /// - /// Processes the given text and if needed, replaces sensitive data with a placeholder. - /// - string ReplaceSensitiveData(string text); - } - - internal sealed class SimpleSensitiveDataProcessor : ISensitiveDataProcessor - { - public const string DefaultReplacementPattern = "*******"; - private readonly (string token, string replacement)[] _secretsToRedact; - - public SimpleSensitiveDataProcessor(string[] secretsToRedact, bool identifyReplacements) + public BinlogRedactorOptions(string inputPath) { - _secretsToRedact = secretsToRedact.Select((secret, cnt) => - (token: secret, identifyReplacements ? $"REDACTED__TKN{(cnt + 1):00}" : DefaultReplacementPattern)) - .ToArray(); + InputPath = inputPath; } - public string ReplaceSensitiveData(string text) - { - foreach ((string token, string replacement) secret in _secretsToRedact) - { - text = text.Replace(secret.token, secret.replacement); - } - - return text; - } + public string[]? TokensToRedact { get; set; } + public string InputPath { get; } + public string? OutputFileName { get; set; } + public bool ProcessEmbeddedFiles { get; set; } = true; + public bool IdentifyReplacemenets { get; set; } = true; + public bool AutodetectCommonPatterns { get; set; } = true; + public bool AutodetectUsername { get; set; } = true; } public class BinlogRedactor { - private readonly ISensitiveDataProcessor _sensitiveDataProcessor; + private readonly ISensitiveDataRedactor _sensitiveDataRedactor; public static void RedactSecrets( string binlogPath, string[] secrets) - => RedactSecrets(binlogPath, secrets, processEmbeddedFiles: true, progress: null); + => RedactSecrets( + new BinlogRedactorOptions(binlogPath) { TokensToRedact = secrets, }, progress: null); public static void RedactSecrets( - string binlogPath, - string[] secrets, - bool processEmbeddedFiles, + BinlogRedactorOptions redactorOptions, Progress progress) { - string outputFile = Path.Combine(PathUtils.TempPath, Path.GetFileName(Path.GetTempFileName()) + ".binlog"); - RedactSecrets(binlogPath, outputFile, secrets, processEmbeddedFiles, progress); - File.Delete(binlogPath); - File.Move(outputFile, binlogPath); - } + string outputFile; + bool replaceInPlace = false; - public static void RedactSecrets( - string binlogPath, - string outputFile, - string[] secrets, - bool processEmbeddedFiles, - Progress progress) - { - if (string.IsNullOrEmpty(outputFile) || - string.Equals(binlogPath, outputFile, StringComparison.OrdinalIgnoreCase)) + if (string.IsNullOrEmpty(redactorOptions.OutputFileName) || + string.Equals(redactorOptions.InputPath, redactorOptions.OutputFileName, StringComparison.OrdinalIgnoreCase)) + { + outputFile = Path.Combine(PathUtils.TempPath, Path.GetFileName(Path.GetTempFileName()) + ".binlog"); + replaceInPlace = true; + } + else + { + outputFile = redactorOptions.OutputFileName; + } + + SensitiveDataKind sensitiveDataKind = SensitiveDataKind.ExplicitSecrets; + if (redactorOptions.AutodetectCommonPatterns) + { + sensitiveDataKind |= SensitiveDataKind.CommonSecrets; + } + if (redactorOptions.AutodetectUsername) + { + sensitiveDataKind |= SensitiveDataKind.Username; + } + + ISensitiveDataRedactor sensitiveDataRedactor = SensitiveDataDetectorFactory.GetSecretsDetector( + sensitiveDataKind, + redactorOptions.IdentifyReplacemenets, + redactorOptions.TokensToRedact); + + new BinlogRedactor(sensitiveDataRedactor) { Progress = progress } + .ProcessBinlog(redactorOptions.InputPath, outputFile, !redactorOptions.ProcessEmbeddedFiles); + + if (replaceInPlace) { - // This is in place redaction. - RedactSecrets(binlogPath, secrets, processEmbeddedFiles, progress); - return; + File.Delete(redactorOptions.InputPath); + File.Move(outputFile, redactorOptions.InputPath); } - var sensitivityProcessor = new SimpleSensitiveDataProcessor(secrets, true); - new BinlogRedactor(sensitivityProcessor) { Progress = progress } - .ProcessBinlog(binlogPath, outputFile, skipEmbeddedFiles: !processEmbeddedFiles); } - public BinlogRedactor(ISensitiveDataProcessor sensitiveDataProcessor) + public BinlogRedactor(ISensitiveDataRedactor sensitiveDataRedactor) { - _sensitiveDataProcessor = sensitiveDataProcessor; + _sensitiveDataRedactor = sensitiveDataRedactor; } public Progress Progress { private get; set; } @@ -92,9 +91,9 @@ public void ProcessBinlog( string outputFileName, bool skipEmbeddedFiles) { - BinaryLogReplayEventSource originalEventsSource = new BinaryLogReplayEventSource(); + BinaryLogReplayEventSource originalEventsSource = new(); - Microsoft.Build.Logging.StructuredLogger.BinaryLogger outputBinlog = new Microsoft.Build.Logging.StructuredLogger.BinaryLogger() + Microsoft.Build.Logging.StructuredLogger.BinaryLogger outputBinlog = new() { Parameters = $"LogFile={outputFileName};OmitInitialInfo", }; @@ -140,7 +139,7 @@ public void ProcessBinlog( void HandleStringRead(StringReadEventArgs args) { - args.StringToBeUsed = _sensitiveDataProcessor.ReplaceSensitiveData(args.OriginalString); + args.StringToBeUsed = _sensitiveDataRedactor.Redact(args.OriginalString); } } } diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/CleanupScope.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/CleanupScope.cs deleted file mode 100644 index e44afc523..000000000 --- a/src/StructuredLogger/BinaryLogger/Postprocessing/CleanupScope.cs +++ /dev/null @@ -1,15 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; - -namespace Microsoft.Build.Logging; - -internal readonly struct CleanupScope : IDisposable -{ - private readonly Action _disposeAction; - - public CleanupScope(Action disposeAction) => _disposeAction = disposeAction; - - public void Dispose() => _disposeAction(); -} diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/ConcatenatedReadStream.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/ConcatenatedReadStream.cs deleted file mode 100644 index 1fecf03f3..000000000 --- a/src/StructuredLogger/BinaryLogger/Postprocessing/ConcatenatedReadStream.cs +++ /dev/null @@ -1,100 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; - -namespace StructuredLogger.BinaryLogger.Postprocessing -{ - internal class ConcatenatedReadStream : Stream - { - private readonly Queue _streams; - private long _position; - - public ConcatenatedReadStream(IEnumerable streams) - => _streams = EnsureStreamsAreReadable(streams); - - public ConcatenatedReadStream(params Stream[] streams) - => _streams = EnsureStreamsAreReadable(streams); - - private static Queue EnsureStreamsAreReadable(IEnumerable streams) - { - var result = (streams is ICollection collection) ? new Queue(collection.Count) : new Queue(); - - foreach (Stream stream in streams) - { - if (!stream.CanRead) - { - throw new ArgumentException("All streams must be readable", nameof(streams)); - } - - if (stream is ConcatenatedReadStream concatenatedStream) - { - foreach (Stream subStream in concatenatedStream._streams) - { - result.Enqueue(subStream); - } - } - else - { - result.Enqueue(stream); - } - } - - return result; - } - - public override void Flush() - { - throw new NotSupportedException("ConcatenatedReadStream is forward-only read-only"); - } - - public override int Read(byte[] buffer, int offset, int count) - { - int totalBytesRead = 0; - - while (count > 0 && _streams.Count > 0) - { - int bytesRead = _streams.Peek().Read(buffer, offset, count); - if (bytesRead == 0) - { - _streams.Dequeue().Dispose(); - continue; - } - - totalBytesRead += bytesRead; - offset += bytesRead; - count -= bytesRead; - } - - _position += totalBytesRead; - return totalBytesRead; - } - - public override long Seek(long offset, SeekOrigin origin) - { - throw new NotSupportedException("ConcatenatedReadStream is forward-only read-only"); - } - - public override void SetLength(long value) - { - throw new NotSupportedException("ConcatenatedReadStream is forward-only read-only"); - } - - public override void Write(byte[] buffer, int offset, int count) - { - throw new NotSupportedException("ConcatenatedReadStream is forward-only read-only"); - } - - public override bool CanRead => true; - public override bool CanSeek => false; - public override bool CanWrite => false; - public override long Length => _streams.Sum(s => s.Length); - - public override long Position - { - get => _position; - set => throw new NotSupportedException("ConcatenatedReadStream is forward-only read-only"); - } - } -} diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/StreamExtensions.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/StreamExtensions.cs deleted file mode 100644 index 180f924bf..000000000 --- a/src/StructuredLogger/BinaryLogger/Postprocessing/StreamExtensions.cs +++ /dev/null @@ -1,118 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Buffers; -using System.Diagnostics; -using System.IO; -using Microsoft.Build.Shared; -using StructuredLogger.BinaryLogger.Postprocessing; - -namespace Microsoft.Build.Logging -{ - internal static class StreamExtensions - { - private static bool CheckIsSkipNeeded(long bytesCount) - { - if(bytesCount is < 0 or > int.MaxValue) - { - throw new ArgumentOutOfRangeException(nameof(bytesCount), $"Attempt to skip {bytesCount} bytes, only non-negative offset up to int.MaxValue is allowed."); - } - - return bytesCount > 0; - } - - public static long SkipBytes(this Stream stream) - => SkipBytes(stream, stream.Length, true); - - public static long SkipBytes(this Stream stream, long bytesCount) - => SkipBytes(stream, bytesCount, true); - - public static long SkipBytes(this Stream stream, long bytesCount, bool throwOnEndOfStream) - { - if (!CheckIsSkipNeeded(bytesCount)) - { - return 0; - } - - byte[] buffer = ArrayPool.Shared.Rent(4096); - using var _ = new CleanupScope(() => ArrayPool.Shared.Return(buffer)); - return SkipBytes(stream, bytesCount, throwOnEndOfStream, buffer); - } - - public static long SkipBytes(this Stream stream, long bytesCount, bool throwOnEndOfStream, byte[] buffer) - { - if (!CheckIsSkipNeeded(bytesCount)) - { - return 0; - } - - long totalRead = 0; - while (totalRead < bytesCount) - { - int read = stream.Read(buffer, 0, (int)Math.Min(bytesCount - totalRead, buffer.Length)); - if (read == 0) - { - if (throwOnEndOfStream) - { - throw new InvalidDataException("Unexpected end of stream."); - } - - return totalRead; - } - - totalRead += read; - } - - return totalRead; - } - - public static byte[] ReadToEnd(this Stream stream) - { - if (stream.TryGetLength(out long length)) - { - BinaryReader reader = new(stream); - return reader.ReadBytes((int)length); - } - - using var ms = new MemoryStream(); - stream.CopyTo(ms); - return ms.ToArray(); - } - - public static bool TryGetLength(this Stream stream, out long length) - { - try - { - length = stream.Length; - return true; - } - catch (NotSupportedException) - { - length = 0; - return false; - } - } - - public static Stream ToReadableSeekableStream(this Stream stream) - { - return TransparentReadStream.EnsureSeekableStream(stream); - } - - /// - /// Creates bounded read-only, forward-only view over an underlying stream. - /// - /// - /// - /// - public static Stream Slice(this Stream stream, long length) - { - return new SubStream(stream, length); - } - - public static Stream Concat(this Stream stream, Stream other) - { - return new ConcatenatedReadStream(stream, other); - } - } -} diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/SubStream.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/SubStream.cs deleted file mode 100644 index eb2713d24..000000000 --- a/src/StructuredLogger/BinaryLogger/Postprocessing/SubStream.cs +++ /dev/null @@ -1,55 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.IO; -using Microsoft.Build.Shared; - -namespace Microsoft.Build.Logging -{ - /// - /// Bounded read-only, forward-only view over an underlying stream. - /// - internal sealed class SubStream : Stream - { - // Do not Dispose/Close on Dispose/Close !! - private readonly Stream _stream; - private readonly long _length; - private long _position; - - public SubStream(Stream stream, long length) - { - _stream = stream; - _length = length; - - if (!stream.CanRead) - { - throw new NotSupportedException("Stream must be readable."); - } - } - - public bool IsAtEnd => _position >= _length; - - public override bool CanRead => true; - - public override bool CanSeek => false; - - public override bool CanWrite => false; - - public override long Length => _length; - - public override long Position { get => _position; set => throw new NotImplementedException(); } - - public override void Flush() { } - public override int Read(byte[] buffer, int offset, int count) - { - count = Math.Min((int)Math.Max(Length - _position, 0), count); - int read = _stream.Read(buffer, offset, count); - _position += read; - return read; - } - public override long Seek(long offset, SeekOrigin origin) => throw new NotImplementedException(); - public override void SetLength(long value) => throw new NotImplementedException(); - public override void Write(byte[] buffer, int offset, int count) => throw new NotImplementedException(); - } -} diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/TransparentReadStream.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/TransparentReadStream.cs deleted file mode 100644 index 14f7eb2d1..000000000 --- a/src/StructuredLogger/BinaryLogger/Postprocessing/TransparentReadStream.cs +++ /dev/null @@ -1,115 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.IO; -using Microsoft.Build.Shared; - -namespace Microsoft.Build.Logging -{ - /// - /// A wrapper stream that allows position tracking and forward seeking. - /// - internal sealed class TransparentReadStream : Stream - { - private readonly Stream _stream; - private long _position; - private long _maxAllowedPosition = long.MaxValue; - - public static Stream EnsureSeekableStream(Stream stream) - { - if (stream.CanSeek) - { - return stream; - } - - if (!stream.CanRead) - { - throw new InvalidOperationException("Stream must be readable."); - } - - return new TransparentReadStream(stream); - } - - public static TransparentReadStream EnsureTransparentReadStream(Stream stream) - { - if (stream is TransparentReadStream transparentReadStream) - { - return transparentReadStream; - } - - if (!stream.CanRead) - { - throw new InvalidOperationException("Stream must be readable."); - } - - return new TransparentReadStream(stream); - } - - private TransparentReadStream(Stream stream) - { - _stream = stream; - } - - public int? BytesCountAllowedToRead - { - set { _maxAllowedPosition = value.HasValue ? _position + value.Value : long.MaxValue; } - } - - // if we haven't constrained the allowed read size - do not report it being unfinished either. - public int BytesCountAllowedToReadRemaining => - _maxAllowedPosition == long.MaxValue ? 0 : (int)(_maxAllowedPosition - _position); - - public override bool CanRead => _stream.CanRead; - public override bool CanSeek => true; - public override bool CanWrite => false; - public override long Length => _stream.Length; - public override long Position - { - get => _position; - set => this.SkipBytes(value - _position, true); - } - - public override void Flush() - { - _stream.Flush(); - } - - public override int Read(byte[] buffer, int offset, int count) - { - if (_position + count > _maxAllowedPosition) - { - throw new StreamChunkOverReadException( - $"Attempt to read {count} bytes, when only {_maxAllowedPosition - _position} are allowed to be read."); - } - - int cnt = _stream.Read(buffer, offset, count); - _position += cnt; - return cnt; - } - - public override long Seek(long offset, SeekOrigin origin) - { - if (origin != SeekOrigin.Current) - { - throw new NotSupportedException("Only seeking from SeekOrigin.Current is supported."); - } - - this.SkipBytes(offset, true); - - return _position; - } - - public override void SetLength(long value) - { - throw new NotSupportedException("Expanding stream is not supported."); - } - - public override void Write(byte[] buffer, int offset, int count) - { - throw new NotSupportedException("Writing is not supported."); - } - - public override void Close() => _stream.Close(); - } -} diff --git a/src/StructuredLogger/StructuredLogger.cs b/src/StructuredLogger/StructuredLogger.cs index 4d7fc1cad..dd64f55ae 100644 --- a/src/StructuredLogger/StructuredLogger.cs +++ b/src/StructuredLogger/StructuredLogger.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using DotUtils.StreamUtils; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; using static Microsoft.Build.Logging.StructuredLogger.BinaryLogger; diff --git a/src/StructuredLogger/StructuredLogger.csproj b/src/StructuredLogger/StructuredLogger.csproj index a6f31fb42..18456b0e4 100644 --- a/src/StructuredLogger/StructuredLogger.csproj +++ b/src/StructuredLogger/StructuredLogger.csproj @@ -28,6 +28,8 @@ + + From 85decc9119c01feaca47d2822a3190065866af7f Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Fri, 3 Nov 2023 21:02:10 +0100 Subject: [PATCH 11/13] Fix the tokens parsing --- src/StructuredLogViewer/MainWindow.xaml.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/StructuredLogViewer/MainWindow.xaml.cs b/src/StructuredLogViewer/MainWindow.xaml.cs index 71aa22d4f..6b9d9fc02 100644 --- a/src/StructuredLogViewer/MainWindow.xaml.cs +++ b/src/StructuredLogViewer/MainWindow.xaml.cs @@ -668,7 +668,12 @@ private async void RedactSecrets() RedactInputControl redactInputControl = new RedactInputControl(GetSaveAsDestination); if (redactInputControl.ShowDialog() == true) { - List stringsToRedact = new(redactInputControl.SecretsBlock?.Split() ?? new string[] { }); + List stringsToRedact = + new(redactInputControl.SecretsBlock? + .Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries) + .Select(s => s.Trim()) + .Where(s => !string.IsNullOrWhiteSpace(s)) + ?? new string[] { }); if ( !stringsToRedact.Any() && From 0ac3e4cb1bd00f2e9c5520bc9b4fa9218725046f Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Sat, 4 Nov 2023 10:39:52 +0100 Subject: [PATCH 12/13] net472 syntax fix --- src/StructuredLogViewer/MainWindow.xaml.cs | 2 +- src/StructuredLogger/StructuredLogger.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/StructuredLogViewer/MainWindow.xaml.cs b/src/StructuredLogViewer/MainWindow.xaml.cs index 6b9d9fc02..00c76e8be 100644 --- a/src/StructuredLogViewer/MainWindow.xaml.cs +++ b/src/StructuredLogViewer/MainWindow.xaml.cs @@ -670,7 +670,7 @@ private async void RedactSecrets() { List stringsToRedact = new(redactInputControl.SecretsBlock? - .Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries) + .Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries) .Select(s => s.Trim()) .Where(s => !string.IsNullOrWhiteSpace(s)) ?? new string[] { }); diff --git a/src/StructuredLogger/StructuredLogger.csproj b/src/StructuredLogger/StructuredLogger.csproj index 18456b0e4..3b1c87667 100644 --- a/src/StructuredLogger/StructuredLogger.csproj +++ b/src/StructuredLogger/StructuredLogger.csproj @@ -29,7 +29,7 @@ - + From 7dc12569da075bfa9a276986967660c5667aae41 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Wed, 8 Nov 2023 15:04:31 +0100 Subject: [PATCH 13/13] Reflect on PR comments; improving binlogtool --- MSBuildStructuredLog.sln | 6 + src/BinlogTool/BinlogTool.csproj | 5 +- src/BinlogTool/Program.cs | 54 ++++++- src/BinlogTool/Redact.cs | 79 ++++++++++ src/BinlogTool/Search.cs | 43 ++++-- .../MSBuildStructuredLogViewer.nuspec | 2 + src/StructuredLogViewer/MainWindow.xaml.cs | 135 +++++++++--------- .../StructuredLogViewer.csproj | 1 + src/StructuredLogViewer/TaskExtensions.cs | 14 ++ .../BinaryLoggerTests.cs | 8 +- .../BinlogRedactor.cs | 11 +- .../StructuredLogger.Utils.csproj | 26 ++++ .../BinaryLogReplayEventSource.cs | 2 +- .../BinaryLogger/BuildEventArgsReader.cs | 7 +- .../Postprocessing/ArchiveFileEventArgs.cs | 4 +- .../ArchiveFileEventArgsExtensions.cs | 3 +- .../EmbeddedContentEventArgs.cs | 3 +- .../IBuildEventArgsReaderNotifications.cs | 2 +- .../IBuildEventStringsReader.cs | 2 +- .../Postprocessing/IBuildFileReader.cs | 2 +- .../Postprocessing/IEmbeddedContentSource.cs | 2 +- .../StreamChunkOverreadException.cs | 2 +- .../Postprocessing/StringReadEventArgs.cs | 2 +- .../BinaryLogger/ProjectImportsCollector.cs | 4 +- src/StructuredLogger/ErrorReporting.cs | 1 - src/StructuredLogger/PathUtils.cs | 2 +- src/StructuredLogger/StructuredLogger.cs | 2 - src/StructuredLogger/StructuredLogger.csproj | 4 +- 28 files changed, 300 insertions(+), 128 deletions(-) create mode 100644 src/BinlogTool/Redact.cs create mode 100644 src/StructuredLogViewer/TaskExtensions.cs rename src/{StructuredLogger/BinaryLogger/Postprocessing => StructuredLogger.Utils}/BinlogRedactor.cs (94%) create mode 100644 src/StructuredLogger.Utils/StructuredLogger.Utils.csproj diff --git a/MSBuildStructuredLog.sln b/MSBuildStructuredLog.sln index c77296dd1..30bd7127b 100644 --- a/MSBuildStructuredLog.sln +++ b/MSBuildStructuredLog.sln @@ -28,6 +28,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ResourcesGenerator", "src\R EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BinlogTool", "src\BinlogTool\BinlogTool.csproj", "{35F44EA6-7259-43CC-8C17-E058F3EB86D3}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "StructuredLogger.Utils", "src\StructuredLogger.Utils\StructuredLogger.Utils.csproj", "{AC634B46-D57C-44C5-BF56-480843182F21}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Mixed Platforms = Debug|Mixed Platforms @@ -66,6 +68,10 @@ Global {35F44EA6-7259-43CC-8C17-E058F3EB86D3}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU {35F44EA6-7259-43CC-8C17-E058F3EB86D3}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU {35F44EA6-7259-43CC-8C17-E058F3EB86D3}.Release|Mixed Platforms.Build.0 = Release|Any CPU + {AC634B46-D57C-44C5-BF56-480843182F21}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {AC634B46-D57C-44C5-BF56-480843182F21}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {AC634B46-D57C-44C5-BF56-480843182F21}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {AC634B46-D57C-44C5-BF56-480843182F21}.Release|Mixed Platforms.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/BinlogTool/BinlogTool.csproj b/src/BinlogTool/BinlogTool.csproj index 4946edf75..172b2a1d9 100644 --- a/src/BinlogTool/BinlogTool.csproj +++ b/src/BinlogTool/BinlogTool.csproj @@ -2,7 +2,7 @@ Exe - 1.0.6 + 1.0.7 net7.0 latest embedded @@ -23,11 +23,12 @@ + - + diff --git a/src/BinlogTool/Program.cs b/src/BinlogTool/Program.cs index 3302e9b7f..8ad53ca3b 100644 --- a/src/BinlogTool/Program.cs +++ b/src/BinlogTool/Program.cs @@ -1,8 +1,9 @@ using System; +using System.Collections.Generic; using System.IO; using System.Linq; using Microsoft.Build.Logging.StructuredLogger; -using StructuredLogger.BinaryLogger.Postprocessing; +using StructuredLogger.Utils; namespace BinlogTool { @@ -18,7 +19,7 @@ binlogtool savefiles input.binlog output_path binlogtool reconstruct input.binlog output_path binlogtool savestrings input.binlog output.txt binlogtool search *.binlog search string - binlogtool redact input.binlog list of secrets to redact"); + binlogtool redact --input:path --recurse --in-place -p:list -p:of -p:secrets -p:to -p:redact"); return; } @@ -75,14 +76,53 @@ binlogtool search *.binlog search string if (firstArg == "redact") { - if (args.Length < 3) + List redactTokens = new List(); + List inputPaths = new List(); + bool recurse = false; + bool inPlace = false; + + foreach (var arg in args.Skip(1)) { - Console.Error.WriteLine("binlogtool redact input.binlog list of secrets to redact"); - return; + if (arg.StartsWith("--input:", StringComparison.OrdinalIgnoreCase)) + { + var input = arg.Substring("--input:".Length); + if (string.IsNullOrEmpty(input)) + { + Console.Error.WriteLine("Invalid input path"); + return; + } + + inputPaths.Add(input); + } + else if (arg.StartsWith("-p:", StringComparison.OrdinalIgnoreCase)) + { + var redactToken = arg.Substring("-p:".Length); + if (string.IsNullOrEmpty(redactToken)) + { + Console.Error.WriteLine("Invalid redact token"); + return; + } + + redactTokens.Add(redactToken); + } + else if (arg.Equals("--recurse", StringComparison.OrdinalIgnoreCase)) + { + recurse = true; + } + else if (arg.Equals("--in-place", StringComparison.OrdinalIgnoreCase)) + { + inPlace = true; + } + else + { + Console.Error.WriteLine($"Invalid argument: {arg}"); + Console.Error.WriteLine("binlogtool redact --input:path --recurse --in-place -p:list -p:of -p:secrets -p:to -p:redact"); + Console.Error.WriteLine("All arguments are optional (missing input assumes current working directory. Missing tokens lead only to autoredactions. Missing --in-place will create new logs with suffix.)"); + return; + } } - var binlog = args[1]; - BinlogRedactor.RedactSecrets(binlog, args.Skip(2).ToArray()); + Redact.Run(inputPaths, redactTokens, inPlace, recurse); return; } diff --git a/src/BinlogTool/Redact.cs b/src/BinlogTool/Redact.cs new file mode 100644 index 000000000..c1e5fbe6c --- /dev/null +++ b/src/BinlogTool/Redact.cs @@ -0,0 +1,79 @@ +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using StructuredLogger.Utils; + +namespace BinlogTool +{ + internal static class Redact + { + public static void Run(List inputs, List tokens, bool inPlace, bool recurse) + { + if (!inputs.Any()) + { + // Default - current directory + inputs.Add(string.Empty); + } + + var inputBinlogs = inputs.SelectMany(input => Searcher.FindBinlogs(input, recurse)).ToList(); + + if (!inputBinlogs.Any()) + { + Log.WriteError("No binlogs found."); + return; + } + + if (inputBinlogs.Count > 1) + { + Log.WriteLine( + $"Found {inputBinlogs.Count} binlog files. Will redact secrets in all. (found files: {(string.Join(',', inputBinlogs))})"); + } + + BinlogRedactorOptions options = new BinlogRedactorOptions(string.Empty) + { + TokensToRedact = tokens.ToArray(), + IdentifyReplacemenets = true, + AutodetectCommonPatterns = true, + AutodetectUsername = true, + ProcessEmbeddedFiles = true, + }; + + var overallStopwatch = Stopwatch.StartNew(); + + foreach (var inputBinlog in inputBinlogs) + { + options.InputPath = inputBinlog; + options.OutputFileName = GetOutputFileName(inputBinlog); + + Log.WriteLine($"Redacting binlog {inputBinlog} to {options.OutputFileName} ({GetFileSizeInKB(inputBinlog)} KB)"); + + var stopwatch = Stopwatch.StartNew(); + + BinlogRedactor.RedactSecrets(options, progress: null); + + stopwatch.Stop(); + Log.WriteLine($"Redacting done. Duration: {stopwatch.Elapsed}"); + } + + overallStopwatch.Stop(); + if(inputBinlogs.Count > 1) + { + Log.WriteLine($"Redacting all binlogs done. Duration: {overallStopwatch.Elapsed}"); + } + + string GetOutputFileName(string inputFileName) + { + if (inPlace) + { + return inputFileName; + } + + return Path.ChangeExtension(inputFileName, ".redacted.binlog"); + } + + long GetFileSizeInKB(string path) + => new FileInfo(path).Length / 1024; + } + } +} diff --git a/src/BinlogTool/Search.cs b/src/BinlogTool/Search.cs index efa124a72..50d39808f 100644 --- a/src/BinlogTool/Search.cs +++ b/src/BinlogTool/Search.cs @@ -1,4 +1,5 @@ -using System; +using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading; @@ -7,23 +8,39 @@ namespace BinlogTool { - public class Searcher + public static class Searcher { public static void Search(string binlogs, string search) { - if (string.IsNullOrEmpty(binlogs)) + var files = FindBinlogs(binlogs, recurse: true).ToList(); + Search(files, search); + } + + public static IEnumerable FindBinlogs(string inputPath, bool recurse) + { + if (string.IsNullOrEmpty(inputPath)) { - binlogs = "*.binlog"; + inputPath = "*.binlog"; } - binlogs = binlogs.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar); + inputPath = inputPath.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar); + + if (File.Exists(inputPath)) + { + return new[] { inputPath }; + } + + if (Directory.Exists(inputPath)) + { + inputPath = Path.Combine(inputPath, "*.binlog"); + } string fileName; string directory; - if (binlogs.Contains(Path.DirectorySeparatorChar)) + if (inputPath.Contains(Path.DirectorySeparatorChar)) { - fileName = Path.GetFileName(binlogs); - directory = Path.GetDirectoryName(binlogs); + fileName = Path.GetFileName(inputPath); + directory = Path.GetDirectoryName(inputPath); if (!Path.IsPathRooted(directory)) { directory = Path.GetFullPath(directory); @@ -31,15 +48,15 @@ public static void Search(string binlogs, string search) } else { - fileName = binlogs; + fileName = inputPath; directory = Environment.CurrentDirectory; } - var files = Directory.GetFiles(directory, fileName, SearchOption.AllDirectories); - Search(files, search); + return Directory.EnumerateFiles(directory, fileName, + new EnumerationOptions() { IgnoreInaccessible = true, RecurseSubdirectories = recurse, }); } - private static void Search(string[] files, string search) + private static void Search(IEnumerable files, string search) { foreach (var file in files) { @@ -88,4 +105,4 @@ public static void PrintTree(BaseNode node, int indent = 0) } } } -} \ No newline at end of file +} diff --git a/src/StructuredLogViewer/MSBuildStructuredLogViewer.nuspec b/src/StructuredLogViewer/MSBuildStructuredLogViewer.nuspec index b47e2368b..cf8b394d0 100644 --- a/src/StructuredLogViewer/MSBuildStructuredLogViewer.nuspec +++ b/src/StructuredLogViewer/MSBuildStructuredLogViewer.nuspec @@ -32,6 +32,7 @@ + @@ -42,6 +43,7 @@ + diff --git a/src/StructuredLogViewer/MainWindow.xaml.cs b/src/StructuredLogViewer/MainWindow.xaml.cs index 00c76e8be..440f5d9b8 100644 --- a/src/StructuredLogViewer/MainWindow.xaml.cs +++ b/src/StructuredLogViewer/MainWindow.xaml.cs @@ -14,7 +14,7 @@ using Microsoft.Build.Logging.StructuredLogger; using Microsoft.Win32; using Squirrel; -using StructuredLogger.BinaryLogger.Postprocessing; +using StructuredLogger.Utils; using StructuredLogViewer.Controls; namespace StructuredLogViewer @@ -358,7 +358,7 @@ private void UpdateRecentItemsMenu(WelcomeScreen welcomeScreen = null) private void SetContent(object content) { // We save build control to allow to bring back states to new one - if (mainContent.Content is BuildControl current) + if (mainContent.Content is BuildControl current) { lastSearchText = current.searchLogControl.SearchText; } @@ -663,80 +663,81 @@ private void Reload() OpenLogFile(logFilePath); } - private async void RedactSecrets() + private async System.Threading.Tasks.Task RedactSecrets() { RedactInputControl redactInputControl = new RedactInputControl(GetSaveAsDestination); - if (redactInputControl.ShowDialog() == true) - { - List stringsToRedact = - new(redactInputControl.SecretsBlock? - .Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries) - .Select(s => s.Trim()) - .Where(s => !string.IsNullOrWhiteSpace(s)) - ?? new string[] { }); - - if ( - !stringsToRedact.Any() && - !redactInputControl.RedactUsername && - !redactInputControl.RedactCommonCredentials) - { - MessageBox.Show("No secrets to redact - no action will be performed"); - return; - } + if (redactInputControl.ShowDialog() != true) + { + return; + } + + List stringsToRedact = + new(redactInputControl.SecretsBlock? + .Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries) + .Select(s => s.Trim()) + .Where(s => !string.IsNullOrWhiteSpace(s)) + ?? new string[] { }); - var progress = new BuildProgress(); - progress.Progress.Updated += update => - { - Dispatcher.InvokeAsync(() => - { - progress.Value = update.Ratio; - }, DispatcherPriority.Background); - }; - progress.ProgressText = "Performing the log redaction ..."; - SetContent(progress); + if ( + !stringsToRedact.Any() && + !redactInputControl.RedactUsername && + !redactInputControl.RedactCommonCredentials) + { + MessageBox.Show("No secrets to redact - no action will be performed"); + return; + } - string error = await System.Threading.Tasks.Task.Run(() => + var progress = new BuildProgress(); + progress.Progress.Updated += update => + { + Dispatcher.InvokeAsync(() => { - try - { - BinlogRedactorOptions redactorOptions = new BinlogRedactorOptions(logFilePath) - { - OutputFileName = redactInputControl.DestinationFile, - ProcessEmbeddedFiles = redactInputControl.RedactEmbeddedFiles, - AutodetectUsername = redactInputControl.RedactUsername, - AutodetectCommonPatterns = redactInputControl.RedactCommonCredentials, - IdentifyReplacemenets = redactInputControl.DistinguishSecretsReplacements, - TokensToRedact = stringsToRedact.ToArray(), - }; - - BinlogRedactor.RedactSecrets( - redactorOptions, - progress.Progress); - } - catch(Exception e) - { - return e.ToString(); - } - - return null; - }); + progress.Value = update.Ratio; + }, DispatcherPriority.Background); + }; + progress.ProgressText = "Performing the log redaction ..."; + SetContent(progress); - if (!string.IsNullOrEmpty(error)) - { - MessageBox.Show($"Redaction failed:{Environment.NewLine}{error}"); - SetContent(currentBuild); - } - else if (string.IsNullOrEmpty(redactInputControl.DestinationFile)) + string error = await System.Threading.Tasks.Task.Run(() => + { + try { - // Reload - OpenLogFile(logFilePath); + BinlogRedactorOptions redactorOptions = new BinlogRedactorOptions(logFilePath) + { + OutputFileName = redactInputControl.DestinationFile, + ProcessEmbeddedFiles = redactInputControl.RedactEmbeddedFiles, + AutodetectUsername = redactInputControl.RedactUsername, + AutodetectCommonPatterns = redactInputControl.RedactCommonCredentials, + IdentifyReplacemenets = redactInputControl.DistinguishSecretsReplacements, + TokensToRedact = stringsToRedact.ToArray(), + }; + + BinlogRedactor.RedactSecrets( + redactorOptions, + progress.Progress); } - else + catch(Exception e) { - SetContent(currentBuild); - AnnounceFileSaved(redactInputControl.DestinationFile); + return e.ToString(); } - + + return null; + }); + + if (!string.IsNullOrEmpty(error)) + { + MessageBox.Show($"Redaction failed:{Environment.NewLine}{error}"); + SetContent(currentBuild); + } + else if (string.IsNullOrEmpty(redactInputControl.DestinationFile)) + { + // Reload + OpenLogFile(logFilePath); + } + else + { + SetContent(currentBuild); + AnnounceFileSaved(redactInputControl.DestinationFile); } } @@ -904,7 +905,7 @@ private void SaveAs_Click(object sender, RoutedEventArgs e) private void RedactSecrets_Click(object sender, RoutedEventArgs e) { - RedactSecrets(); + RedactSecrets().Ignore(); } private void HelpLink_Click(object sender, RoutedEventArgs e) diff --git a/src/StructuredLogViewer/StructuredLogViewer.csproj b/src/StructuredLogViewer/StructuredLogViewer.csproj index 41f6df8e2..771c7d319 100644 --- a/src/StructuredLogViewer/StructuredLogViewer.csproj +++ b/src/StructuredLogViewer/StructuredLogViewer.csproj @@ -32,6 +32,7 @@ + diff --git a/src/StructuredLogViewer/TaskExtensions.cs b/src/StructuredLogViewer/TaskExtensions.cs new file mode 100644 index 000000000..89b7ab523 --- /dev/null +++ b/src/StructuredLogViewer/TaskExtensions.cs @@ -0,0 +1,14 @@ +using System.Threading.Tasks; + +namespace StructuredLogViewer +{ + internal static class TaskExtensions + { + /// + /// Consumes a task and doesn't do anything with it. Useful for fire-and-forget calls to async methods within sync methods. + /// + /// The task whose result is to be ignored. + public static void Ignore(this Task task) + { /* this is it */ } + } +} diff --git a/src/StructuredLogger.Tests/BinaryLoggerTests.cs b/src/StructuredLogger.Tests/BinaryLoggerTests.cs index a58690d17..f6579709f 100644 --- a/src/StructuredLogger.Tests/BinaryLoggerTests.cs +++ b/src/StructuredLogger.Tests/BinaryLoggerTests.cs @@ -185,14 +185,14 @@ private static void AssertBinlogsHaveEqualContent(string firstPath, string secon reader2.ArchiveFileEncountered += arg => AddArchiveFile(embedFiles2, arg); - + // Pull events from both logs simultaneously and compare them int i = 0; - while (reader1.Read() is { } ev1) + while (reader1.Read() is { } logRecord1) { i++; - var ev2 = reader2.Read(); + var logRecord2 = reader2.Read(); - ev1.Should().BeEquivalentTo(ev2, + logRecord1.Should().BeEquivalentTo(logRecord2, $"Binlogs ({firstPath} and {secondPath}) should be equal at event {i}"); } // Read the second reader - to confirm there are no more events diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/BinlogRedactor.cs b/src/StructuredLogger.Utils/BinlogRedactor.cs similarity index 94% rename from src/StructuredLogger/BinaryLogger/Postprocessing/BinlogRedactor.cs rename to src/StructuredLogger.Utils/BinlogRedactor.cs index 4b30ee89b..eeb1a5c87 100644 --- a/src/StructuredLogger/BinaryLogger/Postprocessing/BinlogRedactor.cs +++ b/src/StructuredLogger.Utils/BinlogRedactor.cs @@ -1,14 +1,11 @@ using System; -using System.Linq; -using System.Collections.Generic; -using System.Text; -using Microsoft.Build.Logging; -using Microsoft.Build.Logging.StructuredLogger; using System.IO; using System.Threading; +using Microsoft.Build.Logging; +using Microsoft.Build.Logging.StructuredLogger; using Microsoft.Build.SensitiveDataDetector; -namespace StructuredLogger.BinaryLogger.Postprocessing +namespace StructuredLogger.Utils { public sealed class BinlogRedactorOptions { @@ -18,7 +15,7 @@ public BinlogRedactorOptions(string inputPath) } public string[]? TokensToRedact { get; set; } - public string InputPath { get; } + public string InputPath { get; set; } public string? OutputFileName { get; set; } public bool ProcessEmbeddedFiles { get; set; } = true; public bool IdentifyReplacemenets { get; set; } = true; diff --git a/src/StructuredLogger.Utils/StructuredLogger.Utils.csproj b/src/StructuredLogger.Utils/StructuredLogger.Utils.csproj new file mode 100644 index 000000000..40cc85604 --- /dev/null +++ b/src/StructuredLogger.Utils/StructuredLogger.Utils.csproj @@ -0,0 +1,26 @@ + + + netstandard2.0 + true + true + true + + + + + + $(DefineConstants);NETCORE + + + + + + + + + + True + ..\StructuredLogger\key.snk + False + + diff --git a/src/StructuredLogger/BinaryLogger/BinaryLogReplayEventSource.cs b/src/StructuredLogger/BinaryLogger/BinaryLogReplayEventSource.cs index dfb725a84..6bffde7ac 100644 --- a/src/StructuredLogger/BinaryLogger/BinaryLogReplayEventSource.cs +++ b/src/StructuredLogger/BinaryLogger/BinaryLogReplayEventSource.cs @@ -25,7 +25,7 @@ internal interface IBinaryLogReplaySource : /// by implementing IEventSource and raising corresponding events. /// /// The class is public so that we can call it from MSBuild.exe when replaying a log file. - public sealed class BinaryLogReplayEventSource : EventArgsDispatcher, IBinaryLogReplaySource + internal sealed class BinaryLogReplayEventSource : EventArgsDispatcher, IBinaryLogReplaySource { /// /// Read the provided binary log file and raise corresponding events for each BuildEventArgs diff --git a/src/StructuredLogger/BinaryLogger/BuildEventArgsReader.cs b/src/StructuredLogger/BinaryLogger/BuildEventArgsReader.cs index b91efce5c..8999bee58 100644 --- a/src/StructuredLogger/BinaryLogger/BuildEventArgsReader.cs +++ b/src/StructuredLogger/BinaryLogger/BuildEventArgsReader.cs @@ -237,7 +237,7 @@ private void ReadBlob(BinaryLogRecordKind kind) if (EmbeddedContentRead != null) { projectImportsCollector = - new ProjectImportsCollector(Path.GetRandomFileName(), false, runOnBackground: false); + new ProjectImportsCollector(PathUtils.TempPath, false, runOnBackground: false); } // We are intentionally not grace handling corrupt embedded stream @@ -363,11 +363,6 @@ private Stream SliceOfEmdeddedContent(bool canHaveCorruptedSize) } } - private void SkipBytes(int count) - { - binaryReader.BaseStream.SkipBytes(count, true); - } - private readonly List<(int name, int value)> nameValues = new List<(int name, int value)>(4096); private void ReadNameValueList() diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs index 0d7421ab2..5e9782db6 100644 --- a/src/StructuredLogger/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs +++ b/src/StructuredLogger/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs @@ -2,9 +2,8 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; -using Microsoft.Build.Logging.StructuredLogger; -namespace Microsoft.Build.Logging; +namespace Microsoft.Build.Logging.StructuredLogger; /// /// Event arguments for event. @@ -14,6 +13,5 @@ public sealed class ArchiveFileEventArgs : EventArgs public ArchiveFileEventArgs(ArchiveFile archiveFile) => ArchiveFile = archiveFile; - public ArchiveFile ArchiveFile { get; set; } } diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/ArchiveFileEventArgsExtensions.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/ArchiveFileEventArgsExtensions.cs index 9580f7b20..cee62f61e 100644 --- a/src/StructuredLogger/BinaryLogger/Postprocessing/ArchiveFileEventArgsExtensions.cs +++ b/src/StructuredLogger/BinaryLogger/Postprocessing/ArchiveFileEventArgsExtensions.cs @@ -2,9 +2,8 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; -using Microsoft.Build.Logging.StructuredLogger; -namespace Microsoft.Build.Logging; +namespace Microsoft.Build.Logging.StructuredLogger; public static class ArchiveFileEventArgsExtensions { diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs index 08b927bd6..0ba1f64a5 100644 --- a/src/StructuredLogger/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs +++ b/src/StructuredLogger/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs @@ -3,9 +3,8 @@ using System; using System.IO; -using Microsoft.Build.Logging.StructuredLogger; -namespace Microsoft.Build.Logging +namespace Microsoft.Build.Logging.StructuredLogger { internal sealed class EmbeddedContentEventArgs : EventArgs { diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs index 1ad97a6ed..7f4053197 100644 --- a/src/StructuredLogger/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs +++ b/src/StructuredLogger/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs @@ -1,7 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -namespace Microsoft.Build.Logging +namespace Microsoft.Build.Logging.StructuredLogger { /// /// An interface for notifications from BuildEventArgsReader diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/IBuildEventStringsReader.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/IBuildEventStringsReader.cs index 83d60f4fe..6378eff07 100644 --- a/src/StructuredLogger/BinaryLogger/Postprocessing/IBuildEventStringsReader.cs +++ b/src/StructuredLogger/BinaryLogger/Postprocessing/IBuildEventStringsReader.cs @@ -3,7 +3,7 @@ using System; -namespace Microsoft.Build.Logging +namespace Microsoft.Build.Logging.StructuredLogger { /// /// An interface for notifications about reading strings from the binary log. diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/IBuildFileReader.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/IBuildFileReader.cs index 8015083e5..5735c2d6d 100644 --- a/src/StructuredLogger/BinaryLogger/Postprocessing/IBuildFileReader.cs +++ b/src/StructuredLogger/BinaryLogger/Postprocessing/IBuildFileReader.cs @@ -3,7 +3,7 @@ using System; -namespace Microsoft.Build.Logging; +namespace Microsoft.Build.Logging.StructuredLogger; public interface IBuildFileReader { diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/IEmbeddedContentSource.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/IEmbeddedContentSource.cs index e867c5577..cb5895f7e 100644 --- a/src/StructuredLogger/BinaryLogger/Postprocessing/IEmbeddedContentSource.cs +++ b/src/StructuredLogger/BinaryLogger/Postprocessing/IEmbeddedContentSource.cs @@ -3,7 +3,7 @@ using System; -namespace Microsoft.Build.Logging +namespace Microsoft.Build.Logging.StructuredLogger { internal interface IEmbeddedContentSource { diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/StreamChunkOverreadException.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/StreamChunkOverreadException.cs index 4d4fe21b5..1ba262441 100644 --- a/src/StructuredLogger/BinaryLogger/Postprocessing/StreamChunkOverreadException.cs +++ b/src/StructuredLogger/BinaryLogger/Postprocessing/StreamChunkOverreadException.cs @@ -3,7 +3,7 @@ using System; -namespace Microsoft.Build.Logging +namespace Microsoft.Build.Logging.StructuredLogger { public class StreamChunkOverReadException : Exception { diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/StringReadEventArgs.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/StringReadEventArgs.cs index 977006015..7eeadb23e 100644 --- a/src/StructuredLogger/BinaryLogger/Postprocessing/StringReadEventArgs.cs +++ b/src/StructuredLogger/BinaryLogger/Postprocessing/StringReadEventArgs.cs @@ -3,7 +3,7 @@ using System; -namespace Microsoft.Build.Logging +namespace Microsoft.Build.Logging.StructuredLogger { /// /// An event args for callback. diff --git a/src/StructuredLogger/BinaryLogger/ProjectImportsCollector.cs b/src/StructuredLogger/BinaryLogger/ProjectImportsCollector.cs index e85d6984f..3f7183a14 100644 --- a/src/StructuredLogger/BinaryLogger/ProjectImportsCollector.cs +++ b/src/StructuredLogger/BinaryLogger/ProjectImportsCollector.cs @@ -5,8 +5,8 @@ using System.IO.Compression; using System.Text; using System.Threading.Tasks; -using Microsoft.Build.Shared; -using StructuredLogger; +using Microsoft.Build.Logging.StructuredLogger; +using Task = System.Threading.Tasks.Task; namespace Microsoft.Build.Logging { diff --git a/src/StructuredLogger/ErrorReporting.cs b/src/StructuredLogger/ErrorReporting.cs index 9ad55da18..ce6d2a08f 100644 --- a/src/StructuredLogger/ErrorReporting.cs +++ b/src/StructuredLogger/ErrorReporting.cs @@ -1,6 +1,5 @@ using System; using System.IO; -using StructuredLogger; namespace Microsoft.Build.Logging.StructuredLogger { diff --git a/src/StructuredLogger/PathUtils.cs b/src/StructuredLogger/PathUtils.cs index de25a3421..ce94c55ab 100644 --- a/src/StructuredLogger/PathUtils.cs +++ b/src/StructuredLogger/PathUtils.cs @@ -3,7 +3,7 @@ using System.IO; using System.Text; -namespace StructuredLogger +namespace Microsoft.Build.Logging.StructuredLogger { public static class PathUtils { diff --git a/src/StructuredLogger/StructuredLogger.cs b/src/StructuredLogger/StructuredLogger.cs index dd64f55ae..5b5e39370 100644 --- a/src/StructuredLogger/StructuredLogger.cs +++ b/src/StructuredLogger/StructuredLogger.cs @@ -106,8 +106,6 @@ public override void Shutdown() if (projectImportsCollector != null) { projectImportsCollector.Close(); - - projectImportsCollector.ProcessResult( streamToEmbed => construction.Build.SourceFilesArchive = streamToEmbed.ReadToEnd(), _ => {}); diff --git a/src/StructuredLogger/StructuredLogger.csproj b/src/StructuredLogger/StructuredLogger.csproj index 3b1c87667..b53272eda 100644 --- a/src/StructuredLogger/StructuredLogger.csproj +++ b/src/StructuredLogger/StructuredLogger.csproj @@ -28,8 +28,7 @@ - - + @@ -63,6 +62,7 @@ False +