diff --git a/MSBuildStructuredLog.sln b/MSBuildStructuredLog.sln index c77296dd1..30bd7127b 100644 --- a/MSBuildStructuredLog.sln +++ b/MSBuildStructuredLog.sln @@ -28,6 +28,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ResourcesGenerator", "src\R EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BinlogTool", "src\BinlogTool\BinlogTool.csproj", "{35F44EA6-7259-43CC-8C17-E058F3EB86D3}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "StructuredLogger.Utils", "src\StructuredLogger.Utils\StructuredLogger.Utils.csproj", "{AC634B46-D57C-44C5-BF56-480843182F21}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Mixed Platforms = Debug|Mixed Platforms @@ -66,6 +68,10 @@ Global {35F44EA6-7259-43CC-8C17-E058F3EB86D3}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU {35F44EA6-7259-43CC-8C17-E058F3EB86D3}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU {35F44EA6-7259-43CC-8C17-E058F3EB86D3}.Release|Mixed Platforms.Build.0 = Release|Any CPU + {AC634B46-D57C-44C5-BF56-480843182F21}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {AC634B46-D57C-44C5-BF56-480843182F21}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {AC634B46-D57C-44C5-BF56-480843182F21}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {AC634B46-D57C-44C5-BF56-480843182F21}.Release|Mixed Platforms.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/BinlogTool/BinlogTool.csproj b/src/BinlogTool/BinlogTool.csproj index 4946edf75..172b2a1d9 100644 --- a/src/BinlogTool/BinlogTool.csproj +++ b/src/BinlogTool/BinlogTool.csproj @@ -2,7 +2,7 @@ Exe - 1.0.6 + 1.0.7 net7.0 latest embedded @@ -23,11 +23,12 @@ + - + diff --git a/src/BinlogTool/Program.cs b/src/BinlogTool/Program.cs index cbef55236..8ad53ca3b 100644 --- a/src/BinlogTool/Program.cs +++ b/src/BinlogTool/Program.cs @@ -1,7 +1,9 @@ using System; +using System.Collections.Generic; using System.IO; using System.Linq; using Microsoft.Build.Logging.StructuredLogger; +using StructuredLogger.Utils; namespace BinlogTool { @@ -16,7 +18,8 @@ binlogtool listtools input.binlog binlogtool savefiles input.binlog output_path binlogtool reconstruct input.binlog output_path binlogtool savestrings input.binlog output.txt - binlogtool search *.binlog search string"); + binlogtool search *.binlog search string + binlogtool redact --input:path --recurse --in-place -p:list -p:of -p:secrets -p:to -p:redact"); return; } @@ -71,6 +74,58 @@ binlogtool savestrings input.binlog output.txt return; } + if (firstArg == "redact") + { + List redactTokens = new List(); + List inputPaths = new List(); + bool recurse = false; + bool inPlace = false; + + foreach (var arg in args.Skip(1)) + { + if (arg.StartsWith("--input:", StringComparison.OrdinalIgnoreCase)) + { + var input = arg.Substring("--input:".Length); + if (string.IsNullOrEmpty(input)) + { + Console.Error.WriteLine("Invalid input path"); + return; + } + + inputPaths.Add(input); + } + else if (arg.StartsWith("-p:", StringComparison.OrdinalIgnoreCase)) + { + var redactToken = arg.Substring("-p:".Length); + if (string.IsNullOrEmpty(redactToken)) + { + Console.Error.WriteLine("Invalid redact token"); + return; + } + + redactTokens.Add(redactToken); + } + else if (arg.Equals("--recurse", StringComparison.OrdinalIgnoreCase)) + { + recurse = true; + } + else if (arg.Equals("--in-place", StringComparison.OrdinalIgnoreCase)) + { + inPlace = true; + } + else + { + Console.Error.WriteLine($"Invalid argument: {arg}"); + Console.Error.WriteLine("binlogtool redact --input:path --recurse --in-place -p:list -p:of -p:secrets -p:to -p:redact"); + Console.Error.WriteLine("All arguments are optional (missing input assumes current working directory. Missing tokens lead only to autoredactions. Missing --in-place will create new logs with suffix.)"); + return; + } + } + + Redact.Run(inputPaths, redactTokens, inPlace, recurse); + return; + } + Console.Error.WriteLine("Invalid arguments"); } diff --git a/src/BinlogTool/Redact.cs b/src/BinlogTool/Redact.cs new file mode 100644 index 000000000..c1e5fbe6c --- /dev/null +++ b/src/BinlogTool/Redact.cs @@ -0,0 +1,79 @@ +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using StructuredLogger.Utils; + +namespace BinlogTool +{ + internal static class Redact + { + public static void Run(List inputs, List tokens, bool inPlace, bool recurse) + { + if (!inputs.Any()) + { + // Default - current directory + inputs.Add(string.Empty); + } + + var inputBinlogs = inputs.SelectMany(input => Searcher.FindBinlogs(input, recurse)).ToList(); + + if (!inputBinlogs.Any()) + { + Log.WriteError("No binlogs found."); + return; + } + + if (inputBinlogs.Count > 1) + { + Log.WriteLine( + $"Found {inputBinlogs.Count} binlog files. Will redact secrets in all. (found files: {(string.Join(',', inputBinlogs))})"); + } + + BinlogRedactorOptions options = new BinlogRedactorOptions(string.Empty) + { + TokensToRedact = tokens.ToArray(), + IdentifyReplacemenets = true, + AutodetectCommonPatterns = true, + AutodetectUsername = true, + ProcessEmbeddedFiles = true, + }; + + var overallStopwatch = Stopwatch.StartNew(); + + foreach (var inputBinlog in inputBinlogs) + { + options.InputPath = inputBinlog; + options.OutputFileName = GetOutputFileName(inputBinlog); + + Log.WriteLine($"Redacting binlog {inputBinlog} to {options.OutputFileName} ({GetFileSizeInKB(inputBinlog)} KB)"); + + var stopwatch = Stopwatch.StartNew(); + + BinlogRedactor.RedactSecrets(options, progress: null); + + stopwatch.Stop(); + Log.WriteLine($"Redacting done. Duration: {stopwatch.Elapsed}"); + } + + overallStopwatch.Stop(); + if(inputBinlogs.Count > 1) + { + Log.WriteLine($"Redacting all binlogs done. Duration: {overallStopwatch.Elapsed}"); + } + + string GetOutputFileName(string inputFileName) + { + if (inPlace) + { + return inputFileName; + } + + return Path.ChangeExtension(inputFileName, ".redacted.binlog"); + } + + long GetFileSizeInKB(string path) + => new FileInfo(path).Length / 1024; + } + } +} diff --git a/src/BinlogTool/Search.cs b/src/BinlogTool/Search.cs index efa124a72..50d39808f 100644 --- a/src/BinlogTool/Search.cs +++ b/src/BinlogTool/Search.cs @@ -1,4 +1,5 @@ -using System; +using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading; @@ -7,23 +8,39 @@ namespace BinlogTool { - public class Searcher + public static class Searcher { public static void Search(string binlogs, string search) { - if (string.IsNullOrEmpty(binlogs)) + var files = FindBinlogs(binlogs, recurse: true).ToList(); + Search(files, search); + } + + public static IEnumerable FindBinlogs(string inputPath, bool recurse) + { + if (string.IsNullOrEmpty(inputPath)) { - binlogs = "*.binlog"; + inputPath = "*.binlog"; } - binlogs = binlogs.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar); + inputPath = inputPath.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar); + + if (File.Exists(inputPath)) + { + return new[] { inputPath }; + } + + if (Directory.Exists(inputPath)) + { + inputPath = Path.Combine(inputPath, "*.binlog"); + } string fileName; string directory; - if (binlogs.Contains(Path.DirectorySeparatorChar)) + if (inputPath.Contains(Path.DirectorySeparatorChar)) { - fileName = Path.GetFileName(binlogs); - directory = Path.GetDirectoryName(binlogs); + fileName = Path.GetFileName(inputPath); + directory = Path.GetDirectoryName(inputPath); if (!Path.IsPathRooted(directory)) { directory = Path.GetFullPath(directory); @@ -31,15 +48,15 @@ public static void Search(string binlogs, string search) } else { - fileName = binlogs; + fileName = inputPath; directory = Environment.CurrentDirectory; } - var files = Directory.GetFiles(directory, fileName, SearchOption.AllDirectories); - Search(files, search); + return Directory.EnumerateFiles(directory, fileName, + new EnumerationOptions() { IgnoreInaccessible = true, RecurseSubdirectories = recurse, }); } - private static void Search(string[] files, string search) + private static void Search(IEnumerable files, string search) { foreach (var file in files) { @@ -88,4 +105,4 @@ public static void PrintTree(BaseNode node, int indent = 0) } } } -} \ No newline at end of file +} diff --git a/src/StructuredLogViewer/Controls/RedactInputControl.xaml b/src/StructuredLogViewer/Controls/RedactInputControl.xaml new file mode 100644 index 000000000..32f4415ae --- /dev/null +++ b/src/StructuredLogViewer/Controls/RedactInputControl.xaml @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/StructuredLogViewer/Controls/RedactInputControl.xaml.cs b/src/StructuredLogViewer/Controls/RedactInputControl.xaml.cs new file mode 100644 index 000000000..74ee75a9c --- /dev/null +++ b/src/StructuredLogViewer/Controls/RedactInputControl.xaml.cs @@ -0,0 +1,92 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Windows; +using System.Windows.Controls; +using System.Windows.Data; +using System.Windows.Documents; +using System.Windows.Input; +using System.Windows.Media; +using System.Windows.Media.Imaging; +using System.Windows.Shapes; + +namespace StructuredLogViewer.Controls +{ + /// + /// Interaction logic for RedactInputControl.xaml + /// + public partial class RedactInputControl : Window + { + private readonly Func _getSaveAsDestination; + public string DestinationFile { get; private set; } + public bool RedactCommonCredentials { get; private set; } = true; + public bool RedactUsername { get; set; } = true; + public bool RedactEmbeddedFiles { get; set; } = true; + public bool DistinguishSecretsReplacements { get; set; } = true; + public string SecretsBlock + { + get { return ChckbxCustomSecrets.IsChecked == true ? TxtSecrets.Text : null; } + } + + public RedactInputControl(Func getSaveAsDestination) + { + _getSaveAsDestination = getSaveAsDestination; + InitializeComponent(); + } + + private void btnDialogOk_Click(object sender, RoutedEventArgs e) + { + this.DialogResult = true; + } + + private void btnSaveAs_Click(object sender, RoutedEventArgs e) + { + var destination = _getSaveAsDestination(); + if (destination != null) + { + this.DestinationFile = destination; + this.DialogResult = true; + } + } + + private void Window_ContentRendered(object sender, EventArgs e) + { + ChckbxUsername.IsChecked = RedactUsername; + ChckbxCommonCredentials.IsChecked = RedactCommonCredentials; + ChckbxCustomSecrets.IsChecked = false; + TxtSecrets.IsEnabled = false; + ChckbxEmbeddedFiles.IsChecked = RedactEmbeddedFiles; + ChckbxDistinguishReplacements.IsChecked = DistinguishSecretsReplacements; + + TxtSecrets.SelectAll(); + TxtSecrets.Focus(); + } + + private void ChckbxCustomSecrets_OnChanged(object sender, RoutedEventArgs e) + { + TxtSecrets.IsEnabled = ChckbxCustomSecrets.IsChecked == true; + } + + private void ChckbxUsername_OnChanged(object sender, RoutedEventArgs e) + { + RedactUsername = ChckbxUsername.IsChecked == true; + } + + private void ChckbxCommonCredentials_OnChanged(object sender, RoutedEventArgs e) + { + RedactCommonCredentials = ChckbxCommonCredentials.IsChecked == true; + } + + private void ChckbxEmbeddedFiles_OnChanged(object sender, RoutedEventArgs e) + { + RedactEmbeddedFiles = ChckbxEmbeddedFiles.IsChecked == true; + } + + private void ChckbxDistinguishReplacements_OnChanged(object sender, RoutedEventArgs e) + { + DistinguishSecretsReplacements = ChckbxDistinguishReplacements.IsChecked == true; + } + } +} diff --git a/src/StructuredLogViewer/MSBuildStructuredLogViewer.nuspec b/src/StructuredLogViewer/MSBuildStructuredLogViewer.nuspec index b47e2368b..cf8b394d0 100644 --- a/src/StructuredLogViewer/MSBuildStructuredLogViewer.nuspec +++ b/src/StructuredLogViewer/MSBuildStructuredLogViewer.nuspec @@ -32,6 +32,7 @@ + @@ -42,6 +43,7 @@ + diff --git a/src/StructuredLogViewer/MainWindow.xaml b/src/StructuredLogViewer/MainWindow.xaml index 41715ffde..e7ef22b09 100644 --- a/src/StructuredLogViewer/MainWindow.xaml +++ b/src/StructuredLogViewer/MainWindow.xaml @@ -1,4 +1,4 @@ - - + + diff --git a/src/StructuredLogViewer/MainWindow.xaml.cs b/src/StructuredLogViewer/MainWindow.xaml.cs index 83ce8eb09..440f5d9b8 100644 --- a/src/StructuredLogViewer/MainWindow.xaml.cs +++ b/src/StructuredLogViewer/MainWindow.xaml.cs @@ -5,6 +5,7 @@ using System.IO; using System.Linq; using System.Net.Http; +using System.Threading; using System.Windows; using System.Windows.Controls; using System.Windows.Input; @@ -13,6 +14,7 @@ using Microsoft.Build.Logging.StructuredLogger; using Microsoft.Win32; using Squirrel; +using StructuredLogger.Utils; using StructuredLogViewer.Controls; namespace StructuredLogViewer @@ -374,11 +376,13 @@ private void SetContent(object content) { ReloadMenu.Visibility = logFilePath != null ? Visibility.Visible : Visibility.Collapsed; SaveAsMenu.Visibility = Visibility.Visible; + RedactSecretsMenu.Visibility = Visibility.Visible; } else { ReloadMenu.Visibility = Visibility.Collapsed; SaveAsMenu.Visibility = Visibility.Collapsed; + RedactSecretsMenu.Visibility = Visibility.Collapsed; } // If we had text inside search log control bring it back @@ -659,31 +663,120 @@ private void Reload() OpenLogFile(logFilePath); } - private void SaveAs() + private async System.Threading.Tasks.Task RedactSecrets() { - if (currentBuild != null) + RedactInputControl redactInputControl = new RedactInputControl(GetSaveAsDestination); + if (redactInputControl.ShowDialog() != true) { - string currentFilePath = currentBuild.LogFilePath; + return; + } + + List stringsToRedact = + new(redactInputControl.SecretsBlock? + .Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries) + .Select(s => s.Trim()) + .Where(s => !string.IsNullOrWhiteSpace(s)) + ?? new string[] { }); + + if ( + !stringsToRedact.Any() && + !redactInputControl.RedactUsername && + !redactInputControl.RedactCommonCredentials) + { + MessageBox.Show("No secrets to redact - no action will be performed"); + return; + } - var saveFileDialog = new SaveFileDialog(); - saveFileDialog.Filter = currentFilePath != null && currentFilePath.EndsWith(".binlog", StringComparison.OrdinalIgnoreCase) ? Serialization.BinlogFileDialogFilter : Serialization.FileDialogFilter; - saveFileDialog.Title = "Save log file as"; - saveFileDialog.CheckFileExists = false; - saveFileDialog.OverwritePrompt = true; - saveFileDialog.ValidateNames = true; - var result = saveFileDialog.ShowDialog(this); - if (result != true) + var progress = new BuildProgress(); + progress.Progress.Updated += update => + { + Dispatcher.InvokeAsync(() => { - return; + progress.Value = update.Ratio; + }, DispatcherPriority.Background); + }; + progress.ProgressText = "Performing the log redaction ..."; + SetContent(progress); + + string error = await System.Threading.Tasks.Task.Run(() => + { + try + { + BinlogRedactorOptions redactorOptions = new BinlogRedactorOptions(logFilePath) + { + OutputFileName = redactInputControl.DestinationFile, + ProcessEmbeddedFiles = redactInputControl.RedactEmbeddedFiles, + AutodetectUsername = redactInputControl.RedactUsername, + AutodetectCommonPatterns = redactInputControl.RedactCommonCredentials, + IdentifyReplacemenets = redactInputControl.DistinguishSecretsReplacements, + TokensToRedact = stringsToRedact.ToArray(), + }; + + BinlogRedactor.RedactSecrets( + redactorOptions, + progress.Progress); + } + catch(Exception e) + { + return e.ToString(); } - string newFilePath = saveFileDialog.FileName; + return null; + }); + + if (!string.IsNullOrEmpty(error)) + { + MessageBox.Show($"Redaction failed:{Environment.NewLine}{error}"); + SetContent(currentBuild); + } + else if (string.IsNullOrEmpty(redactInputControl.DestinationFile)) + { + // Reload + OpenLogFile(logFilePath); + } + else + { + SetContent(currentBuild); + AnnounceFileSaved(redactInputControl.DestinationFile); + } + } + + private string GetSaveAsDestination() + { + string currentFilePath = currentBuild.LogFilePath; + + var saveFileDialog = new SaveFileDialog(); + saveFileDialog.Filter = currentFilePath != null && currentFilePath.EndsWith(".binlog", StringComparison.OrdinalIgnoreCase) ? Serialization.BinlogFileDialogFilter : Serialization.FileDialogFilter; + saveFileDialog.Title = "Save log file as"; + saveFileDialog.CheckFileExists = false; + saveFileDialog.OverwritePrompt = true; + saveFileDialog.ValidateNames = true; + var result = saveFileDialog.ShowDialog(this); + + return result == true ? saveFileDialog.FileName : null; + } + + private void AnnounceFileSaved(string filePath) + { + Dispatcher.InvokeAsync(() => + { + currentBuild.UpdateBreadcrumb(new Message { Text = $"Saved {filePath}" }); + }); + SettingsService.AddRecentLogFile(filePath); + } + + private void SaveAs() + { + if (currentBuild != null) + { + string currentFilePath = currentBuild.LogFilePath; + string newFilePath = GetSaveAsDestination(); if (string.IsNullOrEmpty(newFilePath) || string.Equals(currentFilePath, newFilePath, StringComparison.OrdinalIgnoreCase)) { return; } - logFilePath = saveFileDialog.FileName; + logFilePath = newFilePath; lock (inProgressOperationLock) { @@ -702,11 +795,7 @@ private void SaveAs() currentBuild.Build.LogFilePath = logFilePath; - Dispatcher.InvokeAsync(() => - { - currentBuild.UpdateBreadcrumb(new Message { Text = $"Saved {logFilePath}" }); - }); - SettingsService.AddRecentLogFile(logFilePath); + AnnounceFileSaved(logFilePath); } catch { @@ -814,6 +903,11 @@ private void SaveAs_Click(object sender, RoutedEventArgs e) SaveAs(); } + private void RedactSecrets_Click(object sender, RoutedEventArgs e) + { + RedactSecrets().Ignore(); + } + private void HelpLink_Click(object sender, RoutedEventArgs e) { Process.Start(new ProcessStartInfo("https://github.com/KirillOsenkov/MSBuildStructuredLog") { UseShellExecute = true }); diff --git a/src/StructuredLogViewer/StructuredLogViewer.csproj b/src/StructuredLogViewer/StructuredLogViewer.csproj index 41f6df8e2..771c7d319 100644 --- a/src/StructuredLogViewer/StructuredLogViewer.csproj +++ b/src/StructuredLogViewer/StructuredLogViewer.csproj @@ -32,6 +32,7 @@ + diff --git a/src/StructuredLogViewer/TaskExtensions.cs b/src/StructuredLogViewer/TaskExtensions.cs new file mode 100644 index 000000000..89b7ab523 --- /dev/null +++ b/src/StructuredLogViewer/TaskExtensions.cs @@ -0,0 +1,14 @@ +using System.Threading.Tasks; + +namespace StructuredLogViewer +{ + internal static class TaskExtensions + { + /// + /// Consumes a task and doesn't do anything with it. Useful for fire-and-forget calls to async methods within sync methods. + /// + /// The task whose result is to be ignored. + public static void Ignore(this Task task) + { /* this is it */ } + } +} diff --git a/src/StructuredLogger.Tests/BinaryLoggerTests.cs b/src/StructuredLogger.Tests/BinaryLoggerTests.cs index e86b12a3c..f6579709f 100644 --- a/src/StructuredLogger.Tests/BinaryLoggerTests.cs +++ b/src/StructuredLogger.Tests/BinaryLoggerTests.cs @@ -3,6 +3,8 @@ using System.IO; using System.Linq; using System.Runtime.CompilerServices; +using FluentAssertions; +using Microsoft.Build.Logging; using Microsoft.Build.Logging.StructuredLogger; using StructuredLogger.Tests; using Xunit; @@ -55,16 +57,20 @@ public BinaryLoggerTests(ITestOutputHelper output) { this.output = output; } + private bool BuildProject(string projectFile, string binLog, bool useInMemoryProject) + { + File.Delete(binLog); + var binaryLogger = new BinaryLogger { Parameters = binLog }; + return useInMemoryProject + ? MSBuild.BuildProjectInMemory(projectFile, binaryLogger) + : MSBuild.BuildProjectFromFile(projectFile, binaryLogger); + } [Fact] public void TestBuildTreeStructureCount() { var binLog = GetTestFile("1.binlog"); - var binaryLogger = new BinaryLogger(); - binaryLogger.Parameters = binLog; - var buildSuccessful = MSBuild.BuildProjectFromFile(s_testProject, binaryLogger); - - Assert.True(buildSuccessful); + Assert.True(BuildProject(s_testProject, binLog, false)); var build = Serialization.Read(binLog); BuildAnalyzer.AnalyzeBuild(build); @@ -93,13 +99,7 @@ public void TestBuildTreeStructureCount() public void TestBinaryLoggerRoundtrip(bool useInMemoryProject) { var binLog = GetTestFile("1.binlog"); - var binaryLogger = new BinaryLogger(); - binaryLogger.Parameters = binLog; - var buildSuccessful = useInMemoryProject - ? MSBuild.BuildProjectInMemory(s_testProject, binaryLogger) - : MSBuild.BuildProjectFromFile(s_testProject, binaryLogger); - - Assert.True(buildSuccessful); + Assert.True(BuildProject(s_testProject, binLog, useInMemoryProject)); var build = Serialization.Read(binLog); var xml1 = GetTestFile("1.xml"); @@ -124,6 +124,90 @@ public void TestBinaryLoggerRoundtrip(bool useInMemoryProject) AssertEx.EqualOrDiff(File.ReadAllText(xml1), File.ReadAllText(GetTestFile("4.xml"))); } + [Fact] + public void TestReaderWriterRoundtripEquality() + { + var binLog = GetTestFile("1.binlog"); + Assert.True(BuildProject(s_testProject, binLog, false)); + var replayedBinlog = GetTestFile("1-replayed.binlog"); + File.Delete(replayedBinlog); + + //need to have in this repo + var logReader = new Logging.StructuredLogger.BinaryLogReplayEventSource(); + + BinaryLogger outputBinlog = new BinaryLogger() + { + Parameters = $"LogFile={replayedBinlog};OmitInitialInfo" + }; + outputBinlog.Initialize(logReader); + logReader.Replay(binLog); + outputBinlog.Shutdown(); + + //assert here + AssertBinlogsHaveEqualContent(binLog, replayedBinlog); + + //TODO: temporarily disabling - as we do not have guarantee for binary equality of events + // there are few mismatches between null and empty - will be fixed along with porting in-flight changes in MSBuild (needs log version update) + //// If this assertation complicates development - it can possibly be removed + //// The structured equality above should be enough. + //AssertFilesAreBinaryEqualAfterUnpack(binLog, replayedBinlog); + } + + private static void AssertFilesAreBinaryEqualAfterUnpack(string firstPath, string secondPath) + { + using var br1 = Logging.StructuredLogger.BinaryLogReplayEventSource.OpenReader(firstPath); + using var br2 = Logging.StructuredLogger.BinaryLogReplayEventSource.OpenReader(secondPath); + const int bufferSize = 4096; + + int readCount = 0; + while (br1.ReadBytes(bufferSize) is { Length: > 0 } bytes1) + { + var bytes2 = br2.ReadBytes(bufferSize); + + bytes1.Should().BeEquivalentTo(bytes2, + $"Buffers starting at position {readCount} differ. First:{Environment.NewLine}{string.Join(",", bytes1)}{Environment.NewLine}Second:{Environment.NewLine}{string.Join(",", bytes2)}"); + readCount += bufferSize; + } + + br2.ReadBytes(bufferSize).Length.Should().Be(0, "Second buffer contains bytes after first file end"); + } + + private static void AssertBinlogsHaveEqualContent(string firstPath, string secondPath) + { + using var reader1 = Logging.StructuredLogger.BinaryLogReplayEventSource.OpenBuildEventsReader(firstPath); + using var reader2 = Logging.StructuredLogger.BinaryLogReplayEventSource.OpenBuildEventsReader(secondPath); + + Dictionary embedFiles1 = new(); + Dictionary embedFiles2 = new(); + + reader1.ArchiveFileEncountered += arg + => AddArchiveFile(embedFiles1, arg); + reader2.ArchiveFileEncountered += arg + => AddArchiveFile(embedFiles2, arg); + + // Pull events from both logs simultaneously and compare them + int i = 0; + while (reader1.Read() is { } logRecord1) + { + i++; + var logRecord2 = reader2.Read(); + + logRecord1.Should().BeEquivalentTo(logRecord2, + $"Binlogs ({firstPath} and {secondPath}) should be equal at event {i}"); + } + // Read the second reader - to confirm there are no more events + // and to force the embedded files to be read. + reader2.Read().Should().BeNull($"Binlogs ({firstPath} and {secondPath}) are not equal - second has more events >{i + 1}"); + + Assert.Equal(embedFiles1, embedFiles2); + embedFiles1.Should().NotBeEmpty(); + + void AddArchiveFile(Dictionary files, ArchiveFileEventArgs arg) + { + files.Add(arg.ArchiveFile.FullPath, arg.ArchiveFile.Text); + } + } + private static string GetProperty(Logging.StructuredLogger.Build build) { var property = build.FindFirstDescendant().FindChild("Properties").FindChild(p => p.Name == "FrameworkSDKRoot").Value; @@ -134,4 +218,4 @@ public void Dispose() { } } -} \ No newline at end of file +} diff --git a/src/StructuredLogger.Tests/StructuredLogger.Tests.csproj b/src/StructuredLogger.Tests/StructuredLogger.Tests.csproj index b5aa3af9b..11099a746 100644 --- a/src/StructuredLogger.Tests/StructuredLogger.Tests.csproj +++ b/src/StructuredLogger.Tests/StructuredLogger.Tests.csproj @@ -21,6 +21,7 @@ + @@ -32,4 +33,9 @@ + + True + ..\StructuredLogger\key.snk + False + \ No newline at end of file diff --git a/src/StructuredLogger.Utils/BinlogRedactor.cs b/src/StructuredLogger.Utils/BinlogRedactor.cs new file mode 100644 index 000000000..eeb1a5c87 --- /dev/null +++ b/src/StructuredLogger.Utils/BinlogRedactor.cs @@ -0,0 +1,143 @@ +using System; +using System.IO; +using System.Threading; +using Microsoft.Build.Logging; +using Microsoft.Build.Logging.StructuredLogger; +using Microsoft.Build.SensitiveDataDetector; + +namespace StructuredLogger.Utils +{ + public sealed class BinlogRedactorOptions + { + public BinlogRedactorOptions(string inputPath) + { + InputPath = inputPath; + } + + public string[]? TokensToRedact { get; set; } + public string InputPath { get; set; } + public string? OutputFileName { get; set; } + public bool ProcessEmbeddedFiles { get; set; } = true; + public bool IdentifyReplacemenets { get; set; } = true; + public bool AutodetectCommonPatterns { get; set; } = true; + public bool AutodetectUsername { get; set; } = true; + } + + public class BinlogRedactor + { + private readonly ISensitiveDataRedactor _sensitiveDataRedactor; + + public static void RedactSecrets( + string binlogPath, + string[] secrets) + => RedactSecrets( + new BinlogRedactorOptions(binlogPath) { TokensToRedact = secrets, }, progress: null); + + public static void RedactSecrets( + BinlogRedactorOptions redactorOptions, + Progress progress) + { + string outputFile; + bool replaceInPlace = false; + + if (string.IsNullOrEmpty(redactorOptions.OutputFileName) || + string.Equals(redactorOptions.InputPath, redactorOptions.OutputFileName, StringComparison.OrdinalIgnoreCase)) + { + outputFile = Path.Combine(PathUtils.TempPath, Path.GetFileName(Path.GetTempFileName()) + ".binlog"); + replaceInPlace = true; + } + else + { + outputFile = redactorOptions.OutputFileName; + } + + SensitiveDataKind sensitiveDataKind = SensitiveDataKind.ExplicitSecrets; + if (redactorOptions.AutodetectCommonPatterns) + { + sensitiveDataKind |= SensitiveDataKind.CommonSecrets; + } + if (redactorOptions.AutodetectUsername) + { + sensitiveDataKind |= SensitiveDataKind.Username; + } + + ISensitiveDataRedactor sensitiveDataRedactor = SensitiveDataDetectorFactory.GetSecretsDetector( + sensitiveDataKind, + redactorOptions.IdentifyReplacemenets, + redactorOptions.TokensToRedact); + + new BinlogRedactor(sensitiveDataRedactor) { Progress = progress } + .ProcessBinlog(redactorOptions.InputPath, outputFile, !redactorOptions.ProcessEmbeddedFiles); + + if (replaceInPlace) + { + File.Delete(redactorOptions.InputPath); + File.Move(outputFile, redactorOptions.InputPath); + } + } + + public BinlogRedactor(ISensitiveDataRedactor sensitiveDataRedactor) + { + _sensitiveDataRedactor = sensitiveDataRedactor; + } + + public Progress Progress { private get; set; } + + public void ProcessBinlog( + string inputFileName, + string outputFileName, + bool skipEmbeddedFiles) + { + BinaryLogReplayEventSource originalEventsSource = new(); + + Microsoft.Build.Logging.StructuredLogger.BinaryLogger outputBinlog = new() + { + Parameters = $"LogFile={outputFileName};OmitInitialInfo", + }; + + ((IBuildEventStringsReader) originalEventsSource).StringReadDone += HandleStringRead; + if (!skipEmbeddedFiles) + { + ((IBuildFileReader)originalEventsSource).ArchiveFileEncountered += + ((Action)HandleStringRead).ToArchiveFileHandler(); + } + + outputBinlog.Initialize(originalEventsSource); + + var inputStream = new FileStream(inputFileName, FileMode.Open, FileAccess.Read, FileShare.Read); + + CancellationTokenSource cts = null; + if (Progress != null) + { + cts = new CancellationTokenSource(); + long streamLength = inputStream.Length; + System.Threading.Tasks.Task.Run(async () => + { + while (!cts.IsCancellationRequested) + { + await System.Threading.Tasks.Task.Delay(200, cts.Token); + Progress.Report((double)inputStream.Position / streamLength); + } + }, cts.Token); + } + + originalEventsSource.Replay(inputStream, CancellationToken.None); + outputBinlog.Shutdown(); + + // TODO: error handling + + if (Progress != null) + { + cts.Cancel(); + Progress.Report(1.0); + } + + ((IBuildEventStringsReader)originalEventsSource).StringReadDone -= HandleStringRead; + + void HandleStringRead(StringReadEventArgs args) + { + args.StringToBeUsed = _sensitiveDataRedactor.Redact(args.OriginalString); + } + } + } +} diff --git a/src/StructuredLogger.Utils/StructuredLogger.Utils.csproj b/src/StructuredLogger.Utils/StructuredLogger.Utils.csproj new file mode 100644 index 000000000..40cc85604 --- /dev/null +++ b/src/StructuredLogger.Utils/StructuredLogger.Utils.csproj @@ -0,0 +1,26 @@ + + + netstandard2.0 + true + true + true + + + + + + $(DefineConstants);NETCORE + + + + + + + + + + True + ..\StructuredLogger\key.snk + False + + diff --git a/src/StructuredLogger/BinaryLogger/BinaryLogReplayEventSource.cs b/src/StructuredLogger/BinaryLogger/BinaryLogReplayEventSource.cs new file mode 100644 index 000000000..6bffde7ac --- /dev/null +++ b/src/StructuredLogger/BinaryLogger/BinaryLogReplayEventSource.cs @@ -0,0 +1,182 @@ +using System; +using System.Collections.Generic; +using System.IO.Compression; +using System.IO; +using System.Text; +using Microsoft.Build.BackEnd; +using Microsoft.Build.Shared; +using System.Threading; +using Microsoft.Build.Framework; + +namespace Microsoft.Build.Logging.StructuredLogger +{ + /// + /// Interface for replaying a binary log file (*.binlog) + /// + internal interface IBinaryLogReplaySource : + IEventSource, + IBuildEventStringsReader, + IBuildFileReader, + IEmbeddedContentSource + { } + + /// + /// Provides a method to read a binary log file (*.binlog) and replay all stored BuildEventArgs + /// by implementing IEventSource and raising corresponding events. + /// + /// The class is public so that we can call it from MSBuild.exe when replaying a log file. + internal sealed class BinaryLogReplayEventSource : EventArgsDispatcher, IBinaryLogReplaySource + { + /// + /// Read the provided binary log file and raise corresponding events for each BuildEventArgs + /// + /// The full file path of the binary log file + public void Replay(string sourceFilePath) + { + Replay(sourceFilePath, CancellationToken.None); + } + + /// + /// Read the provided binary log file opened as a stream and raise corresponding events for each BuildEventArgs + /// + /// Stream over the binlog content. + /// + public void Replay(Stream sourceFileStream, CancellationToken cancellationToken) + { + using var binaryReader = OpenReader(sourceFileStream); + Replay(binaryReader, cancellationToken); + } + + /// + /// Creates a for the provided binary log file. + /// Performs decompression and buffering in the optimal way. + /// Caller is responsible for disposing the returned reader. + /// + /// + /// BinaryReader of the given binlog file. + public static BinaryReader OpenReader(string sourceFilePath) + { + Stream? stream = null; + try + { + stream = new FileStream(sourceFilePath, FileMode.Open, FileAccess.Read, FileShare.Read); + return OpenReader(stream); + } + catch (Exception) + { + stream?.Dispose(); + throw; + } + } + + /// + /// Creates a for the provided binary log file. + /// Performs decompression and buffering in the optimal way. + /// Caller is responsible for disposing the returned reader. + /// + /// Stream over the binlog file + /// BinaryReader of the given binlog file. + public static BinaryReader OpenReader(Stream sourceFileStream) + { + var gzipStream = new GZipStream(sourceFileStream, CompressionMode.Decompress, leaveOpen: false); + + // wrapping the GZipStream in a buffered stream significantly improves performance + // and the max throughput is reached with a 32K buffer. See details here: + // https://github.com/dotnet/runtime/issues/39233#issuecomment-745598847 + var bufferedStream = new BufferedStream(gzipStream, 32768); + return new BinaryReader(bufferedStream); + } + + /// + /// Creates a for the provided binary log file. + /// Performs decompression and buffering in the optimal way. + /// + /// + /// BinaryReader of the given binlog file. + internal static BuildEventArgsReader OpenBuildEventsReader(string sourceFilePath) + => OpenBuildEventsReader(OpenReader(sourceFilePath), true); + + /// + /// Creates a for the provided binary reader over binary log file. + /// Caller is responsible for disposing the returned reader. + /// + /// + /// Indicates whether the passed BinaryReader should be closed on disposing. + /// BuildEventArgsReader over the given binlog file binary reader. + internal static BuildEventArgsReader OpenBuildEventsReader( + BinaryReader binaryReader, + bool closeInput) + { + int fileFormatVersion = binaryReader.ReadInt32(); + + // the log file is written using a newer version of file format + // that we don't know how to read + if (fileFormatVersion > BinaryLogger.FileFormatVersion) + { + var text = $"The log file format version is {fileFormatVersion}, whereas this version of MSBuild only supports versions up to {BinaryLogger.FileFormatVersion}."; + throw new NotSupportedException(text); + } + + return new BuildEventArgsReader(binaryReader, fileFormatVersion) + { + CloseInput = closeInput, + }; + } + + /// + /// Read the provided binary log file and raise corresponding events for each BuildEventArgs + /// + /// The full file path of the binary log file + /// A indicating the replay should stop as soon as possible. + public void Replay(string sourceFilePath, CancellationToken cancellationToken) + { + using var binaryReader = OpenReader(sourceFilePath); + Replay(binaryReader, cancellationToken); + } + + /// + /// Read the provided binary log file and raise corresponding events for each BuildEventArgs + /// + /// The binary log content binary reader - caller is responsible for disposing. + /// A indicating the replay should stop as soon as possible. + public void Replay(BinaryReader binaryReader, CancellationToken cancellationToken) + { + using BuildEventArgsReader reader = OpenBuildEventsReader(binaryReader, false); + + reader.EmbeddedContentRead += _embeddedContentRead; + reader.StringReadDone += _stringReadDone; + reader.ArchiveFileEncountered += _archiveFileEncountered; + + while (!cancellationToken.IsCancellationRequested && reader.Read() is { } instance) + { + Dispatch(instance); + } + } + + private Action? _embeddedContentRead; + /// + event Action? IEmbeddedContentSource.EmbeddedContentRead + { + // Explicitly implemented event has to declare explicit add/remove accessors + // https://stackoverflow.com/a/2268472/2308106 + add => _embeddedContentRead += value; + remove => _embeddedContentRead -= value; + } + + private Action? _archiveFileEncountered; + /// + event Action? IBuildFileReader.ArchiveFileEncountered + { + add => _archiveFileEncountered += value; + remove => _archiveFileEncountered -= value; + } + + private Action? _stringReadDone; + /// + event Action? IBuildEventStringsReader.StringReadDone + { + add => _stringReadDone += value; + remove => _stringReadDone -= value; + } + } +} diff --git a/src/StructuredLogger/BinaryLogger/BinaryLogger.cs b/src/StructuredLogger/BinaryLogger/BinaryLogger.cs index 6d8b85800..35624ff92 100644 --- a/src/StructuredLogger/BinaryLogger/BinaryLogger.cs +++ b/src/StructuredLogger/BinaryLogger/BinaryLogger.cs @@ -125,7 +125,8 @@ public void Initialize(IEventSource eventSource) Environment.SetEnvironmentVariable("MSBUILDLOGIMPORTS", "1"); bool logPropertiesAndItemsAfterEvaluation = true; - ProcessParameters(); + ProcessParameters(out bool omitInitialInfo); + var replayEventsSource = eventSource as IBinaryLogReplaySource; try { @@ -147,7 +148,7 @@ public void Initialize(IEventSource eventSource) stream = new FileStream(FilePath, FileMode.Create); - if (CollectProjectImports != ProjectImportsCollectionMode.None) + if (CollectProjectImports != ProjectImportsCollectionMode.None && replayEventsSource == null) { projectImportsCollector = new ProjectImportsCollector(FilePath, CollectProjectImports == ProjectImportsCollectionMode.ZipFile); } @@ -186,7 +187,24 @@ public void Initialize(IEventSource eventSource) binaryWriter.Write(FileFormatVersion); - LogInitialInfo(); + if (replayEventsSource != null) + { + if (CollectProjectImports == ProjectImportsCollectionMode.Embed) + { + replayEventsSource.EmbeddedContentRead += args => + eventArgsWriter.WriteBlob(args.ContentKind, args.ContentStream); + } + else if (CollectProjectImports == ProjectImportsCollectionMode.ZipFile) + { + replayEventsSource.EmbeddedContentRead += args => + ProjectImportsCollector.FlushBlobToFile(FilePath, args.ContentStream); + } + } + + if (!omitInitialInfo) + { + LogInitialInfo(); + } eventSource.AnyEventRaised += EventSource_AnyEventRaised; } @@ -222,12 +240,17 @@ public void Shutdown() if (projectImportsCollector != null) { + projectImportsCollector.Close(); + if (CollectProjectImports == ProjectImportsCollectionMode.Embed) { - eventArgsWriter.WriteBlob(BinaryLogRecordKind.ProjectImportArchive, projectImportsCollector.GetAllBytes()); + projectImportsCollector.ProcessResult( + streamToEmbed => eventArgsWriter.WriteBlob(BinaryLogRecordKind.ProjectImportArchive, streamToEmbed), + LogMessage); + + projectImportsCollector.DeleteArchive(); } - projectImportsCollector.Close(); projectImportsCollector = null; } @@ -285,13 +308,14 @@ private void CollectImports(BuildEventArgs e) /// /// /// - private void ProcessParameters() + private void ProcessParameters(out bool omitInitialInfo) { if (Parameters == null) { throw new LoggerException(ResourceUtilities.FormatResourceStringStripCodeAndKeyword("InvalidBinaryLoggerParameters", "")); } + omitInitialInfo = false; var parameters = Parameters.Split(MSBuildConstants.SemicolonChar, StringSplitOptions.RemoveEmptyEntries); foreach (var parameter in parameters) { @@ -307,6 +331,10 @@ private void ProcessParameters() { CollectProjectImports = ProjectImportsCollectionMode.ZipFile; } + else if (string.Equals(parameter, "OmitInitialInfo", StringComparison.OrdinalIgnoreCase)) + { + omitInitialInfo = true; + } else if (parameter.EndsWith(".binlog", StringComparison.OrdinalIgnoreCase)) { FilePath = parameter; diff --git a/src/StructuredLogger/BinaryLogger/BuildEventArgsReader.cs b/src/StructuredLogger/BinaryLogger/BuildEventArgsReader.cs index d10284990..8999bee58 100644 --- a/src/StructuredLogger/BinaryLogger/BuildEventArgsReader.cs +++ b/src/StructuredLogger/BinaryLogger/BuildEventArgsReader.cs @@ -5,10 +5,12 @@ using System.Collections; using System.Collections.Generic; using System.IO; +using System.IO.Compression; using System.Linq; using System.Reflection; using System.Runtime.InteropServices; using System.Text; +using DotUtils.StreamUtils; using Microsoft.Build.BackEnd; using Microsoft.Build.Collections; using Microsoft.Build.Framework; @@ -21,7 +23,7 @@ namespace Microsoft.Build.Logging.StructuredLogger /// /// Deserializes and returns BuildEventArgs-derived objects from a BinaryReader /// - internal partial class BuildEventArgsReader : IDisposable + internal partial class BuildEventArgsReader : IBuildEventArgsReaderNotifications, IDisposable { private readonly BinaryReader binaryReader; private readonly int fileFormatVersion; @@ -59,6 +61,12 @@ public BuildEventArgsReader(BinaryReader binaryReader, int fileFormatVersion) this.fileFormatVersion = fileFormatVersion; } + /// + /// Directs whether the passed should be closed when this instance is disposed. + /// Defaults to "false". + /// + public bool CloseInput { private get; set; } = false; + public void Dispose() { if (stringStorage != null) @@ -66,8 +74,21 @@ public void Dispose() stringStorage.Dispose(); stringStorage = null; } + if (CloseInput) + { + binaryReader.Dispose(); + } } + /// + public event Action? StringReadDone; + + /// + internal event Action? EmbeddedContentRead; + + /// + public event Action? ArchiveFileEncountered; + /// /// Raised when the log reader encounters a binary blob embedded in the stream. /// The arguments include the blob kind and the byte buffer with the contents. @@ -203,10 +224,91 @@ private static bool IsAuxiliaryRecord(BinaryLogRecordKind recordKind) private void ReadBlob(BinaryLogRecordKind kind) { // Work around bad logs caused by https://github.com/dotnet/msbuild/pull/9022#discussion_r1271468212 - if (kind == BinaryLogRecordKind.ProjectImportArchive && fileFormatVersion == 16) + var canHaveCorruptedSize = kind == BinaryLogRecordKind.ProjectImportArchive && fileFormatVersion == 16; + Stream embeddedStream = SliceOfEmdeddedContent(canHaveCorruptedSize); + + if (ArchiveFileEncountered != null) + { + // We could create ZipArchive over the target stream, and write to that directly, + // however, binlog format needs to know stream size upfront - which is unknown, + // so we would need to write the size after - and that would require the target stream to be seekable (which it's not) + ProjectImportsCollector? projectImportsCollector = null; + + if (EmbeddedContentRead != null) + { + projectImportsCollector = + new ProjectImportsCollector(PathUtils.TempPath, false, runOnBackground: false); + } + + // We are intentionally not grace handling corrupt embedded stream + + using var zipArchive = new ZipArchive(embeddedStream, ZipArchiveMode.Read); + + foreach (var entry in zipArchive.Entries/*.OrderBy(e => e.LastWriteTime)*/) + { + var file = ArchiveFile.From(entry, adjustPath: false); + ArchiveFileEventArgs archiveFileEventArgs = new(file); + ArchiveFileEncountered(archiveFileEventArgs); + + if (projectImportsCollector != null) + { + var resultFile = archiveFileEventArgs.ArchiveFile; + + projectImportsCollector.AddFileFromMemory( + resultFile.FullPath, + resultFile.Text, + makePathAbsolute: false, + entryCreationStamp: entry.LastWriteTime); + } + } + + // Once embedded files are replayed one by one - we can send the resulting stream to subscriber + if (OnBlobRead != null || EmbeddedContentRead != null) + { + projectImportsCollector!.ProcessResult( + streamToEmbed => InvokeEmbeddedDataListeners(kind, streamToEmbed), + error => throw new InvalidDataException(error)); + projectImportsCollector.DeleteArchive(); + } + } + else if (OnBlobRead != null || EmbeddedContentRead != null) + { + InvokeEmbeddedDataListeners(kind, embeddedStream); + } + else + { + embeddedStream.SkipBytes(); + } + } + + private void InvokeEmbeddedDataListeners(BinaryLogRecordKind kind, Stream embeddedStream) + { + byte[] bytes = null; + // we need to materialize the stream into a byte array + if (OnBlobRead != null) + { + bytes = embeddedStream.ReadToEnd(); + OnBlobRead(kind, bytes); + } + + if (EmbeddedContentRead != null) + { + // the embed stream was already read - we need to simulate the stream + if (bytes != null) + { + embeddedStream = new MemoryStream(bytes, writable: false); + } + + EmbeddedContentRead(new EmbeddedContentEventArgs(kind, embeddedStream)); + } + } + + private Stream SliceOfEmdeddedContent(bool canHaveCorruptedSize) + { + // Work around bad logs caused by https://github.com/dotnet/msbuild/pull/9022#discussion_r1271468212 + if (canHaveCorruptedSize) { int length; - byte[] bytes; // We have to preread some bytes to figure out if the log is buggy, // so store bytes to backfill for the "real" read later. @@ -250,20 +352,14 @@ private void ReadBlob(BinaryLogRecordKind kind) prefixBytes = reader.ReadBytes(12 - bytesRead); } - bytes = binaryReader.ReadBytes(length - prefixBytes.Length); - - byte[] fullBytes = new byte[length]; - prefixBytes.CopyTo(fullBytes, 0); - bytes.CopyTo(fullBytes, prefixBytes.Length); - bytes = fullBytes; - - OnBlobRead?.Invoke(kind, bytes); + Stream prefixStream = new MemoryStream(prefixBytes); + Stream dataStream = binaryReader.BaseStream.Slice(length - prefixBytes.Length); + return prefixStream.Concat(dataStream); } else { int length = ReadInt32(); - byte[] bytes = binaryReader.ReadBytes(length); - OnBlobRead?.Invoke(kind, bytes); + return binaryReader.BaseStream.Slice(length); } } @@ -1448,9 +1544,17 @@ private IEnumerable ReadTaskItemList() return list; } + private readonly StringReadEventArgs stringReadEventArgs = new StringReadEventArgs(string.Empty); private string ReadString() { - return binaryReader.ReadString(); + string text = binaryReader.ReadString(); + if (this.StringReadDone != null) + { + stringReadEventArgs.Reuse(text); + StringReadDone(stringReadEventArgs); + text = stringReadEventArgs.StringToBeUsed; + } + return text; } private string ReadOptionalString() diff --git a/src/StructuredLogger/BinaryLogger/BuildEventArgsWriter.cs b/src/StructuredLogger/BinaryLogger/BuildEventArgsWriter.cs index 7e83a77d7..64e92c45e 100644 --- a/src/StructuredLogger/BinaryLogger/BuildEventArgsWriter.cs +++ b/src/StructuredLogger/BinaryLogger/BuildEventArgsWriter.cs @@ -6,6 +6,7 @@ using System.Collections.Generic; using System.Globalization; using System.IO; +using System.Linq; using Microsoft.Build.Framework; using Microsoft.Build.Framework.Profiler; using Microsoft.Build.Internal; @@ -133,43 +134,47 @@ public void Write(BuildEventArgs e) currentRecordStream.SetLength(0); } -/* -Base types and inheritance ("EventArgs" suffix omitted): - -Build - Telemetry - LazyFormattedBuild - BuildMessage - CriticalBuildMessage - EnvironmentVariableRead - FileUsed - MetaprojectGenerated - ProjectImported - PropertyInitialValueSet - PropertyReassignment - TargetSkipped - TaskCommandLine - TaskParameter - UninitializedPropertyRead - AssemblyLoaded - BuildStatus - TaskStarted - TaskFinished - TargetStarted - TargetFinished - ProjectStarted - ProjectFinished - BuildStarted - BuildFinished - ProjectEvaluationStarted - ProjectEvaluationFinished - BuildError - BuildWarning - CustomBuild - ExternalProjectStarted - ExternalProjectFinished - -*/ + /* + Base types and inheritance ("EventArgs" suffix omitted): + + Build + Telemetry + LazyFormattedBuild + BuildMessage + CriticalBuildMessage + EnvironmentVariableRead + FileUsed + MetaprojectGenerated + ProjectImported + PropertyInitialValueSet + PropertyReassignment + TargetSkipped + TaskCommandLine + TaskParameter + UninitializedPropertyRead + AssemblyLoaded + ExtendedMessage + BuildStatus + TaskStarted + TaskFinished + TargetStarted + TargetFinished + ProjectStarted + ProjectFinished + BuildStarted + BuildFinished + ProjectEvaluationStarted + ProjectEvaluationFinished + BuildError + ExtendedBuildError + BuildWarning + ExtendedBuildWarning + CustomBuild + ExternalProjectStarted + ExternalProjectFinished + ExtendedCustomBuild + + */ private void WriteCore(BuildEventArgs e) { @@ -191,12 +196,31 @@ private void WriteCore(BuildEventArgs e) default: // convert all unrecognized objects to message // and just preserve the message - var buildMessageEventArgs = new BuildMessageEventArgs( - e.Message, - e.HelpKeyword, - e.SenderName, - MessageImportance.Normal, - e.Timestamp); + BuildMessageEventArgs buildMessageEventArgs; + if (e is IExtendedBuildEventArgs extendedData) + { + // For Extended events convert to ExtendedBuildMessageEventArgs + buildMessageEventArgs = new ExtendedBuildMessageEventArgs( + extendedData.ExtendedType, + e.Message, + e.HelpKeyword, + e.SenderName, + MessageImportance.Normal, + e.Timestamp) + { + ExtendedData = extendedData.ExtendedData, + ExtendedMetadata = extendedData.ExtendedMetadata, + }; + } + else + { + buildMessageEventArgs = new BuildMessageEventArgs( + e.Message, + e.HelpKeyword, + e.SenderName, + MessageImportance.Normal, + e.Timestamp); + } buildMessageEventArgs.BuildEventContext = e.BuildEventContext ?? BuildEventContext.Invalid; Write(buildMessageEventArgs); break; @@ -214,6 +238,22 @@ public void WriteBlob(BinaryLogRecordKind kind, byte[] bytes) Write(bytes); } + public void WriteBlob(BinaryLogRecordKind kind, Stream stream) + { + if (stream.Length > int.MaxValue) + { + throw new ArgumentOutOfRangeException(nameof(stream)); + } + + // write the blob directly to the underlying writer, + // bypassing the memory stream + using var redirection = RedirectWritesToOriginalWriter(); + + Write(kind); + Write((int)stream.Length); + Write(stream); + } + /// /// Switches the binaryWriter used by the Write* methods to the direct underlying stream writer /// until the disposable is disposed. Useful to bypass the currentRecordWriter to write a string, @@ -250,10 +290,17 @@ private void Write(BuildStartedEventArgs e) } else { - Write(0); + Write(e.BuildEnvironment?.Where(kvp => IsWellKnownEnvironmentDerivedProperty(kvp.Key))); } } + private static bool IsWellKnownEnvironmentDerivedProperty(string propertyName) + { + return propertyName.StartsWith("MSBUILD") || + propertyName.StartsWith("COMPLUS_") || + propertyName.StartsWith("DOTNET_"); + } + private void Write(BuildFinishedEventArgs e) { Write(BinaryLogRecordKind.BuildFinished); @@ -372,7 +419,9 @@ private void Write(TargetFinishedEventArgs e) private void Write(TaskStartedEventArgs e) { Write(BinaryLogRecordKind.TaskStarted); - WriteBuildEventArgsFields(e, writeMessage: false); + WriteBuildEventArgsFields(e, writeMessage: false, writeLineAndColumn: true); + Write(e.LineNumber); + Write(e.ColumnNumber); WriteDeduplicatedString(e.TaskName); WriteDeduplicatedString(e.ProjectFile); WriteDeduplicatedString(e.TaskFile); @@ -422,6 +471,7 @@ private void Write(BuildMessageEventArgs e) { switch (e) { + case FileUsedEventArgs responseFileUsed: Write(responseFileUsed); break; case TaskParameterEventArgs taskParameter: Write(taskParameter); break; case ProjectImportedEventArgs projectImported: Write(projectImported); break; case TargetSkippedEventArgs targetSkipped: Write(targetSkipped); break; @@ -431,8 +481,7 @@ private void Write(BuildMessageEventArgs e) case EnvironmentVariableReadEventArgs environmentVariableRead: Write(environmentVariableRead); break; case PropertyInitialValueSetEventArgs propertyInitialValueSet: Write(propertyInitialValueSet); break; case CriticalBuildMessageEventArgs criticalBuildMessage: Write(criticalBuildMessage); break; - case FileUsedEventArgs fileUsed: Write(fileUsed); break; - case AssemblyLoadBuildEventArgs assemblyLoaded: Write(assemblyLoaded); break; + case AssemblyLoadBuildEventArgs assemblyLoad: Write(assemblyLoad); break; default: // actual BuildMessageEventArgs Write(BinaryLogRecordKind.Message); WriteMessageFields(e, writeImportance: true); @@ -456,12 +505,24 @@ private void Write(TargetSkippedEventArgs e) WriteDeduplicatedString(e.TargetFile); WriteDeduplicatedString(e.TargetName); WriteDeduplicatedString(e.ParentTarget); - WriteDeduplicatedString(""); // Condition - WriteDeduplicatedString(""); // EvaluatedCondition - Write(true); // OriginallySucceeded + WriteDeduplicatedString(e.Condition); // Condition + WriteDeduplicatedString(e.EvaluatedCondition); // EvaluatedCondition + Write(e.OriginallySucceeded); // OriginallySucceeded Write((int)e.BuildReason); - Write((int)0); // SkipReason - Write(false); // OptionalBuildEventContext + Write((int)e.SkipReason); // SkipReason + binaryWriter.WriteOptionalBuildEventContext(e.OriginalBuildEventContext); // OptionalBuildEventContext + } + + private void Write(AssemblyLoadBuildEventArgs e) + { + Write(BinaryLogRecordKind.AssemblyLoad); + WriteMessageFields(e, writeMessage: false, writeImportance: false); + Write((int)e.LoadingContext); + WriteDeduplicatedString(e.LoadingInitiator); + WriteDeduplicatedString(e.AssemblyName); + WriteDeduplicatedString(e.AssemblyPath); + Write(e.MVID); + WriteDeduplicatedString(e.AppDomainDescriptor); } private void Write(CriticalBuildMessageEventArgs e) @@ -506,22 +567,10 @@ private void Write(EnvironmentVariableReadEventArgs e) private void Write(FileUsedEventArgs e) { Write(BinaryLogRecordKind.FileUsed); - WriteMessageFields(e, writeImportance: false); + WriteMessageFields(e); WriteDeduplicatedString(e.FilePath); } - private void Write(AssemblyLoadBuildEventArgs e) - { - Write(BinaryLogRecordKind.AssemblyLoad); - WriteMessageFields(e, writeMessage: false, writeImportance: false); - Write((int)e.LoadingContext); - WriteDeduplicatedString(e.LoadingInitiator); - WriteDeduplicatedString(e.AssemblyName); - WriteDeduplicatedString(e.AssemblyPath); - Write(e.MVID); - WriteDeduplicatedString(e.AppDomainDescriptor); - } - private void Write(TaskCommandLineEventArgs e) { Write(BinaryLogRecordKind.TaskCommandLine); @@ -537,7 +586,8 @@ private void Write(TaskParameterEventArgs e) Write((int)e.Kind); WriteDeduplicatedString(e.ItemType); WriteTaskItemList(e.Items, e.LogItemMetadata); - if (e.Kind == TaskParameterMessageKind.AddItem) + if (e.Kind == TaskParameterMessageKind.AddItem + || e.Kind == TaskParameterMessageKind.TaskOutput) { CheckForFilesToEmbed(e.ItemType, e.Items); } @@ -587,6 +637,11 @@ private void WriteBaseFields(BuildEventArgs e, BuildEventArgsFieldFlags flags) { Write(e.Timestamp); } + + if ((flags & BuildEventArgsFieldFlags.Extended) != 0) + { + Write(e as IExtendedBuildEventArgs); + } } private void WriteMessageFields(BuildMessageEventArgs e, bool writeMessage = true, bool writeImportance = false) @@ -753,6 +808,11 @@ private static BuildEventArgsFieldFlags GetBuildEventArgsFieldFlags(BuildEventAr flags |= BuildEventArgsFieldFlags.Timestamp; } + if (e is IExtendedBuildEventArgs extendedData) + { + flags |= BuildEventArgsFieldFlags.Extended; + } + return flags; } @@ -1113,6 +1173,11 @@ private void Write(byte[] bytes) binaryWriter.Write(bytes); } + private void Write(Stream stream) + { + stream.CopyTo(binaryWriter.BaseStream); + } + private void Write(byte b) { binaryWriter.Write(b); @@ -1213,6 +1278,16 @@ private void Write(ProfiledLocation e) Write(e.InclusiveTime); } + private void Write(IExtendedBuildEventArgs extendedData) + { + if (extendedData?.ExtendedType != null) + { + WriteDeduplicatedString(extendedData.ExtendedType); + Write(extendedData.ExtendedMetadata); + WriteDeduplicatedString(extendedData.ExtendedData); + } + } + internal readonly struct HashKey : IEquatable { private readonly ulong value; diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs new file mode 100644 index 000000000..5e9782db6 --- /dev/null +++ b/src/StructuredLogger/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Build.Logging.StructuredLogger; + +/// +/// Event arguments for event. +/// +public sealed class ArchiveFileEventArgs : EventArgs +{ + public ArchiveFileEventArgs(ArchiveFile archiveFile) => + ArchiveFile = archiveFile; + + public ArchiveFile ArchiveFile { get; set; } +} diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/ArchiveFileEventArgsExtensions.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/ArchiveFileEventArgsExtensions.cs new file mode 100644 index 000000000..cee62f61e --- /dev/null +++ b/src/StructuredLogger/BinaryLogger/Postprocessing/ArchiveFileEventArgsExtensions.cs @@ -0,0 +1,22 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Build.Logging.StructuredLogger; + +public static class ArchiveFileEventArgsExtensions +{ + public static Action ToArchiveFileHandler(this Action stringHandler) + { + return args => + { + var pathArgs = new StringReadEventArgs(args.ArchiveFile.FullPath); + stringHandler(pathArgs); + var contentArgs = new StringReadEventArgs(args.ArchiveFile.Text); + stringHandler(contentArgs); + + args.ArchiveFile = new ArchiveFile(pathArgs.StringToBeUsed, contentArgs.StringToBeUsed); + }; + } +} diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs new file mode 100644 index 000000000..0ba1f64a5 --- /dev/null +++ b/src/StructuredLogger/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; + +namespace Microsoft.Build.Logging.StructuredLogger +{ + internal sealed class EmbeddedContentEventArgs : EventArgs + { + public EmbeddedContentEventArgs(BinaryLogRecordKind contentKind, Stream contentStream) + { + ContentKind = contentKind; + ContentStream = contentStream; + } + + public BinaryLogRecordKind ContentKind { get; } + public Stream ContentStream { get; } + } +} diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs new file mode 100644 index 000000000..7f4053197 --- /dev/null +++ b/src/StructuredLogger/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.Build.Logging.StructuredLogger +{ + /// + /// An interface for notifications from BuildEventArgsReader + /// + public interface IBuildEventArgsReaderNotifications : + IBuildEventStringsReader, + IBuildFileReader + { } +} diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/IBuildEventStringsReader.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/IBuildEventStringsReader.cs new file mode 100644 index 000000000..6378eff07 --- /dev/null +++ b/src/StructuredLogger/BinaryLogger/Postprocessing/IBuildEventStringsReader.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Build.Logging.StructuredLogger +{ + /// + /// An interface for notifications about reading strings from the binary log. + /// + public interface IBuildEventStringsReader + { + /// + /// An event that allows the subscriber to be notified when a string is read from the binary log. + /// Subscriber may adjust the string by setting property. + /// The passed event arg can be reused and should not be stored. + /// + public event Action? StringReadDone; + } +} diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/IBuildFileReader.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/IBuildFileReader.cs new file mode 100644 index 000000000..5735c2d6d --- /dev/null +++ b/src/StructuredLogger/BinaryLogger/Postprocessing/IBuildFileReader.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Build.Logging.StructuredLogger; + +public interface IBuildFileReader +{ + /// + /// An event that allows the caller to be notified when an embedded file is encountered in the binary log. + /// Subscribing to this event obligates the subscriber to read the file content and set the resulting content + /// via or . + /// When subscriber is OK with greedy reading entire content of the file, it can simplify subscribing to this event, + /// by using handler with same signature as handler for and wrapping it via + /// extension. + /// + /// + /// + /// private void OnStringReadDone(StringReadEventArgs e) + /// { + /// e.StringToBeUsed = e.StringToBeUsed.Replace("foo", "bar"); + /// } + /// + /// private void SubscribeToEvents() + /// { + /// reader.StringReadDone += OnStringReadDone; + /// reader.ArchiveFileEncountered += ((Action<StringReadEventArgs>)OnStringReadDone).ToArchiveFileHandler(); + /// } + /// + /// + /// + public event Action? ArchiveFileEncountered; +} diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/IEmbeddedContentSource.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/IEmbeddedContentSource.cs new file mode 100644 index 000000000..cb5895f7e --- /dev/null +++ b/src/StructuredLogger/BinaryLogger/Postprocessing/IEmbeddedContentSource.cs @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Build.Logging.StructuredLogger +{ + internal interface IEmbeddedContentSource + { + /// + /// Raised when the log reader encounters a project import archive (embedded content) in the stream. + /// The subscriber must read the exactly given length of binary data from the stream - otherwise exception is raised. + /// If no subscriber is attached, the data is skipped. + /// + event Action EmbeddedContentRead; + } +} diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/StreamChunkOverreadException.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/StreamChunkOverreadException.cs new file mode 100644 index 000000000..1ba262441 --- /dev/null +++ b/src/StructuredLogger/BinaryLogger/Postprocessing/StreamChunkOverreadException.cs @@ -0,0 +1,22 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Build.Logging.StructuredLogger +{ + public class StreamChunkOverReadException : Exception + { + public StreamChunkOverReadException() + { + } + + public StreamChunkOverReadException(string message) : base(message) + { + } + + public StreamChunkOverReadException(string message, Exception inner) : base(message, inner) + { + } + } +} diff --git a/src/StructuredLogger/BinaryLogger/Postprocessing/StringReadEventArgs.cs b/src/StructuredLogger/BinaryLogger/Postprocessing/StringReadEventArgs.cs new file mode 100644 index 000000000..7eeadb23e --- /dev/null +++ b/src/StructuredLogger/BinaryLogger/Postprocessing/StringReadEventArgs.cs @@ -0,0 +1,35 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Build.Logging.StructuredLogger +{ + /// + /// An event args for callback. + /// + public sealed class StringReadEventArgs : EventArgs + { + /// + /// The original string that was read from the binary log. + /// + public string OriginalString { get; private set; } + + /// + /// The adjusted string (or the original string of none subscriber replaced it) that will be used by the reader. + /// + public string StringToBeUsed { get; set; } + + public StringReadEventArgs(string str) + { + OriginalString = str; + StringToBeUsed = str; + } + + internal void Reuse(string newValue) + { + OriginalString = newValue; + StringToBeUsed = newValue; + } + } +} diff --git a/src/StructuredLogger/BinaryLogger/ProjectImportsCollector.cs b/src/StructuredLogger/BinaryLogger/ProjectImportsCollector.cs index 5b2b1ae4c..3f7183a14 100644 --- a/src/StructuredLogger/BinaryLogger/ProjectImportsCollector.cs +++ b/src/StructuredLogger/BinaryLogger/ProjectImportsCollector.cs @@ -1,9 +1,12 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.IO.Compression; using System.Text; using System.Threading.Tasks; +using Microsoft.Build.Logging.StructuredLogger; +using Task = System.Threading.Tasks.Task; namespace Microsoft.Build.Logging { @@ -15,30 +18,11 @@ namespace Microsoft.Build.Logging /// internal class ProjectImportsCollector { - private Stream _stream; - public byte[] GetAllBytes() - { - if (_stream == null) - { - return Array.Empty(); - } - else if (ArchiveFilePath == null) - { - var stream = _stream as MemoryStream; - // Before we can use the zip archive, it must be closed. - Close(false); - return stream.ToArray(); - } - else - { - Close(); - return File.ReadAllBytes(ArchiveFilePath); - } - } - + private Stream _fileStream; private ZipArchive _zipArchive; - - private string ArchiveFilePath { get; set; } + private readonly string _archiveFilePath; + private readonly bool _runOnBackground; + private const string DefaultSourcesArchiveExtension = ".ProjectImports.zip"; /// /// Avoid visiting each file more than once. @@ -48,71 +32,126 @@ public byte[] GetAllBytes() // this will form a chain of file write tasks, running sequentially on a background thread private Task _currentTask = Task.CompletedTask; - public ProjectImportsCollector(string logFilePath, bool createFile, string sourcesArchiveExtension = ".ProjectImports.zip") + internal static void FlushBlobToFile( + string logFilePath, + Stream contentStream) { - try + string archiveFilePath = GetArchiveFilePath(logFilePath, DefaultSourcesArchiveExtension); + + using var fileStream = new FileStream(archiveFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.Delete); + contentStream.CopyTo(fileStream); + } + + // Archive file will be stored alongside the binlog + private static string GetArchiveFilePath(string logFilePath, string sourcesArchiveExtension) + => Path.ChangeExtension(logFilePath, sourcesArchiveExtension); + + public ProjectImportsCollector( + string logFilePath, + bool createFile, + string sourcesArchiveExtension = DefaultSourcesArchiveExtension, + bool runOnBackground = true) + { + if (createFile) { - if (createFile) - { - ArchiveFilePath = Path.ChangeExtension(logFilePath, sourcesArchiveExtension); - _stream = new FileStream(ArchiveFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.Delete); - } - else + // Archive file will be stored alongside the binlog + _archiveFilePath = GetArchiveFilePath(logFilePath, sourcesArchiveExtension); + } + else + { + string cacheDirectory = GetCacheDirectory(); + if (!Directory.Exists(cacheDirectory)) { - _stream = new MemoryStream(); + Directory.CreateDirectory(cacheDirectory); } - _zipArchive = new ZipArchive(_stream, ZipArchiveMode.Create, true); + + // Archive file will be temporarily stored in MSBuild cache folder and deleted when no longer needed + _archiveFilePath = Path.Combine( + cacheDirectory, + GetArchiveFilePath( + Path.GetFileName(logFilePath), + sourcesArchiveExtension)); + } + + try + { + _fileStream = new FileStream(_archiveFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.Delete); + _zipArchive = new ZipArchive(_fileStream, ZipArchiveMode.Create); } catch { // For some reason we weren't able to create a file for the archive. // Disable the file collector. - _stream = null; + _fileStream = null; _zipArchive = null; } + + _runOnBackground = runOnBackground; + } + + private static string GetCacheDirectory() + { + string dir = Path.Combine(PathUtils.TempPath, + $"MSBuildTemp-{Environment.UserName}-{Process.GetCurrentProcess().Id}-{AppDomain.CurrentDomain.Id}"); + Directory.CreateDirectory(dir); + return dir; + } + + public void AddFile(string? filePath) + { + AddFileHelper(filePath, AddFileCore); } - public void AddFile(string filePath) + public void AddFileFromMemory( + string? filePath, + string data, + DateTimeOffset? entryCreationStamp = null, + bool makePathAbsolute = true) { - if (filePath != null && _stream != null) + AddFileHelper(filePath, path => + AddFileFromMemoryCore(path, data, makePathAbsolute, entryCreationStamp)); + } + + public void AddFileFromMemory( + string? filePath, + Stream data, + DateTimeOffset? entryCreationStamp = null, + bool makePathAbsolute = true) + { + AddFileHelper(filePath, path => AddFileFromMemoryCore(path, data, makePathAbsolute, entryCreationStamp)); + } + + private void AddFileHelper( + string? filePath, + Action addFileWorker) + { + if (filePath != null && _fileStream != null) { - lock (_stream) + lock (_fileStream) { - // enqueue the task to add a file and return quickly - // to avoid holding up the current thread - _currentTask = _currentTask.ContinueWith(t => + if (_runOnBackground) + { + // enqueue the task to add a file and return quickly + // to avoid holding up the current thread + _currentTask = _currentTask.ContinueWith( + t => { TryAddFile(); }, + TaskScheduler.Default); + } + else { - try - { - AddFileCore(filePath); - } - catch - { - } - }, TaskScheduler.Default); + TryAddFile(); + } } } - } - public void AddFileFromMemory(string filePath, string data) - { - if (filePath != null && data != null && _stream != null) + void TryAddFile() { - lock (_stream) + try { - // enqueue the task to add a file and return quickly - // to avoid holding up the current thread - _currentTask = _currentTask.ContinueWith(t => - { - try - { - AddFileFromMemoryCore(filePath, data); - } - catch - { - } - }, TaskScheduler.Default); + addFileWorker(filePath); } + catch + { } } } @@ -123,64 +162,82 @@ public void AddFileFromMemory(string filePath, string data) private void AddFileCore(string filePath) { // quick check to avoid repeated disk access for Exists etc. - if (_processedFiles.Contains(filePath)) + if (!ShouldAddFile(ref filePath, true, true)) { return; } - var fileInfo = new FileInfo(filePath); - if (!fileInfo.Exists || fileInfo.Length == 0) + using FileStream content = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete); + AddFileData(filePath, content, null); + } + + /// + /// This method doesn't need locking/synchronization because it's only called + /// from a task that is chained linearly + /// + private void AddFileFromMemoryCore(string filePath, string data, bool makePathAbsolute, DateTimeOffset? entryCreationStamp) + { + // quick check to avoid repeated disk access for Exists etc. + if (!ShouldAddFile(ref filePath, false, makePathAbsolute)) { - _processedFiles.Add(filePath); return; } - filePath = Path.GetFullPath(filePath); + using var content = new MemoryStream(Encoding.UTF8.GetBytes(data)); + AddFileData(filePath, content, entryCreationStamp); + } - // if the file is already included, don't include it again - if (!_processedFiles.Add(filePath)) + private void AddFileFromMemoryCore(string filePath, Stream data, bool makePathAbsolute, DateTimeOffset? entryCreationStamp) + { + // quick check to avoid repeated disk access for Exists etc. + if (!ShouldAddFile(ref filePath, false, makePathAbsolute)) { return; } - using (Stream entryStream = OpenArchiveEntry(filePath)) - using (FileStream content = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete)) - { - content.CopyTo(entryStream); - } + AddFileData(filePath, data, entryCreationStamp); } - /// - /// This method doesn't need locking/synchronization because it's only called - /// from a task that is chained linearly - /// - private void AddFileFromMemoryCore(string filePath, string data) + private void AddFileData(string filePath, Stream data, DateTimeOffset? entryCreationStamp) + { + using Stream entryStream = OpenArchiveEntry(filePath, entryCreationStamp); + data.CopyTo(entryStream); + } + + private bool ShouldAddFile(ref string filePath, bool checkFileExistence, bool makeAbsolute) { // quick check to avoid repeated disk access for Exists etc. if (_processedFiles.Contains(filePath)) { - return; + return false; } - filePath = Path.GetFullPath(filePath); - - // if the file is already included, don't include it again - if (!_processedFiles.Add(filePath)) + if (checkFileExistence && !File.Exists(filePath)) { - return; + _processedFiles.Add(filePath); + return false; } - using (Stream entryStream = OpenArchiveEntry(filePath)) - using (var content = new MemoryStream(Encoding.UTF8.GetBytes(data))) + // Only make the path absolute if it's request. In the replay scenario, the file entries + // are read from zip archive - where ':' is stripped and path can then seem relative. + if (makeAbsolute) { - content.CopyTo(entryStream); + filePath = Path.GetFullPath(filePath); } + + // if the file is already included, don't include it again + return _processedFiles.Add(filePath); } - private Stream OpenArchiveEntry(string filePath) + private Stream OpenArchiveEntry(string filePath, DateTimeOffset? entryCreationStamp) { string archivePath = CalculateArchivePath(filePath); - var archiveEntry = _zipArchive.CreateEntry(archivePath); + var archiveEntry = _zipArchive!.CreateEntry(archivePath); + if (entryCreationStamp.HasValue) + { + archiveEntry.LastWriteTime = entryCreationStamp.Value; + } + return archiveEntry.Open(); } @@ -195,7 +252,28 @@ private static string CalculateArchivePath(string filePath) return archivePath; } - public void Close(bool closeStream = true) + public void ProcessResult(Action consumeStream, Action onError) + { + Close(); + + // It is possible that the archive couldn't be created for some reason. + // Only embed it if it actually exists. + if (File.Exists(_archiveFilePath)) + { + using FileStream fileStream = File.OpenRead(_archiveFilePath); + + if (fileStream.Length > int.MaxValue) + { + onError("Imported files archive exceeded 2GB limit and it's not embedded."); + } + else + { + consumeStream(fileStream); + } + } + } + + public void Close() { // wait for all pending file writes to complete _currentTask.Wait(); @@ -206,11 +284,17 @@ public void Close(bool closeStream = true) _zipArchive = null; } - if (closeStream && (_stream != null)) + if (_fileStream != null) { - _stream.Dispose(); - _stream = null; + _fileStream.Dispose(); + _fileStream = null; } } + + public void DeleteArchive() + { + Close(); + File.Delete(_archiveFilePath); + } } } diff --git a/src/StructuredLogger/ErrorReporting.cs b/src/StructuredLogger/ErrorReporting.cs index f83f13f46..ce6d2a08f 100644 --- a/src/StructuredLogger/ErrorReporting.cs +++ b/src/StructuredLogger/ErrorReporting.cs @@ -5,19 +5,7 @@ namespace Microsoft.Build.Logging.StructuredLogger { public class ErrorReporting { - private static readonly string logFilePath = Path.Combine(GetRootPath(), "LoggerExceptions.txt"); - - private static string GetRootPath() - { -#if NETCORE - var path = Path.GetTempPath(); -#else - var path = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData); -#endif - - path = Path.Combine(path, "Microsoft", "MSBuildStructuredLog"); - return path; - } + private static readonly string logFilePath = Path.Combine(PathUtils.RootPath, "LoggerExceptions.txt"); public static void ReportException(Exception ex) { diff --git a/src/StructuredLogger/ObjectModel/ArchiveFile.cs b/src/StructuredLogger/ObjectModel/ArchiveFile.cs index 10e4d95e2..e7f76678f 100644 --- a/src/StructuredLogger/ObjectModel/ArchiveFile.cs +++ b/src/StructuredLogger/ObjectModel/ArchiveFile.cs @@ -1,4 +1,4 @@ -using System.IO; +using System.IO; using System.IO.Compression; namespace Microsoft.Build.Logging.StructuredLogger @@ -15,8 +15,11 @@ public ArchiveFile(string fullPath, string text) public string Text { get; } public static ArchiveFile From(ZipArchiveEntry entry) + => From(entry, adjustPath: true); + + public static ArchiveFile From(ZipArchiveEntry entry, bool adjustPath) { - var filePath = CalculateArchivePath(entry.FullName); + var filePath = adjustPath ? CalculateArchivePath(entry.FullName) : entry.FullName; var text = GetText(entry); var file = new ArchiveFile(filePath, text); return file; @@ -51,4 +54,4 @@ public static string CalculateArchivePath(string filePath) return archivePath; } } -} \ No newline at end of file +} diff --git a/src/StructuredLogger/PathUtils.cs b/src/StructuredLogger/PathUtils.cs new file mode 100644 index 000000000..ce94c55ab --- /dev/null +++ b/src/StructuredLogger/PathUtils.cs @@ -0,0 +1,25 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace Microsoft.Build.Logging.StructuredLogger +{ + public static class PathUtils + { + public static readonly string RootPath = GetRootPath(); + public static readonly string TempPath = Path.Combine(RootPath, "Temp"); + + private static string GetRootPath() + { +#if NETCORE + var path = Path.GetTempPath(); +#else + var path = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData); +#endif + + path = Path.Combine(path, "Microsoft", "MSBuildStructuredLog"); + return path; + } + } +} diff --git a/src/StructuredLogger/Progress.cs b/src/StructuredLogger/Progress.cs index 3c6fa8aed..40f0e8653 100644 --- a/src/StructuredLogger/Progress.cs +++ b/src/StructuredLogger/Progress.cs @@ -9,6 +9,11 @@ public class Progress : IProgress public event Action Updated; + public virtual void Report(double ratio) + { + Report(new ProgressUpdate { Ratio = ratio }); + } + public virtual void Report(ProgressUpdate progressUpdate) { Updated?.Invoke(progressUpdate); @@ -20,4 +25,4 @@ public struct ProgressUpdate public double Ratio; public int BufferLength; } -} \ No newline at end of file +} diff --git a/src/StructuredLogger/StructuredLogger.cs b/src/StructuredLogger/StructuredLogger.cs index e88817d83..5b5e39370 100644 --- a/src/StructuredLogger/StructuredLogger.cs +++ b/src/StructuredLogger/StructuredLogger.cs @@ -1,7 +1,9 @@ using System; using System.IO; +using DotUtils.StreamUtils; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; +using static Microsoft.Build.Logging.StructuredLogger.BinaryLogger; namespace Microsoft.Build.Logging.StructuredLogger { @@ -103,10 +105,12 @@ public override void Shutdown() if (projectImportsCollector != null) { - var bytes = projectImportsCollector.GetAllBytes(); - construction.Build.SourceFilesArchive = bytes; - projectImportsCollector.Close(); + projectImportsCollector.ProcessResult( + streamToEmbed => construction.Build.SourceFilesArchive = streamToEmbed.ReadToEnd(), + _ => {}); + + projectImportsCollector.DeleteArchive(); projectImportsCollector = null; } diff --git a/src/StructuredLogger/StructuredLogger.csproj b/src/StructuredLogger/StructuredLogger.csproj index f1388227b..b53272eda 100644 --- a/src/StructuredLogger/StructuredLogger.csproj +++ b/src/StructuredLogger/StructuredLogger.csproj @@ -28,6 +28,7 @@ + @@ -60,4 +61,8 @@ key.snk False + + + +