Sensitive data redacting PoC

MSBuildStructuredLog.sln

@@ -28,6 +28,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ResourcesGenerator", "src\R
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BinlogTool", "src\BinlogTool\BinlogTool.csproj", "{35F44EA6-7259-43CC-8C17-E058F3EB86D3}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "StructuredLogger.Utils", "src\StructuredLogger.Utils\StructuredLogger.Utils.csproj", "{AC634B46-D57C-44C5-BF56-480843182F21}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Mixed Platforms = Debug|Mixed Platforms
@@ -66,6 +68,10 @@ Global
 		{35F44EA6-7259-43CC-8C17-E058F3EB86D3}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
 		{35F44EA6-7259-43CC-8C17-E058F3EB86D3}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
 		{35F44EA6-7259-43CC-8C17-E058F3EB86D3}.Release|Mixed Platforms.Build.0 = Release|Any CPU
+		{AC634B46-D57C-44C5-BF56-480843182F21}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
+		{AC634B46-D57C-44C5-BF56-480843182F21}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
+		{AC634B46-D57C-44C5-BF56-480843182F21}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
+		{AC634B46-D57C-44C5-BF56-480843182F21}.Release|Mixed Platforms.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

src/BinlogTool/BinlogTool.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <Version>1.0.6</Version>
+    <Version>1.0.7</Version>
     <TargetFramework>net7.0</TargetFramework>
     <LangVersion>latest</LangVersion>
     <DebugType>embedded</DebugType>
@@ -23,11 +23,12 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\StructuredLogger.Utils\StructuredLogger.Utils.csproj" />
     <ProjectReference Include="..\StructuredLogger\StructuredLogger.csproj" />
   </ItemGroup>
 
   <ItemGroup>
-    <None Include="$(IconFilePath)" Pack="true" PackagePath=""/>
+    <None Include="$(IconFilePath)" Pack="true" PackagePath="" />
   </ItemGroup>
 
 </Project>

src/BinlogTool/Program.cs

@@ -1,7 +1,9 @@
 using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using Microsoft.Build.Logging.StructuredLogger;
+using StructuredLogger.Utils;
 
 namespace BinlogTool
 {
@@ -16,7 +18,8 @@ binlogtool listtools input.binlog
     binlogtool savefiles input.binlog output_path
     binlogtool reconstruct input.binlog output_path
     binlogtool savestrings input.binlog output.txt
-    binlogtool search *.binlog search string");
+    binlogtool search *.binlog search string
+    binlogtool redact --input:path --recurse --in-place -p:list -p:of -p:secrets -p:to -p:redact");
                 return;
             }
 
@@ -71,6 +74,58 @@ binlogtool savestrings input.binlog output.txt
                 return;
             }
 
+            if (firstArg == "redact")
+            {
+                List<string> redactTokens = new List<string>();
+                List<string> inputPaths = new List<string>();
+                bool recurse = false;
+                bool inPlace = false;
+
+                foreach (var arg in args.Skip(1))
+                {
+                    if (arg.StartsWith("--input:", StringComparison.OrdinalIgnoreCase))
+                    {
+                        var input = arg.Substring("--input:".Length);
+                        if (string.IsNullOrEmpty(input))
+                        {
+                            Console.Error.WriteLine("Invalid input path");
+                            return;
+                        }
+
+                        inputPaths.Add(input);
+                    }
+                    else if (arg.StartsWith("-p:", StringComparison.OrdinalIgnoreCase))
+                    {
+                        var redactToken = arg.Substring("-p:".Length);
+                        if (string.IsNullOrEmpty(redactToken))
+                        {
+                            Console.Error.WriteLine("Invalid redact token");
+                            return;
+                        }
+
+                        redactTokens.Add(redactToken);
+                    }
+                    else if (arg.Equals("--recurse", StringComparison.OrdinalIgnoreCase))
+                    {
+                        recurse = true;
+                    }
+                    else if (arg.Equals("--in-place", StringComparison.OrdinalIgnoreCase))
+                    {
+                        inPlace = true;
+                    }
+                    else
+                    {
+                        Console.Error.WriteLine($"Invalid argument: {arg}");
+                        Console.Error.WriteLine("binlogtool redact --input:path --recurse --in-place -p:list -p:of -p:secrets -p:to -p:redact");
+                        Console.Error.WriteLine("All arguments are optional (missing input assumes current working directory. Missing tokens lead only to autoredactions. Missing --in-place will create new logs with suffix.)");
+                        return;
+                    }
+                }
+
+                Redact.Run(inputPaths, redactTokens, inPlace, recurse);
+                return;
+            }
+
             Console.Error.WriteLine("Invalid arguments");
         }
 

src/BinlogTool/Redact.cs

@@ -0,0 +1,79 @@
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using StructuredLogger.Utils;
+
+namespace BinlogTool
+{
+    internal static class Redact
+    {
+        public static void Run(List<string> inputs, List<string> tokens, bool inPlace, bool recurse)
+        {
+            if (!inputs.Any())
+            {
+                // Default - current directory
+                inputs.Add(string.Empty);
+            }
+
+            var inputBinlogs = inputs.SelectMany(input => Searcher.FindBinlogs(input, recurse)).ToList();
+
+            if (!inputBinlogs.Any())
+            {
+                Log.WriteError("No binlogs found.");
+                return;
+            }
+
+            if (inputBinlogs.Count > 1)
+            {
+                Log.WriteLine(
+                    $"Found {inputBinlogs.Count} binlog files. Will redact secrets in all. (found files: {(string.Join(',', inputBinlogs))})");
+            }
+
+            BinlogRedactorOptions options = new BinlogRedactorOptions(string.Empty)
+            {
+                TokensToRedact = tokens.ToArray(),
+                IdentifyReplacemenets = true,
+                AutodetectCommonPatterns = true,
+                AutodetectUsername = true,
+                ProcessEmbeddedFiles = true,
+            };
+
+            var overallStopwatch = Stopwatch.StartNew();
+
+            foreach (var inputBinlog in inputBinlogs)
+            {
+                options.InputPath = inputBinlog;
+                options.OutputFileName = GetOutputFileName(inputBinlog);
+
+                Log.WriteLine($"Redacting binlog {inputBinlog} to {options.OutputFileName} ({GetFileSizeInKB(inputBinlog)} KB)");
+
+                var stopwatch = Stopwatch.StartNew();
+
+                BinlogRedactor.RedactSecrets(options, progress: null);
+
+                stopwatch.Stop();
+                Log.WriteLine($"Redacting done. Duration: {stopwatch.Elapsed}");
+            }
+
+            overallStopwatch.Stop();
+            if(inputBinlogs.Count > 1)
+            {
+                Log.WriteLine($"Redacting all binlogs done. Duration: {overallStopwatch.Elapsed}");
+            }
+
+            string GetOutputFileName(string inputFileName)
+            {
+                if (inPlace)
+                {
+                    return inputFileName;
+                }
+
+                return Path.ChangeExtension(inputFileName, ".redacted.binlog");
+            }
+
+            long GetFileSizeInKB(string path)
+                => new FileInfo(path).Length / 1024;
+        }
+    }
+}

src/BinlogTool/Search.cs

@@ -1,4 +1,5 @@
-using System;
+using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Threading;
@@ -7,39 +8,55 @@
 
 namespace BinlogTool
 {
-    public class Searcher
+    public static class Searcher
     {
         public static void Search(string binlogs, string search)
         {
-            if (string.IsNullOrEmpty(binlogs))
+            var files = FindBinlogs(binlogs, recurse: true).ToList();
+            Search(files, search);
+        }
+
+        public static IEnumerable<string> FindBinlogs(string inputPath, bool recurse)
+        {
+            if (string.IsNullOrEmpty(inputPath))
             {
-                binlogs = "*.binlog";
+                inputPath = "*.binlog";
             }
 
-            binlogs = binlogs.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar);
+            inputPath = inputPath.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar);
+
+            if (File.Exists(inputPath))
+            {
+                return new[] { inputPath };
+            }
+
+            if (Directory.Exists(inputPath))
+            {
+                inputPath = Path.Combine(inputPath, "*.binlog");
+            }
 
             string fileName;
             string directory;
-            if (binlogs.Contains(Path.DirectorySeparatorChar))
+            if (inputPath.Contains(Path.DirectorySeparatorChar))
             {
-                fileName = Path.GetFileName(binlogs);
-                directory = Path.GetDirectoryName(binlogs);
+                fileName = Path.GetFileName(inputPath);
+                directory = Path.GetDirectoryName(inputPath);
                 if (!Path.IsPathRooted(directory))
                 {
                     directory = Path.GetFullPath(directory);
                 }
             }
             else
             {
-                fileName = binlogs;
+                fileName = inputPath;
                 directory = Environment.CurrentDirectory;
             }
 
-            var files = Directory.GetFiles(directory, fileName, SearchOption.AllDirectories);
-            Search(files, search);
+            return Directory.EnumerateFiles(directory, fileName,
+                new EnumerationOptions() { IgnoreInaccessible = true, RecurseSubdirectories = recurse, });
         }
 
-        private static void Search(string[] files, string search)
+        private static void Search(IEnumerable<string> files, string search)
         {
             foreach (var file in files)
             {
@@ -88,4 +105,4 @@ public static void PrintTree(BaseNode node, int indent = 0)
             }
         }
     }
-}
+}

src/StructuredLogViewer/Controls/RedactInputControl.xaml

@@ -0,0 +1,42 @@
+<Window x:Class="StructuredLogViewer.Controls.RedactInputControl"
+        xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
+        xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
+        xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
+        xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
+        xmlns:local="clr-namespace:StructuredLogViewer.Controls"
+        mc:Ignorable="d"
+        ContentRendered="Window_ContentRendered"
+        Title="Redact Binlog" SizeToContent="WidthAndHeight" Height="310">
+    <Grid Margin="5">
+        <Grid.RowDefinitions>
+            <RowDefinition Height="*" />
+            <RowDefinition Height="Auto" />
+        </Grid.RowDefinitions>
+
+        <StackPanel Margin="10">
+            <CheckBox Content="Process embedded files" Margin="0 5" IsChecked="True" Name="ChckbxEmbeddedFiles" Unchecked="ChckbxEmbeddedFiles_OnChanged" Checked="ChckbxEmbeddedFiles_OnChanged" />
+            <CheckBox Content="Use distinct replacements per secret type" Margin="0 5" IsChecked="True" Name="ChckbxDistinguishReplacements" Unchecked="ChckbxDistinguishReplacements_OnChanged" Checked="ChckbxDistinguishReplacements_OnChanged" />
+            <CheckBox Content="Autodetect and redact common credentials" Margin="0 5" Name="ChckbxCommonCredentials" Unchecked="ChckbxCommonCredentials_OnChanged" Checked="ChckbxCommonCredentials_OnChanged" IsChecked="True" />
+            <CheckBox Content="Autodetect and redact username" Margin="0 5" IsChecked="True" Name="ChckbxUsername" Unchecked="ChckbxUsername_OnChanged" Checked="ChckbxUsername_OnChanged" />
+            <CheckBox Content="Redact following newline-separated explicit strings" Name="ChckbxCustomSecrets" Margin="0 5" IsChecked="False" Unchecked="ChckbxCustomSecrets_OnChanged" Checked="ChckbxCustomSecrets_OnChanged" />
+            <TextBox Name="TxtSecrets" Margin="0 5" IsEnabled="True" MinLines="3" MinHeight="50" TextWrapping="Wrap" AcceptsReturn="True"/>
+        </StackPanel>
+
+
+
+        <Grid Grid.Row="1">
+            <Grid.ColumnDefinitions>
+                <ColumnDefinition Width="*"/>
+                <ColumnDefinition Width="Auto"/>
+                <ColumnDefinition Width="Auto"/>
+                <ColumnDefinition Width="Auto"/>
+            </Grid.ColumnDefinitions>
+
+            <Button Grid.Column="1" IsDefault="True" Name="BtnDialogOk" Click="btnDialogOk_Click" Margin="3 5" Padding="15, 2">_Redact In Place</Button>
+            <Button Grid.Column="2" Name="BtnSaveAs" Click="btnSaveAs_Click" Margin="3 5" Padding="15, 2">_Save As</Button>
+            <Button Grid.Column="3" IsCancel="True" Margin="3 5" Padding="15, 2">_Cancel</Button>
+        </Grid>
+
+    </Grid>
+
+</Window>