Add source build patch support to VMR manager

src/Microsoft.DotNet.Darc/src/DarcLib/Models/VirtualMonoRepo/SourceMapping.cs

@@ -16,4 +16,5 @@ public record SourceMapping(
     string DefaultRef,
     IReadOnlyCollection<string> Include,
     IReadOnlyCollection<string> Exclude,
+    IReadOnlyCollection<string> VmrPatches,
     string? Version = null);

src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/SourceMappingParser.cs

@@ -21,17 +21,15 @@ public interface ISourceMappingParser
 
 public class SourceMappingParser : ISourceMappingParser
 {
-    private const string SourceMappingsFileName = "source-mappings.json";
-
     public async Task<IReadOnlyCollection<SourceMapping>> ParseMappings(string vmrPath)
     {
-        var mappingFilePath = Path.Combine(vmrPath, "src", SourceMappingsFileName);
+        var mappingFilePath = Path.Combine(vmrPath, VmrManagerBase.VmrSourcesPath, VmrManagerBase.SourceMappingsFileName);
         var mappingFile = new FileInfo(mappingFilePath);
 
         if (!mappingFile.Exists)
         {
             throw new FileNotFoundException(
-                $"Failed to find {SourceMappingsFileName} file in the VMR directory",
+                $"Failed to find {VmrManagerBase.SourceMappingsFileName} file in the VMR directory",
                 mappingFilePath);
         }
 
@@ -43,14 +41,20 @@ public async Task<IReadOnlyCollection<SourceMapping>> ParseMappings(string vmrPa
 
         using var stream = File.Open(mappingFile.FullName, FileMode.Open);
         var settings = await JsonSerializer.DeserializeAsync<SourceMappingFile>(stream, options)
-            ?? throw new Exception($"Failed to deserialize {SourceMappingsFileName}");
+            ?? throw new Exception($"Failed to deserialize {VmrManagerBase.SourceMappingsFileName}");
+
+        var patchesPath = settings.PatchesPath;
+        if (patchesPath is not null)
+        {
+            patchesPath = Path.Combine(vmrPath, patchesPath.Replace('/', Path.DirectorySeparatorChar));
+        }
 
         return settings.Mappings
-            .Select(m => CreateMapping(settings.Defaults, m))
+            .Select(mapping => CreateMapping(settings.Defaults, mapping, Path.Join(patchesPath, mapping.Name)))
             .ToImmutableArray();
     }
 
-    private static SourceMapping CreateMapping(SourceMappingSetting defaults, SourceMappingSetting setting)
+    private static SourceMapping CreateMapping(SourceMappingSetting defaults, SourceMappingSetting setting, string? patchesPath)
     {
         IEnumerable<string> include = setting.Include ?? Enumerable.Empty<string>();
         IEnumerable<string> exclude = setting.Exclude ?? Enumerable.Empty<string>();
@@ -68,13 +72,18 @@ private static SourceMapping CreateMapping(SourceMappingSetting defaults, Source
             }
         }
 
+        var vmrPatches = patchesPath is not null && Directory.Exists(patchesPath)
+            ? Directory.GetFiles(patchesPath)
+            : Array.Empty<string>();
+
         return new SourceMapping(
-            Name: setting.Name ?? throw new InvalidOperationException($"Missing `name` in {SourceMappingsFileName}"),
+            Name: setting.Name ?? throw new InvalidOperationException($"Missing `{nameof(SourceMapping.Name).ToLower()}` in {VmrManagerBase.SourceMappingsFileName}"),
             Version: setting.Version,
-            DefaultRemote: setting.DefaultRemote ?? throw new InvalidOperationException($"Missing `defaultRemote` in {SourceMappingsFileName}"),
+            DefaultRemote: setting.DefaultRemote ?? throw new InvalidOperationException($"Missing `{nameof(SourceMapping.DefaultRemote).ToLower()}` in {VmrManagerBase.SourceMappingsFileName}"),
             DefaultRef: setting.DefaultRef ?? defaults.DefaultRef ?? "main",
             Include: include.ToImmutableArray(),
-            Exclude: exclude.ToImmutableArray());
+            Exclude: exclude.ToImmutableArray(),
+            VmrPatches: vmrPatches.ToImmutableArray());
     }
 
     private class SourceMappingFile
@@ -86,6 +95,8 @@ private class SourceMappingFile
             Exclude = Array.Empty<string>(),
         };
 
+        public string? PatchesPath { get; set; }
+
         public List<SourceMappingSetting> Mappings { get; set; } = new();
     }
 

src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrInitializer.cs

@@ -44,26 +44,31 @@ public async Task InitializeVmr(SourceMapping mapping, string? targetRevision, C
             throw new EmptySyncException($"Repository {mapping.Name} already exists");
         }
 
-        _logger.LogInformation("Initializing {name}", mapping.Name);
+        _logger.LogInformation("Initializing {name} at {revision}..", mapping.Name, targetRevision ?? mapping.DefaultRef);
 
         string clonePath = await CloneOrPull(mapping);
-        string patchPath = GetPatchFilePath(mapping);
-
         cancellationToken.ThrowIfCancellationRequested();
 
         using var clone = new Repository(clonePath);
         var commit = GetCommit(clone, (targetRevision is null || targetRevision == HEAD) ? null : targetRevision);
 
+        string patchPath = GetPatchFilePath(mapping);
         await CreatePatch(mapping, clonePath, Constants.EmptyGitObject, commit.Id.Sha, patchPath, cancellationToken);
         cancellationToken.ThrowIfCancellationRequested();
+
         await ApplyPatch(mapping, patchPath, cancellationToken);
         cancellationToken.ThrowIfCancellationRequested();
+
         await TagRepo(mapping, commit.Id.Sha);
+        cancellationToken.ThrowIfCancellationRequested();
 
-        var description = PrepareCommitMessage(InitializationCommitMessage, mapping, null, commit.Id.Sha, null);
+        await ApplyVmrPatches(mapping, cancellationToken);
+        cancellationToken.ThrowIfCancellationRequested();
 
         // Commit but do not add files (they were added to index directly)
-        cancellationToken.ThrowIfCancellationRequested();
-        Commit(description, DotnetBotCommitSignature);
+        var message = PrepareCommitMessage(InitializationCommitMessage, mapping, newSha: commit.Id.Sha);
+        Commit(message, DotnetBotCommitSignature);
+
+        _logger.LogInformation("Initialization of {name} finished", mapping.Name);
     }
 }

src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrManagerBase.cs

@@ -23,13 +23,17 @@ public abstract class VmrManagerBase
     private const string KeepAttribute = "vmr-preserve";
     private const string IgnoreAttribute = "vmr-ignore";
 
+    public const string VmrSourcesPath = "src";
+    public const string SourceMappingsFileName = "source-mappings.json";
+
     private readonly ILogger<VmrUpdater> _logger;
     private readonly IProcessManager _processManager;
     private readonly IRemoteFactory _remoteFactory;
-    private readonly string _vmrPath;
     private readonly string _tagsPath;
     private readonly string _tmpPath;
 
+    protected string VmrPath { get; }
+
     protected string SourcesPath { get; }
 
     public IReadOnlyCollection<SourceMapping> Mappings { get; }
@@ -46,9 +50,9 @@ protected VmrManagerBase(
         _processManager = processManager;
         _remoteFactory = remoteFactory;
         _tmpPath = tmpPath;
-        _vmrPath = vmrPath;
-        SourcesPath = Path.Join(vmrPath, "src");
-        _tagsPath = Path.Join(SourcesPath, ".tags");
+        VmrPath = vmrPath;
+        SourcesPath = Path.Combine(vmrPath, "src");
+        _tagsPath = Path.Combine(SourcesPath, ".tags");
 
         Mappings = mappings;
     }
@@ -60,7 +64,7 @@ protected VmrManagerBase(
     /// <returns>Path to the cloned repo</returns>
     protected async Task<string> CloneOrPull(SourceMapping mapping)
     {
-        var clonePath = Path.Combine(_tmpPath, mapping.Name);
+        var clonePath = GetClonePath(mapping);
         if (Directory.Exists(clonePath))
         {
             _logger.LogInformation("Clone of {repo} found, pulling new changes...", mapping.DefaultRemote);
@@ -97,7 +101,7 @@ protected async Task TagRepo(SourceMapping mapping, string commitId)
         await File.WriteAllTextAsync(tagFile, commitId);
 
         // Stage the tag file
-        using var repository = new Repository(_vmrPath);
+        using var repository = new Repository(VmrPath);
         Commands.Stage(repository, tagFile);
     }
 
@@ -172,14 +176,14 @@ protected async Task ApplyPatch(SourceMapping mapping, string patchPath, Cancell
     {
         // We have to give git a relative path with forward slashes where to apply the patch
         var destPath = Path.Combine(SourcesPath, mapping.Name)
-            .Replace(_vmrPath, null)
+            .Replace(VmrPath, null)
             .Replace("\\", "/")
             [1..];
 
-        _logger.LogInformation("Applying patch to {path}...", destPath);
+        _logger.LogInformation("Applying patch {patchPath} to {path}...", patchPath, destPath);
 
         // This will help ignore some CR/LF issues (e.g. files with both endings)
-        (await _processManager.ExecuteGit(_vmrPath, new[] { "config", "apply.ignoreWhitespace", "change" }, cancellationToken: cancellationToken))
+        (await _processManager.ExecuteGit(VmrPath, new[] { "config", "apply.ignoreWhitespace", "change" }, cancellationToken: cancellationToken))
             .ThrowIfFailed("Failed to set git config whitespace settings");
 
         Directory.CreateDirectory(destPath);
@@ -205,7 +209,7 @@ protected async Task ApplyPatch(SourceMapping mapping, string patchPath, Cancell
             patchPath,
         };
 
-        var result = await _processManager.ExecuteGit(_vmrPath, args, cancellationToken: CancellationToken.None);
+        var result = await _processManager.ExecuteGit(VmrPath, args, cancellationToken: CancellationToken.None);
         result.ThrowIfFailed($"Failed to apply the patch for {destPath}");
         _logger.LogDebug("{output}", result.ToString());
 
@@ -214,17 +218,39 @@ protected async Task ApplyPatch(SourceMapping mapping, string patchPath, Cancell
         // This will end up having the working tree all staged
         _logger.LogInformation("Resetting the working tree...");
         args = new[] { "checkout", destPath };
-        result = await _processManager.ExecuteGit(_vmrPath, args, cancellationToken: CancellationToken.None);
+        result = await _processManager.ExecuteGit(VmrPath, args, cancellationToken: CancellationToken.None);
         result.ThrowIfFailed($"Failed to clean the working tree");
         _logger.LogDebug("{output}", result.ToString());
     }
 
+    /// <summary>
+    /// Applies VMR patches onto files of given mapping's subrepository.
+    /// </summary>
+    /// <param name="mapping">Mapping</param>
+    protected async Task ApplyVmrPatches(SourceMapping mapping, CancellationToken cancellationToken)
+    {
+        if (!mapping.VmrPatches.Any())
+        {
+            return;
+        }
+
+        _logger.LogInformation("Applying VMR patches for {mappingName}..", mapping.Name);
+
+        foreach (var patchFile in mapping.VmrPatches)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+
+            _logger.LogDebug("Applying {patch}..", patchFile);
+            await ApplyPatch(mapping, patchFile, cancellationToken);
+        }
+    }
+
     protected void Commit(string commitMessage, Signature author)
     {
         _logger.LogInformation("Committing..");
 
         var watch = Stopwatch.StartNew();
-        using var repository = new Repository(_vmrPath);
+        using var repository = new Repository(VmrPath);
         var commit = repository.Commit(commitMessage, author, DotnetBotCommitSignature);
 
         _logger.LogInformation("Created {sha} in {duration} seconds", ShortenId(commit.Id.Sha), (int) watch.Elapsed.TotalSeconds);
@@ -233,18 +259,33 @@ protected void Commit(string commitMessage, Signature author)
     protected string GetPatchFilePath(SourceMapping mapping) => Path.Combine(_tmpPath, $"{mapping.Name}.patch");
 
     protected string GetTagFilePath(SourceMapping mapping) => Path.Combine(_tagsPath, $".{mapping.Name}");
+
+    protected string GetClonePath(SourceMapping mapping) => Path.Combine(_tmpPath, mapping.Name);
 
-    protected static string PrepareCommitMessage(string template, SourceMapping mapping, string? oldSha, string? newSha, string? commitMessage)
+    /// <summary>
+    /// Takes a given commit message template and populates it with given values, URLs and others.
+    /// </summary>
+    /// <param name="template">Template into which the values are filled into</param>
+    /// <param name="mapping">Repository mapping</param>
+    /// <param name="oldSha">SHA we are updating from</param>
+    /// <param name="newSha">SHA we are updating to</param>
+    /// <param name="additionalMessage">Additional message inserted in the commit body</param>
+    protected static string PrepareCommitMessage(
+        string template,
+        SourceMapping mapping,
+        string? oldSha = null,
+        string? newSha = null,
+        string? additionalMessage = null)
     {
         var replaces = new Dictionary<string, string?>
         {
             { "name", mapping.Name },
             { "remote", mapping.DefaultRemote },
             { "oldSha", oldSha },
             { "newSha", newSha },
-            { "oldShaShort", oldSha is null ? null : ShortenId(oldSha) },
-            { "newShaShort", newSha is null ? null : ShortenId(newSha) },
-            { "commitMessage", commitMessage },
+            { "oldShaShort", oldSha is null ? string.Empty : ShortenId(oldSha) },
+            { "newShaShort", newSha is null ? string.Empty : ShortenId(newSha) },
+            { "commitMessage", additionalMessage ?? string.Empty },
         };
 
         foreach (var replace in replaces)

src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrManagerFactory.cs

@@ -12,9 +12,9 @@ namespace Microsoft.DotNet.DarcLib.VirtualMonoRepo;
 public interface IVmrManagerFactory
 {
     Task<IVmrInitializer> CreateVmrInitializer();
-    Task<IVmrInitializer> CreateVmrInitializer(string vmrPath, string tmpPath);
+    Task<IVmrInitializer> CreateVmrInitializer(IVmrManagerConfiguration configuration);
     Task<IVmrUpdater> CreateVmrUpdater();
-    Task<IVmrUpdater> CreateVmrUpdater(string vmrPath, string tmpPath);
+    Task<IVmrUpdater> CreateVmrUpdater(IVmrManagerConfiguration configuration);
 }
 
 public class VmrManagerFactory : IVmrManagerFactory
@@ -36,24 +36,24 @@ public VmrManagerFactory(
     public Task<IVmrInitializer> CreateVmrInitializer()
         => CreateVmrManager<IVmrInitializer, VmrInitializer>();
 
-    public Task<IVmrInitializer> CreateVmrInitializer(string vmrPath, string tmpPath)
-        => CreateVmrManager<IVmrInitializer, VmrInitializer>(vmrPath, tmpPath);
+    public Task<IVmrInitializer> CreateVmrInitializer(IVmrManagerConfiguration configuration)
+        => CreateVmrManager<IVmrInitializer, VmrInitializer>(configuration);
 
     public Task<IVmrUpdater> CreateVmrUpdater()
         => CreateVmrManager<IVmrUpdater, VmrUpdater>();
 
-    public Task<IVmrUpdater> CreateVmrUpdater(string vmrPath, string tmpPath)
-        => CreateVmrManager<IVmrUpdater, VmrUpdater>(vmrPath, tmpPath);
+    public Task<IVmrUpdater> CreateVmrUpdater(IVmrManagerConfiguration configuration)
+        => CreateVmrManager<IVmrUpdater, VmrUpdater>(configuration);
 
     private async Task<R> CreateVmrManager<R, T>() where T : R
     {
         var mappings = await _sourceMappingParser.ParseMappings(_configuration.VmrPath);
         return ActivatorUtilities.CreateInstance<T>(_serviceProvider, mappings);
     }
 
-    private async Task<R> CreateVmrManager<R, T>(string vmrPath, string tmpPath) where T : R
+    private async Task<R> CreateVmrManager<R, T>(IVmrManagerConfiguration configuration) where T : R
     {
-        var mappings = await _sourceMappingParser.ParseMappings(_configuration.VmrPath);
-        return ActivatorUtilities.CreateInstance<T>(_serviceProvider, new VmrManagerConfiguration(vmrPath, tmpPath), mappings);
+        var mappings = await _sourceMappingParser.ParseMappings(configuration.VmrPath);
+        return ActivatorUtilities.CreateInstance<T>(_serviceProvider, configuration, mappings);
     }
 }