From 267384301e3089523bd7dd2525323821dd15209a Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Tue, 23 Aug 2022 16:36:31 +0200 Subject: [PATCH 1/6] Add source build patch support to VMR manager --- .../Models/VirtualMonoRepo/SourceMapping.cs | 1 + .../VirtualMonoRepo/SourceMappingParser.cs | 25 +++-- .../DarcLib/VirtualMonoRepo/VmrInitializer.cs | 6 +- .../DarcLib/VirtualMonoRepo/VmrManagerBase.cs | 53 +++++++--- .../src/DarcLib/VirtualMonoRepo/VmrUpdater.cs | 96 ++++++++++++++++++- 5 files changed, 156 insertions(+), 25 deletions(-) diff --git a/src/Microsoft.DotNet.Darc/src/DarcLib/Models/VirtualMonoRepo/SourceMapping.cs b/src/Microsoft.DotNet.Darc/src/DarcLib/Models/VirtualMonoRepo/SourceMapping.cs index 4c5c7bfa45..6543500557 100644 --- a/src/Microsoft.DotNet.Darc/src/DarcLib/Models/VirtualMonoRepo/SourceMapping.cs +++ b/src/Microsoft.DotNet.Darc/src/DarcLib/Models/VirtualMonoRepo/SourceMapping.cs @@ -16,4 +16,5 @@ public record SourceMapping( string DefaultRef, IReadOnlyCollection Include, IReadOnlyCollection Exclude, + IReadOnlyCollection VmrPatches, string? Version = null); diff --git a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/SourceMappingParser.cs b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/SourceMappingParser.cs index dc91c37a1d..9fac43231a 100644 --- a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/SourceMappingParser.cs +++ b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/SourceMappingParser.cs @@ -21,20 +21,20 @@ public interface ISourceMappingParser public class SourceMappingParser : ISourceMappingParser { - private const string SourceMappingsFileName = "source-mappings.json"; - public async Task> ParseMappings(string vmrPath) { - var mappingFilePath = Path.Combine(vmrPath, "src", SourceMappingsFileName); + var mappingFilePath = Path.Combine(vmrPath, VmrManagerBase.VmrSourcesPath, VmrManagerBase.SourceMappingsFileName); var mappingFile = new FileInfo(mappingFilePath); if (!mappingFile.Exists) { throw new FileNotFoundException( - $"Failed to find {SourceMappingsFileName} file in the VMR directory", + $"Failed to find {VmrManagerBase.SourceMappingsFileName} file in the VMR directory", mappingFilePath); } + var patchesPath = Path.Join(vmrPath, VmrManagerBase.VmrPatchesPath); + var options = new JsonSerializerOptions { AllowTrailingCommas = true, @@ -43,14 +43,14 @@ public async Task> ParseMappings(string vmrPa using var stream = File.Open(mappingFile.FullName, FileMode.Open); var settings = await JsonSerializer.DeserializeAsync(stream, options) - ?? throw new Exception($"Failed to deserialize {SourceMappingsFileName}"); + ?? throw new Exception($"Failed to deserialize {VmrManagerBase.SourceMappingsFileName}"); return settings.Mappings - .Select(m => CreateMapping(settings.Defaults, m)) + .Select(mapping => CreateMapping(settings.Defaults, mapping, Path.Join(patchesPath, mapping.Name))) .ToImmutableArray(); } - private static SourceMapping CreateMapping(SourceMappingSetting defaults, SourceMappingSetting setting) + private static SourceMapping CreateMapping(SourceMappingSetting defaults, SourceMappingSetting setting, string patchesPath) { IEnumerable include = setting.Include ?? Enumerable.Empty(); IEnumerable exclude = setting.Exclude ?? Enumerable.Empty(); @@ -68,13 +68,18 @@ private static SourceMapping CreateMapping(SourceMappingSetting defaults, Source } } + var vmrPatches = Directory.Exists(patchesPath) + ? Directory.GetFiles(patchesPath) + : Array.Empty(); + return new SourceMapping( - Name: setting.Name ?? throw new InvalidOperationException($"Missing `name` in {SourceMappingsFileName}"), + Name: setting.Name ?? throw new InvalidOperationException($"Missing `name` in {VmrManagerBase.SourceMappingsFileName}"), Version: setting.Version, - DefaultRemote: setting.DefaultRemote ?? throw new InvalidOperationException($"Missing `defaultRemote` in {SourceMappingsFileName}"), + DefaultRemote: setting.DefaultRemote ?? throw new InvalidOperationException($"Missing `defaultRemote` in {VmrManagerBase.SourceMappingsFileName}"), DefaultRef: setting.DefaultRef ?? defaults.DefaultRef ?? "main", Include: include.ToImmutableArray(), - Exclude: exclude.ToImmutableArray()); + Exclude: exclude.ToImmutableArray(), + VmrPatches: vmrPatches.ToImmutableArray()); } private class SourceMappingFile diff --git a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrInitializer.cs b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrInitializer.cs index eb04324c4e..7802877e8c 100644 --- a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrInitializer.cs +++ b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrInitializer.cs @@ -59,11 +59,13 @@ public async Task InitializeVmr(SourceMapping mapping, string? targetRevision, C await ApplyPatch(mapping, patchPath, cancellationToken); cancellationToken.ThrowIfCancellationRequested(); await TagRepo(mapping, commit.Id.Sha); + cancellationToken.ThrowIfCancellationRequested(); + await ApplyVmrPatches(mapping, cancellationToken); - var description = PrepareCommitMessage(InitializationCommitMessage, mapping, null, commit.Id.Sha, null); + var message = PrepareCommitMessage(InitializationCommitMessage, mapping, oldSha: null, commit.Id.Sha, null); // Commit but do not add files (they were added to index directly) cancellationToken.ThrowIfCancellationRequested(); - Commit(description, DotnetBotCommitSignature); + Commit(message, DotnetBotCommitSignature); } } diff --git a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrManagerBase.cs b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrManagerBase.cs index a15cb00d4c..464b86169c 100644 --- a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrManagerBase.cs +++ b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrManagerBase.cs @@ -23,13 +23,18 @@ public abstract class VmrManagerBase private const string KeepAttribute = "vmr-preserve"; private const string IgnoreAttribute = "vmr-ignore"; + public const string VmrSourcesPath = "src"; + public const string VmrPatchesPath = "patches"; + public const string SourceMappingsFileName = "source-mappings.json"; + private readonly ILogger _logger; private readonly IProcessManager _processManager; private readonly IRemoteFactory _remoteFactory; - private readonly string _vmrPath; private readonly string _tagsPath; private readonly string _tmpPath; + protected string VmrPath { get; } + protected string SourcesPath { get; } public IReadOnlyCollection Mappings { get; } @@ -46,9 +51,9 @@ protected VmrManagerBase( _processManager = processManager; _remoteFactory = remoteFactory; _tmpPath = tmpPath; - _vmrPath = vmrPath; - SourcesPath = Path.Join(vmrPath, "src"); - _tagsPath = Path.Join(SourcesPath, ".tags"); + VmrPath = vmrPath; + SourcesPath = Path.Combine(vmrPath, "src"); + _tagsPath = Path.Combine(SourcesPath, ".tags"); Mappings = mappings; } @@ -60,7 +65,7 @@ protected VmrManagerBase( /// Path to the cloned repo protected async Task CloneOrPull(SourceMapping mapping) { - var clonePath = Path.Combine(_tmpPath, mapping.Name); + var clonePath = GetClonePath(mapping); if (Directory.Exists(clonePath)) { _logger.LogInformation("Clone of {repo} found, pulling new changes...", mapping.DefaultRemote); @@ -97,7 +102,7 @@ protected async Task TagRepo(SourceMapping mapping, string commitId) await File.WriteAllTextAsync(tagFile, commitId); // Stage the tag file - using var repository = new Repository(_vmrPath); + using var repository = new Repository(VmrPath); Commands.Stage(repository, tagFile); } @@ -172,14 +177,14 @@ protected async Task ApplyPatch(SourceMapping mapping, string patchPath, Cancell { // We have to give git a relative path with forward slashes where to apply the patch var destPath = Path.Combine(SourcesPath, mapping.Name) - .Replace(_vmrPath, null) + .Replace(VmrPath, null) .Replace("\\", "/") [1..]; - _logger.LogInformation("Applying patch to {path}...", destPath); + _logger.LogInformation("Applying patch {patchPath} to {path}...", patchPath, destPath); // This will help ignore some CR/LF issues (e.g. files with both endings) - (await _processManager.ExecuteGit(_vmrPath, new[] { "config", "apply.ignoreWhitespace", "change" }, cancellationToken: cancellationToken)) + (await _processManager.ExecuteGit(VmrPath, new[] { "config", "apply.ignoreWhitespace", "change" }, cancellationToken: cancellationToken)) .ThrowIfFailed("Failed to set git config whitespace settings"); Directory.CreateDirectory(destPath); @@ -205,7 +210,7 @@ protected async Task ApplyPatch(SourceMapping mapping, string patchPath, Cancell patchPath, }; - var result = await _processManager.ExecuteGit(_vmrPath, args, cancellationToken: CancellationToken.None); + var result = await _processManager.ExecuteGit(VmrPath, args, cancellationToken: CancellationToken.None); result.ThrowIfFailed($"Failed to apply the patch for {destPath}"); _logger.LogDebug("{output}", result.ToString()); @@ -214,17 +219,39 @@ protected async Task ApplyPatch(SourceMapping mapping, string patchPath, Cancell // This will end up having the working tree all staged _logger.LogInformation("Resetting the working tree..."); args = new[] { "checkout", destPath }; - result = await _processManager.ExecuteGit(_vmrPath, args, cancellationToken: CancellationToken.None); + result = await _processManager.ExecuteGit(VmrPath, args, cancellationToken: CancellationToken.None); result.ThrowIfFailed($"Failed to clean the working tree"); _logger.LogDebug("{output}", result.ToString()); } + /// + /// Applies VMR patches onto files of given mapping's subrepository. + /// + /// Mapping + protected async Task ApplyVmrPatches(SourceMapping mapping, CancellationToken cancellationToken) + { + if (!mapping.VmrPatches.Any()) + { + return; + } + + _logger.LogInformation("Applying VMR patches for {mappingName}..", mapping.Name); + + foreach (var patchFile in mapping.VmrPatches) + { + cancellationToken.ThrowIfCancellationRequested(); + + _logger.LogDebug("Applying {patch}..", patchFile); + await ApplyPatch(mapping, patchFile, cancellationToken); + } + } + protected void Commit(string commitMessage, Signature author) { _logger.LogInformation("Committing.."); var watch = Stopwatch.StartNew(); - using var repository = new Repository(_vmrPath); + using var repository = new Repository(VmrPath); var commit = repository.Commit(commitMessage, author, DotnetBotCommitSignature); _logger.LogInformation("Created {sha} in {duration} seconds", ShortenId(commit.Id.Sha), (int) watch.Elapsed.TotalSeconds); @@ -233,6 +260,8 @@ protected void Commit(string commitMessage, Signature author) protected string GetPatchFilePath(SourceMapping mapping) => Path.Combine(_tmpPath, $"{mapping.Name}.patch"); protected string GetTagFilePath(SourceMapping mapping) => Path.Combine(_tagsPath, $".{mapping.Name}"); + + protected string GetClonePath(SourceMapping mapping) => Path.Combine(_tmpPath, mapping.Name); protected static string PrepareCommitMessage(string template, SourceMapping mapping, string? oldSha, string? newSha, string? commitMessage) { diff --git a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrUpdater.cs b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrUpdater.cs index 7e787ac2ce..fc530a8b3f 100644 --- a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrUpdater.cs +++ b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrUpdater.cs @@ -5,7 +5,9 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Text; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using LibGit2Sharp; @@ -39,7 +41,16 @@ public class VmrUpdater : VmrManagerBase, IVmrUpdater {commitMessage} """; + /// + /// Matches output of `git apply --numstat` which lists files contained in a patch file. + /// Example output: + /// 0 14 /s/vmr/src/roslyn-analyzers/eng/Versions.props + /// - - /s/vmr/src/roslyn-analyzers/some-binary.dll + /// + private static readonly Regex GitPatchSummaryLine = new(@"^[\-0-9]+\s+[\-0-9]+\s+(?[^\s]+)$", RegexOptions.Compiled); + private readonly ILogger _logger; + private readonly IProcessManager _processManager; private readonly IRemoteFactory _remoteFactory; public VmrUpdater( @@ -51,6 +62,7 @@ public VmrUpdater( : base(processManager, remoteFactory, logger, mappings, configuration.VmrPath, configuration.TmpPath) { _logger = logger; + _processManager = processManager; _remoteFactory = remoteFactory; } @@ -217,6 +229,9 @@ private async Task UpdateRepoToRevision( throw new InvalidOperationException($"Failed to find the patch at {patchPath}"); } + await RestorePatchedFilesFromRepo(mapping, fromRevision, cancellationToken); + cancellationToken.ThrowIfCancellationRequested(); + if (info.Length == 0) { _logger.LogInformation("No changes for {repo} in given commits (maybe only excluded files changed?)", mapping.Name); @@ -227,7 +242,9 @@ private async Task UpdateRepoToRevision( } await TagRepo(mapping, toRevision); - + cancellationToken.ThrowIfCancellationRequested(); + await ApplyVmrPatches(mapping, cancellationToken); + cancellationToken.ThrowIfCancellationRequested(); Commit(commitMessage, author); } @@ -240,4 +257,81 @@ private async Task HasRemoteUpdates(SourceMapping mapping, string currentS var lastCommit = await remoteRepo.GetLatestCommitAsync(mapping.DefaultRemote, mapping.DefaultRef); return !lastCommit.Equals(currentSha, StringComparison.InvariantCultureIgnoreCase); } + + /// + /// For all files for which we have patches in VMR, restore their original version from the repo. + /// This is because VMR contains already patched versions of these files and new updates from the repo wouldn't apply. + /// + /// Mapping + /// Revision from which we were updating + private async Task RestorePatchedFilesFromRepo(SourceMapping mapping, string originalRevision, CancellationToken cancellationToken) + { + if (!mapping.VmrPatches.Any()) + { + return; + } + + _logger.LogInformation("Restoring files with patches for {mappingName}..", mapping.Name); + + // We checkout the clone to the given revision once for all its patches + var clonePath = GetClonePath(mapping); + if (!Directory.Exists(clonePath)) + { + await CloneOrPull(mapping); + } + + var localRepo = new LocalGitClient(_processManager.GitExecutable, _logger); + localRepo.Checkout(clonePath, originalRevision); + + foreach (var patch in mapping.VmrPatches) + { + _logger.LogDebug("Processing VMR patch `{patch}`..", patch); + + foreach (var patchedFile in await GetFilesInPatch(clonePath, patch, cancellationToken)) + { + // git always works with forward slashes (even on Windows) + string relativePath = Path.DirectorySeparatorChar != '/' + ? patchedFile.Replace('/', Path.DirectorySeparatorChar) + : patchedFile; + + var originalFile = Path.Combine(clonePath, relativePath); + var destination = Path.Combine(SourcesPath, mapping.Name, relativePath); + + _logger.LogDebug("Restoring file `{originalFile}` to `{destination}`..", originalFile, destination); + + // Copy old revision to VMR + File.Copy(originalFile, destination, overwrite: true); + } + } + + // Stage the restored files (all future patches are applied to index directly) + using var repository = new Repository(VmrPath); + Commands.Stage(repository, $"{VmrSourcesPath}/{mapping.Name}"); + + _logger.LogDebug("Files from VMR patches for {mappingName} restored", mapping.Name); + } + + /// + /// Resolves a list of all files that are part of a given patch diff. + /// + /// Path (to the repo) the patch applies onto + /// Path to the patch file + /// List of all files (paths relative to repo's root) that are part of a given patch diff + private async Task> GetFilesInPatch(string repoPath, string patchPath, CancellationToken cancellationToken) + { + var result = await _processManager.ExecuteGit(repoPath, new[] { "apply", "--numstat", patchPath }, cancellationToken); + result.ThrowIfFailed($"Failed to enumerate files from a patch at `{patchPath}`"); + + var files = new List(); + foreach (var line in result.StandardOutput.Split(Environment.NewLine)) + { + var match = GitPatchSummaryLine.Match(line); + if (match.Success) + { + files.Add(match.Groups["file"].Value); + } + } + + return files; + } } From 352b2fd8630799315d43a00d2461921b01371d03 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Tue, 23 Aug 2022 18:34:45 +0200 Subject: [PATCH 2/6] Read path with patches from source-mappings.json --- .../VirtualMonoRepo/SourceMappingParser.cs | 18 ++++++++++++------ .../DarcLib/VirtualMonoRepo/VmrManagerBase.cs | 1 - .../VirtualMonoRepo/VmrManagerFactory.cs | 18 +++++++++--------- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/SourceMappingParser.cs b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/SourceMappingParser.cs index 9fac43231a..bfc3a3813e 100644 --- a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/SourceMappingParser.cs +++ b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/SourceMappingParser.cs @@ -33,8 +33,6 @@ public async Task> ParseMappings(string vmrPa mappingFilePath); } - var patchesPath = Path.Join(vmrPath, VmrManagerBase.VmrPatchesPath); - var options = new JsonSerializerOptions { AllowTrailingCommas = true, @@ -45,12 +43,18 @@ public async Task> ParseMappings(string vmrPa var settings = await JsonSerializer.DeserializeAsync(stream, options) ?? throw new Exception($"Failed to deserialize {VmrManagerBase.SourceMappingsFileName}"); + var patchesPath = settings.PatchesPath; + if (patchesPath is not null) + { + patchesPath = Path.Combine(vmrPath, patchesPath.Replace('/', Path.DirectorySeparatorChar)); + } + return settings.Mappings .Select(mapping => CreateMapping(settings.Defaults, mapping, Path.Join(patchesPath, mapping.Name))) .ToImmutableArray(); } - private static SourceMapping CreateMapping(SourceMappingSetting defaults, SourceMappingSetting setting, string patchesPath) + private static SourceMapping CreateMapping(SourceMappingSetting defaults, SourceMappingSetting setting, string? patchesPath) { IEnumerable include = setting.Include ?? Enumerable.Empty(); IEnumerable exclude = setting.Exclude ?? Enumerable.Empty(); @@ -68,14 +72,14 @@ private static SourceMapping CreateMapping(SourceMappingSetting defaults, Source } } - var vmrPatches = Directory.Exists(patchesPath) + var vmrPatches = patchesPath is not null && Directory.Exists(patchesPath) ? Directory.GetFiles(patchesPath) : Array.Empty(); return new SourceMapping( - Name: setting.Name ?? throw new InvalidOperationException($"Missing `name` in {VmrManagerBase.SourceMappingsFileName}"), + Name: setting.Name ?? throw new InvalidOperationException($"Missing `{nameof(SourceMapping.Name).ToLower()}` in {VmrManagerBase.SourceMappingsFileName}"), Version: setting.Version, - DefaultRemote: setting.DefaultRemote ?? throw new InvalidOperationException($"Missing `defaultRemote` in {VmrManagerBase.SourceMappingsFileName}"), + DefaultRemote: setting.DefaultRemote ?? throw new InvalidOperationException($"Missing `{nameof(SourceMapping.DefaultRemote).ToLower()}` in {VmrManagerBase.SourceMappingsFileName}"), DefaultRef: setting.DefaultRef ?? defaults.DefaultRef ?? "main", Include: include.ToImmutableArray(), Exclude: exclude.ToImmutableArray(), @@ -91,6 +95,8 @@ private class SourceMappingFile Exclude = Array.Empty(), }; + public string? PatchesPath { get; set; } + public List Mappings { get; set; } = new(); } diff --git a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrManagerBase.cs b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrManagerBase.cs index 464b86169c..6d6d8b27d7 100644 --- a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrManagerBase.cs +++ b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrManagerBase.cs @@ -24,7 +24,6 @@ public abstract class VmrManagerBase private const string IgnoreAttribute = "vmr-ignore"; public const string VmrSourcesPath = "src"; - public const string VmrPatchesPath = "patches"; public const string SourceMappingsFileName = "source-mappings.json"; private readonly ILogger _logger; diff --git a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrManagerFactory.cs b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrManagerFactory.cs index 80757f7832..b98fce9dab 100644 --- a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrManagerFactory.cs +++ b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrManagerFactory.cs @@ -12,9 +12,9 @@ namespace Microsoft.DotNet.DarcLib.VirtualMonoRepo; public interface IVmrManagerFactory { Task CreateVmrInitializer(); - Task CreateVmrInitializer(string vmrPath, string tmpPath); + Task CreateVmrInitializer(IVmrManagerConfiguration configuration); Task CreateVmrUpdater(); - Task CreateVmrUpdater(string vmrPath, string tmpPath); + Task CreateVmrUpdater(IVmrManagerConfiguration configuration); } public class VmrManagerFactory : IVmrManagerFactory @@ -36,14 +36,14 @@ public VmrManagerFactory( public Task CreateVmrInitializer() => CreateVmrManager(); - public Task CreateVmrInitializer(string vmrPath, string tmpPath) - => CreateVmrManager(vmrPath, tmpPath); + public Task CreateVmrInitializer(IVmrManagerConfiguration configuration) + => CreateVmrManager(configuration); public Task CreateVmrUpdater() => CreateVmrManager(); - public Task CreateVmrUpdater(string vmrPath, string tmpPath) - => CreateVmrManager(vmrPath, tmpPath); + public Task CreateVmrUpdater(IVmrManagerConfiguration configuration) + => CreateVmrManager(configuration); private async Task CreateVmrManager() where T : R { @@ -51,9 +51,9 @@ private async Task CreateVmrManager() where T : R return ActivatorUtilities.CreateInstance(_serviceProvider, mappings); } - private async Task CreateVmrManager(string vmrPath, string tmpPath) where T : R + private async Task CreateVmrManager(IVmrManagerConfiguration configuration) where T : R { - var mappings = await _sourceMappingParser.ParseMappings(_configuration.VmrPath); - return ActivatorUtilities.CreateInstance(_serviceProvider, new VmrManagerConfiguration(vmrPath, tmpPath), mappings); + var mappings = await _sourceMappingParser.ParseMappings(configuration.VmrPath); + return ActivatorUtilities.CreateInstance(_serviceProvider, configuration, mappings); } } From 102c1369ad7e41d30c6344c054f9d1fa55ab7a50 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Wed, 24 Aug 2022 14:23:00 +0200 Subject: [PATCH 3/6] Make nullable PrepareCommitMessage parameters optional --- .../DarcLib/VirtualMonoRepo/VmrInitializer.cs | 2 +- .../DarcLib/VirtualMonoRepo/VmrManagerBase.cs | 21 +++++++++++++++---- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrInitializer.cs b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrInitializer.cs index 7802877e8c..ff98d01051 100644 --- a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrInitializer.cs +++ b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrInitializer.cs @@ -62,7 +62,7 @@ public async Task InitializeVmr(SourceMapping mapping, string? targetRevision, C cancellationToken.ThrowIfCancellationRequested(); await ApplyVmrPatches(mapping, cancellationToken); - var message = PrepareCommitMessage(InitializationCommitMessage, mapping, oldSha: null, commit.Id.Sha, null); + var message = PrepareCommitMessage(InitializationCommitMessage, mapping, newSha: commit.Id.Sha); // Commit but do not add files (they were added to index directly) cancellationToken.ThrowIfCancellationRequested(); diff --git a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrManagerBase.cs b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrManagerBase.cs index 6d6d8b27d7..ea79138e39 100644 --- a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrManagerBase.cs +++ b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrManagerBase.cs @@ -262,7 +262,20 @@ protected void Commit(string commitMessage, Signature author) protected string GetClonePath(SourceMapping mapping) => Path.Combine(_tmpPath, mapping.Name); - protected static string PrepareCommitMessage(string template, SourceMapping mapping, string? oldSha, string? newSha, string? commitMessage) + /// + /// Takes a given commit message template and populates it with given values, URLs and others. + /// + /// Template into which the values are filled into + /// Repository mapping + /// SHA we are updating from + /// SHA we are updating to + /// Additional message inserted in the commit body + protected static string PrepareCommitMessage( + string template, + SourceMapping mapping, + string? oldSha = null, + string? newSha = null, + string? additionalMessage = null) { var replaces = new Dictionary { @@ -270,9 +283,9 @@ protected static string PrepareCommitMessage(string template, SourceMapping mapp { "remote", mapping.DefaultRemote }, { "oldSha", oldSha }, { "newSha", newSha }, - { "oldShaShort", oldSha is null ? null : ShortenId(oldSha) }, - { "newShaShort", newSha is null ? null : ShortenId(newSha) }, - { "commitMessage", commitMessage }, + { "oldShaShort", oldSha is null ? string.Empty : ShortenId(oldSha) }, + { "newShaShort", newSha is null ? string.Empty : ShortenId(newSha) }, + { "commitMessage", additionalMessage ?? string.Empty }, }; foreach (var replace in replaces) From c04d1e9281e611970a2df918882be018191c7aaa Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Wed, 24 Aug 2022 15:28:14 +0200 Subject: [PATCH 4/6] Log which revision is being initialized --- .../src/DarcLib/VirtualMonoRepo/VmrInitializer.cs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrInitializer.cs b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrInitializer.cs index ff98d01051..d21fd6873f 100644 --- a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrInitializer.cs +++ b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrInitializer.cs @@ -44,7 +44,7 @@ public async Task InitializeVmr(SourceMapping mapping, string? targetRevision, C throw new EmptySyncException($"Repository {mapping.Name} already exists"); } - _logger.LogInformation("Initializing {name}", mapping.Name); + _logger.LogInformation("Initializing {name} at {revision}", mapping.Name, targetRevision ?? mapping.DefaultRef); string clonePath = await CloneOrPull(mapping); string patchPath = GetPatchFilePath(mapping); @@ -56,16 +56,18 @@ public async Task InitializeVmr(SourceMapping mapping, string? targetRevision, C await CreatePatch(mapping, clonePath, Constants.EmptyGitObject, commit.Id.Sha, patchPath, cancellationToken); cancellationToken.ThrowIfCancellationRequested(); + await ApplyPatch(mapping, patchPath, cancellationToken); cancellationToken.ThrowIfCancellationRequested(); + await TagRepo(mapping, commit.Id.Sha); cancellationToken.ThrowIfCancellationRequested(); - await ApplyVmrPatches(mapping, cancellationToken); - var message = PrepareCommitMessage(InitializationCommitMessage, mapping, newSha: commit.Id.Sha); + await ApplyVmrPatches(mapping, cancellationToken); + cancellationToken.ThrowIfCancellationRequested(); // Commit but do not add files (they were added to index directly) - cancellationToken.ThrowIfCancellationRequested(); + var message = PrepareCommitMessage(InitializationCommitMessage, mapping, newSha: commit.Id.Sha); Commit(message, DotnetBotCommitSignature); } } From dd784f24ebca4e9ab09f7140a75484e7e8614754 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Wed, 24 Aug 2022 15:29:52 +0200 Subject: [PATCH 5/6] Log which revision is being initialized --- .../src/DarcLib/VirtualMonoRepo/VmrInitializer.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrInitializer.cs b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrInitializer.cs index d21fd6873f..ebe5110b3b 100644 --- a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrInitializer.cs +++ b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrInitializer.cs @@ -47,13 +47,12 @@ public async Task InitializeVmr(SourceMapping mapping, string? targetRevision, C _logger.LogInformation("Initializing {name} at {revision}", mapping.Name, targetRevision ?? mapping.DefaultRef); string clonePath = await CloneOrPull(mapping); - string patchPath = GetPatchFilePath(mapping); - cancellationToken.ThrowIfCancellationRequested(); using var clone = new Repository(clonePath); var commit = GetCommit(clone, (targetRevision is null || targetRevision == HEAD) ? null : targetRevision); + string patchPath = GetPatchFilePath(mapping); await CreatePatch(mapping, clonePath, Constants.EmptyGitObject, commit.Id.Sha, patchPath, cancellationToken); cancellationToken.ThrowIfCancellationRequested(); From bac5df2adbf7ee31c51eb8148a0fcd70c1f6663d Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Wed, 24 Aug 2022 15:30:47 +0200 Subject: [PATCH 6/6] Log which revision is being initialized --- .../src/DarcLib/VirtualMonoRepo/VmrInitializer.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrInitializer.cs b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrInitializer.cs index ebe5110b3b..00b30167fa 100644 --- a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrInitializer.cs +++ b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrInitializer.cs @@ -44,7 +44,7 @@ public async Task InitializeVmr(SourceMapping mapping, string? targetRevision, C throw new EmptySyncException($"Repository {mapping.Name} already exists"); } - _logger.LogInformation("Initializing {name} at {revision}", mapping.Name, targetRevision ?? mapping.DefaultRef); + _logger.LogInformation("Initializing {name} at {revision}..", mapping.Name, targetRevision ?? mapping.DefaultRef); string clonePath = await CloneOrPull(mapping); cancellationToken.ThrowIfCancellationRequested(); @@ -68,5 +68,7 @@ public async Task InitializeVmr(SourceMapping mapping, string? targetRevision, C // Commit but do not add files (they were added to index directly) var message = PrepareCommitMessage(InitializationCommitMessage, mapping, newSha: commit.Id.Sha); Commit(message, DotnetBotCommitSignature); + + _logger.LogInformation("Initialization of {name} finished", mapping.Name); } }