diff --git a/src/Microsoft.DotNet.Darc/src/Darc/Operations/VirtualMonoRepo/InitializeOperation.cs b/src/Microsoft.DotNet.Darc/src/Darc/Operations/VirtualMonoRepo/InitializeOperation.cs index 02d1e420dd..3d002e8834 100644 --- a/src/Microsoft.DotNet.Darc/src/Darc/Operations/VirtualMonoRepo/InitializeOperation.cs +++ b/src/Microsoft.DotNet.Darc/src/Darc/Operations/VirtualMonoRepo/InitializeOperation.cs @@ -25,5 +25,5 @@ protected override async Task ExecuteInternalAsync( bool recursive, CancellationToken cancellationToken) => - await vmrManager.InitializeRepository(mapping, targetRevision, recursive, cancellationToken); + await vmrManager.InitializeRepository(mapping, targetRevision, null, recursive, cancellationToken); } diff --git a/src/Microsoft.DotNet.Darc/src/Darc/Operations/VirtualMonoRepo/UpdateOperation.cs b/src/Microsoft.DotNet.Darc/src/Darc/Operations/VirtualMonoRepo/UpdateOperation.cs index ab73e8f8ca..366520ddcc 100644 --- a/src/Microsoft.DotNet.Darc/src/Darc/Operations/VirtualMonoRepo/UpdateOperation.cs +++ b/src/Microsoft.DotNet.Darc/src/Darc/Operations/VirtualMonoRepo/UpdateOperation.cs @@ -28,5 +28,5 @@ protected override async Task ExecuteInternalAsync( bool recursive, CancellationToken cancellationToken) => - await vmrManager.UpdateRepository(mapping, targetRevision, _options.NoSquash, recursive, cancellationToken); + await vmrManager.UpdateRepository(mapping, targetRevision, null, _options.NoSquash, recursive, cancellationToken); } diff --git a/src/Microsoft.DotNet.Darc/src/Darc/Operations/VirtualMonoRepo/VmrOperationBase.cs b/src/Microsoft.DotNet.Darc/src/Darc/Operations/VirtualMonoRepo/VmrOperationBase.cs index 5eca0ac81f..5c135c49a3 100644 --- a/src/Microsoft.DotNet.Darc/src/Darc/Operations/VirtualMonoRepo/VmrOperationBase.cs +++ b/src/Microsoft.DotNet.Darc/src/Darc/Operations/VirtualMonoRepo/VmrOperationBase.cs @@ -43,7 +43,7 @@ public override async Task ExecuteAsync() return Constants.ErrorCode; } - var vmrManager = Provider.GetRequiredService(); + TVmrManager vmrManager = Provider.GetRequiredService(); IEnumerable<(SourceMapping Mapping, string? Revision)> reposToSync; @@ -61,7 +61,7 @@ public override async Task ExecuteAsync() SourceMapping ResolveMapping(string repo) { - return vmrManager!.Mappings.FirstOrDefault(m => m.Name.Equals(repo, StringComparison.InvariantCultureIgnoreCase)) + return vmrManager.Mappings.FirstOrDefault(m => m.Name.Equals(repo, StringComparison.InvariantCultureIgnoreCase)) ?? throw new Exception($"No mapping named '{repo}' found"); } diff --git a/src/Microsoft.DotNet.Darc/src/DarcLib/Models/MsBuildPropsFile.cs b/src/Microsoft.DotNet.Darc/src/DarcLib/Models/MsBuildPropsFile.cs index a144ed48af..1db0e995f2 100644 --- a/src/Microsoft.DotNet.Darc/src/DarcLib/Models/MsBuildPropsFile.cs +++ b/src/Microsoft.DotNet.Darc/src/DarcLib/Models/MsBuildPropsFile.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Xml; using System.Xml.Linq; @@ -48,6 +49,12 @@ protected MsBuildPropsFile(bool? orderPropertiesAscending) public void SerializeToXml(string path) { + var directory = Path.GetDirectoryName(path) ?? throw new ArgumentException($"'{path}' is not a valid path."); + if (!Directory.Exists(directory)) + { + Directory.CreateDirectory(directory); + } + XmlSerializer serializer = new XmlSerializer(typeof(XmlElement)); var xmlDocument = new XmlDocument(); XmlElement root = xmlDocument.CreateElement(ProjectPropertyName); diff --git a/src/Microsoft.DotNet.Darc/src/DarcLib/Models/VirtualMonoRepo/AllVersionsPropsFile.cs b/src/Microsoft.DotNet.Darc/src/DarcLib/Models/VirtualMonoRepo/AllVersionsPropsFile.cs index 6a745f4d86..0fa8434aec 100644 --- a/src/Microsoft.DotNet.Darc/src/DarcLib/Models/VirtualMonoRepo/AllVersionsPropsFile.cs +++ b/src/Microsoft.DotNet.Darc/src/DarcLib/Models/VirtualMonoRepo/AllVersionsPropsFile.cs @@ -6,6 +6,7 @@ using System.Collections.Generic; using System.Xml; using Microsoft.DotNet.DarcLib.Models; +using Microsoft.DotNet.DarcLib.VirtualMonoRepo; #nullable enable namespace Microsoft.DotNet.Darc.Models.VirtualMonoRepo; @@ -14,8 +15,8 @@ public interface IAllVersionsPropsFile : IMsBuildPropsFile { Dictionary Versions { get; } - (string? Sha, string? Version) GetVersion(string repository); - void UpdateVersion(string repository, string sha, string version); + VmrDependencyVersion? GetVersion(string repository); + void UpdateVersion(string repository, string sha, string packageVersion); } /// @@ -36,21 +37,27 @@ public AllVersionsPropsFile(Dictionary versions) Versions = versions; } - public (string? Sha, string? Version) GetVersion(string repository) + public VmrDependencyVersion? GetVersion(string repository) { var key = SanitizePropertyName(repository) + ShaPropertyName; Versions.TryGetValue(key, out var sha); + if (sha is null) + { + return null; + } + key = SanitizePropertyName(repository) + PackageVersionPropertyName; Versions.TryGetValue(key, out var version); - return (sha, version); + + return new(sha, version); } - public void UpdateVersion(string repository, string sha, string version) + public void UpdateVersion(string repository, string sha, string packageVersion) { var key = SanitizePropertyName(repository); Versions[key + ShaPropertyName] = sha; - Versions[key + PackageVersionPropertyName] = version; + Versions[key + PackageVersionPropertyName] = packageVersion; } public static AllVersionsPropsFile DeserializeFromXml(string path) diff --git a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/IVmrInitializer.cs b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/IVmrInitializer.cs index d186adfeab..a606ce19c3 100644 --- a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/IVmrInitializer.cs +++ b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/IVmrInitializer.cs @@ -18,14 +18,20 @@ public interface IVmrInitializer : IVmrManager /// /// Name of a repository mapping /// Revision (commit SHA, branch, tag..) onto which to synchronize, leave empty for HEAD + /// Version of packages, that the SHA we're updating to, produced /// When true, initializes dependencies (from Version.Details.xml) recursively /// Cancellation token - Task InitializeRepository(string mappingName, string? targetRevision, bool initializeDependencies, CancellationToken cancellationToken) + Task InitializeRepository( + string mappingName, + string? targetRevision, + string? targetVersion, + bool initializeDependencies, + CancellationToken cancellationToken) { var mapping = Mappings.FirstOrDefault(m => m.Name == mappingName) ?? throw new Exception($"No repository mapping named `{mappingName}` found!"); - return InitializeRepository(mapping, targetRevision, initializeDependencies, cancellationToken); + return InitializeRepository(mapping, targetRevision, targetVersion, initializeDependencies, cancellationToken); } /// @@ -33,7 +39,13 @@ Task InitializeRepository(string mappingName, string? targetRevision, bool initi /// /// Repository mapping /// Revision (commit SHA, branch, tag..) onto which to synchronize, leave empty for HEAD + /// Version of packages, that the SHA we're updating to, produced /// When true, initializes dependencies (from Version.Details.xml) recursively /// Cancellation token - Task InitializeRepository(SourceMapping mapping, string? targetRevision, bool initializeDependencies, CancellationToken cancellationToken); + Task InitializeRepository( + SourceMapping mapping, + string? targetRevision, + string? targetVersion, + bool initializeDependencies, + CancellationToken cancellationToken); } diff --git a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/IVmrUpdater.cs b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/IVmrUpdater.cs index 7448f8d793..19a7ed4d14 100644 --- a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/IVmrUpdater.cs +++ b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/IVmrUpdater.cs @@ -18,12 +18,14 @@ public interface IVmrUpdater : IVmrManager /// /// Name of a repository mapping /// Revision (commit SHA, branch, tag..) onto which to synchronize, leave empty for HEAD + /// Version of packages, that the SHA we're updating to, produced /// Whether to pull changes commit by commit instead of squashing all updates into one /// When true, updates dependencies (from Version.Details.xml) recursively /// Cancellation token Task UpdateRepository( string mappingName, string? targetRevision, + string? targetVersion, bool noSquash, bool updateDependencies, CancellationToken cancellationToken) @@ -31,7 +33,7 @@ Task UpdateRepository( var mapping = Mappings.FirstOrDefault(m => m.Name == mappingName) ?? throw new Exception($"No repository mapping named `{mappingName}` found!"); - return UpdateRepository(mapping, targetRevision, noSquash, updateDependencies, cancellationToken); + return UpdateRepository(mapping, targetRevision, null, noSquash, updateDependencies, cancellationToken); } /// @@ -39,12 +41,14 @@ Task UpdateRepository( /// /// Repository mapping /// Revision (commit SHA, branch, tag..) onto which to synchronize, leave empty for HEAD + /// Version of packages, that the SHA we're updating to, produced /// Whether to pull changes commit by commit instead of squashing all updates into one /// When true, updates dependencies (from Version.Details.xml) recursively /// Cancellation token Task UpdateRepository( SourceMapping mapping, string? targetRevision, + string? targetVersion, bool noSquash, bool updateDependencies, CancellationToken cancellationToken); diff --git a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/SourceMappingParser.cs b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/SourceMappingParser.cs index bfc3a3813e..d09c4e823b 100644 --- a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/SourceMappingParser.cs +++ b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/SourceMappingParser.cs @@ -23,13 +23,13 @@ public class SourceMappingParser : ISourceMappingParser { public async Task> ParseMappings(string vmrPath) { - var mappingFilePath = Path.Combine(vmrPath, VmrManagerBase.VmrSourcesPath, VmrManagerBase.SourceMappingsFileName); + var mappingFilePath = Path.Combine(vmrPath, VmrDependencyTracker.VmrSourcesDir, VmrDependencyTracker.SourceMappingsFileName); var mappingFile = new FileInfo(mappingFilePath); if (!mappingFile.Exists) { throw new FileNotFoundException( - $"Failed to find {VmrManagerBase.SourceMappingsFileName} file in the VMR directory", + $"Failed to find {VmrDependencyTracker.SourceMappingsFileName} file in the VMR directory", mappingFilePath); } @@ -41,7 +41,7 @@ public async Task> ParseMappings(string vmrPa using var stream = File.Open(mappingFile.FullName, FileMode.Open); var settings = await JsonSerializer.DeserializeAsync(stream, options) - ?? throw new Exception($"Failed to deserialize {VmrManagerBase.SourceMappingsFileName}"); + ?? throw new Exception($"Failed to deserialize {VmrDependencyTracker.SourceMappingsFileName}"); var patchesPath = settings.PatchesPath; if (patchesPath is not null) @@ -56,6 +56,18 @@ public async Task> ParseMappings(string vmrPa private static SourceMapping CreateMapping(SourceMappingSetting defaults, SourceMappingSetting setting, string? patchesPath) { + if (setting.Name is null) + { + throw new InvalidOperationException( + $"Missing `{nameof(SourceMapping.Name).ToLower()}` in {VmrDependencyTracker.SourceMappingsFileName}"); + } + + if (setting.DefaultRemote is null) + { + throw new InvalidOperationException( + $"Missing `{nameof(SourceMapping.DefaultRemote).ToLower()}` in {VmrDependencyTracker.SourceMappingsFileName}"); + } + IEnumerable include = setting.Include ?? Enumerable.Empty(); IEnumerable exclude = setting.Exclude ?? Enumerable.Empty(); @@ -77,9 +89,9 @@ private static SourceMapping CreateMapping(SourceMappingSetting defaults, Source : Array.Empty(); return new SourceMapping( - Name: setting.Name ?? throw new InvalidOperationException($"Missing `{nameof(SourceMapping.Name).ToLower()}` in {VmrManagerBase.SourceMappingsFileName}"), + Name: setting.Name, Version: setting.Version, - DefaultRemote: setting.DefaultRemote ?? throw new InvalidOperationException($"Missing `{nameof(SourceMapping.DefaultRemote).ToLower()}` in {VmrManagerBase.SourceMappingsFileName}"), + DefaultRemote: setting.DefaultRemote, DefaultRef: setting.DefaultRef ?? defaults.DefaultRef ?? "main", Include: include.ToImmutableArray(), Exclude: exclude.ToImmutableArray(), diff --git a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrDependencyTracker.cs b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrDependencyTracker.cs new file mode 100644 index 0000000000..0fb9018dcc --- /dev/null +++ b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrDependencyTracker.cs @@ -0,0 +1,104 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using Microsoft.DotNet.Darc.Models.VirtualMonoRepo; + +#nullable enable +namespace Microsoft.DotNet.DarcLib.VirtualMonoRepo; + +public record VmrDependencyVersion(string Sha, string? PackageVersion); + +public interface IVmrDependencyTracker +{ + string VmrPath { get; } + + string SourcesPath { get; } + + IReadOnlyCollection Mappings { get; } + + string GetRepoSourcesPath(SourceMapping mapping) => Path.Combine(SourcesPath, mapping.Name); + + void UpdateDependencyVersion(SourceMapping mapping, VmrDependencyVersion version); + + VmrDependencyVersion? GetDependencyVersion(SourceMapping mapping); +} + +/// +/// Holds information about versions of individual repositories synchronized in the VMR. +/// Uses the AllRepoVersions.props file as source of truth and propagates changes into the git-info files. +/// +public class VmrDependencyTracker : IVmrDependencyTracker +{ + public const string SourceMappingsFileName = "source-mappings.json"; + public const string VmrSourcesDir = "src"; + public const string GitInfoSourcesDir = "git-info"; + + // TODO: https://github.com/dotnet/source-build/issues/2250 + private const string DefaultVersion = "8.0.100"; + + private readonly Lazy _repoVersions; + private readonly string _allVersionsFilePath; + + public string VmrPath { get; } + + public string SourcesPath { get; } + + public IReadOnlyCollection Mappings { get; } + + public VmrDependencyTracker( + IVmrManagerConfiguration configuration, + IReadOnlyCollection mappings) + { + VmrPath = configuration.VmrPath; + SourcesPath = Path.Combine(configuration.VmrPath, VmrSourcesDir); + Mappings = mappings; + + _allVersionsFilePath = Path.Combine(VmrPath, GitInfoSourcesDir, AllVersionsPropsFile.FileName); + _repoVersions = new Lazy(LoadAllVersionsFile, LazyThreadSafetyMode.ExecutionAndPublication); + } + + public VmrDependencyVersion? GetDependencyVersion(SourceMapping mapping) + => _repoVersions.Value.GetVersion(mapping.Name); + + public void UpdateDependencyVersion(SourceMapping mapping, VmrDependencyVersion version) + { + // TODO: https://github.com/dotnet/source-build/issues/2250 + if (version.PackageVersion is null) + { + version = version with { PackageVersion = DefaultVersion }; + } + + _repoVersions.Value.UpdateVersion(mapping.Name, version.Sha, version.PackageVersion); + _repoVersions.Value.SerializeToXml(_allVersionsFilePath); + + var (buildId, releaseLabel) = VersionFiles.DeriveBuildInfo(mapping.Name, version.PackageVersion); + + var gitInfo = new GitInfoFile + { + GitCommitHash = version.Sha, + OfficialBuildId = buildId, + PreReleaseVersionLabel = releaseLabel, + IsStable = string.IsNullOrWhiteSpace(releaseLabel), + OutputPackageVersion = version.PackageVersion, + }; + + gitInfo.SerializeToXml(GetGitInfoFilePath(mapping)); + } + + private AllVersionsPropsFile LoadAllVersionsFile() + { + if (!File.Exists(_allVersionsFilePath)) + { + return new AllVersionsPropsFile(new()); + } + + return AllVersionsPropsFile.DeserializeFromXml(_allVersionsFilePath); + } + + private string GetGitInfoFilePath(SourceMapping mapping) => Path.Combine(VmrPath, GitInfoSourcesDir, $"{mapping.Name}.props"); +} diff --git a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrInitializer.cs b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrInitializer.cs index 36cebff8ba..d888ceaf8f 100644 --- a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrInitializer.cs +++ b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrInitializer.cs @@ -2,7 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using System.Collections.Generic; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -23,28 +22,31 @@ public class VmrInitializer : VmrManagerBase, IVmrInitializer Original commit: {remote}/commit/{newSha} """; - + + private readonly IVmrDependencyTracker _dependencyTracker; private readonly ILogger _logger; public VmrInitializer( + IVmrDependencyTracker dependencyTracker, IProcessManager processManager, IRemoteFactory remoteFactory, IVersionDetailsParser versionDetailsParser, ILogger logger, - IVmrManagerConfiguration configuration, - IReadOnlyCollection mappings) - : base(processManager, remoteFactory, versionDetailsParser, logger, mappings, configuration.VmrPath, configuration.TmpPath) + IVmrManagerConfiguration configuration) + : base(dependencyTracker, processManager, remoteFactory, versionDetailsParser, logger, configuration.TmpPath) { + _dependencyTracker = dependencyTracker; _logger = logger; } public async Task InitializeRepository( SourceMapping mapping, string? targetRevision, + string? targetVersion, bool initializeDependencies, CancellationToken cancellationToken) { - if (File.Exists(GetTagFilePath(mapping))) + if (_dependencyTracker.GetDependencyVersion(mapping) is not null) { throw new EmptySyncException($"Repository {mapping.Name} already exists"); } @@ -64,7 +66,8 @@ public async Task InitializeRepository( await ApplyPatch(mapping, patchPath, cancellationToken); cancellationToken.ThrowIfCancellationRequested(); - await TagRepo(mapping, commit.Id.Sha); + _dependencyTracker.UpdateDependencyVersion(mapping, new(commit.Id.Sha, targetVersion)); + Commands.Stage(new Repository(_dependencyTracker.VmrPath), VmrDependencyTracker.GitInfoSourcesDir); cancellationToken.ThrowIfCancellationRequested(); await ApplyVmrPatches(mapping, cancellationToken); @@ -89,7 +92,7 @@ private async Task InitializeDependencies(SourceMapping mapping, CancellationTok { foreach (var (dependency, dependencyMapping) in await GetDependencies(mapping, cancellationToken)) { - if (Directory.Exists(GetRepoSourcesPath(dependencyMapping))) + if (Directory.Exists(_dependencyTracker.GetRepoSourcesPath(dependencyMapping))) { _logger.LogDebug("Dependency {repo} has already been initialized", dependencyMapping.Name); continue; @@ -100,7 +103,7 @@ private async Task InitializeDependencies(SourceMapping mapping, CancellationTok dependency.Commit, dependency.Version); - await InitializeRepository(dependencyMapping, dependency.Commit, true, cancellationToken); + await InitializeRepository(dependencyMapping, dependency.Commit, dependency.Version, true, cancellationToken); } } } diff --git a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrManagerBase.cs b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrManagerBase.cs index cb09134426..d837d8ceb8 100644 --- a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrManagerBase.cs +++ b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrManagerBase.cs @@ -18,47 +18,36 @@ #nullable enable namespace Microsoft.DotNet.DarcLib.VirtualMonoRepo; -public abstract class VmrManagerBase +public abstract class VmrManagerBase : IVmrManager { protected const string HEAD = "HEAD"; private const string KeepAttribute = "vmr-preserve"; private const string IgnoreAttribute = "vmr-ignore"; - public const string VmrSourcesPath = "src"; - public const string SourceMappingsFileName = "source-mappings.json"; - - private readonly ILogger _logger; + private readonly IVmrDependencyTracker _dependencyInfo; private readonly IProcessManager _processManager; private readonly IRemoteFactory _remoteFactory; private readonly IVersionDetailsParser _versionDetailsParser; - private readonly string _tagsPath; + private readonly ILogger _logger; + private readonly string _tmpPath; - protected string VmrPath { get; } - - protected string SourcesPath { get; } - - public IReadOnlyCollection Mappings { get; } + public IReadOnlyCollection Mappings => _dependencyInfo.Mappings; protected VmrManagerBase( + IVmrDependencyTracker dependencyInfo, IProcessManager processManager, IRemoteFactory remoteFactory, IVersionDetailsParser versionDetailsParser, ILogger logger, - IReadOnlyCollection mappings, - string vmrPath, string tmpPath) { _logger = logger; + _dependencyInfo = dependencyInfo; _processManager = processManager; _remoteFactory = remoteFactory; _versionDetailsParser = versionDetailsParser; _tmpPath = tmpPath; - VmrPath = vmrPath; - SourcesPath = Path.Combine(vmrPath, VmrSourcesPath); - _tagsPath = Path.Combine(SourcesPath, ".tags"); - - Mappings = mappings; } /// @@ -89,26 +78,6 @@ protected async Task CloneOrPull(SourceMapping mapping) return clonePath; } - /// - /// Notes down the current SHA the given repo is synchronized to into a file inside the VMR. - /// - /// Repository - /// SHA - protected async Task TagRepo(SourceMapping mapping, string commitId) - { - if (!Directory.Exists(_tagsPath)) - { - Directory.CreateDirectory(_tagsPath); - } - - var tagFile = GetTagFilePath(mapping); - await File.WriteAllTextAsync(tagFile, commitId); - - // Stage the tag file - using var repository = new Repository(VmrPath); - Commands.Stage(repository, tagFile); - } - /// /// Creates a patch file (a diff) for given two commits in a repo adhering to the in/exclusion filters of the mapping. /// @@ -179,15 +148,15 @@ protected async Task CreatePatch( protected async Task ApplyPatch(SourceMapping mapping, string patchPath, CancellationToken cancellationToken) { // We have to give git a relative path with forward slashes where to apply the patch - var destPath = GetRepoSourcesPath(mapping) - .Replace(VmrPath, null) + var destPath = _dependencyInfo.GetRepoSourcesPath(mapping) + .Replace(_dependencyInfo.VmrPath, null) .Replace("\\", "/") [1..]; _logger.LogInformation("Applying patch {patchPath} to {path}...", patchPath, destPath); // This will help ignore some CR/LF issues (e.g. files with both endings) - (await _processManager.ExecuteGit(VmrPath, new[] { "config", "apply.ignoreWhitespace", "change" }, cancellationToken: cancellationToken)) + (await _processManager.ExecuteGit(_dependencyInfo.VmrPath, new[] { "config", "apply.ignoreWhitespace", "change" }, cancellationToken: cancellationToken)) .ThrowIfFailed("Failed to set git config whitespace settings"); Directory.CreateDirectory(destPath); @@ -213,7 +182,7 @@ protected async Task ApplyPatch(SourceMapping mapping, string patchPath, Cancell patchPath, }; - var result = await _processManager.ExecuteGit(VmrPath, args, cancellationToken: CancellationToken.None); + var result = await _processManager.ExecuteGit(_dependencyInfo.VmrPath, args, cancellationToken: CancellationToken.None); result.ThrowIfFailed($"Failed to apply the patch for {destPath}"); _logger.LogDebug("{output}", result.ToString()); @@ -222,7 +191,7 @@ protected async Task ApplyPatch(SourceMapping mapping, string patchPath, Cancell // This will end up having the working tree all staged _logger.LogInformation("Resetting the working tree..."); args = new[] { "checkout", destPath }; - result = await _processManager.ExecuteGit(VmrPath, args, cancellationToken: CancellationToken.None); + result = await _processManager.ExecuteGit(_dependencyInfo.VmrPath, args, cancellationToken: CancellationToken.None); result.ThrowIfFailed($"Failed to clean the working tree"); _logger.LogDebug("{output}", result.ToString()); } @@ -264,14 +233,14 @@ protected async Task UpdateGitmodules(CancellationToken cancellationToken) // Matches the 'path = ' setting from the .gitmodules file so that we can prefix it var pathSettingRegex = new Regex(@"(\bpath[ \t]*\=[ \t]*\b)"); - using (var vmrGitmodule = File.Open(Path.Combine(VmrPath, gitmodulesFileName), FileMode.Create)) + using (var vmrGitmodule = File.Open(Path.Combine(_dependencyInfo.VmrPath, gitmodulesFileName), FileMode.Create)) using (var writer = new StreamWriter(vmrGitmodule) { NewLine = "\n" }) { foreach (var mapping in Mappings) { cancellationToken.ThrowIfCancellationRequested(); - var repoGitmodulePath = Path.Combine(GetRepoSourcesPath(mapping), gitmodulesFileName); + var repoGitmodulePath = Path.Combine(_dependencyInfo.GetRepoSourcesPath(mapping), gitmodulesFileName); if (!File.Exists(repoGitmodulePath)) { continue; @@ -289,7 +258,7 @@ protected async Task UpdateGitmodules(CancellationToken cancellationToken) // Add src/[repo]/ prefixes to paths content = pathSettingRegex - .Replace(content, $"$1{VmrSourcesPath}/{mapping.Name}/") + .Replace(content, $"$1{VmrDependencyTracker.VmrSourcesDir}/{mapping.Name}/") .Replace("\r\n", "\n"); await writer.WriteAsync(content); @@ -300,7 +269,7 @@ protected async Task UpdateGitmodules(CancellationToken cancellationToken) } } - (await _processManager.ExecuteGit(VmrPath, new[] { "add", gitmodulesFileName }, cancellationToken)) + (await _processManager.ExecuteGit(_dependencyInfo.VmrPath, new[] { "add", gitmodulesFileName }, cancellationToken)) .ThrowIfFailed("Failed to stage the .gitmodules file!"); } @@ -309,7 +278,7 @@ protected void Commit(string commitMessage, Signature author) _logger.LogInformation("Committing.."); var watch = Stopwatch.StartNew(); - using var repository = new Repository(VmrPath); + using var repository = new Repository(_dependencyInfo.VmrPath); var commit = repository.Commit(commitMessage, author, DotnetBotCommitSignature); _logger.LogInformation("Created {sha} in {duration} seconds", ShortenId(commit.Id.Sha), (int) watch.Elapsed.TotalSeconds); @@ -323,7 +292,7 @@ protected void Commit(string commitMessage, Signature author) CancellationToken cancellationToken) { var versionDetailsPath = Path.Combine( - GetRepoSourcesPath(mapping), + _dependencyInfo.GetRepoSourcesPath(mapping), VersionFiles.VersionDetailsXml.Replace('/', Path.DirectorySeparatorChar)); var versionDetailsContent = await File.ReadAllTextAsync(versionDetailsPath, cancellationToken); @@ -352,12 +321,8 @@ protected void Commit(string commitMessage, Signature author) protected string GetPatchFilePath(SourceMapping mapping) => Path.Combine(_tmpPath, $"{mapping.Name}.patch"); - protected string GetTagFilePath(SourceMapping mapping) => Path.Combine(_tagsPath, $".{mapping.Name}"); - protected string GetClonePath(SourceMapping mapping) => Path.Combine(_tmpPath, mapping.Name); - protected string GetRepoSourcesPath(SourceMapping mapping) => Path.Combine(SourcesPath, mapping.Name); - /// /// Takes a given commit message template and populates it with given values, URLs and others. /// diff --git a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrManagerFactory.cs b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrManagerFactory.cs deleted file mode 100644 index b98fce9dab..0000000000 --- a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrManagerFactory.cs +++ /dev/null @@ -1,59 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Threading.Tasks; -using Microsoft.Extensions.DependencyInjection; - -#nullable enable -namespace Microsoft.DotNet.DarcLib.VirtualMonoRepo; - -public interface IVmrManagerFactory -{ - Task CreateVmrInitializer(); - Task CreateVmrInitializer(IVmrManagerConfiguration configuration); - Task CreateVmrUpdater(); - Task CreateVmrUpdater(IVmrManagerConfiguration configuration); -} - -public class VmrManagerFactory : IVmrManagerFactory -{ - private readonly IServiceProvider _serviceProvider; - private readonly ISourceMappingParser _sourceMappingParser; - private readonly IVmrManagerConfiguration _configuration; - - public VmrManagerFactory( - IServiceProvider serviceProvider, - ISourceMappingParser sourceMappingParser, - IVmrManagerConfiguration configuration) - { - _serviceProvider = serviceProvider; - _sourceMappingParser = sourceMappingParser; - _configuration = configuration; - } - - public Task CreateVmrInitializer() - => CreateVmrManager(); - - public Task CreateVmrInitializer(IVmrManagerConfiguration configuration) - => CreateVmrManager(configuration); - - public Task CreateVmrUpdater() - => CreateVmrManager(); - - public Task CreateVmrUpdater(IVmrManagerConfiguration configuration) - => CreateVmrManager(configuration); - - private async Task CreateVmrManager() where T : R - { - var mappings = await _sourceMappingParser.ParseMappings(_configuration.VmrPath); - return ActivatorUtilities.CreateInstance(_serviceProvider, mappings); - } - - private async Task CreateVmrManager(IVmrManagerConfiguration configuration) where T : R - { - var mappings = await _sourceMappingParser.ParseMappings(configuration.VmrPath); - return ActivatorUtilities.CreateInstance(_serviceProvider, configuration, mappings); - } -} diff --git a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrRegistrations.cs b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrRegistrations.cs index d48fd65647..b49a55aec4 100644 --- a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrRegistrations.cs +++ b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrRegistrations.cs @@ -3,6 +3,8 @@ // See the LICENSE file in the project root for more information. using System; +using Microsoft.DotNet.Darc.Models.VirtualMonoRepo; +using System.Collections.Generic; using Microsoft.DotNet.DarcLib.Helpers; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; @@ -38,8 +40,14 @@ private static void RegisterManagers(IServiceCollection services, string gitLoca services.TryAddTransient(sp => ActivatorUtilities.CreateInstance(sp, gitLocation)); services.TryAddTransient(); services.TryAddTransient(); - services.TryAddTransient(); - services.TryAddTransient(sp => sp.GetRequiredService().CreateVmrUpdater().GetAwaiter().GetResult()); - services.TryAddTransient(sp => sp.GetRequiredService().CreateVmrInitializer().GetAwaiter().GetResult()); + services.TryAddTransient(); + services.TryAddTransient(); + services.TryAddTransient(); + services.TryAddSingleton>(sp => + { + var configuration = sp.GetRequiredService(); + var mappingParser = sp.GetRequiredService(); + return mappingParser.ParseMappings(configuration.VmrPath).GetAwaiter().GetResult(); + }); } } diff --git a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrUpdater.cs b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrUpdater.cs index c501b6330e..a81e01460b 100644 --- a/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrUpdater.cs +++ b/src/Microsoft.DotNet.Darc/src/DarcLib/VirtualMonoRepo/VmrUpdater.cs @@ -50,19 +50,21 @@ public class VmrUpdater : VmrManagerBase, IVmrUpdater private static readonly Regex GitPatchSummaryLine = new(@"^[\-0-9]+\s+[\-0-9]+\s+(?[^\s]+)$", RegexOptions.Compiled); private readonly ILogger _logger; + private readonly IVmrDependencyTracker _dependencyTracker; private readonly IProcessManager _processManager; private readonly IRemoteFactory _remoteFactory; public VmrUpdater( + IVmrDependencyTracker dependencyTracker, IProcessManager processManager, IRemoteFactory remoteFactory, IVersionDetailsParser versionDetailsParser, ILogger logger, - IVmrManagerConfiguration configuration, - IReadOnlyCollection mappings) - : base(processManager, remoteFactory, versionDetailsParser, logger, mappings, configuration.VmrPath, configuration.TmpPath) + IVmrManagerConfiguration configuration) + : base(dependencyTracker, processManager, remoteFactory, versionDetailsParser, logger, configuration.TmpPath) { _logger = logger; + _dependencyTracker = dependencyTracker; _processManager = processManager; _remoteFactory = remoteFactory; } @@ -70,22 +72,24 @@ public VmrUpdater( public Task UpdateRepository( SourceMapping mapping, string? targetRevision, + string? targetVersion, bool noSquash, bool updateDependencies, CancellationToken cancellationToken) { return updateDependencies - ? UpdateRepositoryRecursively(mapping, targetRevision, noSquash, cancellationToken) - : UpdateRepository(mapping, targetRevision, noSquash, cancellationToken); + ? UpdateRepositoryRecursively(mapping, targetRevision, targetVersion, noSquash, cancellationToken) + : UpdateRepository(mapping, targetRevision, targetVersion, noSquash, cancellationToken); } private async Task UpdateRepository( SourceMapping mapping, string? targetRevision, + string? targetVersion, bool noSquash, CancellationToken cancellationToken) { - var currentSha = await GetCurrentVersion(mapping); + var currentSha = GetCurrentVersion(mapping); if (!await HasRemoteUpdates(mapping, currentSha)) { @@ -172,6 +176,7 @@ await UpdateRepoToRevision( mapping, currentSha, commitToCopy.Sha, + commitToCopy.Sha == targetCommit.Sha ? targetVersion : null, clonePath, message, commitToCopy.Author, @@ -201,6 +206,7 @@ await UpdateRepoToRevision( mapping, currentSha, targetRevision, + targetVersion, clonePath, message, DotnetBotCommitSignature, @@ -215,13 +221,14 @@ await UpdateRepoToRevision( private async Task UpdateRepositoryRecursively( SourceMapping mapping, string? targetRevision, + string? targetVersion, bool noSquash, CancellationToken cancellationToken) { - var reposToUpdate = new Queue<(SourceMapping mapping, string? targetRevision)>(); - reposToUpdate.Enqueue((mapping, targetRevision)); + var reposToUpdate = new Queue<(SourceMapping mapping, string? targetRevision, string? targetVersion)>(); + reposToUpdate.Enqueue((mapping, targetRevision, targetVersion)); - var updatedDependencies = new HashSet<(SourceMapping mapping, string? targetRevision)>(); + var updatedDependencies = new HashSet<(SourceMapping mapping, string? targetRevision, string? targetVersion)>(); while (reposToUpdate.TryDequeue(out var repoToUpdate)) { @@ -231,7 +238,7 @@ private async Task UpdateRepositoryRecursively( mappingToUpdate.Name, repoToUpdate.targetRevision ?? HEAD); - await UpdateRepository(mappingToUpdate, repoToUpdate.targetRevision, noSquash, cancellationToken); + await UpdateRepository(mappingToUpdate, repoToUpdate.targetRevision, repoToUpdate.targetVersion, noSquash, cancellationToken); updatedDependencies.Add(repoToUpdate); foreach (var (dependency, dependencyMapping) in await GetDependencies(mappingToUpdate, cancellationToken)) @@ -241,14 +248,14 @@ private async Task UpdateRepositoryRecursively( continue; } - var dependencySha = await GetCurrentVersion(dependencyMapping); + var dependencySha = GetCurrentVersion(dependencyMapping); if (dependencySha == dependency.Commit) { _logger.LogDebug("Dependency {name} is already at {sha}, skipping..", dependency.Name, dependencySha); continue; } - reposToUpdate.Enqueue((dependencyMapping, dependency.Commit)); + reposToUpdate.Enqueue((dependencyMapping, dependency.Commit, dependency.Version)); } } @@ -270,6 +277,7 @@ private async Task UpdateRepoToRevision( SourceMapping mapping, string fromRevision, string toRevision, + string? targetVersion, string clonePath, string commitMessage, Signature author, @@ -300,7 +308,8 @@ private async Task UpdateRepoToRevision( await ApplyPatch(mapping, patchPath, cancellationToken); } - await TagRepo(mapping, toRevision); + _dependencyTracker.UpdateDependencyVersion(mapping, new(toRevision, targetVersion)); + Commands.Stage(new Repository(_dependencyTracker.VmrPath), VmrDependencyTracker.GitInfoSourcesDir); cancellationToken.ThrowIfCancellationRequested(); await ApplyVmrPatches(mapping, cancellationToken); @@ -347,6 +356,8 @@ private async Task RestorePatchedFilesFromRepo(SourceMapping mapping, string ori var localRepo = new LocalGitClient(_processManager.GitExecutable, _logger); localRepo.Checkout(clonePath, originalRevision); + var repoSourcesPath = _dependencyTracker.GetRepoSourcesPath(mapping); + foreach (var patch in mapping.VmrPatches) { _logger.LogDebug("Processing VMR patch `{patch}`..", patch); @@ -359,7 +370,7 @@ private async Task RestorePatchedFilesFromRepo(SourceMapping mapping, string ori : patchedFile; var originalFile = Path.Combine(clonePath, relativePath); - var destination = Path.Combine(SourcesPath, mapping.Name, relativePath); + var destination = Path.Combine(repoSourcesPath, relativePath); _logger.LogDebug("Restoring file `{originalFile}` to `{destination}`..", originalFile, destination); @@ -369,8 +380,8 @@ private async Task RestorePatchedFilesFromRepo(SourceMapping mapping, string ori } // Stage the restored files (all future patches are applied to index directly) - using var repository = new Repository(VmrPath); - Commands.Stage(repository, $"{VmrSourcesPath}/{mapping.Name}"); + using var repository = new Repository(_dependencyTracker.VmrPath); + Commands.Stage(repository, repoSourcesPath); _logger.LogDebug("Files from VMR patches for {mappingName} restored", mapping.Name); } @@ -399,19 +410,15 @@ private async Task> GetFilesInPatch(string repoPath, return files; } - private async Task GetCurrentVersion(SourceMapping mapping) + private string GetCurrentVersion(SourceMapping mapping) { - var tagFile = GetTagFilePath(mapping); - string currentSha; - try - { - currentSha = (await File.ReadAllTextAsync(tagFile)).Trim(); - } - catch (FileNotFoundException) + var version = _dependencyTracker.GetDependencyVersion(mapping); + + if (version is null) { - throw new InvalidOperationException($"Missing tag file for {mapping.Name} - please initialize the individual repo first"); + throw new InvalidOperationException($"Repository {mapping.Name} has not been initialized yet"); } - return currentSha; + return version.Sha; } } diff --git a/src/Microsoft.DotNet.Darc/tests/Microsoft.DotNet.DarcLib.Tests/Models/VirtualMonoRepo/AllVersionsPropsFileTests.cs b/src/Microsoft.DotNet.Darc/tests/Microsoft.DotNet.DarcLib.Tests/Models/VirtualMonoRepo/AllVersionsPropsFileTests.cs index a2a736675f..80c0968c7c 100644 --- a/src/Microsoft.DotNet.Darc/tests/Microsoft.DotNet.DarcLib.Tests/Models/VirtualMonoRepo/AllVersionsPropsFileTests.cs +++ b/src/Microsoft.DotNet.Darc/tests/Microsoft.DotNet.DarcLib.Tests/Models/VirtualMonoRepo/AllVersionsPropsFileTests.cs @@ -6,6 +6,7 @@ using System.IO; using FluentAssertions; using Microsoft.DotNet.Darc.Models.VirtualMonoRepo; +using Microsoft.DotNet.DarcLib.VirtualMonoRepo; using NUnit.Framework; #nullable enable @@ -41,21 +42,21 @@ public void CleanUpOutputFile() [Test] public void AllVersionsPropsFileIsDeSerializedTest() { - string runtimeSha = "26a71c61fbda229f151afb14e274604b4926df5c"; - string runtimeVersion = "7.0.0-rc.1.22403.8"; + var runtimeVersion = new VmrDependencyVersion("26a71c61fbda229f151afb14e274604b4926df5c", "7.0.0-rc.1.22403.8"); + var sdkVersion = new VmrDependencyVersion("6e00e543bbeb8e0491420e2f6b3f7d235166596d", "7.0.100-rc.1.22404.18"); var allVersionsPropsFile = new AllVersionsPropsFile(new() { - { "runtimeGitCommitHash", runtimeSha }, - { "runtimeOutputPackageVersion", runtimeVersion }, - { "sdkGitCommitHash", "6e00e543bbeb8e0491420e2f6b3f7d235166596d" }, - { "sdkOutputPackageVersion", "7.0.100-rc.1.22404.18" }, + { "runtimeGitCommitHash", runtimeVersion.Sha }, + { "runtimeOutputPackageVersion", runtimeVersion.PackageVersion! }, + { "sdkGitCommitHash", sdkVersion.Sha }, + { "sdkOutputPackageVersion", sdkVersion.PackageVersion! }, }); void VerifyVersions() { - allVersionsPropsFile.GetVersion("runtime").Should().Be((runtimeSha, runtimeVersion)); - allVersionsPropsFile.GetVersion("sdk").Should().Be(("6e00e543bbeb8e0491420e2f6b3f7d235166596d", "7.0.100-rc.1.22404.18")); + allVersionsPropsFile.GetVersion("runtime").Should().Be(runtimeVersion); + allVersionsPropsFile.GetVersion("sdk").Should().Be(sdkVersion); } VerifyVersions(); @@ -68,10 +69,10 @@ void VerifyVersions() - {runtimeSha} - {runtimeVersion} - 6e00e543bbeb8e0491420e2f6b3f7d235166596d - 7.0.100-rc.1.22404.18 + {runtimeVersion.Sha} + {runtimeVersion.PackageVersion} + {sdkVersion.Sha} + {sdkVersion.PackageVersion} """); @@ -80,12 +81,12 @@ void VerifyVersions() VerifyVersions(); - var newRuntimeSha = "225ce682f0578db2db5644df5e7024276b39785e"; - var newRuntimeVersion = "7.0.0-rc.1.22403.8"; - allVersionsPropsFile.UpdateVersion("runtime", newRuntimeSha, newRuntimeVersion); + runtimeVersion = new VmrDependencyVersion("225ce682f0578db2db5644df5e7024276b39785e", "7.0.0-rc.1.22444.8"); - allVersionsPropsFile.GetVersion("runtime").Should().Be((newRuntimeSha, newRuntimeVersion)); - allVersionsPropsFile.GetVersion("sdk").Should().Be(("6e00e543bbeb8e0491420e2f6b3f7d235166596d", "7.0.100-rc.1.22404.18")); + allVersionsPropsFile.UpdateVersion("runtime", runtimeVersion.Sha, runtimeVersion.PackageVersion!); + + allVersionsPropsFile.GetVersion("runtime").Should().Be(runtimeVersion); + allVersionsPropsFile.GetVersion("sdk").Should().Be(sdkVersion); allVersionsPropsFile.SerializeToXml(_outputFile); @@ -95,10 +96,10 @@ void VerifyVersions() - {newRuntimeSha} - {newRuntimeVersion} - 6e00e543bbeb8e0491420e2f6b3f7d235166596d - 7.0.100-rc.1.22404.18 + {runtimeVersion.Sha} + {runtimeVersion.PackageVersion} + {sdkVersion.Sha} + {sdkVersion.PackageVersion} """);