diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 933d7c0..630ca5f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -49,13 +49,15 @@ jobs: run: dotnet build src/AggregateConfigBuildTask.sln --configuration Release -warnaserror -p:Version=${{ steps.get_version.outputs.VERSION }} - name: Run tests for AggregateConfigBuildTask solution - run: dotnet test src/AggregateConfigBuildTask.sln --configuration Release -warnaserror -p:Version=${{ steps.get_version.outputs.VERSION }} + run: dotnet test src/AggregateConfigBuildTask.sln --configuration Release -warnaserror -p:Version=${{ steps.get_version.outputs.VERSION }} -p:CollectCoverage=true - name: Upload NuGetPackage artifact uses: actions/upload-artifact@v4 with: name: NuGetPackage - path: src/Task/bin/Release/AggregateConfigBuildTask.${{ steps.get_version.outputs.VERSION }}.nupkg + path: | + src/Task/bin/Release/AggregateConfigBuildTask.${{ steps.get_version.outputs.VERSION }}.nupkg + src/Task/bin/Release/AggregateConfigBuildTask.${{ steps.get_version.outputs.VERSION }}.snupkg integration_tests: needs: build @@ -89,7 +91,7 @@ jobs: run: dotnet build test/IntegrationTests.sln --configuration Release -warnaserror -p:Version=${{ needs.build.outputs.VERSION }} - name: Run IntegrationTests - run: dotnet test test/IntegrationTests.sln --configuration Release -warnaserror -p:Version=${{ needs.build.outputs.VERSION }} + run: dotnet test test/IntegrationTests.sln --configuration Release -warnaserror -p:Version=${{ needs.build.outputs.VERSION }} -p:CollectCoverage=true - name: Upload integration results artifact uses: actions/upload-artifact@v4 diff --git a/README.md b/README.md index fa2d4f8..e87954d 100644 --- a/README.md +++ b/README.md @@ -25,10 +25,7 @@ dotnet add package AggregateConfigBuildTask Alternatively, add the following line to your `.csproj` file: ```xml - - all - native;contentFiles;analyzers;runtime - + ``` `{latest}` can be found [here](https://www.nuget.org/packages/AggregateConfigBuildTask#versions-body-tab). diff --git a/src/AggregateConfigBuildTask.sln b/src/AggregateConfigBuildTask.sln index 29c4e17..17957ab 100644 --- a/src/AggregateConfigBuildTask.sln +++ b/src/AggregateConfigBuildTask.sln @@ -12,6 +12,7 @@ EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{29D0AE56-C184-4741-824F-521198552928}" ProjectSection(SolutionItems) = preProject .editorconfig = .editorconfig + Directory.Packages.props = Directory.Packages.props EndProjectSection EndProject Global diff --git a/src/Contracts/Contracts.csproj b/src/Contracts/Contracts.csproj index 261ad5b..56a26c6 100644 --- a/src/Contracts/Contracts.csproj +++ b/src/Contracts/Contracts.csproj @@ -2,6 +2,7 @@ netstandard2.0 + true diff --git a/src/Contracts/InputOutputEnums.cs b/src/Contracts/InputOutputEnums.cs index 3ac3e74..e81f362 100644 --- a/src/Contracts/InputOutputEnums.cs +++ b/src/Contracts/InputOutputEnums.cs @@ -1,4 +1,4 @@ -namespace AggregateConfig.Contracts +namespace AggregateConfigBuildTask.Contracts { public enum OutputTypeEnum { diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props new file mode 100644 index 0000000..6233193 --- /dev/null +++ b/src/Directory.Packages.props @@ -0,0 +1,30 @@ + + + true + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/Task/AggregateConfig.cs b/src/Task/AggregateConfig.cs index 34c6228..0c1a350 100644 --- a/src/Task/AggregateConfig.cs +++ b/src/Task/AggregateConfig.cs @@ -1,17 +1,15 @@ -using AggregateConfig.Contracts; -using AggregateConfig.FileHandlers; -using AggregateConfigBuildTask; +using AggregateConfigBuildTask.Contracts; +using AggregateConfigBuildTask.FileHandlers; using Microsoft.Build.Framework; using System; using System.IO; -using System.Linq; +using System.Reflection; using System.Runtime.CompilerServices; -using System.Text.Json; using Task = Microsoft.Build.Utilities.Task; [assembly: InternalsVisibleTo("AggregateConfig.Tests.UnitTests")] -namespace AggregateConfig +namespace AggregateConfigBuildTask { public class AggregateConfig : Task { @@ -46,8 +44,7 @@ public override bool Execute() { try { - bool hasError = false; - JsonElement? finalResult = null; + EmitHeader(); OutputFile = Path.GetFullPath(OutputFile); @@ -74,48 +71,7 @@ public override bool Execute() fileSystem.CreateDirectory(directoryPath); } - var expectedExtensions = FileHandlerFactory.GetExpectedFileExtensions(inputType); - var files = fileSystem.GetFiles(InputDirectory, "*.*") - .Where(file => expectedExtensions.Contains(Path.GetExtension(file).ToLower())) - .ToList(); - - foreach (var file in files) - { - Log.LogMessage(MessageImportance.High, "- Found file {0}", file); - - IInputReader outputWriter; - try - { - outputWriter = FileHandlerFactory.GetInputReader(fileSystem, inputType); - } - catch (ArgumentException ex) - { - hasError = true; - Log.LogError("No reader found for file {0}: {1} Stacktrace: {2}", file, ex.Message, ex.StackTrace); - continue; - } - - JsonElement fileData; - try - { - fileData = outputWriter.ReadInput(file); - } - catch (Exception ex) - { - hasError = true; - Log.LogError("Could not parse {0}: {1}", file, ex.Message); - Log.LogErrorFromException(ex, true, true, file); - continue; - } - - // Merge the deserialized object into the final result - finalResult = ObjectManager.MergeObjects(finalResult, fileData, file, AddSourceProperty); - } - - if (hasError) - { - return false; - } + var finalResult = ObjectManager.MergeFileObjects(InputDirectory, inputType, AddSourceProperty, fileSystem, Log).GetAwaiter().GetResult(); if (finalResult == null) { @@ -124,11 +80,7 @@ public override bool Execute() } var additionalPropertiesDictionary = JsonHelper.ParseAdditionalProperties(AdditionalProperties); - if (!ObjectManager.InjectAdditionalProperties(ref finalResult, additionalPropertiesDictionary)) - { - Log.LogError("Additional properties could not be injected since the top-level is not a JSON object."); - return false; - } + finalResult = ObjectManager.InjectAdditionalProperties(finalResult, additionalPropertiesDictionary, Log).GetAwaiter().GetResult(); var writer = FileHandlerFactory.GetOutputWriter(fileSystem, outputType); writer.WriteOutput(finalResult, OutputFile); @@ -138,10 +90,20 @@ public override bool Execute() } catch (Exception ex) { - Log.LogError("An unknown exception occured: {0}", ex.Message); + Log.LogError("An unknown exception occurred: {0}", ex.Message); Log.LogErrorFromException(ex, true, true, null); return false; } } + + private void EmitHeader() + { + var assembly = Assembly.GetExecutingAssembly(); + var informationalVersion = assembly + .GetCustomAttribute()? + .InformationalVersion; + + Log.LogMessage($"AggregateConfig Version: {informationalVersion}"); + } } } diff --git a/src/Task/AggregateConfigBuildTask.csproj b/src/Task/AggregateConfigBuildTask.csproj index 696ba52..a6c67da 100644 --- a/src/Task/AggregateConfigBuildTask.csproj +++ b/src/Task/AggregateConfigBuildTask.csproj @@ -4,6 +4,15 @@ netstandard2.0 true true + true + true + true + true + NU5100 + + + + true @@ -20,6 +29,10 @@ https://github.com/richardsondev/AggregateConfigBuildTask/releases/tag/v$(Version) docs/README.md true + true + snupkg + true + tasks @@ -28,28 +41,28 @@ - - - + + + - - - - + + + + - - - - + + + + licenses/YamlDotNet/LICENSE.txt - + licenses/LICENSE diff --git a/src/Task/FileHandlers/ArmParametersFileHandler.cs b/src/Task/FileHandlers/ArmParametersFileHandler.cs index 089b922..51ef876 100644 --- a/src/Task/FileHandlers/ArmParametersFileHandler.cs +++ b/src/Task/FileHandlers/ArmParametersFileHandler.cs @@ -2,12 +2,13 @@ using System.Collections.Generic; using System.Text.Json; using System.Text.Json.Nodes; +using System.Threading.Tasks; -namespace AggregateConfig.FileHandlers +namespace AggregateConfigBuildTask.FileHandlers { public class ArmParametersFileHandler : IOutputWriter, IInputReader { - IFileSystem fileSystem; + readonly IFileSystem fileSystem; internal ArmParametersFileHandler(IFileSystem fileSystem) { @@ -15,7 +16,7 @@ internal ArmParametersFileHandler(IFileSystem fileSystem) } /// - public JsonElement ReadInput(string inputPath) + public ValueTask ReadInput(string inputPath) { using (var stream = fileSystem.OpenRead(inputPath)) { @@ -40,10 +41,10 @@ public JsonElement ReadInput(string inputPath) } var modifiedJson = modifiedParameters.ToJsonString(); - return JsonSerializer.Deserialize(modifiedJson); + return new ValueTask(Task.FromResult(JsonSerializer.Deserialize(modifiedJson))); } - return jsonDoc.RootElement.Clone(); + return new ValueTask(Task.FromResult(jsonDoc.RootElement.Clone())); } } } diff --git a/src/Task/FileHandlers/FileHandlerFactory.cs b/src/Task/FileHandlers/FileHandlerFactory.cs index f3b169d..bc13876 100644 --- a/src/Task/FileHandlers/FileHandlerFactory.cs +++ b/src/Task/FileHandlers/FileHandlerFactory.cs @@ -1,8 +1,8 @@ -using AggregateConfig.Contracts; +using AggregateConfigBuildTask.Contracts; using System; using System.Collections.Generic; -namespace AggregateConfig.FileHandlers +namespace AggregateConfigBuildTask.FileHandlers { public static class FileHandlerFactory { diff --git a/src/Task/FileHandlers/IInputReader.cs b/src/Task/FileHandlers/IInputReader.cs index e28ca15..7aed869 100644 --- a/src/Task/FileHandlers/IInputReader.cs +++ b/src/Task/FileHandlers/IInputReader.cs @@ -1,9 +1,10 @@ using System.Text.Json; +using System.Threading.Tasks; -namespace AggregateConfig.FileHandlers +namespace AggregateConfigBuildTask.FileHandlers { public interface IInputReader { - JsonElement ReadInput(string inputPath); + ValueTask ReadInput(string inputPath); } } diff --git a/src/Task/FileHandlers/IOutputWriter.cs b/src/Task/FileHandlers/IOutputWriter.cs index 48d5d52..03cc1e6 100644 --- a/src/Task/FileHandlers/IOutputWriter.cs +++ b/src/Task/FileHandlers/IOutputWriter.cs @@ -1,6 +1,6 @@ using System.Text.Json; -namespace AggregateConfig.FileHandlers +namespace AggregateConfigBuildTask.FileHandlers { public interface IOutputWriter { diff --git a/src/Task/FileHandlers/JsonFileHandler.cs b/src/Task/FileHandlers/JsonFileHandler.cs index 6436e16..5f67e7a 100644 --- a/src/Task/FileHandlers/JsonFileHandler.cs +++ b/src/Task/FileHandlers/JsonFileHandler.cs @@ -1,10 +1,11 @@ using System.Text.Json; +using System.Threading.Tasks; -namespace AggregateConfig.FileHandlers +namespace AggregateConfigBuildTask.FileHandlers { public class JsonFileHandler : IOutputWriter, IInputReader { - IFileSystem fileSystem; + readonly IFileSystem fileSystem; internal JsonFileHandler(IFileSystem fileSystem) { @@ -12,11 +13,11 @@ internal JsonFileHandler(IFileSystem fileSystem) } /// - public JsonElement ReadInput(string inputPath) + public ValueTask ReadInput(string inputPath) { using (var json = fileSystem.OpenRead(inputPath)) { - return JsonSerializer.Deserialize(json); + return JsonSerializer.DeserializeAsync(json); } } diff --git a/src/Task/FileHandlers/YamlFileHandler.cs b/src/Task/FileHandlers/YamlFileHandler.cs index 0ec1bb9..43c6df2 100644 --- a/src/Task/FileHandlers/YamlFileHandler.cs +++ b/src/Task/FileHandlers/YamlFileHandler.cs @@ -1,14 +1,15 @@ using System.IO; using System.Text.Json; +using System.Threading.Tasks; using YamlDotNet.Serialization; using YamlDotNet.Serialization.NamingConventions; using YamlDotNet.System.Text.Json; -namespace AggregateConfig.FileHandlers +namespace AggregateConfigBuildTask.FileHandlers { public class YamlFileHandler : IOutputWriter, IInputReader { - IFileSystem fileSystem; + readonly IFileSystem fileSystem; internal YamlFileHandler(IFileSystem fileSystem) { @@ -16,16 +17,18 @@ internal YamlFileHandler(IFileSystem fileSystem) } /// - public JsonElement ReadInput(string inputPath) + public ValueTask ReadInput(string inputPath) { using (TextReader reader = fileSystem.OpenText(inputPath)) { - return new DeserializerBuilder() - .WithNamingConvention(CamelCaseNamingConvention.Instance) - .WithTypeConverter(new SystemTextJsonYamlTypeConverter()) - .WithTypeInspector(x => new SystemTextJsonTypeInspector(x)) - .Build() - .Deserialize(reader); + return new ValueTask( + Task.FromResult( + new DeserializerBuilder() + .WithNamingConvention(CamelCaseNamingConvention.Instance) + .WithTypeConverter(new SystemTextJsonYamlTypeConverter()) + .WithTypeInspector(x => new SystemTextJsonTypeInspector(x)) + .Build() + .Deserialize(reader))); } } diff --git a/src/Task/FileSystem/FileSystem.cs b/src/Task/FileSystem/FileSystem.cs index 8549bd8..70914bc 100644 --- a/src/Task/FileSystem/FileSystem.cs +++ b/src/Task/FileSystem/FileSystem.cs @@ -1,6 +1,6 @@ using System.IO; -namespace AggregateConfig +namespace AggregateConfigBuildTask { internal class FileSystem : IFileSystem { diff --git a/src/Task/FileSystem/IFileSystem.cs b/src/Task/FileSystem/IFileSystem.cs index 69280e2..33cc783 100644 --- a/src/Task/FileSystem/IFileSystem.cs +++ b/src/Task/FileSystem/IFileSystem.cs @@ -1,6 +1,6 @@ using System.IO; -namespace AggregateConfig +namespace AggregateConfigBuildTask { /// /// Interface for a file system abstraction, allowing various implementations to handle file operations. diff --git a/src/Task/JsonHelper.cs b/src/Task/JsonHelper.cs index 8db4878..46a1073 100644 --- a/src/Task/JsonHelper.cs +++ b/src/Task/JsonHelper.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; +using System.IO; using System.Text.Json; +using System.Threading.Tasks; namespace AggregateConfigBuildTask { @@ -11,7 +13,7 @@ internal static class JsonHelper /// /// A dictionary containing string keys and JsonElement values. /// A JsonElement representing the dictionary. - public static JsonElement ConvertToJsonElement(Dictionary dictionary) + public static Task ConvertToJsonElement(Dictionary dictionary) { return ConvertObjectToJsonElement(dictionary); } @@ -21,7 +23,7 @@ public static JsonElement ConvertToJsonElement(Dictionary d /// /// A list containing JsonElement objects. /// A JsonElement representing the list. - public static JsonElement ConvertToJsonElement(List list) + public static Task ConvertToJsonElement(List list) { return ConvertObjectToJsonElement(list); } @@ -93,10 +95,23 @@ public static Dictionary ParseAdditionalProperties(string[] prop /// /// The object to convert to JsonElement. /// A JsonElement representing the object. - public static JsonElement ConvertObjectToJsonElement(object value) + public static async Task ConvertObjectToJsonElement(object value) { - var json = JsonSerializer.Serialize(value); - return JsonDocument.Parse(json).RootElement; + if (value == null) + { + throw new ArgumentNullException(nameof(value)); + } + + using (var memoryStream = new MemoryStream()) + { + await JsonSerializer.SerializeAsync(memoryStream, value); + memoryStream.Seek(0, SeekOrigin.Begin); + + using (var jsonDocument = await JsonDocument.ParseAsync(memoryStream)) + { + return jsonDocument.RootElement.Clone(); + } + } } /// diff --git a/src/Task/ObjectManager.cs b/src/Task/ObjectManager.cs index 622b5ca..8a5ad02 100644 --- a/src/Task/ObjectManager.cs +++ b/src/Task/ObjectManager.cs @@ -1,25 +1,90 @@ -using System; +using AggregateConfigBuildTask.Contracts; +using AggregateConfigBuildTask.FileHandlers; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text.Json; +using System.Threading.Tasks; namespace AggregateConfigBuildTask { internal static class ObjectManager { - /// - /// Merges two JsonElements into a single JsonElement. Will merge nested objects and lists together. - /// - public static JsonElement MergeObjects(JsonElement? obj1, JsonElement? obj2, string source2, bool injectSourceProperty) + public static async Task MergeFileObjects(string fileObjectDirectoryPath, InputTypeEnum inputType, bool addSourceProperty, IFileSystem fileSystem, TaskLoggingHelper log) + { + var finalResults = new ConcurrentBag(); + JsonElement? finalResult = null; + bool hasError = false; + + var expectedExtensions = FileHandlerFactory.GetExpectedFileExtensions(inputType); + var fileGroups = fileSystem.GetFiles(fileObjectDirectoryPath, "*.*") + .Where(file => expectedExtensions.Contains(Path.GetExtension(file).ToLower())) + .ToList() + .Chunk(100); + + await fileGroups.ForEachAsync(Environment.ProcessorCount, + async (files) => { + JsonElement? intermediateResult = null; + foreach (var file in files) + { + log.LogMessage(MessageImportance.High, "- Found file {0}", file); + + IInputReader outputWriter; + try + { + outputWriter = FileHandlerFactory.GetInputReader(fileSystem, inputType); + } + catch (ArgumentException ex) + { + hasError = true; + log.LogError("No reader found for file {0}: {1} Stacktrace: {2}", file, ex.Message, ex.StackTrace); + continue; + } + + JsonElement fileData; + try + { + fileData = await outputWriter.ReadInput(file); + } + catch (Exception ex) + { + hasError = true; + log.LogError("Could not parse {0}: {1}", file, ex.Message); + log.LogErrorFromException(ex, true, true, file); + continue; + } + + // Merge the deserialized object into the final result + finalResults.Add(await ObjectManager.MergeObjects(intermediateResult, fileData, file, addSourceProperty)); + } + }); + + if (hasError) + { + return null; + } + + foreach (var result in finalResults) + { + finalResult = await ObjectManager.MergeObjects(finalResult, result, null, false); + } + + return finalResult; + } + + private static async Task InjectSourceProperty(JsonElement? obj2, string source2, bool injectSourceProperty) { // If injectSourceProperty is true, inject the source property into the second object - if (injectSourceProperty && obj2.HasValue && obj2.Value.ValueKind == JsonValueKind.Object) + if (obj2 != null && injectSourceProperty && obj2.HasValue && obj2.Value.ValueKind == JsonValueKind.Object) { var obj2Dict = obj2.Value; var jsonObject = obj2Dict.EnumerateObject().ToDictionary(p => p.Name, p => p.Value); - foreach (var kvp in jsonObject) + foreach (var kvp in jsonObject.ToList()) { var key = kvp.Key; var value = kvp.Value; @@ -41,18 +106,29 @@ public static JsonElement MergeObjects(JsonElement? obj1, JsonElement? obj2, str nestedDict["source"] = JsonDocument.Parse($"\"{Path.GetFileNameWithoutExtension(source2)}\"").RootElement; // Update the list at the correct index - obj2NestedList[index] = JsonHelper.ConvertToJsonElement(nestedDict); + obj2NestedList[index] = await JsonHelper.ConvertToJsonElement(nestedDict); } } - jsonObject[key] = JsonHelper.ConvertToJsonElement(obj2NestedList); + jsonObject[key] = await JsonHelper.ConvertToJsonElement(obj2NestedList); } } - obj2 = JsonHelper.ConvertObjectToJsonElement(jsonObject); + obj2 = await JsonHelper.ConvertObjectToJsonElement(jsonObject); } - if (obj1 == null) return obj2.HasValue ? obj2.Value : default; - if (obj2 == null) return obj1.HasValue ? obj1.Value : default; + return obj2; + } + + /// + /// Merges two JsonElements into a single JsonElement. Will merge nested objects and lists together. + /// + public static async Task MergeObjects(JsonElement? obj1, JsonElement? obj2, string source2, bool injectSourceProperty) + { + obj1 = await InjectSourceProperty(obj1, source2, injectSourceProperty); + obj2 = await InjectSourceProperty(obj2, source2, injectSourceProperty); + + if (obj1 == null) return obj2 ?? default; + if (obj2 == null) return obj1 ?? default; // Handle merging of objects if (obj1.Value.ValueKind == JsonValueKind.Object && obj2.Value.ValueKind == JsonValueKind.Object) @@ -64,7 +140,7 @@ public static JsonElement MergeObjects(JsonElement? obj1, JsonElement? obj2, str { if (dict1.ContainsKey(key)) { - dict1[key] = MergeObjects(dict1[key], dict2[key], source2, injectSourceProperty); + dict1[key] = await MergeObjects(dict1[key], dict2[key], source2, injectSourceProperty); } else { @@ -72,7 +148,7 @@ public static JsonElement MergeObjects(JsonElement? obj1, JsonElement? obj2, str } } - return JsonHelper.ConvertToJsonElement(dict1); + return await JsonHelper.ConvertToJsonElement(dict1); } // Handle merging of arrays else if (obj1.Value.ValueKind == JsonValueKind.Array && obj2.Value.ValueKind == JsonValueKind.Array) @@ -85,7 +161,7 @@ public static JsonElement MergeObjects(JsonElement? obj1, JsonElement? obj2, str list1.Add(item); } - return JsonHelper.ConvertToJsonElement(list1); + return await JsonHelper.ConvertToJsonElement(list1); } // For scalar values, obj2 overwrites obj1 else @@ -99,8 +175,9 @@ public static JsonElement MergeObjects(JsonElement? obj1, JsonElement? obj2, str /// /// The object that is expected to be a JSON object (JsonElement) where additional properties will be injected. /// A dictionary of additional properties to inject. + /// Logger reference. /// True if the properties were successfully injected, false otherwise. - public static bool InjectAdditionalProperties(ref JsonElement? finalResult, Dictionary additionalPropertiesDictionary) + public static async Task InjectAdditionalProperties(JsonElement? finalResult, Dictionary additionalPropertiesDictionary, TaskLoggingHelper log) { if (additionalPropertiesDictionary?.Count > 0) { @@ -111,20 +188,19 @@ public static bool InjectAdditionalProperties(ref JsonElement? finalResult, Dict // Add the properties from additionalPropertiesDictionary, converting values to JsonElement foreach (var property in additionalPropertiesDictionary) { - jsonDictionary[property.Key] = JsonHelper.ConvertObjectToJsonElement(property.Value); + jsonDictionary[property.Key] = await JsonHelper.ConvertObjectToJsonElement(property.Value); } - finalResult = JsonHelper.ConvertToJsonElement(jsonDictionary); - return true; + return await JsonHelper.ConvertToJsonElement(jsonDictionary); } else { - Console.Error.WriteLine("Additional properties could not be injected since the top-level is not a JSON object."); - return false; + log.LogWarning("Additional properties could not be injected since the top-level is not a JSON object."); + return finalResult; } } - return true; + return finalResult; } } } diff --git a/src/Task/Polyfills/ListExtensions.cs b/src/Task/Polyfills/ListExtensions.cs new file mode 100644 index 0000000..61a102f --- /dev/null +++ b/src/Task/Polyfills/ListExtensions.cs @@ -0,0 +1,72 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace AggregateConfigBuildTask +{ + public static class ListExtensions + { + /// + /// Splits a list into chunks of the specified size. + /// + /// The list to split into chunks. + /// The size of each chunk. + /// An IEnumerable of string arrays, each containing a chunk of the original list. + public static IEnumerable> Chunk(this List source, int chunkSize) + { + if (source == null) + { + throw new ArgumentNullException(nameof(source)); + } + + if (chunkSize <= 0) + { + throw new ArgumentOutOfRangeException(nameof(chunkSize), "Chunk size must be greater than 0."); + } + + for (int i = 0; i < source.Count; i += chunkSize) + { + yield return source.GetRange(i, Math.Min(chunkSize, source.Count - i)); + } + } + + /// + /// Asynchronously processes each element of a collection in parallel, with a limit on the number of concurrent tasks. + /// + /// The type of elements in the source collection. + /// The collection of elements to process. + /// The maximum number of tasks to run concurrently. + /// The asynchronous delegate to execute for each element in the source collection. + /// A task that represents the asynchronous processing of the collection. + /// Thrown if the source or body is null. + /// Thrown if the degree of parallelism is less than 1. + public static async Task ForEachAsync( + this IEnumerable source, + int degreeOfParallelism, + Func body) + { + if (source == null) throw new ArgumentNullException(nameof(source)); + if (body == null) throw new ArgumentNullException(nameof(body)); + if (degreeOfParallelism < 1) throw new ArgumentOutOfRangeException(nameof(degreeOfParallelism), "Degree of parallelism must be at least 1."); + + var semaphore = new SemaphoreSlim(degreeOfParallelism); + + var tasks = source.Select(async item => + { + await semaphore.WaitAsync(); + try + { + await body(item); + } + finally + { + semaphore.Release(); + } + }); + + await Task.WhenAll(tasks); + } + } +} diff --git a/src/Task/build/AggregateConfigBuildTask.targets b/src/Task/build/AggregateConfigBuildTask.targets index c5a837c..9e5c30d 100644 --- a/src/Task/build/AggregateConfigBuildTask.targets +++ b/src/Task/build/AggregateConfigBuildTask.targets @@ -1,6 +1,6 @@  + AssemblyFile="$(MSBuildThisFileDirectory)..\tasks\netstandard2.0\AggregateConfigBuildTask.dll" /> diff --git a/src/UnitTests/TaskTestBase.cs b/src/UnitTests/TaskTestBase.cs index 7ffb80a..d604c17 100644 --- a/src/UnitTests/TaskTestBase.cs +++ b/src/UnitTests/TaskTestBase.cs @@ -1,4 +1,4 @@ -using AggregateConfig.Contracts; +using AggregateConfigBuildTask.Contracts; using Microsoft.Build.Framework; using Moq; using Newtonsoft.Json; @@ -6,8 +6,9 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Text; -namespace AggregateConfig.Tests.Unit +namespace AggregateConfigBuildTask.Tests.Unit { public class TaskTestBase { @@ -22,10 +23,8 @@ public void TestInitialize(bool isWindowsMode, string testPath) } [TestMethod] - [DataRow(true)] - [DataRow(false)] [Description("Test that YAML files are merged into correct JSON output.")] - public void ShouldGenerateJsonOutput(bool isWindows) + public void ShouldGenerateJsonOutput() { // Arrange: Prepare sample YAML data in the mock file system. mockFileSystem.WriteAllText($"{testPath}\\file1.yml", @" @@ -42,9 +41,9 @@ public void ShouldGenerateJsonOutput(bool isWindows) InputDirectory = testPath, OutputFile = testPath + @"\output.json", OutputType = OutputTypeEnum.Json.ToString(), - AddSourceProperty = true + AddSourceProperty = true, + BuildEngine = Mock.Of() }; - task.BuildEngine = Mock.Of(); // Act: Execute the task bool result = task.Execute(); @@ -77,9 +76,9 @@ public void ShouldGenerateArmParameterOutput() InputDirectory = testPath, OutputFile = testPath + @"\output.parameters.json", OutputType = OutputTypeEnum.Arm.ToString(), - AddSourceProperty = true + AddSourceProperty = true, + BuildEngine = Mock.Of() }; - task.BuildEngine = Mock.Of(); // Act: Execute the task bool result = task.Execute(); @@ -111,9 +110,9 @@ public void ShouldAddSourceProperty() InputDirectory = testPath, OutputFile = testPath + @"\output.json", OutputType = OutputTypeEnum.Json.ToString(), - AddSourceProperty = true + AddSourceProperty = true, + BuildEngine = Mock.Of() }; - task.BuildEngine = Mock.Of(); // Act: Execute the task bool result = task.Execute(); @@ -134,13 +133,17 @@ public void ShouldAddSourcePropertyMultipleFiles() mockFileSystem.WriteAllText($"{testPath}\\file1.yml", @" options: - name: 'Option 1' - description: 'First option'"); + description: 'First option' + additionalOptions: + value: 'Good day'"); mockFileSystem.WriteAllText($"{testPath}\\file2.yml", @" options: - name: 'Option 2' description: 'Second option' - name: 'Option 3' description: 'Third option' + additionalOptions: + value: 'Good night' text: - name: 'Text 1' description: 'Text'"); @@ -151,9 +154,9 @@ public void ShouldAddSourcePropertyMultipleFiles() InputDirectory = testPath, OutputFile = testPath + @"\output.json", OutputType = OutputTypeEnum.Json.ToString(), - AddSourceProperty = true + AddSourceProperty = true, + BuildEngine = Mock.Of() }; - task.BuildEngine = Mock.Of(); // Act: Execute the task bool result = task.Execute(); @@ -161,15 +164,11 @@ public void ShouldAddSourcePropertyMultipleFiles() // Assert: Verify that the source property was added Assert.IsTrue(result); string output = mockFileSystem.ReadAllText($"{testPath}\\output.json"); - var json = JsonConvert.DeserializeObject>>>(output); - Assert.IsTrue(json["options"][0].ContainsKey("source")); - Assert.AreEqual("file1", json["options"][0]["source"]); - Assert.IsTrue(json["options"][1].ContainsKey("source")); - Assert.AreEqual("file2", json["options"][1]["source"]); - Assert.IsTrue(json["options"][2].ContainsKey("source")); - Assert.AreEqual("file2", json["options"][2]["source"]); - Assert.IsTrue(json["text"][0].ContainsKey("source")); - Assert.AreEqual("file2", json["text"][0]["source"]); + var json = JsonConvert.DeserializeObject>>>(output); + Assert.IsTrue(OptionExistsWithSource(json["options"], "Option 1", "file1")); + Assert.IsTrue(OptionExistsWithSource(json["options"], "Option 2", "file2")); + Assert.IsTrue(OptionExistsWithSource(json["options"], "Option 3", "file2")); + Assert.IsTrue(OptionExistsWithSource(json["text"], "Text 1", "file2")); } [TestMethod] @@ -192,9 +191,9 @@ public void ShouldIncludeAdditionalPropertiesInJson() { { "Group", "TestRG" }, { "Environment\\=Key", "Prod\\=West" } - }.Select(q => $"{q.Key}={q.Value}").ToArray() + }.Select(q => $"{q.Key}={q.Value}").ToArray(), + BuildEngine = Mock.Of() }; - task.BuildEngine = Mock.Of(); // Act: Execute the task bool result = task.Execute(); @@ -227,9 +226,9 @@ public void ShouldIncludeAdditionalPropertiesInArmParameters() { { "Group", "TestRG" }, { "Environment", "Prod" } - }.Select(q => $"{q.Key}={q.Value}").ToArray() + }.Select(q => $"{q.Key}={q.Value}").ToArray(), + BuildEngine = Mock.Of() }; - task.BuildEngine = Mock.Of(); // Act: Execute the task bool result = task.Execute(); @@ -253,9 +252,9 @@ public void ShouldHandleEmptyDirectory() { InputDirectory = testPath, OutputFile = testPath + @"\output.json", - OutputType = OutputTypeEnum.Json.ToString() + OutputType = OutputTypeEnum.Json.ToString(), + BuildEngine = Mock.Of() }; - task.BuildEngine = Mock.Of(); // Act: Execute the task bool result = task.Execute(); @@ -280,9 +279,9 @@ public void ShouldHandleInvalidYamlFormat() { InputDirectory = testPath, OutputFile = testPath + @"\output.json", - OutputType = OutputTypeEnum.Json.ToString() + OutputType = OutputTypeEnum.Json.ToString(), + BuildEngine = Mock.Of() }; - task.BuildEngine = Mock.Of(); // Act: Expect the task to fail bool result = task.Execute(); @@ -306,9 +305,9 @@ public void ShouldCorrectlyParseBooleanValues() { InputDirectory = testPath, OutputFile = testPath + @"\output.json", - OutputType = OutputTypeEnum.Arm.ToString() + OutputType = OutputTypeEnum.Arm.ToString(), + BuildEngine = Mock.Of() }; - task.BuildEngine = Mock.Of(); // Act: Execute the task bool result = task.Execute(); @@ -350,9 +349,9 @@ public void ShouldIncludeAdditionalPropertiesInJsonInput() { { "Group", "TestRG" }, { "Environment", "Prod" } - }.Select(q => $"{q.Key}={q.Value}").ToArray() + }.Select(q => $"{q.Key}={q.Value}").ToArray(), + BuildEngine = Mock.Of() }; - task.BuildEngine = Mock.Of(); // Act: Execute the task bool result = task.Execute(); @@ -402,9 +401,9 @@ public void ShouldIncludeAdditionalPropertiesInArmParameterFile() { { "Group", "TestRG" }, { "Environment", "Prod" } - }.Select(q => $"{q.Key}={q.Value}").ToArray() + }.Select(q => $"{q.Key}={q.Value}").ToArray(), + BuildEngine = Mock.Of() }; - task.BuildEngine = Mock.Of(); // Act: Execute the task bool result = task.Execute(); @@ -421,5 +420,69 @@ public void ShouldIncludeAdditionalPropertiesInArmParameterFile() Assert.AreEqual("Boolean", parameters.GetValue("options")["value"].First()["isEnabled"].Type.ToString()); Assert.AreEqual(true, parameters.GetValue("options")["value"].First()["isEnabled"].Value()); } + + [TestMethod] + [Description("Stress test to verify the source property is correctly added for 1,000 files with 10 options each.")] + [Timeout(60000)] + public void StressTest_ShouldAddSourcePropertyManyFiles() + { + // Arrange: Prepare sample YAML data. + const int totalFiles = 1_000; + const int totalOptionsPerFile = 10; + + for (int fileIndex = 1; fileIndex <= totalFiles; fileIndex++) + { + var sb = new StringBuilder(); + sb.AppendLine("options:"); + + for (int optionIndex = 1; optionIndex <= totalOptionsPerFile; optionIndex++) + { + sb.AppendLine($" - name: 'Option {optionIndex}'"); + sb.AppendLine($" description: 'Description for Option {optionIndex}'"); + } + + // Write each YAML file to the mock file system + mockFileSystem.WriteAllText($"{testPath}\\file{fileIndex}.yml", sb.ToString()); + } + + var task = new AggregateConfig(mockFileSystem) + { + InputDirectory = testPath, + OutputFile = testPath + @"\output.json", + OutputType = OutputTypeEnum.Json.ToString(), + AddSourceProperty = true, + BuildEngine = Mock.Of() + }; + + // Act: Execute the task + bool result = task.Execute(); + + // Assert: Verify that the source property was added correctly for all files and options + Assert.IsTrue(result); + string output = mockFileSystem.ReadAllText($"{testPath}\\output.json"); + var json = JsonConvert.DeserializeObject>>>(output); + + int optionIndexInTotal = 0; + + for (int fileIndex = 1; fileIndex <= totalFiles; fileIndex++) + { + for (int optionIndex = 1; optionIndex <= totalOptionsPerFile; optionIndex++, optionIndexInTotal++) + { + Assert.IsTrue(OptionExistsWithSource(json["options"], $"Option {optionIndex}", $"file{fileIndex}")); + } + } + } + + /// + /// Check if an option exists with a given name and source + /// + private static bool OptionExistsWithSource(List> options, string optionName, string expectedSource) + { + return options.Any(option => + option.ContainsKey("name") && + (string)option["name"] == optionName && + option.ContainsKey("source") && + (string)option["source"] == expectedSource); + } } } diff --git a/src/UnitTests/TaskUnixTests.cs b/src/UnitTests/TaskUnixTests.cs index 0526232..f1e44a2 100644 --- a/src/UnitTests/TaskUnixTests.cs +++ b/src/UnitTests/TaskUnixTests.cs @@ -1,4 +1,4 @@ -namespace AggregateConfig.Tests.Unit +namespace AggregateConfigBuildTask.Tests.Unit { [TestClass] public class TaskUnixTests : TaskTestBase diff --git a/src/UnitTests/TaskWindowsTests.cs b/src/UnitTests/TaskWindowsTests.cs index 977907c..a67168e 100644 --- a/src/UnitTests/TaskWindowsTests.cs +++ b/src/UnitTests/TaskWindowsTests.cs @@ -1,4 +1,4 @@ -namespace AggregateConfig.Tests.Unit +namespace AggregateConfigBuildTask.Tests.Unit { [TestClass] public class TaskWindowsTests : TaskTestBase diff --git a/src/UnitTests/UnitTests.csproj b/src/UnitTests/UnitTests.csproj index bddc474..9b933be 100644 --- a/src/UnitTests/UnitTests.csproj +++ b/src/UnitTests/UnitTests.csproj @@ -8,17 +8,17 @@ true false true + CS1591 - - - - - - - - + + + + + + + diff --git a/src/UnitTests/VirtualFileSystem.cs b/src/UnitTests/VirtualFileSystem.cs index f5102fd..c4f5654 100644 --- a/src/UnitTests/VirtualFileSystem.cs +++ b/src/UnitTests/VirtualFileSystem.cs @@ -4,25 +4,19 @@ using System.Text; using System.Text.RegularExpressions; -namespace AggregateConfig.Tests.Unit +namespace AggregateConfigBuildTask.Tests.Unit { - internal class VirtualFileSystem : IFileSystem + internal class VirtualFileSystem(bool isWindowsMode = true) : IFileSystem { - private readonly bool isWindowsMode = false; - private readonly Dictionary fileSystem; + private readonly bool isWindowsMode = isWindowsMode; + private readonly Dictionary fileSystem = new( + isWindowsMode ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal + ); private RegexOptions RegexOptions => isWindowsMode ? RegexOptions.IgnoreCase : RegexOptions.None; private StringComparison StringComparison => isWindowsMode ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; private string EnvironmentLineBreak => isWindowsMode ? "\r\n" : "\n"; - public VirtualFileSystem(bool isWindowsMode = true) - { - this.isWindowsMode = isWindowsMode; - this.fileSystem = new Dictionary( - isWindowsMode ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal - ); - } - /// public string[] GetFiles(string path, string searchPattern) { @@ -44,7 +38,7 @@ public string[] GetFiles(string path, string searchPattern) } } - return files.ToArray(); + return [.. files]; } /// @@ -163,7 +157,7 @@ private string EnsureTrailingDirectorySeparator(string directoryPath) /// /// Converts a file search pattern (with * and ?) to a regex pattern. /// - private string ConvertPatternToRegex(string searchPattern) + private static string ConvertPatternToRegex(string searchPattern) { // Escape special regex characters except for * and ? string escapedPattern = Regex.Escape(searchPattern); diff --git a/test/IntegrationTests/IntegrationTests.csproj b/test/IntegrationTests/IntegrationTests.csproj index 5959ecc..174a210 100644 --- a/test/IntegrationTests/IntegrationTests.csproj +++ b/test/IntegrationTests/IntegrationTests.csproj @@ -12,13 +12,9 @@ - - all - native;contentFiles;analyzers;runtime - + -