diff --git a/README.md b/README.md index ee282a7..a6b108c 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ - Path params, query params, headers, and JSON request bodies - Dot-notation assertions with array index support - Validation for strings, objects, and arrays +- CLI and CI execution mode with environment and file filtering, JSON output, JUnit XML output, and non-zero exit codes on failed runs - Toggleable test selection in the dashboard, including select-all, clear-all, expand-all, collapse-all, and individual test control - Collapsible pass/fail reporting with environment, endpoint, test, and response-preview drill-down - cURL analysis page that scans configured YAML definitions, generates suggested environment and endpoint YAML when missing, and supports response-driven assertion building @@ -43,6 +44,41 @@ dotnet run --project src/ApiTestRunner.App -c Release The app starts the dashboard at `http://localhost:5005` by default, auto-launches the browser if enabled, and executes the split sample suite after the web server is ready. +## CLI and CI mode + +The same executable also supports headless execution for terminal and pipeline use. + +Basic examples: + +```powershell +dotnet run --project src/ApiTestRunner.App -- --ci +dotnet run --project src/ApiTestRunner.App -- --ci --env Local +dotnet run --project src/ApiTestRunner.App -- --ci --env Local,UAT +dotnet run --project src/ApiTestRunner.App -- --ci --file Samples/Environments/sample-api.yaml --file Samples/Endpoints/accounts/get-accounts.yaml +dotnet run --project src/ApiTestRunner.App -- --ci --format json +dotnet run --project src/ApiTestRunner.App -- --ci --format junit --output artifacts/test-results.xml +``` + +Supported CLI options: + +- `--ci` enables headless execution mode +- `--env ` runs only the named environments and can be repeated +- `--file ` overrides `Execution.TestFiles` for this run and can be repeated +- `--format ` emits machine-readable results +- `--output ` writes formatted output to a file instead of standard output + +CLI behavior: + +- exit code `0` means all selected tests passed +- exit code `1` means at least one test failed or the run could not be completed +- relative `--file` and `--output` paths are resolved from the app content root +- `--file` accepts the same exact paths, directories, and glob patterns as `Execution.TestFiles` +- when `--format` is omitted, the CLI still prints the human-readable execution summary + +Practical note: + +- CLI mode still boots the ASP.NET Core host internally before running the suite. That is intentional because the bundled sample suite targets the embedded `/sample-api/*` endpoints. + ## Dashboard workflow The main dashboard now exposes a test-selection panel before execution: @@ -110,11 +146,18 @@ Assumption: - The CI/CD platform is GitHub Actions. If you need Azure DevOps, GitLab CI, or Jenkins instead, the same stages can be ported. +Pipeline example: + +```powershell +dotnet run --project src/ApiTestRunner.App -- --ci --env Local --format junit --output artifacts/test-results.xml +``` + +This is a good default shape for CI jobs because it produces a predictable exit code and a result artifact that most pipeline systems can ingest. + ## Future enhancements The current version is intentionally focused on a local dashboard-driven workflow. The next major enhancements are: -- CLI and CI execution mode with options such as `--ci`, `--env`, and `--file`, plus machine-readable result output like JSON and JUnit XML for pipeline use - Response value capture and run-scoped variable reuse so one request can extract data such as tokens or IDs and pass them into later requests - Live execution progress in the dashboard so long-running suites can stream environment, endpoint, and test updates while the run is still in progress @@ -129,6 +172,12 @@ The current version is intentionally focused on a local dashboard-driven workflo - `Execution.MaxConcurrency` - `Execution.HttpTimeoutSeconds` +CLI mode can override part of this configuration per run: + +- `--file` overrides `Execution.TestFiles` +- `--env` narrows execution to specific loaded environments +- `--format` and `--output` affect only CLI result export and do not change dashboard behavior + `Execution.TestFiles` accepts: - Exact file paths diff --git a/src/ApiTestRunner.App/Options/CliExecutionOptions.cs b/src/ApiTestRunner.App/Options/CliExecutionOptions.cs new file mode 100644 index 0000000..207f151 --- /dev/null +++ b/src/ApiTestRunner.App/Options/CliExecutionOptions.cs @@ -0,0 +1,21 @@ +namespace ApiTestRunner.App.Options; + +public sealed class CliExecutionOptions +{ + public bool Enabled { get; init; } + + public IReadOnlyList EnvironmentNames { get; init; } = []; + + public IReadOnlyList TestFiles { get; init; } = []; + + public CliOutputFormat OutputFormat { get; init; } = CliOutputFormat.None; + + public string? OutputPath { get; init; } +} + +public enum CliOutputFormat +{ + None, + Json, + JUnit +} diff --git a/src/ApiTestRunner.App/Program.cs b/src/ApiTestRunner.App/Program.cs index 15ab19a..219b87e 100644 --- a/src/ApiTestRunner.App/Program.cs +++ b/src/ApiTestRunner.App/Program.cs @@ -6,6 +6,7 @@ using ApiTestRunner.Core.Services; using Microsoft.Extensions.Options; +var cliExecutionOptions = CliArgumentParser.Parse(args); var builder = WebApplication.CreateBuilder(args); var webServerOptions = builder.Configuration @@ -16,12 +17,21 @@ builder.Services.Configure(builder.Configuration.GetSection(WebServerOptions.SectionName)); builder.Services.Configure(builder.Configuration.GetSection(ExecutionOptions.SectionName)); +builder.Services.AddSingleton(Options.Create(cliExecutionOptions)); builder.Services.AddApiTestRunnerCore(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); -builder.Services.AddHostedService(); +builder.Services.AddSingleton(); +if (cliExecutionOptions.Enabled) +{ + builder.Services.AddHostedService(); +} +else +{ + builder.Services.AddHostedService(); +} builder.Services.AddHttpClient((serviceProvider, client) => { diff --git a/src/ApiTestRunner.App/Services/CliArgumentParser.cs b/src/ApiTestRunner.App/Services/CliArgumentParser.cs new file mode 100644 index 0000000..d01b28d --- /dev/null +++ b/src/ApiTestRunner.App/Services/CliArgumentParser.cs @@ -0,0 +1,86 @@ +using ApiTestRunner.App.Options; + +namespace ApiTestRunner.App.Services; + +public static class CliArgumentParser +{ + public static CliExecutionOptions Parse(string[] args) + { + ArgumentNullException.ThrowIfNull(args); + + var environmentNames = new List(); + var testFiles = new List(); + var enabled = false; + var outputFormat = CliOutputFormat.None; + string? outputPath = null; + + for (var index = 0; index < args.Length; index++) + { + var argument = args[index]; + + switch (argument) + { + case "--ci": + enabled = true; + break; + case "--env": + environmentNames.AddRange(ReadDelimitedValues(args, ref index, argument)); + break; + case "--file": + testFiles.Add(ReadSingleValue(args, ref index, argument)); + break; + case "--format": + outputFormat = ParseOutputFormat(ReadSingleValue(args, ref index, argument)); + break; + case "--output": + outputPath = ReadSingleValue(args, ref index, argument); + break; + } + } + + return new CliExecutionOptions + { + Enabled = enabled, + EnvironmentNames = environmentNames + .Where(value => !string.IsNullOrWhiteSpace(value)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(), + TestFiles = testFiles + .Where(value => !string.IsNullOrWhiteSpace(value)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(), + OutputFormat = outputFormat, + OutputPath = string.IsNullOrWhiteSpace(outputPath) ? null : outputPath + }; + } + + private static IReadOnlyList ReadDelimitedValues(string[] args, ref int index, string option) + { + var value = ReadSingleValue(args, ref index, option); + return value.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + } + + private static string ReadSingleValue(string[] args, ref int index, string option) + { + if (index + 1 >= args.Length) + { + throw new InvalidOperationException($"The CLI option '{option}' is missing a value."); + } + + index++; + return args[index]; + } + + private static CliOutputFormat ParseOutputFormat(string value) + { + return value.Trim().ToLowerInvariant() switch + { + "none" => CliOutputFormat.None, + "json" => CliOutputFormat.Json, + "junit" => CliOutputFormat.JUnit, + "junitxml" => CliOutputFormat.JUnit, + _ => throw new InvalidOperationException( + $"Unsupported CLI output format '{value}'. Supported values: none, json, junit.") + }; + } +} diff --git a/src/ApiTestRunner.App/Services/CliExecutionHostedService.cs b/src/ApiTestRunner.App/Services/CliExecutionHostedService.cs new file mode 100644 index 0000000..5ded1ea --- /dev/null +++ b/src/ApiTestRunner.App/Services/CliExecutionHostedService.cs @@ -0,0 +1,118 @@ +using ApiTestRunner.App.Options; +using ApiTestRunner.App.Models; +using ApiTestRunner.Core.Models; +using ApiTestRunner.Core.Services; +using Microsoft.Extensions.Options; + +namespace ApiTestRunner.App.Services; + +public sealed class CliExecutionHostedService : IHostedService +{ + private readonly IHostApplicationLifetime _applicationLifetime; + private readonly IConfiguredTestSuiteProvider _suiteProvider; + private readonly IApiTestExecutor _executor; + private readonly IOptions _executionOptions; + private readonly IOptions _cliExecutionOptions; + private readonly CliResultWriter _resultWriter; + private readonly ILogger _logger; + + public CliExecutionHostedService( + IHostApplicationLifetime applicationLifetime, + IConfiguredTestSuiteProvider suiteProvider, + IApiTestExecutor executor, + IOptions executionOptions, + IOptions cliExecutionOptions, + CliResultWriter resultWriter, + ILogger logger) + { + _applicationLifetime = applicationLifetime; + _suiteProvider = suiteProvider; + _executor = executor; + _executionOptions = executionOptions; + _cliExecutionOptions = cliExecutionOptions; + _resultWriter = resultWriter; + _logger = logger; + } + + public Task StartAsync(CancellationToken cancellationToken) + { + _applicationLifetime.ApplicationStarted.Register(() => + { + _ = Task.Run(async () => + { + try + { + var loadedSuite = await _suiteProvider.LoadAsync(CancellationToken.None); + var executionSuite = FilterSuiteByEnvironmentNames( + loadedSuite.Suite, + _cliExecutionOptions.Value.EnvironmentNames); + var result = await _executor.RunAsync( + executionSuite, + _executionOptions.Value.MaxConcurrency, + CancellationToken.None); + + WriteSummary(result, loadedSuite.FilePaths); + var outputPath = await _resultWriter.WriteAsync(result, CancellationToken.None); + if (!string.IsNullOrWhiteSpace(outputPath)) + { + await Console.Out.WriteLineAsync($"Wrote {_cliExecutionOptions.Value.OutputFormat} results to {outputPath}"); + } + + Environment.ExitCode = result.IsSuccess ? 0 : 1; + } + catch (Exception exception) + { + _logger.LogError(exception, "CLI execution failed"); + await Console.Error.WriteLineAsync(exception.Message); + Environment.ExitCode = 1; + } + finally + { + _applicationLifetime.StopApplication(); + } + }); + }); + + return Task.CompletedTask; + } + + public Task StopAsync(CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + + public static ApiTestSuiteDefinition FilterSuiteByEnvironmentNames( + ApiTestSuiteDefinition suite, + IReadOnlyList environmentNames) + { + if (environmentNames.Count == 0) + { + return suite; + } + + var names = new HashSet(environmentNames, StringComparer.OrdinalIgnoreCase); + var filteredEnvironments = suite.Environments + .Where(environment => names.Contains(environment.Name)) + .ToList(); + + if (filteredEnvironments.Count == 0) + { + throw new InvalidOperationException( + $"None of the requested environments were found: {string.Join(", ", environmentNames)}"); + } + + return new ApiTestSuiteDefinition + { + Environments = filteredEnvironments + }; + } + + private static void WriteSummary(TestRunResult result, IReadOnlyList filePaths) + { + Console.WriteLine($"Loaded {filePaths.Count} YAML files."); + Console.WriteLine($"Executed {result.TotalTests} tests across {result.TotalEndpoints} endpoints."); + Console.WriteLine($"Passed: {result.PassedTests}"); + Console.WriteLine($"Failed: {result.FailedTests}"); + Console.WriteLine($"Duration: {Math.Round(result.TotalDurationMs)} ms"); + } +} diff --git a/src/ApiTestRunner.App/Services/CliResultWriter.cs b/src/ApiTestRunner.App/Services/CliResultWriter.cs new file mode 100644 index 0000000..a052efc --- /dev/null +++ b/src/ApiTestRunner.App/Services/CliResultWriter.cs @@ -0,0 +1,104 @@ +using System.Text; +using System.Text.Json; +using System.Xml.Linq; +using ApiTestRunner.App.Options; +using ApiTestRunner.Core.Models; +using Microsoft.Extensions.Options; + +namespace ApiTestRunner.App.Services; + +public sealed class CliResultWriter +{ + private static readonly JsonSerializerOptions JsonSerializerOptions = new(JsonSerializerDefaults.Web) + { + WriteIndented = true + }; + + private readonly IHostEnvironment _hostEnvironment; + private readonly IOptions _cliExecutionOptions; + + public CliResultWriter( + IHostEnvironment hostEnvironment, + IOptions cliExecutionOptions) + { + _hostEnvironment = hostEnvironment; + _cliExecutionOptions = cliExecutionOptions; + } + + public async Task WriteAsync(TestRunResult result, CancellationToken cancellationToken = default) + { + var options = _cliExecutionOptions.Value; + if (options.OutputFormat == CliOutputFormat.None) + { + return null; + } + + var payload = options.OutputFormat switch + { + CliOutputFormat.Json => JsonSerializer.Serialize(result, JsonSerializerOptions), + CliOutputFormat.JUnit => BuildJUnitDocument(result).ToString(), + _ => throw new InvalidOperationException($"Unsupported output format '{options.OutputFormat}'.") + }; + + if (string.IsNullOrWhiteSpace(options.OutputPath)) + { + await Console.Out.WriteLineAsync(payload); + return null; + } + + var outputPath = ResolvePath(options.OutputPath); + var directory = Path.GetDirectoryName(outputPath); + if (!string.IsNullOrWhiteSpace(directory)) + { + Directory.CreateDirectory(directory); + } + + await File.WriteAllTextAsync(outputPath, payload, Encoding.UTF8, cancellationToken); + return outputPath; + } + + private string ResolvePath(string configuredPath) + { + return Path.IsPathRooted(configuredPath) + ? configuredPath + : Path.GetFullPath(Path.Combine(_hostEnvironment.ContentRootPath, configuredPath)); + } + + private static XDocument BuildJUnitDocument(TestRunResult result) + { + var suiteElement = new XElement( + "testsuite", + new XAttribute("name", "ApiTestRunner"), + new XAttribute("tests", result.TotalTests), + new XAttribute("failures", result.FailedTests), + new XAttribute("errors", 0), + new XAttribute("time", (result.TotalDurationMs / 1000d).ToString("0.###", System.Globalization.CultureInfo.InvariantCulture))); + + foreach (var environment in result.Environments) + { + foreach (var endpoint in environment.Endpoints) + { + foreach (var test in endpoint.Tests) + { + var testCaseElement = new XElement( + "testcase", + new XAttribute("classname", $"{environment.Name}.{endpoint.Name}"), + new XAttribute("name", test.Name), + new XAttribute("time", (endpoint.DurationMs / 1000d).ToString("0.###", System.Globalization.CultureInfo.InvariantCulture))); + + if (!test.IsSuccess) + { + testCaseElement.Add(new XElement( + "failure", + new XAttribute("message", test.ErrorMessage ?? "Test failed"), + test.ErrorMessage ?? "Test failed")); + } + + suiteElement.Add(testCaseElement); + } + } + } + + return new XDocument(new XElement("testsuites", suiteElement)); + } +} diff --git a/src/ApiTestRunner.App/Services/ConfiguredTestSuiteProvider.cs b/src/ApiTestRunner.App/Services/ConfiguredTestSuiteProvider.cs index 1e9d8ea..a3e2f24 100644 --- a/src/ApiTestRunner.App/Services/ConfiguredTestSuiteProvider.cs +++ b/src/ApiTestRunner.App/Services/ConfiguredTestSuiteProvider.cs @@ -12,21 +12,27 @@ public sealed class ConfiguredTestSuiteProvider : IConfiguredTestSuiteProvider private readonly IYamlTestSuiteLoader _loader; private readonly IOptions _executionOptions; + private readonly IOptions _cliExecutionOptions; private readonly IHostEnvironment _hostEnvironment; public ConfiguredTestSuiteProvider( IYamlTestSuiteLoader loader, IOptions executionOptions, + IOptions cliExecutionOptions, IHostEnvironment hostEnvironment) { _loader = loader; _executionOptions = executionOptions; + _cliExecutionOptions = cliExecutionOptions; _hostEnvironment = hostEnvironment; } public async Task LoadAsync(CancellationToken cancellationToken = default) { - var filePaths = ResolveConfiguredFiles(_executionOptions.Value.TestFiles); + var configuredFiles = _cliExecutionOptions.Value.Enabled && _cliExecutionOptions.Value.TestFiles.Count > 0 + ? _cliExecutionOptions.Value.TestFiles + : _executionOptions.Value.TestFiles; + var filePaths = ResolveConfiguredFiles(configuredFiles); var suite = await _loader.LoadAsync(filePaths, cancellationToken); return new LoadedTestSuite(suite, filePaths); } diff --git a/tests/ApiTestRunner.App.Tests/CliArgumentParserTests.cs b/tests/ApiTestRunner.App.Tests/CliArgumentParserTests.cs new file mode 100644 index 0000000..74e3b0d --- /dev/null +++ b/tests/ApiTestRunner.App.Tests/CliArgumentParserTests.cs @@ -0,0 +1,37 @@ +using ApiTestRunner.App.Options; +using ApiTestRunner.App.Services; + +namespace ApiTestRunner.App.Tests; + +public sealed class CliArgumentParserTests +{ + [Fact] + public void Parse_ReadsCiModeEnvironmentFilesAndOutputOptions() + { + var result = CliArgumentParser.Parse( + [ + "--ci", + "--env", "Uat,Prod", + "--env", "Canary", + "--file", "samples/endpoints/accounts.yaml", + "--file", "samples/endpoints/customers.yaml", + "--format", "junit", + "--output", "artifacts/results.xml" + ]); + + Assert.True(result.Enabled); + Assert.Equal(CliOutputFormat.JUnit, result.OutputFormat); + Assert.Equal("artifacts/results.xml", result.OutputPath); + Assert.Equal(["Uat", "Prod", "Canary"], result.EnvironmentNames); + Assert.Equal( + ["samples/endpoints/accounts.yaml", "samples/endpoints/customers.yaml"], + result.TestFiles); + } + + [Fact] + public void Parse_ThrowsForMissingOptionValue() + { + var exception = Assert.Throws(() => CliArgumentParser.Parse(["--format"])); + Assert.Contains("missing a value", exception.Message); + } +} diff --git a/tests/ApiTestRunner.App.Tests/CliExecutionAndResultWriterTests.cs b/tests/ApiTestRunner.App.Tests/CliExecutionAndResultWriterTests.cs new file mode 100644 index 0000000..95c6fd8 --- /dev/null +++ b/tests/ApiTestRunner.App.Tests/CliExecutionAndResultWriterTests.cs @@ -0,0 +1,121 @@ +using ApiTestRunner.App.Options; +using ApiTestRunner.App.Services; +using ApiTestRunner.Core.Models; +using Microsoft.Extensions.FileProviders; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Options; + +namespace ApiTestRunner.App.Tests; + +public sealed class CliExecutionAndResultWriterTests : IDisposable +{ + private readonly string _tempDirectory; + + public CliExecutionAndResultWriterTests() + { + _tempDirectory = Path.Combine(Path.GetTempPath(), "ApiTestRunnerCliTests", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_tempDirectory); + } + + [Fact] + public void FilterSuiteByEnvironmentNames_KeepsOnlyRequestedEnvironments() + { + var suite = new ApiTestSuiteDefinition + { + Environments = + [ + new EnvironmentDefinition { Name = "Local", BaseUrl = "http://localhost:5005" }, + new EnvironmentDefinition { Name = "Uat", BaseUrl = "https://uat.example.com" } + ] + }; + + var filtered = CliExecutionHostedService.FilterSuiteByEnvironmentNames(suite, ["Uat"]); + + var environment = Assert.Single(filtered.Environments); + Assert.Equal("Uat", environment.Name); + } + + [Fact] + public async Task WriteAsync_WritesJUnitXmlToConfiguredOutputPath() + { + var outputPath = Path.Combine(_tempDirectory, "results.xml"); + var writer = new CliResultWriter( + new StubHostEnvironment(_tempDirectory), + Microsoft.Extensions.Options.Options.Create(new CliExecutionOptions + { + Enabled = true, + OutputFormat = CliOutputFormat.JUnit, + OutputPath = outputPath + })); + + var result = new TestRunResult + { + StartedAtUtc = DateTimeOffset.UtcNow, + CompletedAtUtc = DateTimeOffset.UtcNow.AddSeconds(1), + Environments = + [ + new EnvironmentRunResult + { + Name = "Uat", + BaseUrl = "https://api.example.com", + Endpoints = + [ + new EndpointRunResult + { + Name = "Get Customer", + Method = "GET", + RequestUrl = "https://api.example.com/customers/C1001", + DurationMs = 100, + Tests = + [ + new TestCaseRunResult + { + Name = "Customer should load", + ExpectedStatus = 200, + ActualStatus = 500, + StatusMatched = false, + IsSuccess = false, + ErrorMessage = "Expected 200 but received 500" + } + ] + } + ] + } + ] + }; + + var writtenPath = await writer.WriteAsync(result); + + Assert.Equal(outputPath, writtenPath); + var xml = await File.ReadAllTextAsync(outputPath); + Assert.Contains("