feat: add CLI and CI execution mode with JSON and JUnit output

README.md

@@ -20,6 +20,7 @@
 - Path params, query params, headers, and JSON request bodies
 - Dot-notation assertions with array index support
 - Validation for strings, objects, and arrays
+- CLI and CI execution mode with environment and file filtering, JSON output, JUnit XML output, and non-zero exit codes on failed runs
 - Toggleable test selection in the dashboard, including select-all, clear-all, expand-all, collapse-all, and individual test control
 - Collapsible pass/fail reporting with environment, endpoint, test, and response-preview drill-down
 - cURL analysis page that scans configured YAML definitions, generates suggested environment and endpoint YAML when missing, and supports response-driven assertion building
@@ -43,6 +44,41 @@ dotnet run --project src/ApiTestRunner.App -c Release
 
 The app starts the dashboard at `http://localhost:5005` by default, auto-launches the browser if enabled, and executes the split sample suite after the web server is ready.
 
+## CLI and CI mode
+
+The same executable also supports headless execution for terminal and pipeline use.
+
+Basic examples:
+
+```powershell
+dotnet run --project src/ApiTestRunner.App -- --ci
+dotnet run --project src/ApiTestRunner.App -- --ci --env Local
+dotnet run --project src/ApiTestRunner.App -- --ci --env Local,UAT
+dotnet run --project src/ApiTestRunner.App -- --ci --file Samples/Environments/sample-api.yaml --file Samples/Endpoints/accounts/get-accounts.yaml
+dotnet run --project src/ApiTestRunner.App -- --ci --format json
+dotnet run --project src/ApiTestRunner.App -- --ci --format junit --output artifacts/test-results.xml
+```
+
+Supported CLI options:
+
+- `--ci` enables headless execution mode
+- `--env <name[,name2]>` runs only the named environments and can be repeated
+- `--file <path|glob|directory>` overrides `Execution.TestFiles` for this run and can be repeated
+- `--format <none|json|junit>` emits machine-readable results
+- `--output <path>` writes formatted output to a file instead of standard output
+
+CLI behavior:
+
+- exit code `0` means all selected tests passed
+- exit code `1` means at least one test failed or the run could not be completed
+- relative `--file` and `--output` paths are resolved from the app content root
+- `--file` accepts the same exact paths, directories, and glob patterns as `Execution.TestFiles`
+- when `--format` is omitted, the CLI still prints the human-readable execution summary
+
+Practical note:
+
+- CLI mode still boots the ASP.NET Core host internally before running the suite. That is intentional because the bundled sample suite targets the embedded `/sample-api/*` endpoints.
+
 ## Dashboard workflow
 
 The main dashboard now exposes a test-selection panel before execution:
@@ -110,11 +146,18 @@ Assumption:
 
 - The CI/CD platform is GitHub Actions. If you need Azure DevOps, GitLab CI, or Jenkins instead, the same stages can be ported.
 
+Pipeline example:
+
+```powershell
+dotnet run --project src/ApiTestRunner.App -- --ci --env Local --format junit --output artifacts/test-results.xml
+```
+
+This is a good default shape for CI jobs because it produces a predictable exit code and a result artifact that most pipeline systems can ingest.
+
 ## Future enhancements
 
 The current version is intentionally focused on a local dashboard-driven workflow. The next major enhancements are:
 
-- CLI and CI execution mode with options such as `--ci`, `--env`, and `--file`, plus machine-readable result output like JSON and JUnit XML for pipeline use
 - Response value capture and run-scoped variable reuse so one request can extract data such as tokens or IDs and pass them into later requests
 - Live execution progress in the dashboard so long-running suites can stream environment, endpoint, and test updates while the run is still in progress
 
@@ -129,6 +172,12 @@ The current version is intentionally focused on a local dashboard-driven workflo
 - `Execution.MaxConcurrency`
 - `Execution.HttpTimeoutSeconds`
 
+CLI mode can override part of this configuration per run:
+
+- `--file` overrides `Execution.TestFiles`
+- `--env` narrows execution to specific loaded environments
+- `--format` and `--output` affect only CLI result export and do not change dashboard behavior
+
 `Execution.TestFiles` accepts:
 
 - Exact file paths

src/ApiTestRunner.App/Options/CliExecutionOptions.cs

@@ -0,0 +1,21 @@
+namespace ApiTestRunner.App.Options;
+
+public sealed class CliExecutionOptions
+{
+    public bool Enabled { get; init; }
+
+    public IReadOnlyList<string> EnvironmentNames { get; init; } = [];
+
+    public IReadOnlyList<string> TestFiles { get; init; } = [];
+
+    public CliOutputFormat OutputFormat { get; init; } = CliOutputFormat.None;
+
+    public string? OutputPath { get; init; }
+}
+
+public enum CliOutputFormat
+{
+    None,
+    Json,
+    JUnit
+}

src/ApiTestRunner.App/Program.cs

@@ -6,6 +6,7 @@
 using ApiTestRunner.Core.Services;
 using Microsoft.Extensions.Options;
 
+var cliExecutionOptions = CliArgumentParser.Parse(args);
 var builder = WebApplication.CreateBuilder(args);
 
 var webServerOptions = builder.Configuration
@@ -16,12 +17,21 @@
 
 builder.Services.Configure<WebServerOptions>(builder.Configuration.GetSection(WebServerOptions.SectionName));
 builder.Services.Configure<ExecutionOptions>(builder.Configuration.GetSection(ExecutionOptions.SectionName));
+builder.Services.AddSingleton(Options.Create(cliExecutionOptions));
 
 builder.Services.AddApiTestRunnerCore();
 builder.Services.AddSingleton<IConfiguredTestSuiteProvider, ConfiguredTestSuiteProvider>();
 builder.Services.AddSingleton<ICurlCommandAnalyzer, CurlCommandAnalyzer>();
 builder.Services.AddSingleton<TestRunCoordinator>();
-builder.Services.AddHostedService<StartupAutomationHostedService>();
+builder.Services.AddSingleton<CliResultWriter>();
+if (cliExecutionOptions.Enabled)
+{
+    builder.Services.AddHostedService<CliExecutionHostedService>();
+}
+else
+{
+    builder.Services.AddHostedService<StartupAutomationHostedService>();
+}
 
 builder.Services.AddHttpClient<IApiTestExecutor, ApiTestExecutor>((serviceProvider, client) =>
 {

src/ApiTestRunner.App/Services/CliArgumentParser.cs

@@ -0,0 +1,86 @@
+using ApiTestRunner.App.Options;
+
+namespace ApiTestRunner.App.Services;
+
+public static class CliArgumentParser
+{
+    public static CliExecutionOptions Parse(string[] args)
+    {
+        ArgumentNullException.ThrowIfNull(args);
+
+        var environmentNames = new List<string>();
+        var testFiles = new List<string>();
+        var enabled = false;
+        var outputFormat = CliOutputFormat.None;
+        string? outputPath = null;
+
+        for (var index = 0; index < args.Length; index++)
+        {
+            var argument = args[index];
+
+            switch (argument)
+            {
+                case "--ci":
+                    enabled = true;
+                    break;
+                case "--env":
+                    environmentNames.AddRange(ReadDelimitedValues(args, ref index, argument));
+                    break;
+                case "--file":
+                    testFiles.Add(ReadSingleValue(args, ref index, argument));
+                    break;
+                case "--format":
+                    outputFormat = ParseOutputFormat(ReadSingleValue(args, ref index, argument));
+                    break;
+                case "--output":
+                    outputPath = ReadSingleValue(args, ref index, argument);
+                    break;
+            }
+        }
+
+        return new CliExecutionOptions
+        {
+            Enabled = enabled,
+            EnvironmentNames = environmentNames
+                .Where(value => !string.IsNullOrWhiteSpace(value))
+                .Distinct(StringComparer.OrdinalIgnoreCase)
+                .ToArray(),
+            TestFiles = testFiles
+                .Where(value => !string.IsNullOrWhiteSpace(value))
+                .Distinct(StringComparer.OrdinalIgnoreCase)
+                .ToArray(),
+            OutputFormat = outputFormat,
+            OutputPath = string.IsNullOrWhiteSpace(outputPath) ? null : outputPath
+        };
+    }
+
+    private static IReadOnlyList<string> ReadDelimitedValues(string[] args, ref int index, string option)
+    {
+        var value = ReadSingleValue(args, ref index, option);
+        return value.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
+    }
+
+    private static string ReadSingleValue(string[] args, ref int index, string option)
+    {
+        if (index + 1 >= args.Length)
+        {
+            throw new InvalidOperationException($"The CLI option '{option}' is missing a value.");
+        }
+
+        index++;
+        return args[index];
+    }
+
+    private static CliOutputFormat ParseOutputFormat(string value)
+    {
+        return value.Trim().ToLowerInvariant() switch
+        {
+            "none" => CliOutputFormat.None,
+            "json" => CliOutputFormat.Json,
+            "junit" => CliOutputFormat.JUnit,
+            "junitxml" => CliOutputFormat.JUnit,
+            _ => throw new InvalidOperationException(
+                $"Unsupported CLI output format '{value}'. Supported values: none, json, junit.")
+        };
+    }
+}

src/ApiTestRunner.App/Services/CliExecutionHostedService.cs

@@ -0,0 +1,118 @@
+using ApiTestRunner.App.Options;
+using ApiTestRunner.App.Models;
+using ApiTestRunner.Core.Models;
+using ApiTestRunner.Core.Services;
+using Microsoft.Extensions.Options;
+
+namespace ApiTestRunner.App.Services;
+
+public sealed class CliExecutionHostedService : IHostedService
+{
+    private readonly IHostApplicationLifetime _applicationLifetime;
+    private readonly IConfiguredTestSuiteProvider _suiteProvider;
+    private readonly IApiTestExecutor _executor;
+    private readonly IOptions<ExecutionOptions> _executionOptions;
+    private readonly IOptions<CliExecutionOptions> _cliExecutionOptions;
+    private readonly CliResultWriter _resultWriter;
+    private readonly ILogger<CliExecutionHostedService> _logger;
+
+    public CliExecutionHostedService(
+        IHostApplicationLifetime applicationLifetime,
+        IConfiguredTestSuiteProvider suiteProvider,
+        IApiTestExecutor executor,
+        IOptions<ExecutionOptions> executionOptions,
+        IOptions<CliExecutionOptions> cliExecutionOptions,
+        CliResultWriter resultWriter,
+        ILogger<CliExecutionHostedService> logger)
+    {
+        _applicationLifetime = applicationLifetime;
+        _suiteProvider = suiteProvider;
+        _executor = executor;
+        _executionOptions = executionOptions;
+        _cliExecutionOptions = cliExecutionOptions;
+        _resultWriter = resultWriter;
+        _logger = logger;
+    }
+
+    public Task StartAsync(CancellationToken cancellationToken)
+    {
+        _applicationLifetime.ApplicationStarted.Register(() =>
+        {
+            _ = Task.Run(async () =>
+            {
+                try
+                {
+                    var loadedSuite = await _suiteProvider.LoadAsync(CancellationToken.None);
+                    var executionSuite = FilterSuiteByEnvironmentNames(
+                        loadedSuite.Suite,
+                        _cliExecutionOptions.Value.EnvironmentNames);
+                    var result = await _executor.RunAsync(
+                        executionSuite,
+                        _executionOptions.Value.MaxConcurrency,
+                        CancellationToken.None);
+
+                    WriteSummary(result, loadedSuite.FilePaths);
+                    var outputPath = await _resultWriter.WriteAsync(result, CancellationToken.None);
+                    if (!string.IsNullOrWhiteSpace(outputPath))
+                    {
+                        await Console.Out.WriteLineAsync($"Wrote {_cliExecutionOptions.Value.OutputFormat} results to {outputPath}");
+                    }
+
+                    Environment.ExitCode = result.IsSuccess ? 0 : 1;
+                }
+                catch (Exception exception)
+                {
+                    _logger.LogError(exception, "CLI execution failed");
+                    await Console.Error.WriteLineAsync(exception.Message);
+                    Environment.ExitCode = 1;
+                }
+                finally
+                {
+                    _applicationLifetime.StopApplication();
+                }
+            });
+        });
+
+        return Task.CompletedTask;
+    }
+
+    public Task StopAsync(CancellationToken cancellationToken)
+    {
+        return Task.CompletedTask;
+    }
+
+    public static ApiTestSuiteDefinition FilterSuiteByEnvironmentNames(
+        ApiTestSuiteDefinition suite,
+        IReadOnlyList<string> environmentNames)
+    {
+        if (environmentNames.Count == 0)
+        {
+            return suite;
+        }
+
+        var names = new HashSet<string>(environmentNames, StringComparer.OrdinalIgnoreCase);
+        var filteredEnvironments = suite.Environments
+            .Where(environment => names.Contains(environment.Name))
+            .ToList();
+
+        if (filteredEnvironments.Count == 0)
+        {
+            throw new InvalidOperationException(
+                $"None of the requested environments were found: {string.Join(", ", environmentNames)}");
+        }
+
+        return new ApiTestSuiteDefinition
+        {
+            Environments = filteredEnvironments
+        };
+    }
+
+    private static void WriteSummary(TestRunResult result, IReadOnlyList<string> filePaths)
+    {
+        Console.WriteLine($"Loaded {filePaths.Count} YAML files.");
+        Console.WriteLine($"Executed {result.TotalTests} tests across {result.TotalEndpoints} endpoints.");
+        Console.WriteLine($"Passed: {result.PassedTests}");
+        Console.WriteLine($"Failed: {result.FailedTests}");
+        Console.WriteLine($"Duration: {Math.Round(result.TotalDurationMs)} ms");
+    }
+}