diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.cs index 232351c3af..21e715888d 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.cs @@ -332,7 +332,9 @@ internal void TestCompleted( string? errorMessage, Exception? exception, string? expected, - string? actual) + string? actual, + string? standardOutput, + string? errorOutput) { FlatException[] flatExceptions = ExceptionFlattener.Flatten(errorMessage, exception); TestCompleted( @@ -343,7 +345,9 @@ internal void TestCompleted( informativeMessage, flatExceptions, expected, - actual); + actual, + standardOutput, + errorOutput); } private void TestCompleted( @@ -354,7 +358,9 @@ private void TestCompleted( string? informativeMessage, FlatException[] exceptions, string? expected, - string? actual) + string? actual, + string? standardOutput, + string? errorOutput) { if (_testProgressState is null) { @@ -398,7 +404,9 @@ private void TestCompleted( informativeMessage, exceptions, expected, - actual)); + actual, + standardOutput, + errorOutput)); } } @@ -416,7 +424,9 @@ private void RenderTestCompleted( string? informativeMessage, FlatException[] flatExceptions, string? expected, - string? actual) + string? actual, + string? standardOutput, + string? errorOutput) { if (outcome == TestOutcome.Passed && !GetShowPassedTests()) { @@ -458,6 +468,23 @@ private void RenderTestCompleted( FormatExpectedAndActual(terminal, expected, actual); FormatStackTrace(terminal, flatExceptions, 0); FormatInnerExceptions(terminal, flatExceptions); + + bool isFailed = outcome is TestOutcome.Fail or TestOutcome.Error or TestOutcome.Timeout or TestOutcome.Canceled; + string? stdoutToShow = _options.ShowStdout switch + { + OutputShowMode.All => standardOutput, + OutputShowMode.Failed => isFailed ? standardOutput : null, + OutputShowMode.None => null, + _ => throw ApplicationStateGuard.Unreachable(), + }; + string? stderrToShow = _options.ShowStderr switch + { + OutputShowMode.All => errorOutput, + OutputShowMode.Failed => isFailed ? errorOutput : null, + OutputShowMode.None => null, + _ => throw ApplicationStateGuard.Unreachable(), + }; + FormatStandardAndErrorOutput(terminal, stdoutToShow, stderrToShow); } private static void FormatInnerExceptions(ITerminal terminal, FlatException[] exceptions) @@ -545,6 +572,36 @@ private static void FormatStackTrace(ITerminal terminal, FlatException[] excepti terminal.ResetColor(); } + private static void FormatStandardAndErrorOutput(ITerminal terminal, string? standardOutput, string? errorOutput) + { + bool hasStdOut = !RoslynString.IsNullOrWhiteSpace(standardOutput); + bool hasStdErr = !RoslynString.IsNullOrWhiteSpace(errorOutput); + if (!hasStdOut && !hasStdErr) + { + return; + } + + terminal.SetColor(TerminalColor.DarkGray); + + if (hasStdOut) + { + terminal.Append(SingleIndentation); + terminal.AppendLine(PlatformResources.StandardOutput); + string? standardOutputWithoutSpecialChars = MakeControlCharactersVisible(standardOutput, normalizeWhitespaceCharacters: false); + AppendIndentedLine(terminal, standardOutputWithoutSpecialChars, DoubleIndentation); + } + + if (hasStdErr) + { + terminal.Append(SingleIndentation); + terminal.AppendLine(PlatformResources.StandardError); + string? standardErrorWithoutSpecialChars = MakeControlCharactersVisible(errorOutput, normalizeWhitespaceCharacters: false); + AppendIndentedLine(terminal, standardErrorWithoutSpecialChars, DoubleIndentation); + } + + terminal.ResetColor(); + } + private void AppendAssemblyLinkTargetFrameworkAndArchitecture(ITerminal terminal) { terminal.AppendLink(_assembly, lineNumber: null); diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporterCommandLineOptionsProvider.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporterCommandLineOptionsProvider.cs index 5de9570121..f570f46ce7 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporterCommandLineOptionsProvider.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporterCommandLineOptionsProvider.cs @@ -16,6 +16,11 @@ internal sealed class TerminalTestReporterCommandLineOptionsProvider : ICommandL public const string OutputOption = "output"; public const string OutputOptionNormalArgument = "normal"; public const string OutputOptionDetailedArgument = "detailed"; + public const string ShowStdoutOption = "show-stdout"; + public const string ShowStderrOption = "show-stderr"; + public const string ShowOutputAllArgument = "all"; + public const string ShowOutputFailedArgument = "failed"; + public const string ShowOutputNoneArgument = "none"; /// public string Uid => nameof(TerminalTestReporterCommandLineOptionsProvider); @@ -38,6 +43,8 @@ public IReadOnlyCollection GetCommandLineOptions() new(NoProgressOption, PlatformResources.TerminalNoProgressOptionDescription, ArgumentArity.Zero, isHidden: false), new(NoAnsiOption, PlatformResources.TerminalNoAnsiOptionDescription, ArgumentArity.Zero, isHidden: false), new(OutputOption, PlatformResources.TerminalOutputOptionDescription, ArgumentArity.ExactlyOne, isHidden: false), + new(ShowStdoutOption, PlatformResources.TerminalShowStdoutOptionDescription, ArgumentArity.ExactlyOne, isHidden: false), + new(ShowStderrOption, PlatformResources.TerminalShowStderrOptionDescription, ArgumentArity.ExactlyOne, isHidden: false), ]; public Task ValidateOptionArgumentsAsync(CommandLineOption commandOption, string[] arguments) @@ -48,9 +55,17 @@ public Task ValidateOptionArgumentsAsync(CommandLineOption com OutputOption => OutputOptionNormalArgument.Equals(arguments[0], StringComparison.OrdinalIgnoreCase) || OutputOptionDetailedArgument.Equals(arguments[0], StringComparison.OrdinalIgnoreCase) ? ValidationResult.ValidTask : ValidationResult.InvalidTask(PlatformResources.TerminalOutputOptionInvalidArgument), + ShowStdoutOption or ShowStderrOption => IsValidShowOutputArgument(arguments[0]) + ? ValidationResult.ValidTask + : ValidationResult.InvalidTask(PlatformResources.TerminalShowOutputOptionInvalidArgument), _ => throw ApplicationStateGuard.Unreachable(), }; + private static bool IsValidShowOutputArgument(string argument) + => ShowOutputAllArgument.Equals(argument, StringComparison.OrdinalIgnoreCase) + || ShowOutputFailedArgument.Equals(argument, StringComparison.OrdinalIgnoreCase) + || ShowOutputNoneArgument.Equals(argument, StringComparison.OrdinalIgnoreCase); + public Task ValidateCommandLineOptionsAsync(ICommandLineOptions commandLineOptions) => // No problem found ValidationResult.ValidTask; diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporterOptions.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporterOptions.cs index a9eef16b5a..71f8d17ae1 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporterOptions.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporterOptions.cs @@ -32,6 +32,34 @@ internal sealed class TerminalTestReporterOptions /// Gets a value indicating the ANSI mode. /// public AnsiMode AnsiMode { get; init; } + + /// + /// Gets a value indicating when to show standard output. + /// + public OutputShowMode ShowStdout { get; init; } = OutputShowMode.All; + + /// + /// Gets a value indicating when to show standard error output. + /// + public OutputShowMode ShowStderr { get; init; } = OutputShowMode.All; +} + +internal enum OutputShowMode +{ + /// + /// Always show the output. + /// + All, + + /// + /// Show the output only for failed tests. + /// + Failed, + + /// + /// Never show the output. + /// + None, } internal enum AnsiMode diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs index a0984459c3..e219a977f1 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs @@ -150,6 +150,9 @@ await _policiesService.RegisterOnAbortCallbackAsync( showPassed = () => true; } + OutputShowMode showStdout = GetShowOutputMode(_commandLineOptions, TerminalTestReporterCommandLineOptionsProvider.ShowStdoutOption); + OutputShowMode showStderr = GetShowOutputMode(_commandLineOptions, TerminalTestReporterCommandLineOptionsProvider.ShowStderrOption); + Func shouldShowProgress = noProgress || ansiMode is AnsiMode.NoAnsi or AnsiMode.SimpleAnsi // User preference is to not show progress. // Or, we are in terminal that's not capable of changing cursor and we can't update progress in-place. @@ -174,9 +177,21 @@ await _policiesService.RegisterOnAbortCallbackAsync( AnsiMode = ansiMode, ShowActiveTests = true, ShowProgress = shouldShowProgress, + ShowStdout = showStdout, + ShowStderr = showStderr, }); } + private static OutputShowMode GetShowOutputMode(ICommandLineOptions commandLineOptions, string optionName) + => commandLineOptions.TryGetOptionArgumentList(optionName, out string[]? arguments) && arguments is { Length: > 0 } + ? arguments[0] switch + { + string s when TerminalTestReporterCommandLineOptionsProvider.ShowOutputFailedArgument.Equals(s, StringComparison.OrdinalIgnoreCase) => OutputShowMode.Failed, + string s when TerminalTestReporterCommandLineOptionsProvider.ShowOutputNoneArgument.Equals(s, StringComparison.OrdinalIgnoreCase) => OutputShowMode.None, + _ => OutputShowMode.All, + } + : OutputShowMode.All; + private static string GetShortArchitecture(string runtimeIdentifier) => runtimeIdentifier.Contains(Dash) ? runtimeIdentifier.Split(Dash, 2)[1] @@ -391,6 +406,8 @@ public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationTo case TestNodeUpdateMessage testNodeStateChanged: TimeSpan? duration = testNodeStateChanged.TestNode.Properties.SingleOrDefault()?.GlobalTiming.Duration; + string? standardOutput = testNodeStateChanged.TestNode.Properties.SingleOrDefault()?.StandardOutput; + string? standardError = testNodeStateChanged.TestNode.Properties.SingleOrDefault()?.StandardError; foreach (FileArtifactProperty artifact in testNodeStateChanged.TestNode.Properties.OfType()) { @@ -418,7 +435,9 @@ public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationTo errorState.Explanation, errorState.Exception, expected: null, - actual: null); + actual: null, + standardOutput, + standardError); break; case FailedTestNodeStateProperty failedState: @@ -431,7 +450,9 @@ public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationTo failedState.Explanation, failedState.Exception, expected: failedState.Exception?.Data["assert.expected"] as string, - actual: failedState.Exception?.Data["assert.actual"] as string); + actual: failedState.Exception?.Data["assert.actual"] as string, + standardOutput, + standardError); break; case TimeoutTestNodeStateProperty timeoutState: @@ -444,7 +465,9 @@ public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationTo timeoutState.Explanation, timeoutState.Exception, expected: null, - actual: null); + actual: null, + standardOutput, + standardError); break; #pragma warning disable CS0618 // Type or member is obsolete @@ -459,7 +482,9 @@ public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationTo cancelledState.Explanation, cancelledState.Exception, expected: null, - actual: null); + actual: null, + standardOutput, + standardError); break; case PassedTestNodeStateProperty: @@ -472,7 +497,9 @@ public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationTo errorMessage: null, exception: null, expected: null, - actual: null); + actual: null, + standardOutput, + standardError); break; case SkippedTestNodeStateProperty skippedState: @@ -485,7 +512,9 @@ public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationTo errorMessage: null, exception: null, expected: null, - actual: null); + actual: null, + standardOutput, + standardError); break; case DiscoveredTestNodeStateProperty: diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.resx b/src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.resx index 6f3637fe6b..73ae197b6a 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.resx +++ b/src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.resx @@ -606,6 +606,17 @@ Valid values are 'Normal', 'Detailed'. Default is 'Normal'. --output expects a single parameter with value 'Normal' or 'Detailed'. + + Determines when to show captured standard output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + + + Determines when to show captured error output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + + + --show-stdout and --show-stderr expect a single parameter with value 'All', 'Failed', or 'None'. + 'timeout' option should have one argument as string in the format <value>[h|m|s] where 'value' is float @@ -625,6 +636,12 @@ Takes one argument as string in the format <value>[h|m|s] where 'value' is Expected --client-port when jsonRpc protocol is used. + + Error output + + + Standard output + Discovering tests from diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.cs.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.cs.xlf index 2b924dedec..03edc65da1 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.cs.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.cs.xlf @@ -743,6 +743,16 @@ Může mít jenom jeden argument jako řetězec ve formátu <value>[h|m|s] Trasování zásobníku + + Error output + Chybový výstup + + + + Standard output + Standardní výstup + + Starting test session. Spouští se testovací relace. @@ -810,6 +820,25 @@ Platné hodnoty jsou Normal a Detailed. Výchozí hodnota je Normal. --output očekává jeden parametr s hodnotou Normal nebo Detailed. + + --show-stdout and --show-stderr expect a single parameter with value 'All', 'Failed', or 'None'. + --show-stdout and --show-stderr expect a single parameter with value 'All', 'Failed', or 'None'. + + + + Determines when to show captured error output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + Determines when to show captured error output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + + + + Determines when to show captured standard output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + Determines when to show captured standard output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + + Writes test results to terminal. Zapíše výsledky testu do terminálu. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.de.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.de.xlf index ce8e09d268..a3fbb6b162 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.de.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.de.xlf @@ -743,6 +743,16 @@ Nimmt ein Argument als Zeichenfolge im Format <value>[h|m|s], wobei "value Stapelüberwachung + + Error output + Fehlerausgabe + + + + Standard output + Standardausgabe + + Starting test session. Die Testsitzung wird gestartet. @@ -810,6 +820,25 @@ Gültige Werte sind „Normal“, „Detailed“. Der Standardwert ist „Normal „--output“ erwartet einen einzelnen Parameter mit dem Wert „Normal“ oder „Detailed“. + + --show-stdout and --show-stderr expect a single parameter with value 'All', 'Failed', or 'None'. + --show-stdout and --show-stderr expect a single parameter with value 'All', 'Failed', or 'None'. + + + + Determines when to show captured error output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + Determines when to show captured error output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + + + + Determines when to show captured standard output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + Determines when to show captured standard output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + + Writes test results to terminal. Schreibt Testergebnisse in das Terminal. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.es.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.es.xlf index fe210c069c..f9e73e27ac 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.es.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.es.xlf @@ -743,6 +743,16 @@ Toma un argumento como cadena con el formato <value>[h|m|s] donde 'value' Seguimiento de la pila + + Error output + Salida de error + + + + Standard output + Salida estándar + + Starting test session. Iniciando sesión de prueba. @@ -810,6 +820,25 @@ Los valores válidos son 'Normal', 'Detallado'. El valor predeterminado es 'Norm --output espera un único parámetro con el valor "Normal" o "Detallado". + + --show-stdout and --show-stderr expect a single parameter with value 'All', 'Failed', or 'None'. + --show-stdout and --show-stderr expect a single parameter with value 'All', 'Failed', or 'None'. + + + + Determines when to show captured error output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + Determines when to show captured error output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + + + + Determines when to show captured standard output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + Determines when to show captured standard output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + + Writes test results to terminal. Escribe los resultados de pruebas en el terminal. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.fr.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.fr.xlf index a751efeeaa..1a18192bce 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.fr.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.fr.xlf @@ -743,6 +743,16 @@ Prend un argument sous forme de chaîne au format <value>[h|m|s] où « v Rapport des appels de procédure + + Error output + Sortie d’erreur + + + + Standard output + Sortie standard + + Starting test session. Démarrage de la session de test. @@ -810,6 +820,25 @@ Les valeurs valides sont « Normal » et « Détaillé ». La valeur par dé --output attend un seul paramètre avec la valeur « Normal » ou « Détaillé ». + + --show-stdout and --show-stderr expect a single parameter with value 'All', 'Failed', or 'None'. + --show-stdout and --show-stderr expect a single parameter with value 'All', 'Failed', or 'None'. + + + + Determines when to show captured error output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + Determines when to show captured error output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + + + + Determines when to show captured standard output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + Determines when to show captured standard output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + + Writes test results to terminal. Écrit les résultats des tests dans le terminal. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.it.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.it.xlf index affb9fdab4..57671b4855 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.it.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.it.xlf @@ -743,6 +743,16 @@ Acquisisce un argomento come stringa nel formato <value>[h|m|s] dove 'valu Analisi dello stack + + Error output + Output errori + + + + Standard output + Output standard + + Starting test session. Avvio della sessione di test. @@ -810,6 +820,25 @@ I valori validi sono 'Normal', 'Detailed'. L'impostazione predefinita è 'Normal --l’output prevede un singolo parametro con un valore 'Normal' o 'Detailed'. + + --show-stdout and --show-stderr expect a single parameter with value 'All', 'Failed', or 'None'. + --show-stdout and --show-stderr expect a single parameter with value 'All', 'Failed', or 'None'. + + + + Determines when to show captured error output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + Determines when to show captured error output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + + + + Determines when to show captured standard output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + Determines when to show captured standard output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + + Writes test results to terminal. Scrive i risultati del test nel terminale. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ja.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ja.xlf index ad63d8d404..7d669ec126 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ja.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ja.xlf @@ -744,6 +744,16 @@ Takes one argument as string in the format <value>[h|m|s] where 'value' is スタック トレース + + Error output + エラー出力 + + + + Standard output + 標準出力 + + Starting test session. テスト セッションを開始しています。 @@ -811,6 +821,25 @@ Valid values are 'Normal', 'Detailed'. Default is 'Normal'. --output には、値が 'Normal' または 'Detailed' の単一のパラメーターが必要です。 + + --show-stdout and --show-stderr expect a single parameter with value 'All', 'Failed', or 'None'. + --show-stdout and --show-stderr expect a single parameter with value 'All', 'Failed', or 'None'. + + + + Determines when to show captured error output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + Determines when to show captured error output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + + + + Determines when to show captured standard output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + Determines when to show captured standard output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + + Writes test results to terminal. テスト結果をターミナルに書き込みます。 diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ko.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ko.xlf index 223c23822f..e206ccd191 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ko.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ko.xlf @@ -743,6 +743,16 @@ Takes one argument as string in the format <value>[h|m|s] where 'value' is 스택 추적 + + Error output + 오류 출력 + + + + Standard output + 표준 출력 + + Starting test session. 테스트 세션을 시작하는 중입니다. @@ -810,6 +820,25 @@ Valid values are 'Normal', 'Detailed'. Default is 'Normal'. --output에는 값이 'Normal' 또는 'Detailed'인 단일 매개 변수가 필요합니다. + + --show-stdout and --show-stderr expect a single parameter with value 'All', 'Failed', or 'None'. + --show-stdout and --show-stderr expect a single parameter with value 'All', 'Failed', or 'None'. + + + + Determines when to show captured error output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + Determines when to show captured error output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + + + + Determines when to show captured standard output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + Determines when to show captured standard output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + + Writes test results to terminal. 테스트 결과를 터미널에 씁니다. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pl.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pl.xlf index e1ea46816d..7eb4bd2488 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pl.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pl.xlf @@ -743,6 +743,16 @@ Pobiera jeden argument jako ciąg w formacie <value>[h|m|s], gdzie element Śledzenie stosu + + Error output + Dane wyjściowe w przypadku błędu + + + + Standard output + Standardowe dane wyjściowe + + Starting test session. Rozpoczynanie sesji testowej. @@ -810,6 +820,25 @@ Prawidłowe wartości to „Normalne”, „Szczegółowe”. Wartość domyśln Parametr --output oczekuje pojedynczego parametru o wartości „Normalne” lub „Szczegółowe”. + + --show-stdout and --show-stderr expect a single parameter with value 'All', 'Failed', or 'None'. + --show-stdout and --show-stderr expect a single parameter with value 'All', 'Failed', or 'None'. + + + + Determines when to show captured error output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + Determines when to show captured error output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + + + + Determines when to show captured standard output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + Determines when to show captured standard output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + + Writes test results to terminal. Zapisuje wyniki testu w terminalu. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pt-BR.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pt-BR.xlf index 7b73e59a16..8afd49e7c5 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pt-BR.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pt-BR.xlf @@ -743,6 +743,16 @@ Recebe um argumento como cadeia de caracteres no formato <valor>[h|m|s] em Rastreamento de Pilha + + Error output + Saída de erros + + + + Standard output + Saída padrão + + Starting test session. Iniciando sessão de teste. @@ -810,6 +820,25 @@ Os valores válidos são “Normal”, “Detalhado”. O padrão é “Normal --saída espera um único parâmetro com o valor “Normal” ou “Detalhado”. + + --show-stdout and --show-stderr expect a single parameter with value 'All', 'Failed', or 'None'. + --show-stdout and --show-stderr expect a single parameter with value 'All', 'Failed', or 'None'. + + + + Determines when to show captured error output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + Determines when to show captured error output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + + + + Determines when to show captured standard output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + Determines when to show captured standard output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + + Writes test results to terminal. Grava resultados de teste no terminal. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ru.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ru.xlf index b1234a8cc5..ddf18a9a43 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ru.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ru.xlf @@ -743,6 +743,16 @@ Takes one argument as string in the format <value>[h|m|s] where 'value' is Трассировка стека + + Error output + Вывод ошибок + + + + Standard output + Стандартный вывод + + Starting test session. Запуск тестового сеанса. @@ -810,6 +820,25 @@ Valid values are 'Normal', 'Detailed'. Default is 'Normal'. --output ожидает один параметр со значением "Normal" или "Detailed". + + --show-stdout and --show-stderr expect a single parameter with value 'All', 'Failed', or 'None'. + --show-stdout and --show-stderr expect a single parameter with value 'All', 'Failed', or 'None'. + + + + Determines when to show captured error output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + Determines when to show captured error output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + + + + Determines when to show captured standard output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + Determines when to show captured standard output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + + Writes test results to terminal. Записывает результаты теста в терминал. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.tr.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.tr.xlf index d84944175e..9cef9423d8 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.tr.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.tr.xlf @@ -743,6 +743,16 @@ Bir bağımsız değişkeni, 'value' değerinin kayan olduğu <value>[h|m| Yığın İzlemesi + + Error output + Hata çıkışı + + + + Standard output + Standart çıkış + + Starting test session. Test oturumu başlatılıyor. @@ -810,6 +820,25 @@ Geçerli değerler: ‘Normal’, ‘Ayrıntılı’. Varsayılan değer: ‘Nor --çıkışta ‘Normal’ veya ‘Ayrıntılı’ değerine sahip tek bir parametre beklenir. + + --show-stdout and --show-stderr expect a single parameter with value 'All', 'Failed', or 'None'. + --show-stdout and --show-stderr expect a single parameter with value 'All', 'Failed', or 'None'. + + + + Determines when to show captured error output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + Determines when to show captured error output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + + + + Determines when to show captured standard output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + Determines when to show captured standard output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + + Writes test results to terminal. Test sonuçlarını terminale yazar. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hans.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hans.xlf index a770808d02..99cc630fa3 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hans.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hans.xlf @@ -743,6 +743,16 @@ Takes one argument as string in the format <value>[h|m|s] where 'value' is 堆栈跟踪 + + Error output + 错误输出 + + + + Standard output + 标准输出 + + Starting test session. 正在启动测试会话。 @@ -810,6 +820,25 @@ Valid values are 'Normal', 'Detailed'. Default is 'Normal'. -- 输出应为值为“常规”或“详细”的单个参数。 + + --show-stdout and --show-stderr expect a single parameter with value 'All', 'Failed', or 'None'. + --show-stdout and --show-stderr expect a single parameter with value 'All', 'Failed', or 'None'. + + + + Determines when to show captured error output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + Determines when to show captured error output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + + + + Determines when to show captured standard output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + Determines when to show captured standard output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + + Writes test results to terminal. 将测试结果写入终端。 diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hant.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hant.xlf index a8968ac99e..2001fc6a04 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hant.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hant.xlf @@ -743,6 +743,16 @@ Takes one argument as string in the format <value>[h|m|s] where 'value' is 堆疊追蹤 + + Error output + 錯誤輸出 + + + + Standard output + 標準輸出 + + Starting test session. 正在啟動測試會話。 @@ -810,6 +820,25 @@ Valid values are 'Normal', 'Detailed'. Default is 'Normal'. --輸出需要值為 'Normal' 或 'Detailed' 的單一參數。 + + --show-stdout and --show-stderr expect a single parameter with value 'All', 'Failed', or 'None'. + --show-stdout and --show-stderr expect a single parameter with value 'All', 'Failed', or 'None'. + + + + Determines when to show captured error output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + Determines when to show captured error output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + + + + Determines when to show captured standard output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + Determines when to show captured standard output of a test. +Valid values are 'All', 'Failed', 'None'. Default is 'All'. + + Writes test results to terminal. 將測試結果寫入終端機。 diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs index ed1b9f778c..6c53b80a79 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs @@ -81,6 +81,12 @@ Output verbosity when reporting tests. Valid values are 'Normal', 'Detailed'. Default is 'Normal'. --settings The path, relative or absolute, to the .runsettings file. For more information and examples on how to configure test run, see https://learn.microsoft.com/visualstudio/test/configure-unit-tests-by-using-a-dot-runsettings-file#the-runsettings-file + --show-stderr + Determines when to show captured error output of a test. + Valid values are 'All', 'Failed', 'None'. Default is 'All'. + --show-stdout + Determines when to show captured standard output of a test. + Valid values are 'All', 'Failed', 'None'. Default is 'All'. --test-parameter Specify or override a key-value pair parameter. For more information and examples, see https://learn.microsoft.com/visualstudio/test/configure-unit-tests-by-using-a-dot-runsettings-file#testrunparameters """; diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/OutputTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/OutputTests.cs new file mode 100644 index 0000000000..62921fd2d5 --- /dev/null +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/OutputTests.cs @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform.Acceptance.IntegrationTests; +using Microsoft.Testing.Platform.Acceptance.IntegrationTests.Helpers; + +namespace MSTest.Acceptance.IntegrationTests; + +[TestClass] +public sealed class OutputTests : AcceptanceTestBase +{ + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task DetailedOutputIsAsExpected(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.ProjectPath, TestAssetFixture.ProjectName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync("--output detailed", cancellationToken: TestContext.CancellationToken); + + // Assert + testHostResult.AssertOutputContains("Assert.AreEqual failed. Expected:<1>. Actual:<2>."); + testHostResult.AssertOutputContains(""" + Standard output + Console message + TestContext Messages: + TestContext message + """); + } + + public sealed class TestAssetFixture() : TestAssetFixtureBase(AcceptanceFixture.NuGetGlobalPackagesFolder) + { + public const string ProjectName = "TestOutput"; + + public string ProjectPath => GetAssetPath(ProjectName); + + public override (string ID, string Name, string Code) GetAssetsToGenerate() => (ProjectName, ProjectName, + SourceCode + .PatchTargetFrameworks(TargetFrameworks.All) + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)); + + private const string SourceCode = """ +#file TestOutput.csproj + + + + Exe + true + $TargetFrameworks$ + + + + + + + + + +#file UnitTest1.cs +using System; +using System.Diagnostics; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[TestClass] +public class UnitTest1 +{ + public TestContext TestContext { get; set; } + + [TestMethod] + public void TestMethod() + { + Debug.WriteLine("Debug message"); + Console.WriteLine("Console message"); + TestContext.WriteLine("TestContext message"); + + Assert.AreEqual(1, 2); + } +} +"""; + } + + public TestContext TestContext { get; set; } +} diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/ShowOutputOptionTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/ShowOutputOptionTests.cs new file mode 100644 index 0000000000..bf6fd90e5f --- /dev/null +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/ShowOutputOptionTests.cs @@ -0,0 +1,199 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform.Acceptance.IntegrationTests; +using Microsoft.Testing.Platform.Acceptance.IntegrationTests.Helpers; +using Microsoft.Testing.Platform.Helpers; + +namespace MSTest.Acceptance.IntegrationTests; + +[TestClass] +public sealed class ShowOutputOptionTests : AcceptanceTestBase +{ + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task ShowStdout_None_NeverShowsStandardOutput(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.ProjectPath, TestAssetFixture.ProjectName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--show-stdout none --output detailed --no-progress --no-ansi", + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertOutputDoesNotContain("Standard output"); + testHostResult.AssertOutputDoesNotContain("stdout from failing test"); + testHostResult.AssertOutputDoesNotContain("stdout from passing test"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task ShowStdout_Failed_ShowsStandardOutputOnlyForFailedTests(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.ProjectPath, TestAssetFixture.ProjectName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--show-stdout failed --output detailed --no-progress --no-ansi", + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertOutputContains("stdout from failing test"); + testHostResult.AssertOutputDoesNotContain("stdout from passing test"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task ShowStdout_All_ShowsStandardOutputForAllTests(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.ProjectPath, TestAssetFixture.ProjectName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--show-stdout all --output detailed --no-progress --no-ansi", + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertOutputContains("stdout from failing test"); + testHostResult.AssertOutputContains("stdout from passing test"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task ShowStdout_Default_ShowsStandardOutputForAllTests(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.ProjectPath, TestAssetFixture.ProjectName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--output detailed --no-progress --no-ansi", + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertOutputContains("stdout from failing test"); + testHostResult.AssertOutputContains("stdout from passing test"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task ShowStdout_InvalidArgument_ReturnsError(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.ProjectPath, TestAssetFixture.ProjectName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--show-stdout invalid", + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIsNot(ExitCodes.Success); + testHostResult.AssertOutputContains("--show-stdout and --show-stderr expect a single parameter with value 'All', 'Failed', or 'None'."); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task ShowStderr_None_NeverShowsErrorOutput(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.ProjectPath, TestAssetFixture.ProjectName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--show-stderr none --output detailed --no-progress --no-ansi", + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertOutputDoesNotContain("Error output"); + testHostResult.AssertOutputDoesNotContain("stderr from failing test"); + testHostResult.AssertOutputDoesNotContain("stderr from passing test"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task ShowStderr_Failed_ShowsErrorOutputOnlyForFailedTests(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.ProjectPath, TestAssetFixture.ProjectName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--show-stderr failed --output detailed --no-progress --no-ansi", + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertOutputContains("stderr from failing test"); + testHostResult.AssertOutputDoesNotContain("stderr from passing test"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task ShowStderr_All_ShowsErrorOutputForAllTests(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.ProjectPath, TestAssetFixture.ProjectName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--show-stderr all --output detailed --no-progress --no-ansi", + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertOutputContains("stderr from failing test"); + testHostResult.AssertOutputContains("stderr from passing test"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task ShowStderr_Default_ShowsErrorOutputForAllTests(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.ProjectPath, TestAssetFixture.ProjectName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--output detailed --no-progress --no-ansi", + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertOutputContains("stderr from failing test"); + testHostResult.AssertOutputContains("stderr from passing test"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task ShowStderr_InvalidArgument_ReturnsError(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.ProjectPath, TestAssetFixture.ProjectName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--show-stderr invalid", + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIsNot(ExitCodes.Success); + testHostResult.AssertOutputContains("--show-stdout and --show-stderr expect a single parameter with value 'All', 'Failed', or 'None'."); + } + + public sealed class TestAssetFixture() : TestAssetFixtureBase(AcceptanceFixture.NuGetGlobalPackagesFolder) + { + public const string ProjectName = "ShowOutputOptionTest"; + + public string ProjectPath => GetAssetPath(ProjectName); + + public override (string ID, string Name, string Code) GetAssetsToGenerate() => (ProjectName, ProjectName, + SourceCode + .PatchTargetFrameworks(TargetFrameworks.All) + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)); + + private const string SourceCode = """ +#file ShowOutputOptionTest.csproj + + + + Exe + true + $TargetFrameworks$ + + + + + + + + + +#file UnitTest1.cs +using System; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[TestClass] +public class UnitTest1 +{ + [TestMethod] + public void FailingTest() + { + Console.WriteLine("stdout from failing test"); + Console.Error.WriteLine("stderr from failing test"); + Assert.Fail("intentional failure"); + } + + [TestMethod] + public void PassingTest() + { + Console.WriteLine("stdout from passing test"); + Console.Error.WriteLine("stderr from passing test"); + } +} +"""; + } + + public TestContext TestContext { get; set; } +} diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs index 29e749a9fa..dee8716b0a 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs @@ -102,6 +102,12 @@ Output verbosity when reporting tests. Enable generating TRX report --report-trx-filename The name of the generated TRX report + --show-stderr + Determines when to show captured error output of a test. + Valid values are 'All', 'Failed', 'None'. Default is 'All'. + --show-stdout + Determines when to show captured standard output of a test. + Valid values are 'All', 'Failed', 'None'. Default is 'All'. """; testHostResult.AssertOutputMatchesLines(wildcardPattern); @@ -350,6 +356,16 @@ Default type is 'Full' Hidden: False Description: Output verbosity when reporting tests. Valid values are 'Normal', 'Detailed'. Default is 'Normal'. + --show-stderr + Arity: 1 + Hidden: False + Description: Determines when to show captured error output of a test. + Valid values are 'All', 'Failed', 'None'. Default is 'All'. + --show-stdout + Arity: 1 + Hidden: False + Description: Determines when to show captured standard output of a test. + Valid values are 'All', 'Failed', 'None'. Default is 'All'. TrxReportGeneratorCommandLine Name: TRX report generator Version: * diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoTests.cs index 74a6900b10..8fbf6f0457 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoTests.cs @@ -69,6 +69,12 @@ Disable reporting progress to screen. --output Output verbosity when reporting tests. Valid values are 'Normal', 'Detailed'. Default is 'Normal'. + --show-stderr + Determines when to show captured error output of a test. + Valid values are 'All', 'Failed', 'None'. Default is 'All'. + --show-stdout + Determines when to show captured standard output of a test. + Valid values are 'All', 'Failed', 'None'. Default is 'All'. """; testHostResult.AssertOutputMatchesLines(wildcardMatchPattern); @@ -262,6 +268,16 @@ Takes one argument as string in the format \[h\|m\|s\] where 'value' is f Hidden: False Description: Output verbosity when reporting tests. Valid values are 'Normal', 'Detailed'. Default is 'Normal'. + --show-stderr + Arity: 1 + Hidden: False + Description: Determines when to show captured error output of a test. + Valid values are 'All', 'Failed', 'None'. Default is 'All'. + --show-stdout + Arity: 1 + Hidden: False + Description: Determines when to show captured standard output of a test. + Valid values are 'All', 'Failed', 'None'. Default is 'All'. Registered tools: There are no registered tools\. """; diff --git a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/OutputDevice/Terminal/TerminalTestReporterTests.cs b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/OutputDevice/Terminal/TerminalTestReporterTests.cs index 9a7df32cab..3c6ff7ec4f 100644 --- a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/OutputDevice/Terminal/TerminalTestReporterTests.cs +++ b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/OutputDevice/Terminal/TerminalTestReporterTests.cs @@ -121,18 +121,20 @@ public void NonAnsiTerminal_OutputFormattingIsCorrect() string folder = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? @"C:\work\" : "/mnt/work/"; terminalReporter.AssemblyRunStarted(); + string standardOutput = "Hello!"; + string errorOutput = "Oh no!"; terminalReporter.TestCompleted(testNodeUid: "PassedTest1", "PassedTest1", TestOutcome.Passed, TimeSpan.FromSeconds(10), - informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null); + informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput, errorOutput); terminalReporter.TestCompleted(testNodeUid: "SkippedTest1", "SkippedTest1", TestOutcome.Skipped, TimeSpan.FromSeconds(10), - informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null); + informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput, errorOutput); // timed out + canceled + failed should all report as failed in summary terminalReporter.TestCompleted(testNodeUid: "TimedoutTest1", "TimedoutTest1", TestOutcome.Timeout, TimeSpan.FromSeconds(10), - informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null); + informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput, errorOutput); terminalReporter.TestCompleted(testNodeUid: "CanceledTest1", "CanceledTest1", TestOutcome.Canceled, TimeSpan.FromSeconds(10), - informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null); + informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput, errorOutput); terminalReporter.TestCompleted(testNodeUid: "FailedTest1", "FailedTest1", TestOutcome.Fail, TimeSpan.FromSeconds(10), - informativeMessage: null, errorMessage: "Tests failed", exception: new StackTraceException(@$" at FailingTest() in {folder}codefile.cs:line 10"), expected: "ABC", actual: "DEF"); + informativeMessage: null, errorMessage: "Tests failed", exception: new StackTraceException(@$" at FailingTest() in {folder}codefile.cs:line 10"), expected: "ABC", actual: "DEF", standardOutput, errorOutput); terminalReporter.ArtifactAdded(outOfProcess: true, testName: null, @$"{folder}artifact1.txt"); terminalReporter.ArtifactAdded(outOfProcess: false, testName: null, @$"{folder}artifact2.txt"); terminalReporter.AssemblyRunCompleted(); @@ -142,9 +144,25 @@ public void NonAnsiTerminal_OutputFormattingIsCorrect() string expected = $""" passed PassedTest1 (10s 000ms) + Standard output + Hello! + Error output + Oh no! skipped SkippedTest1 (10s 000ms) + Standard output + Hello! + Error output + Oh no! failed (canceled) TimedoutTest1 (10s 000ms) + Standard output + Hello! + Error output + Oh no! failed (canceled) CanceledTest1 (10s 000ms) + Standard output + Hello! + Error output + Oh no! failed FailedTest1 (10s 000ms) Tests failed Expected @@ -152,6 +170,10 @@ Tests failed Actual DEF at FailingTest() in {folder}codefile.cs:10 + Standard output + Hello! + Error output + Oh no! Out of process file artifacts produced: - {folder}artifact1.txt @@ -195,18 +217,20 @@ public void SimpleAnsiTerminal_OutputFormattingIsCorrect() string folder = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? @"C:\work\" : "/mnt/work/"; terminalReporter.AssemblyRunStarted(); + string standardOutput = "Hello!"; + string errorOutput = "Oh no!"; terminalReporter.TestCompleted(testNodeUid: "PassedTest1", "PassedTest1", TestOutcome.Passed, TimeSpan.FromSeconds(10), - informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null); + informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput, errorOutput); terminalReporter.TestCompleted(testNodeUid: "SkippedTest1", "SkippedTest1", TestOutcome.Skipped, TimeSpan.FromSeconds(10), - informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null); + informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput, errorOutput); // timed out + canceled + failed should all report as failed in summary terminalReporter.TestCompleted(testNodeUid: "TimedoutTest1", "TimedoutTest1", TestOutcome.Timeout, TimeSpan.FromSeconds(10), - informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null); + informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput, errorOutput); terminalReporter.TestCompleted(testNodeUid: "CanceledTest1", "CanceledTest1", TestOutcome.Canceled, TimeSpan.FromSeconds(10), - informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null); + informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput, errorOutput); terminalReporter.TestCompleted(testNodeUid: "FailedTest1", "FailedTest1", TestOutcome.Fail, TimeSpan.FromSeconds(10), - informativeMessage: null, errorMessage: "Tests failed", exception: new StackTraceException(@$" at FailingTest() in {folder}codefile.cs:line 10"), expected: "ABC", actual: "DEF"); + informativeMessage: null, errorMessage: "Tests failed", exception: new StackTraceException(@$" at FailingTest() in {folder}codefile.cs:line 10"), expected: "ABC", actual: "DEF", standardOutput, errorOutput); terminalReporter.ArtifactAdded(outOfProcess: true, testName: null, @$"{folder}artifact1.txt"); terminalReporter.ArtifactAdded(outOfProcess: false, testName: null, @$"{folder}artifact2.txt"); terminalReporter.AssemblyRunCompleted(); @@ -216,16 +240,36 @@ public void SimpleAnsiTerminal_OutputFormattingIsCorrect() string expected = $""" ␛[32mpassed␛[m PassedTest1 ␛[90m(10s 000ms)␛[m - ␛[33mskipped␛[m SkippedTest1 ␛[90m(10s 000ms)␛[m - ␛[31mfailed (canceled)␛[m TimedoutTest1 ␛[90m(10s 000ms)␛[m - ␛[31mfailed (canceled)␛[m CanceledTest1 ␛[90m(10s 000ms)␛[m - ␛[31mfailed␛[m FailedTest1 ␛[90m(10s 000ms)␛[m + ␛[90m Standard output + ␛[90m Hello! + ␛[90m Error output + ␛[90m Oh no! + ␛[m␛[33mskipped␛[m SkippedTest1 ␛[90m(10s 000ms)␛[m + ␛[90m Standard output + ␛[90m Hello! + ␛[90m Error output + ␛[90m Oh no! + ␛[m␛[31mfailed (canceled)␛[m TimedoutTest1 ␛[90m(10s 000ms)␛[m + ␛[90m Standard output + ␛[90m Hello! + ␛[90m Error output + ␛[90m Oh no! + ␛[m␛[31mfailed (canceled)␛[m CanceledTest1 ␛[90m(10s 000ms)␛[m + ␛[90m Standard output + ␛[90m Hello! + ␛[90m Error output + ␛[90m Oh no! + ␛[m␛[31mfailed␛[m FailedTest1 ␛[90m(10s 000ms)␛[m ␛[31m Tests failed ␛[m␛[31m Expected ␛[31m ABC ␛[31m Actual ␛[31m DEF ␛[m␛[90m at FailingTest() in {folder}codefile.cs:10␛[90m + ␛[m␛[90m Standard output + ␛[90m Hello! + ␛[90m Error output + ␛[90m Oh no! ␛[m Out of process file artifacts produced: - {folder}artifact1.txt @@ -270,18 +314,20 @@ public void AnsiTerminal_OutputFormattingIsCorrect() string folderLinkNoSlash = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? @"C:/work" : "mnt/work"; terminalReporter.AssemblyRunStarted(); + string standardOutput = "Hello!"; + string errorOutput = "Oh no!"; terminalReporter.TestCompleted(testNodeUid: "PassedTest1", "PassedTest1", TestOutcome.Passed, TimeSpan.FromSeconds(10), - informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null); + informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput, errorOutput); terminalReporter.TestCompleted(testNodeUid: "SkippedTest1", "SkippedTest1", TestOutcome.Skipped, TimeSpan.FromSeconds(10), - informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null); + informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput, errorOutput); // timed out + canceled + failed should all report as failed in summary terminalReporter.TestCompleted(testNodeUid: "TimedoutTest1", "TimedoutTest1", TestOutcome.Timeout, TimeSpan.FromSeconds(10), - informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null); + informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput, errorOutput); terminalReporter.TestCompleted(testNodeUid: "CanceledTest1", "CanceledTest1", TestOutcome.Canceled, TimeSpan.FromSeconds(10), - informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null); + informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput, errorOutput); terminalReporter.TestCompleted(testNodeUid: "FailedTest1", "FailedTest1", TestOutcome.Fail, TimeSpan.FromSeconds(10), - informativeMessage: null, errorMessage: "Tests failed", exception: new StackTraceException(@$" at FailingTest() in {folder}codefile.cs:line 10"), expected: "ABC", actual: "DEF"); + informativeMessage: null, errorMessage: "Tests failed", exception: new StackTraceException(@$" at FailingTest() in {folder}codefile.cs:line 10"), expected: "ABC", actual: "DEF", standardOutput, errorOutput); terminalReporter.ArtifactAdded(outOfProcess: true, testName: null, @$"{folder}artifact1.txt"); terminalReporter.ArtifactAdded(outOfProcess: false, testName: null, @$"{folder}artifact2.txt"); terminalReporter.AssemblyRunCompleted(); @@ -291,16 +337,36 @@ public void AnsiTerminal_OutputFormattingIsCorrect() string expected = $""" ␛[32mpassed␛[m PassedTest1 ␛[90m(10s 000ms)␛[m - ␛[33mskipped␛[m SkippedTest1 ␛[90m(10s 000ms)␛[m - ␛[31mfailed (canceled)␛[m TimedoutTest1 ␛[90m(10s 000ms)␛[m - ␛[31mfailed (canceled)␛[m CanceledTest1 ␛[90m(10s 000ms)␛[m - ␛[31mfailed␛[m FailedTest1 ␛[90m(10s 000ms)␛[m + ␛[90m Standard output + Hello! + Error output + Oh no! + ␛[m␛[33mskipped␛[m SkippedTest1 ␛[90m(10s 000ms)␛[m + ␛[90m Standard output + Hello! + Error output + Oh no! + ␛[m␛[31mfailed (canceled)␛[m TimedoutTest1 ␛[90m(10s 000ms)␛[m + ␛[90m Standard output + Hello! + Error output + Oh no! + ␛[m␛[31mfailed (canceled)␛[m CanceledTest1 ␛[90m(10s 000ms)␛[m + ␛[90m Standard output + Hello! + Error output + Oh no! + ␛[m␛[31mfailed␛[m FailedTest1 ␛[90m(10s 000ms)␛[m ␛[31m Tests failed ␛[m␛[31m Expected ABC Actual DEF ␛[m␛[90m at FailingTest() in ␛[90m␛]8;;file:///{folderLink}codefile.cs␛\{folder}codefile.cs:10␛]8;;␛\␛[m␛[90m + ␛[m␛[90m Standard output + Hello! + Error output + Oh no! ␛[m Out of process file artifacts produced: - ␛[90m␛]8;;file:///{folderLink}artifact1.txt␛\{folder}artifact1.txt␛]8;;␛\␛[m @@ -355,6 +421,8 @@ public void AnsiTerminal_OutputProgressFrameIsCorrect() string folder = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? @"C:\work\" : "/mnt/work/"; terminalReporter.AssemblyRunStarted(); + string standardOutput = "Hello!"; + string errorOutput = "Oh no!"; // Note: Add 1ms to make the order of the progress frame deterministic. // Otherwise all tests that run for 1m31s could show in any order. @@ -370,9 +438,9 @@ public void AnsiTerminal_OutputProgressFrameIsCorrect() stopwatchFactory.AddTime(TimeSpan.FromSeconds(1)); terminalReporter.TestCompleted(testNodeUid: "PassedTest1", "PassedTest1", TestOutcome.Passed, TimeSpan.FromSeconds(10), - informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null); + informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput, errorOutput); terminalReporter.TestCompleted(testNodeUid: "SkippedTest1", "SkippedTest1", TestOutcome.Skipped, TimeSpan.FromSeconds(10), - informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null); + informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput, errorOutput); string output = stringBuilderConsole.Output; startHandle.Set(); @@ -385,7 +453,11 @@ public void AnsiTerminal_OutputProgressFrameIsCorrect() // Note: The progress is drawn after each completed event. string expected = $""" {busyIndicatorString}␛[?25l␛[32mpassed␛[m PassedTest1 ␛[90m(10s 000ms)␛[m - + ␛[90m Standard output + Hello! + Error output + Oh no! + ␛[m [␛[32m✓1␛[m/␛[31mx0␛[m/␛[33m↓0␛[m] assembly.dll (net8.0|x64)␛[242G(1m 31s) SkippedTest1␛[242G(1m 31s) InProgressTest1␛[242G(1m 31s) @@ -393,7 +465,11 @@ public void AnsiTerminal_OutputProgressFrameIsCorrect() InProgressTest3␛[246G(1s) ␛[7F ␛[J␛[33mskipped␛[m SkippedTest1 ␛[90m(10s 000ms)␛[m - + ␛[90m Standard output + Hello! + Error output + Oh no! + ␛[m [␛[32m✓1␛[m/␛[31mx0␛[m/␛[33m↓1␛[m] assembly.dll (net8.0|x64)␛[242G(1m 31s) InProgressTest1␛[242G(1m 31s) InProgressTest2␛[245G(31s) @@ -436,6 +512,85 @@ public void TestProgressStateAwareTerminal_WriteToTerminal_ShouldEraseProgressTh Assert.IsGreaterThan(renderIndex, stopUpdateIndex, "StopUpdate should be called after rendering progress."); } + [TestMethod] + public void NonAnsiTerminal_ShowOutputNone_DoesNotShowOutput() + { + string targetFramework = "net8.0"; + string architecture = "x64"; + string assembly = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? @"C:\work\assembly.dll" : "/mnt/work/assembly.dll"; + + var stringBuilderConsole = new StringBuilderConsole(); + var terminalReporter = new TerminalTestReporter(assembly, targetFramework, architecture, stringBuilderConsole, new CTRLPlusCCancellationTokenSource(), new TerminalTestReporterOptions + { + ShowPassedTests = () => true, + AnsiMode = AnsiMode.NoAnsi, + ShowProgress = () => false, + ShowStdout = OutputShowMode.None, + ShowStderr = OutputShowMode.None, + }); + + DateTimeOffset startTime = DateTimeOffset.MinValue; + DateTimeOffset endTime = DateTimeOffset.MaxValue; + terminalReporter.TestExecutionStarted(startTime, 1, isDiscovery: false); + terminalReporter.AssemblyRunStarted(); + + terminalReporter.TestCompleted(testNodeUid: "PassedTest1", "PassedTest1", TestOutcome.Passed, TimeSpan.FromSeconds(1), + informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput: "Hello!", errorOutput: "Oh no!"); + terminalReporter.TestCompleted(testNodeUid: "FailedTest1", "FailedTest1", TestOutcome.Fail, TimeSpan.FromSeconds(1), + informativeMessage: null, errorMessage: "Tests failed", exception: null, expected: null, actual: null, standardOutput: "Hello!", errorOutput: "Oh no!"); + + terminalReporter.AssemblyRunCompleted(); + terminalReporter.TestExecutionCompleted(endTime); + + string output = stringBuilderConsole.Output; + + Assert.DoesNotContain("Standard output", output); + Assert.DoesNotContain("Error output", output); + Assert.DoesNotContain("Hello!", output); + Assert.DoesNotContain("Oh no!", output); + } + + [TestMethod] + public void NonAnsiTerminal_ShowOutputFailed_ShowsOutputOnlyForFailedTests() + { + string targetFramework = "net8.0"; + string architecture = "x64"; + string assembly = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? @"C:\work\assembly.dll" : "/mnt/work/assembly.dll"; + + var stringBuilderConsole = new StringBuilderConsole(); + var terminalReporter = new TerminalTestReporter(assembly, targetFramework, architecture, stringBuilderConsole, new CTRLPlusCCancellationTokenSource(), new TerminalTestReporterOptions + { + ShowPassedTests = () => true, + AnsiMode = AnsiMode.NoAnsi, + ShowProgress = () => false, + ShowStdout = OutputShowMode.Failed, + ShowStderr = OutputShowMode.Failed, + }); + + DateTimeOffset startTime = DateTimeOffset.MinValue; + DateTimeOffset endTime = DateTimeOffset.MaxValue; + terminalReporter.TestExecutionStarted(startTime, 1, isDiscovery: false); + terminalReporter.AssemblyRunStarted(); + + terminalReporter.TestCompleted(testNodeUid: "PassedTest1", "PassedTest1", TestOutcome.Passed, TimeSpan.FromSeconds(1), + informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput: "passed-stdout", errorOutput: "passed-stderr"); + terminalReporter.TestCompleted(testNodeUid: "FailedTest1", "FailedTest1", TestOutcome.Fail, TimeSpan.FromSeconds(1), + informativeMessage: null, errorMessage: "Tests failed", exception: null, expected: null, actual: null, standardOutput: "failed-stdout", errorOutput: "failed-stderr"); + + terminalReporter.AssemblyRunCompleted(); + terminalReporter.TestExecutionCompleted(endTime); + + string output = stringBuilderConsole.Output; + + // stdout/stderr for passed tests should NOT appear + Assert.DoesNotContain("passed-stdout", output); + Assert.DoesNotContain("passed-stderr", output); + + // stdout/stderr for failed tests SHOULD appear + Assert.Contains("failed-stdout", output); + Assert.Contains("failed-stderr", output); + } + private static string? ShowEscape(string? text) { string visibleEsc = "\x241b"; @@ -672,7 +827,7 @@ public void TestDisplayNames_WithControlCharacters_AreNormalized(char controlCha // Test display name with the specific control character string testDisplayName = $"Test{controlChar}Name"; terminalReporter.TestCompleted(testNodeUid: "Test1", testDisplayName, TestOutcome.Passed, TimeSpan.FromSeconds(1), - informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null); + informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput: null, errorOutput: null); terminalReporter.AssemblyRunCompleted(); terminalReporter.TestExecutionCompleted(endTime); @@ -907,7 +1062,7 @@ public void AnsiTerminal_ProgressFrame_UseWindowWidthForCursorPositioning_WhenBu stopwatchFactory.AddTime(TimeSpan.FromMinutes(1) + TimeSpan.FromSeconds(31)); terminalReporter.TestCompleted(testNodeUid: "Test1", "Test1", TestOutcome.Passed, TimeSpan.FromSeconds(10), - informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null); + informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput: null, errorOutput: null); string output = stringBuilderConsole.Output; startHandle.Set();