From 3471ad89e02c688f9be280663078fc6199374a1f Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Fri, 25 Sep 2020 13:42:10 +0200 Subject: [PATCH 1/3] [xharness] Use TextReader/Writer instead of StreamReader/Writer in a few places. This makes it easier to write tests, since TextReader/Writer can use in-memory streams (no need to mess with temporary files). --- .../IResultParser.cs | 2 +- .../XmlResultParser.cs | 124 +++++++++--------- 2 files changed, 65 insertions(+), 61 deletions(-) diff --git a/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared/IResultParser.cs b/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared/IResultParser.cs index cbbc2721352f..39b45b2b235c 100644 --- a/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared/IResultParser.cs +++ b/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared/IResultParser.cs @@ -12,7 +12,7 @@ public interface IResultParser { // failure perse but the situation in which the app could not be built, timeout or crashed. void GenerateFailure (ILogs logs, string source, string appName, string variation, string title, string message, string stderrPath, XmlResultJargon jargon); - void GenerateFailure (ILogs logs, string source, string appName, string variation, string title, string message, StreamReader stderrReader, XmlResultJargon jargon); + void GenerateFailure (ILogs logs, string source, string appName, string variation, string title, string message, TextReader stderrReader, XmlResultJargon jargon); // updates a given xml result to contain a list of attachments. This is useful for CI to be able to add logs as part of the attachments of a failing test. void UpdateMissingData (string source, string destination, string applicationName, IEnumerable attachments); diff --git a/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared/XmlResultParser.cs b/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared/XmlResultParser.cs index 8e787e676242..d44eb9b293a4 100644 --- a/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared/XmlResultParser.cs +++ b/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared/XmlResultParser.cs @@ -28,37 +28,45 @@ public bool IsValidXml (string path, out XmlResultJargon type) if (!File.Exists (path)) return false; - using (var stream = File.OpenText (path)) { - string line; - while ((line = stream.ReadLine ()) != null) { // special case when get got the tcp connection - if (line.Contains ("ping")) - continue; - if (line.Contains ("test-run")) { // first element of the NUnitV3 test collection - type = XmlResultJargon.NUnitV3; - return true; - } - if (line.Contains ("TouchUnitTestRun")) { - type = XmlResultJargon.TouchUnit; - return true; - } - if (line.Contains ("test-results")) { // first element of the NUnitV3 test collection - type = XmlResultJargon.NUnitV2; - return true; - } - if (line.Contains ("")) { // first element of the xUnit test collection - type = XmlResultJargon.xUnit; - return true; - } - if (line.Contains ("")) { // first element of the xUnit test collection + type = XmlResultJargon.xUnit; + return true; + } + if (line.Contains (" ("time", "0"), ("asserts", "1")); - static void WriteNUnitV2TestCase (XmlWriter writer, string title, string message, StreamReader stderr) + static void WriteNUnitV2TestCase (XmlWriter writer, string title, string message, TextReader stderr) { writer.WriteStartElement ("test-case"); WriteAttributes (writer, @@ -787,7 +791,7 @@ static void WriteNUnitV2TestCase (XmlWriter writer, string title, string message writer.WriteEndElement (); // test-case } - static void GenerateNUnitV2Failure (XmlWriter writer, string title, string message, StreamReader stderr) + static void GenerateNUnitV2Failure (XmlWriter writer, string title, string message, TextReader stderr) { writer.WriteStartElement ("test-results"); WriteAttributes (writer, @@ -832,7 +836,7 @@ static void WriteNUnitV3TestSuiteAttributes (XmlWriter writer, string title) => ("skipped", "0"), ("asserts", "0")); - static void WriteFailure (XmlWriter writer, string message, StreamReader? stderr = null) + static void WriteFailure (XmlWriter writer, string message, TextReader? stderr = null) { writer.WriteStartElement ("failure"); writer.WriteStartElement ("message"); @@ -846,7 +850,7 @@ static void WriteFailure (XmlWriter writer, string message, StreamReader? stderr writer.WriteEndElement (); // failure } - static void GenerateNUnitV3Failure (XmlWriter writer, string title, string message, StreamReader stderr) + static void GenerateNUnitV3Failure (XmlWriter writer, string title, string message, TextReader stderr) { var date = DateTime.Now; writer.WriteStartElement ("test-run"); @@ -898,7 +902,7 @@ static void GenerateNUnitV3Failure (XmlWriter writer, string title, string messa writer.WriteEndElement (); // test-run } - static void GeneratexUnitFailure (XmlWriter writer, string title, string message, StreamReader stderr) + static void GeneratexUnitFailure (XmlWriter writer, string title, string message, TextReader stderr) { writer.WriteStartElement ("assemblies"); writer.WriteStartElement ("assembly"); @@ -935,7 +939,7 @@ static void GeneratexUnitFailure (XmlWriter writer, string title, string message writer.WriteEndElement (); // assemblies } - static void GenerateFailureXml (string destination, string title, string message, StreamReader stderrReader, XmlResultJargon jargon) + static void GenerateFailureXml (string destination, string title, string message, TextReader stderrReader, XmlResultJargon jargon) { XmlWriterSettings settings = new XmlWriterSettings { Indent = true }; using (var stream = File.CreateText (destination)) @@ -957,7 +961,7 @@ static void GenerateFailureXml (string destination, string title, string message } public void GenerateFailure (ILogs logs, string source, string appName, string variation, string title, - string message, StreamReader stderr, XmlResultJargon jargon) + string message, TextReader stderr, XmlResultJargon jargon) { // VSTS does not provide a nice way to report build errors, create a fake // test result with a failure in the case the build did not work From 782bee091560936ba0efce52798abc8e879c19f1 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Fri, 25 Sep 2020 14:33:32 +0200 Subject: [PATCH 2/3] [xharness] Fix rendering human-readable results for parameterized tests. Fixes #9400. We weren't probably parsing nested test-suite elements in the xml, so implement parsing of test-suite elements using recursion, which is easier and less error-prone (and turns out to work fine too). Fixes https://github.com/xamarin/xamarin-macios/issues/9400. --- .../NUnitV3SampleParameterizedFailure.xml | 3696 +++++++++++++++++ .../XmlResultParserTests.cs | 68 +- .../XmlResultParser.cs | 137 +- 3 files changed, 3824 insertions(+), 77 deletions(-) create mode 100644 tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/Samples/NUnitV3SampleParameterizedFailure.xml diff --git a/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/Samples/NUnitV3SampleParameterizedFailure.xml b/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/Samples/NUnitV3SampleParameterizedFailure.xml new file mode 100644 index 000000000000..49fb7fcf34a3 --- /dev/null +++ b/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/Samples/NUnitV3SampleParameterizedFailure.xml @@ -0,0 +1,3696 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + /Users/builder/jenkins/workspace/xamarin-macios-pr-builder/jenkins-results/tests/xammac-tests/311/execute-Mac_Modern-20200923_052627.txt + + + /Users/builder/jenkins/workspace/xamarin-macios-pr-builder/jenkins-results/tests/xammac-tests/311/main-20200923_052836.log + + + /Users/builder/jenkins/workspace/xamarin-macios-pr-builder/jenkins-results/tests/xammac-tests/311/test-Mac_Modern-20200923_052627-clean.xml + + + /Users/builder/jenkins/workspace/xamarin-macios-pr-builder/jenkins-results/tests/xammac-tests/311/test-Mac_Modern-20200923_052627.xml + + + + + + + + + + + + + + + + + + /Users/builder/jenkins/workspace/xamarin-macios-pr-builder/jenkins-results/tests/xammac-tests/311/execute-Mac_Modern-20200923_052627.txt + + + /Users/builder/jenkins/workspace/xamarin-macios-pr-builder/jenkins-results/tests/xammac-tests/311/main-20200923_052836.log + + + /Users/builder/jenkins/workspace/xamarin-macios-pr-builder/jenkins-results/tests/xammac-tests/311/test-Mac_Modern-20200923_052627-clean.xml + + + /Users/builder/jenkins/workspace/xamarin-macios-pr-builder/jenkins-results/tests/xammac-tests/311/test-Mac_Modern-20200923_052627.xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + /Users/builder/jenkins/workspace/xamarin-macios-pr-builder/jenkins-results/tests/xammac-tests/311/execute-Mac_Modern-20200923_052627.txt + + + /Users/builder/jenkins/workspace/xamarin-macios-pr-builder/jenkins-results/tests/xammac-tests/311/main-20200923_052836.log + + + /Users/builder/jenkins/workspace/xamarin-macios-pr-builder/jenkins-results/tests/xammac-tests/311/test-Mac_Modern-20200923_052627-clean.xml + + + /Users/builder/jenkins/workspace/xamarin-macios-pr-builder/jenkins-results/tests/xammac-tests/311/test-Mac_Modern-20200923_052627.xml + + + + + + diff --git a/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/XmlResultParserTests.cs b/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/XmlResultParserTests.cs index 876061161a4e..1e111f0cc1be 100644 --- a/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/XmlResultParserTests.cs +++ b/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/XmlResultParserTests.cs @@ -308,40 +308,46 @@ public void GenerateFailureTest (XmlResultJargon jargon) Directory.Delete (logsDir, true); } - /// - /// https://github.com/xamarin/xamarin-macios/issues/8214 - /// - [Test] - public void Issue8214Test () + [TestCase ("Issue8214.xml", true, "Tests run: 2376 Passed: 2301 Inconclusive: 13 Failed: 1 Ignored: 74")] // https://github.com/xamarin/xamarin-macios/issues/8214 + [TestCase ("NUnitV2Sample.xml", true, "Tests run: 21 Passed: 4 Inconclusive: 1 Failed: 2 Ignored: 7")] + [TestCase ("NUnitV2SampleFailure.xml", true, "Tests run: 21 Passed: 4 Inconclusive: 1 Failed: 2 Ignored: 7")] + [TestCase ("NUnitV3Sample.xml", true, "Tests run: 25 Passed: 12 Inconclusive: 1 Failed: 2 Ignored: 4")] + [TestCase ("NUnitV3SampleFailure.xml", true, "Tests run: 5 Passed: 3 Inconclusive: 1 Failed: 2 Ignored: 4")] + [TestCase ("TestCaseFailures.xml", true, "Tests run: 440 Passed: 405 Inconclusive: 0 Failed: 23 Ignored: 6")] + [TestCase ("TouchUnitSample.xml", false, "Tests run: 2354 Passed: 2223 Inconclusive: 13 Failed: 0 Ignored: 59", new string [] { "Tests run: 2286 Passed: 2282 Inconclusive: 4 Failed: 0 Ignored: 47" })] // The counting is a bit off here, seems like that's in Touch.Unit + [TestCase ("xUnitSample.xml", false, "Tests run: 53821 Passed: 53801 Inconclusive: 0 Failed: 0 Ignored: 20")] + [TestCase ("NUnitV3SampleParameterizedFailure.xml", true, "Tests run: 2086 Passed: 2041 Inconclusive: 7 Failed: 2 Ignored: 43", new string [] { " [FAIL] GHIssue8342(OK,\"mandel\",\"12345678\",\"mandel\",\"12345678\") : Status not ok" })] + public void HumanReadableResultsTest (string xmlFile, bool expectedFailure, string expectedResultLine, string [] additionalLines = null) { - string expectedResultLine = "Tests run: 2376 Passed: 2301 Inconclusive: 13 Failed: 1 Ignored: 74"; - // get the sample that was added to the issue to validate that we do parse the resuls correctly and copy it to a local - // path to be parsed - var name = GetType ().Assembly.GetManifestResourceNames ().Where (a => a.EndsWith ("Issue8214.xml", StringComparison.Ordinal)).FirstOrDefault (); - var tempPath = Path.GetTempFileName (); - var destinationFile = Path.GetTempFileName (); - using (var outputStream = new StreamWriter (tempPath)) - using (var sampleStream = new StreamReader (GetType ().Assembly.GetManifestResourceStream (name))) { - string line; - while ((line = sampleStream.ReadLine ()) != null) - outputStream.WriteLine (line); + // get the sample xml to parse + var name = GetType ().Assembly.GetManifestResourceNames ().Where (a => a.EndsWith (xmlFile, StringComparison.Ordinal)).FirstOrDefault (); + using var validXmlSource = new StreamReader (GetType ().Assembly.GetManifestResourceStream (name)); + using var source = new StreamReader (GetType ().Assembly.GetManifestResourceStream (name)); + using var destination = new StringWriter (); + + // Get the xml type + Assert.IsTrue (resultParser.IsValidXml (validXmlSource, out var type), "Valid Xml"); + + // generate the results + var (resultLine, failed) = resultParser.GenerateHumanReadableResults (source, destination, type); + var output = destination.ToString (); + + Console.WriteLine (output); + + Assert.AreEqual (expectedFailure, failed, "failed"); + Assert.AreEqual (expectedResultLine, resultLine, "result line"); + + if (additionalLines != null) { + var lines = output.Split ('\n'); + foreach (var line in additionalLines) + Assert.That (lines, Does.Contain (line), "Expected line"); } - var (resultLine, failed) = resultParser.GenerateHumanReadableResults (tempPath, destinationFile, XmlResultJargon.NUnitV3); - Assert.IsTrue (failed, "failed"); - Assert.AreEqual (expectedResultLine, resultLine, "resultLine"); - // verify that the destination does contain the result line - string resultLineInDestinationFile = null; - using (var resultReader = new StreamReader (destinationFile)) { - string line; - while ((line = resultReader.ReadLine ()) != null) { - if (line.Contains ("Tests run:")) { - resultLineInDestinationFile = line; - break; - } - } + + if (expectedFailure) { + Assert.That (output, Does.Contain ("[FAIL]"), "FAIL"); + } else { + Assert.That (output, Does.Not.Contain ("[FAIL]"), "Not FAIL"); } - Assert.IsNotNull (resultLineInDestinationFile, "result file result line"); - Assert.AreEqual (expectedResultLine, resultLineInDestinationFile, "content result file result line"); } [Test] diff --git a/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared/XmlResultParser.cs b/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared/XmlResultParser.cs index d44eb9b293a4..511e6ec3a882 100644 --- a/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared/XmlResultParser.cs +++ b/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared/XmlResultParser.cs @@ -66,6 +66,95 @@ public bool IsValidXml (TextReader stream, out XmlResultJargon type) return false; } + static void ParseNUnitV3XmlTestCase (TextWriter writer, XmlReader reader) + { + if (reader.NodeType != XmlNodeType.Element) + throw new InvalidOperationException (); + if (reader.Name != "test-case") + throw new InvalidOperationException (); + + // read the test cases in the current node + var status = reader ["result"]; + switch (status) { + case "Passed": + writer.Write ("\t[PASS] "); + break; + case "Skipped": + writer.Write ("\t[IGNORED] "); + break; + case "Error": + case "Failed": + writer.Write ("\t[FAIL] "); + break; + case "Inconclusive": + writer.Write ("\t[INCONCLUSIVE] "); + break; + default: + writer.Write ("\t[INFO] "); + break; + } + var name = reader ["name"]; + writer.Write (name); + if (status == "Failed") { // we need to print the message + reader.ReadToDescendant ("failure"); + reader.ReadToDescendant ("message"); + writer.Write ($" : {reader.ReadElementContentAsString ()}"); + if (reader.Name != "stack-trace") + reader.ReadToNextSibling ("stack-trace"); + writer.Write ($" : {reader.ReadElementContentAsString ()}"); + } + if (status == "Skipped") { // nice to have the skip reason + reader.ReadToDescendant ("reason"); + reader.ReadToDescendant ("message"); + writer.Write ($" : {reader.ReadElementContentAsString ()}"); + } + // add a new line + writer.WriteLine (); + } + + static void ParseNUnitV3XmlTestSuite (TextWriter writer, XmlReader reader, bool nested, int nesting = 0) + { + if (reader.NodeType != XmlNodeType.Element) + throw new InvalidOperationException (); + if (reader.Name != "test-suite") + throw new InvalidOperationException (); + + var type = reader ["type"]; + var isFixture = type == "TestFixture" || type == "ParameterizedFixture"; + var testCaseName = reader ["fullname"]; + var time = reader.GetAttribute ("time") ?? "0"; // some nodes might not have the time :/ + + if (isFixture) + writer.WriteLine (testCaseName); + + if (reader.IsEmptyElement) { + if (isFixture) + writer.WriteLine ($"{testCaseName} {time} ms"); + return; + } + + while (reader.Read ()) { + switch (reader.NodeType ) { + case XmlNodeType.Element: + if (reader.Name == "test-case") { + ParseNUnitV3XmlTestCase (writer, reader); + } else if (reader.Name == "test-suite") { + ParseNUnitV3XmlTestSuite (writer, reader, nested || isFixture); + } + break; + case XmlNodeType.EndElement: + if (reader.Name == "test-suite") { + if (isFixture) + writer.WriteLine ($"{testCaseName} {time} ms"); + return; + } + break; + } + } + + throw new InvalidOperationException ("Invalid XML: no test-suite end element"); + } + static (string resultLine, bool failed) ParseNUnitV3Xml (TextReader stream, TextWriter writer) { long testcasecount, passed, failed, inconclusive, skipped; @@ -82,52 +171,8 @@ public bool IsValidXml (TextReader stream, out XmlResultJargon type) long.TryParse (reader ["skipped"], out skipped); failedTestRun = failed != 0; } - if (reader.NodeType == XmlNodeType.Element && reader.Name == "test-suite" && (reader ["type"] == "TestFixture" || reader ["type"] == "ParameterizedFixture")) { - var testCaseName = reader ["fullname"]; - writer.WriteLine (testCaseName); - var time = reader.GetAttribute ("time") ?? "0"; // some nodes might not have the time :/ - // get the first node and then move in the siblings of the same type - reader.ReadToDescendant ("test-case"); - do { - if (reader.Name != "test-case") - break; - // read the test cases in the current node - var status = reader ["result"]; - switch (status) { - case "Passed": - writer.Write ("\t[PASS] "); - break; - case "Skipped": - writer.Write ("\t[IGNORED] "); - break; - case "Error": - case "Failed": - writer.Write ("\t[FAIL] "); - break; - case "Inconclusive": - writer.Write ("\t[INCONCLUSIVE] "); - break; - default: - writer.Write ("\t[INFO] "); - break; - } - writer.Write (reader ["name"]); - if (status == "Failed") { // we need to print the message - reader.ReadToDescendant ("failure"); - reader.ReadToDescendant ("message"); - writer.Write ($" : {reader.ReadElementContentAsString ()}"); - reader.ReadToNextSibling ("stack-trace"); - writer.Write ($" : {reader.ReadElementContentAsString ()}"); - } - if (status == "Skipped") { // nice to have the skip reason - reader.ReadToDescendant ("reason"); - reader.ReadToDescendant ("message"); - writer.Write ($" : {reader.ReadElementContentAsString ()}"); - } - // add a new line - writer.WriteLine (); - } while (reader.ReadToNextSibling ("test-case")); - writer.WriteLine ($"{testCaseName} {time} ms"); + if (reader.NodeType == XmlNodeType.Element && reader.Name == "test-suite") { + ParseNUnitV3XmlTestSuite (writer, reader, false); } } } From 6e027c13608f55bdef0ac2069a7a7d2216e8e7fc Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Fri, 25 Sep 2020 14:52:45 +0200 Subject: [PATCH 3/3] Remove debug spew. --- .../XmlResultParserTests.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/XmlResultParserTests.cs b/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/XmlResultParserTests.cs index 1e111f0cc1be..77bbf93dec5a 100644 --- a/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/XmlResultParserTests.cs +++ b/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/XmlResultParserTests.cs @@ -332,8 +332,6 @@ public void HumanReadableResultsTest (string xmlFile, bool expectedFailure, stri var (resultLine, failed) = resultParser.GenerateHumanReadableResults (source, destination, type); var output = destination.ToString (); - Console.WriteLine (output); - Assert.AreEqual (expectedFailure, failed, "failed"); Assert.AreEqual (expectedResultLine, resultLine, "result line");