Fix stack trace output for TestNG failures

java-extension/com.microsoft.java.test.runner/src/main/java/com/microsoft/java/test/runner/Launcher.java

@@ -57,10 +57,10 @@ public static void main(String[] args) {
             launcher.execute(params);
         } catch (final ParameterException e) {
             exitStatus = EXIT_WITH_INVALID_INPUT_CODE;
-            TestOutputStream.instance().println(new TestMessageItem("Invalid Parameter.", e));
+            logError("Invalid Parameter.", e);
         } catch (final Throwable e) {
             exitStatus = EXIT_WITH_UNKNOWN_EXCEPTION;
-            TestOutputStream.instance().println(new TestMessageItem("Exception happens in the Test Runner.", e));
+            logError("Exception occured while running tests.", e);
         } finally {
             TestOutputStream.instance().close();
             try {
@@ -73,4 +73,10 @@ public static void main(String[] args) {
             System.exit(exitStatus);
         }
     }
+
+    private static void logError(String message, Throwable ex) {
+        System.err.println(message);
+        ex.printStackTrace();
+        TestOutputStream.instance().println(new TestMessageItem(message, ex));
+    }
 }

src/runners/baseRunner/BaseRunner.ts

@@ -12,12 +12,12 @@ import { IProgressReporter } from '../../debugger.api';
 import { IExecutionConfig } from '../../runConfigs';
 import { IRunTestContext } from '../../types';
 import { ITestRunner } from '../ITestRunner';
-import { IRunnerResultAnalyzer } from './IRunnerResultAnalyzer';
+import { RunnerResultAnalyzer } from './RunnerResultAnalyzer';
 
 export abstract class BaseRunner implements ITestRunner {
     protected server: Server;
     protected socket: Socket;
-    protected runnerResultAnalyzer: IRunnerResultAnalyzer;
+    protected runnerResultAnalyzer: RunnerResultAnalyzer;
 
     private disposables: Disposable[] = [];
 
@@ -144,7 +144,7 @@ export abstract class BaseRunner implements ITestRunner {
         return [];
     }
 
-    protected abstract getAnalyzer(): IRunnerResultAnalyzer;
+    protected abstract getAnalyzer(): RunnerResultAnalyzer;
 }
 
 export interface IJUnitLaunchArguments {

src/runners/baseRunner/RunnerResultAnalyzer.ts

@@ -0,0 +1,35 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+
+import * as path from 'path';
+import { Location, MarkdownString, Range, TestItem, TestMessage } from 'vscode';
+import { IRunTestContext } from '../../types';
+import { setTestState, TestResultState } from '../utils';
+
+export abstract class RunnerResultAnalyzer {
+    constructor(protected testContext: IRunTestContext) { }
+
+    public abstract analyzeData(data: string): void;
+    public abstract processData(data: string): void;
+
+    protected processStackTrace(data: string, traces: MarkdownString, assertionFailure: TestMessage | undefined, currentItem: TestItem | undefined, projectName: string): void {
+        const traceRegExp: RegExp = /(\s?at\s+)([\w$\\.]+\/)?((?:[\w$]+\.)+[<\w$>]+)\(([\w-$]+\.java):(\d+)\)/;
+
+        const traceResults: RegExpExecArray | null = traceRegExp.exec(data);
+        if (traceResults && traceResults.length === 6) {
+            traces.appendText(traceResults[1]);
+            traces.appendMarkdown(`${(traceResults[2] || '') + traceResults[3]}([${traceResults[4]}:${traceResults[5]}](command:_java.test.openStackTrace?${encodeURIComponent(JSON.stringify([data, projectName]))}))`);
+            if (assertionFailure && currentItem && path.basename(currentItem.uri?.fsPath || '') === traceResults[4]) {
+                const lineNum: number = parseInt(traceResults[5], 10);
+                if (currentItem.uri) {
+                    assertionFailure.location = new Location(currentItem.uri, new Range(lineNum - 1, 0, lineNum, 0));
+                }
+                setTestState(this.testContext.testRun, currentItem, TestResultState.Failed, assertionFailure);
+            }
+        } else {
+            // in case the message contains message like: 'expected: <..> but was: <..>'
+            traces.appendText(data.replace(/</g, '&lt;').replace(/>/g, '&gt;'));
+        }
+        traces.appendText('\n');
+    }
+}

src/runners/junitRunner/JUnitRunnerResultAnalyzer.ts

@@ -1,16 +1,15 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license.
 
-import * as path from 'path';
-import { Location, MarkdownString, Range, TestItem, TestMessage } from 'vscode';
+import { Location, MarkdownString, TestItem, TestMessage } from 'vscode';
 import { INVOCATION_PREFIX } from '../../constants';
 import { dataCache, ITestItemData } from '../../controller/testItemDataCache';
 import { createTestItem } from '../../controller/utils';
 import { IJavaTestItem, IRunTestContext, TestKind, TestLevel } from '../../types';
-import { IRunnerResultAnalyzer } from '../baseRunner/IRunnerResultAnalyzer';
+import { RunnerResultAnalyzer } from '../baseRunner/RunnerResultAnalyzer';
 import { findTestLocation, setTestState, TestResultState } from '../utils';
 
-export class JUnitRunnerResultAnalyzer implements IRunnerResultAnalyzer {
+export class JUnitRunnerResultAnalyzer extends RunnerResultAnalyzer {
 
     private testOutputMapping: Map<string, ITestInfo> = new Map();
     private triggeredTestsMapping: Map<string, TestItem> = new Map();
@@ -25,7 +24,8 @@ export class JUnitRunnerResultAnalyzer implements IRunnerResultAnalyzer {
     private projectName: string;
     private incompleteTestSuite: ITestInfo[] = [];
 
-    constructor(private testContext: IRunTestContext) {
+    constructor(protected testContext: IRunTestContext) {
+        super(testContext);
         this.projectName = testContext.projectName;
         const queue: TestItem[] = [...testContext.testItems];
         while (queue.length) {
@@ -42,7 +42,7 @@ export class JUnitRunnerResultAnalyzer implements IRunnerResultAnalyzer {
             }
             this.triggeredTestsMapping.set(item.id, item);
         }
-     }
+    }
 
     public analyzeData(data: string): void {
         const lines: string[] = data.split(/\r?\n/);
@@ -146,23 +146,8 @@ export class JUnitRunnerResultAnalyzer implements IRunnerResultAnalyzer {
                     this.assertionFailure = TestMessage.diff(`Expected [${assertionResults[1]}] but was [${assertionResults[2]}]`, assertionResults[1], assertionResults[2]);
                 }
             }
-            const traceRegExp: RegExp = /(\s?at\s+)([\w$\\.]+\/)?((?:[\w$]+\.)+[<\w$>]+)\(([\w-$]+\.java):(\d+)\)/;
-            const traceResults: RegExpExecArray | null = traceRegExp.exec(data);
-            if (traceResults && traceResults.length === 6) {
-                this.traces.appendText(traceResults[1]);
-                this.traces.appendMarkdown(`${(traceResults[2] || '') + traceResults[3]}([${traceResults[4]}:${traceResults[5]}](command:_java.test.openStackTrace?${encodeURIComponent(JSON.stringify([data, this.projectName]))}))`);
-                if (this.assertionFailure && this.currentItem && path.basename(this.currentItem.uri?.fsPath || '') === traceResults[4]) {
-                    const lineNum: number = parseInt(traceResults[5], 10);
-                    if (this.currentItem.uri) {
-                        this.assertionFailure.location = new Location(this.currentItem.uri, new Range(lineNum - 1, 0, lineNum, 0));
-                    }
-                    setTestState(this.testContext.testRun, this.currentItem, TestResultState.Failed, this.assertionFailure);
-                }
-            } else {
-                // in case the message contains message like: 'expected: <..> but was: <..>'
-                this.traces.appendText(data.replace(/</g, '&lt;').replace(/>/g, '&gt;'));
-            }
-            this.traces.appendText('\n');
+
+            this.processStackTrace(data, this.traces, this.assertionFailure, this.currentItem, this.projectName);
         }
     }
 

src/runners/junitRunner/JunitRunner.ts

@@ -5,7 +5,7 @@ import { AddressInfo } from 'net';
 import { CancellationToken, DebugConfiguration } from 'vscode';
 import { IProgressReporter } from '../../debugger.api';
 import { BaseRunner } from '../baseRunner/BaseRunner';
-import { IRunnerResultAnalyzer } from '../baseRunner/IRunnerResultAnalyzer';
+import { RunnerResultAnalyzer } from '../baseRunner/RunnerResultAnalyzer';
 import { JUnitRunnerResultAnalyzer } from './JUnitRunnerResultAnalyzer';
 
 export class JUnitRunner extends BaseRunner {
@@ -25,7 +25,7 @@ export class JUnitRunner extends BaseRunner {
         return super.run(launchConfiguration, token, progressReporter);
     }
 
-    protected getAnalyzer(): IRunnerResultAnalyzer {
+    protected getAnalyzer(): RunnerResultAnalyzer {
         return new JUnitRunnerResultAnalyzer(this.testContext);
     }
 }

src/runners/testngRunner/TestNGRunner.ts

@@ -5,7 +5,7 @@ import { TestItem } from 'vscode';
 import { dataCache } from '../../controller/testItemDataCache';
 import { TestLevel } from '../../types';
 import { BaseRunner } from '../baseRunner/BaseRunner';
-import { IRunnerResultAnalyzer } from '../baseRunner/IRunnerResultAnalyzer';
+import { RunnerResultAnalyzer } from '../baseRunner/RunnerResultAnalyzer';
 import { TestNGRunnerResultAnalyzer } from './TestNGRunnerResultAnalyzer';
 
 export class TestNGRunner extends BaseRunner {
@@ -38,7 +38,7 @@ export class TestNGRunner extends BaseRunner {
         }).filter(Boolean)];
     }
 
-    protected getAnalyzer(): IRunnerResultAnalyzer {
+    protected getAnalyzer(): RunnerResultAnalyzer {
         return new TestNGRunnerResultAnalyzer(this.testContext);
     }
 }

src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts

@@ -4,23 +4,24 @@
 import { Location, MarkdownString, TestItem, TestMessage } from 'vscode';
 import { dataCache } from '../../controller/testItemDataCache';
 import { IRunTestContext, TestLevel } from '../../types';
-import { IRunnerResultAnalyzer } from '../baseRunner/IRunnerResultAnalyzer';
+import { RunnerResultAnalyzer } from '../baseRunner/RunnerResultAnalyzer';
 import { setTestState, TestResultState } from '../utils';
 
 const TEST_START: string = 'testStarted';
 const TEST_FAIL: string = 'testFailed';
 const TEST_FINISH: string = 'testFinished';
 
-export class TestNGRunnerResultAnalyzer implements IRunnerResultAnalyzer {
+export class TestNGRunnerResultAnalyzer extends RunnerResultAnalyzer {
 
-    private readonly regex: RegExp = /@@<TestRunner-({[\s\S]*?})-TestRunner>/;
+    private readonly regex: RegExp = /@@<TestRunner-({[\s\S]*?})-TestRunner>/g;
 
     private triggeredTestsMapping: Map<string, TestItem> = new Map();
     private currentTestState: TestResultState;
     private currentItem: TestItem | undefined;
     private projectName: string;
 
-    constructor(private testContext: IRunTestContext) {
+    constructor(protected testContext: IRunTestContext) {
+        super(testContext);
         this.projectName = testContext.projectName;
         const queue: TestItem[] = [...testContext.testItems];
         while (queue.length) {
@@ -40,32 +41,21 @@ export class TestNGRunnerResultAnalyzer implements IRunnerResultAnalyzer {
     }
 
     public analyzeData(data: string): void {
-        const lines: string[] = data.split(/\r?\n/);
-        for (const line of lines) {
-            if (!line) {
-                continue;
-            }
-            const match: RegExpExecArray | null = this.regex.exec(line);
-            if (match) {
-                // Message from Test Runner executable
-                try {
-                    this.processData(match[1]);
-                } catch (error) {
-                    this.testContext.testRun.appendOutput(`[ERROR] Failed to parse output data: ${line}\n`);
-                }
-            } else {
-                this.testContext.testRun.appendOutput(line + '\r\n');
+        let match: RegExpExecArray | null;
+        // tslint:disable-next-line: no-conditional-assignment
+        while ((match = this.regex.exec(data)) !== null) {
+            try {
+               this.processData(match[1]);
+            } catch (error) {
+                this.testContext.testRun.appendOutput(`[ERROR] Failed to parse output data: ${match[1]}\n`);
             }
         }
     }
 
     public processData(data: string): void {
         const outputData: ITestNGOutputData = JSON.parse(data) as ITestNGOutputData;
-        if (outputData.name.toLocaleLowerCase() === 'error') {
-            this.testContext.testRun.appendOutput(`[ERROR] ${this.unescape(data)}\r\n`);
-        } else {
-            this.testContext.testRun.appendOutput(`${this.unescape(data)}\r\n`);
-        }
+
+        this.testContext.testRun.appendOutput(this.unescape(data).replace(/\r?\n/g, '\r\n'));
 
         const id: string = `${this.projectName}@${outputData.attributes.name}`;
         if (outputData.name === TEST_START) {
@@ -83,37 +73,21 @@ export class TestNGRunnerResultAnalyzer implements IRunnerResultAnalyzer {
             }
             this.currentTestState = TestResultState.Failed;
             const testMessages: TestMessage[] = [];
-            if (outputData.attributes.message) {
-                const message: TestMessage = new TestMessage(outputData.attributes.message.trim());
-                if (item.uri && item.range) {
-                    message.location = new Location(item.uri, item.range);
-                }
-                testMessages.push(message);
-            }
 
             if (outputData.attributes.trace) {
-                const traceString: string = outputData.attributes.trace.trim();
                 const markdownTrace: MarkdownString = new MarkdownString();
                 markdownTrace.isTrusted = true;
-                const traceRegExp: RegExp = /(\s?at\s+)([\w$\\.]+\/)?((?:[\w$]+\.)+[<\w$>]+)\(([\w-$]+\.java):(\d+)\)/;
-                for (const line of traceString.split(/\r?\n/)) {
-                    const traceResults: RegExpExecArray | null = traceRegExp.exec(line);
-                    if (traceResults && traceResults.length === 6) {
-                        markdownTrace.appendText(traceResults[1]);
-                        markdownTrace.appendMarkdown(`${(traceResults[2] || '') + traceResults[3]}([${traceResults[4]}:${traceResults[5]}](command:_java.test.openStackTrace?${encodeURIComponent(JSON.stringify([data, this.projectName]))}))`);
-                    } else {
-                        // in case the message contains message like: 'expected: <..> but was: <..>'
-                        markdownTrace.appendText(line.replace(/</g, '&lt;').replace(/>/g, '&gt;'));
-                    }
-                    markdownTrace.appendText('\n');
+
+                for (const line of outputData.attributes.trace.split(/\r?\n/)) {
+                    this.processStackTrace(line, markdownTrace, undefined, this.currentItem, this.projectName);
                 }
+
                 const testMessage: TestMessage = new TestMessage(markdownTrace);
                 if (item.uri && item.range) {
                     testMessage.location = new Location(item.uri, item.range);
                 }
                 testMessages.push(testMessage);
             }
-
             const duration: number = Number.parseInt(outputData.attributes.duration, 10);
             setTestState(this.testContext.testRun, item, this.currentTestState, testMessages, duration);
         } else if (outputData.name === TEST_FINISH) {
@@ -164,7 +138,7 @@ enum TestOutputType {
     Error,
 }
 
-interface ITestNGAttributes  {
+interface ITestNGAttributes {
     name: string;
     duration: string;
     location: string;