From e061eba5efe586e8408fe5a6635fe195ee73fdaf Mon Sep 17 00:00:00 2001 From: Jonathan Kropfinger Date: Tue, 17 Aug 2021 20:27:58 -0500 Subject: [PATCH 01/13] Fixed the stack trace that is printed out when a TestNG test fails. --- src/runners/baseRunner/BaseRunner.ts | 6 ++-- .../baseRunner/IRunnerResultAnalyzer.ts | 34 +++++++++++++++++-- .../junitRunner/JUnitRunnerResultAnalyzer.ts | 29 ++++------------ src/runners/junitRunner/JunitRunner.ts | 4 +-- src/runners/testngRunner/TestNGRunner.ts | 4 +-- .../TestNGRunnerResultAnalyzer.ts | 26 +++++--------- 6 files changed, 54 insertions(+), 49 deletions(-) diff --git a/src/runners/baseRunner/BaseRunner.ts b/src/runners/baseRunner/BaseRunner.ts index 9a844658..f947973c 100644 --- a/src/runners/baseRunner/BaseRunner.ts +++ b/src/runners/baseRunner/BaseRunner.ts @@ -12,12 +12,12 @@ import { IProgressReporter } from '../../debugger.api'; import { IExecutionConfig } from '../../runConfigs'; import { IRunTestContext } from '../../types'; import { ITestRunner } from '../ITestRunner'; -import { IRunnerResultAnalyzer } from './IRunnerResultAnalyzer'; +import { RunnerResultAnalyzer } from './IRunnerResultAnalyzer'; export abstract class BaseRunner implements ITestRunner { protected server: Server; protected socket: Socket; - protected runnerResultAnalyzer: IRunnerResultAnalyzer; + protected runnerResultAnalyzer: RunnerResultAnalyzer; private disposables: Disposable[] = []; @@ -144,7 +144,7 @@ export abstract class BaseRunner implements ITestRunner { return []; } - protected abstract getAnalyzer(): IRunnerResultAnalyzer; + protected abstract getAnalyzer(): RunnerResultAnalyzer; } export interface IJUnitLaunchArguments { diff --git a/src/runners/baseRunner/IRunnerResultAnalyzer.ts b/src/runners/baseRunner/IRunnerResultAnalyzer.ts index d8b93d1d..43bd1396 100644 --- a/src/runners/baseRunner/IRunnerResultAnalyzer.ts +++ b/src/runners/baseRunner/IRunnerResultAnalyzer.ts @@ -1,7 +1,35 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. -export interface IRunnerResultAnalyzer { - analyzeData(data: string): void; - processData(data: string): void; +import * as path from 'path'; +import { Location, MarkdownString, Range, TestItem, TestMessage } from "vscode"; +import { IRunTestContext } from "../../types"; +import { setTestState, TestResultState } from "../utils"; + +export abstract class RunnerResultAnalyzer { + abstract analyzeData(data: string): void; + abstract processData(data: string): void; + + constructor (protected testContext: IRunTestContext) { } + + protected processStackTrace(data: string, traces: MarkdownString, assertionFailure: TestMessage | undefined, currentItem: TestItem | undefined, projectName: string): void { + const traceRegExp: RegExp = /(\s?at\s+)([\w$\\.]+\/)?((?:[\w$]+\.)+[<\w$>]+)\(([\w-$]+\.java):(\d+)\)/; + + const traceResults: RegExpExecArray | null = traceRegExp.exec(data); + if (traceResults && traceResults.length === 6) { + traces.appendText(traceResults[1]); + traces.appendMarkdown(`${(traceResults[2] || '') + traceResults[3]}([${traceResults[4]}:${traceResults[5]}](command:_java.test.openStackTrace?${encodeURIComponent(JSON.stringify([data, projectName]))}))`); + if (assertionFailure && currentItem && path.basename(currentItem.uri?.fsPath || '') === traceResults[4]) { + const lineNum: number = parseInt(traceResults[5], 10); + if (currentItem.uri) { + assertionFailure.location = new Location(currentItem.uri, new Range(lineNum - 1, 0, lineNum, 0)); + } + setTestState(this.testContext.testRun, currentItem, TestResultState.Failed, assertionFailure); + } + } else { + // in case the message contains message like: 'expected: <..> but was: <..>' + traces.appendText(data.replace(//g, '>')); + } + traces.appendText('\n'); + } } diff --git a/src/runners/junitRunner/JUnitRunnerResultAnalyzer.ts b/src/runners/junitRunner/JUnitRunnerResultAnalyzer.ts index de12fa60..e6c35ecd 100644 --- a/src/runners/junitRunner/JUnitRunnerResultAnalyzer.ts +++ b/src/runners/junitRunner/JUnitRunnerResultAnalyzer.ts @@ -1,16 +1,15 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. -import * as path from 'path'; -import { Location, MarkdownString, Range, TestItem, TestMessage } from 'vscode'; +import { Location, MarkdownString, TestItem, TestMessage } from 'vscode'; import { INVOCATION_PREFIX } from '../../constants'; import { dataCache, ITestItemData } from '../../controller/testItemDataCache'; import { createTestItem } from '../../controller/utils'; import { IJavaTestItem, IRunTestContext, TestKind, TestLevel } from '../../types'; -import { IRunnerResultAnalyzer } from '../baseRunner/IRunnerResultAnalyzer'; +import { RunnerResultAnalyzer } from '../baseRunner/IRunnerResultAnalyzer'; import { findTestLocation, setTestState, TestResultState } from '../utils'; -export class JUnitRunnerResultAnalyzer implements IRunnerResultAnalyzer { +export class JUnitRunnerResultAnalyzer extends RunnerResultAnalyzer { private testOutputMapping: Map = new Map(); private triggeredTestsMapping: Map = new Map(); @@ -25,7 +24,8 @@ export class JUnitRunnerResultAnalyzer implements IRunnerResultAnalyzer { private projectName: string; private incompleteTestSuite: ITestInfo[] = []; - constructor(private testContext: IRunTestContext) { + constructor(protected testContext: IRunTestContext) { + super(testContext); this.projectName = testContext.projectName; const queue: TestItem[] = [...testContext.testItems]; while (queue.length) { @@ -146,23 +146,8 @@ export class JUnitRunnerResultAnalyzer implements IRunnerResultAnalyzer { this.assertionFailure = TestMessage.diff(`Expected [${assertionResults[1]}] but was [${assertionResults[2]}]`, assertionResults[1], assertionResults[2]); } } - const traceRegExp: RegExp = /(\s?at\s+)([\w$\\.]+\/)?((?:[\w$]+\.)+[<\w$>]+)\(([\w-$]+\.java):(\d+)\)/; - const traceResults: RegExpExecArray | null = traceRegExp.exec(data); - if (traceResults && traceResults.length === 6) { - this.traces.appendText(traceResults[1]); - this.traces.appendMarkdown(`${(traceResults[2] || '') + traceResults[3]}([${traceResults[4]}:${traceResults[5]}](command:_java.test.openStackTrace?${encodeURIComponent(JSON.stringify([data, this.projectName]))}))`); - if (this.assertionFailure && this.currentItem && path.basename(this.currentItem.uri?.fsPath || '') === traceResults[4]) { - const lineNum: number = parseInt(traceResults[5], 10); - if (this.currentItem.uri) { - this.assertionFailure.location = new Location(this.currentItem.uri, new Range(lineNum - 1, 0, lineNum, 0)); - } - setTestState(this.testContext.testRun, this.currentItem, TestResultState.Failed, this.assertionFailure); - } - } else { - // in case the message contains message like: 'expected: <..> but was: <..>' - this.traces.appendText(data.replace(//g, '>')); - } - this.traces.appendText('\n'); + + this.processStackTrace(data, this.traces, this.assertionFailure, this.currentItem, this.projectName) } } diff --git a/src/runners/junitRunner/JunitRunner.ts b/src/runners/junitRunner/JunitRunner.ts index f02461bc..8c181079 100644 --- a/src/runners/junitRunner/JunitRunner.ts +++ b/src/runners/junitRunner/JunitRunner.ts @@ -5,7 +5,7 @@ import { AddressInfo } from 'net'; import { CancellationToken, DebugConfiguration } from 'vscode'; import { IProgressReporter } from '../../debugger.api'; import { BaseRunner } from '../baseRunner/BaseRunner'; -import { IRunnerResultAnalyzer } from '../baseRunner/IRunnerResultAnalyzer'; +import { RunnerResultAnalyzer } from '../baseRunner/IRunnerResultAnalyzer'; import { JUnitRunnerResultAnalyzer } from './JUnitRunnerResultAnalyzer'; export class JUnitRunner extends BaseRunner { @@ -25,7 +25,7 @@ export class JUnitRunner extends BaseRunner { return super.run(launchConfiguration, token, progressReporter); } - protected getAnalyzer(): IRunnerResultAnalyzer { + protected getAnalyzer(): RunnerResultAnalyzer { return new JUnitRunnerResultAnalyzer(this.testContext); } } diff --git a/src/runners/testngRunner/TestNGRunner.ts b/src/runners/testngRunner/TestNGRunner.ts index 85c1101a..5a1646e8 100644 --- a/src/runners/testngRunner/TestNGRunner.ts +++ b/src/runners/testngRunner/TestNGRunner.ts @@ -5,7 +5,7 @@ import { TestItem } from 'vscode'; import { dataCache } from '../../controller/testItemDataCache'; import { TestLevel } from '../../types'; import { BaseRunner } from '../baseRunner/BaseRunner'; -import { IRunnerResultAnalyzer } from '../baseRunner/IRunnerResultAnalyzer'; +import { RunnerResultAnalyzer } from '../baseRunner/IRunnerResultAnalyzer'; import { TestNGRunnerResultAnalyzer } from './TestNGRunnerResultAnalyzer'; export class TestNGRunner extends BaseRunner { @@ -38,7 +38,7 @@ export class TestNGRunner extends BaseRunner { }).filter(Boolean)]; } - protected getAnalyzer(): IRunnerResultAnalyzer { + protected getAnalyzer(): RunnerResultAnalyzer { return new TestNGRunnerResultAnalyzer(this.testContext); } } diff --git a/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts b/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts index 0d384431..4bd2459b 100644 --- a/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts +++ b/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts @@ -4,14 +4,14 @@ import { Location, MarkdownString, TestItem, TestMessage } from 'vscode'; import { dataCache } from '../../controller/testItemDataCache'; import { IRunTestContext, TestLevel } from '../../types'; -import { IRunnerResultAnalyzer } from '../baseRunner/IRunnerResultAnalyzer'; +import { RunnerResultAnalyzer } from '../baseRunner/IRunnerResultAnalyzer'; import { setTestState, TestResultState } from '../utils'; const TEST_START: string = 'testStarted'; const TEST_FAIL: string = 'testFailed'; const TEST_FINISH: string = 'testFinished'; -export class TestNGRunnerResultAnalyzer implements IRunnerResultAnalyzer { +export class TestNGRunnerResultAnalyzer extends RunnerResultAnalyzer { private readonly regex: RegExp = /@@/; @@ -20,7 +20,8 @@ export class TestNGRunnerResultAnalyzer implements IRunnerResultAnalyzer { private currentItem: TestItem | undefined; private projectName: string; - constructor(private testContext: IRunTestContext) { + constructor(protected testContext: IRunTestContext) { + super(testContext); this.projectName = testContext.projectName; const queue: TestItem[] = [...testContext.testItems]; while (queue.length) { @@ -40,7 +41,7 @@ export class TestNGRunnerResultAnalyzer implements IRunnerResultAnalyzer { } public analyzeData(data: string): void { - const lines: string[] = data.split(/\r?\n/); + const lines: string[] = this.unescape(data).split(/\r?\n/); for (const line of lines) { if (!line) { continue; @@ -92,22 +93,13 @@ export class TestNGRunnerResultAnalyzer implements IRunnerResultAnalyzer { } if (outputData.attributes.trace) { - const traceString: string = outputData.attributes.trace.trim(); const markdownTrace: MarkdownString = new MarkdownString(); markdownTrace.isTrusted = true; - const traceRegExp: RegExp = /(\s?at\s+)([\w$\\.]+\/)?((?:[\w$]+\.)+[<\w$>]+)\(([\w-$]+\.java):(\d+)\)/; - for (const line of traceString.split(/\r?\n/)) { - const traceResults: RegExpExecArray | null = traceRegExp.exec(line); - if (traceResults && traceResults.length === 6) { - markdownTrace.appendText(traceResults[1]); - markdownTrace.appendMarkdown(`${(traceResults[2] || '') + traceResults[3]}([${traceResults[4]}:${traceResults[5]}](command:_java.test.openStackTrace?${encodeURIComponent(JSON.stringify([data, this.projectName]))}))`); - } else { - // in case the message contains message like: 'expected: <..> but was: <..>' - markdownTrace.appendText(line.replace(//g, '>')); - } - markdownTrace.appendText('\n'); - } const testMessage: TestMessage = new TestMessage(markdownTrace); + + this.processStackTrace(data, markdownTrace, testMessage, this.currentItem, this.projectName); + + if (item.uri && item.range) { testMessage.location = new Location(item.uri, item.range); } From aa89a26aabe9bbd1821a4ef6b2c82d714f258851 Mon Sep 17 00:00:00 2001 From: Jonathan Kropfinger Date: Tue, 17 Aug 2021 21:13:42 -0500 Subject: [PATCH 02/13] Renamed IRunnerResultAnalyzer.ts to RunnerResultAnalyzer.ts. --- src/runners/baseRunner/BaseRunner.ts | 2 +- .../baseRunner/IRunnerResultAnalyzer.ts | 35 ------------------- .../baseRunner/RunnerResultAnalyzer.ts | 35 +++++++++++++++++++ .../junitRunner/JUnitRunnerResultAnalyzer.ts | 8 ++--- src/runners/junitRunner/JunitRunner.ts | 2 +- src/runners/testngRunner/TestNGRunner.ts | 2 +- .../TestNGRunnerResultAnalyzer.ts | 5 ++- 7 files changed, 44 insertions(+), 45 deletions(-) delete mode 100644 src/runners/baseRunner/IRunnerResultAnalyzer.ts create mode 100644 src/runners/baseRunner/RunnerResultAnalyzer.ts diff --git a/src/runners/baseRunner/BaseRunner.ts b/src/runners/baseRunner/BaseRunner.ts index f947973c..e3b96821 100644 --- a/src/runners/baseRunner/BaseRunner.ts +++ b/src/runners/baseRunner/BaseRunner.ts @@ -12,7 +12,7 @@ import { IProgressReporter } from '../../debugger.api'; import { IExecutionConfig } from '../../runConfigs'; import { IRunTestContext } from '../../types'; import { ITestRunner } from '../ITestRunner'; -import { RunnerResultAnalyzer } from './IRunnerResultAnalyzer'; +import { RunnerResultAnalyzer } from './RunnerResultAnalyzer'; export abstract class BaseRunner implements ITestRunner { protected server: Server; diff --git a/src/runners/baseRunner/IRunnerResultAnalyzer.ts b/src/runners/baseRunner/IRunnerResultAnalyzer.ts deleted file mode 100644 index 43bd1396..00000000 --- a/src/runners/baseRunner/IRunnerResultAnalyzer.ts +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. - -import * as path from 'path'; -import { Location, MarkdownString, Range, TestItem, TestMessage } from "vscode"; -import { IRunTestContext } from "../../types"; -import { setTestState, TestResultState } from "../utils"; - -export abstract class RunnerResultAnalyzer { - abstract analyzeData(data: string): void; - abstract processData(data: string): void; - - constructor (protected testContext: IRunTestContext) { } - - protected processStackTrace(data: string, traces: MarkdownString, assertionFailure: TestMessage | undefined, currentItem: TestItem | undefined, projectName: string): void { - const traceRegExp: RegExp = /(\s?at\s+)([\w$\\.]+\/)?((?:[\w$]+\.)+[<\w$>]+)\(([\w-$]+\.java):(\d+)\)/; - - const traceResults: RegExpExecArray | null = traceRegExp.exec(data); - if (traceResults && traceResults.length === 6) { - traces.appendText(traceResults[1]); - traces.appendMarkdown(`${(traceResults[2] || '') + traceResults[3]}([${traceResults[4]}:${traceResults[5]}](command:_java.test.openStackTrace?${encodeURIComponent(JSON.stringify([data, projectName]))}))`); - if (assertionFailure && currentItem && path.basename(currentItem.uri?.fsPath || '') === traceResults[4]) { - const lineNum: number = parseInt(traceResults[5], 10); - if (currentItem.uri) { - assertionFailure.location = new Location(currentItem.uri, new Range(lineNum - 1, 0, lineNum, 0)); - } - setTestState(this.testContext.testRun, currentItem, TestResultState.Failed, assertionFailure); - } - } else { - // in case the message contains message like: 'expected: <..> but was: <..>' - traces.appendText(data.replace(//g, '>')); - } - traces.appendText('\n'); - } -} diff --git a/src/runners/baseRunner/RunnerResultAnalyzer.ts b/src/runners/baseRunner/RunnerResultAnalyzer.ts new file mode 100644 index 00000000..fc2af811 --- /dev/null +++ b/src/runners/baseRunner/RunnerResultAnalyzer.ts @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +import * as path from 'path'; +import { Location, MarkdownString, Range, TestItem, TestMessage } from 'vscode'; +import { IRunTestContext } from '../../types'; +import { setTestState, TestResultState } from '../utils'; + +export abstract class RunnerResultAnalyzer { + constructor(protected testContext: IRunTestContext) { } + + public abstract analyzeData(data: string): void; + public abstract processData(data: string): void; + + protected processStackTrace(data: string, traces: MarkdownString, assertionFailure: TestMessage | undefined, currentItem: TestItem | undefined, projectName: string): void { + const traceRegExp: RegExp = /(\s?at\s+)([\w$\\.]+\/)?((?:[\w$]+\.)+[<\w$>]+)\(([\w-$]+\.java):(\d+)\)/; + + const traceResults: RegExpExecArray | null = traceRegExp.exec(data); + if (traceResults && traceResults.length === 6) { + traces.appendText(traceResults[1]); + traces.appendMarkdown(`${(traceResults[2] || '') + traceResults[3]}([${traceResults[4]}:${traceResults[5]}](command:_java.test.openStackTrace?${encodeURIComponent(JSON.stringify([data, projectName]))}))`); + if (assertionFailure && currentItem && path.basename(currentItem.uri?.fsPath || '') === traceResults[4]) { + const lineNum: number = parseInt(traceResults[5], 10); + if (currentItem.uri) { + assertionFailure.location = new Location(currentItem.uri, new Range(lineNum - 1, 0, lineNum, 0)); + } + setTestState(this.testContext.testRun, currentItem, TestResultState.Failed, assertionFailure); + } + } else { + // in case the message contains message like: 'expected: <..> but was: <..>' + traces.appendText(data.replace(//g, '>')); + } + traces.appendText('\n'); + } +} diff --git a/src/runners/junitRunner/JUnitRunnerResultAnalyzer.ts b/src/runners/junitRunner/JUnitRunnerResultAnalyzer.ts index e6c35ecd..dfff3cdd 100644 --- a/src/runners/junitRunner/JUnitRunnerResultAnalyzer.ts +++ b/src/runners/junitRunner/JUnitRunnerResultAnalyzer.ts @@ -6,7 +6,7 @@ import { INVOCATION_PREFIX } from '../../constants'; import { dataCache, ITestItemData } from '../../controller/testItemDataCache'; import { createTestItem } from '../../controller/utils'; import { IJavaTestItem, IRunTestContext, TestKind, TestLevel } from '../../types'; -import { RunnerResultAnalyzer } from '../baseRunner/IRunnerResultAnalyzer'; +import { RunnerResultAnalyzer } from '../baseRunner/RunnerResultAnalyzer'; import { findTestLocation, setTestState, TestResultState } from '../utils'; export class JUnitRunnerResultAnalyzer extends RunnerResultAnalyzer { @@ -42,7 +42,7 @@ export class JUnitRunnerResultAnalyzer extends RunnerResultAnalyzer { } this.triggeredTestsMapping.set(item.id, item); } - } + } public analyzeData(data: string): void { const lines: string[] = data.split(/\r?\n/); @@ -146,8 +146,8 @@ export class JUnitRunnerResultAnalyzer extends RunnerResultAnalyzer { this.assertionFailure = TestMessage.diff(`Expected [${assertionResults[1]}] but was [${assertionResults[2]}]`, assertionResults[1], assertionResults[2]); } } - - this.processStackTrace(data, this.traces, this.assertionFailure, this.currentItem, this.projectName) + + this.processStackTrace(data, this.traces, this.assertionFailure, this.currentItem, this.projectName); } } diff --git a/src/runners/junitRunner/JunitRunner.ts b/src/runners/junitRunner/JunitRunner.ts index 8c181079..4ee80ef7 100644 --- a/src/runners/junitRunner/JunitRunner.ts +++ b/src/runners/junitRunner/JunitRunner.ts @@ -5,7 +5,7 @@ import { AddressInfo } from 'net'; import { CancellationToken, DebugConfiguration } from 'vscode'; import { IProgressReporter } from '../../debugger.api'; import { BaseRunner } from '../baseRunner/BaseRunner'; -import { RunnerResultAnalyzer } from '../baseRunner/IRunnerResultAnalyzer'; +import { RunnerResultAnalyzer } from '../baseRunner/RunnerResultAnalyzer'; import { JUnitRunnerResultAnalyzer } from './JUnitRunnerResultAnalyzer'; export class JUnitRunner extends BaseRunner { diff --git a/src/runners/testngRunner/TestNGRunner.ts b/src/runners/testngRunner/TestNGRunner.ts index 5a1646e8..2350905d 100644 --- a/src/runners/testngRunner/TestNGRunner.ts +++ b/src/runners/testngRunner/TestNGRunner.ts @@ -5,7 +5,7 @@ import { TestItem } from 'vscode'; import { dataCache } from '../../controller/testItemDataCache'; import { TestLevel } from '../../types'; import { BaseRunner } from '../baseRunner/BaseRunner'; -import { RunnerResultAnalyzer } from '../baseRunner/IRunnerResultAnalyzer'; +import { RunnerResultAnalyzer } from '../baseRunner/RunnerResultAnalyzer'; import { TestNGRunnerResultAnalyzer } from './TestNGRunnerResultAnalyzer'; export class TestNGRunner extends BaseRunner { diff --git a/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts b/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts index 4bd2459b..55481ed8 100644 --- a/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts +++ b/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts @@ -4,7 +4,7 @@ import { Location, MarkdownString, TestItem, TestMessage } from 'vscode'; import { dataCache } from '../../controller/testItemDataCache'; import { IRunTestContext, TestLevel } from '../../types'; -import { RunnerResultAnalyzer } from '../baseRunner/IRunnerResultAnalyzer'; +import { RunnerResultAnalyzer } from '../baseRunner/RunnerResultAnalyzer'; import { setTestState, TestResultState } from '../utils'; const TEST_START: string = 'testStarted'; @@ -99,7 +99,6 @@ export class TestNGRunnerResultAnalyzer extends RunnerResultAnalyzer { this.processStackTrace(data, markdownTrace, testMessage, this.currentItem, this.projectName); - if (item.uri && item.range) { testMessage.location = new Location(item.uri, item.range); } @@ -156,7 +155,7 @@ enum TestOutputType { Error, } -interface ITestNGAttributes { +interface ITestNGAttributes { name: string; duration: string; location: string; From 324fb5d37ae0c03e03598238bf8592ab5d7877db Mon Sep 17 00:00:00 2001 From: Jonathan Kropfinger Date: Fri, 20 Aug 2021 08:36:59 -0500 Subject: [PATCH 03/13] Resolved peer review findings from pull request. Additionally the failure analysis logic was refactored to generate clearer test report info when "peeking" a test failure. --- .../TestNGRunnerResultAnalyzer.ts | 49 +++++++------------ 1 file changed, 17 insertions(+), 32 deletions(-) diff --git a/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts b/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts index 55481ed8..917e5336 100644 --- a/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts +++ b/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts @@ -13,7 +13,7 @@ const TEST_FINISH: string = 'testFinished'; export class TestNGRunnerResultAnalyzer extends RunnerResultAnalyzer { - private readonly regex: RegExp = /@@/; + private readonly regex: RegExp = /@@/g; private triggeredTestsMapping: Map = new Map(); private currentTestState: TestResultState; @@ -41,31 +41,26 @@ export class TestNGRunnerResultAnalyzer extends RunnerResultAnalyzer { } public analyzeData(data: string): void { - const lines: string[] = this.unescape(data).split(/\r?\n/); - for (const line of lines) { - if (!line) { - continue; - } - const match: RegExpExecArray | null = this.regex.exec(line); - if (match) { - // Message from Test Runner executable - try { - this.processData(match[1]); - } catch (error) { - this.testContext.testRun.appendOutput(`[ERROR] Failed to parse output data: ${line}\n`); - } - } else { - this.testContext.testRun.appendOutput(line + '\r\n'); + let match: RegExpExecArray | null; + // tslint:disable-next-line: no-conditional-assignment + while ((match = this.regex.exec(data)) !== null) { + try { + this.processData(match[1]); + } catch (error) { + this.testContext.testRun.appendOutput(`[ERROR] Failed to parse output data: ${match[1]}\n`); } } } public processData(data: string): void { const outputData: ITestNGOutputData = JSON.parse(data) as ITestNGOutputData; - if (outputData.name.toLocaleLowerCase() === 'error') { - this.testContext.testRun.appendOutput(`[ERROR] ${this.unescape(data)}\r\n`); - } else { - this.testContext.testRun.appendOutput(`${this.unescape(data)}\r\n`); + + for (const line of this.unescape(data).split(/\r?\n/)) { + if (outputData.name.toLocaleLowerCase() === 'error') { + this.testContext.testRun.appendOutput(`[ERROR] ${line}\r\n`); + } else { + this.testContext.testRun.appendOutput(`${line}\r\n`); + } } const id: string = `${this.projectName}@${outputData.attributes.name}`; @@ -84,25 +79,15 @@ export class TestNGRunnerResultAnalyzer extends RunnerResultAnalyzer { } this.currentTestState = TestResultState.Failed; const testMessages: TestMessage[] = []; - if (outputData.attributes.message) { - const message: TestMessage = new TestMessage(outputData.attributes.message.trim()); - if (item.uri && item.range) { - message.location = new Location(item.uri, item.range); - } - testMessages.push(message); - } if (outputData.attributes.trace) { const markdownTrace: MarkdownString = new MarkdownString(); markdownTrace.isTrusted = true; const testMessage: TestMessage = new TestMessage(markdownTrace); - this.processStackTrace(data, markdownTrace, testMessage, this.currentItem, this.projectName); - - if (item.uri && item.range) { - testMessage.location = new Location(item.uri, item.range); + for (const line of outputData.attributes.trace.split(/\r?\n/)) { + this.processStackTrace(line, markdownTrace, testMessage, this.currentItem, this.projectName); } - testMessages.push(testMessage); } const duration: number = Number.parseInt(outputData.attributes.duration, 10); From d0085b9d66586d87d1c540b2176ad5e7814db1c9 Mon Sep 17 00:00:00 2001 From: Jonathan Kropfinger Date: Fri, 20 Aug 2021 09:02:46 -0500 Subject: [PATCH 04/13] Removed unused import from TestNGRunnerResultAnalyzer.ts --- src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts b/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts index 917e5336..f5e6ce20 100644 --- a/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts +++ b/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. -import { Location, MarkdownString, TestItem, TestMessage } from 'vscode'; +import { MarkdownString, TestItem, TestMessage } from 'vscode'; import { dataCache } from '../../controller/testItemDataCache'; import { IRunTestContext, TestLevel } from '../../types'; import { RunnerResultAnalyzer } from '../baseRunner/RunnerResultAnalyzer'; From 30565fe486b83b9f3a17f0b7221a1ed80353c79c Mon Sep 17 00:00:00 2001 From: Jonathan Kropfinger Date: Sat, 21 Aug 2021 01:15:16 -0500 Subject: [PATCH 05/13] Added logging statements to Launcher.java to allow for exceptions to be logged in the debug console if the tests cannot be ran for some reason. --- .../java/com/microsoft/java/test/runner/Launcher.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/java-extension/com.microsoft.java.test.runner/src/main/java/com/microsoft/java/test/runner/Launcher.java b/java-extension/com.microsoft.java.test.runner/src/main/java/com/microsoft/java/test/runner/Launcher.java index 5c5a4a79..7c00025e 100644 --- a/java-extension/com.microsoft.java.test.runner/src/main/java/com/microsoft/java/test/runner/Launcher.java +++ b/java-extension/com.microsoft.java.test.runner/src/main/java/com/microsoft/java/test/runner/Launcher.java @@ -57,10 +57,10 @@ public static void main(String[] args) { launcher.execute(params); } catch (final ParameterException e) { exitStatus = EXIT_WITH_INVALID_INPUT_CODE; - TestOutputStream.instance().println(new TestMessageItem("Invalid Parameter.", e)); + logError("Invalid Parameter.", e); } catch (final Throwable e) { exitStatus = EXIT_WITH_UNKNOWN_EXCEPTION; - TestOutputStream.instance().println(new TestMessageItem("Exception happens in the Test Runner.", e)); + logError("Exception occured while running tests.", e); } finally { TestOutputStream.instance().close(); try { @@ -73,4 +73,10 @@ public static void main(String[] args) { System.exit(exitStatus); } } + + private static void logError(String message, Throwable ex) { + System.err.println(message); + ex.printStackTrace(); + TestOutputStream.instance().println(new TestMessageItem(message, ex)); + } } From 38e287e95554013ecb0277c50248755df4c4370d Mon Sep 17 00:00:00 2001 From: Jonathan Kropfinger Date: Tue, 24 Aug 2021 20:08:21 -0500 Subject: [PATCH 06/13] Fixed the stack trace issues that resulted in only a single element being shown in the error preview window --- .../baseRunner/RunnerResultAnalyzer.ts | 28 ++++++++++--- .../junitRunner/JUnitRunnerResultAnalyzer.ts | 39 +++++++------------ .../TestNGRunnerResultAnalyzer.ts | 13 +++---- 3 files changed, 44 insertions(+), 36 deletions(-) diff --git a/src/runners/baseRunner/RunnerResultAnalyzer.ts b/src/runners/baseRunner/RunnerResultAnalyzer.ts index fc2af811..45bd576f 100644 --- a/src/runners/baseRunner/RunnerResultAnalyzer.ts +++ b/src/runners/baseRunner/RunnerResultAnalyzer.ts @@ -3,8 +3,9 @@ import * as path from 'path'; import { Location, MarkdownString, Range, TestItem, TestMessage } from 'vscode'; +import { INVOCATION_PREFIX } from '../../constants'; import { IRunTestContext } from '../../types'; -import { setTestState, TestResultState } from '../utils'; +import { findTestLocation, setTestState, TestResultState } from '../utils'; export abstract class RunnerResultAnalyzer { constructor(protected testContext: IRunTestContext) { } @@ -12,19 +13,20 @@ export abstract class RunnerResultAnalyzer { public abstract analyzeData(data: string): void; public abstract processData(data: string): void; - protected processStackTrace(data: string, traces: MarkdownString, assertionFailure: TestMessage | undefined, currentItem: TestItem | undefined, projectName: string): void { + protected processStackTrace(data: string, traces: MarkdownString, testMessage: TestMessage | undefined, currentItem: TestItem | undefined, projectName: string): void { const traceRegExp: RegExp = /(\s?at\s+)([\w$\\.]+\/)?((?:[\w$]+\.)+[<\w$>]+)\(([\w-$]+\.java):(\d+)\)/; const traceResults: RegExpExecArray | null = traceRegExp.exec(data); if (traceResults && traceResults.length === 6) { traces.appendText(traceResults[1]); traces.appendMarkdown(`${(traceResults[2] || '') + traceResults[3]}([${traceResults[4]}:${traceResults[5]}](command:_java.test.openStackTrace?${encodeURIComponent(JSON.stringify([data, projectName]))}))`); - if (assertionFailure && currentItem && path.basename(currentItem.uri?.fsPath || '') === traceResults[4]) { + if (testMessage && currentItem && path.basename(currentItem.uri?.fsPath || '') === traceResults[4]) { + testMessage.message = data.trim(); const lineNum: number = parseInt(traceResults[5], 10); if (currentItem.uri) { - assertionFailure.location = new Location(currentItem.uri, new Range(lineNum - 1, 0, lineNum, 0)); + testMessage.location = new Location(currentItem.uri, new Range(lineNum - 1, 0, lineNum, 0)); } - setTestState(this.testContext.testRun, currentItem, TestResultState.Failed, assertionFailure); + setTestState(this.testContext.testRun, currentItem, TestResultState.Failed, testMessage); } } else { // in case the message contains message like: 'expected: <..> but was: <..>' @@ -32,4 +34,20 @@ export abstract class RunnerResultAnalyzer { } traces.appendText('\n'); } + + protected async finishFailureMessage(item: TestItem | undefined, testMessage: TestMessage, duration?: number): Promise { + if (item) { + if (item.uri && item.range) { + testMessage.location = new Location(item.uri, item.range); + } else { + let id: string = item.id; + if (id.startsWith(INVOCATION_PREFIX)) { + id = id.substring(INVOCATION_PREFIX.length); + } + const location: Location | undefined = await findTestLocation(id); + testMessage.location = location; + } + setTestState(this.testContext.testRun, item, TestResultState.Failed, testMessage, duration); + } + } } diff --git a/src/runners/junitRunner/JUnitRunnerResultAnalyzer.ts b/src/runners/junitRunner/JUnitRunnerResultAnalyzer.ts index dfff3cdd..917f8b29 100644 --- a/src/runners/junitRunner/JUnitRunnerResultAnalyzer.ts +++ b/src/runners/junitRunner/JUnitRunnerResultAnalyzer.ts @@ -1,13 +1,13 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. -import { Location, MarkdownString, TestItem, TestMessage } from 'vscode'; +import { MarkdownString, TestItem, TestMessage } from 'vscode'; import { INVOCATION_PREFIX } from '../../constants'; import { dataCache, ITestItemData } from '../../controller/testItemDataCache'; import { createTestItem } from '../../controller/utils'; import { IJavaTestItem, IRunTestContext, TestKind, TestLevel } from '../../types'; import { RunnerResultAnalyzer } from '../baseRunner/RunnerResultAnalyzer'; -import { findTestLocation, setTestState, TestResultState } from '../utils'; +import { setTestState, TestResultState } from '../utils'; export class JUnitRunnerResultAnalyzer extends RunnerResultAnalyzer { @@ -17,7 +17,7 @@ export class JUnitRunnerResultAnalyzer extends RunnerResultAnalyzer { private currentItem: TestItem | undefined; private currentDuration: number = 0; private traces: MarkdownString; - private assertionFailure: TestMessage | undefined; + private testMessage: TestMessage | undefined; private recordingType: RecordingType; private expectString: string; private actualString: string; @@ -116,7 +116,7 @@ export class JUnitRunnerResultAnalyzer extends RunnerResultAnalyzer { } const testMessage: TestMessage = new TestMessage(this.traces); - this.tryAppendMessage(this.currentItem, testMessage); + this.finishFailureMessage(this.currentItem, testMessage); this.recordingType = RecordingType.None; if (this.currentTestState === TestResultState.Errored) { setTestState(this.testContext.testRun, this.currentItem, this.currentTestState); @@ -131,23 +131,28 @@ export class JUnitRunnerResultAnalyzer extends RunnerResultAnalyzer { } else if (data.startsWith(MessageId.ActualEnd)) { this.recordingType = RecordingType.None; this.actualString = this.actualString.replace(/\n$/, ''); - if (!this.assertionFailure && this.expectString && this.actualString) { - this.assertionFailure = TestMessage.diff(`Expected [${this.expectString}] but was [${this.actualString}]`, this.expectString, this.actualString); + if (!this.testMessage && this.expectString && this.actualString) { + this.testMessage = TestMessage.diff(`Expected [${this.expectString}] but was [${this.actualString}]`, this.expectString, this.actualString); } } else if (this.recordingType === RecordingType.ExpectMessage) { this.expectString += data + '\n'; } else if (this.recordingType === RecordingType.ActualMessage) { this.actualString += data + '\n'; } else if (this.recordingType === RecordingType.StackTrace) { - if (!this.assertionFailure) { + if (!this.testMessage) { const assertionRegExp: RegExp = /expected.*:.*<(.+?)>.*but.*:.*<(.+?)>/mi; const assertionResults: RegExpExecArray | null = assertionRegExp.exec(data); if (assertionResults && assertionResults.length === 3) { - this.assertionFailure = TestMessage.diff(`Expected [${assertionResults[1]}] but was [${assertionResults[2]}]`, assertionResults[1], assertionResults[2]); + this.testMessage = TestMessage.diff(`Expected [${assertionResults[1]}] but was [${assertionResults[2]}]`, assertionResults[1], assertionResults[2]); + } else { + const markdownTrace: MarkdownString = new MarkdownString(); + markdownTrace.isTrusted = true; + this.testMessage = new TestMessage(data); } } - this.processStackTrace(data, this.traces, this.assertionFailure, this.currentItem, this.projectName); + this.processStackTrace(data, this.traces, this.testMessage, this.currentItem, this.projectName); + this.testMessage = undefined; } } @@ -184,7 +189,7 @@ export class JUnitRunnerResultAnalyzer extends RunnerResultAnalyzer { this.currentTestState = TestResultState.Running; this.currentItem = item; this.currentDuration = 0; - this.assertionFailure = undefined; + this.testMessage = undefined; this.expectString = ''; this.actualString = ''; this.recordingType = RecordingType.None; @@ -275,20 +280,6 @@ export class JUnitRunnerResultAnalyzer extends RunnerResultAnalyzer { }); } } - - private async tryAppendMessage(item: TestItem, testMessage: TestMessage): Promise { - if (item.uri && item.range) { - testMessage.location = new Location(item.uri, item.range); - } else { - let id: string = item.id; - if (id.startsWith(INVOCATION_PREFIX)) { - id = id.substring(INVOCATION_PREFIX.length); - } - const location: Location | undefined = await findTestLocation(id); - testMessage.location = location; - } - setTestState(this.testContext.testRun, item, TestResultState.Failed, testMessage); - } } enum MessageId { diff --git a/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts b/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts index f5e6ce20..a95324c3 100644 --- a/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts +++ b/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts @@ -78,20 +78,19 @@ export class TestNGRunnerResultAnalyzer extends RunnerResultAnalyzer { return; } this.currentTestState = TestResultState.Failed; - const testMessages: TestMessage[] = []; + const traces: MarkdownString = new MarkdownString(); if (outputData.attributes.trace) { - const markdownTrace: MarkdownString = new MarkdownString(); - markdownTrace.isTrusted = true; - const testMessage: TestMessage = new TestMessage(markdownTrace); - for (const line of outputData.attributes.trace.split(/\r?\n/)) { - this.processStackTrace(line, markdownTrace, testMessage, this.currentItem, this.projectName); + traces.isTrusted = true; + const testMessage: TestMessage = new TestMessage(line); + + this.processStackTrace(line, traces, testMessage, this.currentItem, this.projectName); } } const duration: number = Number.parseInt(outputData.attributes.duration, 10); - setTestState(this.testContext.testRun, item, this.currentTestState, testMessages, duration); + this.finishFailureMessage(this.currentItem, new TestMessage(traces), duration); } else if (outputData.name === TEST_FINISH) { const item: TestItem | undefined = this.getTestItem(data); if (!item) { From ef8b17a4b12b5e209c0393090e835b174356ac00 Mon Sep 17 00:00:00 2001 From: Jonathan Kropfinger Date: Tue, 24 Aug 2021 21:38:40 -0500 Subject: [PATCH 07/13] Fixed the test message to allow diffs for JUnit tests --- src/runners/baseRunner/RunnerResultAnalyzer.ts | 1 - src/runners/junitRunner/JUnitRunnerResultAnalyzer.ts | 1 - src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts | 5 ++--- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/runners/baseRunner/RunnerResultAnalyzer.ts b/src/runners/baseRunner/RunnerResultAnalyzer.ts index 45bd576f..3721dcac 100644 --- a/src/runners/baseRunner/RunnerResultAnalyzer.ts +++ b/src/runners/baseRunner/RunnerResultAnalyzer.ts @@ -21,7 +21,6 @@ export abstract class RunnerResultAnalyzer { traces.appendText(traceResults[1]); traces.appendMarkdown(`${(traceResults[2] || '') + traceResults[3]}([${traceResults[4]}:${traceResults[5]}](command:_java.test.openStackTrace?${encodeURIComponent(JSON.stringify([data, projectName]))}))`); if (testMessage && currentItem && path.basename(currentItem.uri?.fsPath || '') === traceResults[4]) { - testMessage.message = data.trim(); const lineNum: number = parseInt(traceResults[5], 10); if (currentItem.uri) { testMessage.location = new Location(currentItem.uri, new Range(lineNum - 1, 0, lineNum, 0)); diff --git a/src/runners/junitRunner/JUnitRunnerResultAnalyzer.ts b/src/runners/junitRunner/JUnitRunnerResultAnalyzer.ts index 917f8b29..15e8191d 100644 --- a/src/runners/junitRunner/JUnitRunnerResultAnalyzer.ts +++ b/src/runners/junitRunner/JUnitRunnerResultAnalyzer.ts @@ -152,7 +152,6 @@ export class JUnitRunnerResultAnalyzer extends RunnerResultAnalyzer { } this.processStackTrace(data, this.traces, this.testMessage, this.currentItem, this.projectName); - this.testMessage = undefined; } } diff --git a/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts b/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts index a95324c3..e5b8daf6 100644 --- a/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts +++ b/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts @@ -80,11 +80,10 @@ export class TestNGRunnerResultAnalyzer extends RunnerResultAnalyzer { this.currentTestState = TestResultState.Failed; const traces: MarkdownString = new MarkdownString(); + traces.isTrusted = true; + const testMessage: TestMessage = new TestMessage(outputData.attributes.message); if (outputData.attributes.trace) { for (const line of outputData.attributes.trace.split(/\r?\n/)) { - traces.isTrusted = true; - const testMessage: TestMessage = new TestMessage(line); - this.processStackTrace(line, traces, testMessage, this.currentItem, this.projectName); } } From 857fc91cb3fe206e144120f5c863f7ed5a27c157 Mon Sep 17 00:00:00 2001 From: Jonathan Kropfinger Date: Wed, 25 Aug 2021 21:24:13 -0500 Subject: [PATCH 08/13] Revert "Fixed the test message to allow diffs for JUnit tests" This reverts commit ef8b17a4b12b5e209c0393090e835b174356ac00. --- src/runners/baseRunner/RunnerResultAnalyzer.ts | 1 + src/runners/junitRunner/JUnitRunnerResultAnalyzer.ts | 1 + src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts | 5 +++-- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/runners/baseRunner/RunnerResultAnalyzer.ts b/src/runners/baseRunner/RunnerResultAnalyzer.ts index 3721dcac..45bd576f 100644 --- a/src/runners/baseRunner/RunnerResultAnalyzer.ts +++ b/src/runners/baseRunner/RunnerResultAnalyzer.ts @@ -21,6 +21,7 @@ export abstract class RunnerResultAnalyzer { traces.appendText(traceResults[1]); traces.appendMarkdown(`${(traceResults[2] || '') + traceResults[3]}([${traceResults[4]}:${traceResults[5]}](command:_java.test.openStackTrace?${encodeURIComponent(JSON.stringify([data, projectName]))}))`); if (testMessage && currentItem && path.basename(currentItem.uri?.fsPath || '') === traceResults[4]) { + testMessage.message = data.trim(); const lineNum: number = parseInt(traceResults[5], 10); if (currentItem.uri) { testMessage.location = new Location(currentItem.uri, new Range(lineNum - 1, 0, lineNum, 0)); diff --git a/src/runners/junitRunner/JUnitRunnerResultAnalyzer.ts b/src/runners/junitRunner/JUnitRunnerResultAnalyzer.ts index 15e8191d..917f8b29 100644 --- a/src/runners/junitRunner/JUnitRunnerResultAnalyzer.ts +++ b/src/runners/junitRunner/JUnitRunnerResultAnalyzer.ts @@ -152,6 +152,7 @@ export class JUnitRunnerResultAnalyzer extends RunnerResultAnalyzer { } this.processStackTrace(data, this.traces, this.testMessage, this.currentItem, this.projectName); + this.testMessage = undefined; } } diff --git a/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts b/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts index e5b8daf6..a95324c3 100644 --- a/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts +++ b/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts @@ -80,10 +80,11 @@ export class TestNGRunnerResultAnalyzer extends RunnerResultAnalyzer { this.currentTestState = TestResultState.Failed; const traces: MarkdownString = new MarkdownString(); - traces.isTrusted = true; - const testMessage: TestMessage = new TestMessage(outputData.attributes.message); if (outputData.attributes.trace) { for (const line of outputData.attributes.trace.split(/\r?\n/)) { + traces.isTrusted = true; + const testMessage: TestMessage = new TestMessage(line); + this.processStackTrace(line, traces, testMessage, this.currentItem, this.projectName); } } From e6fafc5f688ebc445e538eb1d897bf0ae8eec615 Mon Sep 17 00:00:00 2001 From: Jonathan Kropfinger Date: Wed, 25 Aug 2021 21:25:01 -0500 Subject: [PATCH 09/13] Revert "Fixed the stack trace issues that resulted in only a single element being shown in the error preview window" This reverts commit 38e287e95554013ecb0277c50248755df4c4370d. --- .../baseRunner/RunnerResultAnalyzer.ts | 28 +++---------- .../junitRunner/JUnitRunnerResultAnalyzer.ts | 39 ++++++++++++------- .../TestNGRunnerResultAnalyzer.ts | 13 ++++--- 3 files changed, 36 insertions(+), 44 deletions(-) diff --git a/src/runners/baseRunner/RunnerResultAnalyzer.ts b/src/runners/baseRunner/RunnerResultAnalyzer.ts index 45bd576f..fc2af811 100644 --- a/src/runners/baseRunner/RunnerResultAnalyzer.ts +++ b/src/runners/baseRunner/RunnerResultAnalyzer.ts @@ -3,9 +3,8 @@ import * as path from 'path'; import { Location, MarkdownString, Range, TestItem, TestMessage } from 'vscode'; -import { INVOCATION_PREFIX } from '../../constants'; import { IRunTestContext } from '../../types'; -import { findTestLocation, setTestState, TestResultState } from '../utils'; +import { setTestState, TestResultState } from '../utils'; export abstract class RunnerResultAnalyzer { constructor(protected testContext: IRunTestContext) { } @@ -13,20 +12,19 @@ export abstract class RunnerResultAnalyzer { public abstract analyzeData(data: string): void; public abstract processData(data: string): void; - protected processStackTrace(data: string, traces: MarkdownString, testMessage: TestMessage | undefined, currentItem: TestItem | undefined, projectName: string): void { + protected processStackTrace(data: string, traces: MarkdownString, assertionFailure: TestMessage | undefined, currentItem: TestItem | undefined, projectName: string): void { const traceRegExp: RegExp = /(\s?at\s+)([\w$\\.]+\/)?((?:[\w$]+\.)+[<\w$>]+)\(([\w-$]+\.java):(\d+)\)/; const traceResults: RegExpExecArray | null = traceRegExp.exec(data); if (traceResults && traceResults.length === 6) { traces.appendText(traceResults[1]); traces.appendMarkdown(`${(traceResults[2] || '') + traceResults[3]}([${traceResults[4]}:${traceResults[5]}](command:_java.test.openStackTrace?${encodeURIComponent(JSON.stringify([data, projectName]))}))`); - if (testMessage && currentItem && path.basename(currentItem.uri?.fsPath || '') === traceResults[4]) { - testMessage.message = data.trim(); + if (assertionFailure && currentItem && path.basename(currentItem.uri?.fsPath || '') === traceResults[4]) { const lineNum: number = parseInt(traceResults[5], 10); if (currentItem.uri) { - testMessage.location = new Location(currentItem.uri, new Range(lineNum - 1, 0, lineNum, 0)); + assertionFailure.location = new Location(currentItem.uri, new Range(lineNum - 1, 0, lineNum, 0)); } - setTestState(this.testContext.testRun, currentItem, TestResultState.Failed, testMessage); + setTestState(this.testContext.testRun, currentItem, TestResultState.Failed, assertionFailure); } } else { // in case the message contains message like: 'expected: <..> but was: <..>' @@ -34,20 +32,4 @@ export abstract class RunnerResultAnalyzer { } traces.appendText('\n'); } - - protected async finishFailureMessage(item: TestItem | undefined, testMessage: TestMessage, duration?: number): Promise { - if (item) { - if (item.uri && item.range) { - testMessage.location = new Location(item.uri, item.range); - } else { - let id: string = item.id; - if (id.startsWith(INVOCATION_PREFIX)) { - id = id.substring(INVOCATION_PREFIX.length); - } - const location: Location | undefined = await findTestLocation(id); - testMessage.location = location; - } - setTestState(this.testContext.testRun, item, TestResultState.Failed, testMessage, duration); - } - } } diff --git a/src/runners/junitRunner/JUnitRunnerResultAnalyzer.ts b/src/runners/junitRunner/JUnitRunnerResultAnalyzer.ts index 917f8b29..dfff3cdd 100644 --- a/src/runners/junitRunner/JUnitRunnerResultAnalyzer.ts +++ b/src/runners/junitRunner/JUnitRunnerResultAnalyzer.ts @@ -1,13 +1,13 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. -import { MarkdownString, TestItem, TestMessage } from 'vscode'; +import { Location, MarkdownString, TestItem, TestMessage } from 'vscode'; import { INVOCATION_PREFIX } from '../../constants'; import { dataCache, ITestItemData } from '../../controller/testItemDataCache'; import { createTestItem } from '../../controller/utils'; import { IJavaTestItem, IRunTestContext, TestKind, TestLevel } from '../../types'; import { RunnerResultAnalyzer } from '../baseRunner/RunnerResultAnalyzer'; -import { setTestState, TestResultState } from '../utils'; +import { findTestLocation, setTestState, TestResultState } from '../utils'; export class JUnitRunnerResultAnalyzer extends RunnerResultAnalyzer { @@ -17,7 +17,7 @@ export class JUnitRunnerResultAnalyzer extends RunnerResultAnalyzer { private currentItem: TestItem | undefined; private currentDuration: number = 0; private traces: MarkdownString; - private testMessage: TestMessage | undefined; + private assertionFailure: TestMessage | undefined; private recordingType: RecordingType; private expectString: string; private actualString: string; @@ -116,7 +116,7 @@ export class JUnitRunnerResultAnalyzer extends RunnerResultAnalyzer { } const testMessage: TestMessage = new TestMessage(this.traces); - this.finishFailureMessage(this.currentItem, testMessage); + this.tryAppendMessage(this.currentItem, testMessage); this.recordingType = RecordingType.None; if (this.currentTestState === TestResultState.Errored) { setTestState(this.testContext.testRun, this.currentItem, this.currentTestState); @@ -131,28 +131,23 @@ export class JUnitRunnerResultAnalyzer extends RunnerResultAnalyzer { } else if (data.startsWith(MessageId.ActualEnd)) { this.recordingType = RecordingType.None; this.actualString = this.actualString.replace(/\n$/, ''); - if (!this.testMessage && this.expectString && this.actualString) { - this.testMessage = TestMessage.diff(`Expected [${this.expectString}] but was [${this.actualString}]`, this.expectString, this.actualString); + if (!this.assertionFailure && this.expectString && this.actualString) { + this.assertionFailure = TestMessage.diff(`Expected [${this.expectString}] but was [${this.actualString}]`, this.expectString, this.actualString); } } else if (this.recordingType === RecordingType.ExpectMessage) { this.expectString += data + '\n'; } else if (this.recordingType === RecordingType.ActualMessage) { this.actualString += data + '\n'; } else if (this.recordingType === RecordingType.StackTrace) { - if (!this.testMessage) { + if (!this.assertionFailure) { const assertionRegExp: RegExp = /expected.*:.*<(.+?)>.*but.*:.*<(.+?)>/mi; const assertionResults: RegExpExecArray | null = assertionRegExp.exec(data); if (assertionResults && assertionResults.length === 3) { - this.testMessage = TestMessage.diff(`Expected [${assertionResults[1]}] but was [${assertionResults[2]}]`, assertionResults[1], assertionResults[2]); - } else { - const markdownTrace: MarkdownString = new MarkdownString(); - markdownTrace.isTrusted = true; - this.testMessage = new TestMessage(data); + this.assertionFailure = TestMessage.diff(`Expected [${assertionResults[1]}] but was [${assertionResults[2]}]`, assertionResults[1], assertionResults[2]); } } - this.processStackTrace(data, this.traces, this.testMessage, this.currentItem, this.projectName); - this.testMessage = undefined; + this.processStackTrace(data, this.traces, this.assertionFailure, this.currentItem, this.projectName); } } @@ -189,7 +184,7 @@ export class JUnitRunnerResultAnalyzer extends RunnerResultAnalyzer { this.currentTestState = TestResultState.Running; this.currentItem = item; this.currentDuration = 0; - this.testMessage = undefined; + this.assertionFailure = undefined; this.expectString = ''; this.actualString = ''; this.recordingType = RecordingType.None; @@ -280,6 +275,20 @@ export class JUnitRunnerResultAnalyzer extends RunnerResultAnalyzer { }); } } + + private async tryAppendMessage(item: TestItem, testMessage: TestMessage): Promise { + if (item.uri && item.range) { + testMessage.location = new Location(item.uri, item.range); + } else { + let id: string = item.id; + if (id.startsWith(INVOCATION_PREFIX)) { + id = id.substring(INVOCATION_PREFIX.length); + } + const location: Location | undefined = await findTestLocation(id); + testMessage.location = location; + } + setTestState(this.testContext.testRun, item, TestResultState.Failed, testMessage); + } } enum MessageId { diff --git a/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts b/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts index a95324c3..f5e6ce20 100644 --- a/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts +++ b/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts @@ -78,19 +78,20 @@ export class TestNGRunnerResultAnalyzer extends RunnerResultAnalyzer { return; } this.currentTestState = TestResultState.Failed; + const testMessages: TestMessage[] = []; - const traces: MarkdownString = new MarkdownString(); if (outputData.attributes.trace) { - for (const line of outputData.attributes.trace.split(/\r?\n/)) { - traces.isTrusted = true; - const testMessage: TestMessage = new TestMessage(line); + const markdownTrace: MarkdownString = new MarkdownString(); + markdownTrace.isTrusted = true; + const testMessage: TestMessage = new TestMessage(markdownTrace); - this.processStackTrace(line, traces, testMessage, this.currentItem, this.projectName); + for (const line of outputData.attributes.trace.split(/\r?\n/)) { + this.processStackTrace(line, markdownTrace, testMessage, this.currentItem, this.projectName); } } const duration: number = Number.parseInt(outputData.attributes.duration, 10); - this.finishFailureMessage(this.currentItem, new TestMessage(traces), duration); + setTestState(this.testContext.testRun, item, this.currentTestState, testMessages, duration); } else if (outputData.name === TEST_FINISH) { const item: TestItem | undefined = this.getTestItem(data); if (!item) { From 42e2c07a1ec4324f34e33ff35b58565430d69d43 Mon Sep 17 00:00:00 2001 From: Jonathan Kropfinger Date: Wed, 25 Aug 2021 21:40:53 -0500 Subject: [PATCH 10/13] Fixed issues with only getting partial stack trace when TestNG tests fail --- .../testngRunner/TestNGRunnerResultAnalyzer.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts b/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts index f5e6ce20..c0d15162 100644 --- a/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts +++ b/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. -import { MarkdownString, TestItem, TestMessage } from 'vscode'; +import { Location, MarkdownString, TestItem, TestMessage } from 'vscode'; import { dataCache } from '../../controller/testItemDataCache'; import { IRunTestContext, TestLevel } from '../../types'; import { RunnerResultAnalyzer } from '../baseRunner/RunnerResultAnalyzer'; @@ -83,13 +83,17 @@ export class TestNGRunnerResultAnalyzer extends RunnerResultAnalyzer { if (outputData.attributes.trace) { const markdownTrace: MarkdownString = new MarkdownString(); markdownTrace.isTrusted = true; - const testMessage: TestMessage = new TestMessage(markdownTrace); for (const line of outputData.attributes.trace.split(/\r?\n/)) { - this.processStackTrace(line, markdownTrace, testMessage, this.currentItem, this.projectName); + this.processStackTrace(line, markdownTrace, undefined, this.currentItem, this.projectName); + } + + const testMessage: TestMessage = new TestMessage(markdownTrace); + if (item.uri && item.range) { + testMessage.location = new Location(item.uri, item.range); } + testMessages.push(testMessage); } - const duration: number = Number.parseInt(outputData.attributes.duration, 10); setTestState(this.testContext.testRun, item, this.currentTestState, testMessages, duration); } else if (outputData.name === TEST_FINISH) { From b2316be3b3c0f397e524f93ca6f15692b8683afa Mon Sep 17 00:00:00 2001 From: Jonathan Kropfinger Date: Wed, 25 Aug 2021 22:02:36 -0500 Subject: [PATCH 11/13] Fixed linting errors --- src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts b/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts index c0d15162..c071123d 100644 --- a/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts +++ b/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts @@ -87,7 +87,7 @@ export class TestNGRunnerResultAnalyzer extends RunnerResultAnalyzer { for (const line of outputData.attributes.trace.split(/\r?\n/)) { this.processStackTrace(line, markdownTrace, undefined, this.currentItem, this.projectName); } - + const testMessage: TestMessage = new TestMessage(markdownTrace); if (item.uri && item.range) { testMessage.location = new Location(item.uri, item.range); From 096870b66dc06240c19b321dc364d86252ef1b68 Mon Sep 17 00:00:00 2001 From: Jonathan Kropfinger Date: Fri, 27 Aug 2021 08:33:24 -0500 Subject: [PATCH 12/13] Resolving peer review comments --- src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts b/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts index c071123d..02f0b6e8 100644 --- a/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts +++ b/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts @@ -55,13 +55,7 @@ export class TestNGRunnerResultAnalyzer extends RunnerResultAnalyzer { public processData(data: string): void { const outputData: ITestNGOutputData = JSON.parse(data) as ITestNGOutputData; - for (const line of this.unescape(data).split(/\r?\n/)) { - if (outputData.name.toLocaleLowerCase() === 'error') { - this.testContext.testRun.appendOutput(`[ERROR] ${line}\r\n`); - } else { - this.testContext.testRun.appendOutput(`${line}\r\n`); - } - } + this.testContext.testRun.appendOutput(this.unescape(data).replace(/\r?\n/g, "\r\n")); const id: string = `${this.projectName}@${outputData.attributes.name}`; if (outputData.name === TEST_START) { From 21382946ebe2ace696826bc567515b0a4685a5d2 Mon Sep 17 00:00:00 2001 From: Jonathan Kropfinger Date: Fri, 27 Aug 2021 08:34:26 -0500 Subject: [PATCH 13/13] Fixed linting errors --- src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts b/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts index 02f0b6e8..a6ee53f1 100644 --- a/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts +++ b/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts @@ -55,7 +55,7 @@ export class TestNGRunnerResultAnalyzer extends RunnerResultAnalyzer { public processData(data: string): void { const outputData: ITestNGOutputData = JSON.parse(data) as ITestNGOutputData; - this.testContext.testRun.appendOutput(this.unescape(data).replace(/\r?\n/g, "\r\n")); + this.testContext.testRun.appendOutput(this.unescape(data).replace(/\r?\n/g, '\r\n')); const id: string = `${this.projectName}@${outputData.attributes.name}`; if (outputData.name === TEST_START) {