From 2b6530c5d40d620c3fd07a7ffd8a30d0e8405f8f Mon Sep 17 00:00:00 2001 From: fladdimir Date: Wed, 11 Jan 2023 21:46:21 +0100 Subject: [PATCH] support Junit5 parallel execution #1472 --- .../junitRunner/JUnitRunnerResultAnalyzer.ts | 129 +++++++---- test/suite/JUnitAnalyzer.parallel.test.ts | 212 ++++++++++++++++++ test/suite/utils.ts | 8 +- .../java/junit5/ParallelExecutionTest.java | 45 ++++ .../test/resources/junit-platform.properties | 7 + 5 files changed, 350 insertions(+), 51 deletions(-) create mode 100644 test/suite/JUnitAnalyzer.parallel.test.ts create mode 100644 test/test-projects/junit/src/test/java/junit5/ParallelExecutionTest.java create mode 100644 test/test-projects/junit/src/test/resources/junit-platform.properties diff --git a/src/runners/junitRunner/JUnitRunnerResultAnalyzer.ts b/src/runners/junitRunner/JUnitRunnerResultAnalyzer.ts index 7c5939e9..d0dc8522 100644 --- a/src/runners/junitRunner/JUnitRunnerResultAnalyzer.ts +++ b/src/runners/junitRunner/JUnitRunnerResultAnalyzer.ts @@ -9,20 +9,24 @@ import { IJavaTestItem, IRunTestContext, TestKind, TestLevel } from '../../types import { RunnerResultAnalyzer } from '../baseRunner/RunnerResultAnalyzer'; import { findTestLocation, setTestState, TestResultState } from '../utils'; + export class JUnitRunnerResultAnalyzer extends RunnerResultAnalyzer { private testOutputMapping: Map = new Map(); private triggeredTestsMapping: Map = new Map(); - private currentTestState: TestResultState; - private currentItem: TestItem | undefined; - private currentDuration: number = 0; + private projectName: string; + private incompleteTestSuite: ITestInfo[] = []; + + // tests may be run concurrently, so each item's current state needs to be remembered + private currentStates: Map = new Map(); + + // failure info for a test is received consecutively: + private tracingItem: TestItem | undefined; private traces: MarkdownString; private assertionFailure: TestMessage | undefined; private recordingType: RecordingType; private expectString: string; private actualString: string; - private projectName: string; - private incompleteTestSuite: ITestInfo[] = []; constructor(protected testContext: IRunTestContext) { super(testContext); @@ -60,40 +64,26 @@ export class JUnitRunnerResultAnalyzer extends RunnerResultAnalyzer { if (!item) { return; } - if (item.id !== this.currentItem?.id) { - this.initializeCache(item); - } - this.testContext.testRun.started(item); - - const start: number = Date.now(); - if (this.currentDuration === 0) { - this.currentDuration = -start; - } else if (this.currentDuration > 0) { - // Some test cases may executed multiple times (@RepeatedTest), we need to calculate the time for each execution - this.currentDuration -= start; - } + this.setCurrentState(item, TestResultState.Running, 0); + this.setDurationAtStart(this.getCurrentState(item)); + setTestState(this.testContext.testRun, item, this.getCurrentState(item).resultState); } else if (data.startsWith(MessageId.TestEnd)) { - if (!this.currentItem) { + const item: TestItem | undefined = this.getTestItem(data.substr(MessageId.TestEnd.length)); + if (!item) { return; } - - if (this.currentDuration < 0) { - const end: number = Date.now(); - this.currentDuration += end; - } - - if (data.indexOf(MessageId.IGNORE_TEST_PREFIX) > -1) { - this.currentTestState = TestResultState.Skipped; - } else if (this.currentTestState === TestResultState.Running) { - this.currentTestState = TestResultState.Passed; - } - setTestState(this.testContext.testRun, this.currentItem, this.currentTestState, undefined, this.currentDuration); + const currentState: CurrentItemState = this.getCurrentState(item); + this.calcDurationAtEnd(currentState); + this.determineResultStateAtEnd(data, currentState); + setTestState(this.testContext.testRun, item, currentState.resultState, undefined, currentState.duration); } else if (data.startsWith(MessageId.TestFailed)) { - if (data.indexOf(MessageId.ASSUMPTION_FAILED_TEST_PREFIX) > -1) { - this.currentTestState = TestResultState.Skipped; - } else { - this.currentTestState = TestResultState.Failed; + const item: TestItem | undefined = this.getTestItem(data.substr(MessageId.TestFailed.length)); + if (!item) { + return; } + const currentState: CurrentItemState = this.getCurrentState(item); + this.determineResultStateOnFailure(data, currentState); + this.initializeTracingItemProcessingCache(item); // traces or comparison failure info might follow immediately } else if (data.startsWith(MessageId.TestError)) { let item: TestItem | undefined = this.getTestItem(data.substr(MessageId.TestError.length)); if (!item) { @@ -104,25 +94,25 @@ export class JUnitRunnerResultAnalyzer extends RunnerResultAnalyzer { return; } } - if (item.id !== this.currentItem?.id) { - this.initializeCache(item); + this.getCurrentState(item).resultState = TestResultState.Errored; + if (item.id !== this.tracingItem?.id) { + this.initializeTracingItemProcessingCache(item); } - this.currentTestState = TestResultState.Errored; } else if (data.startsWith(MessageId.TraceStart)) { this.traces = new MarkdownString(); this.traces.isTrusted = true; this.traces.supportHtml = true; this.recordingType = RecordingType.StackTrace; } else if (data.startsWith(MessageId.TraceEnd)) { - if (!this.currentItem) { + if (!this.tracingItem) { return; } - const testMessage: TestMessage = new TestMessage(this.traces); - this.tryAppendMessage(this.currentItem, testMessage, this.currentTestState); + const currentResultState: TestResultState = this.getCurrentState(this.tracingItem).resultState; + this.tryAppendMessage(this.tracingItem, testMessage, currentResultState); this.recordingType = RecordingType.None; - if (this.currentTestState === TestResultState.Errored) { - setTestState(this.testContext.testRun, this.currentItem, this.currentTestState); + if (currentResultState === TestResultState.Errored) { + setTestState(this.testContext.testRun, this.tracingItem, currentResultState); } } else if (data.startsWith(MessageId.ExpectStart)) { this.recordingType = RecordingType.ExpectMessage; @@ -150,7 +140,38 @@ export class JUnitRunnerResultAnalyzer extends RunnerResultAnalyzer { } } - this.processStackTrace(data, this.traces, this.assertionFailure, this.currentItem, this.projectName); + this.processStackTrace(data, this.traces, this.assertionFailure, this.tracingItem, this.projectName); + } + } + + private determineResultStateOnFailure(data: string, currentState: CurrentItemState): void { + const isSkip: boolean = data.indexOf(MessageId.ASSUMPTION_FAILED_TEST_PREFIX) > -1; + currentState.resultState = isSkip ? TestResultState.Skipped : TestResultState.Failed; + } + + private determineResultStateAtEnd(data: string, currentState: CurrentItemState): void { + const isIgnore: boolean = data.indexOf(MessageId.IGNORE_TEST_PREFIX) > -1; + if (isIgnore) { + currentState.resultState = TestResultState.Skipped; + } else if (currentState.resultState === TestResultState.Running) { + currentState.resultState = TestResultState.Passed; + } + } + + private setDurationAtStart(currentState: CurrentItemState): void { + const start: number = Date.now(); + if (currentState.duration === 0) { + currentState.duration = -start; + } else if (currentState.duration > 0) { + // Some test cases may executed multiple times (@RepeatedTest), we need to calculate the time for each execution + currentState.duration -= start; + } + } + + private calcDurationAtEnd(currentState: CurrentItemState): void { + if (currentState.duration < 0) { + const end: number = Date.now(); + currentState.duration += end; } } @@ -182,10 +203,17 @@ export class JUnitRunnerResultAnalyzer extends RunnerResultAnalyzer { return `${this.projectName}@${message}`; } - protected initializeCache(item: TestItem): void { - this.currentTestState = TestResultState.Running; - this.currentItem = item; - this.currentDuration = 0; + private setCurrentState(testItem: TestItem, resultState: TestResultState, duration: number): void { + this.currentStates.set(testItem, { resultState, duration }); + } + + private getCurrentState(testItem: TestItem): CurrentItemState { + if (!this.currentStates.has(testItem)) this.setCurrentState(testItem, TestResultState.Running, 0); + return this.currentStates.get(testItem)!; + } + + private initializeTracingItemProcessingCache(item: TestItem): void { + this.tracingItem = item; this.assertionFailure = undefined; this.expectString = ''; this.actualString = ''; @@ -288,7 +316,7 @@ export class JUnitRunnerResultAnalyzer extends RunnerResultAnalyzer { if (testItem) { if (dataCache.get(testItem)?.testKind === TestKind.JUnit5 && - this.getLabelWithoutCodicon(testItem.label) !== displayName) { + this.getLabelWithoutCodicon(testItem.label) !== displayName) { testItem.description = displayName; } else { testItem.description = ''; @@ -390,3 +418,8 @@ enum RecordingType { ExpectMessage, ActualMessage, } + +interface CurrentItemState { + resultState: TestResultState; + duration: number; +} diff --git a/test/suite/JUnitAnalyzer.parallel.test.ts b/test/suite/JUnitAnalyzer.parallel.test.ts new file mode 100644 index 00000000..9cd47bb0 --- /dev/null +++ b/test/suite/JUnitAnalyzer.parallel.test.ts @@ -0,0 +1,212 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +'use strict'; + +import * as sinon from 'sinon'; +import { Range, TestController, TestItem, TestMessage, TestRunRequest, tests, workspace } from 'vscode'; +import { JUnitRunnerResultAnalyzer } from '../../src/runners/junitRunner/JUnitRunnerResultAnalyzer'; +import { TestKind } from '../../src/types'; +import { generateTestItem } from './utils'; + +// tslint:disable: only-arrow-functions +// tslint:disable: no-object-literal-type-assertion +suite('JUnit Runner Analyzer Tests for JUnit5 Parallel Tests', () => { + + let testController: TestController; + + let testItem1: TestItem; + let testItem2: TestItem; + + let startedSpy: sinon.SinonSpy<[test: TestItem], void>; + let passedSpy: sinon.SinonSpy<[test: TestItem, duration?: number | undefined], void>; + let failedSpy: sinon.SinonSpy<[test: TestItem, message: TestMessage | readonly TestMessage[], duration?: number | undefined], void>; + + let analyzer: JUnitRunnerResultAnalyzer; + + setup(() => { + testController = tests.createTestController('testController', 'Mock Test'); + const testItemParent = generateTestItem(testController, 'junit@junit5.ParallelExecutionTest', TestKind.JUnit5, new Range(0, 0, 0, 0), undefined, 'ParallelExecutionTest.java'); + testItem1 = generateTestItem(testController, 'junit@junit5.ParallelExecutionTest#test1', TestKind.JUnit5, new Range(1, 0, 1, 0), undefined, 'ParallelExecutionTest.java'); + testItem2 = generateTestItem(testController, 'junit@junit5.ParallelExecutionTest#test2', TestKind.JUnit5, new Range(2, 0, 2, 0), undefined, 'ParallelExecutionTest.java'); + testItemParent.children.add(testItem1); + testItemParent.children.add(testItem2); + const testRunRequest = new TestRunRequest([testItemParent], []); + const testRun = testController.createTestRun(testRunRequest); + startedSpy = sinon.spy(testRun, 'started'); + passedSpy = sinon.spy(testRun, 'passed'); + failedSpy = sinon.spy(testRun, 'failed'); + const runnerContext = { + isDebug: false, + kind: TestKind.JUnit5, + projectName: 'junit', + testItems: [testItemParent], + testRun: testRun, + workspaceFolder: workspace.workspaceFolders?.[0]!, + }; + analyzer = new JUnitRunnerResultAnalyzer(runnerContext); + }); + + teardown(() => { + testController.dispose(); + }); + + test("successfull parallel execution of 2 test methods within a class", () => { + + + let testRunnerOutput = + ` +%TESTC 2 v2 + +%TSTTREE2,junit5.ParallelExecutionTest,true,2,false,1,ParallelExecutionTest,,[engine:junit-jupiter]/[class:junit5.ParallelExecutionTest] + +%TSTTREE3,test1(junit5.ParallelExecutionTest),false,1,false,2,test1(),,[engine:junit-jupiter]/[class:junit5.ParallelExecutionTest]/[method:test1()] +%TSTTREE4,test2(junit5.ParallelExecutionTest),false,1,false,2,test2(),,[engine:junit-jupiter]/[class:junit5.ParallelExecutionTest]/[method:test2()] + +%TESTS 3,test1(junit5.ParallelExecutionTest) +%TESTS 4,test2(junit5.ParallelExecutionTest) + +%TESTE 3,test1(junit5.ParallelExecutionTest) +%TESTE 4,test2(junit5.ParallelExecutionTest) + +%RUNTIME58 +`; + + analyzer.analyzeData(testRunnerOutput); + + sinon.assert.calledWith(startedSpy, testItem1); + sinon.assert.calledWith(startedSpy, testItem2); + sinon.assert.calledWith(passedSpy, testItem1); + sinon.assert.calledWith(passedSpy, testItem2); + }); + + test("failed parallel execution with traces", () => { + + let testRunnerOutput = + ` +%TESTC 2 v2 +%TSTTREE2,junit5.ParallelExecutionTest,true,2,false,1,ParallelExecutionTest,,[engine:junit-jupiter]/[class:junit5.ParallelExecutionTest] + +%TSTTREE3,test1(junit5.ParallelExecutionTest),false,1,false,2,test1(),,[engine:junit-jupiter]/[class:junit5.ParallelExecutionTest]/[method:test1()] +%TSTTREE4,test2(junit5.ParallelExecutionTest),false,1,false,2,test2(),,[engine:junit-jupiter]/[class:junit5.ParallelExecutionTest]/[method:test2()] + +%TESTS 3,test1(junit5.ParallelExecutionTest) +%TESTS 4,test2(junit5.ParallelExecutionTest) + +%FAILED 3,test1(junit5.ParallelExecutionTest) +%TRACES +org.opentest4j.AssertionFailedError: test1 failed +at org.junit.jupiter.api.Assertions.fail(Assertions.java:109) +at junit5.ParallelExecutionTest.fail(ParallelExecutionTest.java:53) +at junit5.ParallelExecutionTest.test1(ParallelExecutionTest.java:40) +at java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) +%TRACEE + +%FAILED 4,test2(junit5.ParallelExecutionTest) +%TRACES +org.opentest4j.AssertionFailedError: test2 failed +at org.junit.jupiter.api.Assertions.fail(Assertions.java:109) +at junit5.ParallelExecutionTest.fail(ParallelExecutionTest.java:53) +at junit5.ParallelExecutionTest.test1(ParallelExecutionTest.java:48) +at java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) +%TRACEE + +%TESTE 3,test1(junit5.ParallelExecutionTest) +%TESTE 4,test2(junit5.ParallelExecutionTest) + +%RUNTIME79 +`; + + analyzer.analyzeData(testRunnerOutput); + + sinon.assert.calledWith(startedSpy, testItem1); + sinon.assert.calledWith(startedSpy, testItem2); + + sinon.assert.calledWith(failedSpy, testItem2, sinon.match.any, sinon.match.number); + sinon.assert.calledWith(failedSpy, testItem2, sinon.match({ + message: sinon.match({ + value: sinon.match(/AssertionFailedError.*test2/) + }) + })); + + sinon.assert.calledWith(failedSpy, testItem1, sinon.match.any, sinon.match.number); + sinon.assert.calledWith(failedSpy, testItem1, sinon.match({ + message: sinon.match({ + value: sinon.match(/AssertionFailedError.*test1/) + }) + })); + + }); + + test("failed parallel execution with comparison failure", () => { + + let testRunnerOutput = + ` +%TESTC 2 v2 +%TSTTREE2,junit5.ParallelExecutionTest,true,2,false,1,ParallelExecutionTest,,[engine:junit-jupiter]/[class:junit5.ParallelExecutionTest] +%TSTTREE3,test1(junit5.ParallelExecutionTest),false,1,false,2,test1(),,[engine:junit-jupiter]/[class:junit5.ParallelExecutionTest]/[method:test1()] +%TSTTREE4,test2(junit5.ParallelExecutionTest),false,1,false,2,test2(),,[engine:junit-jupiter]/[class:junit5.ParallelExecutionTest]/[method:test2()] + +%TESTS 3,test1(junit5.ParallelExecutionTest) +%TESTS 4,test2(junit5.ParallelExecutionTest) + +%FAILED 3,test1(junit5.ParallelExecutionTest) +%EXPECTS +expected1 +%EXPECTE +%ACTUALS +actual1 +%ACTUALE +%TRACES +org.opentest4j.AssertionFailedError: comparison1 failed + at junit5.ParallelExecutionTest.test1(ParallelExecutionTest.java:40) +%TRACEE + +%FAILED 4,test2(junit5.ParallelExecutionTest) +%EXPECTS +expected2 +%EXPECTE +%ACTUALS +actual2 +%ACTUALE +%TRACES +org.opentest4j.AssertionFailedError: comparison2 failed + at junit5.ParallelExecutionTest.test1(ParallelExecutionTest.java:48) +%TRACEE + +%TESTE 3,test1(junit5.ParallelExecutionTest) +%TESTE 4,test2(junit5.ParallelExecutionTest) + +%RUNTIME87 +`; + + analyzer.analyzeData(testRunnerOutput); + + sinon.assert.calledWith(startedSpy, testItem1); + sinon.assert.calledWith(startedSpy, testItem2); + + // testItem1 + sinon.assert.calledWith(failedSpy, testItem1, sinon.match.any, sinon.match.number); + sinon.assert.calledWith(failedSpy, testItem1, sinon.match({ + expectedOutput: "expected1", actualOutput: "actual1" + })); + sinon.assert.calledWith(failedSpy, testItem1, sinon.match({ + message: sinon.match({ + value: sinon.match(/comparison1/) + }) + })); + + // testItem2 + sinon.assert.calledWith(failedSpy, testItem2, sinon.match.any, sinon.match.number); + sinon.assert.calledWith(failedSpy, testItem2, sinon.match({ + expectedOutput: "expected2", actualOutput: "actual2" + })); + sinon.assert.calledWith(failedSpy, testItem2, sinon.match({ + message: sinon.match({ + value: sinon.match(/comparison2/) + }) + })); + + }); + +}); diff --git a/test/suite/utils.ts b/test/suite/utils.ts index 8a00951b..27dd6194 100644 --- a/test/suite/utils.ts +++ b/test/suite/utils.ts @@ -5,11 +5,13 @@ import * as fse from 'fs-extra'; import * as path from 'path'; -import { TestController, TestItem, Uri, Range, extensions, commands, workspace } from "vscode"; +import { commands, extensions, Range, TestController, TestItem, Uri, workspace } from "vscode"; import { dataCache } from "../../src/controller/testItemDataCache"; import { TestKind, TestLevel } from "../../src/types"; -export function generateTestItem(testController: TestController, id: string, testKind: TestKind, range?: Range, jdtHandler?: string): TestItem { +export function generateTestItem(testController: TestController, id: string, testKind: TestKind, + range?: Range, jdtHandler?: string, uri: string = '/mock/test/TestAnnotation.java'): TestItem { + if (!id) { throw new Error('id cannot be null'); } @@ -18,7 +20,7 @@ export function generateTestItem(testController: TestController, id: string, tes const fullName = id.substring(id.indexOf('@') + 1); const label = id.substring(id.indexOf('#') + 1) + '()'; - const testItem = testController.createTestItem(id, label, Uri.file('/mock/test/TestAnnotation.java')); + const testItem = testController.createTestItem(id, label, Uri.file(uri)); testItem.range = range || new Range(0, 0, 0, 0); dataCache.set(testItem, { jdtHandler: jdtHandler || '', diff --git a/test/test-projects/junit/src/test/java/junit5/ParallelExecutionTest.java b/test/test-projects/junit/src/test/java/junit5/ParallelExecutionTest.java new file mode 100644 index 00000000..1be0f905 --- /dev/null +++ b/test/test-projects/junit/src/test/java/junit5/ParallelExecutionTest.java @@ -0,0 +1,45 @@ +package junit5; + +import java.util.concurrent.Semaphore; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; + +/** + * Test class with 3 test methods which are executed concurrently, so that the + * correct result processing of overlapping test runs can be verified. + */ +@Execution(ExecutionMode.CONCURRENT) +class ParallelExecutionTest { + + private static final Semaphore s1 = new Semaphore(0); + private static final Semaphore s2 = new Semaphore(0); + private static final Semaphore s3 = new Semaphore(0); + + @Test + void test1() { + s1.release(); + s2.acquireUninterruptibly(); + assertComparison("expected1", "actual1"); + } + + @Test + void test2() { + s2.release(); + s3.acquireUninterruptibly(); + assertComparison("expected2", "actual2"); + } + + @Test + void test3() { + s3.release(); + s1.acquireUninterruptibly(); + } + + private void assertComparison(String expected, String actual) { + Assertions.assertEquals(expected, actual); + } + +} diff --git a/test/test-projects/junit/src/test/resources/junit-platform.properties b/test/test-projects/junit/src/test/resources/junit-platform.properties new file mode 100644 index 00000000..b0fa4c18 --- /dev/null +++ b/test/test-projects/junit/src/test/resources/junit-platform.properties @@ -0,0 +1,7 @@ +junit.jupiter.execution.parallel.enabled = true +# run all test sequentially by default +# can be changed e.g. by annotating certain classes with @Execution(ExecutionMode.CONCURRENT) +junit.jupiter.execution.parallel.mode.default = same_thread +junit.jupiter.execution.parallel.mode.classes.default = same_thread +# run at least 3 tests in parallel +junit.jupiter.execution.parallel.config.dynamic.factor = 3