From 8d8cfba065900c6a8b87630297ac6930d604fcf5 Mon Sep 17 00:00:00 2001 From: Sidharth Vinod Date: Sun, 26 Sep 2021 09:38:04 +0530 Subject: [PATCH 1/5] feat(reporters): Add error position to JSON Report --- src/test/reporters/base.ts | 2 +- src/test/reporters/json.ts | 30 +++++++++++++++++---- tests/playwright-test/json-reporter.spec.ts | 17 ++++++++++++ 3 files changed, 43 insertions(+), 6 deletions(-) diff --git a/src/test/reporters/base.ts b/src/test/reporters/base.ts index 18e6ce5d0aefe..f1f3c5dffb764 100644 --- a/src/test/reporters/base.ts +++ b/src/test/reporters/base.ts @@ -277,7 +277,7 @@ function indent(lines: string, tab: string) { return lines.replace(/^(?=.+$)/gm, tab); } -function positionInFile(stackLines: string[], file: string): { column: number; line: number; } | undefined { +export function positionInFile(stackLines: string[], file: string): { column: number; line: number; } | undefined { // Stack will have /private/var/folders instead of /var/folders on Mac. file = fs.realpathSync(file); for (const line of stackLines) { diff --git a/src/test/reporters/json.ts b/src/test/reporters/json.ts index ae255d617adbc..beb3a978e4d30 100644 --- a/src/test/reporters/json.ts +++ b/src/test/reporters/json.ts @@ -17,6 +17,7 @@ import fs from 'fs'; import path from 'path'; import { FullConfig, TestCase, Suite, TestResult, TestError, TestStep, FullResult, TestStatus, Location, Reporter } from '../../../types/testReporter'; +import { positionInFile } from './base'; export interface JSONReport { config: Omit & { @@ -65,11 +66,18 @@ export interface JSONReportTestResult { status: TestStatus | undefined; duration: number; error: TestError | undefined; - stdout: JSONReportSTDIOEntry[], - stderr: JSONReportSTDIOEntry[], + stdout: JSONReportSTDIOEntry[]; + stderr: JSONReportSTDIOEntry[]; retry: number; steps?: JSONReportTestStep[]; - attachments: { name: string, path?: string, body?: string, contentType: string }[]; + attachments: { + name: string; + path?: string; + body?: string; + contentType: string; + }[]; + line?: number; + column?: number; } export interface JSONReportTestStep { title: string; @@ -216,13 +224,24 @@ class JSONReporter implements Reporter { annotations: test.annotations, expectedStatus: test.expectedStatus, projectName: test.titlePath()[1], - results: test.results.map(r => this._serializeTestResult(r)), + results: test.results.map(r => this._serializeTestResult(r, test.location.file)), status: test.outcome(), }; } - private _serializeTestResult(result: TestResult): JSONReportTestResult { + private _serializeTestResult(result: TestResult, file: string): JSONReportTestResult { const steps = result.steps.filter(s => s.category === 'test.step'); + let position: { column?: number; line?: number } = {}; + if (file && result.status === 'failed'){ + const lines = result.error?.stack?.split('\n') ?? []; + const firstStackLine = lines.findIndex(line => + line.startsWith(' at ') + ); + if (firstStackLine !== -1) { + const stackLines = lines.slice(firstStackLine); + position = positionInFile(stackLines, file) ?? {}; + } + } return { workerIndex: result.workerIndex, status: result.status, @@ -238,6 +257,7 @@ class JSONReporter implements Reporter { path: a.path, body: a.body?.toString('base64') })), + ...position }; } diff --git a/tests/playwright-test/json-reporter.spec.ts b/tests/playwright-test/json-reporter.spec.ts index da280f1c3c5e2..3b103b2ef8ae6 100644 --- a/tests/playwright-test/json-reporter.spec.ts +++ b/tests/playwright-test/json-reporter.spec.ts @@ -183,3 +183,20 @@ test('should have relative always-posix paths', async ({ runInlineTest }) => { expect(result.report.suites[0].specs[0].line).toBe(6); expect(result.report.suites[0].specs[0].column).toBe(7); }); + +test('should have error position in results', async ({ + runInlineTest, +}) => { + const result = await runInlineTest({ + 'a.test.js': ` + const { test } = pwt; + test('math works!', async ({}) => { + expect(1 + 1).toBe(3); + }); + `, + }); + expect(result.exitCode).toBe(1); + expect(result.report.suites[0].specs[0].file).toBe('a.test.js'); + expect(result.report.suites[0].specs[0].tests[0].results[0].line).toBe(7); + expect(result.report.suites[0].specs[0].tests[0].results[0].column).toBe(23); +}); From 0bcd092e4073796e86f3beaa78e7c8bce4b7e3a6 Mon Sep 17 00:00:00 2001 From: Sidharth Vinod Date: Sun, 26 Sep 2021 09:42:23 +0530 Subject: [PATCH 2/5] feat(reporters): Add error position to JSON Report --- src/test/reporters/json.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/reporters/json.ts b/src/test/reporters/json.ts index beb3a978e4d30..3ea964e345c8e 100644 --- a/src/test/reporters/json.ts +++ b/src/test/reporters/json.ts @@ -232,8 +232,8 @@ class JSONReporter implements Reporter { private _serializeTestResult(result: TestResult, file: string): JSONReportTestResult { const steps = result.steps.filter(s => s.category === 'test.step'); let position: { column?: number; line?: number } = {}; - if (file && result.status === 'failed'){ - const lines = result.error?.stack?.split('\n') ?? []; + if (file && result.error?.stack !== undefined) { + const lines = result.error.stack.split('\n') ?? []; const firstStackLine = lines.findIndex(line => line.startsWith(' at ') ); From 5a2205425afb2888188e623e5e22cacf793a826a Mon Sep 17 00:00:00 2001 From: Sidharth Vinod Date: Tue, 28 Sep 2021 10:49:43 +0530 Subject: [PATCH 3/5] chore(reporter): Add prepareErrorStack --- src/test/reporters/base.ts | 59 +++++++++++++++++++++++++++----------- src/test/reporters/json.ts | 19 ++++++------ 2 files changed, 51 insertions(+), 27 deletions(-) diff --git a/src/test/reporters/base.ts b/src/test/reporters/base.ts index f1f3c5dffb764..6945a02a9d650 100644 --- a/src/test/reporters/base.ts +++ b/src/test/reporters/base.ts @@ -26,6 +26,7 @@ import { FullConfig, TestCase, Suite, TestResult, TestError, Reporter, FullResul const stackUtils = new StackUtils(); type TestResultOutput = { chunk: string | Buffer, type: 'stdout' | 'stderr' }; +export type PositionInFile = { column: number; line: number }; const kOutputSymbol = Symbol('output'); export class BaseReporter implements Reporter { @@ -240,28 +241,22 @@ function formatTestHeader(config: FullConfig, test: TestCase, indent: string, in export function formatError(error: TestError, file?: string) { const stack = error.stack; - const tokens = []; + const tokens = ['']; if (stack) { - tokens.push(''); - const lines = stack.split('\n'); - let firstStackLine = lines.findIndex(line => line.startsWith(' at ')); - if (firstStackLine === -1) - firstStackLine = lines.length; - tokens.push(lines.slice(0, firstStackLine).join('\n')); - const stackLines = lines.slice(firstStackLine); - const position = file ? positionInFile(stackLines, file) : null; - if (position) { - const source = fs.readFileSync(file!, 'utf8'); + const { message, stackLines, codeFrame } = prepareErrorStack( + stack, + file + ); + tokens.push(message); + if (codeFrame) { tokens.push(''); - tokens.push(codeFrameColumns(source, { start: position }, { highlightCode: colors.enabled })); + tokens.push(codeFrame); } tokens.push(''); tokens.push(colors.dim(stackLines.join('\n'))); } else if (error.message) { - tokens.push(''); tokens.push(error.message); - } else { - tokens.push(''); + } else if (error.value) { tokens.push(error.value); } return tokens.join('\n'); @@ -277,7 +272,39 @@ function indent(lines: string, tab: string) { return lines.replace(/^(?=.+$)/gm, tab); } -export function positionInFile(stackLines: string[], file: string): { column: number; line: number; } | undefined { +export function prepareErrorStack( + stack: string, + file?: string +): { + message: string; + stackLines: string[]; + position?: PositionInFile; + codeFrame?: string; +} { + const lines = stack.split('\n'); + let firstStackLine = lines.findIndex(line => line.startsWith(' at ')); + if (firstStackLine === -1) firstStackLine = lines.length; + const message = lines.slice(0, firstStackLine).join('\n'); + const stackLines = lines.slice(firstStackLine); + const position = file ? positionInFile(stackLines, file) : undefined; + let codeFrame; + if (position) { + const source = fs.readFileSync(file!, 'utf8'); + codeFrame = codeFrameColumns( + source, + { start: position }, + { highlightCode: colors.enabled } + ); + } + return { + message, + stackLines, + position, + codeFrame, + }; +} + +function positionInFile(stackLines: string[], file: string): { column: number; line: number; } | undefined { // Stack will have /private/var/folders instead of /var/folders on Mac. file = fs.realpathSync(file); for (const line of stackLines) { diff --git a/src/test/reporters/json.ts b/src/test/reporters/json.ts index 3ea964e345c8e..b343e98663e15 100644 --- a/src/test/reporters/json.ts +++ b/src/test/reporters/json.ts @@ -17,7 +17,7 @@ import fs from 'fs'; import path from 'path'; import { FullConfig, TestCase, Suite, TestResult, TestError, TestStep, FullResult, TestStatus, Location, Reporter } from '../../../types/testReporter'; -import { positionInFile } from './base'; +import { PositionInFile, prepareErrorStack } from './base'; export interface JSONReport { config: Omit & { @@ -231,16 +231,13 @@ class JSONReporter implements Reporter { private _serializeTestResult(result: TestResult, file: string): JSONReportTestResult { const steps = result.steps.filter(s => s.category === 'test.step'); - let position: { column?: number; line?: number } = {}; - if (file && result.error?.stack !== undefined) { - const lines = result.error.stack.split('\n') ?? []; - const firstStackLine = lines.findIndex(line => - line.startsWith(' at ') + let errorPosition: PositionInFile | undefined; + if (result.error?.stack){ + const { position } = prepareErrorStack( + result.error.stack, + file ); - if (firstStackLine !== -1) { - const stackLines = lines.slice(firstStackLine); - position = positionInFile(stackLines, file) ?? {}; - } + errorPosition = position; } return { workerIndex: result.workerIndex, @@ -257,7 +254,7 @@ class JSONReporter implements Reporter { path: a.path, body: a.body?.toString('base64') })), - ...position + ...errorPosition }; } From 1864c8d344864051ce1cb05bf63ba25595b82a19 Mon Sep 17 00:00:00 2001 From: Sidharth Vinod Date: Thu, 30 Sep 2021 08:27:15 +0530 Subject: [PATCH 4/5] chore(reporter): Separate codeframe generation --- src/test/reporters/base.ts | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/src/test/reporters/base.ts b/src/test/reporters/base.ts index 6945a02a9d650..686f48f45ce5a 100644 --- a/src/test/reporters/base.ts +++ b/src/test/reporters/base.ts @@ -243,11 +243,13 @@ export function formatError(error: TestError, file?: string) { const stack = error.stack; const tokens = ['']; if (stack) { - const { message, stackLines, codeFrame } = prepareErrorStack( + const { message, stackLines, position } = prepareErrorStack( stack, file ); tokens.push(message); + + const codeFrame = generateCodeFrame(file, position); if (codeFrame) { tokens.push(''); tokens.push(codeFrame); @@ -272,14 +274,24 @@ function indent(lines: string, tab: string) { return lines.replace(/^(?=.+$)/gm, tab); } -export function prepareErrorStack( - stack: string, - file?: string -): { +function generateCodeFrame(file?: string, position?: PositionInFile): string | undefined { + if (!position || !file) + return; + + const source = fs.readFileSync(file!, 'utf8'); + const codeFrame = codeFrameColumns( + source, + { start: position }, + { highlightCode: colors.enabled } + ); + + return codeFrame; +} + +export function prepareErrorStack(stack: string, file?: string): { message: string; stackLines: string[]; position?: PositionInFile; - codeFrame?: string; } { const lines = stack.split('\n'); let firstStackLine = lines.findIndex(line => line.startsWith(' at ')); @@ -287,20 +299,10 @@ export function prepareErrorStack( const message = lines.slice(0, firstStackLine).join('\n'); const stackLines = lines.slice(firstStackLine); const position = file ? positionInFile(stackLines, file) : undefined; - let codeFrame; - if (position) { - const source = fs.readFileSync(file!, 'utf8'); - codeFrame = codeFrameColumns( - source, - { start: position }, - { highlightCode: colors.enabled } - ); - } return { message, stackLines, position, - codeFrame, }; } From 32d1de94fe45f031fc9cda5da4c46e589d28f501 Mon Sep 17 00:00:00 2001 From: Sidharth Vinod Date: Thu, 30 Sep 2021 08:33:58 +0530 Subject: [PATCH 5/5] feat(json-reporter): Add errorLocation. --- src/test/reporters/json.ts | 23 ++++++++++----------- tests/playwright-test/json-reporter.spec.ts | 4 ++-- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/test/reporters/json.ts b/src/test/reporters/json.ts index b343e98663e15..69baddbdc925f 100644 --- a/src/test/reporters/json.ts +++ b/src/test/reporters/json.ts @@ -76,8 +76,7 @@ export interface JSONReportTestResult { body?: string; contentType: string; }[]; - line?: number; - column?: number; + errorLocation?: PositionInFile } export interface JSONReportTestStep { title: string; @@ -231,15 +230,7 @@ class JSONReporter implements Reporter { private _serializeTestResult(result: TestResult, file: string): JSONReportTestResult { const steps = result.steps.filter(s => s.category === 'test.step'); - let errorPosition: PositionInFile | undefined; - if (result.error?.stack){ - const { position } = prepareErrorStack( - result.error.stack, - file - ); - errorPosition = position; - } - return { + const jsonResult: JSONReportTestResult = { workerIndex: result.workerIndex, status: result.status, duration: result.duration, @@ -254,8 +245,16 @@ class JSONReporter implements Reporter { path: a.path, body: a.body?.toString('base64') })), - ...errorPosition }; + if (result.error?.stack) { + const { position } = prepareErrorStack( + result.error.stack, + file + ); + if (position) + jsonResult.errorLocation = position; + } + return jsonResult; } private _serializeTestStep(step: TestStep): JSONReportTestStep { diff --git a/tests/playwright-test/json-reporter.spec.ts b/tests/playwright-test/json-reporter.spec.ts index 3b103b2ef8ae6..f9b0d5714bd8b 100644 --- a/tests/playwright-test/json-reporter.spec.ts +++ b/tests/playwright-test/json-reporter.spec.ts @@ -197,6 +197,6 @@ test('should have error position in results', async ({ }); expect(result.exitCode).toBe(1); expect(result.report.suites[0].specs[0].file).toBe('a.test.js'); - expect(result.report.suites[0].specs[0].tests[0].results[0].line).toBe(7); - expect(result.report.suites[0].specs[0].tests[0].results[0].column).toBe(23); + expect(result.report.suites[0].specs[0].tests[0].results[0].errorLocation.line).toBe(7); + expect(result.report.suites[0].specs[0].tests[0].results[0].errorLocation.column).toBe(23); });