From dc61c0d05bc64c66ef6ad72c2c150926cb65320c Mon Sep 17 00:00:00 2001
From: Sidharth Vinod <sidharthv96@gmail.com>
Date: Sun, 26 Sep 2021 18:48:42 +0530
Subject: [PATCH 01/21] feat(reporter): Add github actions reporter

---
 src/test/reporters/base.ts   |  19 +-
 src/test/reporters/github.ts | 349 +++++++++++++++++++++++++++++++++++
 2 files changed, 360 insertions(+), 8 deletions(-)
 create mode 100644 src/test/reporters/github.ts

diff --git a/src/test/reporters/base.ts b/src/test/reporters/base.ts
index 18e6ce5d0aefe..0d243fb25e9ea 100644
--- a/src/test/reporters/base.ts
+++ b/src/test/reporters/base.ts
@@ -25,9 +25,12 @@ import { FullConfig, TestCase, Suite, TestResult, TestError, Reporter, FullResul
 
 const stackUtils = new StackUtils();
 
-type TestResultOutput = { chunk: string | Buffer, type: 'stdout' | 'stderr' };
-const kOutputSymbol = Symbol('output');
-
+export type TestResultOutput = { chunk: string | Buffer, type: 'stdout' | 'stderr' };
+export const kOutputSymbol = Symbol('output');
+export interface Position {
+  column: number;
+  line: number;
+}
 export class BaseReporter implements Reporter  {
   duration = 0;
   config!: FullConfig;
@@ -215,11 +218,11 @@ export function formatResultFailure(test: TestCase, result: TestResult, initialI
   return resultTokens;
 }
 
-function relativeTestPath(config: FullConfig, test: TestCase): string {
+export function relativeTestPath(config: FullConfig, test: TestCase): string {
   return path.relative(config.rootDir, test.location.file) || path.basename(test.location.file);
 }
 
-function stepSuffix(step: TestStep | undefined) {
+export function stepSuffix(step: TestStep | undefined) {
   const stepTitles = step ? step.titlePath() : [];
   return stepTitles.map(t => ' › ' + t).join('');
 }
@@ -232,7 +235,7 @@ export function formatTestTitle(config: FullConfig, test: TestCase, step?: TestS
   return `${projectTitle}${location} › ${titles.join(' ')}${stepSuffix(step)}`;
 }
 
-function formatTestHeader(config: FullConfig, test: TestCase, indent: string, index?: number): string {
+export function formatTestHeader(config: FullConfig, test: TestCase, indent: string, index?: number): string {
   const title = formatTestTitle(config, test);
   const header = `${indent}${index ? index + ') ' : ''}${title}`;
   return pad(header, '=');
@@ -273,11 +276,11 @@ function pad(line: string, char: string): string {
   return line + colors.gray(char.repeat(Math.max(0, 100 - line.length)));
 }
 
-function indent(lines: string, tab: string) {
+export function indent(lines: string, tab: string) {
   return lines.replace(/^(?=.+$)/gm, tab);
 }
 
-function positionInFile(stackLines: string[], file: string): { column: number; line: number; } | undefined {
+export function positionInFile(stackLines: string[], file: string): Position | undefined {
   // Stack will have /private/var/folders instead of /var/folders on Mac.
   file = fs.realpathSync(file);
   for (const line of stackLines) {
diff --git a/src/test/reporters/github.ts b/src/test/reporters/github.ts
new file mode 100644
index 0000000000000..0922af15aedd3
--- /dev/null
+++ b/src/test/reporters/github.ts
@@ -0,0 +1,349 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { codeFrameColumns } from '@babel/code-frame';
+import fs from 'fs';
+// @ts-ignore
+import milliseconds from 'ms';
+import path from 'path';
+import {
+  FullConfig,
+  TestCase,
+  TestResult,
+  TestError,
+  FullResult,
+  TestStep,
+} from '../../../types/testReporter';
+import {
+  relativeTestPath,
+  Position,
+  stripAnsiEscapes,
+  indent,
+  stepSuffix,
+  positionInFile,
+  formatTestHeader,
+  BaseReporter,
+  kOutputSymbol,
+  TestResultOutput,
+} from './base';
+
+type GithubLogType = 'debug' | 'notice' | 'warning' | 'error';
+
+type GithubLogOptions = Partial<{
+  title: string;
+  file: string;
+  col: number;
+  endColumn: number;
+  line: number;
+  endLine: number;
+}>;
+
+class GithubLogger {
+	isCI: boolean = process.env.CI === 'true';
+  isGithubAction: boolean = process.env.GITHUB_ACTION !== undefined;
+  shouldLog = (this.isCI && this.isGithubAction) || process.env.PW_GH_ACTION_DEBUG === 'true' ;
+
+  log(
+    message: string,
+    type: GithubLogType = 'notice',
+    options: GithubLogOptions = {}
+  ) {
+    if (this.shouldLog) {
+      if (this.isGithubAction) message = message.replace(/\n/g, '%0A');
+
+      const configs = Object.entries(options)
+          .map(([key, option]) => `${key}=${option}`)
+          .join(',');
+      console.log(`::${type} ${configs}::${message}`);
+    }
+  }
+
+  debug(message: string, options?: GithubLogOptions) {
+    this.log(message, 'debug', options);
+  }
+
+  error(message: string, options?: GithubLogOptions) {
+    this.log(message, 'error', options);
+  }
+
+  notice(message: string, options?: GithubLogOptions) {
+    this.log(message, 'notice', options);
+  }
+
+  warning(message: string, options?: GithubLogOptions) {
+    this.log(message, 'warning', options);
+  }
+}
+
+
+interface Annotation {
+  filePath: string;
+  title: string;
+  message: string;
+  position?: Position;
+}
+
+interface FailureDetails {
+  position?: Position;
+  tokens: string[];
+}
+
+interface ErrorDetails {
+  position?: Position;
+  message: string;
+}
+
+export class GithubReporter extends BaseReporter {
+  githubLogger = new GithubLogger();
+
+  override async onEnd(result: FullResult) {
+    super.onEnd(result);
+    this.epilogue(true);
+  }
+
+  private _printSlowTestAnnotations() {
+    if (!this.config.reportSlowTests) return;
+    const fileDurations = [...this.fileDurations.entries()];
+    fileDurations.sort((a, b) => b[1] - a[1]);
+    const count = Math.min(
+        fileDurations.length,
+        this.config.reportSlowTests.max || Number.POSITIVE_INFINITY
+    );
+    for (let i = 0; i < count; ++i) {
+      const duration = fileDurations[i][1];
+      if (duration <= this.config.reportSlowTests.threshold) break;
+      const filePath = workspaceRelativePath(
+          path.join(process.cwd(), fileDurations[i][0])
+      );
+      this.githubLogger.warning(`${filePath} (${milliseconds(duration)})`, {
+        title: 'Slow Test',
+        file: filePath,
+      });
+    }
+  }
+
+  override epilogue(full: boolean) {
+    let skipped = 0;
+    let expected = 0;
+    const skippedWithError: TestCase[] = [];
+    const unexpected: TestCase[] = [];
+    const flaky: TestCase[] = [];
+
+    this.suite.allTests().forEach(test => {
+      switch (test.outcome()) {
+        case 'skipped': {
+          ++skipped;
+          if (test.results.some(result => !!result.error))
+            skippedWithError.push(test);
+          break;
+        }
+        case 'expected':
+          ++expected;
+          break;
+        case 'unexpected':
+          unexpected.push(test);
+          break;
+        case 'flaky':
+          flaky.push(test);
+          break;
+      }
+    });
+
+    const noticeLines: string[] = [];
+    noticeLines.push('');
+    if (unexpected.length) {
+      noticeLines.push(`  ${unexpected.length} failed`);
+      for (const test of unexpected)
+        noticeLines.push(formatTestHeader(this.config, test, '    '));
+    }
+    if (flaky.length) {
+      noticeLines.push(`  ${flaky.length} flaky`);
+      for (const test of flaky)
+        noticeLines.push(formatTestHeader(this.config, test, '    '));
+    }
+    if (skipped) noticeLines.push(`  ${skipped} skipped`);
+    if (expected) {
+      noticeLines.push(
+          `  ${expected} passed` + ` (${milliseconds(this.duration)})`
+      );
+    }
+    if (this.result.status === 'timedout') {
+      noticeLines.push(
+          `  Timed out waiting ${
+            this.config.globalTimeout / 1000
+          }s for the entire test run`
+      );
+    }
+
+    this.githubLogger.notice(noticeLines.join('\n'), {
+      title: '🎭 Playwright Run Summary',
+    });
+
+    const failuresToPrint = [...unexpected, ...flaky, ...skippedWithError];
+    if (full && failuresToPrint.length)
+      this._printFailureAnnotations(failuresToPrint);
+
+    this._printSlowTestAnnotations();
+  }
+
+  private _printFailureAnnotations(failures: TestCase[]) {
+    failures.forEach((test, index) => {
+      const annotations = formatFailure(this.config, test, index + 1, true);
+      annotations.forEach(({ filePath, title, message, position }) => {
+        const options: GithubLogOptions = {
+          file: filePath,
+          title,
+        };
+        if (position) {
+          options.line = position.line;
+          options.col = position.column;
+        }
+        this.githubLogger.error(message, options);
+      });
+    });
+  }
+}
+
+function workspaceRelativePath(filePath: string): string {
+  return path.relative(process.env['GITHUB_WORKSPACE'] ?? '', filePath);
+}
+
+export function formatFailure(
+  config: FullConfig,
+  test: TestCase,
+  index?: number,
+  stdio?: boolean
+): Annotation[] {
+  const title = formatTestTitle(config, test);
+  const filePath = workspaceRelativePath(test.location.file);
+  const annotations: Annotation[] = [];
+  for (const result of test.results) {
+    const lines: string[] = [];
+    lines.push(formatTestHeader(config, test, '  ', index));
+    const failureDetails = formatResultFailure(test, result, '    ');
+    const resultTokens = failureDetails.tokens;
+    const position = failureDetails.position;
+    if (!resultTokens.length) continue;
+    if (result.retry) {
+      lines.push('');
+      lines.push(`    Retry #${result.retry}`);
+    }
+    lines.push(...resultTokens);
+
+    const output = ((result as any)[kOutputSymbol] || []) as TestResultOutput[];
+    if (stdio && output.length) {
+      const outputText = output
+          .map(({ chunk, type }) => {
+            const text = chunk.toString('utf8');
+            if (type === 'stderr') return stripAnsiEscapes(text);
+            return text;
+          })
+          .join('');
+      lines.push('');
+      lines.push('--- Test output ---' + '\n\n' + outputText + '\n');
+    }
+
+    lines.push('');
+    annotations.push({
+      filePath,
+      position,
+      title,
+      message: lines.join('\n'),
+    });
+  }
+
+  return annotations;
+}
+
+export function formatResultFailure(
+  test: TestCase,
+  result: TestResult,
+  initialIndent: string
+): FailureDetails {
+  const resultTokens: string[] = [];
+  if (result.status === 'timedOut') {
+    resultTokens.push('');
+    resultTokens.push(
+        indent(`Timeout of ${test.timeout}ms exceeded.`, initialIndent)
+    );
+  }
+  if (result.status === 'passed' && test.expectedStatus === 'failed') {
+    resultTokens.push('');
+    resultTokens.push(indent(`Expected to fail, but passed.`, initialIndent));
+  }
+  let error: ErrorDetails | undefined = undefined;
+  if (result.error !== undefined) {
+    error = formatError(result.error, test.location.file);
+    resultTokens.push(indent(error.message, initialIndent));
+  }
+  return {
+    tokens: resultTokens,
+    position: error?.position,
+  };
+}
+
+export function formatTestTitle(
+  config: FullConfig,
+  test: TestCase,
+  step?: TestStep
+): string {
+  // root, project, file, ...describes, test
+  const [, projectName, , ...titles] = test.titlePath();
+  const location = `${relativeTestPath(config, test)}:${test.location.line}:${
+    test.location.column
+  }`;
+  const projectTitle = projectName ? `[${projectName}] › ` : '';
+  return `${projectTitle}${location} › ${titles.join(' ')}${stepSuffix(step)}`;
+}
+
+
+export function formatError(error: TestError, file?: string): ErrorDetails {
+  const stack = error.stack;
+  const tokens = [''];
+  let position: Position | undefined;
+
+  if (stack) {
+    const lines = stack.split('\n');
+    let firstStackLine = lines.findIndex(line => line.startsWith('    at '));
+    if (firstStackLine === -1) firstStackLine = lines.length;
+    tokens.push(lines.slice(0, firstStackLine).join('\n'));
+    const stackLines = lines.slice(firstStackLine);
+    position = file ? positionInFile(stackLines, file) : undefined;
+    if (position) {
+      const source = fs.readFileSync(file!, 'utf8');
+      tokens.push('');
+      tokens.push(
+          codeFrameColumns(
+              source,
+              { start: position },
+              { highlightCode: false }
+          )
+      );
+    }
+    tokens.push('');
+    tokens.push(stackLines.join('\n'));
+  } else if (error.message) {
+    tokens.push(error.message);
+  } else if (error.value) {
+    tokens.push(error.value);
+  }
+  return {
+    position,
+    message: tokens.join('\n'),
+  };
+}
+
+export default GithubReporter;

From 4accc8115861fdee76f5dca27e0a98a0a2d56c55 Mon Sep 17 00:00:00 2001
From: Sidharth Vinod <sidharthv96@gmail.com>
Date: Sun, 26 Sep 2021 19:01:36 +0530
Subject: [PATCH 02/21] chore: Remove unnecessary exports

---
 src/test/reporters/github.ts | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/src/test/reporters/github.ts b/src/test/reporters/github.ts
index 0922af15aedd3..b4edc5d7d5539 100644
--- a/src/test/reporters/github.ts
+++ b/src/test/reporters/github.ts
@@ -88,7 +88,6 @@ class GithubLogger {
   }
 }
 
-
 interface Annotation {
   filePath: string;
   title: string;
@@ -221,7 +220,7 @@ function workspaceRelativePath(filePath: string): string {
   return path.relative(process.env['GITHUB_WORKSPACE'] ?? '', filePath);
 }
 
-export function formatFailure(
+function formatFailure(
   config: FullConfig,
   test: TestCase,
   index?: number,
@@ -268,7 +267,7 @@ export function formatFailure(
   return annotations;
 }
 
-export function formatResultFailure(
+function formatResultFailure(
   test: TestCase,
   result: TestResult,
   initialIndent: string
@@ -295,7 +294,7 @@ export function formatResultFailure(
   };
 }
 
-export function formatTestTitle(
+function formatTestTitle(
   config: FullConfig,
   test: TestCase,
   step?: TestStep
@@ -309,8 +308,7 @@ export function formatTestTitle(
   return `${projectTitle}${location} › ${titles.join(' ')}${stepSuffix(step)}`;
 }
 
-
-export function formatError(error: TestError, file?: string): ErrorDetails {
+function formatError(error: TestError, file?: string): ErrorDetails {
   const stack = error.stack;
   const tokens = [''];
   let position: Position | undefined;

From edeed6fa68a0977faf09e2a6c712b1163d79c33b Mon Sep 17 00:00:00 2001
From: Sidharth Vinod <sidharthv96@gmail.com>
Date: Tue, 28 Sep 2021 09:59:40 +0530
Subject: [PATCH 03/21] chore(reporter): Change interface to type

---
 src/test/reporters/github.ts | 33 ++++++++++++++++-----------------
 1 file changed, 16 insertions(+), 17 deletions(-)

diff --git a/src/test/reporters/github.ts b/src/test/reporters/github.ts
index b4edc5d7d5539..6bb0a01b7406c 100644
--- a/src/test/reporters/github.ts
+++ b/src/test/reporters/github.ts
@@ -51,6 +51,22 @@ type GithubLogOptions = Partial<{
   endLine: number;
 }>;
 
+type Annotation = {
+  filePath: string;
+  title: string;
+  message: string;
+  position?: Position;
+};
+
+type FailureDetails = {
+  tokens: string[];
+  position?: Position;
+};
+
+type ErrorDetails = {
+  message: string;
+  position?: Position;
+};
 class GithubLogger {
 	isCI: boolean = process.env.CI === 'true';
   isGithubAction: boolean = process.env.GITHUB_ACTION !== undefined;
@@ -88,23 +104,6 @@ class GithubLogger {
   }
 }
 
-interface Annotation {
-  filePath: string;
-  title: string;
-  message: string;
-  position?: Position;
-}
-
-interface FailureDetails {
-  position?: Position;
-  tokens: string[];
-}
-
-interface ErrorDetails {
-  position?: Position;
-  message: string;
-}
-
 export class GithubReporter extends BaseReporter {
   githubLogger = new GithubLogger();
 

From de95d3167eaa81629e53539fcf9c744a000f879a Mon Sep 17 00:00:00 2001
From: Sidharth Vinod <sidharthv96@gmail.com>
Date: Tue, 28 Sep 2021 10:24:03 +0530
Subject: [PATCH 04/21] chore(github-reporter): Make members private

---
 src/test/reporters/github.ts | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/test/reporters/github.ts b/src/test/reporters/github.ts
index 6bb0a01b7406c..c4d334970b46c 100644
--- a/src/test/reporters/github.ts
+++ b/src/test/reporters/github.ts
@@ -68,17 +68,17 @@ type ErrorDetails = {
   position?: Position;
 };
 class GithubLogger {
-	isCI: boolean = process.env.CI === 'true';
-  isGithubAction: boolean = process.env.GITHUB_ACTION !== undefined;
-  shouldLog = (this.isCI && this.isGithubAction) || process.env.PW_GH_ACTION_DEBUG === 'true' ;
+	private _isCI: boolean = process.env.CI === 'true';
+  private _isGithubAction: boolean = process.env.GITHUB_ACTION !== undefined;
+  private _shouldLog = (this._isCI && this._isGithubAction) || process.env.PW_GH_ACTION_DEBUG === 'true' ;
 
-  log(
+  private _log(
     message: string,
     type: GithubLogType = 'notice',
     options: GithubLogOptions = {}
   ) {
-    if (this.shouldLog) {
-      if (this.isGithubAction) message = message.replace(/\n/g, '%0A');
+    if (this._shouldLog) {
+      if (this._isGithubAction) message = message.replace(/\n/g, '%0A');
 
       const configs = Object.entries(options)
           .map(([key, option]) => `${key}=${option}`)
@@ -88,19 +88,19 @@ class GithubLogger {
   }
 
   debug(message: string, options?: GithubLogOptions) {
-    this.log(message, 'debug', options);
+    this._log(message, 'debug', options);
   }
 
   error(message: string, options?: GithubLogOptions) {
-    this.log(message, 'error', options);
+    this._log(message, 'error', options);
   }
 
   notice(message: string, options?: GithubLogOptions) {
-    this.log(message, 'notice', options);
+    this._log(message, 'notice', options);
   }
 
   warning(message: string, options?: GithubLogOptions) {
-    this.log(message, 'warning', options);
+    this._log(message, 'warning', options);
   }
 }
 

From 8bc0a50db97ac28ec3cad289655123f5bb54969b Mon Sep 17 00:00:00 2001
From: Sidharth Vinod <sidharthv96@gmail.com>
Date: Tue, 28 Sep 2021 12:13:03 +0530
Subject: [PATCH 05/21] feat(reporters): Add github action reporter

---
 src/test/reporters/base.ts   | 227 ++++++++++++++++++++++----------
 src/test/reporters/github.ts | 246 ++---------------------------------
 src/test/reporters/junit.ts  |   2 +-
 src/test/reporters/line.ts   |   2 +-
 4 files changed, 169 insertions(+), 308 deletions(-)

diff --git a/src/test/reporters/base.ts b/src/test/reporters/base.ts
index 75921e97f1f56..8f0bddfa433e7 100644
--- a/src/test/reporters/base.ts
+++ b/src/test/reporters/base.ts
@@ -27,10 +27,38 @@ const stackUtils = new StackUtils();
 
 export type TestResultOutput = { chunk: string | Buffer, type: 'stdout' | 'stderr' };
 export const kOutputSymbol = Symbol('output');
-export interface Position {
+
+export type Position = {
   column: number;
   line: number;
-}
+};
+
+
+type Annotation = {
+  filePath: string;
+  title: string;
+  message: string;
+  position?: Position;
+};
+
+type FailureDetails = {
+  tokens: string[];
+  position?: Position;
+};
+
+type ErrorDetails = {
+  message: string;
+  position?: Position;
+};
+
+type TestSummary = {
+  skipped: number;
+  expected: number;
+  skippedWithError: TestCase[];
+  unexpected: TestCase[];
+  flaky: TestCase[];
+};
+
 export class BaseReporter implements Reporter  {
   duration = 0;
   config!: FullConfig;
@@ -78,18 +106,48 @@ export class BaseReporter implements Reporter  {
     this.result = result;
   }
 
-  private _printSlowTests() {
+  protected getSlowTests(): [string, number][] {
     if (!this.config.reportSlowTests)
-      return;
+      return [];
     const fileDurations = [...this.fileDurations.entries()];
     fileDurations.sort((a, b) => b[1] - a[1]);
     const count = Math.min(fileDurations.length, this.config.reportSlowTests.max || Number.POSITIVE_INFINITY);
-    for (let i = 0; i < count; ++i) {
-      const duration = fileDurations[i][1];
-      if (duration <= this.config.reportSlowTests.threshold)
-        break;
-      console.log(colors.yellow('  Slow test: ') + fileDurations[i][0] + colors.yellow(` (${milliseconds(duration)})`));
+    const threshold =  this.config.reportSlowTests.threshold;
+    return fileDurations.filter(([,duration]) => duration > threshold).slice(0, count);
+  }
+
+  protected printSlowTests() {
+    this.getSlowTests().forEach(([file, duration]) => {
+      console.log(colors.yellow('  Slow test: ') + file + colors.yellow(` (${milliseconds(duration)})`));
+    });
+  }
+
+  protected generateSummaryMessage({
+    skipped,
+    expected,
+    unexpected,
+    flaky
+  }: TestSummary) {
+    const tokens: string[] = [];
+    tokens.push('');
+    if (unexpected.length) {
+      tokens.push(colors.red(`  ${unexpected.length} failed`));
+      for (const test of unexpected)
+        tokens.push(colors.red(formatTestHeader(this.config, test, '    ')));
     }
+    if (flaky.length) {
+      tokens.push(colors.yellow(`  ${flaky.length} flaky`));
+      for (const test of flaky)
+        tokens.push(colors.yellow(formatTestHeader(this.config, test, '    ')));
+    }
+    if (skipped)
+      tokens.push(colors.yellow(`  ${skipped} skipped`));
+    if (expected)
+      tokens.push(colors.green(`  ${expected} passed`) + colors.dim(` (${milliseconds(this.duration)})`));
+    if (this.result.status === 'timedout')
+      tokens.push(colors.red(`  Timed out waiting ${this.config.globalTimeout / 1000}s for the entire test run`));
+
+    return tokens.join('\n');
   }
 
   epilogue(full: boolean) {
@@ -114,35 +172,28 @@ export class BaseReporter implements Reporter  {
     });
 
     const failuresToPrint = [...unexpected, ...flaky, ...skippedWithError];
-    if (full && failuresToPrint.length) {
-      console.log('');
-      this._printFailures(failuresToPrint);
-    }
-
-    this._printSlowTests();
+    if (full && failuresToPrint.length)
+      this.printFailures(failuresToPrint);
+    this.printSlowTests();
+    const summaryMessage = this.generateSummaryMessage({
+      skipped,
+      expected,
+      skippedWithError,
+      unexpected,
+      flaky
+    });
+    this.printSummary(summaryMessage);
+  }
 
+  protected printSummary(summary: string){
     console.log('');
-    if (unexpected.length) {
-      console.log(colors.red(`  ${unexpected.length} failed`));
-      for (const test of unexpected)
-        console.log(colors.red(formatTestHeader(this.config, test, '    ')));
-    }
-    if (flaky.length) {
-      console.log(colors.yellow(`  ${flaky.length} flaky`));
-      for (const test of flaky)
-        console.log(colors.yellow(formatTestHeader(this.config, test, '    ')));
-    }
-    if (skipped)
-      console.log(colors.yellow(`  ${skipped} skipped`));
-    if (expected)
-      console.log(colors.green(`  ${expected} passed`) + colors.dim(` (${milliseconds(this.duration)})`));
-    if (this.result.status === 'timedout')
-      console.log(colors.red(`  Timed out waiting ${this.config.globalTimeout / 1000}s for the entire test run`));
+    console.log(summary);
   }
 
-  private _printFailures(failures: TestCase[]) {
+  protected printFailures(failures: TestCase[]) {
+    console.log('');
     failures.forEach((test, index) => {
-      console.log(formatFailure(this.config, test, index + 1, this.printTestOutput));
+      console.log(formatFailure(this.config, test, index + 1, this.printTestOutput).message);
     });
   }
 
@@ -151,41 +202,56 @@ export class BaseReporter implements Reporter  {
   }
 }
 
-export function formatFailure(config: FullConfig, test: TestCase, index?: number, stdio?: boolean): string {
+export function workspaceRelativePath(filePath: string): string {
+  return path.relative(process.env['GITHUB_WORKSPACE'] ?? '', filePath);
+}
+
+export function formatFailure(config: FullConfig, test: TestCase, index?: number, stdio?: boolean, attachment = true): {
+  message: string,
+  annotations: Annotation[]
+} {
   const lines: string[] = [];
-  lines.push(colors.red(formatTestHeader(config, test, '  ', index)));
+  const title = formatTestTitle(config, test);
+  const filePath = workspaceRelativePath(test.location.file);
+  const annotations: Annotation[] = [];
+  const header = formatTestHeader(config, test, '  ', index);
+  lines.push(colors.red(header));
   for (const result of test.results) {
-    const resultTokens = formatResultFailure(test, result, '    ');
+    const resultLines: string[] = [];
+
+    const { tokens: resultTokens, position } = formatResultFailure(test, result, '    ');
     if (!resultTokens.length)
       continue;
     if (result.retry) {
-      lines.push('');
-      lines.push(colors.gray(pad(`    Retry #${result.retry}`, '-')));
+      resultLines.push('');
+      resultLines.push(colors.gray(pad(`    Retry #${result.retry}`, '-')));
     }
-    lines.push(...resultTokens);
-    for (let i = 0; i < result.attachments.length; ++i) {
-      const attachment = result.attachments[i];
-      lines.push('');
-      lines.push(colors.cyan(pad(`    attachment #${i + 1}: ${attachment.name} (${attachment.contentType})`, '-')));
-      if (attachment.path) {
-        const relativePath = path.relative(process.cwd(), attachment.path);
-        lines.push(colors.cyan(`    ${relativePath}`));
-        // Make this extensible
-        if (attachment.name === 'trace') {
-          lines.push(colors.cyan(`    Usage:`));
-          lines.push('');
-          lines.push(colors.cyan(`        npx playwright show-trace ${relativePath}`));
-          lines.push('');
-        }
-      } else {
-        if (attachment.contentType.startsWith('text/')) {
-          let text = attachment.body!.toString();
-          if (text.length > 300)
-            text = text.slice(0, 300) + '...';
-          lines.push(colors.cyan(`    ${text}`));
+    resultLines.push(...resultTokens);
+    if (attachment){
+      for (let i = 0; i < result.attachments.length; ++i) {
+        const attachment = result.attachments[i];
+        resultLines.push('');
+        resultLines.push(colors.cyan(pad(`    attachment #${i + 1}: ${attachment.name} (${attachment.contentType})`, '-')));
+        if (attachment.path) {
+          const relativePath = path.relative(process.cwd(), attachment.path);
+          resultLines.push(colors.cyan(`    ${relativePath}`));
+          // Make this extensible
+          if (attachment.name === 'trace') {
+            resultLines.push(colors.cyan(`    Usage:`));
+            resultLines.push('');
+            resultLines.push(colors.cyan(`        npx playwright show-trace ${relativePath}`));
+            resultLines.push('');
+          }
+        } else {
+          if (attachment.contentType.startsWith('text/')) {
+            let text = attachment.body!.toString();
+            if (text.length > 300)
+              text = text.slice(0, 300) + '...';
+            resultLines.push(colors.cyan(`    ${text}`));
+          }
         }
+        resultLines.push(colors.cyan(pad('   ', '-')));
       }
-      lines.push(colors.cyan(pad('   ', '-')));
     }
     const output = ((result as any)[kOutputSymbol] || []) as TestResultOutput[];
     if (stdio && output.length) {
@@ -195,15 +261,25 @@ export function formatFailure(config: FullConfig, test: TestCase, index?: number
           return colors.red(stripAnsiEscapes(text));
         return text;
       }).join('');
-      lines.push('');
-      lines.push(colors.gray(pad('--- Test output', '-')) + '\n\n' + outputText + '\n' + pad('', '-'));
+      resultLines.push('');
+      resultLines.push(colors.gray(pad('--- Test output', '-')) + '\n\n' + outputText + '\n' + pad('', '-'));
     }
+    annotations.push({
+      filePath,
+      position,
+      title,
+      message: [header, ...resultLines].join('\n'),
+    });
+    lines.push(...resultLines);
   }
   lines.push('');
-  return lines.join('\n');
+  return {
+    message: lines.join('\n'),
+    annotations
+  };
 }
 
-export function formatResultFailure(test: TestCase, result: TestResult, initialIndent: string): string[] {
+export function formatResultFailure(test: TestCase, result: TestResult, initialIndent: string): FailureDetails {
   const resultTokens: string[] = [];
   if (result.status === 'timedOut') {
     resultTokens.push('');
@@ -213,9 +289,15 @@ export function formatResultFailure(test: TestCase, result: TestResult, initialI
     resultTokens.push('');
     resultTokens.push(indent(colors.red(`Expected to fail, but passed.`), initialIndent));
   }
-  if (result.error !== undefined)
-    resultTokens.push(indent(formatError(result.error, test.location.file), initialIndent));
-  return resultTokens;
+  let error: ErrorDetails | undefined = undefined;
+  if (result.error !== undefined) {
+    error = formatError(result.error, test.location.file);
+    resultTokens.push(indent(error.message, initialIndent));
+  }
+  return {
+    tokens: resultTokens,
+    position: error?.position,
+  };
 }
 
 export function relativeTestPath(config: FullConfig, test: TestCase): string {
@@ -241,9 +323,11 @@ export function formatTestHeader(config: FullConfig, test: TestCase, indent: str
   return pad(header, '=');
 }
 
-export function formatError(error: TestError, file?: string) {
+export function formatError(error: TestError, file?: string): ErrorDetails {
   const stack = error.stack;
   const tokens = [];
+  let position: Position | undefined;
+
   if (stack) {
     tokens.push('');
     const lines = stack.split('\n');
@@ -252,7 +336,7 @@ export function formatError(error: TestError, file?: string) {
       firstStackLine = lines.length;
     tokens.push(lines.slice(0, firstStackLine).join('\n'));
     const stackLines = lines.slice(firstStackLine);
-    const position = file ? positionInFile(stackLines, file) : null;
+    position = file ? positionInFile(stackLines, file) : undefined;
     if (position) {
       const source = fs.readFileSync(file!, 'utf8');
       tokens.push('');
@@ -267,7 +351,10 @@ export function formatError(error: TestError, file?: string) {
     tokens.push('');
     tokens.push(error.value);
   }
-  return tokens.join('\n');
+  return {
+    position,
+    message: tokens.join('\n'),
+  };
 }
 
 function pad(line: string, char: string): string {
diff --git a/src/test/reporters/github.ts b/src/test/reporters/github.ts
index c4d334970b46c..98aa54f9d0079 100644
--- a/src/test/reporters/github.ts
+++ b/src/test/reporters/github.ts
@@ -14,31 +14,14 @@
  * limitations under the License.
  */
 
-import { codeFrameColumns } from '@babel/code-frame';
-import fs from 'fs';
 // @ts-ignore
 import milliseconds from 'ms';
 import path from 'path';
+import { BaseReporter, formatFailure, workspaceRelativePath } from './base';
 import {
-  FullConfig,
   TestCase,
-  TestResult,
-  TestError,
   FullResult,
-  TestStep,
 } from '../../../types/testReporter';
-import {
-  relativeTestPath,
-  Position,
-  stripAnsiEscapes,
-  indent,
-  stepSuffix,
-  positionInFile,
-  formatTestHeader,
-  BaseReporter,
-  kOutputSymbol,
-  TestResultOutput,
-} from './base';
 
 type GithubLogType = 'debug' | 'notice' | 'warning' | 'error';
 
@@ -51,22 +34,6 @@ type GithubLogOptions = Partial<{
   endLine: number;
 }>;
 
-type Annotation = {
-  filePath: string;
-  title: string;
-  message: string;
-  position?: Position;
-};
-
-type FailureDetails = {
-  tokens: string[];
-  position?: Position;
-};
-
-type ErrorDetails = {
-  message: string;
-  position?: Position;
-};
 class GithubLogger {
 	private _isCI: boolean = process.env.CI === 'true';
   private _isGithubAction: boolean = process.env.GITHUB_ACTION !== undefined;
@@ -112,94 +79,29 @@ export class GithubReporter extends BaseReporter {
     this.epilogue(true);
   }
 
-  private _printSlowTestAnnotations() {
-    if (!this.config.reportSlowTests) return;
-    const fileDurations = [...this.fileDurations.entries()];
-    fileDurations.sort((a, b) => b[1] - a[1]);
-    const count = Math.min(
-        fileDurations.length,
-        this.config.reportSlowTests.max || Number.POSITIVE_INFINITY
-    );
-    for (let i = 0; i < count; ++i) {
-      const duration = fileDurations[i][1];
-      if (duration <= this.config.reportSlowTests.threshold) break;
+  protected override printSlowTests() {
+    this.getSlowTests().forEach(([file, duration]) => {
       const filePath = workspaceRelativePath(
-          path.join(process.cwd(), fileDurations[i][0])
+          path.join(process.cwd(), file)
       );
       this.githubLogger.warning(`${filePath} (${milliseconds(duration)})`, {
         title: 'Slow Test',
         file: filePath,
       });
-    }
-  }
-
-  override epilogue(full: boolean) {
-    let skipped = 0;
-    let expected = 0;
-    const skippedWithError: TestCase[] = [];
-    const unexpected: TestCase[] = [];
-    const flaky: TestCase[] = [];
-
-    this.suite.allTests().forEach(test => {
-      switch (test.outcome()) {
-        case 'skipped': {
-          ++skipped;
-          if (test.results.some(result => !!result.error))
-            skippedWithError.push(test);
-          break;
-        }
-        case 'expected':
-          ++expected;
-          break;
-        case 'unexpected':
-          unexpected.push(test);
-          break;
-        case 'flaky':
-          flaky.push(test);
-          break;
-      }
     });
 
-    const noticeLines: string[] = [];
-    noticeLines.push('');
-    if (unexpected.length) {
-      noticeLines.push(`  ${unexpected.length} failed`);
-      for (const test of unexpected)
-        noticeLines.push(formatTestHeader(this.config, test, '    '));
-    }
-    if (flaky.length) {
-      noticeLines.push(`  ${flaky.length} flaky`);
-      for (const test of flaky)
-        noticeLines.push(formatTestHeader(this.config, test, '    '));
-    }
-    if (skipped) noticeLines.push(`  ${skipped} skipped`);
-    if (expected) {
-      noticeLines.push(
-          `  ${expected} passed` + ` (${milliseconds(this.duration)})`
-      );
-    }
-    if (this.result.status === 'timedout') {
-      noticeLines.push(
-          `  Timed out waiting ${
-            this.config.globalTimeout / 1000
-          }s for the entire test run`
-      );
-    }
 
-    this.githubLogger.notice(noticeLines.join('\n'), {
+  }
+
+  protected override printSummary(summary: string){
+    this.githubLogger.notice(summary, {
       title: '🎭 Playwright Run Summary',
     });
-
-    const failuresToPrint = [...unexpected, ...flaky, ...skippedWithError];
-    if (full && failuresToPrint.length)
-      this._printFailureAnnotations(failuresToPrint);
-
-    this._printSlowTestAnnotations();
   }
 
-  private _printFailureAnnotations(failures: TestCase[]) {
+  protected override printFailures(failures: TestCase[]) {
     failures.forEach((test, index) => {
-      const annotations = formatFailure(this.config, test, index + 1, true);
+      const { annotations } = formatFailure(this.config, test, index + 1, true);
       annotations.forEach(({ filePath, title, message, position }) => {
         const options: GithubLogOptions = {
           file: filePath,
@@ -215,132 +117,4 @@ export class GithubReporter extends BaseReporter {
   }
 }
 
-function workspaceRelativePath(filePath: string): string {
-  return path.relative(process.env['GITHUB_WORKSPACE'] ?? '', filePath);
-}
-
-function formatFailure(
-  config: FullConfig,
-  test: TestCase,
-  index?: number,
-  stdio?: boolean
-): Annotation[] {
-  const title = formatTestTitle(config, test);
-  const filePath = workspaceRelativePath(test.location.file);
-  const annotations: Annotation[] = [];
-  for (const result of test.results) {
-    const lines: string[] = [];
-    lines.push(formatTestHeader(config, test, '  ', index));
-    const failureDetails = formatResultFailure(test, result, '    ');
-    const resultTokens = failureDetails.tokens;
-    const position = failureDetails.position;
-    if (!resultTokens.length) continue;
-    if (result.retry) {
-      lines.push('');
-      lines.push(`    Retry #${result.retry}`);
-    }
-    lines.push(...resultTokens);
-
-    const output = ((result as any)[kOutputSymbol] || []) as TestResultOutput[];
-    if (stdio && output.length) {
-      const outputText = output
-          .map(({ chunk, type }) => {
-            const text = chunk.toString('utf8');
-            if (type === 'stderr') return stripAnsiEscapes(text);
-            return text;
-          })
-          .join('');
-      lines.push('');
-      lines.push('--- Test output ---' + '\n\n' + outputText + '\n');
-    }
-
-    lines.push('');
-    annotations.push({
-      filePath,
-      position,
-      title,
-      message: lines.join('\n'),
-    });
-  }
-
-  return annotations;
-}
-
-function formatResultFailure(
-  test: TestCase,
-  result: TestResult,
-  initialIndent: string
-): FailureDetails {
-  const resultTokens: string[] = [];
-  if (result.status === 'timedOut') {
-    resultTokens.push('');
-    resultTokens.push(
-        indent(`Timeout of ${test.timeout}ms exceeded.`, initialIndent)
-    );
-  }
-  if (result.status === 'passed' && test.expectedStatus === 'failed') {
-    resultTokens.push('');
-    resultTokens.push(indent(`Expected to fail, but passed.`, initialIndent));
-  }
-  let error: ErrorDetails | undefined = undefined;
-  if (result.error !== undefined) {
-    error = formatError(result.error, test.location.file);
-    resultTokens.push(indent(error.message, initialIndent));
-  }
-  return {
-    tokens: resultTokens,
-    position: error?.position,
-  };
-}
-
-function formatTestTitle(
-  config: FullConfig,
-  test: TestCase,
-  step?: TestStep
-): string {
-  // root, project, file, ...describes, test
-  const [, projectName, , ...titles] = test.titlePath();
-  const location = `${relativeTestPath(config, test)}:${test.location.line}:${
-    test.location.column
-  }`;
-  const projectTitle = projectName ? `[${projectName}] › ` : '';
-  return `${projectTitle}${location} › ${titles.join(' ')}${stepSuffix(step)}`;
-}
-
-function formatError(error: TestError, file?: string): ErrorDetails {
-  const stack = error.stack;
-  const tokens = [''];
-  let position: Position | undefined;
-
-  if (stack) {
-    const lines = stack.split('\n');
-    let firstStackLine = lines.findIndex(line => line.startsWith('    at '));
-    if (firstStackLine === -1) firstStackLine = lines.length;
-    tokens.push(lines.slice(0, firstStackLine).join('\n'));
-    const stackLines = lines.slice(firstStackLine);
-    position = file ? positionInFile(stackLines, file) : undefined;
-    if (position) {
-      const source = fs.readFileSync(file!, 'utf8');
-      tokens.push('');
-      tokens.push(
-          codeFrameColumns(
-              source,
-              { start: position },
-              { highlightCode: false }
-          )
-      );
-    }
-    tokens.push('');
-    tokens.push(stackLines.join('\n'));
-  } else if (error.message) {
-    tokens.push(error.message);
-  } else if (error.value) {
-    tokens.push(error.value);
-  }
-  return {
-    position,
-    message: tokens.join('\n'),
-  };
-}
-
 export default GithubReporter;
diff --git a/src/test/reporters/junit.ts b/src/test/reporters/junit.ts
index aa4875825e9e0..d4e93b317bb01 100644
--- a/src/test/reporters/junit.ts
+++ b/src/test/reporters/junit.ts
@@ -142,7 +142,7 @@ class JUnitReporter implements Reporter {
           message: `${path.basename(test.location.file)}:${test.location.line}:${test.location.column} ${test.title}`,
           type: 'FAILURE',
         },
-        text: stripAnsiEscapes(formatFailure(this.config, test))
+        text: stripAnsiEscapes(formatFailure(this.config, test).message)
       });
     }
 
diff --git a/src/test/reporters/line.ts b/src/test/reporters/line.ts
index 547fd653a4d73..a45103476a1e0 100644
--- a/src/test/reporters/line.ts
+++ b/src/test/reporters/line.ts
@@ -61,7 +61,7 @@ class LineReporter extends BaseReporter {
     process.stdout.write(`\u001B[1A\u001B[2K${title}\n`);
     if (!this.willRetry(test) && (test.outcome() === 'flaky' || test.outcome() === 'unexpected')) {
       process.stdout.write(`\u001B[1A\u001B[2K`);
-      console.log(formatFailure(this.config, test, ++this._failures));
+      console.log(formatFailure(this.config, test, ++this._failures).message);
       console.log();
     }
   }

From 65aa52ea58bf0f8b45e85bc8fc1014bcad4a4d75 Mon Sep 17 00:00:00 2001
From: Sidharth Vinod <sidharthv96@gmail.com>
Date: Tue, 28 Sep 2021 12:23:45 +0530
Subject: [PATCH 06/21] chore(reporters): Remove unused exports

---
 src/test/reporters/base.ts | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/test/reporters/base.ts b/src/test/reporters/base.ts
index 8f0bddfa433e7..99b3f996534b2 100644
--- a/src/test/reporters/base.ts
+++ b/src/test/reporters/base.ts
@@ -300,16 +300,16 @@ export function formatResultFailure(test: TestCase, result: TestResult, initialI
   };
 }
 
-export function relativeTestPath(config: FullConfig, test: TestCase): string {
+function relativeTestPath(config: FullConfig, test: TestCase): string {
   return path.relative(config.rootDir, test.location.file) || path.basename(test.location.file);
 }
 
-export function stepSuffix(step: TestStep | undefined) {
+function stepSuffix(step: TestStep | undefined) {
   const stepTitles = step ? step.titlePath() : [];
   return stepTitles.map(t => ' › ' + t).join('');
 }
 
-export function formatTestTitle(config: FullConfig, test: TestCase, step?: TestStep): string {
+function formatTestTitle(config: FullConfig, test: TestCase, step?: TestStep): string {
   // root, project, file, ...describes, test
   const [, projectName, , ...titles] = test.titlePath();
   const location = `${relativeTestPath(config, test)}:${test.location.line}:${test.location.column}`;
@@ -317,7 +317,7 @@ export function formatTestTitle(config: FullConfig, test: TestCase, step?: TestS
   return `${projectTitle}${location} › ${titles.join(' ')}${stepSuffix(step)}`;
 }
 
-export function formatTestHeader(config: FullConfig, test: TestCase, indent: string, index?: number): string {
+function formatTestHeader(config: FullConfig, test: TestCase, indent: string, index?: number): string {
   const title = formatTestTitle(config, test);
   const header = `${indent}${index ? index + ') ' : ''}${title}`;
   return pad(header, '=');
@@ -363,11 +363,11 @@ function pad(line: string, char: string): string {
   return line + colors.gray(char.repeat(Math.max(0, 100 - line.length)));
 }
 
-export function indent(lines: string, tab: string) {
+function indent(lines: string, tab: string) {
   return lines.replace(/^(?=.+$)/gm, tab);
 }
 
-export function positionInFile(stackLines: string[], file: string): Position | undefined {
+function positionInFile(stackLines: string[], file: string): Position | undefined {
   // Stack will have /private/var/folders instead of /var/folders on Mac.
   file = fs.realpathSync(file);
   for (const line of stackLines) {

From 2841db439f97e82c8dd68e96cb5a7fe941123db9 Mon Sep 17 00:00:00 2001
From: Sidharth Vinod <sidharthv96@gmail.com>
Date: Tue, 28 Sep 2021 13:29:22 +0530
Subject: [PATCH 07/21] fix(reporters): Export formatTestTitle

---
 src/test/reporters/base.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/reporters/base.ts b/src/test/reporters/base.ts
index 99b3f996534b2..b2a8b87290a15 100644
--- a/src/test/reporters/base.ts
+++ b/src/test/reporters/base.ts
@@ -309,7 +309,7 @@ function stepSuffix(step: TestStep | undefined) {
   return stepTitles.map(t => ' › ' + t).join('');
 }
 
-function formatTestTitle(config: FullConfig, test: TestCase, step?: TestStep): string {
+export function formatTestTitle(config: FullConfig, test: TestCase, step?: TestStep): string {
   // root, project, file, ...describes, test
   const [, projectName, , ...titles] = test.titlePath();
   const location = `${relativeTestPath(config, test)}:${test.location.line}:${test.location.column}`;

From 38e8bc0bfe3f8f4cd931b3da410bbb2890957261 Mon Sep 17 00:00:00 2001
From: Sidharth Vinod <sidharthv96@gmail.com>
Date: Tue, 28 Sep 2021 13:40:38 +0530
Subject: [PATCH 08/21] fix(reporters): Use tokens in raw reporter

---
 src/test/reporters/raw.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/reporters/raw.ts b/src/test/reporters/raw.ts
index 260c9d8f73e0f..ae306b1a5c31c 100644
--- a/src/test/reporters/raw.ts
+++ b/src/test/reporters/raw.ts
@@ -181,7 +181,7 @@ class RawReporter {
       startTime: result.startTime.toISOString(),
       duration: result.duration,
       status: result.status,
-      error: formatResultFailure(test, result, '').join('').trim(),
+      error: formatResultFailure(test, result, '').tokens.join('').trim(),
       attachments: this._createAttachments(result),
       steps: this._serializeSteps(test, result.steps)
     };

From db5e34511a625e503b9741c3055ef66b5871c86c Mon Sep 17 00:00:00 2001
From: Sidharth Vinod <sidharthv96@gmail.com>
Date: Wed, 29 Sep 2021 10:37:17 +0530
Subject: [PATCH 09/21] chore(reporter): Fix formatting comments

---
 package-lock.json            | 13 ++++++++++
 package.json                 |  1 +
 src/test/reporters/base.ts   | 33 ++++++++++++------------
 src/test/reporters/github.ts | 49 +++++++++++++++++-------------------
 src/test/reporters/line.ts   |  4 ++-
 5 files changed, 56 insertions(+), 44 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index eb294f8ebd557..c9d6a989ecc89 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -62,6 +62,7 @@
         "@types/extract-zip": "^1.6.2",
         "@types/mime": "^2.0.3",
         "@types/minimatch": "^3.0.3",
+        "@types/ms": "^0.7.31",
         "@types/node": "^10.17.28",
         "@types/pixelmatch": "^5.2.1",
         "@types/pngjs": "^3.4.2",
@@ -1485,6 +1486,12 @@
       "integrity": "sha512-1z8k4wzFnNjVK/tlxvrWuK5WMt6mydWWP7+zvH5eFep4oj+UkrfiJTRtjCeBXNpwaA/FYqqtb4/QS4ianFpIRA==",
       "dev": true
     },
+    "node_modules/@types/ms": {
+      "version": "0.7.31",
+      "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.31.tgz",
+      "integrity": "sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==",
+      "dev": true
+    },
     "node_modules/@types/node": {
       "version": "10.17.60",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.60.tgz",
@@ -11935,6 +11942,12 @@
       "integrity": "sha512-1z8k4wzFnNjVK/tlxvrWuK5WMt6mydWWP7+zvH5eFep4oj+UkrfiJTRtjCeBXNpwaA/FYqqtb4/QS4ianFpIRA==",
       "dev": true
     },
+    "@types/ms": {
+      "version": "0.7.31",
+      "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.31.tgz",
+      "integrity": "sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==",
+      "dev": true
+    },
     "@types/node": {
       "version": "10.17.60",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.60.tgz",
diff --git a/package.json b/package.json
index aad5538e5586a..7fc9c436aab71 100644
--- a/package.json
+++ b/package.json
@@ -91,6 +91,7 @@
     "@types/extract-zip": "^1.6.2",
     "@types/mime": "^2.0.3",
     "@types/minimatch": "^3.0.3",
+    "@types/ms": "^0.7.31",
     "@types/node": "^10.17.28",
     "@types/pixelmatch": "^5.2.1",
     "@types/pngjs": "^3.4.2",
diff --git a/src/test/reporters/base.ts b/src/test/reporters/base.ts
index b2a8b87290a15..69f6f88d2d0cb 100644
--- a/src/test/reporters/base.ts
+++ b/src/test/reporters/base.ts
@@ -17,7 +17,6 @@
 import { codeFrameColumns } from '@babel/code-frame';
 import colors from 'colors/safe';
 import fs from 'fs';
-// @ts-ignore
 import milliseconds from 'ms';
 import path from 'path';
 import StackUtils from 'stack-utils';
@@ -193,7 +192,10 @@ export class BaseReporter implements Reporter  {
   protected printFailures(failures: TestCase[]) {
     console.log('');
     failures.forEach((test, index) => {
-      console.log(formatFailure(this.config, test, index + 1, this.printTestOutput).message);
+      console.log(formatFailure(this.config, test, {
+        index: index + 1,
+        includeStdio: this.printTestOutput
+      }).message);
     });
   }
 
@@ -202,23 +204,18 @@ export class BaseReporter implements Reporter  {
   }
 }
 
-export function workspaceRelativePath(filePath: string): string {
-  return path.relative(process.env['GITHUB_WORKSPACE'] ?? '', filePath);
-}
-
-export function formatFailure(config: FullConfig, test: TestCase, index?: number, stdio?: boolean, attachment = true): {
+export function formatFailure(config: FullConfig, test: TestCase, options: {index?: number, includeStdio?: boolean, includeAttachments?: boolean, filePath?: string} = {}): {
   message: string,
   annotations: Annotation[]
 } {
+  const { index, includeStdio, includeAttachments = true, filePath } = options;
   const lines: string[] = [];
   const title = formatTestTitle(config, test);
-  const filePath = workspaceRelativePath(test.location.file);
   const annotations: Annotation[] = [];
   const header = formatTestHeader(config, test, '  ', index);
   lines.push(colors.red(header));
   for (const result of test.results) {
     const resultLines: string[] = [];
-
     const { tokens: resultTokens, position } = formatResultFailure(test, result, '    ');
     if (!resultTokens.length)
       continue;
@@ -227,7 +224,7 @@ export function formatFailure(config: FullConfig, test: TestCase, index?: number
       resultLines.push(colors.gray(pad(`    Retry #${result.retry}`, '-')));
     }
     resultLines.push(...resultTokens);
-    if (attachment){
+    if (includeAttachments) {
       for (let i = 0; i < result.attachments.length; ++i) {
         const attachment = result.attachments[i];
         resultLines.push('');
@@ -254,7 +251,7 @@ export function formatFailure(config: FullConfig, test: TestCase, index?: number
       }
     }
     const output = ((result as any)[kOutputSymbol] || []) as TestResultOutput[];
-    if (stdio && output.length) {
+    if (includeStdio && output.length) {
       const outputText = output.map(({ chunk, type }) => {
         const text = chunk.toString('utf8');
         if (type === 'stderr')
@@ -264,12 +261,14 @@ export function formatFailure(config: FullConfig, test: TestCase, index?: number
       resultLines.push('');
       resultLines.push(colors.gray(pad('--- Test output', '-')) + '\n\n' + outputText + '\n' + pad('', '-'));
     }
-    annotations.push({
-      filePath,
-      position,
-      title,
-      message: [header, ...resultLines].join('\n'),
-    });
+    if (filePath) {
+      annotations.push({
+        filePath,
+        position,
+        title,
+        message: [header, ...resultLines].join('\n'),
+      });
+    }
     lines.push(...resultLines);
   }
   lines.push('');
diff --git a/src/test/reporters/github.ts b/src/test/reporters/github.ts
index 98aa54f9d0079..b4e7466ce02ee 100644
--- a/src/test/reporters/github.ts
+++ b/src/test/reporters/github.ts
@@ -14,10 +14,9 @@
  * limitations under the License.
  */
 
-// @ts-ignore
 import milliseconds from 'ms';
 import path from 'path';
-import { BaseReporter, formatFailure, workspaceRelativePath } from './base';
+import { BaseReporter, formatFailure } from './base';
 import {
   TestCase,
   FullResult,
@@ -35,23 +34,15 @@ type GithubLogOptions = Partial<{
 }>;
 
 class GithubLogger {
-	private _isCI: boolean = process.env.CI === 'true';
-  private _isGithubAction: boolean = process.env.GITHUB_ACTION !== undefined;
-  private _shouldLog = (this._isCI && this._isGithubAction) || process.env.PW_GH_ACTION_DEBUG === 'true' ;
-
-  private _log(
-    message: string,
-    type: GithubLogType = 'notice',
-    options: GithubLogOptions = {}
-  ) {
-    if (this._shouldLog) {
-      if (this._isGithubAction) message = message.replace(/\n/g, '%0A');
-
-      const configs = Object.entries(options)
-          .map(([key, option]) => `${key}=${option}`)
-          .join(',');
-      console.log(`::${type} ${configs}::${message}`);
-    }
+  private _isGithubAction: boolean = !!process.env.GITHUB_ACTION;
+
+  private _log(message: string, type: GithubLogType = 'notice', options: GithubLogOptions = {}) {
+    if (this._isGithubAction)
+      message = message.replace(/\n/g, '%0A');
+    const configs = Object.entries(options)
+        .map(([key, option]) => `${key}=${option}`)
+        .join(',');
+    console.log(`::${type} ${configs}::${message}`);
   }
 
   debug(message: string, options?: GithubLogOptions) {
@@ -81,27 +72,29 @@ export class GithubReporter extends BaseReporter {
 
   protected override printSlowTests() {
     this.getSlowTests().forEach(([file, duration]) => {
-      const filePath = workspaceRelativePath(
-          path.join(process.cwd(), file)
-      );
+      const filePath = workspaceRelativePath(path.join(process.cwd(), file));
       this.githubLogger.warning(`${filePath} (${milliseconds(duration)})`, {
         title: 'Slow Test',
         file: filePath,
       });
     });
-
-
   }
 
   protected override printSummary(summary: string){
     this.githubLogger.notice(summary, {
-      title: '🎭 Playwright Run Summary',
+      title: '🎭 Playwright Run Summary'
     });
   }
 
   protected override printFailures(failures: TestCase[]) {
     failures.forEach((test, index) => {
-      const { annotations } = formatFailure(this.config, test, index + 1, true);
+      const filePath = workspaceRelativePath(test.location.file);
+      const { annotations } = formatFailure(this.config, test, {
+        filePath,
+        index: index + 1,
+        includeStdio: true,
+        includeAttachments: false,
+      });
       annotations.forEach(({ filePath, title, message, position }) => {
         const options: GithubLogOptions = {
           file: filePath,
@@ -117,4 +110,8 @@ export class GithubReporter extends BaseReporter {
   }
 }
 
+function workspaceRelativePath(filePath: string): string {
+  return path.relative(process.env['GITHUB_WORKSPACE'] ?? '', filePath);
+}
+
 export default GithubReporter;
diff --git a/src/test/reporters/line.ts b/src/test/reporters/line.ts
index a45103476a1e0..cf442c6c8a89b 100644
--- a/src/test/reporters/line.ts
+++ b/src/test/reporters/line.ts
@@ -61,7 +61,9 @@ class LineReporter extends BaseReporter {
     process.stdout.write(`\u001B[1A\u001B[2K${title}\n`);
     if (!this.willRetry(test) && (test.outcome() === 'flaky' || test.outcome() === 'unexpected')) {
       process.stdout.write(`\u001B[1A\u001B[2K`);
-      console.log(formatFailure(this.config, test, ++this._failures).message);
+      console.log(formatFailure(this.config, test, {
+        index: ++this._failures
+      }).message);
       console.log();
     }
   }

From 492ca952a5ad447d4ac1fde27e040bd9786728d0 Mon Sep 17 00:00:00 2001
From: Sidharth Vinod <sidharthv96@gmail.com>
Date: Wed, 29 Sep 2021 10:41:07 +0530
Subject: [PATCH 10/21] chore(reporter): Fix formatting comments

---
 src/test/reporters/github.ts | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/test/reporters/github.ts b/src/test/reporters/github.ts
index b4e7466ce02ee..adce1267b7c62 100644
--- a/src/test/reporters/github.ts
+++ b/src/test/reporters/github.ts
@@ -17,10 +17,7 @@
 import milliseconds from 'ms';
 import path from 'path';
 import { BaseReporter, formatFailure } from './base';
-import {
-  TestCase,
-  FullResult,
-} from '../../../types/testReporter';
+import { TestCase, FullResult } from '../../../types/testReporter';
 
 type GithubLogType = 'debug' | 'notice' | 'warning' | 'error';
 

From f0838b745c7fdb30ff8d158f23a0f426d7b0ea0e Mon Sep 17 00:00:00 2001
From: Sidharth Vinod <sidharthv96@gmail.com>
Date: Wed, 29 Sep 2021 10:41:43 +0530
Subject: [PATCH 11/21] chore(reporter): Remove ts-ignore from list

---
 src/test/reporters/list.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/test/reporters/list.ts b/src/test/reporters/list.ts
index f5edf1f175b58..83f2122df10df 100644
--- a/src/test/reporters/list.ts
+++ b/src/test/reporters/list.ts
@@ -16,7 +16,6 @@
 
 /* eslint-disable no-console */
 import colors from 'colors/safe';
-// @ts-ignore
 import milliseconds from 'ms';
 import { BaseReporter, formatTestTitle } from './base';
 import { FullConfig, FullResult, Suite, TestCase, TestResult, TestStep } from '../../../types/testReporter';

From e8fafa04befee9866c0a69552dfe7d1df9b91476 Mon Sep 17 00:00:00 2001
From: Sidharth Vinod <sidharthv96@gmail.com>
Date: Wed, 29 Sep 2021 10:45:33 +0530
Subject: [PATCH 12/21] chore(reporter): Remove extra line

---
 src/test/reporters/base.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/test/reporters/base.ts b/src/test/reporters/base.ts
index 69f6f88d2d0cb..e2a36a735e07f 100644
--- a/src/test/reporters/base.ts
+++ b/src/test/reporters/base.ts
@@ -32,7 +32,6 @@ export type Position = {
   line: number;
 };
 
-
 type Annotation = {
   filePath: string;
   title: string;

From 4882d0bc53b679d66344dd452c41c23b3b9ff0e5 Mon Sep 17 00:00:00 2001
From: Sidharth Vinod <sidharthv96@gmail.com>
Date: Wed, 29 Sep 2021 22:09:58 +0530
Subject: [PATCH 13/21] fix: Remove base -> github calls.

---
 src/test/reporters/base.ts   | 43 +++++++++++++++++++++---------------
 src/test/reporters/github.ts | 17 ++++++++++----
 2 files changed, 38 insertions(+), 22 deletions(-)

diff --git a/src/test/reporters/base.ts b/src/test/reporters/base.ts
index e2a36a735e07f..9664a522d1492 100644
--- a/src/test/reporters/base.ts
+++ b/src/test/reporters/base.ts
@@ -55,6 +55,7 @@ type TestSummary = {
   skippedWithError: TestCase[];
   unexpected: TestCase[];
   flaky: TestCase[];
+  failuresToPrint: TestCase[];
 };
 
 export class BaseReporter implements Reporter  {
@@ -114,12 +115,6 @@ export class BaseReporter implements Reporter  {
     return fileDurations.filter(([,duration]) => duration > threshold).slice(0, count);
   }
 
-  protected printSlowTests() {
-    this.getSlowTests().forEach(([file, duration]) => {
-      console.log(colors.yellow('  Slow test: ') + file + colors.yellow(` (${milliseconds(duration)})`));
-    });
-  }
-
   protected generateSummaryMessage({
     skipped,
     expected,
@@ -148,7 +143,7 @@ export class BaseReporter implements Reporter  {
     return tokens.join('\n');
   }
 
-  epilogue(full: boolean) {
+  protected generateSummary(): TestSummary {
     let skipped = 0;
     let expected = 0;
     const skippedWithError: TestCase[] = [];
@@ -170,25 +165,26 @@ export class BaseReporter implements Reporter  {
     });
 
     const failuresToPrint = [...unexpected, ...flaky, ...skippedWithError];
-    if (full && failuresToPrint.length)
-      this.printFailures(failuresToPrint);
-    this.printSlowTests();
-    const summaryMessage = this.generateSummaryMessage({
+    return {
       skipped,
       expected,
       skippedWithError,
       unexpected,
-      flaky
-    });
-    this.printSummary(summaryMessage);
+      flaky,
+      failuresToPrint
+    };
   }
 
-  protected printSummary(summary: string){
-    console.log('');
-    console.log(summary);
+  epilogue(full: boolean) {
+    const summary = this.generateSummary();
+    const summaryMessage = this.generateSummaryMessage(summary);
+    if (full && summary.failuresToPrint.length)
+      this._printFailures(summary.failuresToPrint);
+    this._printSlowTests();
+    this._printSummary(summaryMessage);
   }
 
-  protected printFailures(failures: TestCase[]) {
+  private _printFailures(failures: TestCase[]) {
     console.log('');
     failures.forEach((test, index) => {
       console.log(formatFailure(this.config, test, {
@@ -198,6 +194,17 @@ export class BaseReporter implements Reporter  {
     });
   }
 
+  private _printSlowTests() {
+    this.getSlowTests().forEach(([file, duration]) => {
+      console.log(colors.yellow('  Slow test: ') + file + colors.yellow(` (${milliseconds(duration)})`));
+    });
+  }
+
+  private _printSummary(summary: string){
+    console.log('');
+    console.log(summary);
+  }
+
   willRetry(test: TestCase): boolean {
     return test.outcome() === 'unexpected' && test.results.length <= test.retries;
   }
diff --git a/src/test/reporters/github.ts b/src/test/reporters/github.ts
index adce1267b7c62..3202acd34fba4 100644
--- a/src/test/reporters/github.ts
+++ b/src/test/reporters/github.ts
@@ -64,10 +64,19 @@ export class GithubReporter extends BaseReporter {
 
   override async onEnd(result: FullResult) {
     super.onEnd(result);
-    this.epilogue(true);
+    this.generateAnnotations();
   }
 
-  protected override printSlowTests() {
+  private generateAnnotations() {
+    const summary = this.generateSummary();
+    const summaryMessage = this.generateSummaryMessage(summary);
+    if (summary.failuresToPrint.length)
+      this._printFailureAnnotations(summary.failuresToPrint);
+    this._printSlowTestAnnotations();
+    this._printSummaryAnnotation(summaryMessage);
+  }
+
+  private _printSlowTestAnnotations() {
     this.getSlowTests().forEach(([file, duration]) => {
       const filePath = workspaceRelativePath(path.join(process.cwd(), file));
       this.githubLogger.warning(`${filePath} (${milliseconds(duration)})`, {
@@ -77,13 +86,13 @@ export class GithubReporter extends BaseReporter {
     });
   }
 
-  protected override printSummary(summary: string){
+  private _printSummaryAnnotation(summary: string){
     this.githubLogger.notice(summary, {
       title: '🎭 Playwright Run Summary'
     });
   }
 
-  protected override printFailures(failures: TestCase[]) {
+  private _printFailureAnnotations(failures: TestCase[]) {
     failures.forEach((test, index) => {
       const filePath = workspaceRelativePath(test.location.file);
       const { annotations } = formatFailure(this.config, test, {

From 42f452522012563caa5f8d3c81ca6d71990d9dd1 Mon Sep 17 00:00:00 2001
From: Sidharth Vinod <sidharthv96@gmail.com>
Date: Wed, 29 Sep 2021 22:14:03 +0530
Subject: [PATCH 14/21] fix: Formatting

---
 src/test/reporters/base.ts | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/src/test/reporters/base.ts b/src/test/reporters/base.ts
index 9664a522d1492..16266863b2383 100644
--- a/src/test/reporters/base.ts
+++ b/src/test/reporters/base.ts
@@ -115,12 +115,7 @@ export class BaseReporter implements Reporter  {
     return fileDurations.filter(([,duration]) => duration > threshold).slice(0, count);
   }
 
-  protected generateSummaryMessage({
-    skipped,
-    expected,
-    unexpected,
-    flaky
-  }: TestSummary) {
+  protected generateSummaryMessage({ skipped, expected, unexpected, flaky }: TestSummary) {
     const tokens: string[] = [];
     tokens.push('');
     if (unexpected.length) {

From 47c246559d23eecab7e2c8f20b1c8830faed5a23 Mon Sep 17 00:00:00 2001
From: Sidharth Vinod <sidharthv96@gmail.com>
Date: Wed, 29 Sep 2021 22:15:27 +0530
Subject: [PATCH 15/21] fix: Formatting github

---
 src/test/reporters/github.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/test/reporters/github.ts b/src/test/reporters/github.ts
index 3202acd34fba4..5ab121dc21820 100644
--- a/src/test/reporters/github.ts
+++ b/src/test/reporters/github.ts
@@ -64,10 +64,10 @@ export class GithubReporter extends BaseReporter {
 
   override async onEnd(result: FullResult) {
     super.onEnd(result);
-    this.generateAnnotations();
+    this._generateAnnotations();
   }
 
-  private generateAnnotations() {
+  private _generateAnnotations() {
     const summary = this.generateSummary();
     const summaryMessage = this.generateSummaryMessage(summary);
     if (summary.failuresToPrint.length)

From 3dec7e757fd52c9166feabf78a048edc6a71e473 Mon Sep 17 00:00:00 2001
From: Sidharth Vinod <sidharthv96@gmail.com>
Date: Thu, 30 Sep 2021 08:11:25 +0530
Subject: [PATCH 16/21] chore(github-reporter): Rename identifiers

---
 src/test/reporters/github.ts | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/src/test/reporters/github.ts b/src/test/reporters/github.ts
index 5ab121dc21820..964b1ea72fd02 100644
--- a/src/test/reporters/github.ts
+++ b/src/test/reporters/github.ts
@@ -19,9 +19,9 @@ import path from 'path';
 import { BaseReporter, formatFailure } from './base';
 import { TestCase, FullResult } from '../../../types/testReporter';
 
-type GithubLogType = 'debug' | 'notice' | 'warning' | 'error';
+type GitHubLogType = 'debug' | 'notice' | 'warning' | 'error';
 
-type GithubLogOptions = Partial<{
+type GitHubLogOptions = Partial<{
   title: string;
   file: string;
   col: number;
@@ -30,11 +30,11 @@ type GithubLogOptions = Partial<{
   endLine: number;
 }>;
 
-class GithubLogger {
-  private _isGithubAction: boolean = !!process.env.GITHUB_ACTION;
+class GitHubLogger {
+  private _isGitHubAction: boolean = !!process.env.GITHUB_ACTION;
 
-  private _log(message: string, type: GithubLogType = 'notice', options: GithubLogOptions = {}) {
-    if (this._isGithubAction)
+  private _log(message: string, type: GitHubLogType = 'notice', options: GitHubLogOptions = {}) {
+    if (this._isGitHubAction)
       message = message.replace(/\n/g, '%0A');
     const configs = Object.entries(options)
         .map(([key, option]) => `${key}=${option}`)
@@ -42,32 +42,32 @@ class GithubLogger {
     console.log(`::${type} ${configs}::${message}`);
   }
 
-  debug(message: string, options?: GithubLogOptions) {
+  debug(message: string, options?: GitHubLogOptions) {
     this._log(message, 'debug', options);
   }
 
-  error(message: string, options?: GithubLogOptions) {
+  error(message: string, options?: GitHubLogOptions) {
     this._log(message, 'error', options);
   }
 
-  notice(message: string, options?: GithubLogOptions) {
+  notice(message: string, options?: GitHubLogOptions) {
     this._log(message, 'notice', options);
   }
 
-  warning(message: string, options?: GithubLogOptions) {
+  warning(message: string, options?: GitHubLogOptions) {
     this._log(message, 'warning', options);
   }
 }
 
-export class GithubReporter extends BaseReporter {
-  githubLogger = new GithubLogger();
+export class GitHubReporter extends BaseReporter {
+  githubLogger = new GitHubLogger();
 
   override async onEnd(result: FullResult) {
     super.onEnd(result);
-    this._generateAnnotations();
+    this._printAnnotations();
   }
 
-  private _generateAnnotations() {
+  private _printAnnotations() {
     const summary = this.generateSummary();
     const summaryMessage = this.generateSummaryMessage(summary);
     if (summary.failuresToPrint.length)
@@ -102,7 +102,7 @@ export class GithubReporter extends BaseReporter {
         includeAttachments: false,
       });
       annotations.forEach(({ filePath, title, message, position }) => {
-        const options: GithubLogOptions = {
+        const options: GitHubLogOptions = {
           file: filePath,
           title,
         };
@@ -120,4 +120,4 @@ function workspaceRelativePath(filePath: string): string {
   return path.relative(process.env['GITHUB_WORKSPACE'] ?? '', filePath);
 }
 
-export default GithubReporter;
+export default GitHubReporter;

From e153aefc05493e568bfa393790d6a4d450b40e80 Mon Sep 17 00:00:00 2001
From: Sidharth Vinod <sidharthv96@gmail.com>
Date: Thu, 30 Sep 2021 09:32:35 +0530
Subject: [PATCH 17/21] test(github-reporter): Add Unit tests

---
 src/test/reporters/github.ts                  |  2 +-
 src/test/runner.ts                            |  4 +-
 tests/playwright-test/github-reporter.spec.ts | 86 +++++++++++++++++++
 3 files changed, 90 insertions(+), 2 deletions(-)
 create mode 100644 tests/playwright-test/github-reporter.spec.ts

diff --git a/src/test/reporters/github.ts b/src/test/reporters/github.ts
index 964b1ea72fd02..b345fd2f59e88 100644
--- a/src/test/reporters/github.ts
+++ b/src/test/reporters/github.ts
@@ -79,7 +79,7 @@ export class GitHubReporter extends BaseReporter {
   private _printSlowTestAnnotations() {
     this.getSlowTests().forEach(([file, duration]) => {
       const filePath = workspaceRelativePath(path.join(process.cwd(), file));
-      this.githubLogger.warning(`${filePath} (${milliseconds(duration)})`, {
+      this.githubLogger.warning(`${filePath} took ${milliseconds(duration)}`, {
         title: 'Slow Test',
         file: filePath,
       });
diff --git a/src/test/runner.ts b/src/test/runner.ts
index bc2c49ac7f5e5..e29c80572f217 100644
--- a/src/test/runner.ts
+++ b/src/test/runner.ts
@@ -27,6 +27,7 @@ import { Loader } from './loader';
 import { Reporter } from '../../types/testReporter';
 import { Multiplexer } from './reporters/multiplexer';
 import DotReporter from './reporters/dot';
+import GitHubReporter from './reporters/github';
 import LineReporter from './reporters/line';
 import ListReporter from './reporters/list';
 import JSONReporter from './reporters/json';
@@ -68,6 +69,7 @@ export class Runner {
       dot: list ? ListModeReporter : DotReporter,
       line: list ? ListModeReporter : LineReporter,
       list: list ? ListModeReporter : ListReporter,
+      github: GitHubReporter,
       json: JSONReporter,
       junit: JUnitReporter,
       null: EmptyReporter,
@@ -539,5 +541,5 @@ class ListModeReporter implements Reporter {
   }
 }
 
-export const builtInReporters = ['list', 'line', 'dot', 'json', 'junit', 'null'] as const;
+export const builtInReporters = ['list', 'line', 'dot', 'json', 'junit', 'null', 'github'] as const;
 export type BuiltInReporter = typeof builtInReporters[number];
diff --git a/tests/playwright-test/github-reporter.spec.ts b/tests/playwright-test/github-reporter.spec.ts
new file mode 100644
index 0000000000000..a5860b6ba0e4a
--- /dev/null
+++ b/tests/playwright-test/github-reporter.spec.ts
@@ -0,0 +1,86 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect, stripAscii } from './playwright-test-fixtures';
+
+test('print GitHub annotations for success', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const { test } = pwt;
+      test('example1', async ({}) => {
+        expect(1 + 1).toBe(2);
+      });
+    `
+  }, { reporter: 'github' }, { GITHUB_ACTION: 'true' });
+  const text = stripAscii(result.output);
+  expect(text).not.toContain('::error');
+  expect(text).toContain('::notice title=🎭 Playwright Run Summary::%0A  1 passed');
+  expect(result.exitCode).toBe(0);
+});
+
+test('print GitHub annotations with newline if not in CI', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const { test } = pwt;
+      test('example1', async ({}) => {
+        expect(1 + 1).toBe(2);
+      });
+    `
+  }, { reporter: 'github' });
+  const text = stripAscii(result.output);
+  expect(text).not.toContain('::error');
+  expect(text).toContain(`::notice title=🎭 Playwright Run Summary::
+  1 passed `);
+  expect(result.exitCode).toBe(0);
+});
+
+
+test('print GitHub annotations for failed tests', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const { test } = pwt;
+      test('example', async ({}) => {
+        expect(1 + 1).toBe(3);
+      });
+    `
+  }, { retries: 3, reporter: 'github' }, { GITHUB_ACTION: 'true' });
+  const text = stripAscii(result.output);
+  expect(text).toContain('::error file=a.test.js,title=a.test.js:6:7 › example,line=7,col=23::  1) a.test.js:6:7 › example =======================================================================%0A%0A    Retry #1');
+  expect(text).toContain('::error file=a.test.js,title=a.test.js:6:7 › example,line=7,col=23::  1) a.test.js:6:7 › example =======================================================================%0A%0A    Retry #2');
+  expect(text).toContain('::error file=a.test.js,title=a.test.js:6:7 › example,line=7,col=23::  1) a.test.js:6:7 › example =======================================================================%0A%0A    Retry #3');
+  expect(result.exitCode).toBe(1);
+});
+
+
+test.only('print GitHub annotations for slow tests', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = {
+        reportSlowTests: { max: 0, threshold: 100 }
+      };
+    `,
+    'a.test.js': `
+      const { test } = pwt;
+      test('slow test', async ({}) => {
+        await new Promise(f => setTimeout(f, 200));
+      });
+    `
+  }, { retries: 3, reporter: 'github' }, { GITHUB_ACTION: 'true' });
+  const text = stripAscii(result.output);
+  expect(text).toContain('::warning title=Slow Test,file=a.test.js::a.test.js took 2');
+  expect(text).toContain('::notice title=🎭 Playwright Run Summary::%0A  1 passed');
+  expect(result.exitCode).toBe(0);
+});
\ No newline at end of file

From e07820dc740fc1e1bec5702643ccde98440892f2 Mon Sep 17 00:00:00 2001
From: Sidharth Vinod <sidharthv96@gmail.com>
Date: Thu, 30 Sep 2021 09:44:48 +0530
Subject: [PATCH 18/21] docs: Add github reporter docs

---
 docs/src/test-advanced-js.md  |  2 +-
 docs/src/test-reporters-js.md | 28 ++++++++++++++++++++++++++++
 2 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/docs/src/test-advanced-js.md b/docs/src/test-advanced-js.md
index 5d606d91f4cc3..85f105de1a1c6 100644
--- a/docs/src/test-advanced-js.md
+++ b/docs/src/test-advanced-js.md
@@ -38,7 +38,7 @@ These options would be typically different between local development and CI oper
   - `'failures-only'` - only preserve output for failed tests.
 - `projects: Project[]` - Multiple [projects](#projects) configuration.
 - `quiet: boolean` - Whether to suppress stdout and stderr from the tests.
-- `reporter: 'list' | 'line' | 'dot' | 'json' | 'junit'` - The reporter to use. See [reporters](./test-reporters.md) for details.
+- `reporter: 'list' | 'line' | 'dot' | 'json' | 'junit' | 'github'` - The reporter to use. See [reporters](./test-reporters.md) for details.
 - `reportSlowTests: { max: number, threshold: number } | null` - Whether to report slow tests. When `null`, slow tests are not reported. Otherwise, tests that took more than `threshold` milliseconds are reported as slow, but no more than `max` number of them. Passing zero as `max` reports all slow tests that exceed the threshold.
 - `shard: { total: number, current: number } | null` - [Shard](./test-parallel.md#shard-tests-between-multiple-machines) information.
 - `updateSnapshots: boolean` - Whether to update expected snapshots with the actual results produced by the test run.
diff --git a/docs/src/test-reporters-js.md b/docs/src/test-reporters-js.md
index 486ed8de42c03..db8464e2bf381 100644
--- a/docs/src/test-reporters-js.md
+++ b/docs/src/test-reporters-js.md
@@ -98,6 +98,34 @@ const config: PlaywrightTestConfig = {
 export default config;
 ```
 
+### Reporter for GitHub Actions
+
+You can use the built in `github` reporter to get automatic failure annotations when running in GitHub actions.
+
+```js js-flavor=js
+// playwright.config.js
+// @ts-check
+
+/** @type {import('@playwright/test').PlaywrightTestConfig} */
+const config = {
+  // 'github' for GitHub Actions CI to generate annotations, default 'list' when running locally
+  reporter: process.env.CI ? 'github' : 'list',
+};
+
+module.exports = config;
+```
+
+```js js-flavor=ts
+// playwright.config.ts
+import { PlaywrightTestConfig } from '@playwright/test';
+
+const config: PlaywrightTestConfig = {
+  // 'github' for GitHub Actions CI to generate annotations, default 'list' when running locally
+  reporter: process.env.CI ? 'github' : 'list',
+};
+export default config;
+```
+
 ## Built-in reporters
 
 All built-in reporters show detailed information about failures, and mostly differ in verbosity for successful runs.

From 4529cbcc459566ab3b757e4d73f2560e4b3f15d9 Mon Sep 17 00:00:00 2001
From: Sidharth Vinod <sidharthv96@gmail.com>
Date: Thu, 30 Sep 2021 09:59:37 +0530
Subject: [PATCH 19/21] test(github-reporter): Remove only

---
 tests/playwright-test/github-reporter.spec.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/playwright-test/github-reporter.spec.ts b/tests/playwright-test/github-reporter.spec.ts
index a5860b6ba0e4a..cf9ca5854b447 100644
--- a/tests/playwright-test/github-reporter.spec.ts
+++ b/tests/playwright-test/github-reporter.spec.ts
@@ -65,7 +65,7 @@ test('print GitHub annotations for failed tests', async ({ runInlineTest }) => {
 });
 
 
-test.only('print GitHub annotations for slow tests', async ({ runInlineTest }) => {
+test('print GitHub annotations for slow tests', async ({ runInlineTest }) => {
   const result = await runInlineTest({
     'playwright.config.ts': `
       module.exports = {

From b1314e8e1bc3f4b0f4f7f81c1f1473de9f9ff177 Mon Sep 17 00:00:00 2001
From: Sidharth Vinod <sidharthv96@gmail.com>
Date: Thu, 30 Sep 2021 10:50:37 +0530
Subject: [PATCH 20/21] test(github-reporter): Fix test in CI

---
 tests/playwright-test/github-reporter.spec.ts | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tests/playwright-test/github-reporter.spec.ts b/tests/playwright-test/github-reporter.spec.ts
index cf9ca5854b447..3358f3bbacf18 100644
--- a/tests/playwright-test/github-reporter.spec.ts
+++ b/tests/playwright-test/github-reporter.spec.ts
@@ -39,7 +39,7 @@ test('print GitHub annotations with newline if not in CI', async ({ runInlineTes
         expect(1 + 1).toBe(2);
       });
     `
-  }, { reporter: 'github' });
+  }, { reporter: 'github' }, { GITHUB_ACTION: '' });
   const text = stripAscii(result.output);
   expect(text).not.toContain('::error');
   expect(text).toContain(`::notice title=🎭 Playwright Run Summary::
@@ -56,11 +56,11 @@ test('print GitHub annotations for failed tests', async ({ runInlineTest }) => {
         expect(1 + 1).toBe(3);
       });
     `
-  }, { retries: 3, reporter: 'github' }, { GITHUB_ACTION: 'true' });
+  }, { retries: 3, reporter: 'github' }, { GITHUB_ACTION: 'true', GITHUB_WORKSPACE: process.cwd() });
   const text = stripAscii(result.output);
-  expect(text).toContain('::error file=a.test.js,title=a.test.js:6:7 › example,line=7,col=23::  1) a.test.js:6:7 › example =======================================================================%0A%0A    Retry #1');
-  expect(text).toContain('::error file=a.test.js,title=a.test.js:6:7 › example,line=7,col=23::  1) a.test.js:6:7 › example =======================================================================%0A%0A    Retry #2');
-  expect(text).toContain('::error file=a.test.js,title=a.test.js:6:7 › example,line=7,col=23::  1) a.test.js:6:7 › example =======================================================================%0A%0A    Retry #3');
+  expect(text).toContain('::error file=test-results/github-reporter-print-GitHub-annotations-for-failed-tests-playwright-test/a.test.js,title=a.test.js:6:7 › example,line=7,col=23::  1) a.test.js:6:7 › example =======================================================================%0A%0A    Retry #1');
+  expect(text).toContain('::error file=test-results/github-reporter-print-GitHub-annotations-for-failed-tests-playwright-test/a.test.js,title=a.test.js:6:7 › example,line=7,col=23::  1) a.test.js:6:7 › example =======================================================================%0A%0A    Retry #2');
+  expect(text).toContain('::error file=test-results/github-reporter-print-GitHub-annotations-for-failed-tests-playwright-test/a.test.js,title=a.test.js:6:7 › example,line=7,col=23::  1) a.test.js:6:7 › example =======================================================================%0A%0A    Retry #3');
   expect(result.exitCode).toBe(1);
 });
 
@@ -78,7 +78,7 @@ test('print GitHub annotations for slow tests', async ({ runInlineTest }) => {
         await new Promise(f => setTimeout(f, 200));
       });
     `
-  }, { retries: 3, reporter: 'github' }, { GITHUB_ACTION: 'true' });
+  }, { retries: 3, reporter: 'github' }, { GITHUB_ACTION: 'true', GITHUB_WORKSPACE: '' });
   const text = stripAscii(result.output);
   expect(text).toContain('::warning title=Slow Test,file=a.test.js::a.test.js took 2');
   expect(text).toContain('::notice title=🎭 Playwright Run Summary::%0A  1 passed');

From 70785cc0e1d7850e2f9fee79deeefee08e95a0f2 Mon Sep 17 00:00:00 2001
From: Sidharth Vinod <sidharthv96@gmail.com>
Date: Sat, 2 Oct 2021 12:42:15 +0530
Subject: [PATCH 21/21] fix(reporter): Path issue on windows

---
 tests/playwright-test/github-reporter.spec.ts | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/tests/playwright-test/github-reporter.spec.ts b/tests/playwright-test/github-reporter.spec.ts
index 3358f3bbacf18..685b440cc2042 100644
--- a/tests/playwright-test/github-reporter.spec.ts
+++ b/tests/playwright-test/github-reporter.spec.ts
@@ -15,6 +15,7 @@
  */
 
 import { test, expect, stripAscii } from './playwright-test-fixtures';
+import { relativeFilePath } from '../../src/test/util';
 
 test('print GitHub annotations for success', async ({ runInlineTest }) => {
   const result = await runInlineTest({
@@ -48,7 +49,7 @@ test('print GitHub annotations with newline if not in CI', async ({ runInlineTes
 });
 
 
-test('print GitHub annotations for failed tests', async ({ runInlineTest }) => {
+test('print GitHub annotations for failed tests', async ({ runInlineTest }, testInfo) => {
   const result = await runInlineTest({
     'a.test.js': `
       const { test } = pwt;
@@ -58,9 +59,10 @@ test('print GitHub annotations for failed tests', async ({ runInlineTest }) => {
     `
   }, { retries: 3, reporter: 'github' }, { GITHUB_ACTION: 'true', GITHUB_WORKSPACE: process.cwd() });
   const text = stripAscii(result.output);
-  expect(text).toContain('::error file=test-results/github-reporter-print-GitHub-annotations-for-failed-tests-playwright-test/a.test.js,title=a.test.js:6:7 › example,line=7,col=23::  1) a.test.js:6:7 › example =======================================================================%0A%0A    Retry #1');
-  expect(text).toContain('::error file=test-results/github-reporter-print-GitHub-annotations-for-failed-tests-playwright-test/a.test.js,title=a.test.js:6:7 › example,line=7,col=23::  1) a.test.js:6:7 › example =======================================================================%0A%0A    Retry #2');
-  expect(text).toContain('::error file=test-results/github-reporter-print-GitHub-annotations-for-failed-tests-playwright-test/a.test.js,title=a.test.js:6:7 › example,line=7,col=23::  1) a.test.js:6:7 › example =======================================================================%0A%0A    Retry #3');
+  const testPath =  relativeFilePath(testInfo.outputPath('a.test.js'));
+  expect(text).toContain(`::error file=${testPath},title=a.test.js:6:7 › example,line=7,col=23::  1) a.test.js:6:7 › example =======================================================================%0A%0A    Retry #1`);
+  expect(text).toContain(`::error file=${testPath},title=a.test.js:6:7 › example,line=7,col=23::  1) a.test.js:6:7 › example =======================================================================%0A%0A    Retry #2`);
+  expect(text).toContain(`::error file=${testPath},title=a.test.js:6:7 › example,line=7,col=23::  1) a.test.js:6:7 › example =======================================================================%0A%0A    Retry #3`);
   expect(result.exitCode).toBe(1);
 });