feat(reporter): Add github actions reporter

src/test/reporters/base.ts

@@ -25,9 +25,12 @@ import { FullConfig, TestCase, Suite, TestResult, TestError, Reporter, FullResul
 
 const stackUtils = new StackUtils();
 
-type TestResultOutput = { chunk: string | Buffer, type: 'stdout' | 'stderr' };
-const kOutputSymbol = Symbol('output');
-
+export type TestResultOutput = { chunk: string | Buffer, type: 'stdout' | 'stderr' };
+export const kOutputSymbol = Symbol('output');
+export interface Position {
+  column: number;
+  line: number;
+}
 export class BaseReporter implements Reporter  {
   duration = 0;
   config!: FullConfig;
@@ -215,11 +218,11 @@ export function formatResultFailure(test: TestCase, result: TestResult, initialI
   return resultTokens;
 }
 
-function relativeTestPath(config: FullConfig, test: TestCase): string {
+export function relativeTestPath(config: FullConfig, test: TestCase): string {
   return path.relative(config.rootDir, test.location.file) || path.basename(test.location.file);
 }
 
-function stepSuffix(step: TestStep | undefined) {
+export function stepSuffix(step: TestStep | undefined) {
   const stepTitles = step ? step.titlePath() : [];
   return stepTitles.map(t => ' › ' + t).join('');
 }
@@ -232,7 +235,7 @@ export function formatTestTitle(config: FullConfig, test: TestCase, step?: TestS
   return `${projectTitle}${location} › ${titles.join(' ')}${stepSuffix(step)}`;
 }
 
-function formatTestHeader(config: FullConfig, test: TestCase, indent: string, index?: number): string {
+export function formatTestHeader(config: FullConfig, test: TestCase, indent: string, index?: number): string {
   const title = formatTestTitle(config, test);
   const header = `${indent}${index ? index + ') ' : ''}${title}`;
   return pad(header, '=');
@@ -273,11 +276,11 @@ function pad(line: string, char: string): string {
   return line + colors.gray(char.repeat(Math.max(0, 100 - line.length)));
 }
 
-function indent(lines: string, tab: string) {
+export function indent(lines: string, tab: string) {
   return lines.replace(/^(?=.+$)/gm, tab);
 }
 
-function positionInFile(stackLines: string[], file: string): { column: number; line: number; } | undefined {
+export function positionInFile(stackLines: string[], file: string): Position | undefined {
   // Stack will have /private/var/folders instead of /var/folders on Mac.
   file = fs.realpathSync(file);
   for (const line of stackLines) {

src/test/reporters/github.ts

@@ -0,0 +1,346 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { codeFrameColumns } from '@babel/code-frame';
+import fs from 'fs';
+// @ts-ignore
+import milliseconds from 'ms';
+import path from 'path';
+import {
+  FullConfig,
+  TestCase,
+  TestResult,
+  TestError,
+  FullResult,
+  TestStep,
+} from '../../../types/testReporter';
+import {
+  relativeTestPath,
+  Position,
+  stripAnsiEscapes,
+  indent,
+  stepSuffix,
+  positionInFile,
+  formatTestHeader,
+  BaseReporter,
+  kOutputSymbol,
+  TestResultOutput,
+} from './base';
+
+type GithubLogType = 'debug' | 'notice' | 'warning' | 'error';
+
+type GithubLogOptions = Partial<{
+  title: string;
+  file: string;
+  col: number;
+  endColumn: number;
+  line: number;
+  endLine: number;
+}>;
+
+type Annotation = {
+  filePath: string;
+  title: string;
+  message: string;
+  position?: Position;
+};
+
+type FailureDetails = {
+  tokens: string[];
+  position?: Position;
+};
+
+type ErrorDetails = {
+  message: string;
+  position?: Position;
+};
+class GithubLogger {
+	private _isCI: boolean = process.env.CI === 'true';
+  private _isGithubAction: boolean = process.env.GITHUB_ACTION !== undefined;
+  private _shouldLog = (this._isCI && this._isGithubAction) || process.env.PW_GH_ACTION_DEBUG === 'true' ;
+
+  private _log(
+    message: string,
+    type: GithubLogType = 'notice',
+    options: GithubLogOptions = {}
+  ) {
+    if (this._shouldLog) {
+      if (this._isGithubAction) message = message.replace(/\n/g, '%0A');
+
+      const configs = Object.entries(options)
+          .map(([key, option]) => `${key}=${option}`)
+          .join(',');
+      console.log(`::${type} ${configs}::${message}`);
+    }
+  }
+
+  debug(message: string, options?: GithubLogOptions) {
+    this._log(message, 'debug', options);
+  }
+
+  error(message: string, options?: GithubLogOptions) {
+    this._log(message, 'error', options);
+  }
+
+  notice(message: string, options?: GithubLogOptions) {
+    this._log(message, 'notice', options);
+  }
+
+  warning(message: string, options?: GithubLogOptions) {
+    this._log(message, 'warning', options);
+  }
+}
+
+export class GithubReporter extends BaseReporter {
+  githubLogger = new GithubLogger();
+
+  override async onEnd(result: FullResult) {
+    super.onEnd(result);
+    this.epilogue(true);
+  }
+
+  private _printSlowTestAnnotations() {
+    if (!this.config.reportSlowTests) return;
+    const fileDurations = [...this.fileDurations.entries()];
+    fileDurations.sort((a, b) => b[1] - a[1]);
+    const count = Math.min(
+        fileDurations.length,
+        this.config.reportSlowTests.max || Number.POSITIVE_INFINITY
+    );
+    for (let i = 0; i < count; ++i) {
+      const duration = fileDurations[i][1];
+      if (duration <= this.config.reportSlowTests.threshold) break;
+      const filePath = workspaceRelativePath(
+          path.join(process.cwd(), fileDurations[i][0])
+      );
+      this.githubLogger.warning(`${filePath} (${milliseconds(duration)})`, {
+        title: 'Slow Test',
+        file: filePath,
+      });
+    }
+  }
+
+  override epilogue(full: boolean) {
+    let skipped = 0;
+    let expected = 0;
+    const skippedWithError: TestCase[] = [];
+    const unexpected: TestCase[] = [];
+    const flaky: TestCase[] = [];
+
+    this.suite.allTests().forEach(test => {
+      switch (test.outcome()) {
+        case 'skipped': {
+          ++skipped;
+          if (test.results.some(result => !!result.error))
+            skippedWithError.push(test);
+          break;
+        }
+        case 'expected':
+          ++expected;
+          break;
+        case 'unexpected':
+          unexpected.push(test);
+          break;
+        case 'flaky':
+          flaky.push(test);
+          break;
+      }
+    });
+
+    const noticeLines: string[] = [];
+    noticeLines.push('');
+    if (unexpected.length) {
+      noticeLines.push(`  ${unexpected.length} failed`);
+      for (const test of unexpected)
+        noticeLines.push(formatTestHeader(this.config, test, '    '));
+    }
+    if (flaky.length) {
+      noticeLines.push(`  ${flaky.length} flaky`);
+      for (const test of flaky)
+        noticeLines.push(formatTestHeader(this.config, test, '    '));
+    }
+    if (skipped) noticeLines.push(`  ${skipped} skipped`);
+    if (expected) {
+      noticeLines.push(
+          `  ${expected} passed` + ` (${milliseconds(this.duration)})`
+      );
+    }
+    if (this.result.status === 'timedout') {
+      noticeLines.push(
+          `  Timed out waiting ${
+            this.config.globalTimeout / 1000
+          }s for the entire test run`
+      );
+    }
+
+    this.githubLogger.notice(noticeLines.join('\n'), {
+      title: '🎭 Playwright Run Summary',
+    });
+
+    const failuresToPrint = [...unexpected, ...flaky, ...skippedWithError];
+    if (full && failuresToPrint.length)
+      this._printFailureAnnotations(failuresToPrint);
+
+    this._printSlowTestAnnotations();
+  }
+
+  private _printFailureAnnotations(failures: TestCase[]) {
+    failures.forEach((test, index) => {
+      const annotations = formatFailure(this.config, test, index + 1, true);
+      annotations.forEach(({ filePath, title, message, position }) => {
+        const options: GithubLogOptions = {
+          file: filePath,
+          title,
+        };
+        if (position) {
+          options.line = position.line;
+          options.col = position.column;
+        }
+        this.githubLogger.error(message, options);
+      });
+    });
+  }
+}
+
+function workspaceRelativePath(filePath: string): string {
+  return path.relative(process.env['GITHUB_WORKSPACE'] ?? '', filePath);
+}
+
+function formatFailure(
+  config: FullConfig,
+  test: TestCase,
+  index?: number,
+  stdio?: boolean
+): Annotation[] {
+  const title = formatTestTitle(config, test);
+  const filePath = workspaceRelativePath(test.location.file);
+  const annotations: Annotation[] = [];
+  for (const result of test.results) {
+    const lines: string[] = [];
+    lines.push(formatTestHeader(config, test, '  ', index));
+    const failureDetails = formatResultFailure(test, result, '    ');
+    const resultTokens = failureDetails.tokens;
+    const position = failureDetails.position;
+    if (!resultTokens.length) continue;
+    if (result.retry) {
+      lines.push('');
+      lines.push(`    Retry #${result.retry}`);
+    }
+    lines.push(...resultTokens);
+
+    const output = ((result as any)[kOutputSymbol] || []) as TestResultOutput[];
+    if (stdio && output.length) {
+      const outputText = output
+          .map(({ chunk, type }) => {
+            const text = chunk.toString('utf8');
+            if (type === 'stderr') return stripAnsiEscapes(text);
+            return text;
+          })
+          .join('');
+      lines.push('');
+      lines.push('--- Test output ---' + '\n\n' + outputText + '\n');
+    }
+
+    lines.push('');
+    annotations.push({
+      filePath,
+      position,
+      title,
+      message: lines.join('\n'),
+    });
+  }
+
+  return annotations;
+}
+
+function formatResultFailure(
+  test: TestCase,
+  result: TestResult,
+  initialIndent: string
+): FailureDetails {
+  const resultTokens: string[] = [];
+  if (result.status === 'timedOut') {
+    resultTokens.push('');
+    resultTokens.push(
+        indent(`Timeout of ${test.timeout}ms exceeded.`, initialIndent)
+    );
+  }
+  if (result.status === 'passed' && test.expectedStatus === 'failed') {
+    resultTokens.push('');
+    resultTokens.push(indent(`Expected to fail, but passed.`, initialIndent));
+  }
+  let error: ErrorDetails | undefined = undefined;
+  if (result.error !== undefined) {
+    error = formatError(result.error, test.location.file);
+    resultTokens.push(indent(error.message, initialIndent));
+  }
+  return {
+    tokens: resultTokens,
+    position: error?.position,
+  };
+}
+
+function formatTestTitle(
+  config: FullConfig,
+  test: TestCase,
+  step?: TestStep
+): string {
+  // root, project, file, ...describes, test
+  const [, projectName, , ...titles] = test.titlePath();
+  const location = `${relativeTestPath(config, test)}:${test.location.line}:${
+    test.location.column
+  }`;
+  const projectTitle = projectName ? `[${projectName}] › ` : '';
+  return `${projectTitle}${location} › ${titles.join(' ')}${stepSuffix(step)}`;
+}
+
+function formatError(error: TestError, file?: string): ErrorDetails {
+  const stack = error.stack;
+  const tokens = [''];
+  let position: Position | undefined;
+
+  if (stack) {
+    const lines = stack.split('\n');
+    let firstStackLine = lines.findIndex(line => line.startsWith('    at '));
+    if (firstStackLine === -1) firstStackLine = lines.length;
+    tokens.push(lines.slice(0, firstStackLine).join('\n'));
+    const stackLines = lines.slice(firstStackLine);
+    position = file ? positionInFile(stackLines, file) : undefined;
+    if (position) {
+      const source = fs.readFileSync(file!, 'utf8');
+      tokens.push('');
+      tokens.push(
+          codeFrameColumns(
+              source,
+              { start: position },
+              { highlightCode: false }
+          )
+      );
+    }
+    tokens.push('');
+    tokens.push(stackLines.join('\n'));
+  } else if (error.message) {
+    tokens.push(error.message);
+  } else if (error.value) {
+    tokens.push(error.value);
+  }
+  return {
+    position,
+    message: tokens.join('\n'),
+  };
+}
+
+export default GithubReporter;