From d0a12f2509a9f4887247a2d507bead3659b65897 Mon Sep 17 00:00:00 2001 From: Guilherme Gazzo Date: Wed, 12 Jun 2024 10:48:08 -0300 Subject: [PATCH 1/8] report flaky tests to jira --- .github/workflows/ci-test-e2e.yml | 3 + .github/workflows/ci.yml | 3 + apps/meteor/playwright.config.ts | 11 +++ apps/meteor/reporters/jira.ts | 134 ++++++++++++++++++++++++++++++ 4 files changed, 151 insertions(+) create mode 100644 apps/meteor/reporters/jira.ts diff --git a/.github/workflows/ci-test-e2e.yml b/.github/workflows/ci-test-e2e.yml index 1dc8993bfa870..f1389834ed019 100644 --- a/.github/workflows/ci-test-e2e.yml +++ b/.github/workflows/ci-test-e2e.yml @@ -67,6 +67,8 @@ on: required: false CODECOV_TOKEN: required: false + REPORTER_JIRA_ROCKETCHAT_API_KEY: + required: false env: MONGO_URL: mongodb://localhost:27017/rocketchat?replicaSet=rs0&directConnection=true @@ -243,6 +245,7 @@ jobs: IS_EE: ${{ inputs.release == 'ee' && 'true' || '' }} REPORTER_ROCKETCHAT_API_KEY: ${{ secrets.REPORTER_ROCKETCHAT_API_KEY }} REPORTER_ROCKETCHAT_URL: ${{ secrets.REPORTER_ROCKETCHAT_URL }} + REPORTER_JIRA_ROCKETCHAT_API_KEY: ${{ secrets.REPORTER_JIRA_ROCKETCHAT_API_KEY }} REPORTER_ROCKETCHAT_REPORT: ${{ github.event.pull_request.draft != 'true' && 'true' || '' }} REPORTER_ROCKETCHAT_RUN: ${{ github.run_number }} REPORTER_ROCKETCHAT_BRANCH: ${{ github.ref }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b16ab459d6bd4..344d6372c596f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -337,6 +337,7 @@ jobs: QASE_API_TOKEN: ${{ secrets.QASE_API_TOKEN }} REPORTER_ROCKETCHAT_API_KEY: ${{ secrets.REPORTER_ROCKETCHAT_API_KEY }} REPORTER_ROCKETCHAT_URL: ${{ secrets.REPORTER_ROCKETCHAT_URL }} + REPORTER_JIRA_ROCKETCHAT_API_KEY: ${{ secrets.REPORTER_JIRA_ROCKETCHAT_API_KEY }} test-api-ee: name: 🔨 Test API (EE) @@ -388,6 +389,7 @@ jobs: REPORTER_ROCKETCHAT_API_KEY: ${{ secrets.REPORTER_ROCKETCHAT_API_KEY }} REPORTER_ROCKETCHAT_URL: ${{ secrets.REPORTER_ROCKETCHAT_URL }} CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + REPORTER_JIRA_ROCKETCHAT_API_KEY: ${{ secrets.REPORTER_JIRA_ROCKETCHAT_API_KEY }} test-ui-ee-no-watcher: name: 🔨 Test UI (EE) @@ -418,6 +420,7 @@ jobs: REPORTER_ROCKETCHAT_API_KEY: ${{ secrets.REPORTER_ROCKETCHAT_API_KEY }} REPORTER_ROCKETCHAT_URL: ${{ secrets.REPORTER_ROCKETCHAT_URL }} CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + REPORTER_JIRA_ROCKETCHAT_API_KEY: ${{ secrets.REPORTER_JIRA_ROCKETCHAT_API_KEY }} tests-done: name: ✅ Tests Done diff --git a/apps/meteor/playwright.config.ts b/apps/meteor/playwright.config.ts index d40592b8f71f9..10b670dccadf8 100644 --- a/apps/meteor/playwright.config.ts +++ b/apps/meteor/playwright.config.ts @@ -32,6 +32,17 @@ export default { draft: process.env.REPORTER_ROCKETCHAT_DRAFT === 'true', }, ], + // process.env.REPORTER_ROCKETCHAT_REPORT === 'true' && + [ + './reporters/jira.ts', + { + url: `https://rocketchat.atlassian.net`, + apiKey: process.env.REPORTER_JIRA_ROCKETCHAT_API_KEY ?? process.env.JIRA_TOKEN, + branch: process.env.REPORTER_ROCKETCHAT_BRANCH, + run: Number(process.env.REPORTER_ROCKETCHAT_RUN), + // draft: process.env.REPORTER_ROCKETCHAT_DRAFT === 'true', + }, + ], [ 'playwright-qase-reporter', { diff --git a/apps/meteor/reporters/jira.ts b/apps/meteor/reporters/jira.ts new file mode 100644 index 0000000000000..bce98e553ba5a --- /dev/null +++ b/apps/meteor/reporters/jira.ts @@ -0,0 +1,134 @@ +import type { Reporter, TestCase, TestResult } from '@playwright/test/reporter'; +import fetch from 'node-fetch'; + +class JIRAReporter implements Reporter { + private url: string; + + private apiKey: string; + + private branch: string; + + private draft: boolean; + + private run: number; + + constructor(options: { url: string; apiKey: string; branch: string; draft: boolean; run: number }) { + this.url = options.url; + this.apiKey = options.apiKey; + this.branch = options.branch; + this.draft = options.draft; + this.run = options.run; + } + + async onTestEnd(test: TestCase, result: TestResult) { + if (process.env.REPORTER_ROCKETCHAT_REPORT !== 'true') { + return; + } + + if (this.draft === true) { + return; + } + + if (result.status === 'passed' || result.status === 'skipped') { + return; + } + + const payload = { + name: test.title, + status: result.status, + duration: result.duration, + branch: this.branch, + draft: this.draft, + run: this.run, + }; + + console.log(`Sending test result to JIRA: ${JSON.stringify(payload)}`); + + // first search and check if there is an existing issue + + const search = await fetch( + `${this.url}/rest/api/2/search?${new URLSearchParams({ + jql: `project = FLAKY AND summary ~ '${payload.name}'`, + })}`, + { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Basic ${this.apiKey}`, + }, + }, + ); + + if (!search.ok) { + throw new Error( + `JIRA: Failed to search for existing issue: ${search.statusText}.` + + `${this.url}/rest/api/2/search${new URLSearchParams({ + jql: `project = FLAKY AND summary ~ '${payload.name}'`, + })}`, + ); + } + + const { issues } = await search.json(); + + if ( + issues.some( + (issue: { + fields: { + summary: string; + }; + }) => issue.fields.summary === payload.name, + ) + ) { + return; + } + + const data: { + fields: { + summary: string; + description: string; + issuetype: { + name: string; + }; + project: { + key: string; + }; + }; + } = { + fields: { + summary: payload.name, + description: '', + issuetype: { + name: 'Tech Debt', + }, + project: { + key: 'FLAKY', + }, + }, + }; + + const responseIssue = await fetch(`${this.url}/rest/api/2/issue`, { + method: 'POST', + body: JSON.stringify(data), + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Basic ${this.apiKey}`, + }, + }); + + const issue = (await responseIssue.json()).key; + + await fetch(`${this.url}/rest/api/2/issue/${issue}/comment`, { + method: 'POST', + body: JSON.stringify({ + Body: `Test run ${payload.run} failed +branch: ${payload.branch}`, + }), + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Basic ${this.apiKey}`, + }, + }); + } +} + +export default JIRAReporter; From dfdf36af600e9f1fe6e7cd35705ee50b256470c6 Mon Sep 17 00:00:00 2001 From: Martin Schoeler Date: Thu, 13 Jun 2024 15:58:56 -0300 Subject: [PATCH 2/8] chore: add head sha to jira reporter (#32598) --- .github/workflows/ci-test-e2e.yml | 1 + apps/meteor/playwright.config.ts | 1 + apps/meteor/reporters/jira.ts | 6 +++++- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-test-e2e.yml b/.github/workflows/ci-test-e2e.yml index f1389834ed019..404aef400e8b4 100644 --- a/.github/workflows/ci-test-e2e.yml +++ b/.github/workflows/ci-test-e2e.yml @@ -250,6 +250,7 @@ jobs: REPORTER_ROCKETCHAT_RUN: ${{ github.run_number }} REPORTER_ROCKETCHAT_BRANCH: ${{ github.ref }} REPORTER_ROCKETCHAT_DRAFT: ${{ github.event.pull_request.draft }} + REPORTER_ROCKETCHAT_HEAD_SHA: ${{ github.event.pull_request.head.sha }} QASE_API_TOKEN: ${{ secrets.QASE_API_TOKEN }} QASE_REPORT: ${{ github.ref == 'refs/heads/develop' && 'true' || '' }} CI: true diff --git a/apps/meteor/playwright.config.ts b/apps/meteor/playwright.config.ts index 10b670dccadf8..d084d3a5fca84 100644 --- a/apps/meteor/playwright.config.ts +++ b/apps/meteor/playwright.config.ts @@ -30,6 +30,7 @@ export default { branch: process.env.REPORTER_ROCKETCHAT_BRANCH, run: Number(process.env.REPORTER_ROCKETCHAT_RUN), draft: process.env.REPORTER_ROCKETCHAT_DRAFT === 'true', + headSha: process.env.REPORTER_ROCKETCHAT_HEAD_SHA, }, ], // process.env.REPORTER_ROCKETCHAT_REPORT === 'true' && diff --git a/apps/meteor/reporters/jira.ts b/apps/meteor/reporters/jira.ts index bce98e553ba5a..5bb00b1586326 100644 --- a/apps/meteor/reporters/jira.ts +++ b/apps/meteor/reporters/jira.ts @@ -12,12 +12,15 @@ class JIRAReporter implements Reporter { private run: number; - constructor(options: { url: string; apiKey: string; branch: string; draft: boolean; run: number }) { + private headSha: string; + + constructor(options: { url: string; apiKey: string; branch: string; draft: boolean; run: number; headSha: string }) { this.url = options.url; this.apiKey = options.apiKey; this.branch = options.branch; this.draft = options.draft; this.run = options.run; + this.headSha = options.headSha; } async onTestEnd(test: TestCase, result: TestResult) { @@ -40,6 +43,7 @@ class JIRAReporter implements Reporter { branch: this.branch, draft: this.draft, run: this.run, + headSha: this.headSha, }; console.log(`Sending test result to JIRA: ${JSON.stringify(payload)}`); From 82d597b9fb0aa1890f9bcd5fac95d87f6941151d Mon Sep 17 00:00:00 2001 From: Guilherme Gazzo Date: Thu, 13 Jun 2024 16:00:32 -0300 Subject: [PATCH 3/8] add head sha to comment --- apps/meteor/reporters/jira.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/meteor/reporters/jira.ts b/apps/meteor/reporters/jira.ts index 5bb00b1586326..9f8631e738baa 100644 --- a/apps/meteor/reporters/jira.ts +++ b/apps/meteor/reporters/jira.ts @@ -124,8 +124,9 @@ class JIRAReporter implements Reporter { await fetch(`${this.url}/rest/api/2/issue/${issue}/comment`, { method: 'POST', body: JSON.stringify({ - Body: `Test run ${payload.run} failed -branch: ${payload.branch}`, + body: `Test run ${payload.run} failed +branch: ${payload.branch} +headSha: ${payload.headSha}`, }), headers: { 'Content-Type': 'application/json', From 7675339c88ff305b5a8bfb22f8aaa916692f1d4b Mon Sep 17 00:00:00 2001 From: Guilherme Gazzo Date: Thu, 13 Jun 2024 16:13:15 -0300 Subject: [PATCH 4/8] add comment to previous task --- apps/meteor/reporters/jira.ts | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/apps/meteor/reporters/jira.ts b/apps/meteor/reporters/jira.ts index 9f8631e738baa..5540617e1baca 100644 --- a/apps/meteor/reporters/jira.ts +++ b/apps/meteor/reporters/jira.ts @@ -74,15 +74,27 @@ class JIRAReporter implements Reporter { const { issues } = await search.json(); - if ( - issues.some( - (issue: { - fields: { - summary: string; - }; - }) => issue.fields.summary === payload.name, - ) - ) { + const existing = issues.find( + (issue: { + fields: { + summary: string; + }; + }) => issue.fields.summary === payload.name, + ); + + if (existing) { + await fetch(`${this.url}/rest/api/2/issue/${existing.key}/comment`, { + method: 'POST', + body: JSON.stringify({ + body: `Test run ${payload.run} failed + branch: ${payload.branch} + headSha: ${payload.headSha}`, + }), + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Basic ${this.apiKey}`, + }, + }); return; } From 7301b7ae4e07f665aae77a429b98b07600ed3661 Mon Sep 17 00:00:00 2001 From: Guilherme Gazzo Date: Thu, 13 Jun 2024 17:49:06 -0300 Subject: [PATCH 5/8] add status --- apps/meteor/playwright.config.ts | 1 + apps/meteor/reporters/jira.ts | 2 ++ 2 files changed, 3 insertions(+) diff --git a/apps/meteor/playwright.config.ts b/apps/meteor/playwright.config.ts index d084d3a5fca84..40ca797e1b95c 100644 --- a/apps/meteor/playwright.config.ts +++ b/apps/meteor/playwright.config.ts @@ -41,6 +41,7 @@ export default { apiKey: process.env.REPORTER_JIRA_ROCKETCHAT_API_KEY ?? process.env.JIRA_TOKEN, branch: process.env.REPORTER_ROCKETCHAT_BRANCH, run: Number(process.env.REPORTER_ROCKETCHAT_RUN), + headSha: process.env.REPORTER_ROCKETCHAT_HEAD_SHA, // draft: process.env.REPORTER_ROCKETCHAT_DRAFT === 'true', }, ], diff --git a/apps/meteor/reporters/jira.ts b/apps/meteor/reporters/jira.ts index 5540617e1baca..92fb1e025364d 100644 --- a/apps/meteor/reporters/jira.ts +++ b/apps/meteor/reporters/jira.ts @@ -87,6 +87,7 @@ class JIRAReporter implements Reporter { method: 'POST', body: JSON.stringify({ body: `Test run ${payload.run} failed + status: ${payload.status} branch: ${payload.branch} headSha: ${payload.headSha}`, }), @@ -137,6 +138,7 @@ class JIRAReporter implements Reporter { method: 'POST', body: JSON.stringify({ body: `Test run ${payload.run} failed +status: ${payload.status} branch: ${payload.branch} headSha: ${payload.headSha}`, }), From 6abdcc3aca74048695f202555bfaa10025e3f623 Mon Sep 17 00:00:00 2001 From: Guilherme Gazzo Date: Fri, 21 Jun 2024 15:38:01 -0300 Subject: [PATCH 6/8] .. --- .github/workflows/ci-test-e2e.yml | 3 +++ apps/meteor/playwright.config.ts | 8 +++++--- apps/meteor/reporters/jira.ts | 31 +++++++++++++++++++++++++++---- 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci-test-e2e.yml b/.github/workflows/ci-test-e2e.yml index fea82abc98509..93fa85d902c88 100644 --- a/.github/workflows/ci-test-e2e.yml +++ b/.github/workflows/ci-test-e2e.yml @@ -258,6 +258,9 @@ jobs: REPORTER_ROCKETCHAT_BRANCH: ${{ github.ref }} REPORTER_ROCKETCHAT_DRAFT: ${{ github.event.pull_request.draft }} REPORTER_ROCKETCHAT_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + REPORTER_ROCKETCHAT_AUTHOR: ${{ github.event.pull_request.user.login }} + REPORTER_ROCKETCHAT_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs/${{ github.job }}?pull_number=${{ github.event.pull_request.number }} + REPORTER_ROCKETCHAT_PR: ${{ github.event.pull_request.number }} QASE_API_TOKEN: ${{ secrets.QASE_API_TOKEN }} QASE_REPORT: ${{ github.ref == 'refs/heads/develop' && 'true' || '' }} CI: true diff --git a/apps/meteor/playwright.config.ts b/apps/meteor/playwright.config.ts index 40ca797e1b95c..822f78e28741f 100644 --- a/apps/meteor/playwright.config.ts +++ b/apps/meteor/playwright.config.ts @@ -33,8 +33,7 @@ export default { headSha: process.env.REPORTER_ROCKETCHAT_HEAD_SHA, }, ], - // process.env.REPORTER_ROCKETCHAT_REPORT === 'true' && - [ + process.env.REPORTER_ROCKETCHAT_REPORT === 'true' && [ './reporters/jira.ts', { url: `https://rocketchat.atlassian.net`, @@ -42,7 +41,10 @@ export default { branch: process.env.REPORTER_ROCKETCHAT_BRANCH, run: Number(process.env.REPORTER_ROCKETCHAT_RUN), headSha: process.env.REPORTER_ROCKETCHAT_HEAD_SHA, - // draft: process.env.REPORTER_ROCKETCHAT_DRAFT === 'true', + author: process.env.REPORTER_ROCKETCHAT_AUTHOR, + run_url: process.env.REPORTER_ROCKETCHAT_RUN_URL, + pr: Number(process.env.REPORTER_ROCKETCHAT_PR), + draft: process.env.REPORTER_ROCKETCHAT_DRAFT === 'true', }, ], [ diff --git a/apps/meteor/reporters/jira.ts b/apps/meteor/reporters/jira.ts index 92fb1e025364d..4bf59b9f87ecc 100644 --- a/apps/meteor/reporters/jira.ts +++ b/apps/meteor/reporters/jira.ts @@ -14,13 +14,32 @@ class JIRAReporter implements Reporter { private headSha: string; - constructor(options: { url: string; apiKey: string; branch: string; draft: boolean; run: number; headSha: string }) { + private author: string; + + private run_url: string; + + private pr: number; + + constructor(options: { + url: string; + apiKey: string; + branch: string; + draft: boolean; + run: number; + headSha: string; + author: string; + run_url: string; + pr: number; + }) { this.url = options.url; this.apiKey = options.apiKey; this.branch = options.branch; this.draft = options.draft; this.run = options.run; this.headSha = options.headSha; + this.author = options.author; + this.run_url = options.run_url; + this.pr = options.pr; } async onTestEnd(test: TestCase, result: TestResult) { @@ -134,13 +153,17 @@ class JIRAReporter implements Reporter { const issue = (await responseIssue.json()).key; + const { location } = test; + await fetch(`${this.url}/rest/api/2/issue/${issue}/comment`, { method: 'POST', body: JSON.stringify({ body: `Test run ${payload.run} failed -status: ${payload.status} -branch: ${payload.branch} -headSha: ${payload.headSha}`, +author: ${this.author} +PR: ${this.pr} +https://github.com/RocketChat/Rocket.Chat/blob/${payload.headSha}/apps/meteor/${location.file}#L${location.line}:${location.column} +${this.run_url} +`, }), headers: { 'Content-Type': 'application/json', From 42566c41881caf434b77be94b4d79355ea3d52f3 Mon Sep 17 00:00:00 2001 From: Guilherme Gazzo Date: Fri, 21 Jun 2024 18:41:48 -0300 Subject: [PATCH 7/8] .. --- apps/meteor/reporters/jira.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/apps/meteor/reporters/jira.ts b/apps/meteor/reporters/jira.ts index 4bf59b9f87ecc..8d079a5887b8e 100644 --- a/apps/meteor/reporters/jira.ts +++ b/apps/meteor/reporters/jira.ts @@ -102,13 +102,17 @@ class JIRAReporter implements Reporter { ); if (existing) { + const { location } = test; + await fetch(`${this.url}/rest/api/2/issue/${existing.key}/comment`, { method: 'POST', body: JSON.stringify({ body: `Test run ${payload.run} failed - status: ${payload.status} - branch: ${payload.branch} - headSha: ${payload.headSha}`, +author: ${this.author} +PR: ${this.pr} +https://github.com/RocketChat/Rocket.Chat/blob/${payload.headSha}/apps/meteor/${location.file}#L${location.line}:${location.column} +${this.run_url} +`, }), headers: { 'Content-Type': 'application/json', From f1c950b03c34e414bcd19adec2194d500b2a1ccc Mon Sep 17 00:00:00 2001 From: Guilherme Gazzo Date: Sat, 22 Jun 2024 00:36:34 -0300 Subject: [PATCH 8/8] .. --- .github/workflows/ci-test-e2e.yml | 2 +- apps/meteor/reporters/jira.ts | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-test-e2e.yml b/.github/workflows/ci-test-e2e.yml index 93fa85d902c88..920aea0aa3087 100644 --- a/.github/workflows/ci-test-e2e.yml +++ b/.github/workflows/ci-test-e2e.yml @@ -259,7 +259,7 @@ jobs: REPORTER_ROCKETCHAT_DRAFT: ${{ github.event.pull_request.draft }} REPORTER_ROCKETCHAT_HEAD_SHA: ${{ github.event.pull_request.head.sha }} REPORTER_ROCKETCHAT_AUTHOR: ${{ github.event.pull_request.user.login }} - REPORTER_ROCKETCHAT_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs/${{ github.job }}?pull_number=${{ github.event.pull_request.number }} + REPORTER_ROCKETCHAT_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} REPORTER_ROCKETCHAT_PR: ${{ github.event.pull_request.number }} QASE_API_TOKEN: ${{ secrets.QASE_API_TOKEN }} QASE_REPORT: ${{ github.ref == 'refs/heads/develop' && 'true' || '' }} diff --git a/apps/meteor/reporters/jira.ts b/apps/meteor/reporters/jira.ts index 8d079a5887b8e..706856389003a 100644 --- a/apps/meteor/reporters/jira.ts +++ b/apps/meteor/reporters/jira.ts @@ -110,7 +110,10 @@ class JIRAReporter implements Reporter { body: `Test run ${payload.run} failed author: ${this.author} PR: ${this.pr} -https://github.com/RocketChat/Rocket.Chat/blob/${payload.headSha}/apps/meteor/${location.file}#L${location.line}:${location.column} +https://github.com/RocketChat/Rocket.Chat/blob/${payload.headSha}/${location.file.replace( + '/home/runner/work/Rocket.Chat/Rocket.Chat', + '', + )}#L${location.line}:${location.column} ${this.run_url} `, }), @@ -165,7 +168,10 @@ ${this.run_url} body: `Test run ${payload.run} failed author: ${this.author} PR: ${this.pr} -https://github.com/RocketChat/Rocket.Chat/blob/${payload.headSha}/apps/meteor/${location.file}#L${location.line}:${location.column} +https://github.com/RocketChat/Rocket.Chat/blob/${payload.headSha}/${location.file.replace( + '/home/runner/work/Rocket.Chat/Rocket.Chat', + '', + )}#L${location.line}:${location.column}, ${this.run_url} `, }),