diff --git a/README.md b/README.md index 216a8d7..04f66c3 100644 --- a/README.md +++ b/README.md @@ -9,11 +9,13 @@ Produce an easy-to-read summary of your project's test data as part of your GitH * Integrates easily with your existing GitHub Actions workflow * Produces summaries from JUnit XML and TAP test output * Compatible with most testing tools for most development platforms -* Customizable to show just a summary, just failed tests, or all test results. +* Produces step outputs, so you can pass summary data to other actions +* Customizable to show just a summary, just failed tests, or all test results +* Output can go to the [GitHub job summary](https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-job-summary) (default), to a file or `stdout` Getting Started --------------- -To set up the test summary action, just add a few lines of YAML to your GitHub Actions workflow. For example, if your test harness produces JUnit XML outputs in the `test/results/` directory, and you want to produce a test summary in a file named `test-summary.md`, add a new step to your workflow YAML after your build and test step: +To set up the test summary action, just add a few lines of YAML to your GitHub Actions workflow. For example, if your test harness produces JUnit XML outputs in the `test/results/` directory, and you want the output attached to the job summary, add a new step to your workflow YAML after your build and test step: ```yaml - name: Test Summary @@ -37,11 +39,20 @@ Update `paths` to match the test output file(s) that your test harness produces. > Note the `if: always()` conditional in this workflow step: you should always use this so that the test summary creation step runs _even if_ the previous steps have failed. This allows your test step to fail -- due to failing tests -- but still produce a test summary. -Upload the markdown -------------------- -The prior "getting started" step generates a summary in GitHub-flavored Markdown (GFM). Once the markdown is generated, you can upload it as a build artifact, add it to a pull request comment, or add it to an issue. For example, to upload the markdown generated in the prior example as a build artifact: +Generating and uploading a markdown file +---------------------------------------- +You can also generate the summary in a GitHub-flavored Markdown (GFM) file, and upload it as a build artifact, add it to a pull request comment, or add it to an issue. Use the `output` parameter to define the target file. + +For example, to create a summary and upload the markdown as a build artifact: ```yaml +- name: Test Summary + uses: test-summary/action@v1 + with: + paths: "test/results/**/TEST-*.xml" + output: test-summary.md + if: always() + - name: Upload test summary uses: actions/upload-artifact@v3 with: @@ -50,6 +61,30 @@ The prior "getting started" step generates a summary in GitHub-flavored Markdown if: always() ``` +Outputs +------- +This action also generates several outputs you can reference in other steps, or even from your job or workflow. These outputs are `passed`, `failed`, `skipped`, and `total`. + +For example, you may want to send a summary to Slack: + +```yaml +- name: Test Summary + id: test_summary + uses: test-summary/action@v1 + with: + paths: "test/results/**/TEST-*.xml" + if: always() +- name: Notify Slack + uses: slackapi/slack-github-action@v1.19.0 + with: + payload: |- + { + "message": "${{ steps.test_summary.outputs.passed }}/${{ steps.test_summary.outputs.total }} tests passed" + } + if: always() +``` + + Examples -------- There are examples for setting up a GitHub Actions step with many different platforms [in the examples repository](https://github.com/test-summary/examples). @@ -94,6 +129,7 @@ Options are specified on the [`with` map](https://docs.github.com/en/actions/usi ```yaml - uses: test-summary/action@v2 with: + paths: "test/results/**/TEST-*.xml" output: "test/results/summary.md" ``` @@ -101,6 +137,16 @@ Options are specified on the [`with` map](https://docs.github.com/en/actions/usi This file is [GitHub Flavored Markdown (GFM)](https://github.github.com/gfm/) and may include permitted HTML. +* **`show`: the test results to summarize in a table** (optional) + This controls whether a test summary table is created or not, as well as what tests are included. It could be `all`, `none`, `pass`, `skip`, or `fail`. The default is `fail` - that is, the summary table will only show the failed tests. For example, if you wanted to show failed and skipped tests: + + ```yaml + - uses: test-summary/action@v1 + with: + paths: "test/results/**/TEST-*.xml" + show: "fail, skip" + ``` + FAQ --- * **How is the summary graphic generated? Does any of my data ever leave GitHub?** diff --git a/package-lock.json b/package-lock.json index 8116fde..3da7c08 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "0.0.1", "license": "MIT", "dependencies": { - "@actions/core": "^1.6.0", + "@actions/core": "^1.10.0", "glob": "^7.2.0", "glob-promise": "^4.2.2", "xml2js": "^0.4.23" @@ -27,7 +27,7 @@ "eslint-plugin-github": "^4.3.5", "eslint-plugin-jest": "^26.1.1", "jest": "^27.5.1", - "mocha": "^9.2.1", + "mocha": "^9.2.2", "mocha-junit-reporter": "^2.0.2", "mocha-multi-reporters": "^1.5.1", "prettier": "^2.5.1", @@ -36,19 +36,20 @@ } }, "node_modules/@actions/core": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.6.0.tgz", - "integrity": "sha512-NB1UAZomZlCV/LmJqkLhNTqtKfFXJZAUPcfl/zqG7EfsQdeUJtaWO98SGbuQ3pydJ3fHl2CvI/51OKYlCYYcaw==", + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.10.0.tgz", + "integrity": "sha512-2aZDDa3zrrZbP5ZYg159sNoLRb61nQ7awl5pSvIq5Qpj81vwDzdMRKzkWJGJuwVvWpvZKx7vspJALyvaaIQyug==", "dependencies": { - "@actions/http-client": "^1.0.11" + "@actions/http-client": "^2.0.1", + "uuid": "^8.3.2" } }, "node_modules/@actions/http-client": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-1.0.11.tgz", - "integrity": "sha512-VRYHGQV1rqnROJqdMvGUbY/Kn8vriQe/F9HR2AlYHzmKuM/p3kjNuXhmdBfcVgsvRWTz5C5XW5xvndZrVBuAYg==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.0.1.tgz", + "integrity": "sha512-PIXiMVtz6VvyaRsGY268qvj57hXQEpsYogYOu2nrQhlf+XCGmZstmuZBbAybUl1nQGnvS1k1eEsQ69ZoD7xlSw==", "dependencies": { - "tunnel": "0.0.6" + "tunnel": "^0.0.6" } }, "node_modules/@ampproject/remapping": { @@ -5028,9 +5029,9 @@ } }, "node_modules/mocha": { - "version": "9.2.1", - "resolved": "https://registry.npmjs.org/mocha/-/mocha-9.2.1.tgz", - "integrity": "sha512-T7uscqjJVS46Pq1XDXyo9Uvey9gd3huT/DD9cYBb4K2Xc/vbKRPUWK067bxDQRK0yIz6Jxk73IrnimvASzBNAQ==", + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-9.2.2.tgz", + "integrity": "sha512-L6XC3EdwT6YrIk0yXpavvLkn8h+EU+Y5UcCHKECyMbdUIxyMuZj4bX4U9e1nvnvUUvQVsV2VHQr5zLdcUkhW/g==", "dev": true, "dependencies": { "@ungap/promise-all-settled": "1.1.2", @@ -5046,9 +5047,9 @@ "he": "1.2.0", "js-yaml": "4.1.0", "log-symbols": "4.1.0", - "minimatch": "3.0.4", + "minimatch": "4.2.1", "ms": "2.1.3", - "nanoid": "3.2.0", + "nanoid": "3.3.1", "serialize-javascript": "6.0.0", "strip-json-comments": "3.1.1", "supports-color": "8.1.1", @@ -5149,15 +5150,15 @@ } }, "node_modules/mocha/node_modules/minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-4.2.1.tgz", + "integrity": "sha512-9Uq1ChtSZO+Mxa/CL1eGizn2vRn3MlLgzhT0Iz8zaY8NdvxvB0d5QdPFmCKf7JKA9Lerx5vRrnwO03jsSfGG9g==", "dev": true, "dependencies": { "brace-expansion": "^1.1.7" }, "engines": { - "node": "*" + "node": ">=10" } }, "node_modules/mocha/node_modules/ms": { @@ -5236,9 +5237,9 @@ "dev": true }, "node_modules/nanoid": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.2.0.tgz", - "integrity": "sha512-fmsZYa9lpn69Ad5eDn7FMcnnSR+8R34W9qJEijxYhTbfOWzr22n1QxCMzXLK+ODyW2973V3Fux959iQoUxzUIA==", + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.1.tgz", + "integrity": "sha512-n6Vs/3KGyxPQd6uO0eH4Bv0ojGSUvuLlIHtC3Y0kEO23YRge8H9x1GCzLn28YX0H66pMkxuaeESFq4tKISKwdw==", "dev": true, "bin": { "nanoid": "bin/nanoid.cjs" @@ -6463,6 +6464,14 @@ "punycode": "^2.1.0" } }, + "node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/v8-compile-cache": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz", @@ -6798,19 +6807,20 @@ }, "dependencies": { "@actions/core": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.6.0.tgz", - "integrity": "sha512-NB1UAZomZlCV/LmJqkLhNTqtKfFXJZAUPcfl/zqG7EfsQdeUJtaWO98SGbuQ3pydJ3fHl2CvI/51OKYlCYYcaw==", + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.10.0.tgz", + "integrity": "sha512-2aZDDa3zrrZbP5ZYg159sNoLRb61nQ7awl5pSvIq5Qpj81vwDzdMRKzkWJGJuwVvWpvZKx7vspJALyvaaIQyug==", "requires": { - "@actions/http-client": "^1.0.11" + "@actions/http-client": "^2.0.1", + "uuid": "^8.3.2" } }, "@actions/http-client": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-1.0.11.tgz", - "integrity": "sha512-VRYHGQV1rqnROJqdMvGUbY/Kn8vriQe/F9HR2AlYHzmKuM/p3kjNuXhmdBfcVgsvRWTz5C5XW5xvndZrVBuAYg==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.0.1.tgz", + "integrity": "sha512-PIXiMVtz6VvyaRsGY268qvj57hXQEpsYogYOu2nrQhlf+XCGmZstmuZBbAybUl1nQGnvS1k1eEsQ69ZoD7xlSw==", "requires": { - "tunnel": "0.0.6" + "tunnel": "^0.0.6" } }, "@ampproject/remapping": { @@ -10586,9 +10596,9 @@ } }, "mocha": { - "version": "9.2.1", - "resolved": "https://registry.npmjs.org/mocha/-/mocha-9.2.1.tgz", - "integrity": "sha512-T7uscqjJVS46Pq1XDXyo9Uvey9gd3huT/DD9cYBb4K2Xc/vbKRPUWK067bxDQRK0yIz6Jxk73IrnimvASzBNAQ==", + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-9.2.2.tgz", + "integrity": "sha512-L6XC3EdwT6YrIk0yXpavvLkn8h+EU+Y5UcCHKECyMbdUIxyMuZj4bX4U9e1nvnvUUvQVsV2VHQr5zLdcUkhW/g==", "dev": true, "requires": { "@ungap/promise-all-settled": "1.1.2", @@ -10604,9 +10614,9 @@ "he": "1.2.0", "js-yaml": "4.1.0", "log-symbols": "4.1.0", - "minimatch": "3.0.4", + "minimatch": "4.2.1", "ms": "2.1.3", - "nanoid": "3.2.0", + "nanoid": "3.3.1", "serialize-javascript": "6.0.0", "strip-json-comments": "3.1.1", "supports-color": "8.1.1", @@ -10637,9 +10647,9 @@ } }, "minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-4.2.1.tgz", + "integrity": "sha512-9Uq1ChtSZO+Mxa/CL1eGizn2vRn3MlLgzhT0Iz8zaY8NdvxvB0d5QdPFmCKf7JKA9Lerx5vRrnwO03jsSfGG9g==", "dev": true, "requires": { "brace-expansion": "^1.1.7" @@ -10739,9 +10749,9 @@ "dev": true }, "nanoid": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.2.0.tgz", - "integrity": "sha512-fmsZYa9lpn69Ad5eDn7FMcnnSR+8R34W9qJEijxYhTbfOWzr22n1QxCMzXLK+ODyW2973V3Fux959iQoUxzUIA==", + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.1.tgz", + "integrity": "sha512-n6Vs/3KGyxPQd6uO0eH4Bv0ojGSUvuLlIHtC3Y0kEO23YRge8H9x1GCzLn28YX0H66pMkxuaeESFq4tKISKwdw==", "dev": true }, "natural-compare": { @@ -11622,6 +11632,11 @@ "punycode": "^2.1.0" } }, + "uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" + }, "v8-compile-cache": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz", diff --git a/package.json b/package.json index 8201f94..ba546fb 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "author": "Edward Thomson", "license": "MIT", "dependencies": { - "@actions/core": "^1.6.0", + "@actions/core": "^1.10.0", "glob": "^7.2.0", "glob-promise": "^4.2.2", "xml2js": "^0.4.23" @@ -47,7 +47,7 @@ "eslint-plugin-github": "^4.3.5", "eslint-plugin-jest": "^26.1.1", "jest": "^27.5.1", - "mocha": "^9.2.1", + "mocha": "^9.2.2", "mocha-junit-reporter": "^2.0.2", "mocha-multi-reporters": "^1.5.1", "prettier": "^2.5.1", diff --git a/src/index.ts b/src/index.ts index f906127..8e91ba2 100644 --- a/src/index.ts +++ b/src/index.ts @@ -115,6 +115,12 @@ async function run(): Promise { const writefile = util.promisify(fs.writeFile) await writefile(outputFile, output) } + + core.setOutput('passed', total.counts.passed) + core.setOutput('failed', total.counts.failed) + core.setOutput('skipped', total.counts.skipped) + core.setOutput('total', total.counts.passed + total.counts.failed + total.counts.skipped) + } catch (error) { if (error instanceof Error) { core.setFailed(error.message) diff --git a/src/test_parser.ts b/src/test_parser.ts index bc79e42..3d114b0 100644 --- a/src/test_parser.ts +++ b/src/test_parser.ts @@ -241,9 +241,9 @@ async function parseJunitXml(xml: any): Promise { status = TestStatus.Skip counts.skipped++ - } else if (testcase.failure) { + } else if (testcase.failure || testcase.error) { status = TestStatus.Fail - details = testcase.failure[0]._ + details = (testcase.failure || testcase.error)[0]._ counts.failed++ } else { diff --git a/test/junit.ts b/test/junit.ts index 8680f54..841326d 100644 --- a/test/junit.ts +++ b/test/junit.ts @@ -87,4 +87,23 @@ describe("junit", async () => { expect(result.suites[0].cases[8].name).to.eql("skipsTestNine") expect(result.suites[0].cases[9].name).to.eql("skipsTestTen") }) + + it("parses bazel", async() => { + // Not a perfect example of Bazel JUnit output - it typically does one file + // per test target, and aggregates all the test cases from the test tooling + // into one Junit testsuite / testcase. This does depend on the actual + // test platform; my experience is mostly with py_test() targets. + const result = await parseJunitFile(`${resourcePath}/04-bazel-junit.xml`) + + expect(result.counts.passed).to.eql(1) + expect(result.counts.failed).to.eql(1) + expect(result.counts.skipped).to.eql(0) + + expect(result.suites.length).to.eql(2) + + expect(result.suites[0].cases[0].name).to.eql("dummy/path/to/project/and/failing_test_target") + expect(result.suites[0].cases[0].status).to.eql(TestStatus.Fail) + expect(result.suites[1].cases[0].name).to.eql("dummy/path/to/project/and/passing_test_target") + expect(result.suites[1].cases[0].status).to.eql(TestStatus.Pass) + }) }) diff --git a/test/resources/junit/04-bazel-junit.xml b/test/resources/junit/04-bazel-junit.xml new file mode 100644 index 0000000..400af08 --- /dev/null +++ b/test/resources/junit/04-bazel-junit.xml @@ -0,0 +1,29 @@ + + + + + +Generated test.log (if the file is not UTF-8, then this may be unreadable): + + + + + + +Generated test.log (if the file is not UTF-8, then this may be unreadable): + + + +