From 7c3560a7ebeae8be96e5256f259518724aea67ab Mon Sep 17 00:00:00 2001 From: Brandon Kimberly Date: Tue, 16 Jun 2020 15:33:52 -0400 Subject: [PATCH 01/25] Added code coverage upload job to CI workflow --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0ddda0f986..77332dedc4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -170,3 +170,9 @@ jobs: ./ci/setup_windows_ci_environment.ps1 - name: run tests run: ./ci/do_ci.ps1 cmake.test_example_plugin + + code_coverage: + name: Code coverage report + steps: + - uses: actions/checkout@v2 + - uses: codecov/codecov-action@v1 From df4286d07944c9f2bf2db51ee85aed640b4401a4 Mon Sep 17 00:00:00 2001 From: Brandon Kimberly Date: Tue, 16 Jun 2020 15:36:01 -0400 Subject: [PATCH 02/25] Created codecov.yml --- codecov.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 codecov.yml diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 0000000000..947d9617f8 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,21 @@ +codecov: + require_ci_to_pass: yes + +coverage: + precision: 2 + round: down + range: "70...100" + +parsers: + gcov: + branch_detection: + conditional: yes + loop: yes + method: no + macro: no + +comment: + layout: "reach,diff,flags,tree" + behavior: default + require_changes: no + From ffa72faf6c496363ba646fc7ca1494c780799bc0 Mon Sep 17 00:00:00 2001 From: Brandon Kimberly Date: Tue, 16 Jun 2020 15:37:06 -0400 Subject: [PATCH 03/25] Fixed codecov job --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 77332dedc4..f6aa3c6dfe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -173,6 +173,7 @@ jobs: code_coverage: name: Code coverage report + runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - uses: codecov/codecov-action@v1 From 6eefc24fa41c801220341851ab695b07c22ca296 Mon Sep 17 00:00:00 2001 From: Brandon Kimberly Date: Tue, 16 Jun 2020 15:44:19 -0400 Subject: [PATCH 04/25] Adjusted codecov job to run tests --- .github/workflows/ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f6aa3c6dfe..f3bda3f86c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -172,8 +172,11 @@ jobs: run: ./ci/do_ci.ps1 cmake.test_example_plugin code_coverage: - name: Code coverage report + name: Code coverage runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + - name: run tests + run: ./ci/do_ci.sh code.coverage + - name: upload reports to codecov - uses: codecov/codecov-action@v1 From f4139b2fc8dc0e4daf4a23ad22890fd9ee9885ee Mon Sep 17 00:00:00 2001 From: Brandon Kimberly Date: Tue, 16 Jun 2020 15:54:57 -0400 Subject: [PATCH 05/25] Added setup portion to code coverage job --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f3bda3f86c..81434d5423 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -176,6 +176,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + - name: setup + run: | + sudo ./ci/setup_cmake.sh + sudo ./ci/setup_ci_environment.sh - name: run tests run: ./ci/do_ci.sh code.coverage - name: upload reports to codecov From 5b13fd902a5c3510269a35089d22467a43c7191f Mon Sep 17 00:00:00 2001 From: Brandon Kimberly Date: Tue, 16 Jun 2020 15:57:01 -0400 Subject: [PATCH 06/25] Fixed syntax error --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 81434d5423..767fffe5ce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -183,4 +183,4 @@ jobs: - name: run tests run: ./ci/do_ci.sh code.coverage - name: upload reports to codecov - - uses: codecov/codecov-action@v1 + uses: codecov/codecov-action@v1 From bf36455d3076827e1e146d806d202b7418166ea0 Mon Sep 17 00:00:00 2001 From: Brandon Kimberly Date: Tue, 16 Jun 2020 15:58:37 -0400 Subject: [PATCH 07/25] Added new target for codecov --- ci/do_ci.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index f80ca7d749..e60ea8aa5d 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -106,6 +106,15 @@ elif [[ "$1" == "format" ]]; then exit 1 fi exit 0 +elif [[ "$1" == "code.coverage" ]]; then + cd "${BUILD_DIR}" + rm -rf * + cmake -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_CXX_FLAGS="-Werror --coverage" \ + "${SRC_DIR}" + make + make test + exit 0 fi echo "Invalid do_ci.sh target, see ci/README.md for valid targets." From e0254578616efa152a04bc68b099f4092872b91e Mon Sep 17 00:00:00 2001 From: Brandon Kimberly Date: Tue, 16 Jun 2020 16:10:13 -0400 Subject: [PATCH 08/25] Increased goal range for coverage --- codecov.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codecov.yml b/codecov.yml index 947d9617f8..bbe5f3ae08 100644 --- a/codecov.yml +++ b/codecov.yml @@ -4,7 +4,7 @@ codecov: coverage: precision: 2 round: down - range: "70...100" + range: "80...100" parsers: gcov: From 92a63b1f350b1164c4fd0d0950821b3f0a053b52 Mon Sep 17 00:00:00 2001 From: Brandon Kimberly Date: Tue, 16 Jun 2020 16:17:22 -0400 Subject: [PATCH 09/25] Combined run tests and upload coverage jobs --- .github/workflows/ci.yml | 4 +--- ci/do_ci.sh | 1 + 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 767fffe5ce..e60a5970db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -180,7 +180,5 @@ jobs: run: | sudo ./ci/setup_cmake.sh sudo ./ci/setup_ci_environment.sh - - name: run tests + - name: run tests and upload report run: ./ci/do_ci.sh code.coverage - - name: upload reports to codecov - uses: codecov/codecov-action@v1 diff --git a/ci/do_ci.sh b/ci/do_ci.sh index e60ea8aa5d..e6954adffe 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -114,6 +114,7 @@ elif [[ "$1" == "code.coverage" ]]; then "${SRC_DIR}" make make test + bash <(curl -s https://codecov.io/bash) exit 0 fi From 81257758687d14c3f52b43160cee4a8997a9c394 Mon Sep 17 00:00:00 2001 From: Brandon Kimberly Date: Wed, 17 Jun 2020 11:03:30 -0400 Subject: [PATCH 10/25] Added lcov to generate coverage reports --- ci/do_ci.sh | 3 ++- ci/setup_ci_environment.sh | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index e6954adffe..a1843a9655 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -114,7 +114,8 @@ elif [[ "$1" == "code.coverage" ]]; then "${SRC_DIR}" make make test - bash <(curl -s https://codecov.io/bash) + lcov --directory . --capture --output-file coverage.info + bash <(curl -s https://codecov.io/bash) -f coverage.info exit 0 fi diff --git a/ci/setup_ci_environment.sh b/ci/setup_ci_environment.sh index 50c7b373e9..9d7df6a1d5 100755 --- a/ci/setup_ci_environment.sh +++ b/ci/setup_ci_environment.sh @@ -7,3 +7,4 @@ apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates \ wget \ git +apt-get install -y lcov From 591423c40dc60f7d2e698bbaf9245f80eeeff44f Mon Sep 17 00:00:00 2001 From: Brandon Kimberly Date: Wed, 17 Jun 2020 11:13:31 -0400 Subject: [PATCH 11/25] Moved codecov config file --- codecov.yml => .github/.codecov.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename codecov.yml => .github/.codecov.yml (93%) diff --git a/codecov.yml b/.github/.codecov.yml similarity index 93% rename from codecov.yml rename to .github/.codecov.yml index bbe5f3ae08..ac49457742 100644 --- a/codecov.yml +++ b/.github/.codecov.yml @@ -1,5 +1,6 @@ codecov: require_ci_to_pass: yes + max_report_age: off coverage: precision: 2 @@ -18,4 +19,3 @@ comment: layout: "reach,diff,flags,tree" behavior: default require_changes: no - From 491114b64e1f3dcd46a67d1b7e1727af7c48efad Mon Sep 17 00:00:00 2001 From: Brandon Kimberly Date: Wed, 17 Jun 2020 11:26:03 -0400 Subject: [PATCH 12/25] Added file path fixing --- .github/.codecov.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/.codecov.yml b/.github/.codecov.yml index ac49457742..f62e0381c6 100644 --- a/.github/.codecov.yml +++ b/.github/.codecov.yml @@ -19,3 +19,6 @@ comment: layout: "reach,diff,flags,tree" behavior: default require_changes: no + +fixes: + - "/home/runner/::" From a61bee01ccfe8e406a96b1e26d750534535c3b04 Mon Sep 17 00:00:00 2001 From: Brandon Kimberly Date: Wed, 17 Jun 2020 11:31:15 -0400 Subject: [PATCH 13/25] Moved codecov yaml file --- .github/.codecov.yml | 24 ------------------------ 1 file changed, 24 deletions(-) delete mode 100644 .github/.codecov.yml diff --git a/.github/.codecov.yml b/.github/.codecov.yml deleted file mode 100644 index f62e0381c6..0000000000 --- a/.github/.codecov.yml +++ /dev/null @@ -1,24 +0,0 @@ -codecov: - require_ci_to_pass: yes - max_report_age: off - -coverage: - precision: 2 - round: down - range: "80...100" - -parsers: - gcov: - branch_detection: - conditional: yes - loop: yes - method: no - macro: no - -comment: - layout: "reach,diff,flags,tree" - behavior: default - require_changes: no - -fixes: - - "/home/runner/::" From 483a5650e8c07c3af1daa63ba20b677b94d260ad Mon Sep 17 00:00:00 2001 From: Brandon Kimberly Date: Wed, 17 Jun 2020 11:34:06 -0400 Subject: [PATCH 14/25] Revert previous commit --- .github/.codecov.yaml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/.codecov.yaml diff --git a/.github/.codecov.yaml b/.github/.codecov.yaml new file mode 100644 index 0000000000..f62e0381c6 --- /dev/null +++ b/.github/.codecov.yaml @@ -0,0 +1,24 @@ +codecov: + require_ci_to_pass: yes + max_report_age: off + +coverage: + precision: 2 + round: down + range: "80...100" + +parsers: + gcov: + branch_detection: + conditional: yes + loop: yes + method: no + macro: no + +comment: + layout: "reach,diff,flags,tree" + behavior: default + require_changes: no + +fixes: + - "/home/runner/::" From b67d65b91b0db88c8a60f277b2ad6d8db8782da7 Mon Sep 17 00:00:00 2001 From: Brandon Kimberly Date: Wed, 17 Jun 2020 11:42:40 -0400 Subject: [PATCH 15/25] Specify yaml file location --- ci/do_ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index a1843a9655..25aea7c011 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -115,7 +115,7 @@ elif [[ "$1" == "code.coverage" ]]; then make make test lcov --directory . --capture --output-file coverage.info - bash <(curl -s https://codecov.io/bash) -f coverage.info + bash <(curl -s https://codecov.io/bash) -e yaml=".github/.codecov.yaml" -f coverage.info exit 0 fi From 9d6c5a0a5fe73d90fb3be072eb4bebba4e274a99 Mon Sep 17 00:00:00 2001 From: Brandon Kimberly Date: Wed, 17 Jun 2020 11:55:42 -0400 Subject: [PATCH 16/25] Moved report upload step to workflow job --- .github/workflows/ci.yml | 6 +++++- ci/do_ci.sh | 1 - 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e60a5970db..7a175d2c95 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -180,5 +180,9 @@ jobs: run: | sudo ./ci/setup_cmake.sh sudo ./ci/setup_ci_environment.sh - - name: run tests and upload report + - name: run tests and generate report run: ./ci/do_ci.sh code.coverage + - name: upload report + - uses: codecov/codecov-action@v1 + with: + file: coverage.info diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 25aea7c011..bb3cac5de7 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -115,7 +115,6 @@ elif [[ "$1" == "code.coverage" ]]; then make make test lcov --directory . --capture --output-file coverage.info - bash <(curl -s https://codecov.io/bash) -e yaml=".github/.codecov.yaml" -f coverage.info exit 0 fi From a821e769e72f95e698706bfe40b53b4291e2ba29 Mon Sep 17 00:00:00 2001 From: Brandon Kimberly Date: Wed, 17 Jun 2020 11:57:52 -0400 Subject: [PATCH 17/25] Fixed syntax error --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a175d2c95..3b763b25f1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -183,6 +183,6 @@ jobs: - name: run tests and generate report run: ./ci/do_ci.sh code.coverage - name: upload report - - uses: codecov/codecov-action@v1 + uses: codecov/codecov-action@v1 with: file: coverage.info From a3cf07cb48d5e8c90135c469b8e31ec344c1f670 Mon Sep 17 00:00:00 2001 From: Brandon Kimberly Date: Wed, 17 Jun 2020 12:04:26 -0400 Subject: [PATCH 18/25] Specify file path to coverage report --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3b763b25f1..1fa49ab0af 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -185,4 +185,4 @@ jobs: - name: upload report uses: codecov/codecov-action@v1 with: - file: coverage.info + file: /home/runner/coverage.info From 72627842983112bf5b643484ceab1118c49fcf0c Mon Sep 17 00:00:00 2001 From: Brandon Kimberly Date: Wed, 17 Jun 2020 13:43:24 -0400 Subject: [PATCH 19/25] Changed target dir for coverage report --- ci/do_ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index bb3cac5de7..8cbcafe02d 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -114,7 +114,7 @@ elif [[ "$1" == "code.coverage" ]]; then "${SRC_DIR}" make make test - lcov --directory . --capture --output-file coverage.info + lcov --directory /home/runner --capture --output-file coverage.info exit 0 fi From 071f128809f20b2173c4cadad9449614ee8629f5 Mon Sep 17 00:00:00 2001 From: Brandon Kimberly Date: Wed, 17 Jun 2020 13:46:32 -0400 Subject: [PATCH 20/25] Changed target dir for coverage report --- ci/do_ci.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 8cbcafe02d..757309df56 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -114,7 +114,8 @@ elif [[ "$1" == "code.coverage" ]]; then "${SRC_DIR}" make make test - lcov --directory /home/runner --capture --output-file coverage.info + lcov --directory $PWD --capture --output-file coverage.info + echo $PWD exit 0 fi From 84f9030fe92c4f9781a0cd108aeb1c38fd824374 Mon Sep 17 00:00:00 2001 From: Brandon Kimberly Date: Wed, 17 Jun 2020 13:49:52 -0400 Subject: [PATCH 21/25] Changed path to coverage report upload --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1fa49ab0af..64ea21ebc0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -185,4 +185,4 @@ jobs: - name: upload report uses: codecov/codecov-action@v1 with: - file: /home/runner/coverage.info + file: /home/runner/build/coverage.info From a9f9b5d6f9aa53bd13c8779c92b6b3496d4ae2c5 Mon Sep 17 00:00:00 2001 From: Brandon Kimberly Date: Wed, 17 Jun 2020 14:00:47 -0400 Subject: [PATCH 22/25] Removed echo debug statement --- ci/do_ci.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 757309df56..8f85330778 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -115,7 +115,6 @@ elif [[ "$1" == "code.coverage" ]]; then make make test lcov --directory $PWD --capture --output-file coverage.info - echo $PWD exit 0 fi From 8ef9f058391cff3ec0822472c4f0d45a722daa2d Mon Sep 17 00:00:00 2001 From: Brandon Kimberly Date: Wed, 17 Jun 2020 14:04:33 -0400 Subject: [PATCH 23/25] Added code.coverage job description to README --- ci/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/README.md b/ci/README.md index f755d3f54a..7f0e39df6e 100644 --- a/ci/README.md +++ b/ci/README.md @@ -12,6 +12,7 @@ CI tests can be run on docker by invoking the script `./ci/run_docker.sh ./ci/do * `bazel.tsan`: build bazel targets and run tests with ThreadSanitizer. * `benchmark`: run all benchmarks. * `format`: use `tools/format.sh` to enforce text formatting. +* `code.coverage`: build cmake targets and run tests. Then upload coverage report to [codecov.io](https://codecov.io/) Additionally, `./ci/run_docker.sh` can be invoked with no arguments to get a docker shell where tests can be run manually. From 0f3ee893ff461c62acc16422d3ae1bf4d34e3ea2 Mon Sep 17 00:00:00 2001 From: Brandon Kimberly Date: Thu, 18 Jun 2020 11:50:54 -0400 Subject: [PATCH 24/25] Added comments about file path fixing --- .github/.codecov.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/.codecov.yaml b/.github/.codecov.yaml index f62e0381c6..354ed22dd4 100644 --- a/.github/.codecov.yaml +++ b/.github/.codecov.yaml @@ -20,5 +20,9 @@ comment: behavior: default require_changes: no +# Relative file path fixing. +# CI file paths must match Git file paths. +# This fix removes the "/home/runner/" prefix +# to coverage report file paths. fixes: - "/home/runner/::" From c6ad30bf91693f8a4871cb00be0239b84db64e82 Mon Sep 17 00:00:00 2001 From: Brandon Kimberly Date: Thu, 18 Jun 2020 17:55:37 -0400 Subject: [PATCH 25/25] Fix minor typo Co-authored-by: Reiley Yang --- ci/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/README.md b/ci/README.md index 7f0e39df6e..81b1f40fe4 100644 --- a/ci/README.md +++ b/ci/README.md @@ -12,7 +12,7 @@ CI tests can be run on docker by invoking the script `./ci/run_docker.sh ./ci/do * `bazel.tsan`: build bazel targets and run tests with ThreadSanitizer. * `benchmark`: run all benchmarks. * `format`: use `tools/format.sh` to enforce text formatting. -* `code.coverage`: build cmake targets and run tests. Then upload coverage report to [codecov.io](https://codecov.io/) +* `code.coverage`: build cmake targets and run tests. Then upload coverage report to [codecov.io](https://codecov.io/). Additionally, `./ci/run_docker.sh` can be invoked with no arguments to get a docker shell where tests can be run manually.