From bfd39952d83eeaad9c2567284fe582fbfd747aa1 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 26 Aug 2022 11:15:22 +0100 Subject: [PATCH 1/5] Print complement failure results last Since github always scrolls to the bottom of any test output, let's put the failed tests last and hide any successful packages. --- .ci/complement_package.gotpl | 14 ++++++-------- .github/workflows/latest_deps.yml | 2 +- .github/workflows/tests.yml | 2 +- .github/workflows/twisted_trunk.yml | 2 +- changelog.d/13639.misc | 1 + 5 files changed, 10 insertions(+), 11 deletions(-) create mode 100644 changelog.d/13639.misc diff --git a/.ci/complement_package.gotpl b/.ci/complement_package.gotpl index e1625fd31fe4..2dd5e5e0e695 100644 --- a/.ci/complement_package.gotpl +++ b/.ci/complement_package.gotpl @@ -27,10 +27,10 @@ which is under the Unlicense licence. {{- . -}}{{- "\n" -}} {{- end -}} {{- with .TestCases -}} - {{- /* Failing tests are first */ -}} + {{- /* Passing tests are first */ -}} {{- range . -}} - {{- if and (ne .Result "PASS") (ne .Result "SKIP") -}} - ::group::{{ "\033" }}[0;31m❌{{ " " }}{{- .Name -}} + {{- if eq .Result "PASS" -}} + ::group::{{ "\033" }}[0;32m✅{{ " " }}{{- .Name -}} {{- "\033" -}}[0;37m ({{if $settings.ShowTestStatus}}{{.Result}}; {{end}}{{ .Duration -}} {{- with .Coverage -}} , coverage: {{ . }}% @@ -47,7 +47,6 @@ which is under the Unlicense licence. {{- end -}} {{- end -}} - {{- /* Then skipped tests are second */ -}} {{- range . -}} {{- if eq .Result "SKIP" -}} @@ -68,11 +67,10 @@ which is under the Unlicense licence. {{- end -}} {{- end -}} - - {{- /* Then passing tests are last */ -}} + {{- /* and failing tests are last */ -}} {{- range . -}} - {{- if eq .Result "PASS" -}} - ::group::{{ "\033" }}[0;32m✅{{ " " }}{{- .Name -}} + {{- if and (ne .Result "PASS") (ne .Result "SKIP") -}} + ::group::{{ "\033" }}[0;31m❌{{ " " }}{{- .Name -}} {{- "\033" -}}[0;37m ({{if $settings.ShowTestStatus}}{{.Result}}; {{end}}{{ .Duration -}} {{- with .Coverage -}} , coverage: {{ . }}% diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index f263cf612dc5..270947f34909 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -163,7 +163,7 @@ jobs: - run: | set -o pipefail - TEST_ONLY_IGNORE_POETRY_LOCKFILE=1 POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | gotestfmt + TEST_ONLY_IGNORE_POETRY_LOCKFILE=1 POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | gotestfmt -hide successful-packages shell: bash name: Run Complement Tests diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 144cb9ffaac2..3b9af8bb62b2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -354,7 +354,7 @@ jobs: - run: | set -o pipefail - POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | gotestfmt + POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | gotestfmt -hide successful-packages shell: bash name: Run Complement Tests diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index dd8e6fbb1cc9..efea07d5113b 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -137,7 +137,7 @@ jobs: - run: | set -o pipefail - TEST_ONLY_SKIP_DEP_HASH_VERIFICATION=1 POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | gotestfmt + TEST_ONLY_SKIP_DEP_HASH_VERIFICATION=1 POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | gotestfmt -hide successful-packages shell: bash name: Run Complement Tests diff --git a/changelog.d/13639.misc b/changelog.d/13639.misc new file mode 100644 index 000000000000..de4e4d120632 --- /dev/null +++ b/changelog.d/13639.misc @@ -0,0 +1 @@ +Improve readability of Complement CI logs by printing failure results last. From 8c7a03b1215d61695cfc023b014eadf11a6a6a02 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 26 Aug 2022 15:19:03 +0100 Subject: [PATCH 2/5] rerun gotestfmt if the tests passed --- .ci/scripts/gotestfmt | 15 +++++++++++++++ .github/workflows/latest_deps.yml | 2 +- .github/workflows/tests.yml | 2 +- .github/workflows/twisted_trunk.yml | 2 +- 4 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 .ci/scripts/gotestfmt diff --git a/.ci/scripts/gotestfmt b/.ci/scripts/gotestfmt new file mode 100644 index 000000000000..f4ddb1cf2b81 --- /dev/null +++ b/.ci/scripts/gotestfmt @@ -0,0 +1,15 @@ +#!/bin/sh +# +# wraps `gotestfmt` + +set -o pipefail +set -e + +# tee the test results to a log, and pipe them into gotestfmt, telling it to hide successful results, +# so that we can clearly see unsuccessful results +tee complement.json | gotestfmt -hide successful-packages + +# gotestfmt will exit non-zero if there were any failures, so if we got to this +# point, we must have had a successful result. Pipe the results back through +# gotestfmt, showing all results. +gotestfmt -hide successful-downloads < complement.json diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index 270947f34909..7dac617c4bec 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -163,7 +163,7 @@ jobs: - run: | set -o pipefail - TEST_ONLY_IGNORE_POETRY_LOCKFILE=1 POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | gotestfmt -hide successful-packages + TEST_ONLY_IGNORE_POETRY_LOCKFILE=1 POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | synapse/.ci/scripts/gotestfmt shell: bash name: Run Complement Tests diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3b9af8bb62b2..3e6d718a7c4d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -354,7 +354,7 @@ jobs: - run: | set -o pipefail - POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | gotestfmt -hide successful-packages + POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | synapse/.ci/scripts/gotestfmt shell: bash name: Run Complement Tests diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index efea07d5113b..0906101cc1c1 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -137,7 +137,7 @@ jobs: - run: | set -o pipefail - TEST_ONLY_SKIP_DEP_HASH_VERIFICATION=1 POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | gotestfmt -hide successful-packages + TEST_ONLY_SKIP_DEP_HASH_VERIFICATION=1 POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | synapse/.ci/scripts/gotestfmt shell: bash name: Run Complement Tests From 083714a6cdbeb3acecbcb520796f95f8be10ebb5 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 26 Aug 2022 15:33:26 +0100 Subject: [PATCH 3/5] fix perms --- .ci/scripts/gotestfmt | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 .ci/scripts/gotestfmt diff --git a/.ci/scripts/gotestfmt b/.ci/scripts/gotestfmt old mode 100644 new mode 100755 From 52a2e84ea22351bfb3764f11ab5751d6162ba6ff Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 26 Aug 2022 16:03:39 +0100 Subject: [PATCH 4/5] bash --- .ci/scripts/gotestfmt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/scripts/gotestfmt b/.ci/scripts/gotestfmt index f4ddb1cf2b81..d667ee0d87e7 100755 --- a/.ci/scripts/gotestfmt +++ b/.ci/scripts/gotestfmt @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # # wraps `gotestfmt` From 4f765c6a215ceb1a267169a8d57ad5a0b31d8dcf Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sun, 28 Aug 2022 18:17:28 +0100 Subject: [PATCH 5/5] Avoid repeating successful output --- .ci/scripts/gotestfmt | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/.ci/scripts/gotestfmt b/.ci/scripts/gotestfmt index d667ee0d87e7..83e0ec63613c 100755 --- a/.ci/scripts/gotestfmt +++ b/.ci/scripts/gotestfmt @@ -1,15 +1,21 @@ #!/bin/bash # -# wraps `gotestfmt` +# wraps `gotestfmt`, hiding output from successful packages unless +# all tests passed. set -o pipefail set -e -# tee the test results to a log, and pipe them into gotestfmt, telling it to hide successful results, -# so that we can clearly see unsuccessful results -tee complement.json | gotestfmt -hide successful-packages +# tee the test results to a log, whilst also piping them into gotestfmt, +# telling it to hide successful results, so that we can clearly see +# unsuccessful results. +tee complement.log | gotestfmt -hide successful-packages # gotestfmt will exit non-zero if there were any failures, so if we got to this -# point, we must have had a successful result. Pipe the results back through -# gotestfmt, showing all results. -gotestfmt -hide successful-downloads < complement.json +# point, we must have had a successful result. +echo "All tests successful; showing all test results" + +# Pipe the test results back through gotestfmt, showing all results. +# The log file consists of JSON lines giving the test results, interspersed +# with regular stdout lines (including reports of downloaded packages). +grep '^{"Time":' complement.log | gotestfmt