From 61ddc95e908f5d5bbe74acb3b8bfcf697226a6e0 Mon Sep 17 00:00:00 2001 From: Michael Kramer Date: Sat, 30 Mar 2024 09:03:28 +0100 Subject: [PATCH 1/8] Fix parameter expansion (SC2145) --- bin/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/run.sh b/bin/run.sh index 462a3a2..8858bd2 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -41,7 +41,7 @@ function installed { } function die { - >&2 echo "❌ Fatal: ${@}" + >&2 echo "❌ Fatal: $*" exit 1 } From 4b8f7e39997032eeb73ac3817ff7f5e1a4a1c73c Mon Sep 17 00:00:00 2001 From: Michael Kramer Date: Sat, 30 Mar 2024 12:54:05 +0100 Subject: [PATCH 2/8] Catch syntax-error before running PHPUnit This is required, because PHPUnit 9.6+ no longer fails for syntax errors with exit code 255. Instead, it converts the errors to exceptions and handles them the same as every other exception (like the ones we use in students stubs). Use "all-fail" to show, that exceptions are now reported as individual failing tests (for all tests here). This was based on a syntax error, too. But that does not work like this test expected anymore. --- bin/run.sh | 5 +++++ tests/all-fail/Leap.php | 2 +- tests/all-fail/expected_results.json | 2 +- tests/syntax-error/expected_results.json | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/bin/run.sh b/bin/run.sh index 8858bd2..2a0f4f2 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -13,6 +13,11 @@ function main { test_files=$(find "${solution_dir}" -type f -name '*Test.php' | tr '\n' ' ') set +e + if ! PHP_OUTPUT=$(php -l "${solution_dir}"/*.php 2>&1 1>/dev/null); then + jo version=3 status=error message="${PHP_OUTPUT/"$solution_dir/"/""}" tests="[]" > "${output_dir%/}/${JSON_RESULTS}" + return 0; + fi + phpunit_output=$(eval "${PHPUNIT_BIN}" \ -d memory_limit=300M \ --log-junit "${output_dir%/}/${XML_RESULTS}" \ diff --git a/tests/all-fail/Leap.php b/tests/all-fail/Leap.php index c92708a..621c314 100644 --- a/tests/all-fail/Leap.php +++ b/tests/all-fail/Leap.php @@ -4,5 +4,5 @@ function isLeap(int $year): bool { - return 1(!($year % 4) && (!!($year % 100) || !($year % 400))); + throw new \BadFunctionCallException("Implement the isLeap function"); } diff --git a/tests/all-fail/expected_results.json b/tests/all-fail/expected_results.json index 0618e8c..da3b947 100644 --- a/tests/all-fail/expected_results.json +++ b/tests/all-fail/expected_results.json @@ -1 +1 @@ -{"version":3,"status":"fail","tests":[{"name":"testLeapYear","status":"error","test_code":"$this->assertTrue(isLeap(1996));\n","message":"LeapTest::testLeapYear\nParseError: syntax error, unexpected token \"(\", expecting \";\"\n\n\/opt\/test-runner\/tests\/all-fail\/Leap.php:7"},{"name":"testNonLeapYear","status":"pass","test_code":"$this->assertFalse(isLeap(1997));\n"},{"name":"testNonLeapEvenYear","status":"pass","test_code":"$this->assertFalse(isLeap(1998));\n"},{"name":"testCentury","status":"pass","test_code":"$this->assertFalse(isLeap(1900));\n"},{"name":"testFourthCentury","status":"pass","test_code":"$this->assertTrue(isLeap(2400));\n"}]} +{"version":3,"status":"fail","tests":[{"name":"testLeapYear","status":"error","test_code":"$this->assertTrue(isLeap(1996));\n","message":"LeapTest::testLeapYear\nBadFunctionCallException: Implement the isLeap function\n\n\/opt\/test-runner\/tests\/all-fail\/Leap.php:7\n\/opt\/test-runner\/tests\/all-fail\/LeapTest.php:36"},{"name":"testNonLeapYear","status":"error","test_code":"$this->assertFalse(isLeap(1997));\n","message":"LeapTest::testNonLeapYear\nBadFunctionCallException: Implement the isLeap function\n\n\/opt\/test-runner\/tests\/all-fail\/Leap.php:7\n\/opt\/test-runner\/tests\/all-fail\/LeapTest.php:41"},{"name":"testNonLeapEvenYear","status":"error","test_code":"$this->assertFalse(isLeap(1998));\n","message":"LeapTest::testNonLeapEvenYear\nBadFunctionCallException: Implement the isLeap function\n\n\/opt\/test-runner\/tests\/all-fail\/Leap.php:7\n\/opt\/test-runner\/tests\/all-fail\/LeapTest.php:46"},{"name":"testCentury","status":"error","test_code":"$this->assertFalse(isLeap(1900));\n","message":"LeapTest::testCentury\nBadFunctionCallException: Implement the isLeap function\n\n\/opt\/test-runner\/tests\/all-fail\/Leap.php:7\n\/opt\/test-runner\/tests\/all-fail\/LeapTest.php:51"},{"name":"testFourthCentury","status":"error","test_code":"$this->assertTrue(isLeap(2400));\n","message":"LeapTest::testFourthCentury\nBadFunctionCallException: Implement the isLeap function\n\n\/opt\/test-runner\/tests\/all-fail\/Leap.php:7\n\/opt\/test-runner\/tests\/all-fail\/LeapTest.php:56"}]} diff --git a/tests/syntax-error/expected_results.json b/tests/syntax-error/expected_results.json index 076dfaf..04d0525 100644 --- a/tests/syntax-error/expected_results.json +++ b/tests/syntax-error/expected_results.json @@ -1 +1 @@ -{"version":3,"status":"fail","tests":[{"name":"testLeapYear","status":"error","test_code":"$this->assertTrue(isLeap(1996));\n","message":"LeapTest::testLeapYear\nParseError: syntax error, unexpected token \"@\", expecting \"(\"\n\n\/opt\/test-runner\/tests\/syntax-error\/Leap.php:5"},{"name":"testNonLeapYear","status":"pass","test_code":"$this->assertFalse(isLeap(1997));\n"},{"name":"testNonLeapEvenYear","status":"pass","test_code":"$this->assertFalse(isLeap(1998));\n"},{"name":"testCentury","status":"pass","test_code":"$this->assertFalse(isLeap(1900));\n"},{"name":"testFourthCentury","status":"pass","test_code":"$this->assertTrue(isLeap(2400));\n"}]} +{"version":3,"status":"error","message":"PHP Parse error: syntax error, unexpected token \"@\", expecting \"(\" in Leap.php on line 5","tests":[]} From 098fe441852fd39423a574847fbe1fe39096656c Mon Sep 17 00:00:00 2001 From: Michael Kramer Date: Sat, 30 Mar 2024 13:38:44 +0100 Subject: [PATCH 3/8] Strip absolute path prefix from result messages This is recommended by test runner specification. --- bin/run.sh | 2 +- junit-handler/src/Handler.php | 8 ++++++-- tests/all-fail/expected_results.json | 2 +- tests/empty-file/expected_results.json | 2 +- tests/partial-fail/expected_results.json | 2 +- tests/table-test/expected_results.json | 2 +- 6 files changed, 11 insertions(+), 7 deletions(-) diff --git a/bin/run.sh b/bin/run.sh index 2a0f4f2..8d4d760 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -29,7 +29,7 @@ function main { set -e if [[ "${phpunit_exit_code}" -eq 255 ]]; then - jo version=2 status=error message="${phpunit_output}" tests="[]" > "${output_dir%/}/${JSON_RESULTS}" + jo version=3 status=error message="${phpunit_output/"$solution_dir/"/""}" tests="[]" > "${output_dir%/}/${JSON_RESULTS}" return 0; fi diff --git a/junit-handler/src/Handler.php b/junit-handler/src/Handler.php index 31f2d83..01666c2 100644 --- a/junit-handler/src/Handler.php +++ b/junit-handler/src/Handler.php @@ -14,6 +14,7 @@ class Handler private const STATUS_ERROR = 'error'; private const STATUS_PASS = 'pass'; private const STATUS_FAIL = 'fail'; + private string $test_file_path = ''; public function run(string $xml_path, $json_path): void { @@ -38,6 +39,9 @@ public function run(string $xml_path, $json_path): void $test_class = $testsuite_attrs['name']; $test_file_path = $testsuite_attrs['file']; + $test_file_name = \basename($test_file_path); + $this->test_file_path = \str_replace($test_file_name, '', $test_file_path); + $testcase_error_count = (int) $testsuite_attrs['errors']; $testcase_failure_count = (int) $testsuite_attrs['failures']; @@ -165,10 +169,10 @@ private function parseTestCases( $output['output'] = (string) $data; } elseif ($name === 'error') { $output['status'] = self::STATUS_ERROR; - $output['message'] = (string) $data; + $output['message'] = \str_replace($this->test_file_path, '', (string) $data); } elseif ($name === 'failure') { $output['status'] = self::STATUS_FAIL; - $output['message'] = (string) $data; + $output['message'] = \str_replace($this->test_file_path, '', (string) $data); } } diff --git a/tests/all-fail/expected_results.json b/tests/all-fail/expected_results.json index da3b947..484eff0 100644 --- a/tests/all-fail/expected_results.json +++ b/tests/all-fail/expected_results.json @@ -1 +1 @@ -{"version":3,"status":"fail","tests":[{"name":"testLeapYear","status":"error","test_code":"$this->assertTrue(isLeap(1996));\n","message":"LeapTest::testLeapYear\nBadFunctionCallException: Implement the isLeap function\n\n\/opt\/test-runner\/tests\/all-fail\/Leap.php:7\n\/opt\/test-runner\/tests\/all-fail\/LeapTest.php:36"},{"name":"testNonLeapYear","status":"error","test_code":"$this->assertFalse(isLeap(1997));\n","message":"LeapTest::testNonLeapYear\nBadFunctionCallException: Implement the isLeap function\n\n\/opt\/test-runner\/tests\/all-fail\/Leap.php:7\n\/opt\/test-runner\/tests\/all-fail\/LeapTest.php:41"},{"name":"testNonLeapEvenYear","status":"error","test_code":"$this->assertFalse(isLeap(1998));\n","message":"LeapTest::testNonLeapEvenYear\nBadFunctionCallException: Implement the isLeap function\n\n\/opt\/test-runner\/tests\/all-fail\/Leap.php:7\n\/opt\/test-runner\/tests\/all-fail\/LeapTest.php:46"},{"name":"testCentury","status":"error","test_code":"$this->assertFalse(isLeap(1900));\n","message":"LeapTest::testCentury\nBadFunctionCallException: Implement the isLeap function\n\n\/opt\/test-runner\/tests\/all-fail\/Leap.php:7\n\/opt\/test-runner\/tests\/all-fail\/LeapTest.php:51"},{"name":"testFourthCentury","status":"error","test_code":"$this->assertTrue(isLeap(2400));\n","message":"LeapTest::testFourthCentury\nBadFunctionCallException: Implement the isLeap function\n\n\/opt\/test-runner\/tests\/all-fail\/Leap.php:7\n\/opt\/test-runner\/tests\/all-fail\/LeapTest.php:56"}]} +{"version":3,"status":"fail","tests":[{"name":"testLeapYear","status":"error","test_code":"$this->assertTrue(isLeap(1996));\n","message":"LeapTest::testLeapYear\nBadFunctionCallException: Implement the isLeap function\n\nLeap.php:7\nLeapTest.php:36"},{"name":"testNonLeapYear","status":"error","test_code":"$this->assertFalse(isLeap(1997));\n","message":"LeapTest::testNonLeapYear\nBadFunctionCallException: Implement the isLeap function\n\nLeap.php:7\nLeapTest.php:41"},{"name":"testNonLeapEvenYear","status":"error","test_code":"$this->assertFalse(isLeap(1998));\n","message":"LeapTest::testNonLeapEvenYear\nBadFunctionCallException: Implement the isLeap function\n\nLeap.php:7\nLeapTest.php:46"},{"name":"testCentury","status":"error","test_code":"$this->assertFalse(isLeap(1900));\n","message":"LeapTest::testCentury\nBadFunctionCallException: Implement the isLeap function\n\nLeap.php:7\nLeapTest.php:51"},{"name":"testFourthCentury","status":"error","test_code":"$this->assertTrue(isLeap(2400));\n","message":"LeapTest::testFourthCentury\nBadFunctionCallException: Implement the isLeap function\n\nLeap.php:7\nLeapTest.php:56"}]} diff --git a/tests/empty-file/expected_results.json b/tests/empty-file/expected_results.json index 4b03d29..d30fa33 100644 --- a/tests/empty-file/expected_results.json +++ b/tests/empty-file/expected_results.json @@ -1 +1 @@ -{"version":3,"status":"fail","tests":[{"name":"testLeapYear","status":"error","test_code":"$this->assertTrue(isLeap(1996));\n","message":"LeapTest::testLeapYear\nError: Call to undefined function isLeap()\n\n\/opt\/test-runner\/tests\/empty-file\/LeapTest.php:36"},{"name":"testNonLeapYear","status":"error","test_code":"$this->assertFalse(isLeap(1997));\n","message":"LeapTest::testNonLeapYear\nError: Call to undefined function isLeap()\n\n\/opt\/test-runner\/tests\/empty-file\/LeapTest.php:41"},{"name":"testNonLeapEvenYear","status":"error","test_code":"$this->assertFalse(isLeap(1998));\n","message":"LeapTest::testNonLeapEvenYear\nError: Call to undefined function isLeap()\n\n\/opt\/test-runner\/tests\/empty-file\/LeapTest.php:46"},{"name":"testCentury","status":"error","test_code":"$this->assertFalse(isLeap(1900));\n","message":"LeapTest::testCentury\nError: Call to undefined function isLeap()\n\n\/opt\/test-runner\/tests\/empty-file\/LeapTest.php:51"},{"name":"testFourthCentury","status":"error","test_code":"$this->assertTrue(isLeap(2400));\n","message":"LeapTest::testFourthCentury\nError: Call to undefined function isLeap()\n\n\/opt\/test-runner\/tests\/empty-file\/LeapTest.php:56"}]} +{"version":3,"status":"fail","tests":[{"name":"testLeapYear","status":"error","test_code":"$this->assertTrue(isLeap(1996));\n","message":"LeapTest::testLeapYear\nError: Call to undefined function isLeap()\n\nLeapTest.php:36"},{"name":"testNonLeapYear","status":"error","test_code":"$this->assertFalse(isLeap(1997));\n","message":"LeapTest::testNonLeapYear\nError: Call to undefined function isLeap()\n\nLeapTest.php:41"},{"name":"testNonLeapEvenYear","status":"error","test_code":"$this->assertFalse(isLeap(1998));\n","message":"LeapTest::testNonLeapEvenYear\nError: Call to undefined function isLeap()\n\nLeapTest.php:46"},{"name":"testCentury","status":"error","test_code":"$this->assertFalse(isLeap(1900));\n","message":"LeapTest::testCentury\nError: Call to undefined function isLeap()\n\nLeapTest.php:51"},{"name":"testFourthCentury","status":"error","test_code":"$this->assertTrue(isLeap(2400));\n","message":"LeapTest::testFourthCentury\nError: Call to undefined function isLeap()\n\nLeapTest.php:56"}]} diff --git a/tests/partial-fail/expected_results.json b/tests/partial-fail/expected_results.json index e92e5c8..8d35881 100644 --- a/tests/partial-fail/expected_results.json +++ b/tests/partial-fail/expected_results.json @@ -1 +1 @@ -{"version":3,"status":"fail","tests":[{"name":"testLeapYear","status":"pass","test_code":"$this->assertTrue(isLeap(1996));\n"},{"name":"testNonLeapYear","status":"pass","test_code":"$this->assertFalse(isLeap(1997));\n"},{"name":"testNonLeapEvenYear","status":"pass","test_code":"$this->assertFalse(isLeap(1998));\n"},{"name":"testCentury","status":"pass","test_code":"$this->assertFalse(isLeap(1900));\n"},{"name":"testFourthCentury","status":"fail","test_code":"$this->assertTrue(isLeap(2400));\n","message":"LeapTest::testFourthCentury\nFailed asserting that false is true.\n\n\/opt\/test-runner\/tests\/partial-fail\/LeapTest.php:56"}]} +{"version":3,"status":"fail","tests":[{"name":"testLeapYear","status":"pass","test_code":"$this->assertTrue(isLeap(1996));\n"},{"name":"testNonLeapYear","status":"pass","test_code":"$this->assertFalse(isLeap(1997));\n"},{"name":"testNonLeapEvenYear","status":"pass","test_code":"$this->assertFalse(isLeap(1998));\n"},{"name":"testCentury","status":"pass","test_code":"$this->assertFalse(isLeap(1900));\n"},{"name":"testFourthCentury","status":"fail","test_code":"$this->assertTrue(isLeap(2400));\n","message":"LeapTest::testFourthCentury\nFailed asserting that false is true.\n\nLeapTest.php:56"}]} diff --git a/tests/table-test/expected_results.json b/tests/table-test/expected_results.json index d8dbf3c..c2f6c7c 100644 --- a/tests/table-test/expected_results.json +++ b/tests/table-test/expected_results.json @@ -1 +1 @@ -{"version":3,"status":"fail","tests":[{"name":"testFrom with data set #0","status":"error","test_code":"$date = $this->dateSetup($inputDate);\n$gs = from($date);\n\n$this->assertSame($expected, $gs->format('Y-m-d H:i:s'));\n","message":"GigasecondTest::testFrom with data set #0 ('2011-04-25', '2043-01-01 01:46:40')\nBadFunctionCallException: Implement the from function\n\n\/opt\/test-runner\/tests\/table-test\/Gigasecond.php:29\n\/opt\/test-runner\/tests\/table-test\/GigasecondTest.php:70"},{"name":"testFrom with data set #1","status":"error","test_code":"$date = $this->dateSetup($inputDate);\n$gs = from($date);\n\n$this->assertSame($expected, $gs->format('Y-m-d H:i:s'));\n","message":"GigasecondTest::testFrom with data set #1 ('1977-06-13', '2009-02-19 01:46:40')\nBadFunctionCallException: Implement the from function\n\n\/opt\/test-runner\/tests\/table-test\/Gigasecond.php:29\n\/opt\/test-runner\/tests\/table-test\/GigasecondTest.php:70"},{"name":"testFrom with data set #2","status":"error","test_code":"$date = $this->dateSetup($inputDate);\n$gs = from($date);\n\n$this->assertSame($expected, $gs->format('Y-m-d H:i:s'));\n","message":"GigasecondTest::testFrom with data set #2 ('1959-07-19', '1991-03-27 01:46:40')\nBadFunctionCallException: Implement the from function\n\n\/opt\/test-runner\/tests\/table-test\/Gigasecond.php:29\n\/opt\/test-runner\/tests\/table-test\/GigasecondTest.php:70"},{"name":"testFrom with data set #3","status":"error","test_code":"$date = $this->dateSetup($inputDate);\n$gs = from($date);\n\n$this->assertSame($expected, $gs->format('Y-m-d H:i:s'));\n","message":"GigasecondTest::testFrom with data set #3 ('2015-01-24 22:00:00', '2046-10-02 23:46:40')\nBadFunctionCallException: Implement the from function\n\n\/opt\/test-runner\/tests\/table-test\/Gigasecond.php:29\n\/opt\/test-runner\/tests\/table-test\/GigasecondTest.php:70"},{"name":"testFrom with data set #4","status":"error","test_code":"$date = $this->dateSetup($inputDate);\n$gs = from($date);\n\n$this->assertSame($expected, $gs->format('Y-m-d H:i:s'));\n","message":"GigasecondTest::testFrom with data set #4 ('2015-01-24 23:59:59', '2046-10-03 01:46:39')\nBadFunctionCallException: Implement the from function\n\n\/opt\/test-runner\/tests\/table-test\/Gigasecond.php:29\n\/opt\/test-runner\/tests\/table-test\/GigasecondTest.php:70"},{"name":"testFromReturnType with data set #0","status":"error","test_code":"$date = $this->dateSetup($inputDate);\n$this->assertInstanceOf(DateTimeImmutable::class, from($date));\n","message":"GigasecondTest::testFromReturnType with data set #0 ('2011-04-25')\nBadFunctionCallException: Implement the from function\n\n\/opt\/test-runner\/tests\/table-test\/Gigasecond.php:29\n\/opt\/test-runner\/tests\/table-test\/GigasecondTest.php:82"},{"name":"testFromReturnType with data set #1","status":"error","test_code":"$date = $this->dateSetup($inputDate);\n$this->assertInstanceOf(DateTimeImmutable::class, from($date));\n","message":"GigasecondTest::testFromReturnType with data set #1 ('1977-06-13')\nBadFunctionCallException: Implement the from function\n\n\/opt\/test-runner\/tests\/table-test\/Gigasecond.php:29\n\/opt\/test-runner\/tests\/table-test\/GigasecondTest.php:82"},{"name":"testFromReturnType with data set #2","status":"error","test_code":"$date = $this->dateSetup($inputDate);\n$this->assertInstanceOf(DateTimeImmutable::class, from($date));\n","message":"GigasecondTest::testFromReturnType with data set #2 ('1959-07-19')\nBadFunctionCallException: Implement the from function\n\n\/opt\/test-runner\/tests\/table-test\/Gigasecond.php:29\n\/opt\/test-runner\/tests\/table-test\/GigasecondTest.php:82"},{"name":"testFromReturnType with data set #3","status":"error","test_code":"$date = $this->dateSetup($inputDate);\n$this->assertInstanceOf(DateTimeImmutable::class, from($date));\n","message":"GigasecondTest::testFromReturnType with data set #3 ('2015-01-24 22:00:00')\nBadFunctionCallException: Implement the from function\n\n\/opt\/test-runner\/tests\/table-test\/Gigasecond.php:29\n\/opt\/test-runner\/tests\/table-test\/GigasecondTest.php:82"},{"name":"testFromReturnType with data set #4","status":"error","test_code":"$date = $this->dateSetup($inputDate);\n$this->assertInstanceOf(DateTimeImmutable::class, from($date));\n","message":"GigasecondTest::testFromReturnType with data set #4 ('2015-01-24 23:59:59')\nBadFunctionCallException: Implement the from function\n\n\/opt\/test-runner\/tests\/table-test\/Gigasecond.php:29\n\/opt\/test-runner\/tests\/table-test\/GigasecondTest.php:82"},{"name":"testRegularTest","status":"pass","test_code":"$this->assertEquals(1, 1);\n"}]} +{"version":3,"status":"fail","tests":[{"name":"testFrom with data set #0","status":"error","test_code":"$date = $this->dateSetup($inputDate);\n$gs = from($date);\n\n$this->assertSame($expected, $gs->format('Y-m-d H:i:s'));\n","message":"GigasecondTest::testFrom with data set #0 ('2011-04-25', '2043-01-01 01:46:40')\nBadFunctionCallException: Implement the from function\n\nGigasecond.php:29\nGigasecondTest.php:70"},{"name":"testFrom with data set #1","status":"error","test_code":"$date = $this->dateSetup($inputDate);\n$gs = from($date);\n\n$this->assertSame($expected, $gs->format('Y-m-d H:i:s'));\n","message":"GigasecondTest::testFrom with data set #1 ('1977-06-13', '2009-02-19 01:46:40')\nBadFunctionCallException: Implement the from function\n\nGigasecond.php:29\nGigasecondTest.php:70"},{"name":"testFrom with data set #2","status":"error","test_code":"$date = $this->dateSetup($inputDate);\n$gs = from($date);\n\n$this->assertSame($expected, $gs->format('Y-m-d H:i:s'));\n","message":"GigasecondTest::testFrom with data set #2 ('1959-07-19', '1991-03-27 01:46:40')\nBadFunctionCallException: Implement the from function\n\nGigasecond.php:29\nGigasecondTest.php:70"},{"name":"testFrom with data set #3","status":"error","test_code":"$date = $this->dateSetup($inputDate);\n$gs = from($date);\n\n$this->assertSame($expected, $gs->format('Y-m-d H:i:s'));\n","message":"GigasecondTest::testFrom with data set #3 ('2015-01-24 22:00:00', '2046-10-02 23:46:40')\nBadFunctionCallException: Implement the from function\n\nGigasecond.php:29\nGigasecondTest.php:70"},{"name":"testFrom with data set #4","status":"error","test_code":"$date = $this->dateSetup($inputDate);\n$gs = from($date);\n\n$this->assertSame($expected, $gs->format('Y-m-d H:i:s'));\n","message":"GigasecondTest::testFrom with data set #4 ('2015-01-24 23:59:59', '2046-10-03 01:46:39')\nBadFunctionCallException: Implement the from function\n\nGigasecond.php:29\nGigasecondTest.php:70"},{"name":"testFromReturnType with data set #0","status":"error","test_code":"$date = $this->dateSetup($inputDate);\n$this->assertInstanceOf(DateTimeImmutable::class, from($date));\n","message":"GigasecondTest::testFromReturnType with data set #0 ('2011-04-25')\nBadFunctionCallException: Implement the from function\n\nGigasecond.php:29\nGigasecondTest.php:82"},{"name":"testFromReturnType with data set #1","status":"error","test_code":"$date = $this->dateSetup($inputDate);\n$this->assertInstanceOf(DateTimeImmutable::class, from($date));\n","message":"GigasecondTest::testFromReturnType with data set #1 ('1977-06-13')\nBadFunctionCallException: Implement the from function\n\nGigasecond.php:29\nGigasecondTest.php:82"},{"name":"testFromReturnType with data set #2","status":"error","test_code":"$date = $this->dateSetup($inputDate);\n$this->assertInstanceOf(DateTimeImmutable::class, from($date));\n","message":"GigasecondTest::testFromReturnType with data set #2 ('1959-07-19')\nBadFunctionCallException: Implement the from function\n\nGigasecond.php:29\nGigasecondTest.php:82"},{"name":"testFromReturnType with data set #3","status":"error","test_code":"$date = $this->dateSetup($inputDate);\n$this->assertInstanceOf(DateTimeImmutable::class, from($date));\n","message":"GigasecondTest::testFromReturnType with data set #3 ('2015-01-24 22:00:00')\nBadFunctionCallException: Implement the from function\n\nGigasecond.php:29\nGigasecondTest.php:82"},{"name":"testFromReturnType with data set #4","status":"error","test_code":"$date = $this->dateSetup($inputDate);\n$this->assertInstanceOf(DateTimeImmutable::class, from($date));\n","message":"GigasecondTest::testFromReturnType with data set #4 ('2015-01-24 23:59:59')\nBadFunctionCallException: Implement the from function\n\nGigasecond.php:29\nGigasecondTest.php:82"},{"name":"testRegularTest","status":"pass","test_code":"$this->assertEquals(1, 1);\n"}]} From 31f9748871c74e7202497294ec29875bd68aed31 Mon Sep 17 00:00:00 2001 From: Michael Kramer Date: Sat, 30 Mar 2024 13:42:26 +0100 Subject: [PATCH 4/8] Use local variables in functions main, installed --- bin/run.sh | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/bin/run.sh b/bin/run.sh index 8d4d760..213efca 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -7,18 +7,22 @@ XML_RESULTS='results.xml' JSON_RESULTS='results.json' function main { - exercise_slug="${1}" - solution_dir="${2}" - output_dir="${3}" + local output="" + local test_files="" + local -i phpunit_exit_code + + # local exercise_slug="${1}" + local solution_dir="${2}" + local output_dir="${3}" test_files=$(find "${solution_dir}" -type f -name '*Test.php' | tr '\n' ' ') set +e - if ! PHP_OUTPUT=$(php -l "${solution_dir}"/*.php 2>&1 1>/dev/null); then - jo version=3 status=error message="${PHP_OUTPUT/"$solution_dir/"/""}" tests="[]" > "${output_dir%/}/${JSON_RESULTS}" + if ! output=$(php -l "${solution_dir}"/*.php 2>&1 1>/dev/null); then + jo version=3 status=error message="${output/"$solution_dir/"/""}" tests="[]" > "${output_dir%/}/${JSON_RESULTS}" return 0; fi - phpunit_output=$(eval "${PHPUNIT_BIN}" \ + output=$(eval "${PHPUNIT_BIN}" \ -d memory_limit=300M \ --log-junit "${output_dir%/}/${XML_RESULTS}" \ --verbose \ @@ -29,7 +33,7 @@ function main { set -e if [[ "${phpunit_exit_code}" -eq 255 ]]; then - jo version=3 status=error message="${phpunit_output/"$solution_dir/"/""}" tests="[]" > "${output_dir%/}/${JSON_RESULTS}" + jo version=3 status=error message="${output/"$solution_dir/"/""}" tests="[]" > "${output_dir%/}/${JSON_RESULTS}" return 0; fi @@ -39,6 +43,8 @@ function main { } function installed { + local cmd + cmd=$(command -v "${1}") [[ -n "${cmd}" ]] && [[ -f "${cmd}" ]] From c79d01598a357bf32060f613f8b4809dc812b897 Mon Sep 17 00:00:00 2001 From: Michael Kramer Date: Sat, 30 Mar 2024 15:04:32 +0100 Subject: [PATCH 5/8] Update README --- README.md | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index a8c8999..f851ba9 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,45 @@ -![Jest Tests](https://github.com/exercism/php-test-runner/workflows/Test%20JUnit-to-JSON/badge.svg) ![Smoke Test](https://github.com/exercism/php-test-runner/workflows/Smoke%20Test/badge.svg) ![Tooling image pushed](https://github.com/exercism/php-test-runner/workflows/Push%20Docker%20images%20to%20DockerHub%20and%20ECR/badge.svg) +![JUnit to JSON Tests](https://github.com/exercism/php-test-runner/workflows/Test%20JUnit-to-JSON/badge.svg) ![Smoke Test](https://github.com/exercism/php-test-runner/workflows/Smoke%20Test/badge.svg) ![Tooling image pushed](https://github.com/exercism/php-test-runner/workflows/Deploy/badge.svg) # PHP Test Runner -This is a minimal test runner for Exercism's v3 platform. It meets the minimal spec for testing _practice exercises_. It does not currently parse the test case code being run, therefore it does not meet the standard for testing _concept exercises_. +TODO: This seems to be outdated, but I don't know the current state: + +This is a minimal test runner for Exercism's v3 platform. +It meets the minimal spec for testing _practice exercises_. +It does not currently parse the test case code being run, therefore it does not meet the standard for testing _concept exercises_. ## Basic components -### Dockerimage +### Docker image -The website uses isolated docker images to run untrusted code in a sandbox. Image consists of PHP 8.2.7 (PHPUnit 9/10). All final assets are built into the image, because the image does not have network access once in use. +The website uses isolated docker images to run untrusted code in a sandbox. +Image consists of PHP 8.2.7 (PHPUnit 9/10). +All final assets are built into the image, because the image does not have network access once in use. -Includes php extensions: ds, intl +Includes PHP extensions: ds, intl ### Test runner Test running a solution is coordinated by a bash script at `bin/run.sh` taking 3 positional arguments: ```text -> bin/run.sh +bin/run.sh ``` +This is what runs inside the production Docker image when students submit their code. + +### Testing the test runner + +In `./tests/` are golden tests to verify test runner behaves as expected. +The CI uses `bin/run-tests.sh` to execute them. + +### Running tests in Docker locally + +This is the recommended way to use this locally. +Use `bin/run-in-docker.sh ` and `bin/run-tests-in-docker.sh` to locally build and run the Docker image. + ### JUnit to JSON -PHPUnit can natively output tests run to junit xml format, but Exercism requires output in json format. A php-based app is located in the `junit-handler` folder. It provides a translation layer from one format to the other incorporating task_id identification and test code inclusion. +PHPUnit can natively output tests run to JUnit XML format, but Exercism requires output in json format. +A PHP-based app is located in the `junit-handler` folder. +It provides a translation layer from one format to the other incorporating `task_id` identification and test code inclusion. From 0fa2b2cf02317de1e086342807443e6319e4d324 Mon Sep 17 00:00:00 2001 From: Michael Kramer Date: Sat, 30 Mar 2024 16:12:30 +0100 Subject: [PATCH 6/8] Note that we cannot remove PHPUnit from image --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d169f3b..c117a0f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,8 @@ RUN curl -L -o phpunit-9.phar https://phar.phpunit.de/phpunit-9.phar && \ WORKDIR /usr/local/bin/junit-handler/ COPY --from=composer:2.5.8 /usr/bin/composer /usr/local/bin/composer COPY junit-handler/ . -RUN composer install --no-interaction +# We need PHPUnit from junit-handler/ to run test-runner tests in CI / locally +RUN composer install --no-interaction FROM php:8.2.7-cli-alpine3.18 AS runtime From cd70775e7ffb713b1b136dc3926ceaba87e19d0c Mon Sep 17 00:00:00 2001 From: Michael Kramer Date: Sat, 30 Mar 2024 16:23:08 +0100 Subject: [PATCH 7/8] Update README --- README.md | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index f851ba9..2a5ea5c 100644 --- a/README.md +++ b/README.md @@ -2,18 +2,15 @@ # PHP Test Runner -TODO: This seems to be outdated, but I don't know the current state: - -This is a minimal test runner for Exercism's v3 platform. -It meets the minimal spec for testing _practice exercises_. -It does not currently parse the test case code being run, therefore it does not meet the standard for testing _concept exercises_. +This is the test runner for Exercism's v3 platform. +It meets the complete spec for testing all exercises. ## Basic components ### Docker image The website uses isolated docker images to run untrusted code in a sandbox. -Image consists of PHP 8.2.7 (PHPUnit 9/10). +The image provided by this repository consists of PHP 8.2.7 (PHPUnit 9/10). All final assets are built into the image, because the image does not have network access once in use. Includes PHP extensions: ds, intl @@ -30,7 +27,7 @@ This is what runs inside the production Docker image when students submit their ### Testing the test runner -In `./tests/` are golden tests to verify test runner behaves as expected. +In `./tests/` are golden tests to verify that the test runner behaves as expected. The CI uses `bin/run-tests.sh` to execute them. ### Running tests in Docker locally @@ -40,6 +37,6 @@ Use `bin/run-in-docker.sh Date: Sat, 30 Mar 2024 18:47:09 +0100 Subject: [PATCH 8/8] Note we cannot test for PHPUnit crashes --- bin/run.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bin/run.sh b/bin/run.sh index 213efca..b7fd569 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -32,6 +32,9 @@ function main { phpunit_exit_code=$? set -e + # This is only a theoretical failure case. This exit code is generated, when + # PHPUnit fails to catch some issue in its internals. It cannot be provoked + # by us for testing our code if [[ "${phpunit_exit_code}" -eq 255 ]]; then jo version=3 status=error message="${output/"$solution_dir/"/""}" tests="[]" > "${output_dir%/}/${JSON_RESULTS}" return 0;