Support user output again

bin/run.sh

@@ -3,8 +3,9 @@
 set -euo pipefail
 
 PHPUNIT_BIN="./bin/phpunit-10.phar"
-XML_RESULTS='results.xml'
-JSON_RESULTS='results.json'
+JUNIT_RESULTS='results.xml'
+TEAMCITY_RESULTS='teamcity.txt'
+EXERCISM_RESULTS='results.json'
 # shellcheck disable=SC2034 # Modifies XDebug behaviour when invoking PHP
 XDEBUG_MODE='off'
 
@@ -20,15 +21,22 @@ function main {
 
   set +e
   if ! output=$(php -l "${solution_dir}"/*.php 2>&1 1>/dev/null); then
-    jo version=3 status=error message="${output/"$solution_dir/"/""}" tests="[]" > "${output_dir%/}/${JSON_RESULTS}"
+    jo version=3 status=error message="${output/"$solution_dir/"/""}" tests="[]" > "${output_dir%/}/${EXERCISM_RESULTS}"
     return 0;
   fi
 
-  output=$(eval "${PHPUNIT_BIN}" \
+  # JUnit results contain the unit test failures only. But they contain `@testdox` - readable test names.
+  # Teamcity results contain user output in addition to failures as "failed risky tests"
+  #   (command line arguments --disallow-test-output and --fail-on-risky).
+  # At the moment we require both logs to provide all information to the website.
+  output=$("${PHPUNIT_BIN}" \
     -d memory_limit=300M \
-    --log-junit "${output_dir%/}/${XML_RESULTS}" \
+    --log-junit "${output_dir%/}/${JUNIT_RESULTS}" \
+    --log-teamcity "${output_dir%/}/${TEAMCITY_RESULTS}" \
     --no-configuration \
     --do-not-cache-result \
+    --disallow-test-output \
+    --fail-on-risky \
     "${test_files%%*( )}" 2>&1)
   phpunit_exit_code=$?
   set -e
@@ -37,13 +45,14 @@ function main {
   # PHPUnit fails to catch some issue in its internals. It cannot be provoked
   # by us for testing our code
   if [[ "${phpunit_exit_code}" -eq 255 ]]; then
-    jo version=3 status=error message="${output/"$solution_dir/"/""}" tests="[]" > "${output_dir%/}/${JSON_RESULTS}"
+    jo version=3 status=error message="${output/"$solution_dir/"/""}" tests="[]" > "${output_dir%/}/${EXERCISM_RESULTS}"
     return 0;
   fi
 
   php junit-handler/run.php \
-    "${output_dir%/}/${XML_RESULTS}" \
-    "${output_dir%/}/${JSON_RESULTS}"
+    "${output_dir%/}/${EXERCISM_RESULTS}" \
+    "${output_dir%/}/${JUNIT_RESULTS}" \
+    "${output_dir%/}/${TEAMCITY_RESULTS}"
 }
 
 function installed {

junit-handler/run.php

@@ -2,8 +2,9 @@
 
 require __DIR__ . '/vendor/autoload.php';
 
-$xml_in = $argv[1];
-$json_out = $argv[2];
+$json_out = $argv[1];
+$xml_in = $argv[2];
+$teamcity_in = $argv[3];
 
 $handler = new \Exercism\JunitHandler\Handler();
-$handler->run($xml_in, $json_out);
+$handler->run($json_out, $xml_in, $teamcity_in);

junit-handler/src/Handler.php

@@ -15,19 +15,24 @@ class Handler
     private const STATUS_PASS = 'pass';
     private const STATUS_FAIL = 'fail';
     private string $test_file_path = '';
+    private ?TeamcityResult $teamcityResult = null;
 
-    public function run(string $xml_path, $json_path): void
-    {
+    public function run(
+        string $json_path,
+        string $xml_path,
+        string $teamcity_path,
+    ): void {
         $testsuites = simplexml_load_file($xml_path);
-        if ($testsuites === false) {
+        $this->teamcityResult = new TeamcityResult($teamcity_path);
+        if ($testsuites === false || !$this->teamcityResult->hasResults()) {
             $output = [
                 'version' => self::VERSION,
                 'tests' => [],
                 'status' => self::STATUS_ERROR,
                 'message' => <<<ERROR_MESSAGE
                     Test run did not produce any output. Check your code to see if the code exits unexpectedly before the report is generated.
 
-                    E.g. Using the `die` function will cause the test runner to exist unexpectedly.
+                    E.g. Using the `die` function will cause the test runner to exit unexpectedly.
                     ERROR_MESSAGE
             ];
             $this->write_json($json_path, $output);
@@ -42,7 +47,6 @@ public function run(string $xml_path, $json_path): void
         $test_file_name = \basename($test_file_path);
         $this->test_file_path = \str_replace($test_file_name, '', $test_file_path);
 
-
         $testcase_error_count = (int) $testsuite_attrs['errors'];
         $testcase_failure_count = (int) $testsuite_attrs['failures'];
 
@@ -164,10 +168,13 @@ private function parseTestCases(
                 $output['name'] = $testdox->getDescription();
             }
 
+            $testName = $method->getName();
+            if ($this->teamcityResult->hasOutputOf($testName)) {
+                $output['output'] = $this->teamcityResult->outputOf($testName);
+            }
+
             foreach ($testcase->children() ?? [] as $name => $data) {
-                if ($name === 'system-out') {
-                    $output['output'] = (string) $data;
-                } elseif ($name === 'error') {
+                if ($name === 'error') {
                     $output['status'] = self::STATUS_ERROR;
                     $output['message'] = \str_replace($this->test_file_path, '', (string) $data);
                 } elseif ($name === 'failure') {

junit-handler/src/TeamcityResult.php

@@ -0,0 +1,97 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Exercism\JunitHandler;
+
+final class TeamcityResult
+{
+    private const OUTPUT_LINE_START = '##teamcity[testFailed';
+    private const OUTPUT_FIELD_START = "message='This test printed output: ";
+    private const OUTPUT_FIELD_END = "' details='";
+    private const NAME_FIELD = "name='";
+
+    private ?array $outputCollection = null;
+
+    public function __construct(
+        private readonly string $teamcityFile,
+    ) {
+        $this->fillOutputCollection();
+    }
+
+    public function hasResults(): bool
+    {
+        return $this->outputCollection !== null;
+    }
+
+    public function hasOutputOf(string $method): bool
+    {
+        return isset($this->outputCollection[$method]);
+    }
+
+    public function outputOf(string $method): ?string
+    {
+        return $this->outputCollection[$method] ?? null;
+    }
+
+    private function fillOutputCollection(): void
+    {
+        try {
+            $linesWithOutput = \array_filter(
+                \file($this->teamcityFile, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES),
+                $this->islineWithOutput(...)
+            );
+            $this->outputCollection = \array_combine(
+                $this->testNamesFrom($linesWithOutput),
+                $this->outputFrom($linesWithOutput),
+            );
+        } catch (\Throwable $exception) {
+            // Intentionally empty.
+        }
+    }
+
+    private function islineWithOutput(string $line): bool
+    {
+        return \str_starts_with($line, self::OUTPUT_LINE_START)
+            && \str_contains($line, self::OUTPUT_FIELD_START)
+            ;
+    }
+
+    private function testNamesFrom(array $lines): array
+    {
+        return \array_map($this->testNameFromThisLine(...), $lines);
+    }
+
+    private function testNameFromThisLine(string $line): string
+    {
+        $startOfTestName = \mb_strpos($line, self::NAME_FIELD) + \mb_strlen(self::NAME_FIELD);
+        $endOfTestName = \mb_strpos($line, "'", $startOfTestName);
+
+        return \mb_substr($line, $startOfTestName, $endOfTestName - $startOfTestName);
+    }
+
+    private function outputFrom(array $lines): array
+    {
+        return \array_map($this->outputFromThisLine(...), $lines);
+    }
+
+    private function outputFromThisLine(string $line): string
+    {
+        $startOfOutput = \mb_strpos($line, self::OUTPUT_FIELD_START) + \mb_strlen(self::OUTPUT_FIELD_START);
+        $endOfOutput = \mb_strpos($line, self::OUTPUT_FIELD_END, $startOfOutput);
+        $rawOutput = \mb_substr($line, $startOfOutput, $endOfOutput - $startOfOutput);
+
+        return $this->unescape($rawOutput);
+    }
+
+    private function unescape(string $text): string
+    {
+        return \str_replace(
+            // Keep this in sync with PHPUnit Teamcity escape()
+            // https://github.com/sebastianbergmann/phpunit/blob/main/src/Logging/TeamCity/TeamCityLogger.php#L331
+            ['||', "|'", '|n', '|r', '|]', '|['],
+            ['|', "'", "\n", "\r", ']', '['],
+            $text,
+        );
+    }
+}

tests/error-with-user-output/HelloWorld.php

@@ -0,0 +1,10 @@
+<?php
+
+function helloWorld()
+{
+    echo "Some 'user üâ`|| \r\toutput\n"
+        . 'containing \\ various "problematic" and UTF-8 chars' . PHP_EOL;
+    var_dump(new stdClass());
+
+    throw new \BadFunctionCallException("Implement the helloWorld() function");
+}

tests/error-with-user-output/HelloWorldTest.php

@@ -0,0 +1,38 @@
+<?php
+
+/*
+ * By adding type hints and enabling strict type checking, code can become
+ * easier to read, self-documenting and reduce the number of potential bugs.
+ * By default, type declarations are non-strict, which means they will attempt
+ * to change the original type to match the type specified by the
+ * type-declaration.
+ *
+ * In other words, if you pass a string to a function requiring a float,
+ * it will attempt to convert the string value to a float.
+ *
+ * To enable strict mode, a single declare directive must be placed at the top
+ * of the file.
+ * This means that the strictness of typing is configured on a per-file basis.
+ * This directive not only affects the type declarations of parameters, but also
+ * a function's return type.
+ *
+ * For more info review the Concept on strict type checking in the PHP track
+ * <link>.
+ *
+ * To disable strict typing, comment out the directive below.
+ */
+
+declare(strict_types=1);
+
+class HelloWorldTest extends PHPUnit\Framework\TestCase
+{
+    public static function setUpBeforeClass(): void
+    {
+        require_once 'HelloWorld.php';
+    }
+
+    public function testHelloWorld(): void
+    {
+        $this->assertEquals('Hello, World!', helloWorld());
+    }
+}

tests/error-with-user-output/expected_results.json

@@ -0,0 +1 @@
+{"version":3,"status":"fail","tests":[{"name":"testHelloWorld","status":"error","test_code":"$this->assertEquals('Hello, World!', helloWorld());\n","output":"Some 'user \u00fc\u00e2`|| \r\toutput\ncontaining \\ various \"problematic\" and UTF-8 chars\nobject(stdClass)#79 (0) {\n}\n","message":"HelloWorldTest::testHelloWorld\nBadFunctionCallException: Implement the helloWorld() function\n\nHelloWorld.php:9\nHelloWorldTest.php:36"}]}

tests/fail-with-user-output/HelloWorld.php

@@ -0,0 +1,10 @@
+<?php
+
+function helloWorld()
+{
+    echo "Some 'user üâ`|| \r\toutput\n"
+        . 'containing \\ various "problematic" and UTF-8 chars' . PHP_EOL;
+    var_dump(new stdClass());
+
+    return "Goodbye, Mars!";
+}

tests/fail-with-user-output/HelloWorldTest.php

@@ -0,0 +1,38 @@
+<?php
+
+/*
+ * By adding type hints and enabling strict type checking, code can become
+ * easier to read, self-documenting and reduce the number of potential bugs.
+ * By default, type declarations are non-strict, which means they will attempt
+ * to change the original type to match the type specified by the
+ * type-declaration.
+ *
+ * In other words, if you pass a string to a function requiring a float,
+ * it will attempt to convert the string value to a float.
+ *
+ * To enable strict mode, a single declare directive must be placed at the top
+ * of the file.
+ * This means that the strictness of typing is configured on a per-file basis.
+ * This directive not only affects the type declarations of parameters, but also
+ * a function's return type.
+ *
+ * For more info review the Concept on strict type checking in the PHP track
+ * <link>.
+ *
+ * To disable strict typing, comment out the directive below.
+ */
+
+declare(strict_types=1);
+
+class HelloWorldTest extends PHPUnit\Framework\TestCase
+{
+    public static function setUpBeforeClass(): void
+    {
+        require_once 'HelloWorld.php';
+    }
+
+    public function testHelloWorld(): void
+    {
+        $this->assertEquals('Hello, World!', helloWorld());
+    }
+}

tests/fail-with-user-output/expected_results.json

@@ -0,0 +1 @@
+{"version":3,"status":"fail","tests":[{"name":"testHelloWorld","status":"fail","test_code":"$this->assertEquals('Hello, World!', helloWorld());\n","output":"Some 'user \u00fc\u00e2`|| \r\toutput\ncontaining \\ various \"problematic\" and UTF-8 chars\nobject(stdClass)#79 (0) {\n}\n","message":"HelloWorldTest::testHelloWorld\nFailed asserting that two strings are equal.\n--- Expected\n+++ Actual\n@@ @@\n-'Hello, World!'\n+'Goodbye, Mars!'\n\nHelloWorldTest.php:36"}]}

tests/success-with-user-output/HelloWorld.php

@@ -0,0 +1,10 @@
+<?php
+
+function helloWorld()
+{
+    echo "Some 'user üâ`|| \r\toutput\n"
+        . 'containing \\ various "problematic" and UTF-8 chars' . PHP_EOL;
+    var_dump(new stdClass());
+
+    return "Hello, World!";
+}

tests/success-with-user-output/HelloWorldTest.php

@@ -0,0 +1,38 @@
+<?php
+
+/*
+ * By adding type hints and enabling strict type checking, code can become
+ * easier to read, self-documenting and reduce the number of potential bugs.
+ * By default, type declarations are non-strict, which means they will attempt
+ * to change the original type to match the type specified by the
+ * type-declaration.
+ *
+ * In other words, if you pass a string to a function requiring a float,
+ * it will attempt to convert the string value to a float.
+ *
+ * To enable strict mode, a single declare directive must be placed at the top
+ * of the file.
+ * This means that the strictness of typing is configured on a per-file basis.
+ * This directive not only affects the type declarations of parameters, but also
+ * a function's return type.
+ *
+ * For more info review the Concept on strict type checking in the PHP track
+ * <link>.
+ *
+ * To disable strict typing, comment out the directive below.
+ */
+
+declare(strict_types=1);
+
+class HelloWorldTest extends PHPUnit\Framework\TestCase
+{
+    public static function setUpBeforeClass(): void
+    {
+        require_once 'HelloWorld.php';
+    }
+
+    public function testHelloWorld(): void
+    {
+        $this->assertEquals('Hello, World!', helloWorld());
+    }
+}

tests/success-with-user-output/expected_results.json

@@ -0,0 +1 @@
+{"version":3,"status":"pass","tests":[{"name":"testHelloWorld","status":"pass","test_code":"$this->assertEquals('Hello, World!', helloWorld());\n","output":"Some 'user \u00fc\u00e2`|| \r\toutput\ncontaining \\ various \"problematic\" and UTF-8 chars\nobject(stdClass)#79 (0) {\n}\n"}]}