feat(teamcity): report the exact status, assertion count, suite taxonomy and test count

bridge/rector/composer.json

@@ -30,6 +30,7 @@
     "require-dev": {
         "testo/assert": "^0.1.12",
         "testo/data": "^0.1.7",
+        "testo/filter": "^0.1.6",
         "testo/testo": "0.10.39 - 1"
     },
     "autoload": {

bridge/rector/src/Testing/Internal/Middleware/RectorFixtureInterceptor.php

@@ -19,6 +19,7 @@
 use Testo\Event\Test\TestBatchStarting;
 use Testo\Event\Test\TestDataSetFinished;
 use Testo\Event\Test\TestDataSetStarting;
+use Testo\Filter\DataPointer;
 use Testo\Pipeline\Attribute\InterceptorOptions;
 use Testo\Pipeline\Middleware\TestRunInterceptor;
 
@@ -61,10 +62,26 @@ public function runTest(TestInfo $info, callable $next): TestResult
             if ($reflection !== null && $fixtures !== []) {
                 $runner = new RectorRunner($this->messenger, [$reflection->getName()]);
 
+                # A fixture occupies the data set slot of the address, so `--filter=Rule::fixture:0:2`
+                # selects the third fixture — the coordinates the IDE sends back for one data set.
+                $dataPointer = $info->getAttribute(DataPointer::class);
+
                 $num = -1;
                 foreach ($fixtures as $label => $path) {
                     ++$num;
-                    $dsInfo = $info->with(arguments: [$runner, $path]);
+                    if ($dataPointer !== null && (
+                        ($dataPointer->provider !== 0)
+                        || ($dataPointer->dataset !== null && $dataPointer->dataset !== $num)
+                    )) {
+                        continue;
+                    }
+
+                    # Each fixture needs its own address, or consumers that key on it collide: TeamCity
+                    # would reuse the batch's node for every data set and nest none of them under it.
+                    $dsInfo = $info->with(
+                        arguments: [$runner, $path],
+                        identity: $info->identity->toDataSet(dataProvider: 0, dataSet: $num),
+                    );
 
                     $this->eventDispatcher->dispatch(new TestDataSetStarting($dsInfo, $label, null, $num));
                     try {

core/Output/Teamcity/Teamcity/Formatter.php

@@ -6,7 +6,9 @@
 
 use Testo\Core\Context\Identity;
 use Testo\Core\Context\Identity\CaseIdentity;
+use Testo\Core\Context\Identity\SuiteIdentity;
 use Testo\Core\Context\Identity\TestIdentity;
+use Testo\Core\Value\Status;
 
 /**
  * Formats TeamCity service messages.
@@ -59,19 +61,41 @@ public static function suiteStarted(string $name, ?Identity $identity = null): s
         $locationHint = self::locationHint($identity);
         $locationHint === null or $attributes['locationHint'] = $locationHint;
 
-        return self::formatMessage('testSuiteStarted', $attributes + self::placement($identity));
+        return self::formatMessage(
+            'testSuiteStarted',
+            $attributes + self::taxonomy($identity) + self::placement($identity),
+        );
+    }
+
+    /**
+     * Formats a message announcing how many tests are about to run.
+     *
+     * Feeds the progress bar of IntelliJ-based IDEs, which is the only consumer — the TeamCity server
+     * ignores it. Counts accumulate rather than replace, so one message per suite is the intended
+     * shape rather than a single total up front.
+     *
+     * @param int<0, max> $count
+     * @return non-empty-string
+     */
+    public static function testCount(int $count): string
+    {
+        return self::formatMessage('testCount', ['count' => (string) $count]);
     }
 
     /**
      * Formats a test suite finished message.
      *
      * @param non-empty-string $name Suite name
      * @param Identity|null $identity Address of the node this message closes. {@see placement()}
+     * @param Status|null $status Aggregated outcome of the node. {@see status()}
      * @return non-empty-string
      */
-    public static function suiteFinished(string $name, ?Identity $identity = null): string
+    public static function suiteFinished(string $name, ?Identity $identity = null, ?Status $status = null): string
     {
-        return self::formatMessage('testSuiteFinished', ['name' => $name] + self::placement($identity));
+        return self::formatMessage(
+            'testSuiteFinished',
+            ['name' => $name] + self::status($status) + self::placement($identity),
+        );
     }
 
     /**
@@ -96,7 +120,10 @@ public static function testStarted(string $name, bool $captureStandardOutput = f
 
         $description !== null and $attributes['metainfo'] = $description;
 
-        return self::formatMessage('testStarted', $attributes + self::placement($identity));
+        return self::formatMessage(
+            'testStarted',
+            $attributes + self::taxonomy($identity) + self::placement($identity),
+        );
     }
 
     /**
@@ -105,15 +132,25 @@ public static function testStarted(string $name, bool $captureStandardOutput = f
      * @param non-empty-string $name Test name
      * @param int<0, max>|null $duration Duration in milliseconds
      * @param TestIdentity|null $identity Address of the test this message closes. {@see placement()}
+     * @param Status|null $status Outcome of the test. {@see status()}
+     * @param int<0, max>|null $assertions Number of assertions the test performed. Null when nothing
+     *        counted them — no assertion plugin is active — which is not the same as a test that
+     *        counted zero.
      * @return non-empty-string
      */
-    public static function testFinished(string $name, ?int $duration = null, ?TestIdentity $identity = null): string
-    {
+    public static function testFinished(
+        string $name,
+        ?int $duration = null,
+        ?TestIdentity $identity = null,
+        ?Status $status = null,
+        ?int $assertions = null,
+    ): string {
         $attributes = ['name' => $name];
 
         $duration !== null and $attributes['duration'] = (string) $duration;
+        $assertions !== null and $attributes['assertions'] = (string) $assertions;
 
-        return self::formatMessage('testFinished', $attributes + self::placement($identity));
+        return self::formatMessage('testFinished', $attributes + self::status($status) + self::placement($identity));
     }
 
     /**
@@ -422,6 +459,44 @@ private static function placement(?Identity $identity): array
         return $placement;
     }
 
+    /**
+     * Which suite the node belongs to and which kind of test it holds — the two things `--suite` and
+     * `--type` select on, so a consumer can offer the same slicing without parsing anything out of a
+     * name or a path.
+     *
+     * Only stated where the address knows it: a suite of the run has no type of its own, since one
+     * suite can hold cases of several ({@see CaseIdentity::$type}).
+     *
+     * @return array<non-empty-string, non-empty-string>
+     */
+    private static function taxonomy(?Identity $identity): array
+    {
+        return match (true) {
+            $identity instanceof CaseIdentity,
+            $identity instanceof TestIdentity => [
+                'testSuite' => $identity->suite,
+                'testType' => $identity->type,
+            ],
+            $identity instanceof SuiteIdentity => ['testSuite' => $identity->suite],
+            default => [],
+        };
+    }
+
+    /**
+     * The exact outcome, which the standard protocol cannot express: it distinguishes only ignored,
+     * failed and everything else, so `Flaky` is indistinguishable from `Passed` and `Risky` from a
+     * clean pass. Consumers that understand the attribute get the {@see Status} verbatim; standard
+     * parsers ignore it and keep reading the run as before.
+     *
+     * Lowercased case name, the same wire format the JSON report speaks.
+     *
+     * @return array<non-empty-string, non-empty-string>
+     */
+    private static function status(?Status $status): array
+    {
+        return $status === null ? [] : ['status' => \strtolower($status->name)];
+    }
+
     /**
      * Location hint for whatever the address names.
      *
@@ -430,12 +505,16 @@ private static function placement(?Identity $identity): array
      * php_qn://path/to/BarTest.php::\Ns\BarTest::itWorks        a test, or its DataProvider batch node
      * php_qn://path/to/BarTest.php::\Ns\BarTest::itWorks:0:1    one data set of it
      * php_qn://path/to/functions.php::\Ns\itWorksToo            a free test function
+     * file://path/to/functions.php                             a case of free functions
      * ```
      *
      * The tail is {@see TestIdentity::fqn()} verbatim, so a hint pastes straight back into `--filter`.
      *
-     * Null when there is no code to point at: a suite of the run is a configuration entry, and a case
-     * of free functions has no class of its own.
+     * A case of free functions names no class, and the file it groups holds several functions rather
+     * than one to point at — so it answers with the file itself under `file://`, the scheme IDEs
+     * resolve to a whole file. Clickable all the same, which a hintless node is not.
+     *
+     * Null only for a suite of the run: it is a configuration entry, with no file of its own to name.
      *
      * @return non-empty-string|null
      */
@@ -447,6 +526,6 @@ private static function locationHint(?Identity $identity): ?string
 
         $fqn = $identity->fqn();
 
-        return $fqn === null ? null : "php_qn://{$identity->file}::\\{$fqn}";
+        return $fqn === null ? "file://{$identity->file}" : "php_qn://{$identity->file}::\\{$fqn}";
     }
 }

core/Output/Teamcity/Teamcity/TeamcityLogger.php

@@ -107,18 +107,29 @@ public function logEnvironment(): void
 
     /**
      * Publishes test suite started message using SuiteInfo.
+     *
+     * Announces the suite's size first, so an IDE can size its progress bar before the first test
+     * reports. The count is the number of located tests: a DataProvider test counts once here but
+     * reports one node per data set, so it is a lower bound rather than an exact total.
      */
     public function suiteStartedFromInfo(SuiteInfo $info): void
     {
+        $count = 0;
+        foreach ($info->testCases->getCases() as $case) {
+            $count += \count($case->tests->getTests());
+        }
+
+        $count > 0 and $this->publish(Formatter::testCount($count));
+
         $this->publish(Formatter::suiteStarted($info->name, $info->identity));
     }
 
     /**
      * Publishes test suite finished message using SuiteInfo.
      */
-    public function suiteFinishedFromInfo(SuiteInfo $info): void
+    public function suiteFinishedFromInfo(SuiteInfo $info, ?Status $status = null): void
     {
-        $this->publish(Formatter::suiteFinished($info->name, $info->identity));
+        $this->publish(Formatter::suiteFinished($info->name, $info->identity, $status));
     }
 
     /**
@@ -132,9 +143,9 @@ public function batchStartedFromInfo(TestInfo $info): void
     /**
      * Publishes test batch finished message (for DataProvider tests).
      */
-    public function batchFinishedFromInfo(TestInfo $info): void
+    public function batchFinishedFromInfo(TestInfo $info, ?Status $status = null): void
     {
-        $this->publish(Formatter::suiteFinished($info->name, $info->identity));
+        $this->publish(Formatter::suiteFinished($info->name, $info->identity, $status));
     }
 
     /**
@@ -156,7 +167,7 @@ public function handleSuiteResult(SuiteInfo $info, SuiteResult $result): void
             );
         }
 
-        $this->suiteFinishedFromInfo($info);
+        $this->suiteFinishedFromInfo($info, $result->status);
     }
 
     /**
@@ -174,9 +185,9 @@ public function caseStartedFromInfo(CaseInfo $info): void
      *
      * Test case is treated as a suite in TeamCity (a class containing tests).
      */
-    public function caseFinishedFromInfo(CaseInfo $info): void
+    public function caseFinishedFromInfo(CaseInfo $info, ?Status $status = null): void
     {
-        $this->publish(Formatter::suiteFinished($info->name, $info->identity));
+        $this->publish(Formatter::suiteFinished($info->name, $info->identity, $status));
     }
 
     /**
@@ -200,7 +211,7 @@ public function handleCaseResult(CaseInfo $caseInfo, CaseResult $result, ?int $d
             );
         }
 
-        $this->caseFinishedFromInfo($caseInfo);
+        $this->caseFinishedFromInfo($caseInfo, $result->status);
     }
 
     /**
@@ -368,6 +379,17 @@ private static function formatTrace(array $trace): string
         return \implode("\n", $lines);
     }
 
+    /**
+     * How many assertions the test performed, or `null` when nothing counted them — the metric is
+     * contributed by the Assert plugin, and a suite running without it says nothing rather than zero.
+     *
+     * @return int<0, max>|null
+     */
+    private static function assertionsOf(TestResult $result): ?int
+    {
+        return $result->summary->metrics['assertions'] ?? null;
+    }
+
     private static function key(string $name): string
     {
         return "\033[36;1m{$name}:\033[0m ";
@@ -387,7 +409,13 @@ private function handlePassedTest(TestResult $result, ?int $duration, ?string $o
     {
         $name = $overrideName ?? $result->info->name;
 
-        $this->publish(Formatter::testFinished($name, $duration, $result->info->identity));
+        $this->publish(Formatter::testFinished(
+            $name,
+            $duration,
+            $result->info->identity,
+            $result->status,
+            self::assertionsOf($result),
+        ));
     }
 
     /**
@@ -399,8 +427,8 @@ private function handleSkippedTest(TestResult $result, ?int $duration, ?string $
     {
         $name = $overrideName ?? $result->info->name;
         $identity = $result->info->identity;
-        $this->publish(Formatter::testIgnored($name, identity: $identity));
-        $this->publish(Formatter::testFinished($name, $duration, $identity));
+        $this->publish(Formatter::testIgnored($name, $result->failure?->getMessage() ?? '', $identity));
+        $this->publish(Formatter::testFinished($name, $duration, $identity, $result->status));
     }
 
     /**
@@ -412,8 +440,9 @@ private function handleCancelledTest(TestResult $result, ?int $duration, ?string
     {
         $name = $overrideName ?? $result->info->name;
         $identity = $result->info->identity;
-        $this->publish(Formatter::testIgnored($name, 'Test cancelled', $identity));
-        $this->publish(Formatter::testFinished($name, $duration, $identity));
+        $message = $result->failure?->getMessage() ?? '';
+        $this->publish(Formatter::testIgnored($name, $message === '' ? 'Test cancelled' : $message, $identity));
+        $this->publish(Formatter::testFinished($name, $duration, $identity, $result->status));
     }
 
     /**
@@ -445,7 +474,7 @@ private function handleFailedTest(TestResult $result, ?int $duration, ?string $o
                 identity: $identity,
             ),
         );
-        $this->publish(Formatter::testFinished($name, $duration, $identity));
+        $this->publish(Formatter::testFinished($name, $duration, $identity, $result->status));
     }
 
     /**
@@ -470,7 +499,7 @@ private function handleAbortedTest(TestResult $result, ?int $duration, ?string $
                 identity: $identity,
             ),
         );
-        $this->publish(Formatter::testFinished($name, $duration, $identity));
+        $this->publish(Formatter::testFinished($name, $duration, $identity, $result->status));
     }
 
     /**
@@ -490,7 +519,7 @@ private function handleRiskyTest(TestResult $result, ?int $duration, ?string $ov
                 identity: $identity,
             ),
         );
-        $this->publish(Formatter::testFinished($name, $duration, $identity));
+        $this->publish(Formatter::testFinished($name, $duration, $identity, $result->status));
     }
 
     /**

core/Output/Teamcity/TeamcityPlugin.php

@@ -187,7 +187,7 @@ private function onTestBatchStarting(TestBatchStarting $event): void
     private function onTestBatchFinished(TestBatchFinished $event): void
     {
         // For DataProvider tests, close the test suite
-        $this->logger->batchFinishedFromInfo($event->testInfo);
+        $this->logger->batchFinishedFromInfo($event->testInfo, $event->testResult->status);
     }
 
     private function onTestDataSetStarting(TestDataSetStarting $event): void

plugin/bench/composer.json

@@ -22,6 +22,7 @@
     "require": {
         "php": ">=8.2",
         "testo/data": "^0.1.7",
+        "testo/filter": "^0.1.6",
         "testo/inline": "^0.1.7",
         "testo/testo": "0.10.39 - 1"
     },