diff --git a/bridge/rector/composer.json b/bridge/rector/composer.json index e1556b58..132e0896 100644 --- a/bridge/rector/composer.json +++ b/bridge/rector/composer.json @@ -30,6 +30,7 @@ "require-dev": { "testo/assert": "^0.1.12", "testo/data": "^0.1.7", + "testo/filter": "^0.1.6", "testo/testo": "0.10.39 - 1" }, "autoload": { diff --git a/bridge/rector/src/Testing/Internal/Middleware/RectorFixtureInterceptor.php b/bridge/rector/src/Testing/Internal/Middleware/RectorFixtureInterceptor.php index 2131afde..8881160f 100644 --- a/bridge/rector/src/Testing/Internal/Middleware/RectorFixtureInterceptor.php +++ b/bridge/rector/src/Testing/Internal/Middleware/RectorFixtureInterceptor.php @@ -19,6 +19,7 @@ use Testo\Event\Test\TestBatchStarting; use Testo\Event\Test\TestDataSetFinished; use Testo\Event\Test\TestDataSetStarting; +use Testo\Filter\DataPointer; use Testo\Pipeline\Attribute\InterceptorOptions; use Testo\Pipeline\Middleware\TestRunInterceptor; @@ -61,10 +62,26 @@ public function runTest(TestInfo $info, callable $next): TestResult if ($reflection !== null && $fixtures !== []) { $runner = new RectorRunner($this->messenger, [$reflection->getName()]); + # A fixture occupies the data set slot of the address, so `--filter=Rule::fixture:0:2` + # selects the third fixture — the coordinates the IDE sends back for one data set. + $dataPointer = $info->getAttribute(DataPointer::class); + $num = -1; foreach ($fixtures as $label => $path) { ++$num; - $dsInfo = $info->with(arguments: [$runner, $path]); + if ($dataPointer !== null && ( + ($dataPointer->provider !== 0) + || ($dataPointer->dataset !== null && $dataPointer->dataset !== $num) + )) { + continue; + } + + # Each fixture needs its own address, or consumers that key on it collide: TeamCity + # would reuse the batch's node for every data set and nest none of them under it. + $dsInfo = $info->with( + arguments: [$runner, $path], + identity: $info->identity->toDataSet(dataProvider: 0, dataSet: $num), + ); $this->eventDispatcher->dispatch(new TestDataSetStarting($dsInfo, $label, null, $num)); try { diff --git a/core/Output/Teamcity/Teamcity/Formatter.php b/core/Output/Teamcity/Teamcity/Formatter.php index 849e0cb7..c74be6be 100644 --- a/core/Output/Teamcity/Teamcity/Formatter.php +++ b/core/Output/Teamcity/Teamcity/Formatter.php @@ -6,7 +6,9 @@ use Testo\Core\Context\Identity; use Testo\Core\Context\Identity\CaseIdentity; +use Testo\Core\Context\Identity\SuiteIdentity; use Testo\Core\Context\Identity\TestIdentity; +use Testo\Core\Value\Status; /** * Formats TeamCity service messages. @@ -59,7 +61,25 @@ public static function suiteStarted(string $name, ?Identity $identity = null): s $locationHint = self::locationHint($identity); $locationHint === null or $attributes['locationHint'] = $locationHint; - return self::formatMessage('testSuiteStarted', $attributes + self::placement($identity)); + return self::formatMessage( + 'testSuiteStarted', + $attributes + self::taxonomy($identity) + self::placement($identity), + ); + } + + /** + * Formats a message announcing how many tests are about to run. + * + * Feeds the progress bar of IntelliJ-based IDEs, which is the only consumer — the TeamCity server + * ignores it. Counts accumulate rather than replace, so one message per suite is the intended + * shape rather than a single total up front. + * + * @param int<0, max> $count + * @return non-empty-string + */ + public static function testCount(int $count): string + { + return self::formatMessage('testCount', ['count' => (string) $count]); } /** @@ -67,11 +87,15 @@ public static function suiteStarted(string $name, ?Identity $identity = null): s * * @param non-empty-string $name Suite name * @param Identity|null $identity Address of the node this message closes. {@see placement()} + * @param Status|null $status Aggregated outcome of the node. {@see status()} * @return non-empty-string */ - public static function suiteFinished(string $name, ?Identity $identity = null): string + public static function suiteFinished(string $name, ?Identity $identity = null, ?Status $status = null): string { - return self::formatMessage('testSuiteFinished', ['name' => $name] + self::placement($identity)); + return self::formatMessage( + 'testSuiteFinished', + ['name' => $name] + self::status($status) + self::placement($identity), + ); } /** @@ -96,7 +120,10 @@ public static function testStarted(string $name, bool $captureStandardOutput = f $description !== null and $attributes['metainfo'] = $description; - return self::formatMessage('testStarted', $attributes + self::placement($identity)); + return self::formatMessage( + 'testStarted', + $attributes + self::taxonomy($identity) + self::placement($identity), + ); } /** @@ -105,15 +132,25 @@ public static function testStarted(string $name, bool $captureStandardOutput = f * @param non-empty-string $name Test name * @param int<0, max>|null $duration Duration in milliseconds * @param TestIdentity|null $identity Address of the test this message closes. {@see placement()} + * @param Status|null $status Outcome of the test. {@see status()} + * @param int<0, max>|null $assertions Number of assertions the test performed. Null when nothing + * counted them — no assertion plugin is active — which is not the same as a test that + * counted zero. * @return non-empty-string */ - public static function testFinished(string $name, ?int $duration = null, ?TestIdentity $identity = null): string - { + public static function testFinished( + string $name, + ?int $duration = null, + ?TestIdentity $identity = null, + ?Status $status = null, + ?int $assertions = null, + ): string { $attributes = ['name' => $name]; $duration !== null and $attributes['duration'] = (string) $duration; + $assertions !== null and $attributes['assertions'] = (string) $assertions; - return self::formatMessage('testFinished', $attributes + self::placement($identity)); + return self::formatMessage('testFinished', $attributes + self::status($status) + self::placement($identity)); } /** @@ -422,6 +459,44 @@ private static function placement(?Identity $identity): array return $placement; } + /** + * Which suite the node belongs to and which kind of test it holds — the two things `--suite` and + * `--type` select on, so a consumer can offer the same slicing without parsing anything out of a + * name or a path. + * + * Only stated where the address knows it: a suite of the run has no type of its own, since one + * suite can hold cases of several ({@see CaseIdentity::$type}). + * + * @return array + */ + private static function taxonomy(?Identity $identity): array + { + return match (true) { + $identity instanceof CaseIdentity, + $identity instanceof TestIdentity => [ + 'testSuite' => $identity->suite, + 'testType' => $identity->type, + ], + $identity instanceof SuiteIdentity => ['testSuite' => $identity->suite], + default => [], + }; + } + + /** + * The exact outcome, which the standard protocol cannot express: it distinguishes only ignored, + * failed and everything else, so `Flaky` is indistinguishable from `Passed` and `Risky` from a + * clean pass. Consumers that understand the attribute get the {@see Status} verbatim; standard + * parsers ignore it and keep reading the run as before. + * + * Lowercased case name, the same wire format the JSON report speaks. + * + * @return array + */ + private static function status(?Status $status): array + { + return $status === null ? [] : ['status' => \strtolower($status->name)]; + } + /** * Location hint for whatever the address names. * @@ -430,12 +505,16 @@ private static function placement(?Identity $identity): array * php_qn://path/to/BarTest.php::\Ns\BarTest::itWorks a test, or its DataProvider batch node * php_qn://path/to/BarTest.php::\Ns\BarTest::itWorks:0:1 one data set of it * php_qn://path/to/functions.php::\Ns\itWorksToo a free test function + * file://path/to/functions.php a case of free functions * ``` * * The tail is {@see TestIdentity::fqn()} verbatim, so a hint pastes straight back into `--filter`. * - * Null when there is no code to point at: a suite of the run is a configuration entry, and a case - * of free functions has no class of its own. + * A case of free functions names no class, and the file it groups holds several functions rather + * than one to point at — so it answers with the file itself under `file://`, the scheme IDEs + * resolve to a whole file. Clickable all the same, which a hintless node is not. + * + * Null only for a suite of the run: it is a configuration entry, with no file of its own to name. * * @return non-empty-string|null */ @@ -447,6 +526,6 @@ private static function locationHint(?Identity $identity): ?string $fqn = $identity->fqn(); - return $fqn === null ? null : "php_qn://{$identity->file}::\\{$fqn}"; + return $fqn === null ? "file://{$identity->file}" : "php_qn://{$identity->file}::\\{$fqn}"; } } diff --git a/core/Output/Teamcity/Teamcity/TeamcityLogger.php b/core/Output/Teamcity/Teamcity/TeamcityLogger.php index 6ade50e6..ba776fb7 100644 --- a/core/Output/Teamcity/Teamcity/TeamcityLogger.php +++ b/core/Output/Teamcity/Teamcity/TeamcityLogger.php @@ -107,18 +107,29 @@ public function logEnvironment(): void /** * Publishes test suite started message using SuiteInfo. + * + * Announces the suite's size first, so an IDE can size its progress bar before the first test + * reports. The count is the number of located tests: a DataProvider test counts once here but + * reports one node per data set, so it is a lower bound rather than an exact total. */ public function suiteStartedFromInfo(SuiteInfo $info): void { + $count = 0; + foreach ($info->testCases->getCases() as $case) { + $count += \count($case->tests->getTests()); + } + + $count > 0 and $this->publish(Formatter::testCount($count)); + $this->publish(Formatter::suiteStarted($info->name, $info->identity)); } /** * Publishes test suite finished message using SuiteInfo. */ - public function suiteFinishedFromInfo(SuiteInfo $info): void + public function suiteFinishedFromInfo(SuiteInfo $info, ?Status $status = null): void { - $this->publish(Formatter::suiteFinished($info->name, $info->identity)); + $this->publish(Formatter::suiteFinished($info->name, $info->identity, $status)); } /** @@ -132,9 +143,9 @@ public function batchStartedFromInfo(TestInfo $info): void /** * Publishes test batch finished message (for DataProvider tests). */ - public function batchFinishedFromInfo(TestInfo $info): void + public function batchFinishedFromInfo(TestInfo $info, ?Status $status = null): void { - $this->publish(Formatter::suiteFinished($info->name, $info->identity)); + $this->publish(Formatter::suiteFinished($info->name, $info->identity, $status)); } /** @@ -156,7 +167,7 @@ public function handleSuiteResult(SuiteInfo $info, SuiteResult $result): void ); } - $this->suiteFinishedFromInfo($info); + $this->suiteFinishedFromInfo($info, $result->status); } /** @@ -174,9 +185,9 @@ public function caseStartedFromInfo(CaseInfo $info): void * * Test case is treated as a suite in TeamCity (a class containing tests). */ - public function caseFinishedFromInfo(CaseInfo $info): void + public function caseFinishedFromInfo(CaseInfo $info, ?Status $status = null): void { - $this->publish(Formatter::suiteFinished($info->name, $info->identity)); + $this->publish(Formatter::suiteFinished($info->name, $info->identity, $status)); } /** @@ -200,7 +211,7 @@ public function handleCaseResult(CaseInfo $caseInfo, CaseResult $result, ?int $d ); } - $this->caseFinishedFromInfo($caseInfo); + $this->caseFinishedFromInfo($caseInfo, $result->status); } /** @@ -368,6 +379,17 @@ private static function formatTrace(array $trace): string return \implode("\n", $lines); } + /** + * How many assertions the test performed, or `null` when nothing counted them — the metric is + * contributed by the Assert plugin, and a suite running without it says nothing rather than zero. + * + * @return int<0, max>|null + */ + private static function assertionsOf(TestResult $result): ?int + { + return $result->summary->metrics['assertions'] ?? null; + } + private static function key(string $name): string { return "\033[36;1m{$name}:\033[0m "; @@ -387,7 +409,13 @@ private function handlePassedTest(TestResult $result, ?int $duration, ?string $o { $name = $overrideName ?? $result->info->name; - $this->publish(Formatter::testFinished($name, $duration, $result->info->identity)); + $this->publish(Formatter::testFinished( + $name, + $duration, + $result->info->identity, + $result->status, + self::assertionsOf($result), + )); } /** @@ -399,8 +427,8 @@ private function handleSkippedTest(TestResult $result, ?int $duration, ?string $ { $name = $overrideName ?? $result->info->name; $identity = $result->info->identity; - $this->publish(Formatter::testIgnored($name, identity: $identity)); - $this->publish(Formatter::testFinished($name, $duration, $identity)); + $this->publish(Formatter::testIgnored($name, $result->failure?->getMessage() ?? '', $identity)); + $this->publish(Formatter::testFinished($name, $duration, $identity, $result->status)); } /** @@ -412,8 +440,9 @@ private function handleCancelledTest(TestResult $result, ?int $duration, ?string { $name = $overrideName ?? $result->info->name; $identity = $result->info->identity; - $this->publish(Formatter::testIgnored($name, 'Test cancelled', $identity)); - $this->publish(Formatter::testFinished($name, $duration, $identity)); + $message = $result->failure?->getMessage() ?? ''; + $this->publish(Formatter::testIgnored($name, $message === '' ? 'Test cancelled' : $message, $identity)); + $this->publish(Formatter::testFinished($name, $duration, $identity, $result->status)); } /** @@ -445,7 +474,7 @@ private function handleFailedTest(TestResult $result, ?int $duration, ?string $o identity: $identity, ), ); - $this->publish(Formatter::testFinished($name, $duration, $identity)); + $this->publish(Formatter::testFinished($name, $duration, $identity, $result->status)); } /** @@ -470,7 +499,7 @@ private function handleAbortedTest(TestResult $result, ?int $duration, ?string $ identity: $identity, ), ); - $this->publish(Formatter::testFinished($name, $duration, $identity)); + $this->publish(Formatter::testFinished($name, $duration, $identity, $result->status)); } /** @@ -490,7 +519,7 @@ private function handleRiskyTest(TestResult $result, ?int $duration, ?string $ov identity: $identity, ), ); - $this->publish(Formatter::testFinished($name, $duration, $identity)); + $this->publish(Formatter::testFinished($name, $duration, $identity, $result->status)); } /** diff --git a/core/Output/Teamcity/TeamcityPlugin.php b/core/Output/Teamcity/TeamcityPlugin.php index cde3118d..72b62b84 100644 --- a/core/Output/Teamcity/TeamcityPlugin.php +++ b/core/Output/Teamcity/TeamcityPlugin.php @@ -187,7 +187,7 @@ private function onTestBatchStarting(TestBatchStarting $event): void private function onTestBatchFinished(TestBatchFinished $event): void { // For DataProvider tests, close the test suite - $this->logger->batchFinishedFromInfo($event->testInfo); + $this->logger->batchFinishedFromInfo($event->testInfo, $event->testResult->status); } private function onTestDataSetStarting(TestDataSetStarting $event): void diff --git a/plugin/bench/composer.json b/plugin/bench/composer.json index 0ba6dec0..31853b61 100644 --- a/plugin/bench/composer.json +++ b/plugin/bench/composer.json @@ -22,6 +22,7 @@ "require": { "php": ">=8.2", "testo/data": "^0.1.7", + "testo/filter": "^0.1.6", "testo/inline": "^0.1.7", "testo/testo": "0.10.39 - 1" }, diff --git a/plugin/bench/src/Internal/Pipeline/BenchInterceptor.php b/plugin/bench/src/Internal/Pipeline/BenchInterceptor.php index 763e287a..644e0ebb 100644 --- a/plugin/bench/src/Internal/Pipeline/BenchInterceptor.php +++ b/plugin/bench/src/Internal/Pipeline/BenchInterceptor.php @@ -16,6 +16,7 @@ use Testo\Event\Test\TestBatchStarting; use Testo\Event\Test\TestDataSetFinished; use Testo\Event\Test\TestDataSetStarting; +use Testo\Filter\DataPointer; use Testo\Pipeline\Attribute\InterceptorOptions; use Testo\Pipeline\Middleware\TestRunInterceptor; use Testo\Pipeline\Policy\ConflictPolicy; @@ -53,12 +54,24 @@ public function runTest(TestInfo $info, callable $next): TestResult # Dispatch batch starting event $this->eventDispatcher->dispatch(new TestBatchStarting($info)); + # Load Filters + $dataPointer = $info->getAttribute(DataPointer::class); + # Run the test for each data set $results = []; $status = Status::Passed; foreach ($attributes as $index => $attr) { + # Check Filters + if ($dataPointer !== null && ( + $dataPointer->provider !== $index + || ($dataPointer->dataset !== null && $dataPointer->dataset !== 0) + )) { + continue; + } + # Each attribute occupies the provider slot of the address, as it does for inline tests, - # with a single data set inside it. + # with a single data set inside it — matching the filter check above, so + # `--filter=method:2` and `--filter=method:2:0` both select the third one. $newInfo = $info ->with(arguments: $attr->arguments, identity: $info->identity->toDataSet(dataProvider: $index, dataSet: 0)) ->withAttribute(Bench::class, $attr); @@ -90,17 +103,27 @@ public function runTest(TestInfo $info, callable $next): TestResult $results[] = $result; } - # Each benchmark case counts as a test, so the aggregate is the sum of their summaries, - # each stamped with the case's final status. - $summary = Summary::combine(\array_map( - static fn(TestResult $r): Summary => $r->summary->withStatus($r->status), - $results, - )); - $results = new MultipleResult($results); - - $finalResult = new TestResult(info: $info, status: $status, result: $results, attributes: [ - MultipleResult::class => $results, - ], summary: $summary); + if ($results === []) { + # No benchmark cases matched the filter — count as a single Risky test; leave counts empty + # so the final status is stamped late by the TestRunner. + $status->isFailure() or $status = Status::Risky; + $finalResult = new TestResult( + info: $info, + status: $status, + result: new \RuntimeException('No benchmark cases were provided.'), + ); + } else { + # Each benchmark case counts as a test, so the aggregate is the sum of their summaries, + # each stamped with the case's final status. + $summary = Summary::combine(\array_map( + static fn(TestResult $r): Summary => $r->summary->withStatus($r->status), + $results, + )); + $multiple = new MultipleResult($results); + $finalResult = new TestResult(info: $info, status: $status, result: $multiple, attributes: [ + MultipleResult::class => $multiple, + ], summary: $summary); + } # Dispatch batch finished event $this->eventDispatcher->dispatch(new TestBatchFinished($info, $finalResult)); diff --git a/plugin/bench/tests/Unit/BenchInterceptorTest.php b/plugin/bench/tests/Unit/BenchInterceptorTest.php new file mode 100644 index 00000000..fb1d763b --- /dev/null +++ b/plugin/bench/tests/Unit/BenchInterceptorTest.php @@ -0,0 +1,147 @@ +runTest( + self::createTestInfo([self::bench(), self::bench()]), + $next, + ); + + Assert::same($callCount, 2); + Assert::same($result->status, Status::Passed); + + # Each benchmark case counts as a test; the aggregate folds their summaries. + Assert::same($result->summary->total(), 2); + Assert::same($result->summary->count(Status::Passed), 2); + + $multiple = $result->getAttribute(MultipleResult::class); + Assert::instanceOf($multiple, MultipleResult::class); + Assert::same(\count($multiple->results), 2); + } + + public function noMatchingBenchmarkCasesYieldASingleRiskyTest(): void + { + $called = false; + $next = static function (TestInfo $info) use (&$called): TestResult { + $called = true; + return new TestResult(info: $info, status: Status::Passed); + }; + + # A DataPointer that matches no benchmark case index leaves the result set empty. + $result = (new BenchInterceptor(self::createDispatcher()))->runTest( + self::createTestInfo( + [self::bench(), self::bench()], + new DataPointer(provider: 99, dataset: null), + ), + $next, + ); + + Assert::false($called); + Assert::same($result->status, Status::Risky); + Assert::instanceOf($result->result, \RuntimeException::class); + Assert::null($result->getAttribute(MultipleResult::class)); + + # Counts are left empty on purpose — the TestRunner stamps the final status late. + Assert::same($result->summary->total(), 0); + } + + public function theDataSetCoordinateSelectsTheOnlySetABenchmarkCaseHas(): void + { + $reached = []; + $next = static function (TestInfo $info) use (&$reached): TestResult { + $reached[] = [$info->identity->dataProvider, $info->identity->dataSet]; + return new TestResult(info: $info, status: Status::Passed); + }; + + $interceptor = new BenchInterceptor(self::createDispatcher()); + $benches = [self::bench(), self::bench(), self::bench()]; + + # A benchmark case is a provider slot holding a single data set, so `:1:0` names the second one. + $hit = $interceptor->runTest( + self::createTestInfo($benches, new DataPointer(provider: 1, dataset: 0)), + $next, + ); + + Assert::same($hit->status, Status::Passed); + Assert::same($hit->summary->total(), 1); + Assert::same($reached, [[1, 0]]); + + # There is no second data set inside that slot, so `:1:1` names nothing. + $miss = $interceptor->runTest( + self::createTestInfo($benches, new DataPointer(provider: 1, dataset: 1)), + $next, + ); + + Assert::same($miss->status, Status::Risky); + Assert::same($reached, [[1, 0]]); + } + + private static function bench(): Bench + { + return new Bench([static fn(): int => 1]); + } + + private static function createDispatcher(): EventDispatcherInterface + { + return new class implements EventDispatcherInterface { + #[\Override] + public function dispatch(object $event): object + { + return $event; + } + }; + } + + /** + * @param list $benches + */ + private static function createTestInfo(array $benches, ?DataPointer $dataPointer = null): TestInfo + { + $caseInfo = new CaseInfo( + definition: new CaseDefinition(name: 'TestCase', type: 'bench', file: Path::create(__FILE__)), + suiteIdentity: new SuiteIdentity('Bench/Unit'), + ); + $testDefinition = new TestDefinition(reflection: new \ReflectionFunction(static fn() => null)); + + $attributes = [Bench::class => $benches]; + $dataPointer === null or $attributes[DataPointer::class] = $dataPointer; + + return new TestInfo( + name: 'target', + caseInfo: $caseInfo, + testDefinition: $testDefinition, + attributes: $attributes, + ); + } +} diff --git a/plugin/bench/tests/suites.php b/plugin/bench/tests/suites.php index 0e33b58d..c3c544a7 100644 --- a/plugin/bench/tests/suites.php +++ b/plugin/bench/tests/suites.php @@ -15,6 +15,12 @@ include: [__DIR__ . '/Self'], ), ), + new SuiteConfig( + name: 'Bench/Unit', + location: new FinderConfig( + include: [__DIR__ . '/Unit'], + ), + ), new SuiteConfig( name: 'Bench/Inline', location: new FinderConfig( diff --git a/plugin/inline/src/Internal/InlineInterceptor.php b/plugin/inline/src/Internal/InlineInterceptor.php index d378452d..43a21ecd 100644 --- a/plugin/inline/src/Internal/InlineInterceptor.php +++ b/plugin/inline/src/Internal/InlineInterceptor.php @@ -63,12 +63,16 @@ public function runTest(TestInfo $info, callable $next): TestResult $status = Status::Passed; foreach ($attributes as $index => $inline) { # Check Filters - if ($dataPointer !== null && $dataPointer->provider !== $index) { + if ($dataPointer !== null && ( + $dataPointer->provider !== $index + || ($dataPointer->dataset !== null && $dataPointer->dataset !== 0) + )) { continue; } - # Each attribute occupies the provider slot of the address, matching the filter check above - # (`--filter=method:2` selects the third one), with a single data set inside it. + # Each attribute occupies the provider slot of the address, with a single data set inside + # it — matching the filter check above, so `--filter=method:2` and `--filter=method:2:0` + # both select the third one. $newInfo = $info ->with(arguments: $inline->arguments, identity: $info->identity->toDataSet(dataProvider: $index, dataSet: 0)) ->withAttribute(TestInline::class, $inline); diff --git a/plugin/inline/tests/Unit/InlineInterceptorTest.php b/plugin/inline/tests/Unit/InlineInterceptorTest.php index 66d5845d..310e92c8 100644 --- a/plugin/inline/tests/Unit/InlineInterceptorTest.php +++ b/plugin/inline/tests/Unit/InlineInterceptorTest.php @@ -76,6 +76,37 @@ public function noMatchingInlineCasesYieldASingleRiskyTest(): void Assert::same($result->summary->total(), 0); } + public function theDataSetCoordinateSelectsTheOnlySetAnInlineCaseHas(): void + { + $reached = []; + $next = static function (TestInfo $info) use (&$reached): TestResult { + $reached[] = [$info->identity->dataProvider, $info->identity->dataSet]; + return new TestResult(info: $info, status: Status::Passed); + }; + + $interceptor = new InlineInterceptor(self::createDispatcher()); + $inlines = [new TestInline([1]), new TestInline([2]), new TestInline([3])]; + + # An inline case is a provider slot holding a single data set, so `:1:0` names the second case. + $hit = $interceptor->runTest( + self::createTestInfo($inlines, new DataPointer(provider: 1, dataset: 0)), + $next, + ); + + Assert::same($hit->status, Status::Passed); + Assert::same($hit->summary->total(), 1); + Assert::same($reached, [[1, 0]]); + + # There is no second data set inside that slot, so `:1:1` names nothing. + $miss = $interceptor->runTest( + self::createTestInfo($inlines, new DataPointer(provider: 1, dataset: 1)), + $next, + ); + + Assert::same($miss->status, Status::Risky); + Assert::same($reached, [[1, 0]]); + } + private static function createDispatcher(): EventDispatcherInterface { return new class implements EventDispatcherInterface { diff --git a/skills/testo-plugin-author/SKILL.md b/skills/testo-plugin-author/SKILL.md index 3318fd05..febb1baf 100644 --- a/skills/testo-plugin-author/SKILL.md +++ b/skills/testo-plugin-author/SKILL.md @@ -137,7 +137,8 @@ Two independent things live on it: run to run. `dataProvider`/`dataSet` are set only for a data set, and address it by **index** — provider keys may repeat, so only the index tells two data sets apart. `fqn()` is the machine-facing form: no suite, no type, pastes straight into `--filter`, and is the tail of TeamCity's - `locationHint` (`php_qn://::\`). + `locationHint` (`php_qn://::\`). A case of free functions has no `fqn()` — no class to + qualify — and is hinted as `file://` instead, so its node stays clickable. - **`runtimeId`** says *which run of it* is in flight, **`pipelineId`** which test run that one is part of — its own for a test, the batch's for each of its data sets — and **`parentId`** which run it opened inside (`null` at a suite). All three are process-local: never persist them or match on them. @@ -155,6 +156,25 @@ from `runtimeId` and its parent from `parentId`, the same two fields at every le tree off the order events arrive in: concurrent tests interleave, so a consumer that nests by "whatever opened last" puts one test's node inside another's. +The built-in TeamCity output carries the **exact** `Status` as a `status` attribute (lowercased case +name: `passed`, `failed`, `skipped`, `error`, `risky`, `flaky`, `cancelled`, `aborted`) on every +`testFinished`, and the aggregated one on `testSuiteFinished` for a suite, a case and a DataProvider +batch. The standard protocol collapses those eight into ignored/failed/neither, so a consumer that +needs `Flaky` apart from `Passed`, or `Risky` apart from a clean pass, reads them there; standard +parsers ignore the attribute. `testFinished` also carries `assertions` — the count the Assert plugin +records under that metric name — omitted entirely when no plugin counted them, which is not the same +as `assertions='0'` for a test that asserted nothing. + +Every opening message — `testSuiteStarted` for a suite, a case or a DataProvider batch, and +`testStarted` for a test or a data set — carries `testSuite` and `testType`, the two things `--suite` +and `--type` select on. A suite of the run states only `testSuite`: it holds cases of several types +and has none of its own. + +Each suite opens with `##teamcity[testCount count='N']` — the tests located for it, read off +`SuiteInfo::$testCases` before the first one runs. Counts accumulate across suites in IntelliJ-based +IDEs (the TeamCity server ignores the message), so one per suite is the intended shape. A DataProvider +test counts once but reports a node per data set, so the number is a lower bound. + ### Passing state down the pipeline — prefer attributes over mutable fields `TestInfo`, `CaseInfo`, and `TestResult` use the `Attributed` trait: `withAttribute(string $name, diff --git a/tests/Output/Unit/Teamcity/FormatterTest.php b/tests/Output/Unit/Teamcity/FormatterTest.php index 3967022f..33f5808f 100644 --- a/tests/Output/Unit/Teamcity/FormatterTest.php +++ b/tests/Output/Unit/Teamcity/FormatterTest.php @@ -8,6 +8,7 @@ use Testo\Assert; use Testo\Core\Context\Identity\SuiteIdentity; use Testo\Core\Context\Identity\TestIdentity; +use Testo\Core\Value\Status; use Testo\Output\Teamcity\Teamcity\Formatter; use Testo\Test; @@ -69,6 +70,108 @@ public function aFinishedMessageNamesTheNodeItCloses(): void Assert::string($msg)->contains("duration='12'"); } + public function aFinishedMessageCarriesTheExactStatus(): void + { + $msg = Formatter::testFinished('itWorks', 12, self::test(), Status::Flaky); + + // The protocol itself cannot say "flaky" — without the attribute this message is byte-identical + // to a clean pass, and a consumer has no way to tell the two apart. + Assert::string($msg)->contains("status='flaky'"); + } + + public function aFinishedMessageWithoutAStatusCarriesNoStatusAttribute(): void + { + $msg = Formatter::testFinished('itWorks', 12, self::test()); + + Assert::string($msg)->notContains('status='); + } + + public function aFinishedMessageCountsTheAssertionsTheTestPerformed(): void + { + $msg = Formatter::testFinished('itWorks', 12, self::test(), Status::Passed, assertions: 3); + + Assert::string($msg)->contains("assertions='3'"); + } + + public function aTestThatCountedNoAssertionsSaysSoRatherThanStayingSilent(): void + { + // Zero is a fact about the test — an unasserted pass — and has to survive as one; only an + // uncounted test (no assertion plugin) omits the attribute. + $counted = Formatter::testFinished('itWorks', 12, self::test(), Status::Risky, assertions: 0); + $uncounted = Formatter::testFinished('itWorks', 12, self::test(), Status::Passed); + + Assert::string($counted)->contains("assertions='0'"); + Assert::string($uncounted)->notContains('assertions='); + } + + public function aFinishedSuiteCarriesItsAggregatedStatus(): void + { + $suite = new SuiteIdentity('Core/Unit'); + + $msg = Formatter::suiteFinished('Core/Unit', $suite, Status::Failed); + + Assert::string($msg)->contains("status='failed'"); + } + + public function aCaseOfFreeFunctionsPointsAtItsFile(): void + { + $case = (new SuiteIdentity('Core/Unit')) + ->toCase(null, 'test', Path::create('/app/tests/functions.php')); + + $msg = Formatter::suiteStarted('functions.php', $case); + + // No class to qualify, and the file holds several functions rather than one to name — so the + // hint points at the file. Without it the node is the only one in the tree nobody can click. + Assert::string($msg)->contains("locationHint='file:///app/tests/functions.php'"); + } + + public function aSuiteOfTheRunStillHasNothingToPointAt(): void + { + $msg = Formatter::suiteStarted('Core/Unit', new SuiteIdentity('Core/Unit')); + + // A configuration entry, with no file of its own. + Assert::string($msg)->notContains('locationHint'); + } + + public function anOpeningNodeNamesTheSuiteAndTypeItBelongsTo(): void + { + $suite = new SuiteIdentity('Core/Unit'); + $case = $suite->toCase('Tests\Foo\BarTest', 'bench', Path::create('/app/tests/BarTest.php')); + + $caseMsg = Formatter::suiteStarted('BarTest', $case); + $testMsg = Formatter::testStarted('itWorks', identity: $case->toTest('itWorks')); + + // What `--suite` and `--type` select on, stated rather than left to be parsed out of a name. + Assert::string($caseMsg)->contains("testSuite='Core/Unit'"); + Assert::string($caseMsg)->contains("testType='bench'"); + Assert::string($testMsg)->contains("testSuite='Core/Unit'"); + Assert::string($testMsg)->contains("testType='bench'"); + } + + public function aSuiteOfTheRunNamesItselfButClaimsNoType(): void + { + $msg = Formatter::suiteStarted('Core/Unit', new SuiteIdentity('Core/Unit')); + + // One suite holds cases of several types, so it has none of its own to report. + Assert::string($msg)->contains("testSuite='Core/Unit'"); + Assert::string($msg)->notContains('testType='); + } + + public function anOpeningNodeWithoutAnAddressClaimsNeither(): void + { + $msg = Formatter::testStarted('itWorks'); + + Assert::string($msg)->notContains('testSuite='); + Assert::string($msg)->notContains('testType='); + } + + public function testCountAnnouncesHowManyTestsAreAboutToRun(): void + { + $msg = Formatter::testCount(42); + + Assert::same($msg, "##teamcity[testCount count='42']"); + } + public function testFailedWithoutComparisonHasNoExtraAttributes(): void { $msg = Formatter::testFailed( diff --git a/tests/Output/Unit/Teamcity/TeamcityLoggerTest.php b/tests/Output/Unit/Teamcity/TeamcityLoggerTest.php index 5364e182..bd3dcf40 100644 --- a/tests/Output/Unit/Teamcity/TeamcityLoggerTest.php +++ b/tests/Output/Unit/Teamcity/TeamcityLoggerTest.php @@ -10,13 +10,19 @@ use Testo\Assert\State\Assertion\ComparisonFailure; use Testo\Core\Context\CaseInfo; use Testo\Core\Context\Identity\SuiteIdentity; +use Testo\Core\Context\SuiteInfo; use Testo\Core\Context\TestInfo; use Testo\Core\Context\TestResult; use Testo\Core\Definition\CaseDefinition; +use Testo\Core\Definition\CaseDefinitions; +use Testo\Core\Definition\TestDefinitions; use Testo\Core\Definition\TestDefinition; +use Testo\Core\Exception\CancelTest; +use Testo\Core\Exception\SkipTest; use Testo\Core\Log\Level; use Testo\Core\Log\Message; use Testo\Core\Value\Status; +use Testo\Core\Value\Summary; use Testo\Output\Teamcity\Teamcity\TeamcityLogger; use Testo\Test; use Tests\Output\Stub\Teamcity\ConcreteSampleTestCase; @@ -148,6 +154,35 @@ public function testStartedFromInfoOmitsMetainfoWhenNoPhpDoc(): void Assert::string($output)->notContains('metainfo='); } + public function aStartingSuiteAnnouncesHowManyTestsItHolds(): void + { + $info = new SuiteInfo( + name: 'Output/Unit', + testCases: CaseDefinitions::fromArray( + self::makeCase('passingTest', 'failingTest'), + self::makeCase('describedTest'), + ), + ); + + $output = self::capture(static fn(TeamcityLogger $logger) => $logger->suiteStartedFromInfo($info)); + + // Counts across every case of the suite, and lands before the suite opens so an IDE can size + // its progress bar before the first test reports. + Assert::string($output)->contains("##teamcity[testCount count='3']"); + Assert::true( + \strpos($output, 'testCount') < \strpos($output, 'testSuiteStarted'), + ); + } + + public function anEmptySuiteAnnouncesNoCount(): void + { + $info = new SuiteInfo(name: 'Output/Unit', testCases: CaseDefinitions::fromArray()); + + $output = self::capture(static fn(TeamcityLogger $logger) => $logger->suiteStartedFromInfo($info)); + + Assert::string($output)->notContains('testCount'); + } + public function logEmptyRunEmitsBuildProblem(): void { $output = self::capture(static fn(TeamcityLogger $logger) => $logger->logEmptyRun()); @@ -188,6 +223,93 @@ public function handleSingleTestResultStampsFlowIdFromIdentity(): void Assert::same(\substr_count($output, "flowId='{$result->info->identity->pipelineId}'"), 2); } + public function everyStatusReachesTheConsumerOnTheFinishMessage(): void + { + foreach (Status::cases() as $status) { + $result = new TestResult( + info: self::makeInfo('passingTest'), + status: $status, + failure: $status->isFailure() ? new \RuntimeException('boom') : null, + attributes: ['duration' => 0], + ); + + $output = self::capture(static fn(TeamcityLogger $logger) => $logger->handleSingleTestResult($result)); + + // The standard messages collapse eight outcomes into three shapes — a consumer that needs the + // exact one reads it off `testFinished`, which every branch emits. + $expected = \strtolower($status->name); + Assert::string($output)->contains("##teamcity[testFinished"); + Assert::string($output)->contains("status='{$expected}'"); + } + } + + public function aCancelledTestCarriesTheReasonFromTheException(): void + { + $result = self::makeResult(Status::Cancelled, new CancelTest('deadline exceeded while waiting for the queue')); + + $output = self::capture(static fn(TeamcityLogger $logger) => $logger->handleSingleTestResult($result)); + + // The reason lives in the thrown exception; a generic stand-in would hide why the run was aborted. + Assert::string($output)->contains("message='deadline exceeded while waiting for the queue'"); + } + + public function aCancelledTestWithoutAReasonFallsBackToAGenericMessage(): void + { + $result = self::makeResult(Status::Cancelled, new CancelTest()); + + $output = self::capture(static fn(TeamcityLogger $logger) => $logger->handleSingleTestResult($result)); + + Assert::string($output)->contains("message='Test cancelled'"); + } + + public function aSkippedTestCarriesTheReasonFromTheException(): void + { + $result = self::makeResult(Status::Skipped, new SkipTest('sqlite extension is missing')); + + $output = self::capture(static fn(TeamcityLogger $logger) => $logger->handleSingleTestResult($result)); + + Assert::string($output)->contains("message='sqlite extension is missing'"); + } + + public function aSkippedTestWithoutAReasonOmitsTheMessage(): void + { + $result = self::makeResult(Status::Skipped, new SkipTest()); + + $output = self::capture(static fn(TeamcityLogger $logger) => $logger->handleSingleTestResult($result)); + + Assert::string($output)->contains('##teamcity[testIgnored'); + Assert::string($output)->notContains('message='); + } + + public function aPassedTestReportsHowManyAssertionsItPerformed(): void + { + $result = new TestResult( + info: self::makeInfo('passingTest'), + status: Status::Passed, + attributes: ['duration' => 0], + summary: new Summary(metrics: ['assertions' => 7]), + ); + + $output = self::capture(static fn(TeamcityLogger $logger) => $logger->handleSingleTestResult($result)); + + Assert::string($output)->contains("assertions='7'"); + } + + public function aPassedTestNobodyCountedAssertionsForOmitsTheAttribute(): void + { + $result = new TestResult( + info: self::makeInfo('passingTest'), + status: Status::Passed, + attributes: ['duration' => 0], + ); + + $output = self::capture(static fn(TeamcityLogger $logger) => $logger->handleSingleTestResult($result)); + + // The count comes from the Assert plugin; without it there is no number to report, and a + // fabricated zero would read as an unasserted test. + Assert::string($output)->notContains('assertions='); + } + public function logMessagePlacesTheOutputOnItsTestsNode(): void { $info = self::makeInfo('passingTest'); @@ -266,6 +388,37 @@ private static function makeFailedResult(\Throwable $failure): TestResult ); } + private static function makeResult(Status $status, ?\Throwable $failure = null): TestResult + { + return new TestResult( + info: self::makeInfo('passingTest'), + status: $status, + failure: $failure, + attributes: ['duration' => 0], + ); + } + + /** + * A case of {@see SampleTestClass} holding exactly the named methods as its tests. + * + * @param non-empty-string ...$methods + */ + private static function makeCase(string ...$methods): CaseDefinition + { + $tests = new TestDefinitions(); + foreach ($methods as $method) { + $tests->define(new \ReflectionMethod(SampleTestClass::class, $method)); + } + + return new CaseDefinition( + name: SampleTestClass::class, + type: 'test', + file: Path::create(__FILE__), + reflection: new \ReflectionClass(SampleTestClass::class), + tests: $tests, + ); + } + /** * @param non-empty-string $method Method of {@see SampleTestClass} backing the test definition. */ diff --git a/tests/Sandbox/Self/AssertTest.php b/tests/Sandbox/Self/AssertTest.php index b482768f..60035583 100644 --- a/tests/Sandbox/Self/AssertTest.php +++ b/tests/Sandbox/Self/AssertTest.php @@ -4,11 +4,12 @@ namespace Tests\Sandbox\Self; -use Testo; use Testo\Assert; use Testo\Assert\ExpectException; use Testo\Assert\State\Assertion\AssertionException; use Testo\Common\Messenger; +use Testo\Core\Exception\CancelTest; +use Testo\Core\Exception\SkipTest; use Testo\Data\DataProvider; use Testo\Data\DataSet; use Testo\Expect; @@ -20,7 +21,7 @@ use Tests\Fixture\ClassDataProvider; /** - * Aassertions sandbox + * Assertions sandbox */ #[Group('sandbox')] final class AssertTest @@ -41,6 +42,31 @@ public static function dataForProvider(): iterable yield 'Any warrior can change the world.' => ['yep']; } + #[Test] + public static function loggerFacade(): void + { + $r = new \ReflectionMethod(self::class, __FUNCTION__); + $code = \array_slice(\file(__FILE__), $r->getStartLine() - 1, $r->getEndLine() - $r->getStartLine() + 1); + \Testo::logger('test.log')->emergency( + <<messenger->channel('query.sql')->debug(<<messenger->channel('response.json')->write(<<messenger->channel('docs.md')->write(<<getStartLine() - 1, $r->getEndLine() - $r->getStartLine() + 1); - Testo::logger('test.log')->emergency(<<messenger->channel('query.sql')->debug( + <<messenger->channel('response.json')->write( + <<messenger->channel('docs.md')->write( + <<