From 5d1907bb1bcdcb5173803cd04b000d7b383c57b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Wed, 15 Jul 2026 16:28:16 +0200 Subject: [PATCH 1/7] GitHubActionsReport: surface exit-code verdict and per-assembly clarity Addresses feedback on issue #9003 for the GitHub Actions report extension: - Map MTP exit codes (zero tests, --minimum-expected-tests, aborted, --maximum-failed-tests, adapter/host failures) to friendly reasons. - Step summary now flips to failure icon and renders an exit-code callout when the run failed without any failed test. - Annotation reporter emits a run-level ::error for non-test-result failures so they surface in the Annotations tab. - Each assembly's step-summary section carries its own verdict for multi-assembly (dotnet test) clarity. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b1e3944b-d2cf-4ab7-ac53-c8ace4ddc260 --- .../GitHubActionsAnnotationReporter.cs | 70 ++++++++++++++++ .../GitHubActionsExitCode.cs | 62 ++++++++++++++ .../GitHubActionsExtensions.cs | 7 +- .../GitHubActionsSummaryReporter.cs | 26 +++++- .../InternalAPI/InternalAPI.Unshipped.txt | 14 ++++ .../PACKAGE.md | 4 +- .../Resources/GitHubActionsResources.resx | 52 ++++++++++++ .../xlf/GitHubActionsResources.cs.xlf | 80 +++++++++++++++++++ .../xlf/GitHubActionsResources.de.xlf | 80 +++++++++++++++++++ .../xlf/GitHubActionsResources.es.xlf | 80 +++++++++++++++++++ .../xlf/GitHubActionsResources.fr.xlf | 80 +++++++++++++++++++ .../xlf/GitHubActionsResources.it.xlf | 80 +++++++++++++++++++ .../xlf/GitHubActionsResources.ja.xlf | 80 +++++++++++++++++++ .../xlf/GitHubActionsResources.ko.xlf | 80 +++++++++++++++++++ .../xlf/GitHubActionsResources.pl.xlf | 80 +++++++++++++++++++ .../xlf/GitHubActionsResources.pt-BR.xlf | 80 +++++++++++++++++++ .../xlf/GitHubActionsResources.ru.xlf | 80 +++++++++++++++++++ .../xlf/GitHubActionsResources.tr.xlf | 80 +++++++++++++++++++ .../xlf/GitHubActionsResources.zh-Hans.xlf | 80 +++++++++++++++++++ .../xlf/GitHubActionsResources.zh-Hant.xlf | 80 +++++++++++++++++++ .../GitHubActionsExitCodeTests.cs | 61 ++++++++++++++ .../GitHubActionsSummaryReporterTests.cs | 49 +++++++++++- 22 files changed, 1375 insertions(+), 10 deletions(-) create mode 100644 src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsExitCode.cs create mode 100644 test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsExitCodeTests.cs diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsAnnotationReporter.cs b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsAnnotationReporter.cs index ec19d823b3..5f4b2587c1 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsAnnotationReporter.cs +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsAnnotationReporter.cs @@ -6,9 +6,11 @@ using Microsoft.Testing.Platform.Extensions; using Microsoft.Testing.Platform.Extensions.Messages; using Microsoft.Testing.Platform.Extensions.OutputDevice; +using Microsoft.Testing.Platform.Extensions.TestHost; using Microsoft.Testing.Platform.Helpers; using Microsoft.Testing.Platform.Logging; using Microsoft.Testing.Platform.OutputDevice; +using Microsoft.Testing.Platform.Services; namespace Microsoft.Testing.Extensions.GitHubActionsReport; @@ -18,13 +20,21 @@ namespace Microsoft.Testing.Extensions.GitHubActionsReport; /// pull request's "Files changed" diff gutter. Skipped tests are surfaced as title-only ::warning /// workflow commands so they are visible in the Annotations tab too. /// +/// +/// It also implements so that, at session end, it can emit one +/// extra ::error for a non-test-result failure exit code (e.g. a --minimum-expected-tests +/// violation or a run that discovered zero tests). Those outcomes carry no failing , +/// so without this they would leave the Annotations tab empty despite the run failing. +/// internal sealed class GitHubActionsAnnotationReporter : IDataConsumer, + ITestSessionLifetimeHandler, IOutputDeviceDataProducer { private readonly IEnvironment _environment; private readonly IFileSystem _fileSystem; private readonly IOutputDevice _outputDisplay; + private readonly ITestApplicationProcessExitCode _testApplicationProcessExitCode; private readonly ILogger _logger; private readonly bool _isEnabled; @@ -33,11 +43,13 @@ public GitHubActionsAnnotationReporter( IEnvironment environment, IFileSystem fileSystem, IOutputDevice outputDisplay, + ITestApplicationProcessExitCode testApplicationProcessExitCode, ILoggerFactory loggerFactory) { _environment = environment; _fileSystem = fileSystem; _outputDisplay = outputDisplay; + _testApplicationProcessExitCode = testApplicationProcessExitCode; _logger = loggerFactory.CreateLogger(); _isEnabled = GitHubActionsFeature.IsEnabled(commandLine, environment, GitHubActionsCommandLineOptions.GitHubActionsAnnotations); } @@ -117,6 +129,64 @@ public async Task ConsumeAsync(IDataProducer dataProducer, IData value, Cancella } } + public Task OnTestSessionStartingAsync(ITestSessionContext sessionUid) => Task.CompletedTask; + + public async Task OnTestSessionFinishingAsync(ITestSessionContext testSessionContext) + { + try + { + testSessionContext.CancellationToken.ThrowIfCancellationRequested(); + + if (!_isEnabled) + { + return; + } + + int exitCode = _testApplicationProcessExitCode.GetProcessExitCode(); + + // Only emit the run-level annotation for a non-test-result failure. Plain "at least one test failed" + // (and success) are already conveyed by the per-test annotations above, so we would otherwise add a + // redundant, location-less error on top of the real ones. + if (GitHubActionsExitCode.IsTestResultOutcome(exitCode)) + { + return; + } + + string line = GetExitCodeAnnotation(exitCode); + + if (_logger.IsEnabled(LogLevel.Trace)) + { + _logger.LogTrace($"Showing exit-code annotation '{line}'."); + } + + await DisplayAnnotationLineAsync(line, testSessionContext.CancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + _logger.LogUnexpectedException(nameof(OnTestSessionFinishingAsync), ex); + } + } + + internal static /* for testing */ string GetExitCodeAnnotation(int exitCode) + { + string title = string.Format( + CultureInfo.InvariantCulture, + GitHubActionsResources.ExitCodeAnnotationTitle, + exitCode.ToString(CultureInfo.InvariantCulture), + GitHubActionsExitCode.GetName(exitCode)); + string message = GitHubActionsExitCode.GetReason(exitCode); + + return string.Format( + CultureInfo.InvariantCulture, + "::error title={0}::{1}", + GitHubActionsEscaper.EscapeProperty(title), + GitHubActionsEscaper.EscapeData(message)); + } + private Task WriteAnnotationAsync(string testName, string? explanation, Exception? exception, CancellationToken cancellationToken) { if (_logger.IsEnabled(LogLevel.Trace)) diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsExitCode.cs b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsExitCode.cs new file mode 100644 index 0000000000..75d8222568 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsExitCode.cs @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Extensions.GitHubActionsReport.Resources; +using Microsoft.Testing.Platform.Helpers; + +namespace Microsoft.Testing.Extensions.GitHubActionsReport; + +/// +/// Helpers for turning a Microsoft.Testing.Platform process exit code into a GitHub-friendly verdict. +/// Used by the step-summary and annotation reporters so that non-test-result failures — e.g. a +/// --minimum-expected-tests violation, a run that discovered zero tests, an aborted session or a +/// --maximum-failed-tests stop — are surfaced instead of silently looking like a passing run. +/// See . +/// +internal static class GitHubActionsExitCode +{ + /// + /// Returns when the exit code is a normal test-result outcome (everything passed, + /// or at least one test failed). Those outcomes are already reflected by the passed/failed totals and the + /// per-test failure annotations, so callers do not surface a separate exit-code callout for them. + /// + public static bool IsTestResultOutcome(int exitCode) + => exitCode is (int)ExitCode.Success or (int)ExitCode.AtLeastOneTestFailed; + + /// + /// Returns when the run did not succeed (any non-zero exit code). + /// + public static bool IndicatesFailure(int exitCode) + => exitCode != (int)ExitCode.Success; + + /// + /// Returns the enum name for a known exit code (e.g. MinimumExpectedTestsPolicyViolation) or + /// Unknown for a value outside the documented set. + /// + public static string GetName(int exitCode) + => Enum.IsDefined(typeof(ExitCode), exitCode) + ? ((ExitCode)exitCode).ToString() + : "Unknown"; + + /// + /// Returns a short, human-readable, localized explanation of the exit code. + /// + public static string GetReason(int exitCode) + => exitCode switch + { + (int)ExitCode.Success => GitHubActionsResources.ExitCodeReasonSuccess, + (int)ExitCode.GenericFailure => GitHubActionsResources.ExitCodeReasonGenericFailure, + (int)ExitCode.AtLeastOneTestFailed => GitHubActionsResources.ExitCodeReasonAtLeastOneTestFailed, + (int)ExitCode.TestSessionAborted => GitHubActionsResources.ExitCodeReasonTestSessionAborted, + (int)ExitCode.InvalidPlatformSetup => GitHubActionsResources.ExitCodeReasonInvalidPlatformSetup, + (int)ExitCode.InvalidCommandLine => GitHubActionsResources.ExitCodeReasonInvalidCommandLine, + (int)ExitCode.TestHostProcessExitedNonGracefully => GitHubActionsResources.ExitCodeReasonTestHostProcessExitedNonGracefully, + (int)ExitCode.ZeroTests => GitHubActionsResources.ExitCodeReasonZeroTests, + (int)ExitCode.MinimumExpectedTestsPolicyViolation => GitHubActionsResources.ExitCodeReasonMinimumExpectedTestsPolicyViolation, + (int)ExitCode.TestAdapterTestSessionFailure => GitHubActionsResources.ExitCodeReasonTestAdapterTestSessionFailure, + (int)ExitCode.DependentProcessExited => GitHubActionsResources.ExitCodeReasonDependentProcessExited, + (int)ExitCode.IncompatibleProtocolVersion => GitHubActionsResources.ExitCodeReasonIncompatibleProtocolVersion, + (int)ExitCode.TestExecutionStoppedForMaxFailedTests => GitHubActionsResources.ExitCodeReasonTestExecutionStoppedForMaxFailedTests, + _ => GitHubActionsResources.ExitCodeReasonUnknown, + }; +} diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsExtensions.cs b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsExtensions.cs index f339915f01..b209f61c15 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsExtensions.cs +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsExtensions.cs @@ -27,6 +27,7 @@ public static void AddGitHubActionsProvider(this ITestApplicationBuilder builder serviceProvider.GetFileSystem(), serviceProvider.GetOutputDevice(), serviceProvider.GetTestApplicationModuleInfo(), + serviceProvider.GetTestApplicationProcessExitCode(), serviceProvider.GetLoggerFactory())); var compositeSlowTestReporter = new CompositeExtensionFactory(serviceProvider => @@ -46,14 +47,18 @@ public static void AddGitHubActionsProvider(this ITestApplicationBuilder builder serviceProvider.GetTestApplicationModuleInfo(), serviceProvider.GetLoggerFactory())); - builder.TestHost.AddDataConsumer(serviceProvider => + var compositeAnnotationReporter = new CompositeExtensionFactory(serviceProvider => new GitHubActionsAnnotationReporter( serviceProvider.GetCommandLineOptions(), serviceProvider.GetEnvironment(), serviceProvider.GetFileSystem(), serviceProvider.GetOutputDevice(), + serviceProvider.GetTestApplicationProcessExitCode(), serviceProvider.GetLoggerFactory())); + builder.TestHost.AddDataConsumer(compositeAnnotationReporter); + builder.TestHost.AddTestSessionLifetimeHandler(compositeAnnotationReporter); + builder.TestHost.AddDataConsumer(compositeSummaryReporter); builder.TestHost.AddTestSessionLifetimeHandler(compositeSummaryReporter); builder.TestHost.AddDataConsumer(compositeSlowTestReporter); diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsSummaryReporter.cs b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsSummaryReporter.cs index e7a33b7b38..05a346f87f 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsSummaryReporter.cs +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsSummaryReporter.cs @@ -34,6 +34,7 @@ internal sealed class GitHubActionsSummaryReporter : private readonly IFileSystem _fileSystem; private readonly IOutputDevice _outputDevice; private readonly ITestApplicationModuleInfo _testApplicationModuleInfo; + private readonly ITestApplicationProcessExitCode _testApplicationProcessExitCode; private readonly ILogger _logger; private readonly Lazy _targetFrameworkMoniker; private readonly bool _isEnabled; @@ -53,12 +54,14 @@ public GitHubActionsSummaryReporter( IFileSystem fileSystem, IOutputDevice outputDevice, ITestApplicationModuleInfo testApplicationModuleInfo, + ITestApplicationProcessExitCode testApplicationProcessExitCode, ILoggerFactory loggerFactory) { _environment = environment; _fileSystem = fileSystem; _outputDevice = outputDevice; _testApplicationModuleInfo = testApplicationModuleInfo; + _testApplicationProcessExitCode = testApplicationProcessExitCode; _logger = loggerFactory.CreateLogger(); _targetFrameworkMoniker = new(TargetFrameworkMonikerHelper.GetTargetFrameworkMonikerIncludingPlatform); _isEnabled = GitHubActionsFeature.IsEnabled(commandLineOptions, environment, GitHubActionsCommandLineOptions.GitHubActionsStepSummary); @@ -171,7 +174,7 @@ public async Task OnTestSessionFinishingAsync(ITestSessionContext testSessionCon snapshot = [.. _records.Values]; } - string markdown = BuildMarkdown(snapshot, _testApplicationModuleInfo.TryGetAssemblyName() ?? "unknown assembly name", _targetFrameworkMoniker.Value); + string markdown = BuildMarkdown(snapshot, _testApplicationModuleInfo.TryGetAssemblyName() ?? "unknown assembly name", _targetFrameworkMoniker.Value, _testApplicationProcessExitCode.GetProcessExitCode()); try { @@ -200,7 +203,7 @@ public async Task OnTestSessionFinishingAsync(ITestSessionContext testSessionCon } } - internal static /* for testing */ string BuildMarkdown(IReadOnlyList records, string assemblyName, string targetFrameworkMoniker) + internal static /* for testing */ string BuildMarkdown(IReadOnlyList records, string assemblyName, string targetFrameworkMoniker, int exitCode) { int total = records.Count; int passed = 0; @@ -231,7 +234,10 @@ public async Task OnTestSessionFinishingAsync(ITestSessionContext testSessionCon } } - string statusIcon = failed > 0 ? "❌" : "✅"; + // Reflect the process verdict, not just the failed-test count: a run can end in failure with zero failed + // tests (e.g. zero tests discovered or a --minimum-expected-tests violation), which must not show ✅. + bool runFailed = failed > 0 || GitHubActionsExitCode.IndicatesFailure(exitCode); + string statusIcon = runFailed ? "❌" : "✅"; var builder = new StringBuilder(); builder.Append("## ").Append(statusIcon).Append(" Test Run Summary — ").Append(assemblyName).Append(" (").Append(targetFrameworkMoniker).Append(")\n\n"); @@ -243,6 +249,20 @@ public async Task OnTestSessionFinishingAsync(ITestSessionContext testSessionCon .Append(" | ").Append(skipped.ToString(CultureInfo.InvariantCulture)) .Append(" | ").Append(FormatDuration(totalDuration)).Append(" |\n\n"); + // Surface a non-test-result failure (aborted, zero tests, --minimum-expected-tests, --maximum-failed-tests, + // adapter/host failures, …) as a GitHub alert callout. Plain pass / at-least-one-failed outcomes are already + // conveyed by the totals table and the failures section, so no callout is added for them. + if (!GitHubActionsExitCode.IsTestResultOutcome(exitCode)) + { + string calloutText = string.Format( + CultureInfo.InvariantCulture, + GitHubActionsResources.ExitCodeCallout, + exitCode.ToString(CultureInfo.InvariantCulture), + GitHubActionsExitCode.GetName(exitCode), + GitHubActionsExitCode.GetReason(exitCode)); + builder.Append("> [!WARNING]\n> ").Append(EscapeInlineCode(calloutText)).Append("\n\n"); + } + if (failures.Count > 0) { builder.Append("### ❌ Failures (").Append(failed.ToString(CultureInfo.InvariantCulture)).Append(")\n\n"); diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/InternalAPI/InternalAPI.Unshipped.txt b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/InternalAPI/InternalAPI.Unshipped.txt index 5aca711f5b..74d64b7eb5 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/InternalAPI/InternalAPI.Unshipped.txt +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/InternalAPI/InternalAPI.Unshipped.txt @@ -1,7 +1,21 @@ #nullable enable abstract Microsoft.Testing.Extensions.SlowTestReporterBase.EmitSlowTestAsync(string! testName, string! displayLabel, System.TimeSpan elapsed, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task! abstract Microsoft.Testing.Extensions.SlowTestReporterBase.GetDisplayLabel(Microsoft.Testing.Platform.Extensions.Messages.TestNode! testNode) -> string! +Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsAnnotationReporter.GitHubActionsAnnotationReporter(Microsoft.Testing.Platform.CommandLine.ICommandLineOptions! commandLine, Microsoft.Testing.Platform.Helpers.IEnvironment! environment, Microsoft.Testing.Platform.Helpers.IFileSystem! fileSystem, Microsoft.Testing.Platform.OutputDevice.IOutputDevice! outputDisplay, Microsoft.Testing.Platform.Services.ITestApplicationProcessExitCode! testApplicationProcessExitCode, Microsoft.Testing.Platform.Logging.ILoggerFactory! loggerFactory) -> void +Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsAnnotationReporter.OnTestSessionFinishingAsync(Microsoft.Testing.Platform.Services.ITestSessionContext! testSessionContext) -> System.Threading.Tasks.Task! +Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsAnnotationReporter.OnTestSessionStartingAsync(Microsoft.Testing.Platform.Services.ITestSessionContext! sessionUid) -> System.Threading.Tasks.Task! +Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsExitCode +Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsSummaryReporter.GitHubActionsSummaryReporter(Microsoft.Testing.Platform.CommandLine.ICommandLineOptions! commandLineOptions, Microsoft.Testing.Platform.Helpers.IEnvironment! environment, Microsoft.Testing.Platform.Helpers.IFileSystem! fileSystem, Microsoft.Testing.Platform.OutputDevice.IOutputDevice! outputDevice, Microsoft.Testing.Platform.Services.ITestApplicationModuleInfo! testApplicationModuleInfo, Microsoft.Testing.Platform.Services.ITestApplicationProcessExitCode! testApplicationProcessExitCode, Microsoft.Testing.Platform.Logging.ILoggerFactory! loggerFactory) -> void +static Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsAnnotationReporter.GetExitCodeAnnotation(int exitCode) -> string! +static Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsExitCode.GetName(int exitCode) -> string! +static Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsExitCode.GetReason(int exitCode) -> string! +static Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsExitCode.IndicatesFailure(int exitCode) -> bool +static Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsExitCode.IsTestResultOutcome(int exitCode) -> bool static Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsSlowTestReporter.BuildNoticeLine(string! testLabel, System.TimeSpan elapsed) -> string! +static Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsSummaryReporter.BuildMarkdown(System.Collections.Generic.IReadOnlyList! records, string! assemblyName, string! targetFrameworkMoniker, int exitCode) -> string! static Microsoft.Testing.Extensions.TestNodeIdentity.GetDisplayLabel(Microsoft.Testing.Platform.Extensions.Messages.TestNode! testNode) -> string! *REMOVED*abstract Microsoft.Testing.Extensions.SlowTestReporterBase.EmitSlowTestAsync(string! testName, System.TimeSpan elapsed, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task! +*REMOVED*Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsAnnotationReporter.GitHubActionsAnnotationReporter(Microsoft.Testing.Platform.CommandLine.ICommandLineOptions! commandLine, Microsoft.Testing.Platform.Helpers.IEnvironment! environment, Microsoft.Testing.Platform.Helpers.IFileSystem! fileSystem, Microsoft.Testing.Platform.OutputDevice.IOutputDevice! outputDisplay, Microsoft.Testing.Platform.Logging.ILoggerFactory! loggerFactory) -> void +*REMOVED*Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsSummaryReporter.GitHubActionsSummaryReporter(Microsoft.Testing.Platform.CommandLine.ICommandLineOptions! commandLineOptions, Microsoft.Testing.Platform.Helpers.IEnvironment! environment, Microsoft.Testing.Platform.Helpers.IFileSystem! fileSystem, Microsoft.Testing.Platform.OutputDevice.IOutputDevice! outputDevice, Microsoft.Testing.Platform.Services.ITestApplicationModuleInfo! testApplicationModuleInfo, Microsoft.Testing.Platform.Logging.ILoggerFactory! loggerFactory) -> void *REMOVED*static Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsSlowTestReporter.BuildNoticeLine(string! testName, System.TimeSpan elapsed) -> string! +*REMOVED*static Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsSummaryReporter.BuildMarkdown(System.Collections.Generic.IReadOnlyList! records, string! assemblyName, string! targetFrameworkMoniker) -> string! diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/PACKAGE.md b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/PACKAGE.md index 850cb292f9..18280d0be9 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/PACKAGE.md +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/PACKAGE.md @@ -15,8 +15,8 @@ dotnet add package Microsoft.Testing.Extensions.GitHubActionsReport This package extends Microsoft.Testing.Platform with: - **Per-assembly log groups**: emits `::group::` / `::endgroup::` workflow commands so each test assembly's output is collapsed by default in the runner UI -- **Failure annotations**: emits an `::error` workflow command for each failing test so failures appear in the workflow Annotations tab and, when the source location can be resolved, on the pull request's "Files changed" diff gutter. Skipped tests are surfaced as title-only `::warning` annotations so they are visible in the Annotations tab too -- **Job summary**: appends a markdown roll-up (totals, failures, slowest tests) to the file pointed to by `GITHUB_STEP_SUMMARY`, which GitHub renders on the workflow run summary page +- **Failure annotations**: emits an `::error` workflow command for each failing test so failures appear in the workflow Annotations tab and, when the source location can be resolved, on the pull request's "Files changed" diff gutter. Skipped tests are surfaced as title-only `::warning` annotations so they are visible in the Annotations tab too. When the run fails for a reason that is not a plain test failure — e.g. a `--minimum-expected-tests` violation, a run that discovered zero tests, an aborted session or a `--maximum-failed-tests` stop — a single run-level `::error` is emitted describing the [Microsoft.Testing.Platform exit code](https://learn.microsoft.com/dotnet/core/testing/microsoft-testing-platform-troubleshooting#exit-codes) +- **Job summary**: appends a markdown roll-up (totals, failures, slowest tests) to the file pointed to by `GITHUB_STEP_SUMMARY`, which GitHub renders on the workflow run summary page. When running `dotnet test` across multiple assemblies each assembly appends its own section (labelled with the assembly name and target framework), and any non-success exit code is called out so a failure is not hidden behind a green ✅ - **Slow-test notices**: emits a `::notice` workflow command for any test still running past a threshold (default 60 seconds) The extension activates when the test run is on GitHub Actions (`GITHUB_ACTIONS=true`) and the `--report-gh` switch is passed; it no-ops otherwise. When active, each feature is enabled by default and can be toggled individually: diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/GitHubActionsResources.resx b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/GitHubActionsResources.resx index f7a0470da3..095dbb6973 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/GitHubActionsResources.resx +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/GitHubActionsResources.resx @@ -125,4 +125,56 @@ Enable GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + + Exit code {0} — {1}: {2} + {0} is the numeric exit code, {1} is the exit code name (not localized), {2} is the reason. Rendered as a callout in the job summary. + + + Test run failed (exit code {0}: {1}) + {0} is the numeric exit code, {1} is the exit code name (not localized). + + + The test run succeeded. + + + The test run failed. + + + At least one test failed. + + + The test session was aborted. + + + The test platform setup is invalid. + + + The command line arguments are invalid. + + + The test host process exited non-gracefully. + + + No tests were found to run. This often means a test filter matched nothing. + + + Fewer tests ran than required by --minimum-expected-tests. + {Locked="--minimum-expected-tests"} + + + The test adapter reported a test session failure. + + + A dependent process exited unexpectedly. + + + An incompatible protocol version was negotiated. + + + Test execution stopped after reaching the limit set by --maximum-failed-tests. + {Locked="--maximum-failed-tests"} + + + The test run reported a non-success exit code. + diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.cs.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.cs.xlf index 51843719e4..311f376490 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.cs.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.cs.xlf @@ -22,6 +22,86 @@ Generátor sestav GitHub Actions + + Test run failed (exit code {0}: {1}) + Test run failed (exit code {0}: {1}) + {0} is the numeric exit code, {1} is the exit code name (not localized). + + + Exit code {0} — {1}: {2} + Exit code {0} — {1}: {2} + {0} is the numeric exit code, {1} is the exit code name (not localized), {2} is the reason. Rendered as a callout in the job summary. + + + At least one test failed. + At least one test failed. + + + + A dependent process exited unexpectedly. + A dependent process exited unexpectedly. + + + + The test run failed. + The test run failed. + + + + An incompatible protocol version was negotiated. + An incompatible protocol version was negotiated. + + + + The command line arguments are invalid. + The command line arguments are invalid. + + + + The test platform setup is invalid. + The test platform setup is invalid. + + + + Fewer tests ran than required by --minimum-expected-tests. + Fewer tests ran than required by --minimum-expected-tests. + {Locked="--minimum-expected-tests"} + + + The test run succeeded. + The test run succeeded. + + + + The test adapter reported a test session failure. + The test adapter reported a test session failure. + + + + Test execution stopped after reaching the limit set by --maximum-failed-tests. + Test execution stopped after reaching the limit set by --maximum-failed-tests. + {Locked="--maximum-failed-tests"} + + + The test host process exited non-gracefully. + The test host process exited non-gracefully. + + + + The test session was aborted. + The test session was aborted. + + + + The test run reported a non-success exit code. + The test run reported a non-success exit code. + + + + No tests were found to run. This often means a test filter matched nothing. + No tests were found to run. This often means a test filter matched nothing. + + Tests: {0} ({1}) Testy: {0} ({1}) diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.de.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.de.xlf index 9504b3223a..f713bbd55d 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.de.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.de.xlf @@ -22,6 +22,86 @@ GitHub Actions Bericht-Generator + + Test run failed (exit code {0}: {1}) + Test run failed (exit code {0}: {1}) + {0} is the numeric exit code, {1} is the exit code name (not localized). + + + Exit code {0} — {1}: {2} + Exit code {0} — {1}: {2} + {0} is the numeric exit code, {1} is the exit code name (not localized), {2} is the reason. Rendered as a callout in the job summary. + + + At least one test failed. + At least one test failed. + + + + A dependent process exited unexpectedly. + A dependent process exited unexpectedly. + + + + The test run failed. + The test run failed. + + + + An incompatible protocol version was negotiated. + An incompatible protocol version was negotiated. + + + + The command line arguments are invalid. + The command line arguments are invalid. + + + + The test platform setup is invalid. + The test platform setup is invalid. + + + + Fewer tests ran than required by --minimum-expected-tests. + Fewer tests ran than required by --minimum-expected-tests. + {Locked="--minimum-expected-tests"} + + + The test run succeeded. + The test run succeeded. + + + + The test adapter reported a test session failure. + The test adapter reported a test session failure. + + + + Test execution stopped after reaching the limit set by --maximum-failed-tests. + Test execution stopped after reaching the limit set by --maximum-failed-tests. + {Locked="--maximum-failed-tests"} + + + The test host process exited non-gracefully. + The test host process exited non-gracefully. + + + + The test session was aborted. + The test session was aborted. + + + + The test run reported a non-success exit code. + The test run reported a non-success exit code. + + + + No tests were found to run. This often means a test filter matched nothing. + No tests were found to run. This often means a test filter matched nothing. + + Tests: {0} ({1}) Tests: {0} ({1}) diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.es.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.es.xlf index f8ba919de8..c6f6a37224 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.es.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.es.xlf @@ -22,6 +22,86 @@ Generador de informes de Acciones de GitHub + + Test run failed (exit code {0}: {1}) + Test run failed (exit code {0}: {1}) + {0} is the numeric exit code, {1} is the exit code name (not localized). + + + Exit code {0} — {1}: {2} + Exit code {0} — {1}: {2} + {0} is the numeric exit code, {1} is the exit code name (not localized), {2} is the reason. Rendered as a callout in the job summary. + + + At least one test failed. + At least one test failed. + + + + A dependent process exited unexpectedly. + A dependent process exited unexpectedly. + + + + The test run failed. + The test run failed. + + + + An incompatible protocol version was negotiated. + An incompatible protocol version was negotiated. + + + + The command line arguments are invalid. + The command line arguments are invalid. + + + + The test platform setup is invalid. + The test platform setup is invalid. + + + + Fewer tests ran than required by --minimum-expected-tests. + Fewer tests ran than required by --minimum-expected-tests. + {Locked="--minimum-expected-tests"} + + + The test run succeeded. + The test run succeeded. + + + + The test adapter reported a test session failure. + The test adapter reported a test session failure. + + + + Test execution stopped after reaching the limit set by --maximum-failed-tests. + Test execution stopped after reaching the limit set by --maximum-failed-tests. + {Locked="--maximum-failed-tests"} + + + The test host process exited non-gracefully. + The test host process exited non-gracefully. + + + + The test session was aborted. + The test session was aborted. + + + + The test run reported a non-success exit code. + The test run reported a non-success exit code. + + + + No tests were found to run. This often means a test filter matched nothing. + No tests were found to run. This often means a test filter matched nothing. + + Tests: {0} ({1}) Pruebas: {0} ({1}) diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.fr.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.fr.xlf index 73720bf55d..093c625a8e 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.fr.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.fr.xlf @@ -22,6 +22,86 @@ Générateur de rapports GitHub Actions + + Test run failed (exit code {0}: {1}) + Test run failed (exit code {0}: {1}) + {0} is the numeric exit code, {1} is the exit code name (not localized). + + + Exit code {0} — {1}: {2} + Exit code {0} — {1}: {2} + {0} is the numeric exit code, {1} is the exit code name (not localized), {2} is the reason. Rendered as a callout in the job summary. + + + At least one test failed. + At least one test failed. + + + + A dependent process exited unexpectedly. + A dependent process exited unexpectedly. + + + + The test run failed. + The test run failed. + + + + An incompatible protocol version was negotiated. + An incompatible protocol version was negotiated. + + + + The command line arguments are invalid. + The command line arguments are invalid. + + + + The test platform setup is invalid. + The test platform setup is invalid. + + + + Fewer tests ran than required by --minimum-expected-tests. + Fewer tests ran than required by --minimum-expected-tests. + {Locked="--minimum-expected-tests"} + + + The test run succeeded. + The test run succeeded. + + + + The test adapter reported a test session failure. + The test adapter reported a test session failure. + + + + Test execution stopped after reaching the limit set by --maximum-failed-tests. + Test execution stopped after reaching the limit set by --maximum-failed-tests. + {Locked="--maximum-failed-tests"} + + + The test host process exited non-gracefully. + The test host process exited non-gracefully. + + + + The test session was aborted. + The test session was aborted. + + + + The test run reported a non-success exit code. + The test run reported a non-success exit code. + + + + No tests were found to run. This often means a test filter matched nothing. + No tests were found to run. This often means a test filter matched nothing. + + Tests: {0} ({1}) Tests : {0} ({1}) diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.it.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.it.xlf index 279b4d86a8..0c5579e119 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.it.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.it.xlf @@ -22,6 +22,86 @@ GitHub Actions report generator + + Test run failed (exit code {0}: {1}) + Test run failed (exit code {0}: {1}) + {0} is the numeric exit code, {1} is the exit code name (not localized). + + + Exit code {0} — {1}: {2} + Exit code {0} — {1}: {2} + {0} is the numeric exit code, {1} is the exit code name (not localized), {2} is the reason. Rendered as a callout in the job summary. + + + At least one test failed. + At least one test failed. + + + + A dependent process exited unexpectedly. + A dependent process exited unexpectedly. + + + + The test run failed. + The test run failed. + + + + An incompatible protocol version was negotiated. + An incompatible protocol version was negotiated. + + + + The command line arguments are invalid. + The command line arguments are invalid. + + + + The test platform setup is invalid. + The test platform setup is invalid. + + + + Fewer tests ran than required by --minimum-expected-tests. + Fewer tests ran than required by --minimum-expected-tests. + {Locked="--minimum-expected-tests"} + + + The test run succeeded. + The test run succeeded. + + + + The test adapter reported a test session failure. + The test adapter reported a test session failure. + + + + Test execution stopped after reaching the limit set by --maximum-failed-tests. + Test execution stopped after reaching the limit set by --maximum-failed-tests. + {Locked="--maximum-failed-tests"} + + + The test host process exited non-gracefully. + The test host process exited non-gracefully. + + + + The test session was aborted. + The test session was aborted. + + + + The test run reported a non-success exit code. + The test run reported a non-success exit code. + + + + No tests were found to run. This often means a test filter matched nothing. + No tests were found to run. This often means a test filter matched nothing. + + Tests: {0} ({1}) Test: {0} ({1}) diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.ja.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.ja.xlf index 98c10d0e71..88d75df89f 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.ja.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.ja.xlf @@ -22,6 +22,86 @@ GitHub Actions レポート生成プログラム + + Test run failed (exit code {0}: {1}) + Test run failed (exit code {0}: {1}) + {0} is the numeric exit code, {1} is the exit code name (not localized). + + + Exit code {0} — {1}: {2} + Exit code {0} — {1}: {2} + {0} is the numeric exit code, {1} is the exit code name (not localized), {2} is the reason. Rendered as a callout in the job summary. + + + At least one test failed. + At least one test failed. + + + + A dependent process exited unexpectedly. + A dependent process exited unexpectedly. + + + + The test run failed. + The test run failed. + + + + An incompatible protocol version was negotiated. + An incompatible protocol version was negotiated. + + + + The command line arguments are invalid. + The command line arguments are invalid. + + + + The test platform setup is invalid. + The test platform setup is invalid. + + + + Fewer tests ran than required by --minimum-expected-tests. + Fewer tests ran than required by --minimum-expected-tests. + {Locked="--minimum-expected-tests"} + + + The test run succeeded. + The test run succeeded. + + + + The test adapter reported a test session failure. + The test adapter reported a test session failure. + + + + Test execution stopped after reaching the limit set by --maximum-failed-tests. + Test execution stopped after reaching the limit set by --maximum-failed-tests. + {Locked="--maximum-failed-tests"} + + + The test host process exited non-gracefully. + The test host process exited non-gracefully. + + + + The test session was aborted. + The test session was aborted. + + + + The test run reported a non-success exit code. + The test run reported a non-success exit code. + + + + No tests were found to run. This often means a test filter matched nothing. + No tests were found to run. This often means a test filter matched nothing. + + Tests: {0} ({1}) テスト: {0} ({1}) diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.ko.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.ko.xlf index a734f3256e..47fadfb727 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.ko.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.ko.xlf @@ -22,6 +22,86 @@ GitHub Actions 보고서 생성기 + + Test run failed (exit code {0}: {1}) + Test run failed (exit code {0}: {1}) + {0} is the numeric exit code, {1} is the exit code name (not localized). + + + Exit code {0} — {1}: {2} + Exit code {0} — {1}: {2} + {0} is the numeric exit code, {1} is the exit code name (not localized), {2} is the reason. Rendered as a callout in the job summary. + + + At least one test failed. + At least one test failed. + + + + A dependent process exited unexpectedly. + A dependent process exited unexpectedly. + + + + The test run failed. + The test run failed. + + + + An incompatible protocol version was negotiated. + An incompatible protocol version was negotiated. + + + + The command line arguments are invalid. + The command line arguments are invalid. + + + + The test platform setup is invalid. + The test platform setup is invalid. + + + + Fewer tests ran than required by --minimum-expected-tests. + Fewer tests ran than required by --minimum-expected-tests. + {Locked="--minimum-expected-tests"} + + + The test run succeeded. + The test run succeeded. + + + + The test adapter reported a test session failure. + The test adapter reported a test session failure. + + + + Test execution stopped after reaching the limit set by --maximum-failed-tests. + Test execution stopped after reaching the limit set by --maximum-failed-tests. + {Locked="--maximum-failed-tests"} + + + The test host process exited non-gracefully. + The test host process exited non-gracefully. + + + + The test session was aborted. + The test session was aborted. + + + + The test run reported a non-success exit code. + The test run reported a non-success exit code. + + + + No tests were found to run. This often means a test filter matched nothing. + No tests were found to run. This often means a test filter matched nothing. + + Tests: {0} ({1}) 테스트: {0}({1}) diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.pl.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.pl.xlf index b5c1db3360..bc3506b89b 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.pl.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.pl.xlf @@ -22,6 +22,86 @@ Generator raportów usługi GitHub Actions + + Test run failed (exit code {0}: {1}) + Test run failed (exit code {0}: {1}) + {0} is the numeric exit code, {1} is the exit code name (not localized). + + + Exit code {0} — {1}: {2} + Exit code {0} — {1}: {2} + {0} is the numeric exit code, {1} is the exit code name (not localized), {2} is the reason. Rendered as a callout in the job summary. + + + At least one test failed. + At least one test failed. + + + + A dependent process exited unexpectedly. + A dependent process exited unexpectedly. + + + + The test run failed. + The test run failed. + + + + An incompatible protocol version was negotiated. + An incompatible protocol version was negotiated. + + + + The command line arguments are invalid. + The command line arguments are invalid. + + + + The test platform setup is invalid. + The test platform setup is invalid. + + + + Fewer tests ran than required by --minimum-expected-tests. + Fewer tests ran than required by --minimum-expected-tests. + {Locked="--minimum-expected-tests"} + + + The test run succeeded. + The test run succeeded. + + + + The test adapter reported a test session failure. + The test adapter reported a test session failure. + + + + Test execution stopped after reaching the limit set by --maximum-failed-tests. + Test execution stopped after reaching the limit set by --maximum-failed-tests. + {Locked="--maximum-failed-tests"} + + + The test host process exited non-gracefully. + The test host process exited non-gracefully. + + + + The test session was aborted. + The test session was aborted. + + + + The test run reported a non-success exit code. + The test run reported a non-success exit code. + + + + No tests were found to run. This often means a test filter matched nothing. + No tests were found to run. This often means a test filter matched nothing. + + Tests: {0} ({1}) Testy: {0} ({1}) diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.pt-BR.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.pt-BR.xlf index 8363530c81..48d54aa631 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.pt-BR.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.pt-BR.xlf @@ -22,6 +22,86 @@ Gerador de relatório do GitHub Actions + + Test run failed (exit code {0}: {1}) + Test run failed (exit code {0}: {1}) + {0} is the numeric exit code, {1} is the exit code name (not localized). + + + Exit code {0} — {1}: {2} + Exit code {0} — {1}: {2} + {0} is the numeric exit code, {1} is the exit code name (not localized), {2} is the reason. Rendered as a callout in the job summary. + + + At least one test failed. + At least one test failed. + + + + A dependent process exited unexpectedly. + A dependent process exited unexpectedly. + + + + The test run failed. + The test run failed. + + + + An incompatible protocol version was negotiated. + An incompatible protocol version was negotiated. + + + + The command line arguments are invalid. + The command line arguments are invalid. + + + + The test platform setup is invalid. + The test platform setup is invalid. + + + + Fewer tests ran than required by --minimum-expected-tests. + Fewer tests ran than required by --minimum-expected-tests. + {Locked="--minimum-expected-tests"} + + + The test run succeeded. + The test run succeeded. + + + + The test adapter reported a test session failure. + The test adapter reported a test session failure. + + + + Test execution stopped after reaching the limit set by --maximum-failed-tests. + Test execution stopped after reaching the limit set by --maximum-failed-tests. + {Locked="--maximum-failed-tests"} + + + The test host process exited non-gracefully. + The test host process exited non-gracefully. + + + + The test session was aborted. + The test session was aborted. + + + + The test run reported a non-success exit code. + The test run reported a non-success exit code. + + + + No tests were found to run. This often means a test filter matched nothing. + No tests were found to run. This often means a test filter matched nothing. + + Tests: {0} ({1}) Testes: {0} ({1}) diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.ru.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.ru.xlf index 7cce83381a..b358015478 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.ru.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.ru.xlf @@ -22,6 +22,86 @@ Генератор отчетов GitHub Actions + + Test run failed (exit code {0}: {1}) + Test run failed (exit code {0}: {1}) + {0} is the numeric exit code, {1} is the exit code name (not localized). + + + Exit code {0} — {1}: {2} + Exit code {0} — {1}: {2} + {0} is the numeric exit code, {1} is the exit code name (not localized), {2} is the reason. Rendered as a callout in the job summary. + + + At least one test failed. + At least one test failed. + + + + A dependent process exited unexpectedly. + A dependent process exited unexpectedly. + + + + The test run failed. + The test run failed. + + + + An incompatible protocol version was negotiated. + An incompatible protocol version was negotiated. + + + + The command line arguments are invalid. + The command line arguments are invalid. + + + + The test platform setup is invalid. + The test platform setup is invalid. + + + + Fewer tests ran than required by --minimum-expected-tests. + Fewer tests ran than required by --minimum-expected-tests. + {Locked="--minimum-expected-tests"} + + + The test run succeeded. + The test run succeeded. + + + + The test adapter reported a test session failure. + The test adapter reported a test session failure. + + + + Test execution stopped after reaching the limit set by --maximum-failed-tests. + Test execution stopped after reaching the limit set by --maximum-failed-tests. + {Locked="--maximum-failed-tests"} + + + The test host process exited non-gracefully. + The test host process exited non-gracefully. + + + + The test session was aborted. + The test session was aborted. + + + + The test run reported a non-success exit code. + The test run reported a non-success exit code. + + + + No tests were found to run. This often means a test filter matched nothing. + No tests were found to run. This often means a test filter matched nothing. + + Tests: {0} ({1}) Тесты: {0} ({1}) diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.tr.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.tr.xlf index 3adddf2e8c..a5cc5f0eeb 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.tr.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.tr.xlf @@ -22,6 +22,86 @@ GitHub Actions rapor oluşturucusu + + Test run failed (exit code {0}: {1}) + Test run failed (exit code {0}: {1}) + {0} is the numeric exit code, {1} is the exit code name (not localized). + + + Exit code {0} — {1}: {2} + Exit code {0} — {1}: {2} + {0} is the numeric exit code, {1} is the exit code name (not localized), {2} is the reason. Rendered as a callout in the job summary. + + + At least one test failed. + At least one test failed. + + + + A dependent process exited unexpectedly. + A dependent process exited unexpectedly. + + + + The test run failed. + The test run failed. + + + + An incompatible protocol version was negotiated. + An incompatible protocol version was negotiated. + + + + The command line arguments are invalid. + The command line arguments are invalid. + + + + The test platform setup is invalid. + The test platform setup is invalid. + + + + Fewer tests ran than required by --minimum-expected-tests. + Fewer tests ran than required by --minimum-expected-tests. + {Locked="--minimum-expected-tests"} + + + The test run succeeded. + The test run succeeded. + + + + The test adapter reported a test session failure. + The test adapter reported a test session failure. + + + + Test execution stopped after reaching the limit set by --maximum-failed-tests. + Test execution stopped after reaching the limit set by --maximum-failed-tests. + {Locked="--maximum-failed-tests"} + + + The test host process exited non-gracefully. + The test host process exited non-gracefully. + + + + The test session was aborted. + The test session was aborted. + + + + The test run reported a non-success exit code. + The test run reported a non-success exit code. + + + + No tests were found to run. This often means a test filter matched nothing. + No tests were found to run. This often means a test filter matched nothing. + + Tests: {0} ({1}) Testler: {0} ({1}) diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.zh-Hans.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.zh-Hans.xlf index 67ee05c5d8..2d8a0453fa 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.zh-Hans.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.zh-Hans.xlf @@ -22,6 +22,86 @@ GitHub Actions 报告生成器 + + Test run failed (exit code {0}: {1}) + Test run failed (exit code {0}: {1}) + {0} is the numeric exit code, {1} is the exit code name (not localized). + + + Exit code {0} — {1}: {2} + Exit code {0} — {1}: {2} + {0} is the numeric exit code, {1} is the exit code name (not localized), {2} is the reason. Rendered as a callout in the job summary. + + + At least one test failed. + At least one test failed. + + + + A dependent process exited unexpectedly. + A dependent process exited unexpectedly. + + + + The test run failed. + The test run failed. + + + + An incompatible protocol version was negotiated. + An incompatible protocol version was negotiated. + + + + The command line arguments are invalid. + The command line arguments are invalid. + + + + The test platform setup is invalid. + The test platform setup is invalid. + + + + Fewer tests ran than required by --minimum-expected-tests. + Fewer tests ran than required by --minimum-expected-tests. + {Locked="--minimum-expected-tests"} + + + The test run succeeded. + The test run succeeded. + + + + The test adapter reported a test session failure. + The test adapter reported a test session failure. + + + + Test execution stopped after reaching the limit set by --maximum-failed-tests. + Test execution stopped after reaching the limit set by --maximum-failed-tests. + {Locked="--maximum-failed-tests"} + + + The test host process exited non-gracefully. + The test host process exited non-gracefully. + + + + The test session was aborted. + The test session was aborted. + + + + The test run reported a non-success exit code. + The test run reported a non-success exit code. + + + + No tests were found to run. This often means a test filter matched nothing. + No tests were found to run. This often means a test filter matched nothing. + + Tests: {0} ({1}) 测试: {0} ({1}) diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.zh-Hant.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.zh-Hant.xlf index 716bfd3d00..d9a43ed994 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.zh-Hant.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.zh-Hant.xlf @@ -22,6 +22,86 @@ GitHub Actions 報告產生器 + + Test run failed (exit code {0}: {1}) + Test run failed (exit code {0}: {1}) + {0} is the numeric exit code, {1} is the exit code name (not localized). + + + Exit code {0} — {1}: {2} + Exit code {0} — {1}: {2} + {0} is the numeric exit code, {1} is the exit code name (not localized), {2} is the reason. Rendered as a callout in the job summary. + + + At least one test failed. + At least one test failed. + + + + A dependent process exited unexpectedly. + A dependent process exited unexpectedly. + + + + The test run failed. + The test run failed. + + + + An incompatible protocol version was negotiated. + An incompatible protocol version was negotiated. + + + + The command line arguments are invalid. + The command line arguments are invalid. + + + + The test platform setup is invalid. + The test platform setup is invalid. + + + + Fewer tests ran than required by --minimum-expected-tests. + Fewer tests ran than required by --minimum-expected-tests. + {Locked="--minimum-expected-tests"} + + + The test run succeeded. + The test run succeeded. + + + + The test adapter reported a test session failure. + The test adapter reported a test session failure. + + + + Test execution stopped after reaching the limit set by --maximum-failed-tests. + Test execution stopped after reaching the limit set by --maximum-failed-tests. + {Locked="--maximum-failed-tests"} + + + The test host process exited non-gracefully. + The test host process exited non-gracefully. + + + + The test session was aborted. + The test session was aborted. + + + + The test run reported a non-success exit code. + The test run reported a non-success exit code. + + + + No tests were found to run. This often means a test filter matched nothing. + No tests were found to run. This often means a test filter matched nothing. + + Tests: {0} ({1}) 測試: {0} ({1}) diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsExitCodeTests.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsExitCodeTests.cs new file mode 100644 index 0000000000..14a92182a2 --- /dev/null +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsExitCodeTests.cs @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +extern alias ghactions; + +using ghactions::Microsoft.Testing.Extensions.GitHubActionsReport; + +namespace Microsoft.Testing.Extensions.UnitTests; + +[TestClass] +public sealed class GitHubActionsExitCodeTests +{ + [TestMethod] + [DataRow(0)] // Success + [DataRow(2)] // AtLeastOneTestFailed + public void IsTestResultOutcome_ForTestResultCodes_ReturnsTrue(int exitCode) + => Assert.IsTrue(GitHubActionsExitCode.IsTestResultOutcome(exitCode)); + + [TestMethod] + [DataRow(1)] // GenericFailure + [DataRow(3)] // TestSessionAborted + [DataRow(8)] // ZeroTests + [DataRow(9)] // MinimumExpectedTestsPolicyViolation + [DataRow(13)] // TestExecutionStoppedForMaxFailedTests + [DataRow(255)] // Unknown + public void IsTestResultOutcome_ForNonTestResultCodes_ReturnsFalse(int exitCode) + => Assert.IsFalse(GitHubActionsExitCode.IsTestResultOutcome(exitCode)); + + [TestMethod] + public void IndicatesFailure_OnlySuccessIsNotFailure() + { + Assert.IsFalse(GitHubActionsExitCode.IndicatesFailure(0)); + Assert.IsTrue(GitHubActionsExitCode.IndicatesFailure(2)); + Assert.IsTrue(GitHubActionsExitCode.IndicatesFailure(9)); + } + + [TestMethod] + public void GetName_ReturnsEnumNameForKnownCodeAndUnknownOtherwise() + { + Assert.AreEqual("MinimumExpectedTestsPolicyViolation", GitHubActionsExitCode.GetName(9)); + Assert.AreEqual("ZeroTests", GitHubActionsExitCode.GetName(8)); + Assert.AreEqual("Unknown", GitHubActionsExitCode.GetName(255)); + } + + [TestMethod] + public void GetReason_ForKnownCode_MentionsRelevantOption() + { + Assert.Contains("--minimum-expected-tests", GitHubActionsExitCode.GetReason(9)); + Assert.Contains("--maximum-failed-tests", GitHubActionsExitCode.GetReason(13)); + } + + [TestMethod] + public void GetExitCodeAnnotation_ForZeroTests_EmitsErrorWorkflowCommand() + { + string annotation = GitHubActionsAnnotationReporter.GetExitCodeAnnotation(8); + + Assert.IsTrue(annotation.StartsWith("::error title=", StringComparison.Ordinal), annotation); + Assert.Contains("exit code 8", annotation); + Assert.Contains("ZeroTests", annotation); + } +} diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsSummaryReporterTests.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsSummaryReporterTests.cs index 6c4c53cf72..223a3856ab 100644 --- a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsSummaryReporterTests.cs +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsSummaryReporterTests.cs @@ -13,6 +13,16 @@ namespace Microsoft.Testing.Extensions.UnitTests; [TestClass] public sealed class GitHubActionsSummaryReporterTests { + // ExitCode.Success (0): a normal passing run — no exit-code callout expected. + private const int SuccessExitCode = 0; + + // ExitCode.AtLeastOneTestFailed (2): failures are conveyed by the table/list, not a callout. + private const int AtLeastOneTestFailedExitCode = 2; + + // ExitCode.ZeroTests (8) and MinimumExpectedTestsPolicyViolation (9): non-test-result failures. + private const int ZeroTestsExitCode = 8; + private const int MinimumExpectedTestsExitCode = 9; + [TestMethod] public void BuildMarkdown_AllPassing_UsesSuccessIconAndTotals() { @@ -23,11 +33,12 @@ public void BuildMarkdown_AllPassing_UsesSuccessIconAndTotals() new("Skip", "CalculatorTests.Skip", GitHubActionsTerminalKind.Skipped, TimeSpan.Zero), ]; - string markdown = GitHubActionsSummaryReporter.BuildMarkdown(records, "CalculatorTests", "net9.0"); + string markdown = GitHubActionsSummaryReporter.BuildMarkdown(records, "CalculatorTests", "net9.0", SuccessExitCode); Assert.Contains("## ✅ Test Run Summary — CalculatorTests (net9.0)", markdown); Assert.Contains("| 3 | 2 | 0 | 1 | 30ms |", markdown); Assert.DoesNotContain("### ❌ Failures", markdown); + Assert.DoesNotContain("[!WARNING]", markdown); } [TestMethod] @@ -39,11 +50,14 @@ public void BuildMarkdown_WithFailures_UsesFailureIconAndListsFailures() new("Boom", "StringUtilsTests.Boom", GitHubActionsTerminalKind.Failed, TimeSpan.FromMilliseconds(7)), ]; - string markdown = GitHubActionsSummaryReporter.BuildMarkdown(records, "StringUtilsTests", "net9.0"); + string markdown = GitHubActionsSummaryReporter.BuildMarkdown(records, "StringUtilsTests", "net9.0", AtLeastOneTestFailedExitCode); Assert.Contains("## ❌ Test Run Summary — StringUtilsTests (net9.0)", markdown); Assert.Contains("### ❌ Failures (1)", markdown); Assert.Contains("- `StringUtilsTests.Boom`", markdown); + + // A plain "at least one test failed" outcome is conveyed by the failures list, not an exit-code callout. + Assert.DoesNotContain("[!WARNING]", markdown); } [TestMethod] @@ -55,7 +69,7 @@ public void BuildMarkdown_EmitsSlowestTestsSortedByDuration() new("Slow", "T.Slow", GitHubActionsTerminalKind.Passed, TimeSpan.FromSeconds(65)), ]; - string markdown = GitHubActionsSummaryReporter.BuildMarkdown(records, "T", "net9.0"); + string markdown = GitHubActionsSummaryReporter.BuildMarkdown(records, "T", "net9.0", SuccessExitCode); Assert.Contains("### ⏱ Slowest tests", markdown); int slowIndex = markdown.IndexOf("- `T.Slow` — 1m 05s", StringComparison.Ordinal); @@ -71,9 +85,36 @@ public void BuildMarkdown_EmitsSlowestTestsSortedByDuration() [TestMethod] public void BuildMarkdown_NoTests_StillEmitsHeaderAndZeroTotals() { - string markdown = GitHubActionsSummaryReporter.BuildMarkdown([], "Empty", "net9.0"); + string markdown = GitHubActionsSummaryReporter.BuildMarkdown([], "Empty", "net9.0", SuccessExitCode); Assert.Contains("## ✅ Test Run Summary — Empty (net9.0)", markdown); Assert.Contains("| 0 | 0 | 0 | 0 | 0ms |", markdown); } + + [TestMethod] + public void BuildMarkdown_ZeroTestsExitCode_UsesFailureIconAndEmitsCallout() + { + // No failing tests, but the process exit code says the run failed because nothing ran. + string markdown = GitHubActionsSummaryReporter.BuildMarkdown([], "Empty", "net9.0", ZeroTestsExitCode); + + Assert.Contains("## ❌ Test Run Summary — Empty (net9.0)", markdown); + Assert.Contains("> [!WARNING]", markdown); + Assert.Contains("Exit code 8 — ZeroTests:", markdown); + } + + [TestMethod] + public void BuildMarkdown_MinimumExpectedTestsExitCode_EmitsCalloutEvenWhenTestsPassed() + { + GitHubActionsTestRecord[] records = + [ + new("Add", "CalculatorTests.Add", GitHubActionsTerminalKind.Passed, TimeSpan.FromMilliseconds(10)), + ]; + + string markdown = GitHubActionsSummaryReporter.BuildMarkdown(records, "CalculatorTests", "net9.0", MinimumExpectedTestsExitCode); + + // The single test passed, yet the run failed the minimum-expected-tests policy: icon and callout reflect it. + Assert.Contains("## ❌ Test Run Summary — CalculatorTests (net9.0)", markdown); + Assert.Contains("Exit code 9 — MinimumExpectedTestsPolicyViolation:", markdown); + Assert.Contains("--minimum-expected-tests", markdown); + } } From 94a9bcfce93c7d007ad307e4be53a2105a519bbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Wed, 15 Jul 2026 16:55:26 +0200 Subject: [PATCH 2/7] Address review: scope exit-code docs and add end-to-end acceptance coverage - Qualify PACKAGE.md and code comments: the exit-code callout/annotation covers session-derived outcomes (ZeroTests, MinimumExpectedTests, TestAdapterTestSessionFailure, MaxFailedTests). A hard abort/cancellation short-circuits end-of-session reporting and early/late-host codes are out of reach, so those are no longer over-claimed. - Add GitHubActionsReportTests acceptance tests driving the extension through a real MTP session (GITHUB_ACTIONS=true + temp GITHUB_STEP_SUMMARY) for pass / fail / zero-tests / minimum-expected-tests, asserting emission and suppression of the callout and run-level annotation end to end. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b1e3944b-d2cf-4ab7-ac53-c8ace4ddc260 --- .../GitHubActionsAnnotationReporter.cs | 4 + .../PACKAGE.md | 7 +- .../GitHubActionsReportTests.cs | 200 ++++++++++++++++++ 3 files changed, 209 insertions(+), 2 deletions(-) create mode 100644 test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/GitHubActionsReportTests.cs diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsAnnotationReporter.cs b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsAnnotationReporter.cs index 5f4b2587c1..2e0d5851af 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsAnnotationReporter.cs +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsAnnotationReporter.cs @@ -147,6 +147,10 @@ public async Task OnTestSessionFinishingAsync(ITestSessionContext testSessionCon // Only emit the run-level annotation for a non-test-result failure. Plain "at least one test failed" // (and success) are already conveyed by the per-test annotations above, so we would otherwise add a // redundant, location-less error on top of the real ones. + // NOTE: a hard abort/cancellation (ExitCode.TestSessionAborted) never reaches this handler — the host + // swallows the OperationCanceledException before the end-of-session notification completes — so this + // path only surfaces session-derived outcomes such as ZeroTests, MinimumExpectedTestsPolicyViolation, + // TestAdapterTestSessionFailure and TestExecutionStoppedForMaxFailedTests. if (GitHubActionsExitCode.IsTestResultOutcome(exitCode)) { return; diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/PACKAGE.md b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/PACKAGE.md index 18280d0be9..92efb5311a 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/PACKAGE.md +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/PACKAGE.md @@ -15,8 +15,11 @@ dotnet add package Microsoft.Testing.Extensions.GitHubActionsReport This package extends Microsoft.Testing.Platform with: - **Per-assembly log groups**: emits `::group::` / `::endgroup::` workflow commands so each test assembly's output is collapsed by default in the runner UI -- **Failure annotations**: emits an `::error` workflow command for each failing test so failures appear in the workflow Annotations tab and, when the source location can be resolved, on the pull request's "Files changed" diff gutter. Skipped tests are surfaced as title-only `::warning` annotations so they are visible in the Annotations tab too. When the run fails for a reason that is not a plain test failure — e.g. a `--minimum-expected-tests` violation, a run that discovered zero tests, an aborted session or a `--maximum-failed-tests` stop — a single run-level `::error` is emitted describing the [Microsoft.Testing.Platform exit code](https://learn.microsoft.com/dotnet/core/testing/microsoft-testing-platform-troubleshooting#exit-codes) -- **Job summary**: appends a markdown roll-up (totals, failures, slowest tests) to the file pointed to by `GITHUB_STEP_SUMMARY`, which GitHub renders on the workflow run summary page. When running `dotnet test` across multiple assemblies each assembly appends its own section (labelled with the assembly name and target framework), and any non-success exit code is called out so a failure is not hidden behind a green ✅ +- **Failure annotations**: emits an `::error` workflow command for each failing test so failures appear in the workflow Annotations tab and, when the source location can be resolved, on the pull request's "Files changed" diff gutter. Skipped tests are surfaced as title-only `::warning` annotations so they are visible in the Annotations tab too. When the test session completes with a non-test-result failure — a `--minimum-expected-tests` violation, a run that discovered zero tests, a `--maximum-failed-tests` stop, or a test-adapter session failure — a single run-level `::error` is emitted describing the [Microsoft.Testing.Platform exit code](https://learn.microsoft.com/dotnet/core/testing/microsoft-testing-platform-troubleshooting#exit-codes) +- **Job summary**: appends a markdown roll-up (totals, failures, slowest tests) to the file pointed to by `GITHUB_STEP_SUMMARY`, which GitHub renders on the workflow run summary page. When running `dotnet test` across multiple assemblies each assembly appends its own section (labelled with the assembly name and target framework), and a non-test-result failure exit code is called out so a failure is not hidden behind a green ✅ + +> [!NOTE] +> The exit-code callout and run-level annotation cover the outcomes the extension can observe once the test session has finished: `AtLeastOneTestFailed` (2, conveyed by the per-test failures rather than a callout), `TestSessionAborted` is _not_ covered because a hard abort/cancellation short-circuits end-of-session reporting, `ZeroTests` (8), `MinimumExpectedTestsPolicyViolation` (9), `TestAdapterTestSessionFailure` (10), and `TestExecutionStoppedForMaxFailedTests` (13). Codes raised before or after the in-process session — e.g. `InvalidCommandLine` (5) or `TestHostProcessExitedNonGracefully` (7) — occur outside the extension's reach and are not surfaced here. - **Slow-test notices**: emits a `::notice` workflow command for any test still running past a threshold (default 60 seconds) The extension activates when the test run is on GitHub Actions (`GITHUB_ACTIONS=true`) and the `--report-gh` switch is passed; it no-ops otherwise. When active, each feature is enabled by default and can be toggled individually: diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/GitHubActionsReportTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/GitHubActionsReportTests.cs new file mode 100644 index 0000000000..5e4c6e00e4 --- /dev/null +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/GitHubActionsReportTests.cs @@ -0,0 +1,200 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Testing.Platform.Acceptance.IntegrationTests; + +/// +/// End-to-end coverage for the GitHub Actions report extension: drives the extension through a real +/// Microsoft.Testing.Platform session (with GITHUB_ACTIONS=true and a temporary +/// GITHUB_STEP_SUMMARY file) and asserts that non-test-result exit codes surface both as a +/// job-summary callout and a run-level ::error annotation, while an ordinary passing run stays +/// quiet and an ordinary test failure only produces per-test annotations. +/// +[TestClass] +public sealed class GitHubActionsReportTests : AcceptanceTestBase +{ + private const string AssetName = "GitHubActionsReportBehavior"; + + public TestContext TestContext { get; set; } = null!; + + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + [TestMethod] + public async Task WhenRunPasses_NoExitCodeCalloutAndSummaryUsesSuccessIcon(string tfm) + { + (TestHostResult result, string summary) = await RunAsync(tfm, testMode: "pass"); + + result.AssertExitCodeIs(ExitCode.Success); + + // A clean run must not emit an exit-code annotation or a warning callout. + result.AssertOutputDoesNotContain("::error title=Test run failed"); + Assert.Contains("✅ Test Run Summary", summary); + Assert.DoesNotContain("[!WARNING]", summary); + } + + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + [TestMethod] + public async Task WhenTestFails_EmitsPerTestAnnotationButNoExitCodeCallout(string tfm) + { + (TestHostResult result, string summary) = await RunAsync(tfm, testMode: "fail"); + + result.AssertExitCodeIs(ExitCode.AtLeastOneTestFailed); + + // The per-test failure annotation is emitted, but AtLeastOneTestFailed is a test-result outcome, + // so no additional run-level exit-code annotation/callout should appear. + result.AssertOutputContains("::error"); + result.AssertOutputDoesNotContain("::error title=Test run failed"); + Assert.Contains("❌ Test Run Summary", summary); + Assert.DoesNotContain("[!WARNING]", summary); + } + + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + [TestMethod] + public async Task WhenZeroTestsRan_EmitsExitCodeAnnotationAndCallout(string tfm) + { + (TestHostResult result, string summary) = await RunAsync(tfm, testMode: "zero"); + + result.AssertExitCodeIs(ExitCode.ZeroTests); + + // The run failed without any failing test: expect both a run-level annotation and a summary callout. + result.AssertOutputContains("::error title=Test run failed"); + result.AssertOutputContains("ZeroTests"); + Assert.Contains("❌ Test Run Summary", summary); + Assert.Contains("[!WARNING]", summary); + Assert.Contains("ZeroTests", summary); + } + + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + [TestMethod] + public async Task WhenMinimumExpectedTestsViolated_EmitsExitCodeAnnotationAndCalloutEvenThoughTestPassed(string tfm) + { + (TestHostResult result, string summary) = await RunAsync(tfm, testMode: "pass", extraArgs: "--minimum-expected-tests 5"); + + result.AssertExitCodeIs(ExitCode.MinimumExpectedTestsPolicyViolation); + + // A single test passed, yet the run failed the minimum-expected-tests policy — the failure must surface. + result.AssertOutputContains("::error title=Test run failed"); + result.AssertOutputContains("MinimumExpectedTestsPolicyViolation"); + Assert.Contains("❌ Test Run Summary", summary); + Assert.Contains("MinimumExpectedTestsPolicyViolation", summary); + } + + private async Task<(TestHostResult Result, string Summary)> RunAsync(string tfm, string testMode, string extraArgs = "") + { + var testHost = TestInfrastructure.TestHost.LocateFrom(AssetFixture.TargetAssetPath, AssetName, tfm); + string stepSummaryPath = Path.Combine(TestContext.TestRunDirectory!, $"gh-step-summary-{Guid.NewGuid():N}.md"); + + TestHostResult result = await testHost.ExecuteAsync( + $"--report-gh {extraArgs}".Trim(), + environmentVariables: new Dictionary + { + ["GITHUB_ACTIONS"] = "true", + ["GITHUB_STEP_SUMMARY"] = stepSummaryPath, + ["GH_TEST_MODE"] = testMode, + }, + cancellationToken: TestContext.CancellationToken); + + string summary = File.Exists(stepSummaryPath) ? File.ReadAllText(stepSummaryPath) : string.Empty; + return (result, summary); + } + + public sealed class TestAssetFixture() : TestAssetFixtureBase() + { + private const string Sources = """ +#file GitHubActionsReportBehavior.csproj + + + $TargetFrameworks$ + Exe + enable + enable + preview + + + + + + +#file Program.cs +using Microsoft.Testing.Extensions; +using Microsoft.Testing.Platform.Builder; +using Microsoft.Testing.Platform.Capabilities.TestFramework; +using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.Extensions.TestFramework; + +public class Program +{ + public static async Task Main(string[] args) + { + ITestApplicationBuilder builder = await TestApplication.CreateBuilderAsync(args); + builder.RegisterTestFramework( + _ => new TestFrameworkCapabilities(), + (_, __) => new DummyTestFramework()); +#pragma warning disable TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. + builder.AddGitHubActionsProvider(); +#pragma warning restore TPEXP + using ITestApplication app = await builder.BuildAsync(); + return await app.RunAsync(); + } +} + +public class DummyTestFramework : ITestFramework, IDataProducer +{ + public string Uid => nameof(DummyTestFramework); + public string Version => "2.0.0"; + public string DisplayName => nameof(DummyTestFramework); + public string Description => nameof(DummyTestFramework); + public Type[] DataTypesProduced => [typeof(TestNodeUpdateMessage)]; + + public Task IsEnabledAsync() => Task.FromResult(true); + + public Task CreateTestSessionAsync(CreateTestSessionContext context) + => Task.FromResult(new CreateTestSessionResult() { IsSuccess = true }); + + public Task CloseTestSessionAsync(CloseTestSessionContext context) + => Task.FromResult(new CloseTestSessionResult() { IsSuccess = true }); + + public async Task ExecuteRequestAsync(ExecuteRequestContext context) + { + // The behavior under test is selected by the GH_TEST_MODE environment variable set by the acceptance test: + // "zero" -> publish no tests (exit code ZeroTests) + // "fail" -> publish a single failing test (exit code AtLeastOneTestFailed) + // "pass" -> publish a single passing test (exit code Success, unless --minimum-expected-tests forces a violation) + string mode = Environment.GetEnvironmentVariable("GH_TEST_MODE") ?? "pass"; + + if (mode == "fail") + { + await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage( + context.Request.Session.SessionUid, + new TestNode() + { + Uid = "test-1", + DisplayName = "FailingTest", + Properties = new PropertyBag(new FailedTestNodeStateProperty("Expected 1 but got 2")), + })); + } + else if (mode == "pass") + { + await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage( + context.Request.Session.SessionUid, + new TestNode() + { + Uid = "test-1", + DisplayName = "PassingTest", + Properties = new PropertyBag(PassedTestNodeStateProperty.CachedInstance), + })); + } + + // mode == "zero": publish nothing. + context.Complete(); + } +} +"""; + + public string TargetAssetPath => GetAssetPath(AssetName); + + public override (string ID, string Name, string Code) GetAssetsToGenerate() => (AssetName, AssetName, + Sources + .PatchTargetFrameworks(TargetFrameworks.All) + .PatchCodeWithReplace("$MicrosoftTestingExtensionsGitHubActionsReportVersion$", MicrosoftTestingExtensionsGitHubActionsReportVersion)); + } +} From 928c35c24987e5ad49a544fb9fe773d1cef20158 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Wed, 15 Jul 2026 17:04:07 +0200 Subject: [PATCH 3/7] Expert review round 1: rename param, tighten PACKAGE.md note, add Unknown-reason test Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b1e3944b-d2cf-4ab7-ac53-c8ace4ddc260 --- .../GitHubActionsAnnotationReporter.cs | 2 +- .../InternalAPI/InternalAPI.Unshipped.txt | 2 +- .../PACKAGE.md | 2 +- .../GitHubActionsExitCodeTests.cs | 8 ++++++++ 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsAnnotationReporter.cs b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsAnnotationReporter.cs index 2e0d5851af..59e1dd58dc 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsAnnotationReporter.cs +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsAnnotationReporter.cs @@ -129,7 +129,7 @@ public async Task ConsumeAsync(IDataProducer dataProducer, IData value, Cancella } } - public Task OnTestSessionStartingAsync(ITestSessionContext sessionUid) => Task.CompletedTask; + public Task OnTestSessionStartingAsync(ITestSessionContext testSessionContext) => Task.CompletedTask; public async Task OnTestSessionFinishingAsync(ITestSessionContext testSessionContext) { diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/InternalAPI/InternalAPI.Unshipped.txt b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/InternalAPI/InternalAPI.Unshipped.txt index 74d64b7eb5..b638b43861 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/InternalAPI/InternalAPI.Unshipped.txt +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/InternalAPI/InternalAPI.Unshipped.txt @@ -3,7 +3,7 @@ abstract Microsoft.Testing.Extensions.SlowTestReporterBase.EmitSlowTestAsync(str abstract Microsoft.Testing.Extensions.SlowTestReporterBase.GetDisplayLabel(Microsoft.Testing.Platform.Extensions.Messages.TestNode! testNode) -> string! Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsAnnotationReporter.GitHubActionsAnnotationReporter(Microsoft.Testing.Platform.CommandLine.ICommandLineOptions! commandLine, Microsoft.Testing.Platform.Helpers.IEnvironment! environment, Microsoft.Testing.Platform.Helpers.IFileSystem! fileSystem, Microsoft.Testing.Platform.OutputDevice.IOutputDevice! outputDisplay, Microsoft.Testing.Platform.Services.ITestApplicationProcessExitCode! testApplicationProcessExitCode, Microsoft.Testing.Platform.Logging.ILoggerFactory! loggerFactory) -> void Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsAnnotationReporter.OnTestSessionFinishingAsync(Microsoft.Testing.Platform.Services.ITestSessionContext! testSessionContext) -> System.Threading.Tasks.Task! -Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsAnnotationReporter.OnTestSessionStartingAsync(Microsoft.Testing.Platform.Services.ITestSessionContext! sessionUid) -> System.Threading.Tasks.Task! +Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsAnnotationReporter.OnTestSessionStartingAsync(Microsoft.Testing.Platform.Services.ITestSessionContext! testSessionContext) -> System.Threading.Tasks.Task! Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsExitCode Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsSummaryReporter.GitHubActionsSummaryReporter(Microsoft.Testing.Platform.CommandLine.ICommandLineOptions! commandLineOptions, Microsoft.Testing.Platform.Helpers.IEnvironment! environment, Microsoft.Testing.Platform.Helpers.IFileSystem! fileSystem, Microsoft.Testing.Platform.OutputDevice.IOutputDevice! outputDevice, Microsoft.Testing.Platform.Services.ITestApplicationModuleInfo! testApplicationModuleInfo, Microsoft.Testing.Platform.Services.ITestApplicationProcessExitCode! testApplicationProcessExitCode, Microsoft.Testing.Platform.Logging.ILoggerFactory! loggerFactory) -> void static Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsAnnotationReporter.GetExitCodeAnnotation(int exitCode) -> string! diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/PACKAGE.md b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/PACKAGE.md index 92efb5311a..83372e408f 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/PACKAGE.md +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/PACKAGE.md @@ -19,7 +19,7 @@ This package extends Microsoft.Testing.Platform with: - **Job summary**: appends a markdown roll-up (totals, failures, slowest tests) to the file pointed to by `GITHUB_STEP_SUMMARY`, which GitHub renders on the workflow run summary page. When running `dotnet test` across multiple assemblies each assembly appends its own section (labelled with the assembly name and target framework), and a non-test-result failure exit code is called out so a failure is not hidden behind a green ✅ > [!NOTE] -> The exit-code callout and run-level annotation cover the outcomes the extension can observe once the test session has finished: `AtLeastOneTestFailed` (2, conveyed by the per-test failures rather than a callout), `TestSessionAborted` is _not_ covered because a hard abort/cancellation short-circuits end-of-session reporting, `ZeroTests` (8), `MinimumExpectedTestsPolicyViolation` (9), `TestAdapterTestSessionFailure` (10), and `TestExecutionStoppedForMaxFailedTests` (13). Codes raised before or after the in-process session — e.g. `InvalidCommandLine` (5) or `TestHostProcessExitedNonGracefully` (7) — occur outside the extension's reach and are not surfaced here. +> The exit-code callout and run-level annotation only cover outcomes the extension can observe once the in-process test session has finished. Those are: `ZeroTests` (8), `MinimumExpectedTestsPolicyViolation` (9), `TestAdapterTestSessionFailure` (10), and `TestExecutionStoppedForMaxFailedTests` (13). `AtLeastOneTestFailed` (2) is already conveyed by the per-test failures, so it gets no separate callout. A hard abort/cancellation (`TestSessionAborted`, 3) short-circuits end-of-session reporting, and codes raised before or after the session — e.g. `InvalidCommandLine` (5) or `TestHostProcessExitedNonGracefully` (7) — occur outside the extension's reach, so none of those are surfaced here. - **Slow-test notices**: emits a `::notice` workflow command for any test still running past a threshold (default 60 seconds) The extension activates when the test run is on GitHub Actions (`GITHUB_ACTIONS=true`) and the `--report-gh` switch is passed; it no-ops otherwise. When active, each feature is enabled by default and can be toggled individually: diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsExitCodeTests.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsExitCodeTests.cs index 14a92182a2..7d2ce2ed02 100644 --- a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsExitCodeTests.cs +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsExitCodeTests.cs @@ -42,6 +42,14 @@ public void GetName_ReturnsEnumNameForKnownCodeAndUnknownOtherwise() Assert.AreEqual("Unknown", GitHubActionsExitCode.GetName(255)); } + [TestMethod] + public void GetReason_ForUnknownCode_ReturnsGenericNonSuccessReason() + { + // 255 is outside the documented ExitCode set: it must not throw and must fall back to the generic reason. + string reason = GitHubActionsExitCode.GetReason(255); + Assert.IsFalse(string.IsNullOrWhiteSpace(reason)); + } + [TestMethod] public void GetReason_ForKnownCode_MentionsRelevantOption() { From c8e9aa10cde66e70c28df7d3bba49cf9545c71e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Wed, 15 Jul 2026 17:20:43 +0200 Subject: [PATCH 4/7] Address review: add UTF-8 BOM to new files, drop unreachable-abort claims, fix markdownlint - Add UTF-8 BOM to GitHubActionsExitCodeTests.cs and GitHubActionsReportTests.cs per .editorconfig utf-8-bom requirement for *.cs. - Remove the aborted-session / host-failure examples from GitHubActionsExitCode class doc and the summary callout comment, matching the documented reachable scope (session-derived outcomes only). - Fix PACKAGE.md markdownlint (MD032 list spacing, MD060 table pipe spacing). Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b1e3944b-d2cf-4ab7-ac53-c8ace4ddc260 --- .../GitHubActionsExitCode.cs | 8 +++++--- .../GitHubActionsSummaryReporter.cs | 7 ++++--- .../PACKAGE.md | 4 ++-- .../GitHubActionsReportTests.cs | 2 +- .../GitHubActionsExitCodeTests.cs | 2 +- 5 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsExitCode.cs b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsExitCode.cs index 75d8222568..4e3fa7495b 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsExitCode.cs +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsExitCode.cs @@ -8,9 +8,11 @@ namespace Microsoft.Testing.Extensions.GitHubActionsReport; /// /// Helpers for turning a Microsoft.Testing.Platform process exit code into a GitHub-friendly verdict. -/// Used by the step-summary and annotation reporters so that non-test-result failures — e.g. a -/// --minimum-expected-tests violation, a run that discovered zero tests, an aborted session or a -/// --maximum-failed-tests stop — are surfaced instead of silently looking like a passing run. +/// Used by the step-summary and annotation reporters so that non-test-result failures observable once the +/// test session has finished — e.g. a --minimum-expected-tests violation, a run that discovered zero +/// tests, a --maximum-failed-tests stop or a test-adapter session failure — are surfaced instead of +/// silently looking like a passing run. Outcomes that occur outside the end-of-session path (a hard +/// abort/cancellation, or host failures raised before/after the session) are not reachable here. /// See . /// internal static class GitHubActionsExitCode diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsSummaryReporter.cs b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsSummaryReporter.cs index 05a346f87f..0fd7065b53 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsSummaryReporter.cs +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsSummaryReporter.cs @@ -249,9 +249,10 @@ public async Task OnTestSessionFinishingAsync(ITestSessionContext testSessionCon .Append(" | ").Append(skipped.ToString(CultureInfo.InvariantCulture)) .Append(" | ").Append(FormatDuration(totalDuration)).Append(" |\n\n"); - // Surface a non-test-result failure (aborted, zero tests, --minimum-expected-tests, --maximum-failed-tests, - // adapter/host failures, …) as a GitHub alert callout. Plain pass / at-least-one-failed outcomes are already - // conveyed by the totals table and the failures section, so no callout is added for them. + // Surface a non-test-result failure that this reporter can observe once the session has finished + // (zero tests, --minimum-expected-tests, --maximum-failed-tests, test-adapter session failure) as a + // GitHub alert callout. Plain pass / at-least-one-failed outcomes are already conveyed by the totals + // table and the failures section, so no callout is added for them. if (!GitHubActionsExitCode.IsTestResultOutcome(exitCode)) { string calloutText = string.Format( diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/PACKAGE.md b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/PACKAGE.md index 83372e408f..9547c8729d 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/PACKAGE.md +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/PACKAGE.md @@ -17,15 +17,15 @@ This package extends Microsoft.Testing.Platform with: - **Per-assembly log groups**: emits `::group::` / `::endgroup::` workflow commands so each test assembly's output is collapsed by default in the runner UI - **Failure annotations**: emits an `::error` workflow command for each failing test so failures appear in the workflow Annotations tab and, when the source location can be resolved, on the pull request's "Files changed" diff gutter. Skipped tests are surfaced as title-only `::warning` annotations so they are visible in the Annotations tab too. When the test session completes with a non-test-result failure — a `--minimum-expected-tests` violation, a run that discovered zero tests, a `--maximum-failed-tests` stop, or a test-adapter session failure — a single run-level `::error` is emitted describing the [Microsoft.Testing.Platform exit code](https://learn.microsoft.com/dotnet/core/testing/microsoft-testing-platform-troubleshooting#exit-codes) - **Job summary**: appends a markdown roll-up (totals, failures, slowest tests) to the file pointed to by `GITHUB_STEP_SUMMARY`, which GitHub renders on the workflow run summary page. When running `dotnet test` across multiple assemblies each assembly appends its own section (labelled with the assembly name and target framework), and a non-test-result failure exit code is called out so a failure is not hidden behind a green ✅ +- **Slow-test notices**: emits a `::notice` workflow command for any test still running past a threshold (default 60 seconds) > [!NOTE] > The exit-code callout and run-level annotation only cover outcomes the extension can observe once the in-process test session has finished. Those are: `ZeroTests` (8), `MinimumExpectedTestsPolicyViolation` (9), `TestAdapterTestSessionFailure` (10), and `TestExecutionStoppedForMaxFailedTests` (13). `AtLeastOneTestFailed` (2) is already conveyed by the per-test failures, so it gets no separate callout. A hard abort/cancellation (`TestSessionAborted`, 3) short-circuits end-of-session reporting, and codes raised before or after the session — e.g. `InvalidCommandLine` (5) or `TestHostProcessExitedNonGracefully` (7) — occur outside the extension's reach, so none of those are surfaced here. -- **Slow-test notices**: emits a `::notice` workflow command for any test still running past a threshold (default 60 seconds) The extension activates when the test run is on GitHub Actions (`GITHUB_ACTIONS=true`) and the `--report-gh` switch is passed; it no-ops otherwise. When active, each feature is enabled by default and can be toggled individually: | Option | Description | Default | -|---|---|---| +| --- | --- | --- | | `--report-gh` | Master switch that turns the extension on (required, in addition to running on GitHub Actions) | off | | `--report-gh-groups on\|off` | Per-assembly log groups | on | | `--report-gh-annotations on\|off` | Failure and skip annotations | on | diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/GitHubActionsReportTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/GitHubActionsReportTests.cs index 5e4c6e00e4..2e8a0fe532 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/GitHubActionsReportTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/GitHubActionsReportTests.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. namespace Microsoft.Testing.Platform.Acceptance.IntegrationTests; diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsExitCodeTests.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsExitCodeTests.cs index 7d2ce2ed02..6a9a4ce6e0 100644 --- a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsExitCodeTests.cs +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsExitCodeTests.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. extern alias ghactions; From 23ba0de63c222c4b723969d54ffda164346af7a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Thu, 16 Jul 2026 09:58:33 +0200 Subject: [PATCH 5/7] Make GITHUB_STEP_SUMMARY append cross-process safe (multi-assembly) Addresses review r3588595379: the shared GITHUB_STEP_SUMMARY file was opened with FileMode.Append + FileShare.ReadWrite. Append only seeks to EOF at open (no atomic OS append), so concurrent test-host processes in a multi-assembly dotnet test run could interleave or clobber each other's section. AppendStepSummaryWithRetryAsync now opens with FileShare.Read (denies other writers) and retries on the sharing-violation IOException with bounded backoff, serializing the per-assembly append so every assembly's section is preserved. Added unit tests for first-attempt write, retry-then-succeed, and give-up-and- rethrow (best-effort warning). Acceptance suite (12 tests) still green. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b1e3944b-d2cf-4ab7-ac53-c8ace4ddc260 --- .../GitHubActionsSummaryReporter.cs | 54 +++++++++++++++- .../InternalAPI/InternalAPI.Unshipped.txt | 1 + .../GitHubActionsSummaryReporterTests.cs | 64 +++++++++++++++++++ 3 files changed, 116 insertions(+), 3 deletions(-) diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsSummaryReporter.cs b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsSummaryReporter.cs index 0fd7065b53..e9dfabe24d 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsSummaryReporter.cs +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsSummaryReporter.cs @@ -30,6 +30,14 @@ internal sealed class GitHubActionsSummaryReporter : private const int MaxFailures = 20; private const int MaxSlowestTests = 10; + // GITHUB_STEP_SUMMARY is a single shared file that every test-host process appends to. Under a + // concurrent multi-assembly `dotnet test` run, contention is resolved by an exclusive-append retry loop + // (see AppendStepSummaryWithRetryAsync). Twenty attempts at 50 ms bound the wait to ~1s, which is ample + // to serialize the tiny per-assembly writes while still failing fast (into a best-effort warning) on a + // genuinely unwritable path. + private const int StepSummaryMaxWriteAttempts = 20; + private static readonly TimeSpan StepSummaryRetryDelay = TimeSpan.FromMilliseconds(50); + private readonly IEnvironment _environment; private readonly IFileSystem _fileSystem; private readonly IOutputDevice _outputDevice; @@ -178,9 +186,7 @@ public async Task OnTestSessionFinishingAsync(ITestSessionContext testSessionCon try { - using IFileStream stream = _fileSystem.NewFileStream(path!, FileMode.Append, FileAccess.Write, FileShare.ReadWrite); - using var writer = new StreamWriter(stream.Stream, new UTF8Encoding(false)); - await writer.WriteAsync(markdown).ConfigureAwait(false); + await AppendStepSummaryWithRetryAsync(_fileSystem, path!, markdown, StepSummaryMaxWriteAttempts, StepSummaryRetryDelay, testSessionContext.CancellationToken).ConfigureAwait(false); } catch (Exception ex) when (ex is not OperationCanceledException) { @@ -203,6 +209,48 @@ public async Task OnTestSessionFinishingAsync(ITestSessionContext testSessionCon } } + /// + /// Appends to the shared GITHUB_STEP_SUMMARY file in a way that is safe + /// when multiple test-host processes (one per assembly / target framework in a dotnet test run) write + /// concurrently. + /// + /// + /// only seeks to the end of the file once, at open time, and performs no + /// atomic OS-level append. Opening with would therefore let two processes + /// position at the same offset and interleave or overwrite each other's section. We instead open with + /// — which denies other writers — so at most one process appends at a time, and + /// retry on the resulting sharing violation (an ) until the holder releases the file. + /// Each write is a single small section, so contention clears almost immediately; the bounded attempt count + /// still lets a genuinely unwritable path surface as the caller's best-effort warning rather than looping + /// forever. + /// + internal static /* for testing */ async Task AppendStepSummaryWithRetryAsync( + IFileSystem fileSystem, + string path, + string content, + int maxAttempts, + TimeSpan retryDelay, + CancellationToken cancellationToken) + { + for (int attempt = 1; ; attempt++) + { + cancellationToken.ThrowIfCancellationRequested(); + try + { + using IFileStream stream = fileSystem.NewFileStream(path, FileMode.Append, FileAccess.Write, FileShare.Read); + using var writer = new StreamWriter(stream.Stream, new UTF8Encoding(false)); + await writer.WriteAsync(content).ConfigureAwait(false); + return; + } + catch (IOException) when (attempt < maxAttempts) + { + // Another test-host process currently holds the summary file open for writing. Back off briefly + // and retry so this assembly's section is appended intact once the holder releases the file. + await Task.Delay(retryDelay, cancellationToken).ConfigureAwait(false); + } + } + } + internal static /* for testing */ string BuildMarkdown(IReadOnlyList records, string assemblyName, string targetFrameworkMoniker, int exitCode) { int total = records.Count; diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/InternalAPI/InternalAPI.Unshipped.txt b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/InternalAPI/InternalAPI.Unshipped.txt index b638b43861..4f0f1696ca 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/InternalAPI/InternalAPI.Unshipped.txt +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/InternalAPI/InternalAPI.Unshipped.txt @@ -6,6 +6,7 @@ Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsAnnotationReporter Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsAnnotationReporter.OnTestSessionStartingAsync(Microsoft.Testing.Platform.Services.ITestSessionContext! testSessionContext) -> System.Threading.Tasks.Task! Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsExitCode Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsSummaryReporter.GitHubActionsSummaryReporter(Microsoft.Testing.Platform.CommandLine.ICommandLineOptions! commandLineOptions, Microsoft.Testing.Platform.Helpers.IEnvironment! environment, Microsoft.Testing.Platform.Helpers.IFileSystem! fileSystem, Microsoft.Testing.Platform.OutputDevice.IOutputDevice! outputDevice, Microsoft.Testing.Platform.Services.ITestApplicationModuleInfo! testApplicationModuleInfo, Microsoft.Testing.Platform.Services.ITestApplicationProcessExitCode! testApplicationProcessExitCode, Microsoft.Testing.Platform.Logging.ILoggerFactory! loggerFactory) -> void +static Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsSummaryReporter.AppendStepSummaryWithRetryAsync(Microsoft.Testing.Platform.Helpers.IFileSystem! fileSystem, string! path, string! content, int maxAttempts, System.TimeSpan retryDelay, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task! static Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsAnnotationReporter.GetExitCodeAnnotation(int exitCode) -> string! static Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsExitCode.GetName(int exitCode) -> string! static Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsExitCode.GetReason(int exitCode) -> string! diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsSummaryReporterTests.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsSummaryReporterTests.cs index 223a3856ab..eaae1266f7 100644 --- a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsSummaryReporterTests.cs +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsSummaryReporterTests.cs @@ -5,6 +5,10 @@ using ghactions::Microsoft.Testing.Extensions.GitHubActionsReport; +using Microsoft.Testing.Platform.Helpers; + +using Moq; + using GitHubActionsTerminalKind = ghactions::Microsoft.Testing.Extensions.TerminalKind; using GitHubActionsTestRecord = ghactions::Microsoft.Testing.Extensions.TestRecord; @@ -117,4 +121,64 @@ public void BuildMarkdown_MinimumExpectedTestsExitCode_EmitsCalloutEvenWhenTests Assert.Contains("Exit code 9 — MinimumExpectedTestsPolicyViolation:", markdown); Assert.Contains("--minimum-expected-tests", markdown); } + + [TestMethod] + public async Task AppendStepSummaryWithRetryAsync_WritesContent_OnFirstAttempt() + { + var buffer = new MemoryStream(); + Mock fileSystem = CreateFileSystemWritingTo(buffer); + + await GitHubActionsSummaryReporter.AppendStepSummaryWithRetryAsync( + fileSystem.Object, "summary.md", "hello world", maxAttempts: 5, retryDelay: TimeSpan.Zero, CancellationToken.None); + + // UTF8Encoding(false) is used by the reporter, so there is no BOM to strip. + Assert.AreEqual("hello world", Encoding.UTF8.GetString(buffer.ToArray())); + fileSystem.Verify(f => f.NewFileStream("summary.md", FileMode.Append, FileAccess.Write, FileShare.Read), Times.Once); + } + + [TestMethod] + public async Task AppendStepSummaryWithRetryAsync_RetriesOnSharingViolation_ThenSucceeds() + { + var buffer = new MemoryStream(); + var fileStream = new Mock(); + fileStream.Setup(s => s.Stream).Returns(buffer); + + var fileSystem = new Mock(); + // First open loses the race against another process (sharing violation), the second one wins. + fileSystem.SetupSequence(f => f.NewFileStream("summary.md", FileMode.Append, FileAccess.Write, FileShare.Read)) + .Throws(new IOException("The process cannot access the file because it is being used by another process.")) + .Returns(fileStream.Object); + + await GitHubActionsSummaryReporter.AppendStepSummaryWithRetryAsync( + fileSystem.Object, "summary.md", "second-wins", maxAttempts: 5, retryDelay: TimeSpan.Zero, CancellationToken.None); + + Assert.AreEqual("second-wins", Encoding.UTF8.GetString(buffer.ToArray())); + fileSystem.Verify(f => f.NewFileStream("summary.md", FileMode.Append, FileAccess.Write, FileShare.Read), Times.Exactly(2)); + } + + [TestMethod] + public async Task AppendStepSummaryWithRetryAsync_Rethrows_WhenAllAttemptsFail() + { + var fileSystem = new Mock(); + fileSystem.Setup(f => f.NewFileStream("summary.md", FileMode.Append, FileAccess.Write, FileShare.Read)) + .Throws(new IOException("locked")); + + // After exhausting the bounded attempts the final IOException propagates so the caller can surface its + // best-effort warning rather than looping forever. + await Assert.ThrowsExactlyAsync(() => GitHubActionsSummaryReporter.AppendStepSummaryWithRetryAsync( + fileSystem.Object, "summary.md", "never-written", maxAttempts: 3, retryDelay: TimeSpan.Zero, CancellationToken.None)); + + fileSystem.Verify(f => f.NewFileStream("summary.md", FileMode.Append, FileAccess.Write, FileShare.Read), Times.Exactly(3)); + } + + private static Mock CreateFileSystemWritingTo(Stream target) + { + var fileStream = new Mock(); + fileStream.Setup(s => s.Stream).Returns(target); + + var fileSystem = new Mock(); + fileSystem.Setup(f => f.NewFileStream(It.IsAny(), FileMode.Append, FileAccess.Write, FileShare.Read)) + .Returns(fileStream.Object); + return fileSystem; + } } From 77d4db866912d1140f53a3af9954199d633db530 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Thu, 16 Jul 2026 12:20:10 +0200 Subject: [PATCH 6/7] Scope step-summary append retry to handle acquisition only Addresses review r3593645686: the retry previously wrapped open+write+flush, so a mid-write failure (e.g. disk full) after a partial append would be retried and re-append the full section, corrupting the summary. Contention only occurs while acquiring the exclusive append handle, so retries are now scoped to the open; any failure once writing has started propagates to the best-effort warning path. Added a unit test proving a post-acquire write failure throws after a single attempt (no retry). Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b1e3944b-d2cf-4ab7-ac53-c8ace4ddc260 --- .../GitHubActionsSummaryReporter.cs | 28 +++++++++--- .../GitHubActionsSummaryReporterTests.cs | 44 +++++++++++++++++++ 2 files changed, 67 insertions(+), 5 deletions(-) diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsSummaryReporter.cs b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsSummaryReporter.cs index e9dfabe24d..7897e8dd99 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsSummaryReporter.cs +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsSummaryReporter.cs @@ -221,8 +221,15 @@ public async Task OnTestSessionFinishingAsync(ITestSessionContext testSessionCon /// — which denies other writers — so at most one process appends at a time, and /// retry on the resulting sharing violation (an ) until the holder releases the file. /// Each write is a single small section, so contention clears almost immediately; the bounded attempt count - /// still lets a genuinely unwritable path surface as the caller's best-effort warning rather than looping + /// still lets a genuinely unlockable file surface as the caller's best-effort warning rather than looping /// forever. + /// + /// Retries are scoped to acquiring the exclusive append handle only. Once the handle is acquired the + /// process appends alone, so contention can no longer occur; a failure that happens during the write + /// (e.g. disk full) may already have appended a partial section, and retrying would re-append the full section + /// on top of it and corrupt the summary. Such a mid-write failure is therefore propagated straight to the + /// caller's best-effort warning path instead of being retried. + /// /// internal static /* for testing */ async Task AppendStepSummaryWithRetryAsync( IFileSystem fileSystem, @@ -235,19 +242,30 @@ public async Task OnTestSessionFinishingAsync(ITestSessionContext testSessionCon for (int attempt = 1; ; attempt++) { cancellationToken.ThrowIfCancellationRequested(); + + IFileStream stream; try { - using IFileStream stream = fileSystem.NewFileStream(path, FileMode.Append, FileAccess.Write, FileShare.Read); - using var writer = new StreamWriter(stream.Stream, new UTF8Encoding(false)); - await writer.WriteAsync(content).ConfigureAwait(false); - return; + stream = fileSystem.NewFileStream(path, FileMode.Append, FileAccess.Write, FileShare.Read); } catch (IOException) when (attempt < maxAttempts) { // Another test-host process currently holds the summary file open for writing. Back off briefly // and retry so this assembly's section is appended intact once the holder releases the file. await Task.Delay(retryDelay, cancellationToken).ConfigureAwait(false); + continue; + } + + // The exclusive append handle is acquired: from here on we append alone, so any failure is a genuine + // write error (not contention) and must not be retried — a partial append followed by a full re-append + // would corrupt the summary. Let it propagate to the caller's best-effort warning path. + using (stream) + using (var writer = new StreamWriter(stream.Stream, new UTF8Encoding(false))) + { + await writer.WriteAsync(content).ConfigureAwait(false); } + + return; } } diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsSummaryReporterTests.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsSummaryReporterTests.cs index eaae1266f7..46c83d400a 100644 --- a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsSummaryReporterTests.cs +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsSummaryReporterTests.cs @@ -171,6 +171,25 @@ await Assert.ThrowsExactlyAsync(() => GitHubActionsSummaryReporter. fileSystem.Verify(f => f.NewFileStream("summary.md", FileMode.Append, FileAccess.Write, FileShare.Read), Times.Exactly(3)); } + [TestMethod] + public async Task AppendStepSummaryWithRetryAsync_DoesNotRetry_WhenWriteFailsAfterHandleAcquired() + { + // The handle is acquired successfully but the write/flush fails (e.g. disk full). Retrying would re-append + // the full section on top of a partial one, so the failure must propagate after a single attempt. + var fileStream = new Mock(); + fileStream.Setup(s => s.Stream).Returns(new ThrowOnWriteStream()); + + var fileSystem = new Mock(); + fileSystem.Setup(f => f.NewFileStream("summary.md", FileMode.Append, FileAccess.Write, FileShare.Read)) + .Returns(fileStream.Object); + + await Assert.ThrowsExactlyAsync(() => GitHubActionsSummaryReporter.AppendStepSummaryWithRetryAsync( + fileSystem.Object, "summary.md", "partial", maxAttempts: 5, retryDelay: TimeSpan.Zero, CancellationToken.None)); + + // Exactly one acquisition: a post-acquire write failure is not contention and must not be retried. + fileSystem.Verify(f => f.NewFileStream("summary.md", FileMode.Append, FileAccess.Write, FileShare.Read), Times.Once); + } + private static Mock CreateFileSystemWritingTo(Stream target) { var fileStream = new Mock(); @@ -181,4 +200,29 @@ private static Mock CreateFileSystemWritingTo(Stream target) .Returns(fileStream.Object); return fileSystem; } + + // A writable stream that fails on any attempt to write or flush, simulating a mid-write I/O error (e.g. disk full) + // after the exclusive append handle has already been acquired. + private sealed class ThrowOnWriteStream : Stream + { + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => throw new NotSupportedException(); + + public override long Position { get => 0; set => throw new NotSupportedException(); } + + public override void Flush() => throw new IOException("There is not enough space on the disk."); + + public override void Write(byte[] buffer, int offset, int count) => throw new IOException("There is not enough space on the disk."); + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + } } From f3a6d62dd6fa28115c5fc4e46f31816e64406e3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Thu, 16 Jul 2026 13:35:29 +0200 Subject: [PATCH 7/7] Silence code-quality nit on ThrowOnWriteStream.Position setter Addresses code-scanning nit r3594649673: expand the test helper's Position setter to explicitly consume the assigned value (discard) so the 'value ignored' analysis no longer fires. Test-only helper; behavior unchanged. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b1e3944b-d2cf-4ab7-ac53-c8ace4ddc260 --- .../GitHubActionsSummaryReporterTests.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsSummaryReporterTests.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsSummaryReporterTests.cs index 46c83d400a..4fefa6ff80 100644 --- a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsSummaryReporterTests.cs +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsSummaryReporterTests.cs @@ -213,7 +213,14 @@ private sealed class ThrowOnWriteStream : Stream public override long Length => throw new NotSupportedException(); - public override long Position { get => 0; set => throw new NotSupportedException(); } + public override long Position + { + get => 0; + + // Position is not settable on this write-only, non-seekable test stream. The discard makes the + // otherwise-ignored assigned value explicit so static analysis doesn't flag it. + set => _ = value; + } public override void Flush() => throw new IOException("There is not enough space on the disk.");