From c6edfd08d837efb5ba1c0e526762086ed8e2d515 Mon Sep 17 00:00:00 2001 From: Tess Gauthier Date: Wed, 26 Aug 2026 14:34:56 -0400 Subject: [PATCH 1/8] add code coverage to PR CI --- .azdo/ci.yml | 93 +++ .../code_coverage/Invoke-AzDOCodeCoverage.ps1 | 156 +++++ .../Invoke-OpenSSHCodeCoverage.ps1 | 178 ++++++ .../code_coverage/OpenSSHCodeCoverage.psm1 | 580 ++++++++++++++++++ .../OpenSSHCodeCoverage.tests.ps1 | 181 ++++++ contrib/win32/openssh/code_coverage/README.md | 128 ++++ 6 files changed, 1316 insertions(+) create mode 100644 contrib/win32/openssh/code_coverage/Invoke-AzDOCodeCoverage.ps1 create mode 100644 contrib/win32/openssh/code_coverage/Invoke-OpenSSHCodeCoverage.ps1 create mode 100644 contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.psm1 create mode 100644 contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.tests.ps1 create mode 100644 contrib/win32/openssh/code_coverage/README.md diff --git a/.azdo/ci.yml b/.azdo/ci.yml index 321c9201a6c5..4f9b1235ba21 100644 --- a/.azdo/ci.yml +++ b/.azdo/ci.yml @@ -164,6 +164,99 @@ stages: displayName: Clean up OpenSSH test environment condition: always() + - job: CodeCoverageWin32OpenSSH + pool: + vmImage: windows-latest + displayName: Win32-OpenSSH Code Coverage + # Coverage is informational and must never gate a merge while it stabilizes. + continueOnError: true + # Only run on pull requests. On branch builds CodeQL is enabled and the added + # coverage instrumentation overhead risks pushing the agent past its ~60 min timeout. + condition: eq(variables['Build.Reason'], 'PullRequest') + variables: + coverageDir: '$(Build.SourcesDirectory)/CodeCoverage' + steps: + - task: DownloadBuildArtifacts@0 + displayName: 'Download build artifacts (with symbols)' + inputs: + buildType: current + downloadType: single + artifactName: Win32-OpenSSH-x64 + downloadPath: '$(System.ArtifactsDirectory)' + + - task: DownloadBuildArtifacts@0 + displayName: 'Download unit test artifacts' + inputs: + buildType: current + downloadType: single + artifactName: UnitTests-x64 + downloadPath: '$(System.ArtifactsDirectory)' + + - task: DownloadBuildArtifacts@0 + displayName: 'Download bash test config file artifact' + inputs: + buildType: current + downloadType: single + artifactName: ConfigFile-x64 + downloadPath: '$(System.ArtifactsDirectory)' + + - pwsh: | + Import-Module -Name "$(Build.SourcesDirectory)/contrib/win32/openssh/AzDOBuildTools" -Force + # The build package copies .pdb symbols alongside the binaries, which + # OpenCppCoverage needs to map executed instructions back to source. + Install-OpenSSH -SourceDir "$(System.ArtifactsDirectory)/Win32-OpenSSH-x64/x64/Release" -OpenSSHDir "$env:SystemDrive/OpenSSH" -Verbose + Install-UnitTests -SourceDir "$(System.ArtifactsDirectory)/UnitTests-x64/x64/Release" -OpenSSHDir "$env:SystemDrive/OpenSSH" -Verbose + displayName: Install Win32-OpenSSH and unit tests + + - pwsh: | + $configFileSrc = "$(System.ArtifactsDirectory)/ConfigFile-x64/config.h" + Copy-Item -Path $configFileSrc -Dest "$(Build.SourcesDirectory)" -Force + displayName: Copy config file artifact for bash tests + + - pwsh: | + & "$(Build.SourcesDirectory)/contrib/win32/openssh/code_coverage/Invoke-AzDOCodeCoverage.ps1" ` + -Suite Core ` + -OpenSSHBinPath "$env:SystemDrive/OpenSSH" ` + -SourceRoot "$(Build.SourcesDirectory)" ` + -OutputDirectory "$(coverageDir)" -Verbose + displayName: Collect coverage - core tests (setup + unit + E2E) + + - pwsh: | + & "$(Build.SourcesDirectory)/contrib/win32/openssh/code_coverage/Invoke-AzDOCodeCoverage.ps1" ` + -Suite Bash ` + -OpenSSHBinPath "$env:SystemDrive/OpenSSH" ` + -SourceRoot "$(Build.SourcesDirectory)" ` + -OutputDirectory "$(coverageDir)" -Verbose + displayName: Collect coverage - bash tests + + - pwsh: | + $summary = Join-Path "$(coverageDir)" 'coverage-summary.md' + if (Test-Path $summary) { Get-Content -Raw $summary } + displayName: Print coverage summary + condition: always() + + - task: PublishCodeCoverageResults@2 + displayName: Publish code coverage results + condition: always() + inputs: + summaryFileLocation: '$(coverageDir)/merged/merged.cobertura.xml' + pathToSources: '$(Build.SourcesDirectory)' + + - pwsh: | + if (Test-Path "$(coverageDir)") + { + $artifactName = 'Win32-OpenSSH-CodeCoverage' + Write-Host "##vso[artifact.upload containerfolder=$artifactName;artifactname=$artifactName;]$(coverageDir)" + } + displayName: Upload code coverage artifacts + condition: always() + + - pwsh: | + Import-Module -Name "$(Build.SourcesDirectory)/contrib/win32/openssh/AzDOBuildTools" -Force + Clear-TestEnvironmentSetup + displayName: Clean up OpenSSH test environment + condition: always() + - job: TestPkgWin32OpenSSHBash pool: vmImage: windows-latest diff --git a/contrib/win32/openssh/code_coverage/Invoke-AzDOCodeCoverage.ps1 b/contrib/win32/openssh/code_coverage/Invoke-AzDOCodeCoverage.ps1 new file mode 100644 index 000000000000..ebc2492d6c7e --- /dev/null +++ b/contrib/win32/openssh/code_coverage/Invoke-AzDOCodeCoverage.ps1 @@ -0,0 +1,156 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +<# + .SYNOPSIS + Azure DevOps entry point for Win32-OpenSSH code coverage. + + .DESCRIPTION + Runs one OpenSSH test suite (the "Core" flow - setup + unit + E2E - or the + "Bash" flow) under OpenCppCoverage against an already-installed OpenSSH + directory, using the same AzDOBuildTools entry points the CI test jobs use. + Each run writes a per-suite binary (.cov) and Cobertura report, then merges + every .cov present under -OutputDirectory into a combined, de-duplicated + report plus per-suite/overlap summaries. + + Because the merge scans -OutputDirectory each time, the script is + idempotent: call it once per suite (Core, then Bash) into the same + -OutputDirectory and the final call produces the aggregate report. + + This script assumes the solution is already built and installed (the CI + Build stage produces PDBs alongside the binaries, which OpenCppCoverage + needs). It never builds. + + .PARAMETER Suite + Which suite to measure this invocation: Core or Bash. + + .PARAMETER OpenSSHBinPath + Installed OpenSSH directory containing the binaries, their PDBs and the + unittest-*.exe binaries (CI installs this to $env:SystemDrive\OpenSSH). + + .PARAMETER SourceRoot + Repository root used to scope coverage to OpenSSH source files. + + .PARAMETER OutputDirectory + Where per-suite and merged coverage artifacts are written. + + .EXAMPLE + .\Invoke-AzDOCodeCoverage.ps1 -Suite Core -OutputDirectory C:\cov + .\Invoke-AzDOCodeCoverage.ps1 -Suite Bash -OutputDirectory C:\cov +#> +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [ValidateSet('Core', 'Bash')] + [string] $Suite, + + [string] $OpenSSHBinPath = "$env:SystemDrive\OpenSSH", + + [string] $SourceRoot, + + [Parameter(Mandatory = $true)] + [string] $OutputDirectory +) + +$ErrorActionPreference = 'Stop' + +$opensshDir = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path +if (-not $SourceRoot) { + $SourceRoot = (Resolve-Path (Join-Path $PSScriptRoot '..' '..' '..' '..')).Path +} +$azdoModule = Join-Path $opensshDir 'AzDOBuildTools' + +Import-Module (Join-Path $PSScriptRoot 'OpenSSHCodeCoverage.psm1') -Force + +$null = New-Item -ItemType Directory -Path $OutputDirectory -Force +$openCpp = Install-OpenCppCoverage +Write-Host "OpenCppCoverage : $openCpp" +Write-Host "Suite : $Suite" +Write-Host "OpenSSHBinPath : $OpenSSHBinPath" +Write-Host "SourceRoot : $SourceRoot" + +# The current PowerShell host re-invokes itself so OpenCppCoverage can monitor +# it (and every ssh/sshd/unittest child it spawns). +$pwshHost = (Get-Process -Id $PID).Path + +switch ($Suite) { + 'Core' { + $name = 'core' + $cmd = "Import-Module '$azdoModule' -Force; Invoke-OpenSSHTests -OpenSSHBinPath '$OpenSSHBinPath'" + } + 'Bash' { + $name = 'bash' + $cmd = "Import-Module '$azdoModule' -Force; Invoke-OpenSSHBashTestsOnly -OpenSSHBinPath '$OpenSSHBinPath'" + } +} + +$session = Invoke-CoverageSession -Name $name ` + -Program $pwshHost -ArgumentList @('-NoProfile', '-NonInteractive', '-Command', $cmd) ` + -OutputDirectory (Join-Path $OutputDirectory $name) ` + -SourceRoot $SourceRoot -ModuleFilter $OpenSSHBinPath ` + -OpenCppCoveragePath $openCpp + +Write-Host "Suite '$name' coverage exit code: $($session.ExitCode)" + +# --- Aggregate every per-suite report collected so far -------------------- +$coberturaReports = Get-ChildItem -Path $OutputDirectory -Filter '*.cobertura.xml' -Recurse | + Where-Object { $_.FullName -notmatch '\\merged\\' } + +if (-not $coberturaReports) { + Write-Warning 'No per-suite Cobertura reports were produced; skipping aggregation.' + return +} + +$suiteMaps = @() +$suiteStats = @() +foreach ($report in $coberturaReports) { + $suiteName = [System.IO.Path]::GetFileNameWithoutExtension($report.Name) -replace '\.cobertura$', '' + $map = Import-CoberturaCoverage -Path $report.FullName -RepositoryRoot $SourceRoot + $suiteMaps += , $map + $suiteStats += Get-CoverageStatistic -CoverageMap $map -Name $suiteName +} + +$mergedMap = Merge-CoverageData -CoverageMap $suiteMaps +$combinedStat = Get-CoverageStatistic -CoverageMap $mergedMap -Name 'combined' +$overlap = Measure-CoverageOverlap -CoverageMap $suiteMaps + +$mergedDir = Join-Path $OutputDirectory 'merged' +# Start each run from a clean merged directory. The CI job calls this script +# once per suite into the same -OutputDirectory; regenerating the merge from +# every .cov found (below) keeps the report complete, and clearing first avoids +# OpenCppCoverage refusing to overwrite an existing HTML export directory. +Remove-Item -LiteralPath $mergedDir -Recurse -Force -ErrorAction SilentlyContinue +$null = New-Item -ItemType Directory -Path $mergedDir -Force + +$nativeMerged = Join-Path $mergedDir 'merged.cobertura.xml' +$helperMerged = Join-Path $mergedDir 'merged-helper.cobertura.xml' + +# Authoritative native merge (also emits browsable HTML). +$binaries = Get-ChildItem -Path $OutputDirectory -Filter '*.cov' -Recurse | ForEach-Object { $_.FullName } +$nativeMergeOk = $false +if ($binaries) { + try { + Merge-CoverageBinary -BinaryPath $binaries -OutputDirectory $mergedDir -Html -OpenCppCoveragePath $openCpp | Out-Null + $nativeMergeOk = Test-Path -LiteralPath $nativeMerged + } + catch { + Write-Warning "Native binary merge failed: $($_.Exception.Message)" + } +} + +# Helper-merged Cobertura (independent of the native merge) + summaries. +New-MergedCoberturaReport -CoverageMap $mergedMap -OutputPath $helperMerged | Out-Null + +# Guarantee merged.cobertura.xml exists for the publish step even when the +# native merge was skipped (no .cov) or failed, by falling back to the helper. +if (-not $nativeMergeOk) { + Write-Warning 'Using helper-merged Cobertura as merged.cobertura.xml (native merge unavailable).' + Copy-Item -LiteralPath $helperMerged -Destination $nativeMerged -Force +} + +$summary = Get-CoverageSummary -SuiteStatistic $suiteStats -CombinedStatistic $combinedStat -Overlap $overlap +$summary | ConvertTo-Json -Depth 6 | Set-Content -Path (Join-Path $OutputDirectory 'coverage-summary.json') + +$markdown = Format-CoverageSummaryMarkdown -Summary $summary +$markdown | Set-Content -Path (Join-Path $OutputDirectory 'coverage-summary.md') +Write-Host "`n$markdown" diff --git a/contrib/win32/openssh/code_coverage/Invoke-OpenSSHCodeCoverage.ps1 b/contrib/win32/openssh/code_coverage/Invoke-OpenSSHCodeCoverage.ps1 new file mode 100644 index 000000000000..4b8d4bd2012e --- /dev/null +++ b/contrib/win32/openssh/code_coverage/Invoke-OpenSSHCodeCoverage.ps1 @@ -0,0 +1,178 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +<# + .SYNOPSIS + Generates a C code coverage estimate for the Win32-OpenSSH solution + (contrib\win32\openssh\Win32-OpenSSH.sln) across the unit, Pester (E2E) and + bash test suites, then aggregates them into a single de-duplicated report. + + .DESCRIPTION + For each requested suite the script runs the suite under OpenCppCoverage, + which attaches to the launched process and every child it spawns (ssh.exe, + sshd.exe, sftp.exe, the unittest-*.exe binaries, ...) and records line + coverage from the debug PDBs. Each suite produces: + + \\.cov (binary, mergeable) + \\.cobertura.xml (per-suite report) + + The binary files are merged natively by OpenCppCoverage into + \merged\merged.cobertura.xml (plus an HTML report). Because + merging unions per-line hits, a line exercised by two suites is counted once + - that is the aggregation-with-overlap behaviour requested. + + In parallel, the pure helpers in OpenSSHCodeCoverage.psm1 re-derive the same + numbers from the per-suite Cobertura reports and additionally quantify how + much the suites overlap. The final artifacts are: + + \coverage-summary.json + \coverage-summary.md + + A Debug build is recommended so that full, unoptimized PDBs are available. + + .PARAMETER NativeHostArch + Architecture whose bin\ folder holds the built binaries (x64, x86, arm64, arm). + + .PARAMETER Configuration + Build configuration to measure. Debug is recommended for accurate coverage. + + .PARAMETER Suite + Which suites to measure. Defaults to all three. + + .PARAMETER OutputDirectory + Where coverage artifacts are written. + + .PARAMETER SkipBuild + Assume the solution is already built; do not invoke MSBuild. + + .EXAMPLE + .\Invoke-OpenSSHCodeCoverage.ps1 -Configuration Debug -Suite Unit + + .EXAMPLE + .\Invoke-OpenSSHCodeCoverage.ps1 -Configuration Debug -OutputDirectory C:\cov +#> +[CmdletBinding()] +param( + [ValidateSet('x86', 'x64', 'arm64', 'arm')] + [string] $NativeHostArch = 'x64', + + [ValidateSet('Debug', 'Release')] + [string] $Configuration = 'Debug', + + [ValidateSet('Unit', 'E2E', 'Bash')] + [string[]] $Suite = @('Unit', 'E2E', 'Bash'), + + [string] $OutputDirectory, + + [switch] $SkipBuild +) + +$ErrorActionPreference = 'Stop' + +$repositoryRoot = (Resolve-Path (Join-Path $PSScriptRoot '..' '..' '..' '..')).Path +$opensshDir = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path +Import-Module (Join-Path $PSScriptRoot 'OpenSSHCodeCoverage.psm1') -Force +Import-Module (Join-Path $opensshDir 'OpenSSHBuildHelper.psm1') -Force -ErrorAction SilentlyContinue +Import-Module (Join-Path $opensshDir 'OpenSSHTestHelper.psm1') -Force -ErrorAction SilentlyContinue + +if (-not $OutputDirectory) { + $OutputDirectory = Join-Path $repositoryRoot 'coverage' +} +$null = New-Item -ItemType Directory -Path $OutputDirectory -Force + +$folderName = if ($NativeHostArch -ieq 'x86') { 'Win32' } else { $NativeHostArch } +$binPath = Join-Path $repositoryRoot "bin\$folderName\$Configuration" + +Write-Host "Repository root : $repositoryRoot" +Write-Host "Binaries : $binPath" +Write-Host "Output : $OutputDirectory" +Write-Host "Suites : $($Suite -join ', ')" + +# --- Build ----------------------------------------------------------------- +if (-not $SkipBuild) { + if (-not (Get-Command 'Start-OpenSSHBuild' -ErrorAction SilentlyContinue)) { + throw 'Start-OpenSSHBuild is unavailable; import OpenSSHBuildHelper.psm1 or pass -SkipBuild.' + } + Write-Host "Building Win32-OpenSSH ($NativeHostArch/$Configuration)..." + Start-OpenSSHBuild -NativeHostArch $NativeHostArch -Configuration $Configuration +} + +if (-not (Test-Path $binPath)) { + throw "Binaries not found at $binPath. Build first or correct -NativeHostArch/-Configuration." +} + +$openCpp = Install-OpenCppCoverage +Write-Host "OpenCppCoverage : $openCpp" + +$pwsh = (Get-Process -Id $PID).Path # path to the current PowerShell host +$sessions = @() + +# --- Unit tests ------------------------------------------------------------ +if ($Suite -contains 'Unit') { + Write-Host "`n=== Unit tests ===" + $cmd = "Import-Module '$opensshDir\OpenSSHTestHelper.psm1' -Force; Invoke-OpenSSHUnitTest -UnitTestDirectory '$binPath'" + $sessions += Invoke-CoverageSession -Name 'unit' ` + -Program $pwsh -ArgumentList @('-NoProfile', '-NonInteractive', '-Command', $cmd) ` + -OutputDirectory (Join-Path $OutputDirectory 'unit') ` + -SourceRoot $repositoryRoot -ModuleFilter $binPath ` + -OpenCppCoveragePath $openCpp +} + +# --- Pester E2E tests ------------------------------------------------------ +if ($Suite -contains 'E2E') { + Write-Host "`n=== Pester E2E tests ===" + $cmd = "Import-Module '$opensshDir\OpenSSHTestHelper.psm1' -Force; Set-OpenSSHTestEnvironment -OpenSSHBinPath '$binPath'; Invoke-OpenSSHE2ETest" + $sessions += Invoke-CoverageSession -Name 'e2e' ` + -Program $pwsh -ArgumentList @('-NoProfile', '-NonInteractive', '-Command', $cmd) ` + -OutputDirectory (Join-Path $OutputDirectory 'e2e') ` + -SourceRoot $repositoryRoot -ModuleFilter $binPath ` + -OpenCppCoveragePath $openCpp +} + +# --- Bash tests ------------------------------------------------------------ +if ($Suite -contains 'Bash') { + Write-Host "`n=== Bash tests ===" + $cmd = "Import-Module '$opensshDir\OpenSSHTestHelper.psm1' -Force; Set-OpenSSHTestEnvironment -OpenSSHBinPath '$binPath'; Invoke-OpenSSHBashTests" + $sessions += Invoke-CoverageSession -Name 'bash' ` + -Program $pwsh -ArgumentList @('-NoProfile', '-NonInteractive', '-Command', $cmd) ` + -OutputDirectory (Join-Path $OutputDirectory 'bash') ` + -SourceRoot $repositoryRoot -ModuleFilter $binPath ` + -OpenCppCoveragePath $openCpp +} + +if (-not $sessions) { + throw 'No suites were run; nothing to report.' +} + +# --- Native merge (authoritative combined report) -------------------------- +Write-Host "`n=== Merging coverage ===" +$binaries = $sessions | ForEach-Object { $_.BinaryPath } | Where-Object { Test-Path $_ } +$mergedCobertura = Merge-CoverageBinary -BinaryPath $binaries ` + -OutputDirectory (Join-Path $OutputDirectory 'merged') -Html -OpenCppCoveragePath $openCpp +Write-Host "Merged report : $mergedCobertura" + +# --- Helper aggregation + overlap ------------------------------------------ +$suiteMaps = @() +$suiteStats = @() +foreach ($session in $sessions) { + if (-not (Test-Path $session.CoberturaPath)) { + Write-Warning "No Cobertura report for suite '$($session.Name)'; skipping." + continue + } + $map = Import-CoberturaCoverage -Path $session.CoberturaPath -RepositoryRoot $repositoryRoot + $suiteMaps += , $map + $suiteStats += Get-CoverageStatistic -CoverageMap $map -Name $session.Name +} + +$mergedMap = Merge-CoverageData -CoverageMap $suiteMaps +$combinedStat = Get-CoverageStatistic -CoverageMap $mergedMap -Name 'combined' +$overlap = Measure-CoverageOverlap -CoverageMap $suiteMaps + +$summary = Get-CoverageSummary -SuiteStatistic $suiteStats -CombinedStatistic $combinedStat -Overlap $overlap +$summary | ConvertTo-Json -Depth 6 | Set-Content -Path (Join-Path $OutputDirectory 'coverage-summary.json') + +$markdown = Format-CoverageSummaryMarkdown -Summary $summary +$markdown | Set-Content -Path (Join-Path $OutputDirectory 'coverage-summary.md') + +Write-Host "`n$markdown" +Write-Host "Artifacts written to $OutputDirectory" diff --git a/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.psm1 b/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.psm1 new file mode 100644 index 000000000000..36b19ef9a3b4 --- /dev/null +++ b/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.psm1 @@ -0,0 +1,580 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +<# + OpenSSHCodeCoverage.psm1 + + Helpers to estimate C code coverage of the Win32-OpenSSH solution + (contrib\win32\openssh\Win32-OpenSSH.sln) across the three test suites + that ship with this repository: + + * unit tests (regress\unittests\*, built as unittest-*.exe) + * Pester E2E (regress\pesterTests\*.Tests.ps1) + * bash tests (regress\*.sh, driven by bash_tests_iterator.ps1) + + Native C coverage on MSVC is collected with OpenCppCoverage + (https://github.com/OpenCppCoverage/OpenCppCoverage). OpenCppCoverage + attaches to a process *and its children* using the debug PDBs, so a single + coverage session captures every ssh.exe / sshd.exe / sftp.exe etc. that a + test spawns. Each suite is run under its own coverage session and exported + as a binary (.cov) file plus a Cobertura XML report. The binary files are + then merged natively by OpenCppCoverage, which unions per-line hit counts - + so a line exercised by two different suites is only counted once. That + natural de-duplication is what lets us aggregate the suites and account for + overlap. + + This module intentionally separates *pure* functions (Cobertura parsing, + line merging, overlap measurement, summary formatting) from the functions + that shell out to OpenCppCoverage / MSBuild. The pure functions carry the + aggregation logic and are covered by OpenSSHCodeCoverage.tests.ps1. +#> + +$ErrorActionPreference = 'Stop' + +# Repository root is four levels up from contrib\win32\openssh\code_coverage. +$script:RepositoryRoot = (Resolve-Path (Join-Path $PSScriptRoot '..' '..' '..' '..')).Path + +#region Pure helpers (unit tested) + +<# + .SYNOPSIS + Normalizes a source-file path emitted in a coverage report to a stable, + repository-relative, forward-slash path. + + .DESCRIPTION + OpenCppCoverage records absolute paths for each source file. Those paths + differ between machines (developer box vs. CI agent) and use backslashes. + To merge reports produced on different machines - and to present readable + results - we strip the repository root (or the well-known CI checkout + prefixes) and normalize separators. +#> +function ConvertTo-NormalizedCoverageSourcePath { + [CmdletBinding()] + [OutputType([string])] + param( + [Parameter(Mandatory = $true)] + [AllowEmptyString()] + [string] $RawPath, + + # Repository root to strip. Defaults to this checkout's root. + [string] $RepositoryRoot = $script:RepositoryRoot + ) + + if ([string]::IsNullOrWhiteSpace($RawPath)) { + return $RawPath + } + + # Normalize separators first so every later comparison is forward-slash. + $normalized = $RawPath -replace '\\', '/' + + # Strip a provided repository root if the path lives underneath it. + if (-not [string]::IsNullOrWhiteSpace($RepositoryRoot)) { + $root = ($RepositoryRoot -replace '\\', '/').TrimEnd('/') + if ($root -and $normalized.StartsWith($root, [System.StringComparison]::OrdinalIgnoreCase)) { + $normalized = $normalized.Substring($root.Length) + } + } + + # Strip well-known CI checkout prefixes: + # Azure Pipelines / GitHub Actions Windows: D:/a/// + # GitHub Actions Linux: /home/runner/work/// + # GitHub Actions macOS: /Users/runner/work/// + $ciPatterns = @( + '(?i)^[a-z]:/a/[^/]+/[^/]+/', + '(?i)^/home/[^/]+/work/[^/]+/[^/]+/', + '(?i)^/Users/[^/]+/work/[^/]+/[^/]+/' + ) + foreach ($pattern in $ciPatterns) { + $normalized = [regex]::Replace($normalized, $pattern, '') + } + + return $normalized.TrimStart('/') +} + +<# + .SYNOPSIS + Imports a Cobertura XML file into a coverage map. + + .DESCRIPTION + Returns a hashtable keyed by normalized source path. Each value is itself a + hashtable mapping line number (int) to hit count (int). This is the shape + consumed by Merge-CoverageData / Get-CoverageStatistic. +#> +function Import-CoberturaCoverage { + [CmdletBinding()] + [OutputType([hashtable])] + param( + [Parameter(Mandatory = $true)] + [string] $Path, + + [string] $RepositoryRoot = $script:RepositoryRoot + ) + + if (-not (Test-Path -LiteralPath $Path)) { + throw "Cobertura report not found: $Path" + } + + [xml] $xml = Get-Content -LiteralPath $Path -Raw + $map = New-Object 'System.Collections.Hashtable' ([System.StringComparer]::OrdinalIgnoreCase) + + foreach ($class in $xml.SelectNodes('//class')) { + $file = $class.GetAttribute('filename') + if ([string]::IsNullOrWhiteSpace($file)) { continue } + $key = ConvertTo-NormalizedCoverageSourcePath -RawPath $file -RepositoryRoot $RepositoryRoot + + if (-not $map.ContainsKey($key)) { + $map[$key] = @{} + } + $lines = $map[$key] + + foreach ($line in $class.SelectNodes('lines/line')) { + $number = [int] $line.GetAttribute('number') + $hits = [int] $line.GetAttribute('hits') + if ($lines.ContainsKey($number)) { + # Same line can appear more than once (e.g. inlined). Keep the max. + if ($hits -gt $lines[$number]) { $lines[$number] = $hits } + } + else { + $lines[$number] = $hits + } + } + } + + return $map +} + +<# + .SYNOPSIS + Merges any number of coverage maps into one, summing hit counts per line. + + .DESCRIPTION + This is the aggregation primitive. Where two suites cover the same file and + line, the hit counts are summed - the line ends up covered exactly once in + the merged map (its hit count is > 0), which is precisely how overlap is + absorbed. Lines unique to a single suite are carried through untouched. +#> +function Merge-CoverageData { + [CmdletBinding()] + [OutputType([hashtable])] + param( + [Parameter(Mandatory = $true)] + [AllowEmptyCollection()] + [hashtable[]] $CoverageMap + ) + + $merged = New-Object 'System.Collections.Hashtable' ([System.StringComparer]::OrdinalIgnoreCase) + + foreach ($map in $CoverageMap) { + if ($null -eq $map) { continue } + foreach ($file in $map.Keys) { + if (-not $merged.ContainsKey($file)) { + $merged[$file] = @{} + } + $target = $merged[$file] + foreach ($number in $map[$file].Keys) { + $hits = [int] $map[$file][$number] + if ($target.ContainsKey($number)) { + $target[$number] = [int] $target[$number] + $hits + } + else { + $target[$number] = $hits + } + } + } + } + + return $merged +} + +<# + .SYNOPSIS + Computes line-coverage statistics for a coverage map. +#> +function Get-CoverageStatistic { + [CmdletBinding()] + [OutputType([pscustomobject])] + param( + [Parameter(Mandatory = $true)] + [hashtable] $CoverageMap, + + # Optional label carried through onto the result (e.g. suite name). + [string] $Name + ) + + [int] $totalLines = 0 + [int] $coveredLines = 0 + $files = New-Object System.Collections.Generic.List[object] + + foreach ($file in ($CoverageMap.Keys | Sort-Object)) { + $lines = $CoverageMap[$file] + [int] $fileTotal = $lines.Count + [int] $fileCovered = 0 + foreach ($number in $lines.Keys) { + if ([int] $lines[$number] -gt 0) { $fileCovered++ } + } + $totalLines += $fileTotal + $coveredLines += $fileCovered + $files.Add([pscustomobject]@{ + File = $file + TotalLines = $fileTotal + CoveredLines = $fileCovered + LineRate = if ($fileTotal -gt 0) { [math]::Round($fileCovered / $fileTotal, 4) } else { 0 } + }) + } + + [pscustomobject]@{ + Name = $Name + TotalLines = $totalLines + CoveredLines = $coveredLines + LineRate = if ($totalLines -gt 0) { [math]::Round($coveredLines / $totalLines, 4) } else { 0 } + Percent = if ($totalLines -gt 0) { [math]::Round(100 * $coveredLines / $totalLines, 2) } else { 0 } + Files = $files + } +} + +<# + .SYNOPSIS + Measures how much the suites overlap. + + .DESCRIPTION + Given the per-suite coverage maps, returns: + * SumCoveredLines - naive sum of covered lines across suites + * CombinedCoveredLines - covered lines after merging (de-duplicated) + * OverlapLines - SumCoveredLines - CombinedCoveredLines, i.e. the + number of (file,line) coverage records that were + claimed by more than one suite + * OverlapPercent - OverlapLines as a percentage of SumCoveredLines + The combined figure is the honest aggregate coverage number; the overlap + figure quantifies redundant coverage between the suites. +#> +function Measure-CoverageOverlap { + [CmdletBinding()] + [OutputType([pscustomobject])] + param( + [Parameter(Mandatory = $true)] + [hashtable[]] $CoverageMap + ) + + [int] $sumCovered = 0 + foreach ($map in $CoverageMap) { + if ($null -eq $map) { continue } + $stat = Get-CoverageStatistic -CoverageMap $map + $sumCovered += $stat.CoveredLines + } + + $merged = Merge-CoverageData -CoverageMap $CoverageMap + $combined = (Get-CoverageStatistic -CoverageMap $merged).CoveredLines + $overlap = $sumCovered - $combined + + [pscustomobject]@{ + SumCoveredLines = $sumCovered + CombinedCoveredLines = $combined + OverlapLines = $overlap + OverlapPercent = if ($sumCovered -gt 0) { [math]::Round(100 * $overlap / $sumCovered, 2) } else { 0 } + } +} + +<# + .SYNOPSIS + Writes a coverage map back out as a minimal Cobertura XML report. + + .DESCRIPTION + Used to persist the helper-merged aggregate (independent of the native + OpenCppCoverage merge) so the numbers can be diffed / archived. +#> +function New-MergedCoberturaReport { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [hashtable] $CoverageMap, + + [Parameter(Mandatory = $true)] + [string] $OutputPath + ) + + $stat = Get-CoverageStatistic -CoverageMap $CoverageMap + $inv = [System.Globalization.CultureInfo]::InvariantCulture + $timestamp = [int][double]::Parse((Get-Date -UFormat %s), $inv) + $lineRateStr = $stat.LineRate.ToString($inv) + + $settings = New-Object System.Xml.XmlWriterSettings + $settings.Indent = $true + $settings.Encoding = [System.Text.Encoding]::UTF8 + + $writer = [System.Xml.XmlWriter]::Create($OutputPath, $settings) + try { + $writer.WriteStartDocument() + $writer.WriteStartElement('coverage') + $writer.WriteAttributeString('line-rate', $lineRateStr) + $writer.WriteAttributeString('branch-rate', '0') + $writer.WriteAttributeString('lines-covered', ([string]$stat.CoveredLines)) + $writer.WriteAttributeString('lines-valid', ([string]$stat.TotalLines)) + $writer.WriteAttributeString('version', '0') + $writer.WriteAttributeString('timestamp', ([string]$timestamp)) + + $writer.WriteStartElement('packages') + $writer.WriteStartElement('package') + $writer.WriteAttributeString('name', 'Win32-OpenSSH') + $writer.WriteAttributeString('line-rate', $lineRateStr) + $writer.WriteStartElement('classes') + + foreach ($file in ($CoverageMap.Keys | Sort-Object)) { + $lines = $CoverageMap[$file] + [int] $fileTotal = $lines.Count + [int] $fileCovered = 0 + foreach ($n in $lines.Keys) { if ([int] $lines[$n] -gt 0) { $fileCovered++ } } + $rate = if ($fileTotal -gt 0) { [math]::Round($fileCovered / $fileTotal, 4) } else { 0 } + + $writer.WriteStartElement('class') + $writer.WriteAttributeString('name', (Split-Path -Leaf $file)) + $writer.WriteAttributeString('filename', $file) + $writer.WriteAttributeString('line-rate', ([double]$rate).ToString($inv)) + $writer.WriteStartElement('lines') + foreach ($n in ($lines.Keys | Sort-Object { [int]$_ })) { + $writer.WriteStartElement('line') + $writer.WriteAttributeString('number', ([string]$n)) + $writer.WriteAttributeString('hits', ([string][int]$lines[$n])) + $writer.WriteEndElement() + } + $writer.WriteEndElement() # lines + $writer.WriteEndElement() # class + } + + $writer.WriteEndElement() # classes + $writer.WriteEndElement() # package + $writer.WriteEndElement() # packages + $writer.WriteEndElement() # coverage + $writer.WriteEndDocument() + } + finally { + $writer.Flush() + $writer.Close() + } + + return $OutputPath +} + +<# + .SYNOPSIS + Builds a human/machine readable summary object from per-suite statistics, + the combined statistic and the overlap measurement. +#> +function Get-CoverageSummary { + [CmdletBinding()] + [OutputType([pscustomobject])] + param( + [Parameter(Mandatory = $true)] + [pscustomobject[]] $SuiteStatistic, + + [Parameter(Mandatory = $true)] + [pscustomobject] $CombinedStatistic, + + [Parameter(Mandatory = $true)] + [pscustomobject] $Overlap + ) + + [pscustomobject]@{ + GeneratedOn = (Get-Date).ToString('o') + Suites = @($SuiteStatistic | ForEach-Object { + [pscustomobject]@{ + Name = $_.Name + TotalLines = $_.TotalLines + CoveredLines = $_.CoveredLines + Percent = $_.Percent + } + }) + Combined = [pscustomobject]@{ + TotalLines = $CombinedStatistic.TotalLines + CoveredLines = $CombinedStatistic.CoveredLines + Percent = $CombinedStatistic.Percent + } + Overlap = $Overlap + } +} + +<# + .SYNOPSIS + Renders a coverage summary as a Markdown table string. +#> +function Format-CoverageSummaryMarkdown { + [CmdletBinding()] + [OutputType([string])] + param( + [Parameter(Mandatory = $true)] + [pscustomobject] $Summary + ) + + $sb = New-Object System.Text.StringBuilder + [void] $sb.AppendLine('# Win32-OpenSSH code coverage estimate') + [void] $sb.AppendLine('') + [void] $sb.AppendLine("Generated: $($Summary.GeneratedOn)") + [void] $sb.AppendLine('') + [void] $sb.AppendLine('| Suite | Covered | Total | Line % |') + [void] $sb.AppendLine('|-------|--------:|------:|-------:|') + foreach ($suite in $Summary.Suites) { + [void] $sb.AppendLine("| $($suite.Name) | $($suite.CoveredLines) | $($suite.TotalLines) | $($suite.Percent)% |") + } + [void] $sb.AppendLine("| **Combined (deduped)** | **$($Summary.Combined.CoveredLines)** | **$($Summary.Combined.TotalLines)** | **$($Summary.Combined.Percent)%** |") + [void] $sb.AppendLine('') + [void] $sb.AppendLine('## Overlap between suites') + [void] $sb.AppendLine('') + [void] $sb.AppendLine("- Sum of per-suite covered lines: $($Summary.Overlap.SumCoveredLines)") + [void] $sb.AppendLine("- Combined (de-duplicated) covered lines: $($Summary.Overlap.CombinedCoveredLines)") + [void] $sb.AppendLine("- Overlapping covered lines: $($Summary.Overlap.OverlapLines) ($($Summary.Overlap.OverlapPercent)% of the sum)") + return $sb.ToString() +} + +#endregion Pure helpers + +#region OpenCppCoverage orchestration + +<# + .SYNOPSIS + Ensures OpenCppCoverage is installed and returns the path to its exe. +#> +function Install-OpenCppCoverage { + [CmdletBinding()] + [OutputType([string])] + param( + [switch] $Force + ) + + $existing = Get-Command 'OpenCppCoverage.exe' -ErrorAction SilentlyContinue + if ($existing -and -not $Force) { + return $existing.Source + } + + $defaultPath = Join-Path $env:ProgramFiles 'OpenCppCoverage\OpenCppCoverage.exe' + if ((Test-Path $defaultPath) -and -not $Force) { + return $defaultPath + } + + $choco = Get-Command 'choco.exe' -ErrorAction SilentlyContinue + if (-not $choco) { + throw 'OpenCppCoverage is not installed and Chocolatey is unavailable. Install OpenCppCoverage from https://github.com/OpenCppCoverage/OpenCppCoverage/releases and re-run.' + } + + Write-Verbose 'Installing OpenCppCoverage via Chocolatey...' + & $choco.Source install opencppcoverage -y --no-progress | Write-Verbose + + $found = Get-Command 'OpenCppCoverage.exe' -ErrorAction SilentlyContinue + if ($found) { return $found.Source } + if (Test-Path $defaultPath) { return $defaultPath } + + throw 'Failed to locate OpenCppCoverage after installation.' +} + +<# + .SYNOPSIS + Runs an arbitrary command under OpenCppCoverage, exporting a binary (.cov) + and a Cobertura XML report scoped to the OpenSSH sources and modules. +#> +function Invoke-CoverageSession { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] [string] $Name, + [Parameter(Mandatory = $true)] [string] $Program, + [string[]] $ArgumentList = @(), + [Parameter(Mandatory = $true)] [string] $OutputDirectory, + [string] $SourceRoot = $script:RepositoryRoot, + [string] $ModuleFilter, + [string] $WorkingDirectory, + [string] $OpenCppCoveragePath + ) + + if (-not $OpenCppCoveragePath) { $OpenCppCoveragePath = Install-OpenCppCoverage } + $null = New-Item -ItemType Directory -Path $OutputDirectory -Force + + $binaryOut = Join-Path $OutputDirectory "$Name.cov" + $coberturaOut = Join-Path $OutputDirectory "$Name.cobertura.xml" + + $occArgs = @( + '--sources', $SourceRoot, + '--export_type', "binary:$binaryOut", + '--export_type', "cobertura:$coberturaOut", + '--cover_children', + '--quiet' + ) + if ($ModuleFilter) { $occArgs += @('--modules', $ModuleFilter) } + if ($WorkingDirectory) { $occArgs += @('--working_dir', $WorkingDirectory) } + $occArgs += '--' + $occArgs += $Program + $occArgs += $ArgumentList + + Write-Verbose "OpenCppCoverage $($occArgs -join ' ')" + & $OpenCppCoveragePath @occArgs + $exit = $LASTEXITCODE + + [pscustomobject]@{ + Name = $Name + ExitCode = $exit + BinaryPath = $binaryOut + CoberturaPath = $coberturaOut + } +} + +<# + .SYNOPSIS + Merges binary (.cov) exports natively with OpenCppCoverage, producing a + combined Cobertura XML and (optionally) an HTML report. This is the + authoritative merged report; the pure helpers produce the same numbers and + add the overlap breakdown. +#> +function Merge-CoverageBinary { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] [string[]] $BinaryPath, + [Parameter(Mandatory = $true)] [string] $OutputDirectory, + [switch] $Html, + [string] $OpenCppCoveragePath + ) + + if (-not $OpenCppCoveragePath) { $OpenCppCoveragePath = Install-OpenCppCoverage } + $null = New-Item -ItemType Directory -Path $OutputDirectory -Force + + $coberturaOut = Join-Path $OutputDirectory 'merged.cobertura.xml' + Remove-Item -LiteralPath $coberturaOut -Force -ErrorAction SilentlyContinue + + $occArgs = @() + foreach ($bin in $BinaryPath) { + if (Test-Path $bin) { $occArgs += @('--input_coverage', $bin) } + } + if (-not $occArgs) { throw 'No binary coverage inputs found to merge.' } + + $occArgs += @('--export_type', "cobertura:$coberturaOut") + if ($Html) { + $htmlOut = Join-Path $OutputDirectory 'html' + # OpenCppCoverage refuses to write into an existing HTML export + # directory, so clear any report from a previous run first. + Remove-Item -LiteralPath $htmlOut -Recurse -Force -ErrorAction SilentlyContinue + $occArgs += @('--export_type', "html:$htmlOut") + } + $occArgs += '--quiet' + + Write-Verbose "OpenCppCoverage $($occArgs -join ' ')" + & $OpenCppCoveragePath @occArgs | Write-Verbose + if ($LASTEXITCODE -ne 0) { + throw "OpenCppCoverage merge failed with exit code $LASTEXITCODE." + } + if (-not (Test-Path -LiteralPath $coberturaOut)) { + throw "OpenCppCoverage merge did not produce $coberturaOut." + } + + return $coberturaOut +} + +#endregion OpenCppCoverage orchestration + +Export-ModuleMember -Function @( + 'ConvertTo-NormalizedCoverageSourcePath', + 'Import-CoberturaCoverage', + 'Merge-CoverageData', + 'Get-CoverageStatistic', + 'Measure-CoverageOverlap', + 'New-MergedCoberturaReport', + 'Get-CoverageSummary', + 'Format-CoverageSummaryMarkdown', + 'Install-OpenCppCoverage', + 'Invoke-CoverageSession', + 'Merge-CoverageBinary' +) diff --git a/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.tests.ps1 b/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.tests.ps1 new file mode 100644 index 000000000000..3dc02123fb9e --- /dev/null +++ b/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.tests.ps1 @@ -0,0 +1,181 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# +# Unit tests for the pure helper functions in OpenSSHCodeCoverage.psm1. +# These validate the aggregation / overlap logic without needing a build, +# OpenCppCoverage, or any of the OpenSSH test suites to run. +# +# Run with Pester 5: Invoke-Pester -Path .\OpenSSHCodeCoverage.tests.ps1 +# + +BeforeAll { + Import-Module (Join-Path $PSScriptRoot 'OpenSSHCodeCoverage.psm1') -Force + + # Writes a minimal Cobertura report for one file to $TestDrive and returns the path. + function New-TestCobertura { + param( + [string] $FileName, + [string] $SourcePath, + [hashtable] $Lines, # line number -> hits + [string] $OutFile + ) + $sb = New-Object System.Text.StringBuilder + [void] $sb.AppendLine('') + [void] $sb.AppendLine('') + [void] $sb.AppendLine(' ') + [void] $sb.AppendLine(" ") + foreach ($n in ($Lines.Keys | Sort-Object { [int]$_ })) { + [void] $sb.AppendLine(" ") + } + [void] $sb.AppendLine(' ') + [void] $sb.AppendLine(' ') + [void] $sb.AppendLine('') + Set-Content -Path $OutFile -Value $sb.ToString() -NoNewline + return $OutFile + } +} + +Describe 'ConvertTo-NormalizedCoverageSourcePath' { + Context 'Repository-root-relative paths' { + It 'Strips the provided repository root and normalizes separators' { + $result = ConvertTo-NormalizedCoverageSourcePath -RawPath 'C:\src\openssh-portable\channels.c' -RepositoryRoot 'C:\src\openssh-portable' + $result | Should -Be 'channels.c' + } + + It 'Strips a nested source path under the repository root' { + $result = ConvertTo-NormalizedCoverageSourcePath -RawPath 'C:\src\openssh-portable\openbsd-compat\bsd-misc.c' -RepositoryRoot 'C:\src\openssh-portable' + $result | Should -Be 'openbsd-compat/bsd-misc.c' + } + + It 'Is case-insensitive about the repository root' { + $result = ConvertTo-NormalizedCoverageSourcePath -RawPath 'C:\SRC\OpenSSH-Portable\channels.c' -RepositoryRoot 'C:\src\openssh-portable' + $result | Should -Be 'channels.c' + } + } + + Context 'CI checkout prefixes' { + It 'Strips the Azure Pipelines / GitHub Actions Windows prefix' { + $result = ConvertTo-NormalizedCoverageSourcePath -RawPath 'D:\a\openssh-portable\openssh-portable\sshconnect2.c' -RepositoryRoot '' + $result | Should -Be 'sshconnect2.c' + } + + It 'Strips the GitHub Actions Linux prefix' { + $result = ConvertTo-NormalizedCoverageSourcePath -RawPath '/home/runner/work/openssh-portable/openssh-portable/kex.c' -RepositoryRoot '' + $result | Should -Be 'kex.c' + } + } + + Context 'Already relative paths' { + It 'Normalizes backslashes and leaves the path otherwise unchanged' { + $result = ConvertTo-NormalizedCoverageSourcePath -RawPath 'openbsd-compat\bsd-misc.c' -RepositoryRoot 'C:\src\openssh-portable' + $result | Should -Be 'openbsd-compat/bsd-misc.c' + } + } +} + +Describe 'Import-CoberturaCoverage' { + It 'Parses lines and hits keyed by normalized path' { + $file = Join-Path $TestDrive 'a.cobertura.xml' + New-TestCobertura -FileName 'channels.c' -SourcePath 'C:\repo\channels.c' -Lines @{ 1 = 5; 2 = 0; 3 = 2 } -OutFile $file | Out-Null + + $map = Import-CoberturaCoverage -Path $file -RepositoryRoot 'C:\repo' + $map.Keys.Count | Should -Be 1 + $map['channels.c'][1] | Should -Be 5 + $map['channels.c'][2] | Should -Be 0 + $map['channels.c'][3] | Should -Be 2 + } +} + +Describe 'Merge-CoverageData' { + Context 'Overlapping coverage of the same file/line' { + It 'Sums hit counts so an overlapping line stays covered once' { + $suiteA = @{ 'channels.c' = @{ 1 = 5; 2 = 3; 3 = 0 } } + $suiteB = @{ 'channels.c' = @{ 1 = 1; 2 = 0; 3 = 4 } } + + $merged = Merge-CoverageData -CoverageMap @($suiteA, $suiteB) + $merged['channels.c'][1] | Should -Be 6 # 5 + 1 + $merged['channels.c'][2] | Should -Be 3 # 3 + 0 + $merged['channels.c'][3] | Should -Be 4 # 0 + 4 + + $stat = Get-CoverageStatistic -CoverageMap $merged + $stat.CoveredLines | Should -Be 3 # all three lines covered + $stat.TotalLines | Should -Be 3 + } + } + + Context 'Non-overlapping files from different suites' { + It 'Keeps both files' { + $suiteA = @{ 'foo.c' = @{ 1 = 5 } } + $suiteB = @{ 'bar.c' = @{ 1 = 3 } } + $merged = Merge-CoverageData -CoverageMap @($suiteA, $suiteB) + $merged.Keys.Count | Should -Be 2 + } + } +} + +Describe 'Get-CoverageStatistic' { + It 'Counts only lines with non-zero hits as covered' { + $map = @{ 'x.c' = @{ 1 = 1; 2 = 0; 3 = 0; 4 = 2 } } + $stat = Get-CoverageStatistic -CoverageMap $map -Name 'unit' + $stat.Name | Should -Be 'unit' + $stat.TotalLines | Should -Be 4 + $stat.CoveredLines | Should -Be 2 + $stat.Percent | Should -Be 50 + } +} + +Describe 'Measure-CoverageOverlap' { + It 'Reports overlap as sum-minus-combined covered lines' { + # Suite A covers lines 1,2 of foo.c. Suite B covers lines 2,3 of foo.c. + # Sum covered = 2 + 2 = 4. Combined covered = 3 (lines 1,2,3). Overlap = 1 (line 2). + $suiteA = @{ 'foo.c' = @{ 1 = 1; 2 = 1 } } + $suiteB = @{ 'foo.c' = @{ 2 = 1; 3 = 1 } } + + $overlap = Measure-CoverageOverlap -CoverageMap @($suiteA, $suiteB) + $overlap.SumCoveredLines | Should -Be 4 + $overlap.CombinedCoveredLines | Should -Be 3 + $overlap.OverlapLines | Should -Be 1 + $overlap.OverlapPercent | Should -Be 25 + } + + It 'Reports zero overlap for disjoint suites' { + $suiteA = @{ 'foo.c' = @{ 1 = 1 } } + $suiteB = @{ 'bar.c' = @{ 1 = 1 } } + $overlap = Measure-CoverageOverlap -CoverageMap @($suiteA, $suiteB) + $overlap.OverlapLines | Should -Be 0 + $overlap.OverlapPercent | Should -Be 0 + } +} + +Describe 'New-MergedCoberturaReport' { + It 'Round-trips a coverage map through Cobertura XML' { + $map = @{ 'channels.c' = @{ 1 = 5; 2 = 0; 3 = 2 } } + $out = Join-Path $TestDrive 'merged.cobertura.xml' + New-MergedCoberturaReport -CoverageMap $map -OutputPath $out | Out-Null + + $reloaded = Import-CoberturaCoverage -Path $out -RepositoryRoot '' + $reloaded['channels.c'][1] | Should -Be 5 + $reloaded['channels.c'][2] | Should -Be 0 + $reloaded['channels.c'][3] | Should -Be 2 + } +} + +Describe 'Get-CoverageSummary / Format-CoverageSummaryMarkdown' { + It 'Produces a markdown table with a combined row and overlap section' { + $suiteA = @{ 'foo.c' = @{ 1 = 1; 2 = 1 } } + $suiteB = @{ 'foo.c' = @{ 2 = 1; 3 = 1 } } + $statA = Get-CoverageStatistic -CoverageMap $suiteA -Name 'unit' + $statB = Get-CoverageStatistic -CoverageMap $suiteB -Name 'bash' + $merged = Merge-CoverageData -CoverageMap @($suiteA, $suiteB) + $combined = Get-CoverageStatistic -CoverageMap $merged -Name 'combined' + $overlap = Measure-CoverageOverlap -CoverageMap @($suiteA, $suiteB) + + $summary = Get-CoverageSummary -SuiteStatistic @($statA, $statB) -CombinedStatistic $combined -Overlap $overlap + $summary.Combined.CoveredLines | Should -Be 3 + + $md = Format-CoverageSummaryMarkdown -Summary $summary + $md | Should -Match 'Combined \(deduped\)' + $md | Should -Match 'Overlapping covered lines: 1' + } +} diff --git a/contrib/win32/openssh/code_coverage/README.md b/contrib/win32/openssh/code_coverage/README.md new file mode 100644 index 000000000000..def364436d96 --- /dev/null +++ b/contrib/win32/openssh/code_coverage/README.md @@ -0,0 +1,128 @@ +# Win32-OpenSSH code coverage + +Tooling to produce a **C code coverage estimate** for the Windows solution +(`contrib\win32\openssh\Win32-OpenSSH.sln`) across all three test suites that +ship with this repository, and to **aggregate** them into a single +de-duplicated number that accounts for overlap between the suites. + +## Files + +| File | Purpose | +|------|---------| +| `OpenSSHCodeCoverage.psm1` | Module. Pure aggregation/overlap helpers **and** OpenCppCoverage orchestration. | +| `OpenSSHCodeCoverage.tests.ps1` | Pester 5 tests for the pure helpers (no build required). | +| `Invoke-OpenSSHCodeCoverage.ps1` | Local end-to-end driver: build → run suites under coverage → merge → summarize. | +| `Invoke-AzDOCodeCoverage.ps1` | CI entry point: run one suite (Core/Bash) under coverage against an installed OpenSSH dir, then merge. | + +All files live in `contrib\win32\openssh\code_coverage\`. + +## Why OpenCppCoverage + +[OpenCppCoverage](https://github.com/OpenCppCoverage/OpenCppCoverage) is the +standard open-source, MSVC-compatible C/C++ coverage tool for Windows. It: + +- reads the debug **PDBs** to map executed instructions back to source lines + (no special build flags or instrumentation needed — just a Debug build), +- with `--cover_children`, attaches to a launched process **and every child it + spawns**, so monitoring the test harness captures every `ssh.exe`, + `sshd.exe`, `sftp.exe`, `unittest-*.exe`, etc. the tests launch, +- exports **binary** (`.cov`, re-mergeable), **Cobertura XML**, and **HTML**. + +## How aggregation and overlap work + +Each suite is measured independently and produces its own `.cov` + Cobertura +report. The `.cov` files are then merged natively by OpenCppCoverage. Merging +**unions per-line hit counts**, so a line exercised by two suites is counted +exactly once in the combined total — that is the de-duplication that makes the +aggregate honest. + +The pure helpers additionally quantify the redundancy: + +``` +SumCoveredLines = covered lines summed across suites (double counts overlap) +CombinedCoveredLines = covered lines after merge (each line once) <- the estimate +OverlapLines = SumCoveredLines - CombinedCoveredLines +``` + +## Usage + +Requires: Visual Studio build tools (to build the solution), Chocolatey (to +auto-install OpenCppCoverage), and the test-suite prerequisites (Cygwin for the +bash suite, Pester for E2E — the existing helpers install these). + +```powershell +cd contrib\win32\openssh\code_coverage + +# All suites, Debug build (recommended for accurate PDBs): +.\Invoke-OpenSSHCodeCoverage.ps1 -Configuration Debug + +# A single suite against an already-built tree: +.\Invoke-OpenSSHCodeCoverage.ps1 -Configuration Debug -Suite Unit -SkipBuild + +# Custom output location: +.\Invoke-OpenSSHCodeCoverage.ps1 -Configuration Debug -OutputDirectory C:\cov +``` + +### Output artifacts (under `-OutputDirectory`, default `.\coverage`) + +``` +unit\unit.cov, unit\unit.cobertura.xml per-suite (unit tests) +e2e\e2e.cov, e2e\e2e.cobertura.xml per-suite (Pester E2E) +bash\bash.cov, bash\bash.cobertura.xml per-suite (bash tests) +merged\merged.cobertura.xml combined, de-duplicated (native merge) +merged\html\ browsable HTML report +coverage-summary.json machine-readable summary +coverage-summary.md per-suite + combined + overlap table +``` + +Example `coverage-summary.md`: + +``` +| Suite | Covered | Total | Line % | +|-------|--------:|------:|-------:| +| unit | 4210 | 20144 | 20.9% | +| e2e | 9633 | 20144 | 47.8% | +| bash | 11002 | 20144 | 54.6% | +| **Combined (deduped)** | **13120** | **20144** | **65.1%** | + +## Overlap between suites +- Sum of per-suite covered lines: 24845 +- Combined (de-duplicated) covered lines: 13120 +- Overlapping covered lines: 11725 (47.19% of the sum) +``` + +## Validating the helpers + +The aggregation/overlap logic is unit tested and does **not** require a build, +OpenCppCoverage, or the OpenSSH suites: + +```powershell +Invoke-Pester -Path .\OpenSSHCodeCoverage.tests.ps1 -Output Detailed +``` + +## Continuous integration (Azure DevOps) + +`.azdo/ci.yml` runs a non-gating **Win32-OpenSSH Code Coverage** job in the +Test stage, in parallel with the existing test jobs. It: + +1. downloads the build artifacts (which include `.pdb` symbols) and unit tests, + and installs OpenSSH to `C:\OpenSSH`, +2. runs `Invoke-AzDOCodeCoverage.ps1 -Suite Core` (setup + unit + E2E) and then + `-Suite Bash`, each under OpenCppCoverage, +3. merges the per-suite `.cov` files, then publishes the merged Cobertura report + via `PublishCodeCoverageResults@2` and uploads the full + `Win32-OpenSSH-CodeCoverage` artifact (per-suite reports, HTML, summaries). + +The job is marked `continueOnError: true` so coverage never blocks a merge. The +suites run the exact CI entry points (`Invoke-OpenSSHTests`, +`Invoke-OpenSSHBashTestsOnly`), so coverage reflects what CI already exercises. + +## Notes + +- Use a **Debug** configuration locally for the most faithful line mapping. + Release with full optimizations can fold/reorder lines and understate + coverage; CI measures the Release artifacts it already produces. +- Coverage is scoped to repository sources via `--sources ` and to the + built binaries via `--modules `, so third-party/system code is excluded. +- The tooling reuses the existing suite entry points, so it measures exactly what + CI already runs. From 9741cd520e773114930c0a729ba115dc8be6d47e Mon Sep 17 00:00:00 2001 From: Tess Gauthier Date: Wed, 26 Aug 2026 15:21:03 -0400 Subject: [PATCH 2/8] Migrate coverage tooling to Microsoft.CodeCoverage.Console Replace the archived OpenCppCoverage layer with the VS 2022 Microsoft.CodeCoverage.Console tool (native C/C++ static instrumentation). Add a Debug + /PROFILE build job and consume its artifacts in a PR-only, non-gating coverage job. Pure aggregation/overlap helpers and their Pester tests are unchanged. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .azdo/ci.yml | 36 +- .azdo/templates/build-win32-openssh-job.yml | 41 +- .../code_coverage/Invoke-AzDOCodeCoverage.ps1 | 95 +++-- .../Invoke-OpenSSHCodeCoverage.ps1 | 123 +++--- .../code_coverage/OpenSSHCodeCoverage.psm1 | 372 +++++++++++++----- .../OpenSSHCodeCoverage.tests.ps1 | 2 +- contrib/win32/openssh/code_coverage/README.md | 106 ++--- 7 files changed, 532 insertions(+), 243 deletions(-) diff --git a/.azdo/ci.yml b/.azdo/ci.yml index 4f9b1235ba21..dd9a09b50a9c 100644 --- a/.azdo/ci.yml +++ b/.azdo/ci.yml @@ -70,6 +70,26 @@ stages: dependenciesArtifactName: vcpkg-dependencies-x86 vcpkgCacheDir: '$(vcpkgCacheDir)' + - job: BuildPkg_Coverage_x64 + displayName: Build Coverage Package (x64 Debug + /PROFILE) + pool: + vmImage: windows-latest + # Consumed only by the PR-only coverage job, so don't spend build time on + # branch builds (where CodeQL already competes for the agent budget). + condition: eq(variables['Build.Reason'], 'PullRequest') + # Non-gating: a coverage-specific build failure must never block the pipeline. + continueOnError: true + steps: + - template: ./templates/build-win32-openssh-job.yml + parameters: + nativeHostArch: x64 + buildOutputDir: x64 + artifactSuffix: Coverage-x64 + dependenciesArtifactName: vcpkg-dependencies-x64 + vcpkgCacheDir: '$(vcpkgCacheDir)' + configuration: Debug + linkerOptions: '/PROFILE' + - stage: Test displayName: Test Win32-OpenSSH dependsOn: Build @@ -177,19 +197,19 @@ stages: coverageDir: '$(Build.SourcesDirectory)/CodeCoverage' steps: - task: DownloadBuildArtifacts@0 - displayName: 'Download build artifacts (with symbols)' + displayName: 'Download coverage build artifacts (Debug + symbols)' inputs: buildType: current downloadType: single - artifactName: Win32-OpenSSH-x64 + artifactName: Win32-OpenSSH-Coverage-x64 downloadPath: '$(System.ArtifactsDirectory)' - task: DownloadBuildArtifacts@0 - displayName: 'Download unit test artifacts' + displayName: 'Download coverage unit test artifacts' inputs: buildType: current downloadType: single - artifactName: UnitTests-x64 + artifactName: UnitTests-Coverage-x64 downloadPath: '$(System.ArtifactsDirectory)' - task: DownloadBuildArtifacts@0 @@ -202,10 +222,10 @@ stages: - pwsh: | Import-Module -Name "$(Build.SourcesDirectory)/contrib/win32/openssh/AzDOBuildTools" -Force - # The build package copies .pdb symbols alongside the binaries, which - # OpenCppCoverage needs to map executed instructions back to source. - Install-OpenSSH -SourceDir "$(System.ArtifactsDirectory)/Win32-OpenSSH-x64/x64/Release" -OpenSSHDir "$env:SystemDrive/OpenSSH" -Verbose - Install-UnitTests -SourceDir "$(System.ArtifactsDirectory)/UnitTests-x64/x64/Release" -OpenSSHDir "$env:SystemDrive/OpenSSH" -Verbose + # Install the Debug (/PROFILE) build so Microsoft.CodeCoverage.Console can + # statically instrument the OpenSSH binaries and unit tests. + Install-OpenSSH -SourceDir "$(System.ArtifactsDirectory)/Win32-OpenSSH-Coverage-x64/x64/Debug" -OpenSSHDir "$env:SystemDrive/OpenSSH" -Verbose + Install-UnitTests -SourceDir "$(System.ArtifactsDirectory)/UnitTests-Coverage-x64/x64/Debug" -OpenSSHDir "$env:SystemDrive/OpenSSH" -Verbose displayName: Install Win32-OpenSSH and unit tests - pwsh: | diff --git a/.azdo/templates/build-win32-openssh-job.yml b/.azdo/templates/build-win32-openssh-job.yml index be225e0ec553..97d189820954 100644 --- a/.azdo/templates/build-win32-openssh-job.yml +++ b/.azdo/templates/build-win32-openssh-job.yml @@ -12,6 +12,12 @@ parameters: default: false - name: vcpkgCacheDir type: string +- name: configuration + type: string + default: Release +- name: linkerOptions + type: string + default: '' steps: - pwsh: | @@ -32,23 +38,39 @@ steps: - pwsh: | $nativeArch = '${{ parameters.nativeHostArch }}' + $configuration = '${{ parameters.configuration }}' + $linkerOptions = '${{ parameters.linkerOptions }}' Import-Module -Name "$(Build.SourcesDirectory)/contrib/win32/openssh/AzDOBuildTools" -Force - Invoke-AzDOBuild -NativeHostArch $nativeArch - displayName: Build Win32-OpenSSH (${{ parameters.nativeHostArch }}) + if ($configuration -eq 'Release' -and -not $linkerOptions) { + Invoke-AzDOBuild -NativeHostArch $nativeArch + } + else { + # Coverage build: honor an alternate configuration (Debug) and inject + # extra linker options (e.g. /PROFILE for static instrumentation) via the + # linker's LINK environment variable, without editing any .vcxproj. + Import-Module -Name "$(Build.SourcesDirectory)/contrib/win32/openssh/OpenSSHBuildHelper.psm1" -Force + if ($linkerOptions) { + $env:LINK = ((@($env:LINK, $linkerOptions) | Where-Object { $_ }) -join ' ') + } + Start-OpenSSHBuild -Configuration $configuration -NativeHostArch $nativeArch -Verbose + } + displayName: Build Win32-OpenSSH (${{ parameters.nativeHostArch }} ${{ parameters.configuration }}) env: VCPKG_BINARY_SOURCES: "clear;files,${{ parameters.vcpkgCacheDir }},readwrite" - pwsh: | $buildOutPath = "$(Build.SourcesDirectory)/bin" $buildOutputDir = '${{ parameters.buildOutputDir }}' + $configuration = '${{ parameters.configuration }}' $fullPath = Join-Path -Path $buildOutPath -ChildPath $buildOutputDir - $releasePath = Join-Path -Path $fullPath -ChildPath 'Release' - Get-ChildItem -Path $releasePath + $configPath = Join-Path -Path $fullPath -ChildPath $configuration + Get-ChildItem -Path $configPath displayName: Capture build results (${{ parameters.artifactSuffix }}) - pwsh: | $nativeArch = '${{ parameters.nativeHostArch }}' $artifactSuffix = '${{ parameters.artifactSuffix }}' + $configuration = '${{ parameters.configuration }}' $includeConfig = [System.Convert]::ToBoolean('${{ parameters.includeConfig }}') Import-Module -Name "$(Build.SourcesDirectory)/contrib/win32/openssh/AzDOBuildTools" -Force @@ -59,8 +81,8 @@ steps: Remove-Item -Path $buildDestPath -Recurse -Force -ErrorAction SilentlyContinue } $null = New-Item -ItemType Directory -Path $buildDestPath -Force - $buildDestArchPath = Join-Path -Path $buildDestPath -ChildPath "$nativeArch/Release" - Copy-BuildResults -BuildResultsPath $buildDestArchPath -NativeHostArch $nativeArch -Configuration Release + $buildDestArchPath = Join-Path -Path $buildDestPath -ChildPath "$nativeArch/$configuration" + Copy-BuildResults -BuildResultsPath $buildDestArchPath -NativeHostArch $nativeArch -Configuration $configuration # # Upload build artifacts Write-Verbose -Verbose -Message "Uploading build artifacts" @@ -74,7 +96,7 @@ steps: Remove-Item -Path $unitTestDestPath -Recurse -Force -ErrorAction SilentlyContinue } $null = New-Item -ItemType Directory -Path $unitTestDestPath -Force - Copy-UnitTests -UnitTestsSrcDir $buildOutPath -UnitTestsDestDir $unitTestDestPath -NativeHostArch $nativeArch -Configuration Release + Copy-UnitTests -UnitTestsSrcDir $buildOutPath -UnitTestsDestDir $unitTestDestPath -NativeHostArch $nativeArch -Configuration $configuration # # Upload unit test artifacts Write-Verbose -Verbose -Message "Uploading unit test artifacts" @@ -91,8 +113,9 @@ steps: displayName: Upload Win32-OpenSSH build artifacts (${{ parameters.artifactSuffix }}) - pwsh: | - $artifactSuffix = '${{ parameters.artifactSuffix }}' - $logFileName = "OpenSSHRelease$artifactSuffix.log" + $nativeArch = '${{ parameters.nativeHostArch }}' + $configuration = '${{ parameters.configuration }}' + $logFileName = "OpenSSH$configuration$nativeArch.log" $uniqueLines = [System.Collections.Generic.HashSet[string]]::new() $logFilePath = "$(Build.SourcesDirectory)/contrib/win32/openssh/$logFileName" if (Test-Path -Path $logFilePath) { diff --git a/contrib/win32/openssh/code_coverage/Invoke-AzDOCodeCoverage.ps1 b/contrib/win32/openssh/code_coverage/Invoke-AzDOCodeCoverage.ps1 index ebc2492d6c7e..169958590d0e 100644 --- a/contrib/win32/openssh/code_coverage/Invoke-AzDOCodeCoverage.ps1 +++ b/contrib/win32/openssh/code_coverage/Invoke-AzDOCodeCoverage.ps1 @@ -7,19 +7,27 @@ .DESCRIPTION Runs one OpenSSH test suite (the "Core" flow - setup + unit + E2E - or the - "Bash" flow) under OpenCppCoverage against an already-installed OpenSSH - directory, using the same AzDOBuildTools entry points the CI test jobs use. - Each run writes a per-suite binary (.cov) and Cobertura report, then merges - every .cov present under -OutputDirectory into a combined, de-duplicated - report plus per-suite/overlap summaries. - - Because the merge scans -OutputDirectory each time, the script is + "Bash" flow) under Microsoft.CodeCoverage.Console against an + already-installed OpenSSH directory, using the same AzDOBuildTools entry + points the CI test jobs use. + + The installed OpenSSH binaries (built with /PROFILE by the coverage build + job) are statically instrumented with a per-suite session id, then the suite + runs while a background server-mode collector owns that session. This + captures every instrumented OpenSSH process - including sshd.exe running as a + Windows service, which is not a child of this script. `shutdown` flushes a + per-suite .coverage file that is converted to Cobertura; every per-suite + report found under -OutputDirectory is then aggregated into a combined, + de-duplicated report plus per-suite/overlap summaries. + + Because the aggregation scans -OutputDirectory each time, the script is idempotent: call it once per suite (Core, then Bash) into the same - -OutputDirectory and the final call produces the aggregate report. + -OutputDirectory and the final call produces the aggregate report. The + instrument step restores (uninstruments) each binary first, so sharing an + installed directory across the two invocations is safe. - This script assumes the solution is already built and installed (the CI - Build stage produces PDBs alongside the binaries, which OpenCppCoverage - needs). It never builds. + This script assumes the solution is already built (with /PROFILE) and + installed. It never builds. .PARAMETER Suite Which suite to measure this invocation: Core or Bash. @@ -63,15 +71,11 @@ $azdoModule = Join-Path $opensshDir 'AzDOBuildTools' Import-Module (Join-Path $PSScriptRoot 'OpenSSHCodeCoverage.psm1') -Force $null = New-Item -ItemType Directory -Path $OutputDirectory -Force -$openCpp = Install-OpenCppCoverage -Write-Host "OpenCppCoverage : $openCpp" -Write-Host "Suite : $Suite" -Write-Host "OpenSSHBinPath : $OpenSSHBinPath" -Write-Host "SourceRoot : $SourceRoot" - -# The current PowerShell host re-invokes itself so OpenCppCoverage can monitor -# it (and every ssh/sshd/unittest child it spawns). -$pwshHost = (Get-Process -Id $PID).Path +$tool = Find-CodeCoverageConsole +Write-Host "CodeCoverage.Console : $tool" +Write-Host "Suite : $Suite" +Write-Host "OpenSSHBinPath : $OpenSSHBinPath" +Write-Host "SourceRoot : $SourceRoot" switch ($Suite) { 'Core' { @@ -84,13 +88,33 @@ switch ($Suite) { } } -$session = Invoke-CoverageSession -Name $name ` - -Program $pwshHost -ArgumentList @('-NoProfile', '-NonInteractive', '-Command', $cmd) ` - -OutputDirectory (Join-Path $OutputDirectory $name) ` - -SourceRoot $SourceRoot -ModuleFilter $OpenSSHBinPath ` - -OpenCppCoveragePath $openCpp +# Instrument the installed OpenSSH binaries with a per-suite session id, then +# run the suite while a server-mode collector owns that session. +$targets = Get-OpenSSHCoverageTarget -BinaryDirectory $OpenSSHBinPath +if (-not $targets) { + throw "No OpenSSH binaries to instrument were found under '$OpenSSHBinPath'." +} +Write-Host "Instrumenting $($targets.Count) binaries for suite '$name'..." + +$sessionId = [guid]::NewGuid().ToString() +Invoke-CoverageInstrument -BinaryPath $targets -SessionId $sessionId -ToolPath $tool | Out-Null -Write-Host "Suite '$name' coverage exit code: $($session.ExitCode)" +# The current PowerShell host re-invokes itself so its ssh/sshd/unittest +# children (all instrumented) run inside the collection window. +$pwshHost = (Get-Process -Id $PID).Path + +try { + $session = Invoke-CoverageSession -Name $name ` + -Program $pwshHost -ArgumentList @('-NoProfile', '-NonInteractive', '-Command', $cmd) ` + -SessionId $sessionId ` + -OutputDirectory (Join-Path $OutputDirectory $name) ` + -ToolPath $tool + Write-Host "Suite '$name' coverage exit code: $($session.ExitCode)" +} +finally { + # Restore the binaries so a subsequent suite step starts from clean images. + Invoke-CoverageUninstrument -BinaryPath $targets -ToolPath $tool +} # --- Aggregate every per-suite report collected so far -------------------- $coberturaReports = Get-ChildItem -Path $OutputDirectory -Filter '*.cobertura.xml' -Recurse | @@ -115,26 +139,24 @@ $combinedStat = Get-CoverageStatistic -CoverageMap $mergedMap -Name 'combined' $overlap = Measure-CoverageOverlap -CoverageMap $suiteMaps $mergedDir = Join-Path $OutputDirectory 'merged' -# Start each run from a clean merged directory. The CI job calls this script -# once per suite into the same -OutputDirectory; regenerating the merge from -# every .cov found (below) keeps the report complete, and clearing first avoids -# OpenCppCoverage refusing to overwrite an existing HTML export directory. +# Regenerate the merge from every .coverage found each run so the report stays +# complete as suites accumulate; clear first to avoid stale outputs. Remove-Item -LiteralPath $mergedDir -Recurse -Force -ErrorAction SilentlyContinue $null = New-Item -ItemType Directory -Path $mergedDir -Force $nativeMerged = Join-Path $mergedDir 'merged.cobertura.xml' $helperMerged = Join-Path $mergedDir 'merged-helper.cobertura.xml' -# Authoritative native merge (also emits browsable HTML). -$binaries = Get-ChildItem -Path $OutputDirectory -Filter '*.cov' -Recurse | ForEach-Object { $_.FullName } +# Authoritative native merge of every per-suite .coverage file. +$coverageFiles = Get-ChildItem -Path $OutputDirectory -Filter '*.coverage' -Recurse | ForEach-Object { $_.FullName } $nativeMergeOk = $false -if ($binaries) { +if ($coverageFiles) { try { - Merge-CoverageBinary -BinaryPath $binaries -OutputDirectory $mergedDir -Html -OpenCppCoveragePath $openCpp | Out-Null + Convert-CoverageReport -InputPath $coverageFiles -OutputPath $nativeMerged -Format cobertura -ToolPath $tool | Out-Null $nativeMergeOk = Test-Path -LiteralPath $nativeMerged } catch { - Write-Warning "Native binary merge failed: $($_.Exception.Message)" + Write-Warning "Native coverage merge failed: $($_.Exception.Message)" } } @@ -142,7 +164,8 @@ if ($binaries) { New-MergedCoberturaReport -CoverageMap $mergedMap -OutputPath $helperMerged | Out-Null # Guarantee merged.cobertura.xml exists for the publish step even when the -# native merge was skipped (no .cov) or failed, by falling back to the helper. +# native merge was skipped (no .coverage) or failed, by falling back to the +# helper. if (-not $nativeMergeOk) { Write-Warning 'Using helper-merged Cobertura as merged.cobertura.xml (native merge unavailable).' Copy-Item -LiteralPath $helperMerged -Destination $nativeMerged -Force diff --git a/contrib/win32/openssh/code_coverage/Invoke-OpenSSHCodeCoverage.ps1 b/contrib/win32/openssh/code_coverage/Invoke-OpenSSHCodeCoverage.ps1 index 4b8d4bd2012e..ec62879ffbaf 100644 --- a/contrib/win32/openssh/code_coverage/Invoke-OpenSSHCodeCoverage.ps1 +++ b/contrib/win32/openssh/code_coverage/Invoke-OpenSSHCodeCoverage.ps1 @@ -8,18 +8,21 @@ bash test suites, then aggregates them into a single de-duplicated report. .DESCRIPTION - For each requested suite the script runs the suite under OpenCppCoverage, - which attaches to the launched process and every child it spawns (ssh.exe, - sshd.exe, sftp.exe, the unittest-*.exe binaries, ...) and records line - coverage from the debug PDBs. Each suite produces: - - \\.cov (binary, mergeable) - \\.cobertura.xml (per-suite report) - - The binary files are merged natively by OpenCppCoverage into - \merged\merged.cobertura.xml (plus an HTML report). Because - merging unions per-line hits, a line exercised by two suites is counted once - - that is the aggregation-with-overlap behaviour requested. + The OpenSSH binaries in bin\\ (built with the /PROFILE + linker switch so they can be statically instrumented) are instrumented once + with Microsoft.CodeCoverage.Console. Each requested suite then runs while a + background server-mode collector owns a shared session id, so every + instrumented OpenSSH process that executes - ssh.exe, sshd.exe (including as + a service), sftp.exe, the unittest-*.exe binaries, ... - is captured + regardless of parent. Each suite produces: + + \\.coverage (binary, mergeable) + \\.cobertura.xml (per-suite report) + + The per-suite .coverage files are merged natively into + \merged\merged.cobertura.xml. Because merging unions + per-line hits, a line exercised by two suites is counted once - that is the + aggregation-with-overlap behaviour requested. In parallel, the pure helpers in OpenSSHCodeCoverage.psm1 re-derive the same numbers from the per-suite Cobertura reports and additionally quantify how @@ -28,7 +31,8 @@ \coverage-summary.json \coverage-summary.md - A Debug build is recommended so that full, unoptimized PDBs are available. + A Debug build is recommended so line mapping is accurate (unoptimized code). + Native C/C++ coverage requires Visual Studio 2022 Enterprise (17.3+). .PARAMETER NativeHostArch Architecture whose bin\ folder holds the built binaries (x64, x86, arm64, arm). @@ -93,51 +97,72 @@ if (-not $SkipBuild) { if (-not (Get-Command 'Start-OpenSSHBuild' -ErrorAction SilentlyContinue)) { throw 'Start-OpenSSHBuild is unavailable; import OpenSSHBuildHelper.psm1 or pass -SkipBuild.' } - Write-Host "Building Win32-OpenSSH ($NativeHostArch/$Configuration)..." - Start-OpenSSHBuild -NativeHostArch $NativeHostArch -Configuration $Configuration + Write-Host "Building Win32-OpenSSH ($NativeHostArch/$Configuration) with /PROFILE..." + # /PROFILE (honored via the linker's LINK env var) emits the fixups + full + # PDBs that static instrumentation needs, without editing any .vcxproj. + $previousLink = $env:LINK + $env:LINK = ((@($env:LINK, '/PROFILE') | Where-Object { $_ }) -join ' ') + try { + Start-OpenSSHBuild -NativeHostArch $NativeHostArch -Configuration $Configuration + } + finally { + $env:LINK = $previousLink + } } if (-not (Test-Path $binPath)) { throw "Binaries not found at $binPath. Build first or correct -NativeHostArch/-Configuration." } -$openCpp = Install-OpenCppCoverage -Write-Host "OpenCppCoverage : $openCpp" +$tool = Find-CodeCoverageConsole +Write-Host "CodeCoverage.Console : $tool" + +# Instrument the built binaries once; every suite reuses the same session id. +$targets = Get-OpenSSHCoverageTarget -BinaryDirectory $binPath +if (-not $targets) { + throw "No OpenSSH binaries to instrument were found under '$binPath'." +} +$sessionId = [guid]::NewGuid().ToString() +Write-Host "Instrumenting $($targets.Count) binaries (session $sessionId)..." +Invoke-CoverageInstrument -BinaryPath $targets -SessionId $sessionId -ToolPath $tool | Out-Null $pwsh = (Get-Process -Id $PID).Path # path to the current PowerShell host $sessions = @() -# --- Unit tests ------------------------------------------------------------ -if ($Suite -contains 'Unit') { - Write-Host "`n=== Unit tests ===" - $cmd = "Import-Module '$opensshDir\OpenSSHTestHelper.psm1' -Force; Invoke-OpenSSHUnitTest -UnitTestDirectory '$binPath'" - $sessions += Invoke-CoverageSession -Name 'unit' ` - -Program $pwsh -ArgumentList @('-NoProfile', '-NonInteractive', '-Command', $cmd) ` - -OutputDirectory (Join-Path $OutputDirectory 'unit') ` - -SourceRoot $repositoryRoot -ModuleFilter $binPath ` - -OpenCppCoveragePath $openCpp -} +try { + # --- Unit tests -------------------------------------------------------- + if ($Suite -contains 'Unit') { + Write-Host "`n=== Unit tests ===" + $cmd = "Import-Module '$opensshDir\OpenSSHTestHelper.psm1' -Force; Invoke-OpenSSHUnitTest -UnitTestDirectory '$binPath'" + $sessions += Invoke-CoverageSession -Name 'unit' ` + -Program $pwsh -ArgumentList @('-NoProfile', '-NonInteractive', '-Command', $cmd) ` + -SessionId $sessionId ` + -OutputDirectory (Join-Path $OutputDirectory 'unit') -ToolPath $tool + } -# --- Pester E2E tests ------------------------------------------------------ -if ($Suite -contains 'E2E') { - Write-Host "`n=== Pester E2E tests ===" - $cmd = "Import-Module '$opensshDir\OpenSSHTestHelper.psm1' -Force; Set-OpenSSHTestEnvironment -OpenSSHBinPath '$binPath'; Invoke-OpenSSHE2ETest" - $sessions += Invoke-CoverageSession -Name 'e2e' ` - -Program $pwsh -ArgumentList @('-NoProfile', '-NonInteractive', '-Command', $cmd) ` - -OutputDirectory (Join-Path $OutputDirectory 'e2e') ` - -SourceRoot $repositoryRoot -ModuleFilter $binPath ` - -OpenCppCoveragePath $openCpp -} + # --- Pester E2E tests -------------------------------------------------- + if ($Suite -contains 'E2E') { + Write-Host "`n=== Pester E2E tests ===" + $cmd = "Import-Module '$opensshDir\OpenSSHTestHelper.psm1' -Force; Set-OpenSSHTestEnvironment -OpenSSHBinPath '$binPath'; Invoke-OpenSSHE2ETest" + $sessions += Invoke-CoverageSession -Name 'e2e' ` + -Program $pwsh -ArgumentList @('-NoProfile', '-NonInteractive', '-Command', $cmd) ` + -SessionId $sessionId ` + -OutputDirectory (Join-Path $OutputDirectory 'e2e') -ToolPath $tool + } -# --- Bash tests ------------------------------------------------------------ -if ($Suite -contains 'Bash') { - Write-Host "`n=== Bash tests ===" - $cmd = "Import-Module '$opensshDir\OpenSSHTestHelper.psm1' -Force; Set-OpenSSHTestEnvironment -OpenSSHBinPath '$binPath'; Invoke-OpenSSHBashTests" - $sessions += Invoke-CoverageSession -Name 'bash' ` - -Program $pwsh -ArgumentList @('-NoProfile', '-NonInteractive', '-Command', $cmd) ` - -OutputDirectory (Join-Path $OutputDirectory 'bash') ` - -SourceRoot $repositoryRoot -ModuleFilter $binPath ` - -OpenCppCoveragePath $openCpp + # --- Bash tests -------------------------------------------------------- + if ($Suite -contains 'Bash') { + Write-Host "`n=== Bash tests ===" + $cmd = "Import-Module '$opensshDir\OpenSSHTestHelper.psm1' -Force; Set-OpenSSHTestEnvironment -OpenSSHBinPath '$binPath'; Invoke-OpenSSHBashTests" + $sessions += Invoke-CoverageSession -Name 'bash' ` + -Program $pwsh -ArgumentList @('-NoProfile', '-NonInteractive', '-Command', $cmd) ` + -SessionId $sessionId ` + -OutputDirectory (Join-Path $OutputDirectory 'bash') -ToolPath $tool + } +} +finally { + # Restore the original binaries. + Invoke-CoverageUninstrument -BinaryPath $targets -ToolPath $tool } if (-not $sessions) { @@ -146,9 +171,9 @@ if (-not $sessions) { # --- Native merge (authoritative combined report) -------------------------- Write-Host "`n=== Merging coverage ===" -$binaries = $sessions | ForEach-Object { $_.BinaryPath } | Where-Object { Test-Path $_ } -$mergedCobertura = Merge-CoverageBinary -BinaryPath $binaries ` - -OutputDirectory (Join-Path $OutputDirectory 'merged') -Html -OpenCppCoveragePath $openCpp +$coverageFiles = $sessions | ForEach-Object { $_.CoveragePath } | Where-Object { $_ -and (Test-Path $_) } +$mergedCobertura = Convert-CoverageReport -InputPath $coverageFiles ` + -OutputPath (Join-Path $OutputDirectory 'merged\merged.cobertura.xml') -Format cobertura -ToolPath $tool Write-Host "Merged report : $mergedCobertura" # --- Helper aggregation + overlap ------------------------------------------ diff --git a/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.psm1 b/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.psm1 index 36b19ef9a3b4..fd3536e7e093 100644 --- a/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.psm1 +++ b/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.psm1 @@ -12,21 +12,34 @@ * Pester E2E (regress\pesterTests\*.Tests.ps1) * bash tests (regress\*.sh, driven by bash_tests_iterator.ps1) - Native C coverage on MSVC is collected with OpenCppCoverage - (https://github.com/OpenCppCoverage/OpenCppCoverage). OpenCppCoverage - attaches to a process *and its children* using the debug PDBs, so a single - coverage session captures every ssh.exe / sshd.exe / sftp.exe etc. that a - test spawns. Each suite is run under its own coverage session and exported - as a binary (.cov) file plus a Cobertura XML report. The binary files are - then merged natively by OpenCppCoverage, which unions per-line hit counts - - so a line exercised by two different suites is only counted once. That - natural de-duplication is what lets us aggregate the suites and account for - overlap. + Native C coverage on MSVC is collected with Microsoft.CodeCoverage.Console + (https://learn.microsoft.com/visualstudio/test/microsoft-code-coverage-console-tool), + the Microsoft-maintained coverage tool that ships with Visual Studio 2022 + (17.3+, Enterprise edition provides native C/C++ support). Unlike a + debugger-attach tool, it uses *static instrumentation*: the OpenSSH binaries + are built with the /PROFILE linker switch (see the dedicated coverage build + job) and rewritten on disk by the `instrument` command, embedding a shared + session id. + + Collection uses server mode. For each suite we start a background collector + (`collect --session-id --server-mode`) and then run the suite. Every + instrumented OpenSSH process that executes while the collector owns the + session - including sshd.exe running as a Windows *service*, which is not a + child of our process - rendezvouses with the collector by session id. That + is why server mode is used rather than a simple child-process wrap: the + E2E/bash suites drive a real sshd service. `shutdown ` flushes each + suite's .coverage file, which we convert to Cobertura via `merge`. + + The per-suite .coverage files are then merged (`merge ... -f cobertura`), + which unions per-line hit counts, so a line exercised by two different + suites is only counted once. That natural de-duplication is what lets us + aggregate the suites and account for overlap. This module intentionally separates *pure* functions (Cobertura parsing, line merging, overlap measurement, summary formatting) from the functions - that shell out to OpenCppCoverage / MSBuild. The pure functions carry the - aggregation logic and are covered by OpenSSHCodeCoverage.tests.ps1. + that shell out to Microsoft.CodeCoverage.Console / MSBuild. The pure + functions carry the aggregation logic and are covered by + OpenSSHCodeCoverage.tests.ps1. #> $ErrorActionPreference = 'Stop' @@ -42,7 +55,7 @@ $script:RepositoryRoot = (Resolve-Path (Join-Path $PSScriptRoot '..' '..' '..' ' repository-relative, forward-slash path. .DESCRIPTION - OpenCppCoverage records absolute paths for each source file. Those paths + The coverage tool records absolute paths for each source file. Those paths differ between machines (developer box vs. CI agent) and use backslashes. To merge reports produced on different machines - and to present readable results - we strip the repository root (or the well-known CI checkout @@ -280,7 +293,8 @@ function Measure-CoverageOverlap { .DESCRIPTION Used to persist the helper-merged aggregate (independent of the native - OpenCppCoverage merge) so the numbers can be diffed / archived. + Microsoft.CodeCoverage.Console merge) so the numbers can be diffed / + archived. #> function New-MergedCoberturaReport { [CmdletBinding()] @@ -426,48 +440,200 @@ function Format-CoverageSummaryMarkdown { #endregion Pure helpers -#region OpenCppCoverage orchestration +#region Microsoft.CodeCoverage.Console orchestration + +# Well-known OpenSSH executables that are built with /PROFILE and therefore can +# be statically instrumented. unittest-*.exe are matched separately by wildcard. +$script:OpenSSHCoverageExeNames = @( + 'ssh.exe', 'sshd.exe', 'sshd-session.exe', 'sftp.exe', 'sftp-server.exe', + 'scp.exe', 'ssh-add.exe', 'ssh-agent.exe', 'ssh-keygen.exe', + 'ssh-keyscan.exe', 'ssh-shellhost.exe', 'ssh-sk-helper.exe', + 'ssh-pkcs11-helper.exe' +) <# .SYNOPSIS - Ensures OpenCppCoverage is installed and returns the path to its exe. + Locates Microsoft.CodeCoverage.Console.exe. + + .DESCRIPTION + The tool ships with Visual Studio 2022 (17.3+) under + Common7\IDE\Extensions\Microsoft\CodeCoverage.Console. Native C/C++ coverage + requires the Enterprise edition. Resolution order: an explicit -Path, the + current PATH, vswhere-reported VS installations, then a scan of the standard + install roots. #> -function Install-OpenCppCoverage { +function Find-CodeCoverageConsole { [CmdletBinding()] [OutputType([string])] param( - [switch] $Force + [string] $Path ) - $existing = Get-Command 'OpenCppCoverage.exe' -ErrorAction SilentlyContinue - if ($existing -and -not $Force) { - return $existing.Source + if ($Path) { + if (Test-Path -LiteralPath $Path) { return (Resolve-Path -LiteralPath $Path).Path } + throw "Microsoft.CodeCoverage.Console.exe was not found at '$Path'." } - $defaultPath = Join-Path $env:ProgramFiles 'OpenCppCoverage\OpenCppCoverage.exe' - if ((Test-Path $defaultPath) -and -not $Force) { - return $defaultPath + $existing = Get-Command 'Microsoft.CodeCoverage.Console.exe' -ErrorAction SilentlyContinue + if ($existing) { return $existing.Source } + + $relative = 'Common7\IDE\Extensions\Microsoft\CodeCoverage.Console\Microsoft.CodeCoverage.Console.exe' + + $vswhere = Join-Path ${env:ProgramFiles(x86)} 'Microsoft Visual Studio\Installer\vswhere.exe' + if (Test-Path $vswhere) { + $roots = & $vswhere -all -prerelease -property installationPath 2>$null + foreach ($root in $roots) { + if (-not $root) { continue } + $candidate = Join-Path $root $relative + if (Test-Path $candidate) { return (Resolve-Path -LiteralPath $candidate).Path } + } } - $choco = Get-Command 'choco.exe' -ErrorAction SilentlyContinue - if (-not $choco) { - throw 'OpenCppCoverage is not installed and Chocolatey is unavailable. Install OpenCppCoverage from https://github.com/OpenCppCoverage/OpenCppCoverage/releases and re-run.' + foreach ($base in @($env:ProgramFiles, ${env:ProgramFiles(x86)})) { + if (-not $base) { continue } + $vsRoot = Join-Path $base 'Microsoft Visual Studio' + if (-not (Test-Path $vsRoot)) { continue } + $found = Get-ChildItem -Path $vsRoot -Recurse -Filter 'Microsoft.CodeCoverage.Console.exe' -ErrorAction SilentlyContinue | + Select-Object -First 1 + if ($found) { return $found.FullName } } - Write-Verbose 'Installing OpenCppCoverage via Chocolatey...' - & $choco.Source install opencppcoverage -y --no-progress | Write-Verbose + throw @' +Microsoft.CodeCoverage.Console.exe was not found. It ships with Visual Studio 2022 (17.3 or later); native C/C++ code coverage requires the Enterprise edition. Install the "Code coverage" component, or run on an agent image that includes VS Enterprise (the hosted windows-latest image does). +'@ +} + +<# + .SYNOPSIS + Returns the OpenSSH binaries under a directory that should be instrumented. +#> +function Get-OpenSSHCoverageTarget { + [CmdletBinding()] + [OutputType([string[]])] + param( + [Parameter(Mandatory = $true)] [string] $BinaryDirectory + ) + + if (-not (Test-Path -LiteralPath $BinaryDirectory)) { + throw "Binary directory '$BinaryDirectory' does not exist." + } + + $targets = New-Object System.Collections.Generic.List[string] + foreach ($name in $script:OpenSSHCoverageExeNames) { + $p = Join-Path $BinaryDirectory $name + if (Test-Path -LiteralPath $p) { [void] $targets.Add((Resolve-Path -LiteralPath $p).Path) } + } + Get-ChildItem -Path $BinaryDirectory -Filter 'unittest-*.exe' -Recurse -ErrorAction SilentlyContinue | + ForEach-Object { [void] $targets.Add($_.FullName) } + + return $targets.ToArray() +} + +<# + .SYNOPSIS + Copies the static_covrun*.dll runtime next to the instrumented binaries. - $found = Get-Command 'OpenCppCoverage.exe' -ErrorAction SilentlyContinue - if ($found) { return $found.Source } - if (Test-Path $defaultPath) { return $defaultPath } + .DESCRIPTION + An instrumented native binary references static_covrun64.dll at load time. + The `collect`/`connect` commands add its directory to PATH automatically, but + the sshd *service* is launched by the Service Control Manager and does not + inherit that PATH, so the runtime must sit beside the instrumented binaries. +#> +function Copy-CoverageRuntime { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] [string] $ToolPath, + [Parameter(Mandatory = $true)] [string] $DestinationDirectory + ) - throw 'Failed to locate OpenCppCoverage after installation.' + $toolDir = Split-Path -Parent $ToolPath + $runtimes = Get-ChildItem -Path $toolDir -Recurse -Filter 'static_covrun*.dll' -ErrorAction SilentlyContinue + if (-not $runtimes) { + Write-Warning "No static_covrun*.dll found under '$toolDir'; instrumented service binaries may fail to load." + return + } + foreach ($rt in $runtimes) { + Copy-Item -LiteralPath $rt.FullName -Destination $DestinationDirectory -Force + } } <# .SYNOPSIS - Runs an arbitrary command under OpenCppCoverage, exporting a binary (.cov) - and a Cobertura XML report scoped to the OpenSSH sources and modules. + Statically instruments a set of native binaries with a shared session id. + + .DESCRIPTION + Each binary is first restored (best-effort uninstrument) so the function is + idempotent across repeated CI steps that share an installed OpenSSH + directory, then instrumented in place. Requires the binaries to have been + linked with /PROFILE. +#> +function Invoke-CoverageInstrument { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] [string[]] $BinaryPath, + [Parameter(Mandatory = $true)] [string] $SessionId, + [string] $ToolPath + ) + + if (-not $ToolPath) { $ToolPath = Find-CodeCoverageConsole } + $instrumented = New-Object System.Collections.Generic.List[string] + + foreach ($bin in $BinaryPath) { + if (-not (Test-Path -LiteralPath $bin)) { + Write-Warning "Instrument target '$bin' not found; skipping." + continue + } + # Restore first so re-running against an already-instrumented binary + # (e.g. Core step then Bash step) does not fail. + & $ToolPath uninstrument $bin --nologo 2>$null | Out-Null + + & $ToolPath instrument $bin --session-id $SessionId --nologo | Write-Verbose + if ($LASTEXITCODE -ne 0) { + throw "Instrumenting '$bin' failed with exit code $LASTEXITCODE. Ensure it was linked with /PROFILE." + } + [void] $instrumented.Add($bin) + } + + if ($instrumented.Count -eq 0) { + throw 'No binaries were instrumented; cannot collect coverage.' + } + + # Drop the coverage runtime next to every directory that holds an + # instrumented binary. + $dirs = $instrumented | ForEach-Object { Split-Path -Parent $_ } | Sort-Object -Unique + foreach ($dir in $dirs) { Copy-CoverageRuntime -ToolPath $ToolPath -DestinationDirectory $dir } + + return $instrumented.ToArray() +} + +<# + .SYNOPSIS + Restores (uninstruments) a set of native binaries. Best-effort. +#> +function Invoke-CoverageUninstrument { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] [string[]] $BinaryPath, + [string] $ToolPath + ) + if (-not $ToolPath) { $ToolPath = Find-CodeCoverageConsole } + foreach ($bin in $BinaryPath) { + if (-not (Test-Path -LiteralPath $bin)) { continue } + & $ToolPath uninstrument $bin --nologo 2>$null | Out-Null + } +} + +<# + .SYNOPSIS + Runs a suite under a server-mode coverage collector and returns the + resulting .coverage plus a per-suite Cobertura report. + + .DESCRIPTION + Starts `collect --session-id --server-mode` as a background process so + it owns the session, runs the suite command, then `shutdown ` to flush + the .coverage file. Every instrumented OpenSSH process (including sshd.exe + started as a service, which is not our child) that executes during the + window rendezvouses with the collector by session id. #> function Invoke-CoverageSession { [CmdletBinding()] @@ -475,95 +641,107 @@ function Invoke-CoverageSession { [Parameter(Mandatory = $true)] [string] $Name, [Parameter(Mandatory = $true)] [string] $Program, [string[]] $ArgumentList = @(), + [Parameter(Mandatory = $true)] [string] $SessionId, [Parameter(Mandatory = $true)] [string] $OutputDirectory, - [string] $SourceRoot = $script:RepositoryRoot, - [string] $ModuleFilter, [string] $WorkingDirectory, - [string] $OpenCppCoveragePath + [int] $CollectorReadySeconds = 5, + [string] $ToolPath ) - if (-not $OpenCppCoveragePath) { $OpenCppCoveragePath = Install-OpenCppCoverage } + if (-not $ToolPath) { $ToolPath = Find-CodeCoverageConsole } $null = New-Item -ItemType Directory -Path $OutputDirectory -Force - $binaryOut = Join-Path $OutputDirectory "$Name.cov" + $coverageOut = Join-Path $OutputDirectory "$Name.coverage" $coberturaOut = Join-Path $OutputDirectory "$Name.cobertura.xml" + Remove-Item -LiteralPath $coverageOut -Force -ErrorAction SilentlyContinue + Remove-Item -LiteralPath $coberturaOut -Force -ErrorAction SilentlyContinue - $occArgs = @( - '--sources', $SourceRoot, - '--export_type', "binary:$binaryOut", - '--export_type', "cobertura:$coberturaOut", - '--cover_children', - '--quiet' - ) - if ($ModuleFilter) { $occArgs += @('--modules', $ModuleFilter) } - if ($WorkingDirectory) { $occArgs += @('--working_dir', $WorkingDirectory) } - $occArgs += '--' - $occArgs += $Program - $occArgs += $ArgumentList + $collectorLog = Join-Path $OutputDirectory "$Name.collector.log" + $collector = Start-Process -FilePath $ToolPath -PassThru -NoNewWindow ` + -RedirectStandardOutput $collectorLog ` + -ArgumentList @( + 'collect', '--session-id', $SessionId, '--server-mode', + '--output', $coverageOut, '--output-format', 'coverage', '--nologo' + ) - Write-Verbose "OpenCppCoverage $($occArgs -join ' ')" - & $OpenCppCoveragePath @occArgs - $exit = $LASTEXITCODE + # Let the collector take ownership of the session before we run the suite. + Start-Sleep -Seconds $CollectorReadySeconds + + $exit = $null + try { + if ($WorkingDirectory) { Push-Location $WorkingDirectory } + & $Program @ArgumentList + $exit = $LASTEXITCODE + } + finally { + if ($WorkingDirectory) { Pop-Location } + & $ToolPath shutdown $SessionId --nologo 2>$null | Write-Verbose + } + + if ($collector -and -not $collector.HasExited) { + $null = $collector.WaitForExit(120000) + } + + if (Test-Path -LiteralPath $coverageOut) { + Convert-CoverageReport -InputPath $coverageOut -OutputPath $coberturaOut -Format cobertura -ToolPath $ToolPath | Out-Null + } + else { + Write-Warning "Collector produced no coverage file for suite '$Name'. See $collectorLog." + } [pscustomobject]@{ - Name = $Name - ExitCode = $exit - BinaryPath = $binaryOut + Name = $Name + ExitCode = $exit + CoveragePath = $coverageOut CoberturaPath = $coberturaOut } } <# .SYNOPSIS - Merges binary (.cov) exports natively with OpenCppCoverage, producing a - combined Cobertura XML and (optionally) an HTML report. This is the - authoritative merged report; the pure helpers produce the same numbers and - add the overlap breakdown. + Merges/converts one or more .coverage (or coverage XML) inputs into a single + report, using the tool's native `merge` command. + + .DESCRIPTION + `merge` unions per-line hit counts across inputs, so merging every per-suite + .coverage yields the de-duplicated aggregate. Pass a single input to simply + convert it to another format. Supported -Format values: cobertura, xml, + coverage. #> -function Merge-CoverageBinary { +function Convert-CoverageReport { [CmdletBinding()] + [OutputType([string])] param( - [Parameter(Mandatory = $true)] [string[]] $BinaryPath, - [Parameter(Mandatory = $true)] [string] $OutputDirectory, - [switch] $Html, - [string] $OpenCppCoveragePath + [Parameter(Mandatory = $true)] [string[]] $InputPath, + [Parameter(Mandatory = $true)] [string] $OutputPath, + [ValidateSet('cobertura', 'xml', 'coverage')] + [string] $Format = 'cobertura', + [string] $ToolPath ) - if (-not $OpenCppCoveragePath) { $OpenCppCoveragePath = Install-OpenCppCoverage } - $null = New-Item -ItemType Directory -Path $OutputDirectory -Force + if (-not $ToolPath) { $ToolPath = Find-CodeCoverageConsole } - $coberturaOut = Join-Path $OutputDirectory 'merged.cobertura.xml' - Remove-Item -LiteralPath $coberturaOut -Force -ErrorAction SilentlyContinue + $inputs = @($InputPath | Where-Object { $_ -and (Test-Path -LiteralPath $_) }) + if (-not $inputs) { throw 'No coverage inputs found to merge/convert.' } - $occArgs = @() - foreach ($bin in $BinaryPath) { - if (Test-Path $bin) { $occArgs += @('--input_coverage', $bin) } - } - if (-not $occArgs) { throw 'No binary coverage inputs found to merge.' } + $outDir = Split-Path -Parent $OutputPath + if ($outDir) { $null = New-Item -ItemType Directory -Path $outDir -Force } + Remove-Item -LiteralPath $OutputPath -Force -ErrorAction SilentlyContinue - $occArgs += @('--export_type', "cobertura:$coberturaOut") - if ($Html) { - $htmlOut = Join-Path $OutputDirectory 'html' - # OpenCppCoverage refuses to write into an existing HTML export - # directory, so clear any report from a previous run first. - Remove-Item -LiteralPath $htmlOut -Recurse -Force -ErrorAction SilentlyContinue - $occArgs += @('--export_type', "html:$htmlOut") - } - $occArgs += '--quiet' - - Write-Verbose "OpenCppCoverage $($occArgs -join ' ')" - & $OpenCppCoveragePath @occArgs | Write-Verbose + $mergeArgs = @('merge') + $inputs + @('--output', $OutputPath, '--output-format', $Format, '--nologo') + Write-Verbose "Microsoft.CodeCoverage.Console $($mergeArgs -join ' ')" + & $ToolPath @mergeArgs | Write-Verbose if ($LASTEXITCODE -ne 0) { - throw "OpenCppCoverage merge failed with exit code $LASTEXITCODE." + throw "Coverage merge/convert failed with exit code $LASTEXITCODE." } - if (-not (Test-Path -LiteralPath $coberturaOut)) { - throw "OpenCppCoverage merge did not produce $coberturaOut." + if (-not (Test-Path -LiteralPath $OutputPath)) { + throw "Coverage merge/convert did not produce '$OutputPath'." } - return $coberturaOut + return $OutputPath } -#endregion OpenCppCoverage orchestration +#endregion Microsoft.CodeCoverage.Console orchestration Export-ModuleMember -Function @( 'ConvertTo-NormalizedCoverageSourcePath', @@ -574,7 +752,11 @@ Export-ModuleMember -Function @( 'New-MergedCoberturaReport', 'Get-CoverageSummary', 'Format-CoverageSummaryMarkdown', - 'Install-OpenCppCoverage', + 'Find-CodeCoverageConsole', + 'Get-OpenSSHCoverageTarget', + 'Copy-CoverageRuntime', + 'Invoke-CoverageInstrument', + 'Invoke-CoverageUninstrument', 'Invoke-CoverageSession', - 'Merge-CoverageBinary' + 'Convert-CoverageReport' ) diff --git a/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.tests.ps1 b/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.tests.ps1 index 3dc02123fb9e..dbe733d0857c 100644 --- a/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.tests.ps1 +++ b/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.tests.ps1 @@ -4,7 +4,7 @@ # # Unit tests for the pure helper functions in OpenSSHCodeCoverage.psm1. # These validate the aggregation / overlap logic without needing a build, -# OpenCppCoverage, or any of the OpenSSH test suites to run. +# Microsoft.CodeCoverage.Console, or any of the OpenSSH test suites to run. # # Run with Pester 5: Invoke-Pester -Path .\OpenSSHCodeCoverage.tests.ps1 # diff --git a/contrib/win32/openssh/code_coverage/README.md b/contrib/win32/openssh/code_coverage/README.md index def364436d96..5b7cef30edc9 100644 --- a/contrib/win32/openssh/code_coverage/README.md +++ b/contrib/win32/openssh/code_coverage/README.md @@ -9,32 +9,40 @@ de-duplicated number that accounts for overlap between the suites. | File | Purpose | |------|---------| -| `OpenSSHCodeCoverage.psm1` | Module. Pure aggregation/overlap helpers **and** OpenCppCoverage orchestration. | +| `OpenSSHCodeCoverage.psm1` | Module. Pure aggregation/overlap helpers **and** Microsoft.CodeCoverage.Console orchestration. | | `OpenSSHCodeCoverage.tests.ps1` | Pester 5 tests for the pure helpers (no build required). | | `Invoke-OpenSSHCodeCoverage.ps1` | Local end-to-end driver: build → run suites under coverage → merge → summarize. | | `Invoke-AzDOCodeCoverage.ps1` | CI entry point: run one suite (Core/Bash) under coverage against an installed OpenSSH dir, then merge. | All files live in `contrib\win32\openssh\code_coverage\`. -## Why OpenCppCoverage +## Why Microsoft.CodeCoverage.Console -[OpenCppCoverage](https://github.com/OpenCppCoverage/OpenCppCoverage) is the -standard open-source, MSVC-compatible C/C++ coverage tool for Windows. It: +[Microsoft.CodeCoverage.Console](https://learn.microsoft.com/visualstudio/test/microsoft-code-coverage-console-tool) +is the Microsoft-maintained coverage tool that ships with Visual Studio 2022 +(17.3+). Native C/C++ coverage requires the **Enterprise** edition (the hosted +`windows-latest` Azure DevOps image includes it). It: -- reads the debug **PDBs** to map executed instructions back to source lines - (no special build flags or instrumentation needed — just a Debug build), -- with `--cover_children`, attaches to a launched process **and every child it - spawns**, so monitoring the test harness captures every `ssh.exe`, - `sshd.exe`, `sftp.exe`, `unittest-*.exe`, etc. the tests launch, -- exports **binary** (`.cov`, re-mergeable), **Cobertura XML**, and **HTML**. +- collects **native C/C++** line coverage via **static instrumentation** — the + binaries are built with the `/PROFILE` linker switch and rewritten on disk by + the `instrument` command, +- supports **server mode** (`collect --session-id --server-mode`): every + instrumented process that runs while the collector owns the session reports + in by session id — including `sshd.exe` running as a Windows **service**, + which is *not* a child of the collector, +- emits `.coverage` (re-mergeable) and, via `merge`, **Cobertura XML**. + +Server mode is why coverage captures the E2E/bash suites: those drive a real +sshd service plus `ssh.exe`/`sftp.exe` clients, none of which are children of a +single wrapped process. ## How aggregation and overlap work -Each suite is measured independently and produces its own `.cov` + Cobertura -report. The `.cov` files are then merged natively by OpenCppCoverage. Merging -**unions per-line hit counts**, so a line exercised by two suites is counted -exactly once in the combined total — that is the de-duplication that makes the -aggregate honest. +Each suite is measured independently and produces its own `.coverage` + +Cobertura report. The `.coverage` files are then merged natively +(`merge ... -f cobertura`). Merging **unions per-line hit counts**, so a line +exercised by two suites is counted exactly once in the combined total — that is +the de-duplication that makes the aggregate honest. The pure helpers additionally quantify the redundancy: @@ -46,14 +54,17 @@ OverlapLines = SumCoveredLines - CombinedCoveredLines ## Usage -Requires: Visual Studio build tools (to build the solution), Chocolatey (to -auto-install OpenCppCoverage), and the test-suite prerequisites (Cygwin for the -bash suite, Pester for E2E — the existing helpers install these). +Requires: Visual Studio 2022 **Enterprise** (17.3+) — provides both the build +tools and `Microsoft.CodeCoverage.Console.exe` for native coverage — and the +test-suite prerequisites (Cygwin for the bash suite, Pester for E2E — the +existing helpers install these). The local driver builds `Debug` with +`/PROFILE` automatically (via the linker `LINK` env var) so the binaries can be +instrumented. ```powershell cd contrib\win32\openssh\code_coverage -# All suites, Debug build (recommended for accurate PDBs): +# All suites, Debug build (recommended for accurate line mapping): .\Invoke-OpenSSHCodeCoverage.ps1 -Configuration Debug # A single suite against an already-built tree: @@ -66,13 +77,12 @@ cd contrib\win32\openssh\code_coverage ### Output artifacts (under `-OutputDirectory`, default `.\coverage`) ``` -unit\unit.cov, unit\unit.cobertura.xml per-suite (unit tests) -e2e\e2e.cov, e2e\e2e.cobertura.xml per-suite (Pester E2E) -bash\bash.cov, bash\bash.cobertura.xml per-suite (bash tests) -merged\merged.cobertura.xml combined, de-duplicated (native merge) -merged\html\ browsable HTML report -coverage-summary.json machine-readable summary -coverage-summary.md per-suite + combined + overlap table +unit\unit.coverage, unit\unit.cobertura.xml per-suite (unit tests) +e2e\e2e.coverage, e2e\e2e.cobertura.xml per-suite (Pester E2E) +bash\bash.coverage, bash\bash.cobertura.xml per-suite (bash tests) +merged\merged.cobertura.xml combined, de-duplicated (native merge) +coverage-summary.json machine-readable summary +coverage-summary.md per-suite + combined + overlap table ``` Example `coverage-summary.md`: @@ -94,7 +104,7 @@ Example `coverage-summary.md`: ## Validating the helpers The aggregation/overlap logic is unit tested and does **not** require a build, -OpenCppCoverage, or the OpenSSH suites: +Microsoft.CodeCoverage.Console, or the OpenSSH suites: ```powershell Invoke-Pester -Path .\OpenSSHCodeCoverage.tests.ps1 -Output Detailed @@ -102,27 +112,33 @@ Invoke-Pester -Path .\OpenSSHCodeCoverage.tests.ps1 -Output Detailed ## Continuous integration (Azure DevOps) -`.azdo/ci.yml` runs a non-gating **Win32-OpenSSH Code Coverage** job in the -Test stage, in parallel with the existing test jobs. It: - -1. downloads the build artifacts (which include `.pdb` symbols) and unit tests, - and installs OpenSSH to `C:\OpenSSH`, -2. runs `Invoke-AzDOCodeCoverage.ps1 -Suite Core` (setup + unit + E2E) and then - `-Suite Bash`, each under OpenCppCoverage, -3. merges the per-suite `.cov` files, then publishes the merged Cobertura report - via `PublishCodeCoverageResults@2` and uploads the full - `Win32-OpenSSH-CodeCoverage` artifact (per-suite reports, HTML, summaries). - -The job is marked `continueOnError: true` so coverage never blocks a merge. The -suites run the exact CI entry points (`Invoke-OpenSSHTests`, +Coverage is wired into `.azdo/ci.yml` as two dedicated, **PR-only, non-gating** +jobs (they never run on branch builds, where CodeQL already competes for the +~60 min agent budget, and `continueOnError: true` so they never block a merge): + +1. **Build Coverage Package (x64 Debug + /PROFILE)** — a Build-stage job that + builds the solution `Debug|x64` with `/PROFILE` injected via the linker + `LINK` env var (no `.vcxproj` edits), publishing `Win32-OpenSSH-Coverage-x64` + and `UnitTests-Coverage-x64` (binaries + PDBs). It runs in parallel with the + normal Release build, so it does not eat into the coverage job's budget. +2. **Win32-OpenSSH Code Coverage** — a Test-stage job that installs the coverage + build to `C:\OpenSSH`, then runs `Invoke-AzDOCodeCoverage.ps1 -Suite Core` + (setup + unit + E2E) and `-Suite Bash`. Each invocation instruments the + installed binaries, runs the suite under a server-mode collector, converts to + Cobertura, and aggregates every per-suite report. It publishes the merged + Cobertura via `PublishCodeCoverageResults@2` and uploads the full + `Win32-OpenSSH-CodeCoverage` artifact (per-suite reports + summaries). + +The suites run the exact CI entry points (`Invoke-OpenSSHTests`, `Invoke-OpenSSHBashTestsOnly`), so coverage reflects what CI already exercises. ## Notes -- Use a **Debug** configuration locally for the most faithful line mapping. - Release with full optimizations can fold/reorder lines and understate - coverage; CI measures the Release artifacts it already produces. -- Coverage is scoped to repository sources via `--sources ` and to the - built binaries via `--modules `, so third-party/system code is excluded. +- A **Debug** build is used for coverage so line mapping is accurate; optimized + Release builds can fold/reorder/strip lines and understate coverage. +- Native C/C++ coverage requires **Visual Studio 2022 Enterprise** and binaries + linked with `/PROFILE`; the coverage build job and local driver handle both. +- Coverage is scoped to OpenSSH source files at report time (paths are + normalized to the repository root), so third-party/system code is excluded. - The tooling reuses the existing suite entry points, so it measures exactly what CI already runs. From cc0648582c471edd0a3a50eb0a35145295330b4a Mon Sep 17 00:00:00 2001 From: Tess Gauthier Date: Wed, 26 Aug 2026 15:56:55 -0400 Subject: [PATCH 3/8] Remove local coverage driver; coverage runs only in AzDO pipeline Drop Invoke-OpenSSHCodeCoverage.ps1 (the build+run+merge local driver); coverage is driven exclusively by Invoke-AzDOCodeCoverage.ps1 in CI. Update README to document local reproduction via the AzDO entry point. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Invoke-OpenSSHCodeCoverage.ps1 | 203 ------------------ contrib/win32/openssh/code_coverage/README.md | 42 ++-- 2 files changed, 19 insertions(+), 226 deletions(-) delete mode 100644 contrib/win32/openssh/code_coverage/Invoke-OpenSSHCodeCoverage.ps1 diff --git a/contrib/win32/openssh/code_coverage/Invoke-OpenSSHCodeCoverage.ps1 b/contrib/win32/openssh/code_coverage/Invoke-OpenSSHCodeCoverage.ps1 deleted file mode 100644 index ec62879ffbaf..000000000000 --- a/contrib/win32/openssh/code_coverage/Invoke-OpenSSHCodeCoverage.ps1 +++ /dev/null @@ -1,203 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. - -<# - .SYNOPSIS - Generates a C code coverage estimate for the Win32-OpenSSH solution - (contrib\win32\openssh\Win32-OpenSSH.sln) across the unit, Pester (E2E) and - bash test suites, then aggregates them into a single de-duplicated report. - - .DESCRIPTION - The OpenSSH binaries in bin\\ (built with the /PROFILE - linker switch so they can be statically instrumented) are instrumented once - with Microsoft.CodeCoverage.Console. Each requested suite then runs while a - background server-mode collector owns a shared session id, so every - instrumented OpenSSH process that executes - ssh.exe, sshd.exe (including as - a service), sftp.exe, the unittest-*.exe binaries, ... - is captured - regardless of parent. Each suite produces: - - \\.coverage (binary, mergeable) - \\.cobertura.xml (per-suite report) - - The per-suite .coverage files are merged natively into - \merged\merged.cobertura.xml. Because merging unions - per-line hits, a line exercised by two suites is counted once - that is the - aggregation-with-overlap behaviour requested. - - In parallel, the pure helpers in OpenSSHCodeCoverage.psm1 re-derive the same - numbers from the per-suite Cobertura reports and additionally quantify how - much the suites overlap. The final artifacts are: - - \coverage-summary.json - \coverage-summary.md - - A Debug build is recommended so line mapping is accurate (unoptimized code). - Native C/C++ coverage requires Visual Studio 2022 Enterprise (17.3+). - - .PARAMETER NativeHostArch - Architecture whose bin\ folder holds the built binaries (x64, x86, arm64, arm). - - .PARAMETER Configuration - Build configuration to measure. Debug is recommended for accurate coverage. - - .PARAMETER Suite - Which suites to measure. Defaults to all three. - - .PARAMETER OutputDirectory - Where coverage artifacts are written. - - .PARAMETER SkipBuild - Assume the solution is already built; do not invoke MSBuild. - - .EXAMPLE - .\Invoke-OpenSSHCodeCoverage.ps1 -Configuration Debug -Suite Unit - - .EXAMPLE - .\Invoke-OpenSSHCodeCoverage.ps1 -Configuration Debug -OutputDirectory C:\cov -#> -[CmdletBinding()] -param( - [ValidateSet('x86', 'x64', 'arm64', 'arm')] - [string] $NativeHostArch = 'x64', - - [ValidateSet('Debug', 'Release')] - [string] $Configuration = 'Debug', - - [ValidateSet('Unit', 'E2E', 'Bash')] - [string[]] $Suite = @('Unit', 'E2E', 'Bash'), - - [string] $OutputDirectory, - - [switch] $SkipBuild -) - -$ErrorActionPreference = 'Stop' - -$repositoryRoot = (Resolve-Path (Join-Path $PSScriptRoot '..' '..' '..' '..')).Path -$opensshDir = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path -Import-Module (Join-Path $PSScriptRoot 'OpenSSHCodeCoverage.psm1') -Force -Import-Module (Join-Path $opensshDir 'OpenSSHBuildHelper.psm1') -Force -ErrorAction SilentlyContinue -Import-Module (Join-Path $opensshDir 'OpenSSHTestHelper.psm1') -Force -ErrorAction SilentlyContinue - -if (-not $OutputDirectory) { - $OutputDirectory = Join-Path $repositoryRoot 'coverage' -} -$null = New-Item -ItemType Directory -Path $OutputDirectory -Force - -$folderName = if ($NativeHostArch -ieq 'x86') { 'Win32' } else { $NativeHostArch } -$binPath = Join-Path $repositoryRoot "bin\$folderName\$Configuration" - -Write-Host "Repository root : $repositoryRoot" -Write-Host "Binaries : $binPath" -Write-Host "Output : $OutputDirectory" -Write-Host "Suites : $($Suite -join ', ')" - -# --- Build ----------------------------------------------------------------- -if (-not $SkipBuild) { - if (-not (Get-Command 'Start-OpenSSHBuild' -ErrorAction SilentlyContinue)) { - throw 'Start-OpenSSHBuild is unavailable; import OpenSSHBuildHelper.psm1 or pass -SkipBuild.' - } - Write-Host "Building Win32-OpenSSH ($NativeHostArch/$Configuration) with /PROFILE..." - # /PROFILE (honored via the linker's LINK env var) emits the fixups + full - # PDBs that static instrumentation needs, without editing any .vcxproj. - $previousLink = $env:LINK - $env:LINK = ((@($env:LINK, '/PROFILE') | Where-Object { $_ }) -join ' ') - try { - Start-OpenSSHBuild -NativeHostArch $NativeHostArch -Configuration $Configuration - } - finally { - $env:LINK = $previousLink - } -} - -if (-not (Test-Path $binPath)) { - throw "Binaries not found at $binPath. Build first or correct -NativeHostArch/-Configuration." -} - -$tool = Find-CodeCoverageConsole -Write-Host "CodeCoverage.Console : $tool" - -# Instrument the built binaries once; every suite reuses the same session id. -$targets = Get-OpenSSHCoverageTarget -BinaryDirectory $binPath -if (-not $targets) { - throw "No OpenSSH binaries to instrument were found under '$binPath'." -} -$sessionId = [guid]::NewGuid().ToString() -Write-Host "Instrumenting $($targets.Count) binaries (session $sessionId)..." -Invoke-CoverageInstrument -BinaryPath $targets -SessionId $sessionId -ToolPath $tool | Out-Null - -$pwsh = (Get-Process -Id $PID).Path # path to the current PowerShell host -$sessions = @() - -try { - # --- Unit tests -------------------------------------------------------- - if ($Suite -contains 'Unit') { - Write-Host "`n=== Unit tests ===" - $cmd = "Import-Module '$opensshDir\OpenSSHTestHelper.psm1' -Force; Invoke-OpenSSHUnitTest -UnitTestDirectory '$binPath'" - $sessions += Invoke-CoverageSession -Name 'unit' ` - -Program $pwsh -ArgumentList @('-NoProfile', '-NonInteractive', '-Command', $cmd) ` - -SessionId $sessionId ` - -OutputDirectory (Join-Path $OutputDirectory 'unit') -ToolPath $tool - } - - # --- Pester E2E tests -------------------------------------------------- - if ($Suite -contains 'E2E') { - Write-Host "`n=== Pester E2E tests ===" - $cmd = "Import-Module '$opensshDir\OpenSSHTestHelper.psm1' -Force; Set-OpenSSHTestEnvironment -OpenSSHBinPath '$binPath'; Invoke-OpenSSHE2ETest" - $sessions += Invoke-CoverageSession -Name 'e2e' ` - -Program $pwsh -ArgumentList @('-NoProfile', '-NonInteractive', '-Command', $cmd) ` - -SessionId $sessionId ` - -OutputDirectory (Join-Path $OutputDirectory 'e2e') -ToolPath $tool - } - - # --- Bash tests -------------------------------------------------------- - if ($Suite -contains 'Bash') { - Write-Host "`n=== Bash tests ===" - $cmd = "Import-Module '$opensshDir\OpenSSHTestHelper.psm1' -Force; Set-OpenSSHTestEnvironment -OpenSSHBinPath '$binPath'; Invoke-OpenSSHBashTests" - $sessions += Invoke-CoverageSession -Name 'bash' ` - -Program $pwsh -ArgumentList @('-NoProfile', '-NonInteractive', '-Command', $cmd) ` - -SessionId $sessionId ` - -OutputDirectory (Join-Path $OutputDirectory 'bash') -ToolPath $tool - } -} -finally { - # Restore the original binaries. - Invoke-CoverageUninstrument -BinaryPath $targets -ToolPath $tool -} - -if (-not $sessions) { - throw 'No suites were run; nothing to report.' -} - -# --- Native merge (authoritative combined report) -------------------------- -Write-Host "`n=== Merging coverage ===" -$coverageFiles = $sessions | ForEach-Object { $_.CoveragePath } | Where-Object { $_ -and (Test-Path $_) } -$mergedCobertura = Convert-CoverageReport -InputPath $coverageFiles ` - -OutputPath (Join-Path $OutputDirectory 'merged\merged.cobertura.xml') -Format cobertura -ToolPath $tool -Write-Host "Merged report : $mergedCobertura" - -# --- Helper aggregation + overlap ------------------------------------------ -$suiteMaps = @() -$suiteStats = @() -foreach ($session in $sessions) { - if (-not (Test-Path $session.CoberturaPath)) { - Write-Warning "No Cobertura report for suite '$($session.Name)'; skipping." - continue - } - $map = Import-CoberturaCoverage -Path $session.CoberturaPath -RepositoryRoot $repositoryRoot - $suiteMaps += , $map - $suiteStats += Get-CoverageStatistic -CoverageMap $map -Name $session.Name -} - -$mergedMap = Merge-CoverageData -CoverageMap $suiteMaps -$combinedStat = Get-CoverageStatistic -CoverageMap $mergedMap -Name 'combined' -$overlap = Measure-CoverageOverlap -CoverageMap $suiteMaps - -$summary = Get-CoverageSummary -SuiteStatistic $suiteStats -CombinedStatistic $combinedStat -Overlap $overlap -$summary | ConvertTo-Json -Depth 6 | Set-Content -Path (Join-Path $OutputDirectory 'coverage-summary.json') - -$markdown = Format-CoverageSummaryMarkdown -Summary $summary -$markdown | Set-Content -Path (Join-Path $OutputDirectory 'coverage-summary.md') - -Write-Host "`n$markdown" -Write-Host "Artifacts written to $OutputDirectory" diff --git a/contrib/win32/openssh/code_coverage/README.md b/contrib/win32/openssh/code_coverage/README.md index 5b7cef30edc9..bddeb29aa629 100644 --- a/contrib/win32/openssh/code_coverage/README.md +++ b/contrib/win32/openssh/code_coverage/README.md @@ -11,7 +11,6 @@ de-duplicated number that accounts for overlap between the suites. |------|---------| | `OpenSSHCodeCoverage.psm1` | Module. Pure aggregation/overlap helpers **and** Microsoft.CodeCoverage.Console orchestration. | | `OpenSSHCodeCoverage.tests.ps1` | Pester 5 tests for the pure helpers (no build required). | -| `Invoke-OpenSSHCodeCoverage.ps1` | Local end-to-end driver: build → run suites under coverage → merge → summarize. | | `Invoke-AzDOCodeCoverage.ps1` | CI entry point: run one suite (Core/Bash) under coverage against an installed OpenSSH dir, then merge. | All files live in `contrib\win32\openssh\code_coverage\`. @@ -54,31 +53,29 @@ OverlapLines = SumCoveredLines - CombinedCoveredLines ## Usage -Requires: Visual Studio 2022 **Enterprise** (17.3+) — provides both the build -tools and `Microsoft.CodeCoverage.Console.exe` for native coverage — and the -test-suite prerequisites (Cygwin for the bash suite, Pester for E2E — the -existing helpers install these). The local driver builds `Debug` with -`/PROFILE` automatically (via the linker `LINK` env var) so the binaries can be -instrumented. +Coverage normally runs in CI (see below). To reproduce locally you need +Visual Studio 2022 **Enterprise** (17.3+) — for both the build tools and +`Microsoft.CodeCoverage.Console.exe` — plus the test-suite prerequisites +(Cygwin for the bash suite, Pester for E2E; the existing helpers install these). + +First build and install the solution with `/PROFILE` so the binaries can be +instrumented (the CI build job does this by injecting `/PROFILE` via the linker +`LINK` env var), then point `Invoke-AzDOCodeCoverage.ps1` at the installed +directory. Call it once per suite into the same `-OutputDirectory`; the final +call produces the aggregate report. ```powershell cd contrib\win32\openssh\code_coverage -# All suites, Debug build (recommended for accurate line mapping): -.\Invoke-OpenSSHCodeCoverage.ps1 -Configuration Debug - -# A single suite against an already-built tree: -.\Invoke-OpenSSHCodeCoverage.ps1 -Configuration Debug -Suite Unit -SkipBuild - -# Custom output location: -.\Invoke-OpenSSHCodeCoverage.ps1 -Configuration Debug -OutputDirectory C:\cov +# Core = setup + unit + E2E, then the bash suite, into one output dir: +.\Invoke-AzDOCodeCoverage.ps1 -Suite Core -OpenSSHBinPath C:\OpenSSH -OutputDirectory C:\cov +.\Invoke-AzDOCodeCoverage.ps1 -Suite Bash -OpenSSHBinPath C:\OpenSSH -OutputDirectory C:\cov ``` -### Output artifacts (under `-OutputDirectory`, default `.\coverage`) +### Output artifacts (under `-OutputDirectory`) ``` -unit\unit.coverage, unit\unit.cobertura.xml per-suite (unit tests) -e2e\e2e.coverage, e2e\e2e.cobertura.xml per-suite (Pester E2E) +core\core.coverage, core\core.cobertura.xml per-suite (setup + unit + E2E) bash\bash.coverage, bash\bash.cobertura.xml per-suite (bash tests) merged\merged.cobertura.xml combined, de-duplicated (native merge) coverage-summary.json machine-readable summary @@ -90,15 +87,14 @@ Example `coverage-summary.md`: ``` | Suite | Covered | Total | Line % | |-------|--------:|------:|-------:| -| unit | 4210 | 20144 | 20.9% | -| e2e | 9633 | 20144 | 47.8% | +| core | 9633 | 20144 | 47.8% | | bash | 11002 | 20144 | 54.6% | | **Combined (deduped)** | **13120** | **20144** | **65.1%** | ## Overlap between suites -- Sum of per-suite covered lines: 24845 +- Sum of per-suite covered lines: 20635 - Combined (de-duplicated) covered lines: 13120 -- Overlapping covered lines: 11725 (47.19% of the sum) +- Overlapping covered lines: 7515 (36.42% of the sum) ``` ## Validating the helpers @@ -137,7 +133,7 @@ The suites run the exact CI entry points (`Invoke-OpenSSHTests`, - A **Debug** build is used for coverage so line mapping is accurate; optimized Release builds can fold/reorder/strip lines and understate coverage. - Native C/C++ coverage requires **Visual Studio 2022 Enterprise** and binaries - linked with `/PROFILE`; the coverage build job and local driver handle both. + linked with `/PROFILE`; the coverage build job handles both. - Coverage is scoped to OpenSSH source files at report time (paths are normalized to the repository root), so third-party/system code is excluded. - The tooling reuses the existing suite entry points, so it measures exactly what From 674f85badbbe3a26bad69e67be2ee80f578952a8 Mon Sep 17 00:00:00 2001 From: Tess Gauthier Date: Thu, 27 Aug 2026 10:48:08 -0400 Subject: [PATCH 4/8] Fix bash-suite coverage: locate covrun runtime and re-install services Two fixes for the bash coverage step: - Copy-CoverageRuntime searched only the CodeCoverage.Console tool dir, but static_covrun*.dll ships under '\Team Tools\Dynamic Code Coverage Tools'. Walk up to the VS install root and search there so the runtime is copied beside the instrumented binaries; the sshd/ssh-agent services (no inherited PATH) can then load and record coverage. - The core suite ends with the uninstall test, which removes the sshd and ssh-agent services and the install dir. Re-install the coverage build before the bash coverage step so Invoke-OpenSSHBashTestsOnly finds the services, mirroring the standalone bash test job. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .azdo/ci.yml | 9 ++++ .../code_coverage/OpenSSHCodeCoverage.psm1 | 48 +++++++++++++++++-- 2 files changed, 52 insertions(+), 5 deletions(-) diff --git a/.azdo/ci.yml b/.azdo/ci.yml index dd9a09b50a9c..95c3ad7ef226 100644 --- a/.azdo/ci.yml +++ b/.azdo/ci.yml @@ -241,6 +241,15 @@ stages: -OutputDirectory "$(coverageDir)" -Verbose displayName: Collect coverage - core tests (setup + unit + E2E) + - pwsh: | + Import-Module -Name "$(Build.SourcesDirectory)/contrib/win32/openssh/AzDOBuildTools" -Force + # The core flow ends with the uninstall test, which removes the sshd and + # ssh-agent services and the install dir. Re-install the coverage build so + # the bash suite (which expects those services) has a clean environment, + # mirroring the standalone bash test job. + Install-OpenSSH -SourceDir "$(System.ArtifactsDirectory)/Win32-OpenSSH-Coverage-x64/x64/Debug" -OpenSSHDir "$env:SystemDrive/OpenSSH" -Verbose + displayName: Re-install Win32-OpenSSH for bash tests + - pwsh: | & "$(Build.SourcesDirectory)/contrib/win32/openssh/code_coverage/Invoke-AzDOCodeCoverage.ps1" ` -Suite Bash ` diff --git a/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.psm1 b/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.psm1 index fd3536e7e093..27dae949f40c 100644 --- a/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.psm1 +++ b/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.psm1 @@ -536,8 +536,14 @@ function Get-OpenSSHCoverageTarget { .DESCRIPTION An instrumented native binary references static_covrun64.dll at load time. The `collect`/`connect` commands add its directory to PATH automatically, but - the sshd *service* is launched by the Service Control Manager and does not - inherit that PATH, so the runtime must sit beside the instrumented binaries. + the sshd/ssh-agent *services* are launched by the Service Control Manager and + do not inherit that PATH, so the runtime must sit beside the instrumented + binaries or the services fail to start. + + The runtime does not live next to Microsoft.CodeCoverage.Console.exe; it ships + under "\Team Tools\Dynamic Code Coverage Tools\" (with a + per-architecture subfolder), so the search starts from the VS install root + inferred from the tool path. #> function Copy-CoverageRuntime { [CmdletBinding()] @@ -547,12 +553,44 @@ function Copy-CoverageRuntime { ) $toolDir = Split-Path -Parent $ToolPath - $runtimes = Get-ChildItem -Path $toolDir -Recurse -Filter 'static_covrun*.dll' -ErrorAction SilentlyContinue + + # The tool lives at \Common7\IDE\Extensions\Microsoft\CodeCoverage.Console; + # walk up to the VS install root (the parent of the "Common7" segment) so the + # sibling "Team Tools\Dynamic Code Coverage Tools" folder is in scope. + $installRoot = $null + $probe = $toolDir + while ($probe) { + if ((Split-Path -Leaf $probe) -eq 'Common7') { + $installRoot = Split-Path -Parent $probe + break + } + $parent = Split-Path -Parent $probe + if (-not $parent -or $parent -eq $probe) { break } + $probe = $parent + } + + # Prefer the known runtime folder, then fall back to broader recursive scans. + $runtimes = $null + if ($installRoot) { + $dynamicTools = Join-Path $installRoot 'Team Tools\Dynamic Code Coverage Tools' + if (Test-Path -LiteralPath $dynamicTools) { + $runtimes = Get-ChildItem -Path $dynamicTools -Recurse -Filter 'static_covrun*.dll' -ErrorAction SilentlyContinue + } + } if (-not $runtimes) { - Write-Warning "No static_covrun*.dll found under '$toolDir'; instrumented service binaries may fail to load." + $runtimes = Get-ChildItem -Path $toolDir -Recurse -Filter 'static_covrun*.dll' -ErrorAction SilentlyContinue + } + if (-not $runtimes -and $installRoot) { + $runtimes = Get-ChildItem -Path $installRoot -Recurse -Filter 'static_covrun*.dll' -ErrorAction SilentlyContinue + } + + if (-not $runtimes) { + Write-Warning "No static_covrun*.dll found near '$ToolPath'; instrumented service binaries may fail to load." return } - foreach ($rt in $runtimes) { + + # Copy one DLL per file name (the arch variants have distinct names). + foreach ($rt in ($runtimes | Group-Object Name | ForEach-Object { $_.Group[0] })) { Copy-Item -LiteralPath $rt.FullName -Destination $DestinationDirectory -Force } } From 6b5fdf496e21bec773d36d3d843d47598572f4d3 Mon Sep 17 00:00:00 2001 From: Tess Gauthier Date: Thu, 27 Aug 2026 11:01:04 -0400 Subject: [PATCH 5/8] Address PR review: path-prefix boundary and collector leak - ConvertTo-NormalizedCoverageSourcePath stripped the repository root with a bare StartsWith, which could wrongly strip a sibling that merely shares a prefix (root 'C:/repo' vs path 'C:/repository/...'). Require a directory boundary (root followed by '/' or end-of-string). Adds a regression test. - Invoke-CoverageSession now terminates the server-mode collector (best-effort, with a warning) if it does not exit within 120s of shutdown, so a hung collector is not leaked into subsequent CI steps. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../openssh/code_coverage/OpenSSHCodeCoverage.psm1 | 14 ++++++++++++-- .../code_coverage/OpenSSHCodeCoverage.tests.ps1 | 5 +++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.psm1 b/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.psm1 index 27dae949f40c..3cad74c8b12e 100644 --- a/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.psm1 +++ b/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.psm1 @@ -83,7 +83,12 @@ function ConvertTo-NormalizedCoverageSourcePath { # Strip a provided repository root if the path lives underneath it. if (-not [string]::IsNullOrWhiteSpace($RepositoryRoot)) { $root = ($RepositoryRoot -replace '\\', '/').TrimEnd('/') - if ($root -and $normalized.StartsWith($root, [System.StringComparison]::OrdinalIgnoreCase)) { + # Only strip when $root is an actual directory-boundary prefix (followed + # by '/' or the whole string), so a sibling like 'C:/repository' is not + # mistaken for a path under root 'C:/repo'. + if ($root -and $normalized.Length -ge $root.Length -and + $normalized.Substring(0, $root.Length).Equals($root, [System.StringComparison]::OrdinalIgnoreCase) -and + ($normalized.Length -eq $root.Length -or $normalized[$root.Length] -eq '/')) { $normalized = $normalized.Substring($root.Length) } } @@ -717,7 +722,12 @@ function Invoke-CoverageSession { } if ($collector -and -not $collector.HasExited) { - $null = $collector.WaitForExit(120000) + # shutdown should stop the collector; if it does not exit in time, kill it + # (best-effort) so we do not leak a collector into subsequent CI steps. + if (-not $collector.WaitForExit(120000)) { + Write-Warning "Coverage collector for suite '$Name' did not exit within 120s of shutdown; terminating it." + try { $collector.Kill() } catch { Write-Warning "Failed to terminate collector for suite '$Name': $($_.Exception.Message)" } + } } if (Test-Path -LiteralPath $coverageOut) { diff --git a/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.tests.ps1 b/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.tests.ps1 index dbe733d0857c..61c0a2d8903b 100644 --- a/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.tests.ps1 +++ b/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.tests.ps1 @@ -52,6 +52,11 @@ Describe 'ConvertTo-NormalizedCoverageSourcePath' { $result = ConvertTo-NormalizedCoverageSourcePath -RawPath 'C:\SRC\OpenSSH-Portable\channels.c' -RepositoryRoot 'C:\src\openssh-portable' $result | Should -Be 'channels.c' } + + It 'Does not strip a sibling directory that merely shares a prefix' { + $result = ConvertTo-NormalizedCoverageSourcePath -RawPath 'C:\repository\file.c' -RepositoryRoot 'C:\repo' + $result | Should -Be 'C:/repository/file.c' + } } Context 'CI checkout prefixes' { From b69f6c7d13ff8db4249066622c3d49602635e0c1 Mon Sep 17 00:00:00 2001 From: Tess Gauthier Date: Thu, 27 Aug 2026 11:55:43 -0400 Subject: [PATCH 6/8] Split coverage into separate Core/Bash jobs with dependent aggregate Running Core and Bash coverage sequentially in one job shared a single C:\OpenSSH install and agent, causing two failures: the core flow's uninstall test removed the sshd/ssh-agent services and left the collector/instrumented processes holding libcrypto.dll, so the mid-job re-install hit a file lock, and the bash suite ran without services. Restructure to mirror the normal pipeline: separate PR-only, non-gating CodeCoverageCore and CodeCoverageBash jobs each run on their own agent with a clean install (no lock/state contention, and they run in parallel), publishing per-suite artifacts. A dependent CodeCoverageAggregate job merges them. - Extract the aggregation/merge/summary logic into an exported module function Invoke-CoverageAggregation. - Add -SkipAggregation to Invoke-AzDOCodeCoverage.ps1 (per-suite jobs) and a new Invoke-AzDOCoverageAggregate.ps1 for the dependent job. - Rewrite the ci.yml coverage jobs and update the README. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .azdo/ci.yml | 140 ++++++++++++++---- .../code_coverage/Invoke-AzDOCodeCoverage.ps1 | 89 +++-------- .../Invoke-AzDOCoverageAggregate.ps1 | 67 +++++++++ .../code_coverage/OpenSSHCodeCoverage.psm1 | 103 ++++++++++++- contrib/win32/openssh/code_coverage/README.md | 52 ++++--- 5 files changed, 335 insertions(+), 116 deletions(-) create mode 100644 contrib/win32/openssh/code_coverage/Invoke-AzDOCoverageAggregate.ps1 diff --git a/.azdo/ci.yml b/.azdo/ci.yml index 95c3ad7ef226..8aa002dd15b7 100644 --- a/.azdo/ci.yml +++ b/.azdo/ci.yml @@ -184,10 +184,10 @@ stages: displayName: Clean up OpenSSH test environment condition: always() - - job: CodeCoverageWin32OpenSSH + - job: CodeCoverageCore pool: vmImage: windows-latest - displayName: Win32-OpenSSH Code Coverage + displayName: Win32-OpenSSH Code Coverage (Core) # Coverage is informational and must never gate a merge while it stabilizes. continueOnError: true # Only run on pull requests. On branch builds CodeQL is enabled and the added @@ -212,6 +212,54 @@ stages: artifactName: UnitTests-Coverage-x64 downloadPath: '$(System.ArtifactsDirectory)' + - pwsh: | + Import-Module -Name "$(Build.SourcesDirectory)/contrib/win32/openssh/AzDOBuildTools" -Force + # Install the Debug (/PROFILE) build so Microsoft.CodeCoverage.Console can + # statically instrument the OpenSSH binaries and unit tests. + Install-OpenSSH -SourceDir "$(System.ArtifactsDirectory)/Win32-OpenSSH-Coverage-x64/x64/Debug" -OpenSSHDir "$env:SystemDrive/OpenSSH" -Verbose + Install-UnitTests -SourceDir "$(System.ArtifactsDirectory)/UnitTests-Coverage-x64/x64/Debug" -OpenSSHDir "$env:SystemDrive/OpenSSH" -Verbose + displayName: Install Win32-OpenSSH and unit tests + + - pwsh: | + & "$(Build.SourcesDirectory)/contrib/win32/openssh/code_coverage/Invoke-AzDOCodeCoverage.ps1" ` + -Suite Core ` + -OpenSSHBinPath "$env:SystemDrive/OpenSSH" ` + -SourceRoot "$(Build.SourcesDirectory)" ` + -OutputDirectory "$(coverageDir)" -SkipAggregation -Verbose + displayName: Collect coverage - core tests (setup + unit + E2E) + + - pwsh: | + if (Test-Path "$(coverageDir)") + { + $artifactName = 'Win32-OpenSSH-CodeCoverage-Core' + Write-Host "##vso[artifact.upload containerfolder=$artifactName;artifactname=$artifactName;]$(coverageDir)" + } + displayName: Upload core coverage artifact + condition: always() + + - pwsh: | + Import-Module -Name "$(Build.SourcesDirectory)/contrib/win32/openssh/AzDOBuildTools" -Force + Clear-TestEnvironmentSetup + displayName: Clean up OpenSSH test environment + condition: always() + + - job: CodeCoverageBash + pool: + vmImage: windows-latest + displayName: Win32-OpenSSH Code Coverage (Bash) + continueOnError: true + condition: eq(variables['Build.Reason'], 'PullRequest') + variables: + coverageDir: '$(Build.SourcesDirectory)/CodeCoverage' + steps: + - task: DownloadBuildArtifacts@0 + displayName: 'Download coverage build artifacts (Debug + symbols)' + inputs: + buildType: current + downloadType: single + artifactName: Win32-OpenSSH-Coverage-x64 + downloadPath: '$(System.ArtifactsDirectory)' + - task: DownloadBuildArtifacts@0 displayName: 'Download bash test config file artifact' inputs: @@ -222,11 +270,10 @@ stages: - pwsh: | Import-Module -Name "$(Build.SourcesDirectory)/contrib/win32/openssh/AzDOBuildTools" -Force - # Install the Debug (/PROFILE) build so Microsoft.CodeCoverage.Console can - # statically instrument the OpenSSH binaries and unit tests. + # Install the Debug (/PROFILE) build so the bash suite runs against + # instrumentable binaries with the sshd/ssh-agent services registered. Install-OpenSSH -SourceDir "$(System.ArtifactsDirectory)/Win32-OpenSSH-Coverage-x64/x64/Debug" -OpenSSHDir "$env:SystemDrive/OpenSSH" -Verbose - Install-UnitTests -SourceDir "$(System.ArtifactsDirectory)/UnitTests-Coverage-x64/x64/Debug" -OpenSSHDir "$env:SystemDrive/OpenSSH" -Verbose - displayName: Install Win32-OpenSSH and unit tests + displayName: Install Win32-OpenSSH - pwsh: | $configFileSrc = "$(System.ArtifactsDirectory)/ConfigFile-x64/config.h" @@ -235,31 +282,68 @@ stages: - pwsh: | & "$(Build.SourcesDirectory)/contrib/win32/openssh/code_coverage/Invoke-AzDOCodeCoverage.ps1" ` - -Suite Core ` + -Suite Bash ` -OpenSSHBinPath "$env:SystemDrive/OpenSSH" ` -SourceRoot "$(Build.SourcesDirectory)" ` - -OutputDirectory "$(coverageDir)" -Verbose - displayName: Collect coverage - core tests (setup + unit + E2E) + -OutputDirectory "$(coverageDir)" -SkipAggregation -Verbose + displayName: Collect coverage - bash tests + + - pwsh: | + if (Test-Path "$(coverageDir)") + { + $artifactName = 'Win32-OpenSSH-CodeCoverage-Bash' + Write-Host "##vso[artifact.upload containerfolder=$artifactName;artifactname=$artifactName;]$(coverageDir)" + } + displayName: Upload bash coverage artifact + condition: always() - pwsh: | Import-Module -Name "$(Build.SourcesDirectory)/contrib/win32/openssh/AzDOBuildTools" -Force - # The core flow ends with the uninstall test, which removes the sshd and - # ssh-agent services and the install dir. Re-install the coverage build so - # the bash suite (which expects those services) has a clean environment, - # mirroring the standalone bash test job. - Install-OpenSSH -SourceDir "$(System.ArtifactsDirectory)/Win32-OpenSSH-Coverage-x64/x64/Debug" -OpenSSHDir "$env:SystemDrive/OpenSSH" -Verbose - displayName: Re-install Win32-OpenSSH for bash tests + Clear-TestEnvironmentSetup + displayName: Clean up OpenSSH test environment + condition: always() + + - job: CodeCoverageAggregate + pool: + vmImage: windows-latest + displayName: Win32-OpenSSH Code Coverage (Aggregate) + # Run after both suite jobs regardless of their outcome (they are non-gating); + # aggregate whichever per-suite artifacts were produced. + dependsOn: + - CodeCoverageCore + - CodeCoverageBash + continueOnError: true + condition: eq(variables['Build.Reason'], 'PullRequest') + variables: + aggregateDir: '$(Build.SourcesDirectory)/CodeCoverage' + steps: + - task: DownloadBuildArtifacts@0 + displayName: 'Download core coverage artifact' + continueOnError: true + inputs: + buildType: current + downloadType: single + artifactName: Win32-OpenSSH-CodeCoverage-Core + downloadPath: '$(System.ArtifactsDirectory)/coverage' + + - task: DownloadBuildArtifacts@0 + displayName: 'Download bash coverage artifact' + continueOnError: true + inputs: + buildType: current + downloadType: single + artifactName: Win32-OpenSSH-CodeCoverage-Bash + downloadPath: '$(System.ArtifactsDirectory)/coverage' - pwsh: | - & "$(Build.SourcesDirectory)/contrib/win32/openssh/code_coverage/Invoke-AzDOCodeCoverage.ps1" ` - -Suite Bash ` - -OpenSSHBinPath "$env:SystemDrive/OpenSSH" ` - -SourceRoot "$(Build.SourcesDirectory)" ` - -OutputDirectory "$(coverageDir)" -Verbose - displayName: Collect coverage - bash tests + & "$(Build.SourcesDirectory)/contrib/win32/openssh/code_coverage/Invoke-AzDOCoverageAggregate.ps1" ` + -InputDirectory "$(System.ArtifactsDirectory)/coverage" ` + -OutputDirectory "$(aggregateDir)" ` + -SourceRoot "$(Build.SourcesDirectory)" -Verbose + displayName: Aggregate per-suite coverage - pwsh: | - $summary = Join-Path "$(coverageDir)" 'coverage-summary.md' + $summary = Join-Path "$(aggregateDir)" 'coverage-summary.md' if (Test-Path $summary) { Get-Content -Raw $summary } displayName: Print coverage summary condition: always() @@ -268,24 +352,18 @@ stages: displayName: Publish code coverage results condition: always() inputs: - summaryFileLocation: '$(coverageDir)/merged/merged.cobertura.xml' + summaryFileLocation: '$(aggregateDir)/merged/merged.cobertura.xml' pathToSources: '$(Build.SourcesDirectory)' - pwsh: | - if (Test-Path "$(coverageDir)") + if (Test-Path "$(aggregateDir)") { $artifactName = 'Win32-OpenSSH-CodeCoverage' - Write-Host "##vso[artifact.upload containerfolder=$artifactName;artifactname=$artifactName;]$(coverageDir)" + Write-Host "##vso[artifact.upload containerfolder=$artifactName;artifactname=$artifactName;]$(aggregateDir)" } displayName: Upload code coverage artifacts condition: always() - - pwsh: | - Import-Module -Name "$(Build.SourcesDirectory)/contrib/win32/openssh/AzDOBuildTools" -Force - Clear-TestEnvironmentSetup - displayName: Clean up OpenSSH test environment - condition: always() - - job: TestPkgWin32OpenSSHBash pool: vmImage: windows-latest diff --git a/contrib/win32/openssh/code_coverage/Invoke-AzDOCodeCoverage.ps1 b/contrib/win32/openssh/code_coverage/Invoke-AzDOCodeCoverage.ps1 index 169958590d0e..353649a7762c 100644 --- a/contrib/win32/openssh/code_coverage/Invoke-AzDOCodeCoverage.ps1 +++ b/contrib/win32/openssh/code_coverage/Invoke-AzDOCodeCoverage.ps1 @@ -16,15 +16,15 @@ runs while a background server-mode collector owns that session. This captures every instrumented OpenSSH process - including sshd.exe running as a Windows service, which is not a child of this script. `shutdown` flushes a - per-suite .coverage file that is converted to Cobertura; every per-suite - report found under -OutputDirectory is then aggregated into a combined, - de-duplicated report plus per-suite/overlap summaries. + per-suite .coverage file that is converted to Cobertura. - Because the aggregation scans -OutputDirectory each time, the script is - idempotent: call it once per suite (Core, then Bash) into the same - -OutputDirectory and the final call produces the aggregate report. The - instrument step restores (uninstruments) each binary first, so sharing an - installed directory across the two invocations is safe. + By default the script then aggregates every per-suite report found under + -OutputDirectory into a combined, de-duplicated report plus per-suite/overlap + summaries (useful for local, single-agent runs). In CI the Core and Bash + suites run in separate jobs (each on its own agent with a clean install, so + there is no cross-suite file-lock or service-state contention); those jobs + pass -SkipAggregation and publish their per-suite artifact, and a dependent + job runs Invoke-AzDOCoverageAggregate.ps1 to combine them. This script assumes the solution is already built (with /PROFILE) and installed. It never builds. @@ -42,6 +42,10 @@ .PARAMETER OutputDirectory Where per-suite and merged coverage artifacts are written. + .PARAMETER SkipAggregation + Produce only this suite's per-suite .coverage + Cobertura report and skip the + combined merge/summary (the dedicated CI aggregation job does that). + .EXAMPLE .\Invoke-AzDOCodeCoverage.ps1 -Suite Core -OutputDirectory C:\cov .\Invoke-AzDOCodeCoverage.ps1 -Suite Bash -OutputDirectory C:\cov @@ -57,7 +61,12 @@ param( [string] $SourceRoot, [Parameter(Mandatory = $true)] - [string] $OutputDirectory + [string] $OutputDirectory, + + # Skip the aggregation/merge/summary pass; only produce this suite's per-suite + # .coverage + Cobertura report. Used by the split CI jobs, where a dedicated + # dependent job aggregates the per-suite artifacts from Core and Bash. + [switch] $SkipAggregation ) $ErrorActionPreference = 'Stop' @@ -116,64 +125,10 @@ finally { Invoke-CoverageUninstrument -BinaryPath $targets -ToolPath $tool } -# --- Aggregate every per-suite report collected so far -------------------- -$coberturaReports = Get-ChildItem -Path $OutputDirectory -Filter '*.cobertura.xml' -Recurse | - Where-Object { $_.FullName -notmatch '\\merged\\' } - -if (-not $coberturaReports) { - Write-Warning 'No per-suite Cobertura reports were produced; skipping aggregation.' +if ($SkipAggregation) { + Write-Host "SkipAggregation set; produced per-suite report only under $OutputDirectory." return } -$suiteMaps = @() -$suiteStats = @() -foreach ($report in $coberturaReports) { - $suiteName = [System.IO.Path]::GetFileNameWithoutExtension($report.Name) -replace '\.cobertura$', '' - $map = Import-CoberturaCoverage -Path $report.FullName -RepositoryRoot $SourceRoot - $suiteMaps += , $map - $suiteStats += Get-CoverageStatistic -CoverageMap $map -Name $suiteName -} - -$mergedMap = Merge-CoverageData -CoverageMap $suiteMaps -$combinedStat = Get-CoverageStatistic -CoverageMap $mergedMap -Name 'combined' -$overlap = Measure-CoverageOverlap -CoverageMap $suiteMaps - -$mergedDir = Join-Path $OutputDirectory 'merged' -# Regenerate the merge from every .coverage found each run so the report stays -# complete as suites accumulate; clear first to avoid stale outputs. -Remove-Item -LiteralPath $mergedDir -Recurse -Force -ErrorAction SilentlyContinue -$null = New-Item -ItemType Directory -Path $mergedDir -Force - -$nativeMerged = Join-Path $mergedDir 'merged.cobertura.xml' -$helperMerged = Join-Path $mergedDir 'merged-helper.cobertura.xml' - -# Authoritative native merge of every per-suite .coverage file. -$coverageFiles = Get-ChildItem -Path $OutputDirectory -Filter '*.coverage' -Recurse | ForEach-Object { $_.FullName } -$nativeMergeOk = $false -if ($coverageFiles) { - try { - Convert-CoverageReport -InputPath $coverageFiles -OutputPath $nativeMerged -Format cobertura -ToolPath $tool | Out-Null - $nativeMergeOk = Test-Path -LiteralPath $nativeMerged - } - catch { - Write-Warning "Native coverage merge failed: $($_.Exception.Message)" - } -} - -# Helper-merged Cobertura (independent of the native merge) + summaries. -New-MergedCoberturaReport -CoverageMap $mergedMap -OutputPath $helperMerged | Out-Null - -# Guarantee merged.cobertura.xml exists for the publish step even when the -# native merge was skipped (no .coverage) or failed, by falling back to the -# helper. -if (-not $nativeMergeOk) { - Write-Warning 'Using helper-merged Cobertura as merged.cobertura.xml (native merge unavailable).' - Copy-Item -LiteralPath $helperMerged -Destination $nativeMerged -Force -} - -$summary = Get-CoverageSummary -SuiteStatistic $suiteStats -CombinedStatistic $combinedStat -Overlap $overlap -$summary | ConvertTo-Json -Depth 6 | Set-Content -Path (Join-Path $OutputDirectory 'coverage-summary.json') - -$markdown = Format-CoverageSummaryMarkdown -Summary $summary -$markdown | Set-Content -Path (Join-Path $OutputDirectory 'coverage-summary.md') -Write-Host "`n$markdown" +# Aggregate every per-suite report collected so far (single-job / local usage). +Invoke-CoverageAggregation -OutputDirectory $OutputDirectory -SourceRoot $SourceRoot -ToolPath $tool | Out-Null diff --git a/contrib/win32/openssh/code_coverage/Invoke-AzDOCoverageAggregate.ps1 b/contrib/win32/openssh/code_coverage/Invoke-AzDOCoverageAggregate.ps1 new file mode 100644 index 000000000000..be9a2c76706c --- /dev/null +++ b/contrib/win32/openssh/code_coverage/Invoke-AzDOCoverageAggregate.ps1 @@ -0,0 +1,67 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +<# + .SYNOPSIS + Azure DevOps aggregation entry point for Win32-OpenSSH code coverage. + + .DESCRIPTION + The Core and Bash suites are measured in separate CI jobs (each on its own + agent with a clean OpenSSH install, so there is no cross-suite file-lock or + service-state contention). Each job publishes its per-suite .coverage + + Cobertura report as an artifact. This script runs in a dependent job that has + downloaded those artifacts into -InputDirectory; it aggregates every per-suite + report it finds into a single de-duplicated report plus per-suite/overlap + summaries under -OutputDirectory. + + .PARAMETER InputDirectory + Directory containing the downloaded per-suite coverage artifacts (searched + recursively for *.coverage and *.cobertura.xml). + + .PARAMETER OutputDirectory + Where the merged report and summaries are written. Defaults to + -InputDirectory. + + .PARAMETER SourceRoot + Repository root used to scope coverage to OpenSSH source files. + + .EXAMPLE + .\Invoke-AzDOCoverageAggregate.ps1 -InputDirectory C:\cov -SourceRoot C:\src +#> +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string] $InputDirectory, + + [string] $OutputDirectory, + + [string] $SourceRoot +) + +$ErrorActionPreference = 'Stop' + +if (-not $OutputDirectory) { $OutputDirectory = $InputDirectory } +if (-not $SourceRoot) { + $SourceRoot = (Resolve-Path (Join-Path $PSScriptRoot '..' '..' '..' '..')).Path +} + +Import-Module (Join-Path $PSScriptRoot 'OpenSSHCodeCoverage.psm1') -Force + +$null = New-Item -ItemType Directory -Path $OutputDirectory -Force + +Write-Host "InputDirectory : $InputDirectory" +Write-Host "OutputDirectory : $OutputDirectory" +Write-Host "SourceRoot : $SourceRoot" + +# If aggregating into a different directory than the inputs, bring the per-suite +# artifacts across so the native .coverage merge and the Cobertura scan see them. +if ((Resolve-Path -LiteralPath $OutputDirectory).Path -ne (Resolve-Path -LiteralPath $InputDirectory).Path) { + Copy-Item -Path (Join-Path $InputDirectory '*') -Destination $OutputDirectory -Recurse -Force +} + +$result = Invoke-CoverageAggregation -OutputDirectory $OutputDirectory -SourceRoot $SourceRoot +if (-not $result) { + throw "No per-suite coverage reports were found under '$InputDirectory'; nothing to aggregate." +} + +Write-Host "Merged report : $($result.MergedReport)" diff --git a/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.psm1 b/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.psm1 index 3cad74c8b12e..acdb8adf36da 100644 --- a/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.psm1 +++ b/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.psm1 @@ -789,6 +789,106 @@ function Convert-CoverageReport { return $OutputPath } +<# + .SYNOPSIS + Aggregates every per-suite coverage report found under a directory into a + single de-duplicated report plus per-suite/overlap summaries. + + .DESCRIPTION + Scans -OutputDirectory (recursively) for per-suite *.cobertura.xml reports + (ignoring anything under a 'merged' folder) and *.coverage files. It: + - imports each per-suite Cobertura report and computes per-suite stats, + - merges the maps and measures cross-suite overlap (pure helpers), + - natively merges the .coverage files into merged\merged.cobertura.xml + (unioning per-line hits, so an overlapping line counts once); if the tool + or .coverage files are unavailable it falls back to the helper-merged + Cobertura so merged.cobertura.xml always exists for publishing, + - writes coverage-summary.json / coverage-summary.md. + + This is invoked either inline after a single-suite collection, or by the CI + aggregation job over the per-suite artifacts downloaded from separate + Core/Bash jobs. +#> +function Invoke-CoverageAggregation { + [CmdletBinding()] + [OutputType([pscustomobject])] + param( + [Parameter(Mandatory = $true)] [string] $OutputDirectory, + [Parameter(Mandatory = $true)] [string] $SourceRoot, + [string] $ToolPath + ) + + $coberturaReports = Get-ChildItem -Path $OutputDirectory -Filter '*.cobertura.xml' -Recurse -ErrorAction SilentlyContinue | + Where-Object { $_.FullName -notmatch '[\\/]merged[\\/]' } + + if (-not $coberturaReports) { + Write-Warning 'No per-suite Cobertura reports were found; skipping aggregation.' + return $null + } + + $suiteMaps = @() + $suiteStats = @() + foreach ($report in $coberturaReports) { + $suiteName = [System.IO.Path]::GetFileNameWithoutExtension($report.Name) -replace '\.cobertura$', '' + $map = Import-CoberturaCoverage -Path $report.FullName -RepositoryRoot $SourceRoot + $suiteMaps += , $map + $suiteStats += Get-CoverageStatistic -CoverageMap $map -Name $suiteName + } + + $mergedMap = Merge-CoverageData -CoverageMap $suiteMaps + $combinedStat = Get-CoverageStatistic -CoverageMap $mergedMap -Name 'combined' + $overlap = Measure-CoverageOverlap -CoverageMap $suiteMaps + + $mergedDir = Join-Path $OutputDirectory 'merged' + # Regenerate the merge each run so the report stays complete; clear stale output first. + Remove-Item -LiteralPath $mergedDir -Recurse -Force -ErrorAction SilentlyContinue + $null = New-Item -ItemType Directory -Path $mergedDir -Force + + $nativeMerged = Join-Path $mergedDir 'merged.cobertura.xml' + $helperMerged = Join-Path $mergedDir 'merged-helper.cobertura.xml' + + # Authoritative native merge of every per-suite .coverage file (best-effort). + if (-not $ToolPath) { + try { $ToolPath = Find-CodeCoverageConsole } catch { $ToolPath = $null } + } + $coverageFiles = Get-ChildItem -Path $OutputDirectory -Filter '*.coverage' -Recurse -ErrorAction SilentlyContinue | + ForEach-Object { $_.FullName } + $nativeMergeOk = $false + if ($ToolPath -and $coverageFiles) { + try { + Convert-CoverageReport -InputPath $coverageFiles -OutputPath $nativeMerged -Format cobertura -ToolPath $ToolPath | Out-Null + $nativeMergeOk = Test-Path -LiteralPath $nativeMerged + } + catch { + Write-Warning "Native coverage merge failed: $($_.Exception.Message)" + } + } + + # Helper-merged Cobertura (independent of the native merge) + summaries. + New-MergedCoberturaReport -CoverageMap $mergedMap -OutputPath $helperMerged | Out-Null + + # Guarantee merged.cobertura.xml exists for the publish step even when the + # native merge was skipped (no tool / no .coverage) or failed. + if (-not $nativeMergeOk) { + Write-Warning 'Using helper-merged Cobertura as merged.cobertura.xml (native merge unavailable).' + Copy-Item -LiteralPath $helperMerged -Destination $nativeMerged -Force + } + + $summary = Get-CoverageSummary -SuiteStatistic $suiteStats -CombinedStatistic $combinedStat -Overlap $overlap + $summaryJson = Join-Path $OutputDirectory 'coverage-summary.json' + $summaryMd = Join-Path $OutputDirectory 'coverage-summary.md' + $summary | ConvertTo-Json -Depth 6 | Set-Content -Path $summaryJson + $markdown = Format-CoverageSummaryMarkdown -Summary $summary + $markdown | Set-Content -Path $summaryMd + Write-Host "`n$markdown" + + return [pscustomobject]@{ + MergedReport = $nativeMerged + SummaryJson = $summaryJson + SummaryMd = $summaryMd + } +} + #endregion Microsoft.CodeCoverage.Console orchestration Export-ModuleMember -Function @( @@ -806,5 +906,6 @@ Export-ModuleMember -Function @( 'Invoke-CoverageInstrument', 'Invoke-CoverageUninstrument', 'Invoke-CoverageSession', - 'Convert-CoverageReport' + 'Convert-CoverageReport', + 'Invoke-CoverageAggregation' ) diff --git a/contrib/win32/openssh/code_coverage/README.md b/contrib/win32/openssh/code_coverage/README.md index bddeb29aa629..a6304fc49b95 100644 --- a/contrib/win32/openssh/code_coverage/README.md +++ b/contrib/win32/openssh/code_coverage/README.md @@ -11,7 +11,8 @@ de-duplicated number that accounts for overlap between the suites. |------|---------| | `OpenSSHCodeCoverage.psm1` | Module. Pure aggregation/overlap helpers **and** Microsoft.CodeCoverage.Console orchestration. | | `OpenSSHCodeCoverage.tests.ps1` | Pester 5 tests for the pure helpers (no build required). | -| `Invoke-AzDOCodeCoverage.ps1` | CI entry point: run one suite (Core/Bash) under coverage against an installed OpenSSH dir, then merge. | +| `Invoke-AzDOCodeCoverage.ps1` | Per-suite CI entry point: run one suite (Core/Bash) under coverage against an installed OpenSSH dir, producing a per-suite report. | +| `Invoke-AzDOCoverageAggregate.ps1` | CI aggregation entry point: merge the per-suite artifacts from the Core and Bash jobs into the combined, de-duplicated report + summaries. | All files live in `contrib\win32\openssh\code_coverage\`. @@ -61,17 +62,27 @@ Visual Studio 2022 **Enterprise** (17.3+) — for both the build tools and First build and install the solution with `/PROFILE` so the binaries can be instrumented (the CI build job does this by injecting `/PROFILE` via the linker `LINK` env var), then point `Invoke-AzDOCodeCoverage.ps1` at the installed -directory. Call it once per suite into the same `-OutputDirectory`; the final -call produces the aggregate report. +directory. The core flow ends by uninstalling OpenSSH, so re-install before the +bash suite (in CI each suite runs on its own agent, so this is unnecessary +there): ```powershell cd contrib\win32\openssh\code_coverage -# Core = setup + unit + E2E, then the bash suite, into one output dir: -.\Invoke-AzDOCodeCoverage.ps1 -Suite Core -OpenSSHBinPath C:\OpenSSH -OutputDirectory C:\cov -.\Invoke-AzDOCodeCoverage.ps1 -Suite Bash -OpenSSHBinPath C:\OpenSSH -OutputDirectory C:\cov +# Core = setup + unit + E2E, into one output dir: +.\Invoke-AzDOCodeCoverage.ps1 -Suite Core -OpenSSHBinPath C:\OpenSSH -OutputDirectory C:\cov -SkipAggregation + +# Re-install OpenSSH (the core flow's uninstall test removed it), then bash: +Install-OpenSSH -SourceDir -OpenSSHDir C:\OpenSSH +.\Invoke-AzDOCodeCoverage.ps1 -Suite Bash -OpenSSHBinPath C:\OpenSSH -OutputDirectory C:\cov -SkipAggregation + +# Combine the per-suite reports into the de-duplicated aggregate: +.\Invoke-AzDOCoverageAggregate.ps1 -InputDirectory C:\cov ``` +Omit `-SkipAggregation` on a single-suite run to get the merge/summary for just +that suite. + ### Output artifacts (under `-OutputDirectory`) ``` @@ -108,22 +119,29 @@ Invoke-Pester -Path .\OpenSSHCodeCoverage.tests.ps1 -Output Detailed ## Continuous integration (Azure DevOps) -Coverage is wired into `.azdo/ci.yml` as two dedicated, **PR-only, non-gating** -jobs (they never run on branch builds, where CodeQL already competes for the -~60 min agent budget, and `continueOnError: true` so they never block a merge): +Coverage is wired into `.azdo/ci.yml` as dedicated, **PR-only, non-gating** jobs +(they never run on branch builds, where CodeQL already competes for the ~60 min +agent budget, and `continueOnError: true` so they never block a merge): 1. **Build Coverage Package (x64 Debug + /PROFILE)** — a Build-stage job that builds the solution `Debug|x64` with `/PROFILE` injected via the linker `LINK` env var (no `.vcxproj` edits), publishing `Win32-OpenSSH-Coverage-x64` and `UnitTests-Coverage-x64` (binaries + PDBs). It runs in parallel with the - normal Release build, so it does not eat into the coverage job's budget. -2. **Win32-OpenSSH Code Coverage** — a Test-stage job that installs the coverage - build to `C:\OpenSSH`, then runs `Invoke-AzDOCodeCoverage.ps1 -Suite Core` - (setup + unit + E2E) and `-Suite Bash`. Each invocation instruments the - installed binaries, runs the suite under a server-mode collector, converts to - Cobertura, and aggregates every per-suite report. It publishes the merged - Cobertura via `PublishCodeCoverageResults@2` and uploads the full - `Win32-OpenSSH-CodeCoverage` artifact (per-suite reports + summaries). + normal Release build. +2. **Win32-OpenSSH Code Coverage (Core)** and **(Bash)** — two Test-stage jobs + that each install the coverage build to `C:\OpenSSH` on their own agent and + run `Invoke-AzDOCodeCoverage.ps1 -Suite Core` (setup + unit + E2E) or + `-Suite Bash` with `-SkipAggregation`. Running the suites in separate jobs + (instead of sequentially in one job) gives each a clean install with no + file-lock or service-state contention, and lets them run in parallel. Each + publishes its per-suite report as `Win32-OpenSSH-CodeCoverage-Core` / + `-Bash`. +3. **Win32-OpenSSH Code Coverage (Aggregate)** — a Test-stage job that depends on + the two suite jobs, downloads their per-suite artifacts, and runs + `Invoke-AzDOCoverageAggregate.ps1` to produce the merged, de-duplicated + report. It publishes the merged Cobertura via `PublishCodeCoverageResults@2` + and uploads the full `Win32-OpenSSH-CodeCoverage` artifact (per-suite reports + + summaries). The suites run the exact CI entry points (`Invoke-OpenSSHTests`, `Invoke-OpenSSHBashTestsOnly`), so coverage reflects what CI already exercises. From f7bc8d59fe1bdb0c96baa499a5c6d5d673d58637 Mon Sep 17 00:00:00 2001 From: Tess Gauthier Date: Thu, 27 Aug 2026 13:28:47 -0400 Subject: [PATCH 7/8] Exclude vendored third-party (vcpkg) sources from coverage Vendored dependencies built through vcpkg (e.g. zlib) are linked into the OpenSSH binaries and appear in the raw coverage data, but they are not OpenSSH code and only dilute the estimate. Filter them out both from the summary (Import-CoberturaCoverage skips excluded paths) and from the published merged.cobertura.xml (new Remove-ExcludedCoverageClasses strips excluded classes and recomputes package/root line counters). Exclusion is driven by \ (default (^|/)vcpkg/). Adds Test-CoverageSourceExcluded predicate plus unit tests (20 total). Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../code_coverage/OpenSSHCodeCoverage.psm1 | 136 +++++++++++++++++- .../OpenSSHCodeCoverage.tests.ps1 | 68 +++++++++ contrib/win32/openssh/code_coverage/README.md | 11 ++ 3 files changed, 213 insertions(+), 2 deletions(-) diff --git a/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.psm1 b/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.psm1 index acdb8adf36da..4b7d44e5af85 100644 --- a/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.psm1 +++ b/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.psm1 @@ -47,8 +47,36 @@ $ErrorActionPreference = 'Stop' # Repository root is four levels up from contrib\win32\openssh\code_coverage. $script:RepositoryRoot = (Resolve-Path (Join-Path $PSScriptRoot '..' '..' '..' '..')).Path +# Source paths (normalized, repository-relative, forward-slash) matching this +# pattern are third-party dependencies vendored into the tree - primarily the +# vcpkg-built libraries such as zlib. They are not OpenSSH code, are never +# exercised by our test suites in a meaningful way, and only dilute the +# coverage estimate, so they are excluded from every coverage map. +$script:CoverageExcludePattern = '(?i)(^|/)vcpkg/' + #region Pure helpers (unit tested) +<# + .SYNOPSIS + Returns $true when a normalized source path should be excluded from the + coverage estimate (e.g. vendored third-party dependencies). +#> +function Test-CoverageSourceExcluded { + [CmdletBinding()] + [OutputType([bool])] + param( + [Parameter(Mandatory = $true)] + [AllowEmptyString()] + [string] $NormalizedPath, + + [string] $ExcludePattern = $script:CoverageExcludePattern + ) + + if ([string]::IsNullOrWhiteSpace($NormalizedPath)) { return $false } + if ([string]::IsNullOrWhiteSpace($ExcludePattern)) { return $false } + return [regex]::IsMatch($NormalizedPath, $ExcludePattern) +} + <# .SYNOPSIS Normalizes a source-file path emitted in a coverage report to a stable, @@ -140,6 +168,10 @@ function Import-CoberturaCoverage { if ([string]::IsNullOrWhiteSpace($file)) { continue } $key = ConvertTo-NormalizedCoverageSourcePath -RawPath $file -RepositoryRoot $RepositoryRoot + # Skip vendored third-party sources (e.g. vcpkg/zlib) - they are not + # OpenSSH code and would only dilute the coverage estimate. + if (Test-CoverageSourceExcluded -NormalizedPath $key) { continue } + if (-not $map.ContainsKey($key)) { $map[$key] = @{} } @@ -375,9 +407,92 @@ function New-MergedCoberturaReport { <# .SYNOPSIS - Builds a human/machine readable summary object from per-suite statistics, - the combined statistic and the overlap measurement. + Removes vendored third-party (excluded) classes from a Cobertura XML file + in place and recomputes the aggregate line counters. + + .DESCRIPTION + The native Microsoft.CodeCoverage.Console merge still contains every source + that was linked into the instrumented binaries, including vcpkg-built + dependencies such as zlib. This strips those nodes (matched via + Test-CoverageSourceExcluded against the normalized filename), drops any + package left empty, and recomputes lines-valid / lines-covered / line-rate + on each package and on the root so the published report matches the summary + estimate. Returns the number of classes removed. #> +function Remove-ExcludedCoverageClasses { + [CmdletBinding()] + [OutputType([int])] + param( + [Parameter(Mandatory = $true)] + [string] $Path, + + [string] $RepositoryRoot = $script:RepositoryRoot + ) + + if (-not (Test-Path -LiteralPath $Path)) { + throw "Cobertura report not found: $Path" + } + + [xml] $xml = Get-Content -LiteralPath $Path -Raw + $inv = [System.Globalization.CultureInfo]::InvariantCulture + [int] $removed = 0 + + foreach ($class in @($xml.SelectNodes('//class'))) { + $file = $class.GetAttribute('filename') + if ([string]::IsNullOrWhiteSpace($file)) { continue } + $key = ConvertTo-NormalizedCoverageSourcePath -RawPath $file -RepositoryRoot $RepositoryRoot + if (Test-CoverageSourceExcluded -NormalizedPath $key) { + [void] $class.ParentNode.RemoveChild($class) + $removed++ + } + } + + # Recompute counters per and for the document root from the + # surviving elements so consumers report the excluded totals. + [int] $rootValid = 0 + [int] $rootCovered = 0 + foreach ($package in @($xml.SelectNodes('//package'))) { + $lines = $package.SelectNodes('.//line') + if (-not $lines -or $lines.Count -eq 0) { + [void] $package.ParentNode.RemoveChild($package) + continue + } + [int] $valid = 0 + [int] $covered = 0 + foreach ($line in $lines) { + $valid++ + if ([int] $line.GetAttribute('hits') -gt 0) { $covered++ } + } + $rate = if ($valid -gt 0) { [math]::Round($covered / $valid, 4) } else { 0 } + Set-CoberturaLineCounter -Node $package -Valid $valid -Covered $covered -Rate $rate -Culture $inv + $rootValid += $valid + $rootCovered += $covered + } + + $rootRate = if ($rootValid -gt 0) { [math]::Round($rootCovered / $rootValid, 4) } else { 0 } + Set-CoberturaLineCounter -Node $xml.DocumentElement -Valid $rootValid -Covered $rootCovered -Rate $rootRate -Culture $inv + + $xml.Save($Path) + return $removed +} + +# Sets/updates the lines-valid, lines-covered and line-rate attributes on a +# Cobertura node (only those already present are updated, plus line-rate/counts +# which every consumer expects). Kept private (not exported). +function Set-CoberturaLineCounter { + param( + [Parameter(Mandatory = $true)] [System.Xml.XmlElement] $Node, + [Parameter(Mandatory = $true)] [int] $Valid, + [Parameter(Mandatory = $true)] [int] $Covered, + [Parameter(Mandatory = $true)] [double] $Rate, + [Parameter(Mandatory = $true)] [System.Globalization.CultureInfo] $Culture + ) + + $Node.SetAttribute('line-rate', $Rate.ToString($Culture)) + if ($Node.HasAttribute('lines-valid')) { $Node.SetAttribute('lines-valid', ([string]$Valid)) } + if ($Node.HasAttribute('lines-covered')) { $Node.SetAttribute('lines-covered', ([string]$Covered)) } +} + function Get-CoverageSummary { [CmdletBinding()] [OutputType([pscustomobject])] @@ -874,6 +989,21 @@ function Invoke-CoverageAggregation { Copy-Item -LiteralPath $helperMerged -Destination $nativeMerged -Force } + # Strip vendored third-party sources (e.g. vcpkg/zlib) from the published + # merged report so it matches the summary estimate. The helper-merged path + # is already filtered at import time; this only affects the native merge. + if ($nativeMergeOk) { + try { + $removedClasses = Remove-ExcludedCoverageClasses -Path $nativeMerged -RepositoryRoot $SourceRoot + if ($removedClasses -gt 0) { + Write-Host "Excluded $removedClasses third-party class entries from merged.cobertura.xml." + } + } + catch { + Write-Warning "Could not strip third-party sources from merged report: $($_.Exception.Message)" + } + } + $summary = Get-CoverageSummary -SuiteStatistic $suiteStats -CombinedStatistic $combinedStat -Overlap $overlap $summaryJson = Join-Path $OutputDirectory 'coverage-summary.json' $summaryMd = Join-Path $OutputDirectory 'coverage-summary.md' @@ -893,11 +1023,13 @@ function Invoke-CoverageAggregation { Export-ModuleMember -Function @( 'ConvertTo-NormalizedCoverageSourcePath', + 'Test-CoverageSourceExcluded', 'Import-CoberturaCoverage', 'Merge-CoverageData', 'Get-CoverageStatistic', 'Measure-CoverageOverlap', 'New-MergedCoberturaReport', + 'Remove-ExcludedCoverageClasses', 'Get-CoverageSummary', 'Format-CoverageSummaryMarkdown', 'Find-CodeCoverageConsole', diff --git a/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.tests.ps1 b/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.tests.ps1 index 61c0a2d8903b..d6f6eb2522ff 100644 --- a/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.tests.ps1 +++ b/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.tests.ps1 @@ -90,8 +90,76 @@ Describe 'Import-CoberturaCoverage' { $map['channels.c'][2] | Should -Be 0 $map['channels.c'][3] | Should -Be 2 } + + It 'Skips vendored third-party (vcpkg) sources' { + $file = Join-Path $TestDrive 'tp.cobertura.xml' + New-TestCobertura -FileName 'deflate.c' -SourcePath 'C:\repo\vcpkg\buildtrees\zlib\src\deflate.c' -Lines @{ 1 = 5; 2 = 3 } -OutFile $file | Out-Null + + $map = Import-CoberturaCoverage -Path $file -RepositoryRoot 'C:\repo' + $map.Keys.Count | Should -Be 0 + } } +Describe 'Test-CoverageSourceExcluded' { + It 'Excludes vcpkg-vendored paths' { + Test-CoverageSourceExcluded -NormalizedPath 'vcpkg/buildtrees/zlib/src/deflate.c' | Should -BeTrue + } + + It 'Excludes vcpkg even when nested deeper in the tree' { + Test-CoverageSourceExcluded -NormalizedPath 'contrib/win32/vcpkg/x.c' | Should -BeTrue + } + + It 'Does not exclude OpenSSH sources' { + Test-CoverageSourceExcluded -NormalizedPath 'channels.c' | Should -BeFalse + Test-CoverageSourceExcluded -NormalizedPath 'contrib/win32/win32compat/ansiprsr.c' | Should -BeFalse + } +} + +Describe 'Remove-ExcludedCoverageClasses' { + It 'Removes third-party classes and recomputes counters' { + $xml = @' + + + + + + + + + + + + + + + + + + + + + + +'@ + $file = Join-Path $TestDrive 'merged.cobertura.xml' + Set-Content -Path $file -Value $xml -NoNewline + + $removed = Remove-ExcludedCoverageClasses -Path $file -RepositoryRoot 'C:\repo' + $removed | Should -Be 2 + + [xml] $doc = Get-Content -LiteralPath $file -Raw + # Only the ssh package with channels.c survives. + $doc.SelectNodes('//package').Count | Should -Be 1 + $doc.SelectNodes('//class').Count | Should -Be 1 + $doc.SelectSingleNode('//class').GetAttribute('filename') | Should -Match 'channels\.c$' + # Root counters recomputed from surviving lines: 1 covered of 2 valid. + $doc.coverage.GetAttribute('lines-valid') | Should -Be '2' + $doc.coverage.GetAttribute('lines-covered') | Should -Be '1' + $doc.coverage.GetAttribute('line-rate') | Should -Be '0.5' + } +} + + Describe 'Merge-CoverageData' { Context 'Overlapping coverage of the same file/line' { It 'Sums hit counts so an overlapping line stays covered once' { diff --git a/contrib/win32/openssh/code_coverage/README.md b/contrib/win32/openssh/code_coverage/README.md index a6304fc49b95..83c0fd5973ff 100644 --- a/contrib/win32/openssh/code_coverage/README.md +++ b/contrib/win32/openssh/code_coverage/README.md @@ -52,6 +52,17 @@ CombinedCoveredLines = covered lines after merge (each line once) <- the estim OverlapLines = SumCoveredLines - CombinedCoveredLines ``` +### Third-party exclusion + +Vendored dependencies built through vcpkg (e.g. zlib) are linked into the +OpenSSH binaries and therefore show up in the raw coverage data, but they are +not OpenSSH code and only dilute the estimate. Any source whose +repository-relative path matches `$script:CoverageExcludePattern` (default +`(^|/)vcpkg/`) is dropped: `Import-CoberturaCoverage` skips it in the summary, +and `Remove-ExcludedCoverageClasses` strips it from the published +`merged.cobertura.xml` (recomputing the package/root line counters) so the +Cobertura report matches the summary. + ## Usage Coverage normally runs in CI (see below). To reproduce locally you need From db5d6b064e4f03f141aae20520e6189bc7ec4a7a Mon Sep 17 00:00:00 2001 From: Tess Gauthier Date: Mon, 31 Aug 2026 16:04:45 -0400 Subject: [PATCH 8/8] Publish de-duplicated coverage report to the ADO widget The Azure DevOps Code Coverage widget was fed the native Microsoft.CodeCoverage.Console merge, which counts each shared .c once per binary it is linked into (~3.6x inflation on this solution), so the widget headline (~41%) did not match the de-duplicated estimate in the coverage summary (~62%). Point PublishCodeCoverageResults@2 at the de-duplicated merged-helper.cobertura.xml so the widget headline matches the number we report. The native per-binary report and raw .coverage files remain in the Win32-OpenSSH-CodeCoverage artifact for per-binary / per-method drill-down. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .azdo/ci.yml | 10 ++++++- contrib/win32/openssh/code_coverage/README.md | 26 +++++++++++++++---- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/.azdo/ci.yml b/.azdo/ci.yml index 8aa002dd15b7..50778f9f7f19 100644 --- a/.azdo/ci.yml +++ b/.azdo/ci.yml @@ -352,7 +352,15 @@ stages: displayName: Publish code coverage results condition: always() inputs: - summaryFileLocation: '$(aggregateDir)/merged/merged.cobertura.xml' + # Publish the de-duplicated (per-source-file) report so the Azure + # DevOps "Code Coverage" widget headline matches the number we stand + # behind (see the "Print coverage summary" step). The native + # Microsoft.CodeCoverage.Console report counts each shared .c once per + # binary it is linked into, which inflates the totals ~3.6x. The native + # per-binary report (merged/merged.cobertura.xml) and the raw .coverage + # files are still uploaded in the Win32-OpenSSH-CodeCoverage artifact + # below for per-binary / per-method drill-down (e.g. in Visual Studio). + summaryFileLocation: '$(aggregateDir)/merged/merged-helper.cobertura.xml' pathToSources: '$(Build.SourcesDirectory)' - pwsh: | diff --git a/contrib/win32/openssh/code_coverage/README.md b/contrib/win32/openssh/code_coverage/README.md index 83c0fd5973ff..fa20e69493d0 100644 --- a/contrib/win32/openssh/code_coverage/README.md +++ b/contrib/win32/openssh/code_coverage/README.md @@ -99,11 +99,25 @@ that suite. ``` core\core.coverage, core\core.cobertura.xml per-suite (setup + unit + E2E) bash\bash.coverage, bash\bash.cobertura.xml per-suite (bash tests) -merged\merged.cobertura.xml combined, de-duplicated (native merge) +merged\merged.cobertura.xml combined, native merge (per-binary) +merged\merged-helper.cobertura.xml combined, de-duplicated by source file coverage-summary.json machine-readable summary coverage-summary.md per-suite + combined + overlap table ``` +Two combined reports are emitted because they answer different questions: + +- **`merged-helper.cobertura.xml`** is de-duplicated by unique source file — a + `.c` linked into several binaries is counted once. This matches the + `coverage-summary.md` estimate and is the report published to the Azure DevOps + **Code Coverage widget** (see CI section). +- **`merged.cobertura.xml`** is the native Microsoft.CodeCoverage.Console merge. + It preserves per-binary packages, per-method coverage, cyclomatic complexity + and branch data, but counts each shared `.c` once **per binary** it is linked + into (~3.6x inflation on this solution). It is kept in the uploaded artifact + for per-binary / per-method drill-down (e.g. reopening the `.coverage` files in + Visual Studio). + Example `coverage-summary.md`: ``` @@ -149,10 +163,12 @@ agent budget, and `continueOnError: true` so they never block a merge): `-Bash`. 3. **Win32-OpenSSH Code Coverage (Aggregate)** — a Test-stage job that depends on the two suite jobs, downloads their per-suite artifacts, and runs - `Invoke-AzDOCoverageAggregate.ps1` to produce the merged, de-duplicated - report. It publishes the merged Cobertura via `PublishCodeCoverageResults@2` - and uploads the full `Win32-OpenSSH-CodeCoverage` artifact (per-suite reports - + summaries). + `Invoke-AzDOCoverageAggregate.ps1` to produce the merged reports. It publishes + the **de-duplicated** report (`merged/merged-helper.cobertura.xml`) via + `PublishCodeCoverageResults@2` so the Code Coverage widget headline matches the + `coverage-summary.md` estimate, and uploads the full `Win32-OpenSSH-CodeCoverage` + artifact (per-suite reports, both merged reports, the raw `.coverage` files, and + summaries) for per-binary / per-method drill-down. The suites run the exact CI entry points (`Invoke-OpenSSHTests`, `Invoke-OpenSSHBashTestsOnly`), so coverage reflects what CI already exercises.