diff --git a/.azdo/ci.yml b/.azdo/ci.yml index 321c9201a6c5..50778f9f7f19 100644 --- a/.azdo/ci.yml +++ b/.azdo/ci.yml @@ -70,6 +70,26 @@ stages: dependenciesArtifactName: vcpkg-dependencies-x86 vcpkgCacheDir: '$(vcpkgCacheDir)' + - job: BuildPkg_Coverage_x64 + displayName: Build Coverage Package (x64 Debug + /PROFILE) + pool: + vmImage: windows-latest + # Consumed only by the PR-only coverage job, so don't spend build time on + # branch builds (where CodeQL already competes for the agent budget). + condition: eq(variables['Build.Reason'], 'PullRequest') + # Non-gating: a coverage-specific build failure must never block the pipeline. + continueOnError: true + steps: + - template: ./templates/build-win32-openssh-job.yml + parameters: + nativeHostArch: x64 + buildOutputDir: x64 + artifactSuffix: Coverage-x64 + dependenciesArtifactName: vcpkg-dependencies-x64 + vcpkgCacheDir: '$(vcpkgCacheDir)' + configuration: Debug + linkerOptions: '/PROFILE' + - stage: Test displayName: Test Win32-OpenSSH dependsOn: Build @@ -164,6 +184,194 @@ stages: displayName: Clean up OpenSSH test environment condition: always() + - job: CodeCoverageCore + pool: + vmImage: windows-latest + displayName: Win32-OpenSSH Code Coverage (Core) + # Coverage is informational and must never gate a merge while it stabilizes. + continueOnError: true + # Only run on pull requests. On branch builds CodeQL is enabled and the added + # coverage instrumentation overhead risks pushing the agent past its ~60 min timeout. + condition: eq(variables['Build.Reason'], 'PullRequest') + variables: + coverageDir: '$(Build.SourcesDirectory)/CodeCoverage' + steps: + - task: DownloadBuildArtifacts@0 + displayName: 'Download coverage build artifacts (Debug + symbols)' + inputs: + buildType: current + downloadType: single + artifactName: Win32-OpenSSH-Coverage-x64 + downloadPath: '$(System.ArtifactsDirectory)' + + - task: DownloadBuildArtifacts@0 + displayName: 'Download coverage unit test artifacts' + inputs: + buildType: current + downloadType: single + artifactName: UnitTests-Coverage-x64 + downloadPath: '$(System.ArtifactsDirectory)' + + - pwsh: | + Import-Module -Name "$(Build.SourcesDirectory)/contrib/win32/openssh/AzDOBuildTools" -Force + # Install the Debug (/PROFILE) build so Microsoft.CodeCoverage.Console can + # statically instrument the OpenSSH binaries and unit tests. + Install-OpenSSH -SourceDir "$(System.ArtifactsDirectory)/Win32-OpenSSH-Coverage-x64/x64/Debug" -OpenSSHDir "$env:SystemDrive/OpenSSH" -Verbose + Install-UnitTests -SourceDir "$(System.ArtifactsDirectory)/UnitTests-Coverage-x64/x64/Debug" -OpenSSHDir "$env:SystemDrive/OpenSSH" -Verbose + displayName: Install Win32-OpenSSH and unit tests + + - pwsh: | + & "$(Build.SourcesDirectory)/contrib/win32/openssh/code_coverage/Invoke-AzDOCodeCoverage.ps1" ` + -Suite Core ` + -OpenSSHBinPath "$env:SystemDrive/OpenSSH" ` + -SourceRoot "$(Build.SourcesDirectory)" ` + -OutputDirectory "$(coverageDir)" -SkipAggregation -Verbose + displayName: Collect coverage - core tests (setup + unit + E2E) + + - pwsh: | + if (Test-Path "$(coverageDir)") + { + $artifactName = 'Win32-OpenSSH-CodeCoverage-Core' + Write-Host "##vso[artifact.upload containerfolder=$artifactName;artifactname=$artifactName;]$(coverageDir)" + } + displayName: Upload core coverage artifact + condition: always() + + - pwsh: | + Import-Module -Name "$(Build.SourcesDirectory)/contrib/win32/openssh/AzDOBuildTools" -Force + Clear-TestEnvironmentSetup + displayName: Clean up OpenSSH test environment + condition: always() + + - job: CodeCoverageBash + pool: + vmImage: windows-latest + displayName: Win32-OpenSSH Code Coverage (Bash) + continueOnError: true + condition: eq(variables['Build.Reason'], 'PullRequest') + variables: + coverageDir: '$(Build.SourcesDirectory)/CodeCoverage' + steps: + - task: DownloadBuildArtifacts@0 + displayName: 'Download coverage build artifacts (Debug + symbols)' + inputs: + buildType: current + downloadType: single + artifactName: Win32-OpenSSH-Coverage-x64 + downloadPath: '$(System.ArtifactsDirectory)' + + - task: DownloadBuildArtifacts@0 + displayName: 'Download bash test config file artifact' + inputs: + buildType: current + downloadType: single + artifactName: ConfigFile-x64 + downloadPath: '$(System.ArtifactsDirectory)' + + - pwsh: | + Import-Module -Name "$(Build.SourcesDirectory)/contrib/win32/openssh/AzDOBuildTools" -Force + # Install the Debug (/PROFILE) build so the bash suite runs against + # instrumentable binaries with the sshd/ssh-agent services registered. + Install-OpenSSH -SourceDir "$(System.ArtifactsDirectory)/Win32-OpenSSH-Coverage-x64/x64/Debug" -OpenSSHDir "$env:SystemDrive/OpenSSH" -Verbose + displayName: Install Win32-OpenSSH + + - pwsh: | + $configFileSrc = "$(System.ArtifactsDirectory)/ConfigFile-x64/config.h" + Copy-Item -Path $configFileSrc -Dest "$(Build.SourcesDirectory)" -Force + displayName: Copy config file artifact for bash tests + + - pwsh: | + & "$(Build.SourcesDirectory)/contrib/win32/openssh/code_coverage/Invoke-AzDOCodeCoverage.ps1" ` + -Suite Bash ` + -OpenSSHBinPath "$env:SystemDrive/OpenSSH" ` + -SourceRoot "$(Build.SourcesDirectory)" ` + -OutputDirectory "$(coverageDir)" -SkipAggregation -Verbose + displayName: Collect coverage - bash tests + + - pwsh: | + if (Test-Path "$(coverageDir)") + { + $artifactName = 'Win32-OpenSSH-CodeCoverage-Bash' + Write-Host "##vso[artifact.upload containerfolder=$artifactName;artifactname=$artifactName;]$(coverageDir)" + } + displayName: Upload bash coverage artifact + condition: always() + + - pwsh: | + Import-Module -Name "$(Build.SourcesDirectory)/contrib/win32/openssh/AzDOBuildTools" -Force + Clear-TestEnvironmentSetup + displayName: Clean up OpenSSH test environment + condition: always() + + - job: CodeCoverageAggregate + pool: + vmImage: windows-latest + displayName: Win32-OpenSSH Code Coverage (Aggregate) + # Run after both suite jobs regardless of their outcome (they are non-gating); + # aggregate whichever per-suite artifacts were produced. + dependsOn: + - CodeCoverageCore + - CodeCoverageBash + continueOnError: true + condition: eq(variables['Build.Reason'], 'PullRequest') + variables: + aggregateDir: '$(Build.SourcesDirectory)/CodeCoverage' + steps: + - task: DownloadBuildArtifacts@0 + displayName: 'Download core coverage artifact' + continueOnError: true + inputs: + buildType: current + downloadType: single + artifactName: Win32-OpenSSH-CodeCoverage-Core + downloadPath: '$(System.ArtifactsDirectory)/coverage' + + - task: DownloadBuildArtifacts@0 + displayName: 'Download bash coverage artifact' + continueOnError: true + inputs: + buildType: current + downloadType: single + artifactName: Win32-OpenSSH-CodeCoverage-Bash + downloadPath: '$(System.ArtifactsDirectory)/coverage' + + - pwsh: | + & "$(Build.SourcesDirectory)/contrib/win32/openssh/code_coverage/Invoke-AzDOCoverageAggregate.ps1" ` + -InputDirectory "$(System.ArtifactsDirectory)/coverage" ` + -OutputDirectory "$(aggregateDir)" ` + -SourceRoot "$(Build.SourcesDirectory)" -Verbose + displayName: Aggregate per-suite coverage + + - pwsh: | + $summary = Join-Path "$(aggregateDir)" 'coverage-summary.md' + if (Test-Path $summary) { Get-Content -Raw $summary } + displayName: Print coverage summary + condition: always() + + - task: PublishCodeCoverageResults@2 + displayName: Publish code coverage results + condition: always() + inputs: + # Publish the de-duplicated (per-source-file) report so the Azure + # DevOps "Code Coverage" widget headline matches the number we stand + # behind (see the "Print coverage summary" step). The native + # Microsoft.CodeCoverage.Console report counts each shared .c once per + # binary it is linked into, which inflates the totals ~3.6x. The native + # per-binary report (merged/merged.cobertura.xml) and the raw .coverage + # files are still uploaded in the Win32-OpenSSH-CodeCoverage artifact + # below for per-binary / per-method drill-down (e.g. in Visual Studio). + summaryFileLocation: '$(aggregateDir)/merged/merged-helper.cobertura.xml' + pathToSources: '$(Build.SourcesDirectory)' + + - pwsh: | + if (Test-Path "$(aggregateDir)") + { + $artifactName = 'Win32-OpenSSH-CodeCoverage' + Write-Host "##vso[artifact.upload containerfolder=$artifactName;artifactname=$artifactName;]$(aggregateDir)" + } + displayName: Upload code coverage artifacts + condition: always() + - job: TestPkgWin32OpenSSHBash pool: vmImage: windows-latest diff --git a/.azdo/templates/build-win32-openssh-job.yml b/.azdo/templates/build-win32-openssh-job.yml index be225e0ec553..97d189820954 100644 --- a/.azdo/templates/build-win32-openssh-job.yml +++ b/.azdo/templates/build-win32-openssh-job.yml @@ -12,6 +12,12 @@ parameters: default: false - name: vcpkgCacheDir type: string +- name: configuration + type: string + default: Release +- name: linkerOptions + type: string + default: '' steps: - pwsh: | @@ -32,23 +38,39 @@ steps: - pwsh: | $nativeArch = '${{ parameters.nativeHostArch }}' + $configuration = '${{ parameters.configuration }}' + $linkerOptions = '${{ parameters.linkerOptions }}' Import-Module -Name "$(Build.SourcesDirectory)/contrib/win32/openssh/AzDOBuildTools" -Force - Invoke-AzDOBuild -NativeHostArch $nativeArch - displayName: Build Win32-OpenSSH (${{ parameters.nativeHostArch }}) + if ($configuration -eq 'Release' -and -not $linkerOptions) { + Invoke-AzDOBuild -NativeHostArch $nativeArch + } + else { + # Coverage build: honor an alternate configuration (Debug) and inject + # extra linker options (e.g. /PROFILE for static instrumentation) via the + # linker's LINK environment variable, without editing any .vcxproj. + Import-Module -Name "$(Build.SourcesDirectory)/contrib/win32/openssh/OpenSSHBuildHelper.psm1" -Force + if ($linkerOptions) { + $env:LINK = ((@($env:LINK, $linkerOptions) | Where-Object { $_ }) -join ' ') + } + Start-OpenSSHBuild -Configuration $configuration -NativeHostArch $nativeArch -Verbose + } + displayName: Build Win32-OpenSSH (${{ parameters.nativeHostArch }} ${{ parameters.configuration }}) env: VCPKG_BINARY_SOURCES: "clear;files,${{ parameters.vcpkgCacheDir }},readwrite" - pwsh: | $buildOutPath = "$(Build.SourcesDirectory)/bin" $buildOutputDir = '${{ parameters.buildOutputDir }}' + $configuration = '${{ parameters.configuration }}' $fullPath = Join-Path -Path $buildOutPath -ChildPath $buildOutputDir - $releasePath = Join-Path -Path $fullPath -ChildPath 'Release' - Get-ChildItem -Path $releasePath + $configPath = Join-Path -Path $fullPath -ChildPath $configuration + Get-ChildItem -Path $configPath displayName: Capture build results (${{ parameters.artifactSuffix }}) - pwsh: | $nativeArch = '${{ parameters.nativeHostArch }}' $artifactSuffix = '${{ parameters.artifactSuffix }}' + $configuration = '${{ parameters.configuration }}' $includeConfig = [System.Convert]::ToBoolean('${{ parameters.includeConfig }}') Import-Module -Name "$(Build.SourcesDirectory)/contrib/win32/openssh/AzDOBuildTools" -Force @@ -59,8 +81,8 @@ steps: Remove-Item -Path $buildDestPath -Recurse -Force -ErrorAction SilentlyContinue } $null = New-Item -ItemType Directory -Path $buildDestPath -Force - $buildDestArchPath = Join-Path -Path $buildDestPath -ChildPath "$nativeArch/Release" - Copy-BuildResults -BuildResultsPath $buildDestArchPath -NativeHostArch $nativeArch -Configuration Release + $buildDestArchPath = Join-Path -Path $buildDestPath -ChildPath "$nativeArch/$configuration" + Copy-BuildResults -BuildResultsPath $buildDestArchPath -NativeHostArch $nativeArch -Configuration $configuration # # Upload build artifacts Write-Verbose -Verbose -Message "Uploading build artifacts" @@ -74,7 +96,7 @@ steps: Remove-Item -Path $unitTestDestPath -Recurse -Force -ErrorAction SilentlyContinue } $null = New-Item -ItemType Directory -Path $unitTestDestPath -Force - Copy-UnitTests -UnitTestsSrcDir $buildOutPath -UnitTestsDestDir $unitTestDestPath -NativeHostArch $nativeArch -Configuration Release + Copy-UnitTests -UnitTestsSrcDir $buildOutPath -UnitTestsDestDir $unitTestDestPath -NativeHostArch $nativeArch -Configuration $configuration # # Upload unit test artifacts Write-Verbose -Verbose -Message "Uploading unit test artifacts" @@ -91,8 +113,9 @@ steps: displayName: Upload Win32-OpenSSH build artifacts (${{ parameters.artifactSuffix }}) - pwsh: | - $artifactSuffix = '${{ parameters.artifactSuffix }}' - $logFileName = "OpenSSHRelease$artifactSuffix.log" + $nativeArch = '${{ parameters.nativeHostArch }}' + $configuration = '${{ parameters.configuration }}' + $logFileName = "OpenSSH$configuration$nativeArch.log" $uniqueLines = [System.Collections.Generic.HashSet[string]]::new() $logFilePath = "$(Build.SourcesDirectory)/contrib/win32/openssh/$logFileName" if (Test-Path -Path $logFilePath) { diff --git a/contrib/win32/openssh/code_coverage/Invoke-AzDOCodeCoverage.ps1 b/contrib/win32/openssh/code_coverage/Invoke-AzDOCodeCoverage.ps1 new file mode 100644 index 000000000000..353649a7762c --- /dev/null +++ b/contrib/win32/openssh/code_coverage/Invoke-AzDOCodeCoverage.ps1 @@ -0,0 +1,134 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +<# + .SYNOPSIS + Azure DevOps entry point for Win32-OpenSSH code coverage. + + .DESCRIPTION + Runs one OpenSSH test suite (the "Core" flow - setup + unit + E2E - or the + "Bash" flow) under Microsoft.CodeCoverage.Console against an + already-installed OpenSSH directory, using the same AzDOBuildTools entry + points the CI test jobs use. + + The installed OpenSSH binaries (built with /PROFILE by the coverage build + job) are statically instrumented with a per-suite session id, then the suite + runs while a background server-mode collector owns that session. This + captures every instrumented OpenSSH process - including sshd.exe running as a + Windows service, which is not a child of this script. `shutdown` flushes a + per-suite .coverage file that is converted to Cobertura. + + By default the script then aggregates every per-suite report found under + -OutputDirectory into a combined, de-duplicated report plus per-suite/overlap + summaries (useful for local, single-agent runs). In CI the Core and Bash + suites run in separate jobs (each on its own agent with a clean install, so + there is no cross-suite file-lock or service-state contention); those jobs + pass -SkipAggregation and publish their per-suite artifact, and a dependent + job runs Invoke-AzDOCoverageAggregate.ps1 to combine them. + + This script assumes the solution is already built (with /PROFILE) and + installed. It never builds. + + .PARAMETER Suite + Which suite to measure this invocation: Core or Bash. + + .PARAMETER OpenSSHBinPath + Installed OpenSSH directory containing the binaries, their PDBs and the + unittest-*.exe binaries (CI installs this to $env:SystemDrive\OpenSSH). + + .PARAMETER SourceRoot + Repository root used to scope coverage to OpenSSH source files. + + .PARAMETER OutputDirectory + Where per-suite and merged coverage artifacts are written. + + .PARAMETER SkipAggregation + Produce only this suite's per-suite .coverage + Cobertura report and skip the + combined merge/summary (the dedicated CI aggregation job does that). + + .EXAMPLE + .\Invoke-AzDOCodeCoverage.ps1 -Suite Core -OutputDirectory C:\cov + .\Invoke-AzDOCodeCoverage.ps1 -Suite Bash -OutputDirectory C:\cov +#> +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [ValidateSet('Core', 'Bash')] + [string] $Suite, + + [string] $OpenSSHBinPath = "$env:SystemDrive\OpenSSH", + + [string] $SourceRoot, + + [Parameter(Mandatory = $true)] + [string] $OutputDirectory, + + # Skip the aggregation/merge/summary pass; only produce this suite's per-suite + # .coverage + Cobertura report. Used by the split CI jobs, where a dedicated + # dependent job aggregates the per-suite artifacts from Core and Bash. + [switch] $SkipAggregation +) + +$ErrorActionPreference = 'Stop' + +$opensshDir = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path +if (-not $SourceRoot) { + $SourceRoot = (Resolve-Path (Join-Path $PSScriptRoot '..' '..' '..' '..')).Path +} +$azdoModule = Join-Path $opensshDir 'AzDOBuildTools' + +Import-Module (Join-Path $PSScriptRoot 'OpenSSHCodeCoverage.psm1') -Force + +$null = New-Item -ItemType Directory -Path $OutputDirectory -Force +$tool = Find-CodeCoverageConsole +Write-Host "CodeCoverage.Console : $tool" +Write-Host "Suite : $Suite" +Write-Host "OpenSSHBinPath : $OpenSSHBinPath" +Write-Host "SourceRoot : $SourceRoot" + +switch ($Suite) { + 'Core' { + $name = 'core' + $cmd = "Import-Module '$azdoModule' -Force; Invoke-OpenSSHTests -OpenSSHBinPath '$OpenSSHBinPath'" + } + 'Bash' { + $name = 'bash' + $cmd = "Import-Module '$azdoModule' -Force; Invoke-OpenSSHBashTestsOnly -OpenSSHBinPath '$OpenSSHBinPath'" + } +} + +# Instrument the installed OpenSSH binaries with a per-suite session id, then +# run the suite while a server-mode collector owns that session. +$targets = Get-OpenSSHCoverageTarget -BinaryDirectory $OpenSSHBinPath +if (-not $targets) { + throw "No OpenSSH binaries to instrument were found under '$OpenSSHBinPath'." +} +Write-Host "Instrumenting $($targets.Count) binaries for suite '$name'..." + +$sessionId = [guid]::NewGuid().ToString() +Invoke-CoverageInstrument -BinaryPath $targets -SessionId $sessionId -ToolPath $tool | Out-Null + +# The current PowerShell host re-invokes itself so its ssh/sshd/unittest +# children (all instrumented) run inside the collection window. +$pwshHost = (Get-Process -Id $PID).Path + +try { + $session = Invoke-CoverageSession -Name $name ` + -Program $pwshHost -ArgumentList @('-NoProfile', '-NonInteractive', '-Command', $cmd) ` + -SessionId $sessionId ` + -OutputDirectory (Join-Path $OutputDirectory $name) ` + -ToolPath $tool + Write-Host "Suite '$name' coverage exit code: $($session.ExitCode)" +} +finally { + # Restore the binaries so a subsequent suite step starts from clean images. + Invoke-CoverageUninstrument -BinaryPath $targets -ToolPath $tool +} + +if ($SkipAggregation) { + Write-Host "SkipAggregation set; produced per-suite report only under $OutputDirectory." + return +} + +# Aggregate every per-suite report collected so far (single-job / local usage). +Invoke-CoverageAggregation -OutputDirectory $OutputDirectory -SourceRoot $SourceRoot -ToolPath $tool | Out-Null diff --git a/contrib/win32/openssh/code_coverage/Invoke-AzDOCoverageAggregate.ps1 b/contrib/win32/openssh/code_coverage/Invoke-AzDOCoverageAggregate.ps1 new file mode 100644 index 000000000000..be9a2c76706c --- /dev/null +++ b/contrib/win32/openssh/code_coverage/Invoke-AzDOCoverageAggregate.ps1 @@ -0,0 +1,67 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +<# + .SYNOPSIS + Azure DevOps aggregation entry point for Win32-OpenSSH code coverage. + + .DESCRIPTION + The Core and Bash suites are measured in separate CI jobs (each on its own + agent with a clean OpenSSH install, so there is no cross-suite file-lock or + service-state contention). Each job publishes its per-suite .coverage + + Cobertura report as an artifact. This script runs in a dependent job that has + downloaded those artifacts into -InputDirectory; it aggregates every per-suite + report it finds into a single de-duplicated report plus per-suite/overlap + summaries under -OutputDirectory. + + .PARAMETER InputDirectory + Directory containing the downloaded per-suite coverage artifacts (searched + recursively for *.coverage and *.cobertura.xml). + + .PARAMETER OutputDirectory + Where the merged report and summaries are written. Defaults to + -InputDirectory. + + .PARAMETER SourceRoot + Repository root used to scope coverage to OpenSSH source files. + + .EXAMPLE + .\Invoke-AzDOCoverageAggregate.ps1 -InputDirectory C:\cov -SourceRoot C:\src +#> +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string] $InputDirectory, + + [string] $OutputDirectory, + + [string] $SourceRoot +) + +$ErrorActionPreference = 'Stop' + +if (-not $OutputDirectory) { $OutputDirectory = $InputDirectory } +if (-not $SourceRoot) { + $SourceRoot = (Resolve-Path (Join-Path $PSScriptRoot '..' '..' '..' '..')).Path +} + +Import-Module (Join-Path $PSScriptRoot 'OpenSSHCodeCoverage.psm1') -Force + +$null = New-Item -ItemType Directory -Path $OutputDirectory -Force + +Write-Host "InputDirectory : $InputDirectory" +Write-Host "OutputDirectory : $OutputDirectory" +Write-Host "SourceRoot : $SourceRoot" + +# If aggregating into a different directory than the inputs, bring the per-suite +# artifacts across so the native .coverage merge and the Cobertura scan see them. +if ((Resolve-Path -LiteralPath $OutputDirectory).Path -ne (Resolve-Path -LiteralPath $InputDirectory).Path) { + Copy-Item -Path (Join-Path $InputDirectory '*') -Destination $OutputDirectory -Recurse -Force +} + +$result = Invoke-CoverageAggregation -OutputDirectory $OutputDirectory -SourceRoot $SourceRoot +if (-not $result) { + throw "No per-suite coverage reports were found under '$InputDirectory'; nothing to aggregate." +} + +Write-Host "Merged report : $($result.MergedReport)" diff --git a/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.psm1 b/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.psm1 new file mode 100644 index 000000000000..4b7d44e5af85 --- /dev/null +++ b/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.psm1 @@ -0,0 +1,1043 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +<# + OpenSSHCodeCoverage.psm1 + + Helpers to estimate C code coverage of the Win32-OpenSSH solution + (contrib\win32\openssh\Win32-OpenSSH.sln) across the three test suites + that ship with this repository: + + * unit tests (regress\unittests\*, built as unittest-*.exe) + * Pester E2E (regress\pesterTests\*.Tests.ps1) + * bash tests (regress\*.sh, driven by bash_tests_iterator.ps1) + + Native C coverage on MSVC is collected with Microsoft.CodeCoverage.Console + (https://learn.microsoft.com/visualstudio/test/microsoft-code-coverage-console-tool), + the Microsoft-maintained coverage tool that ships with Visual Studio 2022 + (17.3+, Enterprise edition provides native C/C++ support). Unlike a + debugger-attach tool, it uses *static instrumentation*: the OpenSSH binaries + are built with the /PROFILE linker switch (see the dedicated coverage build + job) and rewritten on disk by the `instrument` command, embedding a shared + session id. + + Collection uses server mode. For each suite we start a background collector + (`collect --session-id --server-mode`) and then run the suite. Every + instrumented OpenSSH process that executes while the collector owns the + session - including sshd.exe running as a Windows *service*, which is not a + child of our process - rendezvouses with the collector by session id. That + is why server mode is used rather than a simple child-process wrap: the + E2E/bash suites drive a real sshd service. `shutdown ` flushes each + suite's .coverage file, which we convert to Cobertura via `merge`. + + The per-suite .coverage files are then merged (`merge ... -f cobertura`), + which unions per-line hit counts, so a line exercised by two different + suites is only counted once. That natural de-duplication is what lets us + aggregate the suites and account for overlap. + + This module intentionally separates *pure* functions (Cobertura parsing, + line merging, overlap measurement, summary formatting) from the functions + that shell out to Microsoft.CodeCoverage.Console / MSBuild. The pure + functions carry the aggregation logic and are covered by + OpenSSHCodeCoverage.tests.ps1. +#> + +$ErrorActionPreference = 'Stop' + +# Repository root is four levels up from contrib\win32\openssh\code_coverage. +$script:RepositoryRoot = (Resolve-Path (Join-Path $PSScriptRoot '..' '..' '..' '..')).Path + +# Source paths (normalized, repository-relative, forward-slash) matching this +# pattern are third-party dependencies vendored into the tree - primarily the +# vcpkg-built libraries such as zlib. They are not OpenSSH code, are never +# exercised by our test suites in a meaningful way, and only dilute the +# coverage estimate, so they are excluded from every coverage map. +$script:CoverageExcludePattern = '(?i)(^|/)vcpkg/' + +#region Pure helpers (unit tested) + +<# + .SYNOPSIS + Returns $true when a normalized source path should be excluded from the + coverage estimate (e.g. vendored third-party dependencies). +#> +function Test-CoverageSourceExcluded { + [CmdletBinding()] + [OutputType([bool])] + param( + [Parameter(Mandatory = $true)] + [AllowEmptyString()] + [string] $NormalizedPath, + + [string] $ExcludePattern = $script:CoverageExcludePattern + ) + + if ([string]::IsNullOrWhiteSpace($NormalizedPath)) { return $false } + if ([string]::IsNullOrWhiteSpace($ExcludePattern)) { return $false } + return [regex]::IsMatch($NormalizedPath, $ExcludePattern) +} + +<# + .SYNOPSIS + Normalizes a source-file path emitted in a coverage report to a stable, + repository-relative, forward-slash path. + + .DESCRIPTION + The coverage tool records absolute paths for each source file. Those paths + differ between machines (developer box vs. CI agent) and use backslashes. + To merge reports produced on different machines - and to present readable + results - we strip the repository root (or the well-known CI checkout + prefixes) and normalize separators. +#> +function ConvertTo-NormalizedCoverageSourcePath { + [CmdletBinding()] + [OutputType([string])] + param( + [Parameter(Mandatory = $true)] + [AllowEmptyString()] + [string] $RawPath, + + # Repository root to strip. Defaults to this checkout's root. + [string] $RepositoryRoot = $script:RepositoryRoot + ) + + if ([string]::IsNullOrWhiteSpace($RawPath)) { + return $RawPath + } + + # Normalize separators first so every later comparison is forward-slash. + $normalized = $RawPath -replace '\\', '/' + + # Strip a provided repository root if the path lives underneath it. + if (-not [string]::IsNullOrWhiteSpace($RepositoryRoot)) { + $root = ($RepositoryRoot -replace '\\', '/').TrimEnd('/') + # Only strip when $root is an actual directory-boundary prefix (followed + # by '/' or the whole string), so a sibling like 'C:/repository' is not + # mistaken for a path under root 'C:/repo'. + if ($root -and $normalized.Length -ge $root.Length -and + $normalized.Substring(0, $root.Length).Equals($root, [System.StringComparison]::OrdinalIgnoreCase) -and + ($normalized.Length -eq $root.Length -or $normalized[$root.Length] -eq '/')) { + $normalized = $normalized.Substring($root.Length) + } + } + + # Strip well-known CI checkout prefixes: + # Azure Pipelines / GitHub Actions Windows: D:/a/// + # GitHub Actions Linux: /home/runner/work/// + # GitHub Actions macOS: /Users/runner/work/// + $ciPatterns = @( + '(?i)^[a-z]:/a/[^/]+/[^/]+/', + '(?i)^/home/[^/]+/work/[^/]+/[^/]+/', + '(?i)^/Users/[^/]+/work/[^/]+/[^/]+/' + ) + foreach ($pattern in $ciPatterns) { + $normalized = [regex]::Replace($normalized, $pattern, '') + } + + return $normalized.TrimStart('/') +} + +<# + .SYNOPSIS + Imports a Cobertura XML file into a coverage map. + + .DESCRIPTION + Returns a hashtable keyed by normalized source path. Each value is itself a + hashtable mapping line number (int) to hit count (int). This is the shape + consumed by Merge-CoverageData / Get-CoverageStatistic. +#> +function Import-CoberturaCoverage { + [CmdletBinding()] + [OutputType([hashtable])] + param( + [Parameter(Mandatory = $true)] + [string] $Path, + + [string] $RepositoryRoot = $script:RepositoryRoot + ) + + if (-not (Test-Path -LiteralPath $Path)) { + throw "Cobertura report not found: $Path" + } + + [xml] $xml = Get-Content -LiteralPath $Path -Raw + $map = New-Object 'System.Collections.Hashtable' ([System.StringComparer]::OrdinalIgnoreCase) + + foreach ($class in $xml.SelectNodes('//class')) { + $file = $class.GetAttribute('filename') + if ([string]::IsNullOrWhiteSpace($file)) { continue } + $key = ConvertTo-NormalizedCoverageSourcePath -RawPath $file -RepositoryRoot $RepositoryRoot + + # Skip vendored third-party sources (e.g. vcpkg/zlib) - they are not + # OpenSSH code and would only dilute the coverage estimate. + if (Test-CoverageSourceExcluded -NormalizedPath $key) { continue } + + if (-not $map.ContainsKey($key)) { + $map[$key] = @{} + } + $lines = $map[$key] + + foreach ($line in $class.SelectNodes('lines/line')) { + $number = [int] $line.GetAttribute('number') + $hits = [int] $line.GetAttribute('hits') + if ($lines.ContainsKey($number)) { + # Same line can appear more than once (e.g. inlined). Keep the max. + if ($hits -gt $lines[$number]) { $lines[$number] = $hits } + } + else { + $lines[$number] = $hits + } + } + } + + return $map +} + +<# + .SYNOPSIS + Merges any number of coverage maps into one, summing hit counts per line. + + .DESCRIPTION + This is the aggregation primitive. Where two suites cover the same file and + line, the hit counts are summed - the line ends up covered exactly once in + the merged map (its hit count is > 0), which is precisely how overlap is + absorbed. Lines unique to a single suite are carried through untouched. +#> +function Merge-CoverageData { + [CmdletBinding()] + [OutputType([hashtable])] + param( + [Parameter(Mandatory = $true)] + [AllowEmptyCollection()] + [hashtable[]] $CoverageMap + ) + + $merged = New-Object 'System.Collections.Hashtable' ([System.StringComparer]::OrdinalIgnoreCase) + + foreach ($map in $CoverageMap) { + if ($null -eq $map) { continue } + foreach ($file in $map.Keys) { + if (-not $merged.ContainsKey($file)) { + $merged[$file] = @{} + } + $target = $merged[$file] + foreach ($number in $map[$file].Keys) { + $hits = [int] $map[$file][$number] + if ($target.ContainsKey($number)) { + $target[$number] = [int] $target[$number] + $hits + } + else { + $target[$number] = $hits + } + } + } + } + + return $merged +} + +<# + .SYNOPSIS + Computes line-coverage statistics for a coverage map. +#> +function Get-CoverageStatistic { + [CmdletBinding()] + [OutputType([pscustomobject])] + param( + [Parameter(Mandatory = $true)] + [hashtable] $CoverageMap, + + # Optional label carried through onto the result (e.g. suite name). + [string] $Name + ) + + [int] $totalLines = 0 + [int] $coveredLines = 0 + $files = New-Object System.Collections.Generic.List[object] + + foreach ($file in ($CoverageMap.Keys | Sort-Object)) { + $lines = $CoverageMap[$file] + [int] $fileTotal = $lines.Count + [int] $fileCovered = 0 + foreach ($number in $lines.Keys) { + if ([int] $lines[$number] -gt 0) { $fileCovered++ } + } + $totalLines += $fileTotal + $coveredLines += $fileCovered + $files.Add([pscustomobject]@{ + File = $file + TotalLines = $fileTotal + CoveredLines = $fileCovered + LineRate = if ($fileTotal -gt 0) { [math]::Round($fileCovered / $fileTotal, 4) } else { 0 } + }) + } + + [pscustomobject]@{ + Name = $Name + TotalLines = $totalLines + CoveredLines = $coveredLines + LineRate = if ($totalLines -gt 0) { [math]::Round($coveredLines / $totalLines, 4) } else { 0 } + Percent = if ($totalLines -gt 0) { [math]::Round(100 * $coveredLines / $totalLines, 2) } else { 0 } + Files = $files + } +} + +<# + .SYNOPSIS + Measures how much the suites overlap. + + .DESCRIPTION + Given the per-suite coverage maps, returns: + * SumCoveredLines - naive sum of covered lines across suites + * CombinedCoveredLines - covered lines after merging (de-duplicated) + * OverlapLines - SumCoveredLines - CombinedCoveredLines, i.e. the + number of (file,line) coverage records that were + claimed by more than one suite + * OverlapPercent - OverlapLines as a percentage of SumCoveredLines + The combined figure is the honest aggregate coverage number; the overlap + figure quantifies redundant coverage between the suites. +#> +function Measure-CoverageOverlap { + [CmdletBinding()] + [OutputType([pscustomobject])] + param( + [Parameter(Mandatory = $true)] + [hashtable[]] $CoverageMap + ) + + [int] $sumCovered = 0 + foreach ($map in $CoverageMap) { + if ($null -eq $map) { continue } + $stat = Get-CoverageStatistic -CoverageMap $map + $sumCovered += $stat.CoveredLines + } + + $merged = Merge-CoverageData -CoverageMap $CoverageMap + $combined = (Get-CoverageStatistic -CoverageMap $merged).CoveredLines + $overlap = $sumCovered - $combined + + [pscustomobject]@{ + SumCoveredLines = $sumCovered + CombinedCoveredLines = $combined + OverlapLines = $overlap + OverlapPercent = if ($sumCovered -gt 0) { [math]::Round(100 * $overlap / $sumCovered, 2) } else { 0 } + } +} + +<# + .SYNOPSIS + Writes a coverage map back out as a minimal Cobertura XML report. + + .DESCRIPTION + Used to persist the helper-merged aggregate (independent of the native + Microsoft.CodeCoverage.Console merge) so the numbers can be diffed / + archived. +#> +function New-MergedCoberturaReport { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [hashtable] $CoverageMap, + + [Parameter(Mandatory = $true)] + [string] $OutputPath + ) + + $stat = Get-CoverageStatistic -CoverageMap $CoverageMap + $inv = [System.Globalization.CultureInfo]::InvariantCulture + $timestamp = [int][double]::Parse((Get-Date -UFormat %s), $inv) + $lineRateStr = $stat.LineRate.ToString($inv) + + $settings = New-Object System.Xml.XmlWriterSettings + $settings.Indent = $true + $settings.Encoding = [System.Text.Encoding]::UTF8 + + $writer = [System.Xml.XmlWriter]::Create($OutputPath, $settings) + try { + $writer.WriteStartDocument() + $writer.WriteStartElement('coverage') + $writer.WriteAttributeString('line-rate', $lineRateStr) + $writer.WriteAttributeString('branch-rate', '0') + $writer.WriteAttributeString('lines-covered', ([string]$stat.CoveredLines)) + $writer.WriteAttributeString('lines-valid', ([string]$stat.TotalLines)) + $writer.WriteAttributeString('version', '0') + $writer.WriteAttributeString('timestamp', ([string]$timestamp)) + + $writer.WriteStartElement('packages') + $writer.WriteStartElement('package') + $writer.WriteAttributeString('name', 'Win32-OpenSSH') + $writer.WriteAttributeString('line-rate', $lineRateStr) + $writer.WriteStartElement('classes') + + foreach ($file in ($CoverageMap.Keys | Sort-Object)) { + $lines = $CoverageMap[$file] + [int] $fileTotal = $lines.Count + [int] $fileCovered = 0 + foreach ($n in $lines.Keys) { if ([int] $lines[$n] -gt 0) { $fileCovered++ } } + $rate = if ($fileTotal -gt 0) { [math]::Round($fileCovered / $fileTotal, 4) } else { 0 } + + $writer.WriteStartElement('class') + $writer.WriteAttributeString('name', (Split-Path -Leaf $file)) + $writer.WriteAttributeString('filename', $file) + $writer.WriteAttributeString('line-rate', ([double]$rate).ToString($inv)) + $writer.WriteStartElement('lines') + foreach ($n in ($lines.Keys | Sort-Object { [int]$_ })) { + $writer.WriteStartElement('line') + $writer.WriteAttributeString('number', ([string]$n)) + $writer.WriteAttributeString('hits', ([string][int]$lines[$n])) + $writer.WriteEndElement() + } + $writer.WriteEndElement() # lines + $writer.WriteEndElement() # class + } + + $writer.WriteEndElement() # classes + $writer.WriteEndElement() # package + $writer.WriteEndElement() # packages + $writer.WriteEndElement() # coverage + $writer.WriteEndDocument() + } + finally { + $writer.Flush() + $writer.Close() + } + + return $OutputPath +} + +<# + .SYNOPSIS + Removes vendored third-party (excluded) classes from a Cobertura XML file + in place and recomputes the aggregate line counters. + + .DESCRIPTION + The native Microsoft.CodeCoverage.Console merge still contains every source + that was linked into the instrumented binaries, including vcpkg-built + dependencies such as zlib. This strips those nodes (matched via + Test-CoverageSourceExcluded against the normalized filename), drops any + package left empty, and recomputes lines-valid / lines-covered / line-rate + on each package and on the root so the published report matches the summary + estimate. Returns the number of classes removed. +#> +function Remove-ExcludedCoverageClasses { + [CmdletBinding()] + [OutputType([int])] + param( + [Parameter(Mandatory = $true)] + [string] $Path, + + [string] $RepositoryRoot = $script:RepositoryRoot + ) + + if (-not (Test-Path -LiteralPath $Path)) { + throw "Cobertura report not found: $Path" + } + + [xml] $xml = Get-Content -LiteralPath $Path -Raw + $inv = [System.Globalization.CultureInfo]::InvariantCulture + [int] $removed = 0 + + foreach ($class in @($xml.SelectNodes('//class'))) { + $file = $class.GetAttribute('filename') + if ([string]::IsNullOrWhiteSpace($file)) { continue } + $key = ConvertTo-NormalizedCoverageSourcePath -RawPath $file -RepositoryRoot $RepositoryRoot + if (Test-CoverageSourceExcluded -NormalizedPath $key) { + [void] $class.ParentNode.RemoveChild($class) + $removed++ + } + } + + # Recompute counters per and for the document root from the + # surviving elements so consumers report the excluded totals. + [int] $rootValid = 0 + [int] $rootCovered = 0 + foreach ($package in @($xml.SelectNodes('//package'))) { + $lines = $package.SelectNodes('.//line') + if (-not $lines -or $lines.Count -eq 0) { + [void] $package.ParentNode.RemoveChild($package) + continue + } + [int] $valid = 0 + [int] $covered = 0 + foreach ($line in $lines) { + $valid++ + if ([int] $line.GetAttribute('hits') -gt 0) { $covered++ } + } + $rate = if ($valid -gt 0) { [math]::Round($covered / $valid, 4) } else { 0 } + Set-CoberturaLineCounter -Node $package -Valid $valid -Covered $covered -Rate $rate -Culture $inv + $rootValid += $valid + $rootCovered += $covered + } + + $rootRate = if ($rootValid -gt 0) { [math]::Round($rootCovered / $rootValid, 4) } else { 0 } + Set-CoberturaLineCounter -Node $xml.DocumentElement -Valid $rootValid -Covered $rootCovered -Rate $rootRate -Culture $inv + + $xml.Save($Path) + return $removed +} + +# Sets/updates the lines-valid, lines-covered and line-rate attributes on a +# Cobertura node (only those already present are updated, plus line-rate/counts +# which every consumer expects). Kept private (not exported). +function Set-CoberturaLineCounter { + param( + [Parameter(Mandatory = $true)] [System.Xml.XmlElement] $Node, + [Parameter(Mandatory = $true)] [int] $Valid, + [Parameter(Mandatory = $true)] [int] $Covered, + [Parameter(Mandatory = $true)] [double] $Rate, + [Parameter(Mandatory = $true)] [System.Globalization.CultureInfo] $Culture + ) + + $Node.SetAttribute('line-rate', $Rate.ToString($Culture)) + if ($Node.HasAttribute('lines-valid')) { $Node.SetAttribute('lines-valid', ([string]$Valid)) } + if ($Node.HasAttribute('lines-covered')) { $Node.SetAttribute('lines-covered', ([string]$Covered)) } +} + +function Get-CoverageSummary { + [CmdletBinding()] + [OutputType([pscustomobject])] + param( + [Parameter(Mandatory = $true)] + [pscustomobject[]] $SuiteStatistic, + + [Parameter(Mandatory = $true)] + [pscustomobject] $CombinedStatistic, + + [Parameter(Mandatory = $true)] + [pscustomobject] $Overlap + ) + + [pscustomobject]@{ + GeneratedOn = (Get-Date).ToString('o') + Suites = @($SuiteStatistic | ForEach-Object { + [pscustomobject]@{ + Name = $_.Name + TotalLines = $_.TotalLines + CoveredLines = $_.CoveredLines + Percent = $_.Percent + } + }) + Combined = [pscustomobject]@{ + TotalLines = $CombinedStatistic.TotalLines + CoveredLines = $CombinedStatistic.CoveredLines + Percent = $CombinedStatistic.Percent + } + Overlap = $Overlap + } +} + +<# + .SYNOPSIS + Renders a coverage summary as a Markdown table string. +#> +function Format-CoverageSummaryMarkdown { + [CmdletBinding()] + [OutputType([string])] + param( + [Parameter(Mandatory = $true)] + [pscustomobject] $Summary + ) + + $sb = New-Object System.Text.StringBuilder + [void] $sb.AppendLine('# Win32-OpenSSH code coverage estimate') + [void] $sb.AppendLine('') + [void] $sb.AppendLine("Generated: $($Summary.GeneratedOn)") + [void] $sb.AppendLine('') + [void] $sb.AppendLine('| Suite | Covered | Total | Line % |') + [void] $sb.AppendLine('|-------|--------:|------:|-------:|') + foreach ($suite in $Summary.Suites) { + [void] $sb.AppendLine("| $($suite.Name) | $($suite.CoveredLines) | $($suite.TotalLines) | $($suite.Percent)% |") + } + [void] $sb.AppendLine("| **Combined (deduped)** | **$($Summary.Combined.CoveredLines)** | **$($Summary.Combined.TotalLines)** | **$($Summary.Combined.Percent)%** |") + [void] $sb.AppendLine('') + [void] $sb.AppendLine('## Overlap between suites') + [void] $sb.AppendLine('') + [void] $sb.AppendLine("- Sum of per-suite covered lines: $($Summary.Overlap.SumCoveredLines)") + [void] $sb.AppendLine("- Combined (de-duplicated) covered lines: $($Summary.Overlap.CombinedCoveredLines)") + [void] $sb.AppendLine("- Overlapping covered lines: $($Summary.Overlap.OverlapLines) ($($Summary.Overlap.OverlapPercent)% of the sum)") + return $sb.ToString() +} + +#endregion Pure helpers + +#region Microsoft.CodeCoverage.Console orchestration + +# Well-known OpenSSH executables that are built with /PROFILE and therefore can +# be statically instrumented. unittest-*.exe are matched separately by wildcard. +$script:OpenSSHCoverageExeNames = @( + 'ssh.exe', 'sshd.exe', 'sshd-session.exe', 'sftp.exe', 'sftp-server.exe', + 'scp.exe', 'ssh-add.exe', 'ssh-agent.exe', 'ssh-keygen.exe', + 'ssh-keyscan.exe', 'ssh-shellhost.exe', 'ssh-sk-helper.exe', + 'ssh-pkcs11-helper.exe' +) + +<# + .SYNOPSIS + Locates Microsoft.CodeCoverage.Console.exe. + + .DESCRIPTION + The tool ships with Visual Studio 2022 (17.3+) under + Common7\IDE\Extensions\Microsoft\CodeCoverage.Console. Native C/C++ coverage + requires the Enterprise edition. Resolution order: an explicit -Path, the + current PATH, vswhere-reported VS installations, then a scan of the standard + install roots. +#> +function Find-CodeCoverageConsole { + [CmdletBinding()] + [OutputType([string])] + param( + [string] $Path + ) + + if ($Path) { + if (Test-Path -LiteralPath $Path) { return (Resolve-Path -LiteralPath $Path).Path } + throw "Microsoft.CodeCoverage.Console.exe was not found at '$Path'." + } + + $existing = Get-Command 'Microsoft.CodeCoverage.Console.exe' -ErrorAction SilentlyContinue + if ($existing) { return $existing.Source } + + $relative = 'Common7\IDE\Extensions\Microsoft\CodeCoverage.Console\Microsoft.CodeCoverage.Console.exe' + + $vswhere = Join-Path ${env:ProgramFiles(x86)} 'Microsoft Visual Studio\Installer\vswhere.exe' + if (Test-Path $vswhere) { + $roots = & $vswhere -all -prerelease -property installationPath 2>$null + foreach ($root in $roots) { + if (-not $root) { continue } + $candidate = Join-Path $root $relative + if (Test-Path $candidate) { return (Resolve-Path -LiteralPath $candidate).Path } + } + } + + foreach ($base in @($env:ProgramFiles, ${env:ProgramFiles(x86)})) { + if (-not $base) { continue } + $vsRoot = Join-Path $base 'Microsoft Visual Studio' + if (-not (Test-Path $vsRoot)) { continue } + $found = Get-ChildItem -Path $vsRoot -Recurse -Filter 'Microsoft.CodeCoverage.Console.exe' -ErrorAction SilentlyContinue | + Select-Object -First 1 + if ($found) { return $found.FullName } + } + + throw @' +Microsoft.CodeCoverage.Console.exe was not found. It ships with Visual Studio 2022 (17.3 or later); native C/C++ code coverage requires the Enterprise edition. Install the "Code coverage" component, or run on an agent image that includes VS Enterprise (the hosted windows-latest image does). +'@ +} + +<# + .SYNOPSIS + Returns the OpenSSH binaries under a directory that should be instrumented. +#> +function Get-OpenSSHCoverageTarget { + [CmdletBinding()] + [OutputType([string[]])] + param( + [Parameter(Mandatory = $true)] [string] $BinaryDirectory + ) + + if (-not (Test-Path -LiteralPath $BinaryDirectory)) { + throw "Binary directory '$BinaryDirectory' does not exist." + } + + $targets = New-Object System.Collections.Generic.List[string] + foreach ($name in $script:OpenSSHCoverageExeNames) { + $p = Join-Path $BinaryDirectory $name + if (Test-Path -LiteralPath $p) { [void] $targets.Add((Resolve-Path -LiteralPath $p).Path) } + } + Get-ChildItem -Path $BinaryDirectory -Filter 'unittest-*.exe' -Recurse -ErrorAction SilentlyContinue | + ForEach-Object { [void] $targets.Add($_.FullName) } + + return $targets.ToArray() +} + +<# + .SYNOPSIS + Copies the static_covrun*.dll runtime next to the instrumented binaries. + + .DESCRIPTION + An instrumented native binary references static_covrun64.dll at load time. + The `collect`/`connect` commands add its directory to PATH automatically, but + the sshd/ssh-agent *services* are launched by the Service Control Manager and + do not inherit that PATH, so the runtime must sit beside the instrumented + binaries or the services fail to start. + + The runtime does not live next to Microsoft.CodeCoverage.Console.exe; it ships + under "\Team Tools\Dynamic Code Coverage Tools\" (with a + per-architecture subfolder), so the search starts from the VS install root + inferred from the tool path. +#> +function Copy-CoverageRuntime { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] [string] $ToolPath, + [Parameter(Mandatory = $true)] [string] $DestinationDirectory + ) + + $toolDir = Split-Path -Parent $ToolPath + + # The tool lives at \Common7\IDE\Extensions\Microsoft\CodeCoverage.Console; + # walk up to the VS install root (the parent of the "Common7" segment) so the + # sibling "Team Tools\Dynamic Code Coverage Tools" folder is in scope. + $installRoot = $null + $probe = $toolDir + while ($probe) { + if ((Split-Path -Leaf $probe) -eq 'Common7') { + $installRoot = Split-Path -Parent $probe + break + } + $parent = Split-Path -Parent $probe + if (-not $parent -or $parent -eq $probe) { break } + $probe = $parent + } + + # Prefer the known runtime folder, then fall back to broader recursive scans. + $runtimes = $null + if ($installRoot) { + $dynamicTools = Join-Path $installRoot 'Team Tools\Dynamic Code Coverage Tools' + if (Test-Path -LiteralPath $dynamicTools) { + $runtimes = Get-ChildItem -Path $dynamicTools -Recurse -Filter 'static_covrun*.dll' -ErrorAction SilentlyContinue + } + } + if (-not $runtimes) { + $runtimes = Get-ChildItem -Path $toolDir -Recurse -Filter 'static_covrun*.dll' -ErrorAction SilentlyContinue + } + if (-not $runtimes -and $installRoot) { + $runtimes = Get-ChildItem -Path $installRoot -Recurse -Filter 'static_covrun*.dll' -ErrorAction SilentlyContinue + } + + if (-not $runtimes) { + Write-Warning "No static_covrun*.dll found near '$ToolPath'; instrumented service binaries may fail to load." + return + } + + # Copy one DLL per file name (the arch variants have distinct names). + foreach ($rt in ($runtimes | Group-Object Name | ForEach-Object { $_.Group[0] })) { + Copy-Item -LiteralPath $rt.FullName -Destination $DestinationDirectory -Force + } +} + +<# + .SYNOPSIS + Statically instruments a set of native binaries with a shared session id. + + .DESCRIPTION + Each binary is first restored (best-effort uninstrument) so the function is + idempotent across repeated CI steps that share an installed OpenSSH + directory, then instrumented in place. Requires the binaries to have been + linked with /PROFILE. +#> +function Invoke-CoverageInstrument { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] [string[]] $BinaryPath, + [Parameter(Mandatory = $true)] [string] $SessionId, + [string] $ToolPath + ) + + if (-not $ToolPath) { $ToolPath = Find-CodeCoverageConsole } + $instrumented = New-Object System.Collections.Generic.List[string] + + foreach ($bin in $BinaryPath) { + if (-not (Test-Path -LiteralPath $bin)) { + Write-Warning "Instrument target '$bin' not found; skipping." + continue + } + # Restore first so re-running against an already-instrumented binary + # (e.g. Core step then Bash step) does not fail. + & $ToolPath uninstrument $bin --nologo 2>$null | Out-Null + + & $ToolPath instrument $bin --session-id $SessionId --nologo | Write-Verbose + if ($LASTEXITCODE -ne 0) { + throw "Instrumenting '$bin' failed with exit code $LASTEXITCODE. Ensure it was linked with /PROFILE." + } + [void] $instrumented.Add($bin) + } + + if ($instrumented.Count -eq 0) { + throw 'No binaries were instrumented; cannot collect coverage.' + } + + # Drop the coverage runtime next to every directory that holds an + # instrumented binary. + $dirs = $instrumented | ForEach-Object { Split-Path -Parent $_ } | Sort-Object -Unique + foreach ($dir in $dirs) { Copy-CoverageRuntime -ToolPath $ToolPath -DestinationDirectory $dir } + + return $instrumented.ToArray() +} + +<# + .SYNOPSIS + Restores (uninstruments) a set of native binaries. Best-effort. +#> +function Invoke-CoverageUninstrument { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] [string[]] $BinaryPath, + [string] $ToolPath + ) + if (-not $ToolPath) { $ToolPath = Find-CodeCoverageConsole } + foreach ($bin in $BinaryPath) { + if (-not (Test-Path -LiteralPath $bin)) { continue } + & $ToolPath uninstrument $bin --nologo 2>$null | Out-Null + } +} + +<# + .SYNOPSIS + Runs a suite under a server-mode coverage collector and returns the + resulting .coverage plus a per-suite Cobertura report. + + .DESCRIPTION + Starts `collect --session-id --server-mode` as a background process so + it owns the session, runs the suite command, then `shutdown ` to flush + the .coverage file. Every instrumented OpenSSH process (including sshd.exe + started as a service, which is not our child) that executes during the + window rendezvouses with the collector by session id. +#> +function Invoke-CoverageSession { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] [string] $Name, + [Parameter(Mandatory = $true)] [string] $Program, + [string[]] $ArgumentList = @(), + [Parameter(Mandatory = $true)] [string] $SessionId, + [Parameter(Mandatory = $true)] [string] $OutputDirectory, + [string] $WorkingDirectory, + [int] $CollectorReadySeconds = 5, + [string] $ToolPath + ) + + if (-not $ToolPath) { $ToolPath = Find-CodeCoverageConsole } + $null = New-Item -ItemType Directory -Path $OutputDirectory -Force + + $coverageOut = Join-Path $OutputDirectory "$Name.coverage" + $coberturaOut = Join-Path $OutputDirectory "$Name.cobertura.xml" + Remove-Item -LiteralPath $coverageOut -Force -ErrorAction SilentlyContinue + Remove-Item -LiteralPath $coberturaOut -Force -ErrorAction SilentlyContinue + + $collectorLog = Join-Path $OutputDirectory "$Name.collector.log" + $collector = Start-Process -FilePath $ToolPath -PassThru -NoNewWindow ` + -RedirectStandardOutput $collectorLog ` + -ArgumentList @( + 'collect', '--session-id', $SessionId, '--server-mode', + '--output', $coverageOut, '--output-format', 'coverage', '--nologo' + ) + + # Let the collector take ownership of the session before we run the suite. + Start-Sleep -Seconds $CollectorReadySeconds + + $exit = $null + try { + if ($WorkingDirectory) { Push-Location $WorkingDirectory } + & $Program @ArgumentList + $exit = $LASTEXITCODE + } + finally { + if ($WorkingDirectory) { Pop-Location } + & $ToolPath shutdown $SessionId --nologo 2>$null | Write-Verbose + } + + if ($collector -and -not $collector.HasExited) { + # shutdown should stop the collector; if it does not exit in time, kill it + # (best-effort) so we do not leak a collector into subsequent CI steps. + if (-not $collector.WaitForExit(120000)) { + Write-Warning "Coverage collector for suite '$Name' did not exit within 120s of shutdown; terminating it." + try { $collector.Kill() } catch { Write-Warning "Failed to terminate collector for suite '$Name': $($_.Exception.Message)" } + } + } + + if (Test-Path -LiteralPath $coverageOut) { + Convert-CoverageReport -InputPath $coverageOut -OutputPath $coberturaOut -Format cobertura -ToolPath $ToolPath | Out-Null + } + else { + Write-Warning "Collector produced no coverage file for suite '$Name'. See $collectorLog." + } + + [pscustomobject]@{ + Name = $Name + ExitCode = $exit + CoveragePath = $coverageOut + CoberturaPath = $coberturaOut + } +} + +<# + .SYNOPSIS + Merges/converts one or more .coverage (or coverage XML) inputs into a single + report, using the tool's native `merge` command. + + .DESCRIPTION + `merge` unions per-line hit counts across inputs, so merging every per-suite + .coverage yields the de-duplicated aggregate. Pass a single input to simply + convert it to another format. Supported -Format values: cobertura, xml, + coverage. +#> +function Convert-CoverageReport { + [CmdletBinding()] + [OutputType([string])] + param( + [Parameter(Mandatory = $true)] [string[]] $InputPath, + [Parameter(Mandatory = $true)] [string] $OutputPath, + [ValidateSet('cobertura', 'xml', 'coverage')] + [string] $Format = 'cobertura', + [string] $ToolPath + ) + + if (-not $ToolPath) { $ToolPath = Find-CodeCoverageConsole } + + $inputs = @($InputPath | Where-Object { $_ -and (Test-Path -LiteralPath $_) }) + if (-not $inputs) { throw 'No coverage inputs found to merge/convert.' } + + $outDir = Split-Path -Parent $OutputPath + if ($outDir) { $null = New-Item -ItemType Directory -Path $outDir -Force } + Remove-Item -LiteralPath $OutputPath -Force -ErrorAction SilentlyContinue + + $mergeArgs = @('merge') + $inputs + @('--output', $OutputPath, '--output-format', $Format, '--nologo') + Write-Verbose "Microsoft.CodeCoverage.Console $($mergeArgs -join ' ')" + & $ToolPath @mergeArgs | Write-Verbose + if ($LASTEXITCODE -ne 0) { + throw "Coverage merge/convert failed with exit code $LASTEXITCODE." + } + if (-not (Test-Path -LiteralPath $OutputPath)) { + throw "Coverage merge/convert did not produce '$OutputPath'." + } + + return $OutputPath +} + +<# + .SYNOPSIS + Aggregates every per-suite coverage report found under a directory into a + single de-duplicated report plus per-suite/overlap summaries. + + .DESCRIPTION + Scans -OutputDirectory (recursively) for per-suite *.cobertura.xml reports + (ignoring anything under a 'merged' folder) and *.coverage files. It: + - imports each per-suite Cobertura report and computes per-suite stats, + - merges the maps and measures cross-suite overlap (pure helpers), + - natively merges the .coverage files into merged\merged.cobertura.xml + (unioning per-line hits, so an overlapping line counts once); if the tool + or .coverage files are unavailable it falls back to the helper-merged + Cobertura so merged.cobertura.xml always exists for publishing, + - writes coverage-summary.json / coverage-summary.md. + + This is invoked either inline after a single-suite collection, or by the CI + aggregation job over the per-suite artifacts downloaded from separate + Core/Bash jobs. +#> +function Invoke-CoverageAggregation { + [CmdletBinding()] + [OutputType([pscustomobject])] + param( + [Parameter(Mandatory = $true)] [string] $OutputDirectory, + [Parameter(Mandatory = $true)] [string] $SourceRoot, + [string] $ToolPath + ) + + $coberturaReports = Get-ChildItem -Path $OutputDirectory -Filter '*.cobertura.xml' -Recurse -ErrorAction SilentlyContinue | + Where-Object { $_.FullName -notmatch '[\\/]merged[\\/]' } + + if (-not $coberturaReports) { + Write-Warning 'No per-suite Cobertura reports were found; skipping aggregation.' + return $null + } + + $suiteMaps = @() + $suiteStats = @() + foreach ($report in $coberturaReports) { + $suiteName = [System.IO.Path]::GetFileNameWithoutExtension($report.Name) -replace '\.cobertura$', '' + $map = Import-CoberturaCoverage -Path $report.FullName -RepositoryRoot $SourceRoot + $suiteMaps += , $map + $suiteStats += Get-CoverageStatistic -CoverageMap $map -Name $suiteName + } + + $mergedMap = Merge-CoverageData -CoverageMap $suiteMaps + $combinedStat = Get-CoverageStatistic -CoverageMap $mergedMap -Name 'combined' + $overlap = Measure-CoverageOverlap -CoverageMap $suiteMaps + + $mergedDir = Join-Path $OutputDirectory 'merged' + # Regenerate the merge each run so the report stays complete; clear stale output first. + Remove-Item -LiteralPath $mergedDir -Recurse -Force -ErrorAction SilentlyContinue + $null = New-Item -ItemType Directory -Path $mergedDir -Force + + $nativeMerged = Join-Path $mergedDir 'merged.cobertura.xml' + $helperMerged = Join-Path $mergedDir 'merged-helper.cobertura.xml' + + # Authoritative native merge of every per-suite .coverage file (best-effort). + if (-not $ToolPath) { + try { $ToolPath = Find-CodeCoverageConsole } catch { $ToolPath = $null } + } + $coverageFiles = Get-ChildItem -Path $OutputDirectory -Filter '*.coverage' -Recurse -ErrorAction SilentlyContinue | + ForEach-Object { $_.FullName } + $nativeMergeOk = $false + if ($ToolPath -and $coverageFiles) { + try { + Convert-CoverageReport -InputPath $coverageFiles -OutputPath $nativeMerged -Format cobertura -ToolPath $ToolPath | Out-Null + $nativeMergeOk = Test-Path -LiteralPath $nativeMerged + } + catch { + Write-Warning "Native coverage merge failed: $($_.Exception.Message)" + } + } + + # Helper-merged Cobertura (independent of the native merge) + summaries. + New-MergedCoberturaReport -CoverageMap $mergedMap -OutputPath $helperMerged | Out-Null + + # Guarantee merged.cobertura.xml exists for the publish step even when the + # native merge was skipped (no tool / no .coverage) or failed. + if (-not $nativeMergeOk) { + Write-Warning 'Using helper-merged Cobertura as merged.cobertura.xml (native merge unavailable).' + Copy-Item -LiteralPath $helperMerged -Destination $nativeMerged -Force + } + + # Strip vendored third-party sources (e.g. vcpkg/zlib) from the published + # merged report so it matches the summary estimate. The helper-merged path + # is already filtered at import time; this only affects the native merge. + if ($nativeMergeOk) { + try { + $removedClasses = Remove-ExcludedCoverageClasses -Path $nativeMerged -RepositoryRoot $SourceRoot + if ($removedClasses -gt 0) { + Write-Host "Excluded $removedClasses third-party class entries from merged.cobertura.xml." + } + } + catch { + Write-Warning "Could not strip third-party sources from merged report: $($_.Exception.Message)" + } + } + + $summary = Get-CoverageSummary -SuiteStatistic $suiteStats -CombinedStatistic $combinedStat -Overlap $overlap + $summaryJson = Join-Path $OutputDirectory 'coverage-summary.json' + $summaryMd = Join-Path $OutputDirectory 'coverage-summary.md' + $summary | ConvertTo-Json -Depth 6 | Set-Content -Path $summaryJson + $markdown = Format-CoverageSummaryMarkdown -Summary $summary + $markdown | Set-Content -Path $summaryMd + Write-Host "`n$markdown" + + return [pscustomobject]@{ + MergedReport = $nativeMerged + SummaryJson = $summaryJson + SummaryMd = $summaryMd + } +} + +#endregion Microsoft.CodeCoverage.Console orchestration + +Export-ModuleMember -Function @( + 'ConvertTo-NormalizedCoverageSourcePath', + 'Test-CoverageSourceExcluded', + 'Import-CoberturaCoverage', + 'Merge-CoverageData', + 'Get-CoverageStatistic', + 'Measure-CoverageOverlap', + 'New-MergedCoberturaReport', + 'Remove-ExcludedCoverageClasses', + 'Get-CoverageSummary', + 'Format-CoverageSummaryMarkdown', + 'Find-CodeCoverageConsole', + 'Get-OpenSSHCoverageTarget', + 'Copy-CoverageRuntime', + 'Invoke-CoverageInstrument', + 'Invoke-CoverageUninstrument', + 'Invoke-CoverageSession', + 'Convert-CoverageReport', + 'Invoke-CoverageAggregation' +) diff --git a/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.tests.ps1 b/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.tests.ps1 new file mode 100644 index 000000000000..d6f6eb2522ff --- /dev/null +++ b/contrib/win32/openssh/code_coverage/OpenSSHCodeCoverage.tests.ps1 @@ -0,0 +1,254 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# +# Unit tests for the pure helper functions in OpenSSHCodeCoverage.psm1. +# These validate the aggregation / overlap logic without needing a build, +# Microsoft.CodeCoverage.Console, or any of the OpenSSH test suites to run. +# +# Run with Pester 5: Invoke-Pester -Path .\OpenSSHCodeCoverage.tests.ps1 +# + +BeforeAll { + Import-Module (Join-Path $PSScriptRoot 'OpenSSHCodeCoverage.psm1') -Force + + # Writes a minimal Cobertura report for one file to $TestDrive and returns the path. + function New-TestCobertura { + param( + [string] $FileName, + [string] $SourcePath, + [hashtable] $Lines, # line number -> hits + [string] $OutFile + ) + $sb = New-Object System.Text.StringBuilder + [void] $sb.AppendLine('') + [void] $sb.AppendLine('') + [void] $sb.AppendLine(' ') + [void] $sb.AppendLine(" ") + foreach ($n in ($Lines.Keys | Sort-Object { [int]$_ })) { + [void] $sb.AppendLine(" ") + } + [void] $sb.AppendLine(' ') + [void] $sb.AppendLine(' ') + [void] $sb.AppendLine('') + Set-Content -Path $OutFile -Value $sb.ToString() -NoNewline + return $OutFile + } +} + +Describe 'ConvertTo-NormalizedCoverageSourcePath' { + Context 'Repository-root-relative paths' { + It 'Strips the provided repository root and normalizes separators' { + $result = ConvertTo-NormalizedCoverageSourcePath -RawPath 'C:\src\openssh-portable\channels.c' -RepositoryRoot 'C:\src\openssh-portable' + $result | Should -Be 'channels.c' + } + + It 'Strips a nested source path under the repository root' { + $result = ConvertTo-NormalizedCoverageSourcePath -RawPath 'C:\src\openssh-portable\openbsd-compat\bsd-misc.c' -RepositoryRoot 'C:\src\openssh-portable' + $result | Should -Be 'openbsd-compat/bsd-misc.c' + } + + It 'Is case-insensitive about the repository root' { + $result = ConvertTo-NormalizedCoverageSourcePath -RawPath 'C:\SRC\OpenSSH-Portable\channels.c' -RepositoryRoot 'C:\src\openssh-portable' + $result | Should -Be 'channels.c' + } + + It 'Does not strip a sibling directory that merely shares a prefix' { + $result = ConvertTo-NormalizedCoverageSourcePath -RawPath 'C:\repository\file.c' -RepositoryRoot 'C:\repo' + $result | Should -Be 'C:/repository/file.c' + } + } + + Context 'CI checkout prefixes' { + It 'Strips the Azure Pipelines / GitHub Actions Windows prefix' { + $result = ConvertTo-NormalizedCoverageSourcePath -RawPath 'D:\a\openssh-portable\openssh-portable\sshconnect2.c' -RepositoryRoot '' + $result | Should -Be 'sshconnect2.c' + } + + It 'Strips the GitHub Actions Linux prefix' { + $result = ConvertTo-NormalizedCoverageSourcePath -RawPath '/home/runner/work/openssh-portable/openssh-portable/kex.c' -RepositoryRoot '' + $result | Should -Be 'kex.c' + } + } + + Context 'Already relative paths' { + It 'Normalizes backslashes and leaves the path otherwise unchanged' { + $result = ConvertTo-NormalizedCoverageSourcePath -RawPath 'openbsd-compat\bsd-misc.c' -RepositoryRoot 'C:\src\openssh-portable' + $result | Should -Be 'openbsd-compat/bsd-misc.c' + } + } +} + +Describe 'Import-CoberturaCoverage' { + It 'Parses lines and hits keyed by normalized path' { + $file = Join-Path $TestDrive 'a.cobertura.xml' + New-TestCobertura -FileName 'channels.c' -SourcePath 'C:\repo\channels.c' -Lines @{ 1 = 5; 2 = 0; 3 = 2 } -OutFile $file | Out-Null + + $map = Import-CoberturaCoverage -Path $file -RepositoryRoot 'C:\repo' + $map.Keys.Count | Should -Be 1 + $map['channels.c'][1] | Should -Be 5 + $map['channels.c'][2] | Should -Be 0 + $map['channels.c'][3] | Should -Be 2 + } + + It 'Skips vendored third-party (vcpkg) sources' { + $file = Join-Path $TestDrive 'tp.cobertura.xml' + New-TestCobertura -FileName 'deflate.c' -SourcePath 'C:\repo\vcpkg\buildtrees\zlib\src\deflate.c' -Lines @{ 1 = 5; 2 = 3 } -OutFile $file | Out-Null + + $map = Import-CoberturaCoverage -Path $file -RepositoryRoot 'C:\repo' + $map.Keys.Count | Should -Be 0 + } +} + +Describe 'Test-CoverageSourceExcluded' { + It 'Excludes vcpkg-vendored paths' { + Test-CoverageSourceExcluded -NormalizedPath 'vcpkg/buildtrees/zlib/src/deflate.c' | Should -BeTrue + } + + It 'Excludes vcpkg even when nested deeper in the tree' { + Test-CoverageSourceExcluded -NormalizedPath 'contrib/win32/vcpkg/x.c' | Should -BeTrue + } + + It 'Does not exclude OpenSSH sources' { + Test-CoverageSourceExcluded -NormalizedPath 'channels.c' | Should -BeFalse + Test-CoverageSourceExcluded -NormalizedPath 'contrib/win32/win32compat/ansiprsr.c' | Should -BeFalse + } +} + +Describe 'Remove-ExcludedCoverageClasses' { + It 'Removes third-party classes and recomputes counters' { + $xml = @' + + + + + + + + + + + + + + + + + + + + + + +'@ + $file = Join-Path $TestDrive 'merged.cobertura.xml' + Set-Content -Path $file -Value $xml -NoNewline + + $removed = Remove-ExcludedCoverageClasses -Path $file -RepositoryRoot 'C:\repo' + $removed | Should -Be 2 + + [xml] $doc = Get-Content -LiteralPath $file -Raw + # Only the ssh package with channels.c survives. + $doc.SelectNodes('//package').Count | Should -Be 1 + $doc.SelectNodes('//class').Count | Should -Be 1 + $doc.SelectSingleNode('//class').GetAttribute('filename') | Should -Match 'channels\.c$' + # Root counters recomputed from surviving lines: 1 covered of 2 valid. + $doc.coverage.GetAttribute('lines-valid') | Should -Be '2' + $doc.coverage.GetAttribute('lines-covered') | Should -Be '1' + $doc.coverage.GetAttribute('line-rate') | Should -Be '0.5' + } +} + + +Describe 'Merge-CoverageData' { + Context 'Overlapping coverage of the same file/line' { + It 'Sums hit counts so an overlapping line stays covered once' { + $suiteA = @{ 'channels.c' = @{ 1 = 5; 2 = 3; 3 = 0 } } + $suiteB = @{ 'channels.c' = @{ 1 = 1; 2 = 0; 3 = 4 } } + + $merged = Merge-CoverageData -CoverageMap @($suiteA, $suiteB) + $merged['channels.c'][1] | Should -Be 6 # 5 + 1 + $merged['channels.c'][2] | Should -Be 3 # 3 + 0 + $merged['channels.c'][3] | Should -Be 4 # 0 + 4 + + $stat = Get-CoverageStatistic -CoverageMap $merged + $stat.CoveredLines | Should -Be 3 # all three lines covered + $stat.TotalLines | Should -Be 3 + } + } + + Context 'Non-overlapping files from different suites' { + It 'Keeps both files' { + $suiteA = @{ 'foo.c' = @{ 1 = 5 } } + $suiteB = @{ 'bar.c' = @{ 1 = 3 } } + $merged = Merge-CoverageData -CoverageMap @($suiteA, $suiteB) + $merged.Keys.Count | Should -Be 2 + } + } +} + +Describe 'Get-CoverageStatistic' { + It 'Counts only lines with non-zero hits as covered' { + $map = @{ 'x.c' = @{ 1 = 1; 2 = 0; 3 = 0; 4 = 2 } } + $stat = Get-CoverageStatistic -CoverageMap $map -Name 'unit' + $stat.Name | Should -Be 'unit' + $stat.TotalLines | Should -Be 4 + $stat.CoveredLines | Should -Be 2 + $stat.Percent | Should -Be 50 + } +} + +Describe 'Measure-CoverageOverlap' { + It 'Reports overlap as sum-minus-combined covered lines' { + # Suite A covers lines 1,2 of foo.c. Suite B covers lines 2,3 of foo.c. + # Sum covered = 2 + 2 = 4. Combined covered = 3 (lines 1,2,3). Overlap = 1 (line 2). + $suiteA = @{ 'foo.c' = @{ 1 = 1; 2 = 1 } } + $suiteB = @{ 'foo.c' = @{ 2 = 1; 3 = 1 } } + + $overlap = Measure-CoverageOverlap -CoverageMap @($suiteA, $suiteB) + $overlap.SumCoveredLines | Should -Be 4 + $overlap.CombinedCoveredLines | Should -Be 3 + $overlap.OverlapLines | Should -Be 1 + $overlap.OverlapPercent | Should -Be 25 + } + + It 'Reports zero overlap for disjoint suites' { + $suiteA = @{ 'foo.c' = @{ 1 = 1 } } + $suiteB = @{ 'bar.c' = @{ 1 = 1 } } + $overlap = Measure-CoverageOverlap -CoverageMap @($suiteA, $suiteB) + $overlap.OverlapLines | Should -Be 0 + $overlap.OverlapPercent | Should -Be 0 + } +} + +Describe 'New-MergedCoberturaReport' { + It 'Round-trips a coverage map through Cobertura XML' { + $map = @{ 'channels.c' = @{ 1 = 5; 2 = 0; 3 = 2 } } + $out = Join-Path $TestDrive 'merged.cobertura.xml' + New-MergedCoberturaReport -CoverageMap $map -OutputPath $out | Out-Null + + $reloaded = Import-CoberturaCoverage -Path $out -RepositoryRoot '' + $reloaded['channels.c'][1] | Should -Be 5 + $reloaded['channels.c'][2] | Should -Be 0 + $reloaded['channels.c'][3] | Should -Be 2 + } +} + +Describe 'Get-CoverageSummary / Format-CoverageSummaryMarkdown' { + It 'Produces a markdown table with a combined row and overlap section' { + $suiteA = @{ 'foo.c' = @{ 1 = 1; 2 = 1 } } + $suiteB = @{ 'foo.c' = @{ 2 = 1; 3 = 1 } } + $statA = Get-CoverageStatistic -CoverageMap $suiteA -Name 'unit' + $statB = Get-CoverageStatistic -CoverageMap $suiteB -Name 'bash' + $merged = Merge-CoverageData -CoverageMap @($suiteA, $suiteB) + $combined = Get-CoverageStatistic -CoverageMap $merged -Name 'combined' + $overlap = Measure-CoverageOverlap -CoverageMap @($suiteA, $suiteB) + + $summary = Get-CoverageSummary -SuiteStatistic @($statA, $statB) -CombinedStatistic $combined -Overlap $overlap + $summary.Combined.CoveredLines | Should -Be 3 + + $md = Format-CoverageSummaryMarkdown -Summary $summary + $md | Should -Match 'Combined \(deduped\)' + $md | Should -Match 'Overlapping covered lines: 1' + } +} diff --git a/contrib/win32/openssh/code_coverage/README.md b/contrib/win32/openssh/code_coverage/README.md new file mode 100644 index 000000000000..fa20e69493d0 --- /dev/null +++ b/contrib/win32/openssh/code_coverage/README.md @@ -0,0 +1,185 @@ +# Win32-OpenSSH code coverage + +Tooling to produce a **C code coverage estimate** for the Windows solution +(`contrib\win32\openssh\Win32-OpenSSH.sln`) across all three test suites that +ship with this repository, and to **aggregate** them into a single +de-duplicated number that accounts for overlap between the suites. + +## Files + +| File | Purpose | +|------|---------| +| `OpenSSHCodeCoverage.psm1` | Module. Pure aggregation/overlap helpers **and** Microsoft.CodeCoverage.Console orchestration. | +| `OpenSSHCodeCoverage.tests.ps1` | Pester 5 tests for the pure helpers (no build required). | +| `Invoke-AzDOCodeCoverage.ps1` | Per-suite CI entry point: run one suite (Core/Bash) under coverage against an installed OpenSSH dir, producing a per-suite report. | +| `Invoke-AzDOCoverageAggregate.ps1` | CI aggregation entry point: merge the per-suite artifacts from the Core and Bash jobs into the combined, de-duplicated report + summaries. | + +All files live in `contrib\win32\openssh\code_coverage\`. + +## Why Microsoft.CodeCoverage.Console + +[Microsoft.CodeCoverage.Console](https://learn.microsoft.com/visualstudio/test/microsoft-code-coverage-console-tool) +is the Microsoft-maintained coverage tool that ships with Visual Studio 2022 +(17.3+). Native C/C++ coverage requires the **Enterprise** edition (the hosted +`windows-latest` Azure DevOps image includes it). It: + +- collects **native C/C++** line coverage via **static instrumentation** — the + binaries are built with the `/PROFILE` linker switch and rewritten on disk by + the `instrument` command, +- supports **server mode** (`collect --session-id --server-mode`): every + instrumented process that runs while the collector owns the session reports + in by session id — including `sshd.exe` running as a Windows **service**, + which is *not* a child of the collector, +- emits `.coverage` (re-mergeable) and, via `merge`, **Cobertura XML**. + +Server mode is why coverage captures the E2E/bash suites: those drive a real +sshd service plus `ssh.exe`/`sftp.exe` clients, none of which are children of a +single wrapped process. + +## How aggregation and overlap work + +Each suite is measured independently and produces its own `.coverage` + +Cobertura report. The `.coverage` files are then merged natively +(`merge ... -f cobertura`). Merging **unions per-line hit counts**, so a line +exercised by two suites is counted exactly once in the combined total — that is +the de-duplication that makes the aggregate honest. + +The pure helpers additionally quantify the redundancy: + +``` +SumCoveredLines = covered lines summed across suites (double counts overlap) +CombinedCoveredLines = covered lines after merge (each line once) <- the estimate +OverlapLines = SumCoveredLines - CombinedCoveredLines +``` + +### Third-party exclusion + +Vendored dependencies built through vcpkg (e.g. zlib) are linked into the +OpenSSH binaries and therefore show up in the raw coverage data, but they are +not OpenSSH code and only dilute the estimate. Any source whose +repository-relative path matches `$script:CoverageExcludePattern` (default +`(^|/)vcpkg/`) is dropped: `Import-CoberturaCoverage` skips it in the summary, +and `Remove-ExcludedCoverageClasses` strips it from the published +`merged.cobertura.xml` (recomputing the package/root line counters) so the +Cobertura report matches the summary. + +## Usage + +Coverage normally runs in CI (see below). To reproduce locally you need +Visual Studio 2022 **Enterprise** (17.3+) — for both the build tools and +`Microsoft.CodeCoverage.Console.exe` — plus the test-suite prerequisites +(Cygwin for the bash suite, Pester for E2E; the existing helpers install these). + +First build and install the solution with `/PROFILE` so the binaries can be +instrumented (the CI build job does this by injecting `/PROFILE` via the linker +`LINK` env var), then point `Invoke-AzDOCodeCoverage.ps1` at the installed +directory. The core flow ends by uninstalling OpenSSH, so re-install before the +bash suite (in CI each suite runs on its own agent, so this is unnecessary +there): + +```powershell +cd contrib\win32\openssh\code_coverage + +# Core = setup + unit + E2E, into one output dir: +.\Invoke-AzDOCodeCoverage.ps1 -Suite Core -OpenSSHBinPath C:\OpenSSH -OutputDirectory C:\cov -SkipAggregation + +# Re-install OpenSSH (the core flow's uninstall test removed it), then bash: +Install-OpenSSH -SourceDir -OpenSSHDir C:\OpenSSH +.\Invoke-AzDOCodeCoverage.ps1 -Suite Bash -OpenSSHBinPath C:\OpenSSH -OutputDirectory C:\cov -SkipAggregation + +# Combine the per-suite reports into the de-duplicated aggregate: +.\Invoke-AzDOCoverageAggregate.ps1 -InputDirectory C:\cov +``` + +Omit `-SkipAggregation` on a single-suite run to get the merge/summary for just +that suite. + +### Output artifacts (under `-OutputDirectory`) + +``` +core\core.coverage, core\core.cobertura.xml per-suite (setup + unit + E2E) +bash\bash.coverage, bash\bash.cobertura.xml per-suite (bash tests) +merged\merged.cobertura.xml combined, native merge (per-binary) +merged\merged-helper.cobertura.xml combined, de-duplicated by source file +coverage-summary.json machine-readable summary +coverage-summary.md per-suite + combined + overlap table +``` + +Two combined reports are emitted because they answer different questions: + +- **`merged-helper.cobertura.xml`** is de-duplicated by unique source file — a + `.c` linked into several binaries is counted once. This matches the + `coverage-summary.md` estimate and is the report published to the Azure DevOps + **Code Coverage widget** (see CI section). +- **`merged.cobertura.xml`** is the native Microsoft.CodeCoverage.Console merge. + It preserves per-binary packages, per-method coverage, cyclomatic complexity + and branch data, but counts each shared `.c` once **per binary** it is linked + into (~3.6x inflation on this solution). It is kept in the uploaded artifact + for per-binary / per-method drill-down (e.g. reopening the `.coverage` files in + Visual Studio). + +Example `coverage-summary.md`: + +``` +| Suite | Covered | Total | Line % | +|-------|--------:|------:|-------:| +| core | 9633 | 20144 | 47.8% | +| bash | 11002 | 20144 | 54.6% | +| **Combined (deduped)** | **13120** | **20144** | **65.1%** | + +## Overlap between suites +- Sum of per-suite covered lines: 20635 +- Combined (de-duplicated) covered lines: 13120 +- Overlapping covered lines: 7515 (36.42% of the sum) +``` + +## Validating the helpers + +The aggregation/overlap logic is unit tested and does **not** require a build, +Microsoft.CodeCoverage.Console, or the OpenSSH suites: + +```powershell +Invoke-Pester -Path .\OpenSSHCodeCoverage.tests.ps1 -Output Detailed +``` + +## Continuous integration (Azure DevOps) + +Coverage is wired into `.azdo/ci.yml` as dedicated, **PR-only, non-gating** jobs +(they never run on branch builds, where CodeQL already competes for the ~60 min +agent budget, and `continueOnError: true` so they never block a merge): + +1. **Build Coverage Package (x64 Debug + /PROFILE)** — a Build-stage job that + builds the solution `Debug|x64` with `/PROFILE` injected via the linker + `LINK` env var (no `.vcxproj` edits), publishing `Win32-OpenSSH-Coverage-x64` + and `UnitTests-Coverage-x64` (binaries + PDBs). It runs in parallel with the + normal Release build. +2. **Win32-OpenSSH Code Coverage (Core)** and **(Bash)** — two Test-stage jobs + that each install the coverage build to `C:\OpenSSH` on their own agent and + run `Invoke-AzDOCodeCoverage.ps1 -Suite Core` (setup + unit + E2E) or + `-Suite Bash` with `-SkipAggregation`. Running the suites in separate jobs + (instead of sequentially in one job) gives each a clean install with no + file-lock or service-state contention, and lets them run in parallel. Each + publishes its per-suite report as `Win32-OpenSSH-CodeCoverage-Core` / + `-Bash`. +3. **Win32-OpenSSH Code Coverage (Aggregate)** — a Test-stage job that depends on + the two suite jobs, downloads their per-suite artifacts, and runs + `Invoke-AzDOCoverageAggregate.ps1` to produce the merged reports. It publishes + the **de-duplicated** report (`merged/merged-helper.cobertura.xml`) via + `PublishCodeCoverageResults@2` so the Code Coverage widget headline matches the + `coverage-summary.md` estimate, and uploads the full `Win32-OpenSSH-CodeCoverage` + artifact (per-suite reports, both merged reports, the raw `.coverage` files, and + summaries) for per-binary / per-method drill-down. + +The suites run the exact CI entry points (`Invoke-OpenSSHTests`, +`Invoke-OpenSSHBashTestsOnly`), so coverage reflects what CI already exercises. + +## Notes + +- A **Debug** build is used for coverage so line mapping is accurate; optimized + Release builds can fold/reorder/strip lines and understate coverage. +- Native C/C++ coverage requires **Visual Studio 2022 Enterprise** and binaries + linked with `/PROFILE`; the coverage build job handles both. +- Coverage is scoped to OpenSSH source files at report time (paths are + normalized to the repository root), so third-party/system code is excluded. +- The tooling reuses the existing suite entry points, so it measures exactly what + CI already runs.