diff --git a/NuGet.config b/NuGet.config index b117cae9d2c7..62c09dfe93dc 100644 --- a/NuGet.config +++ b/NuGet.config @@ -24,6 +24,7 @@ + @@ -33,10 +34,12 @@ + + @@ -63,12 +66,15 @@ + + + diff --git a/THIRD-PARTY-NOTICES.TXT b/THIRD-PARTY-NOTICES.TXT index 859bd1a6d9cd..97426f91879b 100644 --- a/THIRD-PARTY-NOTICES.TXT +++ b/THIRD-PARTY-NOTICES.TXT @@ -8,4 +8,4 @@ bring it to our attention. Post an issue or email us: The attached notices are provided for information only. -No notices are provided at this time. \ No newline at end of file +No notices are provided at this time. diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index e11fa81e92a0..ef6315e9e071 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -15,42 +15,42 @@ f5f6eb5420398b365107b9be9e5cb6290626e39f - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + d839c41c85988aadc213e8e42269ecd7883a1790 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + d839c41c85988aadc213e8e42269ecd7883a1790 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + d839c41c85988aadc213e8e42269ecd7883a1790 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + d839c41c85988aadc213e8e42269ecd7883a1790 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + d839c41c85988aadc213e8e42269ecd7883a1790 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + d839c41c85988aadc213e8e42269ecd7883a1790 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + d839c41c85988aadc213e8e42269ecd7883a1790 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + d839c41c85988aadc213e8e42269ecd7883a1790 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + d839c41c85988aadc213e8e42269ecd7883a1790 @@ -59,18 +59,18 @@ https://github.com/dotnet/core-setup 7d57652f33493fa022125b7f63aad0d70c52d810 - + https://github.com/dotnet/emsdk - b634e009d59f72e9254f984a6b89e685955e0eb8 + 8e475db5d45016c77714cce29077ae8c66d12d53 - + https://github.com/dotnet/emsdk - b634e009d59f72e9254f984a6b89e685955e0eb8 + 8e475db5d45016c77714cce29077ae8c66d12d53 - + https://github.com/dotnet/emsdk - b634e009d59f72e9254f984a6b89e685955e0eb8 + 8e475db5d45016c77714cce29077ae8c66d12d53 @@ -135,13 +135,13 @@ https://github.com/dotnet/roslyn fc52718eccdb37693a40a518b1178b1e23114e68 - - https://github.com/dotnet/aspnetcore - 3c5c32a79a7269c8ee7649d0ea1630f115d8400b + + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore + a95d6599f37762ec8394ba9ca01e8263b2e36a26 - https://github.com/dotnet/aspnetcore - 3c5c32a79a7269c8ee7649d0ea1630f115d8400b + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore + a95d6599f37762ec8394ba9ca01e8263b2e36a26 https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted @@ -230,99 +230,99 @@ bc9161306b23641b0364b8f93d546da4d48da1eb - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + d839c41c85988aadc213e8e42269ecd7883a1790 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + d839c41c85988aadc213e8e42269ecd7883a1790 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + d839c41c85988aadc213e8e42269ecd7883a1790 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + d839c41c85988aadc213e8e42269ecd7883a1790 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + d839c41c85988aadc213e8e42269ecd7883a1790 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + d839c41c85988aadc213e8e42269ecd7883a1790 - + https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - 9c74c8784a90d64bcc5c5c552d5420eafb60fa4f + 889bd3cf7e1c61b6bdee233190da4931118b6e6d - + https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - 9c74c8784a90d64bcc5c5c552d5420eafb60fa4f + 889bd3cf7e1c61b6bdee233190da4931118b6e6d - + https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - 9c74c8784a90d64bcc5c5c552d5420eafb60fa4f + 889bd3cf7e1c61b6bdee233190da4931118b6e6d - + https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - 9c74c8784a90d64bcc5c5c552d5420eafb60fa4f + 889bd3cf7e1c61b6bdee233190da4931118b6e6d - + https://dev.azure.com/dnceng/internal/_git/dotnet-wpf - a865df6e83bf8b34bbd6802fdcf25c5cdfb05191 + 15c7bd7f92ae827be47bfb0ca16660f8cd376189 - https://github.com/dotnet/aspnetcore - 3c5c32a79a7269c8ee7649d0ea1630f115d8400b + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore + a95d6599f37762ec8394ba9ca01e8263b2e36a26 - - https://github.com/dotnet/aspnetcore - 3c5c32a79a7269c8ee7649d0ea1630f115d8400b + + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore + a95d6599f37762ec8394ba9ca01e8263b2e36a26 - https://github.com/dotnet/aspnetcore - 3c5c32a79a7269c8ee7649d0ea1630f115d8400b + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore + a95d6599f37762ec8394ba9ca01e8263b2e36a26 - - https://github.com/dotnet/aspnetcore - 3c5c32a79a7269c8ee7649d0ea1630f115d8400b + + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore + a95d6599f37762ec8394ba9ca01e8263b2e36a26 - - https://github.com/dotnet/aspnetcore - 3c5c32a79a7269c8ee7649d0ea1630f115d8400b + + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore + a95d6599f37762ec8394ba9ca01e8263b2e36a26 - - https://github.com/dotnet/aspnetcore - 3c5c32a79a7269c8ee7649d0ea1630f115d8400b + + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore + a95d6599f37762ec8394ba9ca01e8263b2e36a26 - - https://github.com/dotnet/aspnetcore - 3c5c32a79a7269c8ee7649d0ea1630f115d8400b + + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore + a95d6599f37762ec8394ba9ca01e8263b2e36a26 - - https://github.com/dotnet/aspnetcore - 3c5c32a79a7269c8ee7649d0ea1630f115d8400b + + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore + a95d6599f37762ec8394ba9ca01e8263b2e36a26 - - https://github.com/dotnet/aspnetcore - 3c5c32a79a7269c8ee7649d0ea1630f115d8400b + + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore + a95d6599f37762ec8394ba9ca01e8263b2e36a26 - - https://github.com/dotnet/aspnetcore - 3c5c32a79a7269c8ee7649d0ea1630f115d8400b + + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore + a95d6599f37762ec8394ba9ca01e8263b2e36a26 - - https://github.com/dotnet/aspnetcore - 3c5c32a79a7269c8ee7649d0ea1630f115d8400b + + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore + a95d6599f37762ec8394ba9ca01e8263b2e36a26 - - https://github.com/dotnet/aspnetcore - 3c5c32a79a7269c8ee7649d0ea1630f115d8400b + + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore + a95d6599f37762ec8394ba9ca01e8263b2e36a26 @@ -344,20 +344,20 @@ - https://github.com/dotnet/aspnetcore - 3c5c32a79a7269c8ee7649d0ea1630f115d8400b + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore + a95d6599f37762ec8394ba9ca01e8263b2e36a26 - https://github.com/dotnet/aspnetcore - 3c5c32a79a7269c8ee7649d0ea1630f115d8400b + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore + a95d6599f37762ec8394ba9ca01e8263b2e36a26 - https://github.com/dotnet/aspnetcore - 3c5c32a79a7269c8ee7649d0ea1630f115d8400b + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore + a95d6599f37762ec8394ba9ca01e8263b2e36a26 - https://github.com/dotnet/aspnetcore - 3c5c32a79a7269c8ee7649d0ea1630f115d8400b + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore + a95d6599f37762ec8394ba9ca01e8263b2e36a26 https://github.com/dotnet/test-templates @@ -390,13 +390,13 @@ - + https://dev.azure.com/dnceng/internal/_git/dotnet-winforms - 4569ad0b3bd84191660c74228ad6d18d29fbc1cb + fa8da0fffee2a42efa08aac643276cd0c482c49c - + https://dev.azure.com/dnceng/internal/_git/dotnet-wpf - a865df6e83bf8b34bbd6802fdcf25c5cdfb05191 + 15c7bd7f92ae827be47bfb0ca16660f8cd376189 https://github.com/dotnet/xdt @@ -503,89 +503,89 @@ - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + d839c41c85988aadc213e8e42269ecd7883a1790 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + d839c41c85988aadc213e8e42269ecd7883a1790 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + d839c41c85988aadc213e8e42269ecd7883a1790 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + d839c41c85988aadc213e8e42269ecd7883a1790 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + d839c41c85988aadc213e8e42269ecd7883a1790 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + d839c41c85988aadc213e8e42269ecd7883a1790 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + d839c41c85988aadc213e8e42269ecd7883a1790 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + d839c41c85988aadc213e8e42269ecd7883a1790 - https://github.com/dotnet/aspnetcore - 3c5c32a79a7269c8ee7649d0ea1630f115d8400b + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore + a95d6599f37762ec8394ba9ca01e8263b2e36a26 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + d839c41c85988aadc213e8e42269ecd7883a1790 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + d839c41c85988aadc213e8e42269ecd7883a1790 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + d839c41c85988aadc213e8e42269ecd7883a1790 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + d839c41c85988aadc213e8e42269ecd7883a1790 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + d839c41c85988aadc213e8e42269ecd7883a1790 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + d839c41c85988aadc213e8e42269ecd7883a1790 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + d839c41c85988aadc213e8e42269ecd7883a1790 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + d839c41c85988aadc213e8e42269ecd7883a1790 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + d839c41c85988aadc213e8e42269ecd7883a1790 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + d839c41c85988aadc213e8e42269ecd7883a1790 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + d839c41c85988aadc213e8e42269ecd7883a1790 @@ -619,9 +619,9 @@ efce34e9f9f25af27e2b471fbbf8c21f1ac2e318 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + d839c41c85988aadc213e8e42269ecd7883a1790 https://github.com/dotnet/arcade-services diff --git a/eng/Versions.props b/eng/Versions.props index df86f73b1ef4..b25e3da3eccc 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -77,7 +77,7 @@ - 9.0.17-servicing.26265.1 + 9.0.18-servicing.26316.2 @@ -86,54 +86,54 @@ - 9.0.17 - 9.0.17-servicing.26264.16 - 9.0.17 - 9.0.17 - 9.0.17-servicing.26264.16 - 9.0.17 - 9.0.17 - 9.0.17 - 9.0.17 - 9.0.17 - 9.0.17 + 9.0.18 + 9.0.18-servicing.26315.22 + 9.0.18 + 9.0.18 + 9.0.18-servicing.26315.22 + 9.0.18 + 9.0.18 + 9.0.18 + 9.0.18 + 9.0.18 + 9.0.18 8.0.0-rc.1.23414.4 - 9.0.17-servicing.26264.16 - 9.0.17-servicing.26264.16 - 9.0.17 - 9.0.17 - 9.0.17 - 9.0.17 + 9.0.18-servicing.26315.22 + 9.0.18-servicing.26315.22 + 9.0.18 + 9.0.18 + 9.0.18 + 9.0.18 2.1.0 - 9.0.17 + 9.0.18 8.0.0 - 9.0.17 - 9.0.17 - 9.0.17 - 9.0.17 - 9.0.17 - 9.0.17 - 9.0.17 + 9.0.18 + 9.0.18 + 9.0.18 + 9.0.18 + 9.0.18 + 9.0.18 + 9.0.18 8.0.0 - 9.0.17 - 9.0.17 - 9.0.17 - 9.0.17 - 9.0.17 - 9.0.17 - 9.0.17 + 9.0.18 + 9.0.18 + 9.0.18 + 9.0.18 + 9.0.18 + 9.0.18 + 9.0.18 8.0.5 - 9.0.17 - 9.0.17 + 9.0.18 + 9.0.18 - 9.0.17-servicing.26268.1 - 9.0.17-servicing.26268.1 - 9.0.17 - 9.0.17 + 9.0.18-servicing.26317.1 + 9.0.18-servicing.26317.1 + 9.0.18 + 9.0.18 @@ -216,18 +216,18 @@ 9.0.18 - 9.0.18-servicing.26310.12 - 9.0.18-servicing.26310.12 - 9.0.18-servicing.26310.12 - 9.0.18-servicing.26310.12 - 9.0.18-servicing.26310.12 + 9.0.18-servicing.26323.11 + 9.0.18-servicing.26323.11 + 9.0.18-servicing.26323.11 + 9.0.18-servicing.26323.11 + 9.0.18-servicing.26323.11 9.0.18 9.0.18 - 9.0.18-servicing.26310.12 - 9.0.18-servicing.26310.12 - 9.0.18-servicing.26310.12 - 9.0.18-servicing.26310.12 - 9.0.18-servicing.26310.12 + 9.0.18-servicing.26323.11 + 9.0.18-servicing.26323.11 + 9.0.18-servicing.26323.11 + 9.0.18-servicing.26323.11 + 9.0.18-servicing.26323.11 @@ -237,8 +237,8 @@ - 9.0.17-rtm.26267.1 - 9.0.17-rtm.26267.1 + 9.0.18-rtm.26316.5 + 9.0.18-rtm.26316.5 @@ -317,8 +317,8 @@ 15.0.9617 18.0.9617 - 9.0.17-servicing.26258.3 - 9.0.17 + 9.0.18-servicing.26310.2 + 9.0.18 $(MicrosoftNETWorkloadEmscriptenCurrentManifest90100PackageVersion) 9.0.100$([System.Text.RegularExpressions.Regex]::Match($(EmscriptenWorkloadManifestVersion), `-(?!rtm)[A-z]*[\.]*\d*`)) diff --git a/eng/pipelines/templates/jobs/vmr-build.yml b/eng/pipelines/templates/jobs/vmr-build.yml index 9aeeae442145..95ba36e8a9b6 100644 --- a/eng/pipelines/templates/jobs/vmr-build.yml +++ b/eng/pipelines/templates/jobs/vmr-build.yml @@ -180,6 +180,26 @@ jobs: # Synchronize new content in the VMR during PRs (we expect this to come - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + - task: AzureCLI@2 + displayName: 'Mint AzDO token for VMR push' + inputs: + azureSubscription: 'dnceng-build-rw-code-rw-wif' + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + $token = az account get-access-token --resource 499b84ac-1321-427f-aa17-267ca6975798 --query accessToken -o tsv + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to acquire Azure DevOps access token via 'az account get-access-token'. Exit code: $LASTEXITCODE" + exit 1 + } + $token = $token.Trim() + if ([string]::IsNullOrWhiteSpace($token)) { + Write-Error "Received an empty or whitespace Azure DevOps access token from 'az account get-access-token'." + exit 1 + } + Write-Host "##vso[task.setvariable variable=AzdoToken;issecret=true]$token" + - template: ../steps/vmr-pull-updates.yml@self parameters: vmrPath: $(vmrPath) @@ -397,23 +417,6 @@ jobs: - script: cp "$(sourcesPath)/src/sdk/NuGet.config" "$(sourcesPath)/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/online.NuGet.Config" displayName: Copy Test NuGet Config for Smoke Tests - - ${{ else }}: - - task: Bash@3 - displayName: Add aspnetcore public feed for smoke tests - inputs: - targetType: inline - script: | - set -euo pipefail - - ONLINE_CONFIG="$(sourcesPath)/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/online.NuGet.Config" - FEED_KEY="darc-pub-dotnet-aspnetcore-3c5c32a7" - FEED_URL="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-aspnetcore-3c5c32a7/nuget/v3/index.json" - - # Add only if missing. Do not touch disabledPackageSources. - if ! grep -qi "| \n |" "$ONLINE_CONFIG" - fi - - script: | set -ex diff --git a/eng/pipelines/templates/jobs/vmr-synchronization.yml b/eng/pipelines/templates/jobs/vmr-synchronization.yml index 94d3331e2f62..db05c8491f9c 100644 --- a/eng/pipelines/templates/jobs/vmr-synchronization.yml +++ b/eng/pipelines/templates/jobs/vmr-synchronization.yml @@ -1,93 +1,111 @@ -### This job synchronizes code from product repositories into the VMR (https://github.com/dotnet/dotnet) -### It synchronizes the content of the VMR to this new commit and pushes the changes - -parameters: -- name: targetRef - displayName: Target revision of dotnet/sdk to synchronize - type: string - default: $(Build.SourceVersion) - -- name: vmrBranch - displayName: dotnet/dotnet branch to use - type: string - default: $(Build.SourceBranch) - -- name: noPush - displayName: Don't push changes to dotnet/dotnet - type: boolean - default: false - -jobs: -- job: Synchronize_VMR - displayName: Synchronize VMR's ${{ parameters.vmrBranch }} - timeoutInMinutes: 120 - variables: - - ${{ if eq(variables['System.TeamProject'], 'public') }}: - - template: /eng/common/templates/variables/pool-providers.yml - - ${{ else }}: - - template: /eng/common/templates-official/variables/pool-providers.yml - - name: vmrPath - value: $(Agent.BuildDirectory)/vmr - - ${{ if not(parameters.noPush) }}: - - ${{ if and( eq(variables['System.TeamProject'], 'internal'), or(startswith(variables['Build.SourceBranch'], 'refs/heads/release/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: - - group: DotNetBot-GitHub - - group: DotNetBot-GitHub-No-Scopes - - name: vmrPublicUrl - value: https://github.com/dotnet/dotnet - - ${{ if and( eq(variables['System.TeamProject'], 'internal'), startswith(variables['Build.SourceBranch'], 'refs/heads/internal/release/')) }}: - - name: vmrInternalUrl - value: https://dnceng@dev.azure.com/dnceng/internal/_git/dotnet-dotnet - - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: $(DncEngPublicBuildPool) - image: build.azurelinux.3.amd64.open - os: linux - ${{ else }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals build.azurelinux.3.amd64 - os: linux - - steps: - - template: ../steps/vmr-prepare.yml - parameters: - vmrBranch: ${{ parameters.vmrBranch }} - - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - template: /eng/common/templates-official/steps/get-federated-access-token.yml - parameters: - federatedServiceConnection: VmrSyncPipeline - outputVariableName: AzdoToken - - - template: ../steps/vmr-pull-updates.yml - parameters: - vmrPath: $(vmrPath) - vmrBranch: ${{ parameters.vmrBranch }} - targetRef: ${{ parameters.targetRef }} - - - ${{ if and(not(parameters.noPush), not(in(variables['Build.Reason'], 'PullRequest')), eq(variables['System.TeamProject'], 'internal')) }}: - # Push main and release branches to the public VMR - - ${{ if or(eq(parameters.vmrBranch, 'main'), startsWith(parameters.vmrBranch, 'release/')) }}: - - script: > - ./.dotnet/dotnet darc vmr push - --vmr '$(vmrPath)' - --commit-verification-pat '$(BotAccount-dotnet-maestro-bot-no-scopes-PAT)' - --branch '${{ parameters.vmrBranch }}' - --remote-url '$(vmrPublicUrl)' - --github-pat '$(BotAccount-dotnet-bot-repo-PAT)' - --verbose - displayName: Push changes to dotnet/dotnet (public) - workingDirectory: $(Agent.BuildDirectory)/sdk - - # Push internal/release branches to the internal VMR - - ${{ if startsWith(parameters.vmrBranch, 'internal/release/') }}: - - script: > - ./.dotnet/dotnet darc vmr push - --vmr '$(vmrPath)' - --skip-commit-verification - --branch '${{ parameters.vmrBranch }}' - --remote-url '$(vmrInternalUrl)' - --azdev-pat '$(AzdoToken)' - --verbose - displayName: Push changes to dotnet-dotnet (internal) - workingDirectory: $(Agent.BuildDirectory)/sdk +### This job synchronizes code from product repositories into the VMR (https://github.com/dotnet/dotnet) +### It synchronizes the content of the VMR to this new commit and pushes the changes + +parameters: +- name: targetRef + displayName: Target revision of dotnet/sdk to synchronize + type: string + default: $(Build.SourceVersion) + +- name: vmrBranch + displayName: dotnet/dotnet branch to use + type: string + default: $(Build.SourceBranch) + +- name: noPush + displayName: Don't push changes to dotnet/dotnet + type: boolean + default: false + +jobs: +- job: Synchronize_VMR + displayName: Synchronize VMR's ${{ parameters.vmrBranch }} + timeoutInMinutes: 120 + variables: + - ${{ if eq(variables['System.TeamProject'], 'public') }}: + - template: /eng/common/templates/variables/pool-providers.yml + - ${{ else }}: + - template: /eng/common/templates-official/variables/pool-providers.yml + - name: vmrPath + value: $(Agent.BuildDirectory)/vmr + - ${{ if not(parameters.noPush) }}: + - ${{ if and( eq(variables['System.TeamProject'], 'internal'), or(startswith(variables['Build.SourceBranch'], 'refs/heads/release/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: + - group: DotNetBot-GitHub + - group: DotNetBot-GitHub-No-Scopes + - name: vmrPublicUrl + value: https://github.com/dotnet/dotnet + - ${{ if and( eq(variables['System.TeamProject'], 'internal'), startswith(variables['Build.SourceBranch'], 'refs/heads/internal/release/')) }}: + - name: vmrInternalUrl + value: https://dnceng@dev.azure.com/dnceng/internal/_git/dotnet-dotnet + + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $(DncEngPublicBuildPool) + image: build.azurelinux.3.amd64.open + os: linux + ${{ else }}: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals build.azurelinux.3.amd64 + os: linux + + steps: + - template: ../steps/vmr-prepare.yml + parameters: + vmrBranch: ${{ parameters.vmrBranch }} + + - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + - template: /eng/common/templates-official/steps/get-federated-access-token.yml + parameters: + federatedServiceConnection: VmrSyncPipeline + outputVariableName: AzdoToken + + - template: ../steps/vmr-pull-updates.yml + parameters: + vmrPath: $(vmrPath) + vmrBranch: ${{ parameters.vmrBranch }} + targetRef: ${{ parameters.targetRef }} + + - ${{ if and(not(parameters.noPush), not(in(variables['Build.Reason'], 'PullRequest')), eq(variables['System.TeamProject'], 'internal')) }}: + # Push main and release branches to the public VMR + - ${{ if or(eq(parameters.vmrBranch, 'main'), startsWith(parameters.vmrBranch, 'release/')) }}: + - script: > + ./.dotnet/dotnet darc vmr push + --vmr '$(vmrPath)' + --commit-verification-pat '$(BotAccount-dotnet-maestro-bot-no-scopes-PAT)' + --branch '${{ parameters.vmrBranch }}' + --remote-url '$(vmrPublicUrl)' + --github-pat '$(BotAccount-dotnet-bot-repo-PAT)' + --verbose + displayName: Push changes to dotnet/dotnet (public) + workingDirectory: $(Agent.BuildDirectory)/sdk + + # Push internal/release branches to the internal VMR + - ${{ if startsWith(parameters.vmrBranch, 'internal/release/') }}: + - task: AzureCLI@2 + displayName: Mint AzDO token via WIF + inputs: + azureSubscription: 'dnceng-build-rw-code-rw-wif' + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + $token = az account get-access-token --resource 499b84ac-1321-427f-aa17-267ca6975798 --query accessToken -o tsv + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to acquire Azure DevOps access token via 'az account get-access-token'. Exit code: $LASTEXITCODE" + exit 1 + } + $token = $token.Trim() + if ([string]::IsNullOrWhiteSpace($token)) { + Write-Error "Received an empty or whitespace Azure DevOps access token from 'az account get-access-token'." + exit 1 + } + Write-Host "##vso[task.setvariable variable=WifAzdoToken;issecret=true]$token" + - script: > + ./.dotnet/dotnet darc vmr push + --vmr '$(vmrPath)' + --skip-commit-verification + --branch '${{ parameters.vmrBranch }}' + --remote-url '$(vmrInternalUrl)' + --azdev-pat '$(WifAzdoToken)' + --verbose + displayName: Push changes to dotnet-dotnet (internal) + workingDirectory: $(Agent.BuildDirectory)/sdk diff --git a/eng/pipelines/templates/steps/vmr-prepare.yml b/eng/pipelines/templates/steps/vmr-prepare.yml index 8cb5bae7815c..155ccda03d5c 100644 --- a/eng/pipelines/templates/steps/vmr-prepare.yml +++ b/eng/pipelines/templates/steps/vmr-prepare.yml @@ -1,4 +1,4 @@ -### These steps clone the VMR (https://github.com/dotnet/dotnet) into $(Agent.BuildDirectory)/vmr for installer +### These steps clone the VMR (https://github.com/dotnet/dotnet) into $(Agent.BuildDirectory)/vmr for sdk parameters: - name: vmrBranch diff --git a/src/Containers/Microsoft.NET.Build.Containers/ContentStore.cs b/src/Containers/Microsoft.NET.Build.Containers/ContentStore.cs index 2fb16be3ec5d..488764482f27 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/ContentStore.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/ContentStore.cs @@ -8,30 +8,18 @@ namespace Microsoft.NET.Build.Containers; internal static class ContentStore { - public static string ArtifactRoot { get; set; } = Path.Combine(Path.GetTempPath(), "Containers"); + public static string ArtifactRoot { get; set; } = EnsureCacheDirectory(); public static string ContentRoot { get { string contentPath = Path.Join(ArtifactRoot, "Content"); - - Directory.CreateDirectory(contentPath); - + CreateCacheDirectory(contentPath); return contentPath; } } - public static string TempPath - { - get - { - string tempPath = Path.Join(ArtifactRoot, "Temp"); - - Directory.CreateDirectory(tempPath); - - return tempPath; - } - } + public static string TempPath { get; } = Directory.CreateTempSubdirectory().FullName; public static string PathForDescriptor(Descriptor descriptor) { @@ -62,8 +50,81 @@ public static string GetPathForHash(string contentHash) return Path.Combine(ContentRoot, contentHash); } - public static string GetTempFile() + public static string GetTempFile() => Path.Join(TempPath, Path.GetRandomFileName()); + + /// + /// Gets the path to the user cache directory for SDK container builds, creating it if it does + /// not already exist. + /// + private static string EnsureCacheDirectory() + { + string userCacheDir = GetUserCacheDirectoryPath(); + CreateCacheDirectory(userCacheDir); + + string dotnetCacheDir = Path.Join(userCacheDir, "dotnet"); + CreateCacheDirectory(dotnetCacheDir); + + string containersCacheDir = Path.Join(dotnetCacheDir, "Containers"); + CreateCacheDirectory(containersCacheDir); + + return containersCacheDir; + } + + /// + /// Creates a cache directory with user-scoped permissions based on the OS. + /// + /// + /// Permissions are only set for the leaf directory. The caller is expected to handle + /// permissions for intermediate directories. + /// + private static void CreateCacheDirectory(string path) + { + if (OperatingSystem.IsWindows()) + { + Directory.CreateDirectory(path); + } + else + { + // Per XDG_CACHE_HOME spec: if the cache directory doesn't exist, try to create it with + // permissions 0700. If it already exists, don't try to change the permissions. + // See https://specifications.freedesktop.org/basedir/latest/#referencing + Directory.CreateDirectory(path, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + } + } + + /// + /// Gets the user-scoped cache directory, accounting for OS differences. + /// + private static string GetUserCacheDirectoryPath() { - return Path.Join(TempPath, Path.GetRandomFileName()); + if (OperatingSystem.IsLinux()) + { + // See "XDG Base Directory Specification": + // https://specifications.freedesktop.org/basedir/latest/ + string? xdgCacheHome = Environment.GetEnvironmentVariable("XDG_CACHE_HOME"); + if (!string.IsNullOrEmpty(xdgCacheHome) && Path.IsPathFullyQualified(xdgCacheHome)) + { + return xdgCacheHome; + } + + // From the spec, if XDG_CACHE_HOME is not set then fall back to ~/.cache + string userHomeDirectory = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); + return Path.Join(userHomeDirectory, ".cache"); + } + + if (OperatingSystem.IsMacOS()) + { + // See "macOS Library Directory Details": + // https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/MacOSXDirectories/MacOSXDirectories.html + string userHomeDirectory = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); + return Path.Join(userHomeDirectory, "Library", "Caches"); + } + + // On other platforms, use the local application data folder. + string localAppData = Environment.GetFolderPath( + Environment.SpecialFolder.LocalApplicationData, + Environment.SpecialFolderOption.DoNotVerify); + + return localAppData; } } diff --git a/src/Containers/Microsoft.NET.Build.Containers/DigestUtils.cs b/src/Containers/Microsoft.NET.Build.Containers/DigestUtils.cs index c0c7e16e6793..862c01298f2d 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/DigestUtils.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/DigestUtils.cs @@ -37,4 +37,17 @@ internal static string GetSha(string str) return Convert.ToHexStringLower(hash); } + + /// + /// Returns the encoded (hash) portion of a digest string as raw bytes. + /// + /// + /// GetEncodedValue("sha256:e3b0c4...") returns the bytes for "e3b0c4...". + /// + internal static byte[] GetEncodedValue(string digest) + { + int separatorIndex = digest.IndexOf(':'); + string encoded = separatorIndex >= 0 ? digest[(separatorIndex + 1)..] : digest; + return Convert.FromHexString(encoded); + } } diff --git a/src/Containers/Microsoft.NET.Build.Containers/Exceptions/InvalidDigestException.cs b/src/Containers/Microsoft.NET.Build.Containers/Exceptions/InvalidDigestException.cs new file mode 100644 index 000000000000..071f07682a99 --- /dev/null +++ b/src/Containers/Microsoft.NET.Build.Containers/Exceptions/InvalidDigestException.cs @@ -0,0 +1,24 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Security.Cryptography; + +namespace Microsoft.NET.Build.Containers; + +internal sealed class InvalidDigestException : Exception +{ + public InvalidDigestException(string message) + : base(message) + { + } + + public static void ThrowIfMismatched(ReadOnlySpan expected, ReadOnlySpan actual) + { + if (!CryptographicOperations.FixedTimeEquals(expected, actual)) + { + string expectedHashString = Convert.ToHexString(expected).ToLowerInvariant(); + string actualHashString = Convert.ToHexString(actual).ToLowerInvariant(); + throw new InvalidDigestException($"Expected {expectedHashString}, got {actualHashString}."); + } + } +} diff --git a/src/Containers/Microsoft.NET.Build.Containers/Registry/Registry.cs b/src/Containers/Microsoft.NET.Build.Containers/Registry/Registry.cs index 3472ba1c53b2..b1a11ffa8423 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/Registry/Registry.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/Registry/Registry.cs @@ -4,6 +4,7 @@ using NuGet.Packaging; using System.Diagnostics; using System.Net.Http.Json; +using System.Security.Cryptography; using System.Text.Json.Nodes; using Microsoft.Extensions.Logging; using Microsoft.NET.Build.Containers.Resources; @@ -90,7 +91,7 @@ internal sealed class Registry public string RegistryName { get; } internal Registry(string registryName, ILogger logger, IRegistryAPI registryAPI, RegistrySettings? settings = null, Func? retryDelayProvider = null) : - this(new Uri($"https://{registryName}"), logger, registryAPI, settings) + this(new Uri($"https://{registryName}"), logger, registryAPI, settings, retryDelayProvider) { } internal Registry(string registryName, ILogger logger, RegistryMode mode, RegistrySettings? settings = null) : @@ -99,7 +100,7 @@ internal Registry(string registryName, ILogger logger, RegistryMode mode, Regist internal Registry(Uri baseUri, ILogger logger, IRegistryAPI registryAPI, RegistrySettings? settings = null, Func? retryDelayProvider = null) : - this(baseUri, logger, new RegistryApiFactory(registryAPI), settings) + this(baseUri, logger, new RegistryApiFactory(registryAPI), settings, retryDelayProvider) { } internal Registry(Uri baseUri, ILogger logger, RegistryMode mode, RegistrySettings? settings = null) : @@ -408,15 +409,35 @@ public async Task DownloadBlobAsync(string repository, Descriptor descri { cancellationToken.ThrowIfCancellationRequested(); string localPath = ContentStore.PathForDescriptor(descriptor); - - if (File.Exists(localPath)) + + try { - // Assume file is up to date and just return it + var fileStream = File.OpenRead(localPath); + + var actualHash = SHA256.HashData(fileStream); + var expectedHash = DigestUtils.GetEncodedValue(descriptor.Digest); + InvalidDigestException.ThrowIfMismatched(expectedHash, actualHash); + return localPath; } - + catch (DirectoryNotFoundException) + { + // Cache miss + } + catch (FileNotFoundException) + { + // Cache miss + } + catch (InvalidDigestException exception) + { + // Incorrect digest + _logger.LogTrace( + "Digest validation failed for cached blob {1} ({2}), redownloading from registry.", + localPath, exception.Message); + } + string tempTarballPath = ContentStore.GetTempFile(); - + int retryCount = 0; while (retryCount < MaxDownloadRetries) { @@ -424,12 +445,14 @@ public async Task DownloadBlobAsync(string repository, Descriptor descri { // No local copy, so download one using Stream responseStream = await _registryAPI.Blob.GetStreamAsync(repository, descriptor.Digest, cancellationToken).ConfigureAwait(false); - + using (FileStream fs = File.Create(tempTarballPath)) { - await responseStream.CopyToAsync(fs, cancellationToken).ConfigureAwait(false); + await responseStream + .CopyToAndVerifyAsync(fs, descriptor.Digest, cancellationToken) + .ConfigureAwait(false); } - + // Break the loop if successful break; } @@ -440,16 +463,16 @@ public async Task DownloadBlobAsync(string repository, Descriptor descri { throw new UnableToDownloadFromRepositoryException(repository); } - + _logger.LogTrace("Download attempt {0}/{1} for repository '{2}' failed. Error: {3}", retryCount, MaxDownloadRetries, repository, ex.ToString()); - + // Wait before retrying await Task.Delay(_retryDelayProvider(), cancellationToken).ConfigureAwait(false); } } - + File.Move(tempTarballPath, localPath, overwrite: true); - + return localPath; } @@ -566,7 +589,7 @@ public async Task PushManifestListAsync( _logger.LogInformation(Strings.Registry_TagUploadStarted, tag, RegistryName); await _registryAPI.Manifest.PutAsync(destinationImageReference.Repository, tag, multiArchImage.ImageIndex, multiArchImage.ImageIndexMediaType, cancellationToken).ConfigureAwait(false); _logger.LogInformation(Strings.Registry_TagUploaded, tag, RegistryName); - } + } } public Task PushAsync(BuiltImage builtImage, SourceImageReference source, DestinationImageReference destination, CancellationToken cancellationToken) diff --git a/src/Containers/Microsoft.NET.Build.Containers/StreamExtensions.cs b/src/Containers/Microsoft.NET.Build.Containers/StreamExtensions.cs new file mode 100644 index 000000000000..f66e88080ea8 --- /dev/null +++ b/src/Containers/Microsoft.NET.Build.Containers/StreamExtensions.cs @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Security.Cryptography; + +namespace Microsoft.NET.Build.Containers; + +internal static class StreamExtensions +{ + public static async Task CopyToAndVerifyAsync( + this Stream source, + Stream destination, + string digest, + CancellationToken cancellationToken) + { + byte[] expectedHash = DigestUtils.GetEncodedValue(digest).ToArray(); + + // Assumption: SHA256 is the only supported algorithm. + // See DigestUtils > s_registeredAlgorithms for more details. + using HashAlgorithm hashAlgorithm = SHA256.Create(); + + await using var cryptoStream = new CryptoStream( + stream: source, + transform: hashAlgorithm, + mode: CryptoStreamMode.Read, + // cryptoStream will be disposed at the end of the method, but setting leaveOpen=true + // tells CryptoStream not to dispose the source Stream. The caller is responsible for + // the lifetime of the source Stream. + leaveOpen: true + ); + + await cryptoStream.CopyToAsync(destination, cancellationToken); + + InvalidDigestException.ThrowIfMismatched(expectedHash, hashAlgorithm.Hash ?? []); + } +} diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/SdkContentTests/SdkFileDiffExclusions.txt b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/SdkContentTests/SdkFileDiffExclusions.txt index 33a1265a0d38..fbee143561b1 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/SdkContentTests/SdkFileDiffExclusions.txt +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/SdkContentTests/SdkFileDiffExclusions.txt @@ -61,6 +61,9 @@ ./sdk/x.y.z/DotnetTools/dotnet-format/ ./sdk/x.y.z/DotnetTools/dotnet-watch/ +# SB rebuilds IdentityModel from older source lacking the transitive Microsoft.Bcl.Cryptography dependency +./sdk/x.y.z/DotnetTools/dotnet-user-jwts/x.y.z/tools/netx.y/any/Microsoft.Bcl.Cryptography.dll|msft + ./sdk/x.y.z/Extensions/cs/|msft ./sdk/x.y.z/Extensions/de/|msft ./sdk/x.y.z/Extensions/es/|msft diff --git a/src/VirtualMonoRepo/source-mappings.json b/src/VirtualMonoRepo/source-mappings.json index fcdb99eea232..f7bbbd477bd8 100644 --- a/src/VirtualMonoRepo/source-mappings.json +++ b/src/VirtualMonoRepo/source-mappings.json @@ -57,6 +57,7 @@ "name": "aspnetcore", "defaultRemote": "https://github.com/dotnet/aspnetcore", "exclude": [ + "src/submodules/MessagePack-CSharp/src/MessagePack.UnityClient/UserSettings/EditorUserSettings.asset", // Non-OSS license - https://github.com/dotnet/source-build/issues/3537 "src/Installers/Windows/AspNetCoreModule-Setup/**/*" ] diff --git a/test/Microsoft.NET.Build.Containers.UnitTests/RegistryTests.cs b/test/Microsoft.NET.Build.Containers.UnitTests/RegistryTests.cs index 84025f239c0c..43bd4cc047d8 100644 --- a/test/Microsoft.NET.Build.Containers.UnitTests/RegistryTests.cs +++ b/test/Microsoft.NET.Build.Containers.UnitTests/RegistryTests.cs @@ -552,7 +552,10 @@ public async Task DownloadBlobAsync_RetriesOnFailure() var logger = _loggerFactory.CreateLogger(nameof(DownloadBlobAsync_RetriesOnFailure)); var repoName = "testRepo"; - var descriptor = new Descriptor(SchemaTypes.OciLayerGzipV1, "sha256:testdigest1234", 1234); + // The digest must be the actual SHA-256 of the response bytes so that the internal + // branch's CopyToAndVerifyAsync digest validation passes after the download succeeds. + var responseBytes = new byte[] { 1, 2, 3 }; + var descriptor = new Descriptor(SchemaTypes.OciLayerGzipV1, "sha256:039058c6f2c0cb492c533b0a4d14ef77cc0f78abccced5287d84a1a2011cfb81", 1234); var cancellationToken = CancellationToken.None; var mockRegistryAPI = new Mock(MockBehavior.Strict); @@ -560,7 +563,7 @@ public async Task DownloadBlobAsync_RetriesOnFailure() .SetupSequence(api => api.Blob.GetStreamAsync(repoName, descriptor.Digest, cancellationToken)) .ThrowsAsync(new Exception("Simulated failure 1")) // First attempt fails .ThrowsAsync(new Exception("Simulated failure 2")) // Second attempt fails - .ReturnsAsync(new MemoryStream(new byte[] { 1, 2, 3 })); // Third attempt succeeds + .ReturnsAsync(new MemoryStream(responseBytes)); // Third attempt succeeds Registry registry = new(repoName, logger, mockRegistryAPI.Object, null, () => TimeSpan.Zero); @@ -592,7 +595,7 @@ public async Task DownloadBlobAsync_ThrowsAfterMaxRetries() var logger = _loggerFactory.CreateLogger(nameof(DownloadBlobAsync_ThrowsAfterMaxRetries)); var repoName = "testRepo"; - var descriptor = new Descriptor(SchemaTypes.OciLayerGzipV1, "sha256:testdigest1234", 1234); + var descriptor = new Descriptor(SchemaTypes.OciLayerGzipV1, "sha256:a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2", 1234); var cancellationToken = CancellationToken.None; var mockRegistryAPI = new Mock(MockBehavior.Strict);