diff --git a/NuGet.config b/NuGet.config index 22a4df19c9e3..51768687e27f 100644 --- a/NuGet.config +++ b/NuGet.config @@ -3,11 +3,13 @@ + + @@ -15,11 +17,13 @@ + + @@ -44,6 +48,7 @@ + @@ -51,8 +56,10 @@ + + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 7e592b6831fd..835ab088271a 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -14,46 +14,46 @@ e3c582deeab9ac91e1a6567a16fc80eceb0bb050 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 46295af5828b062bbbf93a9cef50fd8cb9fbcb09 + 18fd75c847399745c43b5970fec840ba71064e80 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 46295af5828b062bbbf93a9cef50fd8cb9fbcb09 + 18fd75c847399745c43b5970fec840ba71064e80 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 46295af5828b062bbbf93a9cef50fd8cb9fbcb09 + 18fd75c847399745c43b5970fec840ba71064e80 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 46295af5828b062bbbf93a9cef50fd8cb9fbcb09 + 18fd75c847399745c43b5970fec840ba71064e80 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 46295af5828b062bbbf93a9cef50fd8cb9fbcb09 + 18fd75c847399745c43b5970fec840ba71064e80 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 46295af5828b062bbbf93a9cef50fd8cb9fbcb09 + 18fd75c847399745c43b5970fec840ba71064e80 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 46295af5828b062bbbf93a9cef50fd8cb9fbcb09 + 18fd75c847399745c43b5970fec840ba71064e80 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime 81cabf2857a01351e5ab578947c7403a5b128ad1 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 46295af5828b062bbbf93a9cef50fd8cb9fbcb09 + 18fd75c847399745c43b5970fec840ba71064e80 - + https://github.com/dotnet/emsdk - 35ebe6f3e6953f878ed92f637ad8d7f27202f7a6 + ac9f8092f4f527cb3ace4aac03f8d1dbadc39983 https://github.com/dotnet/msbuild @@ -111,13 +111,13 @@ https://github.com/dotnet/roslyn 38896ab4e7cee896fcde8a4e26914a777c794e3b - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - bb9eccba9080e07bce32c0bc27c3564c753a7cfe + 03ef277b368eb3b02122ca62f97d155845cfd5fa - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - bb9eccba9080e07bce32c0bc27c3564c753a7cfe + 03ef277b368eb3b02122ca62f97d155845cfd5fa https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted @@ -200,9 +200,9 @@ https://github.com/microsoft/vstest aa59400b11e1aeee2e8af48928dbd48748a8bef9 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 46295af5828b062bbbf93a9cef50fd8cb9fbcb09 + 18fd75c847399745c43b5970fec840ba71064e80 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime @@ -228,70 +228,70 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-runtime 5535e31a712343a63f5d7d796cd874e563e5ac14 - + https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - 432d0577ee8d6a36654d23a83182a0c7da27a69f + 580168d2b82abec4ec1460077b8e62c016c031c4 - + https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - 432d0577ee8d6a36654d23a83182a0c7da27a69f + 580168d2b82abec4ec1460077b8e62c016c031c4 - + https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - 432d0577ee8d6a36654d23a83182a0c7da27a69f + 580168d2b82abec4ec1460077b8e62c016c031c4 - + https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - 432d0577ee8d6a36654d23a83182a0c7da27a69f + 580168d2b82abec4ec1460077b8e62c016c031c4 - + https://dev.azure.com/dnceng/internal/_git/dotnet-wpf - 59524df4bc81ace465f0c2b41c0de26623c4a820 + 90b062461604a96b6bc77309bfe93718ece2c518 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - bb9eccba9080e07bce32c0bc27c3564c753a7cfe + 03ef277b368eb3b02122ca62f97d155845cfd5fa - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - bb9eccba9080e07bce32c0bc27c3564c753a7cfe + 03ef277b368eb3b02122ca62f97d155845cfd5fa - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - bb9eccba9080e07bce32c0bc27c3564c753a7cfe + 03ef277b368eb3b02122ca62f97d155845cfd5fa - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - bb9eccba9080e07bce32c0bc27c3564c753a7cfe + 03ef277b368eb3b02122ca62f97d155845cfd5fa - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - bb9eccba9080e07bce32c0bc27c3564c753a7cfe + 03ef277b368eb3b02122ca62f97d155845cfd5fa - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - bb9eccba9080e07bce32c0bc27c3564c753a7cfe + 03ef277b368eb3b02122ca62f97d155845cfd5fa - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - bb9eccba9080e07bce32c0bc27c3564c753a7cfe + 03ef277b368eb3b02122ca62f97d155845cfd5fa - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - bb9eccba9080e07bce32c0bc27c3564c753a7cfe + 03ef277b368eb3b02122ca62f97d155845cfd5fa - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - bb9eccba9080e07bce32c0bc27c3564c753a7cfe + 03ef277b368eb3b02122ca62f97d155845cfd5fa - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - bb9eccba9080e07bce32c0bc27c3564c753a7cfe + 03ef277b368eb3b02122ca62f97d155845cfd5fa - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - bb9eccba9080e07bce32c0bc27c3564c753a7cfe + 03ef277b368eb3b02122ca62f97d155845cfd5fa https://github.com/dotnet/razor @@ -306,21 +306,21 @@ https://github.com/dotnet/razor e137c0322ddb9e3d4f4a5de385647baa9719f9a6 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - bb9eccba9080e07bce32c0bc27c3564c753a7cfe + 03ef277b368eb3b02122ca62f97d155845cfd5fa - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - bb9eccba9080e07bce32c0bc27c3564c753a7cfe + 03ef277b368eb3b02122ca62f97d155845cfd5fa - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - bb9eccba9080e07bce32c0bc27c3564c753a7cfe + 03ef277b368eb3b02122ca62f97d155845cfd5fa - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - bb9eccba9080e07bce32c0bc27c3564c753a7cfe + 03ef277b368eb3b02122ca62f97d155845cfd5fa https://github.com/dotnet/xdt @@ -349,9 +349,9 @@ 02fe27cd6a9b001c8feb7938e6ef4b3799745759 - - https://github.com/dotnet/source-build-externals - f537cc21040df4a14d26a3045a0d4b6fdc1efc3b + + https://dev.azure.com/dnceng/internal/_git/dotnet-source-build-externals + 15f64ab037253b38c22fa801dccf26d809af95fd @@ -437,9 +437,9 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-runtime 5535e31a712343a63f5d7d796cd874e563e5ac14 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - bb9eccba9080e07bce32c0bc27c3564c753a7cfe + 03ef277b368eb3b02122ca62f97d155845cfd5fa https://dev.azure.com/dnceng/internal/_git/dotnet-runtime @@ -477,9 +477,9 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-runtime 81cabf2857a01351e5ab578947c7403a5b128ad1 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a6bde67c455f2ac219988c7a66171631090b6f65 + 18fd75c847399745c43b5970fec840ba71064e80 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime diff --git a/eng/Versions.props b/eng/Versions.props index 6364e75c865a..36e8ed1f2b9e 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -49,21 +49,21 @@ - 8.0.28 - 8.0.28-servicing.26264.13 - 8.0.28 + 8.0.29 + 8.0.29-servicing.26324.3 + 8.0.29 $(MicrosoftNETCoreAppRuntimewinx64PackageVersion) 8.0.2 - 8.0.28 - 8.0.28-servicing.26264.13 + 8.0.29 + 8.0.29-servicing.26324.3 8.0.0 8.0.1 8.0.3 8.0.1 - 8.0.28 + 8.0.29 8.0.0 8.0.0 - 8.0.28 + 8.0.29 8.0.0 8.0.0 8.0.0 @@ -74,7 +74,7 @@ 8.0.1 8.0.0 8.0.1 - 8.0.3 + 8.0.4 8.0.0 8.0.1 8.0.6 @@ -170,13 +170,13 @@ - 8.0.28 - 8.0.28-servicing.26269.13 - 8.0.28-servicing.26269.13 - 8.0.28-servicing.26269.13 - 8.0.28-servicing.26269.13 - 8.0.28-servicing.26269.13 - 8.0.28 + 8.0.29 + 8.0.29-servicing.26325.17 + 8.0.29-servicing.26325.17 + 8.0.29-servicing.26325.17 + 8.0.29-servicing.26325.17 + 8.0.29-servicing.26325.17 + 8.0.29 @@ -186,7 +186,7 @@ - 8.0.28-servicing.26269.2 + 8.0.29-servicing.26325.4 @@ -230,7 +230,7 @@ - 8.0.28 + 8.0.29 $(MicrosoftNETWorkloadEmscriptenCurrentManifest80100PackageVersion) 8.0.100$([System.Text.RegularExpressions.Regex]::Match($(EmscriptenWorkloadManifestVersion), `-rtm|-[A-z]*\.*\d*`)) diff --git a/src/Containers/Microsoft.NET.Build.Containers/ContentStore.cs b/src/Containers/Microsoft.NET.Build.Containers/ContentStore.cs index 2fb16be3ec5d..488764482f27 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/ContentStore.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/ContentStore.cs @@ -8,30 +8,18 @@ namespace Microsoft.NET.Build.Containers; internal static class ContentStore { - public static string ArtifactRoot { get; set; } = Path.Combine(Path.GetTempPath(), "Containers"); + public static string ArtifactRoot { get; set; } = EnsureCacheDirectory(); public static string ContentRoot { get { string contentPath = Path.Join(ArtifactRoot, "Content"); - - Directory.CreateDirectory(contentPath); - + CreateCacheDirectory(contentPath); return contentPath; } } - public static string TempPath - { - get - { - string tempPath = Path.Join(ArtifactRoot, "Temp"); - - Directory.CreateDirectory(tempPath); - - return tempPath; - } - } + public static string TempPath { get; } = Directory.CreateTempSubdirectory().FullName; public static string PathForDescriptor(Descriptor descriptor) { @@ -62,8 +50,81 @@ public static string GetPathForHash(string contentHash) return Path.Combine(ContentRoot, contentHash); } - public static string GetTempFile() + public static string GetTempFile() => Path.Join(TempPath, Path.GetRandomFileName()); + + /// + /// Gets the path to the user cache directory for SDK container builds, creating it if it does + /// not already exist. + /// + private static string EnsureCacheDirectory() + { + string userCacheDir = GetUserCacheDirectoryPath(); + CreateCacheDirectory(userCacheDir); + + string dotnetCacheDir = Path.Join(userCacheDir, "dotnet"); + CreateCacheDirectory(dotnetCacheDir); + + string containersCacheDir = Path.Join(dotnetCacheDir, "Containers"); + CreateCacheDirectory(containersCacheDir); + + return containersCacheDir; + } + + /// + /// Creates a cache directory with user-scoped permissions based on the OS. + /// + /// + /// Permissions are only set for the leaf directory. The caller is expected to handle + /// permissions for intermediate directories. + /// + private static void CreateCacheDirectory(string path) + { + if (OperatingSystem.IsWindows()) + { + Directory.CreateDirectory(path); + } + else + { + // Per XDG_CACHE_HOME spec: if the cache directory doesn't exist, try to create it with + // permissions 0700. If it already exists, don't try to change the permissions. + // See https://specifications.freedesktop.org/basedir/latest/#referencing + Directory.CreateDirectory(path, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + } + } + + /// + /// Gets the user-scoped cache directory, accounting for OS differences. + /// + private static string GetUserCacheDirectoryPath() { - return Path.Join(TempPath, Path.GetRandomFileName()); + if (OperatingSystem.IsLinux()) + { + // See "XDG Base Directory Specification": + // https://specifications.freedesktop.org/basedir/latest/ + string? xdgCacheHome = Environment.GetEnvironmentVariable("XDG_CACHE_HOME"); + if (!string.IsNullOrEmpty(xdgCacheHome) && Path.IsPathFullyQualified(xdgCacheHome)) + { + return xdgCacheHome; + } + + // From the spec, if XDG_CACHE_HOME is not set then fall back to ~/.cache + string userHomeDirectory = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); + return Path.Join(userHomeDirectory, ".cache"); + } + + if (OperatingSystem.IsMacOS()) + { + // See "macOS Library Directory Details": + // https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/MacOSXDirectories/MacOSXDirectories.html + string userHomeDirectory = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); + return Path.Join(userHomeDirectory, "Library", "Caches"); + } + + // On other platforms, use the local application data folder. + string localAppData = Environment.GetFolderPath( + Environment.SpecialFolder.LocalApplicationData, + Environment.SpecialFolderOption.DoNotVerify); + + return localAppData; } } diff --git a/src/Containers/Microsoft.NET.Build.Containers/DigestUtils.cs b/src/Containers/Microsoft.NET.Build.Containers/DigestUtils.cs index e549defd4635..54458d7a471c 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/DigestUtils.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/DigestUtils.cs @@ -27,4 +27,17 @@ internal static string GetSha(string str) return Convert.ToHexString(hash).ToLowerInvariant(); } + + /// + /// Returns the encoded (hash) portion of a digest string as raw bytes. + /// + /// + /// GetEncodedValue("sha256:e3b0c4...") returns the bytes for "e3b0c4...". + /// + internal static byte[] GetEncodedValue(string digest) + { + int separatorIndex = digest.IndexOf(':'); + string encoded = separatorIndex >= 0 ? digest[(separatorIndex + 1)..] : digest; + return Convert.FromHexString(encoded); + } } diff --git a/src/Containers/Microsoft.NET.Build.Containers/Exceptions/InvalidDigestException.cs b/src/Containers/Microsoft.NET.Build.Containers/Exceptions/InvalidDigestException.cs new file mode 100644 index 000000000000..071f07682a99 --- /dev/null +++ b/src/Containers/Microsoft.NET.Build.Containers/Exceptions/InvalidDigestException.cs @@ -0,0 +1,24 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Security.Cryptography; + +namespace Microsoft.NET.Build.Containers; + +internal sealed class InvalidDigestException : Exception +{ + public InvalidDigestException(string message) + : base(message) + { + } + + public static void ThrowIfMismatched(ReadOnlySpan expected, ReadOnlySpan actual) + { + if (!CryptographicOperations.FixedTimeEquals(expected, actual)) + { + string expectedHashString = Convert.ToHexString(expected).ToLowerInvariant(); + string actualHashString = Convert.ToHexString(actual).ToLowerInvariant(); + throw new InvalidDigestException($"Expected {expectedHashString}, got {actualHashString}."); + } + } +} diff --git a/src/Containers/Microsoft.NET.Build.Containers/Registry/Registry.cs b/src/Containers/Microsoft.NET.Build.Containers/Registry/Registry.cs index b3df405b7457..5d07ca41410c 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/Registry/Registry.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/Registry/Registry.cs @@ -6,6 +6,7 @@ using NuGet.RuntimeModel; using System.Diagnostics; using System.Net.Http.Json; +using System.Security.Cryptography; using System.Text.Json.Nodes; namespace Microsoft.NET.Build.Containers; @@ -257,11 +258,31 @@ public async Task DownloadBlobAsync(string repository, Descriptor descri cancellationToken.ThrowIfCancellationRequested(); string localPath = ContentStore.PathForDescriptor(descriptor); - if (File.Exists(localPath)) + try { - // Assume file is up to date and just return it + var fileStream = File.OpenRead(localPath); + + var actualHash = SHA256.HashData(fileStream); + var expectedHash = DigestUtils.GetEncodedValue(descriptor.Digest); + InvalidDigestException.ThrowIfMismatched(expectedHash, actualHash); + return localPath; } + catch (DirectoryNotFoundException) + { + // Cache miss + } + catch (FileNotFoundException) + { + // Cache miss + } + catch (InvalidDigestException exception) + { + // Incorrect digest + _logger.LogTrace( + "Digest validation failed for cached blob {1} ({2}), redownloading from registry.", + localPath, exception.Message); + } // No local copy, so download one using Stream responseStream = await _registryAPI.Blob.GetStreamAsync(repository, descriptor.Digest, cancellationToken).ConfigureAwait(false); @@ -269,7 +290,9 @@ public async Task DownloadBlobAsync(string repository, Descriptor descri string tempTarballPath = ContentStore.GetTempFile(); using (FileStream fs = File.Create(tempTarballPath)) { - await responseStream.CopyToAsync(fs, cancellationToken).ConfigureAwait(false); + await responseStream + .CopyToAndVerifyAsync(fs, descriptor.Digest, cancellationToken) + .ConfigureAwait(false); } cancellationToken.ThrowIfCancellationRequested(); @@ -319,7 +342,7 @@ internal async Task UploadBlobChunkedAsync(Stream con // manual because ACR throws an error with the .NET type {"Range":"bytes 0-84521/*","Reason":"the Content-Range header format is invalid"} // content.Headers.Add("Content-Range", $"0-{contents.Length - 1}"); Debug.Assert(content.Headers.TryAddWithoutValidation("Content-Range", $"{chunkStart}-{chunkStart + bytesRead - 1}")); - + NextChunkUploadInformation nextChunk = await _registryAPI.Blob.Upload.UploadChunkAsync(patchUri, content, cancellationToken).ConfigureAwait(false); patchUri = nextChunk.UploadUri; diff --git a/src/Containers/Microsoft.NET.Build.Containers/StreamExtensions.cs b/src/Containers/Microsoft.NET.Build.Containers/StreamExtensions.cs new file mode 100644 index 000000000000..f66e88080ea8 --- /dev/null +++ b/src/Containers/Microsoft.NET.Build.Containers/StreamExtensions.cs @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Security.Cryptography; + +namespace Microsoft.NET.Build.Containers; + +internal static class StreamExtensions +{ + public static async Task CopyToAndVerifyAsync( + this Stream source, + Stream destination, + string digest, + CancellationToken cancellationToken) + { + byte[] expectedHash = DigestUtils.GetEncodedValue(digest).ToArray(); + + // Assumption: SHA256 is the only supported algorithm. + // See DigestUtils > s_registeredAlgorithms for more details. + using HashAlgorithm hashAlgorithm = SHA256.Create(); + + await using var cryptoStream = new CryptoStream( + stream: source, + transform: hashAlgorithm, + mode: CryptoStreamMode.Read, + // cryptoStream will be disposed at the end of the method, but setting leaveOpen=true + // tells CryptoStream not to dispose the source Stream. The caller is responsible for + // the lifetime of the source Stream. + leaveOpen: true + ); + + await cryptoStream.CopyToAsync(destination, cancellationToken); + + InvalidDigestException.ThrowIfMismatched(expectedHash, hashAlgorithm.Hash ?? []); + } +}