diff --git a/src/services/Elastic.Changelog/Uploading/ChangelogUploadService.cs b/src/services/Elastic.Changelog/Uploading/ChangelogUploadService.cs index d6e2c76cfc..b859ebb156 100644 --- a/src/services/Elastic.Changelog/Uploading/ChangelogUploadService.cs +++ b/src/services/Elastic.Changelog/Uploading/ChangelogUploadService.cs @@ -121,44 +121,12 @@ public async Task Upload(IDiagnosticsCollector collector, ChangelogUploadA if (result.Failed > 0) collector.EmitError(string.Empty, $"{result.Failed} file(s) failed to upload"); - // On a successful upload, refresh the per-product registry.json so consumers can enumerate - // content without an S3 listing: the bundle index (consumed by the changelog directive in - // cdn: mode) for bundle uploads, and the changelog-entry index (consumed by `changelog - // bundle` when sourcing entries from the CDN) for changelog uploads. - // Failures here are logged but don't fail the upload — the objects themselves are already in S3. - if (result.Failed == 0 && targets.Count > 0) - { - var scope = args.ArtifactType == ArtifactType.Bundle ? RegistryScope.Bundle : RegistryScope.Changelog; - await RefreshRegistries(collector, client, etagCalculator, args, targets, scope, ctx); - } - + // No registry refresh here: the scrubber Lambda is the sole producer of the public + // registry.json, reconciled from actual public bucket state on every S3 event this upload + // just emitted (elastic/docs-eng-team#688). A private-bucket registry no longer exists. return result.Failed == 0; } - private async Task RefreshRegistries( - IDiagnosticsCollector collector, - IAmazonS3 client, - IS3EtagCalculator etagCalculator, - ChangelogUploadArguments args, - IReadOnlyList uploadTargets, - RegistryScope scope, - Cancel ctx) - { - try - { - var builder = new RegistryBuilder(logFactory, _fileSystem, client, etagCalculator, args.S3BucketName); - var result = await builder.RefreshAsync(collector, uploadTargets, ctx, scope); - _logger.LogInformation("Registry refresh ({Scope}): {Updated} updated, {Unchanged} unchanged, {Failed} failed", - scope, result.Updated, result.Unchanged, result.Failed); - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - // Leaving the manifest stale is non-fatal — bundle objects are unaffected. - _logger.LogWarning(ex, "Registry refresh failed; bundles uploaded successfully but manifests may be stale"); - collector.EmitWarning(string.Empty, $"Failed to refresh registry manifest(s): {ex.Message}"); - } - } - internal IReadOnlyList DiscoverUploadTargets(IDiagnosticsCollector collector, string changelogDir, string? org, string? repo, string? branch) { // Option AD: entries live once, under the authoring org/repo/branch pool — independent of which diff --git a/src/services/Elastic.Changelog/Uploading/RegistryBuilder.cs b/src/services/Elastic.Changelog/Uploading/RegistryBuilder.cs deleted file mode 100644 index bdf5eddada..0000000000 --- a/src/services/Elastic.Changelog/Uploading/RegistryBuilder.cs +++ /dev/null @@ -1,343 +0,0 @@ -// Licensed to Elasticsearch B.V under one or more agreements. -// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. -// See the LICENSE file in the project root for more information - -using System.IO.Abstractions; -using System.Net; -using System.Text.Json; -using Amazon.S3; -using Amazon.S3.Model; -using Elastic.Documentation.Configuration.ReleaseNotes; -using Elastic.Documentation.Diagnostics; -using Elastic.Documentation.Integrations.S3; -using Elastic.Documentation.Versions; -using Microsoft.Extensions.Logging; - -namespace Elastic.Changelog.Uploading; - -/// -/// Which per-product manifest a run refreshes. -/// -internal enum RegistryScope -{ - /// The bundle index at bundle/{product}/registry.json, listing scrubbed bundle files. - Bundle, - - /// The changelog-entry index at changelog/{org}/{repo}/{branch}/registry.json, listing individual entry files. - Changelog -} - -/// -/// Refreshes a registry.json manifest in the private bucket after an upload run. -/// Depending on this is either the bundle index -/// (bundle/{product}/registry.json) or the changelog-entry index -/// (changelog/{org}/{repo}/{branch}/registry.json). Each grouping (product, or org/repo/branch) -/// touched in the run gets its manifest merged with what is already known on S3 (read back, merged by -/// file name, written with an optimistic concurrency guard so parallel uploads for the same group cannot -/// clobber each other). -/// -internal sealed class RegistryBuilder( - ILoggerFactory logFactory, - IFileSystem fileSystem, - IAmazonS3 s3Client, - IS3EtagCalculator etagCalculator, - string bucketName, - TimeProvider? timeProvider = null -) -{ - private readonly ILogger _logger = logFactory.CreateLogger(); - private readonly TimeProvider _time = timeProvider ?? TimeProvider.System; - - // Bounds the optimistic-concurrency retry loop. Concurrent uploads for the same product are - // expected to be rare (releases are largely serialized), so a small ceiling is plenty. - private const int MaxWriteAttempts = 5; - - /// Outcome counts for a manifest refresh run, used for logging only. - internal sealed record RefreshResult(int Updated, int Unchanged, int Failed); - - /// - /// Builds and writes per-product manifests for every product touched by . - /// Each manifest is merged with the copy already on S3 and written back with a conditional PUT - /// (If-Match on update, If-None-Match: * on create); a precondition failure means a - /// concurrent writer won the race, so we re-read, re-merge, and retry. - /// - /// Diagnostics sink for non-fatal warnings. - /// Upload targets produced by DiscoverBundleUploadTargets or DiscoverUploadTargets. - /// Cancellation token. - /// Which per-product manifest to refresh (bundle index or changelog-entry index). - public async Task RefreshAsync( - IDiagnosticsCollector collector, - IReadOnlyList uploadTargets, - Cancel ctx, - RegistryScope scope = RegistryScope.Bundle) - { - // Each upload target carries an artifact-root S3 key — "bundle/{product}/{file}" or - // "changelog/{org}/{repo}/{branch}/{file}". Group by the scope's key (product for bundles, the - // {org}/{repo}/{branch} prefix for entries) so we produce one manifest per affected group. - var byProduct = uploadTargets - .Select(t => (Target: t, Product: ExtractGroupKey(t.S3Key, scope))) - .Where(x => x.Product is not null) - .GroupBy(x => x.Product!, StringComparer.Ordinal); - - var updated = 0; - var unchanged = 0; - var failed = 0; - - foreach (var group in byProduct) - { - ctx.ThrowIfCancellationRequested(); - - var product = group.Key; - var localEntries = await BuildLocalEntries(collector, product, group.Select(x => x.Target).ToList(), scope, ctx); - if (localEntries.Count == 0) - { - _logger.LogDebug("No usable manifest entries derived for product {Product}; skipping", product); - continue; - } - - switch (await WriteManifest(collector, product, localEntries, scope, ctx)) - { - case WriteOutcome.Updated: - updated++; - break; - case WriteOutcome.Unchanged: - unchanged++; - break; - default: - failed++; - break; - } - } - - return new RefreshResult(updated, unchanged, failed); - } - - /// Extracts the grouping key (product for bundle/{product}/…, {org}/{repo}/{branch} for changelog/{org}/{repo}/{branch}/…) from an artifact-root S3 key, or null. - private static string? ExtractGroupKey(string s3Key, RegistryScope scope) => scope == RegistryScope.Changelog - ? ChangelogKeys.ExtractChangelogGroup(s3Key) - : ChangelogKeys.ExtractBundleGroup(s3Key); - - /// The S3 key of the manifest for the given and grouping segment. - private static string RegistryKeyFor(string group, RegistryScope scope) => scope == RegistryScope.Changelog - ? ChangelogKeys.ChangelogRegistryKey(group) - : ChangelogKeys.BundleRegistryKey(group); - - /// Builds manifest entries for this run's bundles, recording the per- target for the bundle index. - private async Task> BuildLocalEntries( - IDiagnosticsCollector collector, - string product, - IReadOnlyList targets, - RegistryScope scope, - Cancel ctx) - { - var entries = new List(targets.Count); - foreach (var target in targets) - { - ctx.ThrowIfCancellationRequested(); - - // The changelog-entry index only needs to enumerate files (consumers re-read each entry - // to filter), so target is left unset there; the bundle index records the per-product target. - var targetVersion = scope == RegistryScope.Bundle - ? ReadTargetForProduct(collector, target.LocalPath, product) - : null; - - string etag; - try - { - etag = await etagCalculator.CalculateS3ETag(target.LocalPath, ctx); - } - catch (Exception ex) - { - collector.EmitWarning(target.LocalPath, - $"Could not compute ETag for manifest entry: {ex.Message}"); - continue; - } - - var fileName = fileSystem.Path.GetFileName(target.LocalPath); - entries.Add(new RegistryBundle - { - File = fileName, - Target = targetVersion, - ETag = etag - }); - } - - return entries; - } - - private string? ReadTargetForProduct(IDiagnosticsCollector collector, string localPath, string product) - { - try - { - var content = fileSystem.File.ReadAllText(localPath); - var bundle = ReleaseNotesSerialization.DeserializeBundle(content); - - // Amends published before products were copied from the parent omit them; record the - // parent bundle's target so :version:-filtered consumers still discover the amend. - if (bundle.Products.Count == 0) - return ReadTargetFromParentBundle(localPath, product); - - var match = bundle.Products.FirstOrDefault(p => string.Equals(p.ProductId, product, StringComparison.Ordinal)); - return (match ?? bundle.Products[0]).Target; - } - catch (Exception ex) - { - collector.EmitWarning(localPath, $"Could not read bundle target for manifest: {ex.Message}"); - return null; - } - } - - private string? ReadTargetFromParentBundle(string amendFilePath, string product) - { - var parentPath = BundleAmendMerger.GetParentBundlePath(amendFilePath); - if (parentPath == null || !fileSystem.File.Exists(parentPath)) - return null; - - var parent = ReleaseNotesSerialization.DeserializeBundle(fileSystem.File.ReadAllText(parentPath)); - if (parent.Products.Count == 0) - return null; - - var match = parent.Products.FirstOrDefault(p => string.Equals(p.ProductId, product, StringComparison.Ordinal)); - return (match ?? parent.Products[0]).Target; - } - - private enum WriteOutcome { Updated, Unchanged, Failed } - - private async Task WriteManifest( - IDiagnosticsCollector collector, - string product, - IReadOnlyList localEntries, - RegistryScope scope, - Cancel ctx) - { - var key = RegistryKeyFor(product, scope); - - for (var attempt = 1; attempt <= MaxWriteAttempts; attempt++) - { - ctx.ThrowIfCancellationRequested(); - - var (existing, etag) = await TryFetchExistingManifest(product, scope, ctx); - var merged = Merge(existing, localEntries); - - // Re-uploading the same bundles must not churn the manifest (keeps reruns idempotent). - if (etag is not null && BundlesEqual(existing, merged)) - { - _logger.LogDebug("registry for {Product} already up to date; skipping write", product); - return WriteOutcome.Unchanged; - } - - var manifest = new Registry - { - Product = product, - GeneratedAt = _time.GetUtcNow(), - Bundles = merged - }; - var json = JsonSerializer.Serialize(manifest, RegistryJsonContext.Default.Registry); - - try - { - await PutManifest(key, json, etag, ctx); - _logger.LogInformation("Wrote registry.json for {Product} with {Count} bundle(s)", product, merged.Count); - return WriteOutcome.Updated; - } - catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.PreconditionFailed) - { - _logger.LogInformation( - "registry for {Product} changed concurrently (attempt {Attempt}/{Max}); re-reading and retrying", - product, attempt, MaxWriteAttempts); - } - } - - collector.EmitWarning(string.Empty, - $"registry for {product} could not be updated after {MaxWriteAttempts} attempts due to concurrent writes; the index may be stale."); - return WriteOutcome.Failed; - } - - /// Reads the existing manifest and its ETag (null ETag when absent; live ETag when corrupt, so the conditional write can overwrite). - private async Task<(IReadOnlyList Bundles, string? ETag)> TryFetchExistingManifest(string product, RegistryScope scope, Cancel ctx) - { - var key = RegistryKeyFor(product, scope); - string? etag = null; - try - { - using var response = await s3Client.GetObjectAsync(new GetObjectRequest - { - BucketName = bucketName, - Key = key - }, ctx); - - etag = response.ETag; - await using var stream = response.ResponseStream; - var existing = await JsonSerializer.DeserializeAsync( - stream, - RegistryJsonContext.Default.Registry, - ctx); - - return (existing?.Bundles ?? [], etag); - } - catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.NotFound) - { - return ([], null); - } - catch (JsonException ex) - { - // Only a genuinely corrupt (unparseable) manifest is rebuilt from this run; the captured ETag - // then lets the conditional write overwrite it safely. Transient S3/IO errors must NOT be - // treated as corruption — otherwise the If-Match PUT would replace a valid manifest with only - // this run's bundles and drop previously published entries. Let those bubble up to the - // best-effort handler in ChangelogUploadService instead. - _logger.LogWarning(ex, "Existing manifest for {Product} could not be parsed; recreating", product); - return ([], etag); - } - } - - private async Task PutManifest(string key, string json, string? etag, Cancel ctx) - { - var request = new PutObjectRequest - { - BucketName = bucketName, - Key = key, - ContentBody = json, - ContentType = "application/json" - }; - - // Optimistic concurrency: update only if unchanged, create only if still absent. - if (etag is null) - request.IfNoneMatch = "*"; - else - request.IfMatch = etag; - - _ = await s3Client.PutObjectAsync(request, ctx); - } - - /// Replaces existing entries by file name and sorts newest-target-first (with a file-name tiebreak) for a stable manifest. - private static List Merge( - IReadOnlyList existing, - IReadOnlyList incoming) - { - var byFile = existing.ToDictionary(b => b.File, b => b, StringComparer.Ordinal); - foreach (var entry in incoming) - byFile[entry.File] = entry; - - return byFile.Values - .OrderByDescending(b => VersionOrDate.Parse(b.Target ?? string.Empty)) - .ThenBy(b => b.File, StringComparer.Ordinal) - .ToList(); - } - - private static bool BundlesEqual(IReadOnlyList a, IReadOnlyList b) - { - if (a.Count != b.Count) - return false; - - for (var i = 0; i < a.Count; i++) - { - if (!string.Equals(a[i].File, b[i].File, StringComparison.Ordinal) || - !string.Equals(a[i].Target, b[i].Target, StringComparison.Ordinal) || - !string.Equals(a[i].ETag, b[i].ETag, StringComparison.Ordinal)) - return false; - } - - return true; - } -} diff --git a/tests/Elastic.Changelog.Tests/Changelogs/BundleAmendEndToEndTests.cs b/tests/Elastic.Changelog.Tests/Changelogs/BundleAmendEndToEndTests.cs index 410909bfa4..76c4965051 100644 --- a/tests/Elastic.Changelog.Tests/Changelogs/BundleAmendEndToEndTests.cs +++ b/tests/Elastic.Changelog.Tests/Changelogs/BundleAmendEndToEndTests.cs @@ -3,24 +3,23 @@ // See the LICENSE file in the project root for more information using System.Net; -using Amazon.S3; -using Amazon.S3.Model; using AwesomeAssertions; using Elastic.Changelog.Bundling; +using Elastic.Changelog.Reconciliation; +using Elastic.Changelog.Tests.Reconciliation; using Elastic.Changelog.Uploading; using Elastic.Documentation.Configuration; using Elastic.Documentation.Configuration.ReleaseNotes; -using Elastic.Documentation.Integrations.S3; -using FakeItEasy; using Microsoft.Extensions.Logging.Abstractions; namespace Elastic.Changelog.Tests.Changelogs; /// /// The full amend acceptance chain: bundle-amend materializes a self-contained amend -/// (parent products copied) → upload destination discovery includes the amend → the registry -/// records the amend's target → a :version:-filtered CDN fetch returns the amend and the -/// exclude-entries retraction (matched by file identity) applies. +/// (parent products copied) → upload destination discovery includes the amend → the scrubber-side +/// reconciler records the amend's target in the public registry → a :version:-filtered CDN +/// fetch returns the amend and the exclude-entries retraction (matched by file +/// identity) applies. /// public class BundleAmendEndToEndTests(ITestOutputHelper output) : ChangelogTestBase(output) { @@ -104,9 +103,9 @@ await FileSystem.File.WriteAllTextAsync(parentPath, $""" amend.Products[0].Owner.Should().Be("elastic"); // -- 2. upload destination discovery includes the amend ------------------------------ - var s3Client = A.Fake(); + var s3 = new FakeS3("public-bucket"); var uploadCollector = new TestDiagnosticsCollector(Output); - var uploadService = new ChangelogUploadService(NullLoggerFactory.Instance, fileSystem: FileSystem, s3Client: s3Client); + var uploadService = new ChangelogUploadService(NullLoggerFactory.Instance, fileSystem: FileSystem, s3Client: s3.Client); var targets = uploadService.DiscoverBundleUploadTargets(uploadCollector, bundleDir); targets.Select(t => t.S3Key).Should().BeEquivalentTo( @@ -115,29 +114,28 @@ await FileSystem.File.WriteAllTextAsync(parentPath, $""" uploadCollector.Errors.Should().Be(0); uploadCollector.Warnings.Should().Be(0); - // -- 3. the registry records the amend's target -------------------------------------- - var puts = new List(); - A.CallTo(() => s3Client.GetObjectAsync(A._, A._)) - .Throws(new AmazonS3Exception("Not Found") { StatusCode = HttpStatusCode.NotFound }); - A.CallTo(() => s3Client.PutObjectAsync(A._, A._)) - .Invokes((PutObjectRequest r, CancellationToken _) => puts.Add(r)) - .Returns(new PutObjectResponse()); + // -- 3. the scrubber-side reconciler records the amend's target ----------------------- + // Uploads only emit S3 events; the scrubber Lambda copies the scrubbed YAMLs to the + // public bucket and reconciles the group's registry from the public listing + // (elastic/docs-eng-team#688). Seed the public bucket with the scrubbed copies (this + // content has no private references, so the scrub is a no-op). + foreach (var target in targets) + _ = s3.Seed("public-bucket", target.S3Key, await FileSystem.File.ReadAllTextAsync(target.LocalPath, ct)); - var registryCollector = new TestDiagnosticsCollector(Output); - var etagCalculator = new S3EtagCalculator(NullLoggerFactory.Instance, FileSystem); - var builder = new RegistryBuilder(NullLoggerFactory.Instance, FileSystem, s3Client, etagCalculator, "test-bucket"); - var refresh = await builder.RefreshAsync(registryCollector, targets, ct); + ChangelogScope.TryCreateBundle("elasticsearch", out var scope).Should().BeTrue(); + var reconciler = new BundleRegistryReconciler(NullLoggerFactory.Instance, s3.Client, "public-bucket"); + var outcome = await reconciler.ReconcileGroupAsync(scope!, ct); - refresh.Updated.Should().Be(1); - var registryPut = puts.Single(p => p.Key == "bundle/elasticsearch/registry.json"); - registryPut.ContentBody.Should().Contain("elasticsearch-9.3.0.amend-1.yaml"); + outcome.Should().Be(GroupReconcileOutcome.Written); + var registryContent = s3.ContentOf("public-bucket", "bundle/elasticsearch/registry.json"); + registryContent.Should().Contain("elasticsearch-9.3.0.amend-1.yaml"); // -- 4. :version:-filtered CDN fetch returns the amend and applies the retraction ---- using var handler = new StubHandler(req => { var path = req.RequestUri!.AbsolutePath; if (path.EndsWith("/registry.json", StringComparison.Ordinal)) - return Response(registryPut.ContentBody, "application/json"); + return Response(registryContent, "application/json"); var fileName = path[(path.LastIndexOf('/') + 1)..]; var localPath = FileSystem.Path.Join(bundleDir, fileName); diff --git a/tests/Elastic.Changelog.Tests/Uploading/ChangelogUploadServiceTests.cs b/tests/Elastic.Changelog.Tests/Uploading/ChangelogUploadServiceTests.cs index 2217c36306..a4e5c1cef3 100644 --- a/tests/Elastic.Changelog.Tests/Uploading/ChangelogUploadServiceTests.cs +++ b/tests/Elastic.Changelog.Tests/Uploading/ChangelogUploadServiceTests.cs @@ -707,7 +707,7 @@ public void DiscoverBundleUploadTargets_OrphanLegacyAmend_WarnsAndSkips() } [Fact] - public async Task Upload_BundleArtifactType_UploadsRegistryAlongsideBundle() + public async Task Upload_BundleArtifactType_DoesNotWriteRegistry() { var bundleDir = _mockFileSystem.Path.Join(Paths.WorkingDirectoryRoot.FullName, Guid.NewGuid().ToString(), "releases"); _mockFileSystem.Directory.CreateDirectory(bundleDir); @@ -751,14 +751,16 @@ public async Task Upload_BundleArtifactType_UploadsRegistryAlongsideBundle() A._ )).MustHaveHappenedOnceExactly(); + // The scrubber Lambda is the sole registry producer (docs-eng-team#688 Phase 3): + // uploads write YAML objects only, never a registry.json. A.CallTo(() => _s3Client.PutObjectAsync( - A.That.Matches(r => r.Key == "bundle/elasticsearch/registry.json"), + A.That.Matches(r => r.Key.EndsWith("registry.json", StringComparison.Ordinal)), A._ - )).MustHaveHappenedOnceExactly(); + )).MustNotHaveHappened(); } [Fact] - public async Task Upload_ChangelogArtifactType_RefreshesRepoScopedRegistry() + public async Task Upload_ChangelogArtifactType_DoesNotWriteRegistry() { // language=yaml AddChangelog("entry.yaml", """ @@ -793,14 +795,15 @@ public async Task Upload_ChangelogArtifactType_RefreshesRepoScopedRegistry() result.Should().BeTrue(); - // Changelog uploads refresh the pool-scoped entry index, not a bundle index. A.CallTo(() => _s3Client.PutObjectAsync( - A.That.Matches(r => r.Key == "changelog/elastic/elasticsearch/main/registry.json"), + A.That.Matches(r => r.Key == "changelog/elastic/elasticsearch/main/entry.yaml"), A._ )).MustHaveHappenedOnceExactly(); + // The scrubber Lambda is the sole registry producer (docs-eng-team#688 Phase 3): + // uploads write YAML objects only, never a registry.json. A.CallTo(() => _s3Client.PutObjectAsync( - A.That.Matches(r => r.Key.StartsWith("bundle/", StringComparison.Ordinal)), + A.That.Matches(r => r.Key.EndsWith("registry.json", StringComparison.Ordinal)), A._ )).MustNotHaveHappened(); } diff --git a/tests/Elastic.Changelog.Tests/Uploading/RegistryBuilderTests.cs b/tests/Elastic.Changelog.Tests/Uploading/RegistryBuilderTests.cs deleted file mode 100644 index bf8566e0c9..0000000000 --- a/tests/Elastic.Changelog.Tests/Uploading/RegistryBuilderTests.cs +++ /dev/null @@ -1,545 +0,0 @@ -// Licensed to Elasticsearch B.V under one or more agreements. -// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. -// See the LICENSE file in the project root for more information - -using System.Diagnostics.CodeAnalysis; -using System.IO.Abstractions.TestingHelpers; -using System.Net; -using System.Text; -using System.Text.Json; -using Amazon.S3; -using Amazon.S3.Model; -using AwesomeAssertions; -using Elastic.Changelog.Tests.Changelogs; -using Elastic.Changelog.Uploading; -using Elastic.Documentation.Configuration; -using Elastic.Documentation.Integrations.S3; -using FakeItEasy; -using Microsoft.Extensions.Logging.Abstractions; -using Nullean.ScopedFileSystem; - -namespace Elastic.Changelog.Tests.Uploading; - -[SuppressMessage("Usage", "CA1001:Types that own disposable fields should be disposable")] -public class RegistryBuilderTests -{ - private readonly MockFileSystem _mockFileSystem; - private readonly ScopedFileSystem _fileSystem; - private readonly IAmazonS3 _s3Client = A.Fake(); - private readonly TestDiagnosticsCollector _collector; - private readonly string _bundleDir; - private readonly RegistryBuilder _builder; - private readonly List _puts = []; - - public RegistryBuilderTests(ITestOutputHelper output) - { - _mockFileSystem = new MockFileSystem(new MockFileSystemOptions - { - CurrentDirectory = Paths.WorkingDirectoryRoot.FullName - }); - _fileSystem = FileSystemFactory.ScopeCurrentWorkingDirectory(_mockFileSystem); - _collector = new TestDiagnosticsCollector(output); - _bundleDir = _mockFileSystem.Path.Join(Paths.WorkingDirectoryRoot.FullName, Guid.NewGuid().ToString(), "releases"); - _mockFileSystem.Directory.CreateDirectory(_bundleDir); - var etagCalculator = new S3EtagCalculator(NullLoggerFactory.Instance, _fileSystem); - // Pin time so generated_at is deterministic in tests. - var fixedTime = new FakeTimeProvider(new DateTimeOffset(2026, 5, 6, 12, 0, 0, TimeSpan.Zero)); - _builder = new RegistryBuilder( - NullLoggerFactory.Instance, - _fileSystem, - _s3Client, - etagCalculator, - "test-bucket", - fixedTime); - - // Capture every manifest PUT so tests can inspect the body and conditional headers. - A.CallTo(() => _s3Client.PutObjectAsync(A._, A._)) - .Invokes((PutObjectRequest r, CancellationToken _) => _puts.Add(r)) - .Returns(new PutObjectResponse()); - } - - private string AddBundle(string fileName, string product, string target) - { - var path = _mockFileSystem.Path.Join(_bundleDir, fileName); - // language=yaml - _mockFileSystem.AddFile(path, new MockFileData($$""" - products: - - product: {{product}} - target: {{target}} - repo: {{product}} - owner: elastic - entries: - - file: - name: 1-feature.yaml - checksum: deadbeef - type: enhancement - title: Sample - """)); - return path; - } - - private void StubExistingManifestNotFound() => - A.CallTo(() => _s3Client.GetObjectAsync(A._, A._)) - .Throws(new AmazonS3Exception("Not Found") { StatusCode = HttpStatusCode.NotFound }); - - private void StubExistingManifest(string product, Registry manifest, string etag = "\"existing-etag\"") => - A.CallTo(() => _s3Client.GetObjectAsync( - A.That.Matches(r => r.Key == $"bundle/{product}/registry.json"), - A._)) - .ReturnsLazily(() => MakeManifestResponse(manifest, etag)); - - private static GetObjectResponse MakeManifestResponse(Registry manifest, string etag) - { - var json = JsonSerializer.Serialize(manifest, RegistryJsonContext.Default.Registry); - return new GetObjectResponse - { - ETag = etag, - ResponseStream = new MemoryStream(Encoding.UTF8.GetBytes(json)) - }; - } - - private static Registry Deserialize(string? json) => - JsonSerializer.Deserialize(json!, RegistryJsonContext.Default.Registry)!; - - [Fact] - public async Task Refresh_NoExistingManifest_CreatesManifestWithIfNoneMatch() - { - var path = AddBundle("9.3.0.yaml", "elasticsearch", "9.3.0"); - var targets = new List { new(path, "bundle/elasticsearch/9.3.0.yaml") }; - StubExistingManifestNotFound(); - - var result = await _builder.RefreshAsync(_collector, targets, TestContext.Current.CancellationToken); - - result.Updated.Should().Be(1); - _puts.Should().ContainSingle(); - var put = _puts[0]; - put.Key.Should().Be("bundle/elasticsearch/registry.json"); - put.IfNoneMatch.Should().Be("*"); - put.IfMatch.Should().BeNull(); - - var manifest = Deserialize(put.ContentBody); - manifest.Product.Should().Be("elasticsearch"); - manifest.Bundles.Should().ContainSingle(); - manifest.Bundles[0].File.Should().Be("9.3.0.yaml"); - manifest.Bundles[0].Target.Should().Be("9.3.0"); - manifest.Bundles[0].ETag.Should().NotBeNullOrEmpty(); - } - - [Fact] - public async Task Refresh_ExistingManifest_MergesByFileNameAndUsesIfMatch() - { - var existing = new Registry - { - Product = "elasticsearch", - GeneratedAt = new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero), - Bundles = - [ - new RegistryBundle { File = "9.2.0.yaml", Target = "9.2.0", ETag = "old-etag-1" }, - new RegistryBundle { File = "9.3.0.yaml", Target = "9.3.0", ETag = "old-etag-2" } - ] - }; - StubExistingManifest("elasticsearch", existing, "\"manifest-v1\""); - - // Re-upload of 9.3.0 (new content → new ETag) plus a brand-new 9.4.0. - var newer = AddBundle("9.3.0.yaml", "elasticsearch", "9.3.0"); - var ten = AddBundle("9.4.0.yaml", "elasticsearch", "9.4.0"); - var targets = new List - { - new(newer, "bundle/elasticsearch/9.3.0.yaml"), - new(ten, "bundle/elasticsearch/9.4.0.yaml") - }; - - var result = await _builder.RefreshAsync(_collector, targets, TestContext.Current.CancellationToken); - - result.Updated.Should().Be(1); - _puts.Should().ContainSingle(); - _puts[0].IfMatch.Should().Be("\"manifest-v1\""); - _puts[0].IfNoneMatch.Should().BeNull(); - - var manifest = Deserialize(_puts[0].ContentBody); - manifest.Bundles.Should().HaveCount(3); - manifest.Bundles.Should().Contain(b => b.File == "9.2.0.yaml" && b.ETag == "old-etag-1"); - - var nineThree = manifest.Bundles.Single(b => b.File == "9.3.0.yaml"); - nineThree.ETag.Should().NotBe("old-etag-2"); // replaced by the freshly-uploaded ETag - manifest.Bundles.Should().Contain(b => b.File == "9.4.0.yaml"); - - manifest.Bundles[0].File.Should().Be("9.4.0.yaml"); // sorted target-desc - } - - [Fact] - public async Task Refresh_SortsManifestByVersionNotLexicographically() - { - StubExistingManifestNotFound(); - - // Version order (not byte order): 9.10.0 must come before 9.9.0 in the written manifest. - var v910 = AddBundle("9.10.0.yaml", "elasticsearch", "9.10.0"); - var v99 = AddBundle("9.9.0.yaml", "elasticsearch", "9.9.0"); - var targets = new List - { - new(v99, "bundle/elasticsearch/9.9.0.yaml"), - new(v910, "bundle/elasticsearch/9.10.0.yaml") - }; - - _ = await _builder.RefreshAsync(_collector, targets, TestContext.Current.CancellationToken); - - var manifest = Deserialize(_puts[0].ContentBody); - manifest.Bundles.Select(b => b.Target).Should().Equal("9.10.0", "9.9.0"); - } - - [Fact] - public async Task Refresh_MultipleProducts_WritesOneManifestPerProduct() - { - var es = AddBundle("9.3.0.yaml", "elasticsearch", "9.3.0"); - var kb = AddBundle("kb-9.3.0.yaml", "kibana", "9.3.0"); - var targets = new List - { - new(es, "bundle/elasticsearch/9.3.0.yaml"), - new(kb, "bundle/kibana/kb-9.3.0.yaml") - }; - StubExistingManifestNotFound(); - - var result = await _builder.RefreshAsync(_collector, targets, TestContext.Current.CancellationToken); - - result.Updated.Should().Be(2); - _puts.Should().HaveCount(2); - _puts.Should().Contain(p => p.Key == "bundle/elasticsearch/registry.json"); - _puts.Should().Contain(p => p.Key == "bundle/kibana/registry.json"); - } - - [Fact] - public async Task Refresh_MultiProductBundle_RecordsTargetPerProduct() - { - // One bundle file declaring two products with *different* targets. - var path = _mockFileSystem.Path.Join(_bundleDir, "multi.yaml"); - // language=yaml - _mockFileSystem.AddFile(path, new MockFileData(""" - products: - - product: elasticsearch - target: 9.3.0 - repo: elasticsearch - owner: elastic - - product: kibana - target: 9.4.0 - repo: kibana - owner: elastic - entries: - - file: - name: 1-feature.yaml - checksum: deadbeef - type: enhancement - title: Sample - """)); - var targets = new List - { - new(path, "bundle/elasticsearch/multi.yaml"), - new(path, "bundle/kibana/multi.yaml") - }; - StubExistingManifestNotFound(); - - _ = await _builder.RefreshAsync(_collector, targets, TestContext.Current.CancellationToken); - - var es = Deserialize(_puts.Single(p => p.Key == "bundle/elasticsearch/registry.json").ContentBody); - es.Bundles[0].Target.Should().Be("9.3.0"); - - var kb = Deserialize(_puts.Single(p => p.Key == "bundle/kibana/registry.json").ContentBody); - kb.Bundles[0].Target.Should().Be("9.4.0"); - } - - [Fact] - public async Task Refresh_ExistingManifestUnreadable_OverwritesUsingLiveETag() - { - A.CallTo(() => _s3Client.GetObjectAsync(A._, A._)) - .ReturnsLazily(() => new GetObjectResponse - { - ETag = "\"corrupt-etag\"", - ResponseStream = new MemoryStream(Encoding.UTF8.GetBytes("not json {{{")) - }); - - var path = AddBundle("9.3.0.yaml", "elasticsearch", "9.3.0"); - var targets = new List { new(path, "bundle/elasticsearch/9.3.0.yaml") }; - - var result = await _builder.RefreshAsync(_collector, targets, TestContext.Current.CancellationToken); - - result.Updated.Should().Be(1); - _puts.Should().ContainSingle(); - _puts[0].IfMatch.Should().Be("\"corrupt-etag\""); // conditional overwrite of the corrupt object - var manifest = Deserialize(_puts[0].ContentBody); - manifest.Bundles.Should().ContainSingle(); - manifest.Bundles[0].File.Should().Be("9.3.0.yaml"); - } - - [Fact] - public async Task Refresh_BundleWithoutTarget_RecordsEntryWithoutTarget() - { - var path = _mockFileSystem.Path.Join(_bundleDir, "no-target.yaml"); - // language=yaml - _mockFileSystem.AddFile(path, new MockFileData(""" - entries: [] - """)); - var targets = new List { new(path, "bundle/elasticsearch/no-target.yaml") }; - StubExistingManifestNotFound(); - - var result = await _builder.RefreshAsync(_collector, targets, TestContext.Current.CancellationToken); - - result.Updated.Should().Be(1); - var manifest = Deserialize(_puts[0].ContentBody); - manifest.Bundles.Should().ContainSingle(); - manifest.Bundles[0].Target.Should().BeNull(); - manifest.Bundles[0].ETag.Should().NotBeNullOrEmpty(); - } - - [Fact] - public async Task Refresh_AmendWithProducts_RecordsTargetFromOwnProducts() - { - var path = _mockFileSystem.Path.Join(_bundleDir, "9.3.0.amend-1.yaml"); - // Amend materialized by a current docs-builder: it carries the parent's complete products. - // language=yaml - _mockFileSystem.AddFile(path, new MockFileData(""" - products: - - product: elasticsearch - target: 9.3.0 - repo: elasticsearch - owner: elastic - entries: - - file: - name: 2-late.yaml - checksum: c0ffee - type: enhancement - title: Late addition - """)); - var targets = new List { new(path, "bundle/elasticsearch/9.3.0.amend-1.yaml") }; - StubExistingManifestNotFound(); - - var result = await _builder.RefreshAsync(_collector, targets, TestContext.Current.CancellationToken); - - result.Updated.Should().Be(1); - var manifest = Deserialize(_puts[0].ContentBody); - manifest.Bundles.Should().ContainSingle(); - manifest.Bundles[0].File.Should().Be("9.3.0.amend-1.yaml"); - manifest.Bundles[0].Target.Should().Be("9.3.0"); - } - - [Fact] - public async Task Refresh_LegacyAmendWithoutProducts_RecordsParentTarget() - { - var parent = AddBundle("9.3.0.yaml", "elasticsearch", "9.3.0"); - var amend = _mockFileSystem.Path.Join(_bundleDir, "9.3.0.amend-1.yaml"); - // Amend published before products were copied from the parent: exclusion only, no products. - // language=yaml - _mockFileSystem.AddFile(amend, new MockFileData(""" - exclude-entries: - - file: - name: 1-feature.yaml - checksum: deadbeef - """)); - var targets = new List - { - new(parent, "bundle/elasticsearch/9.3.0.yaml"), - new(amend, "bundle/elasticsearch/9.3.0.amend-1.yaml") - }; - StubExistingManifestNotFound(); - - var result = await _builder.RefreshAsync(_collector, targets, TestContext.Current.CancellationToken); - - result.Updated.Should().Be(1); - var manifest = Deserialize(_puts[0].ContentBody); - manifest.Bundles.Should().HaveCount(2); - var amendEntry = manifest.Bundles.Single(b => b.File == "9.3.0.amend-1.yaml"); - amendEntry.Target.Should().Be("9.3.0", "the amend inherits the parent bundle's target"); - } - - [Fact] - public async Task Refresh_OrphanLegacyAmendWithoutParent_RecordsEntryWithoutTarget() - { - var amend = _mockFileSystem.Path.Join(_bundleDir, "9.3.0.amend-1.yaml"); - // language=yaml - _mockFileSystem.AddFile(amend, new MockFileData(""" - exclude-entries: - - file: - name: 1-feature.yaml - checksum: deadbeef - """)); - var targets = new List { new(amend, "bundle/elasticsearch/9.3.0.amend-1.yaml") }; - StubExistingManifestNotFound(); - - var result = await _builder.RefreshAsync(_collector, targets, TestContext.Current.CancellationToken); - - result.Updated.Should().Be(1); - var manifest = Deserialize(_puts[0].ContentBody); - manifest.Bundles.Should().ContainSingle(); - manifest.Bundles[0].Target.Should().BeNull(); - } - - [Fact] - public async Task Refresh_UnchangedManifest_SkipsWrite() - { - var path = AddBundle("9.3.0.yaml", "elasticsearch", "9.3.0"); - var etagCalculator = new S3EtagCalculator(NullLoggerFactory.Instance, _fileSystem); - var bundleEtag = await etagCalculator.CalculateS3ETag(path, TestContext.Current.CancellationToken); - - // Existing manifest already contains exactly what this run would produce. - var existing = new Registry - { - Product = "elasticsearch", - GeneratedAt = new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero), - Bundles = [new RegistryBundle { File = "9.3.0.yaml", Target = "9.3.0", ETag = bundleEtag }] - }; - StubExistingManifest("elasticsearch", existing, "\"manifest-v1\""); - - var targets = new List { new(path, "bundle/elasticsearch/9.3.0.yaml") }; - var result = await _builder.RefreshAsync(_collector, targets, TestContext.Current.CancellationToken); - - result.Unchanged.Should().Be(1); - result.Updated.Should().Be(0); - _puts.Should().BeEmpty(); - } - - [Fact] - public async Task Refresh_ConcurrentWrite_RetriesAfterPreconditionFailed() - { - var v1 = new Registry - { - Product = "elasticsearch", - GeneratedAt = new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero), - Bundles = [new RegistryBundle { File = "9.2.0.yaml", Target = "9.2.0", ETag = "etag-92" }] - }; - // Second read reflects a concurrent writer that added 9.3.0 and bumped the object ETag. - var v2 = new Registry - { - Product = "elasticsearch", - GeneratedAt = new DateTimeOffset(2026, 1, 2, 0, 0, 0, TimeSpan.Zero), - Bundles = - [ - new RegistryBundle { File = "9.3.0.yaml", Target = "9.3.0", ETag = "etag-93" }, - new RegistryBundle { File = "9.2.0.yaml", Target = "9.2.0", ETag = "etag-92" } - ] - }; - A.CallTo(() => _s3Client.GetObjectAsync(A._, A._)) - .ReturnsNextFromSequence( - MakeManifestResponse(v1, "\"manifest-v1\""), - MakeManifestResponse(v2, "\"manifest-v2\"")); - - // First PUT loses the optimistic-concurrency race; the second succeeds. - A.CallTo(() => _s3Client.PutObjectAsync(A._, A._)) - .Throws(new AmazonS3Exception("Precondition Failed") { StatusCode = HttpStatusCode.PreconditionFailed }) - .Once(); - - var path = AddBundle("9.4.0.yaml", "elasticsearch", "9.4.0"); - var targets = new List { new(path, "bundle/elasticsearch/9.4.0.yaml") }; - - var result = await _builder.RefreshAsync(_collector, targets, TestContext.Current.CancellationToken); - - result.Updated.Should().Be(1); - A.CallTo(() => _s3Client.GetObjectAsync(A._, A._)) - .MustHaveHappenedTwiceExactly(); - A.CallTo(() => _s3Client.PutObjectAsync(A._, A._)) - .MustHaveHappenedTwiceExactly(); - - // Two PUTs were attempted (asserted above); the first threw on the precondition failure before - // _puts.Add ran, so only the successful retry is captured here. It used the re-read ETag and - // merged both the concurrent and local entries. - _puts.Should().ContainSingle(); - _puts[0].IfMatch.Should().Be("\"manifest-v2\""); - var manifest = Deserialize(_puts[0].ContentBody); - manifest.Bundles.Select(b => b.File).Should().BeEquivalentTo(["9.4.0.yaml", "9.3.0.yaml", "9.2.0.yaml"]); - } - - [Fact] - public async Task Refresh_PersistentConcurrentWrite_EmitsWarningAndReportsFailure() - { - var existing = new Registry - { - Product = "elasticsearch", - GeneratedAt = new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero), - Bundles = [new RegistryBundle { File = "9.2.0.yaml", Target = "9.2.0", ETag = "etag-92" }] - }; - StubExistingManifest("elasticsearch", existing, "\"manifest-v1\""); - - // Every PUT loses the race. - A.CallTo(() => _s3Client.PutObjectAsync(A._, A._)) - .Throws(new AmazonS3Exception("Precondition Failed") { StatusCode = HttpStatusCode.PreconditionFailed }); - - var path = AddBundle("9.4.0.yaml", "elasticsearch", "9.4.0"); - var targets = new List { new(path, "bundle/elasticsearch/9.4.0.yaml") }; - - var result = await _builder.RefreshAsync(_collector, targets, TestContext.Current.CancellationToken); - - result.Failed.Should().Be(1); - result.Updated.Should().Be(0); - _collector.Warnings.Should().BeGreaterThan(0); - } - - [Fact] - public async Task Refresh_NoTargets_WritesNothing() - { - var result = await _builder.RefreshAsync(_collector, [], TestContext.Current.CancellationToken); - result.Should().Be(new RegistryBuilder.RefreshResult(0, 0, 0)); - _puts.Should().BeEmpty(); - } - - [Fact] - public async Task Refresh_ChangelogScope_WritesEntryRegistryWithoutTarget() - { - var path = _mockFileSystem.Path.Join(_bundleDir, "1-feature.yaml"); - // language=yaml - _mockFileSystem.AddFile(path, new MockFileData(""" - title: Sample - type: enhancement - products: - - product: elasticsearch - target: 9.3.0 - """)); - var targets = new List { new(path, "changelog/elastic/elasticsearch/main/1-feature.yaml") }; - A.CallTo(() => _s3Client.GetObjectAsync(A._, A._)) - .Throws(new AmazonS3Exception("Not Found") { StatusCode = HttpStatusCode.NotFound }); - - var result = await _builder.RefreshAsync(_collector, targets, TestContext.Current.CancellationToken, RegistryScope.Changelog); - - result.Updated.Should().Be(1); - _puts.Should().ContainSingle(); - _puts[0].Key.Should().Be("changelog/elastic/elasticsearch/main/registry.json"); - - var manifest = Deserialize(_puts[0].ContentBody); - // The grouping identifier for the entry index is the {org}/{repo}/{branch} prefix. - manifest.Product.Should().Be("elastic/elasticsearch/main"); - manifest.Bundles.Should().ContainSingle(); - manifest.Bundles[0].File.Should().Be("1-feature.yaml"); - // The changelog-entry index only enumerates files; per-entry target is not recorded. - manifest.Bundles[0].Target.Should().BeNull(); - manifest.Bundles[0].ETag.Should().NotBeNullOrEmpty(); - } - - [Fact] - public async Task Refresh_ChangelogScope_BranchWithSlashes_GroupsByFullPoolPrefix() - { - var path = _mockFileSystem.Path.Join(_bundleDir, "2-feature.yaml"); - // language=yaml - _mockFileSystem.AddFile(path, new MockFileData(""" - title: Sample on a feature branch - type: enhancement - products: - - product: elasticsearch - """)); - // The branch "feature/foo" contributes two key segments; the registry must live at the pool root. - var targets = new List { new(path, "changelog/elastic/elasticsearch/feature/foo/2-feature.yaml") }; - A.CallTo(() => _s3Client.GetObjectAsync(A._, A._)) - .Throws(new AmazonS3Exception("Not Found") { StatusCode = HttpStatusCode.NotFound }); - - var result = await _builder.RefreshAsync(_collector, targets, TestContext.Current.CancellationToken, RegistryScope.Changelog); - - result.Updated.Should().Be(1); - _puts.Should().ContainSingle(); - _puts[0].Key.Should().Be("changelog/elastic/elasticsearch/feature/foo/registry.json"); - - var manifest = Deserialize(_puts[0].ContentBody); - manifest.Product.Should().Be("elastic/elasticsearch/feature/foo"); - manifest.Bundles.Should().ContainSingle(); - manifest.Bundles[0].File.Should().Be("2-feature.yaml"); - } - - private sealed class FakeTimeProvider(DateTimeOffset now) : TimeProvider - { - public override DateTimeOffset GetUtcNow() => now; - } -}