Add changelog registry reconcile and verify commands

docs/cli-schema.json

@@ -4433,7 +4433,230 @@
           ]
         }
       ],
-      "namespaces": []
+      "namespaces": [
+        {
+          "segment": "registry",
+          "summary": "Operate on the scrubber-owned public changelog registries.",
+          "options": [],
+          "commands": [
+            {
+              "path": [
+                "changelog",
+                "registry"
+              ],
+              "name": "reconcile",
+              "summary": "Send explicit reconcile messages to the scrubber queue so the Lambda performs a full group heal (object-level reconcile over the union of both buckets, then a registry rebuild from public state) for every planned group. This command never mutates S3 itself \u2014 the scrubber Lambda stays the public bucket\u0027s single writer. Convergent: re-running re-plans against current state. Enqueuing is not reconciling \u2014 gate on \u0060changelog registry verify\u0060 after the queue drains and the DLQ is empty.",
+              "usage": "docs-builder changelog registry reconcile --s3-bucket-name \u003Cstring\u003E --public-s3-bucket-name \u003Cstring\u003E --queue-url \u003Cstring\u003E [options]",
+              "examples": [],
+              "parameters": [
+                {
+                  "role": "flag",
+                  "name": "s3-bucket-name",
+                  "type": "string",
+                  "required": true,
+                  "summary": "Private changelog bundles bucket to plan from."
+                },
+                {
+                  "role": "flag",
+                  "name": "public-s3-bucket-name",
+                  "type": "string",
+                  "required": true,
+                  "summary": "Public (CDN) changelog bundles bucket to plan from."
+                },
+                {
+                  "role": "flag",
+                  "name": "queue-url",
+                  "type": "string",
+                  "required": true,
+                  "summary": "URL of the scrubber SQS queue to send reconcile messages to."
+                },
+                {
+                  "role": "flag",
+                  "name": "product",
+                  "type": "string",
+                  "required": false,
+                  "summary": "Only reconcile this bundle group (bundle/{product}/). Mutually exclusive with --owner/--repo/--branch."
+                },
+                {
+                  "role": "flag",
+                  "name": "owner",
+                  "type": "string",
+                  "required": false,
+                  "summary": "GitHub owner of a single changelog-pool group to reconcile."
+                },
+                {
+                  "role": "flag",
+                  "name": "repo",
+                  "type": "string",
+                  "required": false,
+                  "summary": "Repository of a single changelog-pool group to reconcile."
+                },
+                {
+                  "role": "flag",
+                  "name": "branch",
+                  "type": "string",
+                  "required": false,
+                  "summary": "Branch of a single changelog-pool group to reconcile (verbatim; slashes allowed)."
+                },
+                {
+                  "role": "dryRun",
+                  "name": "dry-run",
+                  "type": "boolean",
+                  "required": false,
+                  "summary": "Print the group plan without sending anything.",
+                  "defaultValue": "false"
+                },
+                {
+                  "role": "flag",
+                  "name": "yes",
+                  "type": "boolean",
+                  "required": false,
+                  "summary": "Skip the interactive confirmation (required when stdin is not a terminal).",
+                  "defaultValue": "false"
+                },
+                {
+                  "role": "flag",
+                  "name": "log-level",
+                  "shortName": "l",
+                  "type": "enum",
+                  "required": false,
+                  "summary": "Minimum log level. Default: information",
+                  "enumValues": [
+                    "trace",
+                    "debug",
+                    "information",
+                    "warning",
+                    "error",
+                    "critical",
+                    "none"
+                  ]
+                },
+                {
+                  "role": "flag",
+                  "name": "config-source",
+                  "shortName": "c",
+                  "type": "enum",
+                  "required": false,
+                  "summary": "Override the configuration source: local, remote",
+                  "enumValues": [
+                    "local",
+                    "remote",
+                    "embedded"
+                  ]
+                },
+                {
+                  "role": "flag",
+                  "name": "skip-private-repositories",
+                  "type": "boolean",
+                  "required": false,
+                  "summary": "Skip cloning private repositories"
+                }
+              ],
+              "intent": {
+                "destructive": true,
+                "idempotent": true,
+                "scope": "global",
+                "requiresAuth": true
+              }
+            },
+            {
+              "path": [
+                "changelog",
+                "registry"
+              ],
+              "name": "verify",
+              "summary": "Compare each planned group\u0027s public registry.json against what a reconcile of the current public listing would write, and report divergence (missing, stale, corrupt, object-divergent; unsupported schema reported distinctly). Strictly read-only. Zero divergence across the plan is the cutover completion gate \u2014 and the standing drift-diagnosis tool afterwards.",
+              "usage": "docs-builder changelog registry verify --s3-bucket-name \u003Cstring\u003E --public-s3-bucket-name \u003Cstring\u003E [options]",
+              "examples": [],
+              "parameters": [
+                {
+                  "role": "flag",
+                  "name": "s3-bucket-name",
+                  "type": "string",
+                  "required": true,
+                  "summary": "Private changelog bundles bucket to plan from."
+                },
+                {
+                  "role": "flag",
+                  "name": "public-s3-bucket-name",
+                  "type": "string",
+                  "required": true,
+                  "summary": "Public (CDN) changelog bundles bucket to verify."
+                },
+                {
+                  "role": "flag",
+                  "name": "product",
+                  "type": "string",
+                  "required": false,
+                  "summary": "Only verify this bundle group (bundle/{product}/). Mutually exclusive with --owner/--repo/--branch."
+                },
+                {
+                  "role": "flag",
+                  "name": "owner",
+                  "type": "string",
+                  "required": false,
+                  "summary": "GitHub owner of a single changelog-pool group to verify."
+                },
+                {
+                  "role": "flag",
+                  "name": "repo",
+                  "type": "string",
+                  "required": false,
+                  "summary": "Repository of a single changelog-pool group to verify."
+                },
+                {
+                  "role": "flag",
+                  "name": "branch",
+                  "type": "string",
+                  "required": false,
+                  "summary": "Branch of a single changelog-pool group to verify (verbatim; slashes allowed)."
+                },
+                {
+                  "role": "flag",
+                  "name": "log-level",
+                  "shortName": "l",
+                  "type": "enum",
+                  "required": false,
+                  "summary": "Minimum log level. Default: information",
+                  "enumValues": [
+                    "trace",
+                    "debug",
+                    "information",
+                    "warning",
+                    "error",
+                    "critical",
+                    "none"
+                  ]
+                },
+                {
+                  "role": "flag",
+                  "name": "config-source",
+                  "shortName": "c",
+                  "type": "enum",
+                  "required": false,
+                  "summary": "Override the configuration source: local, remote",
+                  "enumValues": [
+                    "local",
+                    "remote",
+                    "embedded"
+                  ]
+                },
+                {
+                  "role": "flag",
+                  "name": "skip-private-repositories",
+                  "type": "boolean",
+                  "required": false,
+                  "summary": "Skip cloning private repositories"
+                }
+              ],
+              "intent": {
+                "requiresAuth": true
+              }
+            }
+          ],
+          "namespaces": []
+        }
+      ]
     },
     {
       "segment": "codex",

docs/cli/changelog/registry/cmd-reconcile.md

@@ -0,0 +1,31 @@
+## Description
+
+Sends one explicit reconcile message per planned group to the scrubber queue. For each message the Lambda performs a **full group heal**: an object-level reconcile over the union of both buckets' listings (scrub and copy whatever is live in the private bucket, delete public objects nothing backs), then a rebuild of the group's public `registry.json` from the public listing. This recovers drift that no pending S3 event would ever repair — lost or DLQ-expired scrub events, ad-hoc uploads, orphaned public objects.
+
+Without a scope filter, the plan is the union of groups discovered in **both** buckets, so orphan public groups (including groups that only have a leftover manifest) are covered. Use `--product`, or `--owner`/`--repo`/`--branch` together, to reconcile a single group.
+
+The command is convergent: re-running it re-plans against current state and the Lambda's writes are conditional, so overlapping runs cannot corrupt a manifest. It is also deliberately indirect — the CLI only sends queue messages, keeping the scrubber Lambda the sole writer of the public bucket.
+
+Every run stamps one correlation id on all its messages and prints a ledger line per group (`group`, SQS `message-id`, `correlation-id`). **Enqueuing is not reconciling**: after a run, watch the queue drain (oldest-message-age ≈ 0), triage anything that reaches the DLQ, and gate on [](/cli/changelog/registry/verify.md) reporting zero divergence.
+
+## Examples
+
+Preview the full plan without sending anything:
+
+```bash
+docs-builder changelog registry reconcile \
+  --s3-bucket-name elastic-docs-v3-changelog-bundles-private \
+  --public-s3-bucket-name elastic-docs-v3-changelog-bundles \
+  --queue-url https://sqs.us-east-1.amazonaws.com/<account>/elastic-docs-v3-changelog-scrub-queue \
+  --dry-run
+```
+
+Reconcile a single product's bundle registry non-interactively:
+
+```bash
+docs-builder changelog registry reconcile \
+  --s3-bucket-name elastic-docs-v3-changelog-bundles-private \
+  --public-s3-bucket-name elastic-docs-v3-changelog-bundles \
+  --queue-url https://sqs.us-east-1.amazonaws.com/<account>/elastic-docs-v3-changelog-scrub-queue \
+  --product elasticsearch --yes
+```

docs/cli/changelog/registry/cmd-verify.md

@@ -0,0 +1,21 @@
+## Description
+
+Read-only drift diagnosis: for every planned group, compares the public `registry.json` against what a reconcile of the current public listing would write — the exact same listing spec and entry rules the scrubber Lambda uses — and reports each divergence:
+
+| Kind | Meaning |
+|---|---|
+| `Missing` | A public object (or the manifest itself) the registry should describe but doesn't. |
+| `Stale` | A manifest entry (or whole manifest) describing something no longer in the bucket, or manifest metadata a reconcile would rewrite. |
+| `Corrupt` | The manifest exists but cannot be parsed. |
+| `ObjectDivergent` | File present on both sides, but the recorded ETag or target disagrees with the object. |
+| `UnsupportedSchema` | The manifest declares a newer `schema_version` than this tool understands. Reported distinctly and never rewritten. |
+
+The command exits non-zero when any group diverges. Zero divergence across the plan is the completion gate after a [](/cli/changelog/registry/reconcile.md) run — and the standing way to answer "is the registry trustworthy right now?".
+
+## Examples
+
+```bash
+docs-builder changelog registry verify \
+  --s3-bucket-name elastic-docs-v3-changelog-bundles-private \
+  --public-s3-bucket-name elastic-docs-v3-changelog-bundles
+```

docs/cli/changelog/registry/index.md

@@ -0,0 +1,3 @@
+Operate on the public changelog bundle registries owned by the scrubber Lambda.
+
+Each group of published changelog artifacts — `bundle/{product}/` or `changelog/{org}/{repo}/{branch}/` — carries a `registry.json` manifest in the **public** bucket, produced exclusively by the scrubber Lambda from the bucket's actual state. These commands are the operator surface for that machinery: `reconcile` asks the Lambda to converge groups (via explicit queue messages — the CLI never writes to S3 itself), and `verify` reports, read-only, whether each public manifest matches its public listing.

src/infra/docs-lambda-changelog-scrubber/Program.cs

@@ -17,6 +17,10 @@
 var publicBucketName = Environment.GetEnvironmentVariable("PUBLIC_BUCKET_NAME")
 	?? throw new InvalidOperationException("PUBLIC_BUCKET_NAME environment variable is required");
 
+// Optional: only explicit reconcile messages (full group heals) need to list the private bucket
+// by name — S3 events carry their source bucket. Without it, reconcile messages are rejected.
+var privateBucketName = Environment.GetEnvironmentVariable("PRIVATE_BUCKET_NAME");
+
 var allowRepos = BuildAllowlist();
 
 await LambdaBootstrapBuilder
@@ -55,7 +59,7 @@ async Task<SQSBatchResponse> Handler(SQSEvent ev, ILambdaContext context)
 	var metrics = new ReconcileMetrics();
 	var scrubber = new ChangelogContentScrubber(logFactory, allowRepos);
 	var reconciler = new RegistryReconciler(logFactory, s3Client, publicBucketName, metrics: metrics);
-	var processor = new ScrubberProcessor(logFactory, s3Client, publicBucketName, scrubber, reconciler, metrics);
+	var processor = new ScrubberProcessor(logFactory, s3Client, publicBucketName, scrubber, reconciler, metrics, privateBucketName);
 
 	var messages = ev.Records.Select(r => new ScrubberQueueMessage(r.MessageId, r.Body)).ToList();
 	var failedIds = await processor.ProcessAsync(messages, CancellationToken.None);

src/services/Elastic.Changelog/Elastic.Changelog.csproj

@@ -17,6 +17,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <PackageReference Include="AWSSDK.SQS" />
     <PackageReference Include="GitHub.Actions.Core" />
     <PackageReference Include="NetEscapades.EnumGenerators" />
     <PackageReference Include="Vecc.YamlDotNet.Analyzers.StaticGenerator" />

src/services/Elastic.Changelog/Reconciliation/ChangelogGroupDiscovery.cs

@@ -0,0 +1,66 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using Amazon.S3;
+using Amazon.S3.Model;
+using Elastic.Documentation.Configuration.ReleaseNotes;
+
+namespace Elastic.Changelog.Reconciliation;
+
+/// <summary>
+/// Enumerates every registry group present in a bucket by walking the <c>bundle/</c> and
+/// <c>changelog/</c> prefixes and deriving each key's scope. Registry keys count too, so a group
+/// that only has an orphaned manifest left is still planned (its reconcile deletes the manifest).
+/// The reconcile/verify planners union this across both buckets so orphan public groups are
+/// covered as well.
+/// </summary>
+public static class ChangelogGroupDiscovery
+{
+	/// <summary>Every scope with at least one key in <paramref name="bucketName"/>, keyed by prefix.</summary>
+	public static async Task<IReadOnlyDictionary<string, ChangelogScope>> DiscoverGroupsAsync(
+		IAmazonS3 s3Client,
+		string bucketName,
+		Cancel ctx)
+	{
+		var scopes = new Dictionary<string, ChangelogScope>(StringComparer.Ordinal);
+		foreach (var prefix in new[] { ChangelogKeys.BundlePrefix, ChangelogKeys.ChangelogPrefix })
+		{
+			var request = new ListObjectsV2Request
+			{
+				BucketName = bucketName,
+				Prefix = prefix
+			};
+
+			ListObjectsV2Response response;
+			do
+			{
+				response = await s3Client.ListObjectsV2Async(request, ctx);
+				foreach (var obj in response.S3Objects ?? [])
+				{
+					if (ChangelogScope.TryFromKey(obj.Key, out var scope))
+						_ = scopes.TryAdd(scope.Prefix, scope);
+				}
+				request.ContinuationToken = response.NextContinuationToken;
+			} while (response.IsTruncated == true);
+		}
+
+		return scopes;
+	}
+
+	/// <summary>The union of both buckets' groups, ordered by prefix for a stable plan.</summary>
+	public static async Task<IReadOnlyList<ChangelogScope>> DiscoverUnionAsync(
+		IAmazonS3 s3Client,
+		string privateBucketName,
+		string publicBucketName,
+		Cancel ctx)
+	{
+		var union = new Dictionary<string, ChangelogScope>(StringComparer.Ordinal);
+		foreach (var (prefix, scope) in await DiscoverGroupsAsync(s3Client, privateBucketName, ctx))
+			_ = union.TryAdd(prefix, scope);
+		foreach (var (prefix, scope) in await DiscoverGroupsAsync(s3Client, publicBucketName, ctx))
+			_ = union.TryAdd(prefix, scope);
+
+		return [.. union.Values.OrderBy(s => s.Prefix, StringComparer.Ordinal)];
+	}
+}