From d9d7e369fb777b71312367aa75406ad3c1e2279f Mon Sep 17 00:00:00 2001 From: Felipe Cotti Date: Thu, 23 Jul 2026 19:22:57 -0300 Subject: [PATCH] Add changelog registry state discovery and reconciliation commands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Registries merge additively under optimistic concurrency and are not authoritative for removals or discovery, so they can drift from the objects actually in the bucket and nothing could detect or repair that. Backfill planning needs a trustworthy current-state snapshot, so this adds a `changelog registry` command group: - `inspect` (read-only): classifies divergence between a scope's private registry.json and the actual private objects — missing, stale, corrupt, and object-divergent — and can emit a machine-readable RegistryStateSnapshot for the backfill planner. - `repair` (explicit, idempotent): converges the private registry to the actual objects through the same conditional-PUT optimistic concurrency as the live upload path, re-inspecting on 412 so concurrent uploads are never clobbered. Guarded against empty results (--allow-empty) and newer-schema downgrades; fully audited. - `verify-public` (strictly read-only): waits under a bounded retry policy for scrubber propagation and diagnoses public divergence. The public bucket is a hard write boundary enforced structurally: the comparison operates on a reader interface with no write surface. - `republish` (explicit): re-emits the private ObjectCreated event via a metadata-preserving S3 self-copy to recover a lost or DLQ'd scrub event; the scrubber remains the sole public-side writer. Part of elastic/docs-eng-team#670 --- docs/cli-schema.json | 435 +++++++++++++++++- docs/cli/changelog/registry/cmd-inspect.md | 39 ++ docs/cli/changelog/registry/cmd-repair.md | 42 ++ docs/cli/changelog/registry/cmd-republish.md | 38 ++ .../changelog/registry/cmd-verify-public.md | 36 ++ docs/cli/changelog/registry/index.md | 19 + docs/development/changelog-bundle-registry.md | 70 ++- .../ChangelogPublicVerificationService.cs | 246 ++++++++++ .../ChangelogRegistryArguments.cs | 106 +++++ .../ChangelogRegistryInspectionService.cs | 64 +++ .../ChangelogRegistryRepairService.cs | 172 +++++++ .../ChangelogRegistryRepublishService.cs | 150 ++++++ .../Reconciliation/ChangelogScope.cs | 76 +++ .../Reconciliation/RegistryScopeInspector.cs | 322 +++++++++++++ .../Reconciliation/RegistryStateFormatter.cs | 39 ++ .../Reconciliation/RegistryStateSnapshot.cs | 157 +++++++ .../Reconciliation/S3ScopeReader.cs | 85 ++++ .../Commands/ChangelogRegistryCommands.cs | 231 ++++++++++ src/tooling/docs-builder/Program.cs | 2 +- .../Reconciliation/FakeS3Bucket.cs | 161 +++++++ .../PublicVerificationServiceTests.cs | 188 ++++++++ .../RegistryInspectionServiceTests.cs | 76 +++ .../Reconciliation/RegistryInspectionTests.cs | 295 ++++++++++++ .../RegistryRepairServiceTests.cs | 227 +++++++++ .../RegistryRepublishServiceTests.cs | 139 ++++++ 25 files changed, 3409 insertions(+), 6 deletions(-) create mode 100644 docs/cli/changelog/registry/cmd-inspect.md create mode 100644 docs/cli/changelog/registry/cmd-repair.md create mode 100644 docs/cli/changelog/registry/cmd-republish.md create mode 100644 docs/cli/changelog/registry/cmd-verify-public.md create mode 100644 docs/cli/changelog/registry/index.md create mode 100644 src/services/Elastic.Changelog/Reconciliation/ChangelogPublicVerificationService.cs create mode 100644 src/services/Elastic.Changelog/Reconciliation/ChangelogRegistryArguments.cs create mode 100644 src/services/Elastic.Changelog/Reconciliation/ChangelogRegistryInspectionService.cs create mode 100644 src/services/Elastic.Changelog/Reconciliation/ChangelogRegistryRepairService.cs create mode 100644 src/services/Elastic.Changelog/Reconciliation/ChangelogRegistryRepublishService.cs create mode 100644 src/services/Elastic.Changelog/Reconciliation/ChangelogScope.cs create mode 100644 src/services/Elastic.Changelog/Reconciliation/RegistryScopeInspector.cs create mode 100644 src/services/Elastic.Changelog/Reconciliation/RegistryStateFormatter.cs create mode 100644 src/services/Elastic.Changelog/Reconciliation/RegistryStateSnapshot.cs create mode 100644 src/services/Elastic.Changelog/Reconciliation/S3ScopeReader.cs create mode 100644 src/tooling/docs-builder/Commands/ChangelogRegistryCommands.cs create mode 100644 tests/Elastic.Changelog.Tests/Reconciliation/FakeS3Bucket.cs create mode 100644 tests/Elastic.Changelog.Tests/Reconciliation/PublicVerificationServiceTests.cs create mode 100644 tests/Elastic.Changelog.Tests/Reconciliation/RegistryInspectionServiceTests.cs create mode 100644 tests/Elastic.Changelog.Tests/Reconciliation/RegistryInspectionTests.cs create mode 100644 tests/Elastic.Changelog.Tests/Reconciliation/RegistryRepairServiceTests.cs create mode 100644 tests/Elastic.Changelog.Tests/Reconciliation/RegistryRepublishServiceTests.cs diff --git a/docs/cli-schema.json b/docs/cli-schema.json index ef677c35be..4d72e50882 100644 --- a/docs/cli-schema.json +++ b/docs/cli-schema.json @@ -4433,7 +4433,440 @@ ] } ], - "namespaces": [] + "namespaces": [ + { + "segment": "registry", + "summary": "Inspect, repair, and verify per-scope changelog registry.json manifests against actual bucket state.", + "options": [], + "commands": [ + { + "path": [ + "changelog", + "registry" + ], + "name": "inspect", + "summary": "Compare a scope\u0027s private registry.json against the actual private-bucket objects and report every divergence.", + "notes": "Registries merge additively and are not authoritative for removals or discovery, so they can drift\nfrom the objects actually in the bucket. This command detects and classifies that drift: missing\n(object exists, registry lacks it), stale (registry entry, object gone), corrupt\n(unparseable/invalid manifest), and object-divergent (registry metadata disagrees with the object).\n\nStrictly read-only \u2014 nothing is written to any bucket. Exits non-zero when the scope diverged.\nUse changelog registry repair to reconcile.", + "usage": "docs-builder changelog registry inspect --s3-bucket-name \u003Cstring\u003E [options]", + "examples": [], + "parameters": [ + { + "role": "flag", + "name": "s3-bucket-name", + "type": "string", + "required": true, + "summary": "Private changelog bundles S3 bucket to inspect." + }, + { + "role": "flag", + "name": "product", + "type": "string", + "required": false, + "summary": "Product of a bundle scope (bundle/{product}/). Mutually exclusive with --owner/--repo/--branch." + }, + { + "role": "flag", + "name": "owner", + "type": "string", + "required": false, + "summary": "GitHub owner of a changelog scope (changelog/{org}/{repo}/{branch}/). Requires --repo and --branch." + }, + { + "role": "flag", + "name": "repo", + "type": "string", + "required": false, + "summary": "Repository of a changelog scope. Requires --owner and --branch." + }, + { + "role": "flag", + "name": "branch", + "type": "string", + "required": false, + "summary": "Branch of a changelog scope, stored verbatim (slashes become key segments). Requires --owner and --repo." + }, + { + "role": "flag", + "name": "out", + "type": "string", + "required": false, + "summary": "Path to write the machine-readable state snapshot JSON to.", + "validations": [ + { + "kind": "rejectSymbolicLinks" + } + ] + }, + { + "role": "flag", + "name": "log-level", + "shortName": "l", + "type": "enum", + "required": false, + "summary": "Minimum log level. Default: information", + "enumValues": [ + "trace", + "debug", + "information", + "warning", + "error", + "critical", + "none" + ] + }, + { + "role": "flag", + "name": "config-source", + "shortName": "c", + "type": "enum", + "required": false, + "summary": "Override the configuration source: local, remote", + "enumValues": [ + "local", + "remote", + "embedded" + ] + }, + { + "role": "flag", + "name": "skip-private-repositories", + "type": "boolean", + "required": false, + "summary": "Skip cloning private repositories" + } + ], + "intent": { + "requiresAuth": true + } + }, + { + "path": [ + "changelog", + "registry" + ], + "name": "repair", + "summary": "Reconcile a scope\u0027s private registry.json from the actual private-bucket objects.", + "notes": "Rebuilds the manifest from the objects that actually exist in the scope: missing entries are added,\nstale entries removed, and divergent metadata corrected. The write uses the same optimistic-concurrency\nconditional PUT as the live upload path (If-Match on update, If-None-Match: * on create), re-inspecting\nand retrying when a concurrent upload refreshes the manifest, so repair is safe to run alongside live\nuploads. Repair is idempotent: a clean scope writes nothing, and running it twice yields no further change.\n\nOnly the private registry is written. The public copy is scrubber-owned and converges through\nthe scrubber\u0027s pass-through of this write\u0027s own event. Every change is logged (before/after) for audit.", + "usage": "docs-builder changelog registry repair --s3-bucket-name \u003Cstring\u003E [options]", + "examples": [], + "parameters": [ + { + "role": "flag", + "name": "s3-bucket-name", + "type": "string", + "required": true, + "summary": "Private changelog bundles S3 bucket to repair the registry in." + }, + { + "role": "flag", + "name": "product", + "type": "string", + "required": false, + "summary": "Product of a bundle scope (bundle/{product}/). Mutually exclusive with --owner/--repo/--branch." + }, + { + "role": "flag", + "name": "owner", + "type": "string", + "required": false, + "summary": "GitHub owner of a changelog scope (changelog/{org}/{repo}/{branch}/). Requires --repo and --branch." + }, + { + "role": "flag", + "name": "repo", + "type": "string", + "required": false, + "summary": "Repository of a changelog scope. Requires --owner and --branch." + }, + { + "role": "flag", + "name": "branch", + "type": "string", + "required": false, + "summary": "Branch of a changelog scope, stored verbatim (slashes become key segments). Requires --owner and --repo." + }, + { + "role": "flag", + "name": "allow-empty", + "type": "boolean", + "required": false, + "summary": "Allow writing a manifest with zero entries when the scope holds no objects. Without this flag an empty result aborts the repair.", + "defaultValue": "false" + }, + { + "role": "dryRun", + "name": "dry-run", + "type": "boolean", + "required": false, + "summary": "Report what would change without writing.", + "defaultValue": "false" + }, + { + "role": "flag", + "name": "log-level", + "shortName": "l", + "type": "enum", + "required": false, + "summary": "Minimum log level. Default: information", + "enumValues": [ + "trace", + "debug", + "information", + "warning", + "error", + "critical", + "none" + ] + }, + { + "role": "flag", + "name": "config-source", + "shortName": "c", + "type": "enum", + "required": false, + "summary": "Override the configuration source: local, remote", + "enumValues": [ + "local", + "remote", + "embedded" + ] + }, + { + "role": "flag", + "name": "skip-private-repositories", + "type": "boolean", + "required": false, + "summary": "Skip cloning private repositories" + } + ], + "intent": { + "idempotent": true, + "scope": "global", + "requiresAuth": true + } + }, + { + "path": [ + "changelog", + "registry" + ], + "name": "republish", + "summary": "Re-emit the private-bucket ObjectCreated event for scope objects so the scrubber re-processes them.", + "notes": "The recovery path for a lost or dead-lettered scrub event: performs a metadata-preserving S3 self-copy\nof each selected private object (content and ETag unchanged), which produces the ObjectCreated notification\nthe scrubber Lambda reacts to. The scrubber then re-scrubs and re-publishes the object to the public bucket\nitself \u2014 this command never writes to the public bucket.\n\nRepublishing only happens through this explicit command; nothing triggers it implicitly.", + "usage": "docs-builder changelog registry republish --s3-bucket-name \u003Cstring\u003E [options]", + "examples": [], + "parameters": [ + { + "role": "flag", + "name": "s3-bucket-name", + "type": "string", + "required": true, + "summary": "Private changelog bundles S3 bucket holding the objects to republish." + }, + { + "role": "flag", + "name": "product", + "type": "string", + "required": false, + "summary": "Product of a bundle scope (bundle/{product}/). Mutually exclusive with --owner/--repo/--branch." + }, + { + "role": "flag", + "name": "owner", + "type": "string", + "required": false, + "summary": "GitHub owner of a changelog scope (changelog/{org}/{repo}/{branch}/). Requires --repo and --branch." + }, + { + "role": "flag", + "name": "repo", + "type": "string", + "required": false, + "summary": "Repository of a changelog scope. Requires --owner and --branch." + }, + { + "role": "flag", + "name": "branch", + "type": "string", + "required": false, + "summary": "Branch of a changelog scope, stored verbatim (slashes become key segments). Requires --owner and --repo." + }, + { + "role": "flag", + "name": "files", + "type": "array", + "required": false, + "summary": "File name(s) in the scope to republish (comma-separated or repeated). Mutually exclusive with --all.", + "repeatable": true, + "elementType": "string" + }, + { + "role": "flag", + "name": "all", + "type": "boolean", + "required": false, + "summary": "Republish every object in the scope, including its registry.json.", + "defaultValue": "false" + }, + { + "role": "flag", + "name": "log-level", + "shortName": "l", + "type": "enum", + "required": false, + "summary": "Minimum log level. Default: information", + "enumValues": [ + "trace", + "debug", + "information", + "warning", + "error", + "critical", + "none" + ] + }, + { + "role": "flag", + "name": "config-source", + "shortName": "c", + "type": "enum", + "required": false, + "summary": "Override the configuration source: local, remote", + "enumValues": [ + "local", + "remote", + "embedded" + ] + }, + { + "role": "flag", + "name": "skip-private-repositories", + "type": "boolean", + "required": false, + "summary": "Skip cloning private repositories" + } + ], + "intent": { + "idempotent": true, + "scope": "global", + "requiresAuth": true + } + }, + { + "path": [ + "changelog", + "registry" + ], + "name": "verify-public", + "summary": "Verify that the scrubber-owned public bucket converged to the state expected from the private bucket.", + "notes": "Compares the public registry.json (a verbatim pass-through copy of the private one) and the public\nobjects against the private scope, waiting under a bounded retry policy (--max-attempts x --poll-interval-seconds)\nbecause registry and YAML scrub events propagate independently and transient divergence is normal.\n\nStrictly read-only: the public bucket is never written \u2014 it is checked through a reader that has no\nwrite operations. If divergence persists (a lost or dead-lettered scrub event), recover with the explicit\nchangelog registry republish operation on the private side.", + "usage": "docs-builder changelog registry verify-public --s3-bucket-name \u003Cstring\u003E --public-s3-bucket-name \u003Cstring\u003E [options]", + "examples": [], + "parameters": [ + { + "role": "flag", + "name": "s3-bucket-name", + "type": "string", + "required": true, + "summary": "Private changelog bundles S3 bucket the expected state derives from." + }, + { + "role": "flag", + "name": "public-s3-bucket-name", + "type": "string", + "required": true, + "summary": "Public (scrubbed) changelog bundles S3 bucket to check. Only ever read." + }, + { + "role": "flag", + "name": "product", + "type": "string", + "required": false, + "summary": "Product of a bundle scope (bundle/{product}/). Mutually exclusive with --owner/--repo/--branch." + }, + { + "role": "flag", + "name": "owner", + "type": "string", + "required": false, + "summary": "GitHub owner of a changelog scope (changelog/{org}/{repo}/{branch}/). Requires --repo and --branch." + }, + { + "role": "flag", + "name": "repo", + "type": "string", + "required": false, + "summary": "Repository of a changelog scope. Requires --owner and --branch." + }, + { + "role": "flag", + "name": "branch", + "type": "string", + "required": false, + "summary": "Branch of a changelog scope, stored verbatim (slashes become key segments). Requires --owner and --repo." + }, + { + "role": "flag", + "name": "max-attempts", + "type": "integer", + "required": false, + "summary": "Maximum comparison attempts before reporting divergence. Defaults to 12.", + "defaultValue": "12" + }, + { + "role": "flag", + "name": "poll-interval-seconds", + "type": "integer", + "required": false, + "summary": "Seconds to wait between attempts. Defaults to 10 (a two-minute budget with the default attempts).", + "defaultValue": "10" + }, + { + "role": "flag", + "name": "log-level", + "shortName": "l", + "type": "enum", + "required": false, + "summary": "Minimum log level. Default: information", + "enumValues": [ + "trace", + "debug", + "information", + "warning", + "error", + "critical", + "none" + ] + }, + { + "role": "flag", + "name": "config-source", + "shortName": "c", + "type": "enum", + "required": false, + "summary": "Override the configuration source: local, remote", + "enumValues": [ + "local", + "remote", + "embedded" + ] + }, + { + "role": "flag", + "name": "skip-private-repositories", + "type": "boolean", + "required": false, + "summary": "Skip cloning private repositories" + } + ], + "intent": { + "requiresAuth": true + } + } + ], + "namespaces": [] + } + ] }, { "segment": "codex", diff --git a/docs/cli/changelog/registry/cmd-inspect.md b/docs/cli/changelog/registry/cmd-inspect.md new file mode 100644 index 0000000000..61cf9e437d --- /dev/null +++ b/docs/cli/changelog/registry/cmd-inspect.md @@ -0,0 +1,39 @@ +## Description + +Compares a scope's private `registry.json` manifest against the objects actually stored under the scope's key prefix in the private bucket, and classifies every divergence: + +| Class | Meaning | +| ----- | ------- | +| `missing` | An object exists in the scope but the registry has no entry for it. | +| `stale` | The registry lists a file whose object no longer exists. | +| `corrupt` | The manifest itself is unparseable or contains invalid (unsafe or duplicate) entries. | +| `object-divergent` | A registry entry's recorded metadata (ETag or target) disagrees with the actual object. | + +The command is strictly **read-only** — it never writes to any bucket. It exits non-zero when the scope diverged; use [`changelog registry repair`](/cli/changelog/registry/repair.md) to reconcile. + +A manifest that declares a `schema_version` newer than this docs-builder understands is reported as `UnsupportedSchema`: its entries cannot be judged and repair refuses to touch it. + +For bundle scopes, the expected `target` of each entry is derived by reading the bundle YAML from S3 (for legacy amend sidecars without `products`, the parent bundle's target is used, matching the upload-time registry builder). Changelog scopes enumerate files only and never record a target. + +## State snapshot + +`--out ` writes a machine-readable JSON snapshot of the scope: registry health, the actual objects, the registry's current entries, the entries the registry *should* contain, and every divergence. The snapshot is the trustworthy current-state input for backfill planning, which cannot rely on the additive registry for discovery or removals. + +## Examples + +Inspect a product bundle scope: + +```sh +docs-builder changelog registry inspect \ + --s3-bucket-name elastic-docs-v3-changelog-bundles-private \ + --product elasticsearch +``` + +Inspect an authoring pool and write the snapshot: + +```sh +docs-builder changelog registry inspect \ + --s3-bucket-name elastic-docs-v3-changelog-bundles-private \ + --owner elastic --repo elasticsearch --branch main \ + --out ./state-snapshot.json +``` diff --git a/docs/cli/changelog/registry/cmd-repair.md b/docs/cli/changelog/registry/cmd-repair.md new file mode 100644 index 0000000000..0c8d9acfb0 --- /dev/null +++ b/docs/cli/changelog/registry/cmd-repair.md @@ -0,0 +1,42 @@ +## Description + +Reconciles a scope's **private** `registry.json` from the objects actually stored in the private bucket: missing entries are added, stale entries removed, and object-divergent metadata (ETag, target) corrected. A corrupt manifest is rebuilt from scratch. + +Repair is a separate, explicit operation — nothing runs it implicitly — and it is **idempotent**: a clean scope writes nothing, and running repair twice yields no further changes. + +## Concurrency safety + +The write uses the same optimistic-concurrency conditional PUT as the live upload path: + +- **update**: `If-Match: ` — only succeeds if the manifest hasn't changed since the repair read it; +- **create**: `If-None-Match: *` — only succeeds if the manifest still doesn't exist. + +A `412 Precondition Failed` means a concurrent live upload refreshed the manifest; the repair then re-inspects — a fresh registry read **and** a fresh object listing — and retries (bounded attempts). The registry is always read before the objects are listed, so an object uploaded concurrently either appears in the re-listing or its registry refresh invalidates the precondition; either way it survives the repair. + +## Safety rails + +- A repair that would produce an **empty** manifest aborts unless `--allow-empty` is passed. +- A manifest with a `schema_version` newer than this docs-builder is never rewritten (that would silently downgrade it). +- `--dry-run` reports the full audit (what would be added, removed, and corrected) without writing. +- Every applied change is logged entry by entry with before/after values for audit. + +Only the private registry is written. The public copy is scrubber-owned: the repaired manifest reaches the public bucket through the scrubber's verbatim pass-through, triggered by this write's own `ObjectCreated` event. + +## Examples + +Preview a repair: + +```sh +docs-builder changelog registry repair \ + --s3-bucket-name elastic-docs-v3-changelog-bundles-private \ + --product elasticsearch \ + --dry-run +``` + +Repair an authoring pool: + +```sh +docs-builder changelog registry repair \ + --s3-bucket-name elastic-docs-v3-changelog-bundles-private \ + --owner elastic --repo elasticsearch --branch main +``` diff --git a/docs/cli/changelog/registry/cmd-republish.md b/docs/cli/changelog/registry/cmd-republish.md new file mode 100644 index 0000000000..acac7d90c3 --- /dev/null +++ b/docs/cli/changelog/registry/cmd-republish.md @@ -0,0 +1,38 @@ +## Description + +Re-emits the private-bucket `s3:ObjectCreated` event for selected objects in a scope so the changelog scrubber Lambda re-processes them. This is the explicit recovery path when [`changelog registry verify-public`](/cli/changelog/registry/verify-public.md) shows a persistently missing public object — typically because a scrub event was lost or ended up in the dead-letter queue. + +The re-emission is a **metadata-preserving S3 self-copy**: each selected object is copied onto its own key with `MetadataDirective: REPLACE`, re-supplying its original content type and user metadata. Content and ETag are unchanged; the copy produces the `ObjectCreated` notification the scrubber listens for, and the scrubber then re-scrubs and re-publishes the object to the public bucket itself. + +Republishing never writes to the public bucket — the scrubber remains the sole public-side writer — and it only ever happens through this explicit command. + +## Selection + +Exactly one selection is required: + +- `--files [,…]` — specific file names within the scope (for example `9.3.0.yaml`, or `registry.json` to re-trigger the manifest pass-through); +- `--all` — every object in the scope, including its `registry.json`. + +## Examples + +Re-emit one lost bundle scrub event: + +```sh +docs-builder changelog registry republish \ + --s3-bucket-name elastic-docs-v3-changelog-bundles-private \ + --product elasticsearch \ + --files 9.3.0.yaml +``` + +Re-emit everything in an authoring pool: + +```sh +docs-builder changelog registry republish \ + --s3-bucket-name elastic-docs-v3-changelog-bundles-private \ + --owner elastic --repo elasticsearch --branch main \ + --all +``` + +:::{note} +If the objects also exist locally, `changelog upload --skip-etag-check` achieves a similar re-trigger by re-uploading unchanged files. `republish` works purely from bucket state and needs no local checkout. +::: diff --git a/docs/cli/changelog/registry/cmd-verify-public.md b/docs/cli/changelog/registry/cmd-verify-public.md new file mode 100644 index 0000000000..196cc7ee02 --- /dev/null +++ b/docs/cli/changelog/registry/cmd-verify-public.md @@ -0,0 +1,36 @@ +## Description + +Verifies that the scrubber-owned **public** bucket has converged to the state expected from the **private** bucket for one scope: + +- the public `registry.json` must equal the private one (the scrubber passes manifests through verbatim); +- every private YAML object must have a public counterpart at the same key; +- no public object may outlive its private source. + +Registry and YAML scrub events propagate independently, so transient divergence is normal. The command therefore re-checks under a **bounded retry policy** — up to `--max-attempts` comparisons, `--poll-interval-seconds` apart (defaults: 12 × 10 s, a two-minute budget) — and succeeds as soon as the state converges. + +## Read-only by construction + +The public bucket is a hard write boundary: this command never writes to it. Internally the comparison runs against a reader interface that exposes no write operations, so no code path can mutate either bucket. + +If divergence persists after the retry budget — typically a lost or dead-lettered scrub event, or a bundle the scrubber refused to publish (unallowlisted private references) — the command reports each finding and exits non-zero. Recover with the explicit [`changelog registry republish`](/cli/changelog/registry/republish.md) operation on the private side. + +## Divergence classes + +| Finding | Meaning | +| ------- | ------- | +| `MissingPublicRegistry` | The private registry exists but its public pass-through copy does not. | +| `CorruptPublicRegistry` | The public registry cannot be parsed. | +| `RegistryMismatch` | Public registry entries differ from the private registry. | +| `MissingPublicObject` | A private object has no public counterpart. | +| `StalePublicObject` | A public object has no private counterpart. | + +## Examples + +```sh +docs-builder changelog registry verify-public \ + --s3-bucket-name elastic-docs-v3-changelog-bundles-private \ + --public-s3-bucket-name elastic-docs-v3-changelog-bundles \ + --product elasticsearch \ + --max-attempts 12 \ + --poll-interval-seconds 10 +``` diff --git a/docs/cli/changelog/registry/index.md b/docs/cli/changelog/registry/index.md new file mode 100644 index 0000000000..92c0834847 --- /dev/null +++ b/docs/cli/changelog/registry/index.md @@ -0,0 +1,19 @@ +The `changelog registry` commands inspect, repair, and verify the per-scope `registry.json` manifests that index published changelog artifacts in S3 — `bundle/{product}/registry.json` for bundle scopes and `changelog/{org}/{repo}/{branch}/registry.json` for authoring pools. + +Registries merge additively under optimistic concurrency and are not authoritative for removals or discovery, so they can drift from the objects actually in the bucket. These commands detect that drift, reconcile the **private** registry from the actual private objects, and verify (without ever writing) that the scrubber-owned **public** bucket has converged. + +## Scope selection + +Every command addresses exactly one scope: + +- `--product ` — a bundle scope (`bundle/{product}/`) +- `--owner --repo --branch ` — a changelog authoring pool (`changelog/{org}/{repo}/{branch}/`) + +## Typical workflow + +1. **Inspect** — `changelog registry inspect` reports every divergence between a scope's private registry and its actual objects, and can emit a machine-readable state snapshot. +2. **Repair** — `changelog registry repair` reconciles the private registry from the actual objects (explicit, never implicit; idempotent). +3. **Verify** — `changelog registry verify-public` waits with a bounded retry policy for the scrubber to propagate state to the public bucket and diagnoses divergence, strictly read-only. +4. **Republish** — `changelog registry republish` re-emits the private-bucket `ObjectCreated` event for selected objects when a scrub event was lost or dead-lettered. + +See [Changelog bundle registry and CDN delivery](/development/changelog-bundle-registry.md) for the underlying architecture and reconciliation semantics. diff --git a/docs/development/changelog-bundle-registry.md b/docs/development/changelog-bundle-registry.md index c313d956b9..69dec65618 100644 --- a/docs/development/changelog-bundle-registry.md +++ b/docs/development/changelog-bundle-registry.md @@ -75,8 +75,8 @@ Stored at `bundle/{product}/registry.json` (bundle index) or `changelog/{org}/{r "product": "elasticsearch", "generated_at": "2026-05-06T12:00:00+00:00", "bundles": [ - { "file": "9.4.0.yaml", "target": "9.4.0", "etag": "…" }, - { "file": "9.3.0.yaml", "target": "9.3.0", "etag": "…" } + { "file": "9.4.0.yaml", "target": "9.4.0", "e_tag": "…" }, + { "file": "9.3.0.yaml", "target": "9.3.0", "e_tag": "…" } ] } ``` @@ -88,14 +88,14 @@ Stored at `bundle/{product}/registry.json` (bundle index) or `changelog/{org}/{r | `generated_at` | UTC timestamp of the last regeneration. | | `bundles[].file` | Bundle file name, resolved at `bundle/{product}/{file}` (or entry file at `changelog/{org}/{repo}/{branch}/{file}` for the entry index). | | `bundles[].target` | Target version/date from the bundle's declaration of **this** product (may be null). For an amend sidecar (`{name}.amend-{N}.yaml`) that declares no products itself (created by older docs-builder versions), the parent bundle's target is recorded when the parent file is available in the same upload run. | -| `bundles[].etag` | See the ETag caveat below. | +| `bundles[].e_tag` | See the ETag caveat below. | Bundles are sorted by `target` descending (newest first) with a deterministic tiebreak on `file`, so the JSON is stable across reruns. ### ETag caveat -`bundles[].etag` is the ETag of the bundle object **as uploaded to the private bucket** +`bundles[].e_tag` is the ETag of the bundle object **as uploaded to the private bucket** (pre-scrub). The scrubber rewrites any bundle that contains private references, so for scrubbed bundles this value **will not match** the public (CDN) object's ETag. @@ -174,6 +174,68 @@ build that includes this feature. Consumers must therefore treat a missing bundle as non-fatal (skip + warn), not an error. +## State discovery and reconciliation + +The merge-by-filename refresh above is **additive**: a manifest never has entries removed by +the live path, it is best-effort (a failed refresh leaves it stale), and nothing in the live +path ever compares it against the objects actually in the bucket. Registries are therefore not +authoritative for removals or discovery, and they can drift — which matters as soon as +something (an operator, or backfill planning) needs a trustworthy view of a scope's current +state. + +The `changelog registry` command group (`src/services/Elastic.Changelog/Reconciliation/`) +closes that gap per scope (`bundle/{product}/` or `changelog/{org}/{repo}/{branch}/`): + +### Inspection (read-only) + +`changelog registry inspect` lists the actual private objects under the scope prefix, reads the +private manifest, and classifies every divergence into a four-class taxonomy: + +| Class | Meaning | +|---|---| +| **missing** | Object exists in the scope, the registry has no entry for it. | +| **stale** | Registry entry whose object no longer exists. | +| **corrupt** | The manifest is unparseable, or contains unsafe/duplicate entries. | +| **object-divergent** | Entry metadata (ETag or target) disagrees with the actual object. | + +For bundle scopes the expected `target` is re-derived from the bundle YAML in S3 (including the +parent-bundle fallback for legacy amends without `products`); changelog scopes enumerate files +only. `--out` writes a machine-readable `RegistryStateSnapshot` (registry health, actual +objects, current entries, expected entries, divergences) — the current-state input the backfill +planner consumes. A manifest with a newer `schema_version` is reported as *unsupported*, not +corrupt: its entries cannot be judged by an older tool. + +### Repair (explicit, private-side only) + +`changelog registry repair` converges the **private** manifest to the actual objects: it is a +separate, explicit operation (nothing repairs implicitly), idempotent (a clean scope writes +nothing; a second run is a no-op), and audited (every added/removed/corrected entry is logged +with before/after values; `--dry-run` prints the plan). Writes go through the same conditional +PUT as the live refresh — `If-Match` on update, `If-None-Match: *` on create — and a `412` +triggers a full re-inspection (fresh manifest read **and** fresh object listing) before the +bounded retry. Because the manifest is read *before* the objects are listed, a concurrent +upload either shows up in the re-listing or invalidates the precondition; it cannot be dropped. +Two rails guard against operator error: an empty result requires `--allow-empty`, and a +newer-schema manifest is never rewritten (that would silently downgrade it). + +### Public-side verification (strictly read-only) and republish + +The public bucket is scrubber-owned: the scrubber Lambda is the **sole** writer there. So +`changelog registry verify-public` only ever compares — the public registry must equal the +private one (verbatim pass-through) and every private YAML object must have a public +counterpart (and vice versa) — under a bounded retry policy +(`--max-attempts` × `--poll-interval-seconds`), because registry and YAML scrub events +propagate independently and transient divergence is normal. The boundary is structural, not +conventional: the comparison code operates on a read-only S3 reader interface +(`IS3ScopeReader`) that exposes no write operations. + +Recovery from a lost or dead-lettered scrub event is the explicit +`changelog registry republish` operation on the **private** side: a metadata-preserving S3 +self-copy (`CopyObject` onto the same key with `MetadataDirective: REPLACE`, re-supplying the +original content type and user metadata) that leaves content and ETag untouched while emitting +the `ObjectCreated` event the scrubber reacts to. The scrubber then re-scrubs and re-publishes +on its own. + ## `changelog bundle` entry sourcing (org/repo/branch gate) The `changelog bundle` command aggregates individual changelog **entries**. It can read those diff --git a/src/services/Elastic.Changelog/Reconciliation/ChangelogPublicVerificationService.cs b/src/services/Elastic.Changelog/Reconciliation/ChangelogPublicVerificationService.cs new file mode 100644 index 0000000000..07d70c5a63 --- /dev/null +++ b/src/services/Elastic.Changelog/Reconciliation/ChangelogPublicVerificationService.cs @@ -0,0 +1,246 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Text.Json; +using Amazon.S3; +using Elastic.Changelog.Uploading; +using Elastic.Documentation.Configuration.ReleaseNotes; +using Elastic.Documentation.Diagnostics; +using Elastic.Documentation.Services; +using Microsoft.Extensions.Logging; + +namespace Elastic.Changelog.Reconciliation; + +/// How the public bucket's state diverges from the state expected from the private bucket. +public enum PublicDivergenceKind +{ + /// The public bucket has no registry.json for the scope while the private bucket does. + MissingPublicRegistry, + + /// The public registry.json exists but cannot be parsed. + CorruptPublicRegistry, + + /// The public registry.json content disagrees with the private one (pass-through should keep them identical). + RegistryMismatch, + + /// A private object has no public counterpart — its scrub event is pending, failed, or was lost. + MissingPublicObject, + + /// A public object has no private counterpart — its delete event is pending or was lost. + StalePublicObject +} + +/// One public-side divergence finding. +public sealed record PublicDivergence +{ + public required PublicDivergenceKind Kind { get; init; } + public required string File { get; init; } + public required string Detail { get; init; } +} + +/// +/// Verifies that the scrubber-owned public bucket has converged to the state expected from +/// the private bucket for one scope: the public registry.json must equal the private +/// one (the scrubber passes it through verbatim), every private YAML object must have a public +/// counterpart, and no public object may outlive its private source. Divergence is re-checked under +/// a bounded retry policy to tolerate in-flight scrubber propagation (registry and YAML scrub events +/// are independent, so transient divergence is normal). +/// +/// +/// This service is strictly read-only by construction: the comparison operates exclusively on +/// , which exposes no write operations, so no code path can mutate +/// either bucket. Recovery from a persistently missing public object is the separate, explicit +/// changelog registry republish operation on the private side. +/// +public sealed class ChangelogPublicVerificationService( + ILoggerFactory logFactory, + IAmazonS3? privateS3Client = null, + IAmazonS3? publicS3Client = null, + TimeProvider? timeProvider = null +) : IService +{ + private readonly ILogger _logger = logFactory.CreateLogger(); + private readonly TimeProvider _time = timeProvider ?? TimeProvider.System; + + public async Task Verify(IDiagnosticsCollector collector, ChangelogPublicVerifyArguments args, Cancel ctx) + { + if (!args.TryResolveScope(collector, out var scope)) + return false; + + if (args.MaxAttempts < 1) + { + collector.EmitError(string.Empty, "--max-attempts must be at least 1."); + return false; + } + + using var defaultPrivateClient = privateS3Client == null ? new AmazonS3Client() : null; + using var defaultPublicClient = publicS3Client == null ? new AmazonS3Client() : null; + IS3ScopeReader privateReader = new S3ScopeReader(privateS3Client ?? defaultPrivateClient!, args.S3BucketName); + IS3ScopeReader publicReader = new S3ScopeReader(publicS3Client ?? defaultPublicClient!, args.PublicS3BucketName); + + IReadOnlyList findings = []; + for (var attempt = 1; attempt <= args.MaxAttempts; attempt++) + { + ctx.ThrowIfCancellationRequested(); + + findings = await Compare(privateReader, publicReader, scope, ctx); + if (findings.Count == 0) + { + _logger.LogInformation("Public state for {Scope} converged after {Attempt} attempt(s)", scope, attempt); + return true; + } + + _logger.LogInformation( + "Public state for {Scope} diverges in {Count} place(s) (attempt {Attempt}/{Max})", + scope, findings.Count, attempt, args.MaxAttempts); + + if (attempt < args.MaxAttempts) + await Task.Delay(args.PollInterval, _time, ctx); + } + + foreach (var finding in findings) + collector.EmitWarning(string.Empty, $"[{finding.Kind}] {finding.File}: {finding.Detail}"); + + collector.EmitError(string.Empty, + $"Public bucket state for {scope} still diverges from the private bucket after {args.MaxAttempts} attempt(s) " + + $"({findings.Count} finding(s)). For a lost or dead-lettered scrub event, re-emit it explicitly with `changelog registry republish`."); + return false; + } + + private async Task> Compare( + IS3ScopeReader privateReader, IS3ScopeReader publicReader, ChangelogScope scope, Cancel ctx) + { + var findings = new List(); + + await CompareRegistries(privateReader, publicReader, scope, findings, ctx); + + var privateFiles = ListScopeFiles(await privateReader.ListObjectsAsync(scope.Prefix, ctx), scope); + var publicFiles = ListScopeFiles(await publicReader.ListObjectsAsync(scope.Prefix, ctx), scope); + + foreach (var file in privateFiles.Where(f => !publicFiles.Contains(f))) + { + findings.Add(new PublicDivergence + { + Kind = PublicDivergenceKind.MissingPublicObject, + File = file, + Detail = "Private object has no public counterpart; its scrub event is pending, failed scrubbing, or was lost." + }); + } + + foreach (var file in publicFiles.Where(f => !privateFiles.Contains(f))) + { + findings.Add(new PublicDivergence + { + Kind = PublicDivergenceKind.StalePublicObject, + File = file, + Detail = "Public object has no private counterpart; its delete event is pending or was lost." + }); + } + + return findings; + } + + private async Task CompareRegistries( + IS3ScopeReader privateReader, IS3ScopeReader publicReader, ChangelogScope scope, List findings, Cancel ctx) + { + var privateManifest = await privateReader.TryGetObjectAsync(scope.RegistryKey, ctx); + var publicManifest = await publicReader.TryGetObjectAsync(scope.RegistryKey, ctx); + + if (privateManifest is null) + { + // Nothing is expected on the public side; a lingering public manifest is stale. + if (publicManifest is not null) + { + findings.Add(new PublicDivergence + { + Kind = PublicDivergenceKind.StalePublicObject, + File = ChangelogKeys.RegistryFileName, + Detail = "Public registry exists but the private registry is gone." + }); + } + return; + } + + if (publicManifest is null) + { + findings.Add(new PublicDivergence + { + Kind = PublicDivergenceKind.MissingPublicRegistry, + File = ChangelogKeys.RegistryFileName, + Detail = "Private registry exists but its public pass-through copy does not." + }); + return; + } + + var privateRegistry = TryParse(privateManifest.Value.Content); + var publicRegistry = TryParse(publicManifest.Value.Content); + if (publicRegistry is null) + { + findings.Add(new PublicDivergence + { + Kind = PublicDivergenceKind.CorruptPublicRegistry, + File = ChangelogKeys.RegistryFileName, + Detail = "Public registry cannot be parsed." + }); + return; + } + + // The private manifest's health is the inspect command's concern; here it only matters + // that pass-through kept both sides identical. + if (privateRegistry is not null && !EntriesEqual(privateRegistry.Bundles, publicRegistry.Bundles)) + { + findings.Add(new PublicDivergence + { + Kind = PublicDivergenceKind.RegistryMismatch, + File = ChangelogKeys.RegistryFileName, + Detail = "Public registry entries differ from the private registry; its pass-through event is pending or was lost." + }); + } + } + + private static Registry? TryParse(string content) + { + try + { + return JsonSerializer.Deserialize(content, RegistryJsonContext.Default.Registry); + } + catch (JsonException) + { + return null; + } + } + + private static bool EntriesEqual(IReadOnlyList a, IReadOnlyList b) + { + if (a.Count != b.Count) + return false; + + for (var i = 0; i < a.Count; i++) + { + if (!string.Equals(a[i].File, b[i].File, StringComparison.Ordinal) || + !string.Equals(a[i].Target, b[i].Target, StringComparison.Ordinal) || + !string.Equals(a[i].ETag, b[i].ETag, StringComparison.Ordinal)) + return false; + } + + return true; + } + + /// Single-segment YAML file names in the scope, excluding the manifest and nested scopes. + private static HashSet ListScopeFiles(IReadOnlyList listed, ChangelogScope scope) + { + var files = new HashSet(StringComparer.Ordinal); + foreach (var obj in listed) + { + var file = obj.Key[scope.Prefix.Length..]; + if (file.Length == 0 || file.Contains('/', StringComparison.Ordinal)) + continue; + if (!file.EndsWith(".yaml", StringComparison.OrdinalIgnoreCase) && !file.EndsWith(".yml", StringComparison.OrdinalIgnoreCase)) + continue; + _ = files.Add(file); + } + + return files; + } +} diff --git a/src/services/Elastic.Changelog/Reconciliation/ChangelogRegistryArguments.cs b/src/services/Elastic.Changelog/Reconciliation/ChangelogRegistryArguments.cs new file mode 100644 index 0000000000..f667628c59 --- /dev/null +++ b/src/services/Elastic.Changelog/Reconciliation/ChangelogRegistryArguments.cs @@ -0,0 +1,106 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Diagnostics.CodeAnalysis; +using Elastic.Documentation.Diagnostics; + +namespace Elastic.Changelog.Reconciliation; + +/// +/// Scope selection shared by every registry reconciliation operation: a bundle scope is addressed +/// by , a changelog-pool scope by //. +/// Exactly one of the two forms must be provided. +/// +public record ChangelogRegistryScopeArguments +{ + /// Product of a bundle scope (bundle/{product}/). Mutually exclusive with the owner/repo/branch form. + public string? Product { get; init; } + + /// GitHub owner of a changelog-pool scope (changelog/{org}/{repo}/{branch}/). + public string? Owner { get; init; } + + /// Repository of a changelog-pool scope. + public string? Repo { get; init; } + + /// Branch of a changelog-pool scope (verbatim; slashes become key segments). + public string? Branch { get; init; } + + /// The private changelog bundles bucket the scope lives in. + public required string S3BucketName { get; init; } + + /// + /// Resolves the scope from the argument form used, emitting an error when neither or both + /// forms are given or when a segment fails validation. + /// + public bool TryResolveScope(IDiagnosticsCollector collector, [NotNullWhen(true)] out ChangelogScope? scope) + { + scope = null; + var hasProduct = !string.IsNullOrWhiteSpace(Product); + var hasPool = !string.IsNullOrWhiteSpace(Owner) || !string.IsNullOrWhiteSpace(Repo) || !string.IsNullOrWhiteSpace(Branch); + + if (hasProduct == hasPool) + { + collector.EmitError(string.Empty, + "Specify exactly one scope: --product for a bundle scope, or --owner, --repo, and --branch together for a changelog scope."); + return false; + } + + if (hasProduct) + { + if (ChangelogScope.TryCreateBundle(Product, out scope)) + return true; + + collector.EmitError(string.Empty, $"Invalid product \"{Product}\" (must match [a-zA-Z0-9_-]+)."); + return false; + } + + if (ChangelogScope.TryCreateChangelog(Owner, Repo, Branch, out scope)) + return true; + + collector.EmitError(string.Empty, + $"Invalid changelog scope \"{Owner ?? ""}/{Repo ?? ""}/{Branch ?? ""}\": " + + "--owner, --repo, and --branch are all required and each segment must be a valid key segment."); + return false; + } +} + +/// Arguments for . +public sealed record ChangelogRegistryInspectArguments : ChangelogRegistryScopeArguments +{ + /// Optional path to write the machine-readable JSON to. + public string? Out { get; init; } +} + +/// Arguments for . +public sealed record ChangelogRegistryRepairArguments : ChangelogRegistryScopeArguments +{ + /// Allow writing a manifest with zero entries when the scope holds no objects. + public bool AllowEmpty { get; init; } + + /// Report what would change without writing. + public bool DryRun { get; init; } +} + +/// Arguments for . +public sealed record ChangelogPublicVerifyArguments : ChangelogRegistryScopeArguments +{ + /// The scrubber-owned public bucket to check. Only ever read. + public required string PublicS3BucketName { get; init; } + + /// Maximum number of comparison attempts before reporting divergence. + public int MaxAttempts { get; init; } = 12; + + /// Delay between comparison attempts. + public TimeSpan PollInterval { get; init; } = TimeSpan.FromSeconds(10); +} + +/// Arguments for . +public sealed record ChangelogRegistryRepublishArguments : ChangelogRegistryScopeArguments +{ + /// Specific file names in the scope to republish. Mutually exclusive with . + public IReadOnlyList Files { get; init; } = []; + + /// Republish every object in the scope, including its registry.json. + public bool All { get; init; } +} diff --git a/src/services/Elastic.Changelog/Reconciliation/ChangelogRegistryInspectionService.cs b/src/services/Elastic.Changelog/Reconciliation/ChangelogRegistryInspectionService.cs new file mode 100644 index 0000000000..2b9c97e40d --- /dev/null +++ b/src/services/Elastic.Changelog/Reconciliation/ChangelogRegistryInspectionService.cs @@ -0,0 +1,64 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.IO.Abstractions; +using System.Text.Json; +using Amazon.S3; +using Elastic.Documentation.Configuration; +using Elastic.Documentation.Diagnostics; +using Elastic.Documentation.Services; +using Microsoft.Extensions.Logging; +using Nullean.ScopedFileSystem; + +namespace Elastic.Changelog.Reconciliation; + +/// +/// Read-only registry state discovery: compares a scope's private registry.json against the +/// actual objects in the private bucket, reports every divergence (missing, stale, corrupt, +/// object-divergent), and optionally writes the machine-readable +/// for downstream consumers such as backfill planning. +/// Never writes to any bucket. +/// +public sealed class ChangelogRegistryInspectionService( + ILoggerFactory logFactory, + IAmazonS3? s3Client = null, + ScopedFileSystem? fileSystem = null, + TimeProvider? timeProvider = null +) : IService +{ + private readonly ILogger _logger = logFactory.CreateLogger(); + private readonly IFileSystem _fileSystem = fileSystem ?? FileSystemFactory.RealWrite; + + public async Task Inspect(IDiagnosticsCollector collector, ChangelogRegistryInspectArguments args, Cancel ctx) + { + if (!args.TryResolveScope(collector, out var scope)) + return false; + + using var defaultClient = s3Client == null ? new AmazonS3Client() : null; + var reader = new S3ScopeReader(s3Client ?? defaultClient!, args.S3BucketName); + + var inspector = new RegistryScopeInspector(logFactory, timeProvider); + var snapshot = await inspector.InspectAsync(reader, scope, ctx); + + RegistryStateFormatter.Log(_logger, snapshot); + + if (!string.IsNullOrWhiteSpace(args.Out)) + { + var json = JsonSerializer.Serialize(snapshot, RegistryStateJsonContext.Default.RegistryStateSnapshot); + await _fileSystem.File.WriteAllTextAsync(args.Out, json, ctx); + _logger.LogInformation("Wrote state snapshot to {Out}", args.Out); + } + + if (snapshot.IsClean) + return true; + + var hint = snapshot.RegistryHealth == RegistryHealth.UnsupportedSchema + ? "The manifest schema is newer than this tool understands; update docs-builder before reconciling." + : "Run `changelog registry repair` to reconcile the private registry."; + collector.EmitError(string.Empty, + $"Registry scope {scope} diverged from the actual objects: registry is {snapshot.RegistryHealth} " + + $"with {snapshot.Divergences.Count} divergence(s). {hint}"); + return false; + } +} diff --git a/src/services/Elastic.Changelog/Reconciliation/ChangelogRegistryRepairService.cs b/src/services/Elastic.Changelog/Reconciliation/ChangelogRegistryRepairService.cs new file mode 100644 index 0000000000..9df679395a --- /dev/null +++ b/src/services/Elastic.Changelog/Reconciliation/ChangelogRegistryRepairService.cs @@ -0,0 +1,172 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Net; +using System.Text.Json; +using Amazon.S3; +using Amazon.S3.Model; +using Elastic.Changelog.Uploading; +using Elastic.Documentation.Diagnostics; +using Elastic.Documentation.Services; +using Microsoft.Extensions.Logging; + +namespace Elastic.Changelog.Reconciliation; + +/// +/// Reconciles a scope's private registry.json from the actual private objects: +/// adds missing entries, drops stale ones, and corrects object-divergent metadata. Writes go +/// through the same optimistic-concurrency conditional PUT the live upload path uses +/// (If-Match on update, If-None-Match: * on create), so a repair is safe against +/// concurrent uploads: a manifest changed underneath us fails the precondition and the repair +/// re-inspects (fresh registry read and fresh object listing) before retrying. +/// Repair is idempotent — a clean scope writes nothing. +/// +/// +/// The public registry is scrubber-owned and is never written here; the repaired private manifest +/// reaches the public bucket through the scrubber's pass-through, triggered by this write's own +/// ObjectCreated event. +/// +public sealed class ChangelogRegistryRepairService( + ILoggerFactory logFactory, + IAmazonS3? s3Client = null, + TimeProvider? timeProvider = null +) : IService +{ + private readonly ILogger _logger = logFactory.CreateLogger(); + private readonly TimeProvider _time = timeProvider ?? TimeProvider.System; + + // Bounds the optimistic-concurrency retry loop, mirroring RegistryBuilder.MaxWriteAttempts. + private const int MaxWriteAttempts = 5; + + public async Task Repair(IDiagnosticsCollector collector, ChangelogRegistryRepairArguments args, Cancel ctx) + { + if (!args.TryResolveScope(collector, out var scope)) + return false; + + using var defaultClient = s3Client == null ? new AmazonS3Client() : null; + var client = s3Client ?? defaultClient!; + var reader = new S3ScopeReader(client, args.S3BucketName); + var inspector = new RegistryScopeInspector(logFactory, timeProvider); + + for (var attempt = 1; attempt <= MaxWriteAttempts; attempt++) + { + ctx.ThrowIfCancellationRequested(); + + var snapshot = await inspector.InspectAsync(reader, scope, ctx); + RegistryStateFormatter.Log(_logger, snapshot); + + if (snapshot.IsClean) + { + _logger.LogInformation("Registry for {Scope} already matches the actual objects; nothing to repair", scope); + return true; + } + + if (snapshot.RegistryHealth == RegistryHealth.UnsupportedSchema) + { + collector.EmitError(string.Empty, + $"Registry {scope.RegistryKey} declares a schema_version newer than this tool understands; " + + "repairing would silently downgrade it. Update docs-builder instead."); + return false; + } + + if (snapshot.ExpectedEntries.Count == 0 && !args.AllowEmpty) + { + collector.EmitError(string.Empty, + $"Scope {scope} contains no objects; refusing to write an empty registry. " + + "Pass --allow-empty if the scope is intentionally empty."); + return false; + } + + LogAudit(snapshot); + + if (args.DryRun) + { + _logger.LogInformation("Dry run: registry for {Scope} was not modified", scope); + return true; + } + + if (await TryWriteRepairedManifest(client, args.S3BucketName, scope, snapshot, attempt, ctx)) + return true; + } + + collector.EmitError(string.Empty, + $"Registry for {scope} could not be repaired after {MaxWriteAttempts} attempts due to concurrent writes; re-run once uploads quiesce."); + return false; + } + + /// Audit trail: exactly which entries the repair adds, removes, or corrects. + private void LogAudit(RegistryStateSnapshot snapshot) + { + foreach (var divergence in snapshot.Divergences) + { + switch (divergence.Kind) + { + case RegistryDivergenceKind.Missing: + _logger.LogInformation("repair will add \"{File}\" (target {Target}, etag {ETag})", + divergence.File, divergence.ObjectTarget ?? "", divergence.ObjectETag); + break; + case RegistryDivergenceKind.Stale: + _logger.LogInformation("repair will remove \"{File}\" (was target {Target}, etag {ETag})", + divergence.File, divergence.RegistryTarget ?? "", divergence.RegistryETag); + break; + case RegistryDivergenceKind.ObjectDivergent: + _logger.LogInformation( + "repair will correct \"{File}\": target {RegistryTarget} -> {ObjectTarget}, etag {RegistryETag} -> {ObjectETag}", + divergence.File, divergence.RegistryTarget ?? "", divergence.ObjectTarget ?? "", + divergence.RegistryETag, divergence.ObjectETag); + break; + case RegistryDivergenceKind.Corrupt: + _logger.LogInformation("repair will rebuild the manifest from the actual objects: {Detail}", divergence.Detail); + break; + default: + break; + } + } + + _logger.LogInformation("repair result: {Before} entr(ies) before, {After} after", + snapshot.RegistryEntries.Count, snapshot.ExpectedEntries.Count); + } + + private async Task TryWriteRepairedManifest( + IAmazonS3 client, string bucketName, ChangelogScope scope, RegistryStateSnapshot snapshot, int attempt, Cancel ctx) + { + var manifest = new Registry + { + Product = scope.Group, + GeneratedAt = _time.GetUtcNow(), + Bundles = snapshot.ExpectedEntries + }; + var json = JsonSerializer.Serialize(manifest, RegistryJsonContext.Default.Registry); + + var request = new PutObjectRequest + { + BucketName = bucketName, + Key = scope.RegistryKey, + ContentBody = json, + ContentType = "application/json" + }; + + // Optimistic concurrency: update only if the manifest is unchanged since the inspection + // read it, create only if it is still absent. A concurrent live upload's registry refresh + // invalidates the precondition and we re-inspect. + if (snapshot.RegistryETag is null) + request.IfNoneMatch = "*"; + else + request.IfMatch = $"\"{snapshot.RegistryETag}\""; + + try + { + _ = await client.PutObjectAsync(request, ctx); + _logger.LogInformation("Repaired registry {Key} with {Count} entr(ies)", scope.RegistryKey, snapshot.ExpectedEntries.Count); + return true; + } + catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.PreconditionFailed) + { + _logger.LogInformation( + "Registry for {Scope} changed concurrently (attempt {Attempt}/{Max}); re-inspecting and retrying", + scope, attempt, MaxWriteAttempts); + return false; + } + } +} diff --git a/src/services/Elastic.Changelog/Reconciliation/ChangelogRegistryRepublishService.cs b/src/services/Elastic.Changelog/Reconciliation/ChangelogRegistryRepublishService.cs new file mode 100644 index 0000000000..ac0b1207f8 --- /dev/null +++ b/src/services/Elastic.Changelog/Reconciliation/ChangelogRegistryRepublishService.cs @@ -0,0 +1,150 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Net; +using Amazon.S3; +using Amazon.S3.Model; +using Elastic.Documentation.Configuration.ReleaseNotes; +using Elastic.Documentation.Diagnostics; +using Elastic.Documentation.Services; +using Microsoft.Extensions.Logging; + +namespace Elastic.Changelog.Reconciliation; + +/// +/// Explicitly re-emits the s3:ObjectCreated event for objects in a private scope so +/// the changelog scrubber Lambda re-processes them — the recovery path for a lost or dead-lettered +/// scrub event. The re-emission is a metadata-preserving S3 self-copy (CopyObject with the +/// same source and destination key and MetadataDirective: REPLACE carrying the original +/// content type and user metadata), which leaves the object's content and ETag untouched while +/// producing an ObjectCreated:Copy notification, the event type the scrubber listens for. +/// +/// +/// This never writes to the public bucket: the scrubber remains the sole public-side writer. It is +/// also never run implicitly — republishing is only ever triggered by this explicit operation. +/// +public sealed class ChangelogRegistryRepublishService( + ILoggerFactory logFactory, + IAmazonS3? s3Client = null +) : IService +{ + private readonly ILogger _logger = logFactory.CreateLogger(); + + public async Task Republish(IDiagnosticsCollector collector, ChangelogRegistryRepublishArguments args, Cancel ctx) + { + if (!args.TryResolveScope(collector, out var scope)) + return false; + + if (args.All == (args.Files.Count > 0)) + { + collector.EmitError(string.Empty, + "Specify exactly one selection: --files with the file name(s) to republish, or --all for every object in the scope."); + return false; + } + + using var defaultClient = s3Client == null ? new AmazonS3Client() : null; + var client = s3Client ?? defaultClient!; + + var keys = args.All + ? await ResolveAllScopeKeys(client, args.S3BucketName, scope, ctx) + : ResolveExplicitKeys(collector, scope, args.Files); + if (keys is null) + return false; + + if (keys.Count == 0) + { + _logger.LogInformation("Scope {Scope} contains no objects to republish", scope); + return true; + } + + var failed = 0; + foreach (var key in keys) + { + ctx.ThrowIfCancellationRequested(); + if (!await RepublishObject(collector, client, args.S3BucketName, key, ctx)) + failed++; + } + + _logger.LogInformation("Republish complete: {Succeeded} re-emitted, {Failed} failed", keys.Count - failed, failed); + if (failed > 0) + collector.EmitError(string.Empty, $"{failed} of {keys.Count} object(s) could not be republished."); + return failed == 0; + } + + /// Every single-segment object in the scope, the registry.json manifest included. + private static async Task> ResolveAllScopeKeys(IAmazonS3 client, string bucketName, ChangelogScope scope, Cancel ctx) + { + var reader = new S3ScopeReader(client, bucketName); + var listed = await reader.ListObjectsAsync(scope.Prefix, ctx); + return listed + .Where(o => + { + var file = o.Key[scope.Prefix.Length..]; + return file.Length > 0 && !file.Contains('/', StringComparison.Ordinal); + }) + .Select(o => o.Key) + .ToList(); + } + + private static IReadOnlyList? ResolveExplicitKeys(IDiagnosticsCollector collector, ChangelogScope scope, IReadOnlyList files) + { + var keys = new List(files.Count); + foreach (var file in files) + { + if (!ChangelogKeys.IsSafeFileName(file)) + { + collector.EmitError(string.Empty, $"Invalid file name \"{file}\": must be a single path segment."); + return null; + } + keys.Add(scope.Prefix + file); + } + + return keys; + } + + private async Task RepublishObject(IDiagnosticsCollector collector, IAmazonS3 client, string bucketName, string key, Cancel ctx) + { + GetObjectMetadataResponse head; + try + { + head = await client.GetObjectMetadataAsync(new GetObjectMetadataRequest + { + BucketName = bucketName, + Key = key + }, ctx); + } + catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + collector.EmitError(string.Empty, $"Cannot republish {key}: the object does not exist in the private bucket."); + return false; + } + + // Self-copy with a replaced-but-identical metadata set: S3 only allows copying an object onto + // itself when the metadata directive is REPLACE, and re-supplying the original values keeps the + // rewrite content- and metadata-preserving. + var request = new CopyObjectRequest + { + SourceBucket = bucketName, + SourceKey = key, + DestinationBucket = bucketName, + DestinationKey = key, + MetadataDirective = S3MetadataDirective.REPLACE, + ContentType = head.Headers.ContentType + }; + foreach (var metadataKey in head.Metadata.Keys) + request.Metadata.Add(metadataKey, head.Metadata[metadataKey]); + + try + { + _ = await client.CopyObjectAsync(request, ctx); + _logger.LogInformation("Re-emitted ObjectCreated for {Key}", key); + return true; + } + catch (AmazonS3Exception ex) + { + collector.EmitError(string.Empty, $"Failed to republish {key}: {ex.Message}", ex); + return false; + } + } +} diff --git a/src/services/Elastic.Changelog/Reconciliation/ChangelogScope.cs b/src/services/Elastic.Changelog/Reconciliation/ChangelogScope.cs new file mode 100644 index 0000000000..9b42d81d08 --- /dev/null +++ b/src/services/Elastic.Changelog/Reconciliation/ChangelogScope.cs @@ -0,0 +1,76 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Diagnostics.CodeAnalysis; +using System.Text.Json.Serialization; +using Elastic.Documentation.Configuration.ReleaseNotes; + +namespace Elastic.Changelog.Reconciliation; + +/// The two registry scope families in the changelog bucket key layout. +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum ChangelogScopeKind +{ + /// A product bundle scope: bundle/{product}/…. + Bundle, + + /// An authoring-pool scope: changelog/{org}/{repo}/{branch}/…. + Changelog +} + +/// +/// Identifies one registry scope in the changelog bundles bucket — a product bundle pool +/// (bundle/{product}/) or an authoring changelog pool +/// (changelog/{org}/{repo}/{branch}/) — and derives the scope's key prefix and +/// registry.json key. Segments are validated on construction via +/// , so a scope instance can always be composed into safe S3 keys. +/// +public sealed record ChangelogScope +{ + private ChangelogScope(ChangelogScopeKind kind, string group) + { + Kind = kind; + Group = group; + } + + /// Which scope family this is. + public ChangelogScopeKind Kind { get; } + + /// + /// The grouping segment(s): the product for a bundle scope, the + /// {org}/{repo}/{branch} prefix for a changelog scope. + /// + public string Group { get; } + + /// The S3 key prefix of every object in this scope, ending in /. + public string Prefix => Kind == ChangelogScopeKind.Bundle + ? $"{ChangelogKeys.BundlePrefix}{Group}/" + : $"{ChangelogKeys.ChangelogPrefix}{Group}/"; + + /// The S3 key of this scope's registry.json manifest. + public string RegistryKey => Kind == ChangelogScopeKind.Bundle + ? ChangelogKeys.BundleRegistryKey(Group) + : ChangelogKeys.ChangelogRegistryKey(Group); + + /// Creates a bundle scope for ; false when the segment is invalid. + public static bool TryCreateBundle(string? product, [NotNullWhen(true)] out ChangelogScope? scope) + { + scope = ChangelogKeys.IsValidProduct(product) + ? new ChangelogScope(ChangelogScopeKind.Bundle, product) + : null; + return scope is not null; + } + + /// Creates a changelog-pool scope for //; false when any segment is invalid. + public static bool TryCreateChangelog(string? org, string? repo, string? branch, [NotNullWhen(true)] out ChangelogScope? scope) + { + scope = ChangelogKeys.IsValidOrg(org) && ChangelogKeys.IsValidRepo(repo) && ChangelogKeys.IsValidBranch(branch) + ? new ChangelogScope(ChangelogScopeKind.Changelog, $"{org}/{repo}/{branch}") + : null; + return scope is not null; + } + + /// + public override string ToString() => Prefix.TrimEnd('/'); +} diff --git a/src/services/Elastic.Changelog/Reconciliation/RegistryScopeInspector.cs b/src/services/Elastic.Changelog/Reconciliation/RegistryScopeInspector.cs new file mode 100644 index 0000000000..c99debaf65 --- /dev/null +++ b/src/services/Elastic.Changelog/Reconciliation/RegistryScopeInspector.cs @@ -0,0 +1,322 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Text.Json; +using Elastic.Changelog.Uploading; +using Elastic.Documentation.Configuration.ReleaseNotes; +using Elastic.Documentation.ReleaseNotes; +using Elastic.Documentation.Versions; +using Microsoft.Extensions.Logging; + +namespace Elastic.Changelog.Reconciliation; + +/// +/// Compares a scope's registry.json manifest against the actual objects in that scope and +/// classifies every divergence: missing (object exists, registry lacks it), stale +/// (registry entry, object gone), corrupt (unparseable/invalid manifest), and +/// object-divergent (registry metadata disagrees with the object). Built exclusively on +/// , so an inspection can never write to any bucket. +/// +/// +/// Registries merge additively under optimistic concurrency and are not authoritative for removals +/// or discovery — this inspector produces the trustworthy current-state snapshot +/// () that repair and backfill planning start from. +/// +public sealed class RegistryScopeInspector(ILoggerFactory logFactory, TimeProvider? timeProvider = null) +{ + private readonly ILogger _logger = logFactory.CreateLogger(); + private readonly TimeProvider _time = timeProvider ?? TimeProvider.System; + + /// Takes a full state snapshot of in 's bucket. + public async Task InspectAsync(IS3ScopeReader reader, ChangelogScope scope, Cancel ctx) + { + var diagnostics = new List(); + + // Read the registry before listing objects: any concurrent registry write after this read + // changes the manifest ETag, which repair's conditional PUT then detects (see RegistryRepairer). + var (registryEntries, registryETag, health, corruptFindings) = await FetchRegistry(reader, scope, ctx); + var objects = await ListScopeObjects(reader, scope, diagnostics, ctx); + var expected = await DeriveExpectedEntries(reader, scope, objects, registryEntries, diagnostics, ctx); + + // Per-file diffs are only meaningful against a readable manifest: a missing manifest diffs + // against an empty entry list (every object is then "missing"), while corrupt and + // unsupported-schema manifests have unknown entries, so only their scope-level finding stands. + var divergences = new List(corruptFindings); + if (health is RegistryHealth.Valid or RegistryHealth.Missing) + divergences.AddRange(Diff(registryEntries, expected, objects)); + + return new RegistryStateSnapshot + { + ScopeKind = scope.Kind, + Scope = scope.Group, + Bucket = reader.BucketName, + RegistryKey = scope.RegistryKey, + GeneratedAt = _time.GetUtcNow(), + RegistryHealth = health, + RegistryETag = registryETag, + Objects = objects, + RegistryEntries = registryEntries, + ExpectedEntries = expected, + Divergences = divergences, + Diagnostics = diagnostics + }; + } + + /// Reads and validates the scope's manifest; corrupt manifests yield findings instead of throwing. + private async Task<(IReadOnlyList Entries, string? ETag, RegistryHealth Health, List Findings)> FetchRegistry( + IS3ScopeReader reader, ChangelogScope scope, Cancel ctx) + { + var manifest = await reader.TryGetObjectAsync(scope.RegistryKey, ctx); + if (manifest is null) + return ([], null, RegistryHealth.Missing, []); + + var (content, etag) = manifest.Value; + Registry? registry; + try + { + registry = JsonSerializer.Deserialize(content, RegistryJsonContext.Default.Registry); + } + catch (JsonException ex) + { + _logger.LogWarning("Manifest {Key} is unparseable: {Message}", scope.RegistryKey, ex.Message); + return ([], etag, RegistryHealth.Corrupt, [Corrupt($"Manifest is not valid JSON: {ex.Message}")]); + } + + if (registry is null) + return ([], etag, RegistryHealth.Corrupt, [Corrupt("Manifest deserialized to null.")]); + + if (registry.SchemaVersion > 1) + return ([], etag, RegistryHealth.UnsupportedSchema, []); + + var findings = ValidateEntries(registry.Bundles); + return findings.Count > 0 + ? ([], etag, RegistryHealth.Corrupt, findings) + : (registry.Bundles, etag, RegistryHealth.Valid, findings); + } + + private static List ValidateEntries(IReadOnlyList entries) + { + var findings = new List(); + var seen = new HashSet(StringComparer.Ordinal); + foreach (var entry in entries) + { + if (!ChangelogKeys.IsSafeFileName(entry.File)) + findings.Add(Corrupt($"Manifest entry has an unsafe file name: \"{entry.File}\".")); + else if (!seen.Add(entry.File)) + findings.Add(Corrupt($"Manifest lists \"{entry.File}\" more than once.")); + } + return findings; + } + + private static RegistryDivergence Corrupt(string detail) => new() + { + Kind = RegistryDivergenceKind.Corrupt, + File = ChangelogKeys.RegistryFileName, + Detail = detail + }; + + /// + /// Lists the scope's content objects: single-segment YAML keys under the scope prefix, excluding + /// the manifest itself. Deeper keys belong to nested scopes (a changelog pool whose branch extends + /// this one) and are never part of this scope's registry. + /// + private async Task> ListScopeObjects( + IS3ScopeReader reader, ChangelogScope scope, List diagnostics, Cancel ctx) + { + var listed = await reader.ListObjectsAsync(scope.Prefix, ctx); + var objects = new List(listed.Count); + foreach (var obj in listed) + { + var file = obj.Key[scope.Prefix.Length..]; + if (file.Length == 0 || file.Contains('/', StringComparison.Ordinal)) + { + // A nested changelog pool (branch "main" vs "main/foo") is expected; a nested key + // under a bundle scope is not, so surface the latter. + if (scope.Kind == ChangelogScopeKind.Bundle) + diagnostics.Add($"Ignored nested key outside this scope: {obj.Key}"); + continue; + } + + if (string.Equals(file, ChangelogKeys.RegistryFileName, StringComparison.Ordinal)) + continue; + + if (!file.EndsWith(".yaml", StringComparison.OrdinalIgnoreCase) && + !file.EndsWith(".yml", StringComparison.OrdinalIgnoreCase)) + { + diagnostics.Add($"Ignored unexpected non-YAML object: {obj.Key}"); + continue; + } + + objects.Add(new ScopeObject + { + File = file, + Key = obj.Key, + ETag = obj.ETag, + Size = obj.Size, + LastModified = obj.LastModified + }); + } + + return objects; + } + + /// + /// Derives the entries the registry should list from the actual objects. Bundle scopes read each + /// bundle's YAML to record the scope product's target (falling back to the parent bundle for + /// legacy amends that omit products, mirroring ); changelog + /// scopes only enumerate files and never record a target. When a bundle cannot be parsed its + /// expected target is unknown, so the registry's currently recorded target is preserved rather + /// than reported (and repaired) as a divergence. + /// + private async Task> DeriveExpectedEntries( + IS3ScopeReader reader, ChangelogScope scope, IReadOnlyList objects, + IReadOnlyList registryEntries, List diagnostics, Cancel ctx) + { + var contentCache = new Dictionary(StringComparer.Ordinal); + var registryByFile = registryEntries.ToDictionary(e => e.File, e => e, StringComparer.Ordinal); + var entries = new List(objects.Count); + foreach (var obj in objects) + { + string? target = null; + if (scope.Kind == ChangelogScopeKind.Bundle) + { + var (derived, known) = await DeriveBundleTarget(reader, scope, obj, contentCache, diagnostics, ctx); + target = known + ? derived + : registryByFile.TryGetValue(obj.File, out var existing) ? existing.Target : null; + } + + entries.Add(new RegistryBundle + { + File = obj.File, + Target = target, + ETag = obj.ETag + }); + } + + return SortEntries(entries); + } + + private async Task<(string? Target, bool Known)> DeriveBundleTarget( + IS3ScopeReader reader, ChangelogScope scope, ScopeObject obj, + Dictionary contentCache, List diagnostics, Cancel ctx) + { + var bundle = await ReadBundle(reader, obj.Key, contentCache, diagnostics, ctx); + if (bundle is null) + return (null, Known: false); + + // Amends published before products were copied from the parent omit them; record the + // parent bundle's target so :version:-filtered consumers still discover the amend. + if (bundle.Products.Count == 0 && BundleAmendMerger.IsAmendFile(obj.File)) + { + var parentFile = BundleAmendMerger.GetParentBundlePath(obj.File); + if (parentFile is null) + return (null, Known: true); + + var parent = await ReadBundle(reader, scope.Prefix + parentFile, contentCache, diagnostics, ctx); + return (parent is { Products.Count: > 0 } ? TargetForProduct(parent, scope.Group) : null, Known: true); + } + + return (bundle.Products.Count > 0 ? TargetForProduct(bundle, scope.Group) : null, Known: true); + } + + private async Task ReadBundle( + IS3ScopeReader reader, string key, Dictionary contentCache, List diagnostics, Cancel ctx) + { + if (!contentCache.TryGetValue(key, out var content)) + { + var fetched = await reader.TryGetObjectAsync(key, ctx); + content = fetched?.Content; + contentCache[key] = content; + } + + if (content is null) + return null; + + try + { + return ReleaseNotesSerialization.DeserializeBundle(content); + } + catch (Exception ex) + { + _logger.LogWarning("Could not parse bundle {Key}: {Message}", key, ex.Message); + diagnostics.Add($"Could not parse bundle {key}; its expected target is unknown: {ex.Message}"); + return null; + } + } + + private static string? TargetForProduct(Bundle bundle, string product) + { + var match = bundle.Products.FirstOrDefault(p => string.Equals(p.ProductId, product, StringComparison.Ordinal)); + return (match ?? bundle.Products[0]).Target; + } + + /// Same ordering the registry writer produces: target descending, file-name tiebreak. + internal static List SortEntries(IEnumerable entries) => + entries + .OrderByDescending(b => VersionOrDate.Parse(b.Target ?? string.Empty)) + .ThenBy(b => b.File, StringComparer.Ordinal) + .ToList(); + + private static List Diff( + IReadOnlyList registryEntries, + IReadOnlyList expected, + IReadOnlyList objects) + { + var divergences = new List(); + var registryByFile = registryEntries.ToDictionary(e => e.File, e => e, StringComparer.Ordinal); + var objectsByFile = objects.ToDictionary(o => o.File, o => o, StringComparer.Ordinal); + + foreach (var want in expected) + { + if (!registryByFile.TryGetValue(want.File, out var have)) + { + divergences.Add(new RegistryDivergence + { + Kind = RegistryDivergenceKind.Missing, + File = want.File, + Detail = "Object exists in the scope but the registry has no entry for it.", + ObjectETag = want.ETag, + ObjectTarget = want.Target + }); + continue; + } + + var etagMatches = string.Equals(S3ScopeReader.NormalizeETag(have.ETag), want.ETag, StringComparison.OrdinalIgnoreCase); + var targetMatches = string.Equals(have.Target, want.Target, StringComparison.Ordinal); + if (!etagMatches || !targetMatches) + { + divergences.Add(new RegistryDivergence + { + Kind = RegistryDivergenceKind.ObjectDivergent, + File = want.File, + Detail = etagMatches + ? "Registry target disagrees with the target derived from the object." + : "Registry ETag disagrees with the actual object.", + RegistryETag = have.ETag, + ObjectETag = want.ETag, + RegistryTarget = have.Target, + ObjectTarget = want.Target + }); + } + } + + foreach (var have in registryEntries) + { + if (!objectsByFile.ContainsKey(have.File)) + { + divergences.Add(new RegistryDivergence + { + Kind = RegistryDivergenceKind.Stale, + File = have.File, + Detail = "Registry lists a file whose object no longer exists in the scope.", + RegistryETag = have.ETag, + RegistryTarget = have.Target + }); + } + } + + return divergences; + } +} diff --git a/src/services/Elastic.Changelog/Reconciliation/RegistryStateFormatter.cs b/src/services/Elastic.Changelog/Reconciliation/RegistryStateFormatter.cs new file mode 100644 index 0000000000..bd1cf35e6a --- /dev/null +++ b/src/services/Elastic.Changelog/Reconciliation/RegistryStateFormatter.cs @@ -0,0 +1,39 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using Microsoft.Extensions.Logging; + +namespace Elastic.Changelog.Reconciliation; + +/// Human-readable rendering of a for CLI output. +public static class RegistryStateFormatter +{ + /// Logs the snapshot's health, counts, every divergence, and every diagnostic. + public static void Log(ILogger logger, RegistryStateSnapshot snapshot) + { + logger.LogInformation( + "Scope {ScopeKind}/{Scope} in {Bucket}: registry {Health}, {ObjectCount} object(s), {EntryCount} registry entr(ies), {DivergenceCount} divergence(s)", + snapshot.ScopeKind, snapshot.Scope, snapshot.Bucket, + snapshot.RegistryHealth, snapshot.Objects.Count, snapshot.RegistryEntries.Count, snapshot.Divergences.Count); + + foreach (var divergence in snapshot.Divergences) + logger.LogWarning("[{Kind}] {File}: {Detail}{Values}", divergence.Kind, divergence.File, divergence.Detail, FormatValues(divergence)); + + foreach (var diagnostic in snapshot.Diagnostics) + logger.LogInformation("{Diagnostic}", diagnostic); + + if (snapshot.IsClean) + logger.LogInformation("Scope is clean: the registry matches the actual objects."); + } + + private static string FormatValues(RegistryDivergence divergence) + { + var parts = new List(2); + if (divergence.RegistryETag is not null || divergence.ObjectETag is not null) + parts.Add($"etag registry={divergence.RegistryETag ?? ""} object={divergence.ObjectETag ?? ""}"); + if (divergence.RegistryTarget is not null || divergence.ObjectTarget is not null) + parts.Add($"target registry={divergence.RegistryTarget ?? ""} object={divergence.ObjectTarget ?? ""}"); + return parts.Count > 0 ? $" ({string.Join("; ", parts)})" : string.Empty; + } +} diff --git a/src/services/Elastic.Changelog/Reconciliation/RegistryStateSnapshot.cs b/src/services/Elastic.Changelog/Reconciliation/RegistryStateSnapshot.cs new file mode 100644 index 0000000000..80f8229a1c --- /dev/null +++ b/src/services/Elastic.Changelog/Reconciliation/RegistryStateSnapshot.cs @@ -0,0 +1,157 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Text.Json.Serialization; +using Elastic.Changelog.Uploading; + +namespace Elastic.Changelog.Reconciliation; + +/// Overall health of a scope's registry.json manifest object. +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum RegistryHealth +{ + /// The manifest exists and parses as a with safe entries. + Valid, + + /// No manifest object exists at the scope's registry key. + Missing, + + /// The manifest exists but cannot be parsed, or contains invalid entries. + Corrupt, + + /// + /// The manifest declares a schema_version newer than this tool understands. Not corrupt — + /// but repair refuses to touch it, because rebuilding would silently downgrade the schema. + /// + UnsupportedSchema +} + +/// How a registry entry (or the manifest itself) diverges from the actual scoped objects. +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum RegistryDivergenceKind +{ + /// An object exists in the scope but the registry has no entry for it. + Missing, + + /// The registry lists a file whose object no longer exists in the scope. + Stale, + + /// The manifest itself is unparseable or contains invalid (unsafe) entries. + Corrupt, + + /// A registry entry's recorded metadata (ETag or target) disagrees with the actual object. + ObjectDivergent +} + +/// One divergence finding between a registry and its scope's actual objects. +public sealed record RegistryDivergence +{ + /// The divergence class. + public required RegistryDivergenceKind Kind { get; init; } + + /// The file the finding concerns (registry.json for ). + public required string File { get; init; } + + /// Human-readable explanation of the finding. + public required string Detail { get; init; } + + /// The ETag recorded in the registry entry, when one exists. + public string? RegistryETag { get; init; } + + /// The actual object's ETag, when the object exists. + public string? ObjectETag { get; init; } + + /// The target recorded in the registry entry, when one exists. + public string? RegistryTarget { get; init; } + + /// The target derived from the actual object, when derivable. + public string? ObjectTarget { get; init; } +} + +/// One actual content object (bundle or entry YAML) found in the scope. +public sealed record ScopeObject +{ + /// File name (last key segment) of the object. + public required string File { get; init; } + + /// Full S3 key of the object. + public required string Key { get; init; } + + /// The object's S3 ETag, normalized (no surrounding quotes). + public required string ETag { get; init; } + + /// Object size in bytes. + public long Size { get; init; } + + /// Object last-modified timestamp, when the listing reported one. + public DateTimeOffset? LastModified { get; init; } +} + +/// +/// A machine-readable snapshot of one registry scope's current state: the actual content objects +/// in the private bucket, the registry manifest's entries, and every divergence between the two. +/// Produced by ; consumed by the repair operation and — as a +/// serialized artifact — by backfill planning, which needs a trustworthy view of current state +/// because registries merge additively and are not authoritative for removals or discovery. +/// +public sealed record RegistryStateSnapshot +{ + /// Snapshot schema version. Incremented when consumers must change their parser. + public int SchemaVersion { get; init; } = 1; + + /// The scope family: bundle or changelog. + public required ChangelogScopeKind ScopeKind { get; init; } + + /// The scope's grouping segment(s): product, or {org}/{repo}/{branch}. + public required string Scope { get; init; } + + /// The bucket the snapshot was taken from. + public required string Bucket { get; init; } + + /// The S3 key of the scope's registry.json manifest. + public required string RegistryKey { get; init; } + + /// Time the snapshot was taken, in UTC. + public required DateTimeOffset GeneratedAt { get; init; } + + /// Health of the manifest object itself. + public required RegistryHealth RegistryHealth { get; init; } + + /// The manifest object's ETag as read (normalized), when the object exists. + public string? RegistryETag { get; init; } + + /// The actual content objects (YAML files) currently in the scope, excluding the manifest. + public required IReadOnlyList Objects { get; init; } + + /// The entries the registry manifest currently lists (empty when missing or corrupt). + public required IReadOnlyList RegistryEntries { get; init; } + + /// + /// The entries the registry should list, derived purely from the actual objects + /// (file name, object ETag, and — for bundle scopes — the target read from the bundle YAML). + /// This is what a repair converges the registry to. + /// + public required IReadOnlyList ExpectedEntries { get; init; } + + /// Every divergence found between the registry and the actual objects. + public required IReadOnlyList Divergences { get; init; } + + /// Non-classified observations (unreadable bundle YAML, unexpected non-YAML keys, …). + public required IReadOnlyList Diagnostics { get; init; } + + /// + /// True when the registry matches the actual objects exactly. A missing manifest over an empty + /// scope is clean (nothing has been published there yet); a missing manifest over a populated + /// scope produces per-object findings and is not. + /// + public bool IsClean => Divergences.Count == 0 && RegistryHealth is RegistryHealth.Valid or RegistryHealth.Missing; +} + +[JsonSourceGenerationOptions( + WriteIndented = true, + PropertyNamingPolicy = JsonKnownNamingPolicy.SnakeCaseLower, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull +)] +[JsonSerializable(typeof(RegistryStateSnapshot))] +public sealed partial class RegistryStateJsonContext : JsonSerializerContext; diff --git a/src/services/Elastic.Changelog/Reconciliation/S3ScopeReader.cs b/src/services/Elastic.Changelog/Reconciliation/S3ScopeReader.cs new file mode 100644 index 0000000000..252f329809 --- /dev/null +++ b/src/services/Elastic.Changelog/Reconciliation/S3ScopeReader.cs @@ -0,0 +1,85 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Net; +using Amazon.S3; +using Amazon.S3.Model; + +namespace Elastic.Changelog.Reconciliation; + +/// A raw object listing row: key, normalized ETag, size, and last-modified. +public sealed record ListedObject(string Key, string ETag, long Size, DateTimeOffset? LastModified); + +/// +/// The read-only S3 surface the reconciliation code operates on. Inspection and all +/// public-bucket checks are built exclusively against this interface, so they structurally +/// cannot write: the write boundary around the scrubber-owned public bucket is enforced by +/// the type system, not by convention. +/// +public interface IS3ScopeReader +{ + /// The bucket this reader is bound to. + string BucketName { get; } + + /// Lists every object under , paginating to completion. + Task> ListObjectsAsync(string prefix, Cancel ctx); + + /// Reads an object's content and normalized ETag; null when the key does not exist. + Task<(string Content, string ETag)?> TryGetObjectAsync(string key, Cancel ctx); +} + +/// Read-only adapter over bound to a single bucket. +public sealed class S3ScopeReader(IAmazonS3 s3Client, string bucketName) : IS3ScopeReader +{ + /// + public string BucketName => bucketName; + + /// Strips the surrounding quotes S3 returns around ETag values. + public static string NormalizeETag(string? etag) => etag?.Trim('"') ?? string.Empty; + + /// + public async Task> ListObjectsAsync(string prefix, Cancel ctx) + { + var request = new ListObjectsV2Request + { + BucketName = bucketName, + Prefix = prefix, + MaxKeys = 1000 + }; + + var objects = new List(); + ListObjectsV2Response response; + do + { + response = await s3Client.ListObjectsV2Async(request, ctx); + foreach (var obj in response.S3Objects ?? []) + objects.Add(new ListedObject(obj.Key, NormalizeETag(obj.ETag), obj.Size ?? 0, obj.LastModified is { } modified ? modified : null)); + request.ContinuationToken = response.NextContinuationToken; + } while (response.IsTruncated == true); + + return objects; + } + + /// + public async Task<(string Content, string ETag)?> TryGetObjectAsync(string key, Cancel ctx) + { + try + { + using var response = await s3Client.GetObjectAsync(new GetObjectRequest + { + BucketName = bucketName, + Key = key + }, ctx); + + await using var stream = response.ResponseStream; + using var reader = new StreamReader(stream); + var content = await reader.ReadToEndAsync(ctx); + return (content, NormalizeETag(response.ETag)); + } + catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + return null; + } + } +} diff --git a/src/tooling/docs-builder/Commands/ChangelogRegistryCommands.cs b/src/tooling/docs-builder/Commands/ChangelogRegistryCommands.cs new file mode 100644 index 0000000000..a5ec0ddebd --- /dev/null +++ b/src/tooling/docs-builder/Commands/ChangelogRegistryCommands.cs @@ -0,0 +1,231 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using Elastic.Changelog.Reconciliation; +using Elastic.Documentation.Diagnostics; +using Elastic.Documentation.Services; +using Microsoft.Extensions.Logging; +using Nullean.Argh; +using Nullean.Argh.Documentation; + +namespace Documentation.Builder.Commands; + +/// Inspect, repair, and verify per-scope changelog registry.json manifests against actual bucket state. +internal sealed class ChangelogRegistryCommands( + ILoggerFactory logFactory, + IDiagnosticsCollector collector +) +{ + /// Compare a scope's private registry.json against the actual private-bucket objects and report every divergence. + /// + /// Registries merge additively and are not authoritative for removals or discovery, so they can drift + /// from the objects actually in the bucket. This command detects and classifies that drift: missing + /// (object exists, registry lacks it), stale (registry entry, object gone), corrupt + /// (unparseable/invalid manifest), and object-divergent (registry metadata disagrees with the object). + /// Strictly read-only — nothing is written to any bucket. Exits non-zero when the scope diverged. + /// Use changelog registry repair to reconcile. + /// + /// Private changelog bundles S3 bucket to inspect. + /// Product of a bundle scope (bundle/{product}/). Mutually exclusive with --owner/--repo/--branch. + /// GitHub owner of a changelog scope (changelog/{org}/{repo}/{branch}/). Requires --repo and --branch. + /// Repository of a changelog scope. Requires --owner and --branch. + /// Branch of a changelog scope, stored verbatim (slashes become key segments). Requires --owner and --repo. + /// Path to write the machine-readable state snapshot JSON to. + /// Cancellation token. + [RequiresAuth] + [NoOptionsInjection] + public async Task Inspect( + string s3BucketName, + string? product = null, + string? owner = null, + string? repo = null, + string? branch = null, + [ExpandUserProfile, RejectSymbolicLinks] FileInfo? @out = null, + CancellationToken ct = default + ) + { + await using var serviceInvoker = new ServiceInvoker(collector); + var service = new ChangelogRegistryInspectionService(logFactory); + var args = new ChangelogRegistryInspectArguments + { + S3BucketName = s3BucketName, + Product = product, + Owner = owner, + Repo = repo, + Branch = branch, + Out = @out?.FullName + }; + serviceInvoker.AddCommand(service, args, + static async (s, c, state, ctx) => await s.Inspect(c, state, ctx) + ); + return await serviceInvoker.InvokeAsync(ct); + } + + /// Reconcile a scope's private registry.json from the actual private-bucket objects. + /// + /// Rebuilds the manifest from the objects that actually exist in the scope: missing entries are added, + /// stale entries removed, and divergent metadata corrected. The write uses the same optimistic-concurrency + /// conditional PUT as the live upload path (If-Match on update, If-None-Match: * on create), re-inspecting + /// and retrying when a concurrent upload refreshes the manifest, so repair is safe to run alongside live + /// uploads. Repair is idempotent: a clean scope writes nothing, and running it twice yields no further change. + /// Only the private registry is written. The public copy is scrubber-owned and converges through + /// the scrubber's pass-through of this write's own event. Every change is logged (before/after) for audit. + /// + /// Private changelog bundles S3 bucket to repair the registry in. + /// Product of a bundle scope (bundle/{product}/). Mutually exclusive with --owner/--repo/--branch. + /// GitHub owner of a changelog scope (changelog/{org}/{repo}/{branch}/). Requires --repo and --branch. + /// Repository of a changelog scope. Requires --owner and --branch. + /// Branch of a changelog scope, stored verbatim (slashes become key segments). Requires --owner and --repo. + /// Allow writing a manifest with zero entries when the scope holds no objects. Without this flag an empty result aborts the repair. + /// Report what would change without writing. + /// Cancellation token. + [RequiresAuth] + [CommandIntent(Intent.Idempotent)] + [MutationScope(MutationScope.Global)] + [NoOptionsInjection] + public async Task Repair( + string s3BucketName, + string? product = null, + string? owner = null, + string? repo = null, + string? branch = null, + bool allowEmpty = false, + [DryRun] bool dryRun = false, + CancellationToken ct = default + ) + { + await using var serviceInvoker = new ServiceInvoker(collector); + var service = new ChangelogRegistryRepairService(logFactory); + var args = new ChangelogRegistryRepairArguments + { + S3BucketName = s3BucketName, + Product = product, + Owner = owner, + Repo = repo, + Branch = branch, + AllowEmpty = allowEmpty, + DryRun = dryRun + }; + serviceInvoker.AddCommand(service, args, + static async (s, c, state, ctx) => await s.Repair(c, state, ctx) + ); + return await serviceInvoker.InvokeAsync(ct); + } + + /// Verify that the scrubber-owned public bucket converged to the state expected from the private bucket. + /// + /// Compares the public registry.json (a verbatim pass-through copy of the private one) and the public + /// objects against the private scope, waiting under a bounded retry policy (--max-attempts x --poll-interval-seconds) + /// because registry and YAML scrub events propagate independently and transient divergence is normal. + /// Strictly read-only: the public bucket is never written — it is checked through a reader that has no + /// write operations. If divergence persists (a lost or dead-lettered scrub event), recover with the explicit + /// changelog registry republish operation on the private side. + /// + /// Private changelog bundles S3 bucket the expected state derives from. + /// Public (scrubbed) changelog bundles S3 bucket to check. Only ever read. + /// Product of a bundle scope (bundle/{product}/). Mutually exclusive with --owner/--repo/--branch. + /// GitHub owner of a changelog scope (changelog/{org}/{repo}/{branch}/). Requires --repo and --branch. + /// Repository of a changelog scope. Requires --owner and --branch. + /// Branch of a changelog scope, stored verbatim (slashes become key segments). Requires --owner and --repo. + /// Maximum comparison attempts before reporting divergence. Defaults to 12. + /// Seconds to wait between attempts. Defaults to 10 (a two-minute budget with the default attempts). + /// Cancellation token. + [RequiresAuth] + [NoOptionsInjection] + public async Task VerifyPublic( + string s3BucketName, + string publicS3BucketName, + string? product = null, + string? owner = null, + string? repo = null, + string? branch = null, + int maxAttempts = 12, + int pollIntervalSeconds = 10, + CancellationToken ct = default + ) + { + await using var serviceInvoker = new ServiceInvoker(collector); + var service = new ChangelogPublicVerificationService(logFactory); + var args = new ChangelogPublicVerifyArguments + { + S3BucketName = s3BucketName, + PublicS3BucketName = publicS3BucketName, + Product = product, + Owner = owner, + Repo = repo, + Branch = branch, + MaxAttempts = maxAttempts, + PollInterval = TimeSpan.FromSeconds(pollIntervalSeconds) + }; + serviceInvoker.AddCommand(service, args, + static async (s, c, state, ctx) => await s.Verify(c, state, ctx) + ); + return await serviceInvoker.InvokeAsync(ct); + } + + /// Re-emit the private-bucket ObjectCreated event for scope objects so the scrubber re-processes them. + /// + /// The recovery path for a lost or dead-lettered scrub event: performs a metadata-preserving S3 self-copy + /// of each selected private object (content and ETag unchanged), which produces the ObjectCreated notification + /// the scrubber Lambda reacts to. The scrubber then re-scrubs and re-publishes the object to the public bucket + /// itself — this command never writes to the public bucket. + /// Republishing only happens through this explicit command; nothing triggers it implicitly. + /// + /// Private changelog bundles S3 bucket holding the objects to republish. + /// Product of a bundle scope (bundle/{product}/). Mutually exclusive with --owner/--repo/--branch. + /// GitHub owner of a changelog scope (changelog/{org}/{repo}/{branch}/). Requires --repo and --branch. + /// Repository of a changelog scope. Requires --owner and --branch. + /// Branch of a changelog scope, stored verbatim (slashes become key segments). Requires --owner and --repo. + /// File name(s) in the scope to republish (comma-separated or repeated). Mutually exclusive with --all. + /// Republish every object in the scope, including its registry.json. + /// Cancellation token. + [RequiresAuth] + [CommandIntent(Intent.Idempotent)] + [MutationScope(MutationScope.Global)] + [NoOptionsInjection] + public async Task Republish( + string s3BucketName, + string? product = null, + string? owner = null, + string? repo = null, + string? branch = null, + string[]? files = null, + bool all = false, + CancellationToken ct = default + ) + { + await using var serviceInvoker = new ServiceInvoker(collector); + var service = new ChangelogRegistryRepublishService(logFactory); + var args = new ChangelogRegistryRepublishArguments + { + S3BucketName = s3BucketName, + Product = product, + Owner = owner, + Repo = repo, + Branch = branch, + Files = ExpandCommaSeparated(files), + All = all + }; + serviceInvoker.AddCommand(service, args, + static async (s, c, state, ctx) => await s.Republish(c, state, ctx) + ); + return await serviceInvoker.InvokeAsync(ct); + } + + private static List ExpandCommaSeparated(string[]? values) + { + if (values is not { Length: > 0 }) + return []; + + var result = new List(); + foreach (var value in values.Where(v => !string.IsNullOrWhiteSpace(v))) + { + if (value.Contains(',')) + result.AddRange(value.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)); + else + result.Add(value); + } + return result; + } +} diff --git a/src/tooling/docs-builder/Program.cs b/src/tooling/docs-builder/Program.cs index 98eb0bc178..d5965625dd 100644 --- a/src/tooling/docs-builder/Program.cs +++ b/src/tooling/docs-builder/Program.cs @@ -49,7 +49,7 @@ _ = app.Map(); _ = app.Map(); _ = app.Map(); - _ = app.MapNamespace("changelog"); + _ = app.MapNamespace("changelog", g => g.MapNamespace("registry")); _ = app.MapNamespace("inbound-links"); _ = app.Map(); diff --git a/tests/Elastic.Changelog.Tests/Reconciliation/FakeS3Bucket.cs b/tests/Elastic.Changelog.Tests/Reconciliation/FakeS3Bucket.cs new file mode 100644 index 0000000000..bb85fbad5b --- /dev/null +++ b/tests/Elastic.Changelog.Tests/Reconciliation/FakeS3Bucket.cs @@ -0,0 +1,161 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Diagnostics.CodeAnalysis; +using System.Net; +using System.Security.Cryptography; +using System.Text; +using Amazon.S3; +using Amazon.S3.Model; +using FakeItEasy; + +namespace Elastic.Changelog.Tests.Reconciliation; + +/// +/// A stateful in-memory S3 bucket behind a FakeItEasy : objects live in a +/// dictionary, ETags are the MD5 of the content (matching real single-part uploads and +/// S3EtagCalculator), and conditional PUTs (If-Match / If-None-Match) enforce +/// real 412 semantics. Every write call is recorded so tests can assert exactly what mutated — +/// or that nothing did. +/// +internal sealed class FakeS3Bucket +{ + private readonly Dictionary _objects = [with(StringComparer.Ordinal)]; + + public IAmazonS3 Client { get; } = A.Fake(); + + /// Every PutObject call received, in order. + public List Puts { get; } = []; + + /// Every CopyObject call received, in order. + public List Copies { get; } = []; + + /// Runs once immediately before the first PutObject is evaluated — simulates a concurrent writer. + public Action? BeforeFirstPut { get; set; } + + /// Runs before every ListObjects evaluation with the 1-based call number — simulates propagation mid-poll. + public Action? OnList { get; set; } + + /// Number of ListObjects calls received. + public int ListCalls { get; private set; } + + private bool _firstPutSeen; + + public FakeS3Bucket() + { + _ = A.CallTo(() => Client.ListObjectsV2Async(A._, A._)) + .ReturnsLazily((ListObjectsV2Request r, CancellationToken _) => List(r)); + + _ = A.CallTo(() => Client.GetObjectAsync(A._, A._)) + .ReturnsLazily((GetObjectRequest r, CancellationToken _) => Get(r)); + + _ = A.CallTo(() => Client.GetObjectMetadataAsync(A._, A._)) + .ReturnsLazily((GetObjectMetadataRequest r, CancellationToken _) => Head(r)); + + _ = A.CallTo(() => Client.PutObjectAsync(A._, A._)) + .ReturnsLazily((PutObjectRequest r, CancellationToken _) => Put(r)); + + _ = A.CallTo(() => Client.CopyObjectAsync(A._, A._)) + .ReturnsLazily((CopyObjectRequest r, CancellationToken _) => Copy(r)); + } + + // MD5 is what real S3 uses for single-part ETags; this mirrors S3EtagCalculator. + [SuppressMessage("Security", "CA5351:Do Not Use Broken Cryptographic Algorithms")] + public static string ETagOf(string content) => + Convert.ToHexStringLower(MD5.HashData(Encoding.UTF8.GetBytes(content))); + + /// Seeds or replaces an object; returns its (unquoted) ETag. + public string Seed(string key, string content) + { + var etag = ETagOf(content); + _objects[key] = (content, etag); + return etag; + } + + public void Remove(string key) => _objects.Remove(key); + + public bool Exists(string key) => _objects.ContainsKey(key); + + public string ContentOf(string key) => _objects[key].Content; + + private ListObjectsV2Response List(ListObjectsV2Request request) + { + ListCalls++; + OnList?.Invoke(ListCalls); + return new ListObjectsV2Response + { + S3Objects = _objects + .Where(kv => kv.Key.StartsWith(request.Prefix ?? string.Empty, StringComparison.Ordinal)) + .OrderBy(kv => kv.Key, StringComparer.Ordinal) + .Select(kv => new S3Object + { + Key = kv.Key, + ETag = $"\"{kv.Value.ETag}\"", + Size = kv.Value.Content.Length, + LastModified = new DateTime(2026, 5, 6, 12, 0, 0, DateTimeKind.Utc) + }) + .ToList(), + IsTruncated = false + }; + } + + private GetObjectResponse Get(GetObjectRequest request) + { + if (!_objects.TryGetValue(request.Key, out var obj)) + throw NotFound(); + + return new GetObjectResponse + { + ETag = $"\"{obj.ETag}\"", + ResponseStream = new MemoryStream(Encoding.UTF8.GetBytes(obj.Content)) + }; + } + + private GetObjectMetadataResponse Head(GetObjectMetadataRequest request) + { + if (!_objects.ContainsKey(request.Key)) + throw NotFound(); + + var response = new GetObjectMetadataResponse(); + response.Headers.ContentType = "application/yaml"; + response.Metadata.Add("x-amz-meta-origin", "test"); + return response; + } + + private PutObjectResponse Put(PutObjectRequest request) + { + if (!_firstPutSeen) + { + _firstPutSeen = true; + BeforeFirstPut?.Invoke(); + } + + Puts.Add(request); + + var exists = _objects.TryGetValue(request.Key, out var current); + if (request.IfNoneMatch == "*" && exists) + throw PreconditionFailed(); + if (request.IfMatch is { } ifMatch && (!exists || ifMatch.Trim('"') != current.ETag)) + throw PreconditionFailed(); + + _ = Seed(request.Key, request.ContentBody); + return new PutObjectResponse(); + } + + private CopyObjectResponse Copy(CopyObjectRequest request) + { + Copies.Add(request); + if (!_objects.TryGetValue(request.SourceKey, out var value)) + throw NotFound(); + + _objects[request.DestinationKey] = value; + return new CopyObjectResponse(); + } + + private static AmazonS3Exception NotFound() => + new("Not Found") { StatusCode = HttpStatusCode.NotFound }; + + private static AmazonS3Exception PreconditionFailed() => + new("Precondition Failed") { StatusCode = HttpStatusCode.PreconditionFailed }; +} diff --git a/tests/Elastic.Changelog.Tests/Reconciliation/PublicVerificationServiceTests.cs b/tests/Elastic.Changelog.Tests/Reconciliation/PublicVerificationServiceTests.cs new file mode 100644 index 0000000000..0ade15c35b --- /dev/null +++ b/tests/Elastic.Changelog.Tests/Reconciliation/PublicVerificationServiceTests.cs @@ -0,0 +1,188 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using Amazon.S3.Model; +using AwesomeAssertions; +using Elastic.Changelog.Reconciliation; +using Elastic.Changelog.Uploading; +using FakeItEasy; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Elastic.Changelog.Tests.Reconciliation; + +[SuppressMessage("Usage", "CA1001:Types that own disposable fields should be disposable")] +public class PublicVerificationServiceTests(ITestOutputHelper output) +{ + private const string PrivateBucket = "private-bucket"; + private const string PublicBucket = "public-bucket"; + private const string RegistryKey = "bundle/elasticsearch/registry.json"; + private const string BundleKey = "bundle/elasticsearch/9.3.0.yaml"; + + private static readonly DateTimeOffset FixedNow = new(2026, 5, 6, 12, 0, 0, TimeSpan.Zero); + + private readonly FakeS3Bucket _private = new(); + private readonly FakeS3Bucket _public = new(); + private readonly TestDiagnosticsCollector _collector = new(output); + + private ChangelogPublicVerificationService Service => + new(NullLoggerFactory.Instance, _private.Client, _public.Client); + + private static ChangelogPublicVerifyArguments Args(int maxAttempts = 1) => new() + { + S3BucketName = PrivateBucket, + PublicS3BucketName = PublicBucket, + Product = "elasticsearch", + MaxAttempts = maxAttempts, + // Zero interval keeps the bounded-wait tests deterministic and instant. + PollInterval = TimeSpan.Zero + }; + + // language=yaml + private const string BundleYaml = """ + products: + - product: elasticsearch + target: 9.3.0 + repo: elasticsearch + owner: elastic + entries: [] + """; + + private static string RegistryJson(params RegistryBundle[] entries) => + JsonSerializer.Serialize(new Registry + { + Product = "elasticsearch", + GeneratedAt = FixedNow, + Bundles = entries + }, RegistryJsonContext.Default.Registry); + + private void SeedConvergedState() + { + var etag = _private.Seed(BundleKey, BundleYaml); + var registry = RegistryJson(new RegistryBundle { File = "9.3.0.yaml", Target = "9.3.0", ETag = etag }); + _ = _private.Seed(RegistryKey, registry); + // The scrubber copied the (unchanged) bundle and passed the registry through verbatim. + _ = _public.Seed(BundleKey, BundleYaml); + _ = _public.Seed(RegistryKey, registry); + } + + private void AssertNoWrites(FakeS3Bucket bucket) + { + bucket.Puts.Should().BeEmpty(); + bucket.Copies.Should().BeEmpty(); + A.CallTo(() => bucket.Client.PutObjectAsync(A._, A._)).MustNotHaveHappened(); + A.CallTo(() => bucket.Client.CopyObjectAsync(A._, A._)).MustNotHaveHappened(); + A.CallTo(() => bucket.Client.DeleteObjectAsync(A._, A._)).MustNotHaveHappened(); + } + + [Fact] + public async Task Verify_ConvergedState_SucceedsWithoutWriting() + { + SeedConvergedState(); + + var result = await Service.Verify(_collector, Args(), TestContext.Current.CancellationToken); + + result.Should().BeTrue(); + _collector.Errors.Should().Be(0); + AssertNoWrites(_public); + AssertNoWrites(_private); + } + + [Fact] + public async Task Verify_MissingPublicObject_ReportsWithoutWriting() + { + SeedConvergedState(); + _public.Remove(BundleKey); + + var result = await Service.Verify(_collector, Args(maxAttempts: 2), TestContext.Current.CancellationToken); + + result.Should().BeFalse(); + _collector.Errors.Should().BeGreaterThan(0); + _collector.Diagnostics.Should().Contain(d => d.Message.Contains("MissingPublicObject")); + AssertNoWrites(_public); + AssertNoWrites(_private); + } + + [Fact] + public async Task Verify_MissingPublicRegistry_Reports() + { + SeedConvergedState(); + _public.Remove(RegistryKey); + + var result = await Service.Verify(_collector, Args(), TestContext.Current.CancellationToken); + + result.Should().BeFalse(); + _collector.Diagnostics.Should().Contain(d => d.Message.Contains("MissingPublicRegistry")); + } + + [Fact] + public async Task Verify_PublicRegistryDiffersFromPrivate_ReportsMismatch() + { + SeedConvergedState(); + _ = _public.Seed(RegistryKey, RegistryJson(new RegistryBundle { File = "9.2.0.yaml", Target = "9.2.0", ETag = "old" })); + + var result = await Service.Verify(_collector, Args(), TestContext.Current.CancellationToken); + + result.Should().BeFalse(); + _collector.Diagnostics.Should().Contain(d => d.Message.Contains("RegistryMismatch")); + } + + [Fact] + public async Task Verify_StalePublicObject_Reports() + { + SeedConvergedState(); + _ = _public.Seed("bundle/elasticsearch/9.1.0.yaml", BundleYaml); + + var result = await Service.Verify(_collector, Args(), TestContext.Current.CancellationToken); + + result.Should().BeFalse(); + _collector.Diagnostics.Should().Contain(d => d.Message.Contains("StalePublicObject")); + } + + [Fact] + public async Task Verify_PersistentDivergence_StopsAtMaxAttempts() + { + SeedConvergedState(); + _public.Remove(BundleKey); + + var result = await Service.Verify(_collector, Args(maxAttempts: 3), TestContext.Current.CancellationToken); + + result.Should().BeFalse(); + // One public listing per comparison attempt: the bounded policy must stop exactly at the limit. + _public.ListCalls.Should().Be(3); + AssertNoWrites(_public); + } + + [Fact] + public async Task Verify_ScrubberCatchesUpMidPoll_Converges() + { + SeedConvergedState(); + _public.Remove(BundleKey); + // The scrubber "delivers" the object after the first divergent comparison. + _public.OnList = call => + { + if (call == 2) + _ = _public.Seed(BundleKey, BundleYaml); + }; + + var result = await Service.Verify(_collector, Args(maxAttempts: 5), TestContext.Current.CancellationToken); + + result.Should().BeTrue(); + _public.ListCalls.Should().Be(2, "the wait loop must stop as soon as the state converges"); + AssertNoWrites(_public); + } + + [Fact] + public async Task Verify_InvalidMaxAttempts_Errors() + { + SeedConvergedState(); + + var result = await Service.Verify(_collector, Args(maxAttempts: 0), TestContext.Current.CancellationToken); + + result.Should().BeFalse(); + _collector.Errors.Should().BeGreaterThan(0); + } + +} diff --git a/tests/Elastic.Changelog.Tests/Reconciliation/RegistryInspectionServiceTests.cs b/tests/Elastic.Changelog.Tests/Reconciliation/RegistryInspectionServiceTests.cs new file mode 100644 index 0000000000..e9efba95fa --- /dev/null +++ b/tests/Elastic.Changelog.Tests/Reconciliation/RegistryInspectionServiceTests.cs @@ -0,0 +1,76 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Diagnostics.CodeAnalysis; +using System.IO.Abstractions.TestingHelpers; +using System.Text.Json; +using AwesomeAssertions; +using Elastic.Changelog.Reconciliation; +using Elastic.Documentation.Configuration; +using Microsoft.Extensions.Logging.Abstractions; +using Nullean.ScopedFileSystem; + +namespace Elastic.Changelog.Tests.Reconciliation; + +[SuppressMessage("Usage", "CA1001:Types that own disposable fields should be disposable")] +public class RegistryInspectionServiceTests(ITestOutputHelper output) +{ + private readonly FakeS3Bucket _bucket = new(); + private readonly MockFileSystem _mockFileSystem = new(new MockFileSystemOptions + { + CurrentDirectory = Paths.WorkingDirectoryRoot.FullName + }); + private readonly TestDiagnosticsCollector _collector = new(output); + + private ChangelogRegistryInspectionService Service => + new(NullLoggerFactory.Instance, _bucket.Client, FileSystemFactory.ScopeCurrentWorkingDirectory(_mockFileSystem)); + + [Fact] + public async Task Inspect_CleanScope_Succeeds() + { + // An empty scope with no manifest: nothing published, nothing to reconcile. + var args = new ChangelogRegistryInspectArguments { S3BucketName = "private-bucket", Product = "elasticsearch" }; + + var result = await Service.Inspect(_collector, args, TestContext.Current.CancellationToken); + + result.Should().BeTrue(); + _collector.Errors.Should().Be(0); + } + + [Fact] + public async Task Inspect_DivergedScope_ErrorsAndWritesSnapshot() + { + _ = _bucket.Seed("bundle/elasticsearch/9.3.0.yaml", "entries: []"); + var outPath = _mockFileSystem.Path.Join(Paths.WorkingDirectoryRoot.FullName, "snapshot.json"); + var args = new ChangelogRegistryInspectArguments + { + S3BucketName = "private-bucket", + Product = "elasticsearch", + Out = outPath + }; + + var result = await Service.Inspect(_collector, args, TestContext.Current.CancellationToken); + + result.Should().BeFalse(); + _collector.Errors.Should().BeGreaterThan(0); + _bucket.Puts.Should().BeEmpty("inspection is read-only"); + + var snapshot = JsonSerializer.Deserialize( + _mockFileSystem.File.ReadAllText(outPath), + RegistryStateJsonContext.Default.RegistryStateSnapshot); + snapshot!.Divergences.Should().ContainSingle().Which.Kind.Should().Be(RegistryDivergenceKind.Missing); + } + + [Fact] + public async Task Inspect_MissingScopeSelection_Errors() + { + var args = new ChangelogRegistryInspectArguments { S3BucketName = "private-bucket" }; + + var result = await Service.Inspect(_collector, args, TestContext.Current.CancellationToken); + + result.Should().BeFalse(); + _collector.Errors.Should().BeGreaterThan(0); + } + +} diff --git a/tests/Elastic.Changelog.Tests/Reconciliation/RegistryInspectionTests.cs b/tests/Elastic.Changelog.Tests/Reconciliation/RegistryInspectionTests.cs new file mode 100644 index 0000000000..a9df8e1856 --- /dev/null +++ b/tests/Elastic.Changelog.Tests/Reconciliation/RegistryInspectionTests.cs @@ -0,0 +1,295 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Text.Json; +using AwesomeAssertions; +using Elastic.Changelog.Reconciliation; +using Elastic.Changelog.Uploading; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Elastic.Changelog.Tests.Reconciliation; + +public class RegistryInspectionTests +{ + private readonly FakeS3Bucket _bucket = new(); + private readonly RegistryScopeInspector _inspector; + + private static readonly DateTimeOffset FixedNow = new(2026, 5, 6, 12, 0, 0, TimeSpan.Zero); + + public RegistryInspectionTests() => + _inspector = new RegistryScopeInspector(NullLoggerFactory.Instance, new FakeTimeProvider(FixedNow)); + + private IS3ScopeReader Reader => new S3ScopeReader(_bucket.Client, "private-bucket"); + + private static ChangelogScope BundleScope(string product = "elasticsearch") => + ChangelogScope.TryCreateBundle(product, out var scope) ? scope : throw new InvalidOperationException(); + + private static ChangelogScope PoolScope(string org = "elastic", string repo = "elasticsearch", string branch = "main") => + ChangelogScope.TryCreateChangelog(org, repo, branch, out var scope) ? scope : throw new InvalidOperationException(); + + // language=yaml + private static string BundleYaml(string product, string target) => $""" + products: + - product: {product} + target: {target} + repo: {product} + owner: elastic + entries: + - file: + name: 1-feature.yaml + checksum: deadbeef + type: enhancement + title: Sample + """; + + // language=yaml + private const string LegacyAmendYaml = """ + exclude-entries: + - file: + name: 1-feature.yaml + checksum: deadbeef + """; + + private string SeedBundle(string file, string target, string product = "elasticsearch") => + _bucket.Seed($"bundle/{product}/{file}", BundleYaml(product, target)); + + private void SeedRegistry(string product = "elasticsearch", params RegistryBundle[] entries) + { + var registry = new Registry + { + Product = product, + GeneratedAt = FixedNow, + Bundles = entries + }; + _ = _bucket.Seed($"bundle/{product}/registry.json", JsonSerializer.Serialize(registry, RegistryJsonContext.Default.Registry)); + } + + private Task Inspect(ChangelogScope? scope = null) => + _inspector.InspectAsync(Reader, scope ?? BundleScope(), TestContext.Current.CancellationToken); + + [Fact] + public async Task Inspect_CleanScope_ReportsClean() + { + var etag = SeedBundle("9.3.0.yaml", "9.3.0"); + SeedRegistry(entries: new RegistryBundle { File = "9.3.0.yaml", Target = "9.3.0", ETag = etag }); + + var snapshot = await Inspect(); + + snapshot.IsClean.Should().BeTrue(); + snapshot.RegistryHealth.Should().Be(RegistryHealth.Valid); + snapshot.Divergences.Should().BeEmpty(); + snapshot.Objects.Should().ContainSingle(); + snapshot.ExpectedEntries.Should().ContainSingle(); + } + + [Fact] + public async Task Inspect_ObjectWithoutRegistryEntry_ReportsMissing() + { + var kept = SeedBundle("9.3.0.yaml", "9.3.0"); + _ = SeedBundle("9.4.0.yaml", "9.4.0"); + SeedRegistry(entries: new RegistryBundle { File = "9.3.0.yaml", Target = "9.3.0", ETag = kept }); + + var snapshot = await Inspect(); + + snapshot.IsClean.Should().BeFalse(); + var divergence = snapshot.Divergences.Should().ContainSingle().Subject; + divergence.Kind.Should().Be(RegistryDivergenceKind.Missing); + divergence.File.Should().Be("9.4.0.yaml"); + divergence.ObjectTarget.Should().Be("9.4.0"); + } + + [Fact] + public async Task Inspect_RegistryEntryWithoutObject_ReportsStale() + { + var etag = SeedBundle("9.3.0.yaml", "9.3.0"); + SeedRegistry(entries: + [ + new RegistryBundle { File = "9.3.0.yaml", Target = "9.3.0", ETag = etag }, + new RegistryBundle { File = "9.2.0.yaml", Target = "9.2.0", ETag = "gone" } + ]); + + var snapshot = await Inspect(); + + var divergence = snapshot.Divergences.Should().ContainSingle().Subject; + divergence.Kind.Should().Be(RegistryDivergenceKind.Stale); + divergence.File.Should().Be("9.2.0.yaml"); + divergence.RegistryTarget.Should().Be("9.2.0"); + } + + [Fact] + public async Task Inspect_UnparseableRegistry_ReportsCorrupt() + { + _ = SeedBundle("9.3.0.yaml", "9.3.0"); + _ = _bucket.Seed("bundle/elasticsearch/registry.json", "not json {{{"); + + var snapshot = await Inspect(); + + snapshot.RegistryHealth.Should().Be(RegistryHealth.Corrupt); + snapshot.IsClean.Should().BeFalse(); + var divergence = snapshot.Divergences.Should().ContainSingle().Subject; + divergence.Kind.Should().Be(RegistryDivergenceKind.Corrupt); + divergence.File.Should().Be("registry.json"); + // Even with a corrupt manifest the snapshot still knows what the registry should contain. + snapshot.ExpectedEntries.Should().ContainSingle().Which.Target.Should().Be("9.3.0"); + } + + [Fact] + public async Task Inspect_RegistryWithUnsafeFileName_ReportsCorrupt() + { + SeedRegistry(entries: new RegistryBundle { File = "../evil.yaml", Target = "9.3.0", ETag = "etag" }); + + var snapshot = await Inspect(); + + snapshot.RegistryHealth.Should().Be(RegistryHealth.Corrupt); + snapshot.Divergences.Should().ContainSingle().Which.Kind.Should().Be(RegistryDivergenceKind.Corrupt); + } + + [Fact] + public async Task Inspect_RegistryETagDisagreesWithObject_ReportsObjectDivergent() + { + _ = SeedBundle("9.3.0.yaml", "9.3.0"); + SeedRegistry(entries: new RegistryBundle { File = "9.3.0.yaml", Target = "9.3.0", ETag = "outdated-etag" }); + + var snapshot = await Inspect(); + + var divergence = snapshot.Divergences.Should().ContainSingle().Subject; + divergence.Kind.Should().Be(RegistryDivergenceKind.ObjectDivergent); + divergence.RegistryETag.Should().Be("outdated-etag"); + divergence.ObjectETag.Should().Be(FakeS3Bucket.ETagOf(BundleYaml("elasticsearch", "9.3.0"))); + } + + [Fact] + public async Task Inspect_RegistryTargetDisagreesWithObject_ReportsObjectDivergent() + { + // A legacy amend (no products of its own) whose registry entry recorded target: null, + // while the parent bundle in the same scope resolves it to 9.3.0. + var parentETag = SeedBundle("9.3.0.yaml", "9.3.0"); + var amendETag = _bucket.Seed("bundle/elasticsearch/9.3.0.amend-1.yaml", LegacyAmendYaml); + SeedRegistry(entries: + [ + new RegistryBundle { File = "9.3.0.yaml", Target = "9.3.0", ETag = parentETag }, + new RegistryBundle { File = "9.3.0.amend-1.yaml", Target = null, ETag = amendETag } + ]); + + var snapshot = await Inspect(); + + var divergence = snapshot.Divergences.Should().ContainSingle().Subject; + divergence.Kind.Should().Be(RegistryDivergenceKind.ObjectDivergent); + divergence.File.Should().Be("9.3.0.amend-1.yaml"); + divergence.RegistryTarget.Should().BeNull(); + divergence.ObjectTarget.Should().Be("9.3.0", "the amend inherits the parent bundle's target"); + } + + [Fact] + public async Task Inspect_MissingRegistryWithObjects_ReportsEveryObjectMissing() + { + _ = SeedBundle("9.3.0.yaml", "9.3.0"); + _ = SeedBundle("9.4.0.yaml", "9.4.0"); + + var snapshot = await Inspect(); + + snapshot.RegistryHealth.Should().Be(RegistryHealth.Missing); + snapshot.IsClean.Should().BeFalse(); + snapshot.Divergences.Should().HaveCount(2); + snapshot.Divergences.Should().OnlyContain(d => d.Kind == RegistryDivergenceKind.Missing); + } + + [Fact] + public async Task Inspect_MissingRegistryOverEmptyScope_ReportsClean() + { + var snapshot = await Inspect(); + + snapshot.RegistryHealth.Should().Be(RegistryHealth.Missing); + snapshot.IsClean.Should().BeTrue("an empty scope with no manifest has nothing to reconcile"); + } + + [Fact] + public async Task Inspect_NewerSchemaRegistry_ReportsUnsupportedWithoutDivergences() + { + _ = _bucket.Seed("bundle/elasticsearch/registry.json", + /*lang=json,strict*/ + """{ "schema_version": 2, "product": "elasticsearch", "generated_at": "2026-05-06T12:00:00+00:00", "bundles": [] }"""); + + var snapshot = await Inspect(); + + snapshot.RegistryHealth.Should().Be(RegistryHealth.UnsupportedSchema); + snapshot.IsClean.Should().BeFalse(); + snapshot.Divergences.Should().BeEmpty("entries of a newer schema cannot be judged"); + } + + [Fact] + public async Task Inspect_UnparseableBundleObject_KeepsRegistryTargetAndDiagnoses() + { + var etag = _bucket.Seed("bundle/elasticsearch/9.3.0.yaml", "\tnot yaml: ["); + SeedRegistry(entries: new RegistryBundle { File = "9.3.0.yaml", Target = "9.3.0", ETag = etag }); + + var snapshot = await Inspect(); + + // The object's target is unknown, so the recorded target is preserved instead of flagged. + snapshot.Divergences.Should().BeEmpty(); + snapshot.ExpectedEntries.Should().ContainSingle().Which.Target.Should().Be("9.3.0"); + snapshot.Diagnostics.Should().ContainSingle(d => d.Contains("Could not parse bundle")); + } + + [Fact] + public async Task Inspect_ChangelogScope_EnumeratesEntriesWithoutTarget() + { + // language=yaml + var etag = _bucket.Seed("changelog/elastic/elasticsearch/main/1-feature.yaml", """ + title: Sample + type: enhancement + products: + - product: elasticsearch + target: 9.3.0 + """); + // A nested pool (branch "main/foo") shares the key prefix but is a different scope. + _ = _bucket.Seed("changelog/elastic/elasticsearch/main/foo/2-feature.yaml", "title: Nested"); + _ = _bucket.Seed("changelog/elastic/elasticsearch/main/registry.json", JsonSerializer.Serialize(new Registry + { + Product = "elastic/elasticsearch/main", + GeneratedAt = FixedNow, + Bundles = [new RegistryBundle { File = "1-feature.yaml", Target = null, ETag = etag }] + }, RegistryJsonContext.Default.Registry)); + + var snapshot = await Inspect(PoolScope()); + + snapshot.IsClean.Should().BeTrue(); + snapshot.Objects.Should().ContainSingle().Which.File.Should().Be("1-feature.yaml"); + snapshot.ExpectedEntries.Should().ContainSingle().Which.Target.Should().BeNull(); + } + + [Fact] + public async Task Inspect_NonYamlObject_IsIgnoredWithDiagnostic() + { + var etag = SeedBundle("9.3.0.yaml", "9.3.0"); + _ = _bucket.Seed("bundle/elasticsearch/notes.txt", "not yaml"); + SeedRegistry(entries: new RegistryBundle { File = "9.3.0.yaml", Target = "9.3.0", ETag = etag }); + + var snapshot = await Inspect(); + + snapshot.IsClean.Should().BeTrue(); + snapshot.Diagnostics.Should().ContainSingle(d => d.Contains("notes.txt")); + } + + [Fact] + public async Task Inspect_Snapshot_SerializesMachineReadable() + { + var etag = SeedBundle("9.3.0.yaml", "9.3.0"); + SeedRegistry(entries: new RegistryBundle { File = "9.4.0.yaml", Target = "9.4.0", ETag = etag }); + + var snapshot = await Inspect(); + var json = JsonSerializer.Serialize(snapshot, RegistryStateJsonContext.Default.RegistryStateSnapshot); + + json.Should().Contain("\"scope_kind\": \"Bundle\""); + json.Should().Contain("\"registry_health\": \"Valid\""); + json.Should().Contain("\"expected_entries\""); + json.Should().Contain("\"is_clean\": false"); + json.Should().Contain("\"Missing\"").And.Contain("\"Stale\""); + } + + private sealed class FakeTimeProvider(DateTimeOffset now) : TimeProvider + { + public override DateTimeOffset GetUtcNow() => now; + } +} diff --git a/tests/Elastic.Changelog.Tests/Reconciliation/RegistryRepairServiceTests.cs b/tests/Elastic.Changelog.Tests/Reconciliation/RegistryRepairServiceTests.cs new file mode 100644 index 0000000000..9ae74b4ab6 --- /dev/null +++ b/tests/Elastic.Changelog.Tests/Reconciliation/RegistryRepairServiceTests.cs @@ -0,0 +1,227 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using AwesomeAssertions; +using Elastic.Changelog.Reconciliation; +using Elastic.Changelog.Uploading; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Elastic.Changelog.Tests.Reconciliation; + +[SuppressMessage("Usage", "CA1001:Types that own disposable fields should be disposable")] +public class RegistryRepairServiceTests(ITestOutputHelper output) +{ + private const string Bucket = "private-bucket"; + private const string RegistryKey = "bundle/elasticsearch/registry.json"; + + private static readonly DateTimeOffset FixedNow = new(2026, 5, 6, 12, 0, 0, TimeSpan.Zero); + + private readonly FakeS3Bucket _bucket = new(); + private readonly TestDiagnosticsCollector _collector = new(output); + + private ChangelogRegistryRepairService Service => + new(NullLoggerFactory.Instance, _bucket.Client, new FakeTimeProvider(FixedNow)); + + private static ChangelogRegistryRepairArguments Args(bool allowEmpty = false, bool dryRun = false) => new() + { + S3BucketName = Bucket, + Product = "elasticsearch", + AllowEmpty = allowEmpty, + DryRun = dryRun + }; + + // language=yaml + private static string BundleYaml(string target) => $""" + products: + - product: elasticsearch + target: {target} + repo: elasticsearch + owner: elastic + entries: + - file: + name: 1-feature.yaml + checksum: deadbeef + type: enhancement + title: Sample + """; + + private string SeedBundle(string file, string target) => + _bucket.Seed($"bundle/elasticsearch/{file}", BundleYaml(target)); + + private void SeedRegistry(params RegistryBundle[] entries) => + _ = _bucket.Seed(RegistryKey, JsonSerializer.Serialize(new Registry + { + Product = "elasticsearch", + GeneratedAt = FixedNow, + Bundles = entries + }, RegistryJsonContext.Default.Registry)); + + private Registry StoredRegistry() => + JsonSerializer.Deserialize(_bucket.ContentOf(RegistryKey), RegistryJsonContext.Default.Registry)!; + + [Fact] + public async Task Repair_DivergedScope_ConvergesRegistryToActualObjects() + { + // One missing object, one stale entry, one divergent etag. + var kept = SeedBundle("9.3.0.yaml", "9.3.0"); + _ = SeedBundle("9.4.0.yaml", "9.4.0"); + SeedRegistry( + new RegistryBundle { File = "9.3.0.yaml", Target = "9.3.0", ETag = "outdated" }, + new RegistryBundle { File = "9.2.0.yaml", Target = "9.2.0", ETag = "gone" }); + + var result = await Service.Repair(_collector, Args(), TestContext.Current.CancellationToken); + + result.Should().BeTrue(); + _collector.Errors.Should().Be(0); + var registry = StoredRegistry(); + registry.Bundles.Select(b => b.File).Should().Equal("9.4.0.yaml", "9.3.0.yaml"); + registry.Bundles.Single(b => b.File == "9.3.0.yaml").ETag.Should().Be(kept); + registry.GeneratedAt.Should().Be(FixedNow); + _bucket.Puts.Should().ContainSingle().Which.IfMatch.Should().NotBeNull(); + } + + [Fact] + public async Task Repair_RunTwice_SecondRunWritesNothing() + { + _ = SeedBundle("9.3.0.yaml", "9.3.0"); + + _ = await Service.Repair(_collector, Args(), TestContext.Current.CancellationToken); + _bucket.Puts.Should().ContainSingle(); + + var second = await Service.Repair(_collector, Args(), TestContext.Current.CancellationToken); + + second.Should().BeTrue(); + _bucket.Puts.Should().ContainSingle("a repaired scope must be clean; repairing again may not write"); + } + + [Fact] + public async Task Repair_CleanScope_WritesNothing() + { + var etag = SeedBundle("9.3.0.yaml", "9.3.0"); + SeedRegistry(new RegistryBundle { File = "9.3.0.yaml", Target = "9.3.0", ETag = etag }); + + var result = await Service.Repair(_collector, Args(), TestContext.Current.CancellationToken); + + result.Should().BeTrue(); + _bucket.Puts.Should().BeEmpty(); + } + + [Fact] + public async Task Repair_MissingRegistry_CreatesWithIfNoneMatch() + { + _ = SeedBundle("9.3.0.yaml", "9.3.0"); + + var result = await Service.Repair(_collector, Args(), TestContext.Current.CancellationToken); + + result.Should().BeTrue(); + var put = _bucket.Puts.Should().ContainSingle().Subject; + put.IfNoneMatch.Should().Be("*"); + put.IfMatch.Should().BeNull(); + StoredRegistry().Bundles.Should().ContainSingle().Which.Target.Should().Be("9.3.0"); + } + + [Fact] + public async Task Repair_DryRun_WritesNothing() + { + _ = SeedBundle("9.3.0.yaml", "9.3.0"); + + var result = await Service.Repair(_collector, Args(dryRun: true), TestContext.Current.CancellationToken); + + result.Should().BeTrue(); + _bucket.Puts.Should().BeEmpty(); + _bucket.Exists(RegistryKey).Should().BeFalse(); + } + + [Fact] + public async Task Repair_EmptyScope_RefusesWithoutAllowEmpty() + { + SeedRegistry(new RegistryBundle { File = "9.2.0.yaml", Target = "9.2.0", ETag = "gone" }); + + var result = await Service.Repair(_collector, Args(), TestContext.Current.CancellationToken); + + result.Should().BeFalse(); + _collector.Errors.Should().BeGreaterThan(0); + _bucket.Puts.Should().BeEmpty(); + } + + [Fact] + public async Task Repair_EmptyScope_AllowEmptyWritesEmptyManifest() + { + SeedRegistry(new RegistryBundle { File = "9.2.0.yaml", Target = "9.2.0", ETag = "gone" }); + + var result = await Service.Repair(_collector, Args(allowEmpty: true), TestContext.Current.CancellationToken); + + result.Should().BeTrue(); + StoredRegistry().Bundles.Should().BeEmpty(); + } + + [Fact] + public async Task Repair_CorruptRegistry_RebuildsFromObjects() + { + _ = SeedBundle("9.3.0.yaml", "9.3.0"); + _ = _bucket.Seed(RegistryKey, "not json {{{"); + + var result = await Service.Repair(_collector, Args(), TestContext.Current.CancellationToken); + + result.Should().BeTrue(); + // The corrupt object's live ETag guards the overwrite. + _bucket.Puts.Should().ContainSingle().Which.IfMatch.Should().NotBeNull(); + StoredRegistry().Bundles.Should().ContainSingle().Which.File.Should().Be("9.3.0.yaml"); + } + + [Fact] + public async Task Repair_NewerSchemaRegistry_RefusesToDowngrade() + { + _ = SeedBundle("9.3.0.yaml", "9.3.0"); + _ = _bucket.Seed(RegistryKey, + /*lang=json,strict*/ + """{ "schema_version": 2, "product": "elasticsearch", "generated_at": "2026-05-06T12:00:00+00:00", "bundles": [] }"""); + + var result = await Service.Repair(_collector, Args(), TestContext.Current.CancellationToken); + + result.Should().BeFalse(); + _collector.Errors.Should().BeGreaterThan(0); + _bucket.Puts.Should().BeEmpty(); + } + + [Fact] + public async Task Repair_ConcurrentRegistryUpdate_ReInspectsAndKeepsConcurrentEntry() + { + _ = SeedBundle("9.3.0.yaml", "9.3.0"); + + // Between the repair's read and its conditional PUT, a live upload publishes 9.4.0 and + // refreshes the registry: the first PUT must fail its precondition and the retry must + // re-list, so the concurrent object survives the repair. + _bucket.BeforeFirstPut = () => + { + var concurrentETag = SeedBundle("9.4.0.yaml", "9.4.0"); + SeedRegistry(new RegistryBundle { File = "9.4.0.yaml", Target = "9.4.0", ETag = concurrentETag }); + }; + + var result = await Service.Repair(_collector, Args(), TestContext.Current.CancellationToken); + + result.Should().BeTrue(); + _bucket.Puts.Should().HaveCount(2, "the first write must lose the optimistic-concurrency race"); + StoredRegistry().Bundles.Select(b => b.File).Should().Equal("9.4.0.yaml", "9.3.0.yaml"); + } + + [Fact] + public async Task Repair_InvalidScopeSelection_Errors() + { + var args = new ChangelogRegistryRepairArguments { S3BucketName = Bucket, Product = "elasticsearch", Owner = "elastic" }; + + var result = await Service.Repair(_collector, args, TestContext.Current.CancellationToken); + + result.Should().BeFalse(); + _collector.Errors.Should().BeGreaterThan(0); + } + + private sealed class FakeTimeProvider(DateTimeOffset now) : TimeProvider + { + public override DateTimeOffset GetUtcNow() => now; + } + +} diff --git a/tests/Elastic.Changelog.Tests/Reconciliation/RegistryRepublishServiceTests.cs b/tests/Elastic.Changelog.Tests/Reconciliation/RegistryRepublishServiceTests.cs new file mode 100644 index 0000000000..cdfdea3ba7 --- /dev/null +++ b/tests/Elastic.Changelog.Tests/Reconciliation/RegistryRepublishServiceTests.cs @@ -0,0 +1,139 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Diagnostics.CodeAnalysis; +using Amazon.S3; +using Amazon.S3.Model; +using AwesomeAssertions; +using Elastic.Changelog.Reconciliation; +using FakeItEasy; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Elastic.Changelog.Tests.Reconciliation; + +[SuppressMessage("Usage", "CA1001:Types that own disposable fields should be disposable")] +public class RegistryRepublishServiceTests(ITestOutputHelper output) +{ + private const string Bucket = "private-bucket"; + + private readonly FakeS3Bucket _bucket = new(); + private readonly TestDiagnosticsCollector _collector = new(output); + + private ChangelogRegistryRepublishService Service => new(NullLoggerFactory.Instance, _bucket.Client); + + private static ChangelogRegistryRepublishArguments Args(IReadOnlyList? files = null, bool all = false) => new() + { + S3BucketName = Bucket, + Product = "elasticsearch", + Files = files ?? [], + All = all + }; + + [Fact] + public async Task Republish_ExplicitFiles_SelfCopiesOnlyThoseKeys() + { + _ = _bucket.Seed("bundle/elasticsearch/9.3.0.yaml", "entries: []"); + _ = _bucket.Seed("bundle/elasticsearch/9.4.0.yaml", "entries: []"); + + var result = await Service.Republish(_collector, Args(files: ["9.3.0.yaml"]), TestContext.Current.CancellationToken); + + result.Should().BeTrue(); + var copy = _bucket.Copies.Should().ContainSingle().Subject; + copy.SourceBucket.Should().Be(Bucket); + copy.DestinationBucket.Should().Be(Bucket, "republish only ever touches the private bucket"); + copy.SourceKey.Should().Be("bundle/elasticsearch/9.3.0.yaml"); + copy.DestinationKey.Should().Be(copy.SourceKey, "the re-emission is a self-copy"); + copy.MetadataDirective.Should().Be(S3MetadataDirective.REPLACE); + copy.ContentType.Should().Be("application/yaml", "the rewrite must preserve the original content type"); + copy.Metadata["x-amz-meta-origin"].Should().Be("test", "the rewrite must preserve user metadata"); + } + + [Fact] + public async Task Republish_SelfCopy_LeavesContentUntouched() + { + _ = _bucket.Seed("bundle/elasticsearch/9.3.0.yaml", "entries: []"); + + _ = await Service.Republish(_collector, Args(files: ["9.3.0.yaml"]), TestContext.Current.CancellationToken); + + _bucket.ContentOf("bundle/elasticsearch/9.3.0.yaml").Should().Be("entries: []"); + _bucket.Puts.Should().BeEmpty("republish rewrites via CopyObject, never PutObject"); + } + + [Fact] + public async Task Republish_All_IncludesEveryScopeObjectAndTheManifest() + { + _ = _bucket.Seed("bundle/elasticsearch/9.3.0.yaml", "entries: []"); + _ = _bucket.Seed("bundle/elasticsearch/registry.json", "{}"); + _ = _bucket.Seed("bundle/kibana/9.3.0.yaml", "entries: []"); + + var result = await Service.Republish(_collector, Args(all: true), TestContext.Current.CancellationToken); + + result.Should().BeTrue(); + _bucket.Copies.Select(c => c.SourceKey).Should().BeEquivalentTo( + "bundle/elasticsearch/9.3.0.yaml", + "bundle/elasticsearch/registry.json"); + } + + [Fact] + public async Task Republish_MissingObject_Errors() + { + var result = await Service.Republish(_collector, Args(files: ["9.9.9.yaml"]), TestContext.Current.CancellationToken); + + result.Should().BeFalse(); + _collector.Errors.Should().BeGreaterThan(0); + _bucket.Copies.Should().BeEmpty(); + } + + [Fact] + public async Task Republish_NoSelection_Errors() + { + var result = await Service.Republish(_collector, Args(), TestContext.Current.CancellationToken); + + result.Should().BeFalse(); + _collector.Errors.Should().BeGreaterThan(0); + _bucket.Copies.Should().BeEmpty(); + } + + [Fact] + public async Task Republish_BothSelections_Errors() + { + var result = await Service.Republish(_collector, Args(files: ["9.3.0.yaml"], all: true), TestContext.Current.CancellationToken); + + result.Should().BeFalse(); + _collector.Errors.Should().BeGreaterThan(0); + } + + [Fact] + public async Task Republish_UnsafeFileName_Rejected() + { + var result = await Service.Republish(_collector, Args(files: ["../evil.yaml"]), TestContext.Current.CancellationToken); + + result.Should().BeFalse(); + _collector.Errors.Should().BeGreaterThan(0); + A.CallTo(() => _bucket.Client.CopyObjectAsync(A._, A._)).MustNotHaveHappened(); + } + + [Fact] + public async Task Republish_EmptyScopeWithAll_Succeeds() + { + var result = await Service.Republish(_collector, Args(all: true), TestContext.Current.CancellationToken); + + result.Should().BeTrue(); + _bucket.Copies.Should().BeEmpty(); + } + + [Fact] + public async Task Republish_PartialFailure_ReportsAndContinues() + { + _ = _bucket.Seed("bundle/elasticsearch/9.3.0.yaml", "entries: []"); + + var result = await Service.Republish(_collector, + Args(files: ["9.3.0.yaml", "missing.yaml"]), TestContext.Current.CancellationToken); + + result.Should().BeFalse(); + _bucket.Copies.Should().ContainSingle("the existing object must still be republished"); + _collector.Errors.Should().BeGreaterThan(0); + } + +}