diff --git a/docs/cli-schema.json b/docs/cli-schema.json index ef677c35be..f408698fc2 100644 --- a/docs/cli-schema.json +++ b/docs/cli-schema.json @@ -4433,7 +4433,96 @@ ] } ], - "namespaces": [] + "namespaces": [ + { + "segment": "backfill", + "summary": "Backfill historical release-note bundles (docs-eng-team#656): census, planning, and guarded publication.", + "options": [], + "commands": [ + { + "path": [ + "changelog", + "backfill" + ], + "name": "inventory", + "summary": "Build the backfill census: an inventory document covering every release-notes product.", + "notes": "Enumerates every product in products.yml that participates in release notes (the\nrelease-notes feature defaults to enabled, so products.yml alone cannot say which\nproducts have release-note surfaces), merges in the hand-maintained census seed, and writes the\nversioned inventory document that backfill planning consumes. Products the seed does not cover\nstay visible as source-unresolved entries with a warning \u2014 an unresolved scope must never\nsilently produce empty bundles. Attributed repositories are checked against the link allowlist in\nthe local assembler.yml; planning re-validates against the deployed scrubber allowlist\nbefore any upload. This command only reads configuration and writes a local file: no S3 access,\nno writes to any remote system.", + "usage": "docs-builder changelog backfill inventory [options]", + "examples": [], + "parameters": [ + { + "role": "flag", + "name": "sources", + "type": "string", + "required": false, + "summary": "Path to the census seed YAML mapping products to their release-note sources. Without it, every release-notes product is reported as source-unresolved.", + "validations": [ + { + "kind": "rejectSymbolicLinks" + }, + { + "kind": "existing" + }, + { + "kind": "fileExtensions", + "values": [ + "yml", + "yaml" + ] + } + ] + }, + { + "role": "flag", + "name": "output", + "type": "string", + "required": false, + "summary": "Where to write the inventory document JSON.", + "defaultValue": "backfill-inventory.json" + }, + { + "role": "flag", + "name": "log-level", + "shortName": "l", + "type": "enum", + "required": false, + "summary": "Minimum log level. Default: information", + "enumValues": [ + "trace", + "debug", + "information", + "warning", + "error", + "critical", + "none" + ] + }, + { + "role": "flag", + "name": "config-source", + "shortName": "c", + "type": "enum", + "required": false, + "summary": "Override the configuration source: local, remote", + "enumValues": [ + "local", + "remote", + "embedded" + ] + }, + { + "role": "flag", + "name": "skip-private-repositories", + "type": "boolean", + "required": false, + "summary": "Skip cloning private repositories" + } + ] + } + ], + "namespaces": [] + } + ] }, { "segment": "codex", diff --git a/docs/cli/changelog/backfill/cmd-inventory.md b/docs/cli/changelog/backfill/cmd-inventory.md new file mode 100644 index 0000000000..546aec33e8 --- /dev/null +++ b/docs/cli/changelog/backfill/cmd-inventory.md @@ -0,0 +1,59 @@ +## Description + +Build the backfill census: an inventory document covering every release-notes product. + +`config/products.yml` cannot say which products have release-note surfaces on its own, because the `release-notes` feature defaults to enabled. This command enumerates every product that participates in release notes, merges in a hand-maintained census seed mapping products to their sources, and writes the versioned inventory document that backfill planning consumes. + +Products the seed does not cover stay visible as `source-unresolved` entries and produce a warning — "we looked and decided no" must always be distinguishable from "we never looked", and an unresolved scope can never silently produce empty bundles. Products can be deliberately deferred in the seed's `unmapped` list, each with a reason, which records the deferral without a warning. + +The command is read-only apart from the local output file: it reads configuration, no S3 access, no remote writes. + +### Defaults applied by the census + +- Sources whose products are all stack-versioned and whose scheme is `semver` get the epic's default cutoff of `9.0.0` when the seed does not specify one. +- Unresolved products get a target scheme derived from their versioning system (`serverless`/project versioning → `date`, `ech` → `monthly`, everything else → `semver`), always paired with an unresolved note so a guess never reads as a confirmed fact. +- Attributed repositories are checked against the link allowlist in the local `assembler.yml`. Planning re-validates against the **deployed** scrubber allowlist identity before any upload; the census status is advisory. + +## Seed format + +```yaml +sources: + - repository: elastic/docs-content # where the release-note content lives + git_ref: main + docset: docs-content # optional + paths: + - release-notes/elasticsearch + products: [elasticsearch] # products.yml ids + target_scheme: semver # semver | date | monthly + cutoff: # optional; stack semver defaults to 9.0.0 + kind: version # version | date + value: 9.0.0 + notes: optional free text + substitutions: {} # docset variable expansions + link_mappings: {} # source link -> canonical destination + attributed_repositories: # repos entries attribute changes to + - elastic/elasticsearch + default_repository: elastic/elasticsearch + bundle_filename_convention: "{repo}-{target}.yaml" + adoption: not-adopted # not-adopted | partially-adopted | fully-adopted + classification: published-history-found + unresolved: [] # open questions for a human +unmapped: + - product: kibana + reason: Deferred to the stack family pass. +``` + +Valid classifications: `published-history-found`, `native-artifacts-found`, `hybrid-page`, `declared-no-history`, `outside-cutoff`, `already-live`. `source-unresolved` is deliberately not seedable — it is the census's own conclusion for products nobody mapped, never something an operator writes by hand. + +## Examples + +```sh +# Census with a seed; writes the inventory document +docs-builder changelog backfill inventory \ + --sources config/backfill/inventory-sources.yml \ + --output .artifacts/backfill-inventory.json + +# Without a seed: every release-notes product is reported source-unresolved, +# useful to see the full census surface before mapping begins +docs-builder changelog backfill inventory --output .artifacts/backfill-inventory.json +``` diff --git a/docs/cli/changelog/backfill/index.md b/docs/cli/changelog/backfill/index.md new file mode 100644 index 0000000000..692731296f --- /dev/null +++ b/docs/cli/changelog/backfill/index.md @@ -0,0 +1,12 @@ +## Description + +Backfill historical release-note bundles ([docs-eng-team#656](https://github.com/elastic/docs-eng-team/issues/656)). + +The changelog/bundle pipeline only contains data produced since each repository adopted the live workflows. The backfill commands make the public data look as if the current system had been in use throughout the docs-builder era: they census the products with published release notes, plan exactly which resolved bundles to create, and publish them with create-only writes. + +The pipeline is staged, and every stage exchanges versioned, content-addressed JSON documents (see `src/services/Elastic.Changelog/Backfill/README.md`): + +1. `inventory` — the census: which products and release-note sources exist and what was decided about each. +2. Planning, materialization, apply, and verification stages follow as they are implemented. + +No stage writes to S3 except the guarded apply stage; everything before it is read-only and reviewable. diff --git a/src/services/Elastic.Changelog/Backfill/Inventory/InventoryCensusService.cs b/src/services/Elastic.Changelog/Backfill/Inventory/InventoryCensusService.cs new file mode 100644 index 0000000000..640c0cf042 --- /dev/null +++ b/src/services/Elastic.Changelog/Backfill/Inventory/InventoryCensusService.cs @@ -0,0 +1,406 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.IO.Abstractions; +using Elastic.Documentation.Configuration; +using Elastic.Documentation.Configuration.Products; +using Elastic.Documentation.Configuration.Versions; +using Elastic.Documentation.Diagnostics; +using Elastic.Documentation.Services; +using Microsoft.Extensions.Logging; +using YamlDotNet.Core; + +namespace Elastic.Changelog.Backfill.Inventory; + +public record BuildInventoryArguments +{ + /// Path to the hand-maintained seed YAML (see ). Optional: without it every product is reported as unresolved. + public string? SourcesPath { get; init; } + + /// Where to write the inventory document JSON. + public required string OutputPath { get; init; } + + /// + /// The scrubber link allowlist to compute each attributed repository's status against, as + /// owner/repo entries. Sourced from the local assembler.yml today; planning + /// re-validates against the deployed allowlist identity before any upload. + /// + public IReadOnlyList AllowRepos { get; init; } = []; +} + +/// +/// The census (docs-eng-team#673): enumerates every product in products.yml that +/// participates in release notes — the feature defaults to enabled, so products.yml +/// alone cannot say which products have release-note surfaces — merges in the hand-maintained +/// source seed, and writes the inventory document planning consumes. Products the seed does +/// not cover stay visible as source-unresolved entries: "we looked and decided no" +/// must always be distinguishable from "we never looked", and an unresolved scope can never +/// silently produce empty bundles. +/// +public class InventoryCensusService( + ILoggerFactory logFactory, + IConfigurationContext configurationContext, + IFileSystem fileSystem +) : IService +{ + private readonly ILogger _logger = logFactory.CreateLogger(); + + /// Builds the inventory document and writes it to the output path. Returns false after emitting errors when the census cannot be produced. + public async Task BuildInventoryAsync(IDiagnosticsCollector collector, BuildInventoryArguments args, Cancel ctx = default) + { + var releaseNotesProducts = configurationContext.ProductsConfiguration.Products.Values + .Where(p => p.Features.ReleaseNotes) + .OrderBy(p => p.Id, StringComparer.Ordinal) + .ToList(); + + var seed = await LoadSeedAsync(collector, args.SourcesPath, ctx); + if (seed is null) + return false; + + if (!ValidateSeed(collector, seed, releaseNotesProducts)) + return false; + + var sources = new List(); + foreach (var seedSource in seed.Sources) + sources.Add(BuildSource(seedSource, args.AllowRepos, releaseNotesProducts)); + + var mappedProductIds = seed.Sources.SelectMany(s => s.Products) + .Concat(seed.Unmapped.Select(u => u.Product ?? "")) + .ToHashSet(StringComparer.Ordinal); + + foreach (var unmapped in seed.Unmapped) + { + var product = releaseNotesProducts.First(p => p.Id == unmapped.Product); + sources.Add(BuildUnresolvedSource(product, unmapped.Reason!)); + } + + foreach (var product in releaseNotesProducts.Where(p => !mappedProductIds.Contains(p.Id))) + { + collector.EmitWarning(args.SourcesPath ?? string.Empty, + $"Product '{product.Id}' declares release notes but the seed neither maps a source for it nor defers it under 'unmapped'; recording it as source-unresolved."); + sources.Add(BuildUnresolvedSource(product, + "Not covered by the census seed: no release-note source recorded and not explicitly deferred.")); + } + + var inventory = new InventoryDocument { Sources = sources }; + + string json; + try + { + json = BackfillDocuments.Serialize(inventory); + } + catch (BackfillDocumentException e) + { + collector.EmitError(args.OutputPath, $"The census produced an invalid inventory document: {e.Message}", e); + return false; + } + + var outputDirectory = fileSystem.Path.GetDirectoryName(args.OutputPath); + if (!string.IsNullOrEmpty(outputDirectory)) + _ = fileSystem.Directory.CreateDirectory(outputDirectory); + await fileSystem.File.WriteAllTextAsync(args.OutputPath, json, ctx); + + var hash = BackfillDocuments.ComputeHash(json); + LogSummary(sources, args.OutputPath, hash); + return true; + } + + private async Task LoadSeedAsync(IDiagnosticsCollector collector, string? sourcesPath, Cancel ctx) + { + if (string.IsNullOrWhiteSpace(sourcesPath)) + { + _logger.LogWarning("No census seed provided; every release-notes product will be reported as source-unresolved"); + return new InventorySourcesSeed(); + } + + if (!fileSystem.File.Exists(sourcesPath)) + { + collector.EmitError(sourcesPath, "The census seed file does not exist."); + return null; + } + + var yaml = await fileSystem.File.ReadAllTextAsync(sourcesPath, ctx); + try + { + return InventorySourcesSeed.Deserialize(yaml); + } + catch (YamlException e) + { + collector.EmitError(sourcesPath, $"The census seed is not valid YAML: {e.Message}", e); + return null; + } + } + + private static bool ValidateSeed(IDiagnosticsCollector collector, InventorySourcesSeed seed, IReadOnlyList releaseNotesProducts) + { + var valid = true; + var knownIds = releaseNotesProducts.Select(p => p.Id).ToHashSet(StringComparer.Ordinal); + var sourceProductIds = seed.Sources.SelectMany(s => s.Products).ToHashSet(StringComparer.Ordinal); + + for (var i = 0; i < seed.Sources.Count; i++) + valid &= ValidateSeedSource(collector, seed.Sources[i], i, knownIds); + + for (var i = 0; i < seed.Unmapped.Count; i++) + valid &= ValidateSeedUnmapped(collector, seed.Unmapped[i], i, knownIds, sourceProductIds); + + return valid; + } + + private static bool ValidateSeedSource(IDiagnosticsCollector collector, SeedSource source, int index, IReadOnlySet knownIds) + { + var valid = true; + var where = $"sources[{index}]"; + + if (TryParseRepository(source.Repository) is null) + { + collector.EmitError(string.Empty, $"{where}: 'repository' must be owner/name (e.g. elastic/docs-content), but found '{source.Repository}'."); + valid = false; + } + if (string.IsNullOrWhiteSpace(source.GitRef)) + { + collector.EmitError(string.Empty, $"{where}: 'git_ref' is required."); + valid = false; + } + if (source.Products.Count == 0) + { + collector.EmitError(string.Empty, $"{where}: at least one product is required."); + valid = false; + } + foreach (var product in source.Products.Where(p => !knownIds.Contains(p))) + { + collector.EmitError(string.Empty, + $"{where}: product '{product}' is not a release-notes product in products.yml (unknown id, or its release-notes feature is disabled)."); + valid = false; + } + if (ParseTargetScheme(source.TargetScheme) is null) + { + collector.EmitError(string.Empty, $"{where}: 'target_scheme' must be semver, date, or monthly, but found '{source.TargetScheme}'."); + valid = false; + } + if (ParseClassification(source.Classification) is null) + { + collector.EmitError(string.Empty, + $"{where}: 'classification' must be one of published-history-found, native-artifacts-found, hybrid-page, declared-no-history, outside-cutoff, already-live, but found '{source.Classification}'."); + valid = false; + } + if (ParseAdoption(source.Adoption) is null) + { + collector.EmitError(string.Empty, $"{where}: 'adoption' must be not-adopted, partially-adopted, or fully-adopted, but found '{source.Adoption}'."); + valid = false; + } + if (source.Cutoff is { } cutoff && ParseCutoffKind(cutoff.Kind) is null) + { + collector.EmitError(string.Empty, $"{where}: 'cutoff.kind' must be version or date, but found '{cutoff.Kind}'."); + valid = false; + } + foreach (var attributed in source.AttributedRepositories.Where(r => TryParseRepository(r) is null)) + { + collector.EmitError(string.Empty, $"{where}: attributed repository '{attributed}' must be owner/name."); + valid = false; + } + if (source.DefaultRepository is not null && TryParseRepository(source.DefaultRepository) is null) + { + collector.EmitError(string.Empty, $"{where}: 'default_repository' must be owner/name, but found '{source.DefaultRepository}'."); + valid = false; + } + + return valid; + } + + private static bool ValidateSeedUnmapped( + IDiagnosticsCollector collector, + SeedUnmapped unmapped, + int index, + IReadOnlySet knownIds, + IReadOnlySet sourceProductIds) + { + var valid = true; + var where = $"unmapped[{index}]"; + + if (string.IsNullOrWhiteSpace(unmapped.Product) || !knownIds.Contains(unmapped.Product)) + { + collector.EmitError(string.Empty, $"{where}: product '{unmapped.Product}' is not a release-notes product in products.yml."); + valid = false; + } + else if (sourceProductIds.Contains(unmapped.Product)) + { + collector.EmitError(string.Empty, + $"{where}: product '{unmapped.Product}' is both mapped by a source and listed as unmapped; remove one of the two."); + valid = false; + } + if (string.IsNullOrWhiteSpace(unmapped.Reason)) + { + collector.EmitError(string.Empty, $"{where}: a reason is required so the deferral is auditable."); + valid = false; + } + + return valid; + } + + private InventorySource BuildSource(SeedSource seedSource, IReadOnlyList allowRepos, IReadOnlyList releaseNotesProducts) + { + var scheme = ParseTargetScheme(seedSource.TargetScheme)!.Value; + var cutoff = seedSource.Cutoff is { } seedCutoff + ? new BackfillCutoff { Kind = ParseCutoffKind(seedCutoff.Kind)!.Value, Value = seedCutoff.Value ?? "", Notes = seedCutoff.Notes } + : DefaultCutoff(seedSource.Products, releaseNotesProducts, scheme); + + if (seedSource.Cutoff is null && cutoff is null) + { + _logger.LogWarning("Source {Repository} has no cutoff and no stack default applies; planning will require one before this scope can be applied", + seedSource.Repository); + } + + return new InventorySource + { + SourceRepository = TryParseRepository(seedSource.Repository), + GitRef = seedSource.GitRef, + Docset = seedSource.Docset, + Paths = seedSource.Paths, + ProductIds = seedSource.Products, + TargetScheme = scheme, + Cutoff = cutoff, + Substitutions = seedSource.Substitutions, + LinkMappings = seedSource.LinkMappings, + AttributedRepositories = seedSource.AttributedRepositories + .Select(r => TryParseRepository(r)!) + .Select(repository => new AttributedRepository + { + Repository = repository, + OnScrubberAllowlist = allowRepos.Contains($"{repository.Owner}/{repository.Name}", StringComparer.OrdinalIgnoreCase) + }) + .ToList(), + DefaultRepository = TryParseRepository(seedSource.DefaultRepository), + BundleFilenameConvention = seedSource.BundleFilenameConvention, + AdoptionState = ParseAdoption(seedSource.Adoption)!.Value, + Classification = ParseClassification(seedSource.Classification)!.Value, + UnresolvedItems = seedSource.Unresolved + }; + } + + private static InventorySource BuildUnresolvedSource(Product product, string reason) + { + var (scheme, schemeNote) = DeriveTargetScheme(product); + var unresolved = new List { reason }; + if (schemeNote is not null) + unresolved.Add(schemeNote); + + return new InventorySource + { + ProductIds = [product.Id], + TargetScheme = scheme, + AdoptionState = AdoptionState.NotAdopted, + Classification = SourceClassification.SourceUnresolved, + UnresolvedItems = unresolved + }; + } + + /// The epic's default boundary: stack-versioned products backfill from 9.0.0; anything else needs an explicit product-specific cutoff. + private static BackfillCutoff? DefaultCutoff(IReadOnlyList productIds, IReadOnlyList releaseNotesProducts, TargetScheme scheme) + { + if (scheme != TargetScheme.Semver) + return null; + + var allStackVersioned = productIds + .Select(id => releaseNotesProducts.FirstOrDefault(p => p.Id == id)) + .All(p => p?.VersioningSystem?.Id == VersioningSystemId.Stack); + + return allStackVersioned + ? new BackfillCutoff { Kind = CutoffKind.Version, Value = "9.0.0", Notes = "Stack default boundary (docs-builder era starts at stack 9.0)." } + : null; + } + + /// + /// Best guess at how an unresolved product names its releases, from its versioning system. + /// Always paired with an unresolved note — a guess must never read as a confirmed fact. + /// + private static (TargetScheme Scheme, string? Note) DeriveTargetScheme(Product product) + { + var scheme = product.VersioningSystem?.Id switch + { + VersioningSystemId.Serverless or VersioningSystemId.ElasticsearchProject or + VersioningSystemId.ObservabilityProject or VersioningSystemId.SecurityProject => TargetScheme.Date, + VersioningSystemId.Ech => TargetScheme.Monthly, + _ => TargetScheme.Semver + }; + return (scheme, $"Target scheme '{Name(scheme)}' was derived from the '{product.VersioningSystem?.Id.ToString() ?? "unknown"}' versioning system; confirm it when mapping this source."); + } + + private static GitRepository? TryParseRepository(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + return null; + + var parts = value.Split('/'); + if (parts.Length != 2 || string.IsNullOrWhiteSpace(parts[0]) || string.IsNullOrWhiteSpace(parts[1])) + return null; + + return new GitRepository { Owner = parts[0], Name = parts[1] }; + } + + private static TargetScheme? ParseTargetScheme(string? value) => value switch + { + "semver" => TargetScheme.Semver, + "date" => TargetScheme.Date, + "monthly" => TargetScheme.Monthly, + _ => null + }; + + private static CutoffKind? ParseCutoffKind(string? value) => value switch + { + "version" => CutoffKind.Version, + "date" => CutoffKind.Date, + _ => null + }; + + private static AdoptionState? ParseAdoption(string? value) => value switch + { + "not-adopted" => AdoptionState.NotAdopted, + "partially-adopted" => AdoptionState.PartiallyAdopted, + "fully-adopted" => AdoptionState.FullyAdopted, + _ => null + }; + + private static SourceClassification? ParseClassification(string? value) => value switch + { + "published-history-found" => SourceClassification.PublishedHistoryFound, + "native-artifacts-found" => SourceClassification.NativeArtifactsFound, + "hybrid-page" => SourceClassification.HybridPage, + "declared-no-history" => SourceClassification.DeclaredNoHistory, + "outside-cutoff" => SourceClassification.OutsideCutoff, + "already-live" => SourceClassification.AlreadyLive, + // source-unresolved is deliberately not seedable: it is the census's own conclusion + // for products nobody mapped, never something an operator writes by hand. + _ => null + }; + + private static string Name(TargetScheme scheme) => scheme switch + { + TargetScheme.Semver => "semver", + TargetScheme.Date => "date", + TargetScheme.Monthly => "monthly", + _ => scheme.ToString() + }; + + private static string Name(SourceClassification classification) => classification switch + { + SourceClassification.PublishedHistoryFound => "published-history-found", + SourceClassification.NativeArtifactsFound => "native-artifacts-found", + SourceClassification.HybridPage => "hybrid-page", + SourceClassification.DeclaredNoHistory => "declared-no-history", + SourceClassification.SourceUnresolved => "source-unresolved", + SourceClassification.OutsideCutoff => "outside-cutoff", + SourceClassification.AlreadyLive => "already-live", + _ => classification.ToString() + }; + + private void LogSummary(IReadOnlyList sources, string outputPath, string hash) + { + var byClassification = sources + .GroupBy(s => s.Classification) + .OrderBy(g => g.Key) + .Select(g => $"{Name(g.Key)}: {g.Count()}"); + _logger.LogInformation("Census: {SourceCount} sources ({Breakdown})", sources.Count, string.Join(", ", byClassification)); + _logger.LogInformation("Inventory written to {OutputPath} ({Hash})", outputPath, hash); + } +} diff --git a/src/services/Elastic.Changelog/Backfill/Inventory/InventorySourcesSeed.cs b/src/services/Elastic.Changelog/Backfill/Inventory/InventorySourcesSeed.cs new file mode 100644 index 0000000000..3b2dd62c52 --- /dev/null +++ b/src/services/Elastic.Changelog/Backfill/Inventory/InventorySourcesSeed.cs @@ -0,0 +1,112 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +namespace Elastic.Changelog.Backfill.Inventory; + +/// +/// The hand-maintained census mapping the inventory stage consumes: where each product's +/// release notes live, plus products deliberately deferred (each with a reason). The census +/// turns this into an , filling classification gaps so no +/// product silently disappears. Field values use the same kebab-case names the inventory +/// document serializes with (e.g. published-history-found, not-adopted). +/// +public sealed class InventorySourcesSeed +{ + /// One entry per known release-note source. + public List Sources { get; set; } = []; + + /// Products intentionally not mapped yet; each needs a reason so the deferral is auditable. + public List Unmapped { get; set; } = []; + + private static readonly IDeserializer Deserializer = + new StaticDeserializerBuilder(new InventorySeedYamlContext()) + .WithNamingConvention(UnderscoredNamingConvention.Instance) + .Build(); + + /// Parses a seed document from YAML. Malformed YAML throws; semantic problems are reported by the census. + public static InventorySourcesSeed Deserialize(string yaml) => + Deserializer.Deserialize(yaml) ?? new InventorySourcesSeed(); +} + +/// One release-note source in the seed: a repository plus everything the census records about it. +public sealed class SeedSource +{ + /// The repository the release-note content lives in, as owner/name (e.g. elastic/docs-content). + public string? Repository { get; set; } + + /// The git ref (branch, tag, or commit) the content is read at. + public string? GitRef { get; set; } + + /// The docset the content belongs to, when the repository hosts more than one. + public string? Docset { get; set; } + + /// Paths inside the repository where the release-note content lives. + public List Paths { get; set; } = []; + + /// The product IDs (as known to products.yml) this source produces release notes for. + public List Products { get; set; } = []; + + /// How the product names its releases: semver, date, or monthly. + public string? TargetScheme { get; set; } + + /// Where hand-written history ends and live workflow data begins. + public SeedCutoff? Cutoff { get; set; } + + /// Docset variable substitutions (like {{es}} → Elasticsearch) needed to expand the source text. + public Dictionary Substitutions { get; set; } = []; + + /// Known mappings from links as written in the source to the destinations they should resolve to. + public Dictionary LinkMappings { get; set; } = []; + + /// Repositories entries attribute changes to, as owner/name. Allowlist status is computed by the census, not recorded here. + public List AttributedRepositories { get; set; } = []; + + /// The repository to attribute content to when an entry does not say where it came from, as owner/name. + public string? DefaultRepository { get; set; } + + /// The file-name pattern bundles for this source are expected to use, e.g. {repo}-{target}.yaml. + public string? BundleFilenameConvention { get; set; } + + /// Adoption of the live changelog workflows: not-adopted, partially-adopted, or fully-adopted. + public string? Adoption { get; set; } + + /// The census conclusion, e.g. published-history-found or native-artifacts-found. + public string? Classification { get; set; } + + /// Open questions about this source that a human still needs to answer. + public List Unresolved { get; set; } = []; +} + +/// A backfill boundary in the seed: everything on or after it is in scope. +public sealed class SeedCutoff +{ + /// Whether is a version or a date. + public string? Kind { get; set; } + + /// The boundary itself, e.g. 9.0.0 or 2025-01-01. + public string? Value { get; set; } + + /// Optional free-text explanation of why this boundary was chosen. + public string? Notes { get; set; } +} + +/// A product deliberately left unmapped for now, with the reason recorded. +public sealed class SeedUnmapped +{ + /// The product ID as known to products.yml. + public string? Product { get; set; } + + /// Why this product is not mapped yet — required, so the deferral is auditable. + public string? Reason { get; set; } +} + +[YamlStaticContext] +[YamlSerializable(typeof(InventorySourcesSeed))] +[YamlSerializable(typeof(SeedSource))] +[YamlSerializable(typeof(SeedCutoff))] +[YamlSerializable(typeof(SeedUnmapped))] +public partial class InventorySeedYamlContext; diff --git a/src/services/Elastic.Changelog/Backfill/InventoryDocument.cs b/src/services/Elastic.Changelog/Backfill/InventoryDocument.cs index c9b8a0c706..b6b222dd89 100644 --- a/src/services/Elastic.Changelog/Backfill/InventoryDocument.cs +++ b/src/services/Elastic.Changelog/Backfill/InventoryDocument.cs @@ -136,11 +136,11 @@ public sealed record AttributedRepository /// public sealed record InventorySource { - /// The repository the release-note content lives in. - public required GitRepository SourceRepository { get; init; } + /// The repository the release-note content lives in. Null only while the source is unresolved — every other classification implies we know where we looked. + public GitRepository? SourceRepository { get; init; } - /// The git ref (branch, tag, or commit) the census read the content at. - public required string GitRef { get; init; } + /// The git ref (branch, tag, or commit) the census read the content at. Null only while the source is unresolved. + public string? GitRef { get; init; } /// The docset the content belongs to, when the repository hosts more than one. public string? Docset { get; init; } @@ -187,9 +187,14 @@ public sealed record InventorySource /// Adds a plain-English description of every problem in this source to . public void Validate(IList problems) { - SourceRepository.Validate(problems); - if (string.IsNullOrWhiteSpace(GitRef)) - problems.Add("An inventory source needs a non-empty git ref."); + SourceRepository?.Validate(problems); + if (Classification != SourceClassification.SourceUnresolved) + { + if (SourceRepository is null) + problems.Add("An inventory source needs a repository unless its classification is source-unresolved."); + if (string.IsNullOrWhiteSpace(GitRef)) + problems.Add("An inventory source needs a non-empty git ref unless its classification is source-unresolved."); + } if (ProductIds.Count == 0) problems.Add("An inventory source needs at least one product ID."); if (ProductIds.Any(string.IsNullOrWhiteSpace)) diff --git a/src/services/Elastic.Changelog/Backfill/README.md b/src/services/Elastic.Changelog/Backfill/README.md index 072894e892..91bcc7d3f7 100644 --- a/src/services/Elastic.Changelog/Backfill/README.md +++ b/src/services/Elastic.Changelog/Backfill/README.md @@ -18,7 +18,7 @@ reviewing a run — reads and writes the same shapes. ## How they flow -1. The inventory stage takes the census and writes an **inventory**; a human adds **overrides** where the census got something wrong. +1. The inventory stage takes the census and writes an **inventory** (`docs-builder changelog backfill inventory`, implemented by `Inventory/InventoryCensusService`); a human adds **overrides** where the census got something wrong. 2. The parser turns release-note sources into a **semantic-model** (plus **provenance** for every recovered fact). 3. Planning combines inventory + overrides + semantic model + the current S3 state into a **plan**. 4. A human approves the plan; the apply stage executes it and writes a **ledger** of what really happened. diff --git a/src/tooling/docs-builder/Commands/ChangelogBackfillCommand.cs b/src/tooling/docs-builder/Commands/ChangelogBackfillCommand.cs new file mode 100644 index 0000000000..8b716003b1 --- /dev/null +++ b/src/tooling/docs-builder/Commands/ChangelogBackfillCommand.cs @@ -0,0 +1,62 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.ComponentModel.DataAnnotations; +using Elastic.Changelog.Backfill.Inventory; +using Elastic.Changelog.Bundling; +using Elastic.Documentation.Configuration; +using Elastic.Documentation.Configuration.Assembler; +using Elastic.Documentation.Diagnostics; +using Elastic.Documentation.Services; +using Microsoft.Extensions.Logging; +using Nullean.Argh; +using Nullean.Argh.Documentation; + +namespace Documentation.Builder.Commands; + +/// Backfill historical release-note bundles (docs-eng-team#656): census, planning, and guarded publication. +internal sealed class ChangelogBackfillCommands( + ILoggerFactory logFactory, + IDiagnosticsCollector collector, + IConfigurationContext configurationContext, + AssemblyConfiguration assemblyConfiguration +) +{ + /// Build the backfill census: an inventory document covering every release-notes product. + /// + /// Enumerates every product in products.yml that participates in release notes (the + /// release-notes feature defaults to enabled, so products.yml alone cannot say which + /// products have release-note surfaces), merges in the hand-maintained census seed, and writes the + /// versioned inventory document that backfill planning consumes. Products the seed does not cover + /// stay visible as source-unresolved entries with a warning — an unresolved scope must never + /// silently produce empty bundles. Attributed repositories are checked against the link allowlist in + /// the local assembler.yml; planning re-validates against the deployed scrubber allowlist + /// before any upload. This command only reads configuration and writes a local file: no S3 access, + /// no writes to any remote system. + /// + /// Path to the census seed YAML mapping products to their release-note sources. Without it, every release-notes product is reported as source-unresolved. + /// Where to write the inventory document JSON. + /// Cancellation token + [NoOptionsInjection] + public async Task Inventory( + [Existing, ExpandUserProfile, RejectSymbolicLinks, FileExtensions(Extensions = "yml,yaml")] FileInfo? sources = null, + [ExpandUserProfile, RejectSymbolicLinks] string output = "backfill-inventory.json", + CancellationToken ct = default + ) + { + await using var serviceInvoker = new ServiceInvoker(collector); + + var service = new InventoryCensusService(logFactory, configurationContext, FileSystemFactory.RealGitRootForPathWrite(null, output)); + var args = new BuildInventoryArguments + { + SourcesPath = sources?.FullName, + OutputPath = output, + AllowRepos = LinkAllowlistSanitizer.BuildAllowReposFromAssembler(assemblyConfiguration) + }; + serviceInvoker.AddCommand(service, args, + static async (s, c, state, ct) => await s.BuildInventoryAsync(c, state, ct) + ); + return await serviceInvoker.InvokeAsync(ct); + } +} diff --git a/src/tooling/docs-builder/Program.cs b/src/tooling/docs-builder/Program.cs index 98eb0bc178..3b4aa45b1a 100644 --- a/src/tooling/docs-builder/Program.cs +++ b/src/tooling/docs-builder/Program.cs @@ -49,7 +49,10 @@ _ = app.Map(); _ = app.Map(); _ = app.Map(); - _ = app.MapNamespace("changelog"); + _ = app.MapNamespace("changelog", g => + { + _ = g.MapNamespace("backfill"); + }); _ = app.MapNamespace("inbound-links"); _ = app.Map(); diff --git a/tests/Elastic.Changelog.Tests/Backfill/InventoryCensusServiceTests.cs b/tests/Elastic.Changelog.Tests/Backfill/InventoryCensusServiceTests.cs new file mode 100644 index 0000000000..b8a744afd8 --- /dev/null +++ b/tests/Elastic.Changelog.Tests/Backfill/InventoryCensusServiceTests.cs @@ -0,0 +1,366 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Collections.Frozen; +using System.Diagnostics.CodeAnalysis; +using System.IO.Abstractions.TestingHelpers; +using AwesomeAssertions; +using Elastic.Changelog.Backfill; +using Elastic.Changelog.Backfill.Inventory; +using Elastic.Documentation.Configuration; +using Elastic.Documentation.Configuration.Products; +using Elastic.Documentation.Configuration.Versions; +using Elastic.Documentation.Versions; +using FakeItEasy; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Elastic.Changelog.Tests.Backfill; + +[SuppressMessage("Usage", "CA1001:Types that own disposable fields should be disposable")] +public class InventoryCensusServiceTests(ITestOutputHelper output) +{ + private const string SeedPath = "/repo/backfill-sources.yml"; + private const string OutputPath = "/repo/.artifacts/backfill-inventory.json"; + + private readonly MockFileSystem _fileSystem = new(); + private readonly TestDiagnosticsCollector _collector = new(output); + private readonly IConfigurationContext _configurationContext = A.Fake(); + + private static VersioningSystem Versioning(VersioningSystemId id) => new() + { + Id = id, + Current = new SemVersion(9, 2, 0), + Base = new SemVersion(9, 2, 0) + }; + + private static Product MakeProduct(string id, VersioningSystemId? versioning = VersioningSystemId.Stack, bool releaseNotes = true) => new() + { + Id = id, + DisplayName = id, + VersioningSystem = versioning is { } v ? Versioning(v) : null, + Features = new ProductFeatures { PublicReference = true, ReleaseNotes = releaseNotes } + }; + + private void SetupProducts(params Product[] products) + { + var dictionary = products.ToDictionary(p => p.Id, p => p); + var configuration = new ProductsConfiguration + { + Products = dictionary.ToFrozenDictionary(), + PublicReferenceProducts = dictionary.ToFrozenDictionary(), + ProductDisplayNames = dictionary.ToDictionary(p => p.Key, p => p.Value.DisplayName).ToFrozenDictionary() + }; + A.CallTo(() => _configurationContext.ProductsConfiguration).Returns(configuration); + } + + private InventoryCensusService CreateService() => + new(NullLoggerFactory.Instance, _configurationContext, _fileSystem); + + private async Task RunAsync(string? seedYaml = null, IReadOnlyList? allowRepos = null) + { + if (seedYaml is not null) + _fileSystem.AddFile(SeedPath, new MockFileData(seedYaml)); + + return await CreateService().BuildInventoryAsync(_collector, new BuildInventoryArguments + { + SourcesPath = seedYaml is not null ? SeedPath : null, + OutputPath = OutputPath, + AllowRepos = allowRepos ?? [] + }, TestContext.Current.CancellationToken); + } + + private InventoryDocument ReadOutput() => + BackfillDocuments.Deserialize(_fileSystem.File.ReadAllText(OutputPath)); + + [Fact] + public async Task BuildInventoryAsync_NoSeed_EveryProductIsVisibleAsUnresolved() + { + SetupProducts(MakeProduct("elasticsearch"), MakeProduct("kibana")); + + var result = await RunAsync(); + + result.Should().BeTrue(); + var inventory = ReadOutput(); + inventory.Sources.Should().HaveCount(2); + inventory.Sources.Should().OnlyContain(s => s.Classification == SourceClassification.SourceUnresolved); + _collector.Diagnostics.Should().HaveCount(2).And.OnlyContain(d => d.Message.Contains("source-unresolved")); + } + + [Fact] + public async Task BuildInventoryAsync_ProductsWithoutReleaseNotesFeature_AreExcluded() + { + SetupProducts(MakeProduct("elasticsearch"), MakeProduct("docs-internal", releaseNotes: false)); + + var result = await RunAsync(); + + result.Should().BeTrue(); + ReadOutput().Sources.Should().ContainSingle(s => s.ProductIds.Contains("elasticsearch")); + } + + [Fact] + public async Task BuildInventoryAsync_MappedStackProduct_GetsStackDefaultCutoff() + { + SetupProducts(MakeProduct("elasticsearch")); + + var result = await RunAsync(""" + sources: + - repository: elastic/docs-content + git_ref: main + paths: [release-notes/elasticsearch] + products: [elasticsearch] + target_scheme: semver + adoption: not-adopted + classification: published-history-found + """); + + result.Should().BeTrue(); + var source = ReadOutput().Sources.Single(); + source.Classification.Should().Be(SourceClassification.PublishedHistoryFound); + source.SourceRepository.Should().Be(new GitRepository { Owner = "elastic", Name = "docs-content" }); + source.Cutoff.Should().NotBeNull(); + source.Cutoff.Kind.Should().Be(CutoffKind.Version); + source.Cutoff.Value.Should().Be("9.0.0"); + } + + [Fact] + public async Task BuildInventoryAsync_ExplicitCutoff_IsNotOverridden() + { + SetupProducts(MakeProduct("elasticsearch")); + + var result = await RunAsync(""" + sources: + - repository: elastic/docs-content + git_ref: main + products: [elasticsearch] + target_scheme: semver + cutoff: { kind: version, value: 8.15.0, notes: earlier import } + adoption: not-adopted + classification: published-history-found + """); + + result.Should().BeTrue(); + ReadOutput().Sources.Single().Cutoff!.Value.Should().Be("8.15.0"); + } + + [Fact] + public async Task BuildInventoryAsync_NonStackSemverProduct_GetsNoDefaultCutoff() + { + SetupProducts(MakeProduct("edot-java", VersioningSystemId.All)); + + var result = await RunAsync(""" + sources: + - repository: elastic/elastic-otel-java + git_ref: main + products: [edot-java] + target_scheme: semver + adoption: partially-adopted + classification: native-artifacts-found + """); + + result.Should().BeTrue(); + ReadOutput().Sources.Single().Cutoff.Should().BeNull(); + } + + [Fact] + public async Task BuildInventoryAsync_AttributedRepositories_GetAllowlistStatus() + { + SetupProducts(MakeProduct("cloud-hosted", VersioningSystemId.Ech)); + + var result = await RunAsync(""" + sources: + - repository: elastic/docs-content + git_ref: main + products: [cloud-hosted] + target_scheme: monthly + attributed_repositories: [elastic/elasticsearch, elastic/cloud] + adoption: partially-adopted + classification: hybrid-page + """, + allowRepos: ["elastic/elasticsearch"]); + + result.Should().BeTrue(); + var attributed = ReadOutput().Sources.Single().AttributedRepositories; + attributed.Should().HaveCount(2); + attributed.Single(a => a.Repository.Name == "elasticsearch").OnScrubberAllowlist.Should().BeTrue(); + attributed.Single(a => a.Repository.Name == "cloud").OnScrubberAllowlist.Should().BeFalse(); + } + + [Fact] + public async Task BuildInventoryAsync_UnmappedProduct_IsRecordedWithReason() + { + SetupProducts(MakeProduct("elasticsearch"), MakeProduct("kibana")); + + var result = await RunAsync(""" + sources: + - repository: elastic/docs-content + git_ref: main + products: [elasticsearch] + target_scheme: semver + adoption: not-adopted + classification: published-history-found + unmapped: + - product: kibana + reason: Deferred to the stack family pass. + """); + + result.Should().BeTrue(); + var kibana = ReadOutput().Sources.Single(s => s.ProductIds.Contains("kibana")); + kibana.Classification.Should().Be(SourceClassification.SourceUnresolved); + kibana.UnresolvedItems.Should().Contain("Deferred to the stack family pass."); + _collector.Diagnostics.Should().BeEmpty(); + } + + [Fact] + public async Task BuildInventoryAsync_UnresolvedSchemes_DeriveFromVersioningSystem() + { + SetupProducts( + MakeProduct("elasticsearch"), + MakeProduct("cloud-serverless", VersioningSystemId.Serverless), + MakeProduct("cloud-hosted", VersioningSystemId.Ech)); + + var result = await RunAsync(); + + result.Should().BeTrue(); + var sources = ReadOutput().Sources; + sources.Single(s => s.ProductIds.Contains("elasticsearch")).TargetScheme.Should().Be(TargetScheme.Semver); + sources.Single(s => s.ProductIds.Contains("cloud-serverless")).TargetScheme.Should().Be(TargetScheme.Date); + sources.Single(s => s.ProductIds.Contains("cloud-hosted")).TargetScheme.Should().Be(TargetScheme.Monthly); + sources.Should().OnlyContain(s => s.UnresolvedItems.Any(i => i.Contains("derived"))); + } + + [Fact] + public async Task BuildInventoryAsync_UnknownProductInSeed_Fails() + { + SetupProducts(MakeProduct("elasticsearch")); + + var result = await RunAsync(""" + sources: + - repository: elastic/docs-content + git_ref: main + products: [not-a-product] + target_scheme: semver + adoption: not-adopted + classification: published-history-found + """); + + result.Should().BeFalse(); + _collector.Diagnostics.Should().Contain(d => d.Message.Contains("not-a-product")); + _fileSystem.File.Exists(OutputPath).Should().BeFalse(); + } + + [Fact] + public async Task BuildInventoryAsync_ProductBothMappedAndUnmapped_Fails() + { + SetupProducts(MakeProduct("elasticsearch")); + + var result = await RunAsync(""" + sources: + - repository: elastic/docs-content + git_ref: main + products: [elasticsearch] + target_scheme: semver + adoption: not-adopted + classification: published-history-found + unmapped: + - product: elasticsearch + reason: Also deferred? + """); + + result.Should().BeFalse(); + _collector.Diagnostics.Should().Contain(d => d.Message.Contains("both mapped")); + } + + [Fact] + public async Task BuildInventoryAsync_UnmappedWithoutReason_Fails() + { + SetupProducts(MakeProduct("elasticsearch")); + + var result = await RunAsync(""" + unmapped: + - product: elasticsearch + """); + + result.Should().BeFalse(); + _collector.Diagnostics.Should().Contain(d => d.Message.Contains("reason is required")); + } + + [Fact] + public async Task BuildInventoryAsync_SourceUnresolvedClassification_IsNotSeedable() + { + SetupProducts(MakeProduct("elasticsearch")); + + var result = await RunAsync(""" + sources: + - repository: elastic/docs-content + git_ref: main + products: [elasticsearch] + target_scheme: semver + adoption: not-adopted + classification: source-unresolved + """); + + result.Should().BeFalse(); + _collector.Diagnostics.Should().Contain(d => d.Message.Contains("classification")); + } + + [Fact] + public async Task BuildInventoryAsync_MalformedRepository_Fails() + { + SetupProducts(MakeProduct("elasticsearch")); + + var result = await RunAsync(""" + sources: + - repository: just-a-name + git_ref: main + products: [elasticsearch] + target_scheme: semver + adoption: not-adopted + classification: published-history-found + """); + + result.Should().BeFalse(); + _collector.Diagnostics.Should().Contain(d => d.Message.Contains("owner/name")); + } + + [Fact] + public async Task BuildInventoryAsync_MissingSeedFile_Fails() + { + SetupProducts(MakeProduct("elasticsearch")); + + var result = await CreateService().BuildInventoryAsync(_collector, new BuildInventoryArguments + { + SourcesPath = "/repo/nope.yml", + OutputPath = OutputPath + }, TestContext.Current.CancellationToken); + + result.Should().BeFalse(); + _collector.Diagnostics.Should().Contain(d => d.Message.Contains("does not exist")); + } + + [Fact] + public async Task BuildInventoryAsync_SameInputs_ProduceTheSameHash() + { + SetupProducts(MakeProduct("elasticsearch"), MakeProduct("kibana")); + + (await RunAsync()).Should().BeTrue(); + var first = BackfillDocuments.ComputeHash(_fileSystem.File.ReadAllText(OutputPath)); + + (await RunAsync()).Should().BeTrue(); + var second = BackfillDocuments.ComputeHash(_fileSystem.File.ReadAllText(OutputPath)); + + second.Should().Be(first); + } + + [Fact] + public async Task BuildInventoryAsync_Output_RoundTripsThroughBackfillDocuments() + { + SetupProducts(MakeProduct("elasticsearch")); + + (await RunAsync()).Should().BeTrue(); + + // ReadOutput() deserializes with full envelope/version/validation checks. + var inventory = ReadOutput(); + inventory.Sources.Should().ContainSingle(); + } +}