diff --git a/scripts/ci/doc-build-test/README.md b/scripts/ci/doc-build-test/README.md new file mode 100644 index 0000000000..71db6dad65 --- /dev/null +++ b/scripts/ci/doc-build-test/README.md @@ -0,0 +1,183 @@ + + +# Documentation build test + +A proof-of-concept harness that reads a Dasharo build manual from +`docs.dasharo.com` and resolves the exact commands a reader would run to build +one firmware variant, so a resulting binary can be compared against a published +release. + +It is a first step towards +[dasharo-issues#1153](https://github.com/Dasharo/dasharo-issues/issues/1153): +_"Create automatic tests of Dasharo build documentation"_. The goal of that +issue is to verify that following the documentation - as a person would, +not as a hand-maintained CI script would - reproduces every historic release +for every supported device. + +## The problem: build manuals are decision trees + +A `building-manual.md` page is authored for MkDocs Material. It is not a +linear script; it is a decision tree built out of `pymdownx.tabbed` content +tabs, admonitions and fenced code blocks. A single page routinely encodes +several mutually exclusive branches at once: + +```text +=== "Dasharo (UEFI)" + === "PRO Z690-A (WIFI) DDR4" + ./build.sh z690a_ddr4 + === "PRO Z690-A (WIFI)" + ./build.sh z690a_ddr5 +=== "Dasharo (coreboot + Heads)" + ... +``` + +Off-the-shelf "runnable docs" tools such as `codedown`, `doc-detective` and +`tuttest` extract _every_ fenced block on the page and run them in order. On +the page above that means running the DDR4 build, the DDR5 build and the Heads +build back to back - incompatible branches concatenated into one broken +script. Branch concatenation is one failure mode. A separate one the +maintainers hit when trying doc-detective - its container lacked host +dependencies the docs assume, such as `sudo` - is about capturing what a fresh +OS needs, and is out of scope here (see Scope below). + +This harness instead resolves a _single path_ through the tree. Tab +resolution follows MkDocs' own `content.tabs.link` semantics: tabs that share +a title are the same choice, so a device selected once applies to every tab +group that offers it. + +## What the harness does + +The parser (`mkdocs_build_extractor.py`) turns a manual into build recipes. +The CLI (`doc_build_test.py`) exposes five subcommands: + +- `list` - enumerate every build target (leaf path) the page describes. +- `extract` - print the ordered commands, expected artifact and caveats for + one selection, with version placeholders substituted. +- `script` - emit a standalone, fail-fast shell script for one selection. +- `verify` - compare a locally built binary against a published release. +- `diagnose` - report documentation issues that block automated testing. + +The parser and its tests use only the Python standard library, so they run in +CI with no extra dependencies and never touch the network. + +## Reproducibility check: sha256, then romscope + +The issue asks whether a build is "identical to the ones we publish". A naive +`sha256` equality check is not enough, and using it alone would report a +failure on almost every real build. Dasharo release binaries are signed with +the 3mdeb Vboot key while a local build is not, so the `VBLOCK` and `GBB` +regions legitimately differ. This is documented in the +[reproducible build verification guide](https://docs.dasharo.com/guides/reproducible-build-verification/). + +`verify` returns `IDENTICAL` (sha256 match) or `DIFFERS`. A `DIFFERS` result is +_not_ by itself a failure: a legitimately reproducible Dasharo build is not +byte-identical to the release, because the release is Vboot-signed and both +carry version strings and build metadata a local build will not match. Deciding +whether a `DIFFERS` result is functionally reproducible needs +[romscope](https://github.com/Dasharo/romscope) `compare` and a human reading +of its report (string / compression / program-data differences). `verify` +therefore surfaces romscope's raw output for a person to interpret rather than +inventing a pass/fail verdict from it. The romscope call is an injected runner, +so the logic stays unit-testable without romscope or Docker present. + +## What it reveals in the current docs + +Run against the live MSI building manual, the harness already surfaces three +classes of documentation issue - exactly the kind of human-error faults the +issue is about - without any change to the docs: + +- _Version-conditional prose._ The UEFI build tab hides a + "For v1.1.1 and older / For v1.1.2 and newer" choice in prose rather than in + a tab, so the resolved recipe contains two mutually exclusive `build.sh` + commands. A machine cannot pick one without parsing the prose. +- _Unlinkable tabs._ In the Heads branch the checkout step labels a device + `PRO Z690-A` while the build step labels it `PRO Z690-A (WIFI) DDR4`. + Because the labels differ, the two tab groups cannot be linked, and + enumeration produces device combinations that make no sense. +- _Unresolvable choices._ Selecting a firmware type but omitting a required + device choice is reported as an ambiguous path, listing the options that + still need a decision. + +The `diagnose` subcommand reports these for a page and exits non-zero when any +are found, so it can gate CI. The committed self-tests run against small +inlined fixtures; the eight findings above were observed by running `diagnose` +against the live `unified/msi/building-manual.md`, not asserted in the suite. + +## Usage + +```bash +# List every build target in a manual +./doc_build_test.py list building-manual.md + +# Resolve one build and print its commands +./doc_build_test.py extract building-manual.md \ + --select "Dasharo (UEFI)" \ + --select "PRO Z690-A (WIFI) DDR4" \ + --version 1.1.3 --revision msi_ms7d25_v1.1.3 + +# Emit a runnable build script for one selection +./doc_build_test.py script building-manual.md \ + --select "Dasharo (UEFI)" --select "PRO Z690-A (WIFI) DDR4" \ + --version 1.1.3 --revision msi_ms7d25_v1.1.3 -o build.sh + +# Compare a locally built binary against a published release +./doc_build_test.py verify --built out.rom --published release.rom \ + --romscope ./romscope + +# Report documentation issues that block automated testing +./doc_build_test.py diagnose building-manual.md +``` + +## Scope and limitations + +This is a proof of concept focused on the deterministic, testable core: +turning a tabbed manual into a single correct build recipe and deciding a +reproducibility verdict. Deliberately out of scope for now: + +- _Running the build._ Building firmware needs Docker and takes minutes per + target; the issue itself flags CI time as a concern. `script` produces a + runnable recipe, but actually executing it and downloading release binaries + is left to a follow-up, run outside pull-request CI. +- _Release discovery._ Mapping a device to its published releases and hashes + (from the per-device `releases.md` pages) is a follow-up. +- _Fresh-OS dependency capture._ Verifying the documented steps on a clean OS + with nothing missing - the `sudo`/toolchain gap the maintainers hit - is the + harder half of the issue and is not attempted here. +- _Parsing assumptions._ Only fences tagged with a shell language are treated + as commands; heredocs and unusual tab-label conventions are not handled. + Placeholder substitution covers the common `X.Y.Z`, `VERSION` and `REVISION` + tokens only. + +## Relation to existing work + +- Test specifications `[BNO] Build on a fresh OS Installation` and + `[FLB] Firmware locally building and flashing` describe the manual + procedure this harness is meant to automate. +- This "parse the docs, then hash-compare" approach follows the plan macpijan + sketched in + [osfv#545](https://github.com/Dasharo/open-source-firmware-validation/pull/545). + An alternative discussed in that thread is to standardise the build (a + universal script, as trialled in + [coreboot#579](https://github.com/Dasharo/coreboot/pull/579), or + Jinja-generated docs) so that parsing becomes unnecessary; this PoC does not + preclude that direction. +- [docs#1240](https://github.com/Dasharo/docs/pull/1240) is a parallel, + non-executing build-docs command checker in the docs repo. This harness is + complementary - single-path resolution, build-script generation and a + reproducibility comparison - and the two should be reconciled rather than + duplicated. + +## Running the self-tests + +```bash +python3 -m unittest discover -s scripts/ci/doc-build-test \ + -t scripts/ci/doc-build-test -p "*_selftests.py" +``` + +The suite uses inlined markdown fixtures that reproduce the real nesting of +the Dasharo manuals, so it needs no network access and no repository +checkout of the docs. diff --git a/scripts/ci/doc-build-test/doc_build_test.py b/scripts/ci/doc-build-test/doc_build_test.py new file mode 100644 index 0000000000..9be9aad712 --- /dev/null +++ b/scripts/ci/doc-build-test/doc_build_test.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python + +# SPDX-FileCopyrightText: 2026 Amey Pawar +# +# SPDX-License-Identifier: Apache-2.0 + +"""Command-line front end for the Dasharo documentation build test. + +Turns a MkDocs building manual into concrete, single-path build recipes and +verifies a locally built binary against a published release. See the module +docstring in ``mkdocs_build_extractor`` and the directory ``README.md`` for the +design rationale. + +Subcommands +----------- +list Enumerate every build target (leaf path) a document describes. +extract Print the resolved commands, artifacts and caveats for one selection. +script Emit a standalone, fail-fast shell script for one selection. +verify Compare a built binary against a published release (sha256 + romscope). +""" + +import argparse +import subprocess +import sys + +import mkdocs_build_extractor as ext + + +def _read(path: str) -> str: + with open(path, "r", encoding="utf-8") as handle: + return handle.read() + + +def _cmd_list(args: argparse.Namespace) -> int: + targets = ext.list_targets(_read(args.doc)) + if not targets: + print("no build targets found", file=sys.stderr) + return 1 + for target in targets: + print(" / ".join(target.selections) or "(single path)") + if target.artifacts: + print(" artifacts: " + ", ".join(target.artifacts)) + print(f"\n{len(targets)} target(s)") + return 0 + + +def _resolve(args: argparse.Namespace) -> ext.Recipe: + return ext.resolve( + _read(args.doc), + select=args.select or [], + version=args.version, + revision=args.revision, + ) + + +def _cmd_extract(args: argparse.Namespace) -> int: + try: + recipe = _resolve(args) + except ext.AmbiguousSelection as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + print("# selection: " + " / ".join(recipe.selections)) + if recipe.caveats: + print("# caveats: " + ", ".join(recipe.caveats)) + if recipe.artifacts: + print("# artifacts: " + ", ".join(recipe.artifacts)) + print() + for command in recipe.commands: + print(command) + return 0 + + +def _cmd_script(args: argparse.Namespace) -> int: + try: + recipe = _resolve(args) + except ext.AmbiguousSelection as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + script = ext.to_script(recipe) + if args.output: + with open(args.output, "w", encoding="utf-8") as handle: + handle.write(script) + else: + sys.stdout.write(script) + return 0 + + +def _cmd_verify(args: argparse.Namespace) -> int: + runner = None + if args.romscope: + + def runner(published: str, built: str) -> str: + result = subprocess.run( + [args.romscope, "compare", published, built], + capture_output=True, + text=True, + check=False, + ) + return result.stdout + result.stderr + + result = ext.verify(args.built, args.published, romscope_runner=runner) + print(result.verdict) + if result.romscope_report: + print("\nromscope report (interpret per romscope's 'Interpreting results'):") + print(result.romscope_report) + return 0 if result.verdict == ext.IDENTICAL else 1 + + +def _cmd_diagnose(args: argparse.Namespace) -> int: + diagnostics = ext.diagnose(_read(args.doc)) + if not diagnostics: + print("no documentation issues detected") + return 0 + for diagnostic in diagnostics: + print(f"{diagnostic.kind}: {diagnostic.detail}") + print(f"\n{len(diagnostics)} issue(s)") + return 1 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="doc_build_test", description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + + p_list = sub.add_parser("list", help="enumerate build targets") + p_list.add_argument("doc", help="path to a building-manual.md") + p_list.set_defaults(func=_cmd_list) + + def add_select(p: argparse.ArgumentParser) -> None: + p.add_argument("doc", help="path to a building-manual.md") + p.add_argument( + "--select", + action="append", + metavar="LABEL", + help="tab label to choose (repeatable)", + ) + p.add_argument("--version", help="value for X.Y.Z / VERSION placeholders") + p.add_argument("--revision", help="value for the REVISION placeholder") + + p_extract = sub.add_parser("extract", help="print a resolved recipe") + add_select(p_extract) + p_extract.set_defaults(func=_cmd_extract) + + p_script = sub.add_parser("script", help="emit a runnable build script") + add_select(p_script) + p_script.add_argument("-o", "--output", help="write script to this file") + p_script.set_defaults(func=_cmd_script) + + p_verify = sub.add_parser("verify", help="compare built vs published binary") + p_verify.add_argument("--built", required=True, help="locally built .rom") + p_verify.add_argument("--published", required=True, help="published .rom") + p_verify.add_argument( + "--romscope", + help="path to a romscope binary for signature-aware comparison", + ) + p_verify.set_defaults(func=_cmd_verify) + + p_diagnose = sub.add_parser( + "diagnose", help="report documentation issues that block automated testing" + ) + p_diagnose.add_argument("doc", help="path to a building-manual.md") + p_diagnose.set_defaults(func=_cmd_diagnose) + return parser + + +def main(argv=None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci/doc-build-test/doc_build_test_selftests.py b/scripts/ci/doc-build-test/doc_build_test_selftests.py new file mode 100644 index 0000000000..a0d0b38708 --- /dev/null +++ b/scripts/ci/doc-build-test/doc_build_test_selftests.py @@ -0,0 +1,378 @@ +#!/usr/bin/env python + +# SPDX-FileCopyrightText: 2026 Amey Pawar +# +# SPDX-License-Identifier: Apache-2.0 + +"""Self-tests for the documentation build extractor. + +Fixtures are inlined as string literals (mirroring +``scripts/ci/regression-scope/regression_scope_selftests.py``) so the sample +docs are not reformatted by the repository's markdownlint hook. They reproduce +the real nesting of ``docs.dasharo.com`` building manuals: firmware-type tabs, +linked device tabs that repeat across tab groups, admonition caveats, version +placeholders and prose artifact declarations. +""" + +import os +import tempfile +import unittest + +import mkdocs_build_extractor as ext + +# A faithful trim of docs/unified/msi/building-manual.md: a UEFI branch whose +# device choice appears in *two* linked tab groups (source checkout + build), +# plus a separate Heads branch with a different device set. +MSI_DOC = """\ +# Building manual + +=== "Dasharo (UEFI)" + + ## Procedure + + Obtain Dasharo source code: + + === "PRO Z690-A DDR4" + > Replace the `REVISION` with `msi_ms7d25_vVERSION`. + + === "PRO Z690-A DDR5" + > Replace the `REVISION` with `msi_ms7d25_vVERSION`. + + ```bash + git clone https://github.com/Dasharo/coreboot.git -b REVISION + cd coreboot + ``` + + Start the build process: + + === "PRO Z690-A DDR4" + ```bash + ./build.sh z690a_ddr4 + ``` + + The resulting Dasharo firmware image will be placed at `$PWD/msi_ms7d25_VERSION_ddr4.rom`. + + === "PRO Z690-A DDR5" + ```bash + ./build.sh z690a_ddr5 + ``` + + The resulting Dasharo firmware image will be placed at `$PWD/msi_ms7d25_VERSION_ddr5.rom`. + +=== "Dasharo (coreboot + Heads)" + + ## Building + + 1. Clone Dasharo Heads repository + + ```bash + git clone https://github.com/Dasharo/heads.git + cd heads + ``` + + === "PRO Z690-A" + ```bash + git checkout msi_ms7d25_v0.9.0 + BOARD=msi_z690a_ddr4 make + ``` +""" + +# A trim exercising package-manager tabs inside an admonition-bearing path and +# a multi-line docker invocation with backslash continuation. +NOVA_DOC = """\ +# Dasharo firmware building guide + +=== "Dasharo (UEFI)" + + - Git + + === "APT package manager" + + ```bash + sudo apt -y install git + ``` + + === "DNF package manager" + + ```bash + sudo dnf -y install git + ``` + + 1. Clone the Dasharo coreboot repository: + + ```bash + git clone https://github.com/Dasharo/coreboot.git + cd coreboot + ``` + + !!! warning + + Releases earlier than October 2023 might not build in custom shells. + + 1. Start docker container: + + ```bash + docker run --rm -it -u $UID \\ + -v $PWD:/home/coreboot/coreboot \\ + -w /home/coreboot/coreboot \\ + coreboot/coreboot-sdk:2023-11-24 /bin/bash + ``` + + This will produce a Dasharo binary placed in `build/coreboot.rom`. +""" + + +# A compact doc reproducing the three defects the real MSI manual contains: +# a version choice hidden in prose, two build commands on one path, and a +# device labelled differently between the checkout and build steps. +DEFECTIVE_DOC = """\ +# Building manual + +=== "Dasharo (UEFI)" + + Obtain source: + + === "PRO Z690-A DDR4" + ```bash + git clone https://github.com/Dasharo/coreboot.git -b REVISION + ``` + + Start the build process: + + === "PRO Z690-A DDR4" + For v1.1.1 and older: + + ```bash + ./build.sh ddr4 + ``` + + For v1.1.2 and newer: + + ```bash + ./build.sh z690a_ddr4 + ``` + +=== "Dasharo (coreboot + Heads)" + + Checkout: + + === "PRO Z690-A" + ```bash + git checkout msi_ms7d25_heads_v0.9.0 + ``` + + Inside the container: + + === "PRO Z690-A (WIFI) DDR4" + ```bash + BOARD=msi_z690a_ddr4 make + ``` +""" + + +class TestParsing(unittest.TestCase): + def test_top_level_is_prose_then_group(self): + nodes = ext.parse(MSI_DOC) + self.assertIsInstance(nodes[0], ext.Prose) + self.assertIsInstance(nodes[1], ext.TabGroup) + self.assertEqual( + nodes[1].titles, + ["Dasharo (UEFI)", "Dasharo (coreboot + Heads)"], + ) + + def test_code_fence_is_dedented(self): + nodes = ext.parse(MSI_DOC) + uefi = nodes[1].tabs[0] + fences = [n for n in uefi.children if isinstance(n, ext.CodeBlock)] + self.assertTrue(fences) + self.assertIn( + "git clone https://github.com/Dasharo/coreboot.git", fences[0].code + ) + self.assertFalse(fences[0].code.startswith(" ")) + + +class TestEnumeration(unittest.TestCase): + def test_linked_tabs_do_not_multiply_targets(self): + # UEFI: 2 devices (linked across 2 groups) -> 2 targets, not 4. + # Heads: 1 device -> 1 target. Total 3. + targets = ext.list_targets(MSI_DOC) + self.assertEqual(len(targets), 3) + + def test_target_selections(self): + targets = ext.list_targets(MSI_DOC) + selections = sorted(" / ".join(t.selections) for t in targets) + self.assertEqual( + selections, + [ + "Dasharo (UEFI) / PRO Z690-A DDR4", + "Dasharo (UEFI) / PRO Z690-A DDR5", + "Dasharo (coreboot + Heads) / PRO Z690-A", + ], + ) + + +class TestSinglePathResolution(unittest.TestCase): + def test_selects_one_branch_only(self): + recipe = ext.resolve( + MSI_DOC, + select=["Dasharo (UEFI)", "PRO Z690-A DDR4"], + version="1.1.3", + revision="msi_ms7d25_v1.1.3", + ) + self.assertEqual( + recipe.commands, + [ + "git clone https://github.com/Dasharo/coreboot.git -b msi_ms7d25_v1.1.3", + "cd coreboot", + "./build.sh z690a_ddr4", + ], + ) + # The mutually exclusive DDR5 branch must never leak into the recipe. + self.assertNotIn("./build.sh z690a_ddr5", recipe.commands) + + def test_shared_step_present_in_all_paths(self): + for device in ("PRO Z690-A DDR4", "PRO Z690-A DDR5"): + recipe = ext.resolve(MSI_DOC, select=["Dasharo (UEFI)", device]) + self.assertIn( + "git clone https://github.com/Dasharo/coreboot.git -b REVISION", + recipe.commands, + ) + + def test_artifact_extraction_and_version_substitution(self): + recipe = ext.resolve( + MSI_DOC, + select=["Dasharo (UEFI)", "PRO Z690-A DDR5"], + version="1.1.3", + revision="msi_ms7d25_v1.1.3", + ) + self.assertEqual(recipe.artifacts, ["$PWD/msi_ms7d25_1.1.3_ddr5.rom"]) + + def test_version_without_revision_is_rejected(self): + # The MSI clone step is `-b REVISION`; asking for a version but no + # revision leaves an unresolved placeholder and must not silently ship. + with self.assertRaises(ValueError): + ext.resolve( + MSI_DOC, + select=["Dasharo (UEFI)", "PRO Z690-A DDR4"], + version="1.1.3", + ) + + def test_heads_branch_is_independent(self): + recipe = ext.resolve( + MSI_DOC, select=["Dasharo (coreboot + Heads)", "PRO Z690-A"] + ) + self.assertIn("git checkout msi_ms7d25_v0.9.0", recipe.commands) + self.assertIn("BOARD=msi_z690a_ddr4 make", recipe.commands) + self.assertNotIn("./build.sh z690a_ddr4", recipe.commands) + + def test_missing_choice_is_ambiguous(self): + with self.assertRaises(ext.AmbiguousSelection) as ctx: + ext.resolve(MSI_DOC, select=["Dasharo (UEFI)"]) + self.assertEqual(ctx.exception.options, ["PRO Z690-A DDR4", "PRO Z690-A DDR5"]) + + +class TestAdmonitionAndContinuation(unittest.TestCase): + def test_package_manager_choice_and_caveat(self): + recipe = ext.resolve(NOVA_DOC, select=["Dasharo (UEFI)", "APT package manager"]) + self.assertIn("sudo apt -y install git", recipe.commands) + self.assertNotIn("sudo dnf -y install git", recipe.commands) + self.assertEqual(recipe.caveats, ["warning"]) + self.assertEqual(recipe.artifacts, ["build/coreboot.rom"]) + + def test_backslash_continuation_stays_single_command(self): + recipe = ext.resolve(NOVA_DOC, select=["Dasharo (UEFI)", "DNF package manager"]) + docker = [c for c in recipe.commands if c.startswith("docker run")] + self.assertEqual(len(docker), 1) + self.assertIn("coreboot/coreboot-sdk:2023-11-24 /bin/bash", docker[0]) + + +class TestFenceHandling(unittest.TestCase): + def test_unlabeled_output_fence_is_not_a_command(self): + doc = ( + "# Build\n\n" + "```bash\n./build.sh board\n```\n\n" + "Expected output:\n\n" + "```\nBuild complete\nFirmware size: 16M\n```\n" + ) + recipe = ext.resolve(doc, select=[]) + self.assertEqual(recipe.commands, ["./build.sh board"]) + + def test_attribute_list_fence_is_shell(self): + doc = "# Build\n\n```{.bash .no-copy}\n./build.sh board\n```\n" + recipe = ext.resolve(doc, select=[]) + self.assertEqual(recipe.commands, ["./build.sh board"]) + + +class TestScriptRendering(unittest.TestCase): + def test_script_is_fail_fast_and_ordered(self): + recipe = ext.resolve( + MSI_DOC, + select=["Dasharo (UEFI)", "PRO Z690-A DDR4"], + version="1.1.3", + revision="msi_ms7d25_v1.1.3", + ) + script = ext.to_script(recipe) + self.assertIn("set -euo pipefail", script) + self.assertLess( + script.index("git clone"), script.index("./build.sh z690a_ddr4") + ) + + +class TestVerify(unittest.TestCase): + def _write(self, data): + fd, path = tempfile.mkstemp() + os.write(fd, data) + os.close(fd) + self.addCleanup(os.remove, path) + return path + + def test_identical_binaries(self): + a = self._write(b"same-bytes") + b = self._write(b"same-bytes") + self.assertEqual(ext.verify(a, b).verdict, ext.IDENTICAL) + + def test_real_difference(self): + a = self._write(b"one") + b = self._write(b"two") + result = ext.verify(a, b) + self.assertEqual(result.verdict, ext.DIFFERS) + self.assertEqual(result.romscope_report, "") + + def test_differ_surfaces_romscope_report_without_classifying_it(self): + a = self._write(b"built") + b = self._write(b"published") + + def fake_romscope(published, built): + return "String differences: build_info\nCompression differences" + + result = ext.verify(a, b, romscope_runner=fake_romscope) + # DIFFERS is not overridden by a guessed verdict; the raw romscope + # report is surfaced for a human to interpret. + self.assertEqual(result.verdict, ext.DIFFERS) + self.assertIn("Compression differences", result.romscope_report) + + +class TestDiagnose(unittest.TestCase): + def test_flags_all_three_defect_classes(self): + kinds = [d.kind for d in ext.diagnose(DEFECTIVE_DOC)] + self.assertEqual(kinds.count("version-conditional-prose"), 2) + self.assertEqual(kinds.count("multiple-build-commands"), 1) + self.assertEqual(kinds.count("inconsistent-tab-labels"), 1) + + def test_clean_docs_report_nothing(self): + self.assertEqual(ext.diagnose(MSI_DOC), []) + self.assertEqual(ext.diagnose(NOVA_DOC), []) + + def test_identical_repeated_build_command_is_not_flagged(self): + doc = ( + "# Build\n\n" + "```bash\n./build.sh h4\n```\n\n" + "Or, equivalently:\n\n" + "```bash\n./build.sh h4\n```\n" + ) + kinds = [d.kind for d in ext.diagnose(doc)] + self.assertNotIn("multiple-build-commands", kinds) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/ci/doc-build-test/mkdocs_build_extractor.py b/scripts/ci/doc-build-test/mkdocs_build_extractor.py new file mode 100644 index 0000000000..69a53c4e0f --- /dev/null +++ b/scripts/ci/doc-build-test/mkdocs_build_extractor.py @@ -0,0 +1,635 @@ +# SPDX-FileCopyrightText: 2026 Amey Pawar +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tab-aware extractor for Dasharo MkDocs "building manual" pages. + +The Dasharo build documentation (``docs.dasharo.com``) is written for MkDocs +Material. A ``building-manual.md`` page is not a linear script - it is a +*decision tree* built out of ``pymdownx.tabbed`` content tabs, admonitions and +fenced code blocks. A single page typically encodes several mutually exclusive +branches at once, for example:: + + === "Dasharo (UEFI)" + ... + === "PRO Z690-A (WIFI) DDR4" + ./build.sh z690a_ddr4 + === "PRO Z690-A (WIFI)" + ./build.sh z690a_ddr5 + === "Dasharo (coreboot + Heads)" + ... + +Off-the-shelf "runnable docs" tools (``codedown``, ``doc-detective``, +``tuttest``) extract *every* fenced block on the page and run them in order. +On a page like the one above that concatenates incompatible branches - +``./build.sh z690a_ddr4`` and ``./build.sh z690a_ddr5`` and the Heads build - +into a single script. (Branch concatenation is one failure mode. A separate +and equally important one, capturing the exact host dependencies a build needs +from a fresh OS, is out of scope for this module; see the README.) + +This module instead resolves a *single path* through the tree. Given a +selection of tab labels (firmware type, device, package manager, ...) it emits +the ordered shell commands for that one build, the artifact(s) the prose says +will be produced, and any admonition caveats attached to the chosen path. Tab +resolution mirrors MkDocs' own ``content.tabs.link`` semantics: tabs that share +a title are the *same* choice, so selecting a device once applies it to every +tab group that offers it. + +The module is pure standard library so it runs in CI with no extra +dependencies and never touches the network. +""" + +from __future__ import annotations + +import hashlib +import re +from dataclasses import dataclass, field +from typing import Callable, Iterator, Optional + +# Fenced-code languages we treat as executable build steps. +SHELL_LANGS = {"bash", "sh", "shell", "console"} + +# Placeholders the docs use for the release version / revision. +VERSION_PLACEHOLDERS = ("X.Y.Z", "VERSION") +REVISION_PLACEHOLDER = "REVISION" + +# Inline-code spans ending in .rom that prose points at as the build output. +_ROM_RE = re.compile(r"`([^`]*?\.rom)`") +_ARTIFACT_CUES = ( + "resulting", + "placed", + "produce", + "produced", + "will be", + "binary", + "image", + "output", +) + +_TAB_RE = re.compile(r'^===\s+(?:"([^"]*)"|\'([^\']*)\')\s*$') +_ADMONITION_RE = re.compile( + r"^(?P!!!|\?\?\?\+?|\?\?\?)\s+(?P[\w-]+)" + r'(?:\s+"(?P[^"]*)")?\s*$' +) +_FENCE_RE = re.compile(r"^(?P<ticks>`{3,}|~{3,})(?P<info>.*)$") + + +# --------------------------------------------------------------------------- # +# Block tree +# --------------------------------------------------------------------------- # +@dataclass +class CodeBlock: + lang: str + code: str + line: int + + +@dataclass +class Prose: + text: str + line: int + + +@dataclass +class Admonition: + kind: str + title: str + children: list = field(default_factory=list) + + +@dataclass +class Tab: + title: str + children: list = field(default_factory=list) + line: int = 0 + + +@dataclass +class TabGroup: + tabs: list = field(default_factory=list) + + @property + def titles(self) -> list: + return [t.title for t in self.tabs] + + @property + def title_key(self) -> tuple: + """Identity of the choice this group offers (order-independent). + + MkDocs links tabs that share a title, so two groups presenting the + same set of titles represent the same decision. + """ + return tuple(sorted(t.title for t in self.tabs)) + + +def _indent(line: str) -> int: + return len(line) - len(line.lstrip(" ")) + + +def _is_blank(line: str) -> bool: + return line.strip() == "" + + +def _parse_lang(info: str) -> str: + """Extract the language from a fence info string. + + Handles plain (```` ```bash ````), title (```` ```bash title="x" ````) and + MkDocs attribute-list (```` ```{.bash .no-copy} ````) forms. + """ + info = info.strip() + if not info: + return "" + if info.startswith("{"): + m = re.search(r"\.([\w-]+)", info) + return m.group(1).lower() if m else "" + return info.split()[0].lower() + + +class _Parser: + """Indentation-aware recursive-descent parser for MkDocs Material blocks.""" + + def __init__(self, text: str) -> None: + self.lines = text.expandtabs(4).splitlines() + self.i = 0 + + def parse(self) -> list: + return self._sequence(0) + + def _sequence(self, min_indent: int) -> list: + nodes: list = [] + prose: list = [] + prose_line = 0 + + def flush_prose() -> None: + nonlocal prose, prose_line + if prose: + nodes.append(Prose("\n".join(prose), prose_line)) + prose = [] + + while self.i < len(self.lines): + line = self.lines[self.i] + if _is_blank(line): + self.i += 1 + continue + indent = _indent(line) + if indent < min_indent: + break + stripped = line.strip() + + fence = _FENCE_RE.match(stripped) + if fence: + flush_prose() + nodes.append(self._code_block(indent, fence)) + continue + + tab = _TAB_RE.match(stripped) + if tab: + flush_prose() + nodes.append(self._tab_group(indent)) + continue + + adm = _ADMONITION_RE.match(stripped) + if adm: + flush_prose() + nodes.append(self._admonition(indent, adm)) + continue + + if not prose: + prose_line = self.i + 1 + prose.append(stripped) + self.i += 1 + + flush_prose() + return nodes + + def _code_block(self, indent: int, fence) -> CodeBlock: + ticks = fence.group("ticks") + lang = _parse_lang(fence.group("info")) + start = self.i + 1 + body: list = [] + self.i += 1 + close = "`" if ticks[0] == "`" else "~" + while self.i < len(self.lines): + line = self.lines[self.i] + s = line.strip() + if s.startswith(close * len(ticks)) and set(s) <= {close}: + self.i += 1 + break + # Dedent by the fence indent; keep deeper indentation intact. + body.append(line[indent:] if len(line) >= indent else line.lstrip(" ")) + self.i += 1 + return CodeBlock(lang=lang, code="\n".join(body), line=start) + + def _tab_group(self, indent: int) -> TabGroup: + group = TabGroup() + while self.i < len(self.lines): + # Skip blanks between sibling tabs. + while self.i < len(self.lines) and _is_blank(self.lines[self.i]): + self.i += 1 + if self.i >= len(self.lines): + break + line = self.lines[self.i] + if _indent(line) != indent: + break + m = _TAB_RE.match(line.strip()) + if not m: + break + title = m.group(1) if m.group(1) is not None else m.group(2) + tab_line = self.i + 1 + self.i += 1 + children = self._sequence(indent + 1) + group.tabs.append(Tab(title=title, children=children, line=tab_line)) + return group + + def _admonition(self, indent: int, adm) -> Admonition: + self.i += 1 + children = self._sequence(indent + 1) + return Admonition( + kind=adm.group("kind"), + title=adm.group("title") or "", + children=children, + ) + + +def parse(text: str) -> list: + """Parse MkDocs Material markdown into a block tree.""" + return _Parser(text).parse() + + +# --------------------------------------------------------------------------- # +# Path resolution +# --------------------------------------------------------------------------- # +class AmbiguousSelection(Exception): + """Raised when a tab group cannot be resolved from the given selection.""" + + def __init__(self, options: list) -> None: + self.options = options + super().__init__( + "ambiguous build path; unresolved choice between: " + + ", ".join(repr(o) for o in options) + ) + + +@dataclass +class Recipe: + """One fully resolved single-path build.""" + + selections: list = field(default_factory=list) + commands: list = field(default_factory=list) + artifacts: list = field(default_factory=list) + caveats: list = field(default_factory=list) + + +def _norm(label: str) -> str: + return " ".join(label.lower().split()) + + +def _label_matches(title: str, wanted) -> bool: + nt = _norm(title) + return any(_norm(w) == nt for w in wanted) + + +def _collect_artifacts(text: str) -> list: + lowered = text.lower() + if not any(cue in lowered for cue in _ARTIFACT_CUES): + return [] + return _ROM_RE.findall(text) + + +def _walk( + nodes: list, + idx: int, + chosen: dict, + select: Optional[list], + state: Recipe, +) -> Iterator[Recipe]: + if idx >= len(nodes): + yield state + return + + node = nodes[idx] + + if isinstance(node, TabGroup): + key = node.title_key + picked = _pick_tabs(node, chosen, select) + for tab in picked: + branch_chosen = chosen + branch_state = state + if key not in chosen: + branch_chosen = dict(chosen) + branch_chosen[key] = tab.title + branch_state = _clone(state) + branch_state.selections = state.selections + [tab.title] + for sub in _walk(tab.children, 0, branch_chosen, select, branch_state): + yield from _walk(nodes, idx + 1, branch_chosen, select, sub) + return + + next_state = _absorb(node, state) + yield from _walk(nodes, idx + 1, chosen, select, next_state) + + +def _pick_tabs(group: TabGroup, chosen: dict, select: Optional[list]) -> list: + key = group.title_key + # A linked group already decided earlier on this path follows that choice. + if key in chosen: + return [t for t in group.tabs if t.title == chosen[key]] + if select is None: + # Enumeration mode: branch over every tab. + return list(group.tabs) + matches = [t for t in group.tabs if _label_matches(t.title, select)] + if len(matches) >= 1: + return matches[:1] + if len(group.tabs) == 1: + return list(group.tabs) + raise AmbiguousSelection(group.titles) + + +def _absorb(node, state: Recipe) -> Recipe: + if isinstance(node, CodeBlock): + # Only fences explicitly tagged with a shell language are build steps. + # Unlabeled fences are sample output / file listings, not commands. + if node.lang in SHELL_LANGS: + new = _clone(state) + for cmd in _split_commands(node.code): + new.commands.append(cmd) + return new + return state + if isinstance(node, Prose): + arts = _collect_artifacts(node.text) + if arts: + new = _clone(state) + new.artifacts = state.artifacts + arts + return new + return state + if isinstance(node, Admonition): + new = _clone(state) + label = node.title or node.kind + new.caveats = state.caveats + [label] + # Admonition bodies can carry code and artifacts on the path too. + for child in node.children: + new = _absorb(child, new) + return new + return state + + +def _split_commands(code: str) -> list: + """Split a fenced block into individual commands. + + Blank lines and comment-only lines are dropped; continuation lines ending + in a backslash are joined so multi-line ``docker run`` invocations stay a + single command. + """ + commands: list = [] + buffer: list = [] + for raw in code.splitlines(): + line = raw.rstrip() + if not line.strip(): + continue + if line.lstrip().startswith("#"): + continue + buffer.append(line) + if line.endswith("\\"): + continue + commands.append("\n".join(buffer)) + buffer = [] + if buffer: + commands.append("\n".join(buffer)) + return commands + + +def _clone(state: Recipe) -> Recipe: + return Recipe( + selections=list(state.selections), + commands=list(state.commands), + artifacts=list(state.artifacts), + caveats=list(state.caveats), + ) + + +def iter_recipes(nodes: list, select: Optional[list] = None) -> Iterator[Recipe]: + """Yield resolved build recipes. + + With ``select=None`` every leaf path in the tree is yielded (enumeration). + With a list of tab labels, a single deterministic path is resolved; an + :class:`AmbiguousSelection` is raised if a required choice is missing. + """ + yield from _walk(nodes, 0, {}, select, Recipe()) + + +def _leftover_placeholders(commands: list) -> set: + joined = "\n".join(commands) + return {t for t in (*VERSION_PLACEHOLDERS, REVISION_PLACEHOLDER) if t in joined} + + +def resolve( + text: str, + select: list, + version: Optional[str] = None, + revision: Optional[str] = None, +) -> Recipe: + """Resolve exactly one build recipe for a selection, with substitutions. + + Raises :class:`AmbiguousSelection` if the selection does not pin a single + path, and ``ValueError`` if a version/revision placeholder is left + unresolved after substitution (e.g. ``--version`` given without the + ``--revision`` a ``git clone -b REVISION`` step needs). + """ + nodes = parse(text) + recipes = list(iter_recipes(nodes, select=select)) + if not recipes: + raise ValueError("no build path found in document") + if len(recipes) > 1: + raise AmbiguousSelection(sorted({" / ".join(r.selections) for r in recipes})) + recipe = recipes[0] + if version is not None or revision is not None: + recipe.commands = [substitute(c, version, revision) for c in recipe.commands] + recipe.artifacts = [substitute(a, version, revision) for a in recipe.artifacts] + leftover = _leftover_placeholders(recipe.commands) + if leftover: + raise ValueError( + "unresolved placeholder(s) after substitution: " + + ", ".join(sorted(leftover)) + + " (pass --version and/or --revision)" + ) + return recipe + + +def list_targets(text: str) -> list: + """Enumerate every build target (leaf path) described by the document.""" + return list(iter_recipes(parse(text), select=None)) + + +def substitute(value: str, version: Optional[str], revision: Optional[str]) -> str: + """Replace version / revision placeholders in a command or artifact.""" + out = value + if revision is not None: + out = out.replace(REVISION_PLACEHOLDER, revision) + if version is not None: + for token in VERSION_PLACEHOLDERS: + out = out.replace(token, version) + return out + + +# --------------------------------------------------------------------------- # +# Runnable script + reproducibility verdict +# --------------------------------------------------------------------------- # +def to_script(recipe: Recipe) -> str: + """Render a recipe as a standalone, fail-fast shell script.""" + header = [ + "#!/usr/bin/env bash", + "# Generated from Dasharo build documentation by doc_build_test.", + "# Selection: " + " / ".join(recipe.selections), + "set -euo pipefail", + "", + ] + if recipe.artifacts: + header.append("# Expected artifact(s): " + ", ".join(recipe.artifacts)) + header.append("") + return "\n".join(header + recipe.commands) + "\n" + + +# Verdicts returned by :func:`verify`. +IDENTICAL = "IDENTICAL" +DIFFERS = "DIFFERS" + + +@dataclass +class VerifyResult: + """Outcome of comparing a built binary to a published release.""" + + verdict: str # IDENTICAL | DIFFERS + romscope_report: str = "" + + +def sha256_file(path: str) -> str: + h = hashlib.sha256() + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(65536), b""): + h.update(chunk) + return h.hexdigest() + + +def verify( + built: str, + published: str, + romscope_runner: Optional[Callable[[str, str], str]] = None, +) -> VerifyResult: + """Compare a locally built binary against a published release. + + ``sha256`` equality yields ``IDENTICAL``. Any other result is ``DIFFERS``, + which does *not* by itself mean the build is wrong: a legitimately + reproducible Dasharo build is not byte-identical to the release (the release + is Vboot-signed, and both carry version strings and build metadata a local + build will not match). Deciding whether a ``DIFFERS`` result is + *functionally* reproducible needs ``romscope compare`` and human reading of + its report (see romscope's "Interpreting results": string / compression / + program-data differences). This function therefore surfaces romscope's raw + output rather than inventing a pass/fail verdict from it. + """ + if sha256_file(built) == sha256_file(published): + return VerifyResult(IDENTICAL) + report = "" + if romscope_runner is not None: + report = romscope_runner(published, built) + return VerifyResult(DIFFERS, report) + + +# --------------------------------------------------------------------------- # +# Documentation diagnostics +# --------------------------------------------------------------------------- # +_VERSION_CONDITION_RE = re.compile( + r"\bfor\s+v?\d+(?:\.\d+)*\s+and\s+(?:older|newer)\b", re.IGNORECASE +) +_BUILD_INVOCATION_RE = re.compile(r"(?:^|\s)\./build\.sh\b") + + +@dataclass +class Diagnostic: + """A machine-detected weakness that makes a manual hard to test.""" + + kind: str + detail: str + + +def _iter_prose(nodes: list) -> Iterator[Prose]: + for node in nodes: + if isinstance(node, Prose): + yield node + elif isinstance(node, Admonition): + yield from _iter_prose(node.children) + elif isinstance(node, TabGroup): + for tab in node.tabs: + yield from _iter_prose(tab.children) + + +def _diagnose_labels(nodes: list, found: list, seen: set) -> None: + """Flag device labels that differ between build steps at the same level. + + Two tab groups in the same scope with different title sets, where one + title is a prefix of another, cannot be linked by MkDocs - a reader has to + guess which checkout tab pairs with which build tab. + """ + groups = [n for n in nodes if isinstance(n, TabGroup)] + for i, group_a in enumerate(groups): + for group_b in groups[i + 1 :]: + if group_a.title_key == group_b.title_key: + continue + for title_a in group_a.titles: + for title_b in group_b.titles: + norm_a, norm_b = _norm(title_a), _norm(title_b) + if norm_a == norm_b: + continue + short, long = sorted((norm_a, norm_b), key=len) + if len(short) >= 6 and long.startswith(short + " "): + pair = (short, long) + if pair not in seen: + seen.add(pair) + found.append( + Diagnostic( + "inconsistent-tab-labels", + f"'{title_a}' vs '{title_b}'", + ) + ) + for node in nodes: + if isinstance(node, TabGroup): + for tab in node.tabs: + _diagnose_labels(tab.children, found, seen) + elif isinstance(node, Admonition): + _diagnose_labels(node.children, found, seen) + + +def diagnose(text: str) -> list: + """Report machine-detectable weaknesses in a build manual. + + The checks are deliberately conservative so that a report is actionable: + + * ``version-conditional-prose`` - a build step branches on the release + version in prose ("For v1.1.1 and older") instead of a tab, so no single + command can be selected mechanically for a given version. + * ``multiple-build-commands`` - one resolved path contains more than one + *distinct* ``./build.sh`` invocation: mutually exclusive builds, or + several board variants, that were not split into separate tabs. + * ``inconsistent-tab-labels`` - two build steps label the same device + differently, so the tabs cannot be linked across steps. + """ + nodes = parse(text) + found: list = [] + + for prose in _iter_prose(nodes): + for line in prose.text.splitlines(): + if _VERSION_CONDITION_RE.search(line): + found.append(Diagnostic("version-conditional-prose", line.strip())) + + for recipe in iter_recipes(nodes): + builds = [c for c in recipe.commands if _BUILD_INVOCATION_RE.search(c)] + distinct = list(dict.fromkeys(builds)) # order-preserving dedupe + if len(distinct) > 1: + found.append( + Diagnostic( + "multiple-build-commands", + " / ".join(recipe.selections) + ": " + ", ".join(distinct), + ) + ) + + _diagnose_labels(nodes, found, set()) + return found