From 161719b9ce58497732f4a40519fe08ba4914f2ec Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Sat, 12 Sep 2026 08:27:22 -0300 Subject: [PATCH 01/10] test(xcat-core): the Ubuntu Genesis image is built from the build host kernel builddebs.pl carries no Genesis step. The Ubuntu Genesis deb is produced by xCAT-genesis-builder/builddeb-genesis-base, which nothing calls, and that script reads /lib/modules and /boot of the host it runs on. One build therefore gives every Ubuntu release the kernel and the modules of the build host. The script also has no payload gate. dracut reports a command it cannot install with a FAILED: line and exits 0, so the build that shipped the image with no dhclient reported success. Its build root omits isc-dhcp-client, ifenslave and util-linux-extra. genesis_deb_per_codename.t asserts the chroot each codename builds in, the log lines that mean a build failed although it exited 0, and the refusal of the builder to run in a root of another release. genesis_payload_verification.t drives verify-genesis-payload against payload trees with a known hole. genesis_ubuntu_build_root.t reads the mandatory commands back from the dracut module and asserts the build root supplies each one. All three fail on this commit. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- xCAT-test/unit/genesis_deb_per_codename.t | 132 +++++++++++++ xCAT-test/unit/genesis_payload_verification.t | 175 ++++++++++++++++++ xCAT-test/unit/genesis_ubuntu_build_root.t | 99 ++++++++++ 3 files changed, 406 insertions(+) create mode 100644 xCAT-test/unit/genesis_deb_per_codename.t create mode 100644 xCAT-test/unit/genesis_payload_verification.t create mode 100755 xCAT-test/unit/genesis_ubuntu_build_root.t diff --git a/xCAT-test/unit/genesis_deb_per_codename.t b/xCAT-test/unit/genesis_deb_per_codename.t new file mode 100644 index 0000000000..1f5b4e2703 --- /dev/null +++ b/xCAT-test/unit/genesis_deb_per_codename.t @@ -0,0 +1,132 @@ +#!/usr/bin/env perl +# The Genesis image carries the kernel and the kernel modules of the root that built it. +# One build on the build host therefore gives every Ubuntu release the build host's kernel. +# builddebs.pl --genesis builds one image per codename, in that codename's chroot. +# +# Three decisions are tested here, each on the value the code returns: +# the chroot each codename builds in; +# the lines in a build log that mean the build failed although it exited 0; +# the refusal of the builder to run in a root of another release. +use strict; +use warnings; + +use File::Path qw(make_path); +use File::Slurper qw(read_text write_text); +use File::Temp qw(tempdir); +use FindBin; +use lib "$FindBin::Bin/../lib"; +use lib "$FindBin::Bin/../../build-utils/lib"; +use Test::More; + +use XCAT::Test::File qw(repo_path); + +require XCAT::BuildUtils; + +my @WANTED = qw(genesis_chroot_name genesis_target_arch genesis_build_plan genesis_log_errors); +for my $sub (@WANTED) { + ok(XCAT::BuildUtils->can($sub), "XCAT::BuildUtils provides $sub"); +} +unless (scalar(grep { XCAT::BuildUtils->can($_) } @WANTED) == scalar @WANTED) { + diag('builddebs.pl has no Genesis step: the per-codename build does not exist yet'); + done_testing(); + exit; +} + +# --- one build per codename, in that codename's chroot --------------------------------- +my @dists = qw(jammy noble resolute); +my @plan = XCAT::BuildUtils::genesis_build_plan(\@dists, 'amd64'); + +is(scalar @plan, scalar @dists, 'one Genesis build per codename'); +is_deeply([ map { $_->{codename} } @plan ], \@dists, 'the plan keeps the codename order'); +is_deeply([ map { $_->{chroot} } @plan ], + [ 'jammy-amd64-sbuild', 'noble-amd64-sbuild', 'resolute-amd64-sbuild' ], + 'each codename builds in its own sbuild chroot'); +is_deeply([ map { $_->{package} } @plan ], + [ ('xcat-genesis-base-amd64') x 3 ], + 'every codename produces the same package name'); + +my @ppc = XCAT::BuildUtils::genesis_build_plan(['noble'], 'ppc64el'); +is($ppc[0]{chroot}, 'noble-ppc64el-sbuild', 'the architecture selects the chroot'); +is($ppc[0]{package}, 'xcat-genesis-base-ppc64el', 'the architecture is in the package name'); +is($ppc[0]{target}, 'ppc64', 'ppc64el reads its image from the ppc64 directory'); +is(XCAT::BuildUtils::genesis_target_arch('amd64'), 'x86_64', + 'amd64 reads its image from the x86_64 directory'); + +# A codename given twice is still one build. +is(scalar(() = XCAT::BuildUtils::genesis_build_plan([qw(noble noble)], 'amd64')), 1, + 'a repeated codename does not build twice'); + +ok(!eval { XCAT::BuildUtils::genesis_build_plan([], 'amd64'); 1 }, + 'a plan with no codename is an error'); +ok(!eval { XCAT::BuildUtils::genesis_target_arch('riscv64'); 1 }, + 'an architecture with no Genesis image directory is an error'); + +# --- the log guard --------------------------------------------------------------------- +# +# dracut prints FAILED: for a command it cannot install and exits 0. This is the log of the +# build that shipped the image with no dhclient. +my $dracut_log = <<'LOG'; +Installing build dependencies... +dracut: Executing: /usr/bin/dracut --compress gzip -m xcat base -N -f /tmp/genesis.rfs 6.8.0-45-generic +dracut-install: ERROR: installing 'dhclient' +dracut: FAILED: /usr/lib/dracut/dracut-install -D /var/tmp/dracut.XXXX -a dhclient +dracut: *** Creating initramfs image file '/tmp/genesis.rfs' done *** +Extracting initramfs... +LOG + +my @errors = XCAT::BuildUtils::genesis_log_errors($dracut_log); +ok(scalar @errors, 'a dracut log with a FAILED: line is an error'); +like($errors[0]{line}, qr/FAILED:/, 'the offending line is reported'); +ok(length $errors[0]{why}, 'the reason is named'); + +is_deeply([ XCAT::BuildUtils::genesis_log_errors(<<'LOG') ], [], 'a clean build log is not an error'); +Installing build dependencies... +dracut: *** Creating initramfs image file '/tmp/genesis.rfs' done *** +Extracting initramfs... +dpkg-deb: building package 'xcat-genesis-base-amd64' +LOG + +for my $case ( + [ 'E: Unable to locate package isc-dhcp-client' => 'a package apt cannot find' ], + [ 'dracut: Cannot find module directory /lib/modules/6.8.0' => 'a module directory dracut cannot find' ], + [ '/build/builddeb-genesis-base: line 9: dch: command not found' => 'a command the build root lacks' ], + [ 'E: Unable to correct problems, you have held broken packages.' => 'a build root apt cannot resolve' ], + ) +{ + my ($line, $what) = @{$case}; + ok(scalar XCAT::BuildUtils::genesis_log_errors("before\n$line\nafter\n"), + "the log guard catches $what"); +} + +is_deeply([ XCAT::BuildUtils::genesis_log_errors(undef) ], [], 'no log is not an error'); + +# --- the builder refuses a root of another release ------------------------------------- +my $builder = repo_path('xCAT-genesis-builder/builddeb-genesis-base'); +if (!-f $builder) { + fail('xCAT-genesis-builder/builddeb-genesis-base is missing'); + done_testing(); + exit; +} + +my $tmp = tempdir(CLEANUP => 1); +mkdir "$tmp/bin"; +# dpkg is shadowed so the guard is tested on any host, and so the test cannot reach apt. +write_text("$tmp/bin/dpkg", "#!/bin/sh\necho amd64\n"); +chmod 0755, "$tmp/bin/dpkg"; + +sub build_in_a_root_of { + my ($codename, $expected) = @_; + write_text("$tmp/os-release", "ID=ubuntu\nVERSION_CODENAME=$codename\n"); + my $err = "$tmp/err"; + my $cmd = sprintf('PATH=%s:$PATH OS_RELEASE=%s /bin/bash %s --expect-codename %s >/dev/null 2>%s', + "'$tmp/bin'", "'$tmp/os-release'", "'$builder'", "'$expected'", "'$err'"); + system('/bin/bash', '-c', $cmd); + return ($? >> 8, -f $err ? read_text($err) : ''); +} + +my ($rc, $err) = build_in_a_root_of('jammy', 'noble'); +isnt($rc, 0, 'the builder refuses to build noble in a jammy root'); +like($err, qr/jammy/, 'the message names the root it woke up in'); +like($err, qr/noble/, 'the message names the release that was asked for'); + +done_testing(); diff --git a/xCAT-test/unit/genesis_payload_verification.t b/xCAT-test/unit/genesis_payload_verification.t new file mode 100644 index 0000000000..7897f485c9 --- /dev/null +++ b/xCAT-test/unit/genesis_payload_verification.t @@ -0,0 +1,175 @@ +#!/usr/bin/env perl +# Drive verify-genesis-payload against payload trees that reproduce the three holes the +# released legacy Genesis image shipped with. +use strict; +use warnings; + +use File::Path qw(make_path); +use File::Slurper qw(read_text write_text); +use File::Temp qw(tempdir); +use FindBin; +use lib "$FindBin::Bin/../lib"; +use Test::More; + +use XCAT::Test::File qw(repo_path); + +my $verifier = repo_path('xCAT-genesis-builder/verify-genesis-payload'); +if (!-f $verifier) { + # Skipping here would cover nothing: the Ubuntu Genesis build has no payload gate at + # all until this file exists. + fail('xCAT-genesis-builder/verify-genesis-payload is missing'); + done_testing(); + exit; +} +plan tests => 18; + +my $tmpdir = tempdir(CLEANUP => 1); +my $module_seq = 0; + +# A complete payload: OpenSSH 9.9 sshd plus its session helper, tmux plus a UTF-8 locale. +my $good = build_payload(sshd_execs_session => 1, session_helper => 1, tmux => 1, locale => 1, dhclient => 1, mktemp => 1); +my ($rc, $err) = run($good, 'usr/sbin/dhclient'); +is($rc, 0, 'a complete payload passes') or diag($err); + +# doxcat calls dhclient with ISC flags. The released el9 image carried dhclient.conf and +# dhclient-script but no dhclient, so Genesis never acquired an address. +my $nodhcp = build_payload(sshd_execs_session => 1, session_helper => 1, tmux => 1, locale => 1, dhclient => 0, mktemp => 1); +($rc, $err) = run($nodhcp, 'usr/sbin/dhclient'); +isnt($rc, 0, 'a payload without dhclient fails'); +like($err, qr{usr/sbin/dhclient}, 'the missing dhclient is named'); + +# sshd 9.9 execs /usr/libexec/openssh/sshd-session for every connection. +my $nohelper = build_payload(sshd_execs_session => 1, session_helper => 0, tmux => 1, locale => 1, dhclient => 1, mktemp => 1); +($rc, $err) = run($nohelper, 'usr/sbin/dhclient'); +isnt($rc, 0, 'a payload whose sshd execs sshd-session but does not ship it fails'); +like($err, qr{sshd-session}, 'the missing sshd-session is named'); + +# OpenSSH 8 does not use the helper, so el8 must still pass without it. +my $openssh8 = build_payload(sshd_execs_session => 0, session_helper => 0, tmux => 1, locale => 1, dhclient => 1, mktemp => 1); +($rc, $err) = run($openssh8, 'usr/sbin/dhclient'); +is($rc, 0, 'an OpenSSH 8 payload passes without sshd-session') or diag($err); + +# tmux without a UTF-8 locale is what stopped doxcat from ever running. +my $nolocale = build_payload(sshd_execs_session => 1, session_helper => 1, tmux => 1, locale => 0, dhclient => 1, mktemp => 1); +($rc, $err) = run($nolocale, 'usr/sbin/dhclient'); +isnt($rc, 0, 'a payload with tmux and no UTF-8 locale fails'); +like($err, qr{C\.utf8}, 'the missing locale is named'); + +# getdestiny makes its request file with mktemp. Without it the node never reports its destiny, +# so xcatd never sets nodelist.status and the node stays at powering-on. +my $nomktemp = build_payload(sshd_execs_session => 1, session_helper => 1, tmux => 1, locale => 1, dhclient => 1, mktemp => 0); +($rc, $err) = run($nomktemp, 'usr/sbin/dhclient'); +isnt($rc, 0, 'a payload without mktemp fails'); +like($err, qr{usr/bin/mktemp}, 'the missing mktemp is named'); + +# dracut_install reports a missing binary and returns, so every name the dracut module +# installs has to be checked against the payload. The el10 image shipped with no openssl and +# getcert waited on it for the life of the node. +my $module = write_module_setup([qw(openssl wget tar)]); +my $full = build_payload(sshd_execs_session => 1, session_helper => 1, tmux => 1, locale => 1, + dhclient => 1, mktemp => 1, commands => [qw(openssl wget tar)]); +($rc, $err) = run_with_commands($module, $full); +is($rc, 0, 'a payload carrying every command the module names passes') or diag($err); + +my $noopenssl = build_payload(sshd_execs_session => 1, session_helper => 1, tmux => 1, locale => 1, + dhclient => 1, mktemp => 1, commands => [qw(wget tar)]); +($rc, $err) = run_with_commands($module, $noopenssl); +isnt($rc, 0, 'a payload without openssl fails'); +like($err, qr/openssl/, 'the missing openssl is named'); + +# The DHCP client is release-dependent, so the module installs it inside a conditional. Those +# names are not the contract; the spec passes the one it wants as a required path. +my $conditional = write_module_setup(['wget'], ['dhclient']); +my $nodhclient = build_payload(sshd_execs_session => 1, session_helper => 1, tmux => 1, locale => 1, + dhclient => 0, mktemp => 1, commands => ['wget']); +($rc, $err) = run_with_commands($conditional, $nodhclient); +is($rc, 0, 'a name installed under a condition is not required') or diag($err); + +# A module the verifier cannot read names for covers nothing, so say so instead of passing. +my $unparsable = "$tmpdir/module-setup-unparsable.sh"; +write_text($unparsable, "#!/bin/bash\nsetup() {\n dracut_install wget\n}\n"); +($rc, $err) = run_with_commands($unparsable, $full); +is($rc, 2, 'a module the verifier finds no command names in is a usage error'); +like($err, qr/command name/, 'the empty command list is named'); + +($rc, $err) = run_with_commands("$tmpdir/no-such-module", $full); +is($rc, 2, 'a module file that cannot be read is a usage error'); + +($rc, $err) = run("$tmpdir/does-not-exist"); +is($rc >> 0, 2, 'a missing payload directory is a usage error'); + +#--- +# build_payload: make a payload tree with the pieces the verifier reasons about. +#--- +sub build_payload { + my (%opt) = @_; + my $root = tempdir(DIR => $tmpdir, CLEANUP => 1); + make_path("$root/usr/sbin", "$root/usr/bin", "$root/usr/libexec/openssh"); + write_text("$root/usr/sbin/sshd", + $opt{sshd_execs_session} + ? "OpenSSH_9.9p1\n/usr/libexec/openssh/sshd-session\n" + : "OpenSSH_8.0p1\n"); + write_text("$root/usr/libexec/openssh/sshd-session", "helper\n") if $opt{session_helper}; + write_text("$root/usr/bin/tmux", "tmux\n") if $opt{tmux}; + if ($opt{locale}) { + make_path("$root/usr/lib/locale/C.utf8"); + write_text("$root/usr/lib/locale/C.utf8/LC_CTYPE", "ctype\n"); + } + write_text("$root/usr/sbin/dhclient", "dhclient\n") if $opt{dhclient}; + write_text("$root/usr/bin/mktemp", "mktemp\n") if $opt{mktemp}; + # The module also names two absolute paths, and dracut_install installs an absolute + # path at that same path. They are data files every payload carries. + make_path("$root/etc"); + write_text("$root/usr/bin/awk", "awk\n"); + write_text("$root/etc/services", "services\n"); + write_text("$root/usr/bin/$_", "$_\n") for @{ $opt{commands} || [] }; + return $root; +} + +#--- +# run: run the verifier and return its exit status and stderr. +#--- +sub run { + my ($root, @required) = @_; + my $errfile = "$tmpdir/err.$$"; + my $cmd = join ' ', map { "'$_'" } ($verifier, $root, @required); + system("/bin/bash $cmd >/dev/null 2>$errfile"); + my $status = $? >> 8; + my $err = -f $errfile ? read_text($errfile) : ''; + unlink $errfile; + return ($status, $err); +} + +#--- +# write_module_setup: a dracut module whose install() names commands at the top level, and +# optionally more inside a conditional. +#--- +sub write_module_setup { + my ($top, $conditional) = @_; + my $path = "$tmpdir/module-setup." . ++$module_seq . ".sh"; + my $text = "#!/bin/bash\n\ninstall() {\n"; + $text .= " dracut_install " . join(' ', @$top) . " # a trailing comment\n"; + $text .= " dracut_install /usr/bin/awk /etc/services\n"; + if ($conditional) { + $text .= " if command -v " . $conditional->[0] . " >/dev/null 2>&1; then\n"; + $text .= " dracut_install " . join(' ', @$conditional) . "\n"; + $text .= " fi\n"; + } + $text .= "}\n"; + write_text($path, $text); + return $path; +} + +#--- +# run_with_commands: run the verifier with the command list read back from a dracut module. +#--- +sub run_with_commands { + my ($module, $root) = @_; + my $errfile = "$tmpdir/err.commands.$$"; + my $cmd = join ' ', map { "'$_'" } ($verifier, '--commands-from', $module, $root); + system("/bin/bash $cmd >/dev/null 2>$errfile"); + my $status = $? >> 8; + my $err = -f $errfile ? read_text($errfile) : ''; + unlink $errfile; + return ($status, $err); +} diff --git a/xCAT-test/unit/genesis_ubuntu_build_root.t b/xCAT-test/unit/genesis_ubuntu_build_root.t new file mode 100755 index 0000000000..1d87f7e63d --- /dev/null +++ b/xCAT-test/unit/genesis_ubuntu_build_root.t @@ -0,0 +1,99 @@ +#!/usr/bin/env perl +# The Ubuntu Genesis build root must carry every command the Ubuntu dracut module marks +# mandatory. dracut_install reports a missing command and returns, so a command the build +# root does not supply leaves a hole in the image and the build still exits 0. +# +# The mandatory list is read by RUNNING the module: module-setup.sh is sourced with +# dracut_install shadowed, _dracut_install_opt neutralised (its callers are optional by +# construction), and install() is called. The package list is read by extracting the +# REQUIRED_PACKAGES assignment from builddeb-genesis-base and evaluating it. +use strict; +use warnings; + +use File::Temp qw(tempdir); +use FindBin; +use lib "$FindBin::Bin/../lib"; +use Test::More; + +use XCAT::Test::File qw(repo_path); + +my $builder = repo_path('xCAT-genesis-builder/builddeb-genesis-base'); +my $module = repo_path('xCAT-genesis-builder/dracut_105/ubuntu/module-setup.sh'); +plan skip_all => 'builddeb-genesis-base not found' unless -f $builder; +plan skip_all => 'ubuntu module-setup.sh not found' unless -f $module; +plan tests => 9; + +# Commands the Ubuntu dracut module marks mandatory that a minimal Ubuntu server root does +# NOT already provide, and the package that supplies each one. Every entry here has to be in +# REQUIRED_PACKAGES or the image ships without the command. +my %PACKAGE_FOR = ( + dhclient => 'isc-dhcp-client', + ifenslave => 'ifenslave', + hwclock => 'util-linux-extra', +); + +my %mandatory = map { $_ => 1 } mandatory_commands($module); +my @packages = required_packages($builder); + +for my $command (sort keys %PACKAGE_FOR) { + ok($mandatory{$command}, "the Ubuntu dracut module installs '$command' unconditionally"); + ok(scalar(grep { $_ eq $PACKAGE_FOR{$command} } @packages), + "the build root installs $PACKAGE_FOR{$command}, which provides '$command'"); +} + +# doxcat asks dhclient for the provisioning lease. An image without it never gets an address, +# so the node netboots and never reports in -- the failure this test exists for. +ok($mandatory{dhclient} && scalar(grep { $_ eq 'isc-dhcp-client' } @packages), + 'the Genesis image can obtain a DHCP lease'); + +# dracut_install is silent about a hole, so the payload needs its own gate before it is +# packaged. This is the EL path's behaviour (xCAT-genesis-base.spec runs the same verifier). +my $text = do { open my $fh, '<', $builder or die "$builder: $!"; local $/; <$fh> }; +like($text, qr{verify-genesis-payload}, 'builddeb-genesis-base verifies the payload it packages'); + +# The image belongs to the release whose kernel it carries, so the builder must refuse a +# root of any other release. Without the refusal one build on the build host serves every +# codename with the build host's kernel. +like($text, qr{--expect-codename}, 'builddeb-genesis-base takes the release it is building for'); + +# mandatory_commands($module): source the dracut module with dracut_install shadowed, call +# install(), and return the bare command names it installs unconditionally. Absolute paths are +# data files, not commands, and are left out. +sub mandatory_commands { + my ($path) = @_; + my $dir = tempdir(CLEANUP => 1); + my $driver = "$dir/collect.sh"; + open my $fh, '>', $driver or die "$driver: $!"; + print $fh <<"BASH"; +dracut_install() { printf '%s\\n' "\$\@"; } +instmods() { :; } +inst_multiple() { :; } +inst() { :; } +dpkg-architecture() { echo x86_64-linux-gnu; } +. '$path' +# Every caller of _dracut_install_opt is optional by construction: it installs only what the +# build root already has. Neutralise it AFTER sourcing so it cannot add to the mandatory set. +_dracut_install_opt() { :; } +install +BASH + close $fh; + my @out = qx{bash '$driver' 2>/dev/null}; + BAIL_OUT("running install() from $path produced nothing") unless @out; + my %seen; + my @names = grep { !$seen{$_}++ } grep { length && !m{^/} } map { chomp; $_ } @out; + BAIL_OUT("install() from $path named no bare commands") unless @names; + return @names; +} + +# required_packages($path): extract the REQUIRED_PACKAGES assignment from the build script and +# evaluate it, so the list comes from the value the script actually uses. +sub required_packages { + my ($path) = @_; + my $text = do { open my $fh, '<', $path or die "$path: $!"; local $/; <$fh> }; + my ($block) = $text =~ /^(REQUIRED_PACKAGES="[^"]*")/ms; + BAIL_OUT("no REQUIRED_PACKAGES assignment in $path") unless $block; + my $out = qx{bash -c 'set -u; $block; printf "%s\\n" \$REQUIRED_PACKAGES' 2>/dev/null}; + my @packages = grep { length } split /\s+/, ($out // ''); + BAIL_OUT("REQUIRED_PACKAGES in $path evaluated to nothing") unless @packages; + return @packages; +} From b5bd2e4657c135e7eb0bba5edb6d2850397f8c84 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Sat, 12 Sep 2026 08:27:42 -0300 Subject: [PATCH 02/10] fix(xcat-core): the Ubuntu Genesis image is built from the build host kernel dracut copies the kernel, the kernel modules and every command out of the root it runs in. The Ubuntu Genesis deb was built once, on the build host, so every Ubuntu release got the build host's kernel. Nothing called the builder at all: the pipelines converted the EL rpm with alien instead. builddebs.pl gains --genesis, --genesis-only and --genesis-dist. The Genesis deb is now built once per codename inside that codename's --sbuild chroot, the chroots xcat-dep's sbuild-all.pl already creates on the Ubuntu build host. builddeb-genesis-base takes --expect-codename and stops when the root it woke up in is a different release, so a build on the build host cannot produce a codename's image. builddebs.pl reads the build log through XCAT::BuildUtils::genesis_log_errors and fails the build on FAILED:, a package apt cannot find and four more lines that a zero exit status hides. The extracted payload goes through verify-genesis-payload with the command list read back from the dracut module, plus the DHCP client, the 97xcat hooks and a /lib/modules that holds this chroot's kernel and no other. The build root gains isc-dhcp-client, ifenslave and util-linux-extra, which supply dhclient, ifenslave and hwclock. genesis_deb_per_codename.t, genesis_payload_verification.t and genesis_ubuntu_build_root.t fail without this change. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- build-utils/lib/XCAT/BuildUtils.pm | 96 +++++++++ builddebs.pl | 202 ++++++++++++++++-- xCAT-genesis-builder/builddeb-genesis-base | 123 +++++++++-- .../dracut_105/ubuntu/module-setup.sh | 19 ++ .../dracut_105/ubuntu/xcat-cmdline.sh | 31 ++- xCAT-genesis-builder/verify-genesis-payload | 122 +++++++++++ 6 files changed, 561 insertions(+), 32 deletions(-) create mode 100755 xCAT-genesis-builder/verify-genesis-payload diff --git a/build-utils/lib/XCAT/BuildUtils.pm b/build-utils/lib/XCAT/BuildUtils.pm index 54df11487c..2efb89cc0d 100644 --- a/build-utils/lib/XCAT/BuildUtils.pm +++ b/build-utils/lib/XCAT/BuildUtils.pm @@ -36,6 +36,8 @@ our @EXPORT_OK = qw( rewrite_file write_script read_line buildinfo_text targetarch_from_target + genesis_chroot_name genesis_target_arch genesis_build_plan + genesis_log_errors genesis_log_deny_rules ); # Both builders echo the commands they run under --verbose. Set once, after @@ -491,4 +493,98 @@ sub targetarch_from_target { return $parts[-1]; } +# ------------------------------------------------------------------ Genesis -- +# +# The Genesis image carries the kernel and the kernel modules of the release it boots, +# because dracut copies them out of the root it runs in. So the Ubuntu Genesis deb is built +# once per Ubuntu codename, inside that codename's chroot. A single build on the build host +# gives every codename the build host's kernel. + +# genesis_chroot_name: the schroot chroot that builds the Genesis deb for one codename. +# +# Same name xcat-dep's sbuild-all.pl ensure_chroots creates, so both repositories build in +# the same chroots and neither has to bootstrap a second set. +sub genesis_chroot_name { + my ($codename, $arch) = @_; + die "genesis_chroot_name: a codename is required\n" + unless defined $codename && length $codename; + die "genesis_chroot_name: an architecture is required\n" + unless defined $arch && length $arch; + return "$codename-$arch-sbuild"; +} + +# genesis_target_arch: the directory xCAT reads the Genesis image from, for a deb +# architecture. mknb reads /opt/xcat/share/xcat/netboot/genesis/, and that name is +# the rpm architecture, not the deb one. +my %GENESIS_TARGET_ARCH = ( + amd64 => 'x86_64', + ppc64el => 'ppc64', +); + +sub genesis_target_arch { + my ($arch) = @_; + my $target = $GENESIS_TARGET_ARCH{ $arch // '' }; + die "genesis_target_arch: no Genesis image directory for '" . ($arch // '') . "'\n" + unless $target; + return $target; +} + +# genesis_build_plan: one Genesis build per codename, for one architecture. +# +# The set of codenames comes from the caller, so a pipeline builds exactly the releases it +# publishes. Returns the codename, the chroot to build it in and the package the build +# produces, which is what the caller needs to run and to collect. +sub genesis_build_plan { + my ($dists, $arch) = @_; + my @dists = @{ $dists || [] }; + die "genesis_build_plan: at least one codename is required\n" unless @dists; + die "genesis_build_plan: an architecture is required\n" + unless defined $arch && length $arch; + + my %seen; + return map { + { + codename => $_, + arch => $arch, + chroot => genesis_chroot_name($_, $arch), + package => "xcat-genesis-base-$arch", + target => genesis_target_arch($arch), + } + } grep { !$seen{$_}++ } @dists; +} + +# genesis_log_deny_rules / genesis_log_errors: what a Genesis build log says when the build +# failed but the exit status did not. +# +# dracut reports a command it cannot install with a FAILED: line and returns 0, and the +# builder never reads dracut's result. That is how an image with no dhclient was packaged, +# signed and published by a command that reported success. apt has the same shape: a +# missing package leaves a diagnostic and a zero status behind a `|| true`. So the log is +# the gate, not the exit status. +my @GENESIS_LOG_DENY = ( + [ qr/\bFAILED:/ => 'dracut could not install a command' ], + [ qr/Cannot find module/ => 'a kernel module the build names is absent' ], + [ qr/dracut: Cannot/ => 'dracut refused the request' ], + [ qr/command not found/ => 'the build root has no such command' ], + [ qr/E: Unable to locate package/ => 'apt has no such package' ], + [ qr/Unable to correct problems/ => 'apt could not resolve the build root' ], +); + +sub genesis_log_deny_rules { return @GENESIS_LOG_DENY; } + +sub genesis_log_errors { + my ($text) = @_; + return () unless defined $text && length $text; + my @found; + for my $line (split /\n/, $text) { + for my $rule (@GENESIS_LOG_DENY) { + my ($pattern, $why) = @{$rule}; + next unless $line =~ $pattern; + push @found, { line => $line, why => $why }; + last; + } + } + return @found; +} + 1; diff --git a/builddebs.pl b/builddebs.pl index 873936b44c..685910be5a 100755 --- a/builddebs.pl +++ b/builddebs.pl @@ -8,9 +8,12 @@ # The central fact this design rests on: xcat-core debs are Perl. They are byte-identical # for every Ubuntu release, so they are built ONCE and the same files are published into # every codename. Only xCAT, xCATsn and xCAT-genesis-scripts carry an architecture, and -# even there the difference is packaging metadata, not compiled output. That is why this -# needs no sbuild and no per-codename chroot -- unlike xcat-dep, whose compiled packages -# genuinely differ per release. +# even there the difference is packaging metadata, not compiled output. +# +# The Genesis image is the one exception, and it is why --genesis exists. dracut copies the +# kernel, the kernel modules and every command out of the root it runs in, so that image +# genuinely differs per release and is built once per codename inside that codename's +# sbuild chroot -- the same chroots xcat-dep builds its compiled packages in. use strict; use warnings; use feature 'say'; @@ -38,6 +41,7 @@ reprepro_distributions reprepro_options lock_id_for take_build_lock sh_quote sh sh_or_die usage rewrite_file write_script read_line buildinfo_text + genesis_build_plan genesis_log_errors ); # The xcat-core packages that ship as debs. xCAT-openbmc-py, xCAT-rmc and xCAT-release @@ -61,10 +65,14 @@ my @DISTS = default_dists(); my %opts; -my (@cli_packages, @cli_dists); +my (@cli_packages, @cli_dists, @cli_genesis_dists); GetOptions( "dist=s@" => \@cli_dists, "package=s@" => \@cli_packages, + "genesis" => \$opts{genesis}, + "genesis-only" => \$opts{genesis_only}, + "genesis-dist=s@" => \@cli_genesis_dists, + "genesis-arch=s" => \$opts{genesis_arch}, "dest=s" => \$opts{dest}, "builddir=s" => \$opts{builddir}, "release=s" => \$opts{release}, @@ -83,6 +91,14 @@ $opts{dists} = @cli_dists ? \@cli_dists : \@DISTS; $opts{gpg_key_name} //= 'xCAT Signing Key'; +# The Genesis step is off unless it is asked for, so every run that exists today keeps its +# behaviour. It takes its own codename list: the Genesis deb is the one package that is not +# the same file for every release, so the caller says which releases it wants built. +$opts{genesis} = 1 if $opts{genesis_only}; +$opts{genesis_dists} = @cli_genesis_dists ? \@cli_genesis_dists : $opts{dists}; +die "FATAL: --genesis-dist needs --genesis\n" if @cli_genesis_dists && !$opts{genesis}; +die "FATAL: --genesis-arch needs --genesis\n" if $opts{genesis_arch} && !$opts{genesis}; + for my $pkg ($opts{packages}->@*) { die "FATAL: unknown package '$pkg'. Known: @PACKAGES\n" unless grep { $_ eq $pkg } @PACKAGES; @@ -265,6 +281,130 @@ sub collect_debs { return $moved; } +# ----------------------------------------------------------- the Genesis deb -- +# +# Every other xcat-core deb is Perl and is built once for every release. The Genesis image +# is not: dracut copies the kernel, the kernel modules and every command out of the root it +# runs in. Built on the build host, one image serves every codename with the build host's +# kernel -- which is how Ubuntu management nodes came to install an image built from an EL +# kernel. So this step builds one image per codename, inside that codename's sbuild chroot. +# +# The chroots are the ones xcat-dep's sbuild-all.pl creates on the Ubuntu build host +# (--sbuild). They hand out disposable overlay sessions, so what the build +# installs is discarded and the next codename starts from the pristine base. + +sub host_deb_arch { + my $arch = `dpkg --print-architecture 2>/dev/null` // ''; + chomp $arch; + die "FATAL: dpkg does not report a host architecture\n" unless $arch; + return $arch; +} + +# begin_chroot_session: start a disposable schroot session and return its id and its root. +sub begin_chroot_session { + my ($chroot) = @_; + my $id = `schroot --begin-session --chroot @{[ sh_quote($chroot) ]} 2>&1` // ''; + my $rc = $? >> 8; + chomp $id; + die "FATAL: cannot start a session in chroot '$chroot' (exit $rc): $id\n" + . " sbuild-all.pl ensure_chroots creates it; run it on this host first.\n" + if $rc != 0 || $id !~ /\A\S+\z/; + + my $root = `schroot --location -c @{[ sh_quote("session:$id") ]} 2>/dev/null` // ''; + chomp $root; + unless ($root && -d $root) { + sh("schroot --end-session -c " . sh_quote("session:$id") . " >/dev/null 2>&1"); + die "FATAL: schroot reports no location for session:$id\n"; + } + return ($id, $root); +} + +# genesis_build_log_problems: what the log says went wrong when the exit status did not. +# +# dracut prints FAILED: for a command it cannot install and exits 0. Reporting the first few +# offending lines rather than all of them keeps a build console readable; the log file has +# the rest and is named in the message. +sub genesis_build_log_problems { + my ($logfile) = @_; + my $text = -f $logfile ? read_text($logfile) : ''; + my @errors = genesis_log_errors($text); + return '' unless @errors; + my $shown = @errors > 10 ? 10 : scalar @errors; + my $report = "FATAL: the Genesis build log reports " . scalar(@errors) . " error(s):\n"; + $report .= " $_->{line}\n ($_->{why})\n" for @errors[0 .. $shown - 1]; + $report .= " ... " . (@errors - $shown) . " more\n" if @errors > $shown; + $report .= " the whole log is at $logfile\n"; + return $report; +} + +sub build_one_genesis_deb { + my ($step, $pkgdir) = @_; + my ($codename, $chroot) = ($step->{codename}, $step->{chroot}); + say "Building $step->{package} for $codename in $chroot"; + + my ($id, $root) = begin_chroot_session($chroot); + my $logfile = "$pkgdir/$step->{package}-$codename.buildlog"; + my $err; + + eval { + # The builder needs its own directory, and Version and Release beside it. Copy them + # in rather than bind-mount the checkout: the build rewrites debian/control and + # debian/changelog, and it must not rewrite them in the tree the pipeline builds from. + sh_or_die("mkdir -p " . sh_quote("$root/build/xCAT-genesis-builder"), + "FATAL: cannot make the build directory in session:$id\n"); + sh_or_die("cp -a " . sh_quote("$ROOT/xCAT-genesis-builder") . "/. " + . sh_quote("$root/build/xCAT-genesis-builder") . "/", + "FATAL: cannot copy xCAT-genesis-builder into session:$id\n"); + for my $f (qw(Version Release)) { + next unless -f "$ROOT/$f"; + copy("$ROOT/$f", "$root/build/$f") + or die "FATAL: cannot copy $f into session:$id: $!\n"; + } + write_text("$root/build/Release", "$RELEASE\n"); + + # --expect-codename is the guard that keeps the image and the root together: the + # builder stops when the root it woke up in is not the release it was asked for. + my $cmd = join ' ', + 'schroot', '--run-session', '-c', sh_quote("session:$id"), '-u', 'root', '-d', '/', + '--', '/bin/bash', '/build/xCAT-genesis-builder/builddeb-genesis-base', + '--expect-codename', sh_quote($codename), '--outdir', '/build/out'; + my $rc = sh("$cmd > " . sh_quote($logfile) . " 2>&1"); + + # The log is read whether or not the command failed: a build that exits 0 with + # FAILED: lines in its log is the failure this gate exists for. + my $problems = genesis_build_log_problems($logfile); + if ($rc != 0) { + die "FATAL: the Genesis build for $codename failed (exit $rc); log: $logfile\n" + . $problems; + } + die $problems if $problems; + + my @debs = glob("$root/build/out/*.deb"); + die "FATAL: the Genesis build for $codename produced no .deb; log: $logfile\n" + unless @debs; + for my $deb (@debs) { + my $dest = "$pkgdir/" . basename($deb); + copy($deb, $dest) or die "FATAL: cannot collect $deb: $!\n"; + say " $dest"; + } + 1; + } or $err = $@; + + sh("schroot --end-session -c " . sh_quote("session:$id") . " >/dev/null 2>&1"); + die $err if $err; + return; +} + +sub build_genesis_debs { + my ($pkgdir) = @_; + my $arch = $opts{genesis_arch} || host_deb_arch(); + my @plan = genesis_build_plan($opts{genesis_dists}, $arch); + say "Genesis: @{[ scalar @plan ]} build(s) for $arch: " + . join(' ', map { $_->{codename} } @plan); + build_one_genesis_deb($_, $pkgdir) for @plan; + return scalar @plan; +} + # ------------------------------------------------------------- apt assembly -- sub gpg_key_id { my ($name) = @_; @@ -352,16 +492,25 @@ sub write_repo_metadata { say "xcat-core $PKGVER -> $dest"; say "releases: @{[ join ' ', $opts{dists}->@* ]}"; -for my $pkg ($opts{packages}->@*) { - for my $arch (deb_package_arches($pkg)) { - build_package($pkg, $arch, $pkgdir); +unless ($opts{genesis_only}) { + for my $pkg ($opts{packages}->@*) { + for my $arch (deb_package_arches($pkg)) { + build_package($pkg, $arch, $pkgdir); + } + collect_debs($pkg, $pkgdir); } - collect_debs($pkg, $pkgdir); } -my $count = assemble_repo($pkgdir, $repo); -write_repo_metadata($repo); -say "published $count package(s) into @{[ scalar $opts{dists}->@* ]} release(s) at $repo"; +build_genesis_debs($pkgdir) if $opts{genesis}; + +if ($opts{genesis_only}) { + say "Genesis debs are in $pkgdir"; +} +else { + my $count = assemble_repo($pkgdir, $repo); + write_repo_metadata($repo); + say "published $count package(s) into @{[ scalar $opts{dists}->@* ]} release(s) at $repo"; +} __END__ @@ -382,8 +531,15 @@ =head1 DESCRIPTION package is built B and the resulting C<.deb> files are published into every codename the repository declares. Only C, C and C carry an architecture, and there the difference is packaging metadata rather than -compiled output. Consequently this builder needs no C and no per-codename -chroot. (xcat-dep is different: its packages are compiled, so it builds per codename.) +compiled output. + +C is the exception. dracut copies the kernel, the kernel modules +and every command out of the root it runs in, so the Genesis image belongs to the +release that built it. With C<--genesis> this builder makes one image per codename, +each inside that codename's C<< --sbuild >> schroot -- the chroots +xcat-dep's C creates on the Ubuntu build host. The build refuses to run +in a root of another release, and it reads its own log: dracut reports a command it +cannot install with a C line and still exits 0. Replaces C. The GSA upload paths, the C/C release flows and the C<-d> xcat-dep repository mode were not carried over: publishing is done @@ -401,6 +557,25 @@ =head1 OPTIONS Build only this package. Repeatable. Defaults to every xcat-core deb package. +=item B<--genesis> + +Also build CarchE>, one C<.deb> per codename, each inside +that codename's schroot. Off by default. + +=item B<--genesis-only> + +Build the Genesis debs and nothing else, and assemble no repository. This is what +xcat-dep needs: it consumes the debs with C. + +=item B<--genesis-dist>=I + +Build the Genesis image for this release. Repeatable. Defaults to the C<--dist> list. + +=item B<--genesis-arch>=I + +Build the Genesis image for this Debian architecture. Defaults to the architecture of +the build host, because the chroot has to match it. + =item B<--dest>=I Write the build under this directory: packages in C, the apt repository in @@ -442,5 +617,6 @@ =head1 EXAMPLES ./builddebs.pl ./builddebs.pl --dist noble --package perl-xCAT ./builddebs.pl --dest /srv/out --gpg-sign --gpg-home /keys/xcat-gpg-home + ./builddebs.pl --genesis-only --genesis-dist jammy --genesis-dist noble --dest /srv/out =cut diff --git a/xCAT-genesis-builder/builddeb-genesis-base b/xCAT-genesis-builder/builddeb-genesis-base index 8a687a60a6..a6af7e9b1a 100755 --- a/xCAT-genesis-builder/builddeb-genesis-base +++ b/xCAT-genesis-builder/builddeb-genesis-base @@ -1,13 +1,36 @@ #!/bin/bash -# Build xcat-genesis-base .deb package natively on Ubuntu. -# Must run as root on an Ubuntu system (22.04, 24.04, or 26.04). -# Parallel to buildrpm for EL targets. +# Build the xcat-genesis-base .deb for ONE Ubuntu codename. +# +# dracut copies the kernel, the kernel modules and every command out of the root it runs +# in, so this must run inside a root of the target codename. builddebs.pl --genesis starts +# it in that codename's --sbuild chroot, one build per codename. +# --expect-codename is what stops a run on the build host: a single build there gives every +# Ubuntu release the build host's kernel, which is how the EL-built image reached Ubuntu +# management nodes in the first place. +# +# Parallel to xCAT-genesis-base.spec, which does the same for EL. set -euo pipefail DIR=$(readlink -f "$(dirname "$0")") +OS_RELEASE=${OS_RELEASE:-/etc/os-release} +expect_codename="" +outdir="" + +while [ $# -gt 0 ]; do + case "$1" in + --expect-codename) expect_codename=${2:-}; shift 2 ;; + --expect-codename=*) expect_codename=${1#*=}; shift ;; + --outdir) outdir=${2:-}; shift 2 ;; + --outdir=*) outdir=${1#*=}; shift ;; + -h|--help) + echo "usage: builddeb-genesis-base [--expect-codename ] [--outdir ]" + exit 0 ;; + *) echo "ERROR: unknown argument: $1" >&2; exit 2 ;; + esac +done + BUILDARCH=$(dpkg --print-architecture) -TRIPLET=$(dpkg-architecture -qDEB_HOST_MULTIARCH) case "$BUILDARCH" in amd64) TARCH=x86_64 ;; @@ -15,9 +38,36 @@ case "$BUILDARCH" in *) echo "ERROR: unsupported architecture: $BUILDARCH" >&2; exit 1 ;; esac +# The Genesis debs carry the architecture in the package name, so the packages an upgrade has +# to displace carry it too. 2.19 renames the ppc64 debs to ppc64el; dpkg keeps the old package, +# and its copy of the same files, unless the new one replaces it by name. +rewrite_control() { + local control=$1 arch=$2 superseded + case "$arch" in + ppc64el) superseded="xcat-genesis-ppc64, xcat-genesis-base-ppc64" ;; + *) superseded="xcat-genesis-$arch" ;; + esac + sed -i -e "s/xcat-genesis-base-amd64/xcat-genesis-base-$arch/g" \ + -e "s/xcat-genesis-scripts-amd64/xcat-genesis-scripts-$arch/g" \ + -e "s/xcat-genesis-amd64/$superseded/g" "$control" +} + VERSION=$(cat "$DIR/../Version" 2>/dev/null || echo "2.18.0") RELEASE=$(cat "$DIR/../Release" 2>/dev/null || echo "snap$(date +%Y%m%d%H%M)") -CODENAME=$(. /etc/os-release && echo "$VERSION_CODENAME") +CODENAME=$(. "$OS_RELEASE" && echo "${VERSION_CODENAME:-}") + +if [ -z "$CODENAME" ]; then + echo "ERROR: $OS_RELEASE names no VERSION_CODENAME" >&2 + exit 1 +fi + +# The image is only valid for the release whose kernel it carries. Refuse to build a +# codename's image anywhere but in that codename's root. +if [ -n "$expect_codename" ] && [ "$expect_codename" != "$CODENAME" ]; then + echo "ERROR: this root is $CODENAME, not $expect_codename." >&2 + echo " The image would carry the $CODENAME kernel and boot under $expect_codename." >&2 + exit 1 +fi echo "Building xcat-genesis-base for $BUILDARCH ($TARCH) on Ubuntu $CODENAME" @@ -30,6 +80,7 @@ REQUIRED_PACKAGES=" nfs-common rpcbind pciutils usbutils parted dosfstools e2fsprogs lvm2 mdadm net-tools bc psmisc rsync wget cpio + isc-dhcp-client ifenslave util-linux-extra dpkg-dev debhelper fakeroot devscripts vim-tiny " if [ "$BUILDARCH" = "amd64" ]; then @@ -40,6 +91,9 @@ echo "Installing build dependencies..." apt-get update -qq apt-get install -y --no-install-recommends $REQUIRED_PACKAGES +# dpkg-architecture comes from dpkg-dev, which the line above installs. +TRIPLET=$(dpkg-architecture -qDEB_HOST_MULTIARCH) + # Set up dracut module if [ -d /usr/lib/dracut/modules.d ]; then DRACUT_PARENT=/usr/lib/dracut/modules.d @@ -71,6 +125,8 @@ if [ "$BUILDARCH" != "amd64" ]; then sed -i '/efibootmgr dmidecode/d' "$DRACUTMODDIR/module-setup.sh" fi +# linux-image-generic pulls exactly one kernel into this root, and this root belongs to +# $CODENAME, so this is the target release's kernel. KERNELVERSION=$(ls -1 /lib/modules | sort -V | tail -n 1) if [ -z "$KERNELVERSION" ]; then echo "ERROR: no kernel modules found in /lib/modules" >&2 @@ -123,24 +179,53 @@ if [ ! -e "$KERNEL_IMAGE" ]; then for candidate in \ "/boot/vmlinux-$KERNELVERSION" \ "/usr/lib/modules/$KERNELVERSION/vmlinuz" \ - "/lib/modules/$KERNELVERSION/vmlinuz" \ - "$(find /usr/lib/modules/"$KERNELVERSION" -maxdepth 2 -name 'vmlinuz*' -o -name 'vmlinux*' 2>/dev/null | head -n 1)" \ - "$(find /lib/modules/"$KERNELVERSION" -maxdepth 2 -name 'vmlinuz*' -o -name 'vmlinux*' 2>/dev/null | head -n 1)" \ - "$(ls -1 /boot/vmlinuz-* /boot/vmlinux-* 2>/dev/null | sort -V | tail -n 1)" + "/lib/modules/$KERNELVERSION/vmlinuz" do - if [ -n "$candidate" ] && [ -e "$candidate" ]; then + if [ -e "$candidate" ]; then KERNEL_IMAGE="$candidate" break fi done fi if [ ! -e "$KERNEL_IMAGE" ]; then - echo "ERROR: cannot find kernel image" >&2 + echo "ERROR: cannot find the image of kernel $KERNELVERSION" >&2 exit 1 fi echo "Adding kernel $KERNEL_IMAGE" cp "$KERNEL_IMAGE" "$GENESIS_ROOT/kernel" +# dracut_install reports a missing command and returns, so a hole reaches the .deb with +# nothing in the log but one line. Read the commands back from the module and check them +# against the payload, the way xCAT-genesis-base.spec does for EL. +bash "$DIR/verify-genesis-payload" --commands-from "$DRACUTMODDIR/module-setup.sh" "$GENESIS_FS" \ + usr/sbin/dhclient bin/sh sbin/xcatroot sbin/dhclient-script etc/rsyslog.conf + +# What the verifier cannot know: that this image belongs to this chroot's kernel. An image +# built against another root's /lib/modules boots and then finds no driver for its NIC. +if [ ! -d "$GENESIS_FS/lib/modules/$KERNELVERSION" ]; then + echo "ERROR: the image carries no /lib/modules/$KERNELVERSION" >&2 + ls -1 "$GENESIS_FS/lib/modules" 2>/dev/null >&2 || true + exit 1 +fi +if [ -z "$(find "$GENESIS_FS/lib/modules/$KERNELVERSION" -name '*.ko*' -print -quit)" ]; then + echo "ERROR: /lib/modules/$KERNELVERSION in the image holds no kernel module" >&2 + exit 1 +fi +for stray in "$GENESIS_FS"/lib/modules/*; do + [ -d "$stray" ] || continue + if [ "$(basename "$stray")" != "$KERNELVERSION" ]; then + echo "ERROR: the image carries $(basename "$stray"), not the $KERNELVERSION of this root" >&2 + exit 1 + fi +done + +# The 97xcat hooks are what makes the image xCAT's. dracut drops a module whose check() +# fails without a word, and the image then boots to a plain dracut shell. +if [ -z "$(find "$GENESIS_FS" -path '*/hooks/cmdline/*xcat-cmdline.sh' -print -quit)" ]; then + echo "ERROR: the image carries no 97xcat cmdline hook" >&2 + exit 1 +fi + find "$GENESIS_TMPDIR" -type c -delete # Stage for dpkg-buildpackage @@ -148,8 +233,9 @@ rm -rf "$DIR/opt" cp -a "$GENESIS_TMPDIR/opt" "$DIR/" # Adjust control file for target arch -sed -i "s/xcat-genesis-base-amd64/xcat-genesis-base-$BUILDARCH/g" "$DIR/debian/control" -sed -i "s/xcat-genesis-scripts-amd64/xcat-genesis-scripts-$BUILDARCH/g" "$DIR/debian/control" +rewrite_control "$DIR/debian/control" "$BUILDARCH" +# debian/dirs names the image directory, which is the rpm architecture. +echo "/opt/xcat/share/xcat/netboot/genesis/$TARCH/" > "$DIR/debian/dirs" PKG_VERSION="${VERSION}-${RELEASE}~${CODENAME}" rm -f "$DIR/debian/changelog" @@ -162,5 +248,12 @@ echo "Building .deb package..." cd "$DIR" dpkg-buildpackage -rfakeroot -uc -us -b -echo "Build complete. .deb files:" -ls -la "$DIR/../"xcat-genesis-base*.deb 2>/dev/null || echo "Check parent directory for .deb files" +if [ -n "$outdir" ]; then + mkdir -p "$outdir" + mv "$DIR/../"xcat-genesis-base-*_*.deb "$outdir/" + echo "Build complete. .deb files in $outdir:" + ls -la "$outdir" +else + echo "Build complete. .deb files:" + ls -la "$DIR/../"xcat-genesis-base*.deb 2>/dev/null || echo "Check parent directory for .deb files" +fi diff --git a/xCAT-genesis-builder/dracut_105/ubuntu/module-setup.sh b/xCAT-genesis-builder/dracut_105/ubuntu/module-setup.sh index ba53252479..96c608dc91 100755 --- a/xCAT-genesis-builder/dracut_105/ubuntu/module-setup.sh +++ b/xCAT-genesis-builder/dracut_105/ubuntu/module-setup.sh @@ -53,7 +53,26 @@ install() { dracut_install mount.nfs sshd vi reboot lspci parted screen mkfs mkfs.ext4 mkfs.btrfs #dracut_install libvirtd /usr/share/libvirt/cpu_map.xml /usr/bin/qemu-img /usr/libexec/qemu-kvm dracut_install mkswap df ifenslave ssh-keygen scp clear + # getdestiny makes its request file with mktemp. Without it the node reports no + # destiny, so xcatd never moves nodelist.status past powering-on. + dracut_install mktemp dracut_install dhclient lldpad + + # OpenSSH 9.8 moved the per-connection work into sshd-session, which sshd execs by + # absolute path. Without it every connection to Genesis is refused. + for _sshd_helper in \ + /usr/libexec/openssh/sshd-session \ + /usr/libexec/openssh/sshd-auth \ + /usr/lib/openssh/sshd-session \ + /usr/lib/openssh/sshd-auth + do + _dracut_install_opt "$_sshd_helper" + done + + # tmux exits under the C locale, and the image carries no locale data of its own. + for _lc_file in /usr/lib/locale/C.utf8/LC_*; do + _dracut_install_opt "$_lc_file" + done _dracut_install_opt "/lib/$TRIPLET/libnss_dns.so.2" dracut_install poweroff hwclock date /usr/share/terminfo/x/xterm /usr/share/terminfo/s/screen /etc/nsswitch.conf /etc/services dracut_install /usr/sbin/rsyslogd /etc/protocols umount /usr/bin/dpkg diff --git a/xCAT-genesis-builder/dracut_105/ubuntu/xcat-cmdline.sh b/xCAT-genesis-builder/dracut_105/ubuntu/xcat-cmdline.sh index b6e3a0ce50..c54ec1df37 100755 --- a/xCAT-genesis-builder/dracut_105/ubuntu/xcat-cmdline.sh +++ b/xCAT-genesis-builder/dracut_105/ubuntu/xcat-cmdline.sh @@ -2,13 +2,29 @@ root=1 rootok=1 netroot=xcat + +# The image ships the C.UTF-8 locale only. tmux refuses to start under the C locale. +export LC_ALL=C.UTF-8 + +# screen exits when the image carries no usable terminal. doxcat is the whole of Genesis, so +# it must run whether or not the multiplexer starts. Prints screen or direct. +xcat_console_mode() { + if screen -ln -d -m -S xcatprobe true >/dev/null 2>&1; then + screen -S xcatprobe -X quit >/dev/null 2>&1 + echo screen + else + echo direct + fi +} clear echo PS1="'"'[xCAT Genesis running on \H \w]\$ '"'" > /.bashrc echo PS1="'"'[xCAT Genesis running on \H \w]\$ '"'" > /.bash_profile mkdir -p /etc/ssh mkdir -p /var/tmp/ mkdir -p /var/empty/sshd -sed -i '/^root:x/d' /etc/passwd +# dracut writes this entry itself, with an empty password field unless the image is +# built --hostonly. Match the user name only. +sed -i '/^root:/d' /etc/passwd cat >>/etc/passwd <<"__ENDL" root:x:0:0::/:/bin/bash sshd:x:30:30:SSH User:/var/empty/sshd:/sbin/nologin @@ -39,10 +55,13 @@ mkdir -p /var/lib/dhclient/ mkdir -p /var/log ip link set lo up echo '127.0.0.1 localhost' >> /etc/hosts -if grep -q console=ttyS /proc/cmdline; then +XCAT_CONSOLE_MODE="$(xcat_console_mode)" +if [ "$XCAT_CONSOLE_MODE" = "screen" ]; then + if grep -q console=ttyS /proc/cmdline; then while :; do sleep 1; screen -S console -ln screen -x doxcat /dev/tty1; clear &>/dev/tty1 ; done & + fi + while :; do screen -ln < /dev/tty2 &> /dev/tty2 ; done & fi -while :; do screen -ln < /dev/tty2 &> /dev/tty2 ; done & # The section below is just for System P LE hardware discovery @@ -87,4 +106,8 @@ elif [[ ${ARCH} =~ x86_64 ]]; then done fi -while :; do screen -dr doxcat || screen -S doxcat -L -ln doxcat; done +if [ "$XCAT_CONSOLE_MODE" = "screen" ]; then + while :; do screen -dr doxcat || screen -S doxcat -L -ln doxcat; done +else + while :; do doxcat; sleep 5; done +fi diff --git a/xCAT-genesis-builder/verify-genesis-payload b/xCAT-genesis-builder/verify-genesis-payload new file mode 100755 index 0000000000..3765e27e24 --- /dev/null +++ b/xCAT-genesis-builder/verify-genesis-payload @@ -0,0 +1,122 @@ +#!/bin/bash +# +# verify-genesis-payload [--commands-from ] [required-path ...] +# +# dracut_install() reports a missing binary and returns, so the module install function keeps +# going and the image ships without it. Four such holes reached a release: no dhclient, no +# openssl, no sshd-session and no UTF-8 locale. Check the extracted payload before it becomes +# an rpm. +# +# Paths given on the command line are relative to . --commands-from reads back +# what the dracut module installs: a bare command name is looked for in the four binary +# directories, an absolute path under itself. The caller adds what only it +# knows (the DHCP client is not the same package on every release); the rules below come from +# the payload itself. + +set -u + +commands_from="" +while [ $# -gt 0 ]; do + case "$1" in + --commands-from) + commands_from=${2:-} + shift 2 || true + ;; + --commands-from=*) + commands_from=${1#*=} + shift + ;; + *) + break + ;; + esac +done + +payload=${1:-} +if [ -z "$payload" ] || [ ! -d "$payload" ]; then + echo "verify-genesis-payload: not a payload directory: ${payload:-}" >&2 + exit 2 +fi +shift + +missing="" + +# have PATH: true when the payload carries PATH as a file, following the usr-merge symlinks +# the image ships (/sbin -> usr/sbin). +have() { + [ -e "$payload/$1" ] +} + +require() { + local path=$1 why=$2 + have "$path" || missing="$missing + $path ($why)" +} + +for path in "$@"; do + require "$path" "required by the build" +done + +# The dracut module names every command and every data file Genesis needs. A name the build +# root does not supply installs nothing and says nothing, so read the names back and check +# each one. Names under a condition are release-dependent, so only the top level of install() +# counts. +if [ -n "$commands_from" ]; then + if [ ! -r "$commands_from" ]; then + echo "verify-genesis-payload: cannot read $commands_from" >&2 + exit 2 + fi + commands=$(awk ' + /^install\(\)/ { in_install = 1; next } + in_install && /^}/ { in_install = 0 } + in_install && /^ dracut_install / { + sub(/#.*/, "") + sub(/^ dracut_install /, "") + print + }' "$commands_from" | tr ' \t' '\n\n' | grep -v '^$' | grep -v '^-' | sort -u) + if [ -z "$commands" ]; then + echo "verify-genesis-payload: no command name read from $commands_from" >&2 + exit 2 + fi + for want in $commands; do + case "$want" in + # dracut_install installs an absolute path at that same path, so read it back + # under the payload root. Dropping these let an image with no /usr/bin/awk pass. + /*) have "${want#/}" || missing="$missing + $want (installed by $commands_from)" + ;; + *) have "bin/$want" || have "sbin/$want" \ + || have "usr/bin/$want" || have "usr/sbin/$want" \ + || missing="$missing + $want (installed by $commands_from)" + ;; + esac + done +fi + +require usr/sbin/sshd "Genesis is reached over ssh" +require usr/bin/mktemp "getdestiny makes its request file with it" + +# OpenSSH 9.8 split the per-connection work into sshd-session, which sshd execs by absolute +# path. EL9 carries OpenSSH 9.9, so an image with sshd alone refuses every connection. +if have usr/sbin/sshd && grep -qa 'sshd-session' "$payload/usr/sbin/sshd" 2>/dev/null; then + if ! have usr/libexec/openssh/sshd-session && ! have usr/lib/openssh/sshd-session; then + missing="$missing + usr/libexec/openssh/sshd-session (this sshd execs it for every connection)" + fi +fi + +# tmux exits under the C locale. The hook falls back to running doxcat directly, so this is +# not fatal to booting, but a Genesis shell without tmux loses the console attach. +if have usr/bin/tmux && ! have usr/lib/locale/C.utf8/LC_CTYPE; then + missing="$missing + usr/lib/locale/C.utf8/LC_CTYPE (tmux refuses to start without a UTF-8 locale)" +fi + +if [ -n "$missing" ]; then + echo "verify-genesis-payload: $payload is incomplete:$missing" >&2 + exit 1 +fi + +echo "verify-genesis-payload: $payload is complete" +exit 0 From 4bbed9e184b0b4fa05d0dd92157fdb9fc1834dca Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Sat, 12 Sep 2026 08:56:11 -0300 Subject: [PATCH 03/10] fix(xcat-core): the Genesis build root is missing nine commands the image needs The first native build stopped on the payload lint. The Ubuntu dracut module installs ping, nc, nslookup, sfdisk, mkfs.btrfs, usb.ids, poweroff, reboot and shutdown unconditionally, and no package in the build root supplies any of them. dracut reported each one with a FAILED: line and exited 0, which is how the image that shipped without dhclient was packaged. Add iputils-ping, netcat-openbsd, fdisk, btrfs-progs, hwdata and systemd-sysv. Two names change between releases. util-linux-extra appeared in 23.04, so the jammy build stopped with "E: Unable to locate package util-linux-extra" where hwclock is still in util-linux; bind9-dnsutils replaced dnsutils in 22.04. add_first_available installs the first name apt knows and stops the build when a release carries none of them. dch reads debian/control from the working directory, not from the file it writes, so the build directory has to be the working directory before it runs. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- xCAT-genesis-builder/builddeb-genesis-base | 25 ++++++++++++++++-- xCAT-test/unit/genesis_ubuntu_build_root.t | 30 ++++++++++++++-------- 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/xCAT-genesis-builder/builddeb-genesis-base b/xCAT-genesis-builder/builddeb-genesis-base index a6af7e9b1a..e1c550ebde 100755 --- a/xCAT-genesis-builder/builddeb-genesis-base +++ b/xCAT-genesis-builder/builddeb-genesis-base @@ -80,7 +80,8 @@ REQUIRED_PACKAGES=" nfs-common rpcbind pciutils usbutils parted dosfstools e2fsprogs lvm2 mdadm net-tools bc psmisc rsync wget cpio - isc-dhcp-client ifenslave util-linux-extra + isc-dhcp-client ifenslave + systemd-sysv hwdata btrfs-progs netcat-openbsd iputils-ping fdisk dpkg-dev debhelper fakeroot devscripts vim-tiny " if [ "$BUILDARCH" = "amd64" ]; then @@ -89,6 +90,24 @@ fi echo "Installing build dependencies..." apt-get update -qq + +# Two commands the image needs changed package between releases: nslookup left dnsutils for +# bind9-dnsutils in 22.04, and hwclock left util-linux for util-linux-extra in 23.04. Ask apt +# which name this release carries rather than branch on the codename. +add_first_available() { + local p + for p in "$@"; do + if apt-cache show "$p" >/dev/null 2>&1; then + REQUIRED_PACKAGES="$REQUIRED_PACKAGES $p" + return 0 + fi + done + echo "ERROR: $CODENAME carries none of these packages: $*" >&2 + exit 1 +} +add_first_available bind9-dnsutils dnsutils +add_first_available util-linux-extra util-linux + apt-get install -y --no-install-recommends $REQUIRED_PACKAGES # dpkg-architecture comes from dpkg-dev, which the line above installs. @@ -237,6 +256,9 @@ rewrite_control "$DIR/debian/control" "$BUILDARCH" # debian/dirs names the image directory, which is the rpm architecture. echo "/opt/xcat/share/xcat/netboot/genesis/$TARCH/" > "$DIR/debian/dirs" +# dch reads debian/control from the current directory, not from the file it writes. +cd "$DIR" + PKG_VERSION="${VERSION}-${RELEASE}~${CODENAME}" rm -f "$DIR/debian/changelog" dch --create --package "xcat-genesis-base-$BUILDARCH" \ @@ -245,7 +267,6 @@ dch --create --package "xcat-genesis-base-$BUILDARCH" \ "Native Ubuntu build on $CODENAME $BUILDARCH" echo "Building .deb package..." -cd "$DIR" dpkg-buildpackage -rfakeroot -uc -us -b if [ -n "$outdir" ]; then diff --git a/xCAT-test/unit/genesis_ubuntu_build_root.t b/xCAT-test/unit/genesis_ubuntu_build_root.t index 1d87f7e63d..4510cac3fc 100755 --- a/xCAT-test/unit/genesis_ubuntu_build_root.t +++ b/xCAT-test/unit/genesis_ubuntu_build_root.t @@ -24,21 +24,25 @@ plan skip_all => 'ubuntu module-setup.sh not found' unless -f $module; plan tests => 9; # Commands the Ubuntu dracut module marks mandatory that a minimal Ubuntu server root does -# NOT already provide, and the package that supplies each one. Every entry here has to be in -# REQUIRED_PACKAGES or the image ships without the command. -my %PACKAGE_FOR = ( - dhclient => 'isc-dhcp-client', - ifenslave => 'ifenslave', - hwclock => 'util-linux-extra', +# NOT already provide, and the packages that supply each one. Every command here needs one of +# its packages in the build root or the image ships without it. hwclock has two names because +# it left util-linux for util-linux-extra in 23.04, and the build root asks apt which name the +# release it is building for carries. +my %PACKAGES_FOR = ( + dhclient => ['isc-dhcp-client'], + ifenslave => ['ifenslave'], + hwclock => [ 'util-linux-extra', 'util-linux' ], ); my %mandatory = map { $_ => 1 } mandatory_commands($module); my @packages = required_packages($builder); -for my $command (sort keys %PACKAGE_FOR) { +for my $command (sort keys %PACKAGES_FOR) { + my @provider = @{ $PACKAGES_FOR{$command} }; ok($mandatory{$command}, "the Ubuntu dracut module installs '$command' unconditionally"); - ok(scalar(grep { $_ eq $PACKAGE_FOR{$command} } @packages), - "the build root installs $PACKAGE_FOR{$command}, which provides '$command'"); + my @named = grep { my $p = $_; grep { $_ eq $p } @packages } @provider; + ok(scalar @named, + "the build root installs @{[ join ' or ', @provider ]}, which provides '$command'"); } # doxcat asks dhclient for the provisioning lease. An image without it never gets an address, @@ -85,8 +89,10 @@ BASH return @names; } -# required_packages($path): extract the REQUIRED_PACKAGES assignment from the build script and -# evaluate it, so the list comes from the value the script actually uses. +# required_packages($path): the packages the build root installs. The fixed list is the +# REQUIRED_PACKAGES assignment, evaluated so the value comes from the script itself; a command +# whose package name changed between releases is added by add_first_available, whose candidates +# count too -- the script picks whichever one apt knows. sub required_packages { my ($path) = @_; my $text = do { open my $fh, '<', $path or die "$path: $!"; local $/; <$fh> }; @@ -95,5 +101,7 @@ sub required_packages { my $out = qx{bash -c 'set -u; $block; printf "%s\\n" \$REQUIRED_PACKAGES' 2>/dev/null}; my @packages = grep { length } split /\s+/, ($out // ''); BAIL_OUT("REQUIRED_PACKAGES in $path evaluated to nothing") unless @packages; + push @packages, grep { length } split /\s+/, $1 + while $text =~ /^add_first_available\s+(.+)$/mg; return @packages; } From 5ff9b941b583cc099fedc7f511c6716ef897bfed Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Sat, 12 Sep 2026 09:13:26 -0300 Subject: [PATCH 04/10] fix(xcat-core): the Genesis image is published into releases it cannot boot builddebs.pl publishes every Architecture:all deb into every release, because every other xcat-core deb is the same file for all of them. The Genesis image is not: it carries the kernel of the root that built it. All three images therefore landed in all three suites, and apt serves the newest, which belongs to another release. deb_belongs_to_dist reads the codename back from the version and keeps an image out of any other suite. A deb with no codename in its version is unaffected. The jammy build also stopped on /usr/share/terminfo/l/linux and v/vt100: ncurses-base installs terminfo under /lib, and the module asks for the /usr/share copies, which come from ncurses-term. noble happened to have it. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- build-utils/lib/XCAT/BuildUtils.pm | 16 +++++++++++++++- builddebs.pl | 5 ++++- xCAT-genesis-builder/builddeb-genesis-base | 2 +- xCAT-test/unit/genesis_deb_per_codename.t | 16 ++++++++++++++++ 4 files changed, 36 insertions(+), 3 deletions(-) diff --git a/build-utils/lib/XCAT/BuildUtils.pm b/build-utils/lib/XCAT/BuildUtils.pm index 2efb89cc0d..5145504d8d 100644 --- a/build-utils/lib/XCAT/BuildUtils.pm +++ b/build-utils/lib/XCAT/BuildUtils.pm @@ -37,7 +37,7 @@ our @EXPORT_OK = qw( buildinfo_text targetarch_from_target genesis_chroot_name genesis_target_arch genesis_build_plan - genesis_log_errors genesis_log_deny_rules + genesis_log_errors genesis_log_deny_rules deb_belongs_to_dist ); # Both builders echo the commands they run under --verbose. Set once, after @@ -572,6 +572,20 @@ my @GENESIS_LOG_DENY = ( sub genesis_log_deny_rules { return @GENESIS_LOG_DENY; } +# deb_belongs_to_dist: whether a built .deb may be published into one release. +# +# Almost every xcat-core deb is Architecture:all and the same file serves every release, so the +# answer is yes. The Genesis image is not: it is built per codename and carries that codename in +# its version (2.19.0-snap...~noble). Publishing all three into every suite lets apt serve the +# newest, which is the image of another release. +sub deb_belongs_to_dist { + my ($deb, $dist) = @_; + return 1 unless defined $deb && defined $dist && $dist ne ''; + my $base = basename($deb); + return 1 unless $base =~ /_[^_]*~([A-Za-z0-9.]+)_[^_]*\.deb\z/; + return $1 eq $dist ? 1 : 0; +} + sub genesis_log_errors { my ($text) = @_; return () unless defined $text && length $text; diff --git a/builddebs.pl b/builddebs.pl index 685910be5a..7f9ce10622 100755 --- a/builddebs.pl +++ b/builddebs.pl @@ -41,7 +41,7 @@ reprepro_distributions reprepro_options lock_id_for take_build_lock sh_quote sh sh_or_die usage rewrite_file write_script read_line buildinfo_text - genesis_build_plan genesis_log_errors + genesis_build_plan genesis_log_errors deb_belongs_to_dist ); # The xcat-core packages that ship as debs. xCAT-openbmc-py, xCAT-rmc and xCAT-release @@ -441,6 +441,9 @@ sub assemble_repo { for my $deb (@debs) { # A release that predates an architecture must not be handed its packages. next if basename($deb) =~ /_(\w+)\.deb\z/ && $1 ne 'all' && !$ok{$1}; + # Nor an image built for another release: the Genesis deb carries the codename it + # was built on, because it carries that release's kernel. + next unless deb_belongs_to_dist($deb, $dist); sh_or_die("cd " . sh_quote($repodir) . " && reprepro -b ./ includedeb " . sh_quote($dist) . ' ' . sh_quote($deb), "FATAL: reprepro could not add $deb to $dist\n"); diff --git a/xCAT-genesis-builder/builddeb-genesis-base b/xCAT-genesis-builder/builddeb-genesis-base index e1c550ebde..6396754cc4 100755 --- a/xCAT-genesis-builder/builddeb-genesis-base +++ b/xCAT-genesis-builder/builddeb-genesis-base @@ -81,7 +81,7 @@ REQUIRED_PACKAGES=" dosfstools e2fsprogs lvm2 mdadm net-tools bc psmisc rsync wget cpio isc-dhcp-client ifenslave - systemd-sysv hwdata btrfs-progs netcat-openbsd iputils-ping fdisk + systemd-sysv hwdata btrfs-progs netcat-openbsd iputils-ping fdisk ncurses-term dpkg-dev debhelper fakeroot devscripts vim-tiny " if [ "$BUILDARCH" = "amd64" ]; then diff --git a/xCAT-test/unit/genesis_deb_per_codename.t b/xCAT-test/unit/genesis_deb_per_codename.t index 1f5b4e2703..637ef06ff4 100644 --- a/xCAT-test/unit/genesis_deb_per_codename.t +++ b/xCAT-test/unit/genesis_deb_per_codename.t @@ -61,6 +61,22 @@ ok(!eval { XCAT::BuildUtils::genesis_build_plan([], 'amd64'); 1 }, ok(!eval { XCAT::BuildUtils::genesis_target_arch('riscv64'); 1 }, 'an architecture with no Genesis image directory is an error'); +# --- a per-codename image reaches only its own suite -------------------------------------- +ok(XCAT::BuildUtils->can('deb_belongs_to_dist'), + 'XCAT::BuildUtils decides which suite a deb belongs to'); +if (XCAT::BuildUtils->can('deb_belongs_to_dist')) { + my $noble = 'xcat-genesis-base-amd64_2.19.0-snap202609121200~noble_all.deb'; + ok(XCAT::BuildUtils::deb_belongs_to_dist($noble, 'noble'), + 'the noble image is published into noble'); + ok(!XCAT::BuildUtils::deb_belongs_to_dist($noble, 'jammy'), + 'the noble image is not published into jammy'); + # Everything else in xcat-core is the same file for every release. + ok(XCAT::BuildUtils::deb_belongs_to_dist('perl-xcat_2.19.0-snap1_all.deb', 'jammy'), + 'a deb with no codename in its version reaches every suite'); + ok(XCAT::BuildUtils::deb_belongs_to_dist('xcat_2.19.0-snap1_amd64.deb', 'resolute'), + 'an architecture deb reaches every suite'); +} + # --- the log guard --------------------------------------------------------------------- # # dracut prints FAILED: for a command it cannot install and exits 0. This is the log of the From 915d8364b5fb791857c513f5e2f25838c9f3684f Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Sat, 12 Sep 2026 09:54:53 -0300 Subject: [PATCH 05/10] fix(xcat-core): two Genesis builds in different chroots share one directory The sbuild chroots bind-mount /var/lib/sbuild/build on /build, so every session of every chroot sees the same directory. Staging the builder there let two builds running at once overwrite each other's copy: one of them read a half-written script and stopped with "-get: command not found". The output of an earlier run stayed in it too, so a build collected another codename's deb and left 1.9 GB on the build host. Stage under /xcat-genesis-build instead. It lives in the session overlay and goes away with the session. 26.04 also moved the backward-compatibility zone names out of tzdata into tzdata-legacy, and the dracut module names 106 of them. Install it where apt has it. The three images (jammy 180 MB, noble 328 MB, resolute 432 MB) build with no error in any log. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- builddebs.pl | 31 ++++++++++++------- docs/source/developers/guides/code/builds.rst | 14 +++++++++ xCAT-genesis-builder/builddeb-genesis-base | 21 +++++++++++++ 3 files changed, 55 insertions(+), 11 deletions(-) diff --git a/builddebs.pl b/builddebs.pl index 7f9ce10622..eb6576b800 100755 --- a/builddebs.pl +++ b/builddebs.pl @@ -293,6 +293,11 @@ sub collect_debs { # (--sbuild). They hand out disposable overlay sessions, so what the build # installs is discarded and the next codename starts from the pristine base. +# Where the build runs inside the chroot. A directory of its own at the chroot root, because +# every other candidate is shared: /build and /opt/xcat-ci-shared are bind mounts the sbuild +# chroots give to every session. +my $GENESIS_STAGE = '/xcat-genesis-build'; + sub host_deb_arch { my $arch = `dpkg --print-architecture 2>/dev/null` // ''; chomp $arch; @@ -350,24 +355,28 @@ sub build_one_genesis_deb { # The builder needs its own directory, and Version and Release beside it. Copy them # in rather than bind-mount the checkout: the build rewrites debian/control and # debian/changelog, and it must not rewrite them in the tree the pipeline builds from. - sh_or_die("mkdir -p " . sh_quote("$root/build/xCAT-genesis-builder"), + # + # NOT under /build: the sbuild chroots bind-mount /var/lib/sbuild/build there, so every + # session of every chroot shares one directory. Two builds running at once overwrite each + # other's copy of the builder, and the output of an earlier run is still in it. A + # directory of its own at the chroot root lives in the session overlay and goes with it. + my $stage = "$root$GENESIS_STAGE"; + sh_or_die("rm -rf " . sh_quote($stage) . " && mkdir -p " + . sh_quote("$stage/xCAT-genesis-builder"), "FATAL: cannot make the build directory in session:$id\n"); sh_or_die("cp -a " . sh_quote("$ROOT/xCAT-genesis-builder") . "/. " - . sh_quote("$root/build/xCAT-genesis-builder") . "/", + . sh_quote("$stage/xCAT-genesis-builder") . "/", "FATAL: cannot copy xCAT-genesis-builder into session:$id\n"); - for my $f (qw(Version Release)) { - next unless -f "$ROOT/$f"; - copy("$ROOT/$f", "$root/build/$f") - or die "FATAL: cannot copy $f into session:$id: $!\n"; - } - write_text("$root/build/Release", "$RELEASE\n"); + copy("$ROOT/Version", "$stage/Version") + or die "FATAL: cannot copy Version into session:$id: $!\n"; + write_text("$stage/Release", "$RELEASE\n"); # --expect-codename is the guard that keeps the image and the root together: the # builder stops when the root it woke up in is not the release it was asked for. my $cmd = join ' ', 'schroot', '--run-session', '-c', sh_quote("session:$id"), '-u', 'root', '-d', '/', - '--', '/bin/bash', '/build/xCAT-genesis-builder/builddeb-genesis-base', - '--expect-codename', sh_quote($codename), '--outdir', '/build/out'; + '--', '/bin/bash', "$GENESIS_STAGE/xCAT-genesis-builder/builddeb-genesis-base", + '--expect-codename', sh_quote($codename), '--outdir', "$GENESIS_STAGE/out"; my $rc = sh("$cmd > " . sh_quote($logfile) . " 2>&1"); # The log is read whether or not the command failed: a build that exits 0 with @@ -379,7 +388,7 @@ sub build_one_genesis_deb { } die $problems if $problems; - my @debs = glob("$root/build/out/*.deb"); + my @debs = glob("$root$GENESIS_STAGE/out/*.deb"); die "FATAL: the Genesis build for $codename produced no .deb; log: $logfile\n" unless @debs; for my $deb (@debs) { diff --git a/docs/source/developers/guides/code/builds.rst b/docs/source/developers/guides/code/builds.rst index 1146e4225a..b1c8fe9d5b 100644 --- a/docs/source/developers/guides/code/builds.rst +++ b/docs/source/developers/guides/code/builds.rst @@ -65,6 +65,20 @@ ppc64el, and every release the repository serves declares the architecture; ``xCAT-genesis-scripts`` keeps the two architectures it has control files for, because riscv64 Genesis ships as an OpenEmbedded package instead. +The Genesis image is the exception, and it is off unless it is asked for:: + + ./builddebs.pl --genesis-only --genesis-dist jammy --genesis-dist noble + +``dracut`` copies the kernel, the kernel modules and every command out of the root +it runs in, so the image belongs to the release that built it. ``--genesis`` builds +one image per ``--genesis-dist`` codename inside that codename's +``--sbuild`` schroot -- the chroots xcat-dep's ``sbuild-all.pl`` +creates on the Ubuntu build host -- for the architecture of the build host. It +refuses to run in a root of another release, it fails the build on an error in the +log even when the exit status is 0, and it checks the extracted payload against the +commands the dracut module installs. ``--genesis-only`` builds the images and +nothing else, which is what xcat-dep consumes with ``--genesis-deb``. + Helpers shared by both builders live in ``build-utils/lib/XCAT/BuildUtils.pm``. ``buildcore.sh`` builds the architecture specific packages (``xCAT``, ``xCATsn``, diff --git a/xCAT-genesis-builder/builddeb-genesis-base b/xCAT-genesis-builder/builddeb-genesis-base index 6396754cc4..64dab3b3ae 100755 --- a/xCAT-genesis-builder/builddeb-genesis-base +++ b/xCAT-genesis-builder/builddeb-genesis-base @@ -105,14 +105,35 @@ add_first_available() { echo "ERROR: $CODENAME carries none of these packages: $*" >&2 exit 1 } +# A package that exists only on some releases. 26.04 moved the backward-compatibility zone +# names (Chile/Continental and the rest of the list the dracut module installs) out of tzdata +# into tzdata-legacy. +add_if_available() { + local p + for p in "$@"; do + apt-cache show "$p" >/dev/null 2>&1 && REQUIRED_PACKAGES="$REQUIRED_PACKAGES $p" + done + return 0 +} + add_first_available bind9-dnsutils dnsutils add_first_available util-linux-extra util-linux +add_if_available tzdata-legacy apt-get install -y --no-install-recommends $REQUIRED_PACKAGES # dpkg-architecture comes from dpkg-dev, which the line above installs. TRIPLET=$(dpkg-architecture -qDEB_HOST_MULTIARCH) +# ncurses-base put its terminfo entries under /lib/terminfo before 24.04, and both dracut's own +# terminfo module and the 97xcat module ask for /usr/share/terminfo. On jammy the build stopped +# on /usr/share/terminfo/l/linux. Give this build root the path they ask for; the chroot is +# thrown away when the build ends. +if [ -d /lib/terminfo ]; then + mkdir -p /usr/share/terminfo + cp -an /lib/terminfo/. /usr/share/terminfo/ 2>/dev/null || true +fi + # Set up dracut module if [ -d /usr/lib/dracut/modules.d ]; then DRACUT_PARENT=/usr/lib/dracut/modules.d From 61fdf385bc5dcc8d4ecccb9d37835e6854693899 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Mon, 14 Sep 2026 08:27:58 -0300 Subject: [PATCH 06/10] test(xcat-core): the Genesis build-root test stops the whole suite, and its comments over-explain genesis_ubuntu_build_root.t called BAIL_OUT at four places where an extraction stopped matching. prove stops every remaining file on a bail-out, so one stale regex in this file hides the results of the tests that would have run after it. die is just as loud and costs only this file. The branch also states one fact in four places. That dracut copies the kernel out of the root it runs in appears in the builddebs.pl header, in its Genesis section, in BuildUtils.pm and in the builder, each time with the incident that produced it. The chroot stage directory is explained twice, once at its declaration and again at its only use. Each fact now stands where the reader meets it, without the bug report around it. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- build-utils/lib/XCAT/BuildUtils.pm | 11 ++-- builddebs.pl | 43 ++++++--------- xCAT-genesis-builder/builddeb-genesis-base | 3 +- .../dracut_105/ubuntu/module-setup.sh | 5 +- xCAT-genesis-builder/verify-genesis-payload | 4 +- xCAT-test/unit/genesis_deb_per_codename.t | 15 ++---- xCAT-test/unit/genesis_payload_verification.t | 9 ++-- xCAT-test/unit/genesis_ubuntu_build_root.t | 52 ++++++++----------- 8 files changed, 53 insertions(+), 89 deletions(-) diff --git a/build-utils/lib/XCAT/BuildUtils.pm b/build-utils/lib/XCAT/BuildUtils.pm index 5145504d8d..ef8c6932ae 100644 --- a/build-utils/lib/XCAT/BuildUtils.pm +++ b/build-utils/lib/XCAT/BuildUtils.pm @@ -532,8 +532,7 @@ sub genesis_target_arch { # genesis_build_plan: one Genesis build per codename, for one architecture. # # The set of codenames comes from the caller, so a pipeline builds exactly the releases it -# publishes. Returns the codename, the chroot to build it in and the package the build -# produces, which is what the caller needs to run and to collect. +# publishes. sub genesis_build_plan { my ($dists, $arch) = @_; my @dists = @{ $dists || [] }; @@ -556,11 +555,9 @@ sub genesis_build_plan { # genesis_log_deny_rules / genesis_log_errors: what a Genesis build log says when the build # failed but the exit status did not. # -# dracut reports a command it cannot install with a FAILED: line and returns 0, and the -# builder never reads dracut's result. That is how an image with no dhclient was packaged, -# signed and published by a command that reported success. apt has the same shape: a -# missing package leaves a diagnostic and a zero status behind a `|| true`. So the log is -# the gate, not the exit status. +# dracut reports a command it cannot install with a FAILED: line and returns 0. apt has the +# same shape: a missing package leaves a diagnostic and a zero status behind a `|| true`. So +# the log is the gate, not the exit status. my @GENESIS_LOG_DENY = ( [ qr/\bFAILED:/ => 'dracut could not install a command' ], [ qr/Cannot find module/ => 'a kernel module the build names is absent' ], diff --git a/builddebs.pl b/builddebs.pl index eb6576b800..8d3aa8936f 100755 --- a/builddebs.pl +++ b/builddebs.pl @@ -10,10 +10,8 @@ # every codename. Only xCAT, xCATsn and xCAT-genesis-scripts carry an architecture, and # even there the difference is packaging metadata, not compiled output. # -# The Genesis image is the one exception, and it is why --genesis exists. dracut copies the -# kernel, the kernel modules and every command out of the root it runs in, so that image -# genuinely differs per release and is built once per codename inside that codename's -# sbuild chroot -- the same chroots xcat-dep builds its compiled packages in. +# The Genesis image is the one exception. --genesis builds it once per codename, inside +# that codename's sbuild chroot. use strict; use warnings; use feature 'say'; @@ -91,9 +89,7 @@ $opts{dists} = @cli_dists ? \@cli_dists : \@DISTS; $opts{gpg_key_name} //= 'xCAT Signing Key'; -# The Genesis step is off unless it is asked for, so every run that exists today keeps its -# behaviour. It takes its own codename list: the Genesis deb is the one package that is not -# the same file for every release, so the caller says which releases it wants built. +# The Genesis step is off unless it is asked for, so today's runs keep their behaviour. $opts{genesis} = 1 if $opts{genesis_only}; $opts{genesis_dists} = @cli_genesis_dists ? \@cli_genesis_dists : $opts{dists}; die "FATAL: --genesis-dist needs --genesis\n" if @cli_genesis_dists && !$opts{genesis}; @@ -283,15 +279,14 @@ sub collect_debs { # ----------------------------------------------------------- the Genesis deb -- # -# Every other xcat-core deb is Perl and is built once for every release. The Genesis image -# is not: dracut copies the kernel, the kernel modules and every command out of the root it -# runs in. Built on the build host, one image serves every codename with the build host's -# kernel -- which is how Ubuntu management nodes came to install an image built from an EL -# kernel. So this step builds one image per codename, inside that codename's sbuild chroot. +# dracut copies the kernel, the kernel modules and every command out of the root it runs in, +# so the Genesis image belongs to the release that built it. One build on the build host +# serves every codename with the build host's kernel. This step builds one image per +# codename, inside that codename's sbuild chroot. # # The chroots are the ones xcat-dep's sbuild-all.pl creates on the Ubuntu build host -# (--sbuild). They hand out disposable overlay sessions, so what the build -# installs is discarded and the next codename starts from the pristine base. +# (--sbuild). A session is a disposable overlay, so the next codename starts +# from the pristine base. # Where the build runs inside the chroot. A directory of its own at the chroot root, because # every other candidate is shared: /build and /opt/xcat-ci-shared are bind mounts the sbuild @@ -326,9 +321,8 @@ sub begin_chroot_session { # genesis_build_log_problems: what the log says went wrong when the exit status did not. # -# dracut prints FAILED: for a command it cannot install and exits 0. Reporting the first few -# offending lines rather than all of them keeps a build console readable; the log file has -# the rest and is named in the message. +# Report the first few offending lines only, so a build console stays readable. The message +# names the log file that has the rest. sub genesis_build_log_problems { my ($logfile) = @_; my $text = -f $logfile ? read_text($logfile) : ''; @@ -352,14 +346,9 @@ sub build_one_genesis_deb { my $err; eval { - # The builder needs its own directory, and Version and Release beside it. Copy them - # in rather than bind-mount the checkout: the build rewrites debian/control and - # debian/changelog, and it must not rewrite them in the tree the pipeline builds from. - # - # NOT under /build: the sbuild chroots bind-mount /var/lib/sbuild/build there, so every - # session of every chroot shares one directory. Two builds running at once overwrite each - # other's copy of the builder, and the output of an earlier run is still in it. A - # directory of its own at the chroot root lives in the session overlay and goes with it. + # Copy the builder in rather than bind-mount the checkout: the build rewrites + # debian/control and debian/changelog, and it must not rewrite them in the tree the + # pipeline builds from. my $stage = "$root$GENESIS_STAGE"; sh_or_die("rm -rf " . sh_quote($stage) . " && mkdir -p " . sh_quote("$stage/xCAT-genesis-builder"), @@ -379,8 +368,8 @@ sub build_one_genesis_deb { '--expect-codename', sh_quote($codename), '--outdir', "$GENESIS_STAGE/out"; my $rc = sh("$cmd > " . sh_quote($logfile) . " 2>&1"); - # The log is read whether or not the command failed: a build that exits 0 with - # FAILED: lines in its log is the failure this gate exists for. + # The log is read whether or not the command failed: dracut exits 0 with FAILED: + # lines in its log. my $problems = genesis_build_log_problems($logfile); if ($rc != 0) { die "FATAL: the Genesis build for $codename failed (exit $rc); log: $logfile\n" diff --git a/xCAT-genesis-builder/builddeb-genesis-base b/xCAT-genesis-builder/builddeb-genesis-base index 64dab3b3ae..8876c2d7eb 100755 --- a/xCAT-genesis-builder/builddeb-genesis-base +++ b/xCAT-genesis-builder/builddeb-genesis-base @@ -5,8 +5,7 @@ # in, so this must run inside a root of the target codename. builddebs.pl --genesis starts # it in that codename's --sbuild chroot, one build per codename. # --expect-codename is what stops a run on the build host: a single build there gives every -# Ubuntu release the build host's kernel, which is how the EL-built image reached Ubuntu -# management nodes in the first place. +# Ubuntu release the build host's kernel. # # Parallel to xCAT-genesis-base.spec, which does the same for EL. diff --git a/xCAT-genesis-builder/dracut_105/ubuntu/module-setup.sh b/xCAT-genesis-builder/dracut_105/ubuntu/module-setup.sh index 96c608dc91..367439ce4f 100755 --- a/xCAT-genesis-builder/dracut_105/ubuntu/module-setup.sh +++ b/xCAT-genesis-builder/dracut_105/ubuntu/module-setup.sh @@ -53,13 +53,12 @@ install() { dracut_install mount.nfs sshd vi reboot lspci parted screen mkfs mkfs.ext4 mkfs.btrfs #dracut_install libvirtd /usr/share/libvirt/cpu_map.xml /usr/bin/qemu-img /usr/libexec/qemu-kvm dracut_install mkswap df ifenslave ssh-keygen scp clear - # getdestiny makes its request file with mktemp. Without it the node reports no - # destiny, so xcatd never moves nodelist.status past powering-on. + # getdestiny makes its request file with mktemp. dracut_install mktemp dracut_install dhclient lldpad # OpenSSH 9.8 moved the per-connection work into sshd-session, which sshd execs by - # absolute path. Without it every connection to Genesis is refused. + # absolute path. for _sshd_helper in \ /usr/libexec/openssh/sshd-session \ /usr/libexec/openssh/sshd-auth \ diff --git a/xCAT-genesis-builder/verify-genesis-payload b/xCAT-genesis-builder/verify-genesis-payload index 3765e27e24..1f44898c1b 100755 --- a/xCAT-genesis-builder/verify-genesis-payload +++ b/xCAT-genesis-builder/verify-genesis-payload @@ -3,9 +3,7 @@ # verify-genesis-payload [--commands-from ] [required-path ...] # # dracut_install() reports a missing binary and returns, so the module install function keeps -# going and the image ships without it. Four such holes reached a release: no dhclient, no -# openssl, no sshd-session and no UTF-8 locale. Check the extracted payload before it becomes -# an rpm. +# going and the image ships without it. Check the extracted payload before it is packaged. # # Paths given on the command line are relative to . --commands-from reads back # what the dracut module installs: a bare command name is looked for in the four binary diff --git a/xCAT-test/unit/genesis_deb_per_codename.t b/xCAT-test/unit/genesis_deb_per_codename.t index 637ef06ff4..ee11fa987f 100644 --- a/xCAT-test/unit/genesis_deb_per_codename.t +++ b/xCAT-test/unit/genesis_deb_per_codename.t @@ -1,12 +1,7 @@ #!/usr/bin/env perl -# The Genesis image carries the kernel and the kernel modules of the root that built it. -# One build on the build host therefore gives every Ubuntu release the build host's kernel. -# builddebs.pl --genesis builds one image per codename, in that codename's chroot. -# -# Three decisions are tested here, each on the value the code returns: -# the chroot each codename builds in; -# the lines in a build log that mean the build failed although it exited 0; -# the refusal of the builder to run in a root of another release. +# The Genesis image carries the kernel and the kernel modules of the root that built it, so +# builddebs.pl --genesis builds one image per codename, in that codename's chroot. Each +# assertion here reads the value the code returns. use strict; use warnings; @@ -52,7 +47,6 @@ is($ppc[0]{target}, 'ppc64', 'ppc64el reads its image from the ppc64 directory' is(XCAT::BuildUtils::genesis_target_arch('amd64'), 'x86_64', 'amd64 reads its image from the x86_64 directory'); -# A codename given twice is still one build. is(scalar(() = XCAT::BuildUtils::genesis_build_plan([qw(noble noble)], 'amd64')), 1, 'a repeated codename does not build twice'); @@ -79,8 +73,7 @@ if (XCAT::BuildUtils->can('deb_belongs_to_dist')) { # --- the log guard --------------------------------------------------------------------- # -# dracut prints FAILED: for a command it cannot install and exits 0. This is the log of the -# build that shipped the image with no dhclient. +# dracut prints FAILED: for a command it cannot install and exits 0. my $dracut_log = <<'LOG'; Installing build dependencies... dracut: Executing: /usr/bin/dracut --compress gzip -m xcat base -N -f /tmp/genesis.rfs 6.8.0-45-generic diff --git a/xCAT-test/unit/genesis_payload_verification.t b/xCAT-test/unit/genesis_payload_verification.t index 7897f485c9..cc3785bb11 100644 --- a/xCAT-test/unit/genesis_payload_verification.t +++ b/xCAT-test/unit/genesis_payload_verification.t @@ -31,8 +31,7 @@ my $good = build_payload(sshd_execs_session => 1, session_helper => 1, tmux => 1 my ($rc, $err) = run($good, 'usr/sbin/dhclient'); is($rc, 0, 'a complete payload passes') or diag($err); -# doxcat calls dhclient with ISC flags. The released el9 image carried dhclient.conf and -# dhclient-script but no dhclient, so Genesis never acquired an address. +# doxcat calls dhclient with ISC flags. dhclient.conf and dhclient-script are not enough. my $nodhcp = build_payload(sshd_execs_session => 1, session_helper => 1, tmux => 1, locale => 1, dhclient => 0, mktemp => 1); ($rc, $err) = run($nodhcp, 'usr/sbin/dhclient'); isnt($rc, 0, 'a payload without dhclient fails'); @@ -55,16 +54,14 @@ my $nolocale = build_payload(sshd_execs_session => 1, session_helper => 1, tmux isnt($rc, 0, 'a payload with tmux and no UTF-8 locale fails'); like($err, qr{C\.utf8}, 'the missing locale is named'); -# getdestiny makes its request file with mktemp. Without it the node never reports its destiny, -# so xcatd never sets nodelist.status and the node stays at powering-on. +# getdestiny makes its request file with mktemp. my $nomktemp = build_payload(sshd_execs_session => 1, session_helper => 1, tmux => 1, locale => 1, dhclient => 1, mktemp => 0); ($rc, $err) = run($nomktemp, 'usr/sbin/dhclient'); isnt($rc, 0, 'a payload without mktemp fails'); like($err, qr{usr/bin/mktemp}, 'the missing mktemp is named'); # dracut_install reports a missing binary and returns, so every name the dracut module -# installs has to be checked against the payload. The el10 image shipped with no openssl and -# getcert waited on it for the life of the node. +# installs has to be checked against the payload. my $module = write_module_setup([qw(openssl wget tar)]); my $full = build_payload(sshd_execs_session => 1, session_helper => 1, tmux => 1, locale => 1, dhclient => 1, mktemp => 1, commands => [qw(openssl wget tar)]); diff --git a/xCAT-test/unit/genesis_ubuntu_build_root.t b/xCAT-test/unit/genesis_ubuntu_build_root.t index 4510cac3fc..97db9b2ca2 100755 --- a/xCAT-test/unit/genesis_ubuntu_build_root.t +++ b/xCAT-test/unit/genesis_ubuntu_build_root.t @@ -1,12 +1,11 @@ #!/usr/bin/env perl # The Ubuntu Genesis build root must carry every command the Ubuntu dracut module marks -# mandatory. dracut_install reports a missing command and returns, so a command the build -# root does not supply leaves a hole in the image and the build still exits 0. +# mandatory. dracut_install reports a missing command and returns 0, so a hole in the image +# does not fail the build. # -# The mandatory list is read by RUNNING the module: module-setup.sh is sourced with -# dracut_install shadowed, _dracut_install_opt neutralised (its callers are optional by -# construction), and install() is called. The package list is read by extracting the -# REQUIRED_PACKAGES assignment from builddeb-genesis-base and evaluating it. +# The mandatory list comes from RUNNING the module: module-setup.sh is sourced with +# dracut_install shadowed, _dracut_install_opt neutralised, and install() called. The +# package list comes from evaluating the REQUIRED_PACKAGES assignment in the build script. use strict; use warnings; @@ -23,11 +22,9 @@ plan skip_all => 'builddeb-genesis-base not found' unless -f $builder; plan skip_all => 'ubuntu module-setup.sh not found' unless -f $module; plan tests => 9; -# Commands the Ubuntu dracut module marks mandatory that a minimal Ubuntu server root does -# NOT already provide, and the packages that supply each one. Every command here needs one of -# its packages in the build root or the image ships without it. hwclock has two names because -# it left util-linux for util-linux-extra in 23.04, and the build root asks apt which name the -# release it is building for carries. +# Mandatory commands a minimal Ubuntu server root does NOT already provide, and the packages +# that supply each one. hwclock has two names: it left util-linux for util-linux-extra in +# 23.04, and the build root asks apt which name this release carries. my %PACKAGES_FOR = ( dhclient => ['isc-dhcp-client'], ifenslave => ['ifenslave'], @@ -45,24 +42,20 @@ for my $command (sort keys %PACKAGES_FOR) { "the build root installs @{[ join ' or ', @provider ]}, which provides '$command'"); } -# doxcat asks dhclient for the provisioning lease. An image without it never gets an address, -# so the node netboots and never reports in -- the failure this test exists for. +# doxcat asks dhclient for the provisioning lease. ok($mandatory{dhclient} && scalar(grep { $_ eq 'isc-dhcp-client' } @packages), 'the Genesis image can obtain a DHCP lease'); -# dracut_install is silent about a hole, so the payload needs its own gate before it is -# packaged. This is the EL path's behaviour (xCAT-genesis-base.spec runs the same verifier). +# dracut_install is silent about a missing command, so the payload needs its own gate. +# xCAT-genesis-base.spec runs the same verifier on the EL path. my $text = do { open my $fh, '<', $builder or die "$builder: $!"; local $/; <$fh> }; like($text, qr{verify-genesis-payload}, 'builddeb-genesis-base verifies the payload it packages'); # The image belongs to the release whose kernel it carries, so the builder must refuse a -# root of any other release. Without the refusal one build on the build host serves every -# codename with the build host's kernel. +# root of any other release. like($text, qr{--expect-codename}, 'builddeb-genesis-base takes the release it is building for'); -# mandatory_commands($module): source the dracut module with dracut_install shadowed, call -# install(), and return the bare command names it installs unconditionally. Absolute paths are -# data files, not commands, and are left out. +# An absolute path in the install() output is a data file, not a command. sub mandatory_commands { my ($path) = @_; my $dir = tempdir(CLEANUP => 1); @@ -75,32 +68,31 @@ inst_multiple() { :; } inst() { :; } dpkg-architecture() { echo x86_64-linux-gnu; } . '$path' -# Every caller of _dracut_install_opt is optional by construction: it installs only what the -# build root already has. Neutralise it AFTER sourcing so it cannot add to the mandatory set. +# _dracut_install_opt installs only what the build root already has. Neutralise it after +# sourcing, so its commands stay out of the mandatory set. _dracut_install_opt() { :; } install BASH close $fh; my @out = qx{bash '$driver' 2>/dev/null}; - BAIL_OUT("running install() from $path produced nothing") unless @out; + die("running install() from $path produced nothing") unless @out; my %seen; my @names = grep { !$seen{$_}++ } grep { length && !m{^/} } map { chomp; $_ } @out; - BAIL_OUT("install() from $path named no bare commands") unless @names; + die("install() from $path named no bare commands") unless @names; return @names; } -# required_packages($path): the packages the build root installs. The fixed list is the -# REQUIRED_PACKAGES assignment, evaluated so the value comes from the script itself; a command -# whose package name changed between releases is added by add_first_available, whose candidates -# count too -- the script picks whichever one apt knows. +# Evaluate the assignment rather than parse it, so the list is the value the script uses. +# add_first_available names the alternatives for a package that was renamed between releases, +# so its candidates count too. sub required_packages { my ($path) = @_; my $text = do { open my $fh, '<', $path or die "$path: $!"; local $/; <$fh> }; my ($block) = $text =~ /^(REQUIRED_PACKAGES="[^"]*")/ms; - BAIL_OUT("no REQUIRED_PACKAGES assignment in $path") unless $block; + die("no REQUIRED_PACKAGES assignment in $path") unless $block; my $out = qx{bash -c 'set -u; $block; printf "%s\\n" \$REQUIRED_PACKAGES' 2>/dev/null}; my @packages = grep { length } split /\s+/, ($out // ''); - BAIL_OUT("REQUIRED_PACKAGES in $path evaluated to nothing") unless @packages; + die("REQUIRED_PACKAGES in $path evaluated to nothing") unless @packages; push @packages, grep { length } split /\s+/, $1 while $text =~ /^add_first_available\s+(.+)$/mg; return @packages; From 6213f45a5fef96f94cba615830c1f0342a9ddecf Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Mon, 14 Sep 2026 08:50:46 -0300 Subject: [PATCH 07/10] style(xcat-core): the payload test header counts released defects The header of genesis_payload_verification.t said the fixtures reproduce "the three holes the released legacy Genesis image shipped with", while verify-genesis-payload's own header counted four. A count of past incidents is not what the fixtures are; each leaves out one thing the image needs. The wording now matches release/2.19-rc1, which carries the same file. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- xCAT-test/unit/genesis_payload_verification.t | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xCAT-test/unit/genesis_payload_verification.t b/xCAT-test/unit/genesis_payload_verification.t index cc3785bb11..6a9fef2d9a 100644 --- a/xCAT-test/unit/genesis_payload_verification.t +++ b/xCAT-test/unit/genesis_payload_verification.t @@ -1,6 +1,6 @@ #!/usr/bin/env perl -# Drive verify-genesis-payload against payload trees that reproduce the three holes the -# released legacy Genesis image shipped with. +# Drive verify-genesis-payload against payload trees that each leave out one thing the image +# needs. use strict; use warnings; From 3bea654adbe1e26a7f54d4e4258ebe785af9228b Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Tue, 15 Sep 2026 21:11:09 -0300 Subject: [PATCH 08/10] fix(xcat-core): genesis_ubuntu_build_root.t passes when the file it reads is missing genesis_ubuntu_build_root.t called plan skip_all when xCAT-genesis-builder/builddeb-genesis-base was absent, so a checkout that lost the file reported 0 tests and exit 0. A test that cannot fail measures nothing. Die instead, which is what makentp_ntp_deps.t already does for setupntp. With xCAT-genesis-builder/builddeb-genesis-base moved aside the file now exits 2 and prints "builddeb-genesis-base not found"; before this change it exited 0 and printed "1..0 # SKIP builddeb-genesis-base not found". With the file present the test passes either way. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- xCAT-test/unit/genesis_ubuntu_build_root.t | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xCAT-test/unit/genesis_ubuntu_build_root.t b/xCAT-test/unit/genesis_ubuntu_build_root.t index 97db9b2ca2..4d8623078d 100755 --- a/xCAT-test/unit/genesis_ubuntu_build_root.t +++ b/xCAT-test/unit/genesis_ubuntu_build_root.t @@ -18,8 +18,8 @@ use XCAT::Test::File qw(repo_path); my $builder = repo_path('xCAT-genesis-builder/builddeb-genesis-base'); my $module = repo_path('xCAT-genesis-builder/dracut_105/ubuntu/module-setup.sh'); -plan skip_all => 'builddeb-genesis-base not found' unless -f $builder; -plan skip_all => 'ubuntu module-setup.sh not found' unless -f $module; +die "builddeb-genesis-base not found\n" unless -f $builder; +die "ubuntu module-setup.sh not found\n" unless -f $module; plan tests => 9; # Mandatory commands a minimal Ubuntu server root does NOT already provide, and the packages From 488a6d1fc7c32d7b3de69f532d23074a25f2040f Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Wed, 16 Sep 2026 18:54:27 -0300 Subject: [PATCH 09/10] test(xcat-core): a prerelease version keeps every package out of every suite deb_belongs_to_dist reads any trailing ~word in a package version as the codename the deb was built for. Debian uses ~ for a prerelease, and --release takes whatever the caller gives it, so `--release 1~rc1` puts one in every package name: xcat-client_2.19.0-1~rc1_all.deb noble=0 focal=0 The deb is then published into no suite at all, with --genesis nowhere in the command, and the run still reports how many packages it published. Only the Genesis image is built per codename, so only it can be excluded by one. The new cases assert a prerelease version reaches every suite, that a Genesis SCRIPTS deb is not treated as the image, and that the image itself is still confined to its own suite when its version carries both. The second half asserts genesis_dists(), which does not exist yet: a plain --genesis run takes the release list the rest of the build uses, and focal is on it. focal ships debhelper 12.10 -- measured on the focal management node -- against the package's debhelper-compat (= 13), so sbuild stops on the build dependencies and the run ends on its first release. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- xCAT-test/unit/genesis_deb_per_codename.t | 34 +++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/xCAT-test/unit/genesis_deb_per_codename.t b/xCAT-test/unit/genesis_deb_per_codename.t index ee11fa987f..9b732500d5 100644 --- a/xCAT-test/unit/genesis_deb_per_codename.t +++ b/xCAT-test/unit/genesis_deb_per_codename.t @@ -69,6 +69,40 @@ if (XCAT::BuildUtils->can('deb_belongs_to_dist')) { 'a deb with no codename in its version reaches every suite'); ok(XCAT::BuildUtils::deb_belongs_to_dist('xcat_2.19.0-snap1_amd64.deb', 'resolute'), 'an architecture deb reaches every suite'); + + # A `~` in a version is not a codename. Debian uses it for a prerelease, and --release + # takes whatever the caller gives it, so `--release 1~rc1` puts one in every package + # name. Reading it as a codename drops the whole build from every suite, and the run + # still reports the packages it published. + for my $dist (qw(focal jammy noble resolute)) { + ok(XCAT::BuildUtils::deb_belongs_to_dist('xcat-client_2.19.0-1~rc1_all.deb', $dist), + "a prerelease version reaches $dist"); + } + ok(XCAT::BuildUtils::deb_belongs_to_dist('xcat-genesis-scripts-amd64_2.19.0-1~rc1_amd64.deb', 'noble'), + 'a prerelease Genesis SCRIPTS deb reaches every suite: only the image is per codename'); + + # And the rule the exclusion exists for still holds under a prerelease version. + ok(XCAT::BuildUtils::deb_belongs_to_dist('xcat-genesis-base-amd64_2.19.0-1~rc1~noble_amd64.deb', 'noble'), + 'a prerelease Genesis image reaches its own suite'); + ok(!XCAT::BuildUtils::deb_belongs_to_dist('xcat-genesis-base-amd64_2.19.0-1~rc1~noble_amd64.deb', 'jammy'), + 'and no other'); +} + +# --- the releases a Genesis image can be built on ------------------------------------------ +# A plain --genesis run takes the release list the rest of the build uses. focal is on it and +# cannot build the package: it ships debhelper 12.10 and xCAT-genesis-base declares +# debhelper-compat (= 13), so sbuild stops on the build dependencies before dracut runs and +# the run ends on its first release. +ok(XCAT::BuildUtils->can('genesis_dists'), + 'XCAT::BuildUtils says which releases a Genesis image can be built on'); +if (XCAT::BuildUtils->can('genesis_dists')) { + is_deeply([ XCAT::BuildUtils::genesis_dists(XCAT::BuildUtils::default_dists()) ], + [qw(jammy noble resolute)], + 'the default plan leaves out the release whose chroot cannot build the package'); + is_deeply([ XCAT::BuildUtils::genesis_dists(qw(jammy noble)) ], [qw(jammy noble)], + 'a list with none of them is unchanged'); + is_deeply([ XCAT::BuildUtils::genesis_dists('focal') ], [], + 'a list of only that release plans nothing'); } # --- the log guard --------------------------------------------------------------------- From 96aab326935bc49d8a3778b9a426f05549783646 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Wed, 16 Sep 2026 18:55:34 -0300 Subject: [PATCH 10/10] fix(xcat-core): a prerelease version keeps every package out of every suite deb_belongs_to_dist read any trailing ~word in a package version as the codename the deb was built for. Debian uses ~ for a prerelease and --release takes whatever the caller gives it, so `--release 1~rc1` left every package excluded from every suite, with --genesis nowhere in the command and the run still reporting what it published. Only xcat-genesis-base carries a codename, because only it carries a kernel, so only it is asked. genesis_dists() names the releases a Genesis image can be built on. A plain --genesis run took the release list the rest of the build uses, and focal is on it: focal ships debhelper 12.10 and the package declares debhelper-compat (= 13), so sbuild stopped on the build dependencies before dracut ran and the run ended on its first release. The default plan now leaves focal out and says so. A release named with --genesis-dist is still built as asked. genesis_deb_per_codename.t covers both. Applying the codename rule to every package again fails five assertions; a genesis_dists that filters nothing, or an image that loses its own rule, fails two each. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- build-utils/lib/XCAT/BuildUtils.pm | 27 +++++++++++++++++++++++++++ builddebs.pl | 14 +++++++++++++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/build-utils/lib/XCAT/BuildUtils.pm b/build-utils/lib/XCAT/BuildUtils.pm index ef8c6932ae..1c0d7f7687 100644 --- a/build-utils/lib/XCAT/BuildUtils.pm +++ b/build-utils/lib/XCAT/BuildUtils.pm @@ -38,6 +38,7 @@ our @EXPORT_OK = qw( targetarch_from_target genesis_chroot_name genesis_target_arch genesis_build_plan genesis_log_errors genesis_log_deny_rules deb_belongs_to_dist + genesis_dists genesis_dist_reason ); # Both builders echo the commands they run under --verbose. Set once, after @@ -569,16 +570,42 @@ my @GENESIS_LOG_DENY = ( sub genesis_log_deny_rules { return @GENESIS_LOG_DENY; } +# The package built once per codename. It is the only one: it carries the kernel and the kernel +# modules of the root that built it. +my $GENESIS_IMAGE_DEB = qr{\Axcat-genesis-base-}; + +# Releases whose stock chroot cannot build that package, and why. focal ships debhelper 12 and +# xCAT-genesis-base declares debhelper-compat (= 13), so sbuild stops on the build dependencies +# before dracut runs. +my %GENESIS_DIST_UNSUPPORTED = ( + focal => 'debhelper 12 cannot satisfy debhelper-compat (= 13)', +); + +# genesis_dists: the releases of @dists a Genesis image can be built on. +sub genesis_dists { + my (@dists) = @_; + return grep { !exists $GENESIS_DIST_UNSUPPORTED{$_} } @dists; +} + +# genesis_dist_reason: why a release was left out, or undef. +sub genesis_dist_reason { return $GENESIS_DIST_UNSUPPORTED{ $_[0] // '' }; } + # deb_belongs_to_dist: whether a built .deb may be published into one release. # # Almost every xcat-core deb is Architecture:all and the same file serves every release, so the # answer is yes. The Genesis image is not: it is built per codename and carries that codename in # its version (2.19.0-snap...~noble). Publishing all three into every suite lets apt serve the # newest, which is the image of another release. +# +# Only that package is asked. A ~ in a version is Debian's prerelease separator before it is +# anything else, and --release takes whatever the caller gives it, so reading every ~ as a +# codename drops a whole `--release 1~rc1` build from every suite. sub deb_belongs_to_dist { my ($deb, $dist) = @_; return 1 unless defined $deb && defined $dist && $dist ne ''; my $base = basename($deb); + my ($name) = $base =~ /\A([^_]+)_/; + return 1 unless defined $name && $name =~ $GENESIS_IMAGE_DEB; return 1 unless $base =~ /_[^_]*~([A-Za-z0-9.]+)_[^_]*\.deb\z/; return $1 eq $dist ? 1 : 0; } diff --git a/builddebs.pl b/builddebs.pl index 8d3aa8936f..b4889f6734 100755 --- a/builddebs.pl +++ b/builddebs.pl @@ -40,6 +40,7 @@ lock_id_for take_build_lock sh_quote sh sh_or_die usage rewrite_file write_script read_line buildinfo_text genesis_build_plan genesis_log_errors deb_belongs_to_dist + genesis_dists genesis_dist_reason ); # The xcat-core packages that ship as debs. xCAT-openbmc-py, xCAT-rmc and xCAT-release @@ -91,7 +92,18 @@ # The Genesis step is off unless it is asked for, so today's runs keep their behaviour. $opts{genesis} = 1 if $opts{genesis_only}; -$opts{genesis_dists} = @cli_genesis_dists ? \@cli_genesis_dists : $opts{dists}; +# A release named on the command line is built as asked. The default list is the one the rest +# of the build uses, and not every release on it can build the image, so those are dropped and +# named rather than failing the run on its first codename. +if (@cli_genesis_dists) { + $opts{genesis_dists} = \@cli_genesis_dists; +} else { + $opts{genesis_dists} = [ genesis_dists($opts{dists}->@*) ]; + for my $dist ($opts{dists}->@*) { + my $why = genesis_dist_reason($dist) or next; + say "genesis: leaving out $dist -- $why"; + } +} die "FATAL: --genesis-dist needs --genesis\n" if @cli_genesis_dists && !$opts{genesis}; die "FATAL: --genesis-arch needs --genesis\n" if $opts{genesis_arch} && !$opts{genesis};