diff --git a/.github/matrix.json b/.github/matrix.json index f862390..113987c 100644 --- a/.github/matrix.json +++ b/.github/matrix.json @@ -5,6 +5,7 @@ { "host": "ubuntu-22.04", "container": "ghcr.io/tamatebako/tpkg-builder-x86_64-linux-musl", "os": "linux-musl", "arch": "x86_64" }, { "host": "ubuntu-22.04-arm", "container": "ghcr.io/tamatebako/tpkg-builder-aarch64-linux-musl","os": "linux-musl", "arch": "arm64" }, { "host": "macos-15-intel", "container": null, "os": "macos", "arch": "x86_64" }, - { "host": "macos-14", "container": null, "os": "macos", "arch": "arm64" } + { "host": "macos-14", "container": null, "os": "macos", "arch": "arm64" }, + { "host": "windows-2022", "container": null, "os": "windows", "arch": "x86_64" } ] } diff --git a/.github/workflows/_build-platform.yml b/.github/workflows/_build-platform.yml index f945634..1a44296 100644 --- a/.github/workflows/_build-platform.yml +++ b/.github/workflows/_build-platform.yml @@ -274,14 +274,19 @@ jobs: ci/check_symbol_provenance.sh --exe \ "runtime-packages/tebako-runtime-${{ needs.compute.outputs.tebako_version }}-${{ matrix.python }}-${{ matrix.host_id }}" + # The exe name carries .exe on this leg (Platform#exe_suffix) and it + # must be spelled out: msys stat() transparency lets foo.exe satisfy + # `[ -s foo ]`, but the native nm/objdump open the argument verbatim + # — an extensionless spelling is ENOENT to them and reads back as a + # stripped exe (run 34758424181's false wall). The ruby factory + # spells ruby.exe the same way. - name: Check symbol provenance (windows) if: matrix.os == 'windows' shell: msys2 {0} run: | set -euo pipefail - exe="runtime-packages/tebako-runtime-${{ needs.compute.outputs.tebako_version }}-${{ matrix.python }}-${{ matrix.host_id }}" - [ -s "$exe" ] || exe="$exe.exe" - ci/check_symbol_provenance.sh --exe "$exe" + ci/check_symbol_provenance.sh --exe \ + "runtime-packages/tebako-runtime-${{ needs.compute.outputs.tebako_version }}-${{ matrix.python }}-${{ matrix.host_id }}.exe" # The boot smoke self-selects the scenario set per platform (the # POSIX set vs the windows bare + named-error trio) and resolves the diff --git a/.gitignore b/.gitignore index 44cc960..bc617e6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Build artifacts .build/ +.build-inspect/ runtime-packages/ # Ruby tooling (factory convention: the lockfile floats) diff --git a/AGENTS.md b/AGENTS.md index c871b66..2dbf872 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -43,10 +43,14 @@ BOOT_LIVE gate + raw early-boot passthrough the gnu legs), and the `fcntl64` export (tamatebako/tebako#529). The POSIX boot-smoke CI legs were red-by-design on the old v2.1.5 pin and are green on the v2.1.10+ line. The windows row -is descoped out of the 02 matrix (TODO.python/05 — CPython upstream -has zero mingw support; the port is a tamatebako/python msys2/ucrt64 -patch -series), and `tools/boot_smoke` hard-kills a wedged child against a +(63e8336's descope) returns with TODO.python/05 item 2: CPython +upstream has zero mingw support, so the windows leg builds from the +line's msys2/ucrt64 scenario tree +(`tfs-python--src-windows-msys.tar.gz`, tamatebako/python's +patches// series) — `scripts/compute_matrix.rb` skips a windows +leg loudly when the pinned `source_release` ships no such asset for +the line (and skips jit lines on windows outright: upstream's JIT +whitelist admits MSVC targets only), and `tools/boot_smoke` hard-kills a wedged child against a CLOCK_MONOTONIC deadline instead of wedging with it. The linked-driver decision and the preload-shim re-exec rationale are recorded in README.md — do not reopen them here; the wrapper pattern diff --git a/README.md b/README.md index 5b46b97..c10681c 100644 --- a/README.md +++ b/README.md @@ -113,8 +113,11 @@ boundary below). The env image ships the stdlib plus a **declarative site-packages whitelist** (`build/site-packages.yml`; v1 keeps `pip` only, with its -dist-info — pip's `importlib.metadata` self-check reads it). Everything -else ensurepip installed is pruned at image assembly. +dist-info — pip's `importlib.metadata` self-check reads it). pip reaches +the staged site-packages via `make install`'s ensurepip on POSIX and via +an explicit bundled-wheel placement on msys (the install's `--root` +rebase drive-strips the build-tree prefix — `PythonBuild#place_pip`). +Anything else the install staged is pruned at image assembly. **The pip form is `python3 -m pip`.** The image's `bin/` directory is pruned wholesale: the ensurepip console scripts' shebangs spell the @@ -125,15 +128,19 @@ interpreter by convention. ## The windows boundary -The v1 windows leg (ucrt64, `--disable-shared`) is the **driver-contract -surface only**: the exe boots the driver, answers `--tebako-image`/ -`TEBAKO_RUNTIME_IMAGE` with the same named errors, and runs the -interpreter only when nothing was mounted (bare/dev mode). There is no -preload tier on windows — with any mount the fs TU exits **69** with a -named error (roadmap 30 phase 2). The `--disable-shared` choice means no -libpython DLL facet ships; `scripts/upload_release.rb` already models -the dll facet opportunistically if a future `--enable-shared` leg -appears (the ruby factory's issue-40 analog). +The windows leg (ucrt64, `--enable-shared` — issue 40's answer: on PE a +loadable module cannot carry undefined symbols, so the extensions link +`libpython.dll`; the ruby factory ships the same shape) is the +**driver-contract surface only**: the exe boots the driver, answers +`--tebako-image`/`TEBAKO_RUNTIME_IMAGE` with the same named errors, and +runs the interpreter only when nothing was mounted (bare/dev mode). +There is no preload tier on windows — with any mount the fs TU exits +**69** with a named error (roadmap 30 phase 2). The shared build ships +the DLL as a `.dll` release facet (the release manifest's +`dll.install_as` names the PE spelling the store materializes beside +the exe), and the mingw support set (libgcc/libwinpthread) is +statically linked into the runtime's own PE modules — a bare machine +installs nothing. ## The artifacts (per version × triplet) diff --git a/build/lib/tebako_python_builder/builder.rb b/build/lib/tebako_python_builder/builder.rb index 1522d3f..a7fbed3 100644 --- a/build/lib/tebako_python_builder/builder.rb +++ b/build/lib/tebako_python_builder/builder.rb @@ -63,11 +63,13 @@ def initialize(repo_root:, python_version:, tebako_version:, prefix:, output:, # end def run # rubocop:disable Metrics/MethodLength - # A flavored line (x.y.z-jit) consumes the SAME pristine source - # tarball as its base — the flavor is a configure-time ability of - # the line, never a second source artifact (the source factory's - # SHA256SUMS names base versions only). - (tarball, sha256) = fetcher.fetch(@python.base_version) + # A flavored line (x.y.z-jit) consumes the SAME source tarball as its + # base — the flavor is a configure-time ability of the line, never a + # second source artifact (the source factory's SHA256SUMS names base + # versions only). The platform selects the scenario: a mingw/ucrt + # host builds from the line's patched windows-msys tree, every POSIX + # host from the unsuffixed pristine one. + (tarball, sha256) = fetcher.fetch(@python.base_version, platform: @platform) puts "-- Building tebako runtime for python #{@python_version} " \ "(tebako #{@tebako_version}, #{@platform.host_id}, #{File.basename(tarball)})" link_unit_dir = link_unit.stage(File.join(@prefix, "link-unit")) @@ -129,6 +131,32 @@ def finalize(built_exe) FileUtils.mkdir_p(File.dirname(output)) FileUtils.cp(built_exe, output) FileUtils.chmod(0o755, output) + stage_python_dll(built_exe) if @platform.msys? + end + + # msys only (the --enable-shared shape): stage the just-linked + # libpython.dll next to the runtime executable under the + # PACKAGE's name (.dll — unique per leg: two same-ABI legs + # share the PE name and would collide in the merged release + # workspace; the manifest's dll.install_as flows the PE name to the + # store entry, and tools/boot_smoke materializes it in-leg). The DLL + # links next to the exe in the build tree (the Makefile's + # $(DLLLIBRARY) rule); its absence means the build regressed to a + # static shape — a named error, never a silent skip. The name's + # single owner is PythonVersion#msys_dll_name (invariant 10). + def stage_python_dll(built_exe) + dll = File.join(File.dirname(built_exe), @python.msys_dll_name) + unless File.file?(dll) + raise TebakoPythonBuilder::Error.new( + "expected the shared build's #{@python.msys_dll_name} next to #{built_exe} " \ + "(the windows-msys leg must configure --enable-shared — issue 40)", 130 + ) + end + + dest = "#{output.sub(/\.exe\z/, "")}.dll" + FileUtils.cp(dll, dest) + FileUtils.chmod(0o755, dest) + puts "-- Runtime DLL: #{dest} (installs as #{@python.msys_dll_name})" end def assemble_and_pack_image(build, sha256, link_unit, link_unit_dir, tfs) diff --git a/build/lib/tebako_python_builder/image_builder.rb b/build/lib/tebako_python_builder/image_builder.rb index 876c914..0a2ecd9 100644 --- a/build/lib/tebako_python_builder/image_builder.rb +++ b/build/lib/tebako_python_builder/image_builder.rb @@ -59,7 +59,13 @@ class ImageBuilder # effective root, so the interpreter follows TEBAKO_MOUNT_ROOT (the # factory owns both sides of the grant; the boot smoke asserts the # chain end-to-end). The additive preload_shim grant (schema_minor 2) - # names exactly the staged shim's in-image path. + # names exactly the staged shim's in-image path. The additive + # runtime_dll basename (schema_minor 3) is emitted for MSYS builds + # only — flowed from PythonVersion#msys_dll_name, the name's single + # owner (invariant 10); the driver exports it as TEBAKO_RUNTIME_DLL + # into the handoff env, and the tfs PE closure walk excludes a bare + # import name matching it (spec 22 §2.1 — the OS's basename-reuse + # rule binds the already-loaded copy). POSIX builds omit the key. LAYOUT_DECLARATION = { "schema" => "layout", "schema_version" => 1, @@ -192,6 +198,8 @@ def deploy_layout(tree) ) unless @platform.msys? declaration["preload_shim"] = File.join("lib", "tebako", @link_unit.preload_shim_name) + else + declaration["runtime_dll"] = @python.msys_dll_name end File.write(path, YAML.dump(declaration)) puts " ... env image layout declaration: #{path}" diff --git a/build/lib/tebako_python_builder/python_build.rb b/build/lib/tebako_python_builder/python_build.rb index 8089e6a..3950b1e 100644 --- a/build/lib/tebako_python_builder/python_build.rb +++ b/build/lib/tebako_python_builder/python_build.rb @@ -110,10 +110,12 @@ def exe_path end # The runtime's abi facet (the release shard's additive `abi` key): - # the build's own EXT_SUFFIX stem (e.g. "cpython-313-x86_64-linux-gnu") - # — exactly the string native-extension wheels pin. Read off the BUILT - # exe itself (the host is the target: no cross), run bare — dev mode, - # no image — so getpath resolves the build tree. + # the build's own EXT_SUFFIX stem (e.g. "cpython-313-x86_64-linux-gnu"; + # the windows-msys shared shape spells it "cp314-mingw_x86_64_ucrt_gnu.pyd" + # — the source series' 0014 SOABI/EXT_SUFFIX port) — exactly the string + # native-extension wheels pin. Read off the BUILT exe itself (the host + # is the target: no cross), run bare — dev mode, no image — so getpath + # resolves the build tree. def abi @abi ||= begin env = { "PYTHONHOME" => nil, "PYTHONPATH" => nil } @@ -126,7 +128,7 @@ def abi # which is why only the linux legs saw it). Re-tag, scrub, and # keep the EXT_SUFFIX-shaped line — stderr noise never parses. text = out.force_encoding(Encoding::UTF_8).scrub - line = text.lines.map(&:strip).find { |l| l.match?(/\A\.?cpython-\d+[\w.-]*\z/) } + line = text.lines.map(&:strip).find { |l| l.match?(/\A\.?(?:cpython-\d+|cp\d+)[\w.-]*\z/) } unless st.exitstatus&.zero? && line raise TebakoPythonBuilder::Error.new( "the built interpreter did not report its EXT_SUFFIX (#{st}): #{text}", 106 @@ -232,13 +234,18 @@ def write_setup_local # (sysconfig data and the .pyc source paths then spell the runtime VFS # path; PYTHONHOME — set by the fs TU from the driver's effective root # — is what actually drives getpath at boot, TODO.python/01's probe). + # windows-msys builds --enable-shared: on PE a loadable module cannot + # carry undefined symbols, so every stdlib extension links + # libpython3.14.dll (the static shape is impossible by construction — + # issue 40; the ruby factory ships the same way). POSIX stays static + # (the shipped single-exe shape). # A jit line adds --enable-experimental-jit (bare = "yes": the JIT is # compiled in AND on by default; PYTHON_JIT=0/1 override at runtime, # CPython whatsnew 3.13 — selecting the jit line IS opting into it). def configure args = ["./configure", "--prefix=#{@platform.mount_root}", - "--disable-shared", + @platform.msys? ? "--enable-shared" : "--disable-shared", "--with-openssl=#{openssl_prefix}", "--with-openssl-rpath=no"] args << "--enable-experimental-jit" if @python.jit? @@ -261,6 +268,15 @@ def configure # LDFLAGS=-pthread flows through PY_LDFLAGS into PY_CORE_LDFLAGS, the # link side of every rule; the compile side already carries the # image's -pthread. + # windows-msys: static-link the mingw support set INTO the runtime's + # PE modules — libpython3.14.dll otherwise imports + # libgcc_s_seh-1.dll + libwinpthread-1.dll, and nothing on a bare + # windows machine provides them (the audience rule: a user running a + # package installs no toolchain). LIBS lands in the link TAIL of the + # exe/DLL rules (after the objects — LDFLAGS sits before them and a + # static archive there is discarded as unneeded); configure prepends + # its own finds, so this pair stays last. The ruby factory's proven + # recipe (its Mlibs::MSYS_DLL_LIBRARIES). # Everywhere else the system openssl/zlib are found by the default # detection (the containers ship libssl-dev/zlib1g-dev, # openssl-dev/zlib-static, pacman openssl). @@ -273,6 +289,8 @@ def configure_env } elsif @platform.linux_gnu? { "LDFLAGS" => "-pthread" } + elsif @platform.msys? + { "LIBS" => "-static-libgcc -l:libwinpthread.a" } else {} end.merge(@jit_env) @@ -424,8 +442,7 @@ def substitute_makefiles # rubocop:disable Metrics/MethodLength substitutions = [ [%r{^(\$\(BUILDPYTHON\):\s*)Programs/python\.o( \$\(LINK_PYTHON_DEPS\).*)$}, "\\1Programs/tebako_python.o\\2"], - [/^(\t\$\(LINKCC\) \$\(PY_CORE_LDFLAGS\) \$\(LINKFORSHARED\) -o \$@ )Programs\/python\.o( \$\(LINK_PYTHON_OBJS\) \$\(LIBS\) \$\(MODLIBS\) \$\(SYSLIBS\))$/, - "\\1Programs/tebako_python.o\\2 $(TEBAKO_LIBS)"], + exe_recipe_substitution, # libainstall ships the interpreter's main object for embedding: # the fs TU rides under the shipped name python.o (the object # consumers of LIBPL expect; its content is the driver-linked TU). @@ -455,10 +472,36 @@ def substitute_makefiles # rubocop:disable Metrics/MethodLength puts " ... Makefile substitutions applied (#{targets.map { |t| File.basename(t) }.join(", ")})" end + # The exe link recipe substitution, platform-shaped. POSIX keeps + # upstream's spelling; the windows-msys source series (0039 + # win_resources_pythonw_msys) rewrites the rule to + # `$(LINKFORSHARED) -municode -o $@ Programs/python.o ... + # python_exe.o` — the prereq-side regex above still matches (its .* + # absorbs python_exe.o), the recipe-side must name the msys spelling + # or the driver link is silently lost (the prereq rewrite alone makes + # the fs TU compile without ever linking it — verify_substitutions + # guards the recipe for exactly this reason). The msys substitution + # STRIPS -municode: the fs TU (build/resources/tebako_python_main.c) + # defines a plain main — with -municode the mingw CRT demands wmain + # and the link dies. python_exe.o (the version/icon resource) stays. + # (Unicode argv reaches the driver through the CRT's narrow + # conversion, matching the shipped POSIX shape; wide-argv parity is + # a recorded follow-up.) + def exe_recipe_substitution + if @platform.msys? + [/^(\t\$\(LINKCC\) \$\(PY_CORE_LDFLAGS\) \$\(LINKFORSHARED\)) -municode( -o \$@ )Programs\/python\.o( \$\(LINK_PYTHON_OBJS\) \$\(LIBS\) \$\(MODLIBS\) \$\(SYSLIBS\)) python_exe\.o$/, + "\\1\\2Programs/tebako_python.o\\3 python_exe.o $(TEBAKO_LIBS)"] + else + [/^(\t\$\(LINKCC\) \$\(PY_CORE_LDFLAGS\) \$\(LINKFORSHARED\) -o \$@ )Programs\/python\.o( \$\(LINK_PYTHON_OBJS\) \$\(LIBS\) \$\(MODLIBS\) \$\(SYSLIBS\))$/, + "\\1Programs/tebako_python.o\\2 $(TEBAKO_LIBS)"] + end + end + def verify_substitutions(durable) content = File.read(durable) missing = [] missing << "the $(BUILDPYTHON) rule" unless content.include?("$(BUILDPYTHON):\tPrograms/tebako_python.o") + missing << "the $(BUILDPYTHON) link recipe" unless content.include?("-o $@ Programs/tebako_python.o") missing << "the libainstall python.o line" unless content.include?("$(INSTALL_DATA) Programs/tebako_python.o") # The wasm anchor is version-conditional (absent on 3.14+), so the # drift guard is inverted: a SURVIVING un-rewritten binding means the @@ -488,26 +531,89 @@ def write_fs_tu end def make + # msys: upstream's DLL rule ($(DLLLIBRARY) libpython$(LDVERSION).dll.a + # as a two-target rule with -Wl,--out-implib=$@) is parallel-unsafe: + # under -j the two per-target invocations interleave and the + # import-library write can clobber the DLL (observed locally: the + # 39 MB libpython3.14.dll replaced by the 1.2 MB import stub — + # serial make self-heals only because the .dll.a target's expansion + # runs last). Pre-building the implib target SERIALLY materializes + # both files with the correct expansion (the recipe writes both), so + # the parallel make finds the rule up-to-date and never re-runs it. + if @platform.msys? + TebakoPythonBuilder::BuildHelpers.run_with_capture(["make", "#{@python.msys_dll_name}.a"], chdir: src_dir) + end TebakoPythonBuilder::BuildHelpers.run_with_capture(["make", "-j", ncores.to_s], env: @jit_env, chdir: src_dir) rescue TebakoPythonBuilder::Error => e raise TebakoPythonBuilder::Error.new("'build_runtime' build step failed: #{e.message}", 104) end # Staged install: `make install DESTDIR=` lands the prefix tree - # at /. ensurepip rides the install (pip - # in site-packages — the image's one selected site-package); the - # compileall pass writes the stdlib .pyc set the read-only image then - # serves (no .pyc writes at run time). + # at /; the compileall pass writes the + # stdlib .pyc set the read-only image then serves (no .pyc writes at + # run time). CPython's install rules spell every destination + # $(DESTDIR)$(dir) with no separator, which composes only because a + # POSIX prefix starts with a slash. The msys prefix is drive-qualified + # ("A:/t") and DESTDIR + "A:/t/..." concatenates into one junk + # component — the tree never lands (the failing recipes are mostly + # `-`-prefixed, so make exits green over the wreckage) — so the msys + # install overrides prefix to the POSIX spelling "/A/t": every install + # dir var derives unexpanded from $(prefix) (autoconf convention), so + # the override flows through to every rule; recorded .pyc source paths + # then spell /A/t/... instead of A:/t/..., a traceback cosmetic. + # ensurepip cannot ride the msys install (ENSUREPIP=no): pip's --root + # rebase drive-strips the built python's BUILD-TREE prefix, never the + # staged one, so pip would land under / — + # place_pip stages it explicitly instead. def install @stage_dir = File.join(@prefix, "stage") FileUtils.rm_rf(@stage_dir, secure: true) FileUtils.mkdir_p(@stage_dir) - TebakoPythonBuilder::BuildHelpers.run_with_capture(["make", "install", "DESTDIR=#{stage_dir}"], - chdir: src_dir) + argv = ["make", "install", "DESTDIR=#{stage_dir}"] + # "A:/t" -> "/A/t" (the method comment's prefix override). + argv += ["prefix=/#{@platform.mount_root.delete(':')}", "ENSUREPIP=no"] if @platform.msys? + TebakoPythonBuilder::BuildHelpers.run_with_capture(argv, chdir: src_dir) + place_pip if @platform.msys? rescue TebakoPythonBuilder::Error => e raise TebakoPythonBuilder::Error.new("'build_runtime' install step failed: #{e.message}", 105) end + # msys only — the ensurepip replacement (the install comment): the + # bundled wheel's own installer run into the staged site-packages, + # mirroring ensurepip's -c invocation (the wheel prepended to + # sys.path, pip run as __main__) but with --target deciding the + # placement — no scheme math, no --root rebase. The build-tree exe + # runs bare (dev mode — the abi probe's shape), and the paths ride + # argv rather than the -c payload: the msys runtime translates + # whole-argument POSIX paths for the native child exe, but a path + # embedded in the script text crosses verbatim. --no-compile mirrors + # ensurepip: .pyc records written here would spell the stage paths — + # dead links in the mounted image — so pip compiles in memory at + # run time. + def place_pip + bundled = Dir.glob(File.join(src_dir, "Lib", "ensurepip", "_bundled", "pip-*-py3-none-any.whl")) + unless bundled.one? + raise TebakoPythonBuilder::Error.new( + "expected exactly one bundled pip wheel under #{src_dir}/Lib/ensurepip/_bundled " \ + "(found #{bundled.size}) — the wheel is the pip source", 105 + ) + end + + target = File.join(staged_prefix_tree, "lib", @python.libdir_name, "site-packages") + FileUtils.mkdir_p(target) + snippet = "import runpy, sys; wheel, links, target = sys.argv[1:4]; " \ + "sys.path = [wheel] + sys.path; " \ + "sys.argv = ['pip', 'install', '--no-cache-dir', '--no-index', " \ + "'--find-links', links, '--target', target, '--upgrade', '--no-compile', " \ + "'--disable-pip-version-check', 'pip']; " \ + "runpy.run_module('pip', run_name='__main__', alter_sys=True)" + TebakoPythonBuilder::BuildHelpers.run_with_capture( + [exe_path, "-W", "ignore::DeprecationWarning", "-E", "-c", snippet, + bundled.first, File.dirname(bundled.first), target], + env: { "PYTHONHOME" => nil, "PYTHONPATH" => nil }, chdir: src_dir + ) + end + public # The staged prefix tree (the image root's content): stage/ diff --git a/build/lib/tebako_python_builder/python_version.rb b/build/lib/tebako_python_builder/python_version.rb index 578bd5e..4b01d83 100644 --- a/build/lib/tebako_python_builder/python_version.rb +++ b/build/lib/tebako_python_builder/python_version.rb @@ -123,10 +123,14 @@ def libdir_name "python#{major_minor.join(".")}" end - # The shared libpython name of an msys --enable-shared build - # (libpython3.13.dll) — kept for the day the windows leg answers the - # issue-40 question with "shared". The v1 windows leg builds - # --disable-shared (no DLL facet). + # The shared libpython name of the msys --enable-shared build + # (libpython3.14.dll) — the PE name the exe's imports resolve. The + # build stages it beside the runtime package as .dll + # (unique per leg); the store materializes it under THIS name at + # install (the release manifest's dll.install_as), the layout card + # declares it as runtime_dll (schema_minor 3), and tools/boot_smoke + # materializes it in-leg. This method is the name's single owner + # (invariant 10). def msys_dll_name "libpython#{major_minor.join(".")}.dll" end diff --git a/build/lib/tebako_python_builder/source_fetcher.rb b/build/lib/tebako_python_builder/source_fetcher.rb index 6fceea0..9c3ac24 100644 --- a/build/lib/tebako_python_builder/source_fetcher.rb +++ b/build/lib/tebako_python_builder/source_fetcher.rb @@ -28,13 +28,16 @@ require "fileutils" module TebakoPythonBuilder - # Download, SHA256-verification and caching of the pristine CPython - # source published by tamatebako/python: the release carries one - # tfs-python--src.tar.gz asset per published version plus a - # SHA256SUMS manifest (the trust anchor — a checksum mismatch is a hard - # error, never a refetch). Zero patches (the source factory's contract), - # so there are no per-platform scenario assets (the ruby factory's - # scenario_asset_names has no analog here). + # Download, SHA256-verification and caching of the CPython source + # published by tamatebako/python: the release carries one + # tfs-python--src[-].tar.gz asset per published + # version per scenario plus a SHA256SUMS manifest (the trust anchor — a + # checksum mismatch is a hard error, never a refetch). The linux-gnu + # scenario ships unsuffixed for back-compat (the pristine upstream + # tree); windows-msys carries the line's msys2/ucrt64 patch series + # (tamatebako/python patches//, TODO.python/05) — the + # scenario_asset_names convention mirrors the ruby factory's, minus + # its msys two-pass split (CPython needs one tree per scenario). class SourceFetcher REPO = "tamatebako/python" @@ -48,18 +51,30 @@ def asset_name(python_version) "tfs-python-#{python_version}-src.tar.gz" end - # Returns [tarball_path, sha256] for the requested python version - def fetch(python_version) - fetch_asset(asset_name(python_version)) + # The one scenario asset a platform consumes. linux-gnu (and every + # other POSIX host) rides the unsuffixed back-compat asset; a + # mingw/ucrt host needs the line's patched windows-msys tree. + def self.scenario_asset_name(python_version, platform) + base = "tfs-python-#{python_version}-src" + platform.msys? ? "#{base}-windows-msys.tar.gz" : "#{base}.tar.gz" end - # The published sha256 of a version's tarball, read from the pinned - # release's SHA256SUMS. This is the per-version cache-key input of the - # build workflow's .build cache: the key and the download-time + # Returns [tarball_path, sha256] for the requested python version. + # With a platform, fetches that platform's scenario asset (the + # windows-msys tree on a mingw host); without one, the unsuffixed + # linux-gnu asset (the historical default). + def fetch(python_version, platform: nil) + fetch_asset(platform ? self.class.scenario_asset_name(python_version, platform) : asset_name(python_version)) + end + + # The published sha256 of a version's scenario tarball, read from the + # pinned release's SHA256SUMS. This is the per-version cache-key input + # of the build workflow's .build cache: the key and the download-time # verification in #fetch_asset share the same source of truth, so a - # cache restores only trees built from exactly these bytes. - def tarball_sha256(python_version) - expected_sha256(asset_name(python_version)) + # cache restores only trees built from exactly these bytes. The + # platform keyword selects the scenario exactly as #fetch's does. + def tarball_sha256(python_version, platform: nil) + expected_sha256(platform ? self.class.scenario_asset_name(python_version, platform) : asset_name(python_version)) end # Returns [tarball_path, sha256] for the named release asset diff --git a/build/site-packages.yml b/build/site-packages.yml index 1399a31..db69d42 100644 --- a/build/site-packages.yml +++ b/build/site-packages.yml @@ -25,8 +25,10 @@ # The site-packages set the env image ships beyond the stdlib (the # declarative selection — README's follow-up 4, landed). `make install`'s -# ensurepip lands pip; entries not named here are pruned at image -# assembly (a name keeps itself and its `-*.dist-info`). +# staged install lands pip (POSIX rides `make install`'s ensurepip; msys +# places the bundled wheel explicitly — PythonBuild#place_pip); entries +# not named here are pruned at image assembly (a name keeps itself and +# its `-*.dist-info`). --- keep: - pip diff --git a/ci/check_symbol_provenance.sh b/ci/check_symbol_provenance.sh index 3f6e3e3..a9d2cbb 100755 --- a/ci/check_symbol_provenance.sh +++ b/ci/check_symbol_provenance.sh @@ -60,8 +60,15 @@ fail() { echo "::error::provenance: $*"; failures=$((failures + 1)); } note() { echo "provenance: $*"; } # capture-then-test everywhere (never nm | grep -q under pipefail: grep's -# early exit SIGPIPEs nm into a false negative). +# early exit SIGPIPEs nm into a false negative). An empty capture earns +# nm's own diagnostic as evidence — a silent nm failure (a path the +# native tool cannot open, an unreadable file) must not read as +# "stripped". symbols="$(nm "$EXE" 2>/dev/null || true)" +if [ -z "$symbols" ]; then + echo "provenance: nm yielded no symbol table for $EXE; nm's own diagnostic:" + nm "$EXE" 2>&1 | head -5 || true +fi for sym in tebako_driver_boot tebako_mount_point tebako_driver_contract_version main; do line="$(printf '%s\n' "$symbols" | grep -E "[TtWwDd] _?${sym}\$" || true)" @@ -79,14 +86,22 @@ done # objdump that cannot honor --disassemble= yields no body and the # llvm-objdump spelling is tried next; neither producing the body is a # failure — never a silently skipped provenance check. +# +# The greps below are capture-then-test for the same reason the nm side +# is: grep -q exits on its first match, the still-writing printf dies of +# SIGPIPE, and pipefail reads the pipeline as 141 — a false "not found". +# A binutils PE objdump's --disassemble= span runs from the symbol +# to the next data island (thousands of lines — past the pipe buffer), +# so the windows leg hit exactly that (run 34759975017: main's body DOES +# call tebako_driver_boot, printed in the failure's own evidence). dis="" if command -v objdump >/dev/null 2>&1; then dis="$(objdump -d --disassemble=main "$EXE" 2>/dev/null || true)" fi -if ! printf '%s\n' "$dis" | grep -q 'main>:' && command -v llvm-objdump >/dev/null 2>&1; then +if [ -z "$(printf '%s\n' "$dis" | grep 'main>:' || true)" ] && command -v llvm-objdump >/dev/null 2>&1; then dis="$(llvm-objdump -d --disassemble-symbols=main "$EXE" 2>/dev/null || true)" fi -if ! printf '%s\n' "$dis" | grep -q 'main>:' && command -v otool >/dev/null 2>&1; then +if [ -z "$(printf '%s\n' "$dis" | grep 'main>:' || true)" ] && command -v otool >/dev/null 2>&1; then # darwin last resort: /usr/bin/objdump is a deprecation shim and # llvm-objdump sits behind xcrun (off PATH) — otool (cctools) is # always there. Mach-O labels are bare `_symbol:` lines; take the body @@ -95,7 +110,7 @@ if ! printf '%s\n' "$dis" | grep -q 'main>:' && command -v otool >/dev/null 2>&1 fi if [ -z "$dis" ]; then fail "no objdump could disassemble main in $EXE — the forwarding check cannot run" -elif printf '%s\n' "$dis" | grep -q tebako_driver_boot; then +elif [ -n "$(printf '%s\n' "$dis" | grep tebako_driver_boot || true)" ]; then note "main forwards to tebako_driver_boot (the fs TU is the entry point)" else fail "main in $EXE does not reference tebako_driver_boot — CPython's own main, not the fs TU (the Makefile substitution never landed)" diff --git a/contract.yml b/contract.yml index c3c0886..6ac3e57 100644 --- a/contract.yml +++ b/contract.yml @@ -77,14 +77,57 @@ container_version: "v1" link_unit_release: "v2.3.2" # The tamatebako/python SOURCE release this factory consumes -# (tfs-python--src.tar.gz + SHA256SUMS, sha256-verified). -# v0.2.0 (2026-09-05) adds 3.11.16; the v0.1.0 set (3.12.14, 3.13.15, -# 3.14.7) carried forward sha256-verified. A build leg fails closed -# while this is empty. -source_release: "v0.2.0" +# (tfs-python--src[-].tar.gz + SHA256SUMS, +# sha256-verified). v0.3.0 adds the platform-scenario axis: 3.14.7 ships +# tfs-python-3.14.7-src-windows-msys.tar.gz (the msys2/ucrt64 patch +# series, TODO.python/05 item 1) alongside the unsuffixed linux-gnu +# back-compat asset every version carries. v0.3.1 carries the +# MSYS2-MINGW-packages 0007 port (tamatebako/python#16): the checked-in +# PC/errmap.h needs -I$(srcdir)/PC on the msys build path, which +# upstream's Makefile grants MSVC only. v0.3.2 carries the MSYS2 sweep +# for the static ucrt64 shape (tamatebako/python#17): 18 more patches +# ported (0001 nt-threads, 0005 mingw configure defaults, 0008/0016/ +# 0017/0036/0037/0041/0045/0054/0056/0059/0060/0063/0075/0078/0097/0100) +# plus the syslibs deviation — the round-2/3 compile+link walls +# (VPATH, thread_pthread, alarm//dev/ptmx probes, winconsoleio, +# version/pathcch libs). v0.3.3 carries the mingw shared-build +# machinery (tamatebako/python#18): 0009/0010 (declspec exports + the +# enable_shared mingw arms — LDLIBRARY=libpython3.14.dll.a / +# DLLLIBRARY=libpython3.14.dll), the dynload_win cluster (0011-0015 — +# the .cp314-mingw_x86_64_ucrt_gnu.pyd rename), the two interpreter +# boot walls the PR's own smoke leg forced (0018: winreg/msvcrt/ +# _winapi bootstrap-static; 0034: sys.winver/dllhandle under +# defined(MS_WINDOWS) && defined(Py_ENABLE_SHARED)), 0066 +# (_Py_CheckPython3), and 0039 (-municode + win resources + +# pythonw.exe). v0.3.4 carries the sysconfig posix-mode pair +# (tamatebako/python#19): 0003 (the sys._is_mingw flags) + 0020 (mingw +# builds take the posix sysconfig path) — the pybuilddir.txt wall the +# first --enable-shared leg hit (trp run 34748169561: KeyError 'LIBPL' +# — upstream's nt path never merges the Makefile-parsed +# build_time_vars); 0034's hunk 3 returns to MSYS2-verbatim context. +# v0.3.5 carries the module-libs cluster (tamatebako/python#20): 0061 + +# 0073 (_multiprocessing: the win32 enable condition, then -lws2_32), +# 0074 (select: _MSC_VER → MS_WIN32 guards + -lws2_32), and 0076 +# (_ctypes: -lole32 -loleaut32 -luuid) — the shared-extension link +# wall the v0.3.4 leg hit (trp run 34750005177: __WSAFDIsSet/ +# __imp_select/__imp_WSAGetLastError; PE resolves every symbol at link +# time and upstream's PY_STDLIB_MOD stanzas carry no per-module libs). +# v0.3.6 carries the mmap + legacy-DLL-loading trio +# (tamatebako/python#21): 0062 (mmap builds on win32 — its absence +# killed ensurepip in `make install`, trp run 34753278108: pip's +# vendored cachecontrol imports mmap), 0067 +# (PYTHONLEGACYWINDOWSDLLLOADING — the bpo-36085 DEFAULT_DIRS search +# never consults PATH, so the build-tree check removed +# _decimal/_zstd/_testinternalcapi whose dep DLLs live in +# /bin), and 0064 (have_dynamic_loading for the dlopen-gated +# test modules); 0076 returns to MSYS2-verbatim context. +# v0.2.0 (2026-09-05) added +# 3.11.16; the v0.1.0 set (3.12.14, 3.13.15, 3.14.7) carried forward +# sha256-verified. A build leg fails closed while this is empty. +source_release: "v0.3.6" # The python version catalog this factory builds and publishes. Mirrors -# tamatebako/python's versions.yml at source release v0.2.0 (3.11.16, +# tamatebako/python's versions.yml at source release v0.3.6 (3.11.16, # 3.12.14, 3.13.15, 3.14.7). The source factory owns the # version PINS (url/sha256/line); this catalog selects which published # versions the matrix grammar expands over — the same split as diff --git a/scripts/compute_matrix.rb b/scripts/compute_matrix.rb index 7cfa692..26a4029 100755 --- a/scripts/compute_matrix.rb +++ b/scripts/compute_matrix.rb @@ -104,6 +104,10 @@ # default; upstream owner: tamatebako/tebako release.yml matrix.platform). LINK_UNIT_PID = TebakoPythonBuilder::Platform::LINK_UNIT_PIDS +# A stand-in mingw/ucrt host, used only to ask SourceFetcher for the +# windows-msys scenario asset name (its grammar branches on #msys?). +MINGW_PLATFORM = TebakoPythonBuilder::Platform.new("x86_64-w64-mingw32").freeze + def usage_error(message) warn "compute_matrix: #{message}" exit 64 @@ -123,6 +127,9 @@ def usage_error(message) container_version = contract.fetch("container_version") do usage_error "#{CONTRACT_YML} carries no container_version pin" end +source_release = contract.fetch("source_release") do + usage_error "#{CONTRACT_YML} carries no source_release pin" +end python_filter = ENV.fetch("PYTHON_FILTER", "full") pythons = @@ -150,6 +157,29 @@ def usage_error(message) env = JSON.parse(File.read(MATRIX_JSON)).fetch("env") +# A windows leg builds only from the line's msys2/ucrt64 scenario tree — +# upstream CPython has zero mingw support (TODO.python/05), so the leg's +# source is tfs-python--src-windows-msys.tar.gz in the pinned +# source release's SHA256SUMS. Lines whose series has not shipped yet +# skip windows LOUDLY (their POSIX legs are unaffected). The SHA256SUMS +# read is lazy and scoped to a windows env row surviving the filters — +# POSIX-only triggers never fetch. +windows_sums = nil +windows_buildable = lambda do |base_version| + windows_sums ||= begin + TebakoPythonBuilder::SourceFetcher.new( + release: source_release, + cache_dir: File.join(REPO_ROOT, ".build", "downloads") + ).sha256sums + rescue TebakoPythonBuilder::Error => e + # An unreadable pinned release (a 404 while the pin names a release + # that does not exist yet) is a config-class failure — the same + # exit-64 discipline as an unknown filter, never a stack trace. + usage_error "cannot read the pinned source release's SHA256SUMS: #{e.message}" + end + windows_sums.key?(TebakoPythonBuilder::SourceFetcher.scenario_asset_name(base_version, MINGW_PLATFORM)) +end + legs = [] selected_env = [] env.each do |row| @@ -181,6 +211,22 @@ def usage_error(message) warn "note: #{python} skipped on linux-musl/#{arch} — CPython's JIT target whitelist rejects *-linux-musl (upstream)" next end + # The same whitelist admits only MSVC windows targets + # (x86_64-pc-windows-msvc): the ucrt64 leg's x86_64-w64-mingw32 triple + # is rejected upstream, so jit lines do not build on windows either. + if line.jit? && os == "windows" + warn "note: #{python} skipped on windows/#{arch} — CPython's JIT target whitelist admits MSVC windows only (upstream)" + next + end + # A windows leg exists only when the pinned source release ships the + # line's msys2/ucrt64 scenario asset (tamatebako/python's + # patches// series, TODO.python/05). POSIX legs are unaffected. + if os == "windows" && !windows_buildable.call(line.base_version) + warn "note: #{python} skipped on windows/#{arch} — #{source_release} ships no " \ + "#{TebakoPythonBuilder::SourceFetcher.scenario_asset_name(line.base_version, MINGW_PLATFORM)} " \ + "(the line's msys2/ucrt64 series lands in a later tamatebako/python release)" + next + end legs << { python: python, os: os, diff --git a/scripts/upload_release.rb b/scripts/upload_release.rb index 0f8926d..17f2d8c 100755 --- a/scripts/upload_release.rb +++ b/scripts/upload_release.rb @@ -137,11 +137,10 @@ def load_contract_version # one-per-package so consumers match on python_version / platform / # filename, and a .tfs file never becomes a top-level entry of its own. # The additive `contract_version` key (roadmap 45) follows the same - # compat rule, and so would a windows shared build's libpython DLL + # compat rule, and so does a windows leg's libpython DLL # (.dll) folded as `dll` with the PE name the store entry - # materializes (`install_as`) — the v1 windows leg builds - # --disable-shared, so no dll ships today (PythonVersion#msys_dll_name - # keeps the name for the day the leg answers "shared"). + # materializes (`install_as`; PythonVersion#msys_dll_name is the + # name's single owner). def build_manifest_entries(packages) # rubocop:disable Metrics/AbcSize executables, images, dlls = partition_packages(packages) executables.sort_by { |package| package.basename.to_s }.map do |package| @@ -510,8 +509,8 @@ def image_entry(image) } end - # The additive libpython-DLL metadata (windows, the --enable-shared - # fallback shape — nothing ships it today): the asset rides under the + # The additive libpython-DLL metadata (windows legs, --enable-shared + # since v0.3.3's MSYS2-series port): the asset rides under the # package's unique name; `install_as` is the PE name the store entry # materializes next to the exe so the exe's imports resolve (the single # owner of that name is PythonVersion#msys_dll_name — consumers ignore @@ -1081,13 +1080,15 @@ def missing_package_assets(present, name) end # The non-executable artifacts a package is expected to carry: the - # filesystem image everywhere. The windows libpython DLL is NOT - # expected — the v1 windows leg builds --disable-shared, so the dll - # facet is opportunistic (folded into the package's entry when the - # workspace carries one, never required); the day a leg answers the - # shared question, the expectation joins this list in the same PR. + # filesystem image everywhere, plus the runtime DLL on windows — the + # windows-msys legs build --enable-shared (the source release's MSYS2 + # 0009/0010 machinery, v0.3.3+), so the staged .dll is as + # required as the image: a leg that failed to stage it is an + # incomplete release, never an optional facet. def expected_facets(name) - ["#{name}.tfs"] + facets = ["#{name}.tfs"] + facets << "#{name}.dll" if name.include?("windows") + facets end # The metadata a landed package owes: one sidecar per landed asset plus @@ -1098,8 +1099,8 @@ def missing_metadata(present, name, landed) end # An unowned DLL means a leg's upload never landed. (The reverse gap — - # a windows executable with no DLL — is the CORRECT shape here: the - # --disable-shared build links libpython statically.) + # a windows executable with no DLL — is a completeness failure caught + # by expected_facets: the windows-msys legs build --enable-shared.) def report_dll_gaps(executables, dlls) (dlls - executables).sort.each do |name| puts "::warning::libpython DLL #{name}.dll has no matching runtime package; " \ diff --git a/spec/source_fetcher_spec.rb b/spec/source_fetcher_spec.rb new file mode 100644 index 0000000..cffce6e --- /dev/null +++ b/spec/source_fetcher_spec.rb @@ -0,0 +1,106 @@ +# frozen_string_literal: true + +# Copyright (c) 2026 [Ribose Inc](https://www.ribose.com). +# All rights reserved. +# This file is a part of tamatebako +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS +# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +require "spec_helper" +require "digest" +require "fileutils" +require "tmpdir" + +RSpec.describe TebakoPythonBuilder::SourceFetcher do + let(:msys_platform) { TebakoPythonBuilder::Platform.new("x86_64-w64-mingw32") } + let(:posix_platform) { TebakoPythonBuilder::Platform.new("x86_64-pc-linux-gnu") } + + # A staged release mirror: SHA256SUMS + the asset bytes it names, + # served over file:// — no spec touches the network. + def stage_release(dir, assets) + FileUtils.mkdir_p(dir) + sums = assets.map do |name, bytes| + File.binwrite(File.join(dir, name), bytes) + "#{Digest::SHA256.hexdigest(bytes)} #{name}" + end + File.write(File.join(dir, "SHA256SUMS"), "#{sums.join("\n")}\n") + end + + def staged_fetcher(dir, assets) + mirror = File.join(dir, "mirror") + stage_release(mirror, assets) + described_class.new(cache_dir: File.join(dir, "cache"), release: "v9.9.9", mirror: "file://#{mirror}") + end + + describe ".scenario_asset_name" do + it "names the unsuffixed back-compat asset for a POSIX host" do + expect(described_class.scenario_asset_name("3.14.7", posix_platform)) + .to eq("tfs-python-3.14.7-src.tar.gz") + end + + it "names the windows-msys scenario asset for a mingw host" do + expect(described_class.scenario_asset_name("3.14.7", msys_platform)) + .to eq("tfs-python-3.14.7-src-windows-msys.tar.gz") + end + end + + describe "#fetch" do + it "fetches the platform's scenario asset, verified against SHA256SUMS" do + Dir.mktmpdir do |dir| + fetcher = staged_fetcher(dir, "tfs-python-3.14.7-src.tar.gz" => "pristine", + "tfs-python-3.14.7-src-windows-msys.tar.gz" => "patched") + + path, sha = fetcher.fetch("3.14.7", platform: msys_platform) + expect(File.basename(path)).to eq("tfs-python-3.14.7-src-windows-msys.tar.gz") + expect(File.binread(path)).to eq("patched") + expect(sha).to eq(Digest::SHA256.hexdigest("patched")) + + posix_path, = fetcher.fetch("3.14.7", platform: posix_platform) + default_path, = fetcher.fetch("3.14.7") + expect([File.basename(posix_path), File.basename(default_path)]) + .to eq(["tfs-python-3.14.7-src.tar.gz", "tfs-python-3.14.7-src.tar.gz"]) + end + end + end + + describe "#tarball_sha256" do + it "reads the platform's scenario asset sum from the pinned SHA256SUMS" do + Dir.mktmpdir do |dir| + fetcher = staged_fetcher(dir, "tfs-python-3.14.7-src.tar.gz" => "pristine", + "tfs-python-3.14.7-src-windows-msys.tar.gz" => "patched") + + expect(fetcher.tarball_sha256("3.14.7", platform: msys_platform)) + .to eq(Digest::SHA256.hexdigest("patched")) + expect(fetcher.tarball_sha256("3.14.7")).to eq(Digest::SHA256.hexdigest("pristine")) + end + end + + it "fails named when the line's windows-msys asset is absent from SHA256SUMS" do + Dir.mktmpdir do |dir| + fetcher = staged_fetcher(dir, "tfs-python-3.13.15-src.tar.gz" => "pristine") + + expect { fetcher.tarball_sha256("3.13.15", platform: msys_platform) } + .to raise_error(TebakoPythonBuilder::Error, /tfs-python-3\.13\.15-src-windows-msys\.tar\.gz/) + end + end + end +end diff --git a/tools/boot_smoke b/tools/boot_smoke index e24a073..c2c3303 100755 --- a/tools/boot_smoke +++ b/tools/boot_smoke @@ -35,7 +35,11 @@ # stdlib is image-resident (the two-artifact # model), so the driver warns about the bare # boot and the interpreter fails on its own -# terms (non-zero exit; every platform) +# terms (non-zero exit + the encodings fatal on +# POSIX; on windows upstream's winreg fallback +# can hand the child a host-registered stdlib, +# so only the driver's warning and getpath's +# platstdlib warning are pinned there) # boot TEBAKO_RUNTIME_IMAGE handoff: stdlib imports # (json/zlib/ssl) off the mounted image, sys.prefix # == the mount root [POSIX] @@ -265,6 +269,25 @@ rescue TebakoPythonBuilder::Error nil end +# The msys shared build: the exe's PE imports resolve libpython.dll +# in the exe's own directory (the OS's application-dir rule — before any +# driver code runs), but the package dir holds the DLL under the unique +# package name (.dll — two same-ABI legs would collide on the PE +# name). Materialize the PE-named copy next to the exe BEFORE any +# scenario boots it, mirroring the store entry the product's install +# assembles (exe + image + DLL under the manifest's dll.install_as name). +# A missing package DLL degrades to the OS loader's own error — loud +# enough at this layer. +def materialize_python_dll(executable, platform, line) + return unless platform.msys? && line + + source = "#{executable.sub(/\.exe\z/, "")}.dll" + return unless File.file?(source) + + dest = File.join(File.dirname(executable), line.msys_dll_name) + FileUtils.cp(source, dest) unless File.file?(dest) +end + # The JIT capability probe, per line (the header's jit block): 3.14+ # answers sys._jit; 3.13 has no sys._jit, so the probe reads the # configure-time -D_Py_JIT out of the installed sysconfig. Runs under @@ -322,18 +345,35 @@ def run_scenarios(executable, platform, selected, tfs:, line:) # rubocop:disable case scenario when "bare" # The two-artifact model: the exe alone is NOT a dev-mode - # interpreter — the stdlib lives in the env image, so a bare - # boot can only fail (getpath's compiled-in prefix resolves - # empty and the encodings import dies). What this scenario pins - # is the DRIVER's half of the contract: it names the bare boot - # on stderr, then the interpreter fails on its own terms. + # interpreter — the stdlib lives in the env image. What this + # scenario pins on every platform is the DRIVER's half of the + # contract: it names the bare boot on stderr. out, err, st = boot(executable, ["-c", "print('unreached')"], {}) - check("bare: non-zero exit (the stdlib is image-resident)", - !st.nil? && !st.exitstatus&.zero?, "stdout: #{out}\nstderr: #{err}") check("bare: the driver names the bare boot", err.include?("TEBAKO_RUNTIME_IMAGE is not set"), "stderr: #{err}") - check("bare: the interpreter ran and died on the missing stdlib", - err.include?("encodings"), "stderr: #{err}") + if platform.msys? + # Windows startup is environment-dependent by upstream design: + # Modules/getpath.py ("os_name == 'nt' and use_environment and + # winreg") appends the HKCU/HKLM SOFTWARE\Python\PythonCore\\ + # PythonPath registry values to sys.path, so on a host with a + # REGISTERED python (the GHA runner ships one) the child finds a + # stdlib, imports encodings, and RUNS the -c program — observed on + # the 0dc372c windows leg (run 34760958198): exit 0, 'unreached' + # printed. On a registry-clean host it dies with the encodings + # fatal like POSIX. Exit status and the encodings message are + # therefore not assertable here; the deterministic upstream + # surface is getpath's platstdlib warning (the prefix fell back + # to the baked-in mount root, which never exists on the runner). + check("bare: getpath names the missing platstdlib", + err.include?("Could not find platform independent libraries"), "stderr: #{err}") + else + # POSIX: no registry fallback — getpath's compiled-in prefix + # resolves empty and the encodings import dies at Py_Initialize. + check("bare: non-zero exit (the stdlib is image-resident)", + !st.nil? && !st.exitstatus&.zero?, "stdout: #{out}\nstderr: #{err}") + check("bare: the interpreter ran and died on the missing stdlib", + err.include?("encodings"), "stderr: #{err}") + end when "boot" unless File.file?(image) check("boot", false, "no sibling env image #{image}") @@ -402,6 +442,7 @@ executable = resolve_executable(options[:runtime_root], options[:exe]) puts "-- Boot smoke: #{executable} (#{platform.host_id})" selected = selected_scenarios(platform, options[:scenario]) line = expected_line(executable, platform) +materialize_python_dll(executable, platform, line) if selected.include?("jit") && line.nil? # The jit assertion needs the declared line; an --exe path outside the # runtime-name grammar (a dev build) can't carry one. An explicit ask