save reproducer script

docs/metadata.md

@@ -59,7 +59,8 @@ available.
         * **status**: target status: "PASS", "FAIL", or "SKIP" (string).
         * **duration**: duration of this target build, in seconds (number).
     - **artifacts**: key/value with target names (string) as keys, and list of
-      artifacts built for that target (list of strings).
+      artifacts built for that target (list of strings). Two keys are not
+      targets: `log` for the build logs, and `reproducer` for `reproducer.sh`.
     - **errors**: number of errors in the build (integer).
     - **warnings**: number of warnings in the build (integer).
 - **sccache**: sccache statistics.

docs/reproducible_builds.md

@@ -61,6 +61,22 @@ the one Alice did on their side. The only artifact that will be different is
 `metadata.json` and `build.log`, because both include data about their local
 systems.
 
+## The reproducer script
+
+The same command line is also saved as `reproducer.sh` in the output directory.
+It is executable, so person A can just send the file to person B. B runs it
+from the root of the same kernel tree:
+
+```
+linux (master) $ /path/to/artifacts/reproducer.sh
+```
+
+The script does not set up the kernel tree. You have to be in the tree you want
+to build.
+
+The reproducer.sh is written before the build starts, so you also get it when
+the build fails.
+
 ## On container images
 
 As described above, the easiest way of ensuring a consistent build environment

test/test_build.py

@@ -3,6 +3,7 @@
 import os
 import pytest
 import re
+import shlex
 import subprocess
 import shutil
 import urllib
@@ -968,6 +969,24 @@ def test_targets(self, metadata):
     def test_command_line(self, metadata):
         assert type(metadata["build"]["reproducer_cmdline"]) is list
 
+    @pytest.fixture(scope="class")
+    def reproducer(self, build):
+        return build.output_dir / "reproducer.sh"
+
+    def test_reproducer_script_written_for_failed_build(self, reproducer):
+        assert reproducer.read_text().startswith("#!/bin/sh\n")
+
+    def test_reproducer_script_is_executable(self, reproducer):
+        assert os.access(reproducer, os.X_OK)
+
+    def test_reproducer_script_runs_the_reproducer_cmdline(self, reproducer, metadata):
+        command = reproducer.read_text().split("\nexec ", 1)[1]
+        command = command.replace("\\\n", "")
+        assert shlex.split(command) == metadata["build"]["reproducer_cmdline"]
+
+    def test_reproducer_script_is_an_artifact(self, metadata):
+        assert metadata["results"]["artifacts"]["reproducer"] == ["reproducer.sh"]
+
 
 class TestParseLog:
     @pytest.fixture(scope="class")

test/test_cmdline.py

@@ -55,6 +55,16 @@ def test_environment(self, cmdline):
         cmd = cmdline.reproduce(build)
         assert "--environment=FOO=BAR" in cmd
 
+    def test_environment_without_local_kcflags(self, cmdline):
+        build = Build()
+        cmd = cmdline.reproduce(build)
+        assert [o for o in cmd if o.startswith("--environment=KCFLAGS=")] == []
+
+    def test_environment_with_kcflags_from_the_user(self, cmdline):
+        build = Build(environment={"KCFLAGS": "-Werror"})
+        cmd = cmdline.reproduce(build)
+        assert "--environment=KCFLAGS=-Werror" in cmd
+
     def test_kconfig_add(self, cmdline):
         build = Build(kconfig_add=["foo.config", "bar.config"])
         cmd = cmdline.reproduce(build)

tuxmake/build.py

@@ -290,7 +290,10 @@ def __init__(
 
         self.offline = False
 
-        self.artifacts = {"log": ["build.log", "build-debug.log"]}
+        self.artifacts = {
+            "log": ["build.log", "build-debug.log"],
+            "reproducer": ["reproducer.sh"],
+        }
         self.__status__ = {}
         self.__durations__ = {}
         self.metadata_collector = MetadataCollector(self)
@@ -446,6 +449,15 @@ def environment(self):
         self.__environment__ = env
         return self.__environment__
 
+    @property
+    def reproducible_environment(self):
+        # Our KCFLAGS points at the local build dir, so the next build has
+        # to set its own.
+        env = dict(self.environment)
+        if "KCFLAGS" not in self.__environment_input__:
+            del env["KCFLAGS"]
+        return env
+
     def get_silent(self):
         if self.verbose:
             return []
@@ -726,6 +738,15 @@ def save_metadata(self):
         tmp.write_text(json.dumps(self.metadata, indent=4, sort_keys=True) + "\n")
         os.replace(tmp, path)
 
+    def save_reproducer(self):
+        # Atomic rename, so a hard kill mid-write can't leave a half-written file.
+        path = self.output_dir / "reproducer.sh"
+        tmp = self.output_dir / "reproducer.sh.tmp"
+        command = quote_command_line(self.cmdline.reproduce(self), " \\\n  ")
+        tmp.write_text(f"#!/bin/sh\nset -eu\nexec {command}\n")
+        tmp.chmod(0o755)
+        os.replace(tmp, path)
+
     def parse_log(self):
         parser = LogParser()
         parser.parse(self.output_dir / "build.log")
@@ -849,6 +870,7 @@ def run(self):
             with self.measure_duration("Early Metadata Extraction"):
                 self.collect_metadata_early()
             self.save_metadata()
+            self.save_reproducer()
 
             version_full = self.metadata.get("compiler", {}).get("version_full")
             if version_full:

tuxmake/cmdline.py

@@ -326,7 +326,10 @@ def reproduce(self, build):
             if option.key in self.ignore:
                 continue
             if hasattr(build, option.key):
-                value = getattr(build, option.key)
+                if option.key == "environment":
+                    value = build.reproducible_environment
+                else:
+                    value = getattr(build, option.key)
                 if not value:
                     continue
                 for c in option.expand(value):

tuxmake/utils.py

@@ -15,8 +15,8 @@
 PROGRESS_REPORT_CHUNK_SIZE = 1 * MB  # Report progress every 1MB for responsiveness
 
 
-def quote_command_line(cmd: List[str]) -> str:
-    return " ".join([shlex.quote(c) for c in cmd])
+def quote_command_line(cmd: List[str], separator: str = " ") -> str:
+    return separator.join([shlex.quote(c) for c in cmd])
 
 
 def get_directory_timestamp(directory):