Reproducible builds

docs/targets.md

@@ -112,6 +112,10 @@ When this target is built, the `kernel` target is not.
 This target builds the Kernel modules. The modules are compressed in a tarball,
 which is copied into the output directory as `modules.tar.xz`.
 
+The `build` and `source` symlinks that `modules_install` creates are left out
+of the tarball. They point at the local build and source directories, so they
+are broken anywhere else, and they made the tarball different on every build.
+
 
 ## headers
 

test/test_build.py

@@ -1243,6 +1243,35 @@ def test_reproducible_sets_constant_values(self, linux):
         ts = "KBUILD_BUILD_TIMESTAMP"
         assert build1.environment[ts] == build2.environment[ts]
 
+    def test_maps_the_build_dir_and_the_source_tree(self, linux):
+        build = Build(tree=linux)
+        assert build.environment["KCFLAGS"] == (
+            f"-ffile-prefix-map={build.build_dir}=/tuxmake "
+            f"-ffile-prefix-map={build.source_tree}/="
+        )
+
+    def test_maps_the_same_for_assembly(self, linux):
+        env = Build(tree=linux).environment
+        assert env["KAFLAGS"] == env["KCFLAGS"]
+
+
+class TestGitWorktree:
+    @pytest.fixture
+    def worktree(self, linux_rw, tmp_path, mocker):
+        mocker.patch("tuxmake.build.get_directory_timestamp", return_value="1")
+        git_dir = tmp_path / "main" / ".git"
+        (git_dir / "worktrees" / "wt").mkdir(parents=True)
+        (git_dir / "worktrees" / "wt" / "commondir").write_text("../..\n")
+        (linux_rw / ".git").write_text(f"gitdir: {git_dir}/worktrees/wt\n")
+        return linux_rw, git_dir
+
+    def test_mounts_the_git_dir(self, worktree, mocker, Popen):
+        tree, git_dir = worktree
+        add_volume = mocker.patch("tuxmake.runtime.Runtime.add_volume")
+        Build(tree=tree).prepare()
+        mounted = [call[0][0] for call in add_volume.call_args_list]
+        assert git_dir in mounted
+
 
 class TestTerminated:
     def test_signal_handler_raises_exception(self):

test/test_cmdline.py

@@ -55,10 +55,11 @@ def test_environment(self, cmdline):
         cmd = cmdline.reproduce(build)
         assert "--environment=FOO=BAR" in cmd
 
-    def test_environment_without_local_kcflags(self, cmdline):
+    @pytest.mark.parametrize("var", ["KCFLAGS", "KAFLAGS"])
+    def test_environment_without_local_prefix_map(self, cmdline, var):
         build = Build()
         cmd = cmdline.reproduce(build)
-        assert [o for o in cmd if o.startswith("--environment=KCFLAGS=")] == []
+        assert [o for o in cmd if o.startswith(f"--environment={var}=")] == []
 
     def test_environment_with_kcflags_from_the_user(self, cmdline):
         build = Build(environment={"KCFLAGS": "-Werror"})

test/test_target.py

@@ -97,6 +97,11 @@ def test_strip_modules(self, modules):
     def test_depends_on_config(self, modules):
         assert modules.dependencies == ["config"]
 
+    def test_leaves_out_the_build_dir_symlinks(self, modules):
+        tar = modules.commands[2]
+        assert "--exclude=lib/modules/*/build" in tar
+        assert "--exclude=lib/modules/*/source" in tar
+
 
 class TestDtbs:
     def test_commands(self, build):

test/test_utils.py

@@ -3,6 +3,7 @@
 import pytest
 from unittest.mock import patch, MagicMock
 from tuxmake.utils import get_directory_timestamp
+from tuxmake.utils import get_git_dir
 from tuxmake.utils import retry
 from tuxmake.utils import download_file_with_progress
 from tuxmake.utils import prepare_file_from_source
@@ -308,3 +309,18 @@ def test_prepare_local_xz_file_without_logger(self, tmp_path):
         mock_print.assert_called_once()
         print_call = mock_print.call_args[0][0]
         assert "Decompressing" in print_call
+
+
+class TestGetGitDir:
+    def test_worktree(self, tmp_path):
+        git_dir = tmp_path / "main" / ".git"
+        (git_dir / "worktrees" / "wt").mkdir(parents=True)
+        (git_dir / "worktrees" / "wt" / "commondir").write_text("../..\n")
+        tree = tmp_path / "wt"
+        tree.mkdir()
+        (tree / ".git").write_text(f"gitdir: {git_dir}/worktrees/wt\n")
+        assert get_git_dir(tree) == git_dir
+
+    def test_normal_tree(self, tmp_path):
+        (tmp_path / ".git").mkdir()
+        assert get_git_dir(tmp_path) is None

tuxmake/build.py

@@ -36,6 +36,7 @@
 from tuxmake.build_utils import defaults
 from tuxmake.utils import quote_command_line
 from tuxmake.utils import get_directory_timestamp
+from tuxmake.utils import get_git_dir
 from tuxmake.utils import prepare_file_from_source
 
 
@@ -166,6 +167,10 @@ class Build:
         "O",
     ]
 
+    # Set from the local build dir, so they are left out of the reproducer
+    # command line. The next build sets its own.
+    LOCAL_ENVIRONMENT = ["KCFLAGS", "KAFLAGS"]
+
     def __init__(
         self,
         tree=".",
@@ -373,6 +378,9 @@ def prepare(self):
         self.runtime.source_dir = self.source_tree
         self.runtime.output_dir = self.output_dir
         self.runtime.add_volume(self.build_dir)
+        git_dir = get_git_dir(self.source_tree)
+        if git_dir:
+            self.runtime.add_volume(git_dir, ro=True)
         if self.prepare_korg_gcc:
             self.runtime.add_volume(self.korg_toolchains_dir)
         if self.wrapper.path:
@@ -444,18 +452,25 @@ def environment(self):
         env["KBUILD_BUILD_TIMESTAMP"] = "@" + self.timestamp
         env["KBUILD_BUILD_USER"] = "tuxmake"
         env["KBUILD_BUILD_HOST"] = "tuxmake"
-        env["KCFLAGS"] = f"-ffile-prefix-map={self.build_dir}/="
+        # The build dir has no trailing slash: the compilation directory is
+        # the build dir itself, and a map with a slash does not match it. The
+        # source tree has one, so the file names come out relative to it.
+        prefix_map = (
+            f"-ffile-prefix-map={self.build_dir}=/tuxmake "
+            f"-ffile-prefix-map={self.source_tree}/="
+        )
+        for var in self.LOCAL_ENVIRONMENT:
+            env[var] = prefix_map
         env.update(self.__environment_input__)
         self.__environment__ = env
         return self.__environment__
 
     @property
     def reproducible_environment(self):
-        # Our KCFLAGS points at the local build dir, so the next build has
-        # to set its own.
         env = dict(self.environment)
-        if "KCFLAGS" not in self.__environment_input__:
-            del env["KCFLAGS"]
+        for var in self.LOCAL_ENVIRONMENT:
+            if var not in self.__environment_input__:
+                del env[var]
         return env
 
     def get_silent(self):

tuxmake/target/modules.ini

@@ -4,7 +4,9 @@ dependencies = config
 preconditions = grep -q CONFIG_MODULES=y {build_dir}/.config
 commands = rm -rf {build_dir}/modinstall
     && {make} modules_install
-    && {tar_caf} {build_dir}/modules.tar{z_ext} -C {build_dir}/modinstall lib
+    && {tar_caf} {build_dir}/modules.tar{z_ext}
+    --exclude=lib/modules/*/build --exclude=lib/modules/*/source
+    -C {build_dir}/modinstall lib
 
 [makevars]
 INSTALL_MOD_STRIP = 1

tuxmake/utils.py

@@ -19,6 +19,20 @@ def quote_command_line(cmd: List[str], separator: str = " ") -> str:
     return separator.join([shlex.quote(c) for c in cmd])
 
 
+def get_git_dir(directory):
+    # In a git worktree, .git is a file that points at a directory outside
+    # the tree. Return that directory, so it can be made available to the
+    # build. Return None for a normal tree, where .git is inside it.
+    dotgit = directory / ".git"
+    if not dotgit.is_file():
+        return None
+    gitdir = Path(dotgit.read_text().partition("gitdir:")[2].strip())
+    commondir = gitdir / "commondir"
+    if commondir.exists():
+        gitdir = gitdir / commondir.read_text().strip()
+    return gitdir.resolve()
+
+
 def get_directory_timestamp(directory):
     if (directory / ".git").exists():
         try: