Set Partial Clone flags on sparse-checkout

internal/job/checkout.go

@@ -762,7 +762,7 @@ func (e *Executor) getOrUpdateMirrorDir(ctx context.Context, repository string)
 
 // fetchSource fetches the git source for the job. If GitSkipFetchExistingCommits is
 // enabled and the commit already exists locally, the fetch is skipped entirely.
-func (e *Executor) fetchSource(ctx context.Context) error {
+func (e *Executor) fetchSource(ctx context.Context, addBloblessFilter bool) error {
 	// If configured, skip the fetch when the commit already exists locally.
 	// This is useful when a pre-populated git mirror is used with --reference,
 	// as the commit objects are already reachable and fetching is redundant.
@@ -773,6 +773,9 @@ func (e *Executor) fetchSource(ctx context.Context) error {
 	}
 
 	gitFetchFlags := e.GitFetchFlags
+	if addBloblessFilter {
+		gitFetchFlags = "--filter=blob:none " + gitFetchFlags
+	}
 
 	switch {
 	case e.RefSpec != "":
@@ -908,6 +911,9 @@ func (e *Executor) defaultCheckoutPhase(ctx context.Context) (retErr error) {
 		return fmt.Errorf("creating checkout dir: %w", err)
 	}
 
+	// Resolve the cone paths to check out (nil means a full checkout).
+	sparsePaths := e.resolveSparseCheckout(ctx)
+
 	// On mirrors and dissociation:
 	//
 	// --reference makes the clone reuse objects from the mirror, using the
@@ -964,6 +970,24 @@ func (e *Executor) defaultCheckoutPhase(ctx context.Context) (retErr error) {
 			}
 		}
 
+		// When sparse checkout applies, add two clone flags:
+		//   --sparse             clone in sparse mode
+		//   --filter=blob:none   make it a partial clone, so blobs outside the
+		//                        sparse set aren't downloaded up front
+		// Each flag is added only if the user hasn't already supplied their own.
+		if len(sparsePaths) > 0 {
+			if slices.Contains(gitCloneFlags, "--sparse") {
+				e.shell.Commentf("Sparse checkout is configured and BUILDKITE_GIT_CLONE_FLAGS already contains a --sparse flag (preserving user-supplied sparse checkout).")
+			} else {
+				gitCloneFlags = append(gitCloneFlags, "--sparse")
+			}
+			if hasPartialCloneFilter(gitCloneFlags) {
+				e.shell.Commentf("Sparse checkout is configured and BUILDKITE_GIT_CLONE_FLAGS already contains a --filter (preserving user-supplied filter).")
+			} else {
+				gitCloneFlags = append(gitCloneFlags, "--filter=blob:none")
+			}
+		}
+
 		// Do the clone.
 		if err := gitClone(ctx, e.shell, gitCloneFlags, e.Repository, "."); err != nil {
 			return fmt.Errorf("cloning git repository: %w", err)
@@ -991,15 +1015,18 @@ func (e *Executor) defaultCheckoutPhase(ctx context.Context) (retErr error) {
 		}
 	}
 
-	if err := e.fetchSource(ctx); err != nil {
+	addBloblessFilter := len(sparsePaths) > 0 &&
+		!strings.Contains(e.GitCloneFlags, "--filter") &&
+		!strings.Contains(e.GitFetchFlags, "--filter")
+	if err := e.fetchSource(ctx, addBloblessFilter); err != nil {
 		return err
 	}
 
 	if err := e.verifyCommit(ctx); err != nil {
 		return err
 	}
 
-	sparseCheckoutActive, err := e.setupSparseCheckout(ctx)
+	sparseCheckoutActive, err := e.setupSparseCheckout(ctx, sparsePaths)
 	if err != nil {
 		return err
 	}

internal/job/checkout_sparse.go

@@ -12,6 +12,31 @@ import (
 	"github.com/buildkite/agent/v3/internal/shell"
 )
 
+// resolveSparseCheckout returns the cone paths to check out for this build, or
+// nil to check out the full tree — either because no paths were requested or
+// because git is too old (< 2.27).
+func (e *Executor) resolveSparseCheckout(ctx context.Context) []string {
+	paths := cleanGitSparseCheckoutPaths(e.GitSparseCheckoutPaths)
+	if len(paths) == 0 {
+		return nil
+	}
+
+	// We require git >= 2.27 because setupSparseCheckout runs
+	// `git sparse-checkout set --cone <paths>` which is only supported by
+	// 2.27 and newer. On older git versions, fall back to a full checkout
+	// by returning nil.
+	ok, err := gitVersionAtLeast(ctx, e.shell, 2, 27)
+	if err != nil {
+		e.shell.Warningf("Sparse checkout requires git >= 2.27; falling back to full checkout (%v)", err)
+		return nil
+	}
+	if !ok {
+		e.shell.Warningf("Sparse checkout requires git >= 2.27; falling back to full checkout")
+		return nil
+	}
+	return paths
+}
+
 func cleanGitSparseCheckoutPaths(paths []string) []string {
 	cleaned := make([]string, 0, len(paths))
 	for _, path := range paths {
@@ -91,31 +116,19 @@ func (e *Executor) disableSparseCheckoutIfConfigured(ctx context.Context) {
 	}
 }
 
-// setupSparseCheckout configures (or disables) git sparse checkout for the
-// current working tree. It returns true if sparse checkout was successfully
-// applied for this build, so callers can adjust later behaviour (e.g. skip
-// submodule init, which requires the full tree).
-func (e *Executor) setupSparseCheckout(ctx context.Context) (bool, error) {
-	paths := cleanGitSparseCheckoutPaths(e.GitSparseCheckoutPaths)
-	if len(paths) == 0 {
-		e.disableSparseCheckoutIfConfigured(ctx)
-		return false, nil
-	}
-
-	ok, err := gitVersionAtLeast(ctx, e.shell, 2, 26)
-	if err != nil {
-		e.shell.Warningf("Sparse checkout requires git >= 2.26; falling back to full checkout (%v)", err)
-		e.disableSparseCheckoutIfConfigured(ctx)
-		return false, nil
-	}
-	if !ok {
-		e.shell.Warningf("Sparse checkout requires git >= 2.26; falling back to full checkout")
+// setupSparseCheckout configures git sparse checkout for the given cone paths.
+// When sparsePaths is empty it does a full checkout instead, disabling any
+// prior sparse checkout configuration. It returns true when sparse checkout is
+// applied, so callers can skip steps that need the full tree (e.g. submodule
+// init).
+func (e *Executor) setupSparseCheckout(ctx context.Context, sparsePaths []string) (bool, error) {
+	if len(sparsePaths) == 0 {
 		e.disableSparseCheckoutIfConfigured(ctx)
 		return false, nil
 	}
 
-	e.shell.Commentf("Setting up sparse checkout for paths: %s", strings.Join(paths, ","))
-	args := append([]string{"sparse-checkout", "set", "--cone"}, paths...)
+	e.shell.Commentf("Setting up sparse checkout for paths: %s", strings.Join(sparsePaths, ","))
+	args := append([]string{"sparse-checkout", "set", "--cone"}, sparsePaths...)
 	if err := e.shell.Command("git", args...).Run(ctx); err != nil {
 		return false, fmt.Errorf("setting sparse checkout paths: %w", err)
 	}

internal/job/checkout_sparse_test.go

@@ -56,15 +56,14 @@ func TestSetupSparseCheckout_Enable(t *testing.T) {
 	defer git.Close() //nolint:errcheck // Best-effort cleanup.
 	executor.GitSparseCheckoutPaths = []string{".buildkite/", "src/"}
 
-	git.Expect("--version").AndWriteToStdout("git version 2.39.0").AndExitWith(0)
 	git.Expect("sparse-checkout", "set", "--cone", ".buildkite/", "src/").AndExitWith(0)
 
-	active, err := executor.setupSparseCheckout(t.Context())
+	active, err := executor.setupSparseCheckout(t.Context(), []string{".buildkite/", "src/"})
 	if err != nil {
-		t.Fatalf("executor.setupSparseCheckout(ctx) error = %v, want nil", err)
+		t.Fatalf("executor.setupSparseCheckout(ctx, sparsePaths) error = %v, want nil", err)
 	}
 	if !active {
-		t.Fatalf("executor.setupSparseCheckout(ctx) active = false, want true")
+		t.Fatalf("executor.setupSparseCheckout(ctx, sparsePaths) active = false, want true")
 	}
 	if got, want := out.String(), "Setting up sparse checkout for paths: .buildkite/,src/"; !strings.Contains(got, want) {
 		t.Fatalf("shell output = %q, want to contain %q", got, want)
@@ -83,12 +82,12 @@ func TestSetupSparseCheckout_DisableWithPriorSparseConfig(t *testing.T) {
 	git.Expect("config", "--worktree", "--list").AndWriteToStdout("").AndExitWith(0)
 	git.Expect("config", "--unset", "extensions.worktreeConfig").AndExitWith(0)
 
-	active, err := executor.setupSparseCheckout(t.Context())
+	active, err := executor.setupSparseCheckout(t.Context(), nil)
 	if err != nil {
-		t.Fatalf("executor.setupSparseCheckout(ctx) error = %v, want nil", err)
+		t.Fatalf("executor.setupSparseCheckout(ctx, sparsePaths) error = %v, want nil", err)
 	}
 	if active {
-		t.Fatalf("executor.setupSparseCheckout(ctx) active = true, want false")
+		t.Fatalf("executor.setupSparseCheckout(ctx, sparsePaths) active = true, want false")
 	}
 	if got, want := out.String(), "Disabling sparse checkout from previous build"; !strings.Contains(got, want) {
 		t.Fatalf("shell output = %q, want to contain %q", got, want)
@@ -107,8 +106,8 @@ func TestSetupSparseCheckout_DisablePreservesOtherWorktreeConfig(t *testing.T) {
 	git.Expect("config", "--worktree", "--list").AndWriteToStdout("user.something=value\n").AndExitWith(0)
 	git.Expect("config", "--unset", "extensions.worktreeConfig").NotCalled()
 
-	if _, err := executor.setupSparseCheckout(t.Context()); err != nil {
-		t.Fatalf("executor.setupSparseCheckout(ctx) error = %v, want nil", err)
+	if _, err := executor.setupSparseCheckout(t.Context(), nil); err != nil {
+		t.Fatalf("executor.setupSparseCheckout(ctx, sparsePaths) error = %v, want nil", err)
 	}
 
 	git.Check(t)
@@ -121,8 +120,8 @@ func TestSetupSparseCheckout_DisableWithoutPriorSparseConfig(t *testing.T) {
 	git.Expect("config").WithAnyArguments().NotCalled()
 	git.Expect("sparse-checkout").WithAnyArguments().NotCalled()
 
-	if _, err := executor.setupSparseCheckout(t.Context()); err != nil {
-		t.Fatalf("executor.setupSparseCheckout(ctx) error = %v, want nil", err)
+	if _, err := executor.setupSparseCheckout(t.Context(), nil); err != nil {
+		t.Fatalf("executor.setupSparseCheckout(ctx, sparsePaths) error = %v, want nil", err)
 	}
 
 	git.Check(t)
@@ -136,14 +135,11 @@ func TestSetupSparseCheckout_VersionFallback(t *testing.T) {
 	git.Expect("--version").AndWriteToStdout("git version 2.25.4").AndExitWith(0)
 	git.Expect("sparse-checkout").WithAnyArguments().NotCalled()
 
-	active, err := executor.setupSparseCheckout(t.Context())
-	if err != nil {
-		t.Fatalf("executor.setupSparseCheckout(ctx) error = %v, want nil", err)
-	}
-	if active {
-		t.Fatalf("executor.setupSparseCheckout(ctx) active = true, want false")
+	paths := executor.resolveSparseCheckout(t.Context())
+	if len(paths) != 0 {
+		t.Fatalf("resolveSparseCheckout(ctx) = %#v, want nil (fallback to full checkout)", paths)
 	}
-	if got, want := out.String(), "Sparse checkout requires git >= 2.26; falling back to full checkout"; !strings.Contains(got, want) {
+	if got, want := out.String(), "Sparse checkout requires git >= 2.27; falling back to full checkout"; !strings.Contains(got, want) {
 		t.Fatalf("shell output = %q, want to contain %q", got, want)
 	}
 
@@ -156,16 +152,27 @@ func TestSetupSparseCheckout_VersionFallbackDisablesPriorSparseConfig(t *testing
 	executor.GitSparseCheckoutPaths = []string{"src/"}
 	createSparseCheckoutFile(t, executor.shell.Getwd())
 
+	// Old git: resolveSparseCheckout warns and returns nil paths, then
+	// setupSparseCheckout disables the prior sparse config left on disk.
 	git.Expect("--version").AndWriteToStdout("git version 2.25.4").AndExitWith(0)
 	git.Expect("config", "--get", "core.sparseCheckout").AndWriteToStdout("true\n").AndExitWith(0)
 	git.Expect("sparse-checkout", "disable").AndExitWith(0)
 	git.Expect("config", "--worktree", "--list").AndWriteToStdout("").AndExitWith(0)
 	git.Expect("config", "--unset", "extensions.worktreeConfig").AndExitWith(0)
 
-	if _, err := executor.setupSparseCheckout(t.Context()); err != nil {
-		t.Fatalf("executor.setupSparseCheckout(ctx) error = %v, want nil", err)
+	sparsePaths := executor.resolveSparseCheckout(t.Context())
+	if len(sparsePaths) != 0 {
+		t.Fatalf("resolveSparseCheckout(ctx) = %#v, want nil (fallback to full checkout)", sparsePaths)
+	}
+
+	active, err := executor.setupSparseCheckout(t.Context(), sparsePaths)
+	if err != nil {
+		t.Fatalf("executor.setupSparseCheckout(ctx, sparsePaths) error = %v, want nil", err)
+	}
+	if active {
+		t.Fatalf("executor.setupSparseCheckout(ctx, sparsePaths) active = true, want false")
 	}
-	if got, want := out.String(), "Sparse checkout requires git >= 2.26; falling back to full checkout"; !strings.Contains(got, want) {
+	if got, want := out.String(), "Sparse checkout requires git >= 2.27; falling back to full checkout"; !strings.Contains(got, want) {
 		t.Fatalf("shell output = %q, want to contain %q", got, want)
 	}
 	if got, want := out.String(), "Disabling sparse checkout from previous build"; !strings.Contains(got, want) {

internal/job/git.go

@@ -10,6 +10,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"regexp"
+	"slices"
 	"strings"
 	"time"
 
@@ -92,6 +93,26 @@ func gitCheckout(ctx context.Context, sh *shell.Shell, gitCheckoutFlags, referen
 	return nil
 }
 
+// hasPartialCloneFilter returns true if flags contains a
+// --filter=<spec> or --filter <spec> option.
+func hasPartialCloneFilter(flags []string) bool {
+	for i, f := range flags {
+		// Check for --filter=<spec>
+		if strings.HasPrefix(f, "--filter=") {
+			return true
+		}
+		// The --filter <spec> form only counts when a value actually follows it
+		if f == "--filter" && i+1 < len(flags) {
+			return true
+		}
+	}
+	return false
+}
+
+func hasSparseCheckoutFlag(flags []string) bool {
+	return slices.Contains(flags, "--sparse")
+}
+
 func gitClone(ctx context.Context, sh *shell.Shell, gitCloneFlags []string, repository, dir string) error {
 	commandArgs := []string{"clone"}
 	commandArgs = append(commandArgs, gitCloneFlags...)

internal/job/git_test.go

@@ -240,6 +240,36 @@ func TestGitClone(t *testing.T) {
 	}
 }
 
+func TestHasPartialCloneFilter(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name  string
+		flags []string
+		want  bool
+	}{
+		{name: "no filter", flags: []string{"-v", "--reference", "mirror"}, want: false},
+		{name: "filter blob:none with equals", flags: []string{"-v", "--filter=blob:none"}, want: true},
+		{name: "filter blob:none separate arg", flags: []string{"-v", "--filter", "blob:none"}, want: true},
+		{name: "filter tree:0 with equals", flags: []string{"-v", "--filter=tree:0"}, want: true},
+		{name: "filter tree:0 separate arg", flags: []string{"-v", "--filter", "tree:0"}, want: true},
+		{name: "multiple filters with blob:none", flags: []string{"--filter=blob:none", "--filter=tree:0"}, want: true},
+		{name: "multiple filters separate args", flags: []string{"--filter", "blob:none", "--filter", "tree:0"}, want: true},
+		{name: "filter prefix lookalike", flags: []string{"-v", "--filtered"}, want: false},
+		{name: "filter without value", flags: []string{"--filter"}, want: false},
+		{name: "empty", flags: nil, want: false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			if got := hasPartialCloneFilter(tt.flags); got != tt.want {
+				t.Errorf("hasPartialCloneFilter(%v) = %t, want %t", tt.flags, got, tt.want)
+			}
+		})
+	}
+}
+
 func TestGitClean(t *testing.T) {
 	t.Parallel()
 	ctx := t.Context()