Set Partial Clone flags on sparse-checkout

internal/job/checkout.go

@@ -762,7 +762,7 @@ func (e *Executor) getOrUpdateMirrorDir(ctx context.Context, repository string)
 
 // fetchSource fetches the git source for the job. If GitSkipFetchExistingCommits is
 // enabled and the commit already exists locally, the fetch is skipped entirely.
-func (e *Executor) fetchSource(ctx context.Context) error {
+func (e *Executor) fetchSource(ctx context.Context, autoAddBloblessFilter bool) error {
 	// If configured, skip the fetch when the commit already exists locally.
 	// This is useful when a pre-populated git mirror is used with --reference,
 	// as the commit objects are already reachable and fetching is redundant.
@@ -773,6 +773,9 @@ func (e *Executor) fetchSource(ctx context.Context) error {
 	}
 
 	gitFetchFlags := e.GitFetchFlags
+	if autoAddBloblessFilter {
+		gitFetchFlags = "--filter=blob:none " + gitFetchFlags
+	}
 
 	switch {
 	case e.RefSpec != "":
@@ -908,6 +911,9 @@ func (e *Executor) defaultCheckoutPhase(ctx context.Context) (retErr error) {
 		return fmt.Errorf("creating checkout dir: %w", err)
 	}
 
+	// If sparse checkout paths are present, plan the sparse checkout.
+	sparsePlan := e.planSparseCheckout(ctx)
+
 	// On mirrors and dissociation:
 	//
 	// --reference makes the clone reuse objects from the mirror, using the
@@ -964,6 +970,23 @@ func (e *Executor) defaultCheckoutPhase(ctx context.Context) (retErr error) {
 			}
 		}
 
+		// If sparse checkout will actually apply, use a partial clone so blobs
+		// outside the sparse set aren't downloaded up front. Only when the user
+		// hasn't already supplied a --filter — git takes the last --filter on the
+		// command line and would silently override theirs.
+		if sparsePlan.supported {
+			if hasSparseCheckoutFlag(gitCloneFlags) {
+				e.shell.Commentf("Sparse checkout is configured and BUILDKITE_GIT_CLONE_FLAGS already contains a --sparse flag (preserving user-supplied sparse checkout).")
+			} else {
+				gitCloneFlags = append(gitCloneFlags, "--sparse")
+			}
+			if hasPartialCloneFilter(gitCloneFlags) {
+				e.shell.Commentf("Sparse checkout is configured and BUILDKITE_GIT_CLONE_FLAGS already contains a --filter (preserving user-supplied filter).")
+			} else {
+				gitCloneFlags = append(gitCloneFlags, "--filter=blob:none")
+			}
+		}
+
 		// Do the clone.
 		if err := gitClone(ctx, e.shell, gitCloneFlags, e.Repository, "."); err != nil {
 			return fmt.Errorf("cloning git repository: %w", err)
@@ -991,15 +1014,18 @@ func (e *Executor) defaultCheckoutPhase(ctx context.Context) (retErr error) {
 		}
 	}
 
-	if err := e.fetchSource(ctx); err != nil {
+	autoAddBloblessFilter := sparsePlan.supported &&
+		!strings.Contains(e.GitCloneFlags, "--filter") &&
+		!strings.Contains(e.GitFetchFlags, "--filter")
+	if err := e.fetchSource(ctx, autoAddBloblessFilter); err != nil {
 		return err
 	}
 
 	if err := e.verifyCommit(ctx); err != nil {
 		return err
 	}
 
-	sparseCheckoutActive, err := e.setupSparseCheckout(ctx)
+	sparseCheckoutActive, err := e.setupSparseCheckout(ctx, sparsePlan)
 	if err != nil {
 		return err
 	}

internal/job/checkout_sparse.go

@@ -12,6 +12,42 @@ import (
 	"github.com/buildkite/agent/v3/internal/shell"
 )
 
+// sparseCheckoutPlan is the decision, made once at the top of the checkout,
+// about whether sparse checkout applies to this build. Data only — all state
+// mutation happens in setupSparseCheckout.
+type sparseCheckoutPlan struct {
+	paths     []string // cleaned; empty when sparse checkout is not requested
+	supported bool     // requested AND `sparse-checkout set --cone` is available (git >= 2.27)
+}
+
+// planSparseCheckout resolves whether sparse checkout can be applied to this
+// build. Called once, before the clone, so the pre-clone --filter decision
+// and the post-fetch `sparse-checkout set --cone` call share the same answer
+// without running `git --version` twice. Runs `git --version` only when
+// sparse checkout was actually requested.
+func (e *Executor) planSparseCheckout(ctx context.Context) sparseCheckoutPlan {
+	paths := cleanGitSparseCheckoutPaths(e.GitSparseCheckoutPaths)
+	if len(paths) == 0 {
+		return sparseCheckoutPlan{}
+	}
+
+	// 2.27 is the floor because setupSparseCheckout calls
+	// `git sparse-checkout set --cone <paths>` in a single call. Cone mode
+	// shipped in 2.26 but `--cone` was only accepted on the `init` subcommand
+	// then; `set --cone` landed in 2.27. On 2.26.x that call fails, so we
+	// fall back to a full checkout rather than carrying the older two-step.
+	ok, err := gitVersionAtLeast(ctx, e.shell, 2, 27)
+	if err != nil {
+		e.shell.Warningf("Sparse checkout requires git >= 2.27; falling back to full checkout (%v)", err)
+		return sparseCheckoutPlan{paths: paths}
+	}
+	if !ok {
+		e.shell.Warningf("Sparse checkout requires git >= 2.27; falling back to full checkout")
+		return sparseCheckoutPlan{paths: paths}
+	}
+	return sparseCheckoutPlan{paths: paths, supported: true}
+}
+
 func cleanGitSparseCheckoutPaths(paths []string) []string {
 	cleaned := make([]string, 0, len(paths))
 	for _, path := range paths {
@@ -91,31 +127,19 @@ func (e *Executor) disableSparseCheckoutIfConfigured(ctx context.Context) {
 	}
 }
 
-// setupSparseCheckout configures (or disables) git sparse checkout for the
-// current working tree. It returns true if sparse checkout was successfully
-// applied for this build, so callers can adjust later behaviour (e.g. skip
-// submodule init, which requires the full tree).
-func (e *Executor) setupSparseCheckout(ctx context.Context) (bool, error) {
-	paths := cleanGitSparseCheckoutPaths(e.GitSparseCheckoutPaths)
-	if len(paths) == 0 {
-		e.disableSparseCheckoutIfConfigured(ctx)
-		return false, nil
-	}
-
-	ok, err := gitVersionAtLeast(ctx, e.shell, 2, 26)
-	if err != nil {
-		e.shell.Warningf("Sparse checkout requires git >= 2.26; falling back to full checkout (%v)", err)
-		e.disableSparseCheckoutIfConfigured(ctx)
-		return false, nil
-	}
-	if !ok {
-		e.shell.Warningf("Sparse checkout requires git >= 2.26; falling back to full checkout")
+// setupSparseCheckout applies a resolved plan to configure (or disable)
+// git sparse checkout for the current working tree. It returns true if
+// sparse checkout was successfully applied for this build, so callers can
+// adjust later behaviour (e.g. skip submodule init, which requires the full
+// tree).
+func (e *Executor) setupSparseCheckout(ctx context.Context, plan sparseCheckoutPlan) (bool, error) {
+	if len(plan.paths) == 0 || !plan.supported {
 		e.disableSparseCheckoutIfConfigured(ctx)
 		return false, nil
 	}
 
-	e.shell.Commentf("Setting up sparse checkout for paths: %s", strings.Join(paths, ","))
-	args := append([]string{"sparse-checkout", "set", "--cone"}, paths...)
+	e.shell.Commentf("Setting up sparse checkout for paths: %s", strings.Join(plan.paths, ","))
+	args := append([]string{"sparse-checkout", "set", "--cone"}, plan.paths...)
 	if err := e.shell.Command("git", args...).Run(ctx); err != nil {
 		return false, fmt.Errorf("setting sparse checkout paths: %w", err)
 	}

internal/job/checkout_sparse_test.go

@@ -56,15 +56,15 @@ func TestSetupSparseCheckout_Enable(t *testing.T) {
 	defer git.Close() //nolint:errcheck // Best-effort cleanup.
 	executor.GitSparseCheckoutPaths = []string{".buildkite/", "src/"}
 
-	git.Expect("--version").AndWriteToStdout("git version 2.39.0").AndExitWith(0)
 	git.Expect("sparse-checkout", "set", "--cone", ".buildkite/", "src/").AndExitWith(0)
 
-	active, err := executor.setupSparseCheckout(t.Context())
+	plan := sparseCheckoutPlan{paths: []string{".buildkite/", "src/"}, supported: true}
+	active, err := executor.setupSparseCheckout(t.Context(), plan)
 	if err != nil {
-		t.Fatalf("executor.setupSparseCheckout(ctx) error = %v, want nil", err)
+		t.Fatalf("executor.setupSparseCheckout(ctx, plan) error = %v, want nil", err)
 	}
 	if !active {
-		t.Fatalf("executor.setupSparseCheckout(ctx) active = false, want true")
+		t.Fatalf("executor.setupSparseCheckout(ctx, plan) active = false, want true")
 	}
 	if got, want := out.String(), "Setting up sparse checkout for paths: .buildkite/,src/"; !strings.Contains(got, want) {
 		t.Fatalf("shell output = %q, want to contain %q", got, want)
@@ -83,12 +83,13 @@ func TestSetupSparseCheckout_DisableWithPriorSparseConfig(t *testing.T) {
 	git.Expect("config", "--worktree", "--list").AndWriteToStdout("").AndExitWith(0)
 	git.Expect("config", "--unset", "extensions.worktreeConfig").AndExitWith(0)
 
-	active, err := executor.setupSparseCheckout(t.Context())
+	plan := sparseCheckoutPlan{paths: []string{"src/"}, supported: false}
+	active, err := executor.setupSparseCheckout(t.Context(), plan)
 	if err != nil {
-		t.Fatalf("executor.setupSparseCheckout(ctx) error = %v, want nil", err)
+		t.Fatalf("executor.setupSparseCheckout(ctx, plan) error = %v, want nil", err)
 	}
 	if active {
-		t.Fatalf("executor.setupSparseCheckout(ctx) active = true, want false")
+		t.Fatalf("executor.setupSparseCheckout(ctx, plan) active = true, want false")
 	}
 	if got, want := out.String(), "Disabling sparse checkout from previous build"; !strings.Contains(got, want) {
 		t.Fatalf("shell output = %q, want to contain %q", got, want)
@@ -107,8 +108,9 @@ func TestSetupSparseCheckout_DisablePreservesOtherWorktreeConfig(t *testing.T) {
 	git.Expect("config", "--worktree", "--list").AndWriteToStdout("user.something=value\n").AndExitWith(0)
 	git.Expect("config", "--unset", "extensions.worktreeConfig").NotCalled()
 
-	if _, err := executor.setupSparseCheckout(t.Context()); err != nil {
-		t.Fatalf("executor.setupSparseCheckout(ctx) error = %v, want nil", err)
+	plan := sparseCheckoutPlan{paths: []string{"src/"}, supported: false}
+	if _, err := executor.setupSparseCheckout(t.Context(), plan); err != nil {
+		t.Fatalf("executor.setupSparseCheckout(ctx, plan) error = %v, want nil", err)
 	}
 
 	git.Check(t)
@@ -121,8 +123,9 @@ func TestSetupSparseCheckout_DisableWithoutPriorSparseConfig(t *testing.T) {
 	git.Expect("config").WithAnyArguments().NotCalled()
 	git.Expect("sparse-checkout").WithAnyArguments().NotCalled()
 
-	if _, err := executor.setupSparseCheckout(t.Context()); err != nil {
-		t.Fatalf("executor.setupSparseCheckout(ctx) error = %v, want nil", err)
+	plan := sparseCheckoutPlan{paths: []string{"src/"}, supported: false}
+	if _, err := executor.setupSparseCheckout(t.Context(), plan); err != nil {
+		t.Fatalf("executor.setupSparseCheckout(ctx, plan) error = %v, want nil", err)
 	}
 
 	git.Check(t)
@@ -136,14 +139,14 @@ func TestSetupSparseCheckout_VersionFallback(t *testing.T) {
 	git.Expect("--version").AndWriteToStdout("git version 2.25.4").AndExitWith(0)
 	git.Expect("sparse-checkout").WithAnyArguments().NotCalled()
 
-	active, err := executor.setupSparseCheckout(t.Context())
-	if err != nil {
-		t.Fatalf("executor.setupSparseCheckout(ctx) error = %v, want nil", err)
+	plan := executor.planSparseCheckout(t.Context())
+	if plan.supported {
+		t.Fatalf("planSparseCheckout(ctx).supported = true, want false")
 	}
-	if active {
-		t.Fatalf("executor.setupSparseCheckout(ctx) active = true, want false")
+	if len(plan.paths) != 1 || plan.paths[0] != "src/" {
+		t.Fatalf("planSparseCheckout(ctx).paths = %#v, want [\"src/\"]", plan.paths)
 	}
-	if got, want := out.String(), "Sparse checkout requires git >= 2.26; falling back to full checkout"; !strings.Contains(got, want) {
+	if got, want := out.String(), "Sparse checkout requires git >= 2.27; falling back to full checkout"; !strings.Contains(got, want) {
 		t.Fatalf("shell output = %q, want to contain %q", got, want)
 	}
 
@@ -156,17 +159,18 @@ func TestSetupSparseCheckout_VersionFallbackDisablesPriorSparseConfig(t *testing
 	executor.GitSparseCheckoutPaths = []string{"src/"}
 	createSparseCheckoutFile(t, executor.shell.Getwd())
 
-	git.Expect("--version").AndWriteToStdout("git version 2.25.4").AndExitWith(0)
 	git.Expect("config", "--get", "core.sparseCheckout").AndWriteToStdout("true\n").AndExitWith(0)
 	git.Expect("sparse-checkout", "disable").AndExitWith(0)
 	git.Expect("config", "--worktree", "--list").AndWriteToStdout("").AndExitWith(0)
 	git.Expect("config", "--unset", "extensions.worktreeConfig").AndExitWith(0)
 
-	if _, err := executor.setupSparseCheckout(t.Context()); err != nil {
-		t.Fatalf("executor.setupSparseCheckout(ctx) error = %v, want nil", err)
+	plan := sparseCheckoutPlan{paths: []string{"src/"}, supported: false}
+	active, err := executor.setupSparseCheckout(t.Context(), plan)
+	if err != nil {
+		t.Fatalf("executor.setupSparseCheckout(ctx, plan) error = %v, want nil", err)
 	}
-	if got, want := out.String(), "Sparse checkout requires git >= 2.26; falling back to full checkout"; !strings.Contains(got, want) {
-		t.Fatalf("shell output = %q, want to contain %q", got, want)
+	if active {
+		t.Fatalf("executor.setupSparseCheckout(ctx, plan) active = true, want false")
 	}
 	if got, want := out.String(), "Disabling sparse checkout from previous build"; !strings.Contains(got, want) {
 		t.Fatalf("shell output = %q, want to contain %q", got, want)

internal/job/git.go

@@ -92,6 +92,27 @@ func gitCheckout(ctx context.Context, sh *shell.Shell, gitCheckoutFlags, referen
 	return nil
 }
 
+// hasPartialCloneFilter reports whether the given clone flags already include
+// any --filter=<spec> or --filter <spec> option. The caller (sparse checkout
+// setup) uses this to avoid stacking its own --filter=blob:none on top of a
+// user-supplied filter, since git takes the last --filter on the command line
+// and would silently override the user's choice.
+func hasPartialCloneFilter(flags []string) bool {
+	for i, f := range flags {
+		if strings.HasPrefix(f, "--filter=") {
+			return true
+		}
+		if f == "--filter" && i+1 < len(flags) {
+			return true
+		}
+	}
+	return false
+}
+
+func hasSparseCheckoutFlag(flags []string) bool {
+	return slices.Contains(flags, "--sparse")
+}
+
 func gitClone(ctx context.Context, sh *shell.Shell, gitCloneFlags []string, repository, dir string) error {
 	commandArgs := []string{"clone"}
 	commandArgs = append(commandArgs, gitCloneFlags...)

internal/job/git_test.go

@@ -240,6 +240,36 @@ func TestGitClone(t *testing.T) {
 	}
 }
 
+func TestHasPartialCloneFilter(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name  string
+		flags []string
+		want  bool
+	}{
+		{name: "no filter", flags: []string{"-v", "--reference", "mirror"}, want: false},
+		{name: "filter blob:none with equals", flags: []string{"-v", "--filter=blob:none"}, want: true},
+		{name: "filter blob:none separate arg", flags: []string{"-v", "--filter", "blob:none"}, want: true},
+		{name: "filter tree:0 with equals", flags: []string{"-v", "--filter=tree:0"}, want: true},
+		{name: "filter tree:0 separate arg", flags: []string{"-v", "--filter", "tree:0"}, want: true},
+		{name: "multiple filters with blob:none", flags: []string{"--filter=blob:none", "--filter=tree:0"}, want: true},
+		{name: "multiple filters separate args", flags: []string{"--filter", "blob:none", "--filter", "tree:0"}, want: true},
+		{name: "filter prefix lookalike", flags: []string{"-v", "--filtered"}, want: false},
+		{name: "filter without value", flags: []string{"--filter"}, want: false},
+		{name: "empty", flags: nil, want: false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			if got := hasPartialCloneFilter(tt.flags); got != tt.want {
+				t.Errorf("hasPartialCloneFilter(%v) = %t, want %t", tt.flags, got, tt.want)
+			}
+		})
+	}
+}
+
 func TestGitClean(t *testing.T) {
 	t.Parallel()
 	ctx := t.Context()