Set Partial Clone flags on sparse-checkout

internal/job/checkout.go

@@ -762,7 +762,10 @@ func (e *Executor) getOrUpdateMirrorDir(ctx context.Context, repository string)
 
 // fetchSource fetches the git source for the job. If GitSkipFetchExistingCommits is
 // enabled and the commit already exists locally, the fetch is skipped entirely.
-func (e *Executor) fetchSource(ctx context.Context) error {
+// When addBloblessFilter is true, --filter=blob:none is prepended to the fetch
+// flags — the caller decides based on sparse-checkout state and user-supplied
+// filters.
+func (e *Executor) fetchSource(ctx context.Context, addBloblessFilter bool) error {
 	// If configured, skip the fetch when the commit already exists locally.
 	// This is useful when a pre-populated git mirror is used with --reference,
 	// as the commit objects are already reachable and fetching is redundant.
@@ -773,6 +776,9 @@ func (e *Executor) fetchSource(ctx context.Context) error {
 	}
 
 	gitFetchFlags := e.GitFetchFlags
+	if addBloblessFilter {
+		gitFetchFlags = "--filter=blob:none " + gitFetchFlags
+	}
 
 	switch {
 	case e.RefSpec != "":
@@ -908,6 +914,21 @@ func (e *Executor) defaultCheckoutPhase(ctx context.Context) (retErr error) {
 		return fmt.Errorf("creating checkout dir: %w", err)
 	}
 
+	// Resolve the cone paths to check out (nil means a full checkout).
+	sparsePaths := e.resolveSparseCheckout(ctx)
+
+	// Split the git clone flags into an array of strings, so we can append
+	// additional flags if needed (e.g., --reference, --dissociate, --sparse, --filter=blob:none).
+	gitCloneFlags, err := shellwords.Split(e.GitCloneFlags)
+	if err != nil {
+		return fmt.Errorf("splitting --git-clone-flags %q: %w", e.GitCloneFlags, err)
+	}
+
+	// Snapshot whether the user supplied their own --filter before we
+	// potentially append one — the fetch-side decision depends on the
+	// original user-supplied state, not on flags we auto-add here.
+	userSuppliedCloneFilter := hasPartialFilterFlags(gitCloneFlags)
+
 	// On mirrors and dissociation:
 	//
 	// --reference makes the clone reuse objects from the mirror, using the
@@ -953,17 +974,31 @@ func (e *Executor) defaultCheckoutPhase(ctx context.Context) (retErr error) {
 
 		// Compute the clone flags. For mirrors we need --reference, and usually
 		// --dissociate.
-		gitCloneFlags, err := shellwords.Split(e.GitCloneFlags)
-		if err != nil {
-			return fmt.Errorf("splitting --git-clone-flags %q: %w", e.GitCloneFlags, err)
-		}
 		if mirrorDir != "" {
 			gitCloneFlags = append(gitCloneFlags, "--reference", mirrorDir)
 			if e.GitMirrorCheckoutMode == "dissociate" {
 				gitCloneFlags = append(gitCloneFlags, "--dissociate")
 			}
 		}
 
+		// When sparse checkout applies, add two clone flags:
+		//   --sparse             clone in sparse mode
+		//   --filter=blob:none   make it a partial clone, so blobs outside the
+		//                        sparse set aren't downloaded up front
+		// Each flag is added only if the user hasn't already supplied their own.
+		if len(sparsePaths) > 0 {
+			if slices.Contains(gitCloneFlags, "--sparse") {
+				e.shell.Commentf("Sparse checkout is configured and BUILDKITE_GIT_CLONE_FLAGS already contains a --sparse flag (preserving user-supplied sparse checkout).")
+			} else {
+				gitCloneFlags = append(gitCloneFlags, "--sparse")
+			}
+			if userSuppliedCloneFilter {
+				e.shell.Commentf("Sparse checkout is configured and BUILDKITE_GIT_CLONE_FLAGS already contains a --filter (preserving user-supplied filter).")
+			} else {
+				gitCloneFlags = append(gitCloneFlags, "--filter=blob:none")
+			}
+		}
+
 		// Do the clone.
 		if err := gitClone(ctx, e.shell, gitCloneFlags, e.Repository, "."); err != nil {
 			return fmt.Errorf("cloning git repository: %w", err)
@@ -991,15 +1026,24 @@ func (e *Executor) defaultCheckoutPhase(ctx context.Context) (retErr error) {
 		}
 	}
 
-	if err := e.fetchSource(ctx); err != nil {
+	// Parse the fetch flags into tokens so we can check for a user-supplied --filter flag.
+	gitFetchFlags, err := shellwords.Split(e.GitFetchFlags)
+	if err != nil {
+		return fmt.Errorf("splitting --git-fetch-flags %q: %w", e.GitFetchFlags, err)
+	}
+
+	addBloblessFilter := len(sparsePaths) > 0 &&
+		!userSuppliedCloneFilter &&
+		!hasPartialFilterFlags(gitFetchFlags)
+	if err := e.fetchSource(ctx, addBloblessFilter); err != nil {
 		return err
 	}
 
 	if err := e.verifyCommit(ctx); err != nil {
 		return err
 	}
 
-	sparseCheckoutActive, err := e.setupSparseCheckout(ctx)
+	sparseCheckoutActive, err := e.setupSparseCheckout(ctx, sparsePaths)
 	if err != nil {
 		return err
 	}
@@ -1110,7 +1154,7 @@ func (e *Executor) defaultCheckoutPhase(ctx context.Context) (retErr error) {
 
 	// When sparse-checkout is active, scope LFS to the same paths so we don't
 	// pull objects outside the sparse set (SUP-6529). If sparse fell back to a
-	// full checkout (e.g. git < 2.26), fetch unscoped so files outside the
+	// full checkout (e.g. git < 2.27), fetch unscoped so files outside the
 	// requested paths still get their LFS content.
 	if e.GitLFSEnabled {
 		lfsArgs := gitLFSFetchCheckoutArgs{

internal/job/checkout_sparse.go

@@ -12,6 +12,31 @@ import (
 	"github.com/buildkite/agent/v3/internal/shell"
 )
 
+// resolveSparseCheckout returns the cone paths to check out for this build, or
+// nil to check out the full tree — either because no paths were requested or
+// because git is too old (< 2.27).
+func (e *Executor) resolveSparseCheckout(ctx context.Context) []string {
+	paths := cleanGitSparseCheckoutPaths(e.GitSparseCheckoutPaths)
+	if len(paths) == 0 {
+		return nil
+	}
+
+	// We require git >= 2.27 because setupSparseCheckout runs
+	// `git sparse-checkout set --cone <paths>`, which was promoted
+	// from experimental to stable in git 2.27. On older git versions,
+	// fall back to a full checkout by returning nil.
+	ok, got, err := gitVersionAtLeast(ctx, e.shell, 2, 27)
+	if err != nil {
+		e.shell.Warningf("Sparse checkout requires git >= 2.27; falling back to full checkout (%v)", err)
+		return nil
+	}
+	if !ok {
+		e.shell.Warningf("Sparse checkout requires git >= 2.27, got %s; falling back to full checkout", got)
+		return nil
+	}
+	return paths
+}
+
 func cleanGitSparseCheckoutPaths(paths []string) []string {
 	cleaned := make([]string, 0, len(paths))
 	for _, path := range paths {
@@ -30,21 +55,27 @@ func parseGitVersion(output string) (major, minor int, ok bool) {
 	return major, minor, true
 }
 
-func gitVersionAtLeast(ctx context.Context, sh *shell.Shell, major, minor int) (bool, error) {
+// gitVersionAtLeast reports whether the local git binary is at least
+// major.minor. It also returns the parsed "M.m" version string so callers can
+// include it in log output. The err return is reserved for actual failures
+// (git command failure, unparseable version output) — a git that is simply
+// too old returns (false, "M.m", nil), not an error.
+func gitVersionAtLeast(ctx context.Context, sh *shell.Shell, major, minor int) (ok bool, got string, err error) {
 	output, err := sh.Command("git", "--version").RunAndCaptureStdout(ctx)
 	if err != nil {
-		return false, err
+		return false, "", err
 	}
 
-	gitMajor, gitMinor, ok := parseGitVersion(strings.TrimSpace(output))
-	if !ok {
-		return false, fmt.Errorf("parsing git version from %q", strings.TrimSpace(output))
+	gitMajor, gitMinor, parseOK := parseGitVersion(strings.TrimSpace(output))
+	if !parseOK {
+		return false, "", fmt.Errorf("parsing git version from %q", strings.TrimSpace(output))
 	}
 
+	got = fmt.Sprintf("%d.%d", gitMajor, gitMinor)
 	if gitMajor != major {
-		return gitMajor > major, nil
+		return gitMajor > major, got, nil
 	}
-	return gitMinor >= minor, nil
+	return gitMinor >= minor, got, nil
 }
 
 // sparseCheckoutMayBeConfigured does a cheap filesystem check for marker files
@@ -91,31 +122,19 @@ func (e *Executor) disableSparseCheckoutIfConfigured(ctx context.Context) {
 	}
 }
 
-// setupSparseCheckout configures (or disables) git sparse checkout for the
-// current working tree. It returns true if sparse checkout was successfully
-// applied for this build, so callers can adjust later behaviour (e.g. skip
-// submodule init, which requires the full tree).
-func (e *Executor) setupSparseCheckout(ctx context.Context) (bool, error) {
-	paths := cleanGitSparseCheckoutPaths(e.GitSparseCheckoutPaths)
-	if len(paths) == 0 {
-		e.disableSparseCheckoutIfConfigured(ctx)
-		return false, nil
-	}
-
-	ok, err := gitVersionAtLeast(ctx, e.shell, 2, 26)
-	if err != nil {
-		e.shell.Warningf("Sparse checkout requires git >= 2.26; falling back to full checkout (%v)", err)
-		e.disableSparseCheckoutIfConfigured(ctx)
-		return false, nil
-	}
-	if !ok {
-		e.shell.Warningf("Sparse checkout requires git >= 2.26; falling back to full checkout")
+// setupSparseCheckout configures git sparse checkout for the given cone paths.
+// When sparsePaths is empty it does a full checkout instead, disabling any
+// prior sparse checkout configuration. It returns true when sparse checkout is
+// applied, so callers can skip steps that need the full tree (e.g. submodule
+// init).
+func (e *Executor) setupSparseCheckout(ctx context.Context, sparsePaths []string) (bool, error) {
+	if len(sparsePaths) == 0 {
 		e.disableSparseCheckoutIfConfigured(ctx)
 		return false, nil
 	}
 
-	e.shell.Commentf("Setting up sparse checkout for paths: %s", strings.Join(paths, ","))
-	args := append([]string{"sparse-checkout", "set", "--cone"}, paths...)
+	e.shell.Commentf("Setting up sparse checkout for paths: %s", strings.Join(sparsePaths, ","))
+	args := append([]string{"sparse-checkout", "set", "--cone"}, sparsePaths...)
 	if err := e.shell.Command("git", args...).Run(ctx); err != nil {
 		return false, fmt.Errorf("setting sparse checkout paths: %w", err)
 	}

internal/job/checkout_sparse_test.go

@@ -54,17 +54,16 @@ func TestParseGitVersion(t *testing.T) {
 func TestSetupSparseCheckout_Enable(t *testing.T) {
 	executor, git, out := newSparseCheckoutTestExecutor(t)
 	defer git.Close() //nolint:errcheck // Best-effort cleanup.
-	executor.GitSparseCheckoutPaths = []string{".buildkite/", "src/"}
 
-	git.Expect("--version").AndWriteToStdout("git version 2.39.0").AndExitWith(0)
+	paths := []string{".buildkite/", "src/"}
 	git.Expect("sparse-checkout", "set", "--cone", ".buildkite/", "src/").AndExitWith(0)
 
-	active, err := executor.setupSparseCheckout(t.Context())
+	active, err := executor.setupSparseCheckout(t.Context(), paths)
 	if err != nil {
-		t.Fatalf("executor.setupSparseCheckout(ctx) error = %v, want nil", err)
+		t.Fatalf("executor.setupSparseCheckout(ctx, sparsePaths) error = %v, want nil", err)
 	}
 	if !active {
-		t.Fatalf("executor.setupSparseCheckout(ctx) active = false, want true")
+		t.Fatalf("executor.setupSparseCheckout(ctx, sparsePaths) active = false, want true")
 	}
 	if got, want := out.String(), "Setting up sparse checkout for paths: .buildkite/,src/"; !strings.Contains(got, want) {
 		t.Fatalf("shell output = %q, want to contain %q", got, want)
@@ -83,12 +82,12 @@ func TestSetupSparseCheckout_DisableWithPriorSparseConfig(t *testing.T) {
 	git.Expect("config", "--worktree", "--list").AndWriteToStdout("").AndExitWith(0)
 	git.Expect("config", "--unset", "extensions.worktreeConfig").AndExitWith(0)
 
-	active, err := executor.setupSparseCheckout(t.Context())
+	active, err := executor.setupSparseCheckout(t.Context(), nil)
 	if err != nil {
-		t.Fatalf("executor.setupSparseCheckout(ctx) error = %v, want nil", err)
+		t.Fatalf("executor.setupSparseCheckout(ctx, sparsePaths) error = %v, want nil", err)
 	}
 	if active {
-		t.Fatalf("executor.setupSparseCheckout(ctx) active = true, want false")
+		t.Fatalf("executor.setupSparseCheckout(ctx, sparsePaths) active = true, want false")
 	}
 	if got, want := out.String(), "Disabling sparse checkout from previous build"; !strings.Contains(got, want) {
 		t.Fatalf("shell output = %q, want to contain %q", got, want)
@@ -107,8 +106,8 @@ func TestSetupSparseCheckout_DisablePreservesOtherWorktreeConfig(t *testing.T) {
 	git.Expect("config", "--worktree", "--list").AndWriteToStdout("user.something=value\n").AndExitWith(0)
 	git.Expect("config", "--unset", "extensions.worktreeConfig").NotCalled()
 
-	if _, err := executor.setupSparseCheckout(t.Context()); err != nil {
-		t.Fatalf("executor.setupSparseCheckout(ctx) error = %v, want nil", err)
+	if _, err := executor.setupSparseCheckout(t.Context(), nil); err != nil {
+		t.Fatalf("executor.setupSparseCheckout(ctx, sparsePaths) error = %v, want nil", err)
 	}
 
 	git.Check(t)
@@ -121,29 +120,26 @@ func TestSetupSparseCheckout_DisableWithoutPriorSparseConfig(t *testing.T) {
 	git.Expect("config").WithAnyArguments().NotCalled()
 	git.Expect("sparse-checkout").WithAnyArguments().NotCalled()
 
-	if _, err := executor.setupSparseCheckout(t.Context()); err != nil {
-		t.Fatalf("executor.setupSparseCheckout(ctx) error = %v, want nil", err)
+	if _, err := executor.setupSparseCheckout(t.Context(), nil); err != nil {
+		t.Fatalf("executor.setupSparseCheckout(ctx, sparsePaths) error = %v, want nil", err)
 	}
 
 	git.Check(t)
 }
 
-func TestSetupSparseCheckout_VersionFallback(t *testing.T) {
+func TestResolveSparseCheckout_VersionFallback(t *testing.T) {
 	executor, git, out := newSparseCheckoutTestExecutor(t)
 	defer git.Close() //nolint:errcheck // Best-effort cleanup.
 	executor.GitSparseCheckoutPaths = []string{"src/"}
 
 	git.Expect("--version").AndWriteToStdout("git version 2.25.4").AndExitWith(0)
 	git.Expect("sparse-checkout").WithAnyArguments().NotCalled()
 
-	active, err := executor.setupSparseCheckout(t.Context())
-	if err != nil {
-		t.Fatalf("executor.setupSparseCheckout(ctx) error = %v, want nil", err)
-	}
-	if active {
-		t.Fatalf("executor.setupSparseCheckout(ctx) active = true, want false")
+	paths := executor.resolveSparseCheckout(t.Context())
+	if len(paths) != 0 {
+		t.Fatalf("resolveSparseCheckout(ctx) = %#v, want nil (fallback to full checkout)", paths)
 	}
-	if got, want := out.String(), "Sparse checkout requires git >= 2.26; falling back to full checkout"; !strings.Contains(got, want) {
+	if got, want := out.String(), "Sparse checkout requires git >= 2.27, got 2.25; falling back to full checkout"; !strings.Contains(got, want) {
 		t.Fatalf("shell output = %q, want to contain %q", got, want)
 	}
 
@@ -156,16 +152,27 @@ func TestSetupSparseCheckout_VersionFallbackDisablesPriorSparseConfig(t *testing
 	executor.GitSparseCheckoutPaths = []string{"src/"}
 	createSparseCheckoutFile(t, executor.shell.Getwd())
 
+	// Old git: resolveSparseCheckout warns and returns nil paths, then
+	// setupSparseCheckout disables the prior sparse config left on disk.
 	git.Expect("--version").AndWriteToStdout("git version 2.25.4").AndExitWith(0)
 	git.Expect("config", "--get", "core.sparseCheckout").AndWriteToStdout("true\n").AndExitWith(0)
 	git.Expect("sparse-checkout", "disable").AndExitWith(0)
 	git.Expect("config", "--worktree", "--list").AndWriteToStdout("").AndExitWith(0)
 	git.Expect("config", "--unset", "extensions.worktreeConfig").AndExitWith(0)
 
-	if _, err := executor.setupSparseCheckout(t.Context()); err != nil {
-		t.Fatalf("executor.setupSparseCheckout(ctx) error = %v, want nil", err)
+	sparsePaths := executor.resolveSparseCheckout(t.Context())
+	if len(sparsePaths) != 0 {
+		t.Fatalf("resolveSparseCheckout(ctx) = %#v, want nil (fallback to full checkout)", sparsePaths)
+	}
+
+	active, err := executor.setupSparseCheckout(t.Context(), sparsePaths)
+	if err != nil {
+		t.Fatalf("executor.setupSparseCheckout(ctx, sparsePaths) error = %v, want nil", err)
+	}
+	if active {
+		t.Fatalf("executor.setupSparseCheckout(ctx, sparsePaths) active = true, want false")
 	}
-	if got, want := out.String(), "Sparse checkout requires git >= 2.26; falling back to full checkout"; !strings.Contains(got, want) {
+	if got, want := out.String(), "Sparse checkout requires git >= 2.27, got 2.25; falling back to full checkout"; !strings.Contains(got, want) {
 		t.Fatalf("shell output = %q, want to contain %q", got, want)
 	}
 	if got, want := out.String(), "Disabling sparse checkout from previous build"; !strings.Contains(got, want) {

internal/job/git.go

@@ -92,6 +92,22 @@ func gitCheckout(ctx context.Context, sh *shell.Shell, gitCheckoutFlags, referen
 	return nil
 }
 
+// hasPartialFilterFlags returns true if flags contains a
+// --filter=<spec> or --filter <spec> option.
+func hasPartialFilterFlags(flags []string) bool {
+	for i, f := range flags {
+		// Check for --filter=<spec>
+		if strings.HasPrefix(f, "--filter=") {
+			return true
+		}
+		// The --filter <spec> form only counts when a value actually follows it
+		if f == "--filter" && i+1 < len(flags) {
+			return true
+		}
+	}
+	return false
+}
+
 func gitClone(ctx context.Context, sh *shell.Shell, gitCloneFlags []string, repository, dir string) error {
 	commandArgs := []string{"clone"}
 	commandArgs = append(commandArgs, gitCloneFlags...)