diff --git a/internal/job/checkout.go b/internal/job/checkout.go index 97f23d3fcc..0fde09bb24 100644 --- a/internal/job/checkout.go +++ b/internal/job/checkout.go @@ -762,7 +762,10 @@ func (e *Executor) getOrUpdateMirrorDir(ctx context.Context, repository string) // fetchSource fetches the git source for the job. If GitSkipFetchExistingCommits is // enabled and the commit already exists locally, the fetch is skipped entirely. -func (e *Executor) fetchSource(ctx context.Context) error { +// When addBloblessFilter is true, --filter=blob:none is prepended to the fetch +// flags — the caller decides based on sparse-checkout state and user-supplied +// filters. +func (e *Executor) fetchSource(ctx context.Context, addBloblessFilter bool) error { // If configured, skip the fetch when the commit already exists locally. // This is useful when a pre-populated git mirror is used with --reference, // as the commit objects are already reachable and fetching is redundant. @@ -773,6 +776,9 @@ func (e *Executor) fetchSource(ctx context.Context) error { } gitFetchFlags := e.GitFetchFlags + if addBloblessFilter { + gitFetchFlags = "--filter=blob:none " + gitFetchFlags + } switch { case e.RefSpec != "": @@ -908,6 +914,21 @@ func (e *Executor) defaultCheckoutPhase(ctx context.Context) (retErr error) { return fmt.Errorf("creating checkout dir: %w", err) } + // Resolve the cone paths to check out (nil means a full checkout). + sparsePaths := e.resolveSparseCheckout(ctx) + + // Split the git clone flags into an array of strings, so we can append + // additional flags if needed (e.g., --reference, --dissociate, --sparse, --filter=blob:none). + gitCloneFlags, err := shellwords.Split(e.GitCloneFlags) + if err != nil { + return fmt.Errorf("splitting --git-clone-flags %q: %w", e.GitCloneFlags, err) + } + + // Snapshot whether the user supplied their own --filter before we + // potentially append one — the fetch-side decision depends on the + // original user-supplied state, not on flags we auto-add here. + userSuppliedCloneFilter := hasPartialFilterFlags(gitCloneFlags) + // On mirrors and dissociation: // // --reference makes the clone reuse objects from the mirror, using the @@ -953,10 +974,6 @@ func (e *Executor) defaultCheckoutPhase(ctx context.Context) (retErr error) { // Compute the clone flags. For mirrors we need --reference, and usually // --dissociate. - gitCloneFlags, err := shellwords.Split(e.GitCloneFlags) - if err != nil { - return fmt.Errorf("splitting --git-clone-flags %q: %w", e.GitCloneFlags, err) - } if mirrorDir != "" { gitCloneFlags = append(gitCloneFlags, "--reference", mirrorDir) if e.GitMirrorCheckoutMode == "dissociate" { @@ -964,6 +981,24 @@ func (e *Executor) defaultCheckoutPhase(ctx context.Context) (retErr error) { } } + // When sparse checkout applies, add two clone flags: + // --sparse clone in sparse mode + // --filter=blob:none make it a partial clone, so blobs outside the + // sparse set aren't downloaded up front + // Each flag is added only if the user hasn't already supplied their own. + if len(sparsePaths) > 0 { + if slices.Contains(gitCloneFlags, "--sparse") { + e.shell.Commentf("Sparse checkout is configured and BUILDKITE_GIT_CLONE_FLAGS already contains a --sparse flag (preserving user-supplied sparse checkout).") + } else { + gitCloneFlags = append(gitCloneFlags, "--sparse") + } + if userSuppliedCloneFilter { + e.shell.Commentf("Sparse checkout is configured and BUILDKITE_GIT_CLONE_FLAGS already contains a --filter (preserving user-supplied filter).") + } else { + gitCloneFlags = append(gitCloneFlags, "--filter=blob:none") + } + } + // Do the clone. if err := gitClone(ctx, e.shell, gitCloneFlags, e.Repository, "."); err != nil { return fmt.Errorf("cloning git repository: %w", err) @@ -991,7 +1026,16 @@ func (e *Executor) defaultCheckoutPhase(ctx context.Context) (retErr error) { } } - if err := e.fetchSource(ctx); err != nil { + // Parse the fetch flags into tokens so we can check for a user-supplied --filter flag. + gitFetchFlags, err := shellwords.Split(e.GitFetchFlags) + if err != nil { + return fmt.Errorf("splitting --git-fetch-flags %q: %w", e.GitFetchFlags, err) + } + + addBloblessFilter := len(sparsePaths) > 0 && + !userSuppliedCloneFilter && + !hasPartialFilterFlags(gitFetchFlags) + if err := e.fetchSource(ctx, addBloblessFilter); err != nil { return err } @@ -999,7 +1043,7 @@ func (e *Executor) defaultCheckoutPhase(ctx context.Context) (retErr error) { return err } - sparseCheckoutActive, err := e.setupSparseCheckout(ctx) + sparseCheckoutActive, err := e.setupSparseCheckout(ctx, sparsePaths) if err != nil { return err } @@ -1110,7 +1154,7 @@ func (e *Executor) defaultCheckoutPhase(ctx context.Context) (retErr error) { // When sparse-checkout is active, scope LFS to the same paths so we don't // pull objects outside the sparse set (SUP-6529). If sparse fell back to a - // full checkout (e.g. git < 2.26), fetch unscoped so files outside the + // full checkout (e.g. git < 2.27), fetch unscoped so files outside the // requested paths still get their LFS content. if e.GitLFSEnabled { lfsArgs := gitLFSFetchCheckoutArgs{ diff --git a/internal/job/checkout_sparse.go b/internal/job/checkout_sparse.go index 7e381afbc4..a07d349c8f 100644 --- a/internal/job/checkout_sparse.go +++ b/internal/job/checkout_sparse.go @@ -12,6 +12,31 @@ import ( "github.com/buildkite/agent/v3/internal/shell" ) +// resolveSparseCheckout returns the cone paths to check out for this build, or +// nil to check out the full tree — either because no paths were requested or +// because git is too old (< 2.27). +func (e *Executor) resolveSparseCheckout(ctx context.Context) []string { + paths := cleanGitSparseCheckoutPaths(e.GitSparseCheckoutPaths) + if len(paths) == 0 { + return nil + } + + // We require git >= 2.27 because setupSparseCheckout runs + // `git sparse-checkout set --cone `, which was promoted + // from experimental to stable in git 2.27. On older git versions, + // fall back to a full checkout by returning nil. + ok, got, err := gitVersionAtLeast(ctx, e.shell, 2, 27) + if err != nil { + e.shell.Warningf("Sparse checkout requires git >= 2.27; falling back to full checkout (%v)", err) + return nil + } + if !ok { + e.shell.Warningf("Sparse checkout requires git >= 2.27, got %s; falling back to full checkout", got) + return nil + } + return paths +} + func cleanGitSparseCheckoutPaths(paths []string) []string { cleaned := make([]string, 0, len(paths)) for _, path := range paths { @@ -30,21 +55,27 @@ func parseGitVersion(output string) (major, minor int, ok bool) { return major, minor, true } -func gitVersionAtLeast(ctx context.Context, sh *shell.Shell, major, minor int) (bool, error) { +// gitVersionAtLeast reports whether the local git binary is at least +// major.minor. It also returns the parsed "M.m" version string so callers can +// include it in log output. The err return is reserved for actual failures +// (git command failure, unparseable version output) — a git that is simply +// too old returns (false, "M.m", nil), not an error. +func gitVersionAtLeast(ctx context.Context, sh *shell.Shell, major, minor int) (ok bool, got string, err error) { output, err := sh.Command("git", "--version").RunAndCaptureStdout(ctx) if err != nil { - return false, err + return false, "", err } - gitMajor, gitMinor, ok := parseGitVersion(strings.TrimSpace(output)) - if !ok { - return false, fmt.Errorf("parsing git version from %q", strings.TrimSpace(output)) + gitMajor, gitMinor, parseOK := parseGitVersion(strings.TrimSpace(output)) + if !parseOK { + return false, "", fmt.Errorf("parsing git version from %q", strings.TrimSpace(output)) } + got = fmt.Sprintf("%d.%d", gitMajor, gitMinor) if gitMajor != major { - return gitMajor > major, nil + return gitMajor > major, got, nil } - return gitMinor >= minor, nil + return gitMinor >= minor, got, nil } // sparseCheckoutMayBeConfigured does a cheap filesystem check for marker files @@ -91,31 +122,19 @@ func (e *Executor) disableSparseCheckoutIfConfigured(ctx context.Context) { } } -// setupSparseCheckout configures (or disables) git sparse checkout for the -// current working tree. It returns true if sparse checkout was successfully -// applied for this build, so callers can adjust later behaviour (e.g. skip -// submodule init, which requires the full tree). -func (e *Executor) setupSparseCheckout(ctx context.Context) (bool, error) { - paths := cleanGitSparseCheckoutPaths(e.GitSparseCheckoutPaths) - if len(paths) == 0 { - e.disableSparseCheckoutIfConfigured(ctx) - return false, nil - } - - ok, err := gitVersionAtLeast(ctx, e.shell, 2, 26) - if err != nil { - e.shell.Warningf("Sparse checkout requires git >= 2.26; falling back to full checkout (%v)", err) - e.disableSparseCheckoutIfConfigured(ctx) - return false, nil - } - if !ok { - e.shell.Warningf("Sparse checkout requires git >= 2.26; falling back to full checkout") +// setupSparseCheckout configures git sparse checkout for the given cone paths. +// When sparsePaths is empty it does a full checkout instead, disabling any +// prior sparse checkout configuration. It returns true when sparse checkout is +// applied, so callers can skip steps that need the full tree (e.g. submodule +// init). +func (e *Executor) setupSparseCheckout(ctx context.Context, sparsePaths []string) (bool, error) { + if len(sparsePaths) == 0 { e.disableSparseCheckoutIfConfigured(ctx) return false, nil } - e.shell.Commentf("Setting up sparse checkout for paths: %s", strings.Join(paths, ",")) - args := append([]string{"sparse-checkout", "set", "--cone"}, paths...) + e.shell.Commentf("Setting up sparse checkout for paths: %s", strings.Join(sparsePaths, ",")) + args := append([]string{"sparse-checkout", "set", "--cone"}, sparsePaths...) if err := e.shell.Command("git", args...).Run(ctx); err != nil { return false, fmt.Errorf("setting sparse checkout paths: %w", err) } diff --git a/internal/job/checkout_sparse_test.go b/internal/job/checkout_sparse_test.go index 61d05e5336..8df1f66535 100644 --- a/internal/job/checkout_sparse_test.go +++ b/internal/job/checkout_sparse_test.go @@ -54,17 +54,16 @@ func TestParseGitVersion(t *testing.T) { func TestSetupSparseCheckout_Enable(t *testing.T) { executor, git, out := newSparseCheckoutTestExecutor(t) defer git.Close() //nolint:errcheck // Best-effort cleanup. - executor.GitSparseCheckoutPaths = []string{".buildkite/", "src/"} - git.Expect("--version").AndWriteToStdout("git version 2.39.0").AndExitWith(0) + paths := []string{".buildkite/", "src/"} git.Expect("sparse-checkout", "set", "--cone", ".buildkite/", "src/").AndExitWith(0) - active, err := executor.setupSparseCheckout(t.Context()) + active, err := executor.setupSparseCheckout(t.Context(), paths) if err != nil { - t.Fatalf("executor.setupSparseCheckout(ctx) error = %v, want nil", err) + t.Fatalf("executor.setupSparseCheckout(ctx, sparsePaths) error = %v, want nil", err) } if !active { - t.Fatalf("executor.setupSparseCheckout(ctx) active = false, want true") + t.Fatalf("executor.setupSparseCheckout(ctx, sparsePaths) active = false, want true") } if got, want := out.String(), "Setting up sparse checkout for paths: .buildkite/,src/"; !strings.Contains(got, want) { t.Fatalf("shell output = %q, want to contain %q", got, want) @@ -83,12 +82,12 @@ func TestSetupSparseCheckout_DisableWithPriorSparseConfig(t *testing.T) { git.Expect("config", "--worktree", "--list").AndWriteToStdout("").AndExitWith(0) git.Expect("config", "--unset", "extensions.worktreeConfig").AndExitWith(0) - active, err := executor.setupSparseCheckout(t.Context()) + active, err := executor.setupSparseCheckout(t.Context(), nil) if err != nil { - t.Fatalf("executor.setupSparseCheckout(ctx) error = %v, want nil", err) + t.Fatalf("executor.setupSparseCheckout(ctx, sparsePaths) error = %v, want nil", err) } if active { - t.Fatalf("executor.setupSparseCheckout(ctx) active = true, want false") + t.Fatalf("executor.setupSparseCheckout(ctx, sparsePaths) active = true, want false") } if got, want := out.String(), "Disabling sparse checkout from previous build"; !strings.Contains(got, want) { t.Fatalf("shell output = %q, want to contain %q", got, want) @@ -107,8 +106,8 @@ func TestSetupSparseCheckout_DisablePreservesOtherWorktreeConfig(t *testing.T) { git.Expect("config", "--worktree", "--list").AndWriteToStdout("user.something=value\n").AndExitWith(0) git.Expect("config", "--unset", "extensions.worktreeConfig").NotCalled() - if _, err := executor.setupSparseCheckout(t.Context()); err != nil { - t.Fatalf("executor.setupSparseCheckout(ctx) error = %v, want nil", err) + if _, err := executor.setupSparseCheckout(t.Context(), nil); err != nil { + t.Fatalf("executor.setupSparseCheckout(ctx, sparsePaths) error = %v, want nil", err) } git.Check(t) @@ -121,14 +120,14 @@ func TestSetupSparseCheckout_DisableWithoutPriorSparseConfig(t *testing.T) { git.Expect("config").WithAnyArguments().NotCalled() git.Expect("sparse-checkout").WithAnyArguments().NotCalled() - if _, err := executor.setupSparseCheckout(t.Context()); err != nil { - t.Fatalf("executor.setupSparseCheckout(ctx) error = %v, want nil", err) + if _, err := executor.setupSparseCheckout(t.Context(), nil); err != nil { + t.Fatalf("executor.setupSparseCheckout(ctx, sparsePaths) error = %v, want nil", err) } git.Check(t) } -func TestSetupSparseCheckout_VersionFallback(t *testing.T) { +func TestResolveSparseCheckout_VersionFallback(t *testing.T) { executor, git, out := newSparseCheckoutTestExecutor(t) defer git.Close() //nolint:errcheck // Best-effort cleanup. executor.GitSparseCheckoutPaths = []string{"src/"} @@ -136,14 +135,11 @@ func TestSetupSparseCheckout_VersionFallback(t *testing.T) { git.Expect("--version").AndWriteToStdout("git version 2.25.4").AndExitWith(0) git.Expect("sparse-checkout").WithAnyArguments().NotCalled() - active, err := executor.setupSparseCheckout(t.Context()) - if err != nil { - t.Fatalf("executor.setupSparseCheckout(ctx) error = %v, want nil", err) - } - if active { - t.Fatalf("executor.setupSparseCheckout(ctx) active = true, want false") + paths := executor.resolveSparseCheckout(t.Context()) + if len(paths) != 0 { + t.Fatalf("resolveSparseCheckout(ctx) = %#v, want nil (fallback to full checkout)", paths) } - if got, want := out.String(), "Sparse checkout requires git >= 2.26; falling back to full checkout"; !strings.Contains(got, want) { + if got, want := out.String(), "Sparse checkout requires git >= 2.27, got 2.25; falling back to full checkout"; !strings.Contains(got, want) { t.Fatalf("shell output = %q, want to contain %q", got, want) } @@ -156,16 +152,27 @@ func TestSetupSparseCheckout_VersionFallbackDisablesPriorSparseConfig(t *testing executor.GitSparseCheckoutPaths = []string{"src/"} createSparseCheckoutFile(t, executor.shell.Getwd()) + // Old git: resolveSparseCheckout warns and returns nil paths, then + // setupSparseCheckout disables the prior sparse config left on disk. git.Expect("--version").AndWriteToStdout("git version 2.25.4").AndExitWith(0) git.Expect("config", "--get", "core.sparseCheckout").AndWriteToStdout("true\n").AndExitWith(0) git.Expect("sparse-checkout", "disable").AndExitWith(0) git.Expect("config", "--worktree", "--list").AndWriteToStdout("").AndExitWith(0) git.Expect("config", "--unset", "extensions.worktreeConfig").AndExitWith(0) - if _, err := executor.setupSparseCheckout(t.Context()); err != nil { - t.Fatalf("executor.setupSparseCheckout(ctx) error = %v, want nil", err) + sparsePaths := executor.resolveSparseCheckout(t.Context()) + if len(sparsePaths) != 0 { + t.Fatalf("resolveSparseCheckout(ctx) = %#v, want nil (fallback to full checkout)", sparsePaths) + } + + active, err := executor.setupSparseCheckout(t.Context(), sparsePaths) + if err != nil { + t.Fatalf("executor.setupSparseCheckout(ctx, sparsePaths) error = %v, want nil", err) + } + if active { + t.Fatalf("executor.setupSparseCheckout(ctx, sparsePaths) active = true, want false") } - if got, want := out.String(), "Sparse checkout requires git >= 2.26; falling back to full checkout"; !strings.Contains(got, want) { + if got, want := out.String(), "Sparse checkout requires git >= 2.27, got 2.25; falling back to full checkout"; !strings.Contains(got, want) { t.Fatalf("shell output = %q, want to contain %q", got, want) } if got, want := out.String(), "Disabling sparse checkout from previous build"; !strings.Contains(got, want) { diff --git a/internal/job/git.go b/internal/job/git.go index 343385179e..a5610ad557 100644 --- a/internal/job/git.go +++ b/internal/job/git.go @@ -92,6 +92,22 @@ func gitCheckout(ctx context.Context, sh *shell.Shell, gitCheckoutFlags, referen return nil } +// hasPartialFilterFlags returns true if flags contains a +// --filter= or --filter option. +func hasPartialFilterFlags(flags []string) bool { + for i, f := range flags { + // Check for --filter= + if strings.HasPrefix(f, "--filter=") { + return true + } + // The --filter form only counts when a value actually follows it + if f == "--filter" && i+1 < len(flags) { + return true + } + } + return false +} + func gitClone(ctx context.Context, sh *shell.Shell, gitCloneFlags []string, repository, dir string) error { commandArgs := []string{"clone"} commandArgs = append(commandArgs, gitCloneFlags...) diff --git a/internal/job/git_test.go b/internal/job/git_test.go index 8a6df06e28..ca3bdf30e2 100644 --- a/internal/job/git_test.go +++ b/internal/job/git_test.go @@ -240,6 +240,36 @@ func TestGitClone(t *testing.T) { } } +func TestHasPartialFilterFlags(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + flags []string + want bool + }{ + {name: "no filter", flags: []string{"-v", "--reference", "mirror"}, want: false}, + {name: "filter blob:none with equals", flags: []string{"-v", "--filter=blob:none"}, want: true}, + {name: "filter blob:none separate arg", flags: []string{"-v", "--filter", "blob:none"}, want: true}, + {name: "filter tree:0 with equals", flags: []string{"-v", "--filter=tree:0"}, want: true}, + {name: "filter tree:0 separate arg", flags: []string{"-v", "--filter", "tree:0"}, want: true}, + {name: "multiple filters with blob:none", flags: []string{"--filter=blob:none", "--filter=tree:0"}, want: true}, + {name: "multiple filters separate args", flags: []string{"--filter", "blob:none", "--filter", "tree:0"}, want: true}, + {name: "filter prefix lookalike", flags: []string{"-v", "--filtered"}, want: false}, + {name: "filter without value", flags: []string{"--filter"}, want: false}, + {name: "empty", flags: nil, want: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + if got := hasPartialFilterFlags(tt.flags); got != tt.want { + t.Errorf("hasPartialFilterFlags(%v) = %t, want %t", tt.flags, got, tt.want) + } + }) + } +} + func TestGitClean(t *testing.T) { t.Parallel() ctx := t.Context() diff --git a/internal/job/integration/checkout_git_mirrors_integration_test.go b/internal/job/integration/checkout_git_mirrors_integration_test.go index e1c2d80c8e..2b6e41ee92 100644 --- a/internal/job/integration/checkout_git_mirrors_integration_test.go +++ b/internal/job/integration/checkout_git_mirrors_integration_test.go @@ -172,10 +172,10 @@ func TestCheckingOutLocalGitProjectWithSparseCheckout_WithGitMirrors(t *testing. git.ExpectAll([][]any{ {"clone", "--mirror", "--bare", "--", tester.Repo.Path, matchSubDir(tester.GitMirrorsDir)}, + {"--version"}, {"clone", "-v", "--filter=blob:none", "--sparse", "--reference", matchSubDir(tester.GitMirrorsDir), "--", tester.Repo.Path, "."}, {"clean", "-fdq"}, {"fetch", "-v", "--filter=blob:none", "--", "origin", "main"}, - {"--version"}, {"sparse-checkout", "set", "--cone", ".buildkite/", "src/"}, {"-c", "advice.detachedHead=false", "checkout", "-f", "FETCH_HEAD"}, {"clean", "-fdq"}, diff --git a/internal/job/integration/checkout_integration_test.go b/internal/job/integration/checkout_integration_test.go index 7624a2ec52..565114e244 100644 --- a/internal/job/integration/checkout_integration_test.go +++ b/internal/job/integration/checkout_integration_test.go @@ -40,8 +40,8 @@ func skipIfGitSparseCheckoutUnsupported(t *testing.T) { if _, err := fmt.Sscanf(string(out), "git version %d.%d", &major, &minor); err != nil { t.Skipf("couldn't parse git version from %q: %v", strings.TrimSpace(string(out)), err) } - if major < 2 || (major == 2 && minor < 26) { - t.Skipf("git sparse-checkout --cone requires git >= 2.26, got %s", strings.TrimSpace(string(out))) + if major < 2 || (major == 2 && minor < 27) { + t.Skipf("git sparse-checkout set --cone requires git >= 2.27, got %s", strings.TrimSpace(string(out))) } } @@ -157,6 +157,198 @@ func TestCheckingOutLocalGitProject(t *testing.T) { tester.RunAndCheck(t, env...) } +// TestCheckingOutLocalGitProjectWithSparseCheckoutFallsBackOnOldGit exercises +// the fallback path in resolveSparseCheckout when git is older than 2.27 (below +// the cone-mode floor). It overrides `git --version` via AndCallFunc to +// report an old git version, while every other invocation passes through to +// the real binary. +// +// The fallback is observable via: the warning in output, the absence of +// auto-added --filter=blob:none on both clone and fetch, and the absence of +// `sparse-checkout set --cone`. The user's own --sparse in +// BUILDKITE_GIT_CLONE_FLAGS still initialises a sparse config, so the test +// also asserts `sparse-checkout disable` runs to undo it. +func TestCheckingOutLocalGitProjectWithSparseCheckoutFallsBackOnOldGit(t *testing.T) { + t.Parallel() + skipIfGitSparseCheckoutUnsupported(t) + + tester, err := NewExecutorTester(mainCtx) + if err != nil { + t.Fatalf("NewExecutorTester() error = %v", err) + } + defer tester.Close() + addSparseCheckoutFixture(t, tester.Repo) + + env := []string{ + "BUILDKITE_GIT_CLONE_FLAGS=-v --sparse", + "BUILDKITE_GIT_CLEAN_FLAGS=-fdq", + "BUILDKITE_GIT_FETCH_FLAGS=-v", + "BUILDKITE_GIT_SPARSE_CHECKOUT_PATHS=src/", + } + + // bintest's mock dispatch always prefers a mock-level passthrough over any + // per-expectation AndCallFunc (see bintest Mock.Invoke), so we can't set + // PassthroughToLocalCommand on the mock — it would run real git for + // `--version` and skip our lie. Instead, register each expectation + // individually and passthrough to real git per-expectation, leaving + // `--version` free to use AndCallFunc. + realGit, err := exec.LookPath("git") + if err != nil { + t.Fatalf("exec.LookPath(git) error = %v", err) + } + + git := tester.MustMock(t, "git") + + git.Expect("--version").AndCallFunc(func(c *bintest.Call) { + _, _ = fmt.Fprintln(c.Stdout, "git version 2.20.0") + c.Exit(0) + }) + + expect := func(args ...any) *bintest.Expectation { + return git.Expect(args...).AndPassthroughToLocalCommand(realGit) + } + + // `git clone --sparse` initialises a sparse-checkout config, so the + // fallback path runs `sparse-checkout disable` to undo it. Crucially, + // `sparse-checkout set --cone ...` must NOT appear — its absence from + // this sequence is what proves the fallback worked. + expect("clone", "-v", "--sparse", "--", tester.Repo.Path, ".") + expect("clean", "-fdq") + expect("fetch", "-v", "--", "origin", "main") + expect("config", "--get", "core.sparseCheckout") + expect("sparse-checkout", "disable") + expect("config", "--worktree", "--list") + // Whether `config --unset extensions.worktreeConfig` fires depends on the + // real git version: `sparse-checkout disable` on git >= ~2.36 leaves + // core.sparsecheckout=false (and friends) in the worktree config, so + // `config --worktree --list` returns non-empty and the unset is skipped. + // On older git the worktree config is empty and the unset does fire. + expect("config", "--unset", "extensions.worktreeConfig").Optionally() + expect("-c", "advice.detachedHead=false", "checkout", "-f", "FETCH_HEAD") + expect("clean", "-fdq") + expect("--no-pager", "log", "-1", "HEAD", "-s", "--no-color", gitShowFormatArg) + + agent := tester.MockAgent(t) + agent.Expect("meta-data", "exists", job.CommitMetadataKey).AndExitWith(1) + agent.Expect("meta-data", "set", job.CommitMetadataKey).WithStdin(commitPattern) + + tester.RunAndCheck(t, env...) + + if want := "Sparse checkout requires git >= 2.27, got 2.20; falling back to full checkout"; !strings.Contains(tester.Output, want) { + t.Errorf("bootstrap output missing fallback warning %q\noutput:\n%s", want, tester.Output) + } + + // With sparse checkout disabled, every path from the fixture must be on + // disk. + requireCheckoutPath(t, tester.CheckoutDir(), ".buildkite/pipeline.yml", true) + requireCheckoutPath(t, tester.CheckoutDir(), "src/main.txt", true) + requireCheckoutPath(t, tester.CheckoutDir(), "docs/readme.md", true) +} + +// TestCheckingOutLocalGitProjectWithSparseCheckoutAutoAddsBlobNoneFilter +// covers the agent's auto-injection of --filter=blob:none onto the clone when +// sparse checkout paths are configured but the user did NOT supply any +// --filter in BUILDKITE_GIT_CLONE_FLAGS. The other sparse integration tests +// all pre-supply --filter=blob:none, which short-circuits the auto-add and +// therefore doesn't exercise the "add it" branch of the clone-flag logic in +// defaultCheckoutPhase. +func TestCheckingOutLocalGitProjectWithSparseCheckoutAutoAddsBlobNoneFilter(t *testing.T) { + t.Parallel() + skipIfGitSparseCheckoutUnsupported(t) + + tester, err := NewExecutorTester(mainCtx) + if err != nil { + t.Fatalf("NewExecutorTester() error = %v", err) + } + defer tester.Close() + addSparseCheckoutFixture(t, tester.Repo) + + env := []string{ + "BUILDKITE_GIT_CLONE_FLAGS=-v", + "BUILDKITE_GIT_CLEAN_FLAGS=-fdq", + "BUILDKITE_GIT_FETCH_FLAGS=-v", + "BUILDKITE_GIT_SPARSE_CHECKOUT_PATHS=.buildkite/,src/", + } + + git := tester. + MustMock(t, "git"). + PassthroughToLocalCommand() + + git.ExpectAll([][]any{ + {"--version"}, + {"clone", "-v", "--sparse", "--filter=blob:none", "--", tester.Repo.Path, "."}, + {"clean", "-fdq"}, + {"fetch", "--filter=blob:none", "-v", "--", "origin", "main"}, + {"sparse-checkout", "set", "--cone", ".buildkite/", "src/"}, + {"-c", "advice.detachedHead=false", "checkout", "-f", "FETCH_HEAD"}, + {"clean", "-fdq"}, + {"--no-pager", "log", "-1", "HEAD", "-s", "--no-color", gitShowFormatArg}, + }) + + agent := tester.MockAgent(t) + agent.Expect("meta-data", "exists", job.CommitMetadataKey).AndExitWith(1) + agent.Expect("meta-data", "set", job.CommitMetadataKey).WithStdin(commitPattern) + + tester.RunAndCheck(t, env...) + + requireCheckoutPath(t, tester.CheckoutDir(), ".buildkite/pipeline.yml", true) + requireCheckoutPath(t, tester.CheckoutDir(), "src/main.txt", true) + requireCheckoutPath(t, tester.CheckoutDir(), "docs/readme.md", false) +} + +// TestCheckingOutLocalGitProjectWithSparseCheckoutPreservesUserFilter asserts +// that when the user supplies their own --filter in BUILDKITE_GIT_CLONE_FLAGS, +// sparse checkout preserves it on clone and does NOT prepend --filter=blob:none +// to fetch. git clone --filter=X writes remote.origin.partialclonefilter=X into +// the repo config, and subsequent fetches inherit it automatically — so +// re-passing a filter on fetch would silently override the user's choice. +func TestCheckingOutLocalGitProjectWithSparseCheckoutPreservesUserFilter(t *testing.T) { + t.Parallel() + skipIfGitSparseCheckoutUnsupported(t) + + tester, err := NewExecutorTester(mainCtx) + if err != nil { + t.Fatalf("NewExecutorTester() error = %v", err) + } + defer tester.Close() + addSparseCheckoutFixture(t, tester.Repo) + + env := []string{ + "BUILDKITE_GIT_CLONE_FLAGS=-v --filter=tree:0", + "BUILDKITE_GIT_CLEAN_FLAGS=-fdq", + "BUILDKITE_GIT_FETCH_FLAGS=-v", + "BUILDKITE_GIT_SPARSE_CHECKOUT_PATHS=.buildkite/,src/", + } + + git := tester. + MustMock(t, "git"). + PassthroughToLocalCommand() + + git.ExpectAll([][]any{ + {"--version"}, + // User's --filter=tree:0 is preserved; --sparse is still auto-added. + {"clone", "-v", "--filter=tree:0", "--sparse", "--", tester.Repo.Path, "."}, + {"clean", "-fdq"}, + // Fetch does NOT get --filter=blob:none prepended — the user's + // tree:0 filter is already stored in repo config and inherited here. + {"fetch", "-v", "--", "origin", "main"}, + {"sparse-checkout", "set", "--cone", ".buildkite/", "src/"}, + {"-c", "advice.detachedHead=false", "checkout", "-f", "FETCH_HEAD"}, + {"clean", "-fdq"}, + {"--no-pager", "log", "-1", "HEAD", "-s", "--no-color", gitShowFormatArg}, + }) + + agent := tester.MockAgent(t) + agent.Expect("meta-data", "exists", job.CommitMetadataKey).AndExitWith(1) + agent.Expect("meta-data", "set", job.CommitMetadataKey).WithStdin(commitPattern) + + tester.RunAndCheck(t, env...) + + requireCheckoutPath(t, tester.CheckoutDir(), ".buildkite/pipeline.yml", true) + requireCheckoutPath(t, tester.CheckoutDir(), "src/main.txt", true) + requireCheckoutPath(t, tester.CheckoutDir(), "docs/readme.md", false) +} + func TestCheckingOutLocalGitProjectWithSparseCheckout(t *testing.T) { t.Parallel() skipIfGitSparseCheckoutUnsupported(t) @@ -180,10 +372,10 @@ func TestCheckingOutLocalGitProjectWithSparseCheckout(t *testing.T) { PassthroughToLocalCommand() git.ExpectAll([][]any{ + {"--version"}, {"clone", "-v", "--filter=blob:none", "--sparse", "--", tester.Repo.Path, "."}, {"clean", "-fdq"}, {"fetch", "-v", "--filter=blob:none", "--", "origin", "main"}, - {"--version"}, {"sparse-checkout", "set", "--cone", ".buildkite/", "src/"}, {"-c", "advice.detachedHead=false", "checkout", "-f", "FETCH_HEAD"}, {"clean", "-fdq"}, @@ -307,10 +499,10 @@ func TestCheckingOutLocalGitProjectWithSparseCheckoutReconfiguresExistingGitDir( agent.Expect("meta-data", "exists", job.CommitMetadataKey).AndExitWith(1) agent.Expect("meta-data", "set", job.CommitMetadataKey).WithStdin(commitPattern) git.ExpectAll([][]any{ + {"--version"}, {"clone", "-v", "--filter=blob:none", "--sparse", "--", tester.Repo.Path, "."}, {"clean", "-fdq"}, {"fetch", "-v", "--filter=blob:none", "--", "origin", "main"}, - {"--version"}, {"sparse-checkout", "set", "--cone", "src/"}, {"-c", "advice.detachedHead=false", "checkout", "-f", "FETCH_HEAD"}, {"clean", "-fdq"}, @@ -326,10 +518,10 @@ func TestCheckingOutLocalGitProjectWithSparseCheckoutReconfiguresExistingGitDir( env[len(env)-1] = "BUILDKITE_GIT_SPARSE_CHECKOUT_PATHS=.buildkite/" agent.Expect("meta-data", "exists", job.CommitMetadataKey).AndExitWith(0) git.ExpectAll([][]any{ + {"--version"}, {"config", "--get-all", "remote.origin.url"}, {"clean", "-fdq"}, {"fetch", "-v", "--filter=blob:none", "--", "origin", "main"}, - {"--version"}, {"sparse-checkout", "set", "--cone", ".buildkite/"}, {"-c", "advice.detachedHead=false", "checkout", "-f", "FETCH_HEAD"}, {"clean", "-fdq"}, @@ -474,11 +666,11 @@ func TestCheckingOutLocalGitProjectWithSparseCheckoutSkipsSubmodules(t *testing. git.Expect("submodule", "sync", "--recursive").NotCalled() git.Expect("submodule", "update").WithAnyArguments().NotCalled() git.ExpectAll([][]any{ - {"clone", "-v", "--sparse", "--", tester.Repo.Path, "."}, + {"--version"}, + {"clone", "-v", "--sparse", "--filter=blob:none", "--", tester.Repo.Path, "."}, {"clean", "-fdq"}, {"submodule", "foreach", "--recursive", "git clean -fdq"}, - {"fetch", "-v", "--", "origin", "main"}, - {"--version"}, + {"fetch", "--filter=blob:none", "-v", "--", "origin", "main"}, {"sparse-checkout", "set", "--cone", "src/"}, {"-c", "advice.detachedHead=false", "checkout", "-f", "FETCH_HEAD"}, {"clean", "-fdq"},