[JENKINS-70094] Add support for partial clone

src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java

@@ -522,6 +522,7 @@
             private Integer timeout;
             private boolean tags = true;
             private Integer depth = 1;
+            private String filterSpec = null;
 
             @Override
             public FetchCommand from(URIish remote, List<RefSpec> refspecs) {
@@ -566,6 +567,12 @@
                 return this;
             }
 
+            @Override
+            public FetchCommand filter(String filterSpec) {
+                this.filterSpec = filterSpec;
+                return this;
+            }
+
             @Override
             public void execute() throws GitException, InterruptedException {
                 listener.getLogger().println("Fetching upstream changes from " + url);
@@ -592,6 +599,37 @@
                     args.add("--depth=" + depth);
                 }
 
+                if (filterSpec != null) {
+                    // try to get the current filterspec
+                    String currentFilterSpec = null;
+                    String defaultRemote = null;
+                    try {
+                        defaultRemote = getDefaultRemote();
+                        if (defaultRemote != null && !defaultRemote.isEmpty()) {
+                            currentFilterSpec =
+                                    launchCommand("config", "remote." + defaultRemote + ".partialclonefilter");
+                        }
+                        // we might fail if we have no promisor configured, catch the exception and just continue
+                    } catch (GitException e) {
+                        // leave currentFilterSpec null
+                    }
+
+                    if (isAtLeastVersion(2, 22, 0, 0)) {
+                        args.add("--filter=" + filterSpec);
+                    }
+
+                    // if the filterspec has changed
+                    if (defaultRemote != null && !filterSpec.equals(currentFilterSpec)) {
+                        launchCommand("config", "remote." + defaultRemote + ".promisor", "true");
+                        launchCommand("config", "remote." + defaultRemote + ".partialclonefilter", filterSpec);
+
+                        if (isAtLeastVersion(2, 36, 0, 0)) {
+                            // reapply filter and trigger maintenance
+                            args.add("--refetch");
+                        }
+                    }
+                }
+
                 warnIfWindowsTemporaryDirNameHasSpaces();
 
                 StandardCredentials cred = credentials.get(url.toPrivateString());
@@ -727,6 +765,7 @@
             private boolean tags = true;
             private List<RefSpec> refspecs;
             private Integer depth = 1;
+            private String filterSpec = null;
 
             @Override
             public CloneCommand url(String url) {
@@ -802,6 +841,12 @@
                 return this;
             }
 
+            @Override
+            public CloneCommand filter(String filterSpec) {
+                this.filterSpec = filterSpec;
+                return this;
+            }
+
             @Override
             public void execute() throws GitException, InterruptedException {
 
@@ -872,9 +917,14 @@
                 if (refspecs == null) {
                     refspecs = Collections.singletonList(new RefSpec("+refs/heads/*:refs/remotes/" + origin + "/*"));
                 }
+                if (filterSpec != null) {
+                    launchCommand("config", "--add", "remote." + origin + ".promisor", "true");
+                    launchCommand("config", "--add", "remote." + origin + ".partialclonefilter", filterSpec);
+                }
                 fetch_().from(urIish, refspecs)
                         .shallow(shallow)
                         .depth(depth)
+                        .filter(filterSpec)
                         .timeout(timeout)
                         .tags(tags)
                         .execute();
@@ -1761,6 +1811,15 @@
         return new File(workspace, pathJoin(".git", "shallow")).exists();
     }
 
+    /** Returns true if the remote has a promisor configured for missing blobs. */
+    boolean hasPromisor(String name) throws GitException, InterruptedException {
+        try {
+            return launchCommand("config", "remote." + name + ".promisor").contains("true");
+        } catch (GitException ge) {
+            return false;
+        }
+    }
+
     private String pathJoin(String a, String b) {
         return new File(a, b).toString();
     }
@@ -2090,6 +2149,17 @@
             @NonNull URIish url,
             Integer timeout)
             throws GitException, InterruptedException {
+        return launchCommandWithCredentials(args, workDir, environment, credentials, url, timeout);
+    }
+
+    private String launchCommandWithCredentials(
+            ArgumentListBuilder args,
+            File workDir,
+            EnvVars env,
+            StandardCredentials credentials,
+            @NonNull URIish url,
+            Integer timeout)
+            throws GitException, InterruptedException {
 
         Path key = null;
         Path ssh = null;
@@ -2098,7 +2168,6 @@
         Path passwordFile = null;
         Path passphrase = null;
         Path knownHostsTemp = null;
-        EnvVars env = environment;
         if (!PROMPT_FOR_AUTHENTICATION && isAtLeastVersion(2, 3, 0, 0)) {
             env = new EnvVars(env);
             env.put("GIT_TERMINAL_PROMPT", "false"); // Don't prompt for auth from command line git
@@ -3179,7 +3248,32 @@
                         args.add("-f");
                     }
                     args.add(ref);
-                    launchCommandIn(args, workspace, checkoutEnv, timeout);
+
+                    String repoUrl = null;
+                    try {
+                        String defaultRemote = getDefaultRemote();
+                        if (defaultRemote != null && !defaultRemote.isEmpty() && hasPromisor(defaultRemote)) {
+                            repoUrl = getRemoteUrl(defaultRemote);
+                        }
+                    } catch (GitException e) {
+                        /* Nothing to do, just keeping repoUrl = null */
+                    }
+
+                    if (repoUrl != null) {
+                        StandardCredentials cred = credentials.get(repoUrl);
+                        if (cred == null) {
+                            cred = defaultCredentials;
+                        }
+
+                        try {
+                            launchCommandWithCredentials(
+                                    args, workspace, checkoutEnv, cred, new URIish(repoUrl), timeout);
+                        } catch (URISyntaxException e) {
+                            throw new GitException("Invalid URL " + repoUrl, e);
+                        }
+                    } else {
+                        launchCommandIn(args, workspace, checkoutEnv, timeout);
+                    }
 
                     if (lfsRemote != null) {
                         final String url = getRemoteUrl(lfsRemote);

src/main/java/org/jenkinsci/plugins/gitclient/CloneCommand.java

@@ -114,4 +114,13 @@ public interface CloneCommand extends GitCommand {
      * @return a {@link org.jenkinsci.plugins.gitclient.CloneCommand} object.
      */
     CloneCommand depth(Integer depth);
+
+    /**
+     * Apply an object filter to a partial clone. If unset, a full clone is performed.
+     *
+     * @param filterSpec filter of objects to be sent by the server
+     * @return a {@link org.jenkinsci.plugins.gitclient.CloneCommand} object.
+     * @since 6.7.0
+     */
+    CloneCommand filter(String filterSpec);
 }

src/main/java/org/jenkinsci/plugins/gitclient/FetchCommand.java

@@ -64,4 +64,13 @@ public interface FetchCommand extends GitCommand {
      * @return a {@link org.jenkinsci.plugins.gitclient.CloneCommand} object.
      */
     FetchCommand depth(Integer depth);
+
+    /**
+     * Apply an object filter to a partial clone. If unset, a full clone is performed.
+     *
+     * @param filterSpec filter of objects to be sent by the server
+     * @return a {@link org.jenkinsci.plugins.gitclient.CloneCommand} object.
+     * @since 6.7.0
+     */
+    FetchCommand filter(String filterSpec);
 }

src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java

@@ -786,6 +786,15 @@
                 return this;
             }
 
+            @Override
+            public org.jenkinsci.plugins.gitclient.FetchCommand filter(String filterSpec) {
+                if (filterSpec != null) {
+                    listener.getLogger()
+                            .println("[WARNING] JGit does not support partial clone filters; reverting to full clone");
+                }
+                return this;
+            }
+
             @Override
             public void execute() throws GitException {
                 try (Repository repo = getRepository()) {
@@ -1638,6 +1647,15 @@
                 return this;
             }
 
+            @Override
+            public CloneCommand filter(String filterSpec) {
+                if (filterSpec != null) {
+                    listener.getLogger()
+                            .println("[WARNING] JGit does not support partial clone filters; reverting to full clone");
+                }
+                return this;
+            }
+
             private RepositoryBuilder newRepositoryBuilder() {
                 RepositoryBuilder builder = new RepositoryBuilder();
                 builder.setGitDir(new File(workspace, Constants.DOT_GIT)).readEnvironment();

src/test/java/org/jenkinsci/plugins/gitclient/CredentialsTest.java

@@ -516,6 +516,38 @@ void testLfsMergeWithCredentials() throws Exception {
                 "Fast-forward merge failed. master and modified_lfs should be the same.");
     }
 
+    @Test
+    @Issue("JENKINS-70094")
+    void testCheckoutPartialCloneWithCredentials() throws Exception {
+        if (testPeriodExpired() || lfsSpecificTest) {
+            return;
+        }
+        File clonedFile = new File(repo, fileToCheck);
+        git.init_().workspace(repo.getAbsolutePath()).execute();
+        assertFalse(clonedFile.exists(), "file " + fileToCheck + " in " + repo + ", has " + listDir(repo));
+        addCredential();
+        // clone with remote url
+        git.clone_()
+                .url(gitRepoURL)
+                .repositoryName("origin")
+                .filter("blob:none")
+                .execute();
+        ObjectId master = git.getHeadRev(gitRepoURL, "master");
+        git.checkout()
+                .branch("master")
+                .ref(master.getName())
+                .deleteBranchIfExist(true)
+                .execute();
+        assertTrue(git.isCommitInRepo(master), "master: " + master + " not in repo");
+        assertEquals(
+                master,
+                git.withRepository(
+                        (gitRepo, unusedChannel) -> gitRepo.findRef("master").getObjectId()),
+                "Master != HEAD");
+        assertEquals("master", git.withRepository((gitRepo, unusedChanel) -> gitRepo.getBranch()), "Wrong branch");
+        assertTrue(clonedFile.exists(), "No file " + fileToCheck + ", has " + listDir(repo));
+    }
+
     private static boolean isWindows() {
         return File.pathSeparatorChar == ';';
     }

src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java

@@ -40,6 +40,7 @@
 import org.junit.jupiter.params.ParameterizedClass;
 import org.junit.jupiter.params.provider.Arguments;
 import org.junit.jupiter.params.provider.MethodSource;
+import org.jvnet.hudson.test.Issue;
 
 @ParameterizedClass(name = "{0}")
 @MethodSource("gitObjects")
@@ -427,12 +428,42 @@ void test_clone_huge_timeout_logging() throws Exception {
         assertTimeout(testGitClient, "fetch", expectedValue);
     }
 
+    @Test
+    @Issue("JENKINS-70094")
+    void test_clone_filter() throws Exception {
+        WorkspaceWithRepo anotherWorkspace = new WorkspaceWithRepo(secondRepo.getRoot(), "git", listener);
+        GitClient anotherGitClient = anotherWorkspace.getGitClient();
+        CloneCommand cmd = anotherGitClient
+                .clone_()
+                .url(anotherWorkspace.localMirror())
+                .repositoryName("origin")
+                .filter("blob:none");
+        cmd.execute();
+        anotherGitClient.checkout().ref("origin/master").branch("master").execute();
+        check_remote_url(anotherWorkspace, anotherGitClient, "origin");
+        assertBranchesExist(anotherGitClient.getBranches(), "master");
+        if (gitImplName.equals("git")) {
+            assertThat("hasPromisor?", anotherWorkspace.cgit().hasPromisor("origin"), is(true));
+            String filterSpec = anotherWorkspace
+                    .launchCommand("git", "config", "remote." + "origin" + ".partialclonefilter")
+                    .trim();
+            assertThat("filterSpec", filterSpec, is("blob:none"));
+            assertPromisorFilesExist(anotherWorkspace.getGitFileDir());
+        }
+    }
+
     private void assertAlternatesFileExists() {
         final String alternates =
                 ".git" + File.separator + "objects" + File.separator + "info" + File.separator + "alternates";
         assertThat(new File(testGitDir, alternates), is(anExistingFile()));
     }
 
+    private void assertPromisorFilesExist(File anotherTestGitDir) {
+        File pack = new File(anotherTestGitDir, ".git" + File.separator + "objects" + File.separator + "pack");
+        File[] promisors = pack.listFiles((dir, name) -> name.endsWith(".promisor"));
+        assertThat(promisors, is(not(emptyArray())));
+    }
+
     private void assertAlternatesFileNotFound() {
         final String alternates =
                 ".git" + File.separator + "objects" + File.separator + "info" + File.separator + "alternates";

src/test/java/org/jenkinsci/plugins/gitclient/GitClientFetchTest.java

@@ -620,6 +620,63 @@ void test_fetch_default_timeout_logging() throws Exception {
         assertRevParseNotCalled(testGitClient, randomBranchName);
     }
 
+    @Test
+    @Issue("JENKINS-70094")
+    void test_fetch_filter() throws Exception {
+        testGitClient
+                .clone_()
+                .url(workspace.localMirror())
+                .repositoryName("origin")
+                .filter("blob:none")
+                .execute();
+        checkoutRandomBranch();
+        testGitClient
+                .fetch_()
+                .from(new URIish("origin"), null)
+                .filter("blob:limit=1k")
+                .execute();
+        boolean expectedPromisorValue = gitImplName.equals("git");
+        assertThat("hasPromisor?", workspace.cgit().hasPromisor("origin"), is(expectedPromisorValue));
+        if (gitImplName.equals("git")) {
+            String filterSpec = workspace
+                    .launchCommand("git", "config", "remote." + "origin" + ".partialclonefilter")
+                    .trim();
+            assertThat("filterSpec", filterSpec, is("blob:limit=1k"));
+        }
+    }
+
+    @Test
+    @Issue("JENKINS-70094")
+    void test_fetch_filter_changed() throws Exception {
+        testGitClient
+                .clone_()
+                .url(workspace.localMirror())
+                .repositoryName("origin")
+                .filter("blob:none")
+                .execute();
+        checkoutRandomBranch();
+        testGitClient.fetch_().from(new URIish("origin"), null).filter("tree:0").execute();
+        boolean expectedPromisorValue = gitImplName.equals("git");
+        assertThat("hasPromisor?", workspace.cgit().hasPromisor("origin"), is(expectedPromisorValue));
+        if (gitImplName.equals("git")) {
+            String filterSpec = workspace
+                    .launchCommand("git", "config", "remote." + "origin" + ".partialclonefilter")
+                    .trim();
+            assertThat("filterSpec", filterSpec, is("tree:0"));
+        }
+        testGitClient
+                .fetch_()
+                .from(new URIish("origin"), null)
+                .filter("blob:limit=1k")
+                .execute();
+        if (gitImplName.equals("git")) {
+            String filterSpec = workspace
+                    .launchCommand("git", "config", "remote." + "origin" + ".partialclonefilter")
+                    .trim();
+            assertThat("filterSpec", filterSpec, is("blob:limit=1k"));
+        }
+    }
+
     private void check_remote_url(WorkspaceWithRepo workspace, GitClient gitClient, final String repositoryName)
             throws Exception {
         assertThat("Wrong remote URL", gitClient.getRemoteUrl(repositoryName), is(workspace.localMirror()));