Fix GitLab API pagination

modules/nextflow/src/main/groovy/nextflow/scm/GitlabRepositoryProvider.groovy

@@ -21,6 +21,7 @@ import groovy.util.logging.Slf4j
 import org.eclipse.jgit.transport.CredentialsProvider
 import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider
 
+import java.net.http.HttpResponse
 import java.nio.charset.StandardCharsets
 
 import static nextflow.Const.DEFAULT_BRANCH
@@ -36,6 +37,14 @@ import static nextflow.Const.DEFAULT_BRANCH
 @Slf4j
 class GitlabRepositoryProvider extends RepositoryProvider {
 
+    /**
+     * Max number of items that can be requested in a single page by the GitLab API.
+     * The API defaults to 20 items per page when not specified.
+     *
+     * See https://docs.gitlab.com/ee/api/rest/#offset-based-pagination
+     */
+    private static final int MAX_PER_PAGE = 100
+
     GitlabRepositoryProvider(String project, ProviderConfig config=null) {
         this.project = project
         this.config = config ?: new ProviderConfig('gitlab')
@@ -90,17 +99,76 @@ class GitlabRepositoryProvider extends RepositoryProvider {
     @Override
     List<BranchInfo> getBranches() {
         // https://docs.gitlab.com/ee/api/branches.html
-        final url = "${config.endpoint}/api/v4/projects/${getProjectName()}/repository/branches"
+        final url = "${config.endpoint}/api/v4/projects/${getProjectName()}/repository/branches?per_page=${MAX_PER_PAGE}"
         this.<BranchInfo>invokeAndResponseWithPaging(url, { Map branch -> new BranchInfo(branch.name as String, branch.commit?.id as String) })
     }
 
     @Override
     List<TagInfo> getTags() {
         // https://docs.gitlab.com/ee/api/tags.html
-        final url = "${config.endpoint}/api/v4/projects/${getProjectName()}/repository/tags"
+        final url = "${config.endpoint}/api/v4/projects/${getProjectName()}/repository/tags?per_page=${MAX_PER_PAGE}"
         this.<TagInfo>invokeAndResponseWithPaging(url, { Map tag -> new TagInfo(tag.name as String, tag.commit?.id as String) })
     }
 
+    @Override
+    protected <T> List<T> invokeAndResponseWithPaging(String request, Closure<T> parse) {
+        final result = new ArrayList<T>()
+        final visited = new HashSet<String>()
+        String url = request
+
+        while( url ) {
+            if( !visited.add(url) )
+                throw new IOException("Invalid GitLab pagination link cycle detected: $url")
+
+            final response = invokeResponse(url)
+            final body = new String(response.body(), StandardCharsets.UTF_8)
+            final items = (List) new JsonSlurper().parseText(body)
+            for( def item : items )
+                result.add(parse(item))
+
+            url = getNextPageUrl(response)
+        }
+
+        return result
+    }
+
+    private String getNextPageUrl(HttpResponse<byte[]> response) {
+        for( String value : response.headers().allValues('Link') ) {
+            final links = value =~ /<([^>]+)>([^,]*)/
+            while( links.find() ) {
+                final target = links.group(1)
+                final params = links.group(2)
+                final relation = params =~ /(?i)(?:^|;)\s*rel\s*=\s*"?([^";,]+)"?/
+                if( relation.find() && relation.group(1).tokenize().contains('next') )
+                    return validateNextPageUrl(response.uri(), target)
+            }
+        }
+        return null
+    }
+
+    private static String validateNextPageUrl(URI current, String target) {
+        final next = current.resolve(target)
+        if( next.userInfo || !sameOrigin(current, next) )
+            throw new IOException("Invalid GitLab pagination URL: $next")
+        return next.toString()
+    }
+
+    private static boolean sameOrigin(URI left, URI right) {
+        left.scheme?.equalsIgnoreCase(right.scheme) &&
+            left.host?.equalsIgnoreCase(right.host) &&
+            effectivePort(left) == effectivePort(right)
+    }
+
+    private static int effectivePort(URI uri) {
+        if( uri.port != -1 )
+            return uri.port
+        if( uri.scheme?.equalsIgnoreCase('https') )
+            return 443
+        if( uri.scheme?.equalsIgnoreCase('http') )
+            return 80
+        return -1
+    }
+
     /** {@inheritDoc} */
     @Override
     String getContentUrl( String path ) {

modules/nextflow/src/main/groovy/nextflow/scm/RepositoryProvider.groovy

@@ -247,6 +247,16 @@ abstract class RepositoryProvider {
      * @return The remote service response as byte array
      */
     protected byte[] invokeBytes( String api ) {
+        return invokeResponse(api).body()
+    }
+
+    /**
+     * Invoke the API request specified and return the complete HTTP response.
+     *
+     * @param api A API request url e.g. https://api.github.com/repos/nextflow-io/hello
+     * @return The remote service response
+     */
+    protected HttpResponse<byte[]> invokeResponse( String api ) {
         assert api
         log.debug "Request [credentials ${getAuthObfuscated() ?: '-'}] -> $api"
         final request = newRequest(api)
@@ -255,8 +265,7 @@ abstract class RepositoryProvider {
         // check the response code
         checkResponse(resp)
         checkMaxLength(resp)
-        // return the body as byte array
-        return resp.body()
+        return resp
     }
 
     protected String getAuthObfuscated() {

modules/nextflow/src/test/groovy/nextflow/scm/GitlabRepositoryProviderTest.groovy

@@ -16,6 +16,12 @@
 
 package nextflow.scm
 
+import java.net.http.HttpClient
+import java.net.http.HttpHeaders
+import java.net.http.HttpRequest
+import java.net.http.HttpResponse
+import javax.net.ssl.SSLSession
+
 import spock.lang.IgnoreIf
 import spock.lang.Requires
 import spock.lang.Specification
@@ -253,4 +259,116 @@ class GitlabRepositoryProviderTest extends Specification {
         entries.any { it.name == 'test-asset.bin' && it.path.contains('/test/') }
         entries.every { it.path && it.sha }
     }
+
+    def 'should follow GitLab pagination links when listing branches' () {
+        given:
+        def provider = Spy(GitlabRepositoryProvider, constructorArgs: ['pditommaso/hello', new ProviderConfig('gitlab')])
+        and:
+        def first = 'https://gitlab.com/api/v4/projects/pditommaso%2Fhello/repository/branches?per_page=100'
+        def second = 'https://gitlab.com/api/v4/projects/pditommaso%2Fhello/repository/branches?per_page=100&page=2'
+        def last = 'https://gitlab.com/api/v4/projects/pditommaso%2Fhello/repository/branches?per_page=100&page=3'
+
+        when:
+        def branches = provider.getBranches()
+
+        then:
+        1 * provider.invokeResponse(first) >> response(
+            first,
+            '[{"name":"main","commit":{"id":"aaa"}}]',
+            "<${last}>; rel=\"last\", <${second}>; type=\"application/json\"; rel=\"prev next\""
+        )
+        1 * provider.invokeResponse(second) >> response(
+            second,
+            '[{"name":"develop","commit":{"id":"bbb"}}]'
+        )
+        and:
+        branches.name == ['main', 'develop']
+    }
+
+    def 'should request the max page size when listing tags' () {
+        given:
+        def provider = Spy(GitlabRepositoryProvider, constructorArgs: ['pditommaso/hello', new ProviderConfig('gitlab')])
+        and:
+        def url = 'https://gitlab.com/api/v4/projects/pditommaso%2Fhello/repository/tags?per_page=100'
+
+        when:
+        def tags = provider.getTags()
+
+        then:
+        1 * provider.invokeResponse(url) >> response(
+            url,
+            '[{"name":"v1.0","commit":{"id":"aaa"}}]'
+        )
+        and:
+        tags.name == ['v1.0']
+    }
+
+    def 'should reject a cross-origin GitLab pagination link' () {
+        given:
+        def provider = Spy(GitlabRepositoryProvider, constructorArgs: ['pditommaso/hello', new ProviderConfig('gitlab')])
+        and:
+        def url = 'https://gitlab.com/api/v4/projects/pditommaso%2Fhello/repository/branches?per_page=100'
+
+        when:
+        provider.getBranches()
+
+        then:
+        1 * provider.invokeResponse(url) >> response(
+            url,
+            '[{"name":"main","commit":{"id":"aaa"}}]',
+            '<https://example.com/api/v4/projects/1/repository/branches?page=2>; rel="next"'
+        )
+        def error = thrown(IOException)
+        error.message == 'Invalid GitLab pagination URL: https://example.com/api/v4/projects/1/repository/branches?page=2'
+    }
+
+    def 'should reject a GitLab pagination link cycle' () {
+        given:
+        def provider = Spy(GitlabRepositoryProvider, constructorArgs: ['pditommaso/hello', new ProviderConfig('gitlab')])
+        and:
+        def url = 'https://gitlab.com/api/v4/projects/pditommaso%2Fhello/repository/branches?per_page=100'
+
+        when:
+        provider.getBranches()
+
+        then:
+        1 * provider.invokeResponse(url) >> response(
+            url,
+            '[{"name":"main","commit":{"id":"aaa"}}]',
+            "<${url}>; rel=\"next\""
+        )
+        def error = thrown(IOException)
+        error.message == "Invalid GitLab pagination link cycle detected: $url"
+    }
+
+    private static HttpResponse<byte[]> response(String url, String responseBody, String link=null) {
+        return new HttpResponse<byte[]>() {
+            @Override
+            int statusCode() { 200 }
+
+            @Override
+            HttpRequest request() { null }
+
+            @Override
+            Optional<HttpResponse<byte[]>> previousResponse() { Optional.empty() }
+
+            @Override
+            HttpHeaders headers() {
+                final values = link ? ['Link': [link]] : [:]
+                HttpHeaders.of(values, (a, b) -> true)
+            }
+
+            @Override
+            byte[] body() { responseBody.bytes }
+
+            @Override
+            Optional<SSLSession> sslSession() { Optional.empty() }
+
+            @Override
+            URI uri() { new URI(url) }
+
+            @Override
+            HttpClient.Version version() { HttpClient.Version.HTTP_1_1 }
+        }
+    }
 }