diff --git a/modules/nextflow/src/main/groovy/nextflow/scm/GitlabRepositoryProvider.groovy b/modules/nextflow/src/main/groovy/nextflow/scm/GitlabRepositoryProvider.groovy index 624c93bff1..0aebbab6e0 100644 --- a/modules/nextflow/src/main/groovy/nextflow/scm/GitlabRepositoryProvider.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/scm/GitlabRepositoryProvider.groovy @@ -21,6 +21,7 @@ import groovy.util.logging.Slf4j import org.eclipse.jgit.transport.CredentialsProvider import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider +import java.net.http.HttpResponse import java.nio.charset.StandardCharsets import static nextflow.Const.DEFAULT_BRANCH @@ -36,6 +37,14 @@ import static nextflow.Const.DEFAULT_BRANCH @Slf4j class GitlabRepositoryProvider extends RepositoryProvider { + /** + * Max number of items that can be requested in a single page by the GitLab API. + * The API defaults to 20 items per page when not specified. + * + * See https://docs.gitlab.com/ee/api/rest/#offset-based-pagination + */ + private static final int MAX_PER_PAGE = 100 + GitlabRepositoryProvider(String project, ProviderConfig config=null) { this.project = project this.config = config ?: new ProviderConfig('gitlab') @@ -90,17 +99,76 @@ class GitlabRepositoryProvider extends RepositoryProvider { @Override List getBranches() { // https://docs.gitlab.com/ee/api/branches.html - final url = "${config.endpoint}/api/v4/projects/${getProjectName()}/repository/branches" + final url = "${config.endpoint}/api/v4/projects/${getProjectName()}/repository/branches?per_page=${MAX_PER_PAGE}" this.invokeAndResponseWithPaging(url, { Map branch -> new BranchInfo(branch.name as String, branch.commit?.id as String) }) } @Override List getTags() { // https://docs.gitlab.com/ee/api/tags.html - final url = "${config.endpoint}/api/v4/projects/${getProjectName()}/repository/tags" + final url = "${config.endpoint}/api/v4/projects/${getProjectName()}/repository/tags?per_page=${MAX_PER_PAGE}" this.invokeAndResponseWithPaging(url, { Map tag -> new TagInfo(tag.name as String, tag.commit?.id as String) }) } + @Override + protected List invokeAndResponseWithPaging(String request, Closure parse) { + final result = new ArrayList() + final visited = new HashSet() + String url = request + + while( url ) { + if( !visited.add(url) ) + throw new IOException("Invalid GitLab pagination link cycle detected: $url") + + final response = invokeResponse(url) + final body = new String(response.body(), StandardCharsets.UTF_8) + final items = (List) new JsonSlurper().parseText(body) + for( def item : items ) + result.add(parse(item)) + + url = getNextPageUrl(response) + } + + return result + } + + private String getNextPageUrl(HttpResponse response) { + for( String value : response.headers().allValues('Link') ) { + final links = value =~ /<([^>]+)>([^,]*)/ + while( links.find() ) { + final target = links.group(1) + final params = links.group(2) + final relation = params =~ /(?i)(?:^|;)\s*rel\s*=\s*"?([^";,]+)"?/ + if( relation.find() && relation.group(1).tokenize().contains('next') ) + return validateNextPageUrl(response.uri(), target) + } + } + return null + } + + private static String validateNextPageUrl(URI current, String target) { + final next = current.resolve(target) + if( next.userInfo || !sameOrigin(current, next) ) + throw new IOException("Invalid GitLab pagination URL: $next") + return next.toString() + } + + private static boolean sameOrigin(URI left, URI right) { + left.scheme?.equalsIgnoreCase(right.scheme) && + left.host?.equalsIgnoreCase(right.host) && + effectivePort(left) == effectivePort(right) + } + + private static int effectivePort(URI uri) { + if( uri.port != -1 ) + return uri.port + if( uri.scheme?.equalsIgnoreCase('https') ) + return 443 + if( uri.scheme?.equalsIgnoreCase('http') ) + return 80 + return -1 + } + /** {@inheritDoc} */ @Override String getContentUrl( String path ) { diff --git a/modules/nextflow/src/main/groovy/nextflow/scm/RepositoryProvider.groovy b/modules/nextflow/src/main/groovy/nextflow/scm/RepositoryProvider.groovy index b442d24c0e..cb287ad8ed 100644 --- a/modules/nextflow/src/main/groovy/nextflow/scm/RepositoryProvider.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/scm/RepositoryProvider.groovy @@ -247,6 +247,16 @@ abstract class RepositoryProvider { * @return The remote service response as byte array */ protected byte[] invokeBytes( String api ) { + return invokeResponse(api).body() + } + + /** + * Invoke the API request specified and return the complete HTTP response. + * + * @param api A API request url e.g. https://api.github.com/repos/nextflow-io/hello + * @return The remote service response + */ + protected HttpResponse invokeResponse( String api ) { assert api log.debug "Request [credentials ${getAuthObfuscated() ?: '-'}] -> $api" final request = newRequest(api) @@ -255,8 +265,7 @@ abstract class RepositoryProvider { // check the response code checkResponse(resp) checkMaxLength(resp) - // return the body as byte array - return resp.body() + return resp } protected String getAuthObfuscated() { diff --git a/modules/nextflow/src/test/groovy/nextflow/scm/GitlabRepositoryProviderTest.groovy b/modules/nextflow/src/test/groovy/nextflow/scm/GitlabRepositoryProviderTest.groovy index 87ec4f9e51..a50f4e043b 100644 --- a/modules/nextflow/src/test/groovy/nextflow/scm/GitlabRepositoryProviderTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/scm/GitlabRepositoryProviderTest.groovy @@ -16,6 +16,12 @@ package nextflow.scm +import java.net.http.HttpClient +import java.net.http.HttpHeaders +import java.net.http.HttpRequest +import java.net.http.HttpResponse +import javax.net.ssl.SSLSession + import spock.lang.IgnoreIf import spock.lang.Requires import spock.lang.Specification @@ -253,4 +259,116 @@ class GitlabRepositoryProviderTest extends Specification { entries.any { it.name == 'test-asset.bin' && it.path.contains('/test/') } entries.every { it.path && it.sha } } + + def 'should follow GitLab pagination links when listing branches' () { + given: + def provider = Spy(GitlabRepositoryProvider, constructorArgs: ['pditommaso/hello', new ProviderConfig('gitlab')]) + and: + def first = 'https://gitlab.com/api/v4/projects/pditommaso%2Fhello/repository/branches?per_page=100' + def second = 'https://gitlab.com/api/v4/projects/pditommaso%2Fhello/repository/branches?per_page=100&page=2' + def last = 'https://gitlab.com/api/v4/projects/pditommaso%2Fhello/repository/branches?per_page=100&page=3' + + when: + def branches = provider.getBranches() + + then: + 1 * provider.invokeResponse(first) >> response( + first, + '[{"name":"main","commit":{"id":"aaa"}}]', + "<${last}>; rel=\"last\", <${second}>; type=\"application/json\"; rel=\"prev next\"" + ) + 1 * provider.invokeResponse(second) >> response( + second, + '[{"name":"develop","commit":{"id":"bbb"}}]' + ) + and: + branches.name == ['main', 'develop'] + } + + def 'should request the max page size when listing tags' () { + given: + def provider = Spy(GitlabRepositoryProvider, constructorArgs: ['pditommaso/hello', new ProviderConfig('gitlab')]) + and: + def url = 'https://gitlab.com/api/v4/projects/pditommaso%2Fhello/repository/tags?per_page=100' + + when: + def tags = provider.getTags() + + then: + 1 * provider.invokeResponse(url) >> response( + url, + '[{"name":"v1.0","commit":{"id":"aaa"}}]' + ) + and: + tags.name == ['v1.0'] + } + + def 'should reject a cross-origin GitLab pagination link' () { + given: + def provider = Spy(GitlabRepositoryProvider, constructorArgs: ['pditommaso/hello', new ProviderConfig('gitlab')]) + and: + def url = 'https://gitlab.com/api/v4/projects/pditommaso%2Fhello/repository/branches?per_page=100' + + when: + provider.getBranches() + + then: + 1 * provider.invokeResponse(url) >> response( + url, + '[{"name":"main","commit":{"id":"aaa"}}]', + '; rel="next"' + ) + def error = thrown(IOException) + error.message == 'Invalid GitLab pagination URL: https://example.com/api/v4/projects/1/repository/branches?page=2' + } + + def 'should reject a GitLab pagination link cycle' () { + given: + def provider = Spy(GitlabRepositoryProvider, constructorArgs: ['pditommaso/hello', new ProviderConfig('gitlab')]) + and: + def url = 'https://gitlab.com/api/v4/projects/pditommaso%2Fhello/repository/branches?per_page=100' + + when: + provider.getBranches() + + then: + 1 * provider.invokeResponse(url) >> response( + url, + '[{"name":"main","commit":{"id":"aaa"}}]', + "<${url}>; rel=\"next\"" + ) + def error = thrown(IOException) + error.message == "Invalid GitLab pagination link cycle detected: $url" + } + + private static HttpResponse response(String url, String responseBody, String link=null) { + return new HttpResponse() { + @Override + int statusCode() { 200 } + + @Override + HttpRequest request() { null } + + @Override + Optional> previousResponse() { Optional.empty() } + + @Override + HttpHeaders headers() { + final values = link ? ['Link': [link]] : [:] + HttpHeaders.of(values, (a, b) -> true) + } + + @Override + byte[] body() { responseBody.bytes } + + @Override + Optional sslSession() { Optional.empty() } + + @Override + URI uri() { new URI(url) } + + @Override + HttpClient.Version version() { HttpClient.Version.HTTP_1_1 } + } + } }