Feature/pipelines

project-properties.gradle

@@ -16,7 +16,7 @@ project.ext {
     isDebugMode = System.getProperty("DEBUG", "false") == "true"
     releaseMode = project.hasProperty("releaseMode") ? project.releaseMode.toBoolean() : false
     scriptsUrl = commonScriptsUrl + (releaseMode ? '5.14.0' : 'develop')
-    migrationsUrl = migrationsScriptsUrl + (releaseMode ? 'rc/6.0.0' : 'develop')
+    migrationsUrl = migrationsScriptsUrl + (releaseMode ? 'rc/6.0.0' : 'feature/pipelines')
     manifestSchemaUrl = schemaUrl + ('manifest.schema.json')
     //TODO refactor with archive download
     testScriptsSrc = [
@@ -115,7 +115,9 @@ project.ext {
             (migrationsUrl + '/migrations/225_extract_launch_attribute.up.sql')                         : 'V225__extract_launch_attribute.up.sql',
             (migrationsUrl + '/migrations/226_extract_launch_statistics.up.sql')                        : 'V226__extract_launch_statistics.up.sql',
             (migrationsUrl + '/migrations/227_update_merge_launch_after_launch_tables_split.up.sql')    : 'V227__update_merge_launch_after_launch_tables_split.up.sql',
-            (migrationsUrl + '/migrations/228_manual_launch_perf_indexes.up.sql')                       : 'V228__manual_launch_perf_indexes.up.sql'
+            (migrationsUrl + '/migrations/228_add_automation_integration_group.up.sql')                 : 'V228__add_automation_integration_group.up.sql',
+            (migrationsUrl + '/migrations/229_manual_launch_perf_indexes.up.sql')                       : 'V229__manual_launch_perf_indexes.up.sql',
+            (migrationsUrl + '/migrations/230_pipeline_and_tms_quality.up.sql')                         : 'V230__pipeline_and_tms_quality.up.sql'
     ]
     excludeTests = ['**/entity/**',
                     '**/aop/**',

src/main/java/com/epam/reportportal/base/core/aifactory/connector/AbstractCiTriggerConnector.java

@@ -0,0 +1,49 @@
+package com.epam.reportportal.base.core.aifactory.connector;
+
+import com.epam.reportportal.base.core.aifactory.model.PipelineExecution;
+import com.epam.reportportal.base.infrastructure.persistence.entity.aifactory.PipelineStage;
+import com.epam.reportportal.base.infrastructure.persistence.entity.integration.Integration;
+import com.epam.reportportal.base.infrastructure.rules.exception.ErrorType;
+import com.epam.reportportal.base.infrastructure.rules.exception.ReportPortalException;
+
+/**
+ * Shared integration-parameter lookup for {@link CiTriggerConnector} implementations.
+ */
+abstract class AbstractCiTriggerConnector implements CiTriggerConnector {
+
+  protected String requiredParam(Integration integration, String paramName) {
+    var value = stringParam(integration, paramName);
+    if (value == null || value.isBlank()) {
+      throw new ReportPortalException(ErrorType.BAD_REQUEST_ERROR, "Missing required integration parameter: " + paramName);
+    }
+    return value;
+  }
+
+  protected String stringParam(Integration integration, String paramName) {
+    if (integration == null || integration.getParams() == null || integration.getParams().getParams() == null) {
+      return null;
+    }
+    var value = integration.getParams().getParams().get(paramName);
+    return value instanceof String stringValue ? stringValue : null;
+  }
+
+  /**
+   * What the stage's execution reported for the given parameter, or null.
+   */
+  protected String retryParameter(PipelineStage stage, String name) {
+    var execution = execution(stage);
+    return execution == null || execution.getRetryParameters() == null ? null : execution.getRetryParameters().get(name);
+  }
+
+  /**
+   * The identifier of the stage's run in the system that ran it, or null.
+   */
+  protected String externalRunId(PipelineStage stage) {
+    var execution = execution(stage);
+    return execution == null ? null : execution.getExternalRunId();
+  }
+
+  private PipelineExecution execution(PipelineStage stage) {
+    return stage.getExecution() == null ? null : stage.getExecution().getContent();
+  }
+}

src/main/java/com/epam/reportportal/base/core/aifactory/connector/CiHttpClient.java

@@ -0,0 +1,103 @@
+package com.epam.reportportal.base.core.aifactory.connector;
+
+import com.epam.reportportal.base.infrastructure.rules.exception.ErrorType;
+import com.epam.reportportal.base.infrastructure.rules.exception.ReportPortalException;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Map;
+import org.apache.hc.client5.http.classic.methods.HttpPost;
+import org.apache.hc.client5.http.config.RequestConfig;
+import org.apache.hc.client5.http.impl.DefaultHttpRequestRetryStrategy;
+import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
+import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse;
+import org.apache.hc.client5.http.impl.classic.HttpClients;
+import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder;
+import org.apache.hc.core5.http.ContentType;
+import org.apache.hc.core5.http.ParseException;
+import org.apache.hc.core5.http.io.entity.EntityUtils;
+import org.apache.hc.core5.http.io.entity.StringEntity;
+import org.apache.hc.core5.util.TimeValue;
+import org.apache.hc.core5.util.Timeout;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.DisposableBean;
+import org.springframework.stereotype.Component;
+
+/**
+ * Minimal HTTPS-only POST client shared by the CI trigger connectors.
+ */
+@Component
+public class CiHttpClient implements DisposableBean {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(CiHttpClient.class);
+
+  private final CloseableHttpClient httpClient;
+
+  public CiHttpClient() {
+    var requestConfig = RequestConfig.custom()
+        .setConnectTimeout(Timeout.ofSeconds(10))
+        .setResponseTimeout(Timeout.ofSeconds(30))
+        .build();
+
+    var connectionManager = PoolingHttpClientConnectionManagerBuilder.create()
+        .setMaxConnTotal(50)
+        .setMaxConnPerRoute(10)
+        .build();
+
+    this.httpClient = HttpClients.custom()
+        .setConnectionManager(connectionManager)
+        .setDefaultRequestConfig(requestConfig)
+        .setRetryStrategy(new DefaultHttpRequestRetryStrategy(2, TimeValue.ofSeconds(1L)))
+        .disableRedirectHandling()
+        .build();
+  }
+
+  /**
+   * Executes an HTTPS POST with a JSON body and returns the response body
+   * (empty string if there is none), so callers that need it (e.g. to read a
+   * created resource's URL out of the response) don't have to re-request it.
+   *
+   * @param url absolute HTTPS URL
+   * @param headers request headers (e.g. Authorization)
+   * @param jsonBody request body, already serialized as JSON
+   */
+  public String postForStatus(String url, Map<String, String> headers, String jsonBody) {
+    validateHttpsUrl(url);
+    var request = new HttpPost(url);
+    headers.forEach(request::setHeader);
+    request.setHeader("Accept", "application/json");
+    request.setEntity(new StringEntity(jsonBody, ContentType.APPLICATION_JSON));
+
+    try (CloseableHttpResponse response = httpClient.execute(request)) {
+      String body = response.getEntity() == null ? "" : EntityUtils.toString(response.getEntity());
+      if (response.getCode() >= 400) {
+        throw new ReportPortalException(ErrorType.BAD_REQUEST_ERROR,
+            "CI trigger request failed. HTTP code: " + response.getCode() + ". " + body);
+      }
+      return body;
+    } catch (IOException | ParseException e) {
+      throw new ReportPortalException(ErrorType.BAD_REQUEST_ERROR, "Failed to execute CI trigger request: " + e.getMessage());
+    }
+  }
+
+  private void validateHttpsUrl(String url) {
+    try {
+      URI uri = new URI(url);
+      if (!"https".equalsIgnoreCase(uri.getScheme()) || uri.getHost() == null) {
+        throw new URISyntaxException(url, "URL must use HTTPS and include a host");
+      }
+    } catch (URISyntaxException e) {
+      throw new ReportPortalException(ErrorType.BAD_REQUEST_ERROR, "CI trigger URL must be a valid HTTPS URL");
+    }
+  }
+
+  @Override
+  public void destroy() {
+    try {
+      httpClient.close();
+    } catch (IOException e) {
+      LOGGER.error("Failed to close CI HTTP client", e);
+    }
+  }
+}

src/main/java/com/epam/reportportal/base/core/aifactory/connector/CiTriggerConnector.java

@@ -0,0 +1,38 @@
+package com.epam.reportportal.base.core.aifactory.connector;
+
+import com.epam.reportportal.base.infrastructure.persistence.entity.aifactory.PipelineStage;
+import com.epam.reportportal.base.infrastructure.persistence.entity.integration.Integration;
+
+/**
+ * Runs a stage again in the CI system that ran it (the "Retry Upload" button).
+ * <p>
+ * Example: the Upload stage was reported with {@code executorType: "GITHUB_ACTIONS"} and retry parameters
+ * {@code {repo: "org/repo", workflowRef: "ai-pipeline.yml"}}. The connector whose
+ * {@link #getSupportedExecutorType()} is {@code GITHUB_ACTIONS} reads those parameters, calls GitHub with the
+ * project's GitHub Actions integration (token) and returns the URL of the new run.
+ * <p>
+ * This is the outbound direction ReportPortal needs for stage retry, the opposite of the inbound TMS sync
+ * connectors, which only ever pull data in. An execution of any other kind simply has no connector and is
+ * not retryable.
+ */
+public interface CiTriggerConnector {
+
+  /**
+   * The executor type this connector handles, e.g. {@code GITHUB_ACTIONS}.
+   */
+  String getSupportedExecutorType();
+
+  /**
+   * Name of the project integration type holding the credentials this connector needs.
+   */
+  String getIntegrationTypeName();
+
+  /**
+   * Triggers a rerun of the job the stage's execution describes.
+   *
+   * @param integration credentials/config for the system (token, base URL)
+   * @param stage the stage to retry; its execution identifies the job
+   * @return the URL of the newly triggered run
+   */
+  String triggerStageRerun(Integration integration, PipelineStage stage);
+}

src/main/java/com/epam/reportportal/base/core/aifactory/connector/GitHubActionsCiConnector.java

@@ -0,0 +1,81 @@
+package com.epam.reportportal.base.core.aifactory.connector;
+
+import com.epam.reportportal.base.infrastructure.persistence.entity.aifactory.PipelineStage;
+import com.epam.reportportal.base.infrastructure.persistence.entity.integration.Integration;
+import com.epam.reportportal.base.infrastructure.rules.exception.ErrorType;
+import com.epam.reportportal.base.infrastructure.rules.exception.ReportPortalException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+import lombok.RequiredArgsConstructor;
+import org.springframework.stereotype.Component;
+import org.springframework.web.util.UriUtils;
+
+/**
+ * Re-triggers a stage's agent job via the GitHub Actions
+ * <a href="https://docs.github.com/en/rest/actions/workflows#create-a-workflow-dispatch-event">
+ * workflow dispatch</a> API. The retry parameter {@code repo} is {@code "owner/repo"},
+ * {@code workflowRef} is the workflow file name (e.g. {@code ai-pipeline.yml}).
+ */
+@Component
+@RequiredArgsConstructor
+public class GitHubActionsCiConnector extends AbstractCiTriggerConnector {
+
+  private static final String TOKEN_PARAM = "token";
+  private static final String GIT_REF_PARAM = "ref";
+  private static final String REPO_PARAM = "repo";
+  private static final String DEFAULT_GIT_REF = "main";
+  private static final String WORKFLOW_PARAM = "workflowRef";
+
+  private final CiHttpClient ciHttpClient;
+  private final ObjectMapper objectMapper;
+
+  @Override
+  public String getSupportedExecutorType() {
+    return "GITHUB_ACTIONS";
+  }
+
+  @Override
+  public String getIntegrationTypeName() {
+    return "github-actions";
+  }
+
+  @Override
+  public String triggerStageRerun(Integration integration, PipelineStage stage) {
+    var repo = retryParameter(stage, REPO_PARAM);
+    var workflowRef = retryParameter(stage, WORKFLOW_PARAM);
+    if (repo == null || workflowRef == null) {
+      throw new ReportPortalException(ErrorType.BAD_REQUEST_ERROR, "Stage is missing its CI repo/workflow reference");
+    }
+    var token = requiredParam(integration, TOKEN_PARAM);
+    // The stage's repo is client-supplied at ingest time and must never steer which repo the
+    // integration's token is used against - it's only accepted here once it's confirmed to be
+    // the exact repo this integration was configured for.
+    var configuredRepo = requiredParam(integration, REPO_PARAM);
+    if (!configuredRepo.equals(repo)) {
+      throw new ReportPortalException(ErrorType.BAD_REQUEST_ERROR,
+          "Stage CI repo '" + repo + "' does not match the repo configured for this project's GitHub Actions integration");
+    }
+    var gitRef = stringParam(integration, GIT_REF_PARAM);
+
+    var workflowFile = UriUtils.encodePathSegment(workflowRef, StandardCharsets.UTF_8);
+    var url = "https://api.github.com/repos/" + repo + "/actions/workflows/" + workflowFile + "/dispatches";
+
+    ObjectNode body = objectMapper.createObjectNode();
+    body.put("ref", gitRef == null || gitRef.isBlank() ? DEFAULT_GIT_REF : gitRef);
+    ObjectNode inputs = body.putObject("inputs");
+    inputs.put("pipeline_stage_id", String.valueOf(stage.getId()));
+    if (externalRunId(stage) != null) {
+      inputs.put("rerun_job_id", externalRunId(stage));
+    }
+
+    ciHttpClient.postForStatus(url, Map.of(
+        "Authorization", "Bearer " + token,
+        "Accept", "application/vnd.github+json",
+        "X-GitHub-Api-Version", "2022-11-28"
+    ), body.toString());
+
+    return "https://github.com/" + repo + "/actions/workflows/" + workflowFile;
+  }
+}

src/main/java/com/epam/reportportal/base/core/aifactory/connector/GitLabCiConnector.java

@@ -0,0 +1,95 @@
+package com.epam.reportportal.base.core.aifactory.connector;
+
+import com.epam.reportportal.base.infrastructure.persistence.entity.aifactory.PipelineStage;
+import com.epam.reportportal.base.infrastructure.persistence.entity.integration.Integration;
+import com.epam.reportportal.base.infrastructure.rules.exception.ErrorType;
+import com.epam.reportportal.base.infrastructure.rules.exception.ReportPortalException;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import lombok.RequiredArgsConstructor;
+import org.springframework.stereotype.Component;
+import org.springframework.web.util.UriUtils;
+
+/**
+ * Re-triggers a stage's agent job via the GitLab
+ * <a href="https://docs.gitlab.com/ee/ci/triggers/">pipeline trigger</a> API.
+ * the retry parameter {@code repo} holds the GitLab project id, {@code workflowRef}
+ * the git ref (branch/tag) the pipeline should run on.
+ */
+@Component
+@RequiredArgsConstructor
+public class GitLabCiConnector extends AbstractCiTriggerConnector {
+
+  private static final String BASE_URL_PARAM = "url";
+  private static final String TOKEN_PARAM = "triggerToken";
+  private static final String PROJECT_ID_PARAM = "projectId";
+  private static final String DEFAULT_BASE_URL = "https://gitlab.com";
+  private static final String REPO_PARAM = "repo";
+  private static final String REF_PARAM = "workflowRef";
+
+  private final CiHttpClient ciHttpClient;
+  private final ObjectMapper objectMapper;
+
+  @Override
+  public String getSupportedExecutorType() {
+    return "GITLAB_CI";
+  }
+
+  @Override
+  public String getIntegrationTypeName() {
+    return "gitlab-ci";
+  }
+
+  @Override
+  public String triggerStageRerun(Integration integration, PipelineStage stage) {
+    var repo = retryParameter(stage, REPO_PARAM);
+    var ref = retryParameter(stage, REF_PARAM);
+    if (repo == null || ref == null) {
+      throw new ReportPortalException(ErrorType.BAD_REQUEST_ERROR, "Stage is missing its CI project/ref reference");
+    }
+    var token = requiredParam(integration, TOKEN_PARAM);
+    // The stage's project id is client-supplied at ingest time and must never steer which
+    // GitLab project the integration's trigger token is used against - it's only accepted here
+    // once it's confirmed to be the exact project this integration was configured for.
+    var configuredProjectId = requiredParam(integration, PROJECT_ID_PARAM);
+    if (!configuredProjectId.equals(repo)) {
+      throw new ReportPortalException(ErrorType.BAD_REQUEST_ERROR,
+          "Stage CI project '" + repo + "' does not match the project configured for this project's GitLab CI integration");
+    }
+    var baseUrl = stringParam(integration, BASE_URL_PARAM);
+    var effectiveBaseUrl = baseUrl == null || baseUrl.isBlank() ? DEFAULT_BASE_URL : baseUrl;
+
+    var projectId = UriUtils.encodePathSegment(repo, StandardCharsets.UTF_8);
+    var url = effectiveBaseUrl + "/api/v4/projects/" + projectId + "/trigger/pipeline";
+
+    ObjectNode body = objectMapper.createObjectNode();
+    body.put("token", token);
+    body.put("ref", ref);
+    ObjectNode variables = body.putObject("variables");
+    variables.put("PIPELINE_STAGE_ID", String.valueOf(stage.getId()));
+    if (externalRunId(stage) != null) {
+      variables.put("RERUN_JOB_ID", externalRunId(stage));
+    }
+
+    var responseBody = ciHttpClient.postForStatus(url, Collections.emptyMap(), body.toString());
+    var fallbackUrl = effectiveBaseUrl + "/api/v4/projects/" + projectId + "/pipelines";
+    return extractPipelineUrl(responseBody, fallbackUrl);
+  }
+
+  /**
+   * GitLab's trigger-pipeline response body carries the created pipeline's own
+   * {@code web_url} — prefer that (a page a human can open) over the raw REST
+   * endpoint this connector called.
+   */
+  private String extractPipelineUrl(String responseBody, String fallbackUrl) {
+    try {
+      var webUrl = objectMapper.readTree(responseBody).path("web_url").asText(null);
+      return webUrl != null && !webUrl.isBlank() ? webUrl : fallbackUrl;
+    } catch (JsonProcessingException | IllegalArgumentException e) {
+      return fallbackUrl;
+    }
+  }
+}