tests fixes

.vscode/settings.json

@@ -1,3 +1,5 @@
 {
-  "java.compile.nullAnalysis.mode": "disabled"
+  "java.compile.nullAnalysis.mode": "disabled",
+  "java.jdt.ls.vmargs": "-XX:+UseParallelGC -XX:GCTimeRatio=4 -XX:AdaptiveSizePolicyWeight=90 -Dsun.zip.disableMemoryMapping=true -Xmx4G -Xms100m -Xlog:disable",
+  "java.configuration.updateBuildConfiguration": "interactive"
 }

cdap-app-fabric-tests/pom.xml

@@ -29,6 +29,11 @@ the License.
   <name>CDAP App Fabric Tests</name>
 
   <dependencies>
+    <dependency>
+      <groupId>io.cdap.cdap</groupId>
+      <artifactId>cdap-common</artifactId>
+      <version>${project.version}</version>
+    </dependency>
     <dependency>
       <groupId>io.cdap.cdap</groupId>
       <artifactId>cdap-api</artifactId>

cdap-app-fabric/pom.xml

@@ -277,10 +277,10 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-jar-plugin</artifactId>
-        <version>2.4</version>
         <executions>
           <execution>
             <id>test-jar</id>
+            <phase>process-test-classes</phase>
             <goals>
               <goal>test-jar</goal>
             </goals>

cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/AbstractAppLifecycleHttpHandler.java

@@ -47,6 +47,8 @@
 import io.cdap.cdap.security.spi.authorization.UnauthorizedException;
 import io.cdap.http.BodyConsumer;
 import io.cdap.http.HttpResponder;
+import io.netty.handler.codec.http.DefaultHttpHeaders;
+import io.netty.handler.codec.http.HttpHeaders;
 import io.netty.handler.codec.http.HttpResponseStatus;
 import java.io.File;
 import java.io.FileReader;
@@ -172,7 +174,8 @@ protected ApplicationRecord getApplicationRecord(ApplicationWithPrograms deploye
 
   protected BodyConsumer deployAppFromArtifact(
       final ApplicationId appId,
-      final boolean skipMarkingLatest)
+      final boolean skipMarkingLatest,
+      final AppDeployStrategy appDeployStrategy)
       throws IOException {
     return new AbstractBodyConsumer(
         File.createTempFile("apprequest-" + appId, ".json", tmpDir)) {
@@ -183,8 +186,16 @@ protected void onFinish(HttpResponder responder, File uploadedFile) {
 
           try {
             ApplicationWithPrograms app = applicationLifecycleService.deployApp(appId, appRequest,
-                null, createProgramTerminator(), skipMarkingLatest);
-            responder.sendJson(HttpResponseStatus.OK, GSON.toJson(getApplicationRecord(app)));
+                null, createProgramTerminator(), skipMarkingLatest, appDeployStrategy);
+
+            if (app.isDeploySkipped()) {
+              LOG.debug("Application {} is already deployed", appId);
+            }
+
+            HttpHeaders headers = new DefaultHttpHeaders()
+                .add(Constants.Gateway.APP_DEPLOYMENT_SKIPPED_HEADER,
+                String.valueOf(app.isDeploySkipped()));
+            responder.sendString(HttpResponseStatus.OK, GSON.toJson(getApplicationRecord(app)), headers);
           } catch (DatasetManagementException e) {
             if (e.getCause() instanceof UnauthorizedException) {
               throw (UnauthorizedException) e.getCause();

cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/AppDeployStrategy.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright © 2026 CDAP Data, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package io.cdap.cdap.gateway.handlers;
+
+import java.util.Arrays;
+import java.util.stream.Collectors;
+
+/**
+ * Policy to control skipping of duplicate application deployments.
+ */
+public enum AppDeployStrategy {
+  SKIP_ON_NO_CHANGE,
+  ALWAYS_DEPLOY;
+
+  /**
+   * Returns a comma-separated string of all allowed policy values.
+   */
+  public static String getAllowedValues() {
+    return Arrays.stream(values())
+        .map(Enum::name)
+        .collect(Collectors.joining(", "));
+  }
+}

cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/AppLifecycleHttpHandler.java

@@ -156,17 +156,19 @@ public class AppLifecycleHttpHandler extends AbstractAppLifecycleHttpHandler {
   @AuditPolicy(AuditDetail.REQUEST_BODY)
   public BodyConsumer create(HttpRequest request, HttpResponder responder,
       @PathParam("namespace-id") final String namespaceId,
-      @PathParam("app-id") final String appId)
+      @PathParam("app-id") final String appId,
+      @QueryParam("deployStrategy") @DefaultValue("ALWAYS_DEPLOY") String deployStrategy)
       throws BadRequestException, NamespaceNotFoundException, AccessException {
     String versionId = ApplicationId.DEFAULT_VERSION;
     // If LCM flow is enabled - we generate specific versions of the app.
     if (Feature.LIFECYCLE_MANAGEMENT_EDIT.isEnabled(featureFlagsProvider)) {
       versionId = RunIds.generate().getId();
     }
     ApplicationId applicationId = validateApplicationVersionId(namespaceId, appId, versionId);
+    AppDeployStrategy strategy = parseDeployStrategy(deployStrategy);
 
     try {
-      return deployAppFromArtifact(applicationId);
+      return deployAppFromArtifact(applicationId, strategy);
     } catch (Exception ex) {
       responder.sendString(HttpResponseStatus.INTERNAL_SERVER_ERROR,
           "Deploy failed: " + ex.getMessage());
@@ -215,7 +217,8 @@ public BodyConsumer createAppVersion(HttpRequest request, HttpResponder responde
 
     // If LCM flow is enabled - Ignore the version provided by the user. Treating it the same as deploy without version
     if (Feature.LIFECYCLE_MANAGEMENT_EDIT.isEnabled(featureFlagsProvider)) {
-      return create(request, responder, namespaceId, appId);
+      return create(request, responder, namespaceId, appId,
+          String.valueOf(AppDeployStrategy.ALWAYS_DEPLOY));
     }
 
     ApplicationId applicationId = validateApplicationVersionId(namespaceId, appId, versionId);
@@ -755,6 +758,11 @@ private List<ApplicationId> decodeAndValidateBatchApplicationRecord(NamespaceId
   // the other behavior requires a BodyConsumer and only have one method per path is allowed,
   // so we have to use a BodyConsumer
   private BodyConsumer deployAppFromArtifact(final ApplicationId appId) throws IOException {
+    return deployAppFromArtifact(appId, AppDeployStrategy.ALWAYS_DEPLOY);
+  }
+
+  private BodyConsumer deployAppFromArtifact(final ApplicationId appId,
+      final AppDeployStrategy appDeployStrategy) throws IOException {
     // Perform auth checks outside BodyConsumer as only the first http request containing auth header
     // to populate SecurityRequestContext while http chunk doesn't. BodyConsumer runs in the thread
     // that processes the last http chunk.
@@ -763,7 +771,7 @@ private BodyConsumer deployAppFromArtifact(final ApplicationId appId) throws IOE
         appId.getParent(),
         applicationLifecycleService.decodeUserId(authenticationContext));
     // createTempFile() needs a prefix of at least 3 characters
-    return deployAppFromArtifact(appId, false);
+    return deployAppFromArtifact(appId, false, appDeployStrategy);
   }
 
   private BodyConsumer deployApplication(final HttpResponder responder,
@@ -878,4 +886,14 @@ private ApplicationId validateApplicationVersionId(@Nullable String namespace,
       throws BadRequestException, NamespaceNotFoundException, AccessException {
     return validateApplicationVersionId(validateNamespace(namespace), appId, versionId);
   }
+
+  private static AppDeployStrategy parseDeployStrategy(String strategy) throws BadRequestException {
+    try {
+      return AppDeployStrategy.valueOf(strategy.toUpperCase());
+    } catch (IllegalArgumentException e) {
+      throw new BadRequestException(String.format(
+          "Invalid value '%s' for query parameter 'deployStrategy'. Allowed values are: %s",
+          strategy, AppDeployStrategy.getAllowedValues()));
+    }
+  }
 }

cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/AppLifecycleHttpHandlerInternal.java

@@ -261,7 +261,7 @@ public BodyConsumer create(HttpRequest request, HttpResponder responder,
     }
     ApplicationId applicationId = validateApplicationVersionId(validateNamespace(namespaceId), appId, versionId);
 
-    return deployAppFromArtifact(applicationId, skipMarkingLatest);
+    return deployAppFromArtifact(applicationId, skipMarkingLatest, AppDeployStrategy.ALWAYS_DEPLOY);
   }
 
   /**

cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/pipeline/ApplicationWithPrograms.java

@@ -27,9 +27,10 @@
 public class ApplicationWithPrograms extends ApplicationDeployable {
 
   private final List<ProgramDescriptor> programDescriptors;
+  private final boolean deploySkipped;
 
   public ApplicationWithPrograms(ApplicationDeployable applicationDeployable,
-      Iterable<? extends ProgramDescriptor> programDescriptors) {
+      Iterable<? extends ProgramDescriptor> programDescriptors, boolean deploySkipped) {
     super(applicationDeployable.getArtifactId(), applicationDeployable.getArtifactLocation(),
         applicationDeployable.getApplicationId(), applicationDeployable.getSpecification(),
         applicationDeployable.getExistingAppSpec(),
@@ -40,6 +41,19 @@ public ApplicationWithPrograms(ApplicationDeployable applicationDeployable,
         applicationDeployable.getSourceControlMeta(), applicationDeployable.isUpgrade(),
         applicationDeployable.isSkipMarkingLatest());
     this.programDescriptors = ImmutableList.copyOf(programDescriptors);
+    this.deploySkipped = deploySkipped;
+  }
+
+  public ApplicationWithPrograms(ApplicationDeployable applicationDeployable,
+      Iterable<? extends ProgramDescriptor> programDescriptors) {
+    this(applicationDeployable, programDescriptors, false);
+  }
+
+  /**
+   * Returns true if the deployment was skipped because it was a duplicate request.
+   */
+  public boolean isDeploySkipped() {
+    return deploySkipped;
   }
 
   /**

cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/DefaultPreviewManager.java

@@ -191,7 +191,9 @@ public class DefaultPreviewManager extends AbstractIdleService implements Previe
   protected void startUp() throws Exception {
     previewInjector = createPreviewInjector();
     StoreDefinition.createAllTables(previewInjector.getInstance(StructuredTableAdmin.class));
-    metricsCollectionService.startAsync();
+    if (metricsCollectionService.state() == State.NEW) {
+      metricsCollectionService.startAsync();
+    }
     logAppender = previewInjector.getInstance(LogAppender.class);
     logAppender.start();
     LoggingContextAccessor.setLoggingContext(
@@ -202,7 +204,9 @@ protected void startUp() throws Exception {
     logSubscriberService.startAsync().awaitRunning();
     dataSubscriberService = previewInjector.getInstance(PreviewDataSubscriberService.class);
     dataSubscriberService.startAsync().awaitRunning();
-    previewDataCleanupService.startAsync().awaitRunning();
+    if (previewDataCleanupService.state() == State.NEW) {
+      previewDataCleanupService.startAsync().awaitRunning();
+    }
   }
 
   @Override

cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/artifact/ArtifactClassLoaderFactory.java

@@ -85,8 +85,7 @@ CloseableClassLoader createClassLoader(File unpackDir) {
             .createProgramClassLoader(cConf, ProgramType.SPARK);
       } catch (Exception e) {
         // If Spark is not supported, exception is expected. We'll use the default filter.
-        LOG.warn("Spark is not supported. Not using ProgramClassLoader from Spark");
-        LOG.trace("Failed to create spark program runner with error:", e);
+        LOG.warn("Spark is not supported. Not using ProgramClassLoader from Spark", e);
       }
     }
 

cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/artifact/ArtifactStore.java

@@ -59,6 +59,7 @@
 import io.cdap.cdap.spi.data.transaction.TransactionException;
 import io.cdap.cdap.spi.data.transaction.TransactionRunner;
 import io.cdap.cdap.spi.data.transaction.TransactionRunners;
+import io.cdap.cdap.spi.data.transaction.TxRunnable;
 import io.cdap.cdap.store.StoreDefinition;
 import java.io.File;
 import java.io.IOException;
@@ -693,7 +694,7 @@ public void updateArtifactProperties(Id.Artifact artifactId,
       Function<Map<String, String>, Map<String, String>> updateFunction)
       throws ArtifactNotFoundException, IOException {
 
-    TransactionRunners.run(transactionRunner, context -> {
+    TransactionRunners.run(transactionRunner, (TxRunnable) context -> {
       StructuredTable artifactDataTable = getTable(context,
           StoreDefinition.ArtifactStore.ARTIFACT_DATA_TABLE);
       ArtifactCell artifactCell = new ArtifactCell(artifactId);
@@ -741,7 +742,7 @@ public ArtifactDetail write(
 
     // if we're not a snapshot version, check that the artifact doesn't exist already.
     if (!artifactId.getVersion().isSnapshot()) {
-      TransactionRunners.run(transactionRunner, context -> {
+      TransactionRunners.run(transactionRunner, (TxRunnable) context -> {
         StructuredTable table = getTable(context,
             StoreDefinition.ArtifactStore.ARTIFACT_DATA_TABLE);
         ArtifactCell artifactCell = new ArtifactCell(artifactId);
@@ -765,7 +766,7 @@ public ArtifactDetail write(
 
     // now try and write the metadata for the artifact
     try {
-      transactionRunner.run(context -> {
+      transactionRunner.run((TxRunnable) context -> {
         // we have to check that the metadata doesn't exist again since somebody else may have written
         // the artifact while we were copying the artifact to the filesystem.
         StructuredTable artifactDataTable = getTable(context,
@@ -831,7 +832,7 @@ private Location copyFile(Id.Artifact artifactId, File artifactContent) throws I
   public void delete(final Id.Artifact artifactId) throws ArtifactNotFoundException, IOException {
 
     // delete everything in a transaction
-    TransactionRunners.run(transactionRunner, context -> {
+    TransactionRunners.run(transactionRunner, (TxRunnable) context -> {
       // first look up details to get plugins and apps in the artifact
       StructuredTable artifactDataTable = getTable(context,
           StoreDefinition.ArtifactStore.ARTIFACT_DATA_TABLE);
@@ -857,7 +858,7 @@ void clear(final NamespaceId namespace) throws IOException {
     final Id.Namespace namespaceId = Id.Namespace.fromEntityId(namespace);
     namespacePathLocator.get(namespace).append(ARTIFACTS_PATH).delete(true);
 
-    TransactionRunners.run(transactionRunner, context -> {
+    TransactionRunners.run(transactionRunner, (TxRunnable) context -> {
       // delete all rows about artifacts in the namespace
       StructuredTable artifactDataTable = getTable(context,
           StoreDefinition.ArtifactStore.ARTIFACT_DATA_TABLE);

cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/batch/MapReduceClassLoader.java

@@ -148,8 +148,10 @@ public MapReduceTaskContextProvider getTaskContextProvider() {
     synchronized (this) {
       taskContextProvider = Optional.ofNullable(taskContextProvider)
           .orElseGet(taskContextProviderSupplier::get);
+      if (taskContextProvider.state() == Service.State.NEW) {
+        taskContextProvider.startAsync().awaitRunning();
+      }
     }
-    taskContextProvider.startAsync().awaitRunning();
     return taskContextProvider;
   }
 

cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/batch/MapReduceRuntimeService.java

@@ -1013,7 +1013,7 @@ private Location createPluginArchive(Location targetDir) throws IOException {
   private Location copyFileToLocation(File file, Location targetDir) throws IOException {
     Location targetLocation = targetDir.append(file.getName()).getTempFile(".jar");
     try (InputStream in = new FileInputStream(file);
-         OutputStream out = Locations.newOutputSupplier(targetLocation).getOutput()) {
+         OutputStream out = targetLocation.getOutputStream()) {
       ByteStreams.copy(in, out);
     }
     return targetLocation;
@@ -1027,8 +1027,8 @@ private Location copyFileToLocation(File file, Location targetDir) throws IOExce
   private Location copyProgramJar(Location targetDir) throws IOException {
     Location programJarCopy = targetDir.append("program.jar");
 
-    try (InputStream in = Locations.newInputSupplier(programJarLocation).getInput();
-         OutputStream out = Locations.newOutputSupplier(programJarCopy).getOutput()) {
+    try (InputStream in = programJarLocation.getInputStream();
+         OutputStream out = programJarCopy.getOutputStream()) {
       ByteStreams.copy(in, out);
     }
     LOG.debug("Copied Program Jar to {}, source: {}", programJarCopy, programJarLocation);

cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/remote/RemoteExecutionJobMain.java

@@ -127,7 +127,7 @@ private void doMain(String[] args) throws Exception {
   @VisibleForTesting
   RemoteExecutionRuntimeJobEnvironment initialize(CConfiguration cConf) throws Exception {
     zkServer = InMemoryZKServer.builder().build();
-    zkServer.startAsync().awaitRunning();
+    zkServer.startAndWait();
 
     InetSocketAddress zkAddr = ResolvingDiscoverable.resolve(zkServer.getLocalAddress());
     String zkConnectStr = String.format("%s:%d", zkAddr.getHostString(), zkAddr.getPort());
@@ -214,7 +214,7 @@ void destroy() {
 
     if (zkServer != null) {
       try {
-        zkServer.stopAsync().awaitTerminated();
+        zkServer.stopAndWait();
       } catch (Exception e) {
         LOG.warn("Failed to stop ZK server", e);
       }

cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/remote/RemoteExecutionTwillController.java

@@ -66,6 +66,7 @@ class RemoteExecutionTwillController implements TwillController {
   private final RemoteProcessController remoteProcessController;
   private final RemoteExecutionService executionService;
   private final long pollCompletedMillis;
+  private final long stopDelayMillis;
   private final boolean terminateWithController;
   private volatile boolean terminateOnServiceStop;
 
@@ -77,6 +78,7 @@ class RemoteExecutionTwillController implements TwillController {
     this.programRunId = programRunId;
     this.runId = RunIds.fromString(programRunId.getRun());
     this.pollCompletedMillis = cConf.getLong(Constants.RuntimeMonitor.POLL_TIME_MS);
+    this.stopDelayMillis = cConf.getLong(Constants.RuntimeMonitor.REMOTE_STOP_DELAY_SECS, 10) * 1000;
 
     // On start up task succeeded, complete the started stage to unblock the onRunning()
     // On start up task failure, mark this controller as terminated with exception
@@ -121,15 +123,18 @@ public void complete() {
     try {
       RuntimeJobStatus status;
       RetryStrategy retryStrategy = RetryStrategies.timeLimit(
-          5, TimeUnit.SECONDS, RetryStrategies.exponentialDelay(500, 2000, TimeUnit.MILLISECONDS));
+          stopDelayMillis, TimeUnit.MILLISECONDS, RetryStrategies.exponentialDelay(500, 2000,
+              TimeUnit.MILLISECONDS));
 
-      // Make sure the remote execution is completed
-      // Give 5 seconds for the remote process to shutdown. After 5 seconds, issues a kill.
+      // Wait for the remote process (e.g. Dataproc job) to complete.
+      // We give 5 sec for the remote process (e.g. DP's master process) to shutdown and another
+      // 5 sec to account for DP's backend propagation delays (CDAP-21219). If we kill/cancel too
+      // early while the job is finishing, it can transition the Dataproc job to an ERROR state .
       long startTime = System.currentTimeMillis();
       while ((status = Retries.callWithRetries(
           remoteProcessController::getStatus, retryStrategy, Exception.class::isInstance))
           == RuntimeJobStatus.RUNNING) {
-        if (System.currentTimeMillis() - startTime >= 5000) {
+        if (System.currentTimeMillis() - startTime >= stopDelayMillis) {
           throw new IllegalStateException(
               "Remote process for " + programRunId + " is still running");
         }