diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNotFoundTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNotFoundTests.java index 712839b80d95..5723c52b8a24 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNotFoundTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNotFoundTests.java @@ -229,7 +229,7 @@ public void performDocumentOperationOnNonExistentContainerGatewayModeV2(Operatio } } - @Test(groups = {"thinclient"}, timeOut = TIMEOUT) + @Test(groups = {"thinclient"}, timeOut = 2 * TIMEOUT, retryAnalyzer = FlakyTestRetryAnalyzer.class) public void performBulkOnNonExistentContainerGatewayModeV2() { logger.info("Running test: Read item from non-existent container in Gateway Connection Mode"); @@ -306,7 +306,7 @@ public void performDocumentOperationOnDeletedContainer(OperationType operationTy CosmosAsyncContainer testContainer = clientToUse.getDatabase(testAsyncDatabase.getId()).getContainer(testContainerId); - Thread.sleep(5000); + Thread.sleep(15000); // Create a different client instance to delete the container deletingAsyncClient = getClientBuilder() @@ -437,7 +437,8 @@ public void performBulkOnDeletedContainer() throws InterruptedException { } } - @Test(groups = {"thinclient"}, dataProvider = "operationTypeProvider", timeOut = TIMEOUT) + @Test(groups = {"thinclient"}, dataProvider = "operationTypeProvider", + timeOut = 2 * TIMEOUT, retryAnalyzer = FlakyTestRetryAnalyzer.class) public void performDocumentOperationOnDeletedContainerWithGatewayV2(OperationType operationType) throws InterruptedException { logger.info("Running test: Read item from deleted container - Gateway V2 Connection Mode"); diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosTracerTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosTracerTest.java index f8e2979f544d..5e022291aae5 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosTracerTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosTracerTest.java @@ -203,8 +203,10 @@ public void cosmosAsyncClient( createAndInitializeDiagnosticsProvider( mockTracer, useLegacyTracing, enableRequestLevelTracing, ShowQueryMode.NONE, forceThresholdViolations, samplingRate); - CosmosDatabaseResponse cosmosDatabaseResponse = client.createDatabaseIfNotExists(cosmosAsyncDatabase.getId(), - ThroughputProperties.createManualThroughput(5000)).block(); + CosmosDatabaseResponse cosmosDatabaseResponse = executeControlPlaneWithRetry( + () -> client.createDatabaseIfNotExists(cosmosAsyncDatabase.getId(), + ThroughputProperties.createManualThroughput(5000)).block(), + mockTracer::reset); assertThat(cosmosDatabaseResponse).isNotNull(); verifyTracerAttributes( mockTracer, diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/FITests_readAllAfterCreation.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/FITests_readAllAfterCreation.java index aa37a47ae29e..1d493ce80455 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/FITests_readAllAfterCreation.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/FITests_readAllAfterCreation.java @@ -12,7 +12,8 @@ import java.util.function.Function; public class FITests_readAllAfterCreation extends FaultInjectionWithAvailabilityStrategyTestsBase { - @Test(groups = {"fi-multi-master", "fi-thinclient-multi-master"}, dataProvider = "testConfigs_readAllAfterCreation", retryAnalyzer = FlakyTestRetryAnalyzer.class) + @Test(groups = {"fi-multi-master", "fi-thinclient-multi-master"}, + dataProvider = "testConfigs_readAllAfterCreation", retryAnalyzer = SuperFlakyTestRetryAnalyzer.class) public void readAllAfterCreation( String testCaseId, Duration endToEndTimeout, diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/FITests_readManyByPartitionKeysAfterCreation.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/FITests_readManyByPartitionKeysAfterCreation.java index 420652e61154..d9dbce55fc22 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/FITests_readManyByPartitionKeysAfterCreation.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/FITests_readManyByPartitionKeysAfterCreation.java @@ -36,7 +36,8 @@ public class FITests_readManyByPartitionKeysAfterCreation extends FaultInjectionWithAvailabilityStrategyTestsBase { - @Test(groups = {"fi-multi-master"}, dataProvider = "testConfigs_readManyByPartitionKeysAfterCreation", retryAnalyzer = FlakyTestRetryAnalyzer.class) + @Test(groups = {"fi-multi-master"}, dataProvider = "testConfigs_readManyByPartitionKeysAfterCreation", + retryAnalyzer = SuperFlakyTestRetryAnalyzer.class) public void readManyByPartitionKeysAfterCreation( String testCaseId, Duration endToEndTimeout, diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/FaultInjectionWithAvailabilityStrategyTestsBase.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/FaultInjectionWithAvailabilityStrategyTestsBase.java index 157a1d21966f..4798e419f25d 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/FaultInjectionWithAvailabilityStrategyTestsBase.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/FaultInjectionWithAvailabilityStrategyTestsBase.java @@ -5082,7 +5082,8 @@ private CosmosAsyncContainer createTestContainer(CosmosAsyncClient clientWithPre private CosmosAsyncContainer createTestContainer(CosmosAsyncClient clientWithPreferredRegions, String dbId) { String containerId = UUID.randomUUID().toString(); - clientWithPreferredRegions.createDatabaseIfNotExists(dbId).block(); + executeControlPlaneWithRetry( + () -> clientWithPreferredRegions.createDatabaseIfNotExists(dbId).block()); CosmosAsyncDatabase databaseWithSeveralWriteableRegions = clientWithPreferredRegions.getDatabase(dbId); // setup db and container and pass their ids accordingly diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ProactiveConnectionManagementTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ProactiveConnectionManagementTest.java index d0224a038014..2c5461260c06 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ProactiveConnectionManagementTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ProactiveConnectionManagementTest.java @@ -256,7 +256,8 @@ public void openConnectionsAndInitCachesWithContainer(ProactiveConnectionManagem } } - @Test(groups = {"multi-master"}, dataProvider = "proactiveContainerInitConfigs") + @Test(groups = {"multi-master"}, dataProvider = "proactiveContainerInitConfigs", + retryAnalyzer = FlakyTestRetryAnalyzer.class) public void openConnectionsAndInitCachesWithCosmosClient_And_PerContainerConnectionPoolSize_ThroughSystemConfig( ProactiveConnectionManagementTestConfig proactiveConnectionManagementTestConfig) throws InterruptedException { diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/SessionConsistencyWithRegionScopingTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/SessionConsistencyWithRegionScopingTests.java index 5d8aa75d5a8f..ba87d718e36d 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/SessionConsistencyWithRegionScopingTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/SessionConsistencyWithRegionScopingTests.java @@ -1806,7 +1806,8 @@ public void readYouWriteWithNoExplicitRegionSwitching( } } - @Test(groups = {"multi-master"}, dataProvider = "readYouWriteWithExplicitRegionSwitchingTestContext", timeOut = 80 * TIMEOUT) + @Test(groups = {"multi-master"}, dataProvider = "readYouWriteWithExplicitRegionSwitchingTestContext", + timeOut = 80 * TIMEOUT, retryAnalyzer = FlakyTestRetryAnalyzer.class) public void readYouWriteWithExplicitRegionSwitching( BiFunction> func, String testId, diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/SplitTestsRetryAnalyzer.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/SplitTestsRetryAnalyzer.java index 3ee8b90920ca..7da4b1b6dfd9 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/SplitTestsRetryAnalyzer.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/SplitTestsRetryAnalyzer.java @@ -12,10 +12,28 @@ public SplitTestsRetryAnalyzer() { @Override public boolean retry(ITestResult iTestResult) { - if (!(iTestResult.getThrowable() instanceof SplitTimeoutException)) { + Throwable throwable = iTestResult.getThrowable(); + // Forcing a partition split is inherently disruptive. Besides the explicit + // SplitTimeoutException, a split on a shared live-test account can surface transient + // 408 (timeout), 429 (throttling) and 503 (service unavailable) errors while the split + // is in progress. Retry all of these (bounded by retryLimit) so split tests are not + // flaky on transient contention; deterministic failures are still surfaced. + if (!(throwable instanceof SplitTimeoutException) && !isTransientCosmosFailure(throwable)) { return false; } return super.retry(iTestResult); } + + private static boolean isTransientCosmosFailure(Throwable throwable) { + Throwable current = throwable; + while (current != null) { + if (current instanceof CosmosException) { + int statusCode = ((CosmosException) current).getStatusCode(); + return statusCode == 408 || statusCode == 429 || statusCode == 503; + } + current = current.getCause(); + } + return false; + } } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/FaultInjectionMetadataRequestRuleTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/FaultInjectionMetadataRequestRuleTests.java index 1641c8c90213..986046750dbc 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/FaultInjectionMetadataRequestRuleTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/FaultInjectionMetadataRequestRuleTests.java @@ -540,24 +540,31 @@ public void faultInjectionServerErrorRuleTests_PartitionKeyRanges_DelayError( int applyLimit, boolean shouldInjectPreferredRegionsOnClient) throws JsonProcessingException { - // We need to create a new client because client may have marked region unavailable in other tests - // which can impact the test result - CosmosAsyncClient testClient = getClientBuilder() - .contentResponseOnWriteEnabled(true) - .preferredRegions(shouldInjectPreferredRegionsOnClient ? this.writePreferredLocations : Collections.emptyList()) - .endToEndOperationLatencyPolicyConfig( - new CosmosEndToEndOperationLatencyPolicyConfigBuilder(Duration.ofMinutes(10)).build()) - .buildAsyncClient(); + String originalSharedPkRangeCacheSetting = + System.getProperty("COSMOS.SHARED_PARTITION_KEY_RANGE_CACHE_ENABLED"); + System.setProperty("COSMOS.SHARED_PARTITION_KEY_RANGE_CACHE_ENABLED", "false"); - CosmosAsyncContainer container = - testClient - .getDatabase(this.cosmosAsyncContainer.getDatabase().getId()) - .getContainer(this.cosmosAsyncContainer.getId()); + CosmosAsyncClient testClient = null; + FaultInjectionRule pkRangesConnectionDelayRule = null; + FaultInjectionRule dataOperationGoneRule = null; + try { + // We need to create a new client because client may have marked region unavailable in other tests + // which can impact the test result + testClient = getClientBuilder() + .contentResponseOnWriteEnabled(true) + .preferredRegions(shouldInjectPreferredRegionsOnClient ? this.writePreferredLocations : Collections.emptyList()) + .endToEndOperationLatencyPolicyConfig( + new CosmosEndToEndOperationLatencyPolicyConfigBuilder(Duration.ofMinutes(10)).build()) + .buildAsyncClient(); - // Test to validate partition key ranges request is being injected connection timeout - String pkRangesConnectionDelay = "PkRanges-connectionDelay-" + UUID.randomUUID(); - FaultInjectionRule pkRangesConnectionDelayRule = - new FaultInjectionRuleBuilder(pkRangesConnectionDelay) + CosmosAsyncContainer container = + testClient + .getDatabase(this.cosmosAsyncContainer.getDatabase().getId()) + .getContainer(this.cosmosAsyncContainer.getId()); + + // Test to validate partition key ranges request is being injected connection timeout + String pkRangesConnectionDelay = "PkRanges-connectionDelay-" + UUID.randomUUID(); + pkRangesConnectionDelayRule = new FaultInjectionRuleBuilder(pkRangesConnectionDelay) .condition( new FaultInjectionConditionBuilder() .region(this.writePreferredLocations.get(0)) @@ -574,8 +581,7 @@ public void faultInjectionServerErrorRuleTests_PartitionKeyRanges_DelayError( .duration(Duration.ofMinutes(5)) .build(); - FaultInjectionRule dataOperationGoneRule = - new FaultInjectionRuleBuilder("DataOperation-gone-" + UUID.randomUUID()) + dataOperationGoneRule = new FaultInjectionRuleBuilder("DataOperation-gone-" + UUID.randomUUID()) .condition( new FaultInjectionConditionBuilder() .operationType(FaultInjectionOperationType.CREATE_ITEM) @@ -589,7 +595,7 @@ public void faultInjectionServerErrorRuleTests_PartitionKeyRanges_DelayError( ) .duration(Duration.ofMinutes(5)) .build(); - try { + // create few items to first make sure the collection cache, pkRanges cache is being populated for (int i = 0; i < 10; i++) { container.createItem(TestObject.create()).block(); @@ -602,6 +608,8 @@ public void faultInjectionServerErrorRuleTests_PartitionKeyRanges_DelayError( try { CosmosDiagnostics cosmosDiagnostics = container.createItem(TestObject.create()).block().getDiagnostics(); + long ruleHitCount = pkRangesConnectionDelayRule.getHitCount(); + assertThat(ruleHitCount).isGreaterThan(0); // The PkRanges requests may have retried in another region, // but the create request will only be retried locally for PARTITION_IS_SPLITTING assertThat(cosmosDiagnostics.getContactedRegionNames().size()).isEqualTo(1); @@ -627,19 +635,31 @@ public void faultInjectionServerErrorRuleTests_PartitionKeyRanges_DelayError( assertThat(pkRangesLookup).isNotNull(); if (faultInjectionServerErrorType == FaultInjectionServerErrorType.CONNECTION_DELAY) { - assertThat(pkRangesLookup.get("durationinMS").asLong()).isGreaterThanOrEqualTo(45 * 1000 * Math.min(applyLimit, 3)); // the duration will be at least one connection timeout + assertThat(pkRangesLookup.get("durationinMS").asLong()) + .isGreaterThanOrEqualTo(45 * 1000 * Math.min(ruleHitCount, 3)); } if (faultInjectionServerErrorType == FaultInjectionServerErrorType.RESPONSE_DELAY) { - assertThat(pkRangesLookup.get("durationinMS").asLong()).isGreaterThanOrEqualTo(500 * Math.min(applyLimit, 3)); // the duration will be at least one response timeout + assertThat(pkRangesLookup.get("durationinMS").asLong()) + .isGreaterThanOrEqualTo(500 * Math.min(ruleHitCount, 3)); } } catch (CosmosException cosmosException) { fail("CreateItem should have succeeded. " + cosmosException.getDiagnostics()); } } finally { - pkRangesConnectionDelayRule.disable(); - dataOperationGoneRule.disable(); + if (originalSharedPkRangeCacheSetting == null) { + System.clearProperty("COSMOS.SHARED_PARTITION_KEY_RANGE_CACHE_ENABLED"); + } else { + System.setProperty( + "COSMOS.SHARED_PARTITION_KEY_RANGE_CACHE_ENABLED", originalSharedPkRangeCacheSetting); + } + if (pkRangesConnectionDelayRule != null) { + pkRangesConnectionDelayRule.disable(); + } + if (dataOperationGoneRule != null) { + dataOperationGoneRule.disable(); + } safeClose(testClient); } } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConsistencyTests2.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConsistencyTests2.java index ae487d5ad0a4..6f2fa02f41bc 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConsistencyTests2.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConsistencyTests2.java @@ -8,6 +8,7 @@ import com.azure.cosmos.CosmosClientBuilder; import com.azure.cosmos.CosmosException; import com.azure.cosmos.DirectConnectionConfig; +import com.azure.cosmos.FlakyTestRetryAnalyzer; import com.azure.cosmos.GatewayConnectionConfig; import com.azure.cosmos.implementation.clienttelemetry.ClientTelemetry; import com.azure.cosmos.implementation.directconnectivity.ReflectionUtils; @@ -143,7 +144,8 @@ public void validateEventualConsistencyOnAsyncReplicationGateway() { // https://msdata.visualstudio.com/CosmosDB/_workitems/edit/355053 } - @Test(groups = {"direct"}, dataProvider = "regionScopedSessionContainerConfigs", timeOut = CONSISTENCY_TEST_TIMEOUT) + @Test(groups = {"direct"}, dataProvider = "regionScopedSessionContainerConfigs", + timeOut = CONSISTENCY_TEST_TIMEOUT, retryAnalyzer = FlakyTestRetryAnalyzer.class) public void validateSessionContainerAfterCollectionDeletion(boolean shouldRegionScopedSessionContainerEnabled) throws Exception { //TODO Need to test with TCP protocol // https://msdata.visualstudio.com/CosmosDB/_workitems/edit/355057 diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/FeedResponseListValidator.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/FeedResponseListValidator.java index c997fc715d03..1201dfa45d67 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/FeedResponseListValidator.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/FeedResponseListValidator.java @@ -144,6 +144,30 @@ public void validate(List> feedList) { return this; } + /** + * Asserts the results CONTAIN all of the expected resource IDs, allowing additional + * unexpected results. Unlike {@link #exactlyContainsInAnyOrder(List)} this does not + * require the result set to be limited to the expected IDs, so it is safe for + * account-global reads (e.g. readAllDatabases) that run against shared test accounts + * where other resources may exist concurrently. + */ + public Builder containsResourceIds(List expectedIds) { + validators.add(new FeedResponseListValidator() { + @Override + public void validate(List> feedList) { + List actualIds = feedList + .stream() + .flatMap(f -> f.getResults().stream()) + .map(r -> getResource(r).getResourceId()) + .collect(Collectors.toList()); + assertThat(actualIds) + .describedAs("Resource IDs of results") + .containsAll(expectedIds); + } + }); + return this; + } + public Builder exactlyContainsIdsInAnyOrder(List expectedIds) { validators.add(new FeedResponseListValidator() { @Override diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ThinClientProbeWiringTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ThinClientProbeWiringTests.java index 8864bd15b262..2bd33435326f 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ThinClientProbeWiringTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ThinClientProbeWiringTests.java @@ -152,6 +152,7 @@ public void setThinClientHttpClient_triggersProbeOnRefresh() throws Exception { // Probe is fire-and-forget on a scheduler -> wait briefly for it to run. waitForProbeCallCount(probeCallCount, 2, Duration.ofSeconds(5)); + waitForProxyDecision(gem, Boolean.TRUE, Duration.ofSeconds(5)); assertThat(probeCallCount.get()).as("probe was issued for each thin-client region").isGreaterThanOrEqualTo(2); assertThat(gem.getProxyProbeDecision()).as("after all-200 cycle, proxy is healthy").isEqualTo(Boolean.TRUE); diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/GatewayAddressCacheTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/GatewayAddressCacheTest.java index a002a20cd9be..edbf088dcc24 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/GatewayAddressCacheTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/GatewayAddressCacheTest.java @@ -1383,10 +1383,10 @@ public void tryGetAddress_repeatedlySetUnhealthyStatus_forceRefresh() throws Int // 2. replica validation will get triggered in case of unhealthyPending / unknown addresses, replica validation will do a // submitOpenConnectionTaskOutsideLoop for each of these addresses but before that it will also do // isCollectionRidUnderOpenConnectionsFlow check to determine the no. of connections to open - Mockito.verify(proactiveOpenConnectionsProcessorMock, Mockito.times(2)) + Mockito.verify(proactiveOpenConnectionsProcessorMock, Mockito.timeout(5000).times(2)) .isCollectionRidUnderOpenConnectionsFlow(Mockito.any()); - Mockito.verify(proactiveOpenConnectionsProcessorMock, Mockito.times(1)) + Mockito.verify(proactiveOpenConnectionsProcessorMock, Mockito.timeout(5000).times(1)) .submitOpenConnectionTaskOutsideLoop(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.anyInt()); } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ChangeFeedTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ChangeFeedTest.java index 959b596cf244..b1c2f5a40ccc 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ChangeFeedTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ChangeFeedTest.java @@ -5,6 +5,7 @@ import com.azure.cosmos.BridgeInternal; import com.azure.cosmos.ConnectionMode; import com.azure.cosmos.CosmosDiagnostics; +import com.azure.cosmos.FlakyTestRetryAnalyzer; import com.azure.cosmos.implementation.AsyncDocumentClient; import com.azure.cosmos.implementation.ClientSideRequestStatistics; import com.azure.cosmos.implementation.Database; @@ -387,7 +388,7 @@ public void changeFeed_fromStartDate() throws Exception { assertThat(count).as("Change feed should have one newly created document").isEqualTo(1); } - @Test(groups = { "query" }, timeOut = TIMEOUT) + @Test(groups = { "query" }, timeOut = TIMEOUT, retryAnalyzer = FlakyTestRetryAnalyzer.class) public void changesFromPartitionKey_AfterInsertingNewDocuments() throws Exception { String partitionKey = partitionKeyToDocuments.keySet().iterator().next(); FeedRange feedRange = new FeedRangePartitionKeyImpl( diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/MultiMasterConflictResolutionTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/MultiMasterConflictResolutionTest.java index 3298660af6fe..6695f202b6c3 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/MultiMasterConflictResolutionTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/MultiMasterConflictResolutionTest.java @@ -50,7 +50,9 @@ public void conflictResolutionPolicyCRUD() { // default last writer wins, path _ts CosmosContainerProperties containerSettings = new CosmosContainerProperties(UUID.randomUUID().toString(), partitionKeyDef); - database.createContainer(containerSettings, new CosmosContainerRequestOptions()).block(); + final CosmosContainerProperties initialContainerSettings = containerSettings; + executeControlPlaneWithRetry(() -> + database.createContainer(initialContainerSettings, new CosmosContainerRequestOptions()).block()); CosmosAsyncContainer container = database.getContainer(containerSettings.getId()); containerSettings = container.read().block().getProperties(); @@ -124,7 +126,9 @@ private void testConflictResolutionPolicyRequiringPath(ConflictResolutionMode co } else { collectionSettings.setConflictResolutionPolicy(ConflictResolutionPolicy.createCustomPolicy(paths[i])); } - collectionSettings = database.createContainer(collectionSettings, new CosmosContainerRequestOptions()).block().getProperties(); + final CosmosContainerProperties containerToCreate = collectionSettings; + collectionSettings = executeControlPlaneWithRetry(() -> + database.createContainer(containerToCreate, new CosmosContainerRequestOptions()).block().getProperties()); assertThat(collectionSettings.getConflictResolutionPolicy().getMode()).isEqualTo(conflictResolutionMode); if (conflictResolutionMode == ConflictResolutionMode.LAST_WRITER_WINS) { diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedCollectionsTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedCollectionsTest.java index de7a32d39e10..24c1760f07be 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedCollectionsTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedCollectionsTest.java @@ -65,7 +65,7 @@ public void readCollections() throws Exception { } - @BeforeClass(groups = { "query" }, timeOut = SETUP_TIMEOUT) + @BeforeClass(groups = { "query" }, timeOut = 16 * SETUP_TIMEOUT) public void before_ReadFeedCollectionsTest() { client = getClientBuilder().buildAsyncClient(); createdDatabase = createDatabase(client, databaseId); diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedDatabasesTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedDatabasesTest.java index 4a5b6376c975..97a5ca3724b1 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedDatabasesTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedDatabasesTest.java @@ -23,10 +23,11 @@ import java.util.List; import java.util.stream.Collectors; +import static org.assertj.core.api.Assertions.assertThat; + public class ReadFeedDatabasesTest extends TestSuiteBase { private List createdDatabases = new ArrayList<>(); - private List allDatabases = new ArrayList<>(); private CosmosAsyncClient client; @@ -41,11 +42,13 @@ public void readDatabases() throws Exception { CosmosPagedFlux feedObservable = client.readAllDatabases(); - int expectedPageSize = (allDatabases.size() + maxItemCount - 1) / maxItemCount; + // readAllDatabases is an account-global read. Live tests share a fixed account, so + // other test runs may create/delete databases concurrently. Assert only that the + // databases created by this test are present (containment) rather than asserting the + // exact account-wide set/count/page-count, which would be non-deterministic. FeedResponseListValidator validator = new FeedResponseListValidator.Builder() - .totalSize(allDatabases.size()) - .exactlyContainsInAnyOrder(allDatabases.stream().map(d -> d.getResourceId()).collect(Collectors.toList())) - .numberOfPages(expectedPageSize) + .containsResourceIds(createdDatabases.stream().map(d -> d.getResourceId()).collect(Collectors.toList())) + .numberOfPagesIsGreaterThanOrEqualTo(1) .pageSatisfy(0, new FeedResponseValidator.Builder() .requestChargeGreaterThanOrEqualTo(1.0).build()) .build(); @@ -56,13 +59,15 @@ public void readDatabases() throws Exception { @BeforeClass(groups = { "query" }, timeOut = SETUP_TIMEOUT) public void before_ReadFeedDatabasesTest() throws URISyntaxException { client = getClientBuilder().buildAsyncClient(); - allDatabases = client.readAllDatabases() - .collectList() - .block(); for(int i = 0; i < 5; i++) { createdDatabases.add(createDatabase(client)); } - allDatabases.addAll(createdDatabases); + + // Guard against the containment assertion silently degrading to a no-op: if no databases + // were created the readDatabases() test would assert nothing meaningful on a shared account. + assertThat(createdDatabases) + .describedAs("databases created by this test") + .isNotEmpty(); } public CosmosDatabaseProperties createDatabase(CosmosAsyncClient client) { diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedOffersTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedOffersTest.java index f316029bbc84..fc1d7f723b8f 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedOffersTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedOffersTest.java @@ -35,6 +35,8 @@ import java.util.UUID; import java.util.stream.Collectors; +import static org.assertj.core.api.Assertions.assertThat; + //TODO: change to use external TestSuiteBase public class ReadFeedOffersTest extends TestSuiteBase { @@ -45,7 +47,8 @@ public class ReadFeedOffersTest extends TestSuiteBase { public final String databaseId = DatabaseForTest.generateId(); private Database createdDatabase; - private List allOffers = new ArrayList<>(); + private final List createdCollections = new ArrayList<>(); + private List expectedOffers = new ArrayList<>(); private AsyncDocumentClient client; @@ -78,13 +81,13 @@ public void readOffers() throws Exception { Flux> feedObservable = client.readOffers(dummyState); - int maxItemCount = ModelBridgeInternal.getMaxItemCountFromQueryRequestOptions(options); - int expectedPageSize = (allOffers.size() + maxItemCount - 1) / maxItemCount; - + // readOffers is an account-global read. Live tests share a fixed account, so + // other test runs may create/delete containers (and thus offers) concurrently. + // Assert only that the offers for the containers created by this test are present + // (containment) rather than asserting the exact account-wide set/count/page-count. FeedResponseListValidator validator = new FeedResponseListValidator.Builder() - .totalSize(allOffers.size()) - .exactlyContainsInAnyOrder(allOffers.stream().map(d -> d.getResourceId()).collect(Collectors.toList())) - .numberOfPages(expectedPageSize) + .containsResourceIds(expectedOffers.stream().map(d -> d.getResourceId()).collect(Collectors.toList())) + .numberOfPagesIsGreaterThanOrEqualTo(1) .pageSatisfy(0, new FeedResponseValidator.Builder() .requestChargeGreaterThanOrEqualTo(1.0).build()) .build(); @@ -92,13 +95,13 @@ public void readOffers() throws Exception { } } - @BeforeClass(groups = { "query" }, timeOut = SETUP_TIMEOUT) + @BeforeClass(groups = { "query" }, timeOut = 6 * SETUP_TIMEOUT) public void before_ReadFeedOffersTest() { client = clientBuilder().build(); createdDatabase = createDatabase(client, databaseId); for(int i = 0; i < 3; i++) { - createCollections(client); + createdCollections.add(createCollections(client)); } QueryFeedOperationState offerDummyState = TestUtils.createDummyQueryFeedOperationState( @@ -109,15 +112,31 @@ public void before_ReadFeedOffersTest() { ); try { - allOffers = client.readOffers(offerDummyState) + List createdCollectionRids = createdCollections.stream() + .map(DocumentCollection::getResourceId) + .collect(Collectors.toList()); + // An Offer's getOfferResourceId() is the resource id of the collection it applies to, + // whereas getResourceId() is the Offer's own resource id. Filter the account-global + // offer feed down to the offers that belong to the collections this test created. + expectedOffers = client.readOffers(offerDummyState) .map(FeedResponse::getResults) .collectList() .map(list -> list.stream().flatMap(Collection::stream).collect(Collectors.toList())) .single() - .block(); + .block() + .stream() + .filter(o -> createdCollectionRids.contains(o.getOfferResourceId())) + .collect(Collectors.toList()); } finally { safeClose(offerDummyState); } + + // Guard against the containment assertion silently degrading to a no-op: each created + // collection has exactly one dedicated-throughput offer, so we must have resolved one + // offer per collection. If this fails the offer read/filtering is broken, not the account. + assertThat(expectedOffers) + .describedAs("offers resolved for the collections created by this test") + .hasSize(createdCollections.size()); } @AfterClass(groups = { "query" }, timeOut = SHUTDOWN_TIMEOUT, alwaysRun = true) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java index cfc11c6b9212..ef314ab4d27d 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java @@ -145,13 +145,14 @@ public abstract class TestSuiteBase extends CosmosAsyncClientTest { protected static final int SETUP_TIMEOUT = 300_000; protected static final int SHUTDOWN_TIMEOUT = 24000; - private static final int SHARED_SUITE_SETUP_TIMEOUT = 600_000; + private static final int SHARED_SUITE_SETUP_TIMEOUT = 2_400_000; protected static final int SUITE_SHUTDOWN_TIMEOUT = 60000; protected static final int WAIT_REPLICA_CATCH_UP_IN_MILLIS = 4000; - private static final Duration COLLECTION_READINESS_MAX_WAIT = Duration.ofMinutes(2); + private static final Duration COLLECTION_READINESS_MAX_WAIT = Duration.ofSeconds( + Long.getLong("COSMOS.COLLECTION_READINESS_MAX_WAIT_SECONDS", Duration.ofMinutes(2).getSeconds())); private static final Duration COLLECTION_READINESS_PROBE_TIMEOUT = Duration.ofSeconds(10); @@ -163,20 +164,25 @@ public abstract class TestSuiteBase extends CosmosAsyncClientTest { private static final int TRANSIENT_CLEANUP_MAX_RETRY_ATTEMPTS = 30; + private static final Duration TRANSIENT_CREATE_RETRY_DELAY = Duration.ofSeconds(3); + + private static final int TRANSIENT_CREATE_MAX_RETRY_ATTEMPTS = 20; + private static final Duration STORED_PROCEDURE_QUERY_RETRY_DELAY = Duration.ofSeconds(1); private static final int STORED_PROCEDURE_QUERY_ATTEMPT_TIMEOUT = 5_000; private static final Duration STORED_PROCEDURE_QUERY_MAX_RETRY_DURATION = Duration.ofSeconds(30); - private static final Duration FEED_RANGE_WARMUP_MAX_WAIT = COLLECTION_READINESS_MAX_WAIT; + private static final Duration FEED_RANGE_WARMUP_MAX_WAIT = Duration.ofMinutes(2); private static final Duration FEED_RANGE_WARMUP_ATTEMPT_TIMEOUT = Duration.ofSeconds(30); private static boolean isTransientCreateFailure(Throwable t) { - if (t instanceof CosmosException) { - int statusCode = ((CosmosException) t).getStatusCode(); - return statusCode == 408 || statusCode == 429; + CosmosException cosmosException = getCosmosException(t); + if (cosmosException != null) { + int statusCode = cosmosException.getStatusCode(); + return statusCode == 408 || statusCode == 429 || statusCode == 503; } return false; } @@ -185,6 +191,65 @@ private static boolean isConflictException(Throwable t) { return t instanceof CosmosException && ((CosmosException) t).getStatusCode() == 409; } + /** + * True for transient control-plane failures that live tests can hit when many concurrent + * matrix legs share a fixed account: 408 (request timeout), 429 (throttling, including the + * metadata/control-plane 429 substatus 3200 that the default client retry does not cover + * well), 500, and 503. These are safe to retry for idempotent database/container CRUD; + * deterministic failures (e.g. 400 bad request, 409 conflict) are intentionally excluded. + */ + private static boolean isTransientControlPlaneFailure(Throwable t) { + Throwable unwrapped = Exceptions.unwrap(t); + if (!(unwrapped instanceof CosmosException)) { + return false; + } + int statusCode = ((CosmosException) unwrapped).getStatusCode(); + return statusCode == HttpConstants.StatusCodes.REQUEST_TIMEOUT + || statusCode == HttpConstants.StatusCodes.TOO_MANY_REQUESTS + || statusCode == HttpConstants.StatusCodes.INTERNAL_SERVER_ERROR + || statusCode == HttpConstants.StatusCodes.SERVICE_UNAVAILABLE; + } + + /** + * Executes a control-plane operation (database/container CRUD) retrying only transient + * shared-account contention failures (see {@link #isTransientControlPlaneFailure}). The + * default SDK throttling retry (9 attempts / 30s) is not enough to absorb the metadata-429 / + * 408 storms that occur when many live-test legs do CRUD on one shared fixed account. Any + * non-transient failure (e.g. an expected 400/409) is rethrown immediately so negative-path + * assertions are preserved. + * + * @param action the control-plane operation to run + * @param beforeRetry optional hook run before each retry (e.g. to reset a mock tracer so a + * failed attempt's spans do not leak into the next attempt); may be null + */ + protected static T executeControlPlaneWithRetry(Supplier action, Runnable beforeRetry) { + final int maxAttempts = 20; + final long retryDelayMillis = Duration.ofSeconds(3).toMillis(); + for (int attempt = 1; ; attempt++) { + try { + return action.get(); + } catch (RuntimeException e) { + if (attempt >= maxAttempts || !isTransientControlPlaneFailure(e)) { + throw e; + } + logger.warn("Transient control-plane failure (attempt {}/{}): {}", attempt, maxAttempts, e.getMessage()); + if (beforeRetry != null) { + beforeRetry.run(); + } + try { + Thread.sleep(retryDelayMillis); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw e; + } + } + } + } + + protected static T executeControlPlaneWithRetry(Supplier action) { + return executeControlPlaneWithRetry(action, null); + } + /** * Executes an action with retry logic for transient failures in @BeforeClass setup methods. * Retries up to maxRetries times with increasing backoff (1s, 2s, 3s...). @@ -409,7 +474,8 @@ private static boolean isTransientCleanupFailure(Throwable throwable) { } int statusCode = ((CosmosException) unwrappedException).getStatusCode(); - return statusCode == HttpConstants.StatusCodes.TOO_MANY_REQUESTS + return statusCode == HttpConstants.StatusCodes.REQUEST_TIMEOUT + || statusCode == HttpConstants.StatusCodes.TOO_MANY_REQUESTS || statusCode == HttpConstants.StatusCodes.INTERNAL_SERVER_ERROR || statusCode == HttpConstants.StatusCodes.SERVICE_UNAVAILABLE; } @@ -927,11 +993,16 @@ private static void createCollectionIfNotExists( CosmosContainerRequestOptions options, int throughput) { - database.createContainer(cosmosContainerProperties, ThroughputProperties.createManualThroughput(throughput), options) - .retryWhen(Retry.fixedDelay(3, Duration.ofSeconds(5)) + database.createContainer( + cosmosContainerProperties, + ThroughputProperties.createManualThroughput(throughput), + options) + .retryWhen(Retry.fixedDelay(TRANSIENT_CREATE_MAX_RETRY_ATTEMPTS, TRANSIENT_CREATE_RETRY_DELAY) .filter(TestSuiteBase::isTransientCreateFailure)) .onErrorResume(e -> isConflictException(e), e -> { - logger.info("Container {} already exists (409 Conflict), treating as success", cosmosContainerProperties.getId()); + logger.info( + "Container {} already exists (409 Conflict), treating as success", + cosmosContainerProperties.getId()); return Mono.empty(); }) .block(); @@ -1261,10 +1332,12 @@ public static CosmosAsyncContainer createCollection(CosmosAsyncDatabase database public static CosmosAsyncContainer createCollection(CosmosAsyncDatabase database, CosmosContainerProperties cosmosContainerProperties, CosmosContainerRequestOptions options, CosmosAsyncClient probeClient) { database.createContainer(cosmosContainerProperties, options) - .retryWhen(Retry.fixedDelay(3, Duration.ofSeconds(5)) + .retryWhen(Retry.fixedDelay(TRANSIENT_CREATE_MAX_RETRY_ATTEMPTS, TRANSIENT_CREATE_RETRY_DELAY) .filter(TestSuiteBase::isTransientCreateFailure)) .onErrorResume(e -> isConflictException(e), e -> { - logger.info("Container {} already exists (409 Conflict), treating as success", cosmosContainerProperties.getId()); + logger.info( + "Container {} already exists (409 Conflict), treating as success", + cosmosContainerProperties.getId()); return Mono.empty(); }) .block(); @@ -2810,9 +2883,18 @@ protected static void truncateCollection(DocumentCollection collection) { logger.info("Truncating DocumentCollection {} ...", collection.getId()); + // Live tests now share fixed accounts across concurrent CI legs, so cleanup can hit + // transient metadata/data throttling (429). Use the same generous throttling retry as + // the housekeeping client (well above the SDK default of 9) so truncation during + // setup/cleanup absorbs 429s instead of failing the @BeforeClass. + ThrottlingRetryOptions truncateRetryOptions = new ThrottlingRetryOptions(); + truncateRetryOptions.setMaxRetryAttemptsOnThrottledRequests(200); + truncateRetryOptions.setMaxRetryWaitTime(Duration.ofSeconds(SUITE_SETUP_TIMEOUT)); + try (CosmosAsyncClient cosmosClient = new CosmosClientBuilder() .key(TestConfigurations.MASTER_KEY) .endpoint(TestConfigurations.HOST) + .throttlingRetryOptions(truncateRetryOptions) .buildAsyncClient()) { CosmosQueryRequestOptions options = new CosmosQueryRequestOptions(); diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ThinClientQueryE2ETest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ThinClientQueryE2ETest.java index fc03a85cb92a..fa789af9c8c0 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ThinClientQueryE2ETest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ThinClientQueryE2ETest.java @@ -51,6 +51,7 @@ import org.testng.annotations.Test; import reactor.core.publisher.Flux; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -295,7 +296,7 @@ private void seedHierarchicalData() { } } - @AfterClass(groups = {"thinclient", "thinclientEndpointProbe"}, timeOut = SHUTDOWN_TIMEOUT, alwaysRun = true) + @AfterClass(groups = {"thinclient", "thinclientEndpointProbe"}, timeOut = 3 * SHUTDOWN_TIMEOUT, alwaysRun = true) public void afterClass() { if (directContainer != null && !seededDocs.isEmpty()) { try { @@ -303,7 +304,9 @@ public void afterClass() { .map(doc -> CosmosBulkOperations.getDeleteItemOperation( doc.get(ID_FIELD).asText(), new PartitionKey(commonPk))) .collect(Collectors.toList()); - directContainer.executeBulkOperations(Flux.fromIterable(deleteOps)).blockLast(); + directContainer.executeBulkOperations(Flux.fromIterable(deleteOps)) + .timeout(Duration.ofSeconds(20)) + .blockLast(); } catch (Exception e) { logger.warn("Bulk delete of seeded docs failed: {}", e.getMessage()); } @@ -1830,7 +1833,11 @@ public void testHierarchicalReadManyByPartitionKeysPrefixPartitionKey() { private static void safeDeleteContainer(CosmosAsyncContainer container) { if (container != null) { - try { container.delete().block(); } catch (Exception e) { logger.warn("Container cleanup failed: {}", e.getMessage()); } + try { + container.delete().timeout(Duration.ofSeconds(20)).block(); + } catch (Exception e) { + logger.warn("Container cleanup failed: {}", e.getMessage()); + } } } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/changefeed/epkversion/IncrementalChangeFeedProcessorTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/changefeed/epkversion/IncrementalChangeFeedProcessorTest.java index 4f0c58acf135..3506d9c5b32f 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/changefeed/epkversion/IncrementalChangeFeedProcessorTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/changefeed/epkversion/IncrementalChangeFeedProcessorTest.java @@ -358,7 +358,7 @@ public void verifyConsistentTimestamps() throws InterruptedException { } } - @Test(groups = {"multi-master"}, timeOut = 50 * CHANGE_FEED_PROCESSOR_TIMEOUT) + @Test(groups = {"multi-master"}, timeOut = 50 * CHANGE_FEED_PROCESSOR_TIMEOUT, retryAnalyzer = FlakyTestRetryAnalyzer.class) public void readFeedDocumentsStartFromCustomDateForMultiWrite_test() throws InterruptedException { CosmosClientBuilder clientBuilder = getClientBuilder(); @@ -2145,11 +2145,13 @@ public void verifyLeasesOnRestart_AfterSplit() throws InterruptedException { } } + private static final Duration CHANGE_FEED_PROCESSOR_LIFECYCLE_TIMEOUT = Duration.ofSeconds(60); + private void startChangeFeedProcessor(ChangeFeedProcessor changeFeedProcessor) { changeFeedProcessor .start() .subscribeOn(Schedulers.boundedElastic()) - .timeout(Duration.ofMillis(2 * CHANGE_FEED_PROCESSOR_TIMEOUT)) + .timeout(CHANGE_FEED_PROCESSOR_LIFECYCLE_TIMEOUT) .onErrorResume(throwable -> { logger.error("Change feed processor did not start in the expected time", throwable); return Mono.error(throwable); @@ -2161,7 +2163,7 @@ private void safeStopChangeFeedProcessor(ChangeFeedProcessor changeFeedProcessor if (changeFeedProcessor != null && changeFeedProcessor.isStarted()) { changeFeedProcessor .stop() - .timeout(Duration.ofMinutes(2 * CHANGE_FEED_PROCESSOR_TIMEOUT)) + .timeout(CHANGE_FEED_PROCESSOR_LIFECYCLE_TIMEOUT) .onErrorResume(throwable -> { logger.warn("Stop changeFeedProcessor failed", throwable); return Mono.empty(); diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/changefeed/pkversion/IncrementalChangeFeedProcessorTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/changefeed/pkversion/IncrementalChangeFeedProcessorTest.java index 52e01924ab29..24df0a0b5921 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/changefeed/pkversion/IncrementalChangeFeedProcessorTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/changefeed/pkversion/IncrementalChangeFeedProcessorTest.java @@ -11,6 +11,7 @@ import com.azure.cosmos.CosmosClientBuilder; import com.azure.cosmos.CosmosEndToEndOperationLatencyPolicyConfigBuilder; import com.azure.cosmos.CosmosNettyLeakDetectorFactory; +import com.azure.cosmos.FlakyTestRetryAnalyzer; import com.azure.cosmos.SplitTestsRetryAnalyzer; import com.azure.cosmos.SplitTimeoutException; import com.azure.cosmos.ThroughputControlGroupConfig; @@ -243,7 +244,7 @@ public void readFeedDocumentsStartFromCustomDate() throws InterruptedException { } } - @Test(groups = {"multi-master"}, timeOut = 50 * CHANGE_FEED_PROCESSOR_TIMEOUT) + @Test(groups = {"multi-master"}, timeOut = 50 * CHANGE_FEED_PROCESSOR_TIMEOUT, retryAnalyzer = FlakyTestRetryAnalyzer.class) public void readFeedDocumentsStartFromCustomDateForMultiWrite_test() throws InterruptedException { CosmosClientBuilder clientBuilder = getClientBuilder(); @@ -2042,11 +2043,13 @@ private void triggerAndWaitForPartitionSplit( } } + private static final Duration CHANGE_FEED_PROCESSOR_LIFECYCLE_TIMEOUT = Duration.ofSeconds(60); + private void startChangeFeedProcessor(ChangeFeedProcessor changeFeedProcessor) { changeFeedProcessor .start() .subscribeOn(Schedulers.boundedElastic()) - .timeout(Duration.ofMillis(2 * CHANGE_FEED_PROCESSOR_TIMEOUT)) + .timeout(CHANGE_FEED_PROCESSOR_LIFECYCLE_TIMEOUT) .onErrorResume(throwable -> { log.error("Change feed processor did not start in the expected time", throwable); return Mono.error(throwable); @@ -2058,7 +2061,7 @@ private void safeStopChangeFeedProcessor(ChangeFeedProcessor changeFeedProcessor if (changeFeedProcessor != null && changeFeedProcessor.isStarted()) { changeFeedProcessor .stop() - .timeout(Duration.ofMinutes(2 * CHANGE_FEED_PROCESSOR_TIMEOUT)) + .timeout(CHANGE_FEED_PROCESSOR_LIFECYCLE_TIMEOUT) .onErrorResume(throwable -> { logger.warn("Stop changeFeedProcessor failed", throwable); return Mono.empty(); diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/workflows/customer/CustomerWorkflowChangeFeedProcessorTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/workflows/customer/CustomerWorkflowChangeFeedProcessorTest.java index 2d5b88cacd65..b4cac2f4eee1 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/workflows/customer/CustomerWorkflowChangeFeedProcessorTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/workflows/customer/CustomerWorkflowChangeFeedProcessorTest.java @@ -6,6 +6,7 @@ import com.azure.cosmos.ChangeFeedProcessorBuilder; import com.azure.cosmos.CosmosAsyncContainer; import com.azure.cosmos.CosmosClientBuilder; +import com.azure.cosmos.FlakyTestRetryAnalyzer; import com.azure.cosmos.TestObject; import com.azure.cosmos.models.ChangeFeedProcessorItem; import com.azure.cosmos.models.ChangeFeedProcessorOptions; @@ -46,14 +47,17 @@ public void afterClass() { closeClient(); } - @Test(groups = {"fi-customer-workflows"}, timeOut = 2 * TIMEOUT) + @Test(groups = {"fi-customer-workflows"}, timeOut = 5 * TIMEOUT, + retryAnalyzer = FlakyTestRetryAnalyzer.class) public void latestVersionProcessorRestartResumesFromLeasesWorkflow() throws InterruptedException { - CosmosAsyncContainer feedContainer = createTemporaryContainer("customer-cfp-feed", "/mypk"); - CosmosAsyncContainer leaseContainer = createTemporaryContainer("customer-cfp-lease", "/id"); + CosmosAsyncContainer feedContainer = null; + CosmosAsyncContainer leaseContainer = null; ChangeFeedProcessor processor = null; FaultInjectionRule readFeedDelayRule = null; try { + feedContainer = createTemporaryContainer("customer-cfp-feed", "/mypk"); + leaseContainer = createTemporaryContainer("customer-cfp-lease", "/id"); Set expectedIds = Collections.newSetFromMap(new ConcurrentHashMap()); Set receivedIds = Collections.newSetFromMap(new ConcurrentHashMap()); CountDownLatch initialLatch = new CountDownLatch(2); @@ -108,13 +112,16 @@ public void latestVersionProcessorRestartResumesFromLeasesWorkflow() throws Inte } } - @Test(groups = {"fi-customer-workflows"}, timeOut = 2 * TIMEOUT) + @Test(groups = {"fi-customer-workflows"}, timeOut = 5 * TIMEOUT, + retryAnalyzer = FlakyTestRetryAnalyzer.class) public void latestVersionProcessorWithNewLeasePrefixReprocessesFromBeginningWorkflow() throws InterruptedException { - CosmosAsyncContainer feedContainer = createTemporaryContainer("customer-cfp-feed", "/mypk"); - CosmosAsyncContainer leaseContainer = createTemporaryContainer("customer-cfp-lease", "/id"); + CosmosAsyncContainer feedContainer = null; + CosmosAsyncContainer leaseContainer = null; ChangeFeedProcessor processor = null; try { + feedContainer = createTemporaryContainer("customer-cfp-feed", "/mypk"); + leaseContainer = createTemporaryContainer("customer-cfp-lease", "/id"); Set expectedIds = Collections.newSetFromMap(new ConcurrentHashMap()); Set initialReceivedIds = Collections.newSetFromMap(new ConcurrentHashMap()); CountDownLatch initialLatch = new CountDownLatch(2); diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/workflows/customer/CustomerWorkflowHighE2ETimeoutTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/workflows/customer/CustomerWorkflowHighE2ETimeoutTest.java index 3946d0324b1e..a3dacd848c7f 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/workflows/customer/CustomerWorkflowHighE2ETimeoutTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/workflows/customer/CustomerWorkflowHighE2ETimeoutTest.java @@ -7,6 +7,7 @@ import com.azure.cosmos.CosmosEndToEndOperationLatencyPolicyConfig; import com.azure.cosmos.CosmosEndToEndOperationLatencyPolicyConfigBuilder; import com.azure.cosmos.CosmosException; +import com.azure.cosmos.FlakyTestRetryAnalyzer; import com.azure.cosmos.TestObject; import com.azure.cosmos.ThresholdBasedAvailabilityStrategy; import com.azure.cosmos.implementation.ImplementationBridgeHelpers; @@ -99,7 +100,8 @@ public void responseDelayWithAvailabilityStrategyWorkflow(String operation, Faul } } - @Test(groups = {"fi-customer-workflows"}, timeOut = 2 * TIMEOUT) + @Test(groups = {"fi-customer-workflows"}, timeOut = 2 * TIMEOUT, + retryAnalyzer = FlakyTestRetryAnalyzer.class) public void partitionMigratingFaultWithE2EPolicyWorkflow() { TestObject item = TestObject.create(); this.container.createItem(item).block(); diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/workflows/customer/CustomerWorkflowTestBase.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/workflows/customer/CustomerWorkflowTestBase.java index e8d57ca887e2..581ed3c14dfc 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/workflows/customer/CustomerWorkflowTestBase.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/workflows/customer/CustomerWorkflowTestBase.java @@ -21,7 +21,8 @@ import com.azure.cosmos.implementation.directconnectivity.ReflectionUtils; import com.azure.cosmos.models.CosmosItemIdentity; import com.azure.cosmos.models.CosmosItemRequestOptions; -import com.azure.cosmos.models.ThroughputProperties; +import com.azure.cosmos.models.CosmosContainerProperties; +import com.azure.cosmos.models.CosmosContainerRequestOptions; import com.azure.cosmos.rx.TestSuiteBase; import com.azure.cosmos.test.faultinjection.CosmosFaultInjectionHelper; import com.azure.cosmos.test.faultinjection.FaultInjectionCondition; @@ -177,11 +178,29 @@ protected final CosmosAsyncContainer createTemporaryContainer(String prefix, Str CosmosAsyncDatabase database = getSharedCosmosDatabase(this.client); String containerId = prefix + "-" + UUID.randomUUID(); - database - .createContainerIfNotExists(containerId, partitionKeyPath, ThroughputProperties.createManualThroughput(400)) - .block(); - - return database.getContainer(containerId); + CosmosContainerProperties properties = new CosmosContainerProperties(containerId, partitionKeyPath); + CosmosAsyncContainer container = database.getContainer(containerId); + boolean createdAndValidated = false; + try { + CosmosAsyncContainer result = + createCollection(database, properties, new CosmosContainerRequestOptions(), 400); + createdAndValidated = true; + return result; + } finally { + if (!createdAndValidated) { + boolean interrupted = Thread.interrupted(); + try { + container.delete().timeout(Duration.ofSeconds(20)).block(); + } catch (Throwable cleanupError) { + logger.warn("Failed to clean up temporary container {} after setup failure.", + containerId, cleanupError); + } finally { + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + } + } } protected static void deleteTemporaryContainer(CosmosAsyncContainer container) { diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/EndpointProbeClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/EndpointProbeClient.java index a0bcc260b40e..e7d82e897e98 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/EndpointProbeClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/EndpointProbeClient.java @@ -121,7 +121,11 @@ public Mono runProbeCycle(Collection regionalEndpoints) { "Thin-client probe cycle threw an unexpected error; leaving failed regions un-cached.", t); return Mono.just(this.thinClientRoutable); }) - .doFinally(s -> this.cycleInProgress.set(false)); + // Publish completion before downstream observers receive the result. + .doOnNext(ignored -> this.cycleInProgress.set(false)) + // A cancelled cycle emits no value, so release its guard here. Errors are converted + // to a value above and therefore use the doOnNext path. + .doOnCancel(() -> this.cycleInProgress.set(false)); }); } diff --git a/sdk/cosmos/live-platform-matrix.json b/sdk/cosmos/live-platform-matrix.json index baa999d6f14a..294809039317 100644 --- a/sdk/cosmos/live-platform-matrix.json +++ b/sdk/cosmos/live-platform-matrix.json @@ -24,7 +24,16 @@ "@{ enableMultipleWriteLocations = $false; defaultConsistencyLevel = 'Strong' }": "", "@{ enableMultipleWriteLocations = $false; defaultConsistencyLevel = 'Session'; enablePartitionMerge = $true }": "", "@{ enableMultipleWriteLocations = $false; defaultConsistencyLevel = 'Session'; enableMultipleRegions = $true; enablePartitionMerge = $true}": "", - "@{ enableMultipleWriteLocations = $false; defaultConsistencyLevel = 'Strong'; enableMultipleRegions = $true }": "" + "@{ enableMultipleWriteLocations = $false; defaultConsistencyLevel = 'Strong'; enableMultipleRegions = $true }": "", + "single-session": "", + "single-session-pmerge": "", + "single-strong": "", + "single-session-split": "", + "single-strong-split": "", + "multiregion-strong": "", + "multimaster-multiregion-session": "", + "multimaster-multiregion-session-fi": "", + "multimaster-multiregion-session-split": "" }, "include": [ { @@ -32,12 +41,14 @@ "Session": { "DESIRED_CONSISTENCY": "Session", "ACCOUNT_CONSISTENCY": "Session", - "ArmTemplateParameters": "@{ enableMultipleWriteLocations = $false; defaultConsistencyLevel = 'Session'; enablePartitionMerge = $true }" + "ArmTemplateParameters": "@{ enableMultipleWriteLocations = $false; defaultConsistencyLevel = 'Session'; enablePartitionMerge = $true }", + "AccountSelector": "single-session-pmerge" }, "Strong": { "DESIRED_CONSISTENCY": "Strong", "ACCOUNT_CONSISTENCY": "Strong", - "ArmTemplateParameters": "@{ enableMultipleWriteLocations = $false; defaultConsistencyLevel = 'Strong' }" + "ArmTemplateParameters": "@{ enableMultipleWriteLocations = $false; defaultConsistencyLevel = 'Strong' }", + "AccountSelector": "single-strong" } }, "AdditionalArgs": [ @@ -45,31 +56,87 @@ ], "ProfileFlag": "-Pe2e", "Agent": { - "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" } + "ubuntu": { + "OSVmImage": "env:LINUXVMIMAGE", + "Pool": "env:LINUXPOOL" + } } }, { "DESIRED_CONSISTENCIES": "[\"Session\"]", "ACCOUNT_CONSISTENCY": "Session", "PROTOCOLS": "[\"Tcp\"]", - "ProfileFlag": [ "-Pcfp-split", "-Psplit", "-Pquery", "-Pfast", "-Pdirect", "-Pconsistency-overrides" ], + "ProfileFlag": [ "-Pquery", "-Pfast", "-Pdirect", "-Pconsistency-overrides" ], "ArmTemplateParameters": "@{ enableMultipleWriteLocations = $false; defaultConsistencyLevel = 'Session' }", "AdditionalArgs": [ "-DargLine=\"-Dio.netty.handler.ssl.noOpenSsl=true\"" ], "Agent": { - "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" } - } + "ubuntu": { + "OSVmImage": "env:LINUXVMIMAGE", + "Pool": "env:LINUXPOOL" + } + }, + "AccountSelector": "single-session" }, { - "DESIRED_CONSISTENCIES": [ "[\"Strong\", \"Session\"]", "[\"BoundedStaleness\"]", "[\"ConsistentPrefix\"]" ], + "DESIRED_CONSISTENCIES": "[\"Session\"]", + "ACCOUNT_CONSISTENCY": "Session", + "PROTOCOLS": "[\"Tcp\"]", + "ProfileFlag": [ "-Pcfp-split", "-Psplit" ], + "ArmTemplateParameters": "@{ enableMultipleWriteLocations = $false; defaultConsistencyLevel = 'Session' }", + "AdditionalArgs": [ + "-DargLine=\"-Dio.netty.handler.ssl.noOpenSsl=true\"" + ], + "Agent": { + "ubuntu": { + "OSVmImage": "env:LINUXVMIMAGE", + "Pool": "env:LINUXPOOL" + } + }, + "AccountSelector": "single-session-split" + }, + { + "DESIRED_CONSISTENCIES": [ + "[\"Strong\", \"Session\"]", + "[\"BoundedStaleness\"]", + "[\"ConsistentPrefix\"]" + ], "ACCOUNT_CONSISTENCY": "Strong", "ArmTemplateParameters": "@{ enableMultipleWriteLocations = $false; defaultConsistencyLevel = 'Strong' }", "PROTOCOLS": "[\"Tcp\"]", - "ProfileFlag": [ "-Pcfp-split", "-Psplit", "-Pquery", "-Pfast" ], + "ProfileFlag": [ + "-Pquery", + "-Pfast" + ], "Agent": { - "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" } - } + "ubuntu": { + "OSVmImage": "env:LINUXVMIMAGE", + "Pool": "env:LINUXPOOL" + } + }, + "AccountSelector": "single-strong" + }, + { + "DESIRED_CONSISTENCIES": [ + "[\"Strong\", \"Session\"]", + "[\"BoundedStaleness\"]", + "[\"ConsistentPrefix\"]" + ], + "ACCOUNT_CONSISTENCY": "Strong", + "ArmTemplateParameters": "@{ enableMultipleWriteLocations = $false; defaultConsistencyLevel = 'Strong' }", + "PROTOCOLS": "[\"Tcp\"]", + "ProfileFlag": [ + "-Pcfp-split", + "-Psplit" + ], + "Agent": { + "ubuntu": { + "OSVmImage": "env:LINUXVMIMAGE", + "Pool": "env:LINUXPOOL" + } + }, + "AccountSelector": "single-strong-split" }, { "DESIRED_CONSISTENCY": "BoundedStaleness", @@ -78,28 +145,44 @@ "ProfileFlag": "-Pe2e", "ArmTemplateParameters": "@{ enableMultipleWriteLocations = $false; defaultConsistencyLevel = 'Strong' }", "Agent": { - "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" } - } + "ubuntu": { + "OSVmImage": "env:LINUXVMIMAGE", + "Pool": "env:LINUXPOOL" + } + }, + "AccountSelector": "single-strong" }, { "DESIRED_CONSISTENCIES": "[\"Strong\", \"Session\"]", "ACCOUNT_CONSISTENCY": "Strong", "PROTOCOLS": "[\"Tcp\"]", - "ProfileFlag": [ "-Pdirect" ], + "ProfileFlag": [ + "-Pdirect" + ], "ArmTemplateParameters": "@{ enableMultipleWriteLocations = $false; defaultConsistencyLevel = 'Strong' }", "Agent": { - "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" } - } + "ubuntu": { + "OSVmImage": "env:LINUXVMIMAGE", + "Pool": "env:LINUXPOOL" + } + }, + "AccountSelector": "single-strong" }, { "DESIRED_CONSISTENCIES": "[\"Session\"]", "ACCOUNT_CONSISTENCY": "Session", "PROTOCOLS": "[\"Tcp\"]", - "ProfileFlag": [ "-Plong" ], + "ProfileFlag": [ + "-Plong" + ], "ArmTemplateParameters": "@{ enableMultipleWriteLocations = $false; defaultConsistencyLevel = 'Session'; enablePartitionMerge = $true }", "Agent": { - "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" } - } + "ubuntu": { + "OSVmImage": "env:LINUXVMIMAGE", + "Pool": "env:LINUXPOOL" + } + }, + "AccountSelector": "single-session-pmerge" }, { "DESIRED_CONSISTENCIES": "[\"Session\"]", @@ -111,11 +194,17 @@ } }, "PROTOCOLS": "[\"Tcp\"]", - "ProfileFlag": [ "-Pmulti-master" ], + "ProfileFlag": [ + "-Pmulti-master" + ], "AdditionalArgs": "\"-DCOSMOS.PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN=TRUE\"", "Agent": { - "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" } - } + "ubuntu": { + "OSVmImage": "env:LINUXVMIMAGE", + "Pool": "env:LINUXPOOL" + } + }, + "AccountSelector": "multimaster-multiregion-session" }, { "DESIRED_CONSISTENCIES": "[\"Session\"]", @@ -127,11 +216,17 @@ } }, "PROTOCOLS": "[\"Tcp\"]", - "ProfileFlag": [ "-Pfi-multi-master" ], + "ProfileFlag": [ + "-Pfi-multi-master" + ], "AdditionalArgs": "\"-DCOSMOS.PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN=TRUE\"", "Agent": { - "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" } - } + "ubuntu": { + "OSVmImage": "env:LINUXVMIMAGE", + "Pool": "env:LINUXPOOL" + } + }, + "AccountSelector": "multimaster-multiregion-session-fi" }, { "DESIRED_CONSISTENCIES": "[\"Session\"]", @@ -143,11 +238,17 @@ } }, "PROTOCOLS": "[\"Tcp\"]", - "ProfileFlag": [ "-Pmulti-master" ], + "ProfileFlag": [ + "-Pmulti-master" + ], "AdditionalArgs": "\"-DCOSMOS.PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN=FALSE\"", "Agent": { - "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" } - } + "ubuntu": { + "OSVmImage": "env:LINUXVMIMAGE", + "Pool": "env:LINUXPOOL" + } + }, + "AccountSelector": "multimaster-multiregion-session" }, { "DESIRED_CONSISTENCIES": "[\"Session\"]", @@ -159,11 +260,17 @@ } }, "PROTOCOLS": "[\"Tcp\"]", - "ProfileFlag": [ "-Pfi-multi-master" ], + "ProfileFlag": [ + "-Pfi-multi-master" + ], "AdditionalArgs": "\"-DCOSMOS.PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN=FALSE\"", "Agent": { - "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" } - } + "ubuntu": { + "OSVmImage": "env:LINUXVMIMAGE", + "Pool": "env:LINUXPOOL" + } + }, + "AccountSelector": "multimaster-multiregion-session-fi" }, { "DESIRED_CONSISTENCIES": "[\"Session\"]", @@ -175,10 +282,44 @@ } }, "PROTOCOLS": "[\"Tcp\"]", - "ProfileFlag": [ "-Pcfp-split", "-Psplit", "-Pquery", "-Pflaky-multi-master", "-Pcircuit-breaker-misc-direct", "-Pcircuit-breaker-misc-gateway", "-Pcircuit-breaker-read-all-read-many", "-Pfast", "-Pdirect" ], + "ProfileFlag": [ + "-Pquery", + "-Pflaky-multi-master", + "-Pcircuit-breaker-misc-direct", + "-Pcircuit-breaker-misc-gateway", + "-Pcircuit-breaker-read-all-read-many", + "-Pfast", + "-Pdirect" + ], "Agent": { - "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" } - } + "ubuntu": { + "OSVmImage": "env:LINUXVMIMAGE", + "Pool": "env:LINUXPOOL" + } + }, + "AccountSelector": "multimaster-multiregion-session" + }, + { + "DESIRED_CONSISTENCIES": "[\"Session\"]", + "ACCOUNT_CONSISTENCY": "Session", + "ArmConfig": { + "MultiMaster_MultiRegion": { + "ArmTemplateParameters": "@{ enableMultipleWriteLocations = $true; defaultConsistencyLevel = 'Session'; enableMultipleRegions = $true }", + "PREFERRED_LOCATIONS": "[\"East US 2\"]" + } + }, + "PROTOCOLS": "[\"Tcp\"]", + "ProfileFlag": [ + "-Pcfp-split", + "-Psplit" + ], + "Agent": { + "ubuntu": { + "OSVmImage": "env:LINUXVMIMAGE", + "Pool": "env:LINUXPOOL" + } + }, + "AccountSelector": "multimaster-multiregion-session-split" }, { "DESIRED_CONSISTENCIES": "[\"Strong\"]", @@ -189,10 +330,16 @@ } }, "PROTOCOLS": "[\"Tcp\"]", - "ProfileFlag": [ "-Pmulti-region-strong" ], + "ProfileFlag": [ + "-Pmulti-region-strong" + ], "Agent": { - "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" } - } + "ubuntu": { + "OSVmImage": "env:LINUXVMIMAGE", + "Pool": "env:LINUXPOOL" + } + }, + "AccountSelector": "multiregion-strong" } ] } diff --git a/sdk/cosmos/pipeline/README.md b/sdk/cosmos/pipeline/README.md new file mode 100644 index 000000000000..be62cdd4a703 --- /dev/null +++ b/sdk/cosmos/pipeline/README.md @@ -0,0 +1,102 @@ +# Cosmos live-test fixed accounts + +This directory holds the **single-secret JSON configuration** used to point the Cosmos +Java live tests at fixed, self-owned Cosmos DB accounts (no central EngSys +provisioner). It is the Track A mechanism from the retargeting plan. + +## Files + +| File | Purpose | +| --- | --- | +| `live-test-accounts.schema.json` | JSON schema (draft-07) for the single secret. | +| `live-test-accounts.sample.json` | Example with all logical accounts (placeholder values). | +| `resolve-cosmos-test-account.sh` | Pre-step parser: reads the JSON secret + a selector, exports `ACCOUNT_HOST`/`ACCOUNT_KEY`. | +| `resolve-cosmos-test-account.tests.sh` | Local tests for the parser (no ADO required). | +| `resolve-test-account-steps.yml` | Reusable `PreTestRunSteps` template that runs the parser for a selector. | + +## How it works + +1. A **single ADO variable-group secret** (`sub-config-cosmos-azure-cloud-test-resources`) + holds a JSON document (see schema) mapping a **logical account name** to its + `{ endpoint, key, secondaryKey, ... }`. +2. Each pipeline stage passes a **selector** (the logical name it needs). +3. A **pre-step** runs `resolve-cosmos-test-account.sh`, which validates the JSON, + looks up the selected account, and exports `ACCOUNT_HOST`, `ACCOUNT_KEY` (and + `SECONDARY_ACCOUNT_KEY` when present) for the tests to read via `TestConfigurations`. + It does NOT set `ACCOUNT_CONSISTENCY`/`PREFERRED_LOCATIONS` — those are matrix-controlled + per leg. +4. The account provisioning script (in [`account-provisioning/`](https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/cosmos/pipeline/account-provisioning)) + regenerates the accounts + JSON on every ephemeral-tenant rotation + (`New-CosmosLiveTestAccounts.ps1` creates the accounts and outputs the JSON). The + secret is then updated manually with that JSON, so pipelines pick up refreshed + endpoints/keys automatically. + +All Cosmos live-test matrix legs run on **linux** agents, so the parser is bash + jq. + +## Selectors (from the test matrices) + +`single-session`, `single-session-pmerge`, `single-strong`, `single-session-split`, +`single-strong-split`, `multiregion-strong`, `multimaster-multiregion-session`, +`multimaster-multiregion-session-fi`, `multimaster-multiregion-session-split`, +`multiregion-tc-session`, `gsi-single-session`, `kafka-session`. + +The `*-split` accounts are dedicated to the partition-split-forcing profiles +(`-Pcfp-split`, `-Psplit`), which raise container throughput to force splits. Isolating +them keeps that split churn off the shared query/fast/direct accounts. +The `*-fi` account isolates fault-injection profiles from normal traffic so injected +availability failures are not compounded by unrelated shared-account load. + +## Wiring a stage (tests.yml / kafka.yml) + +Use the reusable steps template in `PreTestRunSteps` and set +`DisableAzureResourceCreation: true` so no per-run provisioning/service-connection deploy +happens: + +```yaml +DisableAzureResourceCreation: true +PreTestRunSteps: + - template: /sdk/cosmos/pipeline/resolve-test-account-steps.yml + parameters: + AccountSelector: multimaster-multiregion-session # literal, or $(AccountSelector) from the matrix +``` + +`ACCOUNT_HOST`/`ACCOUNT_KEY` set by the resolve step are consumed by `TestConfigurations` +via environment variables; no test-code change is required. The key is emitted with the +azure-sdk double-set convention (`_ACCOUNT_KEY` secret to mask it in logs + a plain +`ACCOUNT_KEY` so it reaches the Maven task's env). + +### Per-leg selection (multi-account stages) + +The main `Cosmos_live_test` stage spans many consistency/topology configs, so each leg of +`live-platform-matrix.json` carries an `AccountSelector` field and the stage passes +`AccountSelector: $(AccountSelector)`. Single-account stages pass a literal selector. + +### Currently wired + +- `Cosmos_live_test` (main) — per-leg selector. +- `Cosmos_Live_Test_Http2` — `multimaster-multiregion-session`. + +NOT yet wired (need follow-up): the thin-client stages and GSI (their accounts need +thin-client / GSI-preview enablement that plain account creation doesn't do), Kafka +(entangled with AAD — belongs in the separate AAD pipeline), and Spring (uses its own +test-resources with `AZURE_SPRING_TENANT_ID`). + +### Secret upkeep + +Every account in the secret carries a `secondaryKey` — `AzureKeyCredentialTest` (in the +`-Pfast` legs) calls `TestConfigurations.SECONDARY_MASTER_KEY`, so a missing secondary key +would fall back to the emulator key and fail against a live account. + +## Secret variable + +The JSON lives in the ADO variable-group secret +`sub-config-cosmos-azure-cloud-test-resources` (Cosmos "user administered" variable +group). The resolve steps template reads it by default. Refresh it manually whenever the +accounts are re-provisioned (e.g. after an ephemeral-tenant rotation) by pasting the +regenerated JSON from `New-CosmosLiveTestAccounts.ps1`. + +## Local test + +```bash +bash sdk/cosmos/pipeline/resolve-cosmos-test-account.tests.sh +``` diff --git a/sdk/cosmos/pipeline/account-provisioning/New-CosmosLiveTestAccounts.ps1 b/sdk/cosmos/pipeline/account-provisioning/New-CosmosLiveTestAccounts.ps1 new file mode 100644 index 000000000000..8720e37a60e0 --- /dev/null +++ b/sdk/cosmos/pipeline/account-provisioning/New-CosmosLiveTestAccounts.ps1 @@ -0,0 +1,259 @@ +<# +.SYNOPSIS + (Re)creates the fixed Cosmos DB accounts used by the azure-sdk-for-java Cosmos live + tests and outputs the cosmos-live-test-accounts JSON (endpoints + keys). + +.DESCRIPTION + The Java Cosmos live tests run against fixed, self-owned accounts (Track A of the + live-test retargeting). Because the ephemeral tenant is deleted/recreated roughly + every 90 days, this script is re-run after each rotation to: + 1. Ensure the resource group (default: sdk-ci) exists (created if missing). + 2. Create (idempotently) one Cosmos account per entry in the definition file, + with the requested consistency / multi-write / multi-region / thin-client / + partition-merge configuration. + 3. Read each account's endpoint + primary (and optional secondary) key. + 4. Assemble the versioned cosmos-live-test-accounts JSON and emit it (to stdout, + and to -OutputPath if provided). + + This script does NOT touch Key Vault. Update the cosmos-live-test-accounts secret / + ADO variable manually with the JSON it outputs. + + Uses the Az PowerShell modules (Az.Accounts, Az.Resources, Az.CosmosDB). + +.PARAMETER SubscriptionId + Subscription hosting the resource group and the Cosmos accounts. + +.PARAMETER ResourceGroupName + Resource group for the accounts. Created if it does not exist. Defaults to 'sdk-ci'. + +.PARAMETER Location + Optional override for the primary/write region. When omitted, the primary region comes + from the definition's regionDefaults (single source of truth; matches test-resources.json). + +.PARAMETER SecondaryLocation + Optional override for the secondary region of multi-region accounts. When omitted, it + comes from the definition's regionDefaults.multiRegion. + +.PARAMETER DefinitionPath + Path to the account definition JSON. Defaults to the file next to this script. + +.PARAMETER AccountNamePrefix + Prefix for the globally-unique Cosmos account names. Defaults to 'sdkci'. + +.PARAMETER OutputPath + Optional path to write the assembled JSON to. The JSON is always also written to + stdout. NOTE: the JSON contains account keys - treat any file you write as a secret. + +.EXAMPLE + # Create/refresh accounts and write the JSON to a file, then update the secret manually + ./New-CosmosLiveTestAccounts.ps1 -SubscriptionId -OutputPath ./accounts.json + +.EXAMPLE + # Dry run - creates nothing, prints the assembled JSON with keys stubbed + ./New-CosmosLiveTestAccounts.ps1 -SubscriptionId -WhatIf + +.NOTES + Requires: PowerShell 7+, Az modules, and Contributor on the subscription. + Idempotent: safe to re-run. +#> +[CmdletBinding(SupportsShouldProcess = $true)] +param( + [Parameter(Mandatory = $true)] + [string] $SubscriptionId, + + [string] $ResourceGroupName = 'sdk-ci', + + [string] $Location, + + [string] $SecondaryLocation, + + [string] $DefinitionPath = (Join-Path $PSScriptRoot 'cosmos-live-test-accounts.definition.json'), + + [ValidatePattern('^[a-z0-9]{1,10}$')] + [string] $AccountNamePrefix = 'sdkci', + + [string] $OutputPath +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +function Write-Info($msg) { Write-Host "==> $msg" -ForegroundColor Cyan } + +# --- Prerequisites ----------------------------------------------------------- +foreach ($m in @('Az.Accounts', 'Az.Resources', 'Az.CosmosDB')) { + if (-not (Get-Module -ListAvailable -Name $m)) { + throw "Required module '$m' is not installed. Install with: Install-Module $m -Scope CurrentUser" + } +} + +if (-not (Test-Path $DefinitionPath)) { throw "Definition file not found: $DefinitionPath" } +$definition = Get-Content -Raw -Path $DefinitionPath | ConvertFrom-Json + +# Regions are defined once in the definition's regionDefaults so the provisioned topology +# matches sdk/cosmos/test-resources.json (the ARM template the old flow used). The optional +# -Location / -SecondaryLocation params override the primary/secondary region for ad-hoc runs. +if (-not ($definition.PSObject.Properties.Name -contains 'regionDefaults')) { + throw "Definition '$DefinitionPath' is missing 'regionDefaults' (singleRegion / multiRegion)." +} +$singleRegionList = @($definition.regionDefaults.singleRegion) +$multiRegionList = @($definition.regionDefaults.multiRegion) +if ($singleRegionList.Count -lt 1 -or $multiRegionList.Count -lt 2) { + throw "regionDefaults must provide singleRegion (>=1) and multiRegion (>=2) entries." +} +if ($Location) { + $singleRegionList[0] = $Location + $multiRegionList[0] = $Location +} +if ($SecondaryLocation) { + $multiRegionList[1] = $SecondaryLocation +} +$primaryRegion = $multiRegionList[0] + +Write-Info "Selecting subscription $SubscriptionId" +$null = Set-AzContext -Subscription $SubscriptionId + +# --- Resource group (create if missing) -------------------------------------- +if (-not (Get-AzResourceGroup -Name $ResourceGroupName -ErrorAction SilentlyContinue)) { + if ($PSCmdlet.ShouldProcess($ResourceGroupName, 'Create resource group')) { + Write-Info "Creating resource group $ResourceGroupName in $primaryRegion" + $null = New-AzResourceGroup -Name $ResourceGroupName -Location $primaryRegion + } +} else { + Write-Info "Resource group $ResourceGroupName already exists" +} + +# --- Helper: build the -LocationObject list from a region list -------------- +# First region is the write region (failoverPriority 0); the rest are read regions. +function New-LocationObjects([string[]] $regionList) { + $locations = @() + for ($i = 0; $i -lt $regionList.Count; $i++) { + $locations += New-AzCosmosDBLocationObject -LocationName $regionList[$i] -FailoverPriority $i -IsZoneRedundant $false + } + return ,$locations +} + +# --- Create / update each account, then collect endpoint + keys -------------- +$secret = [ordered]@{ + version = 1 + accounts = [ordered]@{} +} + +foreach ($acct in $definition.accounts) { + $selector = $acct.name + $accountName = ("{0}-{1}" -f $AccountNamePrefix, $selector).ToLower() + if ($accountName.Length -gt 44) { + throw "Generated account name '$accountName' exceeds 44 chars. Shorten AccountNamePrefix or the selector '$selector'." + } + + $multiRegion = [bool]$acct.enableMultipleRegions + $multiWrite = [bool]$acct.enableMultipleWriteLocations + $regionList = if ($multiRegion) { $multiRegionList } else { $singleRegionList } + $locations = New-LocationObjects $regionList + + $capabilities = @() + if ($acct.PSObject.Properties.Name -contains 'capabilities') { + # Any account-level capabilities (e.g. thin-client) come from the definition so we + # never ship a guessed capability name. Leave empty if none are specified. + $capabilities = @($acct.capabilities) + } + + $existing = Get-AzCosmosDBAccount -ResourceGroupName $ResourceGroupName -Name $accountName -ErrorAction SilentlyContinue + if (-not $existing) { + if ($PSCmdlet.ShouldProcess($accountName, "Create Cosmos account [$selector]")) { + Write-Info "Creating Cosmos account '$accountName' (selector=$selector, consistency=$($acct.defaultConsistencyLevel), multiWrite=$multiWrite, multiRegion=$multiRegion)" + $params = @{ + ResourceGroupName = $ResourceGroupName + Name = $accountName + LocationObject = $locations + DefaultConsistencyLevel = $acct.defaultConsistencyLevel + EnableMultipleWriteLocations = $multiWrite + ApiKind = 'GlobalDocumentDB' + } + if ($acct.defaultConsistencyLevel -eq 'BoundedStaleness') { + $params['MaxStalenessIntervalInSeconds'] = 5 + $params['MaxStalenessPrefix'] = 100 + } + if ($acct.PSObject.Properties.Name -contains 'enablePartitionMerge' -and $acct.enablePartitionMerge) { + $params['EnablePartitionMerge'] = $true + } + if ($capabilities.Count -gt 0) { $params['Capabilities'] = $capabilities } + $null = New-AzCosmosDBAccount @params + } + } else { + # Account already exists. Reconcile capabilities: Cosmos capabilities are additive + # and cannot be removed, so we only add any desired capability that is missing + # (e.g. EnableNoSQLVectorSearch). This makes the script idempotent for capability + # changes on already-provisioned accounts. + $existingCaps = @() + if ($existing.Capabilities) { $existingCaps = @($existing.Capabilities | ForEach-Object { $_.Name }) } + $missingCaps = @($capabilities | Where-Object { $existingCaps -notcontains $_ }) + if ($missingCaps.Count -gt 0) { + $mergedCaps = @($existingCaps + $missingCaps | Select-Object -Unique) + if ($PSCmdlet.ShouldProcess($accountName, "Add capabilities [$($missingCaps -join ', ')]")) { + Write-Info "Account '$accountName' exists (selector=$selector); adding missing capabilities: $($missingCaps -join ', ')" + # Capabilities cannot be set via Update-AzCosmosDBAccount, so PATCH the account + # through ARM. Capabilities are additive; send the full merged list. + $resourceId = "/subscriptions/$SubscriptionId/resourceGroups/$ResourceGroupName/providers/Microsoft.DocumentDB/databaseAccounts/$accountName" + $body = @{ + properties = @{ + capabilities = @($mergedCaps | ForEach-Object { @{ name = $_ } }) + } + } | ConvertTo-Json -Depth 6 + $resp = Invoke-AzRestMethod -Method PATCH -Path "$($resourceId)?api-version=2024-11-15" -Payload $body + if ($resp.StatusCode -ge 300) { + throw "Failed to add capabilities to '$accountName' (HTTP $($resp.StatusCode)): $($resp.Content)" + } + } + } else { + Write-Info "Cosmos account '$accountName' already exists (selector=$selector); capabilities up to date" + } + } + + # Read endpoint + keys. Under -WhatIf (dry run) never read real keys — stub them so a + # preview never emits secrets, even for already-provisioned accounts. + if ($WhatIfPreference) { + $endpoint = "https://$accountName.documents.azure.com:443/" + $primary = 'WHATIF_KEY' + $secondary = 'WHATIF_SECONDARY_KEY' + } else { + $account = Get-AzCosmosDBAccount -ResourceGroupName $ResourceGroupName -Name $accountName + $endpoint = $account.DocumentEndpoint + $keys = Get-AzCosmosDBAccountKey -ResourceGroupName $ResourceGroupName -Name $accountName -Type 'Keys' + $primary = $keys.PrimaryMasterKey + $secondary = $keys.SecondaryMasterKey + } + + $entry = [ordered]@{ + endpoint = $endpoint + key = $primary + consistency = $acct.defaultConsistencyLevel + multiWrite = $multiWrite + } + if ($acct.PSObject.Properties.Name -contains 'includeSecondaryKey' -and $acct.includeSecondaryKey) { + $entry['secondaryKey'] = $secondary + } + if ($acct.PSObject.Properties.Name -contains 'thinClient' -and $acct.thinClient) { + $entry['thinClient'] = $true + } + if ($acct.PSObject.Properties.Name -contains 'preferredLocations') { + $entry['preferredLocations'] = [string[]]@($acct.preferredLocations) + } + $entry['regions'] = [string[]]$regionList + + $secret.accounts[$selector] = $entry +} + +# --- Emit the assembled JSON ------------------------------------------------- +$secretJson = $secret | ConvertTo-Json -Depth 8 + +if ($OutputPath) { + if ($PSCmdlet.ShouldProcess($OutputPath, 'Write accounts JSON to file')) { + Set-Content -Path $OutputPath -Value $secretJson -NoNewline + Write-Info "Wrote accounts JSON to '$OutputPath' ($($secret.accounts.Count) accounts). Contains keys - treat as secret." + } +} + +Write-Info "Assembled $($secret.accounts.Count) accounts. Update the cosmos-live-test-accounts secret manually with this JSON." +# Emit the JSON to stdout so it can be captured/redirected. +Write-Output $secretJson diff --git a/sdk/cosmos/pipeline/account-provisioning/README.md b/sdk/cosmos/pipeline/account-provisioning/README.md new file mode 100644 index 000000000000..ecd5e7289c06 --- /dev/null +++ b/sdk/cosmos/pipeline/account-provisioning/README.md @@ -0,0 +1,61 @@ +# Cosmos live-test account provisioning + +`New-CosmosLiveTestAccounts.ps1` provisions the fixed Cosmos DB accounts used by the +**azure-sdk-for-java** Cosmos live tests and outputs the single JSON document +(`cosmos-live-test-accounts`) the pipelines read. + +- **`New-CosmosLiveTestAccounts.ps1`** — creates the resource group (if missing) + the + accounts, and **outputs** the accounts JSON (endpoints + keys). It does not touch Key + Vault; publish the JSON to the secret manually (see below). + +## Why + +The Java Cosmos live tests are moving off the central EngSys on-the-fly provisioner to +**fixed, self-owned accounts** (no service-connection / tenant dependency for the +key-based tests). Because the ephemeral tenant is deleted and recreated roughly every +**90 days**, this script is re-run after each rotation to recreate the accounts and +regenerate the fresh endpoints/keys. + +## Files + +| File | Purpose | +| --- | --- | +| `New-CosmosLiveTestAccounts.ps1` | Creates `sdk-ci` RG (if missing) + accounts; outputs accounts JSON. | +| `cosmos-live-test-accounts.definition.json` | Desired accounts (logical selector + config). | + +The emitted JSON conforms to the schema at +`../live-test-accounts.schema.json`, which the pipeline pre-step +`../resolve-cosmos-test-account.sh` parses. + +## Prerequisites + +- PowerShell 7+ +- Az modules: `Az.Accounts`, `Az.Resources`, `Az.CosmosDB` +- Contributor on the subscription that hosts `sdk-ci` +- Signed in: `Connect-AzAccount -Tenant -Subscription ` + +## Usage + +```powershell +# Create/refresh accounts and write the JSON to a file +# (contains keys - treat as secret, delete after publishing) +./New-CosmosLiveTestAccounts.ps1 -SubscriptionId -OutputPath ./accounts.json + +# Dry run: create nothing, print the assembled JSON with keys stubbed +./New-CosmosLiveTestAccounts.ps1 -SubscriptionId -WhatIf +``` + +Idempotent: existing accounts are left in place and missing capabilities are added; the +JSON is regenerated with current endpoints/keys. + +Then **update the Key Vault secret / ADO variable manually** with the contents of +`accounts.json` (paste the JSON into the `cosmos-live-test-accounts` secret). + +## Rotation runbook + +1. Ephemeral tenant is recreated (~every 90 days). +2. `Connect-AzAccount -Tenant -Subscription `. +3. Run the account script (above) to recreate any missing `sdk-ci` accounts and + regenerate `accounts.json`. +4. Manually update the `cosmos-live-test-accounts` secret with the new JSON. +5. Java pipelines pick up the refreshed values on their next run — no YAML edits. diff --git a/sdk/cosmos/pipeline/account-provisioning/cosmos-live-test-accounts.definition.json b/sdk/cosmos/pipeline/account-provisioning/cosmos-live-test-accounts.definition.json new file mode 100644 index 000000000000..5599595f0446 --- /dev/null +++ b/sdk/cosmos/pipeline/account-provisioning/cosmos-live-test-accounts.definition.json @@ -0,0 +1,158 @@ +{ + "comment": "Desired fixed Cosmos accounts for the Java SDK live tests. Consumed by New-CosmosLiveTestAccounts.ps1 to (re)create accounts in the sdk-ci resource group and to emit the cosmos-live-test-accounts Key Vault secret. Account names here are the logical selectors used by the azure-sdk-for-java tests.yml / kafka.yml stages. accountNamePrefix + name form the actual Cosmos account name (must be globally unique, <=44 chars, lowercase); the region is intentionally NOT encoded in the name. regionDefaults defines the provisioned topology and mirrors sdk/cosmos/test-resources.json (singleRegion: Central US; multiRegion: Central US write + East US 2), so preferredLocations values match a region that actually exists on the multi-region accounts. NOTE: thinClient=true only tags the emitted secret; enabling thin-client on the account itself requires the correct account 'capabilities' entry / federation config once confirmed - add it to a per-account \"capabilities\": [\"\"] array.", + "accounts": [ + { + "name": "single-session", + "defaultConsistencyLevel": "Session", + "enableMultipleWriteLocations": false, + "enableMultipleRegions": false, + "enablePartitionMerge": false, + "thinClient": false, + "includeSecondaryKey": true, + "capabilities": [ + "EnableNoSQLVectorSearch" + ] + }, + { + "name": "single-session-pmerge", + "defaultConsistencyLevel": "Session", + "enableMultipleWriteLocations": false, + "enableMultipleRegions": false, + "enablePartitionMerge": true, + "thinClient": false, + "includeSecondaryKey": true, + "capabilities": [ + "EnableNoSQLVectorSearch" + ] + }, + { + "name": "single-strong", + "defaultConsistencyLevel": "Strong", + "enableMultipleWriteLocations": false, + "enableMultipleRegions": false, + "enablePartitionMerge": false, + "thinClient": false, + "includeSecondaryKey": true, + "capabilities": [ + "EnableNoSQLVectorSearch" + ] + }, + { + "name": "single-session-split", + "defaultConsistencyLevel": "Session", + "enableMultipleWriteLocations": false, + "enableMultipleRegions": false, + "enablePartitionMerge": false, + "thinClient": false, + "includeSecondaryKey": true, + "capabilities": [ + "EnableNoSQLVectorSearch" + ] + }, + { + "name": "single-strong-split", + "defaultConsistencyLevel": "Strong", + "enableMultipleWriteLocations": false, + "enableMultipleRegions": false, + "enablePartitionMerge": false, + "thinClient": false, + "includeSecondaryKey": true, + "capabilities": [ + "EnableNoSQLVectorSearch" + ] + }, + { + "name": "multiregion-strong", + "defaultConsistencyLevel": "Strong", + "enableMultipleWriteLocations": false, + "enableMultipleRegions": true, + "enablePartitionMerge": false, + "thinClient": false, + "includeSecondaryKey": true, + "capabilities": [ + "EnableNoSQLVectorSearch" + ] + }, + { + "name": "multimaster-multiregion-session", + "defaultConsistencyLevel": "Session", + "enableMultipleWriteLocations": true, + "enableMultipleRegions": true, + "enablePartitionMerge": false, + "thinClient": false, + "preferredLocations": [ + "East US 2" + ], + "includeSecondaryKey": true, + "capabilities": [ + "EnableNoSQLVectorSearch" + ] + }, + { + "name": "multimaster-multiregion-session-fi", + "defaultConsistencyLevel": "Session", + "enableMultipleWriteLocations": true, + "enableMultipleRegions": true, + "enablePartitionMerge": false, + "thinClient": false, + "preferredLocations": [ + "East US 2" + ], + "includeSecondaryKey": true, + "capabilities": [ + "EnableNoSQLVectorSearch" + ] + }, + { + "name": "multimaster-multiregion-session-split", + "defaultConsistencyLevel": "Session", + "enableMultipleWriteLocations": true, + "enableMultipleRegions": true, + "enablePartitionMerge": false, + "thinClient": false, + "preferredLocations": [ + "East US 2" + ], + "includeSecondaryKey": true, + "capabilities": [ + "EnableNoSQLVectorSearch" + ] + }, + { + "name": "multiregion-tc-session", + "defaultConsistencyLevel": "Session", + "enableMultipleWriteLocations": false, + "enableMultipleRegions": true, + "enablePartitionMerge": false, + "thinClient": true, + "includeSecondaryKey": true + }, + { + "name": "gsi-single-session", + "defaultConsistencyLevel": "Session", + "enableMultipleWriteLocations": false, + "enableMultipleRegions": false, + "enablePartitionMerge": false, + "thinClient": false, + "includeSecondaryKey": true + }, + { + "name": "kafka-session", + "defaultConsistencyLevel": "Session", + "enableMultipleWriteLocations": false, + "enableMultipleRegions": false, + "enablePartitionMerge": false, + "thinClient": false, + "includeSecondaryKey": true + } + ], + "regionDefaults": { + "singleRegion": [ + "Central US" + ], + "multiRegion": [ + "Central US", + "East US 2" + ] + } +} diff --git a/sdk/cosmos/pipeline/cspell.json b/sdk/cosmos/pipeline/cspell.json new file mode 100644 index 000000000000..260efd9bc6f3 --- /dev/null +++ b/sdk/cosmos/pipeline/cspell.json @@ -0,0 +1,13 @@ +{ + "version": "0.2", + "words": [ + "esac", + "issecret", + "Pcfp", + "pmerge", + "Pfast", + "Psplit", + "sdkci", + "WHATIF" + ] +} diff --git a/sdk/cosmos/pipeline/live-test-accounts.sample.json b/sdk/cosmos/pipeline/live-test-accounts.sample.json new file mode 100644 index 000000000000..b2d2084ae16b --- /dev/null +++ b/sdk/cosmos/pipeline/live-test-accounts.sample.json @@ -0,0 +1,140 @@ +{ + "version": 1, + "accounts": { + "single-session": { + "endpoint": "https://REPLACE-single-session.documents.azure.com:443/", + "key": "REPLACE_KEY", + "regions": [ + "Central US" + ], + "consistency": "Session", + "multiWrite": false, + "secondaryKey": "REPLACE_SECONDARY_KEY" + }, + "single-session-pmerge": { + "endpoint": "https://REPLACE-single-session-pmerge.documents.azure.com:443/", + "key": "REPLACE_KEY", + "regions": [ + "Central US" + ], + "consistency": "Session", + "multiWrite": false, + "secondaryKey": "REPLACE_SECONDARY_KEY" + }, + "single-strong": { + "endpoint": "https://REPLACE-single-strong.documents.azure.com:443/", + "key": "REPLACE_KEY", + "regions": [ + "Central US" + ], + "consistency": "Strong", + "multiWrite": false, + "secondaryKey": "REPLACE_SECONDARY_KEY" + }, + "single-session-split": { + "endpoint": "https://REPLACE-single-session-split.documents.azure.com:443/", + "key": "REPLACE_KEY", + "regions": [ + "Central US" + ], + "consistency": "Session", + "multiWrite": false, + "secondaryKey": "REPLACE_SECONDARY_KEY" + }, + "single-strong-split": { + "endpoint": "https://REPLACE-single-strong-split.documents.azure.com:443/", + "key": "REPLACE_KEY", + "regions": [ + "Central US" + ], + "consistency": "Strong", + "multiWrite": false, + "secondaryKey": "REPLACE_SECONDARY_KEY" + }, + "multiregion-strong": { + "endpoint": "https://REPLACE-multiregion-strong.documents.azure.com:443/", + "key": "REPLACE_KEY", + "regions": [ + "Central US", + "East US 2" + ], + "consistency": "Strong", + "multiWrite": false, + "secondaryKey": "REPLACE_SECONDARY_KEY" + }, + "multimaster-multiregion-session": { + "endpoint": "https://REPLACE-multimaster-multiregion-session.documents.azure.com:443/", + "key": "REPLACE_KEY", + "regions": [ + "Central US", + "East US 2" + ], + "preferredLocations": [ + "East US 2" + ], + "consistency": "Session", + "multiWrite": true, + "secondaryKey": "REPLACE_SECONDARY_KEY" + }, + "multimaster-multiregion-session-fi": { + "endpoint": "https://REPLACE-multimaster-multiregion-session-fi.documents.azure.com:443/", + "key": "REPLACE_KEY", + "regions": [ + "Central US", + "East US 2" + ], + "preferredLocations": [ + "East US 2" + ], + "consistency": "Session", + "multiWrite": true, + "secondaryKey": "REPLACE_SECONDARY_KEY" + }, + "multimaster-multiregion-session-split": { + "endpoint": "https://REPLACE-multimaster-multiregion-session-split.documents.azure.com:443/", + "key": "REPLACE_KEY", + "regions": [ + "Central US", + "East US 2" + ], + "preferredLocations": [ + "East US 2" + ], + "consistency": "Session", + "multiWrite": true, + "secondaryKey": "REPLACE_SECONDARY_KEY" + }, + "multiregion-tc-session": { + "endpoint": "https://REPLACE-multiregion-tc-session.documents.azure.com:443/", + "key": "REPLACE_KEY", + "regions": [ + "Central US", + "East US 2" + ], + "consistency": "Session", + "multiWrite": false, + "thinClient": true, + "secondaryKey": "REPLACE_SECONDARY_KEY" + }, + "gsi-single-session": { + "endpoint": "https://REPLACE-gsi-single-session.documents.azure.com:443/", + "key": "REPLACE_KEY", + "regions": [ + "Central US" + ], + "consistency": "Session", + "multiWrite": false, + "secondaryKey": "REPLACE_SECONDARY_KEY" + }, + "kafka-session": { + "endpoint": "https://REPLACE-kafka-session.documents.azure.com:443/", + "key": "REPLACE_KEY", + "secondaryKey": "REPLACE_SECONDARY_KEY", + "regions": [ + "Central US" + ], + "consistency": "Session", + "multiWrite": false + } + } +} diff --git a/sdk/cosmos/pipeline/live-test-accounts.schema.json b/sdk/cosmos/pipeline/live-test-accounts.schema.json new file mode 100644 index 000000000000..0ffaf67aff56 --- /dev/null +++ b/sdk/cosmos/pipeline/live-test-accounts.schema.json @@ -0,0 +1,68 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://azure.github.io/azure-sdk-for-java/cosmos/live-test-accounts.schema.json", + "title": "Cosmos live-test fixed accounts config", + "description": "Single-secret JSON describing the fixed Cosmos DB accounts used by the Java live tests. Stored as one Key Vault secret / ADO variable (cosmos-live-test-accounts) and refreshed by the account-creation script on each ephemeral-tenant rotation. Each pipeline stage selects one account by its logical name.", + "type": "object", + "required": ["version", "accounts"], + "additionalProperties": false, + "properties": { + "version": { + "type": "integer", + "description": "Schema version. The parser rejects versions it does not understand.", + "enum": [1] + }, + "accounts": { + "type": "object", + "description": "Map of logical account selector -> account descriptor.", + "minProperties": 1, + "additionalProperties": { "$ref": "#/definitions/account" } + } + }, + "definitions": { + "account": { + "type": "object", + "required": ["endpoint", "key"], + "additionalProperties": false, + "properties": { + "endpoint": { + "type": "string", + "description": "documentEndpoint of the Cosmos account (https://.documents.azure.com:443/).", + "pattern": "^https://" + }, + "key": { + "type": "string", + "description": "Primary master key. Treated as a secret by the parser.", + "minLength": 1 + }, + "secondaryKey": { + "type": "string", + "description": "Optional secondary master key (exported as SECONDARY_ACCOUNT_KEY)." + }, + "regions": { + "type": "array", + "description": "Regions the account is provisioned in (informational / documentation).", + "items": { "type": "string" } + }, + "preferredLocations": { + "type": "array", + "description": "Optional preferred locations exported as PREFERRED_LOCATIONS.", + "items": { "type": "string" } + }, + "consistency": { + "type": "string", + "description": "Account default consistency level (exported as ACCOUNT_CONSISTENCY).", + "enum": ["Strong", "BoundedStaleness", "Session", "ConsistentPrefix", "Eventual"] + }, + "multiWrite": { + "type": "boolean", + "description": "True if the account has multiple write locations (multi-master)." + }, + "thinClient": { + "type": "boolean", + "description": "True if the account is thin-client enabled." + } + } + } + } +} diff --git a/sdk/cosmos/pipeline/resolve-cosmos-test-account.sh b/sdk/cosmos/pipeline/resolve-cosmos-test-account.sh new file mode 100755 index 000000000000..6d46493e9f77 --- /dev/null +++ b/sdk/cosmos/pipeline/resolve-cosmos-test-account.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# +# Resolves a single Cosmos live-test account from the one JSON secret +# (cosmos-live-test-accounts) and exports ACCOUNT_HOST / ACCOUNT_KEY (and optional +# SECONDARY_ACCOUNT_KEY / PREFERRED_LOCATIONS / ACCOUNT_CONSISTENCY) for the tests. +# +# All Cosmos live-test matrix legs run on linux agents, so this is bash + jq. +# +# Inputs (environment variables): +# COSMOS_TEST_ACCOUNTS_JSON Raw JSON matching live-test-accounts.schema.json +# (the value of the cosmos-live-test-accounts secret). +# COSMOS_ACCOUNT_SELECTOR Logical account name to select (e.g. multimaster-multiregion-session). +# COSMOS_ACCOUNTS_LOCAL Optional. When "true", prints KEY=VALUE to stdout instead of +# emitting Azure DevOps ##vso logging commands (used for local tests). +# +# Exit codes: 0 success; non-zero on any validation failure. +set -euo pipefail + +fail() { echo "ERROR: $*" >&2; exit 1; } + +command -v jq >/dev/null 2>&1 || fail "jq is required but not found on PATH." + +json="${COSMOS_TEST_ACCOUNTS_JSON:-}" +selector="${COSMOS_ACCOUNT_SELECTOR:-}" +local_mode="${COSMOS_ACCOUNTS_LOCAL:-false}" + +[ -n "$json" ] || fail "COSMOS_TEST_ACCOUNTS_JSON is empty. Wire the cosmos-live-test-accounts secret to this variable." +[ -n "$selector" ] || fail "COSMOS_ACCOUNT_SELECTOR is empty. Set it to a logical account name." + +echo "$json" | jq empty 2>/dev/null || fail "COSMOS_TEST_ACCOUNTS_JSON is not valid JSON." + +version="$(echo "$json" | jq -r '.version // empty')" +[ "$version" = "1" ] || fail "Unsupported or missing schema version '$version' (parser supports: 1)." + +if [ "$(echo "$json" | jq --arg s "$selector" 'has("accounts") and (.accounts | has($s))')" != "true" ]; then + available="$(echo "$json" | jq -r '.accounts | keys | join(", ")' 2>/dev/null || echo '')" + fail "Account selector '$selector' not found. Available: ${available}" +fi + +acct="$(echo "$json" | jq -c --arg s "$selector" '.accounts[$s]')" + +endpoint="$(echo "$acct" | jq -r '.endpoint // empty')" +key="$(echo "$acct" | jq -r '.key // empty')" +secondary_key="$(echo "$acct" | jq -r '.secondaryKey // empty')" +consistency="$(echo "$acct" | jq -r '.consistency // empty')" +preferred="$(echo "$acct" | jq -r '(.preferredLocations // []) | join(",")')" + +[ -n "$endpoint" ] || fail "Account '$selector' is missing required 'endpoint'." +[ -n "$key" ] || fail "Account '$selector' is missing required 'key'." +case "$endpoint" in + https://*) : ;; + *) fail "Account '$selector' endpoint must start with https:// (got '$endpoint')." ;; +esac + +emit_public() { # name value + if [ "$local_mode" = "true" ]; then + echo "$1=$2" + else + echo "##vso[task.setvariable variable=$1;issecret=false]$2" + fi +} +# Emit a secret using the azure-sdk double-set convention: register the literal value as +# a secret (variable _NAME) so the log scrubber masks it everywhere, AND set a plain +# variable NAME so it still auto-exports as an environment variable to the Maven test +# task. Marking a variable issecret=true alone would prevent env propagation. +emit_secret() { # name value + if [ "$local_mode" = "true" ]; then + echo "$1=$2" + else + echo "##vso[task.setvariable variable=_$1;issecret=true]$2" + echo "##vso[task.setvariable variable=$1;issecret=false]$2" + fi +} + +# Only ACCOUNT_HOST / ACCOUNT_KEY (+ optional SECONDARY_ACCOUNT_KEY) are emitted. +# ACCOUNT_CONSISTENCY and PREFERRED_LOCATIONS are intentionally NOT set here: they are +# controlled per matrix leg (DESIRED_CONSISTENCIES / ACCOUNT_CONSISTENCY / PREFERRED_LOCATIONS) +# and the fixed accounts provisioned to match, so emitting them here would clobber +# the matrix values. +emit_public ACCOUNT_HOST "$endpoint" +emit_secret ACCOUNT_KEY "$key" +[ -n "$secondary_key" ] && emit_secret SECONDARY_ACCOUNT_KEY "$secondary_key" + +# Masked, secret-free summary for logs. +echo "Resolved Cosmos test account '$selector': endpoint=$endpoint consistency=${consistency:-} preferredLocations=${preferred:-} key=***" >&2 diff --git a/sdk/cosmos/pipeline/resolve-cosmos-test-account.tests.sh b/sdk/cosmos/pipeline/resolve-cosmos-test-account.tests.sh new file mode 100755 index 000000000000..4fae6ab599e2 --- /dev/null +++ b/sdk/cosmos/pipeline/resolve-cosmos-test-account.tests.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# +# Local tests for resolve-cosmos-test-account.sh (no ADO required). +# Run: bash sdk/cosmos/pipeline/resolve-cosmos-test-account.tests.sh +set -uo pipefail + +HERE="$(cd "$(dirname "$0")" && pwd)" +SCRIPT="$HERE/resolve-cosmos-test-account.sh" +SAMPLE="$HERE/live-test-accounts.sample.json" + +pass=0; fail=0 +ok() { echo " PASS: $1"; pass=$((pass+1)); } +no() { echo " FAIL: $1"; fail=$((fail+1)); } + +run() { # selector json -> sets OUT, RC (local mode) + OUT="$(COSMOS_ACCOUNTS_LOCAL=true COSMOS_ACCOUNT_SELECTOR="$1" COSMOS_TEST_ACCOUNTS_JSON="$2" bash "$SCRIPT" 2>/dev/null)" + RC=$? +} + +SAMPLE_JSON="$(cat "$SAMPLE")" + +echo "Test 1: resolves a valid selector and exports host+key (no consistency/preferred)" +run "multimaster-multiregion-session" "$SAMPLE_JSON" +if [ $RC -eq 0 ] && echo "$OUT" | grep -q "^ACCOUNT_HOST=https://REPLACE-multimaster-multiregion-session" \ + && echo "$OUT" | grep -q "^ACCOUNT_KEY=REPLACE_KEY" \ + && ! echo "$OUT" | grep -q "^ACCOUNT_CONSISTENCY=" \ + && ! echo "$OUT" | grep -q "^PREFERRED_LOCATIONS="; then ok "resolved host+key only"; else no "resolve valid selector (rc=$RC): $OUT"; fi + +echo "Test 2: exports SECONDARY_ACCOUNT_KEY when present" +run "kafka-session" "$SAMPLE_JSON" +if [ $RC -eq 0 ] && echo "$OUT" | grep -q "^SECONDARY_ACCOUNT_KEY=REPLACE_SECONDARY_KEY"; then ok "secondary key exported"; else no "secondary key (rc=$RC): $OUT"; fi + +echo "Test 3: unknown selector fails with non-zero" +run "does-not-exist" "$SAMPLE_JSON" +if [ $RC -ne 0 ]; then ok "unknown selector rejected"; else no "unknown selector should fail: $OUT"; fi + +echo "Test 4: invalid JSON fails" +run "single-session" "{not json" +if [ $RC -ne 0 ]; then ok "invalid json rejected"; else no "invalid json should fail"; fi + +echo "Test 5: unsupported version fails" +run "single-session" '{"version":99,"accounts":{"single-session":{"endpoint":"https://x","key":"k"}}}' +if [ $RC -ne 0 ]; then ok "bad version rejected"; else no "bad version should fail"; fi + +echo "Test 6: missing key fails" +run "x" '{"version":1,"accounts":{"x":{"endpoint":"https://x"}}}' +if [ $RC -ne 0 ]; then ok "missing key rejected"; else no "missing key should fail"; fi + +echo "Test 7: non-https endpoint fails" +run "x" '{"version":1,"accounts":{"x":{"endpoint":"http://x","key":"k"}}}' +if [ $RC -ne 0 ]; then ok "non-https rejected"; else no "non-https should fail"; fi + +echo "Test 8: empty selector fails" +OUT="$(COSMOS_ACCOUNTS_LOCAL=true COSMOS_ACCOUNT_SELECTOR="" COSMOS_TEST_ACCOUNTS_JSON="$SAMPLE_JSON" bash "$SCRIPT" 2>/dev/null)"; RC=$? +if [ $RC -ne 0 ]; then ok "empty selector rejected"; else no "empty selector should fail"; fi + +echo "" +echo "Results: $pass passed, $fail failed" +[ $fail -eq 0 ] diff --git a/sdk/cosmos/pipeline/resolve-test-account-steps.yml b/sdk/cosmos/pipeline/resolve-test-account-steps.yml new file mode 100644 index 000000000000..1fade3956749 --- /dev/null +++ b/sdk/cosmos/pipeline/resolve-test-account-steps.yml @@ -0,0 +1,28 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# +# Reusable pre-test step that resolves one fixed Cosmos live-test account from the single +# JSON secret and exports ACCOUNT_HOST / ACCOUNT_KEY (+ optional SECONDARY_ACCOUNT_KEY) +# for the test run. Drop this into a stage's PreTestRunSteps. +# +# The account JSON lives in the ADO variable group secret referenced by AccountsJsonVar +# (default: the Cosmos user-administered variable sub-config-cosmos-azure-cloud-test-resources). +# +# AccountSelector selects which logical account to use. It may be a literal (for +# single-account stages) or a macro like $(AccountSelector) sourced from the matrix leg +# (for the main multi-account stage). +parameters: + - name: AccountSelector + type: string + - name: AccountsJsonVar + type: string + default: 'sub-config-cosmos-azure-cloud-test-resources' + +steps: + - task: Bash@3 + displayName: 'Resolve fixed Cosmos test account (${{ parameters.AccountSelector }})' + inputs: + filePath: $(Build.SourcesDirectory)/sdk/cosmos/pipeline/resolve-cosmos-test-account.sh + env: + COSMOS_TEST_ACCOUNTS_JSON: $(${{ parameters.AccountsJsonVar }}) + COSMOS_ACCOUNT_SELECTOR: ${{ parameters.AccountSelector }} diff --git a/sdk/cosmos/tests.yml b/sdk/cosmos/tests.yml index 1fee90f5c46b..dda2f82ce92d 100644 --- a/sdk/cosmos/tests.yml +++ b/sdk/cosmos/tests.yml @@ -9,6 +9,15 @@ extends: CloudConfig: Public: ServiceConnection: azure-sdk-tests-cosmos + # Cosmos live tests run against fixed, self-owned accounts (RG sdk-ci) rather than + # provisioning an account per run. The resolve pre-step reads the single JSON + # secret (sub-config-cosmos-azure-cloud-test-resources) and selects the account + # for each matrix leg via $(AccountSelector), exporting ACCOUNT_HOST/ACCOUNT_KEY. + DisableAzureResourceCreation: true + PreTestRunSteps: + - template: /sdk/cosmos/pipeline/resolve-test-account-steps.yml + parameters: + AccountSelector: $(AccountSelector) MatrixConfigs: - Name: Cosmos_live_test Path: sdk/cosmos/live-platform-matrix.json @@ -41,6 +50,12 @@ extends: CloudConfig: Public: ServiceConnection: azure-sdk-tests-cosmos + # Fixed self-owned account (RG sdk-ci); no per-run provisioning. + DisableAzureResourceCreation: true + PreTestRunSteps: + - template: /sdk/cosmos/pipeline/resolve-test-account-steps.yml + parameters: + AccountSelector: multimaster-multiregion-session MatrixConfigs: - Name: Cosmos_live_test_http2 Path: sdk/cosmos/live-http2-platform-matrix.json @@ -65,7 +80,7 @@ extends: TestResultsFiles: '**/junitreports/TEST-*.xml' AdditionalVariables: - name: AdditionalArgs - value: '-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -DCOSMOS.HTTP2_ENABLED=true' + value: '-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -DCOSMOS.HTTP2_ENABLED=true -DCOSMOS.THINCLIENT_ENABLED=false' - template: /eng/pipelines/templates/stages/archetype-sdk-tests-isolated.yml parameters: @@ -171,7 +186,7 @@ extends: # with fallback to Gateway V1, rather than a hard opt-in. HTTP2_ENABLED stays true because # thin-client routing requires HTTP/2 to be effectively enabled. - name: AdditionalArgs - value: '-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -DACCOUNT_HOST=$(thinclient-test-endpoint) -DACCOUNT_KEY=$(thinclient-test-key) -DCOSMOS.HTTP2_ENABLED=true' + value: '-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -DACCOUNT_HOST=$(thinclient-test-endpoint) -DACCOUNT_KEY=$(thinclient-test-key) -DCOSMOS.HTTP2_ENABLED=true -DCOSMOS.COLLECTION_READINESS_MAX_WAIT_SECONDS=300' - template: /eng/pipelines/templates/stages/archetype-sdk-tests-isolated.yml parameters: @@ -383,4 +398,3 @@ extends: AdditionalVariables: - name: AdditionalArgs value: '-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true' -