diff --git a/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java b/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java index a22be5989f..c6f93d2215 100644 --- a/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java +++ b/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java @@ -328,8 +328,11 @@ public class IntegrationTestMojo extends AbstractSurefireMojo { * Failed first will run tests that failed on previous run first, as well as new tests for this run. *
*
- * Balanced is only relevant with parallel=classes, and will try to optimize the run-order of the tests reducing the - * overall execution time. Initially a statistics file is created and every next test run will reorder classes. + * Balanced will try to optimize the run-order of the tests reducing the overall execution time by distributing the + * classes across the concurrent test consumers based on their recorded run time. The number of consumers is + * {@code forkCount} multiplied by {@code threadCount} (when {@code parallel} is used), so balanced is effective + * with {@code forkCount > 1} even without {@code parallel=classes}. Initially a statistics file is created and + * every next test run will reorder classes. *
*
* Note that the statistics are stored in a file named .surefire-XXXXXXXXX beside pom.xml and diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java index 676e9cdc6c..0ff19f15d4 100644 --- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java +++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java @@ -1825,6 +1825,10 @@ private ProviderConfiguration createProviderConfiguration(RunOrderParameters run getProperties().setProperty(ProviderParameterNames.STACK_TRACE_MAX_FRAMES, String.valueOf(stackTraceMaxFrames)); + getProperties() + .setProperty( + ProviderParameterNames.FORKCOUNT_PROP, Integer.toString(Math.max(getEffectiveForkCount(), 1))); + Map providerProperties = toStringProperties(getProperties()); return new ProviderConfiguration( diff --git a/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefireMojo.java b/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefireMojo.java index 33f22a9792..42da65ca70 100644 --- a/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefireMojo.java +++ b/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefireMojo.java @@ -308,8 +308,11 @@ public class SurefireMojo extends AbstractSurefireMojo implements SurefireReport * Failed first will run tests that failed on previous run first, as well as new tests for this run. *
*
- * Balanced is only relevant with parallel=classes, and will try to optimize the run-order of the tests reducing the - * overall execution time. Initially a statistics file is created and every next test run will reorder classes. + * Balanced will try to optimize the run-order of the tests reducing the overall execution time by distributing the + * classes across the concurrent test consumers based on their recorded run time. The number of consumers is + * {@code forkCount} multiplied by {@code threadCount} (when {@code parallel} is used), so balanced is effective + * with {@code forkCount > 1} even without {@code parallel=classes}. Initially a statistics file is created and + * every next test run will reorder classes. *
*
* Note that the statistics are stored in a file named .surefire-XXXXXXXXX beside pom.xml and diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/api/booter/BaseProviderFactory.java b/surefire-api/src/main/java/org/apache/maven/surefire/api/booter/BaseProviderFactory.java index 9658ee2b59..0c1245a2c9 100644 --- a/surefire-api/src/main/java/org/apache/maven/surefire/api/booter/BaseProviderFactory.java +++ b/surefire-api/src/main/java/org/apache/maven/surefire/api/booter/BaseProviderFactory.java @@ -105,9 +105,14 @@ private int getThreadCount() { return threadcount == null ? 1 : Math.max(Integer.parseInt(threadcount), 1); } + private int getForkCount() { + final String forkcount = providerProperties.get(ProviderParameterNames.FORKCOUNT_PROP); + return forkcount == null ? 1 : Math.max(Integer.parseInt(forkcount), 1); + } + @Override public RunOrderCalculator getRunOrderCalculator() { - return new DefaultRunOrderCalculator(runOrderParameters, getThreadCount()); + return new DefaultRunOrderCalculator(runOrderParameters, getThreadCount() * getForkCount()); } public void setReporterFactory(ReporterFactory reporterFactory) { diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/api/booter/ProviderParameterNames.java b/surefire-api/src/main/java/org/apache/maven/surefire/api/booter/ProviderParameterNames.java index 9220438d52..7115b0e075 100644 --- a/surefire-api/src/main/java/org/apache/maven/surefire/api/booter/ProviderParameterNames.java +++ b/surefire-api/src/main/java/org/apache/maven/surefire/api/booter/ProviderParameterNames.java @@ -32,6 +32,13 @@ public class ProviderParameterNames { public static final String THREADCOUNT_PROP = "threadcount"; + /** + * Number of forked JVMs executing tests concurrently, used together with {@link #THREADCOUNT_PROP} to size the + * distribution of the {@code balanced} run order. + * @since 3.6.0 + */ + public static final String FORKCOUNT_PROP = "forkcount"; + public static final String PARALLEL_PROP = "parallel"; public static final String THREADCOUNTSUITES_PROP = "threadcountsuites"; diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java b/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java index 5286e603b8..1d2f1c85ca 100644 --- a/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java +++ b/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java @@ -40,13 +40,13 @@ public class DefaultRunOrderCalculator implements RunOrderCalculator { private final RunOrderParameters runOrderParameters; - private final int threadCount; + private final int distributedParallelism; private final Random random; - public DefaultRunOrderCalculator(RunOrderParameters runOrderParameters, int threadCount) { + public DefaultRunOrderCalculator(RunOrderParameters runOrderParameters, int distributedParallelism) { this.runOrderParameters = runOrderParameters; - this.threadCount = threadCount; + this.distributedParallelism = distributedParallelism; this.runOrder = runOrderParameters.getRunOrder(); this.sortOrder = this.runOrder.length > 0 ? getSortOrderComparator(this.runOrder[0]) : null; Long runOrderRandomSeed = runOrderParameters.getRunOrderRandomSeed(); @@ -77,7 +77,7 @@ private void orderTestClasses(List> testClasses, RunOrder runOrder) { } else if (RunOrder.BALANCED.equals(runOrder)) { RunEntryStatisticsMap stat = RunEntryStatisticsMap.fromFile(runOrderParameters.getRunStatisticsFile()); - List> prioritized = stat.getPrioritizedTestsClassRunTime(testClasses, threadCount); + List> prioritized = stat.getPrioritizedTestsClassRunTime(testClasses, distributedParallelism); testClasses.clear(); testClasses.addAll(prioritized); diff --git a/surefire-api/src/test/java/org/apache/maven/surefire/api/booter/BaseProviderFactoryTest.java b/surefire-api/src/test/java/org/apache/maven/surefire/api/booter/BaseProviderFactoryTest.java new file mode 100644 index 0000000000..abe38f187d --- /dev/null +++ b/surefire-api/src/test/java/org/apache/maven/surefire/api/booter/BaseProviderFactoryTest.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.maven.surefire.api.booter; + +import java.io.File; +import java.lang.reflect.Field; +import java.util.HashMap; +import java.util.Map; + +import org.apache.maven.surefire.api.testset.RunOrderParameters; +import org.apache.maven.surefire.api.util.DefaultRunOrderCalculator; +import org.apache.maven.surefire.api.util.RunOrder; +import org.apache.maven.surefire.api.util.RunOrderCalculator; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Verifies that the {@code balanced} run order distribution width is derived from both the parallel + * {@code threadcount} and the {@code forkcount}. + */ +public class BaseProviderFactoryTest { + + @Test + public void distributionWidthDefaultsToForkCountWhenNoThreadCount() { + assertThat(distributedParallelism(properties(null, "3"))).isEqualTo(3); + } + + @Test + public void distributionWidthEqualsThreadCountWhenSingleFork() { + assertThat(distributedParallelism(properties("4", "1"))).isEqualTo(4); + } + + @Test + public void distributionWidthMultipliesThreadCountAndForkCount() { + assertThat(distributedParallelism(properties("4", "3"))).isEqualTo(12); + } + + @Test + public void distributionWidthDefaultsToOneWhenNoPropertiesSet() { + assertThat(distributedParallelism(properties(null, null))).isEqualTo(1); + } + + @Test + public void distributionWidthClampsZeroForkCountToOne() { + assertThat(distributedParallelism(properties("4", "0"))).isEqualTo(4); + } + + private static Map properties(String threadCount, String forkCount) { + Map providerProperties = new HashMap<>(); + if (threadCount != null) { + providerProperties.put(ProviderParameterNames.THREADCOUNT_PROP, threadCount); + } + if (forkCount != null) { + providerProperties.put(ProviderParameterNames.FORKCOUNT_PROP, forkCount); + } + return providerProperties; + } + + private static int distributedParallelism(Map providerProperties) { + BaseProviderFactory factory = new BaseProviderFactory(true); + factory.setProviderProperties(providerProperties); + factory.setRunOrderParameters(new RunOrderParameters(new RunOrder[] {RunOrder.BALANCED}, new File("."))); + RunOrderCalculator calculator = factory.getRunOrderCalculator(); + assertThat(calculator).isInstanceOf(DefaultRunOrderCalculator.class); + return getInternalState(calculator, "distributedParallelism"); + } + + private static int getInternalState(Object target, String fieldName) { + try { + Field field = target.getClass().getDeclaredField(fieldName); + field.setAccessible(true); + return field.getInt(target); + } catch (ReflectiveOperationException e) { + throw new RuntimeException(e); + } + } +} diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/RunOrderIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/RunOrderIT.java index 3a4cfb8fe0..6f0a72a3e9 100644 --- a/surefire-its/src/test/java/org/apache/maven/surefire/its/RunOrderIT.java +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/RunOrderIT.java @@ -88,6 +88,16 @@ public void testReverseAlphabeticalJUnit4() throws VerificationException { assertTestnamesAppearInSpecificOrder(validator, TESTS_IN_REVERSE_ALPHABETICAL_ORDER); } + @Test + public void testBalancedWithMultipleForksWithoutParallel() { + executeBalancedWithForkCount(2); + } + + @Test + public void testBalancedWithoutForkingClampsDistributionWidth() { + executeBalancedWithForkCount(0); + } + @Test public void testNonExistingRunOrderJUnit4() { unpack().forkCount(1) @@ -141,6 +151,14 @@ private OutputValidator executeWithRunOrder(String runOrder, String sourceName, .verifyErrorFree(3); } + private OutputValidator executeBalancedWithForkCount(int forkCount) { + return unpack().forkCount(forkCount) + .reuseForks(reuseForks()) + .runOrder("balanced") + .executeTest() + .verifyErrorFree(3); + } + private OutputValidator executeWithRandomOrder(long seed) { return unpack().forkCount(1) .reuseForks(reuseForks())