Start tests in sorted order also with multiple threads

src/ParallelTestRunner.jl

@@ -1189,10 +1189,12 @@ function _runtests(mod::Module, args::ParsedArgs;
     #
 
     tests_to_start = Threads.Atomic{Int}(length(tests))
+    next_test = Threads.Atomic{Int}(1)
     try
-        @sync for test in tests
+        @sync for _ in 1:length(tests)
             push!(worker_tasks, Threads.@spawn begin
                 local p = nothing
+                local test
                 acquired = false
                 try
                     Base.acquire(sem)
@@ -1202,6 +1204,11 @@ function _runtests(mod::Module, args::ParsedArgs;
 
                     done && return
 
+                    # with multiple threads, tasks reach this point in arbitrary order,
+                    # so pick the next test to run only now, rather than at spawn time,
+                    # to preserve the sorted test order (issue #139)
+                    test = tests[Threads.atomic_add!(next_test, 1)]
+
                     test_t0 = @lock running_tests begin
                         test_t0 = time()
                         running_tests[][test] = test_t0

test/runtests.jl

@@ -5,6 +5,10 @@ cd(@__DIR__)
 
 include(joinpath(@__DIR__, "utils.jl"))
 
+# dummy module to give the "test start order" testset its own history file,
+# without clobbering the history of `ParallelTestRunner` used by other testsets
+module SortingHistoryDummy end
+
 @testset "ParallelTestRunner" verbose=true begin
 
 @testset "basic use" begin
@@ -71,6 +75,34 @@ end
     @test contains(str, "SUCCESS")
 end
 
+@testset "test start order" begin
+    # tests must start in descending order of historical duration, regardless of
+    # the number of threads of this process (issue #139)
+    testsuite = Dict(
+        name => :(@test true)
+        for name in ("sort1", "sort2", "sort3", "sort4", "sort5")
+    )
+    # deliberately not in alphabetical order, so that tests accidentally started in
+    # alphabetical (or insertion) order would not pass the test below
+    expected_order = ["sort3", "sort1", "sort5", "sort2", "sort4"]
+    ParallelTestRunner.save_test_history(SortingHistoryDummy,
+        Dict(name => Float64(length(expected_order) - idx)
+             for (idx, name) in enumerate(expected_order)))
+
+    io = IOBuffer()
+    # with a single job, tests start strictly in acquisition order
+    runtests(SortingHistoryDummy, ["--jobs=1", "--verbose"]; testsuite, stdout=io, stderr=io)
+
+    str = String(take!(io))
+    @test contains(str, "SUCCESS")
+    started_at = map(expected_order) do name
+        m = match(Regex("$(name) .+ started at"), str)
+        @test m !== nothing
+        m === nothing ? typemax(Int) : m.offset
+    end
+    @test issorted(started_at)
+end
+
 @testset "init code" begin
     init_code = quote
         using Test