Fix bicep test exit code on compilation errors

src/Bicep.Cli.IntegrationTests/TestFrameworkCommandTests.cs

@@ -46,6 +46,29 @@ public async Task Test_NonBicepFiles_ShouldFail_WithExpectedErrorMessage()
             }
         }
 
+        [TestMethod]
+        public async Task Test_CompilationErrors_ShouldFail_WithoutReportingOverallSuccess()
+        {
+            var settings = new InvocationSettings(new(TestContext, TestFrameworkEnabled: true, AssertsEnabled: true), BicepTestConstants.ClientFactory, BicepTestConstants.TemplateSpecRepositoryFactory);
+            var outputFileDir = FileHelper.GetResultFilePath(TestContext, "outputdir");
+            Directory.CreateDirectory(outputFileDir);
+
+            // Include a valid test to verify that a compilation error prevents an overall success result.
+            FileHelper.SaveResultFile(TestContext, "test.bicep", "// Valid test target.", outputFileDir);
+            var bicepPath = FileHelper.SaveResultFile(TestContext, "main.bicep", @"test valid 'test.bicep' = {}
+test missing 'missing.bicep' = {}", outputFileDir);
+
+            var (output, error, result) = await Bicep(settings, "test", bicepPath);
+
+            using (new AssertionScope())
+            {
+                result.Should().Be(1);
+                output.Should().Contain("Evaluation valid Passed!");
+                output.Should().NotContain("All 1 evaluations passed!");
+                error.Should().Contain("Error BCP091");
+            }
+        }
+
         [TestMethod]
         public async Task Test_commandNoParams_ShouldSucceed()
         {

src/Bicep.Cli/Commands/TestCommand.cs

@@ -67,13 +67,13 @@ public async Task<int> RunAsync(TestArguments args)
             var declarations = semanticModel.Root.TestDeclarations;
             var testResults = TestRunner.Run(declarations);
 
-            LogResults(testResults);
+            LogResults(testResults, summary.HasErrors);
 
-            // return non-zero exit code on errors
-            return testResults.Success ? 0 : 1;
+            // Return a non-zero exit code for compilation and test evaluation errors.
+            return summary.HasErrors || !testResults.Success ? 1 : 0;
         }
 
-        private void LogResults(TestResults testResults)
+        private void LogResults(TestResults testResults, bool hasCompilationErrors)
         {
             foreach (var (testDeclaration, evaluation) in testResults.Results)
             {
@@ -95,11 +95,12 @@ private void LogResults(TestResults testResults)
                     }
                 }
             }
-            if (testResults.Success)
+            // Do not report overall success when compilation diagnostics contain errors.
+            if (testResults.Success && !hasCompilationErrors)
             {
                 io.Output.Writer.WriteLine($"All {testResults.TotalEvaluations} evaluations passed!");
             }
-            else
+            else if (!testResults.Success)
             {
                 io.Error.Writer.WriteLine($"Evaluation Summary: Failure!");
                 io.Error.Writer.WriteLine($"Total: {testResults.TotalEvaluations} - Success: {testResults.SuccessfulEvaluations} - Skipped: {testResults.SkippedEvaluations} - Failed: {testResults.FailedEvaluations}");