Agent test harness: routing cases, authored history, scope narrowing,…

src/Infrastructure/BotSharp.Abstraction/Conversations/ISyntheticConversationProbe.cs

@@ -0,0 +1,31 @@
+namespace BotSharp.Abstraction.Conversations;
+
+/// <summary>
+/// Reports whether a conversation is synthetic -- driven by an automated harness rather than by a
+/// person -- so that guards aimed at human overuse can stand aside for it.
+///
+/// This exists because per-user volume limits count the wrong thing for a test harness. A regression
+/// suite legitimately opens one conversation per case per model, at machine speed, and does it under
+/// whatever identity the background worker happens to have. Measured as human behaviour that looks
+/// like abuse, and the limit then fails every test with a message about conversation quotas -- which
+/// says nothing about the agent under test and is indistinguishable, in a report, from the agent
+/// having regressed.
+///
+/// Deliberately a query rather than a flag on the message: a flag would have to be threaded through
+/// every call that creates or continues a conversation, and anything that forgot would silently be
+/// treated as human traffic. Asking by conversation id keeps the answer in one place.
+///
+/// Implementations must be cheap and side-effect free -- this is called on the message path -- and
+/// must answer false for anything they do not recognise. Several may be registered; a conversation is
+/// synthetic if any of them claims it. None being registered is the normal case, and then nothing is
+/// exempt.
+/// </summary>
+public interface ISyntheticConversationProbe
+{
+    /// <summary>
+    /// True when this conversation is being driven by a harness. Must not throw, and must not treat
+    /// an unknown or blank id as synthetic: getting this wrong in that direction exempts real user
+    /// traffic from the very limits it is meant to be held to.
+    /// </summary>
+    bool IsSynthetic(string conversationId);
+}

src/Infrastructure/BotSharp.Logger/Hooks/RateLimitConversationHook.cs

@@ -48,6 +48,19 @@ public override async Task OnMessageReceived(RoleDialogModel message)
             return;
         }
 
+        // Everything below this point is a volume guard against human overuse, and neither guard
+        // measures anything meaningful for an automated harness: a regression suite legitimately
+        // opens one conversation per case per model and drives its turns as fast as the model
+        // answers. Left in force they fail every test with a message about quotas, which says nothing
+        // about the agent under test and reads, in a report, exactly like the agent having regressed.
+        //
+        // The input length check above still applies. That one is about a single message being too
+        // large, which is a real condition a test should surface rather than be excused from.
+        if (IsSyntheticConversation(convId))
+        {
+            return;
+        }
+
         // Check message sending frequency
         var userSents = Dialogs.Where(x => x.Role == AgentRole.User)
             .TakeLast(2).ToList();
@@ -86,4 +99,44 @@ public override async Task OnMessageReceived(RoleDialogModel message)
             }
         }
     }
+
+    /// <summary>
+    /// Whether this conversation is driven by a harness rather than a person. False when nothing is
+    /// registered to answer, which is the normal case -- an absent probe must never exempt anyone.
+    /// </summary>
+    private bool IsSyntheticConversation(string conversationId)
+    {
+        if (string.IsNullOrEmpty(conversationId))
+        {
+            return false;
+        }
+
+        var probes = _services.GetServices<ISyntheticConversationProbe>().ToList();
+        if (probes.Count == 0)
+        {
+            return false;
+        }
+
+        foreach (var probe in probes)
+        {
+            try
+            {
+                if (probe.IsSynthetic(conversationId))
+                {
+                    return true;
+                }
+            }
+            catch (Exception ex)
+            {
+                // A probe that throws must not take the message down with it, and must not be read as
+                // a yes: failing closed here means the worst case is a harness message being rate
+                // limited, rather than real traffic escaping the limit.
+                _logger.LogWarning(ex,
+                    "A synthetic conversation probe failed for conversation {ConversationId}; "
+                    + "treating it as real traffic.", conversationId);
+            }
+        }
+
+        return false;
+    }
 }

src/Plugins/BotSharp.Plugin.AgentTesting/AgentTestingPlugin.cs

@@ -44,6 +44,13 @@ public void RegisterDI(IServiceCollection services, IConfiguration config)
         // Singleton: the test context has to be visible across requests and across threads.
         services.AddSingleton<IAgentTestRunRegistry, AgentTestRunRegistry>();
 
+        // Lets BotSharp's own rate limiting recognise a harness conversation and stand aside. Backed
+        // by the registry rather than by the conversation's "test-set" tag on purpose: the tag is
+        // only written once the conversation row exists, which is after the first message has already
+        // been through the rate limit hook, so a tag-based check would still block the first turn of
+        // every case. The registry entry exists before the conversation is opened at all.
+        services.AddSingleton<ISyntheticConversationProbe, AgentTestSyntheticConversationProbe>();
+
         // The seam that takes over function execution. Lose this line and mocking silently stops
         // working -- the runner's canary self-check is what catches that.
         services.AddScoped<IFunctionExecutorProvider, TestMockExecutorProvider>();
@@ -76,6 +83,14 @@ public void RegisterDI(IServiceCollection services, IConfiguration config)
         // AgentTestController, so it needs an explicit registration the same way every other
         // service in this method does (there is no auto-discovery mechanism in this plugin).
         services.AddScoped<ICaseSegmenter, LlmCaseSegmenter>();
+
+        // Scoped, like the segmenter: it resolves IChatCompletion implementations out of the same
+        // scope and holds no state between calls.
+        services.AddScoped<IAgentTestJudge, LlmAgentTestJudge>();
+
+        // Scoped for the same reason, and reads the case store to ground its prompt in the suite's
+        // existing cases and the edited case's last run.
+        services.AddScoped<ICaseAuthor, LlmCaseAuthor>();
         services.AddScoped<AgentTestRecorder>();
 
         // AgentTestRunQueue is both a singleton and a BackgroundService: all three lines point at