From 965f6405dc19b1440b2e585bc1e1c8a24366cda6 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Mon, 6 Jul 2026 14:29:04 +0000 Subject: [PATCH 1/3] Add a RoutingChatClient that can route requests to different chat clients. --- dotnet/agent-framework-dotnet.slnx | 1 + .../Agent_Step22_MultiModelRouting.csproj | 17 + .../Agent_Step22_MultiModelRouting/Program.cs | 119 +++ .../Agent_Step22_MultiModelRouting/README.md | 65 ++ dotnet/samples/02-agents/Agents/README.md | 2 + .../Microsoft.Agents.AI/AgentJsonUtilities.cs | 3 + .../ChatClient/RoutingChatClient.cs | 376 +++++++++ .../ChatClient/RoutingChatClientOptions.cs | 54 ++ .../ChatClient/RoutingContext.cs | 77 ++ .../ChatClient/RoutingState.cs | 32 + .../ChatClient/RoutingChatClientTests.cs | 726 ++++++++++++++++++ 11 files changed, 1472 insertions(+) create mode 100644 dotnet/samples/02-agents/Agents/Agent_Step22_MultiModelRouting/Agent_Step22_MultiModelRouting.csproj create mode 100644 dotnet/samples/02-agents/Agents/Agent_Step22_MultiModelRouting/Program.cs create mode 100644 dotnet/samples/02-agents/Agents/Agent_Step22_MultiModelRouting/README.md create mode 100644 dotnet/src/Microsoft.Agents.AI/ChatClient/RoutingChatClient.cs create mode 100644 dotnet/src/Microsoft.Agents.AI/ChatClient/RoutingChatClientOptions.cs create mode 100644 dotnet/src/Microsoft.Agents.AI/ChatClient/RoutingContext.cs create mode 100644 dotnet/src/Microsoft.Agents.AI/ChatClient/RoutingState.cs create mode 100644 dotnet/tests/Microsoft.Agents.AI.UnitTests/ChatClient/RoutingChatClientTests.cs diff --git a/dotnet/agent-framework-dotnet.slnx b/dotnet/agent-framework-dotnet.slnx index 80c586bf4e6..857a59511bc 100644 --- a/dotnet/agent-framework-dotnet.slnx +++ b/dotnet/agent-framework-dotnet.slnx @@ -65,6 +65,7 @@ + diff --git a/dotnet/samples/02-agents/Agents/Agent_Step22_MultiModelRouting/Agent_Step22_MultiModelRouting.csproj b/dotnet/samples/02-agents/Agents/Agent_Step22_MultiModelRouting/Agent_Step22_MultiModelRouting.csproj new file mode 100644 index 00000000000..5dc952fd9b8 --- /dev/null +++ b/dotnet/samples/02-agents/Agents/Agent_Step22_MultiModelRouting/Agent_Step22_MultiModelRouting.csproj @@ -0,0 +1,17 @@ + + + + Exe + net10.0 + + enable + enable + + + + + + + + + diff --git a/dotnet/samples/02-agents/Agents/Agent_Step22_MultiModelRouting/Program.cs b/dotnet/samples/02-agents/Agents/Agent_Step22_MultiModelRouting/Program.cs new file mode 100644 index 00000000000..a3f6dcca884 --- /dev/null +++ b/dotnet/samples/02-agents/Agents/Agent_Step22_MultiModelRouting/Program.cs @@ -0,0 +1,119 @@ +// Copyright (c) Microsoft. All rights reserved. + +// Multi-model routing with RoutingChatClient +// +// Demonstrates how to back a single agent with multiple chat clients that use +// different models, and switch between them at runtime. +// +// RoutingChatClient is an IChatClient decorator that holds several named inner +// clients and, for each request, routes to one of them based on the active +// destination stored in the session. It also accepts an optional fallback +// factory that builds a client on the fly for any key that is not a registered +// inner client. Here we key the inner clients (and the fallback) by model name. +// +// This sample: +// 1) Registers two inner clients keyed by model name (models A and B). +// 2) Adds a fallback factory that constructs a Foundry client for whatever +// model name (key) is requested but not pre-registered (model C). +// 3) Retrieves the RoutingChatClient back from the agent via GetService. +// 4) Runs the same agent/session three times, switching the active model +// between runs with SetActiveDestinationKey, so a single conversation is +// served by three different models in turn. +// +// Chat history storage +// -------------------- +// Every client is created with AsIChatClientWithStoredOutputDisabled(...), i.e. +// the Responses API "store" flag is set to false and chat history is kept +// client-side by the agent's session. This is required when routing across +// clients: service-stored chat history is tied to the *service* that created the +// conversation, so it is only available when every turn is served by that same +// service. Because routing can send different turns to different clients (and +// potentially different services), the conversation must be carried client-side +// so it is replayed to whichever model handles the next turn. +// +// If you instead route only among clients that all share the same service, you +// can use service-stored history (AsIChatClient(...) with storage enabled) and +// let the service persist the conversation. +// +// Reasoning content +// ----------------- +// Every client also passes includeReasoningEncryptedContent: false. Encrypted +// reasoning content is model-specific: one model cannot necessarily interpret +// another model's encrypted reasoning, so it must not be echoed back in requests +// when a single conversation is served by multiple models. When you route to only +// a single model, you can leave this enabled (the default) to preserve reasoning +// across turns. + +using Azure.AI.Extensions.OpenAI; +using Azure.AI.Projects; +using Azure.Identity; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI; + +string endpoint = Environment.GetEnvironmentVariable("FOUNDRY_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("FOUNDRY_PROJECT_ENDPOINT is not set."); + +// Two pre-registered models (inner clients) and a third model resolved via the +// fallback factory. Set these to models deployed in your Foundry project. +string modelA = Environment.GetEnvironmentVariable("FOUNDRY_MODEL") ?? "gpt-5.4"; +string modelB = Environment.GetEnvironmentVariable("FOUNDRY_MODEL_ALT1") ?? "gpt-5.4-mini"; +string modelC = Environment.GetEnvironmentVariable("FOUNDRY_MODEL_ALT2") ?? "Deepseek-V4-Pro"; + +// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. +// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid +// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. +AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential()); +var responsesClient = aiProjectClient.GetProjectOpenAIClient().GetProjectResponsesClient(); + +// Register two inner clients keyed by model name. Each uses client-side chat +// history (stored output disabled), and disables encrypted reasoning content +// (includeReasoningEncryptedContent: false) since it is not portable across +// models, so the conversation can move freely between models. +var innerClients = new Dictionary +{ + [modelA] = responsesClient.AsIChatClientWithStoredOutputDisabled(modelA, includeReasoningEncryptedContent: false), + [modelB] = responsesClient.AsIChatClientWithStoredOutputDisabled(modelB, includeReasoningEncryptedContent: false), +}; + +// The fallback factory builds a client for any requested model name (key) that +// is not one of the registered inner clients. The created client is used for the +// single request and then disposed by default. You can disable disposal by setting +// the RoutingChatClientOptions.DisableFallbackChatClientDisposal setting to true. +// This allows you to reuse or cache clients created in the fallback factory, +// but you must then manage their disposal yourself. +var routingChatClient = new RoutingChatClient( + innerClients, + fallbackFactory: (destinationKey, _, _) => + { + Console.WriteLine($" (fallback factory building a client for model '{destinationKey}')"); + return new ValueTask(responsesClient.AsIChatClientWithStoredOutputDisabled(destinationKey, includeReasoningEncryptedContent: false)); + }); + +AIAgent agent = new ChatClientAgent( + routingChatClient, + instructions: "You are a helpful assistant. Keep answers to a single short sentence, and always start by stating which model you are.", + name: "MultiModelAgent"); + +AgentSession session = await agent.CreateSessionAsync(); + +// The RoutingChatClient can be retrieved back from the agent via GetService. +// This is uesful when you don't hold a direct reference to it — for example when +// the agent was created elsewhere or resolved from a DI container. +RoutingChatClient routing = agent.GetService() + ?? throw new InvalidOperationException("The agent is not backed by a RoutingChatClient."); + +// Run 1: default destination — the first registered inner client (model A). +Console.WriteLine($"[Active model: {routing.GetActiveDestinationKey(session) ?? "(default)"}]"); +Console.WriteLine(await agent.RunAsync("Give me a fun fact about the ocean.", session)); +Console.WriteLine(); + +// Run 2: switch to the second registered inner client (model B). +routing.SetActiveDestinationKey(session, modelB); +Console.WriteLine($"[Active model: {routing.GetActiveDestinationKey(session)}]"); +Console.WriteLine(await agent.RunAsync("Give me another one, on a different topic.", session)); +Console.WriteLine(); + +// Run 3: switch to a model that is NOT registered as an inner client. The +// fallback factory constructs a client for it on the fly. +routing.SetActiveDestinationKey(session, modelC); +Console.WriteLine($"[Active model: {routing.GetActiveDestinationKey(session)}]"); +Console.WriteLine(await agent.RunAsync("And one more, about space.", session)); diff --git a/dotnet/samples/02-agents/Agents/Agent_Step22_MultiModelRouting/README.md b/dotnet/samples/02-agents/Agents/Agent_Step22_MultiModelRouting/README.md new file mode 100644 index 00000000000..d163bb9f476 --- /dev/null +++ b/dotnet/samples/02-agents/Agents/Agent_Step22_MultiModelRouting/README.md @@ -0,0 +1,65 @@ +# Multi-Model Routing + +This sample demonstrates how to back a single agent with multiple chat clients that use different models, using `RoutingChatClient`, and how to switch the active model at runtime. + +## What This Sample Shows + +`RoutingChatClient` is an `IChatClient` decorator that holds several named inner clients and, for each request, routes to one of them based on the *active destination* stored in the session. It also accepts an optional **fallback factory** that builds a client on the fly for any key that is not a registered inner client. + +In this sample the destination key is the **model name**: + +- Two inner clients are registered, keyed by model name (models A and B). +- A fallback factory constructs a Foundry client for whatever model name (key) is requested but not pre-registered (model C). The created client serves the single request and is disposed afterwards by default. +- The active model is changed between runs with `SetActiveDestinationKey`, and `GetActiveDestinationKey` reports the current model. A single conversation is served by three different models in turn. +- The `RoutingChatClient` is retrieved back from the agent with `agent.GetService()` — useful when you don't hold a direct reference (for example when the agent is created elsewhere or resolved from a DI container). + +No custom `Router` is supplied, so routing simply follows the per-session active destination — exactly what `SetActiveDestinationKey` controls. + +## Chat History Storage + +Every client is created with `AsIChatClientWithStoredOutputDisabled(...)`, which sets the Responses API `store` flag to `false` so chat history is kept **client-side** by the agent's session. + +This matters when routing across clients: + +- **Service-stored chat history** is tied to the **service** that created the conversation, so it is only available when every turn is served by that same service. If your destinations all share one service, you can enable service-stored history (`AsIChatClient(...)`) and let the service persist the conversation. +- **Client-side chat history** (used here) carries the conversation in the agent's session and replays it to whichever model handles the next turn. This is required when routing may send different turns to different clients (and potentially different services), as this sample does. + +## Reasoning Content + +Every client is also created with `includeReasoningEncryptedContent: false`. Encrypted reasoning content is **model-specific** — one model cannot necessarily interpret another model's encrypted reasoning — so it must not be echoed back in requests when a single conversation is routed across multiple models. If you route to only a single model, you can leave this enabled (the default) to preserve reasoning across turns. + +## Prerequisites + +- .NET 10 SDK or later +- Microsoft Foundry project endpoint with the models below deployed +- Azure CLI installed and authenticated + +**Note**: This sample uses `DefaultAzureCredential`. Sign in with `az login` before running. For production, prefer a specific credential such as `ManagedIdentityCredential`. For more information, see the [Azure CLI authentication documentation](https://learn.microsoft.com/cli/azure/authenticate-azure-cli-interactively). + +## Environment Variables + +```powershell +$env:FOUNDRY_PROJECT_ENDPOINT="https://your-project.services.ai.azure.com/api/projects/your-project" # Required +$env:FOUNDRY_MODEL="gpt-5.4" # Optional, model A (default: gpt-5.4) +$env:FOUNDRY_MODEL_ALT1="gpt-5.4-mini" # Optional, model B (default: gpt-5.4-mini) +$env:FOUNDRY_MODEL_ALT2="Deepseek-V4-Pro" # Optional, model C, resolved via the fallback factory (default: Deepseek-V4-Pro) +``` + +Update the model names to match models deployed in your Foundry project. + +## Running the Sample + +```powershell +cd dotnet/samples/02-agents/Agents/Agent_Step22_MultiModelRouting +dotnet run +``` + +## Expected Behavior + +The sample runs three turns of one conversation: + +1. **First turn** — routed to the default destination (model A, the first registered inner client). +2. **Second turn** — `SetActiveDestinationKey` switches to model B (the second registered inner client). +3. **Third turn** — `SetActiveDestinationKey` switches to model C, which is not registered as an inner client, so the fallback factory builds a client for it. + +Each response starts by stating which model produced it, and the conversation context carries across all three models because chat history is stored client-side. diff --git a/dotnet/samples/02-agents/Agents/README.md b/dotnet/samples/02-agents/Agents/README.md index fff6e8a3b6f..2e52fdce2bb 100644 --- a/dotnet/samples/02-agents/Agents/README.md +++ b/dotnet/samples/02-agents/Agents/README.md @@ -47,6 +47,8 @@ Before you begin, ensure you have the following prerequisites: |[Using compaction pipeline with an agent](./Agent_Step18_CompactionPipeline/)|This sample demonstrates how to use a compaction pipeline to efficiently limit the size of the conversation history for an agent.| |[In-function-loop checkpointing](./Agent_Step19_InFunctionLoopCheckpointing/)|This sample demonstrates how to persist chat history after each service call during a tool-calling loop, enabling crash recovery and mid-run observability.| |[Dynamic function tools](./Agent_Step20_DynamicFunctionTools/)|This sample demonstrates how to dynamically expand the set of function tools available to an agent during a function-calling loop using the ambient FunctionInvocationContext.| +|[Shell tool with environment-aware prompt](./Agent_Step21_ShellWithEnvironment/)|This sample demonstrates using the shell tool in stateless and persistent modes paired with an AIContextProvider that injects environment-aware system prompt instructions.| +|[Multi-model routing](./Agent_Step22_MultiModelRouting/)|This sample demonstrates how to back a single agent with multiple chat clients that use different models via RoutingChatClient, switching the active model at runtime.| ## Running the samples from the console diff --git a/dotnet/src/Microsoft.Agents.AI/AgentJsonUtilities.cs b/dotnet/src/Microsoft.Agents.AI/AgentJsonUtilities.cs index 487879a73bf..5347addb644 100644 --- a/dotnet/src/Microsoft.Agents.AI/AgentJsonUtilities.cs +++ b/dotnet/src/Microsoft.Agents.AI/AgentJsonUtilities.cs @@ -108,6 +108,9 @@ private static JsonSerializerOptions CreateDefaultOptions() [JsonSerializable(typeof(BackgroundTaskStatus))] [JsonSerializable(typeof(List), TypeInfoPropertyName = "BackgroundTaskInfoList")] + // RoutingChatClient types + [JsonSerializable(typeof(RoutingState))] + [ExcludeFromCodeCoverage] internal sealed partial class JsonContext : JsonSerializerContext; } diff --git a/dotnet/src/Microsoft.Agents.AI/ChatClient/RoutingChatClient.cs b/dotnet/src/Microsoft.Agents.AI/ChatClient/RoutingChatClient.cs new file mode 100644 index 00000000000..b9336154512 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/ChatClient/RoutingChatClient.cs @@ -0,0 +1,376 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Agents.AI; + +/// +/// An that routes each request to one of several inner chat clients. +/// +/// +/// +/// This decorator holds +/// multiple named inner clients (destinations) and selects one per request. By default, requests are routed +/// to the session's currently active destination, which is stored in the session's +/// as a . Use +/// and to switch or inspect the +/// active destination key for a session. +/// +/// +/// The default destination for a new session is produced by the optional stateInitializer constructor +/// argument. When it is not supplied, the first entry in the inner clients dictionary is used. +/// +/// +/// A custom router can be supplied via to select a destination +/// key per request. For each request, the destination is resolved in the following order: +/// +/// The router (when configured) is invoked to produce a key; otherwise the session's active destination key is used. +/// When the key matches a registered inner client, that client is used. +/// +/// When the key does not match a registered inner client and a fallback factory is configured, a client is created on the fly +/// for that request. By default the created client is disposed after the request completes; set +/// to keep it (for example, when the factory caches or +/// returns shared clients). +/// Otherwise an is thrown. +/// +/// An empty string is treated as an ordinary key (looked up, then routed to the fallback factory); only +/// is routed directly to the fallback factory. +/// +/// +/// It is valid to construct the client with no inner clients by using the constructor that accepts only a +/// fallback factory, in which case every request is served by the fallback factory. +/// +/// +/// This client resolves the current agent and session from , which is +/// set automatically when an agent's run methods are called. It must therefore be invoked within an agent run +/// that has a resolved session; calling it outside of an agent run (or before a session is resolved) throws +/// . +/// +/// +/// Instances are thread-safe across different sessions. A single session must not be used concurrently: the +/// per-session active destination state assumes only one request per session is in flight at a time. +/// +/// +[Experimental(DiagnosticIds.Experiments.AgentsAIExperiments)] +public sealed class RoutingChatClient : IChatClient +{ + private readonly IReadOnlyDictionary _innerClients; + private readonly Func>? _router; + private readonly Func> _fallbackFactory; + private readonly ProviderSessionState _sessionState; + private readonly bool _disableFallbackChatClientDisposal; + + /// + /// Initializes a new instance of the class that routes to a fixed set of + /// inner clients. + /// + /// The inner clients to route to, keyed by destination name. Must be non-empty. + /// + /// An optional function that initializes the for a new session. Use this to + /// specify the default destination a new session is routed to. When , the default + /// initializer selects the first entry in . + /// + /// Optional settings that control routing behavior. When , defaults are used. + /// Thrown when is . + /// + /// Thrown when is empty or contains a value. + /// + public RoutingChatClient( + IReadOnlyDictionary innerClients, + Func? stateInitializer = null, + RoutingChatClientOptions? options = null) + : this(RequireNonEmpty(innerClients), s_noFallbackFactory, stateInitializer, options) + { + } + + /// + /// Initializes a new instance of the class that serves every request from a + /// fallback factory (no fixed inner clients). + /// + /// + /// An asynchronous factory used to construct an on the fly for the routed key. It + /// receives the routed key (which may be for the default destination), the + /// , and a , and returns the client to use. A new + /// client is created for each request that routes to the factory and, by default, disposed after that request + /// completes (see ). + /// + /// + /// An optional function that initializes the for a new session. When + /// , the default initializer sets the active destination to . + /// + /// Optional settings that control routing behavior. When , defaults are used. + /// Thrown when is . + public RoutingChatClient( + Func> fallbackFactory, + Func? stateInitializer = null, + RoutingChatClientOptions? options = null) + : this(new Dictionary(), Throw.IfNull(fallbackFactory), stateInitializer, options) + { + } + + /// + /// Initializes a new instance of the class that routes to a fixed set of + /// inner clients and falls back to a factory for unregistered keys. + /// + /// The inner clients to route to, keyed by destination name. Must be non-empty. + /// + /// An asynchronous factory used to construct an on the fly when the routed key is + /// not one of the registered inner clients. It receives the routed key (which may be + /// for the default destination), the , and a , and + /// returns the client to use. A new client is created for each request that routes to the factory and, by + /// default, disposed after that request completes (see + /// ). + /// + /// + /// An optional function that initializes the for a new session. Use this to + /// specify the default destination a new session is routed to. When , the default + /// initializer selects the first entry in . + /// + /// Optional settings that control routing behavior. When , defaults are used. + /// + /// Thrown when or is . + /// + /// + /// Thrown when contains a value. + /// + public RoutingChatClient( + IReadOnlyDictionary innerClients, + Func> fallbackFactory, + Func? stateInitializer = null, + RoutingChatClientOptions? options = null) + { + Throw.IfNull(innerClients); + Throw.IfNull(fallbackFactory); + + foreach (var pair in innerClients) + { + if (pair.Value is null) + { + throw new ArgumentException($"The inner client for key '{pair.Key}' is null.", nameof(innerClients)); + } + } + + this._innerClients = innerClients; + this._router = options?.Router; + this._fallbackFactory = fallbackFactory; + this._disableFallbackChatClientDisposal = options?.DisableFallbackChatClientDisposal ?? false; + + string? defaultDestination = innerClients.Keys.FirstOrDefault(); + string stateKey = options?.StateKey ?? this.GetType().Name; + this._sessionState = new ProviderSessionState( + stateInitializer ?? (_ => new RoutingState { ActiveDestination = defaultDestination }), + stateKey, + AgentJsonUtilities.DefaultOptions); + } + + /// + /// A fallback factory used by the inner-clients-only constructor. It resolves no client (returns + /// ), so an unresolved key results in an when the + /// request is resolved. + /// + private static readonly Func> s_noFallbackFactory = + (_, _, _) => default; + + /// + /// Validates that is non-null and non-empty, returning it for chaining. + /// + private static IReadOnlyDictionary RequireNonEmpty(IReadOnlyDictionary innerClients) + { + Throw.IfNull(innerClients); + + if (innerClients.Count == 0) + { + throw new ArgumentException("At least one inner client must be provided.", nameof(innerClients)); + } + + return innerClients; + } + + /// + public async Task GetResponseAsync( + IEnumerable messages, + ChatOptions? options = null, + CancellationToken cancellationToken = default) + { + Throw.IfNull(messages); + + var (agent, session) = GetRequiredRunContext(); + var (client, disposeAfterUse) = await this.ResolveClientAsync(messages, options, agent, session, cancellationToken).ConfigureAwait(false); + try + { + return await client.GetResponseAsync(messages, options, cancellationToken).ConfigureAwait(false); + } + finally + { + if (disposeAfterUse) + { + client.Dispose(); + } + } + } + + /// + public async IAsyncEnumerable GetStreamingResponseAsync( + IEnumerable messages, + ChatOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + Throw.IfNull(messages); + + var (agent, session) = GetRequiredRunContext(); + var (client, disposeAfterUse) = await this.ResolveClientAsync(messages, options, agent, session, cancellationToken).ConfigureAwait(false); + + try + { + await foreach (var update in client.GetStreamingResponseAsync(messages, options, cancellationToken).ConfigureAwait(false)) + { + yield return update; + } + } + finally + { + if (disposeAfterUse) + { + client.Dispose(); + } + } + } + + /// + /// Gets the currently active destination key for the specified session. + /// + /// The session whose active destination key should be returned. + /// + /// The active destination key for the session, or when the default destination + /// (the first inner client) is in effect. + /// + /// Thrown when is . + public string? GetActiveDestinationKey(AgentSession session) + { + Throw.IfNull(session); + + return this._sessionState.GetOrInitializeState(session).ActiveDestination; + } + + /// + /// Sets the active destination key for the specified session. + /// + /// The session whose active destination key should be updated. + /// + /// The destination key to make active. May be any string (a registered inner client key or a key handled by + /// the fallback factory), or to use the default destination (the first inner client). + /// + /// Thrown when is . + public void SetActiveDestinationKey(AgentSession session, string? destinationKey) + { + Throw.IfNull(session); + + var state = this._sessionState.GetOrInitializeState(session); + state.ActiveDestination = destinationKey; + this._sessionState.SaveState(session, state); + } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Throw.IfNull(serviceType); + + if (serviceKey is null && serviceType.IsInstanceOfType(this)) + { + return this; + } + + return this.GetActiveClientForService()?.GetService(serviceType, serviceKey); + } + + /// + public void Dispose() + { + foreach (var client in this._innerClients.Values) + { + client.Dispose(); + } + } + + /// + /// Gets the current agent and session from the ambient run context, throwing if either is unavailable. + /// + /// No run context or session is available. + private static (AIAgent Agent, AgentSession Session) GetRequiredRunContext() + { + var runContext = AIAgent.CurrentRunContext + ?? throw new InvalidOperationException( + $"{nameof(RoutingChatClient)} can only be used within the context of a running AIAgent. " + + "Ensure that the chat client is being invoked as part of an AIAgent.RunAsync or AIAgent.RunStreamingAsync call."); + + var session = runContext.Session + ?? throw new InvalidOperationException( + $"{nameof(RoutingChatClient)} requires a session. " + + "Ensure the agent has a resolved session before invoking the chat client."); + + return (runContext.Agent, session); + } + + /// + /// Resolves the inner client that should forward to. This is a best-effort lookup: + /// it uses the active destination of the current run's session when available, otherwise the first inner client. + /// + private IChatClient? GetActiveClientForService() + { + var session = AIAgent.CurrentRunContext?.Session; + string? key = session is not null + ? this._sessionState.GetOrInitializeState(session).ActiveDestination + : null; + + if (key is not null && this._innerClients.TryGetValue(key, out var client)) + { + return client; + } + + return this._innerClients.Count > 0 ? this._innerClients.Values.First() : null; + } + + /// + /// Resolves the destination client for a request using the configured router and fallback factory. + /// + /// + /// A tuple containing the resolved client and a flag indicating whether the caller should dispose it after the + /// request completes. The flag is only for clients created by the fallback factory when + /// disposal is not disabled; registered inner clients are owned by this instance and disposed at teardown. + /// + private async ValueTask<(IChatClient Client, bool DisposeAfterUse)> ResolveClientAsync(IEnumerable messages, ChatOptions? options, AIAgent agent, AgentSession session, CancellationToken cancellationToken) + { + var state = this._sessionState.GetOrInitializeState(session); + + var context = new RoutingContext( + agent, + session, + messages as IReadOnlyList ?? messages.ToList(), + options, + this._innerClients, + state.ActiveDestination); + + string? key = this._router is not null + ? await this._router(context, cancellationToken).ConfigureAwait(false) + : context.ActiveDestination; + + if (key is not null && this._innerClients.TryGetValue(key, out var client)) + { + return (client, false); + } + + var created = await this._fallbackFactory(key, context, cancellationToken).ConfigureAwait(false) + ?? throw new InvalidOperationException($"No inner client is registered for destination '{key ?? "(null)"}' and no fallback client is available."); + + return (created, !this._disableFallbackChatClientDisposal); + } +} diff --git a/dotnet/src/Microsoft.Agents.AI/ChatClient/RoutingChatClientOptions.cs b/dotnet/src/Microsoft.Agents.AI/ChatClient/RoutingChatClientOptions.cs new file mode 100644 index 00000000000..41e4dbed999 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/ChatClient/RoutingChatClientOptions.cs @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Agents.AI; + +/// +/// Options that control the behavior of a . +/// +[Experimental(DiagnosticIds.Experiments.AgentsAIExperiments)] +public sealed class RoutingChatClientOptions +{ + /// + /// Gets or sets a custom asynchronous routing heuristic that selects the destination key for a request. + /// + /// + /// + /// The function receives a and a and returns + /// the key of the destination that should handle the request, or to route to the + /// default destination (the first inner client). It is asynchronous so callers can perform I/O + /// (for example, an inference call) to decide the best destination. When (the default), + /// the currently active destination for the session is used (see ). + /// + /// + /// The returned key is used only for the current request; it does not change the session's active + /// destination. Use to switch the active destination. + /// + /// + public Func>? Router { get; set; } + + /// + /// Gets or sets the key used to store the routing state in the . + /// + /// + /// Defaults to the client's type name. Override this if you need multiple + /// instances with separate state in the same session. + /// + public string? StateKey { get; set; } + + /// + /// Gets or sets a value indicating whether disposal of chat clients created by the fallback factory is + /// disabled. + /// + /// + /// by default, meaning a client created by the fallback factory is disposed after the + /// request that created it completes. Set to to keep such clients alive after use — for + /// example, when the factory caches or returns shared clients whose lifetime it manages itself. + /// + public bool DisableFallbackChatClientDisposal { get; set; } +} diff --git a/dotnet/src/Microsoft.Agents.AI/ChatClient/RoutingContext.cs b/dotnet/src/Microsoft.Agents.AI/ChatClient/RoutingContext.cs new file mode 100644 index 00000000000..afbe2b35f44 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/ChatClient/RoutingContext.cs @@ -0,0 +1,77 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.AI; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Agents.AI; + +/// +/// Provides the information available to a router and fallback factory +/// when deciding which inner client should handle a request. +/// +[Experimental(DiagnosticIds.Experiments.AgentsAIExperiments)] +public sealed class RoutingContext +{ + /// + /// Initializes a new instance of the class. + /// + /// The executing the current run. + /// The associated with the current run. + /// The messages being sent in the current request. + /// The chat options for the current request, if any. + /// The registered inner clients keyed by destination name. + /// The currently active destination key for the session. + [Experimental(DiagnosticIds.Experiments.AgentsAIExperiments)] + public RoutingContext( + AIAgent agent, + AgentSession session, + IReadOnlyList messages, + ChatOptions? options, + IReadOnlyDictionary innerClients, + string? activeDestination) + { + this.Agent = agent; + this.Session = session; + this.Messages = messages; + this.Options = options; + this.InnerClients = innerClients; + this.ActiveDestination = activeDestination; + } + + /// + /// Gets the executing the current run. + /// + public AIAgent Agent { get; } + + /// + /// Gets the associated with the current run. + /// + public AgentSession Session { get; } + + /// + /// Gets the messages being sent in the current request. + /// + public IReadOnlyList Messages { get; } + + /// + /// Gets the chat options for the current request, if any. + /// + public ChatOptions? Options { get; } + + /// + /// Gets the registered inner clients keyed by destination name. + /// + public IReadOnlyDictionary InnerClients { get; } + + /// + /// Gets the currently active destination key for the session, or when the default + /// destination (the first inner client) should be used. + /// + /// + /// This is the value returned by the default router. It reflects the destination stored in the + /// session's (or the value produced by the state initializer for a new session). + /// + public string? ActiveDestination { get; } +} diff --git a/dotnet/src/Microsoft.Agents.AI/ChatClient/RoutingState.cs b/dotnet/src/Microsoft.Agents.AI/ChatClient/RoutingState.cs new file mode 100644 index 00000000000..1dc53084ecf --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/ChatClient/RoutingState.cs @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Diagnostics.CodeAnalysis; +using System.Text.Json.Serialization; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Agents.AI; + +/// +/// Represents the serializable routing state of a , +/// stored in the session's . +/// +/// +/// This state tracks the currently active destination for a session. Use it from a custom +/// stateInitializer to control which inner client a new session is routed to by default. +/// +[Experimental(DiagnosticIds.Experiments.AgentsAIExperiments)] +public sealed class RoutingState +{ + /// + /// Gets or sets the key of the currently active destination for this session, or + /// to use the default destination. + /// + /// + /// When non-, the value corresponds to a key in the inner clients dictionary supplied + /// to the (or a key handled by a fallback factory). It is used by the default + /// router to select a destination. When , the request is routed to the first inner + /// client (the default destination). + /// + [JsonPropertyName("activeDestination")] + public string? ActiveDestination { get; set; } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.UnitTests/ChatClient/RoutingChatClientTests.cs b/dotnet/tests/Microsoft.Agents.AI.UnitTests/ChatClient/RoutingChatClientTests.cs new file mode 100644 index 00000000000..f629b8167dc --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.UnitTests/ChatClient/RoutingChatClientTests.cs @@ -0,0 +1,726 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Moq; + +namespace Microsoft.Agents.AI.UnitTests; + +/// +/// Unit tests for . +/// +public class RoutingChatClientTests +{ + #region Construction + + /// + /// Verify that the constructor throws when there are no inner clients and no fallback factory. + /// + [Fact] + public void Constructor_NullInnerClients_NoFallback_Throws() + { + // Arrange & Act & Assert + Assert.Throws(() => new RoutingChatClient((IReadOnlyDictionary)null!)); + } + + /// + /// Verify that the constructor throws when the inner clients dictionary is empty and no fallback factory + /// is configured. + /// + [Fact] + public void Constructor_EmptyInnerClients_NoFallback_Throws() + { + // Arrange & Act & Assert + Assert.Throws(() => new RoutingChatClient(new Dictionary())); + } + + /// + /// Verify that the constructor allows no inner clients when a fallback factory is configured. + /// + [Fact] + public void Constructor_NoInnerClients_WithFallback_Succeeds() + { + // Arrange + var fallback = CreateMockClient(); + + // Act & Assert (does not throw) + using var routing = new RoutingChatClient( + fallbackFactory: (_, _, _) => new ValueTask(fallback.Object)); + } + + /// + /// Verify that can be constructed via its public constructor so callers can + /// unit-test their own router and fallback factory callbacks. + /// + [Fact] + public async Task RoutingContext_PublicConstructor_ExposesSuppliedValuesAsync() + { + // Arrange + var clientA = CreateMockClient(); + var routing = new RoutingChatClient(new Dictionary { ["a"] = clientA.Object }); + var agent = new ChatClientAgent(routing); + var session = await agent.CreateSessionAsync(); + var messages = new[] { new ChatMessage(ChatRole.User, "hi") }; + var options = new ChatOptions(); + var innerClients = new Dictionary { ["a"] = clientA.Object }; + + // Act + var context = new RoutingContext(agent, session, messages, options, innerClients, "a"); + + // Assert + Assert.Same(agent, context.Agent); + Assert.Same(session, context.Session); + Assert.Same(messages, context.Messages); + Assert.Same(options, context.Options); + Assert.Same(innerClients, context.InnerClients); + Assert.Equal("a", context.ActiveDestination); + } + + #endregion + + #region Routing + + /// + /// Verify that, without a custom router, requests route to the default (first) destination. + /// + [Fact] + public async Task GetResponseAsync_DefaultRoutesToFirstDestinationAsync() + { + // Arrange + var clientA = CreateMockClient(); + var clientB = CreateMockClient(); + var routing = new RoutingChatClient(new Dictionary + { + ["a"] = clientA.Object, + ["b"] = clientB.Object, + }); + var agent = new ChatClientAgent(routing); + var session = await agent.CreateSessionAsync(); + + // Act + await agent.RunAsync([new ChatMessage(ChatRole.User, "hi")], session); + + // Assert + clientA.Verify(c => c.GetResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny()), Times.Once); + clientB.Verify(c => c.GetResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); + } + + /// + /// Verify that a custom router selects the destination for a request. + /// + [Fact] + public async Task GetResponseAsync_CustomRouterSelectsDestinationAsync() + { + // Arrange + var clientA = CreateMockClient(); + var clientB = CreateMockClient(); + var routing = new RoutingChatClient( + new Dictionary { ["a"] = clientA.Object, ["b"] = clientB.Object }, + options: new RoutingChatClientOptions { Router = (_, _) => new ValueTask("b") }); + var agent = new ChatClientAgent(routing); + var session = await agent.CreateSessionAsync(); + + // Act + await agent.RunAsync([new ChatMessage(ChatRole.User, "hi")], session); + + // Assert + clientB.Verify(c => c.GetResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny()), Times.Once); + clientA.Verify(c => c.GetResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); + } + + /// + /// Verify that an asynchronous router is awaited before the selected destination is used. + /// + [Fact] + public async Task GetResponseAsync_AsyncRouterIsAwaitedAsync() + { + // Arrange + var clientA = CreateMockClient(); + var clientB = CreateMockClient(); + var routing = new RoutingChatClient( + new Dictionary { ["a"] = clientA.Object, ["b"] = clientB.Object }, + options: new RoutingChatClientOptions + { + Router = async (_, ct) => + { + await Task.Delay(1, ct); + return "b"; + }, + }); + var agent = new ChatClientAgent(routing); + var session = await agent.CreateSessionAsync(); + + // Act + await agent.RunAsync([new ChatMessage(ChatRole.User, "hi")], session); + + // Assert + clientB.Verify(c => c.GetResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny()), Times.Once); + clientA.Verify(c => c.GetResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); + } + + /// + /// Verify that the router and fallback factory receive the current agent and session on the context. + /// + [Fact] + public async Task Router_ReceivesAgentAndSessionAsync() + { + // Arrange + var clientA = CreateMockClient(); + AIAgent? observedAgent = null; + AgentSession? observedSession = null; + var routing = new RoutingChatClient( + new Dictionary { ["a"] = clientA.Object }, + options: new RoutingChatClientOptions + { + Router = (context, _) => + { + observedAgent = context.Agent; + observedSession = context.Session; + return new ValueTask(context.ActiveDestination); + }, + }); + var agent = new ChatClientAgent(routing); + var session = await agent.CreateSessionAsync(); + + // Act + await agent.RunAsync([new ChatMessage(ChatRole.User, "hi")], session); + + // Assert + Assert.NotNull(observedAgent); + Assert.NotNull(observedSession); + } + + /// + /// Verify that invoking the client outside of an agent run throws. + /// + [Fact] + public async Task GetResponseAsync_WithoutRunContext_ThrowsAsync() + { + // Arrange + var clientA = CreateMockClient(); + var routing = new RoutingChatClient(new Dictionary { ["a"] = clientA.Object }); + + // Act & Assert + await Assert.ThrowsAsync(() => + routing.GetResponseAsync([new ChatMessage(ChatRole.User, "hi")])); + } + + /// + /// Verify that an unknown routed key with no fallback factory throws. + /// + [Fact] + public async Task GetResponseAsync_UnknownKeyWithoutFallback_ThrowsAsync() + { + // Arrange + var clientA = CreateMockClient(); + var routing = new RoutingChatClient( + new Dictionary { ["a"] = clientA.Object }, + options: new RoutingChatClientOptions { Router = (_, _) => new ValueTask("missing") }); + var agent = new ChatClientAgent(routing); + var session = await agent.CreateSessionAsync(); + + // Act & Assert + await Assert.ThrowsAsync(() => + agent.RunAsync([new ChatMessage(ChatRole.User, "hi")], session)); + } + + /// + /// Verify that streaming requests route to the selected destination. + /// + [Fact] + public async Task GetStreamingResponseAsync_RoutesToSelectedDestinationAsync() + { + // Arrange + var clientA = CreateMockClient(); + var clientB = CreateMockClient(); + var routing = new RoutingChatClient( + new Dictionary { ["a"] = clientA.Object, ["b"] = clientB.Object }, + options: new RoutingChatClientOptions { Router = (_, _) => new ValueTask("b") }); + var agent = new ChatClientAgent(routing); + var session = await agent.CreateSessionAsync(); + + // Act + await foreach (var _ in agent.RunStreamingAsync([new ChatMessage(ChatRole.User, "hi")], session)) + { + } + + // Assert + clientB.Verify(c => c.GetStreamingResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny()), Times.Once); + clientA.Verify(c => c.GetStreamingResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); + } + + /// + /// Verify that when the routed key is not registered, the fallback factory constructs a client on the fly. + /// + [Fact] + public async Task GetResponseAsync_FallbackFactoryCreatesClient_WhenKeyUnknownAsync() + { + // Arrange + var clientA = CreateMockClient(); + var fallback = CreateMockClient(); + var routing = new RoutingChatClient( + new Dictionary { ["a"] = clientA.Object }, + fallbackFactory: (_, _, _) => new ValueTask(fallback.Object), + options: new RoutingChatClientOptions + { + Router = (_, _) => new ValueTask("created"), + }); + var agent = new ChatClientAgent(routing); + var session = await agent.CreateSessionAsync(); + + // Act + await agent.RunAsync([new ChatMessage(ChatRole.User, "hi")], session); + + // Assert + fallback.Verify(c => c.GetResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny()), Times.Once); + clientA.Verify(c => c.GetResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); + } + + #endregion + + #region GetService + + /// + /// Verify that GetService returns the routing client itself when the requested type matches. + /// + [Fact] + public void GetService_ReturnsSelf_WhenTypeMatches() + { + // Arrange + var clientA = CreateMockClient(); + var routing = new RoutingChatClient(new Dictionary { ["a"] = clientA.Object }); + + // Act + var service = routing.GetService(typeof(RoutingChatClient)); + + // Assert + Assert.Same(routing, service); + } + + /// + /// Verify that GetService forwards to the first inner client when no run context is active. + /// + [Fact] + public void GetService_ForwardsToFirstInnerClient_WhenNoRunContext() + { + // Arrange + var clientA = CreateMockClient(); + var marker = new object(); + clientA.Setup(c => c.GetService(typeof(string), null)).Returns(marker); + var routing = new RoutingChatClient(new Dictionary { ["a"] = clientA.Object }); + + // Act + var service = routing.GetService(typeof(string)); + + // Assert + Assert.Same(marker, service); + clientA.Verify(c => c.GetService(typeof(string), null), Times.Once); + } + + #endregion + + #region Active destination (session-based) + + /// + /// Verify that switches routing for the session and + /// that reflects the change. + /// + [Fact] + public async Task SetActiveDestinationKey_SwitchesRoutingForSessionAsync() + { + // Arrange + var clientA = CreateMockClient(); + var clientB = CreateMockClient(); + var routing = new RoutingChatClient(new Dictionary + { + ["a"] = clientA.Object, + ["b"] = clientB.Object, + }); + var agent = new ChatClientAgent(routing); + var session = await agent.CreateSessionAsync(); + + // Act + routing.SetActiveDestinationKey(session, "b"); + await agent.RunAsync([new ChatMessage(ChatRole.User, "hi")], session); + + // Assert + Assert.Equal("b", routing.GetActiveDestinationKey(session)); + clientB.Verify(c => c.GetResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny()), Times.Once); + clientA.Verify(c => c.GetResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); + } + + /// + /// Verify that the default active destination key for a new session is the first inner client key. + /// + [Fact] + public async Task GetActiveDestinationKey_DefaultsToFirstInnerClientKeyAsync() + { + // Arrange + var routing = new RoutingChatClient(new Dictionary + { + ["a"] = CreateMockClient().Object, + ["b"] = CreateMockClient().Object, + }); + var agent = new ChatClientAgent(routing); + var session = await agent.CreateSessionAsync(); + + // Act + var key = routing.GetActiveDestinationKey(session); + + // Assert + Assert.Equal("a", key); + } + + /// + /// Verify that a active destination key routes to the fallback factory (invoked with a + /// key), rather than to the first inner client. + /// + [Fact] + public async Task SetActiveDestinationKey_Null_RoutesToFallbackWithNullKeyAsync() + { + // Arrange + var clientA = CreateMockClient(); + var fallback = CreateMockClient(); + string? observedKey = "sentinel"; + var routing = new RoutingChatClient( + new Dictionary { ["a"] = clientA.Object }, + fallbackFactory: (key, _, _) => + { + observedKey = key; + return new ValueTask(fallback.Object); + }); + var agent = new ChatClientAgent(routing); + var session = await agent.CreateSessionAsync(); + + // Act — set the active destination to null. + routing.SetActiveDestinationKey(session, null); + await agent.RunAsync([new ChatMessage(ChatRole.User, "hi")], session); + + // Assert + Assert.Null(routing.GetActiveDestinationKey(session)); + Assert.Null(observedKey); + fallback.Verify(c => c.GetResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny()), Times.Once); + clientA.Verify(c => c.GetResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); + } + + /// + /// Verify that a active destination key throws when no fallback factory is configured + /// (the first inner client is not used as an implicit default). + /// + [Fact] + public async Task SetActiveDestinationKey_Null_NoFallback_ThrowsAsync() + { + // Arrange + var clientA = CreateMockClient(); + var routing = new RoutingChatClient(new Dictionary + { + ["a"] = clientA.Object, + }); + var agent = new ChatClientAgent(routing); + var session = await agent.CreateSessionAsync(); + + // Act + routing.SetActiveDestinationKey(session, null); + + // Assert + await Assert.ThrowsAsync( + () => agent.RunAsync([new ChatMessage(ChatRole.User, "hi")], session)); + clientA.Verify(c => c.GetResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); + } + + /// + /// Verify that an unregistered active destination key routes to the fallback factory (no exception). + /// + [Fact] + public async Task SetActiveDestinationKey_UnregisteredKey_RoutesToFallbackAsync() + { + // Arrange + var clientA = CreateMockClient(); + var fallback = CreateMockClient(); + var routing = new RoutingChatClient( + new Dictionary { ["a"] = clientA.Object }, + fallbackFactory: (_, _, _) => new ValueTask(fallback.Object)); + var agent = new ChatClientAgent(routing); + var session = await agent.CreateSessionAsync(); + + // Act + routing.SetActiveDestinationKey(session, "created"); + await agent.RunAsync([new ChatMessage(ChatRole.User, "hi")], session); + + // Assert + fallback.Verify(c => c.GetResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny()), Times.Once); + clientA.Verify(c => c.GetResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); + } + + /// + /// Verify that an empty-string active destination key is treated as an ordinary (unregistered) key and + /// routes to the fallback factory. + /// + [Fact] + public async Task SetActiveDestinationKey_EmptyString_RoutesToFallbackAsync() + { + // Arrange + var clientA = CreateMockClient(); + var fallback = CreateMockClient(); + string? observedKey = null; + var routing = new RoutingChatClient( + new Dictionary { ["a"] = clientA.Object }, + fallbackFactory: (key, _, _) => + { + observedKey = key; + return new ValueTask(fallback.Object); + }); + var agent = new ChatClientAgent(routing); + var session = await agent.CreateSessionAsync(); + + // Act + routing.SetActiveDestinationKey(session, string.Empty); + await agent.RunAsync([new ChatMessage(ChatRole.User, "hi")], session); + + // Assert + Assert.Equal(string.Empty, observedKey); + fallback.Verify(c => c.GetResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny()), Times.Once); + } + + /// + /// Verify that the active destination is isolated between sessions. + /// + [Fact] + public async Task ActiveDestination_IsolatedBetweenSessionsAsync() + { + // Arrange + var clientA = CreateMockClient(); + var clientB = CreateMockClient(); + var routing = new RoutingChatClient(new Dictionary + { + ["a"] = clientA.Object, + ["b"] = clientB.Object, + }); + var agent = new ChatClientAgent(routing); + var sessionOne = await agent.CreateSessionAsync(); + var sessionTwo = await agent.CreateSessionAsync(); + + // Act — switch only session one to "b"; session two keeps the default "a". + routing.SetActiveDestinationKey(sessionOne, "b"); + await agent.RunAsync([new ChatMessage(ChatRole.User, "hi")], sessionOne); + await agent.RunAsync([new ChatMessage(ChatRole.User, "hi")], sessionTwo); + + // Assert + Assert.Equal("b", routing.GetActiveDestinationKey(sessionOne)); + Assert.Equal("a", routing.GetActiveDestinationKey(sessionTwo)); + clientB.Verify(c => c.GetResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny()), Times.Once); + clientA.Verify(c => c.GetResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny()), Times.Once); + } + + #endregion + + #region Fallback disposal (per request) + + /// + /// Verify that the fallback factory is invoked per request (no caching) and each created client is disposed + /// after the request completes. + /// + [Fact] + public async Task FallbackFactory_CreatesPerRequest_AndDisposesAfterUseAsync() + { + // Arrange + var created = new List>(); + const string RouteKey = "created"; + var routing = new RoutingChatClient( + new Dictionary { ["a"] = CreateMockClient().Object }, + fallbackFactory: (_, _, _) => + { + var mock = CreateMockClient(); + created.Add(mock); + return new ValueTask(mock.Object); + }, + options: new RoutingChatClientOptions + { + Router = (_, _) => new ValueTask(RouteKey), + }); + var agent = new ChatClientAgent(routing); + var session = await agent.CreateSessionAsync(); + + // Act — two runs with the same routed key. + await agent.RunAsync([new ChatMessage(ChatRole.User, "one")], session); + await agent.RunAsync([new ChatMessage(ChatRole.User, "two")], session); + + // Assert — a fresh client is created for each request and disposed after use. + Assert.Equal(2, created.Count); + foreach (var mock in created) + { + mock.Verify(c => c.GetResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny()), Times.Once); + mock.Verify(c => c.Dispose(), Times.Once); + } + } + + /// + /// Verify that a client created by the fallback factory is disposed after the request completes by default. + /// + [Fact] + public async Task FallbackFactory_DisposesCreatedClientAfterUse_ByDefaultAsync() + { + // Arrange + var fallback = CreateMockClient(); + var routing = new RoutingChatClient( + fallbackFactory: (_, _, _) => new ValueTask(fallback.Object)); + var agent = new ChatClientAgent(routing); + var session = await agent.CreateSessionAsync(); + + // Act + await agent.RunAsync([new ChatMessage(ChatRole.User, "hi")], session); + + // Assert + fallback.Verify(c => c.Dispose(), Times.Once); + } + + /// + /// Verify that setting prevents the + /// created fallback client from being disposed after use. + /// + [Fact] + public async Task DisableFallbackChatClientDisposal_DoesNotDisposeCreatedClientAsync() + { + // Arrange + var fallback = CreateMockClient(); + var routing = new RoutingChatClient( + fallbackFactory: (_, _, _) => new ValueTask(fallback.Object), + options: new RoutingChatClientOptions { DisableFallbackChatClientDisposal = true }); + var agent = new ChatClientAgent(routing); + var session = await agent.CreateSessionAsync(); + + // Act + await agent.RunAsync([new ChatMessage(ChatRole.User, "hi")], session); + + // Assert + fallback.Verify(c => c.GetResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny()), Times.Once); + fallback.Verify(c => c.Dispose(), Times.Never); + } + + /// + /// Verify that a registered inner client is not disposed after a request (inner clients are owned by the + /// routing client and disposed only at teardown). + /// + [Fact] + public async Task InnerClient_NotDisposedAfterRequestAsync() + { + // Arrange + var clientA = CreateMockClient(); + var routing = new RoutingChatClient(new Dictionary { ["a"] = clientA.Object }); + var agent = new ChatClientAgent(routing); + var session = await agent.CreateSessionAsync(); + + // Act + await agent.RunAsync([new ChatMessage(ChatRole.User, "hi")], session); + + // Assert + clientA.Verify(c => c.GetResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny()), Times.Once); + clientA.Verify(c => c.Dispose(), Times.Never); + } + + /// + /// Verify that a client with no inner clients routes every request via the fallback factory. + /// + [Fact] + public async Task NoInnerClients_RoutesViaFallbackAsync() + { + // Arrange + var fallback = CreateMockClient(); + var routing = new RoutingChatClient( + fallbackFactory: (_, _, _) => new ValueTask(fallback.Object), + options: new RoutingChatClientOptions + { + Router = (_, _) => new ValueTask("anything"), + }); + var agent = new ChatClientAgent(routing); + var session = await agent.CreateSessionAsync(); + + // Act + await agent.RunAsync([new ChatMessage(ChatRole.User, "hi")], session); + + // Assert + fallback.Verify(c => c.GetResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny()), Times.Once); + } + + /// + /// Verify that a fallback-only client with no router and a default (null) active destination invokes the + /// fallback factory with a key. + /// + [Fact] + public async Task NoInnerClients_NoRouter_InvokesFallbackWithNullKeyAsync() + { + // Arrange + var fallback = CreateMockClient(); + string? observedKey = "sentinel"; + var routing = new RoutingChatClient( + fallbackFactory: (key, _, _) => + { + observedKey = key; + return new ValueTask(fallback.Object); + }); + var agent = new ChatClientAgent(routing); + var session = await agent.CreateSessionAsync(); + + // Act + await agent.RunAsync([new ChatMessage(ChatRole.User, "hi")], session); + + // Assert + Assert.Null(observedKey); + fallback.Verify(c => c.GetResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny()), Times.Once); + } + + #endregion + + #region State key + + /// + /// Verify that a custom stores routing state under that key. + /// + [Fact] + public async Task CustomStateKey_StoresStateUnderProvidedKeyAsync() + { + // Arrange + const string StateKey = "my-routing-key"; + var routing = new RoutingChatClient( + new Dictionary { ["a"] = CreateMockClient().Object, ["b"] = CreateMockClient().Object }, + options: new RoutingChatClientOptions { StateKey = StateKey }); + var agent = new ChatClientAgent(routing); + var session = await agent.CreateSessionAsync(); + + // Act + routing.SetActiveDestinationKey(session, "b"); + + // Assert + Assert.True(session.StateBag.TryGetValue(StateKey, out var state)); + Assert.Equal("b", state!.ActiveDestination); + } + + #endregion + + #region Helpers + + private static Mock CreateMockClient(string responseText = "ok") + { + var mock = new Mock(); + mock.Setup(c => c.GetResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny())) + .ReturnsAsync(new ChatResponse([new ChatMessage(ChatRole.Assistant, responseText)])); + mock.Setup(c => c.GetStreamingResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny())) + .Returns(ToAsyncEnumerableAsync(new ChatResponseUpdate(ChatRole.Assistant, responseText))); + return mock; + } + + private static async IAsyncEnumerable ToAsyncEnumerableAsync(params ChatResponseUpdate[] updates) + { + foreach (var update in updates) + { + yield return update; + } + + await Task.CompletedTask; + } + + #endregion +} From b04abf1646d831c7b997f5295de566055f4fe899 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Mon, 6 Jul 2026 15:06:28 +0000 Subject: [PATCH 2/3] Address PR comments --- .../Agents/Agent_Step22_MultiModelRouting/Program.cs | 2 +- .../Microsoft.Agents.AI/ChatClient/RoutingChatClient.cs | 7 ++++--- .../ChatClient/RoutingChatClientOptions.cs | 9 +++++---- .../src/Microsoft.Agents.AI/ChatClient/RoutingContext.cs | 4 ++-- .../src/Microsoft.Agents.AI/ChatClient/RoutingState.cs | 8 +++++--- 5 files changed, 17 insertions(+), 13 deletions(-) diff --git a/dotnet/samples/02-agents/Agents/Agent_Step22_MultiModelRouting/Program.cs b/dotnet/samples/02-agents/Agents/Agent_Step22_MultiModelRouting/Program.cs index a3f6dcca884..6d97ee17dc6 100644 --- a/dotnet/samples/02-agents/Agents/Agent_Step22_MultiModelRouting/Program.cs +++ b/dotnet/samples/02-agents/Agents/Agent_Step22_MultiModelRouting/Program.cs @@ -96,7 +96,7 @@ AgentSession session = await agent.CreateSessionAsync(); // The RoutingChatClient can be retrieved back from the agent via GetService. -// This is uesful when you don't hold a direct reference to it — for example when +// This is useful when you don't hold a direct reference to it — for example when // the agent was created elsewhere or resolved from a DI container. RoutingChatClient routing = agent.GetService() ?? throw new InvalidOperationException("The agent is not backed by a RoutingChatClient."); diff --git a/dotnet/src/Microsoft.Agents.AI/ChatClient/RoutingChatClient.cs b/dotnet/src/Microsoft.Agents.AI/ChatClient/RoutingChatClient.cs index b9336154512..f2da7ed6a49 100644 --- a/dotnet/src/Microsoft.Agents.AI/ChatClient/RoutingChatClient.cs +++ b/dotnet/src/Microsoft.Agents.AI/ChatClient/RoutingChatClient.cs @@ -250,8 +250,8 @@ public async IAsyncEnumerable GetStreamingResponseAsync( /// /// The session whose active destination key should be returned. /// - /// The active destination key for the session, or when the default destination - /// (the first inner client) is in effect. + /// The active destination key for the session, or when the request is routed directly + /// to the fallback factory. /// /// Thrown when is . public string? GetActiveDestinationKey(AgentSession session) @@ -267,7 +267,8 @@ public async IAsyncEnumerable GetStreamingResponseAsync( /// The session whose active destination key should be updated. /// /// The destination key to make active. May be any string (a registered inner client key or a key handled by - /// the fallback factory), or to use the default destination (the first inner client). + /// the fallback factory), or to route the request directly to the fallback factory + /// (invoked with a key), which throws if no fallback factory is configured. /// /// Thrown when is . public void SetActiveDestinationKey(AgentSession session, string? destinationKey) diff --git a/dotnet/src/Microsoft.Agents.AI/ChatClient/RoutingChatClientOptions.cs b/dotnet/src/Microsoft.Agents.AI/ChatClient/RoutingChatClientOptions.cs index 41e4dbed999..36aeff6e914 100644 --- a/dotnet/src/Microsoft.Agents.AI/ChatClient/RoutingChatClientOptions.cs +++ b/dotnet/src/Microsoft.Agents.AI/ChatClient/RoutingChatClientOptions.cs @@ -20,10 +20,11 @@ public sealed class RoutingChatClientOptions /// /// /// The function receives a and a and returns - /// the key of the destination that should handle the request, or to route to the - /// default destination (the first inner client). It is asynchronous so callers can perform I/O - /// (for example, an inference call) to decide the best destination. When (the default), - /// the currently active destination for the session is used (see ). + /// the key of the destination that should handle the request, or to route the request + /// directly to the fallback factory (invoked with a key). It is asynchronous so callers + /// can perform I/O (for example, an inference call) to decide the best destination. When + /// (the default), the currently active destination for the session is used (see + /// ). /// /// /// The returned key is used only for the current request; it does not change the session's active diff --git a/dotnet/src/Microsoft.Agents.AI/ChatClient/RoutingContext.cs b/dotnet/src/Microsoft.Agents.AI/ChatClient/RoutingContext.cs index afbe2b35f44..3aa53fcc604 100644 --- a/dotnet/src/Microsoft.Agents.AI/ChatClient/RoutingContext.cs +++ b/dotnet/src/Microsoft.Agents.AI/ChatClient/RoutingContext.cs @@ -66,8 +66,8 @@ public RoutingContext( public IReadOnlyDictionary InnerClients { get; } /// - /// Gets the currently active destination key for the session, or when the default - /// destination (the first inner client) should be used. + /// Gets the currently active destination key for the session, or when the request + /// is routed directly to the fallback factory. /// /// /// This is the value returned by the default router. It reflects the destination stored in the diff --git a/dotnet/src/Microsoft.Agents.AI/ChatClient/RoutingState.cs b/dotnet/src/Microsoft.Agents.AI/ChatClient/RoutingState.cs index 1dc53084ecf..b283888f133 100644 --- a/dotnet/src/Microsoft.Agents.AI/ChatClient/RoutingState.cs +++ b/dotnet/src/Microsoft.Agents.AI/ChatClient/RoutingState.cs @@ -19,13 +19,15 @@ public sealed class RoutingState { /// /// Gets or sets the key of the currently active destination for this session, or - /// to use the default destination. + /// to route the request directly to the fallback factory. /// /// /// When non-, the value corresponds to a key in the inner clients dictionary supplied /// to the (or a key handled by a fallback factory). It is used by the default - /// router to select a destination. When , the request is routed to the first inner - /// client (the default destination). + /// router to select a destination. When , the request is routed directly to the fallback + /// factory (invoked with a key) without a dictionary lookup, and an + /// is thrown if no fallback factory is configured. A new + /// session's default destination (the first inner client) is set by the state initializer. /// [JsonPropertyName("activeDestination")] public string? ActiveDestination { get; set; } From b120aefbea686eda0d93c6a4b770a463698d6e27 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Mon, 6 Jul 2026 15:49:31 +0000 Subject: [PATCH 3/3] Add information on setting a custom router --- .../Agents/Agent_Step22_MultiModelRouting/Program.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dotnet/samples/02-agents/Agents/Agent_Step22_MultiModelRouting/Program.cs b/dotnet/samples/02-agents/Agents/Agent_Step22_MultiModelRouting/Program.cs index 6d97ee17dc6..0d4b4ec31ae 100644 --- a/dotnet/samples/02-agents/Agents/Agent_Step22_MultiModelRouting/Program.cs +++ b/dotnet/samples/02-agents/Agents/Agent_Step22_MultiModelRouting/Program.cs @@ -86,6 +86,13 @@ { Console.WriteLine($" (fallback factory building a client for model '{destinationKey}')"); return new ValueTask(responsesClient.AsIChatClientWithStoredOutputDisabled(destinationKey, includeReasoningEncryptedContent: false)); + }, + options: new RoutingChatClientOptions + { + // If set, can be used to override the active destination in session state for a request. + // E.g. you could implement a routing heuristic that inspects the request and chooses a model based on its content, + // or even perform an inference call to a model to decide which model should handle the request. + Router = null, }); AIAgent agent = new ChatClientAgent(