Rebuilt how AI Studio knows what a model can do

.github/workflows/build-and-release.yml

@@ -724,9 +724,94 @@ jobs:
           overwrite: true
           retention-days: ${{ env.RETENTION_INTERMEDIATE_ASSETS }}
 
+  #
+  # The quality gate. Deliberately without an `if` on build_enabled: a pull request only builds
+  # when somebody sets the run-pipeline label, and a gate which is closed exactly while nobody is
+  # looking is not a gate. It is cheap for the same reason it is unconditional -- one platform, no
+  # Tauri bundle, no signing, no artifacts.
+  #
+  verify:
+    name: Verify
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Read the toolchain versions from the metadata
+        run: |
+          # The .NET SDK version. The format is '9.0.205 (commit 3e1383b780)',
+          # so we extract the version number alone:
+          dotnet_sdk_version=$(sed -n '4p' metadata.txt | sed 's/[^0-9.]*\([0-9.]*\).*/\1/')
+
+          # The Rust version, written the same way:
+          rust_version=$(sed -n '6p' metadata.txt | sed 's/[^0-9.]*\([0-9.]*\).*/\1/')
+
+          echo "DOTNET_SDK_VERSION=${dotnet_sdk_version}" >> $GITHUB_ENV
+          echo "RUST_VERSION=${rust_version}" >> $GITHUB_ENV
+
+          echo ".NET SDK version: '${dotnet_sdk_version}'"
+          echo "Rust version: '${rust_version}'"
+
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: ${{ env.DOTNET_SDK_VERSION }}
+          cache: true
+          cache-dependency-path: 'app/MindWork AI Studio/packages.lock.json'
+
+      - name: Cache Rust
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/git/db/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            runtime/target
+
+          key: verify-linux-x64-rust-${{ env.RUST_VERSION }}
+
+      - name: Setup Rust (stable)
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: ${{ env.RUST_VERSION }}
+          components: clippy
+
+      - name: Setup dependencies (Ubuntu-specific)
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libwebkit2gtk-4.1-dev libayatana-appindicator3-dev librsvg2-dev patchelf libfuse2 xdg-utils gstreamer1.0-plugins-base gstreamer1.0-plugins-good
+
+      - name: Place stand-ins for what Tauri's build script expects
+        run: |
+          # Tauri's build script insists that everything the configuration lists is already there,
+          # and refuses to run otherwise -- so nothing Rust compiles without it. Two of those
+          # things are products of a build which has not run here: the .NET app as a sidecar, and
+          # the PDF library which the build downloads into the resources. The other two resource
+          # directories, notices and tokenizers, are in the repository and need nothing.
+          #
+          # This job never bundles anything and never starts the app; it compiles, tests and lints
+          # the Rust code, and no test opens either file. Empty stand-ins are therefore enough,
+          # while publishing the sidecar and downloading the library would cost minutes for files
+          # nobody here reads. Should a Rust test ever need the real library, this job has to
+          # deploy it the way build_main does instead of placing a stand-in.
+          mkdir -p "app/MindWork AI Studio/bin/dist"
+          touch "app/MindWork AI Studio/bin/dist/mindworkAIStudioServer-x86_64-unknown-linux-gnu"
+          chmod +x "app/MindWork AI Studio/bin/dist/mindworkAIStudioServer-x86_64-unknown-linux-gnu"
+
+          mkdir -p runtime/resources/libraries
+          touch runtime/resources/libraries/stand-in-for-verify.txt
+
+      - name: Run the quality gate
+        run: |
+          cd "app/Build"
+          dotnet run verify
+
   build_main:
     name: Build app (${{ matrix.dotnet_runtime }})
-    needs: [determine_run_mode, read_metadata]
+    needs: [determine_run_mode, read_metadata, verify]
     if: needs.determine_run_mode.outputs.build_enabled == 'true'
     permissions:
       contents: read

.gitignore

@@ -172,3 +172,6 @@ orleans.codegen.cs
 
 # Tauri generated schemas/manifests
 /runtime/gen/
+
+# Ignore what a failing snapshot test leaves behind for comparison:
+/app/Tests/Models/Corpus/CapabilitySnapshot.actual.txt

AGENTS.md

@@ -80,7 +80,21 @@ Notes:
   troubleshooting, no matter whether it came from the MCP server or from the user.
 
 ### Running Tests
-Currently, no automated test suite exists in the repository.
+The .NET tests live in `app/Tests`, a single NUnit project that holds the tests of every area; each
+area gets its own folder and namespace below it rather than a project of its own. Agents run them
+through the IDE for the same reason they build there:
+
+```
+mcp__rider__execute_terminal_command  command: "cd app/Tests && dotnet test"
+```
+
+An assembly-wide `[SetUpFixture]` in `app/Tests/TestHost.cs` fills the static application state that
+the app itself only fills while starting up, `Program.LOGGER_FACTORY` above all. Types that
+initialize a static logger from it — `Settings.Provider` among them — otherwise die in their type
+initializer before the first assertion. Prefer writing new code so that it does not reach for such
+statics at all.
+
+The Rust tests run with `cargo test` in `runtime/`, through the `rustrover` MCP server.
 
 ## Architecture Details
 
@@ -141,7 +155,8 @@ Key structure:
 Plugins are written in Lua and provide:
 - **Language plugins** - I18N translations (e.g., German language pack)
 - **Configuration plugins** - Enterprise IT configurations for centrally managed providers, settings
-- **Future:** Assistant plugins for custom assistants
+- **Assistant plugins** - custom assistants and direct-chat launchers, subject to approval or a local security audit
+- **Model plugins** - what an organization's own models can do, see `documentation/Models.md`
 
 **Example configuration plugin:** `app/MindWork AI Studio/Plugins/configuration/plugin.lua`
 
@@ -177,6 +192,21 @@ When adding, changing, or removing model-driven tools, keep these parts in sync:
 
 Tool implementations must treat model-provided arguments as untrusted input. Validate settings and arguments, protect secrets with `SensitiveTraceArgumentNames`, use `ToolExecutionBlockedException` for intentional policy blocks, and check provider confidence before returning sensitive data to the model.
 
+## Model Capabilities
+
+**Documentation:** `documentation/Models.md`
+
+What a model can do is answered in `app/MindWork AI Studio/Models/`, through `provider.GetModelProfile()`. Never ask `ModelRegistry` directly from a component: the extension method is what adds the expert settings and what a provider's model list reported, and the registry alone answers neither.
+
+When adding, changing, or removing model knowledge, keep these parts in sync:
+- `app/MindWork AI Studio/Models/<Vendor>/<Family>.cs` for the family itself. Creating the class is enough — the source generator in `app/SourceGeneratedMappings/` collects every non-abstract `ModelFamily` and `IModelHost` at compile time, so there is no registration list. Do not add reflection here; `PublishTrimmed` is on.
+- `app/Tests/Models/Corpus/` for the model IDs the family covers, marked as either unchanged or expected to change. A porting difference which nobody declared is what the corpus exists to catch.
+- `app/MindWork AI Studio/Models/Kinds/` when the change is about what kind of model something is, rather than what it can do. These are ordinary rules of the same engine.
+- `app/MindWork AI Studio/Models/Hosting/Hosts/` when a provider wraps model names or cannot pass an API through. A host unwraps and trims the transport; it states nothing about the model itself.
+- `app/MindWork AI Studio/Plugins/models/plugin.lua` when a new field can be declared by an organization, and `app/MindWork AI Studio/Plugins/configuration/plugin.lua` when it can be overridden per provider instance.
+
+Rules are never tried in order: specificity is computed from the rule, and two rules of equal specificity on one name fail the test suite. State how a model reasons with `Reasoning(...)` — the three reasoning capabilities are override vocabulary and must never appear in a profile. Every family and every host has to name the page it was read from and the day somebody read it; `dotnet run verify-models` reports the ones which have gone stale.
+
 ## RAG (Retrieval-Augmented Generation)
 
 RAG integration is currently in development (preview feature). Architecture:

README.md

@@ -189,6 +189,8 @@ You want to know how to build MindWork AI Studio from source? [Check out the ins
 
 Do you want to add or maintain model-driven tools? [Read the tool development guide here](documentation/Tools.md).
 
+Do you want to teach AI Studio what a model can do? [Read the model capabilities guide here](documentation/Models.md).
+
 </details>
 
 <details>

app/Build/Commands/UpdateMetadataCommands.cs

@@ -87,8 +87,10 @@ private async Task BuildPreparedRelease(bool offline)
         await new CollectI18NKeysCommand().CollectI18NKeys();
 
         // Build the final release, where Rust knows the updated metadata, the .NET
-        // artifacts are already in place, and .NET knows the updated web assets, etc.:
-        await this.Build(offline);
+        // artifacts are already in place, and .NET knows the updated web assets, etc.
+        // The gate already ran in the first build; running it a second time on the same
+        // sources would only add minutes:
+        await this.Build(offline, skipVerify: true);
     }
 
     [Command("update-metainfo", Description = "Update the AppStream metainfo entry of one release from its changelog")]
@@ -221,11 +223,21 @@ private async Task UpdateReleaseDependenciesAndLicence()
 
     [Command("build", Description = "Build MindWork AI Studio")]
     public async Task Build(
-        [Option("offline", Description = "Skip downloads and use locally available build dependencies")] bool offline = false)
+        [Option("offline", Description = "Skip downloads and use locally available build dependencies")] bool offline = false,
+        [Option("skip-verify", Description = "Skip the quality gate which otherwise runs before anything is built")] bool skipVerify = false)
     {
         if(!Environment.IsWorkingDirectoryValid())
             return;
-
+
+        //
+        // The gate runs before anything is built, and the build stops when it does not pass. That
+        // way the same command answers both questions a person has -- is it sound, and does it
+        // build -- and answers them in that order, because building something the tests reject
+        // takes minutes to produce an artifact nobody should use.
+        //
+        if (!skipVerify && await new VerifyCommand().Verify() is not 0)
+            throw new CommandExitedException(1);
+
         //
         // Build the .NET project:
         //
@@ -596,7 +608,7 @@ private static async Task WriteMetainfoRelease(string appVersion, DateTime relea
 
         // Drop any earlier entry of this version, so that the version stays unique and moves to the top.
         // We remove from the back, so that the index of the remaining matches stays valid:
-        foreach (var previousRelease in ReleaseBlockRegex().Matches(metainfo).Cast<Match>().Where(match => ReleaseTagHasVersion(match.Value, appVersion)).Reverse())
+        foreach (var previousRelease in ReleaseBlockRegex().Matches(metainfo).Where(match => ReleaseTagHasVersion(match.Value, appVersion)).Reverse())
             metainfo = metainfo.Remove(previousRelease.Index, previousRelease.Length);
 
         var lineEnding = metainfo.Contains("\r\n", StringComparison.Ordinal) ? "\r\n" : "\n";

app/Build/Commands/VerifyCommand.cs

@@ -0,0 +1,92 @@
+using Build.Tools;
+
+// ReSharper disable ClassNeverInstantiated.Global
+// ReSharper disable UnusedType.Global
+// ReSharper disable UnusedMember.Global
+namespace Build.Commands;
+
+/// <summary>
+/// The quality gate: one command, the same one locally and in the pipeline.
+/// </summary>
+/// <remarks>
+/// Every check runs, even after one of them has failed. A gate which stops at the first failure
+/// tells you one thing per run, and the next run costs the same minutes again -- while the point of
+/// running the whole thing is to learn everything which is wrong in one go.
+/// </remarks>
+public sealed class VerifyCommand
+{
+    /// <summary>
+    /// How the .NET app is named once it lies where Tauri expects it.
+    /// </summary>
+    private const string SIDECAR_PREFIX = "mindworkAIStudioServer-";
+
+    [Command("verify", Description = "Run the quality gate: .NET tests, Rust tests, Clippy, and the model sources")]
+    public async Task<int> Verify()
+    {
+        if(!Environment.IsWorkingDirectoryValid())
+            return 1;
+
+        Console.WriteLine("==============================");
+        Console.WriteLine("- Quality gate: every check runs, so that the first failure does not hide the next ...");
+
+        var results = new List<(string What, int ExitCode)>
+        {
+            (".NET tests", await CommandRunner.RunAsync(Environment.GetTestsDirectory(), "dotnet", "test --nologo")),
+        };
+
+        var runtimeDirectory = Environment.GetRustRuntimeDirectory();
+        if (WhatTauriExpectsIsThere())
+        {
+            results.Add(("Rust tests", await CommandRunner.RunAsync(runtimeDirectory, "cargo", "test")));
+            results.Add(("Clippy", await CommandRunner.RunAsync(runtimeDirectory, "cargo", "clippy --all-targets -- -D warnings")));
+        }
+        else
+        {
+            //
+            // Tauri's build script insists that everything the configuration lists is already
+            // there and refuses to run otherwise, so nothing Rust compiles until a build has
+            // produced those files once. Failing here would be a trap rather than a gate: the way
+            // to produce them is `dotnet run build`, and that command runs this gate first -- a
+            // fresh clone would never get past it.
+            //
+            Console.WriteLine("- Skipping the Rust tests and Clippy: the .NET sidecar or the downloaded libraries are missing, and Tauri's build script needs both before anything Rust compiles.");
+            Console.WriteLine("   Run 'dotnet run build --skip-verify' once. From then on, this part of the gate runs with the rest.");
+        }
+
+        results.Add(("Model sources", new VerifyModelsCommand().VerifyModels()));
+
+        Console.WriteLine("==============================");
+        Console.WriteLine("- Quality gate:");
+        foreach (var (what, exitCode) in results)
+            Console.WriteLine($"   - {what}: {(exitCode is 0 ? "passed" : $"failed, exit code {exitCode}")}");
+
+        var failed = results.Count(result => result.ExitCode is not 0);
+        if (failed is 0)
+        {
+            Console.WriteLine($"- All {results.Count} checks passed.");
+            return 0;
+        }
+
+        Console.WriteLine($"- {failed} of {results.Count} checks failed.");
+        return 1;
+    }
+
+    /// <summary>
+    /// Whether a build has already produced the files Tauri's build script reads.
+    /// </summary>
+    /// <remarks>
+    /// Both are products of a build rather than of the repository: the .NET app arrives as a
+    /// sidecar, and the PDF library is downloaded into the resources. The other resource
+    /// directories the configuration names are in the repository and are always there.
+    /// </remarks>
+    /// <returns>True, when cargo can get past the build script.</returns>
+    private static bool WhatTauriExpectsIsThere()
+    {
+        var distributionDirectory = Path.Combine(Environment.GetAIStudioDirectory(), "bin", "dist");
+        if (!Directory.Exists(distributionDirectory) || !Directory.EnumerateFiles(distributionDirectory, $"{SIDECAR_PREFIX}*").Any())
+            return false;
+
+        var librariesDirectory = Path.Combine(Environment.GetRustRuntimeDirectory(), "resources", "libraries");
+        return Directory.Exists(librariesDirectory) && Directory.EnumerateFiles(librariesDirectory).Any();
+    }
+}