Issue #651 文字送り中の領域内Busyと最終文字列翻訳を実装

WindowTranslator.Abstractions/TextRect.cs

@@ -32,6 +32,16 @@ public record TextRect(string SourceText, double X, double Y, double Width, doub
     /// </summary>
     public double MaxWidth { get; init; } = double.NaN;
 
+    /// <summary>
+    /// 表示可能な翻訳結果を待っている理由。
+    /// </summary>
+    public TextRegionBusyReason BusyReasons { get; init; }
+
+    /// <summary>
+    /// 領域内にBusyを表示するかどうか。
+    /// </summary>
+    public bool IsBusy => this.BusyReasons != TextRegionBusyReason.None;
+
     /// <summary>
     /// コンストラクタ
     /// </summary>
@@ -163,4 +173,4 @@ public record TextInfo(string SourceText, string? TranslatedText)
     /// このテキストの文脈
     /// </summary>
     public string Context { get; init; } = string.Empty;
-};
+};

WindowTranslator.Abstractions/TextRegionBusyReason.cs

@@ -0,0 +1,23 @@
+namespace WindowTranslator;
+
+/// <summary>
+/// テキスト領域が表示可能な翻訳結果を待っている理由。
+/// </summary>
+[Flags]
+public enum TextRegionBusyReason
+{
+    /// <summary>
+    /// 待機していない。
+    /// </summary>
+    None = 0,
+
+    /// <summary>
+    /// 翻訳結果を待っている。
+    /// </summary>
+    Translation = 1 << 0,
+
+    /// <summary>
+    /// 文字送りの完了を待っている。
+    /// </summary>
+    Typewriter = 1 << 1,
+}

WindowTranslator.Tests/OcrTypewriterTrackingTests.cs

@@ -0,0 +1,172 @@
+using System.Drawing;
+using Microsoft.Extensions.Logging.Abstractions;
+using WindowTranslator.Modules.Ocr;
+
+namespace WindowTranslator.Tests;
+
+public sealed class OcrTypewriterTrackingTests
+{
+    private static readonly Size imageSize = new(1280, 720);
+
+    [Fact]
+    public void ProgressiveTextStaysBusyAndOnlyTheFinalTextBecomesTranslatable()
+    {
+        OcrTextTracker tracker = CreateTracker();
+        List<string> translationRequests = [];
+
+        ObserveAndCollect(tracker, translationRequests, 0, Rect("H"));
+        TextRect he = ObserveAndCollect(tracker, translationRequests, 500, Rect("He")).Single();
+        TextRect hel = ObserveAndCollect(tracker, translationRequests, 1000, Rect("Hel")).Single();
+        TextRect helloBusy = ObserveAndCollect(tracker, translationRequests, 1500, Rect("Hello")).Single();
+        TextRect hello = ObserveAndCollect(tracker, translationRequests, 2000, Rect("Hello")).Single();
+
+        Assert.Equal(TextRegionBusyReason.Typewriter, he.BusyReasons);
+        Assert.Equal(TextRegionBusyReason.Typewriter, hel.BusyReasons);
+        Assert.Equal(TextRegionBusyReason.Typewriter, helloBusy.BusyReasons);
+        Assert.False(hello.IsBusy);
+        Assert.Equal("Hello", hello.SourceText);
+        Assert.Equal(["H", "Hello"], translationRequests);
+    }
+
+    [Fact]
+    public void OscillatingTailSelectsTheMostProgressedCandidateAndLeavesTypewriterBusy()
+    {
+        OcrTextTracker tracker = CreateTracker();
+        List<string> translationRequests = [];
+
+        ObserveAndCollect(tracker, translationRequests, 0, Rect("H"));
+        ObserveAndCollect(tracker, translationRequests, 500, Rect("He"));
+        ObserveAndCollect(tracker, translationRequests, 1000, Rect("Hel"));
+        ObserveAndCollect(tracker, translationRequests, 1500, Rect("Hello"));
+        ObserveAndCollect(tracker, translationRequests, 2000, Rect("Hell0"));
+        ObserveAndCollect(tracker, translationRequests, 2500, Rect("Hello"));
+        TextRect final = ObserveAndCollect(tracker, translationRequests, 3000, Rect("Hell0")).Single();
+
+        Assert.False(final.IsBusy);
+        Assert.Equal("Hello", final.SourceText);
+        Assert.Equal(["H", "Hello"], translationRequests);
+    }
+
+    [Fact]
+    public void ReturningToTheConfirmedTextCancelsTypewriterBusy()
+    {
+        OcrTextTracker tracker = CreateTracker();
+
+        TextRect original = Update(tracker, 0, Rect("Menu")).Single();
+        TextRect progressing = Update(tracker, 500, Rect("Menu...")).Single();
+        TextRect restored = Update(tracker, 1000, Rect("Menu")).Single();
+
+        Assert.False(original.IsBusy);
+        Assert.Equal(TextRegionBusyReason.Typewriter, progressing.BusyReasons);
+        Assert.False(restored.IsBusy);
+        Assert.Equal("Menu", restored.SourceText);
+    }
+
+    [Fact]
+    public void OrdinaryReplacementUsesTheExistingTextConfirmation()
+    {
+        OcrTextTracker tracker = CreateTracker();
+
+        Update(tracker, 0, Rect("Menu"));
+        TextRect candidate = Update(tracker, 500, Rect("Game")).Single();
+        TextRect confirmed = Update(tracker, 1000, Rect("Game")).Single();
+
+        Assert.False(candidate.IsBusy);
+        Assert.Equal("Menu", candidate.SourceText);
+        Assert.False(confirmed.IsBusy);
+        Assert.Equal("Game", confirmed.SourceText);
+    }
+
+    [Fact]
+    public void ProgressionCanStartAfterThePreviousTextWasReplaced()
+    {
+        OcrTextTracker tracker = CreateTracker();
+
+        Update(tracker, 0, Rect("Menu"));
+        TextRect firstCharacter = Update(tracker, 500, Rect("H")).Single();
+        TextRect progressing = Update(tracker, 1000, Rect("He")).Single();
+
+        Assert.False(firstCharacter.IsBusy);
+        Assert.Equal("Menu", firstCharacter.SourceText);
+        Assert.Equal(TextRegionBusyReason.Typewriter, progressing.BusyReasons);
+    }
+
+    [Fact]
+    public void ASingleExtensionAfterCompletionUsesTheExistingStabilization()
+    {
+        OcrTextTracker tracker = CreateTracker();
+
+        Update(tracker, 0, Rect("H"));
+        Update(tracker, 500, Rect("He"));
+        Update(tracker, 1000, Rect("Hello"));
+        TextRect completed = Update(tracker, 1500, Rect("Hello")).Single();
+        TextRect noise = Update(tracker, 2000, Rect("Hello!")).Single();
+
+        Assert.False(completed.IsBusy);
+        Assert.Equal("Hello", completed.SourceText);
+        Assert.False(noise.IsBusy);
+        Assert.Equal("Hello", noise.SourceText);
+    }
+
+    [Fact]
+    public void RapidRepeatedFramesDoNotEndTypewriterBeforeThePauseThreshold()
+    {
+        OcrTextTracker tracker = CreateTracker();
+
+        Update(tracker, 0, Rect("H"));
+        Update(tracker, 50, Rect("He"));
+        TextRect earlyRepeat = Update(tracker, 100, Rect("He")).Single();
+        TextRect stillProgressing = Update(tracker, 400, Rect("He")).Single();
+        TextRect completed = Update(tracker, 550, Rect("He")).Single();
+
+        Assert.True(earlyRepeat.IsBusy);
+        Assert.True(stillProgressing.IsBusy);
+        Assert.False(completed.IsBusy);
+        Assert.Equal("He", completed.SourceText);
+    }
+
+    [Fact]
+    public void ATypewriterTrackDoesNotBlockAnotherLogicalTrack()
+    {
+        OcrTextTracker tracker = CreateTracker();
+
+        Update(tracker, 0, Rect("H"), Rect("Status", x: 400));
+        IReadOnlyList<TextRect> output = Update(
+            tracker,
+            500,
+            Rect("He"),
+            Rect("Status", x: 400));
+
+        TextRect typewriter = Assert.Single(output, text => text.X == 100);
+        TextRect stable = Assert.Single(output, text => text.X == 400);
+        Assert.True(typewriter.IsBusy);
+        Assert.False(stable.IsBusy);
+        Assert.Equal("Status", stable.SourceText);
+    }
+
+    private static OcrTextTracker CreateTracker()
+        => new(NullLogger<OcrTextTracker>.Instance);
+
+    private static IReadOnlyList<TextRect> ObserveAndCollect(
+        OcrTextTracker tracker,
+        List<string> translationRequests,
+        int milliseconds,
+        params TextRect[] observations)
+    {
+        IReadOnlyList<TextRect> output = Update(tracker, milliseconds, observations);
+        translationRequests.AddRange(output
+            .Where(text => !text.IsBusy)
+            .Select(text => text.SourceText)
+            .Where(text => !translationRequests.Contains(text, StringComparer.Ordinal)));
+        return output;
+    }
+
+    private static IReadOnlyList<TextRect> Update(
+        OcrTextTracker tracker,
+        int milliseconds,
+        params TextRect[] observations)
+        => tracker.Update(observations, imageSize, TimeSpan.FromMilliseconds(milliseconds));
+
+    private static TextRect Rect(string text, double x = 100)
+        => new(text, x, 100, 160, 30, 20, false);
+}

WindowTranslator/Data/BoolToDataTemplateConverter.cs

@@ -0,0 +1,19 @@
+using System.Globalization;
+using System.Windows;
+using System.Windows.Data;
+
+namespace WindowTranslator.Data;
+
+[ValueConversion(typeof(bool), typeof(DataTemplate))]
+public sealed class BoolToDataTemplateConverter : IValueConverter
+{
+    public required DataTemplate FalseContent { get; set; }
+
+    public required DataTemplate TrueContent { get; set; }
+
+    public object Convert(object value, Type targetType, object parameter, CultureInfo culture)
+        => value is true ? this.TrueContent : this.FalseContent;
+
+    public object ConvertBack(object value, Type targetType, object parameter, CultureInfo culture)
+        => throw new NotSupportedException();
+}

WindowTranslator/Modules/Main/MainViewModelBase.cs

@@ -212,10 +212,19 @@ private async Task CreateTextOverlayAsync()
                 using var t = this.logger.LogDebugTime("PreTranslate");
                 texts = await tmp.ToArrayAsync();
             }
-            TranslateAsync(texts).Forget();
+            TranslateAsync(texts.Where(t => !t.IsBusy)).Forget();
             texts = texts.Select(t => t switch
             {
-                { TranslatedText: null } when this.cache.Contains(t.SourceText) => t with { TranslatedText = this.cache.Get(t.SourceText) },
+                { IsBusy: true } => t with { TranslatedText = null },
+                { TranslatedText: null } when this.cache.Contains(t.SourceText) => t with
+                {
+                    TranslatedText = this.cache.Get(t.SourceText),
+                    BusyReasons = t.BusyReasons & ~TextRegionBusyReason.Translation,
+                },
+                { TranslatedText: null } => t with
+                {
+                    BusyReasons = t.BusyReasons | TextRegionBusyReason.Translation,
+                },
                 _ => t,
             }).ToArray();
             {
@@ -260,6 +269,7 @@ private async Task TranslateAsync(IEnumerable<TextRect> texts)
                 return;
             }
             requests = requests
+                .Where(t => !t.IsBusy)
                 .Where(t => t.TranslatedText is null)
                 .Where(t => !this.cache.Contains(t.SourceText))
                 .ToArray();