diff --git a/WindowTranslator.Abstractions/TextRect.cs b/WindowTranslator.Abstractions/TextRect.cs index f38b1733..1ddf6d8a 100644 --- a/WindowTranslator.Abstractions/TextRect.cs +++ b/WindowTranslator.Abstractions/TextRect.cs @@ -32,6 +32,16 @@ public record TextRect(string SourceText, double X, double Y, double Width, doub /// public double MaxWidth { get; init; } = double.NaN; + /// + /// 表示可能な翻訳結果を待っている理由。 + /// + public TextRegionBusyReason BusyReasons { get; init; } + + /// + /// 領域内にBusyを表示するかどうか。 + /// + public bool IsBusy => this.BusyReasons != TextRegionBusyReason.None; + /// /// コンストラクタ /// @@ -163,4 +173,4 @@ public record TextInfo(string SourceText, string? TranslatedText) /// このテキストの文脈 /// public string Context { get; init; } = string.Empty; -}; \ No newline at end of file +}; diff --git a/WindowTranslator.Abstractions/TextRegionBusyReason.cs b/WindowTranslator.Abstractions/TextRegionBusyReason.cs new file mode 100644 index 00000000..c1e4ba82 --- /dev/null +++ b/WindowTranslator.Abstractions/TextRegionBusyReason.cs @@ -0,0 +1,23 @@ +namespace WindowTranslator; + +/// +/// テキスト領域が表示可能な翻訳結果を待っている理由。 +/// +[Flags] +public enum TextRegionBusyReason +{ + /// + /// 待機していない。 + /// + None = 0, + + /// + /// 翻訳結果を待っている。 + /// + Translation = 1 << 0, + + /// + /// 文字送りの完了を待っている。 + /// + Typewriter = 1 << 1, +} diff --git a/WindowTranslator.Tests/OcrTypewriterTrackingTests.cs b/WindowTranslator.Tests/OcrTypewriterTrackingTests.cs new file mode 100644 index 00000000..b3bc12d7 --- /dev/null +++ b/WindowTranslator.Tests/OcrTypewriterTrackingTests.cs @@ -0,0 +1,172 @@ +using System.Drawing; +using Microsoft.Extensions.Logging.Abstractions; +using WindowTranslator.Modules.Ocr; + +namespace WindowTranslator.Tests; + +public sealed class OcrTypewriterTrackingTests +{ + private static readonly Size imageSize = new(1280, 720); + + [Fact] + public void ProgressiveTextStaysBusyAndOnlyTheFinalTextBecomesTranslatable() + { + OcrTextTracker tracker = CreateTracker(); + List translationRequests = []; + + ObserveAndCollect(tracker, translationRequests, 0, Rect("H")); + TextRect he = ObserveAndCollect(tracker, translationRequests, 500, Rect("He")).Single(); + TextRect hel = ObserveAndCollect(tracker, translationRequests, 1000, Rect("Hel")).Single(); + TextRect helloBusy = ObserveAndCollect(tracker, translationRequests, 1500, Rect("Hello")).Single(); + TextRect hello = ObserveAndCollect(tracker, translationRequests, 2000, Rect("Hello")).Single(); + + Assert.Equal(TextRegionBusyReason.Typewriter, he.BusyReasons); + Assert.Equal(TextRegionBusyReason.Typewriter, hel.BusyReasons); + Assert.Equal(TextRegionBusyReason.Typewriter, helloBusy.BusyReasons); + Assert.False(hello.IsBusy); + Assert.Equal("Hello", hello.SourceText); + Assert.Equal(["H", "Hello"], translationRequests); + } + + [Fact] + public void OscillatingTailSelectsTheMostProgressedCandidateAndLeavesTypewriterBusy() + { + OcrTextTracker tracker = CreateTracker(); + List translationRequests = []; + + ObserveAndCollect(tracker, translationRequests, 0, Rect("H")); + ObserveAndCollect(tracker, translationRequests, 500, Rect("He")); + ObserveAndCollect(tracker, translationRequests, 1000, Rect("Hel")); + ObserveAndCollect(tracker, translationRequests, 1500, Rect("Hello")); + ObserveAndCollect(tracker, translationRequests, 2000, Rect("Hell0")); + ObserveAndCollect(tracker, translationRequests, 2500, Rect("Hello")); + TextRect final = ObserveAndCollect(tracker, translationRequests, 3000, Rect("Hell0")).Single(); + + Assert.False(final.IsBusy); + Assert.Equal("Hello", final.SourceText); + Assert.Equal(["H", "Hello"], translationRequests); + } + + [Fact] + public void ReturningToTheConfirmedTextCancelsTypewriterBusy() + { + OcrTextTracker tracker = CreateTracker(); + + TextRect original = Update(tracker, 0, Rect("Menu")).Single(); + TextRect progressing = Update(tracker, 500, Rect("Menu...")).Single(); + TextRect restored = Update(tracker, 1000, Rect("Menu")).Single(); + + Assert.False(original.IsBusy); + Assert.Equal(TextRegionBusyReason.Typewriter, progressing.BusyReasons); + Assert.False(restored.IsBusy); + Assert.Equal("Menu", restored.SourceText); + } + + [Fact] + public void OrdinaryReplacementUsesTheExistingTextConfirmation() + { + OcrTextTracker tracker = CreateTracker(); + + Update(tracker, 0, Rect("Menu")); + TextRect candidate = Update(tracker, 500, Rect("Game")).Single(); + TextRect confirmed = Update(tracker, 1000, Rect("Game")).Single(); + + Assert.False(candidate.IsBusy); + Assert.Equal("Menu", candidate.SourceText); + Assert.False(confirmed.IsBusy); + Assert.Equal("Game", confirmed.SourceText); + } + + [Fact] + public void ProgressionCanStartAfterThePreviousTextWasReplaced() + { + OcrTextTracker tracker = CreateTracker(); + + Update(tracker, 0, Rect("Menu")); + TextRect firstCharacter = Update(tracker, 500, Rect("H")).Single(); + TextRect progressing = Update(tracker, 1000, Rect("He")).Single(); + + Assert.False(firstCharacter.IsBusy); + Assert.Equal("Menu", firstCharacter.SourceText); + Assert.Equal(TextRegionBusyReason.Typewriter, progressing.BusyReasons); + } + + [Fact] + public void ASingleExtensionAfterCompletionUsesTheExistingStabilization() + { + OcrTextTracker tracker = CreateTracker(); + + Update(tracker, 0, Rect("H")); + Update(tracker, 500, Rect("He")); + Update(tracker, 1000, Rect("Hello")); + TextRect completed = Update(tracker, 1500, Rect("Hello")).Single(); + TextRect noise = Update(tracker, 2000, Rect("Hello!")).Single(); + + Assert.False(completed.IsBusy); + Assert.Equal("Hello", completed.SourceText); + Assert.False(noise.IsBusy); + Assert.Equal("Hello", noise.SourceText); + } + + [Fact] + public void RapidRepeatedFramesDoNotEndTypewriterBeforeThePauseThreshold() + { + OcrTextTracker tracker = CreateTracker(); + + Update(tracker, 0, Rect("H")); + Update(tracker, 50, Rect("He")); + TextRect earlyRepeat = Update(tracker, 100, Rect("He")).Single(); + TextRect stillProgressing = Update(tracker, 400, Rect("He")).Single(); + TextRect completed = Update(tracker, 550, Rect("He")).Single(); + + Assert.True(earlyRepeat.IsBusy); + Assert.True(stillProgressing.IsBusy); + Assert.False(completed.IsBusy); + Assert.Equal("He", completed.SourceText); + } + + [Fact] + public void ATypewriterTrackDoesNotBlockAnotherLogicalTrack() + { + OcrTextTracker tracker = CreateTracker(); + + Update(tracker, 0, Rect("H"), Rect("Status", x: 400)); + IReadOnlyList output = Update( + tracker, + 500, + Rect("He"), + Rect("Status", x: 400)); + + TextRect typewriter = Assert.Single(output, text => text.X == 100); + TextRect stable = Assert.Single(output, text => text.X == 400); + Assert.True(typewriter.IsBusy); + Assert.False(stable.IsBusy); + Assert.Equal("Status", stable.SourceText); + } + + private static OcrTextTracker CreateTracker() + => new(NullLogger.Instance); + + private static IReadOnlyList ObserveAndCollect( + OcrTextTracker tracker, + List translationRequests, + int milliseconds, + params TextRect[] observations) + { + IReadOnlyList output = Update(tracker, milliseconds, observations); + translationRequests.AddRange(output + .Where(text => !text.IsBusy) + .Select(text => text.SourceText) + .Where(text => !translationRequests.Contains(text, StringComparer.Ordinal))); + return output; + } + + private static IReadOnlyList Update( + OcrTextTracker tracker, + int milliseconds, + params TextRect[] observations) + => tracker.Update(observations, imageSize, TimeSpan.FromMilliseconds(milliseconds)); + + private static TextRect Rect(string text, double x = 100) + => new(text, x, 100, 160, 30, 20, false); +} diff --git a/WindowTranslator/Data/BoolToDataTemplateConverter.cs b/WindowTranslator/Data/BoolToDataTemplateConverter.cs new file mode 100644 index 00000000..f1cbe2ef --- /dev/null +++ b/WindowTranslator/Data/BoolToDataTemplateConverter.cs @@ -0,0 +1,19 @@ +using System.Globalization; +using System.Windows; +using System.Windows.Data; + +namespace WindowTranslator.Data; + +[ValueConversion(typeof(bool), typeof(DataTemplate))] +public sealed class BoolToDataTemplateConverter : IValueConverter +{ + public required DataTemplate FalseContent { get; set; } + + public required DataTemplate TrueContent { get; set; } + + public object Convert(object value, Type targetType, object parameter, CultureInfo culture) + => value is true ? this.TrueContent : this.FalseContent; + + public object ConvertBack(object value, Type targetType, object parameter, CultureInfo culture) + => throw new NotSupportedException(); +} diff --git a/WindowTranslator/Modules/Main/MainViewModelBase.cs b/WindowTranslator/Modules/Main/MainViewModelBase.cs index f6cdfb2c..b647b140 100644 --- a/WindowTranslator/Modules/Main/MainViewModelBase.cs +++ b/WindowTranslator/Modules/Main/MainViewModelBase.cs @@ -212,10 +212,19 @@ private async Task CreateTextOverlayAsync() using var t = this.logger.LogDebugTime("PreTranslate"); texts = await tmp.ToArrayAsync(); } - TranslateAsync(texts).Forget(); + TranslateAsync(texts.Where(t => !t.IsBusy)).Forget(); texts = texts.Select(t => t switch { - { TranslatedText: null } when this.cache.Contains(t.SourceText) => t with { TranslatedText = this.cache.Get(t.SourceText) }, + { IsBusy: true } => t with { TranslatedText = null }, + { TranslatedText: null } when this.cache.Contains(t.SourceText) => t with + { + TranslatedText = this.cache.Get(t.SourceText), + BusyReasons = t.BusyReasons & ~TextRegionBusyReason.Translation, + }, + { TranslatedText: null } => t with + { + BusyReasons = t.BusyReasons | TextRegionBusyReason.Translation, + }, _ => t, }).ToArray(); { @@ -260,6 +269,7 @@ private async Task TranslateAsync(IEnumerable texts) return; } requests = requests + .Where(t => !t.IsBusy) .Where(t => t.TranslatedText is null) .Where(t => !this.cache.Contains(t.SourceText)) .ToArray(); diff --git a/WindowTranslator/Modules/Ocr/OcrTextTracker.cs b/WindowTranslator/Modules/Ocr/OcrTextTracker.cs index d4951043..b44073d7 100644 --- a/WindowTranslator/Modules/Ocr/OcrTextTracker.cs +++ b/WindowTranslator/Modules/Ocr/OcrTextTracker.cs @@ -23,6 +23,8 @@ public sealed class OcrTextTracker(ILogger logger) : IOcrTextTra private const double TextVoteDecay = 0.75; private const double TextVoteThreshold = 1.5; private const int TextVoteHistorySize = 5; + private const int TypewriterCandidateHistorySize = 7; + private const int TypewriterCooldownFrames = 2; private const double MinimumStructureSizeRatio = 0.65; private const double MinimumStructureOverlap = 0.45; private const double MinimumStructureTextSimilarity = 0.65; @@ -31,6 +33,8 @@ public sealed class OcrTextTracker(ILogger logger) : IOcrTextTra private const double StrongOneToOneScore = 0.9; private const double AngleVectorEpsilon = 0.000000000001; private static readonly TimeSpan dormantRetention = TimeSpan.FromSeconds(5); + private static readonly TimeSpan typewriterStableDuration = TimeSpan.FromMilliseconds(500); + private static readonly TimeSpan typewriterTailDuration = TimeSpan.FromMilliseconds(1500); private readonly ILogger logger = logger; private readonly object syncRoot = new(); @@ -1622,6 +1626,8 @@ private sealed class TextTrack( private TextRect? geometryCandidate; private int geometryCandidateCount; private readonly List textVotes = []; + private TypewriterState? typewriter; + private int typewriterCooldownFrames; private string normalizedConfirmedText = NormalizeText(observation.SourceText); private DormantGeometry? dormantGeometry; private double velocityX; @@ -1654,9 +1660,10 @@ public void Observe(TextRect current, TimeSpan timestamp) { this.MissedFrames = 0; this.UpdateMotion(current, timestamp); + string previousText = this.LatestObservation.SourceText; this.LatestObservation = current; this.LastObservationTime = timestamp; - this.UpdateText(current.SourceText); + this.UpdateText(previousText, current.SourceText, timestamp); this.UpdateGeometry(current); this.Stabilized = current with { @@ -1668,6 +1675,9 @@ public void Observe(TextRect current, TimeSpan timestamp) FontSize = this.Stabilized.FontSize, MultiLine = this.Stabilized.MultiLine, Angle = this.Stabilized.Angle, + BusyReasons = this.typewriter is null + ? current.BusyReasons & ~TextRegionBusyReason.Typewriter + : current.BusyReasons | TextRegionBusyReason.Typewriter, }; } @@ -1722,15 +1732,56 @@ public void Reactivate(TimeSpan timestamp) this.ResetMotion(timestamp); } - private void UpdateText(string current) + private void UpdateText(string previous, string current, TimeSpan timestamp) { + string normalizedPrevious = NormalizeText(previous); string normalizedCurrent = NormalizeText(current); + + if (this.typewriter is not null) + { + if (normalizedCurrent == this.typewriter.Start.Normalized) + { + this.typewriter = null; + this.textVotes.Clear(); + return; + } + + if (this.typewriter.Observe(new(normalizedCurrent, current), timestamp, out TextVote? final)) + { + TextVote confirmed = final + ?? throw new InvalidOperationException("文字送りの最終候補がありません。"); + this.ConfirmedText = confirmed.Original; + this.normalizedConfirmedText = confirmed.Normalized; + this.typewriter = null; + this.typewriterCooldownFrames = TypewriterCooldownFrames; + this.textVotes.Clear(); + } + return; + } + + bool canStartTypewriter = this.typewriterCooldownFrames == 0; + if (this.typewriterCooldownFrames > 0) + { + this.typewriterCooldownFrames--; + } + if (normalizedCurrent == this.normalizedConfirmedText) { this.textVotes.Clear(); return; } + if (canStartTypewriter && IsProgressiveTextChange(normalizedPrevious, normalizedCurrent)) + { + this.typewriter = new( + new(this.normalizedConfirmedText, this.ConfirmedText), + new(normalizedPrevious, previous), + new(normalizedCurrent, current), + timestamp); + this.textVotes.Clear(); + return; + } + this.textVotes.Add(new(normalizedCurrent, current)); if (this.textVotes.Count > TextVoteHistorySize) { @@ -1765,6 +1816,11 @@ private void UpdateText(string current) } } + private static bool IsProgressiveTextChange(string previous, string current) + => previous.Length > 0 + && current.Length > previous.Length + && current.StartsWith(previous, StringComparison.Ordinal); + private void UpdateMotion(TextRect current, TimeSpan timestamp) { double seconds = (timestamp - this.LastObservationTime).TotalSeconds; @@ -1941,6 +1997,90 @@ public TextRect Restore(TextRect child, TextRect parent) } } + private sealed class TypewriterState( + TextVote start, + TextVote previous, + TextVote current, + TimeSpan timestamp) + { + private readonly List history = [previous, current]; + private TextVote last = current; + private int consecutiveCount = 1; + private TimeSpan lastProgressTime = timestamp; + + public TextVote Start { get; } = start; + + public TextVote MostProgressed { get; private set; } = current; + + public bool Observe(TextVote current, TimeSpan timestamp, out TextVote? final) + { + this.AddHistory(current); + if (current.Normalized == this.last.Normalized) + { + this.consecutiveCount++; + } + else + { + this.last = current; + this.consecutiveCount = 1; + } + + if (IsProgressiveTextChange(this.MostProgressed.Normalized, current.Normalized)) + { + this.MostProgressed = current; + this.lastProgressTime = timestamp; + final = null; + return false; + } + + if (current.Normalized == this.MostProgressed.Normalized + && this.consecutiveCount >= 2) + { + if (timestamp - this.lastProgressTime >= typewriterStableDuration) + { + final = current; + return true; + } + } + + if (timestamp - this.lastProgressTime >= typewriterTailDuration) + { + final = this.SelectFinal(); + return true; + } + + final = null; + return false; + } + + private void AddHistory(TextVote candidate) + { + this.history.Add(candidate); + if (this.history.Count > TypewriterCandidateHistorySize) + { + this.history.RemoveAt(0); + } + } + + private TextVote SelectFinal() + => this.history + .Where(candidate => candidate.Normalized != this.Start.Normalized) + .Select((candidate, index) => (candidate, index)) + .GroupBy(item => item.candidate.Normalized) + .Select(group => new + { + Candidate = group.Last().candidate, + Count = group.Count(), + IsMostProgressed = group.Key == this.MostProgressed.Normalized, + LastIndex = group.Max(item => item.index), + }) + .OrderByDescending(candidate => candidate.Count) + .ThenByDescending(candidate => candidate.IsMostProgressed) + .ThenByDescending(candidate => candidate.LastIndex) + .Select(candidate => candidate.Candidate) + .FirstOrDefault(this.MostProgressed); + } + private sealed record TextVote(string Normalized, string Original); } } diff --git a/WindowTranslator/Themes/Generic.xaml b/WindowTranslator/Themes/Generic.xaml index 0ae7061c..da42e7af 100644 --- a/WindowTranslator/Themes/Generic.xaml +++ b/WindowTranslator/Themes/Generic.xaml @@ -50,8 +50,8 @@ CornerRadius="{Binding RelativeSource={RelativeSource Self}, Converter={StaticResource s2crConv}}" Opacity="{Binding TranslatedText, Converter={StaticResource n2dConv}}"> - - + + - - + + - - + + - +