Make the boxes reachable, and the settings say what they do

README.md

@@ -23,7 +23,7 @@ See it in action on YouTube: https://www.youtube.com/watch?v=Izu_8pql7cE
 - **Wayland Global Hotkey & Command OSD**: Trigger translation, OSD configurations, and sidebars seamlessly using customizable system-wide leader hotkeys.
 - **Dialogue Mode (Autoplay VNs / Story RPGs)**: Lock onto a screen region, translate automatically, and advance/refresh translations inline with a simple mouse click.
 - **Live In-Place Translation**: Lock onto a region and let MAGE paint translations directly over the original text — Google Lens style — refreshing as the screen changes, with no bubbles in the way. Two engines: a vision model that reads and translates in one call, or **local OCR** (PP-OCRv5 through ONNX Runtime) that reads on your machine and sends only the text — so a screen full of particle effects over unchanged dialogue costs nothing at all, and a line you have seen before costs nothing twice. Run `scripts/export_ppocr_onnx.py` once to fetch the models.
-- **New Interface (optional)**: Instead of a leader key and a menu of letters, place **translation boxes** on screen and click an **orb**. Boxes stay where you put them, each with its own mode — Live, Once, or Ignore, which hides a ticking clock or an animated minimap from everything else. The orb is your translation log, a chat that already knows what is on screen, and a push-to-talk microphone. Turn it on in **Settings → Features**; the classic interface is untouched and stays the default.
+- **Boxes and the orb**: Instead of a leader key and a menu of letters, place **translation boxes** on screen and click an **orb**. Boxes stay where you put them, each with its own mode — Live, Once, or Ignore, which hides a ticking clock or an animated minimap from everything else. The orb is your translation log, a chat that already knows what is on screen, and a push-to-talk microphone. It is the interface: the leader key and its menu of letters are gone.
 - **Session Memory**: MAGE remembers the text it has translated, the dialogue you have read, and the questions you have asked, so you can ask *"what do I do next?"* and get an answer grounded in your actual playthrough rather than generic wiki knowledge.
 - **NPU Offload (Ryzen AI)**: On XDNA 2 hardware, chat, translation, and speech can run on the NPU via Lemonade's FastFlowLM backend, leaving the GPU to the vision model. Check your machine with `./mage.sh --doctor-npu`.
 - **Visual Grounding Target Highlighting**: Ask the assistant *"where do I click?"* or *"where is the exit?"* and watch it highlight the exact physical coordinates on your screen.

apps/mage-client/src/mage/live_common.py

@@ -59,6 +59,7 @@
 import imagehash
 from PIL import Image
 from PyQt6.QtCore import QRect, QThread, pyqtSignal
+from shared_types.state import t
 
 logger = logging.getLogger(__name__)
 
@@ -68,6 +69,7 @@
     "DEFAULT_INTERVAL_MS",
     "LIVE_HASH_SIZE",
     "MAX_CAPTURE_FAILURES",
+    "MAX_INFERENCE_FAILURES",
     "PAINT_MASK_PADDING",
     "REGION_CHANGE_THRESHOLD",
     "REGION_COMPARE_SIZE",
@@ -130,6 +132,24 @@
 #: Consecutive capture failures tolerated before giving up.
 MAX_CAPTURE_FAILURES = 5
 
+#: Consecutive *identical* inference failures tolerated before giving up.
+#:
+#: A missing model, a model that will not load, a backend that is not running:
+#: none of these fix themselves between ticks, and a box grinding through
+#: capture-read-fail several times a second costs the game frames to learn
+#: nothing.  Deliberately larger than the capture allowance — a backend
+#: restarting mid-session is worth waiting out.
+MAX_INFERENCE_FAILURES = 8
+
+#: How often the loop says what it has been doing, in seconds.
+#:
+#: The overlay is silent by design when it has nothing to paint, which makes
+#: "reading, and there is nothing there" and "not reading at all" look
+#: identical from the outside.  This is the line that tells them apart, and it
+#: is at INFO because the person who needs it is the person who cannot see
+#: their translations.
+REPORT_INTERVAL_SECONDS = 30.0
+
 
 class LiveRegion:
     """A translated line plus its background colour, in region coordinates."""
@@ -291,6 +311,17 @@ def __init__(
         self._last_signature: tuple = ()
         self._warned_untranslated = False
 
+        # The last failure reported, and how many times it has repeated.  Kept
+        # so a broken box says so once rather than every tick.
+        self._last_error: str | None = None
+        self._failures = 0
+
+        # What the loop has done since it last said so.
+        self._tally = dict.fromkeys(
+            ("ticks", "captured", "changed", "translated", "painted"), 0
+        )
+        self._reported_at = 0.0
+
         # Translations from earlier frames, each with a hash of the pixels it
         # was read from. A box whose pixels are unchanged is still showing the
         # same text, so its translation stands without another inference —
@@ -359,9 +390,19 @@ async def _run_async(self):
         interval = self.interval_ms / 1000.0
         failures = 0
         inflight = False
+        self._reported_at = time.monotonic()
+        logger.info(
+            "%s watching %s every %d ms via %s",
+            type(self).__name__,
+            self.rect,
+            self.interval_ms,
+            "continuous capture" if self._frame_stream is not None else "screenshots",
+        )
 
         while self._running:
             started = time.monotonic()
+            self._tally["ticks"] += 1
+            self._report(started)
 
             frame = await asyncio.to_thread(self._grab)
             if frame is None:
@@ -372,21 +413,24 @@ async def _run_async(self):
                 await self._sleep_remainder(started, interval)
                 continue
             failures = 0
+            self._tally["captured"] += 1
 
             current = self._masked_hash(frame)
 
             if inflight or not self._should_translate(current):
                 await self._sleep_remainder(started, interval)
                 continue
 
+            self._tally["changed"] += 1
             self._clean_hash = current
 
             # Whatever is painted was read from an older frame. Re-check each
             # box against this one: unchanged text keeps its translation while
             # the call runs, and text that has gone stops being painted now
             # rather than seconds from now when the call returns.
             self._carry = self._surviving_regions(frame)
-            self._publish([], frame)
+            if self._publish([], frame):
+                self._tally["painted"] += 1
 
             inflight = True
             try:
@@ -395,19 +439,25 @@ async def _run_async(self):
                 logger.warning("Live lens inference timed out; skipping frame")
                 regions = None
             except Exception as exc:
-                logger.warning("Live lens inference failed: %s", exc)
-                self.error.emit(str(exc))
                 regions = None
+                if self._note_failure(str(exc)):
+                    return
             finally:
                 inflight = False
 
+            if regions is not None:
+                self._last_error = None
+                self._failures = 0
+                self._tally["translated"] += 1
+
             if regions is None:
                 # The call failed. The carry is still valid for this frame, so
                 # the overlay keeps the lines whose text has not changed
                 # instead of blanking, and they survive into the next tick.
                 self._remember(self._carry, frame)
             else:
-                self._publish(regions, frame)
+                if self._publish(regions, frame):
+                    self._tally["painted"] += 1
                 self._carry = []
                 # The boxes we mask changed with this render, so re-hash the
                 # frame we just translated under the *new* mask. Comparing the
@@ -604,6 +654,53 @@ def _warn_if_nothing_translated(self, live) -> None:
         logger.warning("%s", message)
         self.error.emit(message)
 
+    # ── saying what happened ─────────────────────────────────────────
+
+    def _note_failure(self, message: str) -> bool:
+        """Report an inference failure; return True when the loop should stop.
+
+        Two things a polling loop must not do: report the same failure on
+        every tick, and keep failing forever.  The first buries the message
+        the user needed under a thousand copies of itself; the second burns a
+        core reading a screen it can never translate.
+        """
+        if message != self._last_error:
+            self._last_error = message
+            self._failures = 0
+            logger.warning("%s inference failed: %s", type(self).__name__, message)
+            self.error.emit(message)
+
+        self._failures += 1
+        if self._failures < MAX_INFERENCE_FAILURES:
+            return False
+
+        logger.error(
+            "%s stopping after %d identical failures: %s",
+            type(self).__name__, self._failures, message,
+        )
+        self.error.emit(f"{t('live.error.stopped')} {message}")
+        return True
+
+    def _report(self, now: float) -> None:
+        """Log what the loop has been doing, at an interval.
+
+        Written as counts rather than events because the useful question is
+        which stage the frames stop at: captured but never changed is a gate
+        problem, changed but never translated is the engine, translated but
+        never repainted is the overlay.
+        """
+        if now - self._reported_at < REPORT_INTERVAL_SECONDS:
+            return
+        self._reported_at = now
+        logger.info(
+            "%s: %d ticks, %d captured, %d changed, %d translated, %d repainted",
+            type(self).__name__,
+            self._tally["ticks"], self._tally["captured"], self._tally["changed"],
+            self._tally["translated"], self._tally["painted"],
+        )
+        for key in self._tally:
+            self._tally[key] = 0
+
     def _capture_scale(self, frame: Image.Image) -> float:
         """Capture pixels per logical pixel, measured from the frame itself."""
         logical_width = self._served_rect.width()

apps/mage-client/src/mage/live_engine.py

@@ -0,0 +1,116 @@
+# MAGE — Gaming HUD for real-time screen translation.
+# Copyright (C) 2026  Clementine Pendragon <clem@pendragon.systems>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+# Contact: clem@pendragon.systems (Clementine Pendragon, c/o Xian Project Development)
+
+"""Which live engine actually runs, given the setting and the machine.
+
+Two engines translate a locked region: a vision model that reads and
+translates in one call, and the local PP-OCRv5 reader that sends only text.
+The setting says which one the user wants.  This says which one they can have.
+
+The difference matters because PP-OCRv5 has never had a published ONNX
+release, so its weights exist only once somebody has run
+``scripts/export_ppocr_onnx.py`` — which, on a fresh install, nobody has.  A
+box built on the reader before that happens fails on its first read, in a
+worker thread, on a tick, where all the user sees is a border turning red and
+no text ever arriving.  The vision model needs nothing exported and is already
+loaded for chat and grounding, so that is what an unprepared machine gets,
+with one line saying why.
+
+Resolved here rather than at each call site because there are two — the boxes
+and the classic live lens — and an engine chosen one way in one place and
+another way in the other is how the boxes came to ignore the setting entirely.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+
+from mage.settings_keys import (
+    DEFAULT_LIVE_ENGINE,
+    KEY_LIVE_ENGINE,
+    KEY_OCR_DETECTOR,
+    KEY_SOURCE_LANG,
+    LIVE_ENGINE_GROUNDING,
+    LIVE_ENGINE_OCR,
+)
+
+logger = logging.getLogger(__name__)
+
+__all__ = ["ocr_models_ready", "resolve_live_engine"]
+
+#: Said once per run, not once per box: five boxes starting together would
+#: otherwise report the same missing weights five times.
+_warned = False
+
+
+def ocr_models_ready(settings) -> bool:
+    """Whether the local reader has the weights it needs, on disk, intact.
+
+    Checks the two models a read actually loads — the detector, and the
+    recognizer for the configured source language — rather than the catalog as
+    a whole, because a machine that exported one script's weights and not
+    another's is a machine that works for Chinese and not for Korean.
+
+    A configured distribution host counts as ready: fetching is then
+    ``ensure_model``'s job, and refusing here would fall back to the vision
+    model for a download that was going to succeed.
+    """
+    if os.environ.get("XIAN_OCR_BASE_URL", "").strip():
+        return True
+
+    try:
+        from xian.ocr.engine import DEFAULT_DETECTOR
+        from xian.ocr.models import verify_model
+        from xian.ocr.scripts import SCRIPTS, recognizer_script_for
+    except ImportError as exc:
+        # The OCR extra is optional; without it there is no reader at all.
+        logger.info("local OCR is not installed (%s)", exc)
+        return False
+
+    detector = (settings.value(KEY_OCR_DETECTOR, "") if settings else "") or DEFAULT_DETECTOR
+    source = settings.value(KEY_SOURCE_LANG, "") if settings else ""
+    recognizer = SCRIPTS[recognizer_script_for(source)]
+
+    return verify_model(detector) and verify_model(recognizer)
+
+
+def resolve_live_engine(settings) -> str:
+    """The engine to build a live worker with.
+
+    Returns one of :data:`~mage.settings_keys.LIVE_ENGINE_OCR` or
+    :data:`~mage.settings_keys.LIVE_ENGINE_GROUNDING`.  The vision model is
+    never second-guessed — it is always available — so the only substitution
+    made here is away from a reader that cannot read.
+    """
+    global _warned
+
+    choice = (settings.value(KEY_LIVE_ENGINE, DEFAULT_LIVE_ENGINE) if settings else DEFAULT_LIVE_ENGINE)
+    if choice != LIVE_ENGINE_OCR:
+        return LIVE_ENGINE_GROUNDING
+    if ocr_models_ready(settings):
+        return LIVE_ENGINE_OCR
+
+    if not _warned:
+        _warned = True
+        logger.warning(
+            "The local OCR weights are not exported, so live translation is using the "
+            "vision model instead. To use the local reader, run: "
+            "uv run --package mage-client python scripts/export_ppocr_onnx.py"
+        )
+    return LIVE_ENGINE_GROUNDING

apps/mage-client/src/mage/settings_keys.py

@@ -16,11 +16,41 @@
 #
 # Contact: clem@pendragon.systems (Clementine Pendragon, c/o Xian Project Development)
 
-"""Canonical QSettings keys for the Mage client.
+"""Canonical QSettings keys for the Mage client, and how to read them.
 
-Centralizes key names to avoid typos and drift across files.
+Centralizes key names to avoid typos and drift across files.  The few readers
+that live here too are the ones whose *interpretation* has to be shared — a
+boolean QSettings stores as a string on one platform and a bool on another, a
+URL that has to be normalised the same way everywhere.
 """
 
+from shared_types import constants
+
+
+def normalized_api_url_from_settings(settings) -> str:
+    """The Lemonade base URL as stored, put in canonical form.
+
+    Beside the keys rather than beside its callers because there are five of
+    them, in three modules, and a URL normalised on the way in but not on the
+    way out is a health check that passes against a server the requests never
+    reach.
+    """
+    from xian.lemonade_url import normalize_lemonade_api_base_url
+
+    return normalize_lemonade_api_base_url(str(settings.value(KEY_API_URL, constants.DEFAULT_API_URL)))
+
+
+def parse_styles(settings) -> list[str]:
+    """The saved translation styles as a list.
+
+    QSettings hands back a list on one platform and a comma-joined string on
+    another; every reader has to accept both.
+    """
+    raw = settings.value(KEY_STYLES, constants.DEFAULT_STYLES)
+    if isinstance(raw, str):
+        return [style.strip() for style in raw.split(",") if style.strip()]
+    return raw if isinstance(raw, list) else []
+
 
 def is_true(value) -> bool:
     """Read a QSettings boolean.
@@ -70,7 +100,12 @@ def is_true(value) -> bool:
 KEY_LIVE_ENGINE = "live_engine"
 LIVE_ENGINE_GROUNDING = "grounding"
 LIVE_ENGINE_OCR = "ocr"
-DEFAULT_LIVE_ENGINE = LIVE_ENGINE_GROUNDING
+#: The local reader, which is what the translation boxes were built around and
+#: what they have always run whatever this said.  It falls back to the vision
+#: model on a machine whose weights are not exported — see
+#: :mod:`mage.live_engine` — so this default costs an unprepared machine
+#: nothing.
+DEFAULT_LIVE_ENGINE = LIVE_ENGINE_OCR
 
 # Detection model for the OCR engine: PP-OCRv5_mobile_det or _server_det.
 KEY_OCR_DETECTOR = "ocr_detector"