Japanese: decimals, line breaks and numbers in the speech check

AGENTS.md

@@ -88,7 +88,9 @@ When you ask, put everything you need in one message. Typical reasons to ask:
    check confirmed. If a word is not confirmed, the voice probably said it
    wrong. Fix it with `{shown|spoken}` in the script and run the command
    again. For a recording, make script.md say what was actually said.
-   Only the scenes you changed are made again.
+   Only the scenes you changed are made again. `check`, `still`, `sheet`,
+   `render` and `preview` also make missing narration, so start them after
+   `voice` has finished, not at the same time.
 
 5. **Plan the pictures.** For each scene, decide what is on screen when each
    marker or important word is spoken. Decide where each thing sits and what
@@ -175,7 +177,8 @@ something to a friend at a table.
 Length and pace:
 
 - Aim for 45 to 120 seconds unless the user says otherwise. A finished video
-  has about 150 words a minute, pauses included.
+  has about 150 words a minute, pauses included. In Japanese it is about 270
+  characters a minute, and in Chinese about 200.
 - Start with the question or problem. Explain it in small steps. Give one
   concrete example, then sum up.
 - One idea per scene, usually 1 to 3 sentences (4 to 15 seconds).
@@ -362,7 +365,12 @@ match whole words. Put a `[#marker]` where you need a moment inside a word.
 Sentences end with `。`, `！` or `？`.
 
 Latin words such as API or JavaScript are read in English. For a Japanese
-reading, write it out: `{API|エーピーアイ}`.
+reading, write it out: `{API|エーピーアイ}`. Numbers can stay in digits,
+including decimals like 365.2422. The Japanese reader sometimes picks the
+wrong reading for a word, such as ついたち for 一日 (write `1日` for
+いちにち) or こよみ for 暦 in グレゴリオ暦 (write `{グレゴリオ暦|グレゴリオれき}`).
+When the speech check flags a word, compare what it heard with the script
+before you change anything.
 
 Whisper often writes a word with other characters that sound the same, so
 the speech check compares sounds. Japanese is compared by its kana reading,
@@ -683,8 +691,9 @@ same. `render --out` still sets the file name you choose.
 
 ## Pace
 
-`pace` makes the whole video quicker or slower. At 1.3 the voice speaks 30%
-faster. The pauses, the time before and after each scene, the transitions and
+`pace` makes the whole video quicker or slower. At 1.3 the voice speaks about
+30% faster. The voice model does not speed up exactly, so check the length
+`voice` reports. The pauses, the time before and after each scene, the transitions and
 every animation also get 30% shorter. Word and marker cues follow the voice.
 So if you tie pictures to words, the rhythm stays right at any pace. Times in
 seconds are real seconds of the scene, like `s.t` and `s.cue()`. So

engine/captions.js

@@ -3,6 +3,26 @@
 import { fontString } from './themes.js';
 import { clamp, ease } from './util.js';
 
+// Some words belong to the word before them, so a phrase or line should not
+// start with one: Japanese particles and verb endings (short hiragana pieces
+// such as は, ごと, せん, or ones starting like ません), a one-kanji suffix
+// such as the 暦 of グレゴリオ暦, Chinese particles such as 的, and counters
+// after a number (4年, 365天).
+const JAPANESE_ENDINGS = /^(ませ|まし|でし|です|ます|しょう|られ|させ|され|なかっ|ながら|ような|ように|という|として|について|によって|くらい|ぐらい)/;
+export function clingsToPrevious(language) {
+  const counter = (text, before) => /\d$/.test(before) && /^[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}]/u.test(text);
+  if (language === 'ja') {
+    return (text, before = '') => {
+      const word = text.replace(/[、。！？「」『』（）]+$/, '');
+      const ending = /^[\u3041-\u3096]+$/.test(word) && (word.length <= 2 || JAPANESE_ENDINGS.test(word));
+      const suffix = /^\p{Script=Han}$/u.test(word) && /[\p{Script=Han}\p{Script=Katakana}]$/u.test(before);
+      return ending || suffix || counter(text, before);
+    };
+  }
+  if (language === 'zh') return (text, before = '') => /^[的了吗呢吧啊着过得地们][，。！？、]*$/.test(text) || counter(text, before);
+  return () => false;
+}
+
 export function buildPhrases(timeline, maxChars) {
   const cjk = ['ja', 'zh'].includes(timeline.language);
   const sentenceEnd = cjk ? /[.!?;:।。！？]["'’)」』）]*$/ : /[.!?;:।]["'’)]*$/;
@@ -13,7 +33,7 @@ export function buildPhrases(timeline, maxChars) {
     // phrases as fit, balanced in length, so no phrase is one lonely word.
     let run = [];
     const flush = () => {
-      for (const words of balancePhrases(run, maxChars, cjk ? '' : ' ')) phrases.push({ words, ...(cjk ? { separator: '' } : {}), start: words[0].start, end: words[words.length - 1].end });
+      for (const words of balancePhrases(run, maxChars, cjk ? '' : ' ', clingsToPrevious(timeline.language))) phrases.push({ words, ...(cjk ? { separator: '' } : {}), start: words[0].start, end: words[words.length - 1].end });
       run = [];
     };
     for (const w of sc.words) {
@@ -32,8 +52,9 @@ export function buildPhrases(timeline, maxChars) {
 }
 
 // Splits words into the fewest phrases of at most maxChars characters, with
-// lengths as even as possible and breaks after commas where they fit.
-export function balancePhrases(words, maxChars, separator = ' ') {
+// lengths as even as possible and breaks after commas where they fit, never
+// before a word that clings to the one before it if that can be helped.
+export function balancePhrases(words, maxChars, separator = ' ', clings = () => false) {
   if (!words.length) return [];
   const len = (a, b) => words.slice(a, b).reduce((n, w) => n + w.text.length + separator.length, -separator.length);
   if (len(0, words.length) <= maxChars) return [words];
@@ -47,7 +68,8 @@ export function balancePhrases(words, maxChars, separator = ' ') {
       if (l > maxChars && b - a > 1) break;
       if (!best[a]) continue;
       const comma = b < n && punctuation.test(words[b - 1].text) ? 150 : 0;
-      const c = { parts: best[a].parts + 1, cost: best[a].cost + (maxChars - l) ** 2 - comma, from: a };
+      const glued = b < n && !comma && clings(words[b].text, words[b - 1].text) ? 250 : 0;
+      const c = { parts: best[a].parts + 1, cost: best[a].cost + (maxChars - l) ** 2 - comma + glued, from: a };
       if (!best[b] || c.parts < best[b].parts || (c.parts === best[b].parts && c.cost < best[b].cost)) best[b] = c;
     }
   }

src/g2p.py

@@ -9,16 +9,46 @@
 mode = sys.argv[1]
 
 
+DIGITS = ['ゼロ', 'イチ', 'ニ', 'サン', 'ヨン', 'ゴ', 'ロク', 'ナナ', 'ハチ', 'キュー']
+
+
+def below_10000(n):
+    th, h, t, o = n // 1000, n // 100 % 10, n // 10 % 10, n % 10
+    out = {1: 'セン', 3: 'サンゼン', 8: 'ハッセン'}.get(th, DIGITS[th] + 'セン') if th else ''
+    out += {1: 'ヒャク', 3: 'サンビャク', 6: 'ロッピャク', 8: 'ハッピャク'}.get(h, DIGITS[h] + 'ヒャク') if h else ''
+    out += ('' if t == 1 else DIGITS[t]) + 'ジュー' if t else ''
+    return out + (DIGITS[o] if o else '')
+
+
+def number(digits):
+    # UniDic leaves digits unread. Whisper may write a number in digits or
+    # kana, so both sides get the same reading.
+    n = int(digits)
+    if n == 0 or n >= 10 ** 8 or digits[0] == '0':
+        return ''.join(DIGITS[int(d)] for d in digits)
+    man, rest = divmod(n, 10000)
+    return (below_10000(man) + 'マン' if man else '') + below_10000(rest)
+
+
 def readings(tagger, text):
     # [start, end, katakana reading] for each word, in characters.
-    out, pos = [], 0
+    out, pos, decimal = [], 0, False
     for word in tagger(text):
         start = text.find(word.surface, pos)
         if start < 0:
             continue
         pos = start + len(word.surface)
         pron = getattr(word.feature, 'pron', None)
-        out.append([start, pos, pron if pron and pron != '*' else word.surface])
+        surface = word.surface
+        if surface.isascii() and surface.isdigit():
+            # After a decimal point, digits are read one by one.
+            reading = ''.join(DIGITS[int(d)] for d in surface) if decimal else number(surface)
+        elif surface == '.' and out and text[start - 1:start].isdigit() and text[pos:pos + 1].isdigit():
+            reading = 'テン'
+        else:
+            reading = pron if pron and pron != '*' else surface
+        decimal = reading == 'テン' or (decimal and surface.isdigit())
+        out.append([start, pos, reading])
     return out
 
 

src/phonemes.js

@@ -96,7 +96,14 @@ async function englishPhonemes(text) {
   return (await espeakWasm(text, 'en-us')).join(' ').replace(/ʲ/g, 'j').replace(/r/g, 'ɹ').replace(/x/g, 'k').replace(/ɬ/g, 'l').trim();
 }
 
+// misaki reads "365.2422" as 365, a pause and 2422. In Japanese the digits
+// after the point are read one by one: 365点二四二二.
+function japaneseDecimals(text) {
+  return text.replace(/(\d)\.(\d+)/g, (m, whole, digits) => whole + '点' + Array.from(digits, d => '〇一二三四五六七八九'[d]).join(''));
+}
+
 async function misakiPhonemes(texts, language) {
+  if (language === 'ja') texts = texts.map(japaneseDecimals);
   const parts = texts.map(text => {
     const list = [];
     let last = 0;

src/subtitles.js

@@ -1,9 +1,12 @@
 // Subtitle cues use the shown words and their times from the voice timeline.
+import { clingsToPrevious } from '../engine/captions.js';
+
 const sentenceEnd = /[.!?…।]["'’”)\]]*$/;
 const punctuation = /[,;:.!?…।]["'’”)\]]*$/;
 
 function lines(words, language = 'en') {
   const cjk = ['ja', 'zh'].includes(language);
+  const clings = clingsToPrevious(language);
   const limit = cjk ? 18 : 42;
   const separator = cjk ? '' : ' ';
   const text = words.map((w) => w.text).join(separator);
@@ -15,7 +18,8 @@ function lines(words, language = 'en') {
     if (a.length > limit || b.length > limit) continue;
     // Break after punctuation, unless that leaves a word or two alone on a line.
     const atPunctuation = (cjk ? /[,;:.!?…।。！？、，；：]["'’”)\]」』）]*$/ : punctuation).test(words[i - 1].text) && Math.min(a.length, b.length) >= (cjk ? 5 : 12);
-    const score = (atPunctuation ? 30 : 0) - Math.abs(a.length - b.length);
+    const glued = !atPunctuation && clings(words[i].text, words[i - 1].text) ? 40 : 0;
+    const score = (atPunctuation ? 30 : 0) - Math.abs(a.length - b.length) - glued;
     if (!best || score > best.score) best = { score, text: `${a}\n${b}` };
   }
   return best?.text ?? null;
@@ -27,6 +31,7 @@ export function subtitleCues(timeline, { from = 0, to = Infinity } = {}) {
   const limit = cjk ? 18 : 42;
   const endSentence = cjk ? /[.!?…।。！？]["'’”)\]」』）]*$/ : sentenceEnd;
   const atPunctuation = cjk ? /[,;:.!?…।。！？、，；：]["'’”)\]」』）]*$/ : punctuation;
+  const clings = clingsToPrevious(language);
   const cues = [];
   for (const scene of timeline.scenes) {
     const words = scene.words || [];
@@ -37,9 +42,11 @@ export function subtitleCues(timeline, { from = 0, to = Infinity } = {}) {
       while (j < words.length) {
         if (endSentence.test(words[j - 1].text) || words[j].start - words[j - 1].end > 0.6 + 1e-9) break;
         if (words[j].end + 0.25 - words[i].start > 6 || lines(words.slice(i, j + 1), language) === null) {
-          for (let k = j - 1; k > i; k--) {
-            if (atPunctuation.test(words[k - 1].text)) { j = k; break; }
-          }
+          let k = j - 1;
+          while (k > i && !atPunctuation.test(words[k - 1].text)) k--;
+          if (k > i) j = k;
+          // Without punctuation, start the next cue at a word of its own.
+          else while (j - 1 > i && clings(words[j].text, words[j - 1].text)) j--;
           break;
         }
         j++;

test/cjk.test.js

@@ -6,7 +6,7 @@ import path from 'node:path';
 import { tokenizeNarration, parseScript, speechChunks, normWord, resolveMarks } from '../src/script.js';
 import { alignWords, readAlike, spreadReadings } from '../src/align.js';
 import { subtitleCues, toSrt, toVtt } from '../src/subtitles.js';
-import { buildPhrases, drawCaptions } from '../engine/captions.js';
+import { buildPhrases, drawCaptions, clingsToPrevious } from '../engine/captions.js';
 import { getTheme } from '../engine/themes.js';
 import { phonemize, misakiAvailable } from '../src/phonemes.js';
 import { languageFonts, ensureFonts } from '../src/fonts.js';
@@ -87,6 +87,27 @@ for (const [language, text] of Object.entries(native)) {
   });
 }
 
+test('subtitle lines and caption phrases do not start with a Japanese or Chinese particle', () => {
+  const timed = list => list.map((text, i) => ({ text, start: i * 0.3, end: (i + 1) * 0.3 }));
+  const ja = timed(['地球', 'の', '動き', 'と', 'カレンダー', 'に', 'ズレ', 'が', 'たまり', 'ます。']);
+  const [cue] = subtitleCues({ language: 'ja', scenes: [{ start: 0, words: ja }] });
+  assert.equal(cue.text, '地球の動きとカレンダーに\nズレがたまります。');
+  const phrases = buildPhrases({ language: 'ja', scenes: [{ start: 0, words: ja }] }, 46);
+  assert.equal(phrases.length, 2);
+  assert.ok(phrases.every(p => !/^[\u3041-\u3096]/.test(p.words[0].text)), phrases.map(p => p.words.map(w => w.text).join('')).join(' / '));
+  const [counter] = subtitleCues({ language: 'ja', scenes: [{ start: 0, words: timed(['そこで、', '基本', 'は', '4', '年', 'ごと', 'に', '1', '日', 'を', '足し', 'ます。']) }] });
+  assert.equal(counter.text, 'そこで、基本は4年ごとに\n1日を足します。');
+  const clings = clingsToPrevious('ja');
+  for (const [text, before] of [['は', '基本'], ['せん', 'ありま'], ['ませんでした。', 'あり'], ['暦', 'グレゴリオ'], ['年', '4'], ['しょう', 'ので']]) assert.equal(clings(text, before), true, text);
+  for (const [text, before] of [['うるう年', 'は'], ['ありま', 'では'], ['足', 'を'], ['地球', 'の'], ['カレンダー', 'の'], ['1', 'に']]) assert.equal(clings(text, before), false, text);
+  // Breaking before or after に balances the lines equally; に stays with the word before.
+  const [even] = subtitleCues({ language: 'ja', scenes: [{ start: 0, words: timed(['アイウエオカキクケ', 'に', 'サシスセソタチツテ。']) }] });
+  assert.equal(even.text, 'アイウエオカキクケに\nサシスセソタチツテ。');
+  const zh = timed(['这', '是', '我们', '最', '常用', '的', '密码', '保存', '方法', '之一', '，', '也', '很', '安全', '的', '做法', '吗', '？']);
+  for (const p of buildPhrases({ language: 'zh', scenes: [{ start: 0, words: zh }] }, 46)) assert.ok(!['的', '吗'].includes(p.words[0].text), p.words.map(w => w.text).join(''));
+  for (const c of subtitleCues({ language: 'zh', scenes: [{ start: 0, words: zh }] })) for (const line of c.text.split('\n')) assert.ok(!/^[的吗]/.test(line), line);
+});
+
 test('Mandarin alignment reads Traditional characters as Simplified, Japanese keeps them', () => {
   const rate = language => alignWords([{ spoken: '浏览器' }, { spoken: '会询问网址。' }], [{ text: '瀏覽器會詢問網址', start: 0, end: 1.6 }], 1.6, language).matchRate;
   assert.equal(rate('zh'), 1);
@@ -168,6 +189,7 @@ test('Python batches JSON lines without shell expansion and accepts misaki phone
     const texts = ['解析。\n次の行', '$(　) `＊` "中文" $@'];
     assert.deepEqual(await phonemize(texts, language), texts.map(t => t + '↗'));
     assert.equal(misakiAvailable(language), true);
+    assert.deepEqual(await phonemize(['およそ365.2422日。'], language), [language === 'ja' ? 'およそ365点二四二二日。↗' : 'およそ365.2422日。↗']);
     // Latin words are read as English between the misaki pieces.
     assert.deepEqual(await phonemize(['我用JavaScript写代码。', 'API。'], language), ['我用↗ dʒˈɑːvə skɹˈɪpt 写代码。↗', 'ˌeɪpˌiːˈaɪ 。↗']);
   }