diff --git a/OpenUtau.Core/Classic/ClassicSingerLoader.cs b/OpenUtau.Core/Classic/ClassicSingerLoader.cs index e9f7a5d87..06dd175b0 100644 --- a/OpenUtau.Core/Classic/ClassicSingerLoader.cs +++ b/OpenUtau.Core/Classic/ClassicSingerLoader.cs @@ -13,6 +13,8 @@ static USinger AdjustSingerType(Voicebank v) { return new Core.DiffSinger.DiffSingerSinger(v) as USinger; case USingerType.Voicevox: return new Core.Voicevox.VoicevoxSinger(v) as USinger; + case USingerType.Neutrino: + return new Core.Neutrino.NeutrinoSinger(v) as USinger; default: return new ClassicSinger(v) as USinger; } diff --git a/OpenUtau.Core/Classic/VoicebankInstaller.cs b/OpenUtau.Core/Classic/VoicebankInstaller.cs index 9eca2707f..9cc4fa475 100644 --- a/OpenUtau.Core/Classic/VoicebankInstaller.cs +++ b/OpenUtau.Core/Classic/VoicebankInstaller.cs @@ -82,6 +82,7 @@ public void Install(string path, string singerType) { File.WriteAllText(touch, "\n"); var config = new VoicebankConfig() { TextFileEncoding = textEncoding.WebName, + SingerType = singerType, }; using (var stream = File.Open(touch.Replace(".txt", ".yaml"), FileMode.Create)) { config.Save(stream); diff --git a/OpenUtau.Core/Neutrino/NeutrinoInferenceUtil.cs b/OpenUtau.Core/Neutrino/NeutrinoInferenceUtil.cs new file mode 100644 index 000000000..d3d6b0aa7 --- /dev/null +++ b/OpenUtau.Core/Neutrino/NeutrinoInferenceUtil.cs @@ -0,0 +1,346 @@ +using System; +using System.Collections.Generic; +using System.IO; + +namespace OpenUtau.Core.Neutrino { + internal readonly struct NeutrinoScorePhoneInput { + public long PhonemeId { get; } + public int SourceIndex { get; } + public double? ManualBoundarySeconds { get; } + + public NeutrinoScorePhoneInput( + long phonemeId, + int sourceIndex = -1, + double? manualBoundarySeconds = null) { + + PhonemeId = phonemeId; + SourceIndex = sourceIndex; + ManualBoundarySeconds = manualBoundarySeconds; + } + } + + internal readonly struct NeutrinoScoreNoteInput { + public float PitchHz { get; } + public float DurationSeconds { get; } + public bool IsExtension { get; } + public NeutrinoScorePhoneInput[] Phones { get; } + + public NeutrinoScoreNoteInput( + float pitchHz, + float durationSeconds, + bool isExtension, + NeutrinoScorePhoneInput[] phones) { + + PitchHz = pitchHz; + DurationSeconds = durationSeconds; + IsExtension = isExtension; + Phones = phones ?? Array.Empty(); + } + } + + internal sealed class NeutrinoScoreSequence { + public long[] PhonemeIds { get; } + public float[] ScorePitchesHz { get; } + public float[] ScoreDurations { get; } + public long[] PhonePositions { get; } + public int[] SourcePhoneIndices { get; } + public double?[] ManualBoundaries { get; } + + public NeutrinoScoreSequence( + long[] phonemeIds, + float[] scorePitchesHz, + float[] scoreDurations, + long[] phonePositions, + int[] sourcePhoneIndices, + double?[] manualBoundaries) { + + PhonemeIds = phonemeIds; + ScorePitchesHz = scorePitchesHz; + ScoreDurations = scoreDurations; + PhonePositions = phonePositions; + SourcePhoneIndices = sourcePhoneIndices; + ManualBoundaries = manualBoundaries; + } + } + + internal readonly struct NeutrinoPhoneChunk { + public int PhoneStart { get; } + public int PhoneCount { get; } + public bool IsActive { get; } + + public NeutrinoPhoneChunk(int phoneStart, int phoneCount, bool isActive) { + PhoneStart = phoneStart; + PhoneCount = phoneCount; + IsActive = isActive; + } + } + + internal readonly struct NeutrinoFrameChunk { + public int PhoneStart { get; } + public int PhoneCount { get; } + public int FrameStart { get; } + public int FrameCount { get; } + public bool IsActive { get; } + + public NeutrinoFrameChunk( + int phoneStart, + int phoneCount, + int frameStart, + int frameCount, + bool isActive) { + + PhoneStart = phoneStart; + PhoneCount = phoneCount; + FrameStart = frameStart; + FrameCount = frameCount; + IsActive = isActive; + } + } + + internal static class NeutrinoInferenceUtil { + public static bool IsExtensionLyric(string lyric) { + return lyric == "-" || lyric?.StartsWith("+", StringComparison.Ordinal) == true; + } + + public static NeutrinoScoreSequence BuildScoreSequence( + IReadOnlyList notes) { + + var phonemeIds = new List(); + var scorePitchesHz = new List(); + var scoreDurations = new List(); + var phonePositions = new List(); + var sourcePhoneIndices = new List(); + var manualBoundaries = new List(); + long? sustainPhonemeId = null; + + foreach (var note in notes) { + var phones = note.Phones; + if (phones.Length == 0) { + if (!note.IsExtension || !sustainPhonemeId.HasValue) { + if (!note.IsExtension) { + sustainPhonemeId = null; + } + continue; + } + + // Official long-mark labels repeat the preceding note's final + // phoneme, while keeping the extension note's pitch and duration. + phones = new[] { new NeutrinoScorePhoneInput(sustainPhonemeId.Value) }; + } + + float durationSeconds = Math.Max(0.001f, note.DurationSeconds); + for (int position = 0; position < phones.Length; position++) { + var phone = phones[position]; + phonemeIds.Add(phone.PhonemeId); + scorePitchesHz.Add(phone.PhonemeId == NeutrinoPhoneme.PAU ? 0 : note.PitchHz); + scoreDurations.Add(durationSeconds); + phonePositions.Add(position); + sourcePhoneIndices.Add(phone.SourceIndex); + manualBoundaries.Add(phone.ManualBoundarySeconds); + } + sustainPhonemeId = phones[^1].PhonemeId; + } + + manualBoundaries.Add(null); + return new NeutrinoScoreSequence( + phonemeIds.ToArray(), + scorePitchesHz.ToArray(), + scoreDurations.ToArray(), + phonePositions.ToArray(), + sourcePhoneIndices.ToArray(), + manualBoundaries.ToArray()); + } + + public static float[] RequireLength(float[] values, int expectedLength, string outputName) { + if (values.Length != expectedLength) { + throw new InvalidDataException( + $"{outputName} length mismatch: actual {values.Length}, expected {expectedLength}."); + } + return values; + } + + public static float[] RequireTimingBoundaryLength( + float[] values, + int phonemeCount, + string outputName) { + + return RequireLength(values, checked(phonemeCount + 1), outputName); + } + + public static NeutrinoPhoneChunk[] BuildPhoneChunks(long[] phonemeIds) { + var chunks = new System.Collections.Generic.List(); + if (phonemeIds.Length == 0) { + return chunks.ToArray(); + } + + int chunkStart = 0; + bool chunkIsActive = true; + bool inPause = false; + bool afterBreath = false; + for (int phone = 0; phone < phonemeIds.Length; phone++) { + if (phonemeIds[phone] == NeutrinoPhoneme.PAU) { + if (!inPause) { + if (phone > chunkStart) { + chunks.Add(new NeutrinoPhoneChunk( + chunkStart, phone - chunkStart, chunkIsActive)); + } + chunkStart = phone; + chunkIsActive = false; + inPause = true; + afterBreath = false; + } + continue; + } + + if (phonemeIds[phone] == NeutrinoPhoneme.BR) { + inPause = false; + afterBreath = true; + continue; + } + + if (inPause || afterBreath) { + chunks.Add(new NeutrinoPhoneChunk( + chunkStart, phone - chunkStart, chunkIsActive)); + chunkStart = phone; + chunkIsActive = true; + inPause = false; + afterBreath = false; + } + } + chunks.Add(new NeutrinoPhoneChunk( + chunkStart, phonemeIds.Length - chunkStart, chunkIsActive)); + return chunks.ToArray(); + } + + public static double[] BuildTimingBoundaries( + float[] scoreDurations, + long[] phonePositions, + NeutrinoPhoneChunk[] chunks, + double frameSeconds, + Func predictBoundaryShifts, + double? leadingContextSeconds = null) { + + if (scoreDurations.Length != phonePositions.Length) { + throw new ArgumentException("Score duration and phone position lengths must match."); + } + + var baseBoundaries = BuildBaseBoundaryTimes(scoreDurations, phonePositions); + var globalBoundaryShifts = new float[baseBoundaries.Length]; + foreach (var chunk in chunks) { + if (!chunk.IsActive) { + continue; + } + var chunkShifts = predictBoundaryShifts(chunk); + if (chunkShifts == null || chunkShifts.Length < chunk.PhoneCount) { + throw new InvalidDataException( + $"Timing chunk output is too short: actual {chunkShifts?.Length ?? 0}, " + + $"expected at least {chunk.PhoneCount}."); + } + + // The official loader copies one value per phone and discards the + // model's extra final value before applying shifts globally. + Array.Copy( + chunkShifts, + 0, + globalBoundaryShifts, + chunk.PhoneStart, + chunk.PhoneCount); + } + return ApplyTimingBoundaryShifts( + baseBoundaries, + globalBoundaryShifts, + frameSeconds, + leadingContextSeconds); + } + + public static NeutrinoFrameChunk[] BuildFrameChunks( + NeutrinoPhoneChunk[] phoneChunks, + double[] boundaries, + int totalFrames, + double frameSeconds) { + + var chunks = new NeutrinoFrameChunk[phoneChunks.Length]; + for (int i = 0; i < chunks.Length; i++) { + var chunk = phoneChunks[i]; + int frameStart = Math.Clamp( + (int)Math.Round(boundaries[chunk.PhoneStart] / frameSeconds), + 0, + totalFrames); + int frameEnd = Math.Clamp( + (int)Math.Round(boundaries[chunk.PhoneStart + chunk.PhoneCount] / frameSeconds), + frameStart, + totalFrames); + chunks[i] = new NeutrinoFrameChunk( + chunk.PhoneStart, + chunk.PhoneCount, + frameStart, + frameEnd - frameStart, + chunk.IsActive); + } + return chunks; + } + + public static T[] Slice(T[] values, int start, int length) { + var result = new T[length]; + Array.Copy(values, start, result, 0, length); + return result; + } + + public static double NormalizeBoundaryStart(double[] boundaries) { + if (boundaries.Length == 0) { + return 0; + } + double start = boundaries[0]; + for (int i = 0; i < boundaries.Length; i++) { + boundaries[i] -= start; + } + return start; + } + + static double[] BuildBaseBoundaryTimes(float[] scoreDurations, long[] phonePositions) { + int numPhones = scoreDurations.Length; + var boundaries = new double[numPhones + 1]; + double time = 0; + for (int i = 0; i < numPhones; i++) { + boundaries[i] = time; + long nextPosition = i + 1 < numPhones ? phonePositions[i + 1] : -1; + if (i == numPhones - 1 || nextPosition <= phonePositions[i]) { + time += scoreDurations[i]; + } + } + boundaries[numPhones] = time; + return boundaries; + } + + static double[] ApplyTimingBoundaryShifts( + double[] baseBoundaries, + float[] boundaryShifts, + double frameSeconds, + double? leadingContextSeconds) { + + var boundaries = (double[])baseBoundaries.Clone(); + if (boundaries.Length > 1 && leadingContextSeconds.HasValue) { + // Official labels normally have a leading pau, so the first active + // phone is not global boundary zero and receives gluon[0]. Emulate + // that context when an OpenUtau phrase starts directly with a phone. + double contextSeconds = Math.Max(0, leadingContextSeconds.Value); + double minBoundary = Math.Min(0, -contextSeconds + frameSeconds); + double shifted = baseBoundaries[0] + boundaryShifts[0]; + boundaries[0] = Math.Round( + Math.Max(shifted, minBoundary) * 1000.0) / 1000.0; + } + for (int i = 1; i < boundaries.Length - 1; i++) { + double shifted = baseBoundaries[i] + boundaryShifts[i]; + boundaries[i] = Math.Round( + Math.Max(shifted, boundaries[i - 1] + frameSeconds) * 1000.0) / 1000.0; + } + for (int i = 1; i < boundaries.Length; i++) { + if (boundaries[i] <= boundaries[i - 1]) { + boundaries[i] = Math.Round( + (boundaries[i - 1] + frameSeconds) * 1000.0) / 1000.0; + } + } + return boundaries; + } + } +} diff --git a/OpenUtau.Core/Neutrino/NeutrinoJapaneseDictionary.cs b/OpenUtau.Core/Neutrino/NeutrinoJapaneseDictionary.cs new file mode 100644 index 000000000..f60c9ed5b --- /dev/null +++ b/OpenUtau.Core/Neutrino/NeutrinoJapaneseDictionary.cs @@ -0,0 +1,201 @@ +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace OpenUtau.Core.Neutrino { + internal static class NeutrinoJapaneseDictionary { + static readonly Dictionary entries = + new Dictionary() { + ["くぁ"] = new[] { "k", "w", "a" }, + ["くぃ"] = new[] { "k", "w", "i" }, + ["くぅ"] = new[] { "k", "w", "u" }, + ["くぇ"] = new[] { "k", "w", "e" }, + ["くぉ"] = new[] { "k", "w", "o" }, + ["くゎ"] = new[] { "k", "w", "a" }, + ["ぐぁ"] = new[] { "g", "w", "a" }, + ["ぐぃ"] = new[] { "g", "w", "i" }, + ["ぐぅ"] = new[] { "g", "w", "u" }, + ["ぐぇ"] = new[] { "g", "w", "e" }, + ["ぐぉ"] = new[] { "g", "w", "o" }, + ["ぐゎ"] = new[] { "g", "w", "a" }, + ["ゔょ"] = new[] { "by", "o" }, + ["ゔゅ"] = new[] { "by", "u" }, + ["ゔゃ"] = new[] { "by", "a" }, + ["ゔぉ"] = new[] { "v", "o" }, + ["ゔぇ"] = new[] { "v", "e" }, + ["ゔぃ"] = new[] { "v", "i" }, + ["ゔぁ"] = new[] { "v", "a" }, + ["ゔ"] = new[] { "v", "u" }, + ["ん"] = new[] { "N" }, + ["を"] = new[] { "o" }, + ["ゑ"] = new[] { "e" }, + ["ゐ"] = new[] { "i" }, + ["わ"] = new[] { "w", "a" }, + ["ゎ"] = new[] { "w", "a" }, + ["ろ"] = new[] { "r", "o" }, + ["れ"] = new[] { "r", "e" }, + ["る"] = new[] { "r", "u" }, + ["りょ"] = new[] { "ry", "o" }, + ["りゅ"] = new[] { "ry", "u" }, + ["りゃ"] = new[] { "ry", "a" }, + ["りぇ"] = new[] { "ry", "e" }, + ["り"] = new[] { "r", "i" }, + ["ら"] = new[] { "r", "a" }, + ["よ"] = new[] { "y", "o" }, + ["ょ"] = new[] { "y", "o" }, + ["ゆ"] = new[] { "y", "u" }, + ["ゅ"] = new[] { "y", "u" }, + ["や"] = new[] { "y", "a" }, + ["ゃ"] = new[] { "y", "a" }, + ["も"] = new[] { "m", "o" }, + ["め"] = new[] { "m", "e" }, + ["む"] = new[] { "m", "u" }, + ["みょ"] = new[] { "my", "o" }, + ["みゅ"] = new[] { "my", "u" }, + ["みゃ"] = new[] { "my", "a" }, + ["みぇ"] = new[] { "my", "e" }, + ["み"] = new[] { "m", "i" }, + ["ま"] = new[] { "m", "a" }, + ["ぽ"] = new[] { "p", "o" }, + ["ぼ"] = new[] { "b", "o" }, + ["ほ"] = new[] { "h", "o" }, + ["ぺ"] = new[] { "p", "e" }, + ["べ"] = new[] { "b", "e" }, + ["へ"] = new[] { "h", "e" }, + ["ぷ"] = new[] { "p", "u" }, + ["ぶ"] = new[] { "b", "u" }, + ["ふぉ"] = new[] { "f", "o" }, + ["ふぇ"] = new[] { "f", "e" }, + ["ふぃ"] = new[] { "f", "i" }, + ["ふぁ"] = new[] { "f", "a" }, + ["ふ"] = new[] { "f", "u" }, + ["ぴょ"] = new[] { "py", "o" }, + ["ぴゅ"] = new[] { "py", "u" }, + ["ぴゃ"] = new[] { "py", "a" }, + ["ぴぇ"] = new[] { "py", "e" }, + ["ぴ"] = new[] { "p", "i" }, + ["びょ"] = new[] { "by", "o" }, + ["びゅ"] = new[] { "by", "u" }, + ["びゃ"] = new[] { "by", "a" }, + ["びぇ"] = new[] { "by", "e" }, + ["び"] = new[] { "b", "i" }, + ["ひょ"] = new[] { "hy", "o" }, + ["ひゅ"] = new[] { "hy", "u" }, + ["ひゃ"] = new[] { "hy", "a" }, + ["ひぇ"] = new[] { "hy", "e" }, + ["ひ"] = new[] { "h", "i" }, + ["ぱ"] = new[] { "p", "a" }, + ["ば"] = new[] { "b", "a" }, + ["は"] = new[] { "h", "a" }, + ["の"] = new[] { "n", "o" }, + ["ね"] = new[] { "n", "e" }, + ["ぬ"] = new[] { "n", "u" }, + ["にょ"] = new[] { "ny", "o" }, + ["にゅ"] = new[] { "ny", "u" }, + ["にゃ"] = new[] { "ny", "a" }, + ["にぇ"] = new[] { "ny", "e" }, + ["に"] = new[] { "n", "i" }, + ["な"] = new[] { "n", "a" }, + ["どぅ"] = new[] { "d", "u" }, + ["ど"] = new[] { "d", "o" }, + ["とぅ"] = new[] { "t", "u" }, + ["と"] = new[] { "t", "o" }, + ["でょ"] = new[] { "dy", "o" }, + ["でゅ"] = new[] { "dy", "u" }, + ["でゃ"] = new[] { "dy", "a" }, + ["でぇ"] = new[] { "dy", "e" }, + ["でぃ"] = new[] { "d", "i" }, + ["で"] = new[] { "d", "e" }, + ["てょ"] = new[] { "ty", "o" }, + ["てゅ"] = new[] { "ty", "u" }, + ["てゃ"] = new[] { "ty", "a" }, + ["てぃ"] = new[] { "t", "i" }, + ["て"] = new[] { "t", "e" }, + ["づ"] = new[] { "z", "u" }, + ["つぉ"] = new[] { "ts", "o" }, + ["つぇ"] = new[] { "ts", "e" }, + ["つぃ"] = new[] { "ts", "i" }, + ["つぁ"] = new[] { "ts", "a" }, + ["つ"] = new[] { "ts", "u" }, + ["っ"] = new[] { "cl" }, + ["ぢ"] = new[] { "j", "i" }, + ["ちょ"] = new[] { "ch", "o" }, + ["ちゅ"] = new[] { "ch", "u" }, + ["ちゃ"] = new[] { "ch", "a" }, + ["ちぇ"] = new[] { "ch", "e" }, + ["ち"] = new[] { "ch", "i" }, + ["だ"] = new[] { "d", "a" }, + ["た"] = new[] { "t", "a" }, + ["ぞ"] = new[] { "z", "o" }, + ["そ"] = new[] { "s", "o" }, + ["ぜ"] = new[] { "z", "e" }, + ["せ"] = new[] { "s", "e" }, + ["ずぃ"] = new[] { "z", "i" }, + ["ず"] = new[] { "z", "u" }, + ["すぃ"] = new[] { "s", "i" }, + ["す"] = new[] { "s", "u" }, + ["じょ"] = new[] { "j", "o" }, + ["じゅ"] = new[] { "j", "u" }, + ["じゃ"] = new[] { "j", "a" }, + ["じぇ"] = new[] { "j", "e" }, + ["じ"] = new[] { "j", "i" }, + ["しょ"] = new[] { "sh", "o" }, + ["しゅ"] = new[] { "sh", "u" }, + ["しゃ"] = new[] { "sh", "a" }, + ["しぇ"] = new[] { "sh", "e" }, + ["しぃ"] = new[] { "s", "i" }, + ["し"] = new[] { "sh", "i" }, + ["ざ"] = new[] { "z", "a" }, + ["さ"] = new[] { "s", "a" }, + ["ご"] = new[] { "g", "o" }, + ["こ"] = new[] { "k", "o" }, + ["げ"] = new[] { "g", "e" }, + ["け"] = new[] { "k", "e" }, + ["ぐ"] = new[] { "g", "u" }, + ["く"] = new[] { "k", "u" }, + ["ぎょ"] = new[] { "gy", "o" }, + ["ぎゅ"] = new[] { "gy", "u" }, + ["ぎゃ"] = new[] { "gy", "a" }, + ["ぎぇ"] = new[] { "gy", "e" }, + ["ぎ"] = new[] { "g", "i" }, + ["きょ"] = new[] { "ky", "o" }, + ["きゅ"] = new[] { "ky", "u" }, + ["きゃ"] = new[] { "ky", "a" }, + ["きぇ"] = new[] { "ky", "e" }, + ["き"] = new[] { "k", "i" }, + ["が"] = new[] { "g", "a" }, + ["か"] = new[] { "k", "a" }, + ["お"] = new[] { "o" }, + ["ぉ"] = new[] { "o" }, + ["え"] = new[] { "e" }, + ["ぇ"] = new[] { "e" }, + ["うぉ"] = new[] { "w", "o" }, + ["うぇ"] = new[] { "w", "e" }, + ["うぃ"] = new[] { "w", "i" }, + ["う"] = new[] { "u" }, + ["ぅ"] = new[] { "u" }, + ["いぇ"] = new[] { "y", "e" }, + ["い"] = new[] { "i" }, + ["ぃ"] = new[] { "i" }, + ["あ"] = new[] { "a" }, + ["ぁ"] = new[] { "a" }, + }; + + public static IEnumerable Phonemes => entries.Values.SelectMany(value => value).Distinct(); + + public static bool TryGetValue(string text, out string[] phonemes) { + string key = ToHiragana(text.Normalize(NormalizationForm.FormC)); + return entries.TryGetValue(key, out phonemes); + } + + static string ToHiragana(string text) { + var chars = text.ToCharArray(); + for (int i = 0; i < chars.Length; i++) { + if (chars[i] >= '\u30a1' && chars[i] <= '\u30f6') { + chars[i] = (char)(chars[i] - 0x60); + } + } + return new string(chars); + } + } +} diff --git a/OpenUtau.Core/Neutrino/NeutrinoPhoneme.cs b/OpenUtau.Core/Neutrino/NeutrinoPhoneme.cs new file mode 100644 index 000000000..412f16a66 --- /dev/null +++ b/OpenUtau.Core/Neutrino/NeutrinoPhoneme.cs @@ -0,0 +1,352 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Serilog; + +namespace OpenUtau.Core.Neutrino { + /// + /// Phoneme-to-ID mapping used by NEUTRINO Tau v3.x Japanese models. + /// The IDs are taken from the official binary's ONNX tensor input, not from + /// dictionary order. + /// + public static class NeutrinoPhoneme { + public const int PAU = 0; + public const int BR = 3; + public const int VR = 26; + public const int PAD = 32; + public const int AP = 41; + + public const int VocabSize = 42; + + static readonly Dictionary phonemeToId = new Dictionary() { + {"pau", 0}, + {"sil", 0}, + {"a", 1}, + {"b", 2}, + {"br", 3}, + {"by", 4}, + {"ch", 5}, + {"cl", 6}, + {"d", 7}, + {"dy", 8}, + {"e", 9}, + {"f", 10}, + {"g", 11}, + {"gy", 12}, + {"h", 13}, + {"hy", 14}, + {"i", 15}, + {"j", 16}, + {"k", 17}, + {"ky", 18}, + {"m", 19}, + {"my", 20}, + {"n", 21}, + {"N", 22}, + {"ny", 23}, + {"o", 24}, + {"p", 25}, + {"py", 27}, + {"r", 28}, + {"ry", 29}, + {"s", 30}, + {"sh", 31}, + {"t", 33}, + {"ts", 34}, + {"ty", 35}, + {"u", 36}, + {"v", 37}, + {"w", 38}, + {"y", 39}, + {"z", 40}, + {"AP", 41}, + {"ap", 41}, + }; + + static readonly Dictionary romajiToPhonemes = + new Dictionary(StringComparer.OrdinalIgnoreCase) { + {"a", new[] {"a"}}, + {"i", new[] {"i"}}, + {"u", new[] {"u"}}, + {"e", new[] {"e"}}, + {"o", new[] {"o"}}, + {"ka", new[] {"k", "a"}}, + {"ki", new[] {"k", "i"}}, + {"ku", new[] {"k", "u"}}, + {"ke", new[] {"k", "e"}}, + {"ko", new[] {"k", "o"}}, + {"kya", new[] {"ky", "a"}}, + {"kyi", new[] {"ky", "i"}}, + {"kyu", new[] {"ky", "u"}}, + {"kye", new[] {"ky", "e"}}, + {"kyo", new[] {"ky", "o"}}, + {"kwa", new[] {"k", "w", "a"}}, + {"kwi", new[] {"k", "w", "i"}}, + {"kwu", new[] {"k", "w", "u"}}, + {"kwe", new[] {"k", "w", "e"}}, + {"kwo", new[] {"k", "w", "o"}}, + {"ga", new[] {"g", "a"}}, + {"gi", new[] {"g", "i"}}, + {"gu", new[] {"g", "u"}}, + {"ge", new[] {"g", "e"}}, + {"go", new[] {"g", "o"}}, + {"gya", new[] {"gy", "a"}}, + {"gyi", new[] {"gy", "i"}}, + {"gyu", new[] {"gy", "u"}}, + {"gye", new[] {"gy", "e"}}, + {"gyo", new[] {"gy", "o"}}, + {"gwa", new[] {"g", "w", "a"}}, + {"gwi", new[] {"g", "w", "i"}}, + {"gwu", new[] {"g", "w", "u"}}, + {"gwe", new[] {"g", "w", "e"}}, + {"gwo", new[] {"g", "w", "o"}}, + {"sa", new[] {"s", "a"}}, + {"si", new[] {"s", "i"}}, + {"shi", new[] {"sh", "i"}}, + {"su", new[] {"s", "u"}}, + {"se", new[] {"s", "e"}}, + {"so", new[] {"s", "o"}}, + {"sya", new[] {"sh", "a"}}, + {"sha", new[] {"sh", "a"}}, + {"syu", new[] {"sh", "u"}}, + {"shu", new[] {"sh", "u"}}, + {"sye", new[] {"sh", "e"}}, + {"she", new[] {"sh", "e"}}, + {"syo", new[] {"sh", "o"}}, + {"sho", new[] {"sh", "o"}}, + {"za", new[] {"z", "a"}}, + {"zi", new[] {"z", "i"}}, + {"ji", new[] {"j", "i"}}, + {"zu", new[] {"z", "u"}}, + {"ze", new[] {"z", "e"}}, + {"zo", new[] {"z", "o"}}, + {"zya", new[] {"j", "a"}}, + {"ja", new[] {"j", "a"}}, + {"zyu", new[] {"j", "u"}}, + {"ju", new[] {"j", "u"}}, + {"zye", new[] {"j", "e"}}, + {"je", new[] {"j", "e"}}, + {"zyo", new[] {"j", "o"}}, + {"jo", new[] {"j", "o"}}, + {"ta", new[] {"t", "a"}}, + {"ti", new[] {"t", "i"}}, + {"chi", new[] {"ch", "i"}}, + {"tu", new[] {"t", "u"}}, + {"tsu", new[] {"ts", "u"}}, + {"te", new[] {"t", "e"}}, + {"to", new[] {"t", "o"}}, + {"tya", new[] {"ty", "a"}}, + {"cha", new[] {"ch", "a"}}, + {"tyu", new[] {"ty", "u"}}, + {"chu", new[] {"ch", "u"}}, + {"tye", new[] {"ty", "e"}}, + {"che", new[] {"ch", "e"}}, + {"tyo", new[] {"ty", "o"}}, + {"cho", new[] {"ch", "o"}}, + {"tsa", new[] {"ts", "a"}}, + {"tsi", new[] {"ts", "i"}}, + {"tse", new[] {"ts", "e"}}, + {"tso", new[] {"ts", "o"}}, + {"da", new[] {"d", "a"}}, + {"di", new[] {"d", "i"}}, + {"du", new[] {"d", "u"}}, + {"de", new[] {"d", "e"}}, + {"do", new[] {"d", "o"}}, + {"dya", new[] {"dy", "a"}}, + {"dyi", new[] {"dy", "i"}}, + {"dyu", new[] {"dy", "u"}}, + {"dye", new[] {"dy", "e"}}, + {"dyo", new[] {"dy", "o"}}, + {"na", new[] {"n", "a"}}, + {"ni", new[] {"n", "i"}}, + {"nu", new[] {"n", "u"}}, + {"ne", new[] {"n", "e"}}, + {"no", new[] {"n", "o"}}, + {"nya", new[] {"ny", "a"}}, + {"nyi", new[] {"ny", "i"}}, + {"nyu", new[] {"ny", "u"}}, + {"nye", new[] {"ny", "e"}}, + {"nyo", new[] {"ny", "o"}}, + {"ha", new[] {"h", "a"}}, + {"hi", new[] {"h", "i"}}, + {"hu", new[] {"f", "u"}}, + {"fu", new[] {"f", "u"}}, + {"he", new[] {"h", "e"}}, + {"ho", new[] {"h", "o"}}, + {"hya", new[] {"hy", "a"}}, + {"hyi", new[] {"hy", "i"}}, + {"hyu", new[] {"hy", "u"}}, + {"hye", new[] {"hy", "e"}}, + {"hyo", new[] {"hy", "o"}}, + {"fa", new[] {"f", "a"}}, + {"fi", new[] {"f", "i"}}, + {"fe", new[] {"f", "e"}}, + {"fo", new[] {"f", "o"}}, + {"ba", new[] {"b", "a"}}, + {"bi", new[] {"b", "i"}}, + {"bu", new[] {"b", "u"}}, + {"be", new[] {"b", "e"}}, + {"bo", new[] {"b", "o"}}, + {"bya", new[] {"by", "a"}}, + {"byi", new[] {"by", "i"}}, + {"byu", new[] {"by", "u"}}, + {"bye", new[] {"by", "e"}}, + {"byo", new[] {"by", "o"}}, + {"pa", new[] {"p", "a"}}, + {"pi", new[] {"p", "i"}}, + {"pu", new[] {"p", "u"}}, + {"pe", new[] {"p", "e"}}, + {"po", new[] {"p", "o"}}, + {"pya", new[] {"py", "a"}}, + {"pyi", new[] {"py", "i"}}, + {"pyu", new[] {"py", "u"}}, + {"pye", new[] {"py", "e"}}, + {"pyo", new[] {"py", "o"}}, + {"ma", new[] {"m", "a"}}, + {"mi", new[] {"m", "i"}}, + {"mu", new[] {"m", "u"}}, + {"me", new[] {"m", "e"}}, + {"mo", new[] {"m", "o"}}, + {"mya", new[] {"my", "a"}}, + {"myi", new[] {"my", "i"}}, + {"myu", new[] {"my", "u"}}, + {"mye", new[] {"my", "e"}}, + {"myo", new[] {"my", "o"}}, + {"ya", new[] {"y", "a"}}, + {"yu", new[] {"y", "u"}}, + {"ye", new[] {"y", "e"}}, + {"yo", new[] {"y", "o"}}, + {"ra", new[] {"r", "a"}}, + {"ri", new[] {"r", "i"}}, + {"ru", new[] {"r", "u"}}, + {"re", new[] {"r", "e"}}, + {"ro", new[] {"r", "o"}}, + {"rya", new[] {"ry", "a"}}, + {"ryi", new[] {"ry", "i"}}, + {"ryu", new[] {"ry", "u"}}, + {"rye", new[] {"ry", "e"}}, + {"ryo", new[] {"ry", "o"}}, + {"wa", new[] {"w", "a"}}, + {"wi", new[] {"w", "i"}}, + {"wu", new[] {"w", "u"}}, + {"we", new[] {"w", "e"}}, + {"wo", new[] {"w", "o"}}, + {"va", new[] {"v", "a"}}, + {"vi", new[] {"v", "i"}}, + {"vu", new[] {"v", "u"}}, + {"ve", new[] {"v", "e"}}, + {"vo", new[] {"v", "o"}}, + {"n", new[] {"N"}}, + {"nn", new[] {"N"}}, + }; + + public static IEnumerable AllPhonemes => + phonemeToId + .Where(kv => kv.Key != "sil" && kv.Key != "ap") + .OrderBy(kv => kv.Value) + .ThenBy(kv => kv.Key, StringComparer.Ordinal) + .Select(kv => kv.Key) + .Concat(NeutrinoJapaneseDictionary.Phonemes) + .Distinct(); + + public static int GetPhonemeId(string phoneme) { + phoneme = phoneme?.Trim(); + if (string.IsNullOrEmpty(phoneme)) { + return PAU; + } + if (phoneme == "R" + || phoneme.Equals("SP", StringComparison.OrdinalIgnoreCase) + || phoneme.Equals("rest", StringComparison.OrdinalIgnoreCase)) { + return PAU; + } + if (phonemeToId.TryGetValue(phoneme, out int id)) { + return id; + } + if (phonemeToId.TryGetValue(phoneme.ToLowerInvariant(), out id)) { + return id; + } + Log.Warning($"Unknown NEUTRINO phoneme: {phoneme}"); + return PAU; + } + + public static int[] KanaToPhonemeIds(string kana) { + return KanaToPhonemes(kana).Select(p => GetPhonemeId(p)).ToArray(); + } + + public static string[] RenderPhoneToPhonemes(string phone) { + phone = phone?.Trim(); + if (string.IsNullOrEmpty(phone)) { + return new[] { "pau" }; + } + if (IsKnownPhoneme(phone)) { + return new[] { NormalizePhoneme(phone) }; + } + return KanaToPhonemes(phone); + } + + public static bool IsVowelPhoneme(string phoneme) { + phoneme = NormalizePhoneme(phoneme?.Trim() ?? string.Empty); + return phoneme == "a" + || phoneme == "i" + || phoneme == "u" + || phoneme == "e" + || phoneme == "o" + || phoneme == "N" + || phoneme == "pau" + || phoneme == "AP"; + } + + public static string[] KanaToPhonemes(string kana) { + kana = kana?.Trim(); + if (string.IsNullOrEmpty(kana)) { + return new[] { "pau" }; + } + if (kana == "R" + || kana.Equals("SP", StringComparison.OrdinalIgnoreCase) + || kana.Equals("rest", StringComparison.OrdinalIgnoreCase)) { + return new[] { "pau" }; + } + if (kana.Equals("n", StringComparison.OrdinalIgnoreCase) + || kana.Equals("nn", StringComparison.OrdinalIgnoreCase)) { + return new[] { "N" }; + } + + var parts = kana.Split(new[] { ' ', '\t' }, StringSplitOptions.RemoveEmptyEntries); + if (parts.Length > 1 && parts.All(IsKnownPhoneme)) { + return parts.Select(NormalizePhoneme).ToArray(); + } + if (IsKnownPhoneme(kana)) { + return new[] { NormalizePhoneme(kana) }; + } + + var normalizedKana = kana.Normalize(NormalizationForm.FormC); + if (NeutrinoJapaneseDictionary.TryGetValue(normalizedKana, out var phonemes)) { + return phonemes; + } + if (romajiToPhonemes.TryGetValue(kana, out phonemes)) { + return phonemes; + } + + Log.Warning($"Kana/romaji not in NEUTRINO dictionary: {kana}"); + return new[] { "pau" }; + } + + static bool IsKnownPhoneme(string phoneme) { + phoneme = phoneme?.Trim(); + if (string.IsNullOrEmpty(phoneme)) { + return false; + } + return phonemeToId.ContainsKey(phoneme) + || phonemeToId.ContainsKey(phoneme.ToLowerInvariant()); + } + + static string NormalizePhoneme(string phoneme) { + if (phonemeToId.ContainsKey(phoneme)) { + return phoneme; + } + var lower = phoneme.ToLowerInvariant(); + return phonemeToId.ContainsKey(lower) ? lower : phoneme; + } + } +} diff --git a/OpenUtau.Core/Neutrino/NeutrinoPhonemizer.cs b/OpenUtau.Core/Neutrino/NeutrinoPhonemizer.cs new file mode 100644 index 000000000..492de158a --- /dev/null +++ b/OpenUtau.Core/Neutrino/NeutrinoPhonemizer.cs @@ -0,0 +1,301 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.ML.OnnxRuntime; +using Microsoft.ML.OnnxRuntime.Tensors; +using OpenUtau.Api; +using OpenUtau.Core.Ustx; +using Serilog; + +namespace OpenUtau.Core.Neutrino { + /// + /// NEUTRINO phonemizer: converts lyrics to model phonemes and uses t.bin + /// to place default phoneme boundaries when the singer is available. + /// + [Phonemizer("NEUTRINO Phonemizer", "NEUTRINO", language: "JA")] + public class NeutrinoPhonemizer : Phonemizer { + const double defaultConsonantMs = 60; + const int minPhonemeTicks = 10; + const int sampleRate = 48000; + const int hopSize = 480; + + NeutrinoSinger neutrinoSinger; + readonly Dictionary timedPhonemes = new Dictionary(); + + public override void SetSinger(USinger singer) { + neutrinoSinger = singer as NeutrinoSinger; + } + + public override void SetUp(Note[][] notes, UProject project, UTrack track) { + timedPhonemes.Clear(); + if (neutrinoSinger == null || notes == null || notes.Length == 0 || timeAxis == null) { + return; + } + try { + neutrinoSinger.EnsureTimingSession(); + foreach (var phrase in SplitPhrases(notes)) { + BuildTimedPhonemes(phrase); + } + } catch (Exception e) { + timedPhonemes.Clear(); + Log.Warning(e, "Failed to run NEUTRINO timing model for phoneme panel; using estimated phoneme positions."); + } + } + + public override Result Process(Note[] notes, Note? prev, Note? next, + Note? prevNeighbour, Note? nextNeighbour, Note[] prevs) { + + if (timedPhonemes.TryGetValue(notes[0].position, out var timed)) { + return new Result { + phonemes = timed.ToArray(), + }; + } + + var lyric = string.IsNullOrWhiteSpace(notes[0].phoneticHint) + ? notes[0].lyric ?? "R" + : notes[0].phoneticHint; + var phonemes = LyricToPhonemes(lyric); + var positions = DistributePhonemes(phonemes, notes); + var resultPhonemes = phonemes + .Select((phoneme, index) => new Phoneme { + index = index, + phoneme = phoneme, + position = positions[index], + }) + .ToArray(); + return new Result { + phonemes = PostProcessPhonemePositions(resultPhonemes, notes), + }; + } + + List SplitPhrases(Note[][] noteGroups) { + var phrases = new List(); + List phrase = null; + int previousEnd = int.MinValue; + foreach (var group in noteGroups.Where(group => group.Length > 0)) { + int start = group[0].position; + int end = group[^1].position + group[^1].duration; + if (phrase == null || start > previousEnd) { + phrase = new List(); + int contextStart = previousEnd == int.MinValue ? 0 : previousEnd; + phrases.Add(new TimedPhrase(phrase, Math.Min(start, contextStart))); + } + phrase.Add(group); + previousEnd = Math.Max(previousEnd, end); + } + return phrases; + } + + void BuildTimedPhonemes(TimedPhrase phrase) { + var noteGroups = phrase.NoteGroups; + var scoreNotes = new List(); + var phoneRefs = new List(); + var groupsByPosition = noteGroups.ToDictionary(group => group[0].position); + var groupedPhonemes = groupsByPosition.ToDictionary(pair => pair.Key, _ => new List()); + + foreach (var group in noteGroups) { + var lyric = string.IsNullOrWhiteSpace(group[0].phoneticHint) + ? group[0].lyric ?? "R" + : group[0].phoneticHint; + var phonemes = LyricToPhonemes(lyric); + var modelPhones = new NeutrinoScorePhoneInput[phonemes.Length]; + for (int i = 0; i < phonemes.Length; i++) { + int id = NeutrinoPhoneme.GetPhonemeId(phonemes[i]); + int sourceIndex = phoneRefs.Count; + modelPhones[i] = new NeutrinoScorePhoneInput(id, sourceIndex); + phoneRefs.Add(new TimedPhoneRef { + groupPosition = group[0].position, + index = i, + phoneme = phonemes[i], + }); + } + + for (int noteIndex = 0; noteIndex < group.Length; noteIndex++) { + var note = group[noteIndex]; + scoreNotes.Add(new NeutrinoScoreNoteInput( + (float)MusicMath.ToneToFreq(note.tone), + Math.Max(0.001f, (float)(GetNoteDurationMs(note) / 1000.0)), + noteIndex > 0 && NeutrinoInferenceUtil.IsExtensionLyric(note.lyric), + noteIndex == 0 + ? modelPhones + : Array.Empty())); + } + } + + var sequence = NeutrinoInferenceUtil.BuildScoreSequence(scoreNotes); + int numPhones = sequence.PhonemeIds.Length; + if (numPhones == 0) { + return; + } + + int phraseStartTick = noteGroups[0][0].position; + double leadingContextSeconds = GetLeadingContextSeconds( + phraseStartTick, + phrase.ContextStartTick); + var boundaries = BuildChunkedTimingBoundaries( + sequence.PhonemeIds, + sequence.ScorePitchesHz, + sequence.ScoreDurations, + sequence.PhonePositions, + leadingContextSeconds); + double phraseStartMs = timeAxis.TickPosToMsPos(phraseStartTick); + + for (int modelPhone = 0; modelPhone < sequence.SourcePhoneIndices.Length; modelPhone++) { + int sourceIndex = sequence.SourcePhoneIndices[modelPhone]; + if (sourceIndex < 0) { + continue; + } + var phoneRef = phoneRefs[sourceIndex]; + double positionMs = phraseStartMs + boundaries[modelPhone] * 1000.0; + int position = timeAxis.MsPosToTickPos(positionMs) - phoneRef.groupPosition; + groupedPhonemes[phoneRef.groupPosition].Add(new Phoneme { + index = phoneRef.index, + phoneme = phoneRef.phoneme, + position = position, + }); + } + + foreach (var pair in groupedPhonemes) { + if (pair.Value.Count > 0) { + timedPhonemes[pair.Key] = PostProcessTimedPhonemePositions( + pair.Value.ToArray(), + groupsByPosition[pair.Key]); + } + } + } + + protected virtual string[] LyricToPhonemes(string lyric) { + return NeutrinoPhoneme.KanaToPhonemes(lyric); + } + + protected virtual Phoneme[] PostProcessPhonemePositions(Phoneme[] phonemes, Note[] notes) { + return phonemes; + } + + protected virtual Phoneme[] PostProcessTimedPhonemePositions(Phoneme[] phonemes, Note[] notes) { + return PostProcessPhonemePositions(phonemes, notes); + } + + double GetNoteDurationMs(Note note) { + double startMs = timeAxis.TickPosToMsPos(note.position); + double endMs = timeAxis.TickPosToMsPos(note.position + note.duration); + return Math.Max(1, endMs - startMs); + } + + double[] BuildChunkedTimingBoundaries( + long[] phonemeIds, + float[] scorePitchesHz, + float[] scoreDurations, + long[] phonePositions, + double leadingContextSeconds) { + + var chunks = NeutrinoInferenceUtil.BuildPhoneChunks(phonemeIds); + double frameSeconds = (double)hopSize / sampleRate; + return NeutrinoInferenceUtil.BuildTimingBoundaries( + scoreDurations, + phonePositions, + chunks, + frameSeconds, + chunk => { + var chunkPitches = NeutrinoInferenceUtil.Slice( + scorePitchesHz, chunk.PhoneStart, chunk.PhoneCount); + var chunkDurations = NeutrinoInferenceUtil.Slice( + scoreDurations, chunk.PhoneStart, chunk.PhoneCount); + var chunkPositions = NeutrinoInferenceUtil.Slice( + phonePositions, chunk.PhoneStart, chunk.PhoneCount); + var chunkIds = NeutrinoInferenceUtil.Slice( + phonemeIds, chunk.PhoneStart, chunk.PhoneCount); + var timingInputs = new List { + NamedOnnxValue.CreateFromTensor("electron", + new DenseTensor(chunkIds, new[] { 1, chunk.PhoneCount })), + NamedOnnxValue.CreateFromTensor("muon", + new DenseTensor(chunkPitches, new[] { 1, chunk.PhoneCount })), + NamedOnnxValue.CreateFromTensor("tau", + new DenseTensor(chunkDurations, new[] { 1, chunk.PhoneCount })), + NamedOnnxValue.CreateFromTensor("selectron", + new DenseTensor(chunkPositions, new[] { 1, chunk.PhoneCount })), + }; + return NeutrinoInferenceUtil.RequireTimingBoundaryLength( + neutrinoSinger.RunTiming(timingInputs), + chunk.PhoneCount, + "NEUTRINO v3 t.bin timing output"); + }, + leadingContextSeconds); + } + + double GetLeadingContextSeconds(int phraseStartTick, int contextStartTick) { + contextStartTick = Math.Max( + phraseStartTick - NeutrinoRenderer.headTicks, + Math.Min(phraseStartTick, contextStartTick)); + double phraseStartMs = timeAxis.TickPosToMsPos(phraseStartTick); + double contextStartMs = timeAxis.TickPosToMsPos(contextStartTick); + return Math.Max(0, (phraseStartMs - contextStartMs) / 1000.0); + } + + readonly struct TimedPhrase { + public List NoteGroups { get; } + public int ContextStartTick { get; } + + public TimedPhrase(List noteGroups, int contextStartTick) { + NoteGroups = noteGroups; + ContextStartTick = contextStartTick; + } + } + + struct TimedPhoneRef { + public int groupPosition; + public int index; + public string phoneme; + } + + int[] DistributePhonemes(string[] phonemes, Note[] notes) { + if (phonemes.Length == 0) { + return Array.Empty(); + } + if (phonemes.Length == 1) { + return new[] { 0 }; + } + + int noteStart = notes[0].position; + int noteEnd = notes.Last().position + notes.Last().duration; + int totalDuration = Math.Max(minPhonemeTicks, noteEnd - noteStart); + int lastStart = Math.Max(0, totalDuration - minPhonemeTicks); + int consonantTicks = Math.Clamp(DefaultConsonantTicks(noteStart), minPhonemeTicks, lastStart); + int firstVowel = Array.FindIndex(phonemes, NeutrinoPhoneme.IsVowelPhoneme); + + var positions = new int[phonemes.Length]; + if (firstVowel > 0) { + for (int i = 0; i < phonemes.Length; i++) { + if (i <= firstVowel) { + positions[i] = (int)Math.Round((double)consonantTicks * i / firstVowel); + } else { + int restCount = phonemes.Length - firstVowel; + positions[i] = consonantTicks + + (int)Math.Round((double)(lastStart - consonantTicks) * (i - firstVowel) / restCount); + } + } + } else { + for (int i = 0; i < phonemes.Length; i++) { + positions[i] = (int)Math.Round((double)lastStart * i / (phonemes.Length - 1)); + } + } + + positions[0] = 0; + for (int i = 1; i < positions.Length; i++) { + int maxPosition = Math.Max(positions[i - 1] + minPhonemeTicks, + totalDuration - minPhonemeTicks * (positions.Length - i)); + positions[i] = Math.Min(Math.Max(positions[i], positions[i - 1] + minPhonemeTicks), maxPosition); + } + return positions; + } + + int DefaultConsonantTicks(int notePosition) { + if (timeAxis == null) { + return 60; + } + double noteMs = timeAxis.TickPosToMsPos(notePosition); + return Math.Max(minPhonemeTicks, + timeAxis.TicksBetweenMsPos(noteMs, noteMs + defaultConsonantMs)); + } + } +} diff --git a/OpenUtau.Core/Neutrino/NeutrinoRenderer.cs b/OpenUtau.Core/Neutrino/NeutrinoRenderer.cs new file mode 100644 index 000000000..97c92031f --- /dev/null +++ b/OpenUtau.Core/Neutrino/NeutrinoRenderer.cs @@ -0,0 +1,814 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.ML.OnnxRuntime; +using Microsoft.ML.OnnxRuntime.Tensors; +using NAudio.Wave; +using OpenUtau.Core.Format; +using OpenUtau.Core.Render; +using OpenUtau.Core.SignalChain; +using OpenUtau.Core.Ustx; +using Serilog; + +namespace OpenUtau.Core.Neutrino { + public class NeutrinoRenderer : IRenderer { + public const int headTicks = 480; + public const int tailTicks = 480; + + const int sampleRate = 48000; + const int outputSampleRate = 44100; + const int hopSize = 480; + const int pitchInterval = 5; + const int numMelBins = 100; + const int cacheVersion = 1; + const int edgeSilenceSamples = 240; + const int fadeInSamples = 240; + const int fadeOutSamples = 240; + const float f0Min = 40f; + const float f0Max = 2000f; + const float melspecMin = -7f; + const float melspecMax = 1f; + const float wavScale = 0.9885531068f; + const float wavClamp = 0.9988493919f; + + static readonly HashSet supportedExp = new HashSet() { + Format.Ustx.DYN, + Format.Ustx.PITD, + Format.Ustx.SHFC, + }; + + static readonly object lockObj = new object(); + + sealed class NeutrinoTimingContext { + public long[] PhonemeIds { get; } + public float[] ScorePitchesHz { get; } + public float[] ScoreDurations { get; } + public long[] PhonePositions { get; } + public float[] TimingDurations { get; } + public long[] FramePhonemeMap { get; } + public int TotalFrames { get; } + public double StartOffsetSeconds { get; } + public NeutrinoFrameChunk[] Chunks { get; } + + public NeutrinoTimingContext( + long[] phonemeIds, + float[] scorePitchesHz, + float[] scoreDurations, + long[] phonePositions, + float[] timingDurations, + long[] framePhonemeMap, + int totalFrames, + double startOffsetSeconds, + NeutrinoFrameChunk[] chunks) { + + PhonemeIds = phonemeIds; + ScorePitchesHz = scorePitchesHz; + ScoreDurations = scoreDurations; + PhonePositions = phonePositions; + TimingDurations = timingDurations; + FramePhonemeMap = framePhonemeMap; + TotalFrames = totalFrames; + StartOffsetSeconds = startOffsetSeconds; + Chunks = chunks; + } + } + + + public USingerType SingerType => USingerType.Neutrino; + public bool SupportsRenderPitch => true; + + public bool SupportsExpression(UExpressionDescriptor descriptor) { + return supportedExp.Contains(descriptor.abbr); + } + + public RenderResult Layout(RenderPhrase phrase) { + var headMs = phrase.positionMs - phrase.timeAxis.TickPosToMsPos(phrase.position - headTicks); + var tailMs = phrase.timeAxis.TickPosToMsPos(phrase.end + tailTicks) - phrase.endMs; + return new RenderResult() { + leadingMs = headMs, + positionMs = phrase.positionMs, + estimatedLengthMs = headMs + phrase.durationMs + tailMs, + }; + } + + public Task Render(RenderPhrase phrase, Progress progress, + int trackNo, CancellationTokenSource cancellation, bool isPreRender) { + + return Task.Run(() => { + lock (lockObj) { + if (cancellation.IsCancellationRequested) { + return new RenderResult(); + } + + string progressInfo = $"Track {trackNo + 1}: {this} " + + $"\"{string.Join(" ", phrase.phones.Select(p => p.phoneme))}\""; + progress.Complete(0, progressInfo); + + var result = Layout(phrase); + var wavPath = Path.Join(PathManager.Inst.CachePath, + $"neutrino-v3-native-v{cacheVersion}-{phrase.hash:x16}.wav"); + phrase.AddCacheFile(wavPath); + + if (TryLoadWaveCache(wavPath, out var cachedSamples)) { + result.samples = cachedSamples; + } + + if (result.samples == null) { + result.samples = InvokeNeutrino(phrase, cancellation); + if (result.samples != null) { + Wave.CorrectSampleScale(result.samples); + SaveWaveCache(wavPath, result.samples); + } + } + + if (result.samples != null) { + Renderers.ApplyDynamics(phrase, result); + } + progress.Complete(phrase.phones.Length, progressInfo); + return result; + } + }); + } + + float[] InvokeNeutrino(RenderPhrase phrase, CancellationTokenSource cancellation) { + if (cancellation.IsCancellationRequested) return null; + var timing = BuildTimingContext(phrase); + if (timing.PhonemeIds.Length == 0) { + return Array.Empty(); + } + + if (cancellation.IsCancellationRequested) return null; + + float[] f0 = BuildEditorF0(phrase, timing); + if (HasNonDefaultValue(phrase.toneShift, 0)) { + // Keep the editor pitch authoritative and apply only the contour + // difference produced by p.bin's StyleShift behavior. + var neutralF0 = RunPredictedF0(phrase, timing, applyStyleShift: false); + var shiftedF0 = RunPredictedF0(phrase, timing, applyStyleShift: true); + ApplyStyleShiftContour(f0, neutralF0, shiftedF0); + } + ClampF0(f0); + + if (cancellation.IsCancellationRequested) return null; + + var singer = phrase.singer as NeutrinoSinger; + var waveform = new float[timing.TotalFrames * hopSize]; + foreach (var chunk in timing.Chunks) { + if (!chunk.IsActive || chunk.FrameCount <= 0) { + continue; + } + if (cancellation.IsCancellationRequested) return null; + + var chunkTiming = BuildChunkTimingContext(timing, chunk); + var chunkF0 = NeutrinoInferenceUtil.Slice(f0, chunk.FrameStart, chunk.FrameCount); + var chunkWaveform = RunAcousticChunk(singer, chunkTiming, chunkF0, cancellation); + if (chunkWaveform == null) return null; + Array.Copy( + chunkWaveform, + 0, + waveform, + chunk.FrameStart * hopSize, + chunkWaveform.Length); + } + + var layout = Layout(phrase); + double waveformOffsetMs = layout.leadingMs + timing.StartOffsetSeconds * 1000.0; + int headSamples = Math.Max(0, (int)(waveformOffsetMs / 1000.0 * sampleRate)); + int tailSamples = Math.Max(0, + (int)(layout.estimatedLengthMs / 1000.0 * sampleRate) - headSamples - waveform.Length); + + int totalSamples = headSamples + waveform.Length + tailSamples; + var result = new float[totalSamples]; + Array.Copy(waveform, 0, result, headSamples, waveform.Length); + + if (sampleRate != outputSampleRate) { + var signal = new NWaves.Signals.DiscreteSignal(sampleRate, result); + signal = NWaves.Operations.Operation.Resample(signal, outputSampleRate); + result = signal.Samples; + } + + return result; + } + + NeutrinoTimingContext BuildChunkTimingContext( + NeutrinoTimingContext timing, + NeutrinoFrameChunk chunk) { + + var timingDurations = NeutrinoInferenceUtil.Slice( + timing.TimingDurations, chunk.PhoneStart, chunk.PhoneCount); + return new NeutrinoTimingContext( + NeutrinoInferenceUtil.Slice(timing.PhonemeIds, chunk.PhoneStart, chunk.PhoneCount), + NeutrinoInferenceUtil.Slice(timing.ScorePitchesHz, chunk.PhoneStart, chunk.PhoneCount), + NeutrinoInferenceUtil.Slice(timing.ScoreDurations, chunk.PhoneStart, chunk.PhoneCount), + NeutrinoInferenceUtil.Slice(timing.PhonePositions, chunk.PhoneStart, chunk.PhoneCount), + timingDurations, + BuildFramePhonemeMap(timingDurations, chunk.FrameCount), + chunk.FrameCount, + 0, + Array.Empty()); + } + + float[] RunAcousticChunk( + NeutrinoSinger singer, + NeutrinoTimingContext timing, + float[] f0, + CancellationTokenSource cancellation) { + + int numPhones = timing.PhonemeIds.Length; + int totalFrames = timing.TotalFrames; + var melspecInputs = new List { + NamedOnnxValue.CreateFromTensor("electron", + new DenseTensor(timing.PhonemeIds, new[] { 1, numPhones })), + NamedOnnxValue.CreateFromTensor("muon", + new DenseTensor(timing.TimingDurations, new[] { 1, numPhones })), + NamedOnnxValue.CreateFromTensor("tau", + new DenseTensor(timing.ScorePitchesHz, new[] { 1, numPhones })), + NamedOnnxValue.CreateFromTensor("selectron", + new DenseTensor(timing.ScoreDurations, new[] { 1, numPhones })), + NamedOnnxValue.CreateFromTensor("smuon", + new DenseTensor(timing.PhonePositions, new[] { 1, numPhones })), + NamedOnnxValue.CreateFromTensor("stau", + new DenseTensor(timing.FramePhonemeMap, new[] { 1, totalFrames })), + NamedOnnxValue.CreateFromTensor("photon", + new DenseTensor(f0, new[] { 1, totalFrames })), + }; + + float[] melSpectrogram = NeutrinoInferenceUtil.RequireLength( + singer.RunMelspec(melspecInputs), + totalFrames * numMelBins, + "NEUTRINO v3 s.bin mel output"); + ClampMelspec(melSpectrogram); + + if (cancellation.IsCancellationRequested) return null; + + var vocoderInput = new float[totalFrames * (numMelBins + 1)]; + for (int frame = 0; frame < totalFrames; frame++) { + for (int bin = 0; bin < numMelBins; bin++) { + vocoderInput[frame * (numMelBins + 1) + bin] = + melSpectrogram[frame * numMelBins + bin]; + } + vocoderInput[frame * (numMelBins + 1) + numMelBins] = f0[frame]; + } + + var vocoderInputs = new List { + NamedOnnxValue.CreateFromTensor("input", + new DenseTensor(vocoderInput, new[] { 1, totalFrames, numMelBins + 1 })), + }; + var waveform = NeutrinoInferenceUtil.RequireLength( + singer.RunVocoder(vocoderInputs), + totalFrames * hopSize, + "NEUTRINO v3 v.bin waveform output"); + if (cancellation.IsCancellationRequested) return null; + PostProcessWaveform(waveform); + return waveform; + } + + static bool HasNonDefaultValue(float[] values, float defaultValue) { + if (values == null) { + return false; + } + foreach (float value in values) { + if (Math.Abs(value - defaultValue) > 0.5f) { + return true; + } + } + return false; + } + + NeutrinoTimingContext BuildTimingContext(RenderPhrase phrase) { + var singer = phrase.singer as NeutrinoSinger; + var (phonemeIds, scorePitchesHz, scoreDurations, phonePositions, manualBoundaries) = + BuildPhonemeSequence(phrase); + + int numPhones = phonemeIds.Length; + if (numPhones == 0) { + return new NeutrinoTimingContext( + phonemeIds, + scorePitchesHz, + scoreDurations, + phonePositions, + Array.Empty(), + Array.Empty(), + 0, + 0, + Array.Empty()); + } + + var phoneChunks = NeutrinoInferenceUtil.BuildPhoneChunks(phonemeIds); + double frameSeconds = (double)hopSize / sampleRate; + double scoreOriginMs = GetScoreOriginMs(phrase); + double leadingContextSeconds = GetLeadingContextSeconds(phrase); + var boundaries = NeutrinoInferenceUtil.BuildTimingBoundaries( + scoreDurations, + phonePositions, + phoneChunks, + frameSeconds, + chunk => { + var chunkPitches = NeutrinoInferenceUtil.Slice( + scorePitchesHz, chunk.PhoneStart, chunk.PhoneCount); + var chunkScoreDurations = NeutrinoInferenceUtil.Slice( + scoreDurations, chunk.PhoneStart, chunk.PhoneCount); + var chunkPhonePositions = NeutrinoInferenceUtil.Slice( + phonePositions, chunk.PhoneStart, chunk.PhoneCount); + var chunkPhonemeIds = NeutrinoInferenceUtil.Slice( + phonemeIds, chunk.PhoneStart, chunk.PhoneCount); + var timingInputs = new List { + NamedOnnxValue.CreateFromTensor("electron", + new DenseTensor(chunkPhonemeIds, new[] { 1, chunk.PhoneCount })), + NamedOnnxValue.CreateFromTensor("muon", + new DenseTensor(chunkPitches, new[] { 1, chunk.PhoneCount })), + NamedOnnxValue.CreateFromTensor("tau", + new DenseTensor(chunkScoreDurations, new[] { 1, chunk.PhoneCount })), + NamedOnnxValue.CreateFromTensor("selectron", + new DenseTensor(chunkPhonePositions, new[] { 1, chunk.PhoneCount })), + }; + return NeutrinoInferenceUtil.RequireTimingBoundaryLength( + singer.RunTiming(timingInputs), + chunk.PhoneCount, + "NEUTRINO v3 t.bin timing output"); + }, + leadingContextSeconds); + + ApplyManualBoundaryOverrides( + boundaries, + manualBoundaries, + leadingContextSeconds); + double boundaryStartSeconds = NeutrinoInferenceUtil.NormalizeBoundaryStart(boundaries); + double startOffsetSeconds = + (scoreOriginMs - phrase.positionMs) / 1000.0 + boundaryStartSeconds; + var timingDurations = BuildTimingDurations(boundaries); + int totalFrames = Math.Max(1, (int)Math.Round(boundaries[^1] * sampleRate / hopSize)); + var framePhonemeMap = BuildFramePhonemeMap(timingDurations, totalFrames); + var frameChunks = NeutrinoInferenceUtil.BuildFrameChunks( + phoneChunks, + boundaries, + totalFrames, + (double)hopSize / sampleRate); + + return new NeutrinoTimingContext( + phonemeIds, + scorePitchesHz, + scoreDurations, + phonePositions, + timingDurations, + framePhonemeMap, + totalFrames, + startOffsetSeconds, + frameChunks); + } + + float[] RunPredictedF0( + RenderPhrase phrase, + NeutrinoTimingContext timing, + bool applyStyleShift = true) { + + if (timing.TotalFrames <= 0 || timing.PhonemeIds.Length == 0) { + return Array.Empty(); + } + + var styleShiftCentsByFrame = applyStyleShift + ? BuildStyleShiftCentsByFrame(phrase, timing) + : Array.Empty(); + var singer = phrase.singer as NeutrinoSinger; + var f0 = new float[timing.TotalFrames]; + foreach (var chunk in timing.Chunks) { + if (!chunk.IsActive || chunk.FrameCount <= 0) { + continue; + } + + var chunkTiming = BuildChunkTimingContext(timing, chunk); + var chunkStyleShift = styleShiftCentsByFrame.Length == 0 + ? Array.Empty() + : NeutrinoInferenceUtil.Slice( + styleShiftCentsByFrame, chunk.FrameStart, chunk.FrameCount); + var scorePitchesHz = ApplyStyleShiftToScorePitches( + chunkTiming.ScorePitchesHz, + BuildPhoneStyleShiftCents(chunkTiming, chunkStyleShift)); + int numPhones = chunkTiming.PhonemeIds.Length; + var pitchInputs = new List { + NamedOnnxValue.CreateFromTensor("electron", + new DenseTensor(chunkTiming.PhonemeIds, new[] { 1, numPhones })), + NamedOnnxValue.CreateFromTensor("muon", + new DenseTensor(chunkTiming.TimingDurations, new[] { 1, numPhones })), + NamedOnnxValue.CreateFromTensor("tau", + new DenseTensor(scorePitchesHz, new[] { 1, numPhones })), + NamedOnnxValue.CreateFromTensor("selectron", + new DenseTensor(chunkTiming.ScoreDurations, new[] { 1, numPhones })), + NamedOnnxValue.CreateFromTensor("smuon", + new DenseTensor(chunkTiming.PhonePositions, new[] { 1, numPhones })), + NamedOnnxValue.CreateFromTensor("stau", + new DenseTensor( + chunkTiming.FramePhonemeMap, + new[] { 1, chunkTiming.TotalFrames })), + }; + var chunkF0 = NeutrinoInferenceUtil.RequireLength( + singer.RunPitch(pitchInputs), + chunkTiming.TotalFrames, + "NEUTRINO v3 p.bin F0 output"); + ApplyInverseStyleShiftToF0(chunkF0, chunkStyleShift); + ClampF0(chunkF0); + Array.Copy(chunkF0, 0, f0, chunk.FrameStart, chunkF0.Length); + } + return f0; + } + + internal static void ApplyStyleShiftContour( + float[] editorF0, + float[] neutralF0, + float[] shiftedF0) { + + int frameCount = Math.Min(editorF0.Length, Math.Min(neutralF0.Length, shiftedF0.Length)); + for (int frame = 0; frame < frameCount; frame++) { + if (editorF0[frame] <= 0 || neutralF0[frame] <= 0 || shiftedF0[frame] <= 0) { + continue; + } + float ratio = shiftedF0[frame] / neutralF0[frame]; + if (float.IsFinite(ratio)) { + editorF0[frame] *= Math.Clamp(ratio, 0.5f, 2f); + } + } + } + + float[] BuildStyleShiftCentsByFrame(RenderPhrase phrase, NeutrinoTimingContext timing) { + if (!HasNonDefaultValue(phrase.toneShift, 0)) { + return Array.Empty(); + } + var result = new float[timing.TotalFrames]; + for (int frame = 0; frame < result.Length; frame++) { + result[frame] = GetFrameToneShiftCents(phrase, timing, frame); + } + return result; + } + + float[] BuildPhoneStyleShiftCents(NeutrinoTimingContext timing, float[] styleShiftCentsByFrame) { + if (styleShiftCentsByFrame.Length == 0) { + return Array.Empty(); + } + int numPhones = timing.PhonemeIds.Length; + var sums = new double[numPhones]; + var counts = new int[numPhones]; + int frameCount = Math.Min(styleShiftCentsByFrame.Length, timing.FramePhonemeMap.Length); + for (int frame = 0; frame < frameCount; frame++) { + int phone = Math.Clamp((int)timing.FramePhonemeMap[frame] - 1, 0, numPhones - 1); + sums[phone] += styleShiftCentsByFrame[frame]; + counts[phone]++; + } + var result = new float[numPhones]; + for (int phone = 0; phone < result.Length; phone++) { + if (counts[phone] > 0) { + result[phone] = (float)(sums[phone] / counts[phone]); + } + } + return result; + } + + float[] ApplyStyleShiftToScorePitches(float[] scorePitchesHz, float[] phoneStyleShiftCents) { + if (phoneStyleShiftCents.Length == 0) { + return scorePitchesHz; + } + var result = (float[])scorePitchesHz.Clone(); + for (int i = 0; i < result.Length && i < phoneStyleShiftCents.Length; i++) { + if (result[i] > 0 && Math.Abs(phoneStyleShiftCents[i]) > 0.5f) { + result[i] *= StyleShiftFactor(phoneStyleShiftCents[i]); + } + } + return result; + } + + void ApplyInverseStyleShiftToF0(float[] f0, float[] styleShiftCentsByFrame) { + if (styleShiftCentsByFrame.Length == 0) { + return; + } + for (int frame = 0; frame < f0.Length && frame < styleShiftCentsByFrame.Length; frame++) { + if (f0[frame] > 0 && Math.Abs(styleShiftCentsByFrame[frame]) > 0.5f) { + f0[frame] /= StyleShiftFactor(styleShiftCentsByFrame[frame]); + } + } + } + + static float StyleShiftFactor(float cents) { + return (float)Math.Pow(2.0, cents / 1200.0); + } + + float[] BuildEditorF0(RenderPhrase phrase, NeutrinoTimingContext timing) { + var f0 = new float[timing.TotalFrames]; + for (int frame = 0; frame < f0.Length; frame++) { + int phoneIndex = GetFramePhoneIndex(timing, frame); + if (phoneIndex < 0 + || timing.PhonemeIds[phoneIndex] == NeutrinoPhoneme.PAU + || timing.ScorePitchesHz[phoneIndex] <= 0) { + continue; + } + + if (phrase.pitches == null || phrase.pitches.Length == 0) { + f0[frame] = timing.ScorePitchesHz[phoneIndex]; + continue; + } + + int pitchIndex = GetFramePitchIndex(phrase, timing, frame); + f0[frame] = (float)MusicMath.ToneToFreq(phrase.pitches[pitchIndex] * 0.01); + } + return f0; + } + + int GetFramePhoneIndex(NeutrinoTimingContext timing, int frame) { + if (timing.FramePhonemeMap.Length == 0 || timing.PhonemeIds.Length == 0) { + return -1; + } + int mapIndex = Math.Clamp(frame, 0, timing.FramePhonemeMap.Length - 1); + return Math.Clamp((int)timing.FramePhonemeMap[mapIndex] - 1, 0, timing.PhonemeIds.Length - 1); + } + + int GetFramePitchIndex(RenderPhrase phrase, NeutrinoTimingContext timing, int frame) { + int ticks = GetFramePitchTick(phrase, timing, frame); + return Math.Clamp((int)(ticks / (double)pitchInterval), 0, phrase.pitches.Length - 1); + } + + float GetFrameToneShiftCents( + RenderPhrase phrase, + NeutrinoTimingContext timing, + int frame) { + + if (phrase.toneShift == null || phrase.toneShift.Length == 0) { + return 0; + } + int ticks = GetFramePitchTick(phrase, timing, frame); + int index = Math.Clamp((int)(ticks / (double)pitchInterval), 0, phrase.toneShift.Length - 1); + return phrase.toneShift[index]; + } + + int GetFramePitchTick(RenderPhrase phrase, NeutrinoTimingContext timing, int frame) { + double frameMs = 1000.0 * hopSize / sampleRate; + double posMs = phrase.positionMs - phrase.leadingMs + + timing.StartOffsetSeconds * 1000.0 + + frame * frameMs; + return phrase.timeAxis.MsPosToTickPos(posMs) - (phrase.position - phrase.leading); + } + + int GetFrameResultTick(RenderPhrase phrase, NeutrinoTimingContext timing, int frame) { + double frameMs = 1000.0 * hopSize / sampleRate; + double posMs = phrase.positionMs - phrase.leadingMs + + timing.StartOffsetSeconds * 1000.0 + + frame * frameMs; + return phrase.timeAxis.MsPosToTickPos(posMs) - phrase.position; + } + + static bool TryLoadWaveCache(string path, out float[] samples) { + samples = null; + if (!File.Exists(path)) { + return false; + } + try { + using var waveStream = Wave.OpenFile(path); + samples = Wave.GetSamples(waveStream.ToSampleProvider().ToMono(1, 0)); + return true; + } catch (Exception e) { + Log.Error(e, "Failed to read NEUTRINO cache, re-rendering"); + return false; + } + } + + static void SaveWaveCache(string path, float[] samples) { + var source = new WaveSource(0, 0, 0, 1); + source.SetSamples(samples); + WaveFileWriter.CreateWaveFile16(path, new ExportAdapter(source).ToMono(1, 0)); + } + + (long[] phonemeIds, float[] scorePitchesHz, float[] scoreDurations, long[] phonePositions, double?[] manualBoundaries) + BuildPhonemeSequence(RenderPhrase phrase) { + + double scoreOriginMs = GetScoreOriginMs(phrase); + var phonesByNote = new Dictionary>(); + + foreach (var phone in phrase.phones) { + var phoneStrs = NeutrinoPhoneme.RenderPhoneToPhonemes(phone.phoneme); + int noteIndex = Math.Clamp(phone.noteIndex, 0, phrase.notes.Length - 1); + if (!phonesByNote.TryGetValue(noteIndex, out var modelPhones)) { + modelPhones = new List(); + phonesByNote[noteIndex] = modelPhones; + } + + for (int i = 0; i < phoneStrs.Length; i++) { + int id = NeutrinoPhoneme.GetPhonemeId(phoneStrs[i]); + modelPhones.Add(new NeutrinoScorePhoneInput( + id, + manualBoundarySeconds: phone.positionOverridden && i == 0 + ? (phone.positionMs - scoreOriginMs) / 1000.0 + : null)); + } + } + + var scoreNotes = new List(); + int firstPhoneNoteIndex = GetFirstPhoneNoteIndex(phrase); + int lastPhoneNoteIndex = phrase.phones.Length == 0 + ? firstPhoneNoteIndex + : Math.Clamp( + phrase.phones.Max(phone => phone.noteIndex), + firstPhoneNoteIndex, + phrase.notes.Length - 1); + for (int noteIndex = firstPhoneNoteIndex; noteIndex < phrase.notes.Length; noteIndex++) { + var note = phrase.notes[noteIndex]; + bool isExtension = NeutrinoInferenceUtil.IsExtensionLyric(note.lyric); + bool hasPhones = phonesByNote.TryGetValue(noteIndex, out var modelPhones); + if (noteIndex > lastPhoneNoteIndex && !isExtension) { + break; + } + + float notePitchHz = (float)MusicMath.ToneToFreq( + note.tone + note.tuning * 0.01f); + float noteDurationSec = Math.Max(0.001f, (float)(note.durationMs / 1000.0)); + scoreNotes.Add(new NeutrinoScoreNoteInput( + notePitchHz, + noteDurationSec, + isExtension, + hasPhones ? modelPhones.ToArray() : Array.Empty())); + } + + var sequence = NeutrinoInferenceUtil.BuildScoreSequence(scoreNotes); + return ( + sequence.PhonemeIds, + sequence.ScorePitchesHz, + sequence.ScoreDurations, + sequence.PhonePositions, + sequence.ManualBoundaries + ); + } + + int GetFirstPhoneNoteIndex(RenderPhrase phrase) { + if (phrase.phones.Length == 0 || phrase.notes.Length == 0) { + return 0; + } + return Math.Clamp(phrase.phones[0].noteIndex, 0, phrase.notes.Length - 1); + } + + double GetScoreOriginMs(RenderPhrase phrase) { + if (phrase.notes.Length == 0) { + return phrase.positionMs; + } + return phrase.notes[GetFirstPhoneNoteIndex(phrase)].positionMs; + } + + double GetLeadingContextSeconds(RenderPhrase phrase) { + if (phrase.notes.Length == 0) { + return 0; + } + var firstNote = phrase.notes[GetFirstPhoneNoteIndex(phrase)]; + int scoreOriginTick = phrase.position + firstNote.position; + double contextStartMs = phrase.timeAxis.TickPosToMsPos(scoreOriginTick - headTicks); + double maximumContextMs = Math.Max(0, firstNote.positionMs - contextStartMs); + return Math.Min(maximumContextMs, phrase.availableLeadingMs) / 1000.0; + } + + internal static void ApplyManualBoundaryOverrides( + double[] boundaries, + double?[] manualBoundaries, + double leadingContextSeconds) { + + if (manualBoundaries == null || manualBoundaries.Length == 0) { + return; + } + + double frameSec = (double)hopSize / sampleRate; + int count = Math.Min(boundaries.Length - 1, manualBoundaries.Length - 1); + for (int i = 0; i < count; i++) { + if (!manualBoundaries[i].HasValue) { + continue; + } + + double min = i == 0 + ? Math.Min(0, -Math.Max(0, leadingContextSeconds) + frameSec) + : boundaries[i - 1] + frameSec; + double max = boundaries[i + 1] - frameSec; + if (max < min) { + max = min; + } + boundaries[i] = Math.Round(Math.Clamp(manualBoundaries[i].Value, min, max) * 1000.0) / 1000.0; + } + + for (int i = 1; i < boundaries.Length; i++) { + if (boundaries[i] <= boundaries[i - 1]) { + boundaries[i] = Math.Round((boundaries[i - 1] + frameSec) * 1000.0) / 1000.0; + } + } + } + + float[] BuildTimingDurations(double[] boundaries) { + var durations = new float[boundaries.Length - 1]; + for (int i = 0; i < durations.Length; i++) { + durations[i] = Math.Max(0.001f, (float)(boundaries[i + 1] - boundaries[i])); + } + return durations; + } + + internal static long[] BuildFramePhonemeMap(float[] timingDurations, int totalFrames) { + var stau = new long[totalFrames]; + double frameSec = (double)hopSize / sampleRate; + double time = 0; + for (int phone = 0; phone < timingDurations.Length; phone++) { + int startFrame = (int)Math.Round(time / frameSec); + time += timingDurations[phone]; + int endFrame = Math.Min(totalFrames, (int)Math.Round(time / frameSec)); + for (int frame = startFrame; frame < endFrame; frame++) { + stau[frame] = phone + 1; + } + } + long finalPhone = timingDurations.Length; + for (int frame = 0; frame < totalFrames; frame++) { + if (stau[frame] == 0) { + stau[frame] = finalPhone; + } + } + return stau; + } + + void ClampF0(float[] f0) { + for (int i = 0; i < f0.Length; i++) { + if (!float.IsFinite(f0[i]) || f0[i] < f0Min) { + f0[i] = 0; + } else if (f0[i] > f0Max) { + f0[i] = f0Max; + } + } + } + + void ClampMelspec(float[] melSpectrogram) { + for (int i = 0; i < melSpectrogram.Length; i++) { + if (!float.IsFinite(melSpectrogram[i])) { + melSpectrogram[i] = melspecMin; + } else if (melSpectrogram[i] < melspecMin) { + melSpectrogram[i] = melspecMin; + } else if (melSpectrogram[i] > melspecMax) { + melSpectrogram[i] = melspecMax; + } + } + } + + void PostProcessWaveform(float[] waveform) { + int edge = Math.Min(edgeSilenceSamples, waveform.Length / 2); + for (int i = 0; i < edge; i++) { + waveform[i] = 0; + waveform[waveform.Length - 1 - i] = 0; + } + + int fadeIn = Math.Min(fadeInSamples, Math.Max(0, waveform.Length - edge)); + for (int i = 0; i < fadeIn; i++) { + int index = edge + i; + if (index >= waveform.Length) break; + float gain = (float)Math.Pow((double)i / fadeInSamples, 2.0); + waveform[index] *= gain; + } + + int fadeOut = Math.Min(fadeOutSamples, Math.Max(0, waveform.Length - edge)); + for (int i = 0; i < fadeOut; i++) { + int index = waveform.Length - edge - 1 - i; + if (index < 0) break; + float gain = (float)Math.Pow((double)i / fadeOutSamples, 2.0); + waveform[index] *= gain; + } + + for (int i = 0; i < waveform.Length; i++) { + float value = waveform[i] * wavScale; + if (!float.IsFinite(value)) value = 0; + if (value > wavClamp) value = wavClamp; + if (value < -wavClamp) value = -wavClamp; + waveform[i] = value; + } + } + + public RenderPitchResult LoadRenderedPitch(RenderPhrase phrase) { + var timing = BuildTimingContext(phrase); + if (timing.TotalFrames <= 0) { + return null; + } + + var f0 = RunPredictedF0(phrase, timing); + var result = new RenderPitchResult { + ticks = new float[f0.Length], + tones = new float[f0.Length], + }; + + for (int frame = 0; frame < f0.Length; frame++) { + result.ticks[frame] = GetFrameResultTick(phrase, timing, frame); + int phoneIndex = GetFramePhoneIndex(timing, frame); + bool voiced = phoneIndex >= 0 + && timing.PhonemeIds[phoneIndex] != NeutrinoPhoneme.PAU + && timing.ScorePitchesHz[phoneIndex] > 0 + && f0[frame] > 0; + result.tones[frame] = voiced + ? (float)MusicMath.FreqToTone(f0[frame]) + : -1f; + } + return result; + } + + public UExpressionDescriptor[] GetSuggestedExpressions( + USinger singer, URenderSettings renderSettings) { + return Array.Empty(); + } + + public override string ToString() => Renderers.NEUTRINO; + } +} diff --git a/OpenUtau.Core/Neutrino/NeutrinoSinger.cs b/OpenUtau.Core/Neutrino/NeutrinoSinger.cs new file mode 100644 index 000000000..7a6ad9101 --- /dev/null +++ b/OpenUtau.Core/Neutrino/NeutrinoSinger.cs @@ -0,0 +1,313 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using Microsoft.ML.OnnxRuntime; +using Microsoft.ML.OnnxRuntime.Tensors; +using OpenUtau.Classic; +using OpenUtau.Core.Ustx; +using OpenUtau.Core.Util; +using Serilog; + +namespace OpenUtau.Core.Neutrino { + public class NeutrinoSinger : USinger { + public override string Id => voicebank.Id; + public override string Name => voicebank.Name; + public override Dictionary LocalizedNames => voicebank.LocalizedNames; + public override USingerType SingerType => USingerType.Neutrino; + public override string BasePath => voicebank.BasePath; + public override string Author => voicebank.Author; + public override string Voice => voicebank.Voice; + public override string Location => Path.GetDirectoryName(voicebank.File); + public override string Web => voicebank.Web; + public override string Version => voicebank.Version; + public override string OtherInfo => voicebank.OtherInfo; + public override IList Errors => errors; + public override string Avatar => voicebank.Image == null ? null : Path.Combine(Location, voicebank.Image); + public override byte[] AvatarData => avatarData; + public override string Portrait => voicebank.Portrait == null ? null : Path.Combine(Location, voicebank.Portrait); + public override float PortraitOpacity => voicebank.PortraitOpacity; + public override int PortraitHeight => voicebank.PortraitHeight; + public override string Sample => voicebank.Sample == null ? null : Path.Combine(Location, voicebank.Sample); + public override string DefaultPhonemizer => + voicebank.DefaultPhonemizer ?? "OpenUtau.Core.Neutrino.NeutrinoPhonemizer"; + public override Encoding TextFileEncoding => voicebank.TextFileEncoding; + public override IList Subbanks => subbanks; + public override IList Otos => otos; + + readonly object sessionLock = new object(); + readonly List errors = new List(); + readonly List subbanks = new List(); + readonly List otos = new List(); + readonly Dictionary otoMap = new Dictionary(); + + Voicebank voicebank; + byte[] avatarData; + InferenceSession timingSession; + InferenceSession pitchSession; + InferenceSession melspecSession; + InferenceSession vocoderSession; + string timingModelPath = string.Empty; + string pitchModelPath = string.Empty; + string melspecModelPath = string.Empty; + string vocoderModelPath = string.Empty; + + public NeutrinoSinger(Voicebank voicebank) { + this.voicebank = voicebank; + found = true; + } + + public override void EnsureLoaded() { + if (!Loaded) { + Reload(); + } + } + + public override void Reload() { + if (!Found) { + return; + } + lock (sessionLock) { + loaded = false; + try { + voicebank.Reload(); + Load(); + loaded = true; + } catch (Exception e) { + Log.Error(e, "Failed to load NEUTRINO singer {SingerPath}", voicebank.File); + } + } + } + + void Load() { + FreeSessions(); + errors.Clear(); + subbanks.Clear(); + otos.Clear(); + otoMap.Clear(); + + string modelDirectory = ResolveModelDirectory(); + if (!IsV3ModelDirectory(modelDirectory)) { + errors.Add($"NEUTRINO v3 model files were not found in {modelDirectory}"); + } + + subbanks.Add(new USubbank(new Subbank() { + Prefix = string.Empty, + Suffix = string.Empty, + ToneRanges = new[] { "C1-B7" }, + })); + foreach (string phoneme in NeutrinoPhoneme.AllPhonemes) { + var oto = UOto.OfDummy(phoneme); + if (otoMap.TryAdd(oto.Alias, oto)) { + otos.Add(oto); + } + } + + avatarData = null; + if (Avatar != null && File.Exists(Avatar)) { + try { + avatarData = File.ReadAllBytes(Avatar); + } catch (Exception e) { + Log.Error(e, "Failed to load NEUTRINO avatar"); + } + } + } + + public void EnsureSessions() { + if (timingSession != null + && pitchSession != null + && melspecSession != null + && vocoderSession != null) { + return; + } + lock (sessionLock) { + EnsureModelPaths(); + timingSession ??= LoadSession(timingModelPath, OnnxRunnerChoice.Default); + pitchSession ??= LoadSession(pitchModelPath, OnnxRunnerChoice.Default); + melspecSession ??= LoadSession(melspecModelPath, OnnxRunnerChoice.Default); + vocoderSession ??= LoadSession(vocoderModelPath, OnnxRunnerChoice.Default); + } + } + + public void EnsureTimingSession() { + if (timingSession != null) { + return; + } + lock (sessionLock) { + EnsureModelPaths(); + timingSession ??= LoadSession(timingModelPath, OnnxRunnerChoice.Default); + } + } + + public void EnsurePitchSession() { + if (pitchSession != null) { + return; + } + lock (sessionLock) { + EnsureModelPaths(); + pitchSession ??= LoadSession(pitchModelPath, OnnxRunnerChoice.Default); + } + } + + public void EnsureMelspecSession() { + if (melspecSession != null) { + return; + } + lock (sessionLock) { + EnsureModelPaths(); + melspecSession ??= LoadSession(melspecModelPath, OnnxRunnerChoice.Default); + } + } + + public void EnsureVocoderSession() { + if (vocoderSession != null) { + return; + } + lock (sessionLock) { + EnsureModelPaths(); + vocoderSession ??= LoadSession(vocoderModelPath, OnnxRunnerChoice.Default); + } + } + + void EnsureModelPaths() { + if (!string.IsNullOrEmpty(timingModelPath)) { + return; + } + string modelDirectory = ResolveModelDirectory(); + timingModelPath = RequireModel(modelDirectory, "t.bin"); + pitchModelPath = RequireModel(modelDirectory, "p.bin"); + melspecModelPath = RequireModel(modelDirectory, "s.bin"); + vocoderModelPath = RequireModel(modelDirectory, "v.bin"); + } + + string ResolveModelDirectory() { + string nested = Path.Combine(Location, "model"); + if (IsV3ModelDirectory(nested)) { + return nested; + } + if (IsV3ModelDirectory(Location)) { + return Location; + } + return Directory.Exists(nested) ? nested : Location; + } + + internal static bool IsV3ModelDirectory(string directory) { + return !string.IsNullOrEmpty(directory) + && File.Exists(Path.Combine(directory, "t.bin")) + && File.Exists(Path.Combine(directory, "p.bin")) + && File.Exists(Path.Combine(directory, "s.bin")) + && File.Exists(Path.Combine(directory, "v.bin")); + } + + static string RequireModel(string directory, string fileName) { + string path = Path.Combine(directory, fileName); + if (!File.Exists(path)) { + throw new FileNotFoundException($"NEUTRINO v3 model was not found: {path}", path); + } + return path; + } + + static InferenceSession LoadSession(string path, OnnxRunnerChoice runnerChoice) { + return Onnx.getInferenceSession(path, runnerChoice); + } + + public float[] RunTiming(IReadOnlyCollection inputs) { + lock (sessionLock) { + EnsureTimingSession(); + return RunWithCpuFallback(ref timingSession, timingModelPath, inputs, "gluon", "timing"); + } + } + + public float[] RunPitch(IReadOnlyCollection inputs) { + lock (sessionLock) { + EnsurePitchSession(); + return RunWithCpuFallback(ref pitchSession, pitchModelPath, inputs, "photon", "pitch"); + } + } + + public float[] RunMelspec(IReadOnlyCollection inputs) { + lock (sessionLock) { + EnsureMelspecSession(); + return RunWithCpuFallback(ref melspecSession, melspecModelPath, inputs, "higgs", "melspec"); + } + } + + public float[] RunVocoder(IReadOnlyCollection inputs) { + lock (sessionLock) { + EnsureVocoderSession(); + return RunWithCpuFallback(ref vocoderSession, vocoderModelPath, inputs, "output", "vocoder"); + } + } + + float[] RunWithCpuFallback( + ref InferenceSession session, + string path, + IReadOnlyCollection inputs, + string outputName, + string modelName) { + + lock (sessionLock) { + try { + return RunOutput(session, inputs, outputName); + } catch (OnnxRuntimeException e) when (Preferences.Default.OnnxRunner == "DirectML") { + Log.Warning(e, "NEUTRINO {ModelName} failed on DirectML; retrying on CPU", modelName); + session?.Dispose(); + session = LoadSession(path, OnnxRunnerChoice.CPU); + return RunOutput(session, inputs, outputName); + } + } + } + + static float[] RunOutput( + InferenceSession session, + IReadOnlyCollection inputs, + string outputName) { + + using var outputs = session.Run(inputs, new[] { outputName }); + return outputs.Single().AsTensor().ToArray(); + } + + public void FreeSessions() { + lock (sessionLock) { + timingSession?.Dispose(); + pitchSession?.Dispose(); + melspecSession?.Dispose(); + vocoderSession?.Dispose(); + timingSession = null; + pitchSession = null; + melspecSession = null; + vocoderSession = null; + timingModelPath = string.Empty; + pitchModelPath = string.Empty; + melspecModelPath = string.Empty; + vocoderModelPath = string.Empty; + } + } + + public override void FreeMemory() { + FreeSessions(); + } + + public override bool TryGetOto(string phoneme, out UOto oto) { + oto = UOto.OfDummy(phoneme); + return true; + } + + public override IEnumerable GetSuggestions(string text) { + if (text != null) { + text = text.Replace(" ", ""); + } + bool all = string.IsNullOrEmpty(text); + return otos.Where(oto => all || oto.Alias.Contains(text, StringComparison.OrdinalIgnoreCase)); + } + + public override byte[] LoadPortrait() { + return string.IsNullOrEmpty(Portrait) ? null : File.ReadAllBytes(Portrait); + } + + public override byte[] LoadSample() { + return string.IsNullOrEmpty(Sample) ? null : File.ReadAllBytes(Sample); + } + } +} diff --git a/OpenUtau.Core/Render/RenderPhrase.cs b/OpenUtau.Core/Render/RenderPhrase.cs index 048fd64f6..9eff34bfe 100644 --- a/OpenUtau.Core/Render/RenderPhrase.cs +++ b/OpenUtau.Core/Render/RenderPhrase.cs @@ -42,9 +42,11 @@ public RenderNote(UProject project, UPart part, UNote note, int phrasePosition) public class RenderPhone { // Relative ticks public readonly int position; + public readonly int rawPosition; public readonly int duration; public readonly int end; public readonly int leading; + public readonly bool positionOverridden; // Absolute milliseconds public readonly double positionMs; @@ -78,10 +80,13 @@ public class RenderPhone { public readonly UOto oto; public readonly ulong hash; - internal RenderPhone(UProject project, UTrack track, UVoicePart part, UNote note, UPhoneme phoneme, int phrasePosition) { + internal RenderPhone(UProject project, UTrack track, UVoicePart part, UNote note, + UPhoneme phoneme, int phrasePosition, int noteIndex) { position = part.position + phoneme.position - phrasePosition; + rawPosition = part.position + phoneme.rawPosition - phrasePosition; duration = phoneme.Duration; end = position + duration; + positionOverridden = phoneme.position != phoneme.rawPosition; positionMs = phoneme.PositionMs; durationMs = phoneme.DurationMs; endMs = phoneme.EndMs; @@ -90,6 +95,7 @@ internal RenderPhone(UProject project, UTrack track, UVoicePart part, UNote note this.phoneme = phoneme.phoneme; tone = note.tone; + this.noteIndex = noteIndex; tempos = project.timeAxis.TemposBetweenTicks(part.position + phoneme.position - leading, part.position + phoneme.End); UTempo[] noteTempos = project.timeAxis.TemposBetweenTicks(part.position + phoneme.position, part.position + phoneme.End); tempo = noteTempos.Length > 0 ? noteTempos[0].bpm : project.tempos[0].bpm; @@ -175,6 +181,7 @@ public class RenderPhrase { public readonly double durationMs; public readonly double endMs; public readonly double leadingMs; + public readonly double availableLeadingMs; public readonly RenderNote[] notes; public readonly RenderPhone[] phones; @@ -223,6 +230,16 @@ internal RenderPhrase(UProject project, UTrack track, UVoicePart part, IEnumerab wavtool = track.RendererSettings.wavtool; timeAxis = project.timeAxis.Clone(); + var firstSourcePhone = phonemes.First(); + int scoreOriginTick = part.position + firstSourcePhone.Parent.position; + int contextStartTick = firstSourcePhone.Parent.Prev == null + ? 0 + : part.position + firstSourcePhone.Parent.Prev.End; + contextStartTick = Math.Min(scoreOriginTick, contextStartTick); + availableLeadingMs = Math.Max(0, + timeAxis.TickPosToMsPos(scoreOriginTick) + - timeAxis.TickPosToMsPos(contextStartTick)); + position = part.position + phonemes.First().position; end = part.position + phonemes.Last().End; duration = end - position; @@ -230,8 +247,12 @@ internal RenderPhrase(UProject project, UTrack track, UVoicePart part, IEnumerab notes = uNotes .Select(n => new RenderNote(project, part, n, position)) .ToArray(); + var noteIndexByNote = uNotes + .Select((note, index) => new { note, index }) + .ToDictionary(item => item.note, item => item.index); phones = phonemes - .Select(p => new RenderPhone(project, track, part, p.Parent, p, position)) + .Select(p => new RenderPhone(project, track, part, p.Parent, p, position, + noteIndexByNote.TryGetValue(p.Parent, out int noteIndex) ? noteIndex : 0)) .ToArray(); leading = phones.First().leading; @@ -494,6 +515,7 @@ private ulong Hash(bool postEffect) { writer.Write(renderer?.ToString() ?? ""); writer.Write(wavtool ?? ""); writer.Write(timeAxis.Timestamp); + writer.Write(availableLeadingMs); foreach (var phone in phones) { writer.Write(phone.hash); } diff --git a/OpenUtau.Core/Render/Renderers.cs b/OpenUtau.Core/Render/Renderers.cs index 469a6ba84..7ce000077 100644 --- a/OpenUtau.Core/Render/Renderers.cs +++ b/OpenUtau.Core/Render/Renderers.cs @@ -15,12 +15,14 @@ public static class Renderers { public const string VOGEN = "VOGEN"; public const string DIFFSINGER = "DIFFSINGER"; public const string VOICEVOX = "VOICEVOX"; + public const string NEUTRINO = "NEUTRINO"; static readonly string[] classicRenderers = new[] { WORLDLINE_R, CLASSIC }; static readonly string[] enunuRenderers = new[] { ENUNU }; static readonly string[] vogenRenderers = new[] { VOGEN }; static readonly string[] diffSingerRenderers = new[] { DIFFSINGER }; static readonly string[] voicevoxRenderers = new[] { VOICEVOX }; + static readonly string[] neutrinoRenderers = new[] { NEUTRINO }; static readonly string[] noRenderers = new string[0]; public static string[] GetSupportedRenderers(USingerType singerType) { @@ -35,6 +37,8 @@ public static string[] GetSupportedRenderers(USingerType singerType) { return diffSingerRenderers; case USingerType.Voicevox: return voicevoxRenderers; + case USingerType.Neutrino: + return neutrinoRenderers; default: return noRenderers; } @@ -70,6 +74,8 @@ public static IRenderer CreateRenderer(string renderer) { return new DiffSinger.DiffSingerRenderer(); } else if (renderer == VOICEVOX) { return new Voicevox.VoicevoxRenderer(); + } else if (renderer == NEUTRINO) { + return new Neutrino.NeutrinoRenderer(); } return null; } diff --git a/OpenUtau.Core/Ustx/USinger.cs b/OpenUtau.Core/Ustx/USinger.cs index 9e952b457..5c9daf00d 100644 --- a/OpenUtau.Core/Ustx/USinger.cs +++ b/OpenUtau.Core/Ustx/USinger.cs @@ -193,7 +193,7 @@ public override string ToString() { } } - [Flags] public enum USingerType { Classic = 0x1, Enunu = 0x2, Vogen = 0x4, DiffSinger = 0x5, Voicevox = 0x6 } + [Flags] public enum USingerType { Classic = 0x1, Enunu = 0x2, Vogen = 0x4, DiffSinger = 0x5, Voicevox = 0x6, Neutrino = 0x7 } public static class SingerTypeUtils { public static Dictionary SingerTypeNames = new Dictionary(){ @@ -201,6 +201,7 @@ public static class SingerTypeUtils { {USingerType.Enunu, "enunu"}, {USingerType.DiffSinger, "diffsinger"}, {USingerType.Voicevox, "voicevox"}, + {USingerType.Neutrino, "neutrino"}, }; public static Dictionary SingerTypeFromName = new Dictionary(){ @@ -208,6 +209,7 @@ public static class SingerTypeUtils { {"enunu", USingerType.Enunu}, {"diffsinger", USingerType.DiffSinger}, {"voicevox", USingerType.Voicevox}, + {"neutrino", USingerType.Neutrino}, }; } diff --git a/OpenUtau.Test/Core/Neutrino/NeutrinoRendererTest.cs b/OpenUtau.Test/Core/Neutrino/NeutrinoRendererTest.cs new file mode 100644 index 000000000..f694a6f0e --- /dev/null +++ b/OpenUtau.Test/Core/Neutrino/NeutrinoRendererTest.cs @@ -0,0 +1,450 @@ +using System; +using System.IO; +using System.Linq; +using OpenUtau.Core.Format; +using OpenUtau.Core.Neutrino; +using OpenUtau.Core.Render; +using OpenUtau.Core.Ustx; +using Xunit; +using FormatUstx = OpenUtau.Core.Format.Ustx; + +namespace OpenUtau.Core.Test.Neutrino { + public class NeutrinoRendererTest { + [Theory] + [InlineData(FormatUstx.DYN)] + [InlineData(FormatUstx.PITD)] + [InlineData(FormatUstx.SHFC)] + public void SupportsCoreAndStyleShiftExpressions(string abbr) { + var descriptor = new UExpressionDescriptor(abbr, abbr, -100, 100, 0) { + type = UExpressionType.Curve, + }; + + Assert.True(new NeutrinoRenderer().SupportsExpression(descriptor)); + } + + [Theory] + [InlineData(FormatUstx.GENC)] + [InlineData(FormatUstx.BREC)] + [InlineData(FormatUstx.TENC)] + [InlineData(FormatUstx.VOIC)] + public void DoesNotAdvertiseHnsepExpressions(string abbr) { + var descriptor = new UExpressionDescriptor(abbr, abbr, -100, 100, 0) { + type = UExpressionType.Curve, + }; + + Assert.False(new NeutrinoRenderer().SupportsExpression(descriptor)); + } + + [Fact] + public void RegistersNeutrinoV3Renderer() { + Assert.Equal( + new[] { Renderers.NEUTRINO }, + Renderers.GetSupportedRenderers(USingerType.Neutrino)); + Assert.Equal(Renderers.NEUTRINO, Renderers.GetDefaultRenderer(USingerType.Neutrino)); + Assert.IsType(Renderers.CreateRenderer(Renderers.NEUTRINO)); + } + + [Theory] + [InlineData("か")] + [InlineData("カ")] + [InlineData("ka")] + public void BuiltInDictionarySupportsKanaAndRomaji(string lyric) { + Assert.Equal(new[] { "k", "a" }, NeutrinoPhoneme.KanaToPhonemes(lyric)); + } + + [Fact] + public void StyleShiftAppliesOnlyItsPredictedPitchDifference() { + var editorF0 = new[] { 220f, 330f, 0f }; + NeutrinoRenderer.ApplyStyleShiftContour( + editorF0, + new[] { 200f, 300f, 0f }, + new[] { 210f, 270f, 100f }); + + Assert.Equal(231f, editorF0[0], 3); + Assert.Equal(297f, editorF0[1], 3); + Assert.Equal(0f, editorF0[2]); + } + + [Fact] + public void V3ModelSignatureRequiresAllFourModels() { + string directory = Path.Combine( + Path.GetTempPath(), + $"neutrino-v3-model-{Guid.NewGuid():N}"); + try { + Directory.CreateDirectory(directory); + foreach (string fileName in new[] { "t.bin", "p.bin", "s.bin" }) { + File.WriteAllBytes(Path.Combine(directory, fileName), Array.Empty()); + } + Assert.False(NeutrinoSinger.IsV3ModelDirectory(directory)); + + File.WriteAllBytes(Path.Combine(directory, "v.bin"), Array.Empty()); + Assert.True(NeutrinoSinger.IsV3ModelDirectory(directory)); + } finally { + Directory.Delete(directory, recursive: true); + } + } + + [Fact] + public void RenderPhoneKeepsPanelConsonantN() { + Assert.Equal(new[] { "N" }, NeutrinoPhoneme.KanaToPhonemes("n")); + Assert.Equal(new[] { "n" }, NeutrinoPhoneme.RenderPhoneToPhonemes("n")); + Assert.Equal(new[] { "n", "o" }, NeutrinoPhoneme.RenderPhoneToPhonemes("no")); + } + + [Fact] + public void FrameMapAssignsUncoveredFramesToFinalPhone() { + Assert.Equal( + new long[] { 2 }, + NeutrinoRenderer.BuildFramePhonemeMap(new[] { 0.001f, 0.001f }, 1)); + Assert.Equal( + new long[] { 1, 2 }, + NeutrinoRenderer.BuildFramePhonemeMap(new[] { 0.011f, 0.001f }, 2)); + } + + [Fact] + public void InferenceChunksSplitAfterBreathAndAroundPauses() { + var chunks = NeutrinoInferenceUtil.BuildPhoneChunks(new long[] { + NeutrinoPhoneme.PAU, + NeutrinoPhoneme.PAU, + 1, + NeutrinoPhoneme.BR, + 2, + NeutrinoPhoneme.PAU, + 4, + }); + + Assert.Equal(5, chunks.Length); + AssertChunk(chunks[0], 0, 2, false); + AssertChunk(chunks[1], 2, 2, true); + AssertChunk(chunks[2], 4, 1, true); + AssertChunk(chunks[3], 5, 1, false); + AssertChunk(chunks[4], 6, 1, true); + } + + [Fact] + public void ConsecutiveBreathsStayInTheSameActiveChunk() { + var chunks = NeutrinoInferenceUtil.BuildPhoneChunks(new long[] { + 1, + NeutrinoPhoneme.BR, + NeutrinoPhoneme.BR, + 2, + }); + + Assert.Equal(2, chunks.Length); + AssertChunk(chunks[0], 0, 3, true); + AssertChunk(chunks[1], 3, 1, true); + } + + [Fact] + public void BreathAfterPauseRemainsInTheInactiveChunk() { + var chunks = NeutrinoInferenceUtil.BuildPhoneChunks(new long[] { + NeutrinoPhoneme.PAU, + NeutrinoPhoneme.BR, + 1, + }); + + Assert.Equal(2, chunks.Length); + AssertChunk(chunks[0], 0, 2, false); + AssertChunk(chunks[1], 2, 1, true); + } + + [Fact] + public void FrameChunksUseGlobalRoundedBoundariesWithoutGaps() { + var phoneChunks = NeutrinoInferenceUtil.BuildPhoneChunks(new long[] { + 1, + NeutrinoPhoneme.BR, + NeutrinoPhoneme.PAU, + 2, + }); + var frameChunks = NeutrinoInferenceUtil.BuildFrameChunks( + phoneChunks, + new[] { 0.0, 0.011, 0.024, 0.032, 0.051 }, + totalFrames: 5, + frameSeconds: 0.01); + + Assert.Equal(3, frameChunks.Length); + AssertFrameChunk(frameChunks[0], 0, 2, 0, 2, true); + AssertFrameChunk(frameChunks[1], 2, 1, 2, 1, false); + AssertFrameChunk(frameChunks[2], 3, 1, 3, 2, true); + Assert.Equal(5, frameChunks.Sum(chunk => chunk.FrameCount)); + } + + [Fact] + public void ChunkedTimingKeepsNextActiveChunkInitialShift() { + var chunks = NeutrinoInferenceUtil.BuildPhoneChunks(new long[] { + 1, + NeutrinoPhoneme.PAU, + 2, + }); + + var boundaries = NeutrinoInferenceUtil.BuildTimingBoundaries( + new[] { 0.3f, 0.2f, 0.4f }, + new long[] { 0, 0, 0 }, + chunks, + frameSeconds: 0.01, + chunk => chunk.PhoneStart switch { + 0 => new[] { 0f, 123f }, + 2 => new[] { -0.05f, 123f }, + _ => throw new InvalidOperationException(), + }); + + Assert.Equal(0.0, boundaries[0], 3); + Assert.Equal(0.3, boundaries[1], 3); + Assert.Equal(0.45, boundaries[2], 3); + Assert.Equal(0.9, boundaries[3], 3); + } + + [Fact] + public void LeadingContextKeepsFirstActiveChunkInitialShift() { + var chunks = NeutrinoInferenceUtil.BuildPhoneChunks(new long[] { 2, 24 }); + + var boundaries = NeutrinoInferenceUtil.BuildTimingBoundaries( + new[] { 0.5f, 0.5f }, + new long[] { 0, 1 }, + chunks, + frameSeconds: 0.01, + chunk => new[] { -0.07f, 0.01f, 123f }, + leadingContextSeconds: 0.5); + + Assert.Equal(-0.07, boundaries[0], 3); + Assert.Equal(0.01, boundaries[1], 3); + Assert.Equal(0.5, boundaries[2], 3); + Assert.Equal(0.08, boundaries[1] - boundaries[0], 3); + + double start = NeutrinoInferenceUtil.NormalizeBoundaryStart(boundaries); + Assert.Equal(-0.07, start, 3); + Assert.Equal(0.0, boundaries[0], 3); + Assert.Equal(0.08, boundaries[1], 3); + Assert.Equal(0.57, boundaries[2], 3); + } + + [Fact] + public void LeadingContextClampsFirstPhoneInsideVirtualPause() { + var chunks = NeutrinoInferenceUtil.BuildPhoneChunks(new long[] { 2, 24 }); + + var boundaries = NeutrinoInferenceUtil.BuildTimingBoundaries( + new[] { 0.5f, 0.5f }, + new long[] { 0, 1 }, + chunks, + frameSeconds: 0.01, + chunk => new[] { -1f, 0.01f, 123f }, + leadingContextSeconds: 0.5); + + Assert.Equal(-0.49, boundaries[0], 3); + Assert.Equal(0.01, boundaries[1], 3); + } + + [Fact] + public void ShortLeadingContextCannotOverlapPreviousPhrase() { + var chunks = NeutrinoInferenceUtil.BuildPhoneChunks(new long[] { 2, 24 }); + + var boundaries = NeutrinoInferenceUtil.BuildTimingBoundaries( + new[] { 0.5f, 0.5f }, + new long[] { 0, 1 }, + chunks, + frameSeconds: 0.01, + chunk => new[] { -0.07f, 0.01f, 123f }, + leadingContextSeconds: 0.03); + + Assert.Equal(-0.02, boundaries[0], 3); + Assert.Equal(0.01, boundaries[1], 3); + } + + [Fact] + public void ZeroLeadingContextKeepsFirstPhoneAtScoreStart() { + var chunks = NeutrinoInferenceUtil.BuildPhoneChunks(new long[] { 2, 24 }); + + var boundaries = NeutrinoInferenceUtil.BuildTimingBoundaries( + new[] { 0.5f, 0.5f }, + new long[] { 0, 1 }, + chunks, + frameSeconds: 0.01, + chunk => new[] { -0.07f, 0.01f, 123f }, + leadingContextSeconds: 0); + + Assert.Equal(0, boundaries[0], 3); + Assert.Equal(0.01, boundaries[1], 3); + } + + [Fact] + public void ManualFirstBoundaryCanExtendConsonantIntoLeadingContext() { + var boundaries = new[] { -0.057, 0.003, 0.5 }; + + NeutrinoRenderer.ApplyManualBoundaryOverrides( + boundaries, + new double?[] { -0.1, null, null }, + leadingContextSeconds: 0.5); + + Assert.Equal(-0.1, boundaries[0], 3); + Assert.Equal(0.003, boundaries[1], 3); + Assert.Equal(0.103, boundaries[1] - boundaries[0], 3); + } + + [Fact] + public void ManualFirstBoundaryCannotExceedLeadingContext() { + var boundaries = new[] { -0.02, 0.01, 0.5 }; + + NeutrinoRenderer.ApplyManualBoundaryOverrides( + boundaries, + new double?[] { -0.1, null, null }, + leadingContextSeconds: 0.03); + + Assert.Equal(-0.02, boundaries[0], 3); + Assert.Equal(0.01, boundaries[1], 3); + } + + [Fact] + public void ChunkedTimingDoesNotRepeatOneNoteDuration() { + var chunks = NeutrinoInferenceUtil.BuildPhoneChunks(new long[] { + 1, + NeutrinoPhoneme.PAU, + 2, + }); + + var boundaries = NeutrinoInferenceUtil.BuildTimingBoundaries( + new[] { 0.5f, 0.5f, 0.5f }, + new long[] { 0, 1, 2 }, + chunks, + frameSeconds: 0.01, + chunk => new float[chunk.PhoneCount + 1]); + + Assert.Equal(0.5, boundaries[^1], 3); + } + + [Theory] + [InlineData("+")] + [InlineData("+~")] + [InlineData("+*")] + [InlineData("+anything")] + public void PlusPrefixedLyricsMatchOpenUtauExtensionSemantics(string lyric) { + Assert.True(NeutrinoInferenceUtil.IsExtensionLyric(lyric)); + } + + [Fact] + public void LegacyMinusExtensionRemainsSupported() { + Assert.True(NeutrinoInferenceUtil.IsExtensionLyric("-")); + } + + [Theory] + [InlineData("")] + [InlineData("a")] + [InlineData("~+")] + public void NonExtensionLyricsRemainIndependent(string lyric) { + Assert.False(NeutrinoInferenceUtil.IsExtensionLyric(lyric)); + } + + [Fact] + public void ExtensionNotesRepeatSustainPhoneWithTheirOwnPitchAndDuration() { + int h = NeutrinoPhoneme.GetPhonemeId("h"); + int o = NeutrinoPhoneme.GetPhonemeId("o"); + var sequence = NeutrinoInferenceUtil.BuildScoreSequence(new[] { + new NeutrinoScoreNoteInput( + 392f, + 0.5f, + false, + new[] { + new NeutrinoScorePhoneInput(h, sourceIndex: 0), + new NeutrinoScorePhoneInput(o, sourceIndex: 1), + }), + new NeutrinoScoreNoteInput( + 329.63f, + 0.25f, + true, + Array.Empty()), + new NeutrinoScoreNoteInput( + 293.66f, + 0.75f, + true, + Array.Empty()), + }); + + Assert.Equal(new long[] { h, o, o, o }, sequence.PhonemeIds); + Assert.Equal(new[] { 392f, 392f, 329.63f, 293.66f }, sequence.ScorePitchesHz); + Assert.Equal(new[] { 0.5f, 0.5f, 0.25f, 0.75f }, sequence.ScoreDurations); + Assert.Equal(new long[] { 0, 1, 0, 0 }, sequence.PhonePositions); + Assert.Equal(new[] { 0, 1, -1, -1 }, sequence.SourcePhoneIndices); + Assert.Equal(5, sequence.ManualBoundaries.Length); + + var boundaries = NeutrinoInferenceUtil.BuildTimingBoundaries( + sequence.ScoreDurations, + sequence.PhonePositions, + NeutrinoInferenceUtil.BuildPhoneChunks(sequence.PhonemeIds), + frameSeconds: 0.01, + chunk => new float[chunk.PhoneCount + 1]); + Assert.Equal(1.5, boundaries[^1], 3); + } + + [Fact] + public void IndependentNoteWithoutPhonesStopsExtensionCarry() { + int o = NeutrinoPhoneme.GetPhonemeId("o"); + var sequence = NeutrinoInferenceUtil.BuildScoreSequence(new[] { + new NeutrinoScoreNoteInput( + 392f, + 0.5f, + false, + new[] { new NeutrinoScorePhoneInput(o) }), + new NeutrinoScoreNoteInput( + 349.23f, + 0.5f, + false, + Array.Empty()), + new NeutrinoScoreNoteInput( + 329.63f, + 0.5f, + true, + Array.Empty()), + }); + + Assert.Equal(new long[] { o }, sequence.PhonemeIds); + } + + [Fact] + public void FixedShapeModelOutputsRejectLengthMismatch() { + var output = new[] { 0.1f, 0.2f }; + Assert.Same(output, NeutrinoInferenceUtil.RequireLength(output, 2, "test output")); + + var error = Assert.Throws( + () => NeutrinoInferenceUtil.RequireLength(output, 3, "test output")); + Assert.Equal("test output length mismatch: actual 2, expected 3.", error.Message); + } + + [Fact] + public void TimingModelReturnsOneMoreBoundaryThanPhonemes() { + var boundaries = new[] { 0f, 0.1f, 0.2f }; + Assert.Same( + boundaries, + NeutrinoInferenceUtil.RequireTimingBoundaryLength(boundaries, 2, "timing output")); + + var error = Assert.Throws( + () => NeutrinoInferenceUtil.RequireTimingBoundaryLength( + new[] { 0f, 0.1f }, 2, "timing output")); + Assert.Equal("timing output length mismatch: actual 2, expected 3.", error.Message); + } + + static void AssertChunk( + NeutrinoPhoneChunk chunk, + int phoneStart, + int phoneCount, + bool isActive) { + + Assert.Equal(phoneStart, chunk.PhoneStart); + Assert.Equal(phoneCount, chunk.PhoneCount); + Assert.Equal(isActive, chunk.IsActive); + } + + static void AssertFrameChunk( + NeutrinoFrameChunk chunk, + int phoneStart, + int phoneCount, + int frameStart, + int frameCount, + bool isActive) { + + Assert.Equal(phoneStart, chunk.PhoneStart); + Assert.Equal(phoneCount, chunk.PhoneCount); + Assert.Equal(frameStart, chunk.FrameStart); + Assert.Equal(frameCount, chunk.FrameCount); + Assert.Equal(isActive, chunk.IsActive); + } + } +} diff --git a/OpenUtau/ViewModels/SingerSetupViewModel.cs b/OpenUtau/ViewModels/SingerSetupViewModel.cs index 0a116218b..a635dbd33 100644 --- a/OpenUtau/ViewModels/SingerSetupViewModel.cs +++ b/OpenUtau/ViewModels/SingerSetupViewModel.cs @@ -30,7 +30,7 @@ public class SingerSetupViewModel : ViewModelBase { [Reactive] public Encoding ArchiveEncoding { get; set; } [Reactive] public Encoding TextEncoding { get; set; } [Reactive] public bool MissingInfo { get; set; } - public string[] SingerTypes { get; set; } = new[] { "utau", "enunu", "diffsinger" }; + public string[] SingerTypes { get; set; } = new[] { "utau", "enunu", "diffsinger", "neutrino" }; [Reactive] public string SingerType { get; set; } private ObservableCollectionExtended textItems; diff --git a/OpenUtau/ViewModels/SingersViewModel.cs b/OpenUtau/ViewModels/SingersViewModel.cs index 1b5a65751..42edc0d89 100644 --- a/OpenUtau/ViewModels/SingersViewModel.cs +++ b/OpenUtau/ViewModels/SingersViewModel.cs @@ -115,7 +115,7 @@ void AttachSinger() { } ).ToList(); var singerTypes = new string[] { - "utau", "enunu", "diffsinger", "voicevox" + "utau", "enunu", "diffsinger", "voicevox", "neutrino" }; setSingerTypeMenuItems = singerTypes.Select(singerType => new MenuItemViewModel((SingerTypeUtils.SingerTypeNames.TryGetValue(singer.SingerType, out var name) ? name : "") == singerType) {