From c5d921888b87ea6b72bd2ff1968f6ef2cf28a121 Mon Sep 17 00:00:00 2001 From: KakaruHayate Date: Sun, 26 Jul 2026 18:12:31 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20speaker=20mix=20=E6=9D=83=E9=87=8D?= =?UTF-8?q?=E9=92=B3=E5=88=B0=20[0,1]=EF=BC=8C=E9=98=B2=E8=B4=9F=E6=9D=83?= =?UTF-8?q?=E9=87=8D=E7=A0=B4=E5=9D=8F=E5=87=B8=E7=BB=84=E5=90=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DiffSingerSpeakerMix.Create 对 mix: 采样值只判 NaN、不钳量程。 轨本身声明为 Continuous(color, 0, 0, 1)、宿主 UI 落笔时也钳制，但数据层无硬契约 （锚点+默认值合成、跨引擎同 key 复用、手改工程皆可越界）——与 CombineVariance 早已记录的兜底理由相同。 负权重危害最大：它把 Σ 拉回 ≤1，令下方「Σ>1 才归一」的判据失效，于是既不归一、 默认 suffix 又补上 1-Σ>1，混出凸包外的 embedding（外插而非插值，落到训练分布之外）： default=Alice, Alice=[1,0] Bob=[0,1] Bob=+0.5 -> [0.5, 0.5] 正常插值 Bob=-0.5 -> [1.5, -0.5] 修复前：越界 Bob=-2.0 -> [3.0, -2.0] 修复前：越界 正负混合 Bob=2 / Carol=-1（Σ 恰为 1、不触发归一）-> [-0.5, 1.5] 修复前：越界 同形态的 phoneme_mix 轨在 DiffSingerSynthesisSession 采样处本就 clamp 到 [0,1]， 此处对齐，消除两条同类轨的行为不一致。 测试： - 把零依赖的 DiffSingerSpeakerMix.cs 加入测试项目（无 SDK 亦可跑）。 - 新增 DiffSingerSpeakerMixTests：默认补权、超额归一、NaN 视作未编辑等既有行为， 加 6 个越界回归（负权重、超 1、正负混合、逐帧钳制、凸包内断言）。 已确认这 6 个在修复前失败、修复后通过；其余 38 个基线测试前后皆通过。 44 tests pass；插件/测试/冒烟工具 Release 均 0 warning 0 error。 --- DiffSingerSpeakerMix.cs | 7 +- .../DiffSinger.Tests/DiffSinger.Tests.csproj | 3 +- .../DiffSingerSpeakerMixTests.cs | 150 ++++++++++++++++++ 3 files changed, 158 insertions(+), 2 deletions(-) create mode 100644 tests/DiffSinger.Tests/DiffSingerSpeakerMixTests.cs diff --git a/DiffSingerSpeakerMix.cs b/DiffSingerSpeakerMix.cs index 997e4a1..da00c1c 100644 --- a/DiffSingerSpeakerMix.cs +++ b/DiffSingerSpeakerMix.cs @@ -41,7 +41,12 @@ float[] Weight(string suffix) for (int f = 0; f < nFrames; f++) { double v = f < sampled.Length ? sampled[f] : double.NaN; - if (!double.IsNaN(v)) w[f] += (float)v; + // 用户值钳到 [0,1]：轨声明即 Continuous(color, 0, 0, 1)、宿主 UI 落笔时钳制，但数据层无硬契约 + // （锚点+默认值合成、跨引擎同 key 复用、手改工程皆可越界）——同 CombineVariance 的兜底理由。 + // 负权重尤其危险：它会让下方「Σ>1 才归一」的判据失效（负值把 Σ 拉回 ≤1）， + // 于是既不归一、默认 suffix 又补上 1-Σ>1，混出凸包外的 embedding（外插而非插值）。 + // 同形态的 phoneme_mix 轨在 DiffSingerSynthesisSession 采样处已 clamp，此处对齐。 + if (!double.IsNaN(v)) w[f] += (float)Math.Clamp(v, 0, 1); } } diff --git a/tests/DiffSinger.Tests/DiffSinger.Tests.csproj b/tests/DiffSinger.Tests/DiffSinger.Tests.csproj index 865aa7b..467e7c5 100644 --- a/tests/DiffSinger.Tests/DiffSinger.Tests.csproj +++ b/tests/DiffSinger.Tests/DiffSinger.Tests.csproj @@ -14,10 +14,11 @@ - + diff --git a/tests/DiffSinger.Tests/DiffSingerSpeakerMixTests.cs b/tests/DiffSinger.Tests/DiffSingerSpeakerMixTests.cs new file mode 100644 index 0000000..f23b573 --- /dev/null +++ b/tests/DiffSinger.Tests/DiffSingerSpeakerMixTests.cs @@ -0,0 +1,150 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using DiffSingerForTuneLab; +using Xunit; + +namespace DiffSinger.Tests; + +// DiffSingerSpeakerMix 的逐帧权重合成（零 SDK 依赖，直接编译进测试程序集）。 +// 重点：权重恒为凸组合——每权重 ∈ [0,1] 且逐帧和为 1。破坏凸性会把 embedding 混到 +// 训练分布之外（外插而非插值），模型行为不可预期。 +public class DiffSingerSpeakerMixTests +{ + // 逐帧权重和恒为 1（凸组合的必要条件）。 + static void AssertConvex(DiffSingerSpeakerMix mix, params string[] suffixes) + { + for (int f = 0; f < mix.FrameCount; f++) + { + float sum = 0; + foreach (var s in suffixes) + { + float w = WeightOf(mix, s, f); + Assert.InRange(w, 0f, 1f); + sum += w; + } + Assert.Equal(1f, sum, precision: 4); + } + } + + // 借 ToEmbedding 反读某 suffix 的逐帧权重：给该 suffix 单位基向量、其余零向量， + // 混出的第 0 维即它的权重。 + static float WeightOf(DiffSingerSpeakerMix mix, string suffix, int frame) + { + var spk = mix.ToEmbedding(s => s == suffix ? new[] { 1f } : new[] { 0f }, 1); + return spk[frame]; + } + + [Fact] + public void NoTracks_DefaultTakesFullWeight() + { + var mix = DiffSingerSpeakerMix.Create("Miku", [], 3); + for (int f = 0; f < 3; f++) + Assert.Equal(1f, WeightOf(mix, "Miku", f), precision: 4); + } + + [Fact] + public void PartialWeight_DefaultFillsRemainder() + { + var mix = DiffSingerSpeakerMix.Create("Miku", [("Teto", [0.25])], 1); + Assert.Equal(0.25f, WeightOf(mix, "Teto", 0), precision: 4); + Assert.Equal(0.75f, WeightOf(mix, "Miku", 0), precision: 4); + AssertConvex(mix, "Miku", "Teto"); + } + + [Fact] + public void OverUnity_NormalizesBySum() + { + var mix = DiffSingerSpeakerMix.Create("Miku", [("Teto", [1.0]), ("Luka", [1.0])], 1); + Assert.Equal(0.5f, WeightOf(mix, "Teto", 0), precision: 4); + Assert.Equal(0.5f, WeightOf(mix, "Luka", 0), precision: 4); + AssertConvex(mix, "Miku", "Teto", "Luka"); + } + + [Fact] + public void NaN_TreatedAsUnedited() + { + var mix = DiffSingerSpeakerMix.Create("Miku", [("Teto", [double.NaN])], 1); + Assert.Equal(0f, WeightOf(mix, "Teto", 0), precision: 4); + Assert.Equal(1f, WeightOf(mix, "Miku", 0), precision: 4); + } + + // —— 回归：越界输入必须钳到 [0,1]，否则凸性被破坏 —— + + // 负权重是最危险的一种：它把 Σ 拉回 ≤1，令「Σ>1 才归一」的判据失效， + // 于是既不归一、默认 suffix 又补上 1-Σ>1 —— 混出凸包外的 embedding。 + // 修复前：Teto=-0.5 会得到 Miku=1.5 / Teto=-0.5。 + [Theory] + [InlineData(-0.5)] + [InlineData(-2.0)] + [InlineData(-100.0)] + public void NegativeWeight_ClampedToZero(double drawn) + { + var mix = DiffSingerSpeakerMix.Create("Miku", [("Teto", [drawn])], 1); + Assert.Equal(0f, WeightOf(mix, "Teto", 0), precision: 4); + Assert.Equal(1f, WeightOf(mix, "Miku", 0), precision: 4); + AssertConvex(mix, "Miku", "Teto"); + } + + [Theory] + [InlineData(1.5)] + [InlineData(50.0)] + public void AboveOneWeight_ClampedThenNormalized(double drawn) + { + // 单条轨钳到 1 → Σ=1 → 不触发归一、默认补 0：目标 suffix 独占。 + var mix = DiffSingerSpeakerMix.Create("Miku", [("Teto", [drawn])], 1); + Assert.Equal(1f, WeightOf(mix, "Teto", 0), precision: 4); + Assert.Equal(0f, WeightOf(mix, "Miku", 0), precision: 4); + AssertConvex(mix, "Miku", "Teto"); + } + + // 正负混合：修复前 Σ = 2 + (-1) = 1 ⇒ 不归一，混出 Teto=2 / Luka=-1 的外插结果。 + [Fact] + public void MixedSignWeights_StayConvex() + { + var mix = DiffSingerSpeakerMix.Create("Miku", [("Teto", [2.0]), ("Luka", [-1.0])], 1); + Assert.Equal(1f, WeightOf(mix, "Teto", 0), precision: 4); + Assert.Equal(0f, WeightOf(mix, "Luka", 0), precision: 4); + AssertConvex(mix, "Miku", "Teto", "Luka"); + } + + // 凸性 ⇒ 混出的 embedding 必落在各说话人 emb 的凸包内（逐维不超出 min/max）。 + [Fact] + public void ResultingEmbedding_StaysWithinConvexHull() + { + var embs = new Dictionary(StringComparer.Ordinal) + { + ["Miku"] = [1f, 0f], + ["Teto"] = [0f, 1f], + }; + var mix = DiffSingerSpeakerMix.Create("Miku", [("Teto", [-0.5])], 1); + var spk = mix.ToEmbedding(s => embs.TryGetValue(s, out var e) ? e : new float[2], 2); + for (int i = 0; i < 2; i++) + { + float lo = Math.Min(embs["Miku"][i], embs["Teto"][i]); + float hi = Math.Max(embs["Miku"][i], embs["Teto"][i]); + Assert.InRange(spk[i], lo, hi); + } + } + + [Fact] + public void MultiFrame_ClampsPerFrame() + { + var mix = DiffSingerSpeakerMix.Create("Miku", [("Teto", [-1.0, 0.5, 2.0])], 3); + Assert.Equal(0f, WeightOf(mix, "Teto", 0), precision: 4); + Assert.Equal(0.5f, WeightOf(mix, "Teto", 1), precision: 4); + Assert.Equal(1f, WeightOf(mix, "Teto", 2), precision: 4); + AssertConvex(mix, "Miku", "Teto"); + } + + // 轨比帧数短：缺的帧按未编辑（NaN）处理，不越界。 + [Fact] + public void ShorterTrack_RemainingFramesUseDefault() + { + var mix = DiffSingerSpeakerMix.Create("Miku", [("Teto", [0.5])], 3); + Assert.Equal(0.5f, WeightOf(mix, "Teto", 0), precision: 4); + Assert.Equal(0f, WeightOf(mix, "Teto", 1), precision: 4); + Assert.Equal(0f, WeightOf(mix, "Teto", 2), precision: 4); + AssertConvex(mix, "Miku", "Teto"); + } +}