From 3eccab50bbdd757126f779687805e7164757eebc Mon Sep 17 00:00:00 2001 From: Chris Thompson Date: Sun, 13 Sep 2026 19:52:58 -0600 Subject: [PATCH 1/2] webui: split TTS text on sentences in Latin-script text (#539) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * webui: split TTS text on sentences in Latin-script text splitTtsChunks treats 。！？!?；;… as sentence terminators. ASCII '.' is not among them, so a paragraph of English prose is one unsplittable sentence and falls through to the fixed-width cut, which lands mid-word: splitTtsChunks("The first sentence is here. The second follows it closely. A third arrives now. ...", 60) [60] The first sentence is here. The second follows it closely. A [60] third arrives now. And a fourth, rather longer than the oth [60] ers, continues past the point where a small budget would hav [26] e to cut. Finally a fifth. Each chunk is a separate synthesis request, so a word split across two of them is pronounced as two fragments. '.' now terminates a sentence, with the guards that make it ambiguous in the first place: not between digits, not after an abbreviation or a single-letter initial, and only before whitespace. So 3.14159, Dr. Smith, J. R. R. Tolkien, file.txt and example.com stay whole. Where no sentence boundary fits the budget, the fallback breaks on words rather than characters, so only a token longer than the entire budget is cut mid-word. The same text now gives: [58] The first sentence is here. The second follows it closely. [20] A third arrives now. [59] And a fourth, rather longer than the others, continues past [49] the point where a small budget would have to cut. [16] Finally a fifth. Two smaller fixes alongside: a "Speaker 1:" prefix is counted against the budget, since it is repeated onto every chunk a line produces and those chunks otherwise exceed the caller's limit; and chunks are trimmed, so a prefixed chunk no longer carries a double space. CJK behaviour is unchanged -- the existing terminators still apply, and '.' is additive. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01EkxqpYvUbjCpRDnFiNiVfx * webui: keep the spacing that separated the sentences Packing threw away the whitespace splitSentences had carefully kept and put a single space back in its place. Three coupled lines assumed that separator was always one space: trimEnd() dropped it, the budget check added 1 for it, and the join wrote it. Sentences in Chinese, Japanese and Korean are adjacent -- a full-width terminator and nothing else -- so this invented a space that was never in the text. That is not only an extra request: it changes what the model is asked to speak, and it spends a character of the budget, so three 20-character sentences stopped fitting in two 40-character chunks. before: [20] ...吧。 [20] ...吧。 [20] ...吧。 after: [40] ...吧。...吧。 [20] ...吧。 The separator that actually followed each sentence is carried instead. One other case changes with it, deliberately: "First one. Second one." keeps its four spaces rather than being silently collapsed to one. Collapsing was an edit to the user's text that nobody asked the chunker to make. Checked against a 17-case corpus covering abbreviations, initials, speaker prefixes, over-long tokens, multiple spacing, newlines, CJK and mixed scripts; those two cases are the only ones whose output moves. --------- Co-authored-by: Claude Opus 5 (1M context) --- webui/native/dist/index.html | 18 ++--- webui/native/src/lib/text.ts | 135 +++++++++++++++++++++++++++++++---- 2 files changed, 132 insertions(+), 21 deletions(-) diff --git a/webui/native/dist/index.html b/webui/native/dist/index.html index cb56598b7..6902014dd 100644 --- a/webui/native/dist/index.html +++ b/webui/native/dist/index.html @@ -31,20 +31,20 @@
diff --git a/webui/native/src/lib/text.ts b/webui/native/src/lib/text.ts index 038a715a0..8bf711f5d 100644 --- a/webui/native/src/lib/text.ts +++ b/webui/native/src/lib/text.ts @@ -1,32 +1,143 @@ const speakerLine = /^\s*(Speaker\s+\d+\s*:)\s*(.*)$/i; -const sentenceParts = /[^。！？!?；;…]*[。！？!?；;…]+|[^。！？!?；;…]+$/g; + +// Sentence terminators. '.' is handled separately by endsSentence, because it is +// the only one that is ambiguous: it also marks decimals, abbreviations, +// initials and file extensions. +const terminators = new Set(['。', '！', '？', '!', '?', '；', ';', '…', '.']); + +// Abbreviations that end in a full stop without ending a sentence. Not +// exhaustive -- it cannot be -- but it covers what prose actually contains, and +// a miss costs a split in a slightly wrong place, not a failure. +const abbreviations = new Set([ + 'mr', 'mrs', 'ms', 'dr', 'prof', 'sr', 'jr', 'st', 'mt', 'rev', 'hon', + 'vs', 'etc', 'eg', 'ie', 'approx', 'dept', 'est', 'fig', 'no', 'vol', + 'jan', 'feb', 'mar', 'apr', 'jun', 'jul', 'aug', 'sep', 'sept', 'oct', + 'nov', 'dec', 'inc', 'ltd', 'co', 'corp', +]); + +function isDigit(character: string): boolean { + return character >= '0' && character <= '9'; +} + +function isWordCharacter(character: string): boolean { + return /[\p{L}\p{N}']/u.test(character); +} + +/** Whether a full stop ends a sentence rather than a number or an abbreviation. */ +function endsSentence(text: string, index: number): boolean { + // Must be followed by whitespace or end of text, which keeps file.txt and + // example.com intact. + const next = text[index + 1]; + if (next !== undefined && !/\s/.test(next)) return false; + + // Not a decimal point. + if (index > 0 && isDigit(text[index - 1]) && next !== undefined && isDigit(next)) return false; + + let start = index; + while (start > 0 && isWordCharacter(text[start - 1])) start -= 1; + const word = text.slice(start, index); + + // A single letter is an initial: "J. R. R. Tolkien". + if (word.length === 1 && /\p{L}/u.test(word)) return false; + + return !abbreviations.has(word.toLowerCase()); +} + +/** + * Break text after each sentence terminator, keeping the terminator and any + * following whitespace attached to the sentence it ends. + */ +function splitSentences(text: string): string[] { + const sentences: string[] = []; + let start = 0; + + for (let index = 0; index < text.length; index += 1) { + if (!terminators.has(text[index])) continue; + if (text[index] === '.' && !endsSentence(text, index)) continue; + + let end = index + 1; + while (end < text.length && terminators.has(text[end])) end += 1; + while (end < text.length && /\s/.test(text[end])) end += 1; + + sentences.push(text.slice(start, end)); + start = end; + index = end - 1; + } + + if (start < text.length) sentences.push(text.slice(start)); + return sentences; +} + +/** + * Cut text into budget-sized pieces at whitespace, falling back to a hard cut + * only for a single token that is itself longer than the budget. + */ +function breakOnWords(text: string, budget: number): string[] { + const pieces: string[] = []; + let rest = text.trim(); + + while (rest.length > budget) { + let cut = rest.lastIndexOf(' ', budget); + if (cut <= 0) cut = budget; + + const piece = rest.slice(0, cut).trim(); + if (piece) pieces.push(piece); + rest = rest.slice(cut).trim(); + } + + if (rest) pieces.push(rest); + return pieces; +} function splitLongLine(line: string, budget: number): string[] { const match = speakerLine.exec(line); const prefix = match ? `${match[1]} ` : ''; const body = match ? match[2] : line.trim(); - const sentences = body.match(sentenceParts) || [body]; + + // The prefix is repeated onto every chunk, so it has to come out of the + // budget or chunks carrying one exceed the limit the caller asked for. + const room = Math.max(1, budget - prefix.length); + + const sentences = splitSentences(body); + if (!sentences.length) sentences.push(body); + const chunks: string[] = []; let current = ''; + // Whatever whitespace actually followed the last sentence added to `current`. + // splitSentences keeps it, and it has to be carried rather than replaced with + // a space: sentences in Chinese, Japanese and Korean are adjacent, separated + // by a full-width terminator and nothing else. Inserting a space there both + // changes the text the model is asked to speak and spends a character of the + // budget, so three 20-character sentences stop fitting in two 40-character + // chunks and become three requests instead of two. + let separator = ''; - for (const sentence of sentences) { - if (current && current.length + sentence.length > budget) { - chunks.push(prefix + current); + for (const raw of sentences) { + const sentence = raw.trimEnd(); + if (!sentence) continue; + const trailing = raw.slice(sentence.length); + + if (current && current.length + separator.length + sentence.length > room) { + chunks.push(prefix + current.trim()); current = ''; + separator = ''; } - if (sentence.length <= budget) { - current += sentence; + if (sentence.length <= room) { + current = current ? `${current}${separator}${sentence}` : sentence; + separator = trailing; continue; } if (current) { - chunks.push(prefix + current); + chunks.push(prefix + current.trim()); current = ''; + separator = ''; } - for (let offset = 0; offset < sentence.length; offset += budget) { - chunks.push(prefix + sentence.slice(offset, offset + budget)); - } + // No sentence boundary fits, so fall back to word boundaries. Only a token + // longer than the whole budget is ever cut mid-word. + for (const piece of breakOnWords(sentence, room)) chunks.push(prefix + piece); } - if (current) chunks.push(prefix + current); + if (current) chunks.push(prefix + current.trim()); + return chunks.length ? chunks : [line]; } From 0e6ebe562e4b69bc15a332ee9c5616a70ceef60d Mon Sep 17 00:00:00 2001 From: Chris Thompson Date: Sun, 13 Sep 2026 19:54:43 -0600 Subject: [PATCH 2/2] webui: make the committed bundle reproducible dist/index.html is committed and embedded into the server binary at configure time (CMakeLists.txt:2226), which is what lets the project build a working server without a JavaScript toolchain. But the bundle is not reproducible: SvelteKit defaults kit.version.name to Date.now() and derives the __sveltekit_ global it embeds from it, so two builds of identical source differ. before: build 1 __sveltekit_1ia7gsf sha256 bb3f9c3c... build 2 __sveltekit_1xqgjp8 sha256 6cd91a94... after: build 1 __sveltekit_1vzi1g sha256 65bed68b... build 2 __sveltekit_1vzi1g sha256 65bed68b... Any two branches that rebuild the web UI therefore conflict in that file whether or not their source changes overlap -- which is what happened to #539, where dist/index.html was the only conflict while src/lib/text.ts merged cleanly and no upstream commit had touched either file. That the id is a pure function of this setting was confirmed by building with the timestamp the committed bundle carries: the derived id came back as __sveltekit_ega6lw, and the result was byte-identical to the committed file. The package version is used rather than a constant. It is stable for a given source tree, so the bundle reproduces, and it changes when the version is bumped, so SvelteKit's client-side "app has been updated" check keeps working across releases. A hard-coded string would have disabled that silently. dist/index.html is rebuilt here so the tree is consistent: after this, running npm run build leaves git clean instead of producing a diff every time. Normalising the build id, the version constant and the Vite content hashes -- all three derived from this one setting -- leaves zero differing lines against the committed bundle, so nothing else in the UI changes. Reported as #545. --- webui/native/dist/index.html | 8 ++++---- webui/native/svelte.config.js | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/webui/native/dist/index.html b/webui/native/dist/index.html index 6902014dd..0e958324f 100644 --- a/webui/native/dist/index.html +++ b/webui/native/dist/index.html @@ -31,15 +31,15 @@
diff --git a/webui/native/svelte.config.js b/webui/native/svelte.config.js index a5ce70d05..8a17ed9cd 100644 --- a/webui/native/svelte.config.js +++ b/webui/native/svelte.config.js @@ -1,5 +1,21 @@ +import { readFileSync } from 'node:fs'; + import adapter from '@sveltejs/adapter-static'; +// SvelteKit defaults kit.version.name to Date.now(), and derives the +// __sveltekit_ global it embeds in dist/index.html from it. That makes the +// built bundle different on every run, and dist/index.html is committed -- so +// any two branches that rebuild the web UI conflict in it, whether or not their +// source changes overlap. +// +// The package version is stable for a given source tree, so the bundle is +// reproducible, while still changing when the version is bumped -- which is +// what SvelteKit's client-side "app has been updated" check needs to keep +// working across releases. +const { version } = JSON.parse( + readFileSync(new URL('./package.json', import.meta.url), 'utf8') +); + /** @type {import('@sveltejs/kit').Config} */ const config = { kit: { @@ -17,6 +33,9 @@ const config = { }, paths: { relative: true + }, + version: { + name: version } } };