From 3de9a790172a3c125527bb1d9267ceb87552d0ca Mon Sep 17 00:00:00 2001 From: Bojie Li Date: Tue, 25 Aug 2026 17:30:35 +0800 Subject: [PATCH] chunker: place cuts with Silero rather than frame energy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Silero has been loaded and run on every recording since it replaced the noise-floor heuristic, but only ever as a yes/no gate on whether a chunk contains speech. Where to cut was still decided by frame energy against a 2nd-percentile floor, which fragments a long pause the moment a breath or a keyboard tap crosses it: 226 pauses of two seconds or more across 147 real recordings, against 451 that Silero finds in the same audio. Taking the gaps between finalised speech runs instead, measured on the 60 retained recordings past the splitting threshold: the median pause a cut lands in goes from 0.76 s to 2.14 s and cuts landing in a pause of a second or more from 40% to 77%. The Swift implementation reproduces the offline prediction exactly — median 2.14 s on 54 cuts across the corpus. The cost is a longer final chunk, p50 17.9 s to 27.7 s. On the offline path that is close to free, because request latency barely tracks chunk length: 60 s costs 2.43 s and 120 s costs 2.98 s. The reason this was not done when Silero arrived is that the live segmenter asks for a boundary every 200 ms once a minute of audio is pending, and re-running the model over the pending buffer costs about 0.19 s of CPU per call at that size — roughly a whole core, sustained, for as long as no qualifying pause appears. Silero is recurrent and designed to stream, so the state and the 64-sample context are now carried across calls and only new samples are analysed: 300 s of audio in 1.62 s, fed in the 85 ms pieces the recorder actually delivers, and a test holds that bound. Two edges are load-bearing. A run still open when the boundary is wanted counts, because during capture the newest gap is always followed by one — only its start is used, and Silero does not report a start until 250 ms of speech confirms it, which is exactly the evidence the pause has ended. Excluding it made the newest pause invisible and the segmenter never cut. And fewer than two runs hands the decision back to the energy finder, which needs no model and cannot fail, so audio Silero cannot parse still gets split rather than growing without bound. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_015XKX6zEAGiZFE5wAxJgYbV --- Sources/DoNotTypeCore/AudioChunker.swift | 139 +++++++++++--- Sources/DoNotTypeCore/SpeechActivity.swift | 170 ++++++++++++++++-- .../SpeechStreamTests.swift | 142 +++++++++++++++ 3 files changed, 416 insertions(+), 35 deletions(-) create mode 100644 Tests/DoNotTypeCoreTests/SpeechStreamTests.swift diff --git a/Sources/DoNotTypeCore/AudioChunker.swift b/Sources/DoNotTypeCore/AudioChunker.swift index 2d0f40d..1a42e88 100644 --- a/Sources/DoNotTypeCore/AudioChunker.swift +++ b/Sources/DoNotTypeCore/AudioChunker.swift @@ -86,6 +86,15 @@ public enum AudioChunker { var chunks: [Chunk] = [] var start = 0 + // One pass over the whole recording, reused for every boundary decision below. Silero is + // sequential, so this is also the only correct way to run it here: analysing each tail + // separately would restart the model's state mid-utterance. + let speech: SpeechActivity.Stream? = { + guard let stream = try? SpeechActivity.Stream() else { return nil } + guard (try? stream.append(pcm: body)) != nil else { return nil } + return stream.speechSegments(includingOpenRun: true).isEmpty ? nil : stream + }() + while start < body.count { let remaining = body.count - start @@ -99,7 +108,12 @@ public enum AudioChunker { } let tail = body.subdata(in: start.. [Chunk] { guard !pcm.isEmpty else { return [] } pending.append(pcm) totalBytes += pcm.count + // Fed every tick, whether or not a boundary is wanted yet, because the model's state + // is sequential: skipping audio would leave it reading a discontinuity. + try? speech?.append(pcm: pcm) var ready: [Chunk] = [] while shouldAnalyse, let cut = AudioChunker.bestBoundary( - in: pending, format: format, policy: policy) + in: pending, format: format, policy: policy, pauses: currentPauses()) { let samples = pending.subdata(in: 0.. [Pause]? { + guard let speech else { return nil } + let windowBytes = SpeechActivity.windowSamples * 2 + let analysedBytes = speech.analysedSamples * 2 + guard analysedBytes + windowBytes >= startBytes + pending.count else { return nil } + + // No finalised speech at all, in a minute of audio, means the model is not seeing this + // recording — an unusual timbre, or something that is not speech. Refusing to cut on + // that basis would let one request grow without bound, so the energy finder takes over. + // One segment and no gaps is the opposite: Silero *is* reading it and says the speaker + // has not paused, which is a reason to wait rather than to overrule it. + let segments = speech.speechSegments(includingOpenRun: true) + guard segments.count >= 2 else { return nil } + // The open run counts here, and it has to. During capture the gap worth cutting at is + // always the most recent one, and the run after it has not ended — the speaker is + // still going. Only that run's *start* is used to close the gap, and Silero does not + // report a start until 250 ms of speech has confirmed it, which is exactly the + // evidence that the pause is over. Excluding it made the newest pause invisible and + // the segmenter never cut. + return speech.pauses( + from: startBytes / 2, format: format, includingOpenRun: true) + } + private var canConsiderBoundary: Bool { if !emittedFirst { return Double(totalBytes) / Double(format.bytesPerSecond) > AudioChunker.threshold @@ -217,11 +278,25 @@ public enum AudioChunker { durationSeconds: Double(samples.count) / Double(format.bytesPerSecond)) } - private struct PauseCandidate { - let cut: Int - let seconds: Double - let duration: Double - let depth: Double + /// A place a cut could go, from whichever detector found it. + /// + /// Two produce these — the frame-energy finder below and Silero's speech-segment gaps — and one + /// scorer ranks them, so the two detectors stay swappable and directly comparable. + public struct Pause: Sendable, Equatable { + /// Byte offset of the pause's midpoint, relative to the start of the buffer searched. + public let cut: Int + public let seconds: Double + public let duration: Double + /// How confidently this is not speech, on a 0–20 scale. Decibels below the speech + /// threshold for the energy finder; the model's own confidence for Silero. + public let depth: Double + + public init(cut: Int, seconds: Double, duration: Double, depth: Double) { + self.cut = cut + self.seconds = seconds + self.duration = duration + self.depth = depth + } } /// Returns the best safe boundary, or nil when the audio has no energy-qualified pause. @@ -233,12 +308,39 @@ public enum AudioChunker { /// A run counts as a boundary only when /// it is surrounded by speech; uniform noise therefore cannot masquerade as one enormous /// pause. The middle leaves acoustic context on both sides without duplicating samples. + /// Ranks pauses found by either detector and returns the chosen cut. + /// + /// `pauses` of nil means "find them yourself, from frame energy" — the behaviour every caller + /// had before Silero could supply them, and the fallback when the model will not load. static func bestBoundary( - in body: Data, format: Format = Format(), policy: BoundaryPolicy = defaultPolicy + in body: Data, format: Format = Format(), policy: BoundaryPolicy = defaultPolicy, + pauses: [Pause]? = nil ) -> Int? { + let found = pauses ?? energyPauses(in: body, format: format, policy: policy) + let eligible = found.filter { $0.duration >= policy.minimumPause && $0.seconds >= policy.minimum } + let preferred = eligible.filter { $0.seconds <= policy.horizon } + if !preferred.isEmpty { + return preferred.max { boundaryScore($0, policy: policy) < boundaryScore($1, policy: policy) }?.cut + } + // Past the decision horizon, use the first real pause instead of waiting for a prettier + // one. Still no pause, still no cut. + return eligible.min { $0.seconds < $1.seconds }?.cut + } + + /// Every energy-qualified pause in the buffer. + /// + /// This is deliberately only a pause finder; Silero makes every decision about whether a chunk + /// contains speech. The floor is a low percentile of the recording's own energy so a train and + /// a quiet office are judged relative to themselves. The second percentile is intentional: + /// ordinary speech can contain less than ten percent pause, while splitting needs only one + /// quiet run. A run counts only when it is surrounded by speech, so uniform noise cannot + /// masquerade as one enormous pause. + static func energyPauses( + in body: Data, format: Format = Format(), policy: BoundaryPolicy = defaultPolicy + ) -> [Pause] { let frameMilliseconds = 20 let frameBytes = format.bytesPerSecond * frameMilliseconds / 1_000 - guard frameBytes > 0, body.count >= frameBytes * 3 else { return nil } + guard frameBytes > 0, body.count >= frameBytes * 3 else { return [] } var levels: [Double] = [] levels.reserveCapacity(body.count / frameBytes) @@ -263,7 +365,7 @@ public enum AudioChunker { offset += frameBytes } } - guard !levels.isEmpty else { return nil } + guard !levels.isEmpty else { return [] } let sorted = levels.sorted() let floor = sorted[min(sorted.count - 1, sorted.count / 50)] @@ -273,7 +375,7 @@ public enum AudioChunker { let evidenceFrames = 5 // 100 ms of speech on each side defeats isolated transients. let evidenceWindow = 100 // two seconds - var candidates: [PauseCandidate] = [] + var candidates: [Pause] = [] var frame = 0 while frame < speaking.count { guard !speaking[frame] else { @@ -297,24 +399,17 @@ public enum AudioChunker { let gapLevel = levels[runStart.. Double { + private static func boundaryScore(_ candidate: Pause, policy: BoundaryPolicy) -> Double { let preferredBonus = candidate.duration >= policy.preferredPause ? 3.0 : 0 let duration = min(2, candidate.duration) * 4 let depth = min(20, candidate.depth) / 10 diff --git a/Sources/DoNotTypeCore/SpeechActivity.swift b/Sources/DoNotTypeCore/SpeechActivity.swift index 930812f..3b6d392 100644 --- a/Sources/DoNotTypeCore/SpeechActivity.swift +++ b/Sources/DoNotTypeCore/SpeechActivity.swift @@ -105,17 +105,24 @@ public enum SpeechActivity { // MARK: - Silero segmentation - /// The part of upstream `get_speech_timestamps` that decides whether final speech exists. - /// Padding and maximum-segment splitting do not affect this gate, which needs duration rather - /// than timestamps; preserving the hysteresis and strict minimum-duration comparison does. - private static func finalisedSpeechSamples( - probabilities: [Float], audioLengthSamples: Int - ) -> Int { + /// The part of upstream `get_speech_timestamps` that decides where final speech is. + /// + /// One implementation, used by both the yes/no gate and the boundary finder, because a + /// segmenter that disagreed with the gate about what counts as speech would cut in places the + /// gate then refused to send. Padding and maximum-segment splitting do not affect either + /// caller; preserving the hysteresis and the strict minimum-duration comparison does. + /// + /// - Parameter includeOpenSegment: whether a run still open at the end counts. True for a + /// finished recording, whose end really is the end. False for a live capture, where the + /// speaker has simply not stopped yet and the run's end is not known. + static func finalisedSpeechSegments( + probabilities: [Float], audioLengthSamples: Int, includeOpenSegment: Bool = true + ) -> [Range] { let minimumSpeechSamples = sampleRate * minimumSpeechMilliseconds / 1_000 let minimumSilenceSamples = sampleRate * minimumSilenceMilliseconds / 1_000 var speechStart: Int? var possibleEnd: Int? - var total = 0 + var segments: [Range] = [] for (index, probability) in probabilities.enumerated() { let current = windowSamples * index @@ -130,15 +137,133 @@ public enum SpeechActivity { if possibleEnd == nil { possibleEnd = current } guard let end = possibleEnd, current - end >= minimumSilenceSamples else { continue } - if end - start > minimumSpeechSamples { total += end - start } + if end - start > minimumSpeechSamples { segments.append(start.. minimumSpeechSamples { - total += audioLengthSamples - start + if includeOpenSegment, let start = speechStart, + audioLengthSamples - start > minimumSpeechSamples + { + segments.append(start.. Int { + finalisedSpeechSegments( + probabilities: probabilities, audioLengthSamples: audioLengthSamples + ).reduce(0) { $0 + $1.count } + } + + // MARK: - Streaming + + /// Silero over a capture that is still running, carrying state instead of re-reading the buffer. + /// + /// The live segmenter asks for a boundary every 200 ms once a minute of audio is pending. Doing + /// that by re-running the model over the whole pending buffer costs about 0.19 s of CPU per + /// call at 60 seconds pending — roughly a whole core, sustained, for as long as no qualifying + /// pause appears. Feeding only the new samples costs about 0.6 ms. + /// + /// Probabilities are kept for the whole capture rather than trimmed at each cut. One `Float` + /// per 512 samples is 31 floats a second, so an hour of dictation is under half a megabyte, and + /// keeping absolute indices means a cut never has to re-align the array against the buffer. + public final class Stream: @unchecked Sendable { + private let model: SileroModel + private var carry: SileroModel.Carry + /// Samples arrived but not yet a complete 512-sample window. + private var leftover = Data() + private var probabilities: [Float] = [] + /// Samples represented by `probabilities`, so callers can index in absolute samples. + public private(set) var analysedSamples = 0 + + public init() throws { + switch SpeechActivity.modelState { + case .ready(let loaded): model = loaded + case .failed(let detail): throw DetectorError.unavailable(detail) + } + carry = SileroModel.Carry() + } + + /// Feeds 16 kHz mono 16-bit PCM. Safe to call with any size, including a partial window. + public func append(pcm: Data) throws { + guard !pcm.isEmpty else { return } + leftover.append(pcm) + let windowBytes = SpeechActivity.windowSamples * 2 + let complete = leftover.count / windowBytes + guard complete > 0 else { return } + + let consumed = complete * windowBytes + let block = leftover.prefix(consumed) + leftover.removeSubrange(0.. [Range] { + SpeechActivity.finalisedSpeechSegments( + probabilities: probabilities, audioLengthSamples: analysedSamples, + includeOpenSegment: includingOpenRun) + } + + /// Boundary candidates in the gaps between finalised speech, relative to `originSample`. + func pauses( + from originSample: Int, format: AudioChunker.Format, includingOpenRun: Bool = false + ) -> [AudioChunker.Pause] { + SpeechActivity.pauses( + segments: speechSegments(includingOpenRun: includingOpenRun), + probabilities: probabilities, from: originSample, format: format) + } + } + + /// The gaps between finalised speech runs, as boundary candidates. + /// + /// A gap is bounded by two runs that each cleared Silero's 250 ms minimum, so it is flanked by + /// real speech by construction — the energy finder has to check that separately with a + /// five-frames-in-two-seconds heuristic. + /// + /// `depth` is the model's own confidence that the gap is not speech, scaled to the 0–20 range + /// the energy finder's decibel depth uses, so one scorer can rank candidates from either source. + static func pauses( + segments: [Range], probabilities: [Float], from originSample: Int, + format: AudioChunker.Format + ) -> [AudioChunker.Pause] { + var out: [AudioChunker.Pause] = [] + for (a, b) in zip(segments, segments.dropFirst()) { + let gapStart = a.upperBound + let gapEnd = b.lowerBound + guard gapEnd > gapStart, gapStart >= originSample else { continue } + + let firstWindow = gapStart / windowSamples + let lastWindow = max(firstWindow + 1, gapEnd / windowSamples) + let slice = probabilities[ + min(firstWindow, probabilities.count).. [Float] { + var carry = Carry() + return try probabilities(pcm: pcm, sampleCount: sampleCount, carry: &carry) + } + + /// The recurrent state Silero carries from one 512-sample window to the next. + /// + /// Split out so a live capture can keep feeding the same session instead of re-running the + /// model over the whole buffer on every tick. Upstream's own streaming mode is exactly this: + /// the state and the 64-sample context are the entire history the model needs. + struct Carry { + var state = [Float](repeating: 0, count: 2 * 128) + var context = [Float](repeating: 0, count: 64) + } + + func probabilities(pcm: Data, sampleCount: Int, carry: inout Carry) throws -> [Float] { try pcm.withUnsafeBytes { raw in - var state = [Float](repeating: 0, count: 2 * 128) - var context = [Float](repeating: 0, count: 64) + var state = carry.state + var context = carry.context + defer { + carry.state = state + carry.context = context + } var probabilities: [Float] = [] probabilities.reserveCapacity( (sampleCount + SpeechActivity.windowSamples - 1) diff --git a/Tests/DoNotTypeCoreTests/SpeechStreamTests.swift b/Tests/DoNotTypeCoreTests/SpeechStreamTests.swift new file mode 100644 index 0000000..a5dcd8b --- /dev/null +++ b/Tests/DoNotTypeCoreTests/SpeechStreamTests.swift @@ -0,0 +1,142 @@ +import XCTest + +@testable import DoNotTypeCore + +/// The streaming Silero session, which is the part of the boundary change that can go subtly wrong. +/// +/// Silero is recurrent: every 512-sample window is evaluated with the state and the 64-sample +/// context left by the one before it. Feeding a live capture therefore means carrying that state +/// across calls rather than re-reading the buffer, and a bug in the carry produces probabilities +/// that are *plausible* rather than wrong-looking — which is exactly the kind of error a synthetic +/// fixture never catches. +final class SpeechStreamTests: XCTestCase { + private func realSpeech() throws -> Data { + let url = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent().deletingLastPathComponent() + .deletingLastPathComponent() + .appendingPathComponent("eval/audio/real-acronym.wav") + try XCTSkipUnless( + FileManager.default.fileExists(atPath: url.path), "eval fixture not present") + let wav = try Data(contentsOf: url) + return try XCTUnwrap(AudioChunker.pcmBody(of: wav)) + } + + /// The carry is correct if arrival size cannot change the answer. + /// + /// 85 ms is what the macOS recorder's tap actually delivers, and 1 s and 4 s bracket it either + /// side. All three must produce identical speech segments, because the audio is identical. + func testArrivalSizeDoesNotChangeTheResult() throws { + let pcm = try realSpeech() + var results: [[Range]] = [] + + for chunkSeconds in [0.085, 1.0, 4.0] { + let stream = try SpeechActivity.Stream() + let step = max(2, Int(chunkSeconds * 16_000) * 2) + var offset = 0 + while offset < pcm.count { + let end = min(offset + step, pcm.count) + try stream.append(pcm: pcm.subdata(in: offset..= 2, "this clip has no finalised gap to test") + let pauses = stream.pauses(from: 0, format: AudioChunker.Format()) + XCTAssertEqual(pauses.count, segments.count - 1) + + for pause in pauses { + XCTAssertGreaterThan(pause.duration, 0) + XCTAssertGreaterThanOrEqual(pause.depth, 0) + XCTAssertLessThanOrEqual(pause.depth, 20) + // The midpoint must land inside a gap, never inside a run. + let sample = pause.cut / 2 + for segment in segments { + XCTAssertFalse(segment.contains(sample), "a cut landed inside speech") + } + } + } +}