From ec443b8db6a4b7a592fbd6151b08a80d3f3de544 Mon Sep 17 00:00:00 2001 From: Mustafa Alammar Date: Fri, 24 Jul 2026 23:34:31 -0700 Subject: [PATCH 1/2] fix: bound the voice conditioning cache to stop it exhausting VRAM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _conds_cache is a plain dict keyed by (path, mtime, exaggeration) and cleared only by unload_model(), so every distinct reference voice pins GPU tensors for the lifetime of the process. Each entry costs ~175 MB of VRAM. Measured on a P106-100 (5.93 GiB) with chatterbox-turbo, three 10-minute soaks cycling 28 reference voices at ~45 req/min: cache size requests failed headroom unbounded 453 406 0.11 GiB 2 89 1 0.10-0.49 GiB 0 88 0 0.28-0.40 GiB Live tensors climbed 4.09 -> 5.46 GiB and stayed there; the card then failed 2 MiB allocations while holding ~260 MB reserved-but-free. The same load with a single voice ran clean, which isolates the cache rather than allocator fragmentation. Larger cards are not immune, only slower to get there: /upload_reference stores each uploaded file under its own path, so every distinct voice a deployment ever serves adds a permanent entry. Adds an LRU bound via CONDS_CACHE_MAX (default 4, ~700 MB). A cache hit now refreshes recency, so a rotating workload evicts least-recently-used rather than by insertion order. Set 0 to disable caching entirely — a miss only re-encodes the reference audio, measured at +0.14s per request, a fixed cost that is ~13% on an 84-character request and within noise by 240 characters. Co-Authored-By: Claude Opus 5 --- engine.py | 44 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/engine.py b/engine.py index a2be1e9..f6b5104 100644 --- a/engine.py +++ b/engine.py @@ -5,6 +5,7 @@ import logging import os import random +from collections import OrderedDict import numpy as np import torch from typing import Optional, Tuple @@ -114,7 +115,43 @@ def _resolve_bf16_setting() -> bool: # Voice conditioning cache: avoids re-encoding the same voice file on every request. # Key: (resolved_path, file_mtime, exaggeration) — mtime invalidates if file changes. -_conds_cache: dict = {} +# +# Bounded, because each entry pins GPU tensors for the process lifetime and this +# was previously an unbounded dict cleared only by unload_model(). Measured on a +# P106-100 (5.93 GiB) with chatterbox-turbo: cycling 28 reference voices grew +# live tensors from 4.09 to 5.46 GiB and left 0.11 GiB of headroom, after which +# 406 of 453 requests failed. The same load with one voice ran clean. +# +# This is not only a small-card concern. /upload_reference stores each uploaded +# file under its own path, so every distinct voice a deployment ever serves mints +# a permanent key — a larger card buys a higher ceiling, not a different outcome. +# +# CONDS_CACHE_MAX bounds the entry count. Measured cost is ~175 MB of VRAM per +# entry (live tensors grew 1.37 GiB across exactly 8 cached voices), so this is a +# far more expensive cache than it looks — budget it against free VRAM, not +# against an entry count that sounds small. +# +# The default of 4 costs ~700 MB, which leaves a 6 GB card ~1.1 GiB for +# generation after the 4.09 GiB model. On a 12 GB card 16+ is comfortable. Set 0 +# to disable caching and re-encode the voice on every request. +_CONDS_CACHE_MAX: int = max(0, int(os.environ.get("CONDS_CACHE_MAX", "4"))) +_conds_cache: "OrderedDict[tuple, object]" = OrderedDict() + + +def _conds_cache_store(key: tuple, conds) -> None: + """Insert under an LRU bound, evicting the least recently used entries. + + Eviction drops the last reference to that entry's tensors; the caching + allocator reuses the freed blocks, so VRAM is recovered without an explicit + empty_cache() on the request path. + """ + if _CONDS_CACHE_MAX == 0: + return + _conds_cache[key] = conds + _conds_cache.move_to_end(key) + while len(_conds_cache) > _CONDS_CACHE_MAX: + evicted_key, _ = _conds_cache.popitem(last=False) + logger.debug(f"Voice cache evicted (LRU, max={_CONDS_CACHE_MAX}): {evicted_key[0]}") def _conds_cache_key(path: str, exaggeration: float) -> tuple: @@ -481,6 +518,9 @@ def synthesize( conds_key = _conds_cache_key(audio_prompt_path, ex_for_key) if conds_key in _conds_cache: chatterbox_model.conds = _conds_cache[conds_key] + # Refresh recency, or the LRU bound would evict by insertion order + # and throw away the hottest voice under a rotating workload. + _conds_cache.move_to_end(conds_key) effective_prompt = None # conds already set, skip prepare_conditionals logger.debug(f"Voice cache hit: {audio_prompt_path}") @@ -509,7 +549,7 @@ def synthesize( # Store conds in cache after first compute for this voice. if conds_key is not None and effective_prompt is not None: if chatterbox_model.conds is not None: - _conds_cache[conds_key] = chatterbox_model.conds + _conds_cache_store(conds_key, chatterbox_model.conds) logger.debug(f"Cached voice conditionals for: {audio_prompt_path}") # The ChatterboxTTS.generate method already returns a CPU tensor. From a7a6eb00cf004b2d80069eaae73376f4f271f34c Mon Sep 17 00:00:00 2001 From: Mustafa Alammar Date: Wed, 29 Jul 2026 21:50:41 -0700 Subject: [PATCH 2/2] fix: make the voice cache LRU thread-safe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit synthesize() runs concurrently — server.py dispatches it through loop.run_in_executor() — so several threads touch _conds_cache at once. The cache read was two non-atomic steps (membership check, then index) and the write three (insert, refresh, evict), so a thread could evict a key between another thread's check and its use and raise KeyError. The unbounded version could not hit this: nothing was ever removed, so a key that existed stayed. Adding eviction introduced the failure mode, which makes this a regression in the LRU commit rather than a pre-existing bug. Serializes cache access behind a lock and replaces the check-then-index read with a single locked lookup that also refreshes recency. Verified with 8 threads x 4000 operations over 12 voices at cap 4 (constant eviction): 0 errors, bound held. Also stops a bad CONDS_CACHE_MAX from breaking module import — int() on garbage raised ValueError at import time, which would have failed the server at startup on a typo. Now warns and falls back to the default. Co-Authored-By: Claude Opus 5 --- engine.py | 61 ++++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 49 insertions(+), 12 deletions(-) diff --git a/engine.py b/engine.py index f6b5104..fc48086 100644 --- a/engine.py +++ b/engine.py @@ -5,6 +5,7 @@ import logging import os import random +import threading from collections import OrderedDict import numpy as np import torch @@ -134,9 +135,41 @@ def _resolve_bf16_setting() -> bool: # The default of 4 costs ~700 MB, which leaves a 6 GB card ~1.1 GiB for # generation after the 4.09 GiB model. On a 12 GB card 16+ is comfortable. Set 0 # to disable caching and re-encode the voice on every request. -_CONDS_CACHE_MAX: int = max(0, int(os.environ.get("CONDS_CACHE_MAX", "4"))) +def _resolve_conds_cache_max() -> int: + """Read CONDS_CACHE_MAX, tolerating garbage rather than failing to import.""" + raw = os.environ.get("CONDS_CACHE_MAX", "4").strip() + try: + return max(0, int(raw)) + except ValueError: + logger.warning( + f"CONDS_CACHE_MAX={raw!r} is not an integer; using the default of 4." + ) + return 4 + + +_CONDS_CACHE_MAX: int = _resolve_conds_cache_max() _conds_cache: "OrderedDict[tuple, object]" = OrderedDict() +# synthesize() runs concurrently: server.py dispatches it through +# loop.run_in_executor(), so several threads can touch this cache at once. +# Reads are two steps (look up, then refresh recency) and writes are three +# (insert, refresh, evict), none of which are atomic — without this lock a +# thread can evict a key between another thread's lookup and its use, raising +# KeyError. The unbounded version could not hit this because nothing was ever +# removed. +_conds_cache_lock = threading.Lock() + + +def _conds_cache_get(key: tuple): + """Return cached conds for `key` and mark it most-recently-used, else None.""" + if _CONDS_CACHE_MAX == 0: + return None + with _conds_cache_lock: + conds = _conds_cache.get(key) + if conds is not None: + _conds_cache.move_to_end(key) + return conds + def _conds_cache_store(key: tuple, conds) -> None: """Insert under an LRU bound, evicting the least recently used entries. @@ -147,11 +180,14 @@ def _conds_cache_store(key: tuple, conds) -> None: """ if _CONDS_CACHE_MAX == 0: return - _conds_cache[key] = conds - _conds_cache.move_to_end(key) - while len(_conds_cache) > _CONDS_CACHE_MAX: - evicted_key, _ = _conds_cache.popitem(last=False) - logger.debug(f"Voice cache evicted (LRU, max={_CONDS_CACHE_MAX}): {evicted_key[0]}") + with _conds_cache_lock: + _conds_cache[key] = conds + _conds_cache.move_to_end(key) + while len(_conds_cache) > _CONDS_CACHE_MAX: + evicted_key, _ = _conds_cache.popitem(last=False) + logger.debug( + f"Voice cache evicted (LRU, max={_CONDS_CACHE_MAX}): {evicted_key[0]}" + ) def _conds_cache_key(path: str, exaggeration: float) -> tuple: @@ -516,11 +552,11 @@ def synthesize( if audio_prompt_path and hasattr(chatterbox_model, "conds"): ex_for_key = 0.0 if loaded_model_type == "turbo" else exaggeration conds_key = _conds_cache_key(audio_prompt_path, ex_for_key) - if conds_key in _conds_cache: - chatterbox_model.conds = _conds_cache[conds_key] - # Refresh recency, or the LRU bound would evict by insertion order - # and throw away the hottest voice under a rotating workload. - _conds_cache.move_to_end(conds_key) + # Single locked lookup that also refreshes recency — checking + # membership and then indexing would race with eviction. + cached_conds = _conds_cache_get(conds_key) + if cached_conds is not None: + chatterbox_model.conds = cached_conds effective_prompt = None # conds already set, skip prepare_conditionals logger.debug(f"Voice cache hit: {audio_prompt_path}") @@ -630,7 +666,8 @@ def reload_model() -> bool: MODEL_LOADED = False loaded_model_type = None loaded_model_class_name = None - _conds_cache.clear() + with _conds_cache_lock: + _conds_cache.clear() logger.info("Voice conditioning cache cleared.") # 3. Force Python Garbage Collection